From 32ad722d4973c78de9c3fc682fe08ecc9a15421d Mon Sep 17 00:00:00 2001 From: Master Date: Sun, 9 Aug 2026 17:32:48 -0500 Subject: [PATCH 1/3] docs: update kernelwiki using evolve skills --- .../blogs/flash-attention-4/MANIFEST.yaml | 11 +- .../code/01-software-exp-cody-waite-horner.cu | 23 - ...d-range-reduction-and-rounded-polynomia.cu | 15 + .../code/02-2-cta-cooperative-backward.cu | 16 +- .../flash-attention-4/code/PROVENANCE.yaml | 15 +- artifacts/blogs/flashmla/MANIFEST.yaml | 19 +- .../flashmla/code/01-mla-decode-inner-loop.cu | 31 - .../01-v3-fp8-sparse-decode-byte-check.py | 16 + .../code/02-decode-page-index-round-trip.py | 15 + .../02-sparse-mla-kv-retrieval-kernel-v3-2.cu | 27 - artifacts/blogs/flashmla/code/PROVENANCE.yaml | 17 +- artifacts/blogs/gated-delta-net/MANIFEST.yaml | 16 +- ...hunk-parallel-prefill-reference-pytorch.py | 29 - .../02-triton-decode-step-kernel-streaming.py | 33 - .../gated-delta-net/code/PROVENANCE.yaml | 21 - .../flash-attention-4/full/PROVENANCE.yaml | 4 + .../variants/01-software-exp-skeleton.cu | 28 +- .../variants/PROVENANCE.yaml | 12 +- .../kernels/flashmla/full/PROVENANCE.yaml | 6 + .../variants/01-mla-decode-inner-loop.cu | 34 - .../01-mla-layout-and-index-contract.cu | 32 + .../kernels/flashmla/variants/PROVENANCE.yaml | 9 +- .../kernels/fused-moe/full/PROVENANCE.yaml | 32 +- .../full/moe-grouped-gemm-launch.cpp | 7 +- .../01-routing-plus-fusion-skeleton.py | 246 ++++- .../fused-moe/variants/PROVENANCE.yaml | 17 +- .../gated-delta-net/full/PROVENANCE.yaml | 5 +- ...hunk-parallel-prefill-reference-pytorch.py | 32 - .../variants/01-gdn-recurrence-reference.py | 101 ++ .../02-triton-decode-step-kernel-streaming.py | 36 - .../gated-delta-net/variants/PROVENANCE.yaml | 21 +- .../gated-dual-gemm/full/PROVENANCE.yaml | 33 +- .../blackwell-cutlass-schedules-and-tma.cu | 24 - .../kernels/gated-dual-gemm/full/task.yml | 64 ++ .../full/vllm-PR-23696-gated-dual-gemm.patch | 872 ------------------ .../01-fused-epilogue-swiglu-skeleton.cu | 25 - .../variants/01-gated-dual-gemm-semantics.py | 73 ++ .../gated-dual-gemm/variants/PROVENANCE.yaml | 12 +- artifacts/prs/vllm/PR-23696/PROVENANCE.yaml | 6 +- data/core-prs.yaml | 20 +- data/inclusion-policy.yaml | 30 +- data/schemas.yaml | 7 +- data/tags.yaml | 7 + data/tool-versions.yaml | 33 +- data/triton-3.6-evidence.md | 90 +- data/triton-universe.yaml | 7 +- data/version-claims.yaml | 47 +- queries/by-hardware-feature.md | 28 +- queries/by-kernel-type.md | 33 +- queries/by-language.md | 13 +- queries/by-problem.md | 14 +- queries/by-repo.md | 10 +- queries/by-technique.md | 32 +- references/examples.md | 4 +- references/primer.md | 4 +- scripts/validate.py | 24 +- scripts/verify_core_prs.py | 83 +- scripts/verify_verbatim.py | 119 ++- sources/blogs/amandeep-nvfp4-attempts.md | 198 ++-- .../blogs/colfax-article-source-kernels.md | 12 +- sources/blogs/colfax-cutlass-blackwell.md | 66 +- sources/blogs/colfax-cutlass-kernels.md | 16 +- sources/blogs/deepgemm.md | 93 +- sources/blogs/flash-attention-4.md | 83 +- sources/blogs/flashmla.md | 130 ++- sources/blogs/gated-delta-net.md | 96 +- sources/blogs/modular-blackwell-matmul.md | 104 +-- sources/blogs/nsa.md | 26 - sources/blogs/nvidia-code-samples.md | 20 +- sources/blogs/simon-nvfp4-gemv.md | 148 +-- sources/blogs/simveit-effective-transpose.md | 15 +- sources/blogs/simveit-load-and-store.md | 21 +- sources/blogs/tcgen05-tutorial.md | 107 +-- sources/blogs/yue-nvfp4-hackathon.md | 140 +-- .../flashinfer-mlsys26/track-a-fused-moe.md | 280 ++---- .../track-c-gated-delta-net.md | 295 +----- .../contests/gpu-mode-nvfp4/problem-1-gemv.md | 221 +---- .../contests/gpu-mode-nvfp4/problem-2-gemm.md | 180 +--- .../problem-3-gated-dual-gemm.md | 191 +--- .../gpu-mode-nvfp4/problem-4-grouped-gemm.md | 194 +--- sources/docs/cutlass-clc-documentation.md | 123 +-- sources/docs/deepseek-v2-mla.md | 18 + sources/docs/deepseek-v3-fp8.md | 21 + sources/docs/flash-attention-4.md | 38 +- sources/docs/nsa.md | 39 + sources/docs/nvidia-blackwell-tuning-guide.md | 142 +-- sources/docs/nvidia-cuda-13-0-2-tma.md | 27 + sources/docs/nvidia-cuda-register-controls.md | 29 + sources/docs/nvidia-cutlass-blackwell.md | 157 +--- sources/docs/nvidia-ptx-isa-sm100.md | 243 +---- .../nvidia-transformer-engine-2.13-nvfp4.md | 26 + sources/docs/tfla.md | 31 +- sources/docs/triton-3.3-blackwell.md | 28 + sources/docs/triton-3.6-blackwell.md | 57 +- sources/prs/cccl/PR-3559.md | 9 +- sources/prs/cccl/PR-6152.md | 12 +- sources/prs/cutlass/PR-2139.md | 5 +- sources/prs/cutlass/PR-2161.md | 8 +- sources/prs/flash-attention/PR-2441.md | 43 +- sources/prs/flashinfer/PR-2387.md | 49 +- sources/prs/vllm/PR-16032.md | 50 +- sources/prs/vllm/PR-23696.md | 72 +- sources/prs/vllm/PR-34597.md | 5 +- wiki/hardware/2sm-cooperative.md | 69 +- wiki/hardware/clc.md | 334 ++----- wiki/hardware/mbarrier.md | 143 +-- wiki/hardware/nvfp4.md | 92 +- wiki/hardware/pdl-gdc.md | 119 ++- wiki/hardware/tcgen05-mma.md | 322 ++----- wiki/hardware/tma.md | 444 +++------ wiki/hardware/tmem.md | 342 ++----- wiki/kernels/deepgemm.md | 218 ++--- wiki/kernels/flash-attention-4.md | 193 ++-- .../kernels/flash-attention-sm100-mla-topk.md | 76 +- wiki/kernels/flashmla.md | 257 ++---- wiki/kernels/fp8-block-scale-gemm.md | 124 +-- wiki/kernels/fused-moe.md | 325 ++----- wiki/kernels/gated-delta-net.md | 275 ++---- wiki/kernels/gated-dual-gemm.md | 141 ++- wiki/kernels/grouped-gemm.md | 299 +++--- wiki/kernels/nsa.md | 242 ++--- wiki/kernels/nvfp4-gemm.md | 295 +++--- wiki/kernels/nvfp4-gemv.md | 272 ++---- wiki/kernels/sparse-mla.md | 235 ++--- .../kernels/tensorrt-llm-blackwell-indexer.md | 115 ++- wiki/languages/cuda-cpp.md | 142 ++- wiki/languages/cute-dsl.md | 167 ++-- wiki/languages/ptx-sm100.md | 125 ++- wiki/languages/triton-blackwell.md | 177 ++-- wiki/migration/register-to-tmem.md | 494 ++-------- wiki/migration/wgmma-to-tcgen05.md | 396 +------- wiki/patterns/compute-bound.md | 88 +- wiki/patterns/low-sm-utilization.md | 77 +- wiki/patterns/memory-bound.md | 89 +- wiki/patterns/moe-load-imbalance.md | 126 ++- wiki/patterns/pipeline-stalls.md | 89 +- wiki/patterns/register-pressure.md | 70 +- wiki/patterns/tail-effect.md | 70 +- wiki/techniques/cache-policy.md | 68 +- wiki/techniques/cccl-memory-primitives.md | 76 +- wiki/techniques/chunk-parallelism.md | 70 +- wiki/techniques/double-buffering.md | 310 ++----- wiki/techniques/epilogue-fusion.md | 284 +++--- .../external-source-map-research.md | 67 +- wiki/techniques/fine-grained-quantization.md | 325 +++---- wiki/techniques/kernel-fusion.md | 115 ++- wiki/techniques/persistent-kernels.md | 232 ++--- wiki/techniques/ping-pong-scheduling.md | 105 +-- wiki/techniques/pipeline-stages.md | 238 ++--- wiki/techniques/register-budgeting.md | 75 +- wiki/techniques/software-exp.md | 266 ++---- wiki/techniques/swizzling.md | 240 ++--- wiki/techniques/tile-scheduling.md | 215 ++--- wiki/techniques/vectorized-loads.md | 325 ++----- wiki/techniques/warp-specialization.md | 278 +----- 155 files changed, 5557 insertions(+), 11282 deletions(-) delete mode 100644 artifacts/blogs/flash-attention-4/code/01-software-exp-cody-waite-horner.cu create mode 100644 artifacts/blogs/flash-attention-4/code/01-software-exp-published-range-reduction-and-rounded-polynomia.cu delete mode 100644 artifacts/blogs/flashmla/code/01-mla-decode-inner-loop.cu create mode 100644 artifacts/blogs/flashmla/code/01-v3-fp8-sparse-decode-byte-check.py create mode 100644 artifacts/blogs/flashmla/code/02-decode-page-index-round-trip.py delete mode 100644 artifacts/blogs/flashmla/code/02-sparse-mla-kv-retrieval-kernel-v3-2.cu delete mode 100644 artifacts/blogs/gated-delta-net/code/01-chunk-parallel-prefill-reference-pytorch.py delete mode 100644 artifacts/blogs/gated-delta-net/code/02-triton-decode-step-kernel-streaming.py delete mode 100644 artifacts/blogs/gated-delta-net/code/PROVENANCE.yaml delete mode 100644 artifacts/kernels/flashmla/variants/01-mla-decode-inner-loop.cu create mode 100644 artifacts/kernels/flashmla/variants/01-mla-layout-and-index-contract.cu delete mode 100644 artifacts/kernels/gated-delta-net/variants/01-chunk-parallel-prefill-reference-pytorch.py create mode 100644 artifacts/kernels/gated-delta-net/variants/01-gdn-recurrence-reference.py delete mode 100644 artifacts/kernels/gated-delta-net/variants/02-triton-decode-step-kernel-streaming.py delete mode 100644 artifacts/kernels/gated-dual-gemm/full/blackwell-cutlass-schedules-and-tma.cu create mode 100644 artifacts/kernels/gated-dual-gemm/full/task.yml delete mode 100644 artifacts/kernels/gated-dual-gemm/full/vllm-PR-23696-gated-dual-gemm.patch delete mode 100644 artifacts/kernels/gated-dual-gemm/variants/01-fused-epilogue-swiglu-skeleton.cu create mode 100644 artifacts/kernels/gated-dual-gemm/variants/01-gated-dual-gemm-semantics.py delete mode 100644 sources/blogs/nsa.md create mode 100644 sources/docs/deepseek-v2-mla.md create mode 100644 sources/docs/deepseek-v3-fp8.md create mode 100644 sources/docs/nsa.md create mode 100644 sources/docs/nvidia-cuda-13-0-2-tma.md create mode 100644 sources/docs/nvidia-cuda-register-controls.md create mode 100644 sources/docs/nvidia-transformer-engine-2.13-nvfp4.md create mode 100644 sources/docs/triton-3.3-blackwell.md diff --git a/artifacts/blogs/flash-attention-4/MANIFEST.yaml b/artifacts/blogs/flash-attention-4/MANIFEST.yaml index e6d2b352f..c52760167 100644 --- a/artifacts/blogs/flash-attention-4/MANIFEST.yaml +++ b/artifacts/blogs/flash-attention-4/MANIFEST.yaml @@ -4,11 +4,12 @@ code_present: true total_blocks: 2 generated_by: scripts/extract_blog_code.py files: -- local_path: code/01-software-exp-cody-waite-horner.cu - heading_path: '## Key Code > ### Software exp (Cody-Waite + Horner)' +- local_path: code/01-software-exp-published-range-reduction-and-rounded-polynomia.cu + heading_path: '## Illustrative Code > ### Software exp (published range reduction + and rounded polynomial)' fence_lang: cuda - sha256: 6fba9c537831bba7b175ee3a050e3adb5ddf99193dee550b7122a4663e3f9164 + sha256: 89c085df18aa5ae9b8693d1085bf43881b8bc3644068761e0312c9ee7628e71c - local_path: code/02-2-cta-cooperative-backward.cu - heading_path: '## Key Code > ### 2-CTA cooperative backward' + heading_path: '## Illustrative Code > ### 2-CTA cooperative backward' fence_lang: cuda - sha256: e089feff40c10ebf42fa1ff878f775c21f4f044fc4b6f2f510b43c2666c7e219 + sha256: 5747bb5ef24ec8a82d9fd80d433d0dc231d0de042af83eeec9414b03a1a53de9 diff --git a/artifacts/blogs/flash-attention-4/code/01-software-exp-cody-waite-horner.cu b/artifacts/blogs/flash-attention-4/code/01-software-exp-cody-waite-horner.cu deleted file mode 100644 index 9f329ce70..000000000 --- a/artifacts/blogs/flash-attention-4/code/01-software-exp-cody-waite-horner.cu +++ /dev/null @@ -1,23 +0,0 @@ -// Extracted from sources/blogs/flash-attention-4.md by scripts/extract_blog_code.py -// Heading: ## Key Code > ### Software exp (Cody-Waite + Horner) -// Original fence language: cuda -// See artifacts/blogs/flash-attention-4/code/PROVENANCE.yaml for origin + license metadata. - -// Software-emulated exp2(x) using Cody-Waite range reduction and a -// Horner-scheme polynomial, Sollya-optimized coefficients. Lets FA-4 -// overlap the exp path with tcgen05.mma because it stays off the SFU. -__device__ __forceinline__ float sw_exp2(float x) { - // Range reduction: x = n + r, with n = round(x), r in [-0.5, 0.5] - int n = __float2int_rn(x); - float r = x - (float)n; - // Horner-scheme polynomial for 2^r, r in [-0.5, 0.5] - float p = 0x1.62e430p-1f; // ~ ln(2) - p = fmaf(p, r, 0x1.ebfc1ep-3f); - p = fmaf(p, r, 0x1.c6af98p-5f); - p = fmaf(p, r, 0x1.3b2c9cp-7f); - p = fmaf(p, r, 0x1.62e43ap-10f); - float y = fmaf(r, p, 1.0f); - // Scale by 2^n via direct FP32 bit manipulation - int bits = __float_as_int(y) + (n << 23); - return __int_as_float(bits); -} diff --git a/artifacts/blogs/flash-attention-4/code/01-software-exp-published-range-reduction-and-rounded-polynomia.cu b/artifacts/blogs/flash-attention-4/code/01-software-exp-published-range-reduction-and-rounded-polynomia.cu new file mode 100644 index 000000000..24483414a --- /dev/null +++ b/artifacts/blogs/flash-attention-4/code/01-software-exp-published-range-reduction-and-rounded-polynomia.cu @@ -0,0 +1,15 @@ +// Extracted from sources/blogs/flash-attention-4.md by scripts/extract_blog_code.py +// Heading: ## Illustrative Code > ### Software exp (published range reduction and rounded polynomial) +// Original fence language: cuda +// See artifacts/blogs/flash-attention-4/code/PROVENANCE.yaml for origin + license metadata. + +// KernelWiki scalar illustration derived from the FA4 blog equations. +// This is not verbatim upstream FA4 code and omits selection and clamping. +#include + +__host__ __device__ inline float fa4_blog_exp2_reference(float x) { + const int n = static_cast(floorf(x)); + const float f = x - static_cast(n); // f in [0, 1) + const float p = 1.0f + f * (0.6951f + f * (0.2276f + f * 0.0771f)); + return ldexpf(p, n); +} diff --git a/artifacts/blogs/flash-attention-4/code/02-2-cta-cooperative-backward.cu b/artifacts/blogs/flash-attention-4/code/02-2-cta-cooperative-backward.cu index 8c4fcaf3a..cbf44c0de 100644 --- a/artifacts/blogs/flash-attention-4/code/02-2-cta-cooperative-backward.cu +++ b/artifacts/blogs/flash-attention-4/code/02-2-cta-cooperative-backward.cu @@ -1,10 +1,14 @@ // Extracted from sources/blogs/flash-attention-4.md by scripts/extract_blog_code.py -// Heading: ## Key Code > ### 2-CTA cooperative backward +// Heading: ## Illustrative Code > ### 2-CTA cooperative backward // Original fence language: cuda // See artifacts/blogs/flash-attention-4/code/PROVENANCE.yaml for origin + license metadata. -// 2-CTA cooperative backward: paired CTAs in a cluster share a single TMEM -// accumulator half, halving SMEM traffic for dK/dV accumulation. -asm volatile( - "tcgen05.mma.cta_group::2.kind::f16 [%0], %1, %2, %3, 1;" - : : "r"(tmem_acc_shared), "l"(desc_a), "l"(desc_b), "r"(0)); +// KernelWiki schematic derived from the FA4 paper/blog dimensions. +// This is not upstream inline PTX or a complete kernel. +struct Fa4TwoCtaBackwardShape { + static constexpr int cta_group = 2; + static constexpr int mma_m = 256; + static constexpr int mma_n = 128; + static constexpr int mma_k = 128; + static constexpr int backward_gemm_count = 5; +}; diff --git a/artifacts/blogs/flash-attention-4/code/PROVENANCE.yaml b/artifacts/blogs/flash-attention-4/code/PROVENANCE.yaml index 2d0e28314..b135da698 100644 --- a/artifacts/blogs/flash-attention-4/code/PROVENANCE.yaml +++ b/artifacts/blogs/flash-attention-4/code/PROVENANCE.yaml @@ -1,21 +1,22 @@ origin_url: https://tridao.me/blog/2026/flash4/ upstream_repo: blog -upstream_sha: c9a560f44002da92d82680036e3482d0ac4939a3 +upstream_sha: none license: inherits-from-source-blog -retrieved_at: 2026-04-27 +retrieved_at: 2026-08-08 asset_mode: extracted generated_by: scripts/extract_blog_code.py size_cap_truncated: false files: -- local_path: 01-software-exp-cody-waite-horner.cu +- local_path: 01-software-exp-published-range-reduction-and-rounded-polynomia.cu role: extracted-block mode: extracted upstream_path: inline-in-blog-markdown - heading_path: '## Key Code > ### Software exp (Cody-Waite + Horner)' - sha256: 6fba9c537831bba7b175ee3a050e3adb5ddf99193dee550b7122a4663e3f9164 + heading_path: '## Illustrative Code > ### Software exp (published range reduction + and rounded polynomial)' + sha256: 89c085df18aa5ae9b8693d1085bf43881b8bc3644068761e0312c9ee7628e71c - local_path: 02-2-cta-cooperative-backward.cu role: extracted-block mode: extracted upstream_path: inline-in-blog-markdown - heading_path: '## Key Code > ### 2-CTA cooperative backward' - sha256: e089feff40c10ebf42fa1ff878f775c21f4f044fc4b6f2f510b43c2666c7e219 + heading_path: '## Illustrative Code > ### 2-CTA cooperative backward' + sha256: 5747bb5ef24ec8a82d9fd80d433d0dc231d0de042af83eeec9414b03a1a53de9 diff --git a/artifacts/blogs/flashmla/MANIFEST.yaml b/artifacts/blogs/flashmla/MANIFEST.yaml index 376f737f1..ff05e28c2 100644 --- a/artifacts/blogs/flashmla/MANIFEST.yaml +++ b/artifacts/blogs/flashmla/MANIFEST.yaml @@ -1,14 +1,15 @@ slug: flashmla -origin_url: https://github.com/deepseek-ai/FlashMLA +origin_url: https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232 code_present: true total_blocks: 2 generated_by: scripts/extract_blog_code.py files: -- local_path: code/01-mla-decode-inner-loop.cu - heading_path: '## Key Code > ### MLA decode inner loop' - fence_lang: cuda - sha256: 4d3950b24b3da7a36e1db9d60a0492ed963d1b49ad6a1b97adeab54abaf4548f -- local_path: code/02-sparse-mla-kv-retrieval-kernel-v3-2.cu - heading_path: '## Key Code > ### Sparse-MLA KV-retrieval kernel (V3.2)' - fence_lang: cuda - sha256: 9b0499c0ddb77a9cebd370e84929d7149ebdc03163768369c95a12ae0b5c4f7f +- local_path: code/01-v3-fp8-sparse-decode-byte-check.py + heading_path: '## Exact V3 FP8 Sparse-Decode Layout > ### V3 FP8 sparse-decode byte + check' + fence_lang: python + sha256: bfc0a6df6b2e04ce4f894a9df04926d4120b4e370354d68e80a55c1645565c7e +- local_path: code/02-decode-page-index-round-trip.py + heading_path: '## Sparse Index Contracts > ### Decode page-index round trip' + fence_lang: python + sha256: a0bbc6f0248adc67b8113e3faa66c1be8924aafce665ebf89b43f0e2f0e4a405 diff --git a/artifacts/blogs/flashmla/code/01-mla-decode-inner-loop.cu b/artifacts/blogs/flashmla/code/01-mla-decode-inner-loop.cu deleted file mode 100644 index 6ca3466b5..000000000 --- a/artifacts/blogs/flashmla/code/01-mla-decode-inner-loop.cu +++ /dev/null @@ -1,31 +0,0 @@ -// Extracted from sources/blogs/flashmla.md by scripts/extract_blog_code.py -// Heading: ## Key Code > ### MLA decode inner loop -// Original fence language: cuda -// See artifacts/blogs/flashmla/code/PROVENANCE.yaml for origin + license metadata. - -// MLA collapses K and V into a shared latent matrix of head-dim Dc=128. -// On decode (one query vector against N KV tokens) we alternate TMA load, -// wgmma/tcgen05 into the q@K^T accumulator, online softmax, and the second -// accumulator against V. -constexpr int Dc = 128; // latent head dim -constexpr int BLOCK_N = 64; // paged KV block size -float acc[Dc] = {0}; -float max_val = -INFINITY; -float l = 0.f; -for (int n0 = 0; n0 < seqlen; n0 += BLOCK_N) { - tma_load(smem_kv, KV_pages + n0); - cp_async_wait(); - float scores[BLOCK_N]; - wgmma_or_tcgen05_mma(scores, q, smem_kv); // q @ K^T - float new_max = warp_reduce_max(scores, BLOCK_N); - float scale = expf(max_val - new_max); - for (int j = 0; j < Dc; j++) acc[j] *= scale; - l *= scale; - for (int j = 0; j < BLOCK_N; j++) { - float p = expf(scores[j] - new_max); - l += p; - for (int d = 0; d < Dc; d++) acc[d] += p * smem_kv[j * Dc + d]; - } - max_val = new_max; -} -for (int d = 0; d < Dc; d++) O[d] = acc[d] / l; diff --git a/artifacts/blogs/flashmla/code/01-v3-fp8-sparse-decode-byte-check.py b/artifacts/blogs/flashmla/code/01-v3-fp8-sparse-decode-byte-check.py new file mode 100644 index 000000000..4fc54a919 --- /dev/null +++ b/artifacts/blogs/flashmla/code/01-v3-fp8-sparse-decode-byte-check.py @@ -0,0 +1,16 @@ +# Extracted from sources/blogs/flashmla.md by scripts/extract_blog_code.py +# Heading: ## Exact V3 FP8 Sparse-Decode Layout > ### V3 FP8 sparse-decode byte check +# Original fence language: python +# See artifacts/blogs/flashmla/code/PROVENANCE.yaml for origin + license metadata. + +# KernelWiki-derived contract check; not upstream FlashMLA code. +NOPE_FP8_VALUES = 512 +GROUPS = 4 +FP32_BYTES = 4 +ROPE_BF16_VALUES = 64 +BF16_BYTES = 2 + +V3_FP8_SPARSE_BYTES = ( + NOPE_FP8_VALUES + GROUPS * FP32_BYTES + ROPE_BF16_VALUES * BF16_BYTES +) +assert V3_FP8_SPARSE_BYTES == 656 diff --git a/artifacts/blogs/flashmla/code/02-decode-page-index-round-trip.py b/artifacts/blogs/flashmla/code/02-decode-page-index-round-trip.py new file mode 100644 index 000000000..3ea60829b --- /dev/null +++ b/artifacts/blogs/flashmla/code/02-decode-page-index-round-trip.py @@ -0,0 +1,15 @@ +# Extracted from sources/blogs/flashmla.md by scripts/extract_blog_code.py +# Heading: ## Sparse Index Contracts > ### Decode page-index round trip +# Original fence language: python +# See artifacts/blogs/flashmla/code/PROVENANCE.yaml for origin + license metadata. + +# KernelWiki-derived contract check; not upstream FlashMLA code. +def encode_page_index(physical_page: int, offset: int, page_size: int) -> int: + assert physical_page >= 0 and 0 <= offset < page_size + return physical_page * page_size + offset + +def decode_page_index(encoded: int, page_size: int) -> tuple[int, int]: + assert encoded >= 0 and page_size > 0 + return divmod(encoded, page_size) + +assert decode_page_index(encode_page_index(7, 13, 64), 64) == (7, 13) diff --git a/artifacts/blogs/flashmla/code/02-sparse-mla-kv-retrieval-kernel-v3-2.cu b/artifacts/blogs/flashmla/code/02-sparse-mla-kv-retrieval-kernel-v3-2.cu deleted file mode 100644 index ce8ec82c2..000000000 --- a/artifacts/blogs/flashmla/code/02-sparse-mla-kv-retrieval-kernel-v3-2.cu +++ /dev/null @@ -1,27 +0,0 @@ -// Extracted from sources/blogs/flashmla.md by scripts/extract_blog_code.py -// Heading: ## Key Code > ### Sparse-MLA KV-retrieval kernel (V3.2) -// Original fence language: cuda -// See artifacts/blogs/flashmla/code/PROVENANCE.yaml for origin + license metadata. - -// Sparse MLA selects top-k KV positions per query before running the dense -// MLA kernel on just those positions. Retrieval uses FP8 dot products with -// per-token scale factors. -__global__ void sparse_mla_topk( - const __nv_fp8_e4m3* Q, const __nv_fp8_e4m3* K, - const float* Q_scale, const float* K_scale, - int* topk_idx, float* topk_score, - int N, int K_DIM, int TOPK) -{ - int q_tile = blockIdx.x; - float scores[N]; - for (int n = 0; n < N; n++) { - float s = 0.f; - for (int k = 0; k < K_DIM; k++) { - s += decode_fp8(Q[q_tile * K_DIM + k]) * Q_scale[q_tile] - * decode_fp8(K[n * K_DIM + k]) * K_scale[n]; - } - scores[n] = s; - } - warp_topk_select(scores, N, topk_idx + q_tile * TOPK, - topk_score + q_tile * TOPK, TOPK); -} diff --git a/artifacts/blogs/flashmla/code/PROVENANCE.yaml b/artifacts/blogs/flashmla/code/PROVENANCE.yaml index fcef8cd20..84a2595b3 100644 --- a/artifacts/blogs/flashmla/code/PROVENANCE.yaml +++ b/artifacts/blogs/flashmla/code/PROVENANCE.yaml @@ -1,21 +1,22 @@ -origin_url: https://github.com/deepseek-ai/FlashMLA +origin_url: https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232 upstream_repo: blog -upstream_sha: 71c737929f2567bd0a094ae140f8f60f390b1232 +upstream_sha: none license: inherits-from-source-blog retrieved_at: 2026-04-27 asset_mode: extracted generated_by: scripts/extract_blog_code.py size_cap_truncated: false files: -- local_path: 01-mla-decode-inner-loop.cu +- local_path: 01-v3-fp8-sparse-decode-byte-check.py role: extracted-block mode: extracted upstream_path: inline-in-blog-markdown - heading_path: '## Key Code > ### MLA decode inner loop' - sha256: 4d3950b24b3da7a36e1db9d60a0492ed963d1b49ad6a1b97adeab54abaf4548f -- local_path: 02-sparse-mla-kv-retrieval-kernel-v3-2.cu + heading_path: '## Exact V3 FP8 Sparse-Decode Layout > ### V3 FP8 sparse-decode byte + check' + sha256: bfc0a6df6b2e04ce4f894a9df04926d4120b4e370354d68e80a55c1645565c7e +- local_path: 02-decode-page-index-round-trip.py role: extracted-block mode: extracted upstream_path: inline-in-blog-markdown - heading_path: '## Key Code > ### Sparse-MLA KV-retrieval kernel (V3.2)' - sha256: 9b0499c0ddb77a9cebd370e84929d7149ebdc03163768369c95a12ae0b5c4f7f + heading_path: '## Sparse Index Contracts > ### Decode page-index round trip' + sha256: a0bbc6f0248adc67b8113e3faa66c1be8924aafce665ebf89b43f0e2f0e4a405 diff --git a/artifacts/blogs/gated-delta-net/MANIFEST.yaml b/artifacts/blogs/gated-delta-net/MANIFEST.yaml index 390694f2d..188abb0e0 100644 --- a/artifacts/blogs/gated-delta-net/MANIFEST.yaml +++ b/artifacts/blogs/gated-delta-net/MANIFEST.yaml @@ -1,14 +1,4 @@ slug: gated-delta-net -origin_url: https://github.com/NVlabs/GatedDeltaNet -code_present: true -total_blocks: 2 -generated_by: scripts/extract_blog_code.py -files: -- local_path: code/01-chunk-parallel-prefill-reference-pytorch.py - heading_path: '## Key Code > ### Chunk-parallel prefill reference (PyTorch)' - fence_lang: python - sha256: 709cd74d72c22e764ffb77d85bbecd0b1c47b52f2cc0c48bab279954d0f24a22 -- local_path: code/02-triton-decode-step-kernel-streaming.py - heading_path: '## Key Code > ### Triton decode-step kernel (streaming)' - fence_lang: python - sha256: 09a1bc93abdf8165a3d7809e485ef17a3d1a980ae579e36ba81b47665ba35b22 +origin_url: https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921 +code_present: false +generated_by: KernelWiki verifier remediation diff --git a/artifacts/blogs/gated-delta-net/code/01-chunk-parallel-prefill-reference-pytorch.py b/artifacts/blogs/gated-delta-net/code/01-chunk-parallel-prefill-reference-pytorch.py deleted file mode 100644 index 7708ad9a0..000000000 --- a/artifacts/blogs/gated-delta-net/code/01-chunk-parallel-prefill-reference-pytorch.py +++ /dev/null @@ -1,29 +0,0 @@ -# Extracted from sources/blogs/gated-delta-net.md by scripts/extract_blog_code.py -# Heading: ## Key Code > ### Chunk-parallel prefill reference (PyTorch) -# Original fence language: python -# See artifacts/blogs/gated-delta-net/code/PROVENANCE.yaml for origin + license metadata. - -import torch - -def gated_delta_net_prefill(q, k, v, gate, initial_state, CHUNK_SIZE=64): - """ - Chunk-parallel prefill. Each chunk's state matrix is reused across its - query window, so we pay the O(Dk*Dv) state update once per chunk, not - per token. - q, k: [B, L, Dk] v: [B, L, Dv] gate: [B, L] - """ - B, L, Dk = q.shape - Dv = v.shape[-1] - out = torch.empty(B, L, Dv, device=q.device, dtype=q.dtype) - state = initial_state.clone() # [B, Dk, Dv] - for ci in range(0, L, CHUNK_SIZE): - ce = min(ci + CHUNK_SIZE, L) - k_chunk = k[:, ci:ce] - v_chunk = v[:, ci:ce] - g_chunk = gate[:, ci:ce] - decay = torch.cumprod(g_chunk, dim=1) # adaptive memory decay - for t in range(ce - ci): - state = state * decay[:, t:t+1, None] - state = state + k_chunk[:, t, :, None] * v_chunk[:, t, None, :] - out[:, ci + t] = (q[:, ci + t, :, None] * state).sum(dim=1) - return out, state diff --git a/artifacts/blogs/gated-delta-net/code/02-triton-decode-step-kernel-streaming.py b/artifacts/blogs/gated-delta-net/code/02-triton-decode-step-kernel-streaming.py deleted file mode 100644 index 33a7f7710..000000000 --- a/artifacts/blogs/gated-delta-net/code/02-triton-decode-step-kernel-streaming.py +++ /dev/null @@ -1,33 +0,0 @@ -# Extracted from sources/blogs/gated-delta-net.md by scripts/extract_blog_code.py -# Heading: ## Key Code > ### Triton decode-step kernel (streaming) -# Original fence language: python -# See artifacts/blogs/gated-delta-net/code/PROVENANCE.yaml for origin + license metadata. - -import triton -import triton.language as tl - -@triton.jit -def gdn_decode_step_kernel( - Q, K, V, GATE, STATE, OUT, - stride_qb, stride_kb, stride_vb, - Dk: tl.constexpr, Dv: tl.constexpr): - """ - One-token delta-rule update for decode. STATE is a [Dk, Dv] matrix kept - per sample; we fold in the new (k,v) pair after applying the decay gate. - """ - b = tl.program_id(0) - dk = tl.arange(0, Dk) - dv = tl.arange(0, Dv) - - q = tl.load(Q + b * stride_qb + dk) # [Dk] - k = tl.load(K + b * stride_kb + dk) # [Dk] - v = tl.load(V + b * stride_vb + dv) # [Dv] - g = tl.load(GATE + b) # scalar decay - - state = tl.load(STATE + b * Dk * Dv + dk[:, None] * Dv + dv[None, :]) - state = state * g # apply decay - state = state + k[:, None] * v[None, :] # delta update - tl.store(STATE + b * Dk * Dv + dk[:, None] * Dv + dv[None, :], state) - - out = tl.sum(q[:, None] * state, axis=0) # [Dv] - tl.store(OUT + b * Dv + dv, out) diff --git a/artifacts/blogs/gated-delta-net/code/PROVENANCE.yaml b/artifacts/blogs/gated-delta-net/code/PROVENANCE.yaml deleted file mode 100644 index 8fc34220e..000000000 --- a/artifacts/blogs/gated-delta-net/code/PROVENANCE.yaml +++ /dev/null @@ -1,21 +0,0 @@ -origin_url: https://github.com/NVlabs/GatedDeltaNet -upstream_repo: blog -upstream_sha: none -license: inherits-from-source-blog -retrieved_at: 2026-04-16 -asset_mode: extracted -generated_by: scripts/extract_blog_code.py -size_cap_truncated: false -files: -- local_path: 01-chunk-parallel-prefill-reference-pytorch.py - role: extracted-block - mode: extracted - upstream_path: inline-in-blog-markdown - heading_path: '## Key Code > ### Chunk-parallel prefill reference (PyTorch)' - sha256: 709cd74d72c22e764ffb77d85bbecd0b1c47b52f2cc0c48bab279954d0f24a22 -- local_path: 02-triton-decode-step-kernel-streaming.py - role: extracted-block - mode: extracted - upstream_path: inline-in-blog-markdown - heading_path: '## Key Code > ### Triton decode-step kernel (streaming)' - sha256: 09a1bc93abdf8165a3d7809e485ef17a3d1a980ae579e36ba81b47665ba35b22 diff --git a/artifacts/kernels/flash-attention-4/full/PROVENANCE.yaml b/artifacts/kernels/flash-attention-4/full/PROVENANCE.yaml index c7e7d7be5..429b2680f 100644 --- a/artifacts/kernels/flash-attention-4/full/PROVENANCE.yaml +++ b/artifacts/kernels/flash-attention-4/full/PROVENANCE.yaml @@ -6,6 +6,10 @@ retrieved_at: '2026-04-17' asset_mode: verbatim size_cap_truncated: false generated_by: Phase 3 Round 2 task11 anchor bundle +description: >- + Adjacent NVIDIA CUTLASS SM100 FMHA backward MLA reference. This bundle is + not the Dao-AILab FlashAttention-4 implementation; it is retained only as a + separately pinned comparison implementation. source_pr_id: pr-cutlass-2466 files: - local_path: sm100_fmha_bwd_mla_kernel_tma_warpspecialized.hpp diff --git a/artifacts/kernels/flash-attention-4/variants/01-software-exp-skeleton.cu b/artifacts/kernels/flash-attention-4/variants/01-software-exp-skeleton.cu index 5fac8e365..4faf84947 100644 --- a/artifacts/kernels/flash-attention-4/variants/01-software-exp-skeleton.cu +++ b/artifacts/kernels/flash-attention-4/variants/01-software-exp-skeleton.cu @@ -1,23 +1,13 @@ -// provenance: derived from pr-cutlass-2466, technique-software-exp, kernel-flash-attention-4; not upstream code -// origin: wiki/kernels/flash-attention-4.md Phase 3 variant (software-emulated exponential skeleton) +// KernelWiki-derived scalar reference for the range reduction and rounded +// degree-3 polynomial printed in Tri Dao's FlashAttention-4 blog. +// This is not verbatim FA4 source. The real kernel applies software evaluation +// to only a selected fraction of entries and includes range handling/scheduling. -// FlashAttention-4 software exp trick: exp2f(x * log2(e)) is emitted by -// the compiler as a fused ex2.approx.f32 instruction on SM100, which runs -// on the MUFU (Multi-Function Unit) path and overlaps with the MMA path. -// This avoids the Tensor Core saturation bottleneck that FA-2/FA-3 hit. - -#include #include -__device__ inline float fa4_rescale_exp(float x, float max_old, float max_new) { - // Rescale accumulator when row maximum updates (online softmax). - // exp2f(diff * log2(e)) fuses to ex2.approx.f32 on SM100. - constexpr float LOG2E = 1.44269504088896340736f; - return __expf((max_old - max_new) * 1.0f) * x; - // Equivalent fused form: - // return exp2f((max_old - max_new) * LOG2E) * x; +__host__ __device__ inline float fa4_blog_exp2_reference(float x) { + const int n = static_cast(floorf(x)); + const float f = x - static_cast(n); // f in [0, 1) + const float p = 1.0f + f * (0.6951f + f * (0.2276f + f * 0.0771f)); + return ldexpf(p, n); } - -// See full/ for the upstream FA4-MLA kernel. The ex2-based rescale lives -// inside that kernel's inner softmax loop (grep for 'exp2|ex2' in the -// mainloop collective file). diff --git a/artifacts/kernels/flash-attention-4/variants/PROVENANCE.yaml b/artifacts/kernels/flash-attention-4/variants/PROVENANCE.yaml index 28dbe7621..c92059979 100644 --- a/artifacts/kernels/flash-attention-4/variants/PROVENANCE.yaml +++ b/artifacts/kernels/flash-attention-4/variants/PROVENANCE.yaml @@ -1,16 +1,18 @@ origin_url: derived upstream_repo: derived license: MIT-style-inherited-from-kernelwiki-tooling -retrieved_at: '2026-04-17' +retrieved_at: '2026-08-08' asset_mode: derived derived_from: - - pr-cutlass-2466 - - technique-software-exp + - blog-flash-attention-4 - kernel-flash-attention-4 size_cap_truncated: false -generated_by: Phase 3 Round 2 task11 anchor bundle +generated_by: KernelWiki verifier remediation +description: >- + Scalar teaching reference for the range reduction and rounded polynomial in + the first-party FA4 blog; not upstream FA4 kernel code. files: - local_path: 01-software-exp-skeleton.cu role: derived-source mode: derived - sha256: 91a822a1dedc26daaedfa9432132a30fc2e1b40c0916072a5de0cccf1b4b1074 + sha256: 15df5285418d175ba4bbbfae5a705d49f77b1c7b9fbdfb0542cd0879d1c1dbac diff --git a/artifacts/kernels/flashmla/full/PROVENANCE.yaml b/artifacts/kernels/flashmla/full/PROVENANCE.yaml index d50dc2ab0..779c32a99 100644 --- a/artifacts/kernels/flashmla/full/PROVENANCE.yaml +++ b/artifacts/kernels/flashmla/full/PROVENANCE.yaml @@ -7,11 +7,16 @@ asset_mode: verbatim size_cap_truncated: false generated_by: Phase 3 Round 3 task12 (flashmla anchor) source_pr_id: pr-cutlass-2472 +bundle_scope: adjacent implementations; not deepseek-ai/FlashMLA source +subject_repo: deepseek-ai/FlashMLA files: - local_path: 77_blackwell_mla_fwd.cu sha256: c072ee30b377248e03d6fc0ea56365f0dd6f03874a81f02ff5a2a1686b81023b role: upstream-file mode: verbatim + upstream_repo: NVIDIA/cutlass + upstream_sha: 9baa06dd + origin_url: https://github.com/NVIDIA/cutlass/pull/2472 upstream_path: examples/77_blackwell_fmha/77_blackwell_mla_fwd.cu - local_path: sm100_fmha_mla_tma_warpspecialized.hpp sha256: f35738f40c9092df2ad10c54bd1c51e56165323298713e102995a4cbb29d35ea @@ -19,4 +24,5 @@ files: mode: verbatim upstream_repo: flashinfer-ai/flashinfer upstream_sha: 9a05c92a + origin_url: https://github.com/flashinfer-ai/flashinfer/commit/9a05c92a upstream_path: include/flashinfer/attention/blackwell/kernel/sm100_fmha_mla_tma_warpspecialized.hpp diff --git a/artifacts/kernels/flashmla/variants/01-mla-decode-inner-loop.cu b/artifacts/kernels/flashmla/variants/01-mla-decode-inner-loop.cu deleted file mode 100644 index 345d7df55..000000000 --- a/artifacts/kernels/flashmla/variants/01-mla-decode-inner-loop.cu +++ /dev/null @@ -1,34 +0,0 @@ -// provenance: derived from blog-flashmla, pr-cutlass-2472; not upstream code -// origin: wiki/kernels/flashmla.md Phase 3 variant (copied from extracted blog bundle) - -// Extracted from sources/blogs/flashmla.md by scripts/extract_blog_code.py -// Heading: ## Key Code > ### MLA decode inner loop -// Original fence language: cuda -// See artifacts/blogs/flashmla/code/PROVENANCE.yaml for origin + license metadata. - -// MLA collapses K and V into a shared latent matrix of head-dim Dc=128. -// On decode (one query vector against N KV tokens) we alternate TMA load, -// wgmma/tcgen05 into the q@K^T accumulator, online softmax, and the second -// accumulator against V. -constexpr int Dc = 128; // latent head dim -constexpr int BLOCK_N = 64; // paged KV block size -float acc[Dc] = {0}; -float max_val = -INFINITY; -float l = 0.f; -for (int n0 = 0; n0 < seqlen; n0 += BLOCK_N) { - tma_load(smem_kv, KV_pages + n0); - cp_async_wait(); - float scores[BLOCK_N]; - wgmma_or_tcgen05_mma(scores, q, smem_kv); // q @ K^T - float new_max = warp_reduce_max(scores, BLOCK_N); - float scale = expf(max_val - new_max); - for (int j = 0; j < Dc; j++) acc[j] *= scale; - l *= scale; - for (int j = 0; j < BLOCK_N; j++) { - float p = expf(scores[j] - new_max); - l += p; - for (int d = 0; d < Dc; d++) acc[d] += p * smem_kv[j * Dc + d]; - } - max_val = new_max; -} -for (int d = 0; d < Dc; d++) O[d] = acc[d] / l; diff --git a/artifacts/kernels/flashmla/variants/01-mla-layout-and-index-contract.cu b/artifacts/kernels/flashmla/variants/01-mla-layout-and-index-contract.cu new file mode 100644 index 000000000..7e3ed3d6e --- /dev/null +++ b/artifacts/kernels/flashmla/variants/01-mla-layout-and-index-contract.cu @@ -0,0 +1,32 @@ +// provenance: derived from blog-flashmla at DeepSeek FlashMLA commit 71c7379; +// not upstream code +// Scope: V3-family FP8 sparse-decode byte layout and encoded page-index arithmetic. + +#include +#include + +namespace kernelwiki_flashmla_contract { + +constexpr int kNopeFp8Bytes = 512; +constexpr int kScaleCount = 4; +constexpr int kFp32Bytes = 4; +constexpr int kRopeBf16Values = 64; +constexpr int kBf16Bytes = 2; +constexpr int kV3Fp8SparseBytes = + kNopeFp8Bytes + kScaleCount * kFp32Bytes + kRopeBf16Values * kBf16Bytes; +static_assert(kV3Fp8SparseBytes == 656); + +// FlashMLA sparse decode consumes this physical page/offset encoding. +// The caller must handle -1 as an invalid entry before decoding it. +constexpr int encode_page_index(int physical_page, int offset, int page_size) { + return physical_page * page_size + offset; +} + +constexpr std::pair decode_page_index(int encoded, int page_size) { + return {encoded / page_size, encoded % page_size}; +} + +static_assert(decode_page_index(encode_page_index(7, 13, 64), 64) == + std::pair{7, 13}); + +} // namespace kernelwiki_flashmla_contract diff --git a/artifacts/kernels/flashmla/variants/PROVENANCE.yaml b/artifacts/kernels/flashmla/variants/PROVENANCE.yaml index 0f725acbe..9d251abcb 100644 --- a/artifacts/kernels/flashmla/variants/PROVENANCE.yaml +++ b/artifacts/kernels/flashmla/variants/PROVENANCE.yaml @@ -5,12 +5,11 @@ retrieved_at: '2026-04-17' asset_mode: derived derived_from: - blog-flashmla -- pr-cutlass-2472 -- hw-tmem size_cap_truncated: false -generated_by: Phase 3 Round 3 task12 (flashmla variants) +generated_by: KernelWiki verifier remediation +bundle_scope: verified layout and index contract only; not an attention kernel files: -- local_path: 01-mla-decode-inner-loop.cu +- local_path: 01-mla-layout-and-index-contract.cu role: derived-source mode: derived - sha256: 4f494dc2d282468966a1e2028bc50d0ef0c364b34d24f75ee881608ddef2df69 + sha256: aa886373129daedc09b9dae7a23960b021256647e1002399db0dca82053c6f96 diff --git a/artifacts/kernels/fused-moe/full/PROVENANCE.yaml b/artifacts/kernels/fused-moe/full/PROVENANCE.yaml index e3579ef9e..6dead4b74 100644 --- a/artifacts/kernels/fused-moe/full/PROVENANCE.yaml +++ b/artifacts/kernels/fused-moe/full/PROVENANCE.yaml @@ -1,29 +1,31 @@ -origin_url: https://github.com/vllm-project/vllm/pull/23696 -upstream_repo: vllm-project/vllm -upstream_sha: 074854b2 -license: inherits-from-upstream -retrieved_at: '2026-04-17' -asset_mode: verbatim +origin_url: https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048 +upstream_repo: multiple-adjacent-references +license: mixed-inherits-from-upstreams-and-kernelwiki-tooling +retrieved_at: '2026-08-08' +asset_mode: extracted size_cap_truncated: false -generated_by: Phase 3 Round 3 task12 (fused-moe anchor) +generated_by: KernelWiki factual remediation source_pr_id: pr-vllm-23696 -notes: 'Mixed-source bundle: vllm gated-dual-GEMM patch + sglang flashinfer-cutedsl - MoE runner + DeepGEMM grouped-GEMM extract.' +notes: >- + Mixed adjacent bundle, not a full implementation of the FlashInfer Track A + FP8 benchmark. The vLLM diff is MXFP4 expert compute, the SGLang file is an + FP4 CuteDSL runner, and the C++ file is local illustrative pseudocode. files: - local_path: vllm-PR-23696-dual-gemm.patch sha256: 033d30b15af4f2050189c2e76ba61959e1e5d4bd049dc901fd56f68890b891a0 role: pr-diff mode: upstream-patch + upstream_repo: vllm-project/vllm + upstream_sha: 074854b24f6e0b1e237a004283e1f46d98c0d73c + upstream_path: aggregate merged diff across the five changed paths - local_path: flashinfer_cutedsl.py sha256: de6bd663382b5bf34dba077d90febc9cc80d9dda908d9947966a3cadbc21bf33 role: upstream-file mode: verbatim upstream_repo: sgl-project/sglang - upstream_sha: c554dc5c + upstream_sha: c554dc5c64b661f2c53225b03a76359eaddc39e4 upstream_path: python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py - local_path: moe-grouped-gemm-launch.cpp - sha256: e4d1af249fb7054fa2df64b89e24f63735c9daa2c51d4e1af87e3f27b71b7f97 - role: extracted-block - mode: extracted - upstream_path: sources/blogs/deepgemm.md - heading_path: '## Key Code > ### MoE grouped-GEMM launch' + sha256: 1368d01910bce19816daf9af6963a24de3809a0b12162fce1e5dd9d79064e884 + role: derived-source + mode: derived diff --git a/artifacts/kernels/fused-moe/full/moe-grouped-gemm-launch.cpp b/artifacts/kernels/fused-moe/full/moe-grouped-gemm-launch.cpp index 8a21baa45..4e69db269 100644 --- a/artifacts/kernels/fused-moe/full/moe-grouped-gemm-launch.cpp +++ b/artifacts/kernels/fused-moe/full/moe-grouped-gemm-launch.cpp @@ -1,7 +1,6 @@ -// Extracted from sources/blogs/deepgemm.md by scripts/extract_blog_code.py -// Heading: ## Key Code > ### MoE grouped-GEMM launch -// Original fence language: cpp -// See artifacts/blogs/deepgemm/code/PROVENANCE.yaml for origin + license metadata. +// KernelWiki illustrative pseudocode; not upstream DeepGEMM or contest code. +// It documents only the varying-M segmentation used by a grouped GEMM. +// It is intentionally incomplete and is not a compilable or timed kernel. // Grouped-GEMM packs a variable list of per-expert GEMMs into one kernel // launch via a prefix-sum offset array; layouts are contiguous (M-axis), diff --git a/artifacts/kernels/fused-moe/variants/01-routing-plus-fusion-skeleton.py b/artifacts/kernels/fused-moe/variants/01-routing-plus-fusion-skeleton.py index 204b2a38d..25897aae1 100644 --- a/artifacts/kernels/fused-moe/variants/01-routing-plus-fusion-skeleton.py +++ b/artifacts/kernels/fused-moe/variants/01-routing-plus-fusion-skeleton.py @@ -1,31 +1,215 @@ -# provenance: derived from pr-sglang-21339, pr-vllm-23696, blog-deepgemm; not upstream code -# origin: wiki/kernels/fused-moe.md Phase 3 variant - -# Fused MoE skeleton: router logits -> top-k expert selection -> grouped -# GEMM dispatch with contiguous M-axis packing. - -import torch - -def fused_moe_forward(hidden, router_weights, expert_weights_gate, - expert_weights_up, expert_weights_down, top_k=2): - """Reference PyTorch implementation of the dispatch + fused-dual-GEMM path. - Production kernels (vllm+sglang+DeepGEMM) fuse all three stages.""" - B, L, D = hidden.shape - logits = hidden @ router_weights.T # [B, L, E] - top_vals, top_idx = logits.topk(top_k, dim=-1) - weights = torch.softmax(top_vals, dim=-1) # renormalize - # Scatter tokens to their selected experts (grouped M-axis) - E = expert_weights_gate.shape[0] - out = torch.zeros_like(hidden) - for e in range(E): - mask = (top_idx == e).any(dim=-1) # [B, L] - if not mask.any(): - continue - x_e = hidden[mask] # [Ne, D] - gate = x_e @ expert_weights_gate[e].T - up = x_e @ expert_weights_up[e].T - y_e = (torch.nn.functional.silu(gate) * up) @ expert_weights_down[e].T - # Scatter back, weighted by the routing weight for this expert - w_e = weights[mask * (top_idx == e).any(dim=-1)[..., None]].sum(dim=-1) - out[mask] = out[mask] + w_e[:, None] * y_e - return out +# provenance: derived from contest-flashinfer-track-a; not upstream code +# Scope: CPU-checkable semantics only; not an optimized or fused GPU kernel. + +from __future__ import annotations + +import math +import random + + +def _sigmoid(value: float) -> float: + if value >= 0: + return 1.0 / (1.0 + math.exp(-value)) + exp_value = math.exp(value) + return exp_value / (1.0 + exp_value) + + +def deepseek_grouped_topk( + routing_logits: list[list[float]], + routing_bias: list[float], + *, + top_k: int, + n_group: int, + topk_group: int, + routed_scaling_factor: float, +) -> tuple[list[list[int]], list[list[float]]]: + """Return global expert IDs and combine weights for DeepSeek-V3 routing. + + Bias affects expert selection but not the normalized combine weights. + """ + if not routing_logits or not routing_logits[0]: + raise ValueError("routing_logits must have shape [tokens, experts]") + experts = len(routing_logits[0]) + if any(len(row) != experts for row in routing_logits): + raise ValueError("routing_logits rows must have equal length") + if len(routing_bias) != experts: + raise ValueError("routing_bias must have shape [experts]") + if experts % n_group: + raise ValueError("experts must be divisible by n_group") + group_size = experts // n_group + if not (1 <= topk_group <= n_group and 1 <= top_k <= topk_group * group_size): + raise ValueError("invalid grouped top-k parameters") + if group_size < 2: + raise ValueError("DeepSeek group scoring requires at least two experts per group") + + all_ids: list[list[int]] = [] + all_weights: list[list[float]] = [] + for row in routing_logits: + sigmoid_scores = [_sigmoid(value) for value in row] + selection_scores = [ + score + bias for score, bias in zip(sigmoid_scores, routing_bias, strict=True) + ] + group_scores = [] + for group in range(n_group): + begin = group * group_size + two_largest = sorted( + selection_scores[begin : begin + group_size], reverse=True + )[:2] + group_scores.append((sum(two_largest), group)) + kept_groups = { + group + for _, group in sorted(group_scores, reverse=True)[:topk_group] + } + candidates = [ + expert + for expert in range(experts) + if expert // group_size in kept_groups + ] + chosen = sorted( + candidates, key=lambda expert: selection_scores[expert], reverse=True + )[:top_k] + raw_weights = [sigmoid_scores[expert] for expert in chosen] + denominator = sum(raw_weights) + all_ids.append(chosen) + all_weights.append( + [ + weight / denominator * routed_scaling_factor + for weight in raw_weights + ] + ) + return all_ids, all_weights + + +def _matvec(matrix: list[list[float]], vector: list[float]) -> list[float]: + if any(len(row) != len(vector) for row in matrix): + raise ValueError("matrix/vector dimensions do not match") + return [sum(a * b for a, b in zip(row, vector, strict=True)) for row in matrix] + + +def local_expert_reference( + hidden_states: list[list[float]], + topk_ids: list[list[int]], + combine_weights: list[list[float]], + w13: list[list[list[float]]], + w2: list[list[list[float]]], + *, + local_expert_offset: int, +) -> list[list[float]]: + """Compute local W13/SwiGLU/W2 contributions and weighted accumulation. + + Logical shapes are hidden_states[T,H], w13[E_local,2I,H], and + w2[E_local,H,I]. + """ + if not hidden_states or not hidden_states[0] or len(w13) != len(w2): + raise ValueError("empty or mismatched input") + tokens, hidden = len(hidden_states), len(hidden_states[0]) + if len(topk_ids) != tokens or len(combine_weights) != tokens: + raise ValueError("top-k rows must match tokens") + if any(len(ids) != len(weights) for ids, weights in zip(topk_ids, combine_weights, strict=True)): + raise ValueError("top-k IDs and weights must have equal row lengths") + output = [[0.0] * hidden for _ in range(tokens)] + + for local_id, (expert_w13, expert_w2) in enumerate(zip(w13, w2, strict=True)): + if not expert_w13 or len(expert_w13) % 2: + raise ValueError("w13 rows must equal 2 * intermediate") + intermediate = len(expert_w13) // 2 + if len(expert_w2) != hidden or any(len(row) != intermediate for row in expert_w2): + raise ValueError("w2 must have shape [hidden, intermediate]") + global_id = local_expert_offset + local_id + for token, hidden_row in enumerate(hidden_states): + matching_weight = sum( + weight + for expert, weight in zip( + topk_ids[token], combine_weights[token], strict=True + ) + if expert == global_id + ) + if matching_weight == 0.0: + continue + projected = _matvec(expert_w13, hidden_row) + up, gate = projected[:intermediate], projected[intermediate:] + activated = [ + _sigmoid(gate_value) * gate_value * up_value + for gate_value, up_value in zip(gate, up, strict=True) + ] + expert_output = _matvec(expert_w2, activated) + for column, value in enumerate(expert_output): + output[token][column] += matching_weight * value + return output + + +def fused_moe_reference( + hidden_states: list[list[float]], + routing_logits: list[list[float]], + routing_bias: list[float], + w13: list[list[list[float]]], + w2: list[list[list[float]]], + *, + local_expert_offset: int, + top_k: int, + n_group: int, + topk_group: int, + routed_scaling_factor: float, +) -> list[list[float]]: + topk_ids, combine_weights = deepseek_grouped_topk( + routing_logits, + routing_bias, + top_k=top_k, + n_group=n_group, + topk_group=topk_group, + routed_scaling_factor=routed_scaling_factor, + ) + return local_expert_reference( + hidden_states, + topk_ids, + combine_weights, + w13, + w2, + local_expert_offset=local_expert_offset, + ) + + +def _self_test() -> None: + logits = [ + [2.0, 1.0, -1.0, -2.0], + [-1.5, -0.5, 0.5, 1.5], + [0.1, 0.3, 0.2, 0.0], + ] + bias = [0.0, 0.2, 0.4, -0.1] + ids, weights = deepseek_grouped_topk( + logits, + bias, + top_k=2, + n_group=2, + topk_group=1, + routed_scaling_factor=2.5, + ) + assert ids == [[1, 0], [2, 3], [2, 3]] + assert all(math.isclose(sum(row), 2.5, rel_tol=1e-12) for row in weights) + + # Negative control: biased selection scores must not become combine weights. + first_biased = [_sigmoid(value) + delta for value, delta in zip(logits[0], bias, strict=True)] + wrong_denominator = sum(first_biased[expert] for expert in ids[0]) + wrong = [first_biased[expert] / wrong_denominator * 2.5 for expert in ids[0]] + assert any(not math.isclose(a, b) for a, b in zip(weights[0], wrong, strict=True)) + + rng = random.Random(7) + hidden = [[rng.uniform(-1, 1) for _ in range(4)] for _ in range(3)] + w13 = [ + [[rng.uniform(-1, 1) for _ in range(4)] for _ in range(6)] + for _ in range(4) + ] + w2 = [ + [[rng.uniform(-1, 1) for _ in range(3)] for _ in range(4)] + for _ in range(4) + ] + output = local_expert_reference( + hidden, ids, weights, w13, w2, local_expert_offset=0 + ) + assert len(output) == 3 and all(len(row) == 4 for row in output) + assert all(math.isfinite(value) for row in output for value in row) + + +if __name__ == "__main__": + _self_test() + print("fused-moe derived reference: PASS") diff --git a/artifacts/kernels/fused-moe/variants/PROVENANCE.yaml b/artifacts/kernels/fused-moe/variants/PROVENANCE.yaml index eb45a5bea..285ef1367 100644 --- a/artifacts/kernels/fused-moe/variants/PROVENANCE.yaml +++ b/artifacts/kernels/fused-moe/variants/PROVENANCE.yaml @@ -1,17 +1,18 @@ -origin_url: derived +origin_url: https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048 upstream_repo: derived license: MIT-style-inherited-from-kernelwiki-tooling -retrieved_at: '2026-04-17' +retrieved_at: '2026-08-08' asset_mode: derived derived_from: -- pr-sglang-21339 -- pr-vllm-23696 -- blog-deepgemm -- kernel-fused-moe +- contest-flashinfer-track-a size_cap_truncated: false -generated_by: Phase 3 Round 3 task12 (fused-moe variants) +generated_by: KernelWiki factual remediation +notes: >- + Parameterized standard-library reference for the definition's grouped + DeepSeek routing and local W13/SwiGLU/W2 accumulation. It neither dequantizes + FP8 tensors nor models GPU launches, layouts, or performance. files: - local_path: 01-routing-plus-fusion-skeleton.py role: derived-source mode: derived - sha256: 2cc640b1baa1a6cd42718616ffe2cf78a9b6d4eb3317464f059c6fd442439342 + sha256: 6e4e5889347d9783585d5bef8764e55cdc07c4840cbb56ed337db37911586aa1 diff --git a/artifacts/kernels/gated-delta-net/full/PROVENANCE.yaml b/artifacts/kernels/gated-delta-net/full/PROVENANCE.yaml index 98a4ac3e8..6ff073765 100644 --- a/artifacts/kernels/gated-delta-net/full/PROVENANCE.yaml +++ b/artifacts/kernels/gated-delta-net/full/PROVENANCE.yaml @@ -1,12 +1,15 @@ origin_url: https://github.com/sgl-project/sglang/pull/21019 upstream_repo: sgl-project/sglang -upstream_sha: 5bdc07d9 +upstream_sha: 5bdc07d974f6cf236fa765a685453ea5e587a838 license: inherits-from-upstream retrieved_at: '2026-04-17' asset_mode: verbatim size_cap_truncated: false generated_by: Phase 3 Round 3 task12 (gated-delta-net anchor) source_pr_id: pr-sglang-21019 +bundle_scope: >- + Adjacent fused split/reshape/concatenate preprocessing for Qwen3-Next and + Qwen3.5 projection outputs; not a GDN recurrence, prefill, or decode kernel. files: - local_path: gdn_fused_proj.py sha256: c4ef85167eb11065721e76dd5d63d6f346f309f84822578dd4a3874249acef89 diff --git a/artifacts/kernels/gated-delta-net/variants/01-chunk-parallel-prefill-reference-pytorch.py b/artifacts/kernels/gated-delta-net/variants/01-chunk-parallel-prefill-reference-pytorch.py deleted file mode 100644 index 9957f3a5b..000000000 --- a/artifacts/kernels/gated-delta-net/variants/01-chunk-parallel-prefill-reference-pytorch.py +++ /dev/null @@ -1,32 +0,0 @@ -# provenance: derived from blog-gated-delta-net, pr-sglang-21019; not upstream code -# origin: wiki/kernels/gated-delta-net.md Phase 3 variant (copied from extracted blog bundle) - -# Extracted from sources/blogs/gated-delta-net.md by scripts/extract_blog_code.py -# Heading: ## Key Code > ### Chunk-parallel prefill reference (PyTorch) -# Original fence language: python -# See artifacts/blogs/gated-delta-net/code/PROVENANCE.yaml for origin + license metadata. - -import torch - -def gated_delta_net_prefill(q, k, v, gate, initial_state, CHUNK_SIZE=64): - """ - Chunk-parallel prefill. Each chunk's state matrix is reused across its - query window, so we pay the O(Dk*Dv) state update once per chunk, not - per token. - q, k: [B, L, Dk] v: [B, L, Dv] gate: [B, L] - """ - B, L, Dk = q.shape - Dv = v.shape[-1] - out = torch.empty(B, L, Dv, device=q.device, dtype=q.dtype) - state = initial_state.clone() # [B, Dk, Dv] - for ci in range(0, L, CHUNK_SIZE): - ce = min(ci + CHUNK_SIZE, L) - k_chunk = k[:, ci:ce] - v_chunk = v[:, ci:ce] - g_chunk = gate[:, ci:ce] - decay = torch.cumprod(g_chunk, dim=1) # adaptive memory decay - for t in range(ce - ci): - state = state * decay[:, t:t+1, None] - state = state + k_chunk[:, t, :, None] * v_chunk[:, t, None, :] - out[:, ci + t] = (q[:, ci + t, :, None] * state).sum(dim=1) - return out, state diff --git a/artifacts/kernels/gated-delta-net/variants/01-gdn-recurrence-reference.py b/artifacts/kernels/gated-delta-net/variants/01-gdn-recurrence-reference.py new file mode 100644 index 000000000..1992a6ea2 --- /dev/null +++ b/artifacts/kernels/gated-delta-net/variants/01-gdn-recurrence-reference.py @@ -0,0 +1,101 @@ +"""One-head Gated DeltaNet recurrence; derived, not a tuned GPU kernel. + +Reference semantics: FlashInfer commit +7f614b86470180bab2d22e36fd1775791c6bf3e6, trace/templates/gdn.py. +""" + +from __future__ import annotations + +from math import isclose + + +Matrix = list[list[float]] + + +def dot(left: list[float], right: list[float]) -> float: + assert len(left) == len(right) and left + return sum(a * b for a, b in zip(left, right)) + + +def outer(left: list[float], right: list[float]) -> Matrix: + return [[a * b for b in right] for a in left] + + +def add(left: Matrix, right: Matrix) -> Matrix: + assert len(left) == len(right) and len(left[0]) == len(right[0]) + return [[a + b for a, b in zip(x, y)] for x, y in zip(left, right)] + + +def scale_matrix(factor: float, matrix: Matrix) -> Matrix: + return [[factor * value for value in row] for row in matrix] + + +def vector_matrix(vector: list[float], matrix: Matrix) -> list[float]: + assert len(vector) == len(matrix) and matrix + return [ + sum(vector[row] * matrix[row][column] for row in range(len(matrix))) + for column in range(len(matrix[0])) + ] + + +def gdn_step( + state: Matrix, + q: list[float], + k: list[float], + v: list[float], + decay: float, + beta: float, + scale: float, +) -> tuple[list[float], Matrix]: + """Apply the compact gated-delta update to state shaped [K,V].""" + assert 0.0 <= decay <= 1.0 and 0.0 <= beta <= 1.0 + assert len(state) == len(q) == len(k) and len(state[0]) == len(v) + decayed = scale_matrix(decay, state) + retrieved = vector_matrix(k, decayed) + correction = [beta * (target - old) for target, old in zip(v, retrieved)] + new_state = add(decayed, outer(k, correction)) + output = [scale * value for value in vector_matrix(q, new_state)] + return output, new_state + + +def _expanded_reference( + state: Matrix, k: list[float], v: list[float], decay: float, beta: float +) -> Matrix: + decayed = scale_matrix(decay, state) + old_v = vector_matrix(k, decayed) + new_v = [beta * target + (1.0 - beta) * old for target, old in zip(v, old_v)] + removed = scale_matrix(-1.0, outer(k, old_v)) + return add(add(decayed, removed), outer(k, new_v)) + + +def _close(left: Matrix, right: Matrix) -> bool: + return all( + isclose(a, b, rel_tol=1e-12, abs_tol=1e-12) + for x, y in zip(left, right) + for a, b in zip(x, y) + ) + + +def _self_test() -> None: + state = [[0.2, -0.3], [0.4, 0.1]] + q = [0.6, -0.8] + k = [0.8, 0.6] + v = [0.5, -0.25] + decay, beta, output_scale = 0.75, 0.4, 0.5 + + output, compact = gdn_step(state, q, k, v, decay, beta, output_scale) + expanded = _expanded_reference(state, k, v, decay, beta) + assert _close(compact, expanded) + assert all(isclose(a, b, rel_tol=1e-12, abs_tol=1e-12) for a, b in zip( + output, [output_scale * x for x in vector_matrix(q, expanded)] + )) + + additive_only = add(scale_matrix(decay, state), outer(k, v)) + assert not _close(compact, additive_only) + old_output = [output_scale * x for x in vector_matrix(q, scale_matrix(decay, state))] + assert output != old_output + print("gated-delta recurrence reference: PASS") + + +if __name__ == "__main__": + _self_test() diff --git a/artifacts/kernels/gated-delta-net/variants/02-triton-decode-step-kernel-streaming.py b/artifacts/kernels/gated-delta-net/variants/02-triton-decode-step-kernel-streaming.py deleted file mode 100644 index 4e85fae5e..000000000 --- a/artifacts/kernels/gated-delta-net/variants/02-triton-decode-step-kernel-streaming.py +++ /dev/null @@ -1,36 +0,0 @@ -# provenance: derived from blog-gated-delta-net, pr-sglang-21019; not upstream code -# origin: wiki/kernels/gated-delta-net.md Phase 3 variant (copied from extracted blog bundle) - -# Extracted from sources/blogs/gated-delta-net.md by scripts/extract_blog_code.py -# Heading: ## Key Code > ### Triton decode-step kernel (streaming) -# Original fence language: python -# See artifacts/blogs/gated-delta-net/code/PROVENANCE.yaml for origin + license metadata. - -import triton -import triton.language as tl - -@triton.jit -def gdn_decode_step_kernel( - Q, K, V, GATE, STATE, OUT, - stride_qb, stride_kb, stride_vb, - Dk: tl.constexpr, Dv: tl.constexpr): - """ - One-token delta-rule update for decode. STATE is a [Dk, Dv] matrix kept - per sample; we fold in the new (k,v) pair after applying the decay gate. - """ - b = tl.program_id(0) - dk = tl.arange(0, Dk) - dv = tl.arange(0, Dv) - - q = tl.load(Q + b * stride_qb + dk) # [Dk] - k = tl.load(K + b * stride_kb + dk) # [Dk] - v = tl.load(V + b * stride_vb + dv) # [Dv] - g = tl.load(GATE + b) # scalar decay - - state = tl.load(STATE + b * Dk * Dv + dk[:, None] * Dv + dv[None, :]) - state = state * g # apply decay - state = state + k[:, None] * v[None, :] # delta update - tl.store(STATE + b * Dk * Dv + dk[:, None] * Dv + dv[None, :], state) - - out = tl.sum(q[:, None] * state, axis=0) # [Dv] - tl.store(OUT + b * Dv + dv, out) diff --git a/artifacts/kernels/gated-delta-net/variants/PROVENANCE.yaml b/artifacts/kernels/gated-delta-net/variants/PROVENANCE.yaml index deb860eb7..78dd439f8 100644 --- a/artifacts/kernels/gated-delta-net/variants/PROVENANCE.yaml +++ b/artifacts/kernels/gated-delta-net/variants/PROVENANCE.yaml @@ -1,20 +1,19 @@ -origin_url: derived +origin_url: https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/trace/templates/gdn.py upstream_repo: derived license: MIT-style-inherited-from-kernelwiki-tooling -retrieved_at: '2026-04-17' +retrieved_at: '2026-08-08' asset_mode: derived derived_from: +- contest-flashinfer-track-c - blog-gated-delta-net -- pr-sglang-21019 -- kernel-gated-delta-net size_cap_truncated: false -generated_by: Phase 3 Round 3 task12 (gated-delta-net variants) +generated_by: KernelWiki factual remediation +notes: >- + Standard-library, one-head semantic recurrence check. It does not model + grouped-value head replication, GPU layouts, numeric precision, launches, + or performance and is not an upstream or tuned GPU kernel. files: -- local_path: 01-chunk-parallel-prefill-reference-pytorch.py +- local_path: 01-gdn-recurrence-reference.py role: derived-source mode: derived - sha256: d7e74b02df4974fb082cacbae8a5182dd2c40cb1c13fd1d74505b51cfb15481b -- local_path: 02-triton-decode-step-kernel-streaming.py - role: derived-source - mode: derived - sha256: d5eef14bdbc44e50784c0ca34e7041dcad2151960a85feddba62b097e20079ad + sha256: 11560c14d805f79dbc1025801beed373c0e17e712e054c6f85231071b8d8b971 diff --git a/artifacts/kernels/gated-dual-gemm/full/PROVENANCE.yaml b/artifacts/kernels/gated-dual-gemm/full/PROVENANCE.yaml index 6237c880b..3f0379e81 100644 --- a/artifacts/kernels/gated-dual-gemm/full/PROVENANCE.yaml +++ b/artifacts/kernels/gated-dual-gemm/full/PROVENANCE.yaml @@ -1,22 +1,15 @@ -origin_url: https://github.com/vllm-project/vllm/pull/23696 -upstream_repo: vllm-project/vllm -upstream_sha: 074854b2 -license: inherits-from-upstream -retrieved_at: '2026-04-17' -asset_mode: extracted +origin_url: https://github.com/gpu-mode/reference-kernels/tree/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm +upstream_repo: gpu-mode/reference-kernels +upstream_sha: c5b2f7c062d5015f29c3a1043cfd04954397944c +license: MIT +retrieved_at: '2026-08-08' +asset_mode: verbatim size_cap_truncated: false -generated_by: Phase 3 Round 3 task12 (gated-dual-gemm anchor) -source_pr_id: pr-vllm-23696 +generated_by: KernelWiki verifier remediation +source_id: contest-gpumode-p3 files: -- local_path: vllm-PR-23696-gated-dual-gemm.patch - sha256: 033d30b15af4f2050189c2e76ba61959e1e5d4bd049dc901fd56f68890b891a0 - role: pr-diff - mode: upstream-patch -- local_path: blackwell-cutlass-schedules-and-tma.cu - sha256: e720037bbdb5ee5054259a331cdaf489e7135a44038d66d1fd9d9e5998b75255 - role: extracted-block - mode: extracted - upstream_path: sources/blogs/tflops-gap-fp4-moe.md - heading_path: '# TFLOPS Gap: Why FP4 MoE Kernel Engineering Matters on Blackwell - > ## Three Key Optimization Techniques > ### 2. Blackwell-Specific CUTLASS Schedules - and TMA' +- local_path: task.yml + sha256: 2bf4e5c04cc8a9c283e519437c8324ca9265d9c12d0dfe2b76dc9f0542ada051 + role: upstream-file + mode: verbatim + upstream_path: problems/nvidia/nvfp4_dual_gemm/task.yml diff --git a/artifacts/kernels/gated-dual-gemm/full/blackwell-cutlass-schedules-and-tma.cu b/artifacts/kernels/gated-dual-gemm/full/blackwell-cutlass-schedules-and-tma.cu deleted file mode 100644 index 4e9d7d9dc..000000000 --- a/artifacts/kernels/gated-dual-gemm/full/blackwell-cutlass-schedules-and-tma.cu +++ /dev/null @@ -1,24 +0,0 @@ -// Extracted from sources/blogs/tflops-gap-fp4-moe.md by scripts/extract_blog_code.py -// Heading: # TFLOPS Gap: Why FP4 MoE Kernel Engineering Matters on Blackwell > ## Three Key Optimization Techniques > ### 2. Blackwell-Specific CUTLASS Schedules and TMA -// Original fence languages: and cuda -// See artifacts/blogs/tflops-gap-fp4-moe/code/PROVENANCE.yaml for origin + license metadata. - -// --- Blackwell-optimized CUTLASS schedule (from the unlabeled config fence) --- -// SGLang selects this kernel schedule + thread-block shape for Blackwell NvFP4 MoE: -// -// KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100 -// ThreadBlockShape = Shape<_128, _128, _128> -// -// Key features of this schedule: -// * Warp Specialization for FP4: dedicated warp roles for loading FP4 data, -// dequantizing to FP16/BF16, and accumulating in FP32. -// * TMA Integration: asynchronous bulk tensor loads bypassing L1 cache, -// feeding directly into shared memory with strict 128-byte alignment. -// * 1 SM Grouping: multiple experts processed per SM rather than -// one-expert-per-SM, better for variable expert sizes. -// * Native NvFP4 Support: hardware FP4 instructions instead of software -// emulation. - -// --- TMA alignment padding (from the cuda-labeled fence) --- -// TMA alignment is enforced by padding blockscale offsets to 128-byte boundaries: -blockscale_offsets[expert_id + 1] = (expert_offsets[expert_id + 1] + 127) / 128 * 128; diff --git a/artifacts/kernels/gated-dual-gemm/full/task.yml b/artifacts/kernels/gated-dual-gemm/full/task.yml new file mode 100644 index 000000000..4d36bde62 --- /dev/null +++ b/artifacts/kernels/gated-dual-gemm/full/task.yml @@ -0,0 +1,64 @@ +# name: nvfp4-dual-gemm + +files: + - {"name": "submission.py", "source": "@SUBMISSION@"} + - {"name": "task.py", "source": "task.py"} + - {"name": "utils.py", "source": "../utils.py"} + - {"name": "reference.py", "source": "reference.py"} + - {"name": "eval.py", "source": "../eval_better_bench.py"} + +lang: "py" + +description: | + + You will implement a block scaled dual matrix-matrix multiplication kernel with silu activation optimized for NVIDIA B200. + To be explicit, you will be given a tuple of tensors: + ``` + (a, b1, b2, sfa, sfb1, sfb2, c) + ``` + where: + * `a` is M x K x L in K-major order in nvfp4(e2m1) + * `b1` is N x K x L in K-major order in nvfp4(e2m1) + * `b2` is N x K x L in K-major order in nvfp4(e2m1) + * `sfa` is M x (K // 16) x L in K-major order in fp8(e4m3fnuz) + * `sfb1` is N x (K // 16) x L in K-major order in fp8(e4m3fnuz) + * `sfb2` is N x (K // 16) x L in K-major order in fp8(e4m3fnuz) + * `c` is M x N x L in fp16 + + Matrix sizes `M` is divisible by mma_tiler_mn[0], `N` is divisible by mma_tiler_mn[1], `K` is divisible by 256. + The ranking criteria is the geometric mean of the benchmark results. + For the grand price, your kernel will be evaluated against the speed of light analysis + and the solution closest to the speed of light will be awarded the grand price. + ``` + The speed of light analysis based on the max(FP4 Tensor Core math throughput, DRAM memory throughput) of B200 and tested under 1.5Ghz clock: + M N K L time[us] + 256 4096 7168 1 4.708 + 512 4096 7168 1 8.714 + 256 3072 4096 1 2.125 + 512 3072 7168 1 6.535 + ``` +config: + main: "eval.py" + +templates: + Python: "template.py" + +tests: + - {"m": 1536, "n": 512, "k": 7168, "l": 1, "seed": 1111} + - {"m": 256, "n": 512, "k": 256, "l": 1, "seed": 1111} + - {"m": 1536, "n": 512, "k": 7168, "l": 1, "seed": 1111} + - {"m": 3072, "n": 1024, "k": 1536, "l": 1, "seed": 1111} + - {"m": 7168, "n": 1024, "k": 256, "l": 1, "seed": 1111} + - {"m": 7168, "n": 2304, "k": 2048, "l": 1, "seed": 1111} + - {"m": 4608, "n": 384, "k": 7168, "l": 1, "seed": 1111} + - {"m": 7168, "n": 384, "k": 2304, "l": 1, "seed": 1111} + - {"m": 512, "n": 768, "k": 7168, "l": 1, "seed": 1111} + - {"m": 4096, "n": 768, "k": 512, "l": 1, "seed": 1111} + +benchmarks: + - {"m": 256, "n": 4096, "k": 7168, "l": 1, "seed": 1111} + - {"m": 512, "n": 4096, "k": 7168, "l": 1, "seed": 1111} + - {"m": 256, "n": 3072, "k": 4096, "l": 1, "seed": 1111} + - {"m": 512, "n": 3072, "k": 7168, "l": 1, "seed": 1111} + +ranking_by: "geom" diff --git a/artifacts/kernels/gated-dual-gemm/full/vllm-PR-23696-gated-dual-gemm.patch b/artifacts/kernels/gated-dual-gemm/full/vllm-PR-23696-gated-dual-gemm.patch deleted file mode 100644 index 33114063f..000000000 --- a/artifacts/kernels/gated-dual-gemm/full/vllm-PR-23696-gated-dual-gemm.patch +++ /dev/null @@ -1,872 +0,0 @@ -diff --git a/tests/kernels/moe/test_mxfp4_moe.py b/tests/kernels/moe/test_mxfp4_moe.py -index 882b034e2f23..9fd72ee152b5 100644 ---- a/tests/kernels/moe/test_mxfp4_moe.py -+++ b/tests/kernels/moe/test_mxfp4_moe.py -@@ -11,6 +11,7 @@ - from packaging import version - - from vllm.platforms import current_platform -+from vllm.utils.flashinfer import has_flashinfer - - QUARK_MXFP4_AVAILABLE = importlib.util.find_spec( - "quark") is not None and version.parse( -@@ -19,6 +20,10 @@ - TRTLLM_GEN_MXFP4_AVAILABLE = current_platform.is_cuda( - ) and current_platform.is_device_capability(100) - -+HOPPER_MXFP4_BF16_AVAILABLE = (current_platform.is_cuda() -+ and current_platform.is_device_capability(90) -+ and has_flashinfer()) -+ - if TRTLLM_GEN_MXFP4_AVAILABLE: - from flashinfer import (fp4_quantize, mxfp8_quantize, - next_positive_power_of_2, -@@ -542,3 +547,317 @@ def test_trtllm_gen_mxfp4_fused_moe( - transpose_optimized=transpose_optimized) - # relatively loose check since the mxfp4 quantization is less accurate - check_accuracy(ref_result, tg_result, atol=0, rtol=0.3, percent=0.8) -+ -+ -+def _interleave_scales_lastdim_by4(scales: torch.Tensor) -> torch.Tensor: -+ """Interleave scales on the last dimension by groups of 4, matching -+ the transformation in mxfp4.py's BF16 (Hopper) path.""" -+ s = scales.to(torch.uint8) -+ s_shape = s.shape -+ assert s_shape[-1] % 4 == 0 -+ s = s.reshape(*s_shape[:-1], s_shape[-1] // 4, 4) -+ # Move the 4-group dimension before the row dimension -+ permuted = s.permute(0, 2, 1, 3) -+ # Merge the row dim with the 4-group dim -+ return permuted.reshape(s_shape[0], s_shape[-1] // 4, s_shape[1] * 4) -+ -+ -+@pytest.mark.parametrize("topk", [1, 4]) -+@pytest.mark.parametrize("num_experts", [32]) -+@pytest.mark.parametrize("num_tokens", [1, 128]) -+@pytest.mark.parametrize("intermediate_size,hidden_size", [(3072, 3072)]) -+@pytest.mark.parametrize("alpha,beta,limit", [(1.0, 1.0, None), -+ (1.702, 1.0, 7.0)]) -+@pytest.mark.skipif( -+ not HOPPER_MXFP4_BF16_AVAILABLE, -+ reason="nvidia gpu sm90 and flashinfer are required for this test", -+) -+def test_flashinfer_cutlass_mxfp4_fused_moe( -+ topk: int, -+ num_experts: int, -+ num_tokens: int, -+ intermediate_size: int, -+ hidden_size: int, -+ alpha: float, -+ beta: float, -+ limit: Optional[float], -+): -+ torch.manual_seed(42) -+ device = "cuda:0" -+ -+ # Inputs -+ hidden_states = torch.randn(num_tokens, -+ hidden_size, -+ device=device, -+ dtype=torch.bfloat16) -+ # Random MXFP4 weights and scales (uint8), contiguous [w1; w3] -+ w13_q = torch.randint( -+ 0, -+ 256, (num_experts, 2 * intermediate_size, hidden_size // 2), -+ device=device, -+ dtype=torch.uint8) -+ w13_scale = torch.randint( -+ 118, -+ 123, (num_experts, 2 * intermediate_size, hidden_size // 32), -+ device=device, -+ dtype=torch.uint8) -+ -+ w2_q = torch.randint(0, -+ 256, -+ (num_experts, hidden_size, intermediate_size // 2), -+ device=device, -+ dtype=torch.uint8) -+ w2_scale = torch.randint( -+ 118, -+ 123, (num_experts, hidden_size, intermediate_size // 32), -+ device=device, -+ dtype=torch.uint8) -+ # Bias contiguous [b1; b3] -+ bias13 = (torch.randn(num_experts, -+ 2 * intermediate_size, -+ device=device, -+ dtype=torch.bfloat16) * 10) -+ bias2 = (torch.randn( -+ num_experts, hidden_size, device=device, dtype=torch.bfloat16) * 10) -+ router_logits = torch.rand(num_tokens, -+ num_experts, -+ dtype=torch.float32, -+ device=device) -+ -+ w13_ref = mxfp4_dequantize(w13_q.clone(), w13_scale.clone()).reshape( -+ num_experts, 2 * intermediate_size, hidden_size) -+ w2_ref = mxfp4_dequantize(w2_q.clone(), w2_scale.clone()).reshape( -+ num_experts, hidden_size, intermediate_size) -+ ref = reference_moe(router_logits.to(torch.float32), topk, num_experts, -+ hidden_states.to(torch.float32), w13_ref, -+ bias13.to(torch.float32), w2_ref, -+ bias2.to(torch.float32), alpha, beta, limit, 'bf16') -+ -+ from vllm.utils.flashinfer import flashinfer_cutlass_fused_moe -+ -+ # Swap halves to arrange as [w3; w1] (kernel expectation) -+ w1_w, w3_w = torch.chunk(w13_q, 2, dim=1) -+ w13_q_swapped = torch.cat([w3_w, w1_w], dim=1) -+ -+ b1, b3 = torch.chunk(bias13.to(torch.float32), 2, dim=-1) -+ w13_b = torch.cat([b3, b1], dim=-1).to(torch.bfloat16) -+ -+ w1_s, w3_s = torch.chunk(w13_scale, 2, dim=1) -+ w13_s = torch.cat([w3_s, w1_s], dim=1) -+ w13_s_inter = _interleave_scales_lastdim_by4(w13_s) -+ w2_s_inter = _interleave_scales_lastdim_by4(w2_scale) -+ -+ routing_weights = torch.nn.functional.softmax(router_logits, -+ dim=1, -+ dtype=torch.float32) -+ token_final_scales, token_selected_experts = torch.topk(routing_weights, -+ topk, -+ dim=-1) -+ token_final_scales = (token_final_scales / -+ token_final_scales.sum(dim=-1, keepdim=True)) -+ token_selected_experts = token_selected_experts.to(torch.int).contiguous() -+ -+ out = torch.empty_like(hidden_states, dtype=torch.bfloat16) -+ if alpha is not None: -+ alpha = torch.full((num_experts, ), alpha, device=hidden_states.device) -+ if beta is not None: -+ beta = torch.full((num_experts, ), beta, device=hidden_states.device) -+ if limit is not None: -+ limit = torch.full((num_experts, ), limit, device=hidden_states.device) -+ -+ _ = flashinfer_cutlass_fused_moe( -+ input=hidden_states, -+ token_selected_experts=token_selected_experts, -+ token_final_scales=token_final_scales, -+ fc1_expert_weights=w13_q_swapped, -+ fc2_expert_weights=w2_q, -+ output_dtype=torch.bfloat16, -+ output=out, -+ quant_scales=[w13_s_inter.to(torch.uint8), -+ w2_s_inter.to(torch.uint8)], -+ fc1_expert_biases=w13_b, -+ fc2_expert_biases=bias2.to(torch.bfloat16), -+ swiglu_alpha=alpha, -+ swiglu_beta=beta, -+ swiglu_limit=limit, -+ tp_size=1, -+ tp_rank=0, -+ ep_size=1, -+ ep_rank=0, -+ use_w4_group_scaling=True, -+ ) -+ -+ # Allow some mismatch due to MXFP4 quantization -+ check_accuracy(ref, out, atol=0, rtol=0.3, percent=0.8) -+ -+ -+@pytest.mark.parametrize("topk", [1, 4]) -+@pytest.mark.parametrize("num_experts", [32]) -+@pytest.mark.parametrize("num_tokens", [1, 128]) -+@pytest.mark.parametrize("intermediate_size,hidden_size", [(3072, 3072)]) -+@pytest.mark.parametrize("alpha,beta,limit", [(1.0, 1.0, None), -+ (1.702, 1.0, 7.0)]) -+@pytest.mark.skipif( -+ not (current_platform.is_cuda() -+ and current_platform.is_device_capability(100) and has_flashinfer()), -+ reason="NVIDIA GPU sm100 and flashinfer are required for this test", -+) -+def test_flashinfer_cutlass_mxfp4_mxfp8_fused_moe( -+ topk: int, -+ num_experts: int, -+ num_tokens: int, -+ intermediate_size: int, -+ hidden_size: int, -+ alpha: Optional[float], -+ beta: Optional[float], -+ limit: Optional[float], -+): -+ torch.manual_seed(42) -+ device = "cuda:0" -+ -+ # Inputs -+ hidden_states = torch.randn(num_tokens, -+ hidden_size, -+ device=device, -+ dtype=torch.bfloat16) -+ # Float weights in w13 format [w1; w3] -+ w13 = (torch.randn(num_experts, -+ 2 * intermediate_size, -+ hidden_size, -+ device=device, -+ dtype=torch.bfloat16) / 10) -+ w2 = (torch.randn(num_experts, -+ hidden_size, -+ intermediate_size, -+ device=device, -+ dtype=torch.bfloat16) / 10) -+ # Bias contiguous [b1; b3] -+ bias13 = (torch.randn(num_experts, -+ 2 * intermediate_size, -+ device=device, -+ dtype=torch.bfloat16) * 10) -+ bias2 = (torch.randn( -+ num_experts, hidden_size, device=device, dtype=torch.bfloat16) * 10) -+ router_logits = torch.rand(num_tokens, -+ num_experts, -+ dtype=torch.float32, -+ device=device) -+ -+ # Quantize weights to MXFP4 per expert (SM100 path) -+ from flashinfer import mxfp4_quantize -+ -+ def quant_mxfp4_batches(a: torch.Tensor, e: int): -+ qs, sfs = [], [] -+ for i in range(e): -+ q, sf = mxfp4_quantize(a[i].cuda()) -+ qs.append(q) -+ sfs.append(sf) -+ return torch.stack(qs), torch.stack(sfs) -+ -+ def dequant_mxfp4_batches(mat_fp4: torch.Tensor, -+ scale_tensor: torch.Tensor): -+ num_batches = mat_fp4.size(0) -+ scale_tensor = scale_tensor.view(num_batches, -1) -+ from flashinfer import mxfp4_dequantize -+ return torch.stack([ -+ mxfp4_dequantize(mat_fp4[b, :, :], scale_tensor[b, :]) -+ for b in range(num_batches) -+ ]) -+ -+ w13_q, w13_scale = quant_mxfp4_batches(w13, num_experts) -+ w2_q, w2_scale = quant_mxfp4_batches(w2, num_experts) -+ -+ # Reference result using dequantized tensors and reference_moe -+ w13_ref = dequant_mxfp4_batches( -+ w13_q.view(torch.uint8), -+ w13_scale.view(torch.uint8).reshape(-1)).to(torch.float32).reshape( -+ num_experts, 2 * intermediate_size, hidden_size) -+ w2_ref = dequant_mxfp4_batches( -+ w2_q.view(torch.uint8), -+ w2_scale.view(torch.uint8).reshape(-1)).to(torch.float32).reshape( -+ num_experts, hidden_size, intermediate_size) -+ -+ # Quantize activations for SM100 path and dequantize for reference -+ hidden_states_q, hidden_states_sf = mxfp8_quantize(hidden_states, True, 32) -+ # Reference uses BF16 input but quantizes intermediate activation to MXFP8 -+ ref = reference_moe(router_logits.to(torch.float32), topk, num_experts, -+ hidden_states.to(torch.float32), w13_ref, -+ bias13.to(torch.float32), w2_ref, -+ bias2.to(torch.float32), alpha, beta, limit, 'mxfp8') -+ -+ # Prepare inputs for FlashInfer CUTLASS fused MoE -+ from vllm.utils.flashinfer import flashinfer_cutlass_fused_moe -+ -+ # Swap halves to arrange as [w3; w1] (kernel expectation) -+ w1_w, w3_w = torch.chunk(w13_q, 2, dim=1) -+ w13_q_swapped = torch.cat([w3_w, w1_w], dim=1) -+ -+ # Swap scales halves to match swapped weights -+ s1, s3 = torch.chunk(w13_scale, 2, dim=1) -+ w13_scale_swapped = torch.cat([s3, s1], dim=1) -+ -+ b1, b3 = torch.chunk(bias13.to(torch.float32), 2, dim=-1) -+ w13_b = torch.cat([b3, b1], dim=-1).to(torch.bfloat16) -+ -+ # Build routing for kernel -+ routing_weights = torch.nn.functional.softmax(router_logits, -+ dim=1, -+ dtype=torch.float32) -+ token_final_scales, token_selected_experts = torch.topk(routing_weights, -+ topk, -+ dim=-1) -+ token_final_scales = (token_final_scales / -+ token_final_scales.sum(dim=-1, keepdim=True)) -+ token_selected_experts = token_selected_experts.to(torch.int).contiguous() -+ -+ out = torch.empty_like(hidden_states, dtype=torch.bfloat16) -+ if alpha is not None: -+ alpha_t = torch.full((num_experts, ), -+ alpha, -+ device=hidden_states.device) -+ else: -+ alpha_t = None -+ if beta is not None: -+ beta_t = torch.full((num_experts, ), beta, device=hidden_states.device) -+ else: -+ beta_t = None -+ if limit is not None: -+ limit_t = torch.full((num_experts, ), -+ limit, -+ device=hidden_states.device) -+ else: -+ limit_t = None -+ -+ # Quant scales for SM100 MXFP8+MXFP4 path -+ fake_input_scale = torch.ones(num_experts, device=device) -+ quant_scales = [ -+ w13_scale_swapped.view(torch.int32), -+ fake_input_scale, -+ w2_scale.view(torch.int32), -+ fake_input_scale, -+ ] -+ -+ _ = flashinfer_cutlass_fused_moe( -+ input=hidden_states_q, -+ token_selected_experts=token_selected_experts, -+ token_final_scales=token_final_scales, -+ fc1_expert_weights=w13_q_swapped.contiguous().view(torch.long), -+ fc2_expert_weights=w2_q.contiguous().view(torch.long), -+ output_dtype=torch.bfloat16, -+ output=out, -+ quant_scales=quant_scales, -+ fc1_expert_biases=w13_b, -+ fc2_expert_biases=bias2.to(torch.bfloat16), -+ swiglu_alpha=alpha_t, -+ swiglu_beta=beta_t, -+ swiglu_limit=limit_t, -+ tp_size=1, -+ tp_rank=0, -+ ep_size=1, -+ ep_rank=0, -+ use_mxfp8_act_scaling=True, -+ input_sf=hidden_states_sf, -+ ) -+ -+ # Allow some mismatch due to MXFP4 quantization -+ check_accuracy(ref, out, atol=0, rtol=0.3, percent=0.8) -diff --git a/vllm/envs.py b/vllm/envs.py -index d7956a2adff6..165cd32721fe 100755 ---- a/vllm/envs.py -+++ b/vllm/envs.py -@@ -166,7 +166,8 @@ - VLLM_HAS_FLASHINFER_CUBIN: bool = False - VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8: bool = False - VLLM_USE_FLASHINFER_MOE_MXFP4_BF16: bool = False -- VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True -+ VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS: bool = False -+ VLLM_ALLREDUCE_USE_SYMM_MEM: bool = False - VLLM_TUNED_CONFIG_FOLDER: Optional[str] = None - VLLM_DISABLE_PAD_FOR_CUDAGRAPH: bool = False - VLLM_GPT_OSS_USE_CONTAINER_TOOL: bool = False -@@ -1004,6 +1005,15 @@ def get_vllm_port() -> Optional[int]: - "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8": - lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8", "0"))), - -+ # If set to 1, use the FlashInfer CUTLASS backend for -+ # MXFP8 (activation) x MXFP4 (weight) MoE. -+ # This is separate from the TRTLLMGEN path controlled by -+ # VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8. -+ "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS": -+ lambda: bool(int( -+ os.getenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS", "0") -+ )), -+ - # If set to 1, use the FlashInfer - # BF16 (activation) x MXFP4 (weight) MoE backend. - "VLLM_USE_FLASHINFER_MOE_MXFP4_BF16": -@@ -1296,6 +1306,7 @@ def compute_hash() -> str: - "VLLM_USE_FLASHINFER_MOE_FP8", - "VLLM_USE_FLASHINFER_MOE_FP4", - "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8", -+ "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS", - "VLLM_USE_FLASHINFER_MOE_MXFP4_BF16", - "VLLM_USE_CUDNN_PREFILL", - "VLLM_USE_TRTLLM_ATTENTION", -diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py -index 89676f98cb0e..a90a71159f72 100644 ---- a/vllm/model_executor/layers/fused_moe/layer.py -+++ b/vllm/model_executor/layers/fused_moe/layer.py -@@ -813,9 +813,16 @@ def __init__( - - # we are padding globally so EP buffer allocation works - if quant_config and quant_config.get_name() == "mxfp4": -- from vllm.model_executor.layers.quantization.mxfp4 import ( # noqa: E501 -- should_use_flashinfer_mxfp4) -- if current_platform.is_rocm() or should_use_flashinfer_mxfp4(): -+ from vllm.model_executor.layers.quantization.mxfp4 import ( -+ Mxfp4Backend, get_mxfp4_backend) -+ current_mxfp4_backend = get_mxfp4_backend() -+ if (current_mxfp4_backend == Mxfp4Backend.SM90_FI_MXFP4_BF16 -+ or current_mxfp4_backend -+ == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_CUTLASS): -+ hidden_size = round_up(hidden_size, 128) -+ elif (current_platform.is_rocm() or current_mxfp4_backend -+ == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_TRTLLM or -+ current_mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_BF16): - hidden_size = round_up(hidden_size, 256) - - # For smuggling this layer into the fused moe custom op -diff --git a/vllm/model_executor/layers/quantization/mxfp4.py b/vllm/model_executor/layers/quantization/mxfp4.py -index 74922756afe5..f935bdd84124 100644 ---- a/vllm/model_executor/layers/quantization/mxfp4.py -+++ b/vllm/model_executor/layers/quantization/mxfp4.py -@@ -1,5 +1,6 @@ - # SPDX-License-Identifier: Apache-2.0 - # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -+from enum import Enum - from typing import Callable, Optional, Union - - import torch -@@ -33,33 +34,72 @@ - logger = init_logger(__name__) - - --def _should_use_flashinfer_mxfp4_bf16(): -- """Determine if FlashInfer MXFP4 BF16 should be used.""" -- # If explicitly set, respect the setting -- if envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16"): -- return envs.VLLM_USE_FLASHINFER_MOE_MXFP4_BF16 -- -- # Enable by default on SM100 if MXFP8 is not explicitly enabled -- if (current_platform.is_device_capability(100) and has_flashinfer() -- and not envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8")): -- logger.info_once( -- "Enabling FlashInfer MXFP4 BF16 backend by default for Blackwell. " -- "For faster performance, consider setting " -- "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8=1, " -- "though this may impact accuracy.") -- return True -- -- return False -- -- --def _should_use_flashinfer_mxfp4_mxfp8(): -- """Determine if FlashInfer MXFP4 MXFP8 should be used.""" -- return envs.VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8 -+# enum for mxfp4 backend -+class Mxfp4Backend(Enum): -+ NONE = 0 -+ -+ # FlashInfer Backend -+ SM100_FI_MXFP4_MXFP8_TRTLLM = 1 -+ SM100_FI_MXFP4_MXFP8_CUTLASS = 2 -+ SM100_FI_MXFP4_BF16 = 3 -+ SM90_FI_MXFP4_BF16 = 4 -+ -+ # Marlin Backend -+ MARLIN = 5 -+ -+ # Triton Backend -+ TRITON = 6 -+ -+ -+def get_mxfp4_backend(): -+ # Backend Selection -+ if current_platform.is_cuda(): -+ if (current_platform.is_device_capability(90) and has_flashinfer() -+ and envs.VLLM_USE_FLASHINFER_MOE_MXFP4_BF16): -+ logger.info_once("Using FlashInfer MXFP4 BF16 backend for SM90") -+ return Mxfp4Backend.SM90_FI_MXFP4_BF16 -+ elif (current_platform.is_device_capability(100) and has_flashinfer() -+ and envs.VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS): -+ logger.info_once( -+ "Using FlashInfer MXFP4 MXFP8 CUTLASS backend for SM100") -+ return Mxfp4Backend.SM100_FI_MXFP4_MXFP8_CUTLASS -+ elif (current_platform.is_device_capability(100) and has_flashinfer() -+ and envs.VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8): -+ logger.info_once( -+ "Using FlashInfer MXFP4 MXFP8 TRTLLM backend for SM100, " -+ "for high concurrency throughput workloads consider setting " -+ "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS=1 for better " -+ "performance") -+ return Mxfp4Backend.SM100_FI_MXFP4_MXFP8_TRTLLM -+ elif current_platform.is_device_capability(100) and has_flashinfer(): -+ logger.info_once( -+ "Using FlashInfer MXFP4 BF16 backend for SM100, " -+ "For faster performance on SM100, consider setting " -+ "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8=1, though this may impact " -+ "accuracy.") -+ return Mxfp4Backend.SM100_FI_MXFP4_BF16 -+ elif ((current_platform.is_device_capability(100) -+ or current_platform.is_device_capability(90)) -+ and not has_flashinfer()): -+ logger.warning_once( -+ "MXFP4 MoE is enabled on Hopper/Blackwell but FlashInfer " -+ "is not available. This may result in degraded performance. " -+ "Please `pip install vllm[flashinfer]` for best results.") - -+ # If FlashInfer is not available, try either Marlin or Triton -+ if current_platform.get_device_capability( -+ )[0] < 9 or not has_triton_kernels() or not is_torch_equal_or_newer( -+ "2.8.0"): -+ logger.info_once("Using Marlin backend") -+ return Mxfp4Backend.MARLIN -+ else: -+ logger.info_once("Using Triton backend") -+ return Mxfp4Backend.TRITON -+ elif current_platform.is_rocm() and has_triton_kernels(): -+ logger.info_once("Using Triton backend") -+ return Mxfp4Backend.TRITON - --def should_use_flashinfer_mxfp4(): -- return (_should_use_flashinfer_mxfp4_mxfp8() -- or _should_use_flashinfer_mxfp4_bf16()) -+ return Mxfp4Backend.NONE - - - class Mxfp4Config(QuantizationConfig): -@@ -113,31 +153,15 @@ def __init__(self, moe: FusedMoEConfig): - super().__init__(moe) - self.topk_indices_dtype = None - self.moe = moe -- self.use_marlin = self._should_use_marlin() -+ self.mxfp4_backend = get_mxfp4_backend() - self.max_capture_size = get_current_vllm_config( - ).compilation_config.max_capture_size - -- if current_platform.is_device_capability(100) and not has_flashinfer(): -- logger.warning_once( -- "MXFP4 MoE is enabled on Blackwell but FlashInfer " -- "is not available. This may result in degraded performance. " -- "Please `pip install vllm[flashinfer]` for best results.") -+ assert self.mxfp4_backend != Mxfp4Backend.NONE, ( -+ "No MXFP4 MoE backend (FlashInfer/Marlin/Triton) available." -+ "Please check your environment and try again.") - self._cache_permute_indices: dict[torch.Size, torch.Tensor] = {} - -- def _should_use_marlin(self): -- if envs.VLLM_MXFP4_USE_MARLIN is not None: -- return envs.VLLM_MXFP4_USE_MARLIN -- if current_platform.is_cuda() and \ -- not current_platform.is_device_capability(100): -- if not current_platform.has_device_capability(90): -- # marlin kernel has better performance on ampere -- return True -- if not has_triton_kernels(): -- return True -- if not is_torch_equal_or_newer("2.8.0"): -- return True -- return False -- - def create_weights(self, layer: torch.nn.Module, num_experts: int, - hidden_size: int, intermediate_size_per_partition: int, - params_dtype: torch.dtype, **extra_weight_attrs): -@@ -157,7 +181,7 @@ def create_weights(self, layer: torch.nn.Module, num_experts: int, - - intermediate_size_per_partition_after_pad = \ - intermediate_size_per_partition -- if self.use_marlin: -+ if self.mxfp4_backend == Mxfp4Backend.MARLIN: - # The moe marlin kernel requires that for each linear - # n % 256 == 0 and k % 128 == 0. - # In gate_up_proj: -@@ -175,16 +199,20 @@ def create_weights(self, layer: torch.nn.Module, num_experts: int, - layer.hidden_size = hidden_size - layer.intermediate_size_per_partition = \ - intermediate_size_per_partition_after_pad -- elif should_use_flashinfer_mxfp4(): -+ elif (self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_TRTLLM -+ or self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_BF16): - # pad the intermediate size to be a multiple of 2 * mxfp4_block - # for to hold non-uniform sharded tensor as well as swizzling - # other padding to increase performance - intermediate_size_per_partition_after_pad = round_up( - intermediate_size_per_partition, 256) - hidden_size = round_up(hidden_size, 256) -- elif current_platform.is_rocm(): -+ elif current_platform.is_rocm() or ( -+ self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_CUTLASS -+ or self.mxfp4_backend == Mxfp4Backend.SM90_FI_MXFP4_BF16): - intermediate_size_per_partition_after_pad = round_up( - intermediate_size_per_partition, 128) -+ hidden_size = round_up(hidden_size, 128) - else: - intermediate_size_per_partition_after_pad = round_up( - intermediate_size_per_partition, 64) -@@ -264,9 +292,10 @@ def create_weights(self, layer: torch.nn.Module, num_experts: int, - set_weight_attrs(w2_bias, extra_weight_attrs) - - def process_weights_after_loading(self, layer): -- if self.use_marlin: -+ if self.mxfp4_backend == Mxfp4Backend.MARLIN: - prepare_moe_fp4_layer_for_marlin(layer) -- elif should_use_flashinfer_mxfp4(): -+ elif (self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_TRTLLM -+ or self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_BF16): - from flashinfer.fp4_quantization import ( - nvfp4_block_scale_interleave) - from flashinfer.fused_moe.core import ( -@@ -429,7 +458,116 @@ def swap_every_two_rows(x, axis=-1): - layer.w2_bias = Parameter(torch.stack(gemm2_bias_shuffled).reshape( - self.num_experts, -1), - requires_grad=False) -- else: -+ elif (self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_CUTLASS -+ or self.mxfp4_backend == Mxfp4Backend.SM90_FI_MXFP4_BF16): -+ layer.gemm1_alpha = Parameter(torch.tensor( -+ [1.702] * self.num_experts, dtype=torch.float32).cuda(), -+ requires_grad=False) -+ layer.gemm1_beta = Parameter(torch.tensor( -+ [1.0] * self.num_experts, dtype=torch.float32).cuda(), -+ requires_grad=False) -+ layer.gemm1_clamp_limit = Parameter(torch.tensor( -+ [7.0] * self.num_experts, dtype=torch.float32).cuda(), -+ requires_grad=False) -+ -+ sf_block_size = 32 # mxfp4 block size -+ -+ # Common shape assertions -+ assert (layer.w13_weight.dim() == 3 -+ and layer.w13_weight.shape[0] == self.num_experts -+ and layer.w13_weight.shape[1] == self.intermediate_size * 2 -+ and layer.w13_weight.shape[2] == self.hidden_size // 2) -+ assert (layer.w13_weight_scale.dim() == 3 -+ and layer.w13_weight_scale.shape[0] == self.num_experts -+ and layer.w13_weight_scale.shape[1] -+ == self.intermediate_size * 2 -+ and layer.w13_weight_scale.shape[2] -+ == self.hidden_size // sf_block_size) -+ assert (layer.w2_weight.dim() == 3 -+ and layer.w2_weight.shape[0] == self.num_experts -+ and layer.w2_weight.shape[1] == self.hidden_size and -+ layer.w2_weight.shape[2] == self.intermediate_size // 2) -+ assert (layer.w2_weight_scale.dim() == 3 -+ and layer.w2_weight_scale.shape[1] == self.hidden_size -+ and layer.w2_weight_scale.shape[2] -+ == self.intermediate_size // sf_block_size) -+ assert (layer.w13_bias.dim() == 2 -+ and layer.w13_bias.shape[0] == self.num_experts -+ and layer.w13_bias.shape[1] == self.intermediate_size * 2) -+ assert (layer.w2_bias.dim() == 2 -+ and layer.w2_bias.shape[0] == self.num_experts -+ and layer.w2_bias.shape[1] == self.hidden_size) -+ -+ # De-interleave and swap for w13 weight, bias, and scales -+ w13_w = layer.w13_weight.data -+ gate_w, up_w = w13_w[:, ::2, :], w13_w[:, 1::2, :] -+ deinterleaved_w13_w = torch.cat([gate_w, up_w], dim=1) -+ w1_w, w3_w = torch.chunk(deinterleaved_w13_w, 2, dim=1) -+ w13_weight_swapped = torch.cat([w3_w, w1_w], dim=1) -+ -+ w13_b = layer.w13_bias.data.to(torch.float32) -+ gate_b, up_b = w13_b[:, ::2], w13_b[:, 1::2] -+ deinterleaved_w13_b = torch.cat([gate_b, up_b], dim=1) -+ b1, b3 = torch.chunk(deinterleaved_w13_b, 2, dim=-1) -+ w13_bias_swapped = torch.cat([b3, b1], dim=-1).to(torch.bfloat16) -+ -+ w13_s = layer.w13_weight_scale.data -+ gate_s, up_s = w13_s[:, ::2, :], w13_s[:, 1::2, :] -+ deinterleaved_w13_s = torch.cat([gate_s, up_s], dim=1) -+ s1, s3 = torch.chunk(deinterleaved_w13_s, 2, dim=1) -+ w13_scale_swapped = torch.cat([s3, s1], dim=1) -+ -+ if self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_CUTLASS: -+ from flashinfer import block_scale_interleave -+ -+ orig_shape = w13_scale_swapped.shape -+ w13_scale_interleaved = block_scale_interleave( -+ w13_scale_swapped.view(torch.uint8)).reshape(orig_shape) -+ -+ w2_s = layer.w2_weight_scale.data -+ orig_shape = w2_s.shape -+ w2_scale_interleaved = block_scale_interleave( -+ w2_s.view(torch.uint8)).reshape(orig_shape) -+ -+ layer.w13_weight = Parameter(w13_weight_swapped, -+ requires_grad=False) -+ layer.w13_weight_scale = Parameter(w13_scale_interleaved, -+ requires_grad=False) -+ layer.w13_bias = Parameter(w13_bias_swapped, -+ requires_grad=False) -+ layer.w2_weight_scale = Parameter(w2_scale_interleaved, -+ requires_grad=False) -+ elif self.mxfp4_backend == Mxfp4Backend.SM90_FI_MXFP4_BF16: -+ -+ def _interleave_mxfp4_cutlass_sm90(w): -+ w_shape = w.shape -+ w_interleaved = w.reshape(w_shape[0], w_shape[1], -+ (w_shape[2] // 4), 4) -+ w_interleaved = w_interleaved.permute(0, 2, 1, 3) -+ w_interleaved = w_interleaved.reshape( -+ w_shape[0], w_shape[2] // 4, w_shape[1] * 4) -+ return w_interleaved -+ -+ w31_scales = w13_scale_swapped.to(torch.uint8).view( -+ torch.uint8) -+ w31_scales_interleaved = _interleave_mxfp4_cutlass_sm90( -+ w31_scales) -+ -+ w2_weight_scale = layer.w2_weight_scale.data -+ w2_scales = w2_weight_scale.to(torch.uint8).view(torch.uint8) -+ w2_scales_interleaved = _interleave_mxfp4_cutlass_sm90( -+ w2_scales) -+ -+ layer.w13_weight = torch.nn.Parameter(torch.cat([w3_w, w1_w], -+ dim=1), -+ requires_grad=False) -+ layer.w13_bias = torch.nn.Parameter(w13_bias_swapped, -+ requires_grad=False) -+ layer.w13_weight_scale = torch.nn.Parameter( -+ w31_scales_interleaved, requires_grad=False) -+ layer.w2_weight_scale = torch.nn.Parameter( -+ w2_scales_interleaved, requires_grad=False) -+ elif self.mxfp4_backend == Mxfp4Backend.TRITON: - from triton_kernels.matmul_ogs import FlexCtx, PrecisionConfig - - w13_bias = layer.w13_bias.to(torch.float32) -@@ -464,6 +602,8 @@ def swap_every_two_rows(x, axis=-1): - layer.w13_weight = None - layer.w2_weight = None - torch.cuda.empty_cache() -+ else: -+ raise ValueError(f"Unsupported backend: {self.mxfp4_backend}") - - def _get_tile_tokens_dim(self, x: torch.Tensor, top_k: int): - # Number of tokens in the input tensor. -@@ -500,7 +640,8 @@ def select_gemm_impl( - raise NotImplementedError( - "Mxfp4 does not support batched experts format for EP") - else: -- if should_use_flashinfer_mxfp4(): -+ if (self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_TRTLLM -+ or self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_BF16): - # B200 code-path - kwargs = { - "gemm1_alpha": layer.gemm1_alpha, -@@ -601,7 +742,7 @@ def apply( - if enable_eplb: - raise NotImplementedError("EPLB is not supported for mxfp4") - -- if self.use_marlin: -+ if self.mxfp4_backend == Mxfp4Backend.MARLIN: - topk_weights, topk_ids = FusedMoE.select_experts( - hidden_states=x, - router_logits=router_logits, -@@ -665,16 +806,19 @@ def apply( - logical_replica_count), ( - "MXFP4 are not supported with this configuration.") - -- if should_use_flashinfer_mxfp4(): -- from flashinfer import mxfp8_quantize, trtllm_fp4_block_scale_moe -- if _should_use_flashinfer_mxfp4_bf16(): -+ if (self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_TRTLLM -+ or self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_BF16): -+ from flashinfer import trtllm_fp4_block_scale_moe -+ if self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_BF16: - assert x.dtype == torch.bfloat16 - x_quant = x - x_scale = None -- else: -+ elif self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_TRTLLM: -+ from flashinfer import mxfp8_quantize - x_quant, x_scale = mxfp8_quantize(x, False) # to mxfp8 - x_scale = x_scale.view(torch.float8_e4m3fn).reshape( - *x.shape[:-1], -1) -+ - trtllm_gen_output = trtllm_fp4_block_scale_moe( - router_logits.to(torch.bfloat16), - None, # routing_bias -@@ -706,7 +850,86 @@ def apply( - tune_max_num_tokens=self.max_capture_size, - )[0] - return trtllm_gen_output -- else: -+ elif (self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_CUTLASS -+ or self.mxfp4_backend == Mxfp4Backend.SM90_FI_MXFP4_BF16): -+ from vllm.utils.flashinfer import flashinfer_cutlass_fused_moe -+ -+ topk_weights, topk_ids = FusedMoE.select_experts( -+ hidden_states=x, -+ router_logits=router_logits, -+ use_grouped_topk=use_grouped_topk, -+ top_k=top_k, -+ renormalize=renormalize, -+ topk_group=topk_group, -+ num_expert_group=num_expert_group, -+ custom_routing_function=custom_routing_function, -+ scoring_func=scoring_func, -+ e_score_correction_bias=e_score_correction_bias, -+ ) -+ -+ # Backend-specific preparation -+ if self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_CUTLASS: -+ -+ from flashinfer import mxfp8_quantize -+ -+ x_quant, x_scale = mxfp8_quantize(x, True, 32) -+ -+ fake_input_scale = torch.ones(self.num_experts, -+ device=x.device) -+ quant_scales = [ -+ layer.w13_weight_scale.contiguous().view(torch.int32), -+ fake_input_scale, -+ layer.w2_weight_scale.contiguous().view(torch.int32), -+ fake_input_scale, -+ ] -+ -+ fi_input = x_quant -+ extra_kwargs = dict( -+ use_mxfp8_act_scaling=True, -+ input_sf=x_scale, -+ fc1_expert_weights=layer.w13_weight.contiguous().view( -+ torch.long), -+ fc2_expert_weights=layer.w2_weight.contiguous().view( -+ torch.long), -+ ) -+ elif self.mxfp4_backend == Mxfp4Backend.SM90_FI_MXFP4_BF16: -+ assert x.dtype == torch.bfloat16 -+ -+ quant_scales = [ -+ layer.w13_weight_scale, -+ layer.w2_weight_scale, -+ ] -+ -+ fi_input = x -+ extra_kwargs = dict( -+ use_w4_group_scaling=True, -+ fc1_expert_weights=layer.w13_weight, -+ fc2_expert_weights=layer.w2_weight, -+ ) -+ -+ output = torch.empty_like(x, dtype=torch.bfloat16) -+ _ = flashinfer_cutlass_fused_moe( -+ input=fi_input, -+ token_selected_experts=topk_ids.to(torch.int).contiguous(), -+ token_final_scales=topk_weights, -+ output_dtype=torch.bfloat16, -+ output=output, -+ quant_scales=quant_scales, -+ fc1_expert_biases=layer.w13_bias, -+ fc2_expert_biases=layer.w2_bias, -+ swiglu_alpha=layer.gemm1_alpha, -+ swiglu_beta=layer.gemm1_beta, -+ swiglu_limit=layer.gemm1_clamp_limit, -+ tp_size=self.moe.tp_size, -+ tp_rank=self.moe.tp_rank, -+ ep_size=self.moe.ep_size, -+ ep_rank=self.moe.ep_rank, -+ tune_max_num_tokens=self.max_capture_size, -+ **extra_kwargs, -+ ) -+ -+ return output -+ elif self.mxfp4_backend == Mxfp4Backend.TRITON: - from vllm.model_executor.layers.fused_moe.gpt_oss_triton_kernels_moe import ( # noqa: E501 - triton_kernel_moe_forward) - return triton_kernel_moe_forward( -@@ -724,3 +947,5 @@ def apply( - w2_precision=self.w2_precision_config, - apply_router_weight_on_input=apply_router_weight_on_input, - ) -+ else: -+ raise ValueError(f"Unsupported backend: {self.mxfp4_backend}") -diff --git a/vllm/model_executor/warmup/kernel_warmup.py b/vllm/model_executor/warmup/kernel_warmup.py -index e42e34ebc77b..89ce20308f44 100644 ---- a/vllm/model_executor/warmup/kernel_warmup.py -+++ b/vllm/model_executor/warmup/kernel_warmup.py -@@ -33,8 +33,8 @@ def kernel_warmup(worker: "Worker"): - max_tokens = worker.scheduler_config.max_num_batched_tokens - deep_gemm_warmup(model, max_tokens) - -- # FlashInfer kernel autotune for Blackwell (SM 10.0) GPUs -- if has_flashinfer() and current_platform.is_device_capability(100): -+ # FlashInfer autotune for Hopper (SM 9.0) and Blackwell (SM 10.0) GPUs -+ if has_flashinfer() and current_platform.has_device_capability(90): - flashinfer_autotune(worker.model_runner) - - # FlashInfer attention warmup diff --git a/artifacts/kernels/gated-dual-gemm/variants/01-fused-epilogue-swiglu-skeleton.cu b/artifacts/kernels/gated-dual-gemm/variants/01-fused-epilogue-swiglu-skeleton.cu deleted file mode 100644 index 125821414..000000000 --- a/artifacts/kernels/gated-dual-gemm/variants/01-fused-epilogue-swiglu-skeleton.cu +++ /dev/null @@ -1,25 +0,0 @@ -// provenance: derived from pr-vllm-23696, kernel-gated-dual-gemm, technique-epilogue-fusion; not upstream code -// origin: wiki/kernels/gated-dual-gemm.md Phase 3 variant - -// Gated dual GEMM fused epilogue: gate_out = silu(gate_gemm_out) * up_gemm_out -// Two GEMMs share the same A operand; their outputs are combined in the -// epilogue warpgroup without touching global memory between them. - -#include -#include - -template -__device__ void gated_dual_gemm_epilogue( - float gate_acc[TILE_M][TILE_N], // accumulator from gate GEMM - float up_acc[TILE_M][TILE_N], // accumulator from up GEMM - __half* out) -{ - for (int m = 0; m < TILE_M; ++m) { - for (int n = 0; n < TILE_N; ++n) { - float g = gate_acc[m][n]; - // SiLU(x) = x * sigmoid(x) - float silu_g = g * (1.f / (1.f + __expf(-g))); - out[m * TILE_N + n] = __float2half(silu_g * up_acc[m][n]); - } - } -} diff --git a/artifacts/kernels/gated-dual-gemm/variants/01-gated-dual-gemm-semantics.py b/artifacts/kernels/gated-dual-gemm/variants/01-gated-dual-gemm-semantics.py new file mode 100644 index 000000000..ec3118bbf --- /dev/null +++ b/artifacts/kernels/gated-dual-gemm/variants/01-gated-dual-gemm-semantics.py @@ -0,0 +1,73 @@ +"""Derived CPU reference for the GPU Mode NVFP4 dual-GEMM result semantics. + +This intentionally does not model NVFP4 encoding, scale layout, or GPU execution. +It checks the two shared-A matrix products and the order of SiLU and multiply. +""" + +from __future__ import annotations + +import math + + +Matrix = list[list[float]] + + +def _sigmoid(value: float) -> float: + if value >= 0.0: + z = math.exp(-value) + return 1.0 / (1.0 + z) + z = math.exp(value) + return z / (1.0 + z) + + +def _project(a: Matrix, b_nk: Matrix) -> Matrix: + if not a or not b_nk or not a[0]: + raise ValueError("matrices must be non-empty") + k = len(a[0]) + if any(len(row) != k for row in a): + raise ValueError("A must be rectangular") + if any(len(row) != k for row in b_nk): + raise ValueError("B must have shape [N,K]") + return [[sum(x * y for x, y in zip(a_row, b_row)) for b_row in b_nk] + for a_row in a] + + +def gated_dual_gemm(a: Matrix, b1_nk: Matrix, b2_nk: Matrix) -> Matrix: + """Return SiLU(A @ B1.T) * (A @ B2.T).""" + if len(b1_nk) != len(b2_nk): + raise ValueError("B1 and B2 must have the same N") + gate = _project(a, b1_nk) + up = _project(a, b2_nk) + return [[g * _sigmoid(g) * u for g, u in zip(g_row, u_row)] + for g_row, u_row in zip(gate, up)] + + +def _assert_close(actual: Matrix, expected: Matrix) -> None: + for actual_row, expected_row in zip(actual, expected): + for actual_value, expected_value in zip(actual_row, expected_row): + assert math.isclose(actual_value, expected_value, rel_tol=1e-12, + abs_tol=1e-12) + + +def _self_test() -> None: + a = [[1.0, -2.0], [0.5, 3.0]] + b1 = [[2.0, 1.0], [-1.0, 0.5]] + b2 = [[0.25, 2.0], [1.0, -3.0]] + + gate = _project(a, b1) + up = _project(a, b2) + expanded = [[g * _sigmoid(g) * u for g, u in zip(g_row, u_row)] + for g_row, u_row in zip(gate, up)] + actual = gated_dual_gemm(a, b1, b2) + _assert_close(actual, expanded) + + wrong_branch = [[u * _sigmoid(u) * g for g, u in zip(g_row, u_row)] + for g_row, u_row in zip(gate, up)] + assert any(not math.isclose(x, y, rel_tol=1e-12, abs_tol=1e-12) + for actual_row, wrong_row in zip(actual, wrong_branch) + for x, y in zip(actual_row, wrong_row)) + + +if __name__ == "__main__": + _self_test() + print("gated dual GEMM semantic reference: PASS") diff --git a/artifacts/kernels/gated-dual-gemm/variants/PROVENANCE.yaml b/artifacts/kernels/gated-dual-gemm/variants/PROVENANCE.yaml index 7fddf037b..d3f4ed189 100644 --- a/artifacts/kernels/gated-dual-gemm/variants/PROVENANCE.yaml +++ b/artifacts/kernels/gated-dual-gemm/variants/PROVENANCE.yaml @@ -1,16 +1,14 @@ origin_url: derived upstream_repo: derived license: MIT-style-inherited-from-kernelwiki-tooling -retrieved_at: '2026-04-17' +retrieved_at: '2026-08-08' asset_mode: derived derived_from: -- pr-vllm-23696 -- kernel-gated-dual-gemm -- technique-epilogue-fusion +- contest-gpumode-p3 size_cap_truncated: false -generated_by: Phase 3 Round 3 task12 (gated-dual-gemm variants) +generated_by: KernelWiki verifier remediation files: -- local_path: 01-fused-epilogue-swiglu-skeleton.cu +- local_path: 01-gated-dual-gemm-semantics.py role: derived-source mode: derived - sha256: 85e158c184546912579ac5270451471a96f135efa3584bafbf45064e5fb7e223 + sha256: ac13383732fc5c946ae1698cc0776131dd399c507736b28a1a7d4f45a976daeb diff --git a/artifacts/prs/vllm/PR-23696/PROVENANCE.yaml b/artifacts/prs/vllm/PR-23696/PROVENANCE.yaml index 8b1ee4fb4..bb3c816ce 100644 --- a/artifacts/prs/vllm/PR-23696/PROVENANCE.yaml +++ b/artifacts/prs/vllm/PR-23696/PROVENANCE.yaml @@ -1,12 +1,16 @@ origin_url: https://github.com/vllm-project/vllm/pull/23696 upstream_repo: vllm-project/vllm -upstream_sha: 074854b2 +upstream_sha: 074854b24f6e0b1e237a004283e1f46d98c0d73c license: inherits-from-upstream retrieved_at: '2026-04-17' asset_mode: verbatim size_cap_truncated: false generated_by: scripts/fetch_pr_diff.py source_pr_id: pr-vllm-23696 +notes: >- + diff.patch is the aggregate merged diff across all five changed paths. The + key-files tree intentionally retains only kernel_warmup.py as a representative + byte-verified file; it is not the complete changed-file set. files: - local_path: diff.patch role: pr-diff diff --git a/data/core-prs.yaml b/data/core-prs.yaml index f124707e8..d7a8d17a4 100644 --- a/data/core-prs.yaml +++ b/data/core-prs.yaml @@ -8,8 +8,8 @@ sources: - cute-dsl-tutorial - triton-in-policy - allowlist -total_captured: 252 -checksum_sha256: 2f30f03bf66bf9f4550949e26e5d805420dca14c74a368a153f0ca145b978465 +total_captured: 246 +checksum_sha256: 6809c0043b75aaf43dbecf1df96509b0605f2759505074bf080b60d5269f64c5 prs: - id: pr-TensorRT-LLM-11697 source_of_inclusion: cute-dsl-tutorial @@ -50,9 +50,9 @@ prs: - id: pr-cutlass-2378 source_of_inclusion: cute-dsl-tutorial - id: pr-cutlass-2466 - source_of_inclusion: cute-dsl-tutorial + source_of_inclusion: wiki-graph-closure - id: pr-cutlass-2472 - source_of_inclusion: cute-dsl-tutorial + source_of_inclusion: wiki-graph-closure - id: pr-cutlass-2492 source_of_inclusion: cute-dsl-tutorial - id: pr-cutlass-2599 @@ -75,8 +75,6 @@ prs: source_of_inclusion: cute-dsl-tutorial - id: pr-deepgemm-304 source_of_inclusion: wiki-graph-closure -- id: pr-flash-attention-1236 - source_of_inclusion: wiki-graph-closure - id: pr-flash-attention-1934 source_of_inclusion: cute-dsl-tutorial - id: pr-flash-attention-1940 @@ -146,8 +144,6 @@ prs: - id: pr-flashinfer-1025 source_of_inclusion: triton-in-policy - id: pr-flashinfer-1039 - source_of_inclusion: triton-in-policy -- id: pr-flashinfer-1117 source_of_inclusion: wiki-graph-closure - id: pr-flashinfer-1331 source_of_inclusion: cute-dsl-tutorial @@ -171,8 +167,6 @@ prs: source_of_inclusion: cute-dsl-tutorial - id: pr-flashinfer-1812 source_of_inclusion: cute-dsl-tutorial -- id: pr-flashinfer-1850 - source_of_inclusion: wiki-graph-closure - id: pr-flashinfer-2149 source_of_inclusion: cute-dsl-tutorial - id: pr-flashinfer-2171 @@ -253,8 +247,6 @@ prs: source_of_inclusion: triton-in-policy - id: pr-pytorch-171129 source_of_inclusion: triton-in-policy -- id: pr-pytorch-175826 - source_of_inclusion: wiki-graph-closure - id: pr-pytorch-176495 source_of_inclusion: triton-in-policy - id: pr-sglang-10403 @@ -429,8 +421,6 @@ prs: source_of_inclusion: triton-in-policy - id: pr-vllm-23666 source_of_inclusion: triton-in-policy -- id: pr-vllm-23696 - source_of_inclusion: wiki-graph-closure - id: pr-vllm-25990 source_of_inclusion: cute-dsl-tutorial - id: pr-vllm-26322 @@ -495,8 +485,6 @@ prs: source_of_inclusion: triton-in-policy - id: pr-vllm-39007 source_of_inclusion: triton-in-policy -- id: pr-vllm-39752 - source_of_inclusion: wiki-graph-closure - id: pr-vllm-40941 source_of_inclusion: triton-in-policy - id: pr-vllm-41428 diff --git a/data/inclusion-policy.yaml b/data/inclusion-policy.yaml index d9bde14f5..c47530c88 100644 --- a/data/inclusion-policy.yaml +++ b/data/inclusion-policy.yaml @@ -2,9 +2,8 @@ ## Consumed by scripts/compute_core_prs.py ## ## Two separate lanes (CuTe DSL and Triton) are classified with different -## criteria because Triton on SM100 has no direct tcgen05/TMEM access -## (see wiki/languages/triton-blackwell.md), so Triton's useful scope is -## narrower than CuTe DSL's. +## criteria because their Blackwell kernel coverage and production maturity +## differ by workload. See wiki/languages/triton-blackwell.md. cute-dsl: description: | @@ -62,20 +61,19 @@ cute-dsl: triton: description: | - Triton is a scoped Blackwell lane. As of Triton 3.6 (released - 2026-01-21), Triton has native SM100 lowering through tcgen05 + TMEM - via descriptor/TMA + warp_specialize, tl.dot_scaled, and Gluon - multi-CTA / 2CTA, but production peak-performance leadership for - compute-bound matmul/attention on Blackwell still favors hand-written - CuTe-DSL / CUTLASS / FA-4 / TRT-LLM kernels for many workloads - (downstream evidence: pr-sglang-5390 measured CUTLASS tcgen05_mla - ~27% faster than the Triton MLA decode baseline; pr-sglang-21595 - moved Blackwell datacenter multimodal attention default away from - triton_attn). We therefore capture Triton PRs in three sub-scopes - that remain genuinely useful on Blackwell rather than mirroring the - full CuTe-DSL lane. + Triton is a scoped Blackwell lane. Native TCGen5/TMEM compiler support + enters between Triton v3.2.0 and v3.3.0. Later releases expand explicit + user surfaces and maturity: v3.5 includes Gluon TCGen5/TMEM and + block-scaled matmul tutorials, while v3.6 adds broader layouts/copies, + warp-specialization work, and initial multi-CTA / 2-CTA Gluon support. + Exact instruction selection remains shape-, dtype-, layout-, target-, and + compiler-configuration-dependent. Production routing also remains + workload-specific: pr-sglang-5390 reports one scoped CUTLASS-over-Triton + MLA result, and pr-sglang-21595 moves one SM100 attention default to FA4. + We therefore capture Triton PRs in three useful Blackwell sub-scopes + rather than mirroring the full CuTe-DSL lane. - See data/version-claims.yaml::vs-triton-3.6-blackwell-tcgen05 for the + See data/version-claims.yaml::vs-triton-3.3-blackwell-tcgen05 for the version-sensitive claim record this rationale references. capture_criteria: diff --git a/data/schemas.yaml b/data/schemas.yaml index 215d8b2c1..847e34c54 100644 --- a/data/schemas.yaml +++ b/data/schemas.yaml @@ -150,7 +150,7 @@ wiki-technique: constraints: type: technique id_prefix: technique- - reproducibility_minimum: snippet + reproducibility_minimum: concept wiki-pattern: required: @@ -165,6 +165,8 @@ wiki-pattern: optional: - architectures - confidence + - reproducibility + - evidence_basis - version_sensitive constraints: type: pattern @@ -199,7 +201,7 @@ wiki-kernel: constraints: type: kernel id_prefix: kernel- - reproducibility_minimum: snippet + reproducibility_minimum: concept wiki-language: required: @@ -237,6 +239,7 @@ wiki-migration: - confidence - reproducibility - prerequisites + - evidence_basis - version_sensitive constraints: type: migration diff --git a/data/tags.yaml b/data/tags.yaml index 114146f0a..1456b3afd 100644 --- a/data/tags.yaml +++ b/data/tags.yaml @@ -57,6 +57,12 @@ techniques: - top-k-selection - parallel-scan - stream-k + - k-dimension-parallelism + - reduction + - occupancy + - launch-bounds + - maxrregcount + - spills kernel_types: - gemm @@ -89,6 +95,7 @@ languages: - ptx - python - jax-pallas + - mojo confidence: - verified diff --git a/data/tool-versions.yaml b/data/tool-versions.yaml index a16ae4707..4ec1738d2 100644 --- a/data/tool-versions.yaml +++ b/data/tool-versions.yaml @@ -18,6 +18,13 @@ tools: - tool: triton releases: + - name: "3.7.1" + released_at: "2026-06-18" + channel: stable + release_notes_url: "https://github.com/triton-lang/triton/releases/tag/v3.7.1" + notes: | + Latest checked stable release as of 2026-08-08. The upstream release + describes two regression fixes and no new API or feature. - name: "3.6.0" released_at: "2026-01-21" channel: stable @@ -27,10 +34,9 @@ tools: - pr-sglang-22079 - pr-sglang-21019 notes: | - First Triton release with native Blackwell tcgen05 + TMEM lowering - infrastructure. Strongest user-visible Blackwell paths: descriptor/TMA - warp-specialized matmul, Gluon multi-CTA / 2CTA, tl.dot_scaled / block - scaled. See data/triton-3.6-evidence.md for the per-pathway breakdown. + Incremental Blackwell release: broader TCGen5/TMEM copies and layouts, + more warp-specialization work, and initial multi-CTA / 2-CTA Gluon + support. Native backend support predates this release. - name: "3.5.1" released_at: "2025-11-12" channel: stable @@ -39,6 +45,25 @@ tools: Last 3.5.x patch before the 3.6 Blackwell story. Pages with version_sensitive claims valid for ">=3.5,<3.6" should pin to this release. + - name: "3.3.0" + released_at: "2025-04-09" + channel: stable + release_notes_url: "https://github.com/triton-lang/triton/releases/tag/v3.3.0" + evidence_source_ids: + - doc-triton-3.3-blackwell + notes: | + First checked tag after v3.2.0 whose tree contains native TCGen5 MMA, + TMEM operations and allocation, MMAv5 lowering, and concrete + Blackwell conversion tests. + - name: "3.2.0" + released_at: "2025-01-22" + channel: stable + release_notes_url: "https://github.com/triton-lang/triton/releases/tag/v3.2.0" + evidence_source_ids: + - doc-triton-3.3-blackwell + notes: | + Checked negative side of the Blackwell backend boundary; corresponding + TCGen5 MMA, TMEM, and MMAv5-lowering symbols are absent from this tag. - tool: cutlass releases: diff --git a/data/triton-3.6-evidence.md b/data/triton-3.6-evidence.md index 54d6e2c94..b8a556f77 100644 --- a/data/triton-3.6-evidence.md +++ b/data/triton-3.6-evidence.md @@ -1,79 +1,41 @@ -# Triton 3.6 Blackwell Evidence Memo +# Triton Blackwell Version Evidence -## Releases of Record -- 2026-01-21: Triton 3.6.0 (`v3.6.0`, release commit `7c56a5e`). Blackwell-relevant items in the official release notes include TMEM encoding/layout work (`#8136`, `#8148`, `#8202`), generic `tcgen05` load/store and copy lowering (`#8225`, `#8421`, `#8495`, `#8102`, `#8338`), `tcgen05.mma` generalization (`#8386`), initial 2CTA Gluon support (`#8644`, `#8653`), `reqnctapercluster` emission (`#8645`), warp-specialization end-to-end aref plumbing (`#8262`, `#7826`, `#8009`), and Gluon `tcgen05 mma scaled` support (`#8393`). -- No subsequent `3.6.x` patch release is visible on the official GitHub releases page as of 2026-04-27; the next older release shown there is 3.5.1 on 2025-11-12. If the wiki wants a machine-checked negative claim rather than a page inspection, mark this needs-verification. +## Releases of record -## Lowering Surfaces -- Pathway: plain `tl.dot` / standard MMA without Blackwell-specific warp specialization or descriptor/TMA structure. - Lowers to `tcgen05` / TMEM on SM100: needs-verification. Triton 3.6 clearly contains native Blackwell `tcgen05` + TMEM infrastructure, but the checked sources do not prove that arbitrary plain `tl.dot` kernels now automatically use that path. - Introducing PRs/commits: needs-verification for this exact user-visible surface. - Caveats: do not replace the old wiki sentence with the opposite blanket claim. The evidence supports “Triton 3.6 has native Blackwell lowering paths,” not “all Triton matmuls on SM100 are now TMEM-backed tcgen05 kernels.” +| Release | Exact revision | Evidence-scoped conclusion | +|---|---|---| +| v3.2.0 | `9641643da6c52000c807b5eeed05edaec4402a67` | Checked negative side: the corresponding TCGen5 MMA, TMEM, and MMAv5-lowering symbols are absent. | +| v3.3.0 | `819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c` | Adds TCGen5 MMA/scaled-MMA operations, TMEM operations and allocation, MMAv5 lowering, and conversion tests for concrete TCGen5/TMEM output. | +| v3.5.0 | `c3c476f357f1e9768ea4e45aa5c17528449ab9ef` | Includes explicit Gluon TCGen5/TMEM and Blackwell block-scaled matmul tutorials, plus documented warp-specialization work. | +| v3.6.0 | `7c56a5e40f7fd928dfd5c72902d5def0097db73a` | Incremental generalization of copies/layouts and MMA, aref warp-specialization work, and initial multi-CTA/2-CTA Gluon support. | +| v3.7.0 | `5f3f125e8f63c24613f1f73b937442864f263f94` | Further end-to-end 2-CTA, multicast, and TMA work. | +| v3.7.1 | `f797708` | Latest checked stable release on 2026-08-08; a two-regression patch with no advertised new API or feature. | -- Pathway: descriptor/TMA matmul with `tl.range(..., warp_specialize=True)` and `tl.dot`, as documented in the persistent matmul tutorial. - Lowers to `tcgen05` / TMEM on SM100: yes for the Blackwell warp-specialized path; this is the strongest checked `tl.*`-surface evidence. The official tutorial says this warp-specialized mode “only works on Blackwell right now,” while the 3.6 release notes add the Blackwell TMEM/layout, `tcgen05`, and warp-specialization aref plumbing needed to make that path real on SM100. - Introducing PRs/commits: warp-specialization lowering plumbing `#8262`, `#7826`, `#8009`, `#8123`, `#8534`, `#8451`, `#8651`; Blackwell TMEM / `tcgen05` backend work `#8136`, `#8148`, `#8202`, `#8386`, `#8421`, `#8495`, `#8102`, `#8338`, `#8225`. - Caveats: the verified path is descriptor/TMA-oriented and warp-specialized, not generic. It is also Blackwell-targeted in the checked docs, and the tutorial evidence does not prove parity for every non-persistent or non-descriptor `tl.dot` kernel shape. +The discriminating version boundary is therefore v3.2.0 to v3.3.0, not v3.5.x to v3.6.0. See `sources/docs/triton-3.3-blackwell.md` for the pinned files and scope. -- Pathway: fused attention forward kernels using `warp_specialize=True` in the Triton tutorial path. - Lowers to `tcgen05` / TMEM on SM100: likely yes on the Blackwell forward path, but some of this is inference from the shared warp-specialization/aref/TMEM lowering stack rather than an explicit “this emits `tcgen05.mma`” statement in the tutorial, so exact coverage should be treated as partially needs-verification. - Introducing PRs/commits: same core Blackwell and warp-specialization lowering series as above, especially `#8262`, `#7826`, `#8009`, `#8136`, `#8148`, `#8202`, `#8386`, `#8421`, `#8495`, `#8102`, `#8338`. - Caveats: the tutorial explicitly ties some forward-path behavior to Blackwell, including the FP8 non-transposed-`V` case. This is not evidence that all attention modes, backward paths, or production attention kernels are equally mature on SM100. +## What the compiler evidence proves -- Pathway: `tl.dot_scaled` / block-scaled matmul on Blackwell. - Lowers to `tcgen05` / TMEM on SM100: yes for the supported hardware-accelerated Blackwell path. The official block-scaled matmul tutorial says these kernels are hardware-accelerated by fifth-generation Tensor Cores on compute capability 10, and the 3.6 dialect docs expose `ttng.tc_gen5_mma_scaled` with TMEM-token semantics plus `ttng.tmem_copy`. - Introducing PRs/commits: Gluon NVIDIA `tcgen05 mma scaled` support `#8393`; frontend fixes around `dot_scaled` `#8564` and `#8658`; shared TMEM / `tcgen05` backend work `#8136`, `#8148`, `#8202`. - Caveats: this path is format- and layout-constrained. The checked tutorial is centered on NVFP4 / MXFP formats and notes that mixed-precision extensions are still future work. +The exact v3.3.0 tree proves that native Blackwell TCGen5/TMEM compiler machinery and tested lowering exist. The v3.5.0 tutorials prove explicit user-visible examples for Gluon TCGen5/TMEM and `tl.dot_scaled` block-scaled matmul by that tag. The v3.6.0 notes prove additional Blackwell work, not first introduction. -- Pathway: Gluon front-end `gl.warp_specialize`, `num_ctas`, and multi-CTA / 2CTA Blackwell lowering. - Lowers to `tcgen05` / TMEM on SM100: yes. This is the most explicit Blackwell-native surface in the checked 3.6 materials: the release notes call out initial 2CTA support in Gluon, `num_ctas`, multi-CTA support, and `tcgen05 mma scaled` support, while the dialect docs expose TMEM allocation/copy and `tc_gen5_mma` / `tc_gen5_mma_scaled` ops directly. - Introducing PRs/commits: Gluon API and multi-CTA work `#8527`, `#8468`, `#8587`, `#8602`, `#8644`; Blackwell backend 2CTA / cluster work `#8644`, `#8653`, `#8645`; Gluon NVIDIA `tcgen05 mma scaled` `#8393`. - Caveats: the release notes describe this as initial support, so cluster-scope and 2CTA usage should still be treated as early-stage. This is also a Gluon-first story; it is stronger evidence for “Triton can target Blackwell natively” than for “classic `tl.dot` is universally first-class on SM100.” +None of those facts implies that arbitrary plain `tl.dot` kernels select TCGen5 for every dtype, layout, shape, architecture, or compiler configuration. A target-specific selection claim needs inspectable IR or PTX from that configuration. -## Caveats and Open Questions -- The old wiki claim “Triton compiler generates wgmma, not tcgen05” is no longer globally correct for Triton 3.6+, but the replacement should be qualified: native `tcgen05` + TMEM paths exist on SM100, especially through warp-specialized descriptor/TMA and Gluon flows. -- The old wiki claim “No TMEM: accumulators stay in registers” is also outdated as a blanket statement. The checked 3.6 dialect docs explicitly model TMEM allocation/copy/load/store and `tc_gen5_mma` / `tc_gen5_mma_scaled` ops with TMEM-token semantics. -- What remains weaker than CuTe-DSL / CUTLASS hand-written Blackwell kernels is production peak-performance coverage. In SGLang `pr-sglang-5390`, a CUTLASS `tcgen05_mla` backend reports about 27% higher throughput than the Triton baseline on Blackwell for MLA decode. -- Downstream routing decisions still show Triton is not the universal best path on Blackwell. In `pr-sglang-21595`, SGLang changes Blackwell datacenter multimodal attention default from `triton_attn` to `fa4`; in `pr-sglang-21914`, SGLang sets TRT-LLM kernels as the default for Blackwell. -- The clearest generally-available Blackwell story in checked sources is not “plain `tl.dot` everywhere,” but “warp-specialized descriptor/TMA kernels and Gluon multi-CTA/2CTA kernels.” Anything beyond that should be marked needs-verification until backed by PTX/IR or a downstream merged PR. -- A policy update from “narrow” toward “first-class” is justified, but only with qualifiers. Recommended interpretation: Triton 3.6+ is first-class for supported Blackwell-native lowering paths and for serious prototyping on SM100, but it is still not the default peak-performance answer for all compute-bound production kernels. -- Open question: find a downstream merged PR in `pytorch`, `vllm`, `sglang`, or `flashinfer` that explicitly depends on Triton 3.6+ and shows `ttng.tc_gen5_mma` / `tcgen05.*` emission for an SM100 Triton kernel. I could not verify that exact anchor from checked downstream sources, so this remains needs-verification. +## Downstream evidence and limits -## Evidence References +- `pr-vllm-34597`, pinned at merge SHA `a1257fd1`, adds FP8 KV-cache handling to vLLM's generic Triton MLA decode backend. Its kernel contains `tl.dot`; neither the PR nor exact code supplies an SM100 guard, a Triton-version pin, a TCGen5/TMEM symbol, or emitted PTX. The PR specifically motivates the backend as the MLA option available on SM120. +- `pr-vllm-29339`, pinned at `c17610e2`, gates MXFP4 `triton_kernels` dispatch to SM90 and SM100. It changes dispatch code, not a Triton kernel or compiler lowering. +- `pr-sglang-21019`, pinned at `5bdc07d9`, contains a Triton GatedDeltaNet projection rearrangement using loads and stores, with no `tl.dot`. +- `pr-sglang-22079`, pinned at `5638d40f`, contains an extend-attention Triton kernel with real `tl.dot` operations. It still contains no emitted-PTX witness for a particular MMA instruction. -### Primary anchors +These artifacts verify downstream Triton use. They do not independently verify which backend instruction any target selects. -- `doc-triton-3.6-blackwell` — Triton 3.6 release notes / official tutorial and dialect-doc summary covering TMEM, `tcgen05`, `warp_specialize`, `num_ctas`, 2CTA mode, and `tcgen05 mma scaled` on Blackwell. (`source_category: official-doc`, file at `sources/docs/triton-3.6-blackwell.md`.) +## Scoped ecosystem observations -The complete primary-anchor set comprises one official-doc anchor and one **post-refresh** downstream upstream-code anchor (per AC-1.1's "at least one **new** `sources/prs//PR-.md` page" contract — "new" = not present in `data/refresh-cutoff.yaml::previous_pages_manifest`): -- `pr-vllm-34597` — **primary post-refresh downstream upstream-code anchor**: vLLM PR titled "[Kernel] Add FP8 KV cache support to Triton MLA decode attention" (merged `2026-02-16` on `architectures: [sm100]`, post-Triton-3.6.0 release date `2026-01-21`). This PR directly modifies actual Triton kernel files — `vllm/v1/attention/ops/triton_decode_attention.py` (the `@triton.jit`-decorated MLA decode kernel doing `tl.dot(q, k)` / `tl.dot(qpe, kpe)` / `tl.dot(p, v)` matmul) and `vllm/v1/attention/backends/mla/triton_mla.py` (the backend wrapping it). The kernel is shipped verbatim under `artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/ops/triton_decode_attention.py` (756 lines, pinned at upstream SHA `a1257fd1`), so the AC-1.1 demonstration is grounded in a checked-in, inspectable Triton kernel rather than inference. Tags: `attention`, `decode`, `fp8`, `mla`. (`source_category: upstream-code`, `languages: [triton]`, `inclusion_reason: kernel file changes`.) -- `pr-vllm-29339` — **secondary post-refresh anchor**: vLLM bugfix that scopes the upstream `triton_kernels` library (the `triton-lang/triton/python/triton_kernels` collection shipped with Triton 3.6) to `architectures: [sm100, sm90]` only for the MXFP4 quantization path, after issues appeared on SM110/SM120. Merged `2025-11-24`. This PR's value is showing the Triton 3.6 SM100 kernel library being explicitly gated to Blackwell production codepaths, but it only modifies a dispatch gate (`vllm/model_executor/layers/quantization/mxfp4.py`) rather than a Triton kernel itself — that is why `pr-vllm-34597` is preferred as primary. Retained as supplementary post-refresh evidence. (`source_category: upstream-code`, `languages: [triton]`.) +SGLang PR 5390 reports one DeepSeek-R1 MLA benchmark in which the CUTLASS run records 10,447.34 total tok/s and the Triton run 8,227.35 total tok/s, a 26.98% difference under the PR's recorded 3,000-prompt, TP8/DP8, float16, 1,000-input/1,000-output-token scope. It is not a universal language comparison. -#### Pre-refresh historical anchors (retained for context, no longer count toward AC-1.1) +SGLang PR 21595 changes the SM100 datacenter multimodal-attention default from `triton_attn` to FA4. This is a scoped routing decision, not evidence about every Blackwell workload. -The following anchors existed before the Round-6 refresh (each appears in `data/refresh-cutoff.yaml::previous_pages_manifest`). They remain useful as substantive evidence but do NOT satisfy AC-1.1's "new tracked-repo PR page" contract on their own: -- `pr-sglang-22079` — pre-refresh secondary anchor: a real Triton attention kernel doing actual `tl.dot(q, k)` / `tl.dot(p, v)` matmul on `architectures: [sm100, sm90]`, merged on `2026-04-03` (well after Triton 3.6.0 released `2026-01-21`). The kernel is the SGLang `extend_attention` Triton kernel for the Gemma4 NVFP4 attention path; tags include `attention`, `nvfp4`, `fp4`, `gemm`, `tcgen05`-relevant block-scaled matmul. Ships verbatim under `artifacts/prs/sglang/PR-22079/key-files/python/sglang/srt/layers/attention/triton_ops/extend_attention.py`. This remains the strongest in-corpus example of a real `tl.dot` Triton matmul kernel landing for SM100 post-3.6 in the *pre-refresh* corpus, but it is pre-refresh per the `previous_pages_manifest`. (`source_category: upstream-code`, `languages: [python, triton]`.) -- `pr-sglang-21019` — pre-refresh secondary anchor: `@triton.jit`-decorated kernel (`fused_qkvzba_split_reshape_cat_kernel` for Qwen3.5 GDN projection) landed for `architectures: [sm100]` on `2026-03-20`. This kernel is `tl.load`/`tl.store` only (memory rearrangement, no `tl.dot`), so it demonstrates "Triton on SM100 post-3.6" but not the matmul lowering surface. Retained as supplementary historical context. +The live FlashInfer-Bench author leaderboard retrieved 2026-08-08 reports Gemini 2.5 Pro at 0.628x and 73.1% resolved, GPT-5 at 0.467x and 92.3%, and Claude Opus 4.1 at 0.456x and 73.1%, each over 660 workloads. The page does not attach those rows to a Triton version or a Triton-only language subset. -### Caveat / ecosystem-readiness anchors +## Open proof obligation -- `pr-sglang-5390` — downstream upstream-code anchor (caveat): CUTLASS `tcgen05_mla` backend outperforming the Triton MLA decode baseline by ~27% on Blackwell. Demonstrates that Triton's Blackwell coverage is not yet at peak parity with hand-written CUTLASS for compute-bound workloads. (`source_category: upstream-code`.) -- `pr-sglang-21595` — downstream upstream-code anchor (caveat): Blackwell multimodal attention default changed from `triton_attn` to FA4 in datacenter SKUs. Demonstrates that production routing decisions still favor non-Triton kernels for some Blackwell paths. (`source_category: upstream-code`.) -- `pr-pytorch-175826` — downstream upstream-code anchor (ecosystem-readiness): PyTorch inductor CI's B200 / SM100 lane moved to CUDA 13.0, reflecting the broader Blackwell toolchain maturation. (`source_category: upstream-code`.) - -### Note on anchor scope - -The plan's AC-1.1 positive test reads "At least one **new** `sources/prs//PR-.md` page demonstrates a kernel that lowers through the Triton 3.6 Blackwell path." This contract has two parts: (a) the page must be *new* (not in `data/refresh-cutoff.yaml::previous_pages_manifest`); (b) the page must *demonstrate a kernel*, not just gate logic. - -Round 6 cited `pr-sglang-22079` etc., which were `tl.dot` kernels but pre-refresh — they failed (a). Round 7 swapped to `pr-vllm-29339`, which was post-refresh but only modifies a dispatch gate (`vllm/model_executor/layers/quantization/mxfp4.py`) — it satisfied (a) but only weakly satisfied (b) by inference about the upstream `triton_kernels` library. **Round 8 promotes `pr-vllm-34597` to primary anchor**: the page is post-refresh AND it directly modifies actual Triton kernel files (`vllm/v1/attention/ops/triton_decode_attention.py` containing `@triton.jit` MLA decode kernel with `tl.dot(q, k)` / `tl.dot(qpe, kpe)` / `tl.dot(p, v)` matmul, plus `vllm/v1/attention/backends/mla/triton_mla.py`). The Triton kernel itself is shipped verbatim under `artifacts/prs/vllm/PR-34597/key-files/...` (pinned at merge SHA `a1257fd1`), so the AC-1.1 demonstration is now grounded in inspectable kernel code rather than dispatch-gate inference. `pr-vllm-29339` is retained as supplementary post-refresh evidence; pre-refresh `pr-sglang-22079` / `pr-sglang-21019` are retained as historical context. - -The strongest possible demonstrations — explicit inspectable PTX showing `tcgen05.mma` emission, or kernels using `tl.dot_scaled` / `warp_specialize` with descriptor/TMA structure — were not found in any tracked-repo PR locally. They currently live in the upstream Triton tutorials. A future refresh round should backfill such an anchor if one becomes available in tracked downstream repos. - -## Recommended wiki rewrite framing -- Triton 3.6 materially changes the Blackwell story. -- The old blanket claim that Triton on SM100 only emits WGMMA with register-resident accumulators is no longer correct. -- Triton 3.6 adds native Blackwell lowering infrastructure for `tcgen05` and TMEM, with the clearest documented path running through warp-specialized descriptor/TMA kernels and newer Gluon multi-CTA / 2CTA support. -- The important qualifier is that this is not yet proof that every plain `tl.dot` kernel on SM100 automatically becomes a TMEM-backed `tcgen05` kernel. -- Treat Triton 3.6+ as a first-class Blackwell language for supported warp-specialized matmul/attention building blocks and block-scaled GEMM. -- Keep CuTe-DSL / CUTLASS / FA4 / TRT-LLM as the expected leaders for many peak-performance production attention and decode kernels. -- Replace “no tcgen05 / no TMEM” with “native `tcgen05` + TMEM paths now exist, but coverage and performance leadership are workload-dependent.” +The checked downstream bundles contain no explicit TCGen5 PTX dump or warp-specialized descriptor/TMA lowering record. Until such evidence is captured for a specific kernel, architecture, dtype, shape, and toolchain, the wiki must not infer exact instruction selection from `tl.dot` source alone. diff --git a/data/triton-universe.yaml b/data/triton-universe.yaml index d548534c7..2e205021c 100644 --- a/data/triton-universe.yaml +++ b/data/triton-universe.yaml @@ -4,8 +4,8 @@ lane: triton generated_by: scripts/compute_core_prs.py total: 267 -captured: 117 -skipped: 150 +captured: 116 +skipped: 151 prs: - id: pr-TensorRT-LLM-10327 captured: false @@ -665,7 +665,8 @@ prs: - id: pr-vllm-33529 captured: true - id: pr-vllm-34597 - captured: true + captured: false + skipped_reason: Triton PR outside the three in-policy sub-scopes - id: pr-vllm-35382 captured: false skipped_reason: Triton PR outside the three in-policy sub-scopes diff --git a/data/version-claims.yaml b/data/version-claims.yaml index 398e76333..0da97a675 100644 --- a/data/version-claims.yaml +++ b/data/version-claims.yaml @@ -19,48 +19,23 @@ ## paths exist + carry the matching pointer. claims: - - id: vs-triton-3.6-blackwell-tcgen05 + - id: vs-triton-3.3-blackwell-tcgen05 tool: triton - claim_valid_for: ">=3.6" - last_verified_release: "3.6.0" - last_verified_at: 2026-04-27 + claim_valid_for: ">=3.3" + last_verified_release: "3.7.1" + last_verified_at: 2026-08-08 applies_to: - wiki/languages/triton-blackwell.md - references/primer.md - references/examples.md - data/inclusion-policy.yaml::triton.description source_ids: + - doc-triton-3.3-blackwell - doc-triton-3.6-blackwell - - pr-vllm-34597 - - pr-vllm-29339 - - pr-sglang-22079 - - pr-sglang-21019 - - pr-sglang-5390 - - pr-sglang-21595 - - pr-pytorch-175826 notes: | - Tracks the claim that Triton 3.6+ has native Blackwell (SM100) lowering - paths through tcgen05.mma + TMEM (descriptor/TMA + warp_specialize on - tl.dot, plus Gluon multi-CTA + 2CTA on tcgen05_mma_scaled), replacing - the obsolete 3.5-era "no tcgen05 / no TMEM" framing. Supporting evidence: - data/triton-3.6-evidence.md memo plus the official release notes - (doc-triton-3.6-blackwell). AC-1.1 "new tracked-repo PR page demonstrates - a kernel" anchor: pr-vllm-34597 (post-refresh; vLLM "[Kernel] Add FP8 KV - cache support to Triton MLA decode attention", merged 2026-02-16, sm100, - directly modifying @triton.jit kernels in vllm/v1/attention/ops/ - triton_decode_attention.py with tl.dot matmul + vllm/v1/attention/ - backends/mla/triton_mla.py; Triton kernel shipped verbatim under - artifacts/prs/vllm/PR-34597/ pinned at SHA a1257fd1). Supplementary - post-refresh anchor: pr-vllm-29339 (Triton 3.6 triton_kernels library - scoped to [sm100, sm90] for MXFP4 dispatch-gate, merged 2025-11-24). - Pre-refresh historical anchors retained as supplementary context (each - appears in data/refresh-cutoff.yaml::previous_pages_manifest): - pr-sglang-22079 (Gemma4 NVFP4 Triton attention kernel doing real tl.dot - matmul on [sm100, sm90], merged 2026-04-03), pr-sglang-21019 - (memory-rearrangement Triton kernel on SM100, merged 2026-03-20, no - tl.dot), pr-sglang-5390, pr-sglang-21595, pr-pytorch-175826 (ecosystem - context). The strongest possible demonstrations of the 3.6 Blackwell- - native paths (descriptor/TMA + warp_specialize, tl.dot_scaled, Gluon - multi-CTA/2CTA) have NOT been found in any tracked-downstream merged - PR locally; future refresh rounds should backfill such an anchor when - one becomes available. + Exact tag comparison places native TCGen5/TMEM compiler support between + v3.2.0 and v3.3.0. Triton v3.6.0 is an incremental expansion of copies, + layouts, warp specialization, and initial multi-CTA/2-CTA Gluon paths; + it is not the introduction boundary. The latest checked stable release + is v3.7.1. This version claim does not assert that every plain tl.dot + shape selects TCGen5; that requires configuration-specific IR or PTX. diff --git a/queries/by-hardware-feature.md b/queries/by-hardware-feature.md index 988772a5a..83d61accb 100644 --- a/queries/by-hardware-feature.md +++ b/queries/by-hardware-feature.md @@ -4,18 +4,20 @@ | Feature | Related Pages | |---------|--------------| -| `2sm-cooperative` | [Microbenchmarking NVIDIA's Blackwell Architecture](../sources/blogs/blackwell-microbenchmarking.md), [Colfax CUTLASS Tutorial: GEMM Kernels Using Tensor Memory for Blackwell](../sources/blogs/colfax-cutlass-blackwell.md), [FlashAttention-4 Blog](../sources/blogs/flash-attention-4.md), [Writing High-Performance Matrix Multiplication Kernels for Blackwell with JAX Pallas](../sources/blogs/jax-pallas-blackwell-matmul.md), [Modular: Matrix Multiplication on Blackwell](../sources/blogs/modular-blackwell-matmul.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [CUTLASS Cluster Launch Control (CLC) Documentation](../sources/docs/cutlass-clc-documentation.md), [FlashAttention-4: Hardware-Friendly Attention on Blackwell](../sources/docs/flash-attention-4.md), [NVIDIA Blackwell Tuning Guide](../sources/docs/nvidia-blackwell-tuning-guide.md), [NVIDIA CUTLASS 4.x Blackwell Support](../sources/docs/nvidia-cutlass-blackwell.md), [Triton 3.6.0 Release Notes — Blackwell (SM100) Lowering](../sources/docs/triton-3.6-blackwell.md), [[TRTLLM-9831][perf] Enable 2CTA with autotune for CuteDSL MoE and Grouped GEMM optimizations](../sources/prs/TensorRT-LLM/PR-10201.md), [fix: examples/cute/tutorial/blackwell/04_mma_tma_2sm_sm100.cu GridDim miscalculated](../sources/prs/cutlass/PR-2492.md), [Feat([FA4][CUTE DSL]) Add head_dim=256 support (forward + backward)](../sources/prs/flash-attention/PR-2412.md), [[hd256] Improve forward kernel with exp2 FMA emulation (3% to 9% performance gain)](../sources/prs/flash-attention/PR-2488.md), [[hd256] Add TMA paged KV support to SM100 2CTA forward kernel](../sources/prs/flash-attention/PR-2489.md), [[FA4][hd256] Backward TMA bulk-store epilogue + LSE/dpsum coalesce](../sources/prs/flash-attention/PR-2497.md), [[Feature] 2-SM support for TMA, TMEM and TCGEN5MMA on Blackwell](../sources/prs/tilelang/PR-1882.md), [Two-SM Cooperative MMA](../wiki/hardware/2sm-cooperative.md), [FlashAttention-4](../wiki/kernels/flash-attention-4.md), [Not Reaching Peak FLOPS](../wiki/patterns/compute-bound.md) | -| `block-scale` | [Twelve Attempts at NVFP4 Batched GEMV](../sources/blogs/amandeep-nvfp4-attempts.md), [DeepGEMM — FP8 GEMM Library](../sources/blogs/deepgemm.md), [NVFP4 Format Details](../sources/blogs/nvfp4-format-details.md), [NVFP4 GEMV and Improved NVFP4 GEMV](../sources/blogs/simon-nvfp4-gemv.md), [TFLOPS Gap: Why FP4 MoE Kernel Engineering Matters on Blackwell](../sources/blogs/tflops-gap-fp4-moe.md), [Blackwell NVFP4 Kernel Hackathon Journey](../sources/blogs/yue-nvfp4-hackathon.md), [FlashInfer MLSys 2026 - Track A: Fused MoE FP8](../sources/contests/flashinfer-mlsys26/track-a-fused-moe.md), [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [GPU Mode NVFP4 Hackathon - Problem 1: Batched GEMV](../sources/contests/gpu-mode-nvfp4/problem-1-gemv.md), [GPU Mode NVFP4 Hackathon - Problem 2: NVFP4 GEMM](../sources/contests/gpu-mode-nvfp4/problem-2-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 3: Gated Dual GEMM](../sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 4: Grouped GEMM](../sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [NVIDIA Blackwell Tuning Guide](../sources/docs/nvidia-blackwell-tuning-guide.md), [NVIDIA CUDA Toolkit 13.x for Blackwell](../sources/docs/nvidia-cuda-13.md), [NVIDIA CUTLASS 4.x Blackwell Support](../sources/docs/nvidia-cutlass-blackwell.md), [PTX ISA SM100 Instructions Reference](../sources/docs/nvidia-ptx-isa-sm100.md), [Triton 3.6.0 Release Notes — Blackwell (SM100) Lowering](../sources/docs/triton-3.6-blackwell.md), [[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes](../sources/prs/DeepGEMM/PR-304.md), [[TRTLLM-9457][feat] Add cute dsl fp8 gemm for Blackwell](../sources/prs/TensorRT-LLM/PR-10130.md), [Blockwise and Groupwise GEMM for Blackwell and Improvements for Hopper](../sources/prs/cutlass/PR-2139.md), [[CuTeDSL] Fix: SM100 block-scale gemm overlapping accumulator](../sources/prs/cutlass/PR-2995.md), [perf: Enable SplitK and fix tile-scheduling for moe fp4 fused moe](../sources/prs/flashinfer/PR-1548.md), [feat: enable deepgemm jit for fp8 block-scale on SM90](../sources/prs/flashinfer/PR-1969.md), [feat: Support unpadded output hidden size for trtllm_fp4_block_scale_moe](../sources/prs/flashinfer/PR-2217.md), [fix: trtllm_mxint4_block_scale_moe unit test to index output list](../sources/prs/flashinfer/PR-2627.md), [int16 Block-Scaled State and Stochastic Rounding for SSU (mamba)](../sources/prs/flashinfer/PR-2645.md), [Support in-place update for `trtllm_fp8_block_scale_moe`](../sources/prs/flashinfer/PR-2739.md), [fix: int32 overflow in `trtllm_fp4_block_scale_moe` causing "Unsupported hidden state scale shape" for EP32+ configs](../sources/prs/flashinfer/PR-2853.md), [fix: snap weight_scale_vec_size to handle block_scale_interleave padding for SM120](../sources/prs/flashinfer/PR-2898.md), [fix: Fix autotuner crash on meta-device tensor in trtllm_fp4_block_scale_routed_moe](../sources/prs/flashinfer/PR-2916.md), [Only swizzle on v block scale; rename kv_block_scales to kv_cache_sf](../sources/prs/flashinfer/PR-2954.md), [feat(trace): embed runnable init() in every TraceTemplate](../sources/prs/flashinfer/PR-3221.md), [[Perf] Eliminate the slice op for Flashinfer `trtllm_fp4_block_scale_moe`](../sources/prs/sglang/PR-15731.md), [integrate blockwise fp8 kernel](../sources/prs/sglang/PR-3529.md), [[Feature] Block-scaled GEMM support for MXFP8 on Blackwell](../sources/prs/tilelang/PR-1945.md), [[NVIDIA] Support nvfp4 tcgen05 gemm](../sources/prs/vllm/PR-13571.md), [[FEAT] [ROCm]: Add AITER Block-Scaled GEMM Feature](../sources/prs/vllm/PR-14968.md), [[Kernel][tcgen05] nvfp4 fused tcgen05 moe](../sources/prs/vllm/PR-23696.md), [[Bugfix] Handle num_expert_group=None in flashinfer block-scale FP8 MoE](../sources/prs/vllm/PR-34494.md), [NVFP4 and Block-Scaled Narrow Precision](../wiki/hardware/nvfp4.md), [DeepGEMM — FP8 GEMM with Fine-Grained Scaling](../wiki/kernels/deepgemm.md), [FP8 Block-Scale GEMM](../wiki/kernels/fp8-block-scale-gemm.md), [Fused MoE — FP8 Block-Scale Routing + Dual GEMM](../wiki/kernels/fused-moe.md), [NVFP4 GEMM — 4-bit Floating Point Matrix Multiply](../wiki/kernels/nvfp4-gemm.md), [NVFP4 Batched GEMV](../wiki/kernels/nvfp4-gemv.md), [Fine-Grained FP8/FP4 Quantization](../wiki/techniques/fine-grained-quantization.md) | -| `clc` | [Modular: Matrix Multiplication on Blackwell](../sources/blogs/modular-blackwell-matmul.md), [GPU Mode NVFP4 Hackathon - Problem 4: Grouped GEMM](../sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [CUTLASS Cluster Launch Control (CLC) Documentation](../sources/docs/cutlass-clc-documentation.md), [NVIDIA Blackwell Tuning Guide](../sources/docs/nvidia-blackwell-tuning-guide.md), [NVIDIA CUDA Toolkit 13.x for Blackwell](../sources/docs/nvidia-cuda-13.md), [NVIDIA CUTLASS 4.x Blackwell Support](../sources/docs/nvidia-cutlass-blackwell.md), [PTX ISA SM100 Instructions Reference](../sources/docs/nvidia-ptx-isa-sm100.md), [Blockwise Improvement and Programmatic Dependent Launch](../sources/prs/cutlass/PR-2161.md), [Support for GEMM-K=0 for Blackwell Grouped GEMMs](../sources/prs/cutlass/PR-2746.md), [[Cute-DSL] Add option for issue_clc_query without multicast](../sources/prs/cutlass/PR-3021.md), [[Ai-assisted] CLC work stealing](../sources/prs/flash-attention/PR-2218.md), [Add CLC scheduler heuristic](../sources/prs/flash-attention/PR-2455.md), [[V1][P/D]P2pNcclConnector supports flashinfer](../sources/prs/vllm/PR-23536.md), [Cluster Launch Control (CLC)](../wiki/hardware/clc.md), [PTX Instructions for SM100](../wiki/languages/ptx-sm100.md), [Low SM Utilization](../wiki/patterns/low-sm-utilization.md), [MoE Expert Load Imbalance](../wiki/patterns/moe-load-imbalance.md), [Tail Effect — Last Wave Underutilization](../wiki/patterns/tail-effect.md), [Persistent Kernels with CLC](../wiki/techniques/persistent-kernels.md), [Tile Scheduling Strategies](../wiki/techniques/tile-scheduling.md) | -| `cluster` | [Microbenchmarking NVIDIA's Blackwell Architecture](../sources/blogs/blackwell-microbenchmarking.md), [Qwen3-Next: Hybrid GDN+MoE Architecture on NVIDIA Blackwell](../sources/blogs/qwen3-next-architecture.md), [CUTLASS Cluster Launch Control (CLC) Documentation](../sources/docs/cutlass-clc-documentation.md), [Two-SM Cooperative MMA](../wiki/hardware/2sm-cooperative.md) | -| `fp4` | [Twelve Attempts at NVFP4 Batched GEMV](../sources/blogs/amandeep-nvfp4-attempts.md), [Microbenchmarking NVIDIA's Blackwell Architecture](../sources/blogs/blackwell-microbenchmarking.md), [Anatomy of a Reward Hack](../sources/blogs/gpu-mode-reward-hack.md), [NVFP4 Format Details](../sources/blogs/nvfp4-format-details.md), [NVFP4 GEMV and Improved NVFP4 GEMV](../sources/blogs/simon-nvfp4-gemv.md), [TFLOPS Gap: Why FP4 MoE Kernel Engineering Matters on Blackwell](../sources/blogs/tflops-gap-fp4-moe.md), [Tilus: A Tile-Level GPGPU Programming Language for Low-Precision Computation](../sources/blogs/tilus-nvidia.md), [Blackwell NVFP4 Kernel Hackathon Journey](../sources/blogs/yue-nvfp4-hackathon.md), [GPU Mode NVFP4 Hackathon - Problem 1: Batched GEMV](../sources/contests/gpu-mode-nvfp4/problem-1-gemv.md), [GPU Mode NVFP4 Hackathon - Problem 2: NVFP4 GEMM](../sources/contests/gpu-mode-nvfp4/problem-2-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 3: Gated Dual GEMM](../sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 4: Grouped GEMM](../sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [NVIDIA Blackwell Tuning Guide](../sources/docs/nvidia-blackwell-tuning-guide.md), [NVIDIA CUDA Toolkit 13.x for Blackwell](../sources/docs/nvidia-cuda-13.md), [NVIDIA CUTLASS 4.x Blackwell Support](../sources/docs/nvidia-cutlass-blackwell.md), [PTX ISA SM100 Instructions Reference](../sources/docs/nvidia-ptx-isa-sm100.md), [[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes](../sources/prs/DeepGEMM/PR-304.md), [Sync nv_dev with upstream #316 (Mega MoE optimizations & benchmarks)](../sources/prs/DeepGEMM/PR-328.md), [[TRTLLM-9661][chore] Further reduce tuning time for cuteDSL nvFP4 dense gemm.](../sources/prs/TensorRT-LLM/PR-10339.md), [[None][feat] fuse shared to sparse experts in TRT-LLM Gen MoE](../sources/prs/TensorRT-LLM/PR-11143.md), [[https://nvbugs/5799917][fix] Recover from CUTLASS MoE doActivation perf regression for MXFP4/NVFP4 dtype](../sources/prs/TensorRT-LLM/PR-11165.md), [[None][feat] Optimize super-v3 nvfp4 for better perf](../sources/prs/TensorRT-LLM/PR-11273.md), [[None][feat] Optimize by fuse nvfp4_quant to layernorm_gated for mamba2_mixer](../sources/prs/TensorRT-LLM/PR-11473.md), [[None][feat] Add support for expert_number<=2048 and K<=32](../sources/prs/TensorRT-LLM/PR-11510.md), [[TRTLLM-10004][feat] Enable GEMM -> AR with GEMM output in registered buffers](../sources/prs/TensorRT-LLM/PR-11589.md), [[https://nvbugs/5799917][fix] Recover from CUTLASS MoE doActivation perf regression for MXFP4/NVFP4 dtype](../sources/prs/TensorRT-LLM/PR-11733.md), [[None][fix] Fix SM120 issue for rms_norm with nvfp4_quant_fusion](../sources/prs/TensorRT-LLM/PR-11774.md), [[None][feat] Support update weight for nvfp4](../sources/prs/TensorRT-LLM/PR-12320.md), [[None][feat] Update rms_norm + fp4_qaunt kernel supporting more dim](../sources/prs/TensorRT-LLM/PR-13033.md), [[None][feat] Add FP4 residual quantization kernel without channel reo…](../sources/prs/TensorRT-LLM/PR-13117.md), [[None][feat] Integrate FP4 indexer for DSA on Blackwell](../sources/prs/TensorRT-LLM/PR-13340.md), [[TRTLLM-11127][feat] add W4A8_MXFP4_FP8 MoE unit test support](../sources/prs/TensorRT-LLM/PR-13401.md), [[TRTLLM-12316][feat] Integrate FP4 indexer for DSv4](../sources/prs/TensorRT-LLM/PR-13575.md), [[TRTLLM-12503][feat] Parallel VAE independent scaling and fix arg passing](../sources/prs/TensorRT-LLM/PR-13873.md), [[TRTLLM-35237][feat] Add cute dsl FP4 paged MQA logits decode kernel](../sources/prs/TensorRT-LLM/PR-13929.md), [[https://nvbugs/6025177][test] rcca tests using kimi k2.5 fp4](../sources/prs/TensorRT-LLM/PR-14172.md), [[None][fix] Add SPDX Apache-2.0 headers to auto_deploy test files](../sources/prs/TensorRT-LLM/PR-14193.md), [feat: Add w4a8_mxfp4_fp8 quantization recipe.](../sources/prs/TensorRT-LLM/PR-4867.md), [[OMNIML-2336][feat] Add NVFP4 x FP8](../sources/prs/TensorRT-LLM/PR-6809.md), [[TRTLLM-8535][feat] Support DeepSeek V3.2 with FP8 + BF16 KV cache/NVFP4 + BF16 KV cache](../sources/prs/TensorRT-LLM/PR-8405.md), [[None][feat] Enable nvfp4 cuda core for sm120](../sources/prs/TensorRT-LLM/PR-8620.md), [[None][feat] Update TRTLLM MoE cubins; reduce mxfp4 weight padding requirement; tighten TMA bound](../sources/prs/TensorRT-LLM/PR-9025.md), [[None][feat] add fp4 gemm + allreduce](../sources/prs/TensorRT-LLM/PR-9729.md), [[None][feat] Port fp4 quantization kernel optimization from FlashInfer](../sources/prs/TensorRT-LLM/PR-9854.md), [bugfix: Fix test and output shape of fp4 quantize](../sources/prs/flashinfer/PR-1114.md), [Add fp4 quantization swizzling tests](../sources/prs/flashinfer/PR-1157.md), [Expose fp4 blockscale swizzling kernel](../sources/prs/flashinfer/PR-1176.md), [Feature/sm100 low latency nvfp4 kernels](../sources/prs/flashinfer/PR-1214.md), [feat: Support MXFP8 x MXFP4 CUTLASS grouped GEMM](../sources/prs/flashinfer/PR-1241.md), [Reduce the JIT compilation time of gen_gemm_sm100_module](../sources/prs/flashinfer/PR-1251.md), [add mm_fp4 use cudnn backend](../sources/prs/flashinfer/PR-1288.md), [Update cutlass fp4 moe kernels](../sources/prs/flashinfer/PR-1294.md), [add cutlass backend for mm_fp4](../sources/prs/flashinfer/PR-1296.md), [Refactor Fused Moe Module](../sources/prs/flashinfer/PR-1309.md), [Support loading autotuned results from json for cutlass fp4 moe backends](../sources/prs/flashinfer/PR-1310.md), [feat: support output nvfp4 in trtllm-gen function call.](../sources/prs/flashinfer/PR-1318.md), [feat: masked layout fp4 gemm using cute-dsl](../sources/prs/flashinfer/PR-1331.md), [add torch float4_e2m1fn_x2 check for cudnn fp4 backend](../sources/prs/flashinfer/PR-1333.md), [[Fix] remove torch 2.8 requirement for FP4 GEMM](../sources/prs/flashinfer/PR-1334.md), [feature: add fp4 mm using trtllm backend](../sources/prs/flashinfer/PR-1355.md), [hotfix: update mxfp4 groupwise-scaled gemm unittests](../sources/prs/flashinfer/PR-1359.md), [support trtllm-gen prefill fp4 output](../sources/prs/flashinfer/PR-1360.md), [Update autotune results for the nvfp4 cutlass moe backends for v0.2.9](../sources/prs/flashinfer/PR-1361.md), [Support scale factor start index for fp4 mha prefill/decode](../sources/prs/flashinfer/PR-1363.md), [bugfix: fixed cutlass fused moe usage of FP4QuantizationSFLayout::SWIZZLED](../sources/prs/flashinfer/PR-1371.md), [bugfix: Add guard for fp4/fp8 related include headers](../sources/prs/flashinfer/PR-1376.md), [GPT-OSS Support: Add Blackwell MoE mxfp4 implementation from TRTLLM and Attention Sink](../sources/prs/flashinfer/PR-1389.md), [gpt-oss: Add MXFP8 x MXFP4 CUTLASS MOE for SM100 and BF16 x MXFP4 CUTLASS for SM90 + SwigluBias Activation](../sources/prs/flashinfer/PR-1396.md), [Add Mxfp4 trtllm-gen moe unit tests](../sources/prs/flashinfer/PR-1399.md), [feature: enable cublas for fp4 gemm when cudnn == 9.11.1 or >= 9.13](../sources/prs/flashinfer/PR-1405.md), [Faster weight processing (moe nvfp4)](../sources/prs/flashinfer/PR-1412.md), [Remove getEnvEnablePDL in favor of enable_pdl parameter](../sources/prs/flashinfer/PR-1446.md), [Fix TRTLLM NVFP4-out attention kernel scale factor dim issue](../sources/prs/flashinfer/PR-1460.md), [tuner: Trtllm-gen Fp4 MoE Autotunner](../sources/prs/flashinfer/PR-1475.md), [fix missing enable_pdl argument in trtllm-gen fp4 moe](../sources/prs/flashinfer/PR-1480.md), [Add python API for masked grouped gemm](../sources/prs/flashinfer/PR-1481.md), [fix: update masked moe gemm fp4 tensor reshape](../sources/prs/flashinfer/PR-1495.md), [feat: scaling at fp4 gemm epilogue](../sources/prs/flashinfer/PR-1498.md), [refactor fp4 masked gemm cute-dsl implementation and add manual cache](../sources/prs/flashinfer/PR-1521.md), [Add GeGLU support to trtllm-gen NVFP4 Fused MoE Kernel](../sources/prs/flashinfer/PR-1525.md), [perf: Enable SplitK and fix tile-scheduling for moe fp4 fused moe](../sources/prs/flashinfer/PR-1548.md), [fix: separate out fp4 lib into sm90 and sm100 versions, add oob checking in fused moe](../sources/prs/flashinfer/PR-1565.md), [update trtllm-gen fp4 autotuner and routing](../sources/prs/flashinfer/PR-1573.md), [bugfix: Fix test_fp4_quantize test bug](../sources/prs/flashinfer/PR-1585.md), [feat: initial support for SM103, SM110, SM120, SM121](../sources/prs/flashinfer/PR-1608.md), [feat: cutlass fp4 gemm bringup for SM120 & SM121](../sources/prs/flashinfer/PR-1609.md), [bugfix: fix fp4 quantization with 8x4 scale factor layout](../sources/prs/flashinfer/PR-1611.md), [feat: add support of fp4_batched_quantize](../sources/prs/flashinfer/PR-1633.md), [Added mx_fp4 support using the cudnn backend](../sources/prs/flashinfer/PR-1644.md), [[Hotfix] `test_fp4_quantize.py` failure on sm103](../sources/prs/flashinfer/PR-1666.md), [feat: Benchmark mm_fp4 mxfp4 support and gemm autotune support. Restore mm_fp4 API behavior](../sources/prs/flashinfer/PR-1706.md), [test: skip the unsupported test cases for sm120/121](../sources/prs/flashinfer/PR-1710.md), [Added xfail for mx_fp4 matmul on SM120](../sources/prs/flashinfer/PR-1766.md), [Masked batch nvfp4 quantization](../sources/prs/flashinfer/PR-1774.md), [Support checks PoC](../sources/prs/flashinfer/PR-1809.md), [fix: fp4 moe on sm120](../sources/prs/flashinfer/PR-1817.md), [[Quantization] Add per-expert global scaling factor for fp4 batched quantize](../sources/prs/flashinfer/PR-1835.md), [raise error for group_gemm_fp8_nt_groupwise then num_groups > 1 on sm120/121](../sources/prs/flashinfer/PR-1862.md), [feat: Add FP4 TRTLLM-Gen throughput MOE batched gemms](../sources/prs/flashinfer/PR-1882.md), [silu_and_mul nvfp4 quanization fusion rework](../sources/prs/flashinfer/PR-1927.md), [fix: Add cutlass as an mm_fp4 backend in compute capability 12.0 in benchmark code](../sources/prs/flashinfer/PR-1959.md), [feat: Add backend='auto' to mm_fp4 and enable autotune for backend='cudnn'](../sources/prs/flashinfer/PR-1979.md), [Feature: Support non-gated activation in cutlass fused MoE nvfp4](../sources/prs/flashinfer/PR-2011.md), [fix: Enable SM121 for mm_fp4](../sources/prs/flashinfer/PR-2012.md), [[DSV3] Optimized Router Gemm](../sources/prs/flashinfer/PR-2019.md), [update trtllm cutlass moe ](../sources/prs/flashinfer/PR-2020.md), [perf: Speed up fp4 quantization for small batch with swizzling for cutlass MoE](../sources/prs/flashinfer/PR-2025.md), [[BUG] Fix trtllm-gen fp4 moe renormalize routing](../sources/prs/flashinfer/PR-2049.md), [Patch sm103 for 3xfp4 moe generation](../sources/prs/flashinfer/PR-2082.md), [perf: enable pdl for cutlass fp4 gemm](../sources/prs/flashinfer/PR-2095.md), [feat: MxInt4 x Bf16 TRT-LLM Gen MoE support](../sources/prs/flashinfer/PR-2159.md), [Add data type check for deepseek fp4 moe](../sources/prs/flashinfer/PR-2165.md), [feat: Support unpadded output hidden size for trtllm_fp4_block_scale_moe](../sources/prs/flashinfer/PR-2217.md), [feat: Fused RMSNorm + FP4 Quantization Kernels in CuTe-DSL](../sources/prs/flashinfer/PR-2233.md), [fix: Add global scale support and optional output allocation for RMSNorm+FP4Quant fusion kernels](../sources/prs/flashinfer/PR-2260.md), [[performance]optimize for nvfp4](../sources/prs/flashinfer/PR-2268.md), [[WIP] Refactor: simplify torch -> cute-dsl boilerplate and enable tvm-ffi for cute-dsl kernels](../sources/prs/flashinfer/PR-2279.md), [[Perf][Feature] Add SM103-specific schedulers for NVFP4 CUTLASS kernels](../sources/prs/flashinfer/PR-2303.md), [feat: Support Fused MoE non gated Relu2 NVFP4 & FP8 and support Nemotron](../sources/prs/flashinfer/PR-2304.md), [Optimize quantization function in large problem size](../sources/prs/flashinfer/PR-2343.md), [fix: In-place Residual Update for add_rmsnorm_fp4quant](../sources/prs/flashinfer/PR-2385.md), [feat: Add output_both_sf_layouts option to add_rmsnorm_fp4quant API](../sources/prs/flashinfer/PR-2395.md), [feat: cuteDSL fp4 moe for better DSR1 performance.](../sources/prs/flashinfer/PR-2398.md), [perf: mm_fp4 heuristic prioritizes CUTLASS over cuDNN on SM103](../sources/prs/flashinfer/PR-2404.md), [refactor: simplify fp4 rmsnorm](../sources/prs/flashinfer/PR-2421.md), [refactor: refactoring cuda code to cute-dsl (part 1)](../sources/prs/flashinfer/PR-2428.md), [Add cute-dsl backends to mxfp[8,4]_quantization for future refactor](../sources/prs/flashinfer/PR-2443.md), [perf: add fp4 GEMM tile configs and streamK scheduler for SM120](../sources/prs/flashinfer/PR-2460.md), [feat: Support Fused MoE non gated Relu2 NVFP4 & FP8 and support Nemotron, fixed](../sources/prs/flashinfer/PR-2462.md), [Support NVFP4 KV cache decode on SM120](../sources/prs/flashinfer/PR-2520.md), [feat: cute dsl mmfp4 for blackwell](../sources/prs/flashinfer/PR-2540.md), [[Bugfix][comm] Fix FP4 one-shot launch config instability in trtllm_allreduce_fusion](../sources/prs/flashinfer/PR-2557.md), [[Bug] Fix spark unit test failures for test_add_rmsnorm_fp4_quant_cute_dsl](../sources/prs/flashinfer/PR-2573.md), [tests: add bias testing to nvfp4 moe](../sources/prs/flashinfer/PR-2585.md), [fix: cute dsl nvfp4 moe routing index error](../sources/prs/flashinfer/PR-2629.md), [fix: add SM121 support to SM120 version guards](../sources/prs/flashinfer/PR-2631.md), [benchmark: Add MXFP4/MXFP8 quantization mode support to FP4 MoE benchmark](../sources/prs/flashinfer/PR-2635.md), [Enable sm120f compilation](../sources/prs/flashinfer/PR-2650.md), [[feat] trtllm-gen mxfp8 gemm](../sources/prs/flashinfer/PR-2653.md), [feat: support mxfp4 & mxfp8 entrypoint for blackwell cutedsl dense gemm](../sources/prs/flashinfer/PR-2660.md), [perf: Update trtllm-gen batched GEMM kernels - faster, more NVFP4 tile dims, MXFP8 with relu2 act](../sources/prs/flashinfer/PR-2667.md), [Add NVFP4 KV cache quantization support for SM100](../sources/prs/flashinfer/PR-2702.md), [fix: Add SM120 (RTX Blackwell desktop) support for NVFP4 MoE kernels](../sources/prs/flashinfer/PR-2725.md), [Support for MXFP4 and NVFP4 group GEMMs on GeForce and Spark](../sources/prs/flashinfer/PR-2738.md), [feat: Add FP4 KV cache quant/dequant kernels ](../sources/prs/flashinfer/PR-2757.md), [feat: Add CuTe-DSL backend for NVFP4 quantization](../sources/prs/flashinfer/PR-2838.md), [fix: int32 overflow in `trtllm_fp4_block_scale_moe` causing "Unsupported hidden state scale shape" for EP32+ configs](../sources/prs/flashinfer/PR-2853.md), [fix: snap weight_scale_vec_size to handle block_scale_interleave padding for SM120](../sources/prs/flashinfer/PR-2898.md), [perf: Optimize CuTe-DSL fp4 and fp8 quantization kernels](../sources/prs/flashinfer/PR-2904.md), [fix: Fix autotuner crash on meta-device tensor in trtllm_fp4_block_scale_routed_moe](../sources/prs/flashinfer/PR-2916.md), [CuTe DSL FP4 GEMM Heuristic](../sources/prs/flashinfer/PR-2940.md), [Only swizzle on v block scale; rename kv_block_scales to kv_cache_sf](../sources/prs/flashinfer/PR-2954.md), [[Fmha] support nvfp4 output keepsMmaAb generation kernels](../sources/prs/flashinfer/PR-2988.md), [ Fix MXFP4/MXFP8 failures in SM120 FAST_BUILD and expand all_tiles[] ](../sources/prs/flashinfer/PR-2994.md), [feat: add PDL support to rmsnorm_fp4quant and add_rmsnorm_fp4quant CuTe DSL kernels](../sources/prs/flashinfer/PR-3008.md), [perf: Port TRT-LLM SM120/SM121 FP4 CUTLASS GEMM optimizations. Add PDL](../sources/prs/flashinfer/PR-3026.md), [[feat] Trtllm-gen Per-token Nvfp4 MoE](../sources/prs/flashinfer/PR-3027.md), [feat: Add backend="b12x" for mm_fp4 on SM120](../sources/prs/flashinfer/PR-3051.md), [feat: Add b12x CuTe DSL fused MoE for SM120](../sources/prs/flashinfer/PR-3066.md), [Support NVFP4 KV for prefill and batch attention kernels](../sources/prs/flashinfer/PR-3097.md), [Integrate CUTLASS Small Tile N Blockscaled GEMMs/Grouped GEMMs for SM120 and SM121](../sources/prs/flashinfer/PR-3152.md), [feat(trace): embed runnable init() in every TraceTemplate](../sources/prs/flashinfer/PR-3221.md), [perf: optimize per-token nvfp4 quantization kernel.](../sources/prs/flashinfer/PR-3237.md), [feat(moe): add SM120 W4A16 b12x kernels](../sources/prs/flashinfer/PR-3271.md), [feat: Add FP4 (E2M1) KV Cache Support with Quantization Utilities for MLA](../sources/prs/sglang/PR-10078.md), [Optimize nvfp4 block scaled gemm kernel when M is small.](../sources/prs/sglang/PR-10101.md), [Fix chunked prefix cache for nvfp4](../sources/prs/sglang/PR-10180.md), [Fix correction bias undefined behavior for nvfp4 models](../sources/prs/sglang/PR-10426.md), [feat: add dsv3 fp4 cutlass moe etp ut](../sources/prs/sglang/PR-10433.md), [Fix bias handling in TritonMoeQuantInfo within quantization/mxfp4.py](../sources/prs/sglang/PR-10579.md), [Fix MTP MoE weight loading with NVFP4 target model.](../sources/prs/sglang/PR-10758.md), [[NVIDIA] Add new SMs support for Spark & Thor](../sources/prs/sglang/PR-11287.md), [Support running FP4 Deepseek on SM120.](../sources/prs/sglang/PR-11708.md), [support cutlass fp4 kernel in sm120](../sources/prs/sglang/PR-11737.md), [Use cutlass fp4 gemm by default](../sources/prs/sglang/PR-11813.md), [Support nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8/NVFP4](../sources/prs/sglang/PR-11866.md), [Feature/nano v2 offline modelopt fp8 and nvfp4](../sources/prs/sglang/PR-12018.md), [Replace [silu_and_mul_]scaled_fp4_group_quant by Flashinfer equivalent](../sources/prs/sglang/PR-12376.md), [Use sgl fp4 quant kernel by default](../sources/prs/sglang/PR-12482.md), [[NVIDIA] Fix CUDA arch requirement in nvfp4 cast](../sources/prs/sglang/PR-12581.md), [feat: Add FP4 (E2M1) KV Cache Support for MHA](../sources/prs/sglang/PR-12612.md), [[NVIDIA] Fix wrong symmetric sizes for fp4 cases](../sources/prs/sglang/PR-12640.md), [[Bugfix] Fix illegal memory access](../sources/prs/sglang/PR-12758.md), [ignore the deepgemm check when the model weight with nvfp4 and moe ba…](../sources/prs/sglang/PR-12782.md), [support mtp with deepseek r1 nvfp4 model](../sources/prs/sglang/PR-13115.md), [Fix nan in global scaling factor for large scale nvfp4 EP](../sources/prs/sglang/PR-13162.md), [Fix EPLB + FP4 Quantization Compatibility Issue](../sources/prs/sglang/PR-13715.md), [[Feat][NVFP4] Enable NVFP4 MoE for Qwen series models (eg. Qwen3-Next) #13761](../sources/prs/sglang/PR-13761.md), [Support fp4 fp8 non gated moe](../sources/prs/sglang/PR-13794.md), [Fix flashinfer cutlass MoE output shape for non-FP4-packed inputs](../sources/prs/sglang/PR-14028.md), [[FIX] trtllm-moe-fp4-renorm for Qwen series models](../sources/prs/sglang/PR-14350.md), [[CPU] Implement MXFP4 Gemm kernels for intel AMX to support GPT OSS series.](../sources/prs/sglang/PR-14385.md), [Mistral Large 3 NVFP4 support](../sources/prs/sglang/PR-14485.md), [[AMD] Support fused_rms_mxfp4_quant in the prefill stage for DeepSeek-R1-MXFP4](../sources/prs/sglang/PR-14975.md), [Mistral Large 3 NVFP4 TRTLLM MoE support](../sources/prs/sglang/PR-15049.md), [[NVIDIA] Fixes for NVFP4 all-gather with spec decoding](../sources/prs/sglang/PR-15280.md), [Fix the accuracy issue when running mxfp4 dsv3 model and enable ep](../sources/prs/sglang/PR-15304.md), [Update flashinfer to 0.6.1](../sources/prs/sglang/PR-15551.md), [[Perf] Eliminate the slice op for Flashinfer `trtllm_fp4_block_scale_moe`](../sources/prs/sglang/PR-15731.md), [Tiny fix cannot launch nvfp4 checkpoint with bf16 kv cache](../sources/prs/sglang/PR-15986.md), [[Performance] Force split_k=1 for MXFP4 Triton kernels on Hopper](../sources/prs/sglang/PR-16014.md), [Inclusion of nvfp4 blockscale in EPLB Rebalance](../sources/prs/sglang/PR-17158.md), [[Fix] GLM 4.7 + NVFP4 + MTP](../sources/prs/sglang/PR-17166.md), [[FIX] Always support TP > 4 for FP4 Gemm](../sources/prs/sglang/PR-17300.md), [[feat] Support nvfp4 quantized model of Qwen3-Next](../sources/prs/sglang/PR-17627.md), [fix(quantization): add sgl_kernel fallback for FP4 quantize on Blackwell GPUs](../sources/prs/sglang/PR-17816.md), [[Bugfix] Fix Mistral Large 3 NVFP4 TRTLLM MoE](../sources/prs/sglang/PR-18065.md), [Fix nvfp4 weight update](../sources/prs/sglang/PR-18085.md), [[ModelOpt] Fix broken Qwen3-235B-A22B-Instruct-2507-NVFP4 launch](../sources/prs/sglang/PR-18189.md), [[ModelOPT] Support Qwen 3 Next Coder NVFP4](../sources/prs/sglang/PR-18224.md), [[Kimi-K2.5] Fix NVFP4 Kimi-K2.5 weight mapping and exclude list](../sources/prs/sglang/PR-18370.md), [Nsa trtllm mla sparse fp8 support with Deepseek v3.2 NVFP4](../sources/prs/sglang/PR-18389.md), [[Perf] ~9.5x faster Blackwell MXFP4 MoE weight loading](../sources/prs/sglang/PR-18858.md), [[Qwen3.5] Enable nvfp4 checkpoint](../sources/prs/sglang/PR-18937.md), [feat: Support MXFP4 quantized dense models on AMD CDNA2/CDNA3 GPUs](../sources/prs/sglang/PR-19143.md), [Adjust padding size to improve triton_kernels moe performance](../sources/prs/sglang/PR-19174.md), [Fix nightly Mistral-Large-3 NVFP4 accuracy threshold](../sources/prs/sglang/PR-19402.md), [[AMD] Fix weight load shape mismatch for amd dsr1 0528 mxfp4](../sources/prs/sglang/PR-19425.md), [[Kernel Slimming] Migrate NVFP4 kernels to JIT](../sources/prs/sglang/PR-19437.md), [[Feature] NVFP4 Marlin fallback for non-Blackwell GPUs (SM75+)](../sources/prs/sglang/PR-19652.md), [Support `triton_kernels` for GPT-OSS on SM120](../sources/prs/sglang/PR-19718.md), [[AMD] Fix FP8 assertion failure in aiter MLA decode by falling back to self.k_scale](../sources/prs/sglang/PR-19935.md), [[JIT Kernel] Reland NVFP4 kernels to JIT](../sources/prs/sglang/PR-20012.md), [Fix SM120 `triton_kernels` MXFP4 `block_k` for GPT-OSS](../sources/prs/sglang/PR-20040.md), [MiniMax-M2.5 - Support dp attention, dp reduce scatter, FP4 all gather, AR fusion in prepare_attn](../sources/prs/sglang/PR-20067.md), [[V32/GLM5] Change default setting of V32 nvfp4 on TP4](../sources/prs/sglang/PR-20086.md), [[diffusion] Support nvfp4 for Flux.2](../sources/prs/sglang/PR-20137.md), [[4/n jit_kernel restruct] speed up CI tests and add benchmark workflow](../sources/prs/sglang/PR-20268.md), [[Benchmark] use flashinfer bench_gpu_time instead of triton do_bench](../sources/prs/sglang/PR-20305.md), [[Model] Support Nemotron 3 Super NVFP4](../sources/prs/sglang/PR-20407.md), [[JIT Kernel] Fix NVFP4 multi-arch compilation failure](../sources/prs/sglang/PR-20874.md), [Add SGLang CUDA crash API logging inspired by FlashInfer](../sources/prs/sglang/PR-20910.md), [[Chore] Clean up JIT compilation flags](../sources/prs/sglang/PR-21022.md), [[AMD]: Support MLA with nhead<16 and FP8 KV cache for TP=8 (Kimi K2.5…](../sources/prs/sglang/PR-21213.md), [Refactor JIT kernel CI to use run_suite.py registration system](../sources/prs/sglang/PR-21239.md), [[NVIDIA] Enable FP4 flashinfer trtllm routed moe](../sources/prs/sglang/PR-21240.md), [CUTLASS NVFP4 GEMM improvement of SM120](../sources/prs/sglang/PR-21314.md), [[Kernel] Support FlashInfer TRTLLM-Gen fused MoE for non-gated FP4 & FP8 (Nemotron)](../sources/prs/sglang/PR-21321.md), [[misc] clean up kernel API](../sources/prs/sglang/PR-21325.md), [Add dedicated FlashInferCuteDslMoE layer for standard-path FP4 MoE](../sources/prs/sglang/PR-21339.md), [Migrate all callers from /get_server_info to /server_info](../sources/prs/sglang/PR-21463.md), [Harden FlashInfer FP4 imports in standard dispatcher](../sources/prs/sglang/PR-21776.md), [[Diffusion] Fix weight scale swizzle and add large-M kernel config for FLUX.2-dev-NVFP4](../sources/prs/sglang/PR-22064.md), [[nvidia] Gemma4 nvfp4 fix](../sources/prs/sglang/PR-22079.md), [[diffusion] Default NVFP4 to CUTLASS and add all-model shape benchmarks](../sources/prs/sglang/PR-22091.md), [[Diffusion] Add diffusion NVFP4 scaled-mm correctness test](../sources/prs/sglang/PR-22127.md), [[RL] Refactor NVFP4 shuffling/swizzling to in-place replacement](../sources/prs/sglang/PR-22204.md), [:sparkles: [diffusion][npu][quant] Add MXFP4 quantization support for Wan2.2 Diffusion on Ascend NPU](../sources/prs/sglang/PR-22338.md), [GLM-5/5.1 MXFP4 Checkpoint Inference Compatibility Fix](../sources/prs/sglang/PR-22543.md), [[Diffusion] Add FLUX.1-dev ModelOpt NVFP4 support](../sources/prs/sglang/PR-22574.md), [reland [Diffusion] Add FLUX.1-dev ModelOpt NVFP4 support](../sources/prs/sglang/PR-22672.md), [[Diffusion] Add Wan2.2 ModelOpt NVFP4 support](../sources/prs/sglang/PR-22681.md), [[FlashInfer v0.6.11] [RL] Support FlashInfer per-token NVFP4 MoE](../sources/prs/sglang/PR-22918.md), [Reland Cute-DSL FP4 dense GEMM](../sources/prs/sglang/PR-23590.md), [Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-23686.md), [Use Cute-DSL NVFP4 quantization kernels](../sources/prs/sglang/PR-23745.md), [[MoE] Unify DeepEPMoE+MoriEPMoE through AITER MoeRunner pre/post-permute](../sources/prs/sglang/PR-23760.md), [Port MXFP4 Marlin MoE support to JIT kernel path](../sources/prs/sglang/PR-24490.md), [Add FlashInfer SM90 cutlass MXFP4 MoE backend (W4A16) for GPT-OSS + DeepSeek-V4](../sources/prs/sglang/PR-24816.md), [[rebase]Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-24986.md), [perf(nvfp4): free unused source scales after weight processing](../sources/prs/sglang/PR-25107.md), [[Intel GPU] Enable DeepSeek V4 Inference on XPU](../sources/prs/sglang/PR-25336.md), [[AMD] Enable shared-experts fusion with new KIMI-K2.5-MXFP4 model.](../sources/prs/sglang/PR-25390.md), [[codex] Update Wan2.2 ModelOpt CI checkpoints](../sources/prs/sglang/PR-25483.md), [Add no_combine support to cutlass_moe_fp4](../sources/prs/sglang/PR-25688.md), [[Test] Stage-a sanity kits; consolidate core/ + models_e2e/ tests](../sources/prs/sglang/PR-25831.md), [Support FP4 gemm (1/2)](../sources/prs/sglang/PR-3899.md), [[Build] Fix cuda12.8 build error in nvfp4_scaled_mm_kernels.cu](../sources/prs/sglang/PR-4953.md), [[1/2] Add Kernel support for Cutlass based Fused FP4 MoE](../sources/prs/sglang/PR-6093.md), [[DeepseekR1-FP4] Add Support for nvidia/DeepSeekR1-FP4 model](../sources/prs/sglang/PR-6853.md), [Fix Deepseek R1 0528 FP4 tensor name mismatch issue during weights loading.](../sources/prs/sglang/PR-7164.md), [Support NVFP4 quantized dense models on AMD CDNA2/CDNA3 GPUs](../sources/prs/sglang/PR-7302.md), [FlashInfer NVFP4 MoE with EP & 2-stream shared expert](../sources/prs/sglang/PR-7327.md), [Fix MTP with Deepseek R1 Fp4](../sources/prs/sglang/PR-7376.md), [Add fp4 quantize before all-gather for Flashinfer cutlass MoE DP (max throughput)](../sources/prs/sglang/PR-7667.md), [Qwen FP8/NVFP4 ModelOPT Quantization support](../sources/prs/sglang/PR-7912.md), [[Fix][Ready]Fix register spilling in cutlass nvfp4 gemm kernel on Blackwell](../sources/prs/sglang/PR-8127.md), [[fix] fix modelopt fp4 on b200](../sources/prs/sglang/PR-8195.md), [[NVIDIA] Add Low Latency NVFP4 decode kernels from Flashinfer](../sources/prs/sglang/PR-8552.md), [[Perf] Auto enable best flashinfer mxfp4 kernel in b200](../sources/prs/sglang/PR-8898.md), [Fix hopper launch gpt-oss model illegal memory](../sources/prs/sglang/PR-8908.md), [Faster weight processing (trtllm-gen moe nvfp4)](../sources/prs/sglang/PR-9162.md), [[NVIDIA] [3/N] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked ](../sources/prs/sglang/PR-9199.md), [[NVIDA] [1/N] Nvfp4 Masked Gemm: Add quant op for the flashinfer grouped gemm](../sources/prs/sglang/PR-9200.md), [Fix FP4 inference corruption issue in glm4.5-air model](../sources/prs/sglang/PR-9346.md), [[fix] Fix mxfp4 triton MoE tp bug](../sources/prs/sglang/PR-9473.md), [[NVIDIA] [2/N] Optimize `silu_and_mul_scaled_fp4_grouped_quant` perf](../sources/prs/sglang/PR-9556.md), [move is_sm90_supported/is_sm100_supported to python/sglang/srt/utils.py](../sources/prs/sglang/PR-9679.md), [[ModelOpt] Fix Weight Loading for DSR1-FP4 Quantization](../sources/prs/sglang/PR-9712.md), [Make fp4_quantize kernels work on sm103](../sources/prs/sglang/PR-9807.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [[TMA] Support FP4 TensorMap TMA copies](../sources/prs/tilelang/PR-2107.md), [[TIR][IR] Update to use tirx](../sources/prs/tilelang/PR-2216.md), [[NVIDIA] Support nvfp4 quantization](../sources/prs/vllm/PR-12784.md), [[NVIDIA] Support nvfp4 tcgen05 gemm](../sources/prs/vllm/PR-13571.md), [[Hardware/NVIDIA/Kernel] [Functional Enablement] [1/N] Enable nvidia/DeepSeek-R1-FP4 Model](../sources/prs/vllm/PR-16362.md), [[Kernel] fp4 marlin kernel](../sources/prs/vllm/PR-17687.md), [[Misc] Add compressed-tensors NVFP4A16 emulation support](../sources/prs/vllm/PR-17914.md), [Use NVFP4 Marlin for CompressedTensorsW4A16Fp4](../sources/prs/vllm/PR-18000.md), [[Quantization] Add compressed-tensors NVFP4 support](../sources/prs/vllm/PR-18312.md), [[V1] Support `LLM.apply_model`](../sources/prs/vllm/PR-18465.md), [[Hardware][NVIDIA] FP4 MoE kernel optimization](../sources/prs/vllm/PR-19110.md), [[Hardware][NVIDIA][kernel] Fp4 MOE quant kernel optimization](../sources/prs/vllm/PR-19500.md), [[Quantization] Add compressed-tensors emulations support for NVFP4](../sources/prs/vllm/PR-19879.md), [[Quantization] Add compressed-tensors NVFP4 MoE Support](../sources/prs/vllm/PR-19990.md), [Add ModelOpt Qwen3 nvfp4 support](../sources/prs/vllm/PR-20101.md), [[Bugfix] Fix some narrowing conversion warnings](../sources/prs/vllm/PR-20141.md), [[Kernel][Bugfix] Fixup some warnings in nvfp4_blockwise_moe when CUDA < 12.8](../sources/prs/vllm/PR-20324.md), [Support Llama 4 for cutlass_moe_fp4](../sources/prs/vllm/PR-20453.md), [[Perf] Reuse workspace for FP8+FP4 Marlin MoE](../sources/prs/vllm/PR-20500.md), [Support mnnvl all2allv from Flashinfer](../sources/prs/vllm/PR-21003.md), [[Feature][OCP MX] Support mxfp6 and mixed mxfp6-mxfp4](../sources/prs/vllm/PR-21166.md), [Support CUTLASS NVFP4 (w4a4) for Blackwell Geforce GPUs (SM120)](../sources/prs/vllm/PR-21309.md), [Support Tensorrt-LLM MoE fp4 for low-latency](../sources/prs/vllm/PR-21331.md), [Update flashinfer CUTLASS NVFP4 MoE Kernel to use per expert global scaling factor](../sources/prs/vllm/PR-21408.md), [[Bug] Fix Compressed Tensor NVFP4 `cutlass_fp4_group_mm` illegal memory access](../sources/prs/vllm/PR-21465.md), [[NVIDIA] Fix Llama4 Scout FP4 functionality issues](../sources/prs/vllm/PR-21499.md), [[Feature] Add Flashinfer MoE Support for Compressed Tensor NVFP4](../sources/prs/vllm/PR-21639.md), [[gpt-oss] flashinfer mxfp4](../sources/prs/vllm/PR-22339.md), [[gpt-oss] triton kernel mxfp4](../sources/prs/vllm/PR-22421.md), [Fix Llama4 FlashInfer FP4 MoE issues](../sources/prs/vllm/PR-22511.md), [Quantization: support FP4 quantized models on AMD CDNA2/CDNA3 GPUs](../sources/prs/vllm/PR-22527.md), [Fix torch version check for SM100 mxfp4 ](../sources/prs/vllm/PR-22535.md), [[Quantization] Expand compressed-tensors MoE matching logic to support NFP4 + FP8 MoEs](../sources/prs/vllm/PR-22674.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv NVFP4-out Attention Kernel](../sources/prs/vllm/PR-22703.md), [Use Blackwell FlashInfer MXFP4 MoE by default if available ](../sources/prs/vllm/PR-23008.md), [Add routed_scaling_factor to MoE grouped topk](../sources/prs/vllm/PR-23123.md), [Fix nvfp4 swizzling](../sources/prs/vllm/PR-23140.md), [[Kernels] Overlap shared experts with send/recv](../sources/prs/vllm/PR-23273.md), [fix incompatibililty with non cuda platform for nvfp4](../sources/prs/vllm/PR-23478.md), [Update Flashinfer to 0.2.14.post1](../sources/prs/vllm/PR-23537.md), [DP/EP Support for gpt-oss with deepep-ht comm kernel on SM100](../sources/prs/vllm/PR-23608.md), [[Bugfix] Fix Marlin NVFP4 for modelopt](../sources/prs/vllm/PR-23659.md), [[NVIDIA] Support SiluMul + NVFP4 quant fusion](../sources/prs/vllm/PR-23671.md), [[Kernel][tcgen05] nvfp4 fused tcgen05 moe](../sources/prs/vllm/PR-23696.md), [[Bugfix][Misc] Fix silu_and_mul_nvfp4_quant issue and extract common utils for nvfp4 kernel source files](../sources/prs/vllm/PR-23727.md), [[Model][gpt-oss] Support DP+EP for GPT-OSS with FlashInfer trtllm-gen MoE](../sources/prs/vllm/PR-23819.md), [[BUGFIX ] fix undefined silu_and_mul_nvfp4_quant](../sources/prs/vllm/PR-23929.md), [[Model] Add LongCat-Flash ](../sources/prs/vllm/PR-23991.md), [[Transform] [Quantization] Add QuTLASS support to vLLM](../sources/prs/vllm/PR-24440.md), [[Kernel][Quantization] add w4a8 support for marlin kernel](../sources/prs/vllm/PR-24722.md), [[Bugfix] Fix accuracy issue for silu_mul + nvfp4 quant fusion kernel](../sources/prs/vllm/PR-24833.md), [[Compile] Fix Compile Warning for Ignoring `MIN_BLOCK_PER_SM`](../sources/prs/vllm/PR-25193.md), [[ROCm] Small functional changes for gptoss](../sources/prs/vllm/PR-25201.md), [Enable Fbgemm NVFP4 on Dense models](../sources/prs/vllm/PR-25609.md), [[Bugfix] Enable padded FP4 quantization](../sources/prs/vllm/PR-25947.md), [[Quantization/NVFP4] Speed up TRTLLM NVFP4 MOE weight loading and fix K/V scale loading for MLA Attn](../sources/prs/vllm/PR-25968.md), [[Bugfix] Allow skipping MoE in NVFP4 (fix for MTP)](../sources/prs/vllm/PR-25987.md), [[MoE] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked](../sources/prs/vllm/PR-25990.md), [[NVIDIA] Add support for cudnn fp4 gemm via flashinfer](../sources/prs/vllm/PR-26107.md), [[ModelOpt] Load w13/w2_input_scale for all experts, nvfp4](../sources/prs/vllm/PR-26135.md), [[ROCM] MoE fp4 CK kernel](../sources/prs/vllm/PR-26545.md), [support flashinfer_fp4 moe for 5090 gpu](../sources/prs/vllm/PR-26669.md), [[NVIDIA] [Perf] Update to leverage flashinfer trtllm FP4 MOE throughput kernel](../sources/prs/vllm/PR-26714.md), [[Bugfix] Fix gpt-oss w4a8 DP/EP on B200](../sources/prs/vllm/PR-26729.md), [Flashinfer_CUTLASS_MOE fuses quantization for TP](../sources/prs/vllm/PR-27223.md), [[Attention] Use sparse prefill kernel for fp8 kv-cache in DeepSeek-v3.2](../sources/prs/vllm/PR-27532.md), [[Bugfix] Fix GPT-OSS on AMD after #28603](../sources/prs/vllm/PR-28816.md), [Add TRTLLM MoE NVFP4 kernel to CompressedTensorsW4A4MoeMethod](../sources/prs/vllm/PR-28892.md), [[Feat] Support non-gated activations in NVFP4 modelopt path](../sources/prs/vllm/PR-29004.md), [[Kernel] Add NVFP4 MoE CUTLASS support for SM120](../sources/prs/vllm/PR-29242.md), [[Bugfix] Only use triton_kernels for MXFP4 on SM90 and SM100](../sources/prs/vllm/PR-29339.md), [[Bugfix] Fix mismatched nvfp4 gemm output shape](../sources/prs/vllm/PR-29742.md), [[ROCm][MXFP4] Infer w4a4 quant method in rocm aiter fused moe](../sources/prs/vllm/PR-29775.md), [[EPLB] Support EPLB w/ NVFP4](../sources/prs/vllm/PR-29804.md), [[Perf] Do FP4 quant before All gather on flashinfer trtllmgen MOE ](../sources/prs/vllm/PR-30014.md), [[ROCm][Quantization] GPT OSS Upstream MoE wmxfp4_afp8 with static scales](../sources/prs/vllm/PR-30357.md), [[Feature] Add SM103 (Blackwell Ultra) Support to vLLM](../sources/prs/vllm/PR-30484.md), [[Perf] Set split_k to 1 for triton_kernels](../sources/prs/vllm/PR-30528.md), [[Perf] Eliminate padding and slicing op for GPT-OSS with Flashinfer MXFP4 MXFP8 MoE](../sources/prs/vllm/PR-30647.md), [[SM100] Enable fp8 compute for prefill MLA](../sources/prs/vllm/PR-30746.md), [[Compressed-Tensors] Simplify NVFP4 Conditions, enable marlin support for NVFP4A16 MoEs](../sources/prs/vllm/PR-30881.md), [[Kernel][Performance] Enable smaller Scaling Factor tiling for NVFP4 small-batch decoding](../sources/prs/vllm/PR-30885.md), [[NVFP4][Perf] Tune NVFP4 input quant kernel for small batch size](../sources/prs/vllm/PR-30897.md), [ [FIX] Always support TP > 4 for FP4 Gemm](../sources/prs/vllm/PR-31099.md), [[Bugfix] Fix Broken ModelOpt NVFP4 MoE](../sources/prs/vllm/PR-31742.md), [[Perf] Fuse stride preparation for NVFP4 cutlass_moe](../sources/prs/vllm/PR-31837.md), [[5/N][Attention] Finish eliminating `vllm/attention` folder](../sources/prs/vllm/PR-32064.md), [[Perf][Kernel] Optimize FP4 quantization kernels (SM100F)](../sources/prs/vllm/PR-32520.md), [Support compress-tensors with nvfp4 or fp8 weights and modelopt with nvfp4 weights on Turing](../sources/prs/vllm/PR-33076.md), [fix: Add SM120 (RTX Blackwell) support for FlashInfer CUTLASS NVFP4 MoE kernels](../sources/prs/vllm/PR-33417.md), [[Kernel] Support Flashinfer trtllm fused MoE non gated FP8 & NVFP4](../sources/prs/vllm/PR-33506.md), [[Bugfix] Fix DSV3.2 NVFP4](../sources/prs/vllm/PR-33932.md), [[Bugfix]fix output Nan/Inf in marlin if dtype=float16](../sources/prs/vllm/PR-33972.md), [[ModelBash][DSR1 NVFp4] Avoid Bf16 Bias Cast](../sources/prs/vllm/PR-34298.md), [[Custom Ops] Add functional + out variant for scaled_fp4_quant](../sources/prs/vllm/PR-34389.md), [[BUGFIX] Fix accuracy regression for NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4 with TP>1](../sources/prs/vllm/PR-34476.md), [[Model] Add NVFP4 quantization support for Step3.5-Flash](../sources/prs/vllm/PR-34478.md), [[Bugfix] Rescale NVFP4 weight scales to fix BF16 dequant underflow](../sources/prs/vllm/PR-34577.md), [[torch.compile] Turn on silu+fp4 quant fusion by default for O1+](../sources/prs/vllm/PR-34718.md), [[Bugfix] Fix NVFP4 TRTLLM MoE non-gated support; add gsm8k for Nemotron-3-Nano FP8+NVFP4](../sources/prs/vllm/PR-34725.md), [[BugFix] Fix fp4 quant kernel on CUDA 12.8](../sources/prs/vllm/PR-35210.md), [fix(mxfp4): return is_monolithic=False when LoRA is enabled for Triton backend](../sources/prs/vllm/PR-35382.md), [[NVFP4] Support NVFP4 dense models from `modelopt` and `compressed-tensors` on AMD Instinct MI300, MI355X and Hopper through emulation](../sources/prs/vllm/PR-35733.md), [[Bugfix] Fix passing of activation_type to trtllm fused MoE NVFP4 and FP8](../sources/prs/vllm/PR-36017.md), [[Mamba] Flashinfer selective_state_update](../sources/prs/vllm/PR-36162.md), [[mla] Support fused FP8/NVFP4 output quantization in MLA attention (#35792)](../sources/prs/vllm/PR-36205.md), [[Bug][MoE] Fix TRTLLM NVFP4 Routing Kernel Precision](../sources/prs/vllm/PR-36725.md), [[Bug][MoE] Strengthen _supports_current_device() checks in the TRTLLM FP8, NVFP4, and FlashInfer CuteDSL MoE experts](../sources/prs/vllm/PR-36728.md), [[MoE Refactor] Mxfp4 oracle rebased](../sources/prs/vllm/PR-37128.md), [Fix minimax m2.5 nvfp4 kv scales weight loading](../sources/prs/vllm/PR-37214.md), [[MoE/EPLB] Fix FlashInfer nvfp4 experts + EPLB correctness](../sources/prs/vllm/PR-37217.md), [[Kernel] Add non-gated support for NVFP4 CUTLASS MoE](../sources/prs/vllm/PR-37320.md), [[Bugfix] Fix EP weight filter breaking EPLB and NVFP4 accuracy](../sources/prs/vllm/PR-37322.md), [Add nvfp4 support to reshape_and_cache_flash](../sources/prs/vllm/PR-37332.md), [[Kernel] Add MXFP4 W4A4 CUTLASS MoE kernel for SM100](../sources/prs/vllm/PR-37463.md), [[Bugfix] Remove assertion for NVFP4 scale dynamic range](../sources/prs/vllm/PR-37465.md), [[Bugfix] Fix marlin nvfp4 rescaling](../sources/prs/vllm/PR-37502.md), [[4/n] Migrate FP4/W4A8 CUTLASS kernels to torch stable ABI](../sources/prs/vllm/PR-37503.md), [[Perf] Use torch compile to fuse pack topk in trtllm moe](../sources/prs/vllm/PR-37695.md), [[Bugfix] Preserve CUDA arch suffix (a/f) for SM12x — fixes NVFP4 NaN on desktop Blackwell](../sources/prs/vllm/PR-37725.md), [[MoE] Move FlashInfer CuteDSL experts into fused_moe/experts/](../sources/prs/vllm/PR-37759.md), [[MoE Kernel] Flashinfer nvfp4 cutedsl moe kernel integration](../sources/prs/vllm/PR-38050.md), [[Bugfix] Fix DeepGemm E8M0 accuracy degradation for Qwen3.5 FP8 on Blackwell](../sources/prs/vllm/PR-38083.md), [Fix NaN from stale FP4 scale padding in create_fp4_scale_tensor](../sources/prs/vllm/PR-38148.md), [[Quantization] Add FlashInfer CuteDSL batched experts backend for NVFP4 MoE](../sources/prs/vllm/PR-38251.md), [[MoE] Add RoutingMethodType.Simulated to TRT-LLM FP8/NVFP4 kernel allowlists](../sources/prs/vllm/PR-38329.md), [[NVIDIA] Bugfix NVFP4 DGX Spark and RTX50](../sources/prs/vllm/PR-38423.md), [[Kernels][MoE] Fix legacy_routing to use bitmatrix-based routing path](../sources/prs/vllm/PR-38504.md), [[Compile] Fix nvfp4 compile warning](../sources/prs/vllm/PR-38573.md), [[Bugfix] Fix NVFP4+MTP crash: force unquantized mtp.fc for Qwen3.5](../sources/prs/vllm/PR-38832.md), [[MoE Refactor] Split up compressed_tensors_moe.py](../sources/prs/vllm/PR-38960.md), [[MoE] Move GPT OSS Triton kernel experts into fused_moe/experts/](../sources/prs/vllm/PR-39007.md), [[Refactor] Move NVFP4 GEMM management into NvFp4LinearKernel](../sources/prs/vllm/PR-39129.md), [[Feature] Batch invariant nvfp4 linear support](../sources/prs/vllm/PR-39322.md), [[Kernel] Support TRTLLM GEN NVFP4 MoE for non-512-aligned hidden dims via weight padding](../sources/prs/vllm/PR-39510.md), [[Bugfix] Reject non-nvfp4 dtypes when using the flashinfer_nvlink_one_sided all2all backend](../sources/prs/vllm/PR-39717.md), [[Bug] Fix batch invariance nvfp4 support](../sources/prs/vllm/PR-39820.md), [[Bugfix] Temporarily disable B200 fp4 MoE layer tests](../sources/prs/vllm/PR-40057.md), [Add nvfp4 kv cache support](../sources/prs/vllm/PR-40177.md), [[Bugfix] Guard mxfp4_experts_quant bindings on ENABLE_NVFP4_SM100](../sources/prs/vllm/PR-40191.md), [[MoE] Move cutlass moe to fused_moe/experts/](../sources/prs/vllm/PR-40574.md), [[DSV4] Add BF16 and MXFP8 A2A support for flashinfer a2a one sided](../sources/prs/vllm/PR-40960.md), [[Kernel][MoE] Support GELU on TRT-LLM NvFP4 fused MoE for Gemma4](../sources/prs/vllm/PR-41050.md), [[ROCm][Quantization][3/N] Refactor quark_moe w4a4 w/ oracle](../sources/prs/vllm/PR-41436.md), [[Quantization] Rework quantization_config to use QuantKey and allow for activation override](../sources/prs/vllm/PR-41566.md), [[MXFP4] Support for linear layers + compressed-tensors integration](../sources/prs/vllm/PR-41664.md), [Add NVFP4 all-gather GEMM fusion for AsyncTP](../sources/prs/vllm/PR-41882.md), [[CPU] Add MXFP4 W4A16 MoE support](../sources/prs/vllm/PR-41922.md), [[MoE] Move various experts classes to fused_moe/experts/](../sources/prs/vllm/PR-41979.md), [[Bugfix] Add swiglu limits to deepgemm fp8 methods](../sources/prs/vllm/PR-41986.md), [[Kernel] Pack topk id/weights triton kernel](../sources/prs/vllm/PR-42527.md), [[Bugfix] fix swiglu limit issue for humming backend + deepseek v4](../sources/prs/vllm/PR-42541.md), [[Perf] Padded nvfp4 quant kernel to remove additional copy, 2.4%~5.7% e2e performance improvement](../sources/prs/vllm/PR-42774.md), [[Perf] Re-enable flashinfer autotune by default and cleanup](../sources/prs/vllm/PR-42857.md), [NVFP4 and Block-Scaled Narrow Precision](../wiki/hardware/nvfp4.md), [NVFP4 GEMM — 4-bit Floating Point Matrix Multiply](../wiki/kernels/nvfp4-gemm.md), [NVFP4 Batched GEMV](../wiki/kernels/nvfp4-gemv.md), [TensorRT-LLM Blackwell FP4 DSA Indexer](../wiki/kernels/tensorrt-llm-blackwell-indexer.md), [Fine-Grained FP8/FP4 Quantization](../wiki/techniques/fine-grained-quantization.md) | -| `fp6` | [Microbenchmarking NVIDIA's Blackwell Architecture](../sources/blogs/blackwell-microbenchmarking.md), [Tilus: A Tile-Level GPGPU Programming Language for Low-Precision Computation](../sources/blogs/tilus-nvidia.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [NVIDIA CUDA Toolkit 13.x for Blackwell](../sources/docs/nvidia-cuda-13.md), [NVIDIA CUTLASS 4.x Blackwell Support](../sources/docs/nvidia-cutlass-blackwell.md), [PTX ISA SM100 Instructions Reference](../sources/docs/nvidia-ptx-isa-sm100.md), [NVFP4 and Block-Scaled Narrow Precision](../wiki/hardware/nvfp4.md) | -| `fp8` | [Microbenchmarking NVIDIA's Blackwell Architecture](../sources/blogs/blackwell-microbenchmarking.md), [DeepGEMM — FP8 GEMM Library](../sources/blogs/deepgemm.md), [FlashMLA — Multi-head Latent Attention](../sources/blogs/flashmla.md), [Tilus: A Tile-Level GPGPU Programming Language for Low-Precision Computation](../sources/blogs/tilus-nvidia.md), [DeepSeek-V3.2-Exp in vLLM: Fine-Grained Sparse Attention in Action](../sources/blogs/vllm-deepseek-v3-sparse-attention.md), [FlashInfer MLSys 2026 - Track A: Fused MoE FP8](../sources/contests/flashinfer-mlsys26/track-a-fused-moe.md), [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [NVIDIA Blackwell Tuning Guide](../sources/docs/nvidia-blackwell-tuning-guide.md), [NVIDIA CUDA Toolkit 13.x for Blackwell](../sources/docs/nvidia-cuda-13.md), [NVIDIA CUTLASS 4.x Blackwell Support](../sources/docs/nvidia-cutlass-blackwell.md), [PTX ISA SM100 Instructions Reference](../sources/docs/nvidia-ptx-isa-sm100.md), [[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes](../sources/prs/DeepGEMM/PR-304.md), [Sync nv_dev with upstream #316 (Mega MoE optimizations & benchmarks)](../sources/prs/DeepGEMM/PR-328.md), [ Solving bank conflict via padding and TMA 3D store](../sources/prs/DeepGEMM/PR-78.md), [Use 1D TMA store instead of 3D](../sources/prs/DeepGEMM/PR-83.md), [Use swizzling instead of padding](../sources/prs/DeepGEMM/PR-86.md), [Support TMA multicast on B with m_grouped_gemm_contiguous.](../sources/prs/DeepGEMM/PR-88.md), [[TRTLLM-9457][feat] Add cute dsl fp8 gemm for Blackwell](../sources/prs/TensorRT-LLM/PR-10130.md), [[None][fix] impl fused triton kernel for e8m0 resmooth to reduce memory footprint](../sources/prs/TensorRT-LLM/PR-10327.md), [[None][feat] fuse shared to sparse experts in TRT-LLM Gen MoE](../sources/prs/TensorRT-LLM/PR-11143.md), [[None][feat] Add support for expert_number<=2048 and K<=32](../sources/prs/TensorRT-LLM/PR-11510.md), [[TRTLLM-10004][feat] Enable GEMM -> AR with GEMM output in registered buffers](../sources/prs/TensorRT-LLM/PR-11589.md), [[TRTLLM-10421][perf] Add fused cat+fp8_quantize CUDA kernel for DSA indexer](../sources/prs/TensorRT-LLM/PR-11899.md), [[None][feat] GLM 5 support and DSA MTP fixes](../sources/prs/TensorRT-LLM/PR-11990.md), [[https://nvbugs/5879577][fix] Fix KeyError in DeepSeekV3Lite FP8 MTP weight loading](../sources/prs/TensorRT-LLM/PR-12530.md), [[#12784][feat] AutoDeploy: Optimize DeepSeek-R1 model performance](../sources/prs/TensorRT-LLM/PR-12946.md), [[TRTLLM-34871][feat] Add cute dsl FP8 paged MQA logits decode kernel](../sources/prs/TensorRT-LLM/PR-13219.md), [[TRTLLM-11127][feat] add W4A8_MXFP4_FP8 MoE unit test support](../sources/prs/TensorRT-LLM/PR-13401.md), [[None][feat] Fuse FP8 1x128 quantize + UE8M0 scale pack on SM100](../sources/prs/TensorRT-LLM/PR-13628.md), [[None][fix] Plumb swiglu_limit through DeepGEMM and TRTLLMGen FP8 fused MoE](../sources/prs/TensorRT-LLM/PR-13767.md), [[TRTLLM-12503][feat] Parallel VAE independent scaling and fix arg passing](../sources/prs/TensorRT-LLM/PR-13873.md), [[TRTLLM-35237][feat] Add cute dsl FP4 paged MQA logits decode kernel](../sources/prs/TensorRT-LLM/PR-13929.md), [[None][feat] Keep DSv4 o_a_proj as FP8, and port vLLM's fused_inv_rope_fp8_quant](../sources/prs/TensorRT-LLM/PR-13938.md), [[TRTLLM-12462][fix] Fix FP8 block scaling GEMM autotuner cache growth](../sources/prs/TensorRT-LLM/PR-14165.md), [[None][fix] Add SPDX Apache-2.0 headers to auto_deploy test files](../sources/prs/TensorRT-LLM/PR-14193.md), [feat: Add w4a8_mxfp4_fp8 quantization recipe.](../sources/prs/TensorRT-LLM/PR-4867.md), [[OMNIML-2336][feat] Add NVFP4 x FP8](../sources/prs/TensorRT-LLM/PR-6809.md), [[TRTLLM-8535][feat] Support DeepSeek V3.2 with FP8 + BF16 KV cache/NVFP4 + BF16 KV cache](../sources/prs/TensorRT-LLM/PR-8405.md), [[None][fix] Fix the performance issue of FP8 blockwise grouped GEMM when using attention DP](../sources/prs/TensorRT-LLM/PR-8501.md), [[None][feat] TRT-LLM Gen MoE optimize DeepSeek Fp8 activation kernel](../sources/prs/TensorRT-LLM/PR-9175.md), [[https://nvbugs/5726962][feat] Apply fusion for W4AFP8_AWQ MoE](../sources/prs/TensorRT-LLM/PR-9838.md), [Improve sm90 mixed dtype kernel](../sources/prs/cutlass/PR-1883.md), [Groupwise scaling along M for FP8 gemm](../sources/prs/cutlass/PR-2037.md), [Improvements for: Groupwise scaling along M for FP8 gemm](../sources/prs/cutlass/PR-2095.md), [Hopper Grouped GEMM support for FP8 Accum](../sources/prs/cutlass/PR-2123.md), [Blockwise and Groupwise GEMM for Blackwell and Improvements for Hopper](../sources/prs/cutlass/PR-2139.md), [hopper-blockwise-generalization-optimization](../sources/prs/cutlass/PR-2270.md), [support fp16 accmulator for sm89 fp8 mma](../sources/prs/cutlass/PR-2378.md), [Example 77 add blackwell flash-attention bwd for MLA shape](../sources/prs/cutlass/PR-2466.md), [Add Blackwell MLA forward (shape: d=192, dv=128) implementation](../sources/prs/cutlass/PR-2472.md), [[Cutlass gemm] Fix SM100 FP8 nosmem epilogue-fusion shape_div 'Divisibility Condition' for non-multiple-of-64 N tiles](../sources/prs/cutlass/PR-2946.md), [[Bug Fix]Set NumSplitsM to 1 when TileShapeM < 128 in sm90 fp8 blockwise scaling CollectiveMma](../sources/prs/cutlass/PR-2965.md), [Fp8 kernel with "in-kernel" transpose of V in producer](../sources/prs/flash-attention/PR-1100.md), [FA3 FP8 qkv descales + restore max offset for h128 causal + added sync for producer WG](../sources/prs/flash-attention/PR-1173.md), [FA3 kvcache + split kv + gqa parallelization](../sources/prs/flash-attention/PR-1236.md), [[Cute,Fwd,Sm100] fp8 e4m3 and e5m2 support](../sources/prs/flash-attention/PR-2109.md), [feat: add functional per-head FP8 quantization for FA3](../sources/prs/flashinfer/PR-1033.md), [perf: accelerate blackwell grouped gemm](../sources/prs/flashinfer/PR-1086.md), [bugfix: fix fp8 attention kernels aot compilation issue](../sources/prs/flashinfer/PR-1087.md), [Add CUTLASS fused moe kernels from TensorRT-LLM.](../sources/prs/flashinfer/PR-1113.md), [[Feature] Support PDL for batch Prefill and Decode](../sources/prs/flashinfer/PR-1117.md), [Add DeepGEMM kernels](../sources/prs/flashinfer/PR-1209.md), [Fix test_groupwise_scaled_gemm_fp8.py](../sources/prs/flashinfer/PR-1211.md), [feat: trtllm-gen fp8 moe kernels](../sources/prs/flashinfer/PR-1212.md), [Patch fp8 cubin availability](../sources/prs/flashinfer/PR-1240.md), [feat: Support MXFP8 x MXFP4 CUTLASS grouped GEMM](../sources/prs/flashinfer/PR-1241.md), [Add trtllm-gen attention mha kernel with FP8 Q/K/V and FP8 output](../sources/prs/flashinfer/PR-1242.md), [Reduce the JIT compilation time of gen_gemm_sm100_module](../sources/prs/flashinfer/PR-1251.md), [init add gemm fp8 using cudnn backend](../sources/prs/flashinfer/PR-1264.md), [feat: add masked deepgemm support and benchmarking](../sources/prs/flashinfer/PR-1266.md), [Unify groupwise fp8 GEMM test](../sources/prs/flashinfer/PR-1281.md), [Bug fix: guard fp8 e8m0 and e2m1 compile ](../sources/prs/flashinfer/PR-1287.md), [Update cutlass fp4 moe kernels](../sources/prs/flashinfer/PR-1294.md), [Make Fp8 MoE routing_bias optional](../sources/prs/flashinfer/PR-1319.md), [Add blockwise-scaled FP8 GEMM via TRTLLM-Gen.](../sources/prs/flashinfer/PR-1320.md), [feat: Fused rope fp8 quantize kernel for MLA](../sources/prs/flashinfer/PR-1339.md), [[fix] remove (view) transpose to keep consistent with majorness MN requirement.](../sources/prs/flashinfer/PR-1358.md), [bugfix: Add guard for fp4/fp8 related include headers](../sources/prs/flashinfer/PR-1376.md), [GPT-OSS Support: Add Blackwell MoE mxfp4 implementation from TRTLLM and Attention Sink](../sources/prs/flashinfer/PR-1389.md), [Adding FP8 benchmark on attention and matmul testing](../sources/prs/flashinfer/PR-1390.md), [gpt-oss: Add MXFP8 x MXFP4 CUTLASS MOE for SM100 and BF16 x MXFP4 CUTLASS for SM90 + SwigluBias Activation](../sources/prs/flashinfer/PR-1396.md), [feature: add cutlass as bmm_fp8 backend.](../sources/prs/flashinfer/PR-1397.md), [bugfix: fix perf issue by using fp8 graph that can use cublaslt](../sources/prs/flashinfer/PR-1435.md), [Add alignment in MxFP8Quantization](../sources/prs/flashinfer/PR-1445.md), [Remove getEnvEnablePDL in favor of enable_pdl parameter](../sources/prs/flashinfer/PR-1446.md), [perf: add 1x4x1 cluster shape for fp8 bmm M<16 cases](../sources/prs/flashinfer/PR-1473.md), [refactor: unify autotuner for bmm_fp8](../sources/prs/flashinfer/PR-1479.md), [feat: Support fp8 qkv, fp16/bf16 out MHA for trtllm-gen.](../sources/prs/flashinfer/PR-1490.md), [Perf: support scale_a/scale_b instead of combined scale in cutlass bmm_fp8](../sources/prs/flashinfer/PR-1491.md), [flashinfer_benchmark QoL Improvements and Attention FP8 Support](../sources/prs/flashinfer/PR-1512.md), [bugfix: Fix compile error for undefined swizzle enum.](../sources/prs/flashinfer/PR-1530.md), [feat: Add fp8-qkv, fp16/bf16 output MHA](../sources/prs/flashinfer/PR-1540.md), [bugfix: fix unittest test_fp8_quantize](../sources/prs/flashinfer/PR-1599.md), [feat: initial support for SM103, SM110, SM120, SM121](../sources/prs/flashinfer/PR-1608.md), [feat: cutlass fp8 gemm bringup for SM120 & SM121](../sources/prs/flashinfer/PR-1610.md), [test: pytest.mark.xfail on deepgemm](../sources/prs/flashinfer/PR-1636.md), [Add benchmark for MLARopeQuantize](../sources/prs/flashinfer/PR-1656.md), [perf&bugfix: skip kv-tile computation out of sliding window in FA2; fix __syncthreads in mergestate](../sources/prs/flashinfer/PR-1661.md), [TGV GEMM as a BF16 backend alternative to cuBLAS](../sources/prs/flashinfer/PR-1668.md), [test: better fp8 quantization init for fused_moe test](../sources/prs/flashinfer/PR-1674.md), [Update deepgemm backend for 103a](../sources/prs/flashinfer/PR-1694.md), [test: skip the unsupported test cases for sm120/121](../sources/prs/flashinfer/PR-1710.md), [TVM: support TVM binding for GroupedGemm](../sources/prs/flashinfer/PR-1725.md), [tests: xfail moe quantization classes mxfp8_bf16 UTs on sm103 ](../sources/prs/flashinfer/PR-1754.md), [tests: skip non SM100/103 for grouped deepgemm](../sources/prs/flashinfer/PR-1767.md), [feat: add xqa fp8 mha and fp8 kv cache](../sources/prs/flashinfer/PR-1769.md), [feat:enable fp8 blockscale moe for fused cultass for sm90](../sources/prs/flashinfer/PR-1819.md), [feat: trtrllm-gen global scaled FP8 GEMMs](../sources/prs/flashinfer/PR-1829.md), [raise error for group_gemm_fp8_nt_groupwise then num_groups > 1 on sm120/121](../sources/prs/flashinfer/PR-1862.md), [misc: fix some B200 GEMM bench](../sources/prs/flashinfer/PR-1883.md), [MLA RoPE + quantization fused kernel: shape generalization for MHA / GQA](../sources/prs/flashinfer/PR-1924.md), [Fix: Verify scales are not None for Cutlass FP8 FusedMoE](../sources/prs/flashinfer/PR-1961.md), [feat: enable deepgemm jit for fp8 block-scale on SM90](../sources/prs/flashinfer/PR-1969.md), [fix: correct PDL parameter handling in RopeQuantize kernel](../sources/prs/flashinfer/PR-1982.md), [update trtllm cutlass moe ](../sources/prs/flashinfer/PR-2020.md), [feat: suitable_auto_backends to prune auto backends, bmm_fp8 refactor, heuristic_func intake](../sources/prs/flashinfer/PR-2029.md), [Enable renormalize(naive) routing for fp8 per-tensor](../sources/prs/flashinfer/PR-2030.md), [Added an initial implementation of Q and KV Cache in fp8 and to use t…](../sources/prs/flashinfer/PR-2035.md), [feat: Add flashinfer.rope.rope_quantize_fp8_append_paged_kv_cache (fused RoPE + Q + KV cache, supports MLA/GQA/MHA) ](../sources/prs/flashinfer/PR-2037.md), [Rebase FP8 SM100 Cutlass FMHA Attention to main (original PR#1238)](../sources/prs/flashinfer/PR-2047.md), [test: Skip test_fp8_quantize.py on Hopper](../sources/prs/flashinfer/PR-2052.md), [Fix moe fp8 failure for sm121](../sources/prs/flashinfer/PR-2061.md), [perf: TRT-LLM MoE Block-FP8 activation optimization](../sources/prs/flashinfer/PR-2063.md), [enable xqa fp8 output](../sources/prs/flashinfer/PR-2081.md), [refactor: update dpsk fused_moe test [1]](../sources/prs/flashinfer/PR-2088.md), [refactor: update dpsk fused_moe test [2]](../sources/prs/flashinfer/PR-2097.md), [refactor: update fa3 codebase and fix hopper unittest [part 1]](../sources/prs/flashinfer/PR-2111.md), [fix: Fix bench_mm_fp8.py](../sources/prs/flashinfer/PR-2129.md), [make DeepGEMM swapAB available for linear gemm SM90](../sources/prs/flashinfer/PR-2131.md), [feat: TRTLLM FMHAv2 backend for ctx attention](../sources/prs/flashinfer/PR-2142.md), [Enable Hopper FA3 FP8 attention in decode.py](../sources/prs/flashinfer/PR-2148.md), [Fp8 attention are now part of cuDNN 9.17.1](../sources/prs/flashinfer/PR-2241.md), [feat: RMSNorm/Fused RMSNorm + FP8 Quantization kernels](../sources/prs/flashinfer/PR-2243.md), [fix: support int64 IdType for RoPE part argument in `rope_quantize_fp8_append_paged_kv_cache`](../sources/prs/flashinfer/PR-2255.md), [feat: Add support for bmm mxfp8](../sources/prs/flashinfer/PR-2256.md), [Fix CUTLASS FP8 gemm correctness issue on SM120/SM121 for shapes where N is not divisible by ScaleGranularityN.](../sources/prs/flashinfer/PR-2261.md), [test: use .float() in in F.cosine_similarity() in bmm_fp8 test](../sources/prs/flashinfer/PR-2266.md), [feat: Support Fused MoE non gated Relu2 NVFP4 & FP8 and support Nemotron](../sources/prs/flashinfer/PR-2304.md), [[perf] Improve gemm_fp8_nt_groupwise (cutlass backend) by 10-40% for batch sizes <= 32](../sources/prs/flashinfer/PR-2327.md), [fix: guard batchWarpReduceSum with ENABLE_FP8 to fix compilation without FP8](../sources/prs/flashinfer/PR-2328.md), [fix: Fix NaN output in mxfp8_quantize for very small input values](../sources/prs/flashinfer/PR-2441.md), [Add cute-dsl backends to mxfp[8,4]_quantization for future refactor](../sources/prs/flashinfer/PR-2443.md), [feat: Add TRTLLM fmha_v2 library for SM90 attention with Skip-Softmax ](../sources/prs/flashinfer/PR-2446.md), [feat: Support Fused MoE non gated Relu2 NVFP4 & FP8 and support Nemotron, fixed](../sources/prs/flashinfer/PR-2462.md), [feat: Add MXFP8 GEMM mm_mxfp8 (cutlass)](../sources/prs/flashinfer/PR-2464.md), [Feat: Trtllm-gen MxFP8 MoE integration](../sources/prs/flashinfer/PR-2505.md), [fix: include fp8_blockscale_gemm_90 in AOT jit-cache](../sources/prs/flashinfer/PR-2533.md), [fallback to fa2 (instead of fa3) for unsupported configuration (bf16 Q, Fp8 KV)](../sources/prs/flashinfer/PR-2536.md), [tests: bmm_fp8 for SM110](../sources/prs/flashinfer/PR-2538.md), [Add gen_gemm_sm100_module_cutlass_mxfp8 to jit-cache](../sources/prs/flashinfer/PR-2549.md), [Implement `cutlass_fused_moe` mxfp8](../sources/prs/flashinfer/PR-2581.md), [fix: add SM121 support to SM120 version guards](../sources/prs/flashinfer/PR-2631.md), [benchmark: Add MXFP4/MXFP8 quantization mode support to FP4 MoE benchmark](../sources/prs/flashinfer/PR-2635.md), [[fp8_blockwise]Fix int32 overflow in TRTLLM fused MoE activation kernel](../sources/prs/flashinfer/PR-2642.md), [[feat] trtllm-gen mxfp8 gemm](../sources/prs/flashinfer/PR-2653.md), [feat: support mxfp4 & mxfp8 entrypoint for blackwell cutedsl dense gemm](../sources/prs/flashinfer/PR-2660.md), [benchmarks: Add FP8 input / BF16 output in ragged prefill benchmark](../sources/prs/flashinfer/PR-2666.md), [perf: Update trtllm-gen batched GEMM kernels - faster, more NVFP4 tile dims, MXFP8 with relu2 act](../sources/prs/flashinfer/PR-2667.md), [feat: Add support for TRTLLM MXFP8 non-gated MoE with ReLU2](../sources/prs/flashinfer/PR-2707.md), [Support for MXFP4 and NVFP4 group GEMMs on GeForce and Spark](../sources/prs/flashinfer/PR-2738.md), [Support in-place update for `trtllm_fp8_block_scale_moe`](../sources/prs/flashinfer/PR-2739.md), [Add cute dsl mla decode op](../sources/prs/flashinfer/PR-2743.md), [[Spark unit test debugging] Fix for tests/gemm/test_groupwise_scaled_gemm_fp8.py](../sources/prs/flashinfer/PR-2751.md), [feat: FP8 output support for CUTLASS MLA paged attention](../sources/prs/flashinfer/PR-2779.md), [tests: skip sliding window + fp8 to prevent hang in fmha_v2 unit tests](../sources/prs/flashinfer/PR-2781.md), [[CuTe DSL] Add modular FMHA prefill and MLA decode attention kernels](../sources/prs/flashinfer/PR-2805.md), [Fix silent bug with FP8 per tensor non-gated MoE](../sources/prs/flashinfer/PR-2882.md), [feat: add pdl support for cute dsl mla decode kernel support](../sources/prs/flashinfer/PR-2901.md), [feat: add MXFP8 GEMM support for SM120](../sources/prs/flashinfer/PR-2902.md), [perf: Optimize CuTe-DSL fp4 and fp8 quantization kernels](../sources/prs/flashinfer/PR-2904.md), [[NVIDIA] fix(jit): enable GDC for CUTLASS fused MoE PDL — prevent random crashes on SM12x](../sources/prs/flashinfer/PR-2913.md), [feat: Add cuBLASLt backend for `mm_bf16` and enable multi-tactic autotuning for FP8/MXFP8 runners](../sources/prs/flashinfer/PR-2914.md), [test: xfail cuDNN FP8 prefill on Blackwell with CUDA <= 12.9](../sources/prs/flashinfer/PR-2963.md), [test: skip unsupported mm_mxfp8 configurations on SM12x](../sources/prs/flashinfer/PR-2974.md), [ Fix MXFP4/MXFP8 failures in SM120 FAST_BUILD and expand all_tiles[] ](../sources/prs/flashinfer/PR-2994.md), [Support Allreduce + Norm + Per-token Group Fp8 Quant Fusion](../sources/prs/flashinfer/PR-3059.md), [feat: Enable FP8 (E4M3/E5M2) in concat_mla_k for optimize long-context prefill performance and refactor type dispatch for BF16/FP16](../sources/prs/flashinfer/PR-3129.md), [Integrate CUTLASS Small Tile N Blockscaled GEMMs/Grouped GEMMs for SM120 and SM121](../sources/prs/flashinfer/PR-3152.md), [test: enable bmm_mxfp8 cutlass backend coverage on SM12x](../sources/prs/flashinfer/PR-3183.md), [feat(trace): embed runnable init() in every TraceTemplate](../sources/prs/flashinfer/PR-3221.md), [Support Kimi K2.5 H64 CuTe DSL MLA decode](../sources/prs/flashinfer/PR-3235.md), [fix(fmha_v2): fix FP8 V-scratch pipeline and varlen scheduler on SM90](../sources/prs/flashinfer/PR-3276.md), [Naive Support for Hopper FP8 Prefill Kernel with Per-Head Quantization](../sources/prs/flashinfer/PR-869.md), [perf: Fix python API overhead when CUDAGraph is not enabled](../sources/prs/flashinfer/PR-969.md), [Update CUTLASS. Refine KernelSchedule for fp8 (grouped) gemm.](../sources/prs/sglang/PR-10491.md), [support qwen3-next-fp8 deepep](../sources/prs/sglang/PR-10622.md), [Fix DSR1 accuracy for flashinfer_trtllm MoE with FP8 quantization](../sources/prs/sglang/PR-11081.md), [[sgl-kernel][1/N]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-11432.md), [[DeepseekV32] Enable flashmla_prefill kernel with fp8 kvcache](../sources/prs/sglang/PR-11655.md), [Support running FP4 Deepseek on SM120.](../sources/prs/sglang/PR-11708.md), [Change bf16 to fp8 for some gemms in attention for DeepSeek ckpt v2](../sources/prs/sglang/PR-11805.md), [Support nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8/NVFP4](../sources/prs/sglang/PR-11866.md), [Feature/nano v2 offline modelopt fp8 and nvfp4](../sources/prs/sglang/PR-12018.md), [[sgl-kernel][4/N]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-12080.md), [[hotfix] missing `w13_weight_fp8` and `w2_weight_fp8` in UE8M0 requantization](../sources/prs/sglang/PR-12259.md), [fix: llama 4 + trtllm gen + fp8 kv cache incompatibility](../sources/prs/sglang/PR-12347.md), [Enable Flashinfer TRTLLM-GEN-MoE FP8 blockwise kernel for Qwen3-Next on Blackwell](../sources/prs/sglang/PR-12543.md), [[sgl-kernel] support custom fp8 flashmla kernel](../sources/prs/sglang/PR-13087.md), [Aiter fp8 kv cache](../sources/prs/sglang/PR-13147.md), [[NVIDIA] Fix broken fp8 MoE of deepseek v3](../sources/prs/sglang/PR-13264.md), [[NVIDIA] Fix use case of SGLANG_ENABLE_FLASHINFER_GEMM](../sources/prs/sglang/PR-13274.md), [[ROCM] Optimized deepseek-r1 fp8 model with + triton_gemm_a8w8 + batch_gemm_a8w8 + fused set_mla_kv_buffer kernel](../sources/prs/sglang/PR-13617.md), [[sgl-kernel][Feat][B200][1/N]Support MXFP8 Grouped GEMM in Blackwell](../sources/prs/sglang/PR-13731.md), [Support fp4 fp8 non gated moe](../sources/prs/sglang/PR-13794.md), [Feat: GLM-4.6 supports shared experts fusion](../sources/prs/sglang/PR-13873.md), [Fix update weight error for blackwell DeepGEMM](../sources/prs/sglang/PR-13910.md), [[DeepSeek v3.2] opt Context Parallelism: support fused moe, multi batch and fp8 kvcache](../sources/prs/sglang/PR-13959.md), [Support checking fp8 params in weight_checker](../sources/prs/sglang/PR-14147.md), [Add Mistral Large 3 support.](../sources/prs/sglang/PR-14213.md), [[CPU] Implement MXFP4 Gemm kernels for intel AMX to support GPT OSS series.](../sources/prs/sglang/PR-14385.md), [Support FP8 MLA prefill and 128k context.](../sources/prs/sglang/PR-14395.md), [Add Mistral Large 3 Eagle Support](../sources/prs/sglang/PR-14466.md), [[sgl-kernel][Feat][B200][2/N] Support MXFP8 Grouped GEMM in Blackwell](../sources/prs/sglang/PR-14640.md), [[sgl-kernel] Update flashmla to include fp8 sparse_mla optimizations](../sources/prs/sglang/PR-15242.md), [Flashinfer MOE FP8 support for Mistral Large 3.](../sources/prs/sglang/PR-15422.md), [[sgl-kernel][6/7]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-15471.md), [[Perf] Add Flashinfer DeepGEMM SM90 for SwapAB Optimization](../sources/prs/sglang/PR-15514.md), [Optimize FP8 MLA KV cache writes with Triton kernel](../sources/prs/sglang/PR-15522.md), [[JIT kernel] Apply jit per_tensor_quant_fp8 kernel](../sources/prs/sglang/PR-15836.md), [Fix FP8 MoE NaN with DeepGEMM on Blackwell](../sources/prs/sglang/PR-16622.md), [[GLM 4.7] Add RTX 6000 Pro aka sm120](../sources/prs/sglang/PR-17235.md), [Disable mla persistent kernel when not using fp8 kv_cache](../sources/prs/sglang/PR-17327.md), [Add mxfp8 support for online quantization, Triton dense linear, and CUTLASS MoE](../sources/prs/sglang/PR-17449.md), [[ROCm] Optimize Deepseek R1 on MI300X](../sources/prs/sglang/PR-18242.md), [Nsa trtllm mla sparse fp8 support with Deepseek v3.2 NVFP4](../sources/prs/sglang/PR-18389.md), [[AMD] Update aiter to v0.1.10.post2](../sources/prs/sglang/PR-18423.md), [Fp8 prefill attn kernel integration](../sources/prs/sglang/PR-18528.md), [[AMD] DSR1/V3 use fp8 bmm in MLA for MI300X](../sources/prs/sglang/PR-18624.md), [[RL] Support per-layer mixed FP8/BF16 serving for FP8 checkpoints](../sources/prs/sglang/PR-18742.md), [Fix NSA FP8 KV cache path for both-trtllm MHA one-shot](../sources/prs/sglang/PR-18931.md), [[FlashInfer v0.6.4] [RL] Integrate FlashInfer mxfp8 gemm, MoE, and routed MoE](../sources/prs/sglang/PR-19537.md), [Various SM120 improvements](../sources/prs/sglang/PR-19721.md), [[AMD] Fix FP8 assertion failure in aiter MLA decode by falling back to self.k_scale](../sources/prs/sglang/PR-19935.md), [Enable modelopt quantized FLUX deployment](../sources/prs/sglang/PR-20082.md), [[AMD] Fp8 prefill integration with radix cache path for dpsk models](../sources/prs/sglang/PR-20187.md), [[Benchmark] use flashinfer bench_gpu_time instead of triton do_bench](../sources/prs/sglang/PR-20305.md), [[NVIDIA] Enable fp8 flashinfer_trtllm_routed MoE for MiniMax-M2.5](../sources/prs/sglang/PR-20394.md), [Support Triton MLA FP8 KV cache](../sources/prs/sglang/PR-20479.md), [FIX: (NSA) Compute topk_indices_offset when NSA prefill flashmla_sparse is used with FP8 KV cache](../sources/prs/sglang/PR-20606.md), [CUTLASS FP8 Blockwise GEMM improvement of SM120](../sources/prs/sglang/PR-20887.md), [Add SGLang CUDA crash API logging inspired by FlashInfer](../sources/prs/sglang/PR-20910.md), [ci: run Stage A CUDA tests as stage-a-test-small-1-gpu on 5090](../sources/prs/sglang/PR-20988.md), [[AMD]: Support MLA with nhead<16 and FP8 KV cache for TP=8 (Kimi K2.5…](../sources/prs/sglang/PR-21213.md), [Refactor JIT kernel CI to use run_suite.py registration system](../sources/prs/sglang/PR-21239.md), [[RL] Support mxfp8 DeepSeek V3](../sources/prs/sglang/PR-21280.md), [[Kernel] Support FlashInfer TRTLLM-Gen fused MoE for non-gated FP4 & FP8 (Nemotron)](../sources/prs/sglang/PR-21321.md), [[misc] clean up kernel API](../sources/prs/sglang/PR-21325.md), [Migrate all callers from /get_server_info to /server_info](../sources/prs/sglang/PR-21463.md), [[AMD] Enable FP8 KV cache and FP8 attention kernel for NSA on MI300/MI355 with TileLang backend](../sources/prs/sglang/PR-21511.md), [[FlashInver v0.6.7] Integrate flashinfer_trtllm mxfp8 gemm](../sources/prs/sglang/PR-21576.md), [[AMD] Add GLM-5-FP8 nightly performance benchmarks for MI30x and MI35x](../sources/prs/sglang/PR-21710.md), [[Misc] [MXFP8] Drop sm100 mxfp8 warning](../sources/prs/sglang/PR-21881.md), [fix pcg torch dynamo recompile in mxfp8 Triton path](../sources/prs/sglang/PR-21888.md), [Tiny fix trtllm_fp8_per_tensor_scale_moe_wrapper router_logits dtype](../sources/prs/sglang/PR-22006.md), [[AMD][HIP] NSA: bf16 passthrough from RMSNorm to eliminate FP8 dequantization](../sources/prs/sglang/PR-22258.md), [[AMD] Fix GLM-5 fp8 KV quant path dispatch on MI300](../sources/prs/sglang/PR-22314.md), [[Reland] DeepSeek-R1-0528-w4a8: DeepEP Low Latency Dispatch Adopts FP8 Communication](../sources/prs/sglang/PR-22316.md), [[Lora] Lora quat info re-factor and support deepseekv3 mla lora](../sources/prs/sglang/PR-22323.md), [[AMD] Add GLM-5.1-FP8 nightly accuracy and performance benchmarks for MI30x and MI35x](../sources/prs/sglang/PR-22336.md), [[Diffusion] modelopt diffusion fp8 support for flux1/flux2 and wan2.2](../sources/prs/sglang/PR-22365.md), [[DSA] Hopper FP8 FlashMLA KV padding](../sources/prs/sglang/PR-22372.md), [[RL] Fix weight update for mxfp8 flashinfer_cutlass gemm backend](../sources/prs/sglang/PR-22484.md), [[Diffusion] Add FLUX.1-dev ModelOpt NVFP4 support](../sources/prs/sglang/PR-22574.md), [reland [Diffusion] Add FLUX.1-dev ModelOpt NVFP4 support](../sources/prs/sglang/PR-22672.md), [Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-23686.md), [[MoE] Unify DeepEPMoE+MoriEPMoE through AITER MoeRunner pre/post-permute](../sources/prs/sglang/PR-23760.md), [Refactor device timer, clean up metrics collector, and add fwd occupancy metric](../sources/prs/sglang/PR-24197.md), [Port MXFP4 Marlin MoE support to JIT kernel path](../sources/prs/sglang/PR-24490.md), [[Codex] Opt Mistral Large performace ](../sources/prs/sglang/PR-24611.md), [[Gemma4] Optimize Gemm4 with fused Q/K/V RMSNorm + per-expert FP8 ckpt loader](../sources/prs/sglang/PR-24696.md), [Add FlashInfer SM90 cutlass MXFP4 MoE backend (W4A16) for GPT-OSS + DeepSeek-V4](../sources/prs/sglang/PR-24816.md), [[attn backend] Integrate tokenspeed_mla prefill/decode kernels (fp8 kv cache, blackwell)](../sources/prs/sglang/PR-24925.md), [Amd/deepseek v4 rebase main 0509](../sources/prs/sglang/PR-24933.md), [[rebase]Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-24986.md), [[Gemma4]: Fix FP8 Triton scale layout](../sources/prs/sglang/PR-25286.md), [[Intel GPU] Enable DeepSeek V4 Inference on XPU](../sources/prs/sglang/PR-25336.md), [[fp8] SM90 swap-AB scaled_mm dispatch (~1.16x kernel geomean, +5.8-18.5% end-to-end)](../sources/prs/sglang/PR-25532.md), [[Test] Stage-a sanity kits; consolidate core/ + models_e2e/ tests](../sources/prs/sglang/PR-25831.md), [support w8a8 fp8 kernel with CUTLASS](../sources/prs/sglang/PR-3047.md), [feat: integrate gemm_fp8 kernel into gemm](../sources/prs/sglang/PR-3056.md), [Apply sgl w8a8 fp8 kernel](../sources/prs/sglang/PR-3148.md), [add tensorrt_llm common and cutlass_extensions as 3rdparty](../sources/prs/sglang/PR-3216.md), [support blockwise fp8 matmul kernel](../sources/prs/sglang/PR-3267.md), [integrate blockwise fp8 kernel](../sources/prs/sglang/PR-3529.md), [add control for cutlass fp8 blockwise gemm](../sources/prs/sglang/PR-3727.md), [[tools] add fp8 max/min constant in utils](../sources/prs/sglang/PR-3959.md), [DeepGemm integrate to gemm](../sources/prs/sglang/PR-4165.md), [linear support deepgemm](../sources/prs/sglang/PR-4199.md), [Accelerate FP8 CUDA Kernel by 20-28%](../sources/prs/sglang/PR-4215.md), [Clean up fp8 support](../sources/prs/sglang/PR-4230.md), [fix per_token_group_quant_fp8 illegal memory when num_groups % 16 != 0](../sources/prs/sglang/PR-4231.md), [Support Blackwell Block Scale FP8 Gemm](../sources/prs/sglang/PR-4278.md), [[FIX] fix incorrect output when enable both deepgemm and torch compile](../sources/prs/sglang/PR-4359.md), [[ROCm] fix dtype](../sources/prs/sglang/PR-4510.md), [Create col-major and tma-aligned x_scale for deep_gemm.gemm_fp8_fp8_bf16_nt](../sources/prs/sglang/PR-4515.md), [Support fp8 gemm for blackwell](../sources/prs/sglang/PR-4558.md), [[quantization] fix channelwise conversion with scalar weight scale](../sources/prs/sglang/PR-4596.md), [Set deepgemm to the default value in the hopper architecture.](../sources/prs/sglang/PR-4613.md), [Add DeepSeek V3/R1 shared experts fusion](../sources/prs/sglang/PR-4918.md), [feat: add DeepGEMM build warning](../sources/prs/sglang/PR-5176.md), [[Fix] Turn off DeepGEMM by default](../sources/prs/sglang/PR-5263.md), [[1/2] Add FP8 Blockscale MoE CUTLASS kernel for Blackwell](../sources/prs/sglang/PR-5281.md), [fix: use deepgemm only on hopper](../sources/prs/sglang/PR-5310.md), [[perf] experimental enhance fp8 per-tensor quant](../sources/prs/sglang/PR-5370.md), [[perf] introduce deep gemm group_gemm_masked as gemm](../sources/prs/sglang/PR-5432.md), [[feature] enable pre compile jit deep_gemm](../sources/prs/sglang/PR-5580.md), [ DeepEP normal support deepgemm-contiguous](../sources/prs/sglang/PR-5626.md), [[perf] dsv3 bmm fallback to bf16](../sources/prs/sglang/PR-5662.md), [[2/2] Add python wrapper for CUTLASS FP8 Blockscale MoE Kernel. ](../sources/prs/sglang/PR-5694.md), [cutlass 3.9 supported to improve fp8_blockwise_gemm](../sources/prs/sglang/PR-5820.md), [chore: upgrade cutlass 3.9.2](../sources/prs/sglang/PR-6004.md), [Upgrade CUTLASS 4.0](../sources/prs/sglang/PR-6336.md), [Add fp8 fused_experts kernel for CPU in sgl-kernel and add UT](../sources/prs/sglang/PR-6404.md), [Fix bug of deepseek-v3 under DP+EP mode with large batchsize/seqlen](../sources/prs/sglang/PR-6449.md), [[Feature] Support Flashinfer fp8 blockwise GEMM kernel on Blackwell](../sources/prs/sglang/PR-6479.md), [Set `num_fused_shared_experts` as `num_shared_experts` when shared_experts fusion is not disabled](../sources/prs/sglang/PR-6736.md), [[CPU] add optimizations for INT8 and FP8 DeepSeek](../sources/prs/sglang/PR-6769.md), [feat: integrate deepgemm into EPMoE](../sources/prs/sglang/PR-6821.md), [CPU: map changes from developing branch in sgl-kernel](../sources/prs/sglang/PR-6833.md), [Use deepgemm instead of triton for fused_qkv_a_proj_with_mqa](../sources/prs/sglang/PR-6890.md), [Add a CUDA kernel for fusing mapping and weighted sum for MoE.](../sources/prs/sglang/PR-6916.md), [[Feature] Support Flashinfer fmha on Blackwell](../sources/prs/sglang/PR-6930.md), [Fuse routed scaling factor in deepseek](../sources/prs/sglang/PR-6970.md), [Update default settings for blackwell](../sources/prs/sglang/PR-7023.md), [Fix positional argument](../sources/prs/sglang/PR-7093.md), [fix amd EP MoE FP8 issue](../sources/prs/sglang/PR-7125.md), [Enable ModelOpt Llama4 fp8 checkpoint deployment in SGLang](../sources/prs/sglang/PR-7129.md), [[amd] Opt dsv3 moe](../sources/prs/sglang/PR-7160.md), [Support new DeepGEMM](../sources/prs/sglang/PR-7172.md), [Tiny let DeepGEMM scale checks cover more cases](../sources/prs/sglang/PR-7182.md), [[AMD] Fail gracefully when AITER is unavailable gfx90a GPUs](../sources/prs/sglang/PR-7187.md), [[fix] fix DeepGEMM blackwell input quant & ut & fix style and log](../sources/prs/sglang/PR-7247.md), [Add CUTLASS FP8 Blockscale MoE kernel for Hopper architecture](../sources/prs/sglang/PR-7278.md), [Fix torch compile run](../sources/prs/sglang/PR-7391.md), [[AMD][Quantization] Add `int4fp8_moe` online quantization on ROCm](../sources/prs/sglang/PR-7392.md), [feat: support DeepSeek-R1-W4AFP8 model with ep-moe mode](../sources/prs/sglang/PR-7762.md), [Qwen FP8/NVFP4 ModelOPT Quantization support](../sources/prs/sglang/PR-7912.md), [[feat] Support tp mode for DeepSeek-R1-W4AFP8](../sources/prs/sglang/PR-8118.md), [[sgl-kernel] Opt per_token_quant_fp8 with warp reduce](../sources/prs/sglang/PR-8130.md), [[1/N]Support DeepSeek-R1 w4a8 normal deepep](../sources/prs/sglang/PR-8247.md), [[2/N]Support DeepSeek-R1 w4a8 low latency deepep](../sources/prs/sglang/PR-8464.md), [TRTLLM-MLA FP8 path](../sources/prs/sglang/PR-8638.md), [feat: support cutlass_moe_fp8 kernel for fusedmoe in sm90](../sources/prs/sglang/PR-8678.md), [[Perf] Tunings for SM100 FP8 CUTLASS kernel](../sources/prs/sglang/PR-8818.md), [optimize: reduce shulffle and quantization overhead in cutlass_moe sm90](../sources/prs/sglang/PR-8962.md), [[fix]: fix cutlass moe ut and and Opt H20 cutlass groupGemm performance](../sources/prs/sglang/PR-9272.md), [[sgl-kernel] feat: Support sm120 cutlass fp8 gemm kernel](../sources/prs/sglang/PR-9403.md), [fix: blackwell dsv3 fp8 issue temporary solution](../sources/prs/sglang/PR-9530.md), [Update CUTLASS 4.2 & Enable K-Major Scale Factor for SM90 FP8 Blockwise Group GEMM](../sources/prs/sglang/PR-9559.md), [move is_sm90_supported/is_sm100_supported to python/sglang/srt/utils.py](../sources/prs/sglang/PR-9679.md), [[CPU] Add FP8 Bmm support](../sources/prs/sglang/PR-9744.md), [Make sm100 fp8 kernels available on sm103](../sources/prs/sglang/PR-9789.md), [CUTLASS fp8 blockwise gemm support of sm120](../sources/prs/sglang/PR-9969.md), [[WIP] support more dtypes for tcgen05](../sources/prs/tilelang/PR-1229.md), [[Enhancement] add more dtype and fix mma.ws for fp16 for tcgen05](../sources/prs/tilelang/PR-1327.md), [[Feature] Block-scaled GEMM support for MXFP8 on Blackwell](../sources/prs/tilelang/PR-1945.md), [[Kernel]: Cutlass 2:4 Sparsity + FP8/Int8 Quant Support](../sources/prs/vllm/PR-10995.md), [[Kernel] Update `cutlass_scaled_mm` to support 2d group (blockwise) scaling](../sources/prs/vllm/PR-11868.md), [Add: Support for Sparse24Bitmask Compressed Models](../sources/prs/vllm/PR-12097.md), [Expert Parallelism (EP) Support for DeepSeek Models](../sources/prs/vllm/PR-12583.md), [[Kernel][Quantization] Integrate block-quantized CUTLASS kernels for DeepSeekV3](../sources/prs/vllm/PR-12587.md), [[Attention] Deepseek v3 MLA support with FP8 compute](../sources/prs/vllm/PR-12601.md), [[Attention] MLA with chunked prefill](../sources/prs/vllm/PR-12639.md), [[AMD][ROCm] Enable DeepSeek model on ROCm](../sources/prs/vllm/PR-12662.md), [[Bugfix] Better FP8 supported defaults](../sources/prs/vllm/PR-12796.md), [[Kernel]Add streamK for block-quantized CUTLASS kernels](../sources/prs/vllm/PR-12978.md), [[core] Perf improvement for DSv3 on AMD GPUs](../sources/prs/vllm/PR-13718.md), [[V1] V1 Enablement Oracle ](../sources/prs/vllm/PR-13726.md), [[Bugfix][Quantization] Fix FP8 + EP](../sources/prs/vllm/PR-13784.md), [add tcgen05 support for tcgen05 fp8 gemm](../sources/prs/vllm/PR-13798.md), [[Kernel] CUTLASS grouped gemm fp8 MoE kernel](../sources/prs/vllm/PR-13972.md), [[core] moe fp8 block quant tuning support](../sources/prs/vllm/PR-14068.md), [dynamic distpatch of fp8 kernels](../sources/prs/vllm/PR-14245.md), [Add cutlass support for blackwell fp8 blockwise gemm](../sources/prs/vllm/PR-14383.md), [[BugFix] Illegal Memory Access in the blockwise cutlass fp8 GEMMs](../sources/prs/vllm/PR-14396.md), [[ROCm][Kernel] MoE weights padding](../sources/prs/vllm/PR-14454.md), [permute/unpermute kernel for moe optimization](../sources/prs/vllm/PR-14568.md), [[Attention] Flash Attention 3 - fp8](../sources/prs/vllm/PR-14570.md), [[Quantization][FP8] Adding support for fp8 gemm layer input in fp8](../sources/prs/vllm/PR-14578.md), [[Attention] MLA get rid of materialization](../sources/prs/vllm/PR-14770.md), [[FEAT][ROCm] Integrate Fused MoE Kernels from AITER](../sources/prs/vllm/PR-14967.md), [[FEAT] [ROCm]: Add AITER Block-Scaled GEMM Feature](../sources/prs/vllm/PR-14968.md), [[Quantization] Fp8 Channelwise Dynamic Per Token GroupedGEMM](../sources/prs/vllm/PR-15587.md), [Modularize fused experts and integrate PPLX kernels](../sources/prs/vllm/PR-15956.md), [Upstream Llama4 Support to Main](../sources/prs/vllm/PR-16113.md), [[Kernel] Support W8A8 channel-wise weights and per-token activations in triton fused_moe_kernel](../sources/prs/vllm/PR-16366.md), [Enable PTPC FP8 for CompressedTensorsW8A8Fp8MoEMethod (triton fused_moe)](../sources/prs/vllm/PR-16537.md), [[ROCm] Add aiter tkw1 kernel for Llama4 fp8](../sources/prs/vllm/PR-16727.md), [[Kernel] some optimizations for dense marlin and moe marlin](../sources/prs/vllm/PR-16850.md), [[Kernel] Add expert_map support to Cutlass FP8 MOE](../sources/prs/vllm/PR-16861.md), [[FEAT] [ROCm]: Add AITER CK 2 Stages MoE support](../sources/prs/vllm/PR-17110.md), [[ROCm][FP8][Kernel] FP8 quantization fused into Custom Paged Attention](../sources/prs/vllm/PR-17139.md), [[NVIDIA] Support Cutlass w8a8 FP8 for Blackwell Geforce GPUs (sm120)](../sources/prs/vllm/PR-17280.md), [[Kernel] fp4 marlin kernel](../sources/prs/vllm/PR-17687.md), [use ceil_div in cutlass block scaling shape check](../sources/prs/vllm/PR-17918.md), [[Feature] Expert Parallelism Load Balancer (EPLB)](../sources/prs/vllm/PR-18343.md), [[V1] Support `LLM.apply_model`](../sources/prs/vllm/PR-18465.md), [Sm100 blockwise fp8 swap ab](../sources/prs/vllm/PR-18564.md), [[Kernel] Integrate CUTLASS MoE kernel with PPLX](../sources/prs/vllm/PR-18762.md), [[Perf] Tunings for SM100 FP8 CUTLASS kernel](../sources/prs/vllm/PR-18778.md), [[Kernel] Enable fp8 support for pplx and BatchedTritonExperts.](../sources/prs/vllm/PR-18864.md), [[Kernel] Support deep_gemm for linear methods](../sources/prs/vllm/PR-19085.md), [[Kernels] Add activation chunking logic to FusedMoEModularKernel](../sources/prs/vllm/PR-19168.md), [[Perf] Further tunings for SM100 FP8 CUTLASS kernel](../sources/prs/vllm/PR-19566.md), [[feat]: CUTLASS block scaled group gemm for SM100](../sources/prs/vllm/PR-19757.md), [[Feature] Integrate new deepgemm](../sources/prs/vllm/PR-19820.md), [ [Feature] Integrate SM100 DeepGEMM support](../sources/prs/vllm/PR-20087.md), [[Bugfix] Fix topk_ids indices_type for CUTLASS w8a8 FP8 MoE](../sources/prs/vllm/PR-20166.md), [[V1] [ROCm] Enable EP with AITER Fused MoE](../sources/prs/vllm/PR-20270.md), [[Kernel] SM90 CUTLASS FP8 GEMM: add support for swap AB + kernel tuning](../sources/prs/vllm/PR-20396.md), [[feat]: add SM100 support for cutlass FP8 groupGEMM](../sources/prs/vllm/PR-20447.md), [Support Llama 4 for fused_marlin_moe](../sources/prs/vllm/PR-20457.md), [[Perf] Reuse workspace for FP8+FP4 Marlin MoE](../sources/prs/vllm/PR-20500.md), [[Performance] Performance improvements in non-blockwise fp8 CUTLASS MoE](../sources/prs/vllm/PR-20762.md), [[Perf] Use Triton instead of Torch for DeepGEMM Per Token Group Quant](../sources/prs/vllm/PR-20841.md), [[Perf] Add swap_ab to SM90 FP8 non-block CUTLASS moe grouped gemm](../sources/prs/vllm/PR-20911.md), [[Perf] Cuda Kernel for Per Token Group Quant](../sources/prs/vllm/PR-21083.md), [[perf] Add fused MLA QKV + strided layernorm](../sources/prs/vllm/PR-21116.md), [[Feature][OCP MX] Support mxfp6 and mixed mxfp6-mxfp4](../sources/prs/vllm/PR-21166.md), [[Bug] DeepGemm: Fix TypeError: per_block_cast_to_fp8() missing 1 required positional argument: 'use_ue8m0' for SM100](../sources/prs/vllm/PR-21187.md), [[NVIDIA] Explicitly disable shuffled weights for flashinfer blockscale moe fp8 kernels](../sources/prs/vllm/PR-21411.md), [[Bugfix][CUDA] fixes CUDA FP8 kv cache dtype supported](../sources/prs/vllm/PR-21420.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv/out Attention Kernel](../sources/prs/vllm/PR-21716.md), [[Kernel] Add support for block FP8 on SM120 (NVIDIA 5090 and RTX PRO 6000)](../sources/prs/vllm/PR-22131.md), [Fp8 paged attention update](../sources/prs/vllm/PR-22222.md), [[Bug] Fix B200 DeepGEMM E8M0 Accuracy Issue](../sources/prs/vllm/PR-22399.md), [[Quantization] Expand compressed-tensors MoE matching logic to support NFP4 + FP8 MoEs](../sources/prs/vllm/PR-22674.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv NVFP4-out Attention Kernel](../sources/prs/vllm/PR-22703.md), [fp8 kv cache support fix for torch.compile](../sources/prs/vllm/PR-22758.md), [[Kernel] Added flashinfer fp8 per-tensor gemms](../sources/prs/vllm/PR-22895.md), [[Bugfix] fix qwen3 moe fp8 accuracy issue](../sources/prs/vllm/PR-23031.md), [[Kernel] CUTLASS MoE FP8: Integrate cuda moe permute/unpermute](../sources/prs/vllm/PR-23045.md), [Add routed_scaling_factor to MoE grouped topk](../sources/prs/vllm/PR-23123.md), [[XPU][Feature] fp8 online quantization support for XPU](../sources/prs/vllm/PR-23148.md), [[kernel] Support W4A8 on Hopper](../sources/prs/vllm/PR-23198.md), [[ROCm][Aiter] Add triton fp8 bmm kernel for mla](../sources/prs/vllm/PR-23264.md), [[Perf] Small optimizations for silu_mul_fp8_quant_deep_gemm](../sources/prs/vllm/PR-23265.md), [[Kernels] Overlap shared experts with send/recv](../sources/prs/vllm/PR-23273.md), [[Perf] Use upstream CUTLASS for SM90 Block FP8 kernel](../sources/prs/vllm/PR-23280.md), [[Bug] Fix R1 Accuracy 0 Bug](../sources/prs/vllm/PR-23294.md), [DP/EP Support for gpt-oss with deepep-ht comm kernel on SM100](../sources/prs/vllm/PR-23608.md), [[Flashinfer] Support Flashinfer TRTLLM FP8-qkv BF16/FP16-out Attention Kernel](../sources/prs/vllm/PR-23647.md), [[Feature] Add Hopper DeepGEMM E8M0 for DeepSeekV3.1 scale_fmt](../sources/prs/vllm/PR-23666.md), [[Kernel][tcgen05] nvfp4 fused tcgen05 moe](../sources/prs/vllm/PR-23696.md), [[Model] Add LongCat-Flash ](../sources/prs/vllm/PR-23991.md), [[Performance] Move apply_w8a8_block_fp8_linear to an op class](../sources/prs/vllm/PR-24666.md), [[NVIDIA] Blackwell Family](../sources/prs/vllm/PR-24673.md), [[Kernel][Quantization] add w4a8 support for marlin kernel](../sources/prs/vllm/PR-24722.md), [[Flashinfer][gpt-oss] Support FP8-qkv Flashinfer TRTLLM Sinks Attention](../sources/prs/vllm/PR-25674.md), [[Bugfix] Convert untraceable GroupShape to list for AMD impl](../sources/prs/vllm/PR-26535.md), [[Bugfix] Fix gpt-oss w4a8 DP/EP on B200](../sources/prs/vllm/PR-26729.md), [[Feature] Batch Invariant: Support DeepGEMM and Blackwell](../sources/prs/vllm/PR-27127.md), [[Kernels] Enable FlashInfer FP8 Blockscale on SM90 (for TEP DSR1)](../sources/prs/vllm/PR-27134.md), [[torch.compile] Enable silu_mul_fp8_quant fusion without custom ops enabled](../sources/prs/vllm/PR-27146.md), [Bugfix: Cutlass FP8 FusedMoE bad scaling factors](../sources/prs/vllm/PR-27255.md), [Feature: Support Relu2 in FusedMoE fp8 cutlass path](../sources/prs/vllm/PR-27261.md), [[Perf] SM100 - add swap AB optimization to CUTLASS FP8 GEMM](../sources/prs/vllm/PR-27284.md), [[Performance] Support FP8 flashinfer TRTLLM MOE on Qwen3 and Qwen-3next](../sources/prs/vllm/PR-27492.md), [[Attention] Use sparse prefill kernel for fp8 kv-cache in DeepSeek-v3.2](../sources/prs/vllm/PR-27532.md), [[Feature] Batch invariant torch.compile](../sources/prs/vllm/PR-27660.md), [[Performance] Fused blockwise quant RMS norm](../sources/prs/vllm/PR-27883.md), [[Performance][B200] Fix deepgemm prologue](../sources/prs/vllm/PR-27897.md), [[ROCm][MLA] enable fp8 MLA decode on ROCm](../sources/prs/vllm/PR-28032.md), [[Performance][B200] silu_mul_quant: pack scales in int32](../sources/prs/vllm/PR-28358.md), [[Performance] Reduce DeepGEMM N dim restriction from 128 to 64 multiplier ](../sources/prs/vllm/PR-28687.md), [[Perf][Kernels] Enable FlashInfer DeepGEMM swapAB on SM90 (for W8A8 Linear Op)](../sources/prs/vllm/PR-29213.md), [chore: add RTX_PRO_6000 GLM4.6-FP8 kernel tuning](../sources/prs/vllm/PR-29240.md), [[Perf] Disable DeepGEMM MoE by default when TP=8 is used](../sources/prs/vllm/PR-29346.md), [[Kernel]Support W4A8 Grouped GEMM on Hopper](../sources/prs/vllm/PR-29691.md), [[MoE-FP8-modelopt] Add FlashInfer alignment padding for intermediate dimensions](../sources/prs/vllm/PR-29748.md), [Add Mistral Large 3 and Ministral 3](../sources/prs/vllm/PR-29757.md), [[Perf] Improve fp8 quant in mla; replace ReduceSum with ReduceScatterSum](../sources/prs/vllm/PR-29795.md), [[Bugfix] Fix FP8 MoE LoRA](../sources/prs/vllm/PR-29890.md), [[Kernel][Quantization][MoE] add marlin kernel support for turing (sm75)](../sources/prs/vllm/PR-29901.md), [[ROCm] add fallback for aiter fp8 decode mla](../sources/prs/vllm/PR-30005.md), [[Quantization] Support Quark int4-fp8 w4a8 for MoE](../sources/prs/vllm/PR-30071.md), [Add llmcompressor fp8 kv-cache quant (per-tensor and per-attn_head)](../sources/prs/vllm/PR-30141.md), [[Bugfix] Fix DeepGEMM after #29546 ](../sources/prs/vllm/PR-30267.md), [[LoRA] Support Quantized Adapters](../sources/prs/vllm/PR-30286.md), [[Bugfix] Fix fp8 DeepGemm compilation issues](../sources/prs/vllm/PR-30336.md), [[ROCm][Quantization] GPT OSS Upstream MoE wmxfp4_afp8 with static scales](../sources/prs/vllm/PR-30357.md), [[Feature] Add SM103 (Blackwell Ultra) Support to vLLM](../sources/prs/vllm/PR-30484.md), [[Perf] Eliminate padding and slicing op for GPT-OSS with Flashinfer MXFP4 MXFP8 MoE](../sources/prs/vllm/PR-30647.md), [[SM100] Enable fp8 compute for prefill MLA](../sources/prs/vllm/PR-30746.md), [[Feature]: Support NVIDIA ModelOpt HF FP8 variants FP8_PER_CHANNEL_PER_TOKEN and FP8_PB_WO in vLLM](../sources/prs/vllm/PR-30957.md), [[Bugfix][Hardware][AMD] Consolidate FP8 min/max values helper function](../sources/prs/vllm/PR-31106.md), [[Bugfix][Hardware][AMD] Fix exception types in AITER MLA FP8 check](../sources/prs/vllm/PR-31177.md), [[SM100] Resubmit FMHA FP8 prefill for MLA](../sources/prs/vllm/PR-31195.md), [[Bugfix][ROCm] Fix Static Quant Issue](../sources/prs/vllm/PR-31502.md), [[Bugfix] Fix Broken ModelOpt NVFP4 MoE](../sources/prs/vllm/PR-31742.md), [[1/N][Attention] Restructure attention: move files](../sources/prs/vllm/PR-31916.md), [[5/N][Attention] Finish eliminating `vllm/attention` folder](../sources/prs/vllm/PR-32064.md), [[Perf] Create TMA-aligned input scale tensor for DeepGemm on Hopper](../sources/prs/vllm/PR-32619.md), [[Bugfix] Fix FP8 MoE EP Weight Loading for ModelOpt Llama4](../sources/prs/vllm/PR-32886.md), [Support compress-tensors with nvfp4 or fp8 weights and modelopt with nvfp4 weights on Turing](../sources/prs/vllm/PR-33076.md), [Add support for Mistral Large 3 inference with Flashinfer MoE](../sources/prs/vllm/PR-33174.md), [[Bugfix] Fix quant RMS norm fusion for quantization with TMA-aligned scales](../sources/prs/vllm/PR-33255.md), [[Bugfix] Register fp8 cutlass_group_gemm as supported for only SM90+SM100](../sources/prs/vllm/PR-33285.md), [[Kernel] Support Flashinfer trtllm fused MoE non gated FP8 & NVFP4](../sources/prs/vllm/PR-33506.md), [[Kernel] Add enable_sm120_or_later for SM121 (DGX Spark) CUTLASS support](../sources/prs/vllm/PR-33517.md), [[Perf] Disable clean_logits in deepgemm fp8_mqa_logits kernel](../sources/prs/vllm/PR-33568.md), [enable skipping of SW attention layers when using FP8 KV cache](../sources/prs/vllm/PR-33695.md), [[Kernel] Integrate SM100 MXFP8 blockscaled grouped MM and quant kernels](../sources/prs/vllm/PR-34448.md), [[Bugfix] Handle num_expert_group=None in flashinfer block-scale FP8 MoE](../sources/prs/vllm/PR-34494.md), [[Kernel] Add FP8 KV cache support to Triton MLA decode attention](../sources/prs/vllm/PR-34597.md), [[Bugfix] Fix NVFP4 TRTLLM MoE non-gated support; add gsm8k for Nemotron-3-Nano FP8+NVFP4](../sources/prs/vllm/PR-34725.md), [Integrate flashinfer mm_mxfp8 in ModelOpt MXFP8](../sources/prs/vllm/PR-35053.md), [[Feat] Add CUDA torch fallbacks for fp8_mqa_logits/fp8_paged_mqa_logits_torch function](../sources/prs/vllm/PR-35271.md), [[Attention][Perf] Optimize cp_gather_and_upconvert_fp8_kv_cache - DeepSeek-v3.2](../sources/prs/vllm/PR-35290.md), [[Quant][Feature] Support online MXFP8 quantization for MoE and dense models](../sources/prs/vllm/PR-35448.md), [[ROCm] Support MLA with nhead<16 and FP8 KV cache for TP=8 (Kimi K2.5/Linear)](../sources/prs/vllm/PR-35850.md), [[Perf] Support FP8 KV cache for Flashinfer MLA Sparse](../sources/prs/vllm/PR-35891.md), [Add support for ModelOpt MXFP8 MoE models](../sources/prs/vllm/PR-35986.md), [[Bugfix] Fix passing of activation_type to trtllm fused MoE NVFP4 and FP8](../sources/prs/vllm/PR-36017.md), [[mla] Support fused FP8/NVFP4 output quantization in MLA attention (#35792)](../sources/prs/vllm/PR-36205.md), [[Perf] Add TRTLLM FP8 MoE Modular Kernel](../sources/prs/vllm/PR-36307.md), [[XPU] Support block fp8 moe by fallback to TritonExpert on XPU](../sources/prs/vllm/PR-36458.md), [[Kernel] Fuse FP8 output quantization into merge_attn_states](../sources/prs/vllm/PR-36518.md), [[Bug][MoE] Strengthen _supports_current_device() checks in the TRTLLM FP8, NVFP4, and FlashInfer CuteDSL MoE experts](../sources/prs/vllm/PR-36728.md), [[Bugfix] Fix KV scales inconsistency in fp8 MLA & FlashInfer kv_cache_dtype "auto" leading to gibberish](../sources/prs/vllm/PR-37054.md), [[XPU] support MLA model on Intel GPU](../sources/prs/vllm/PR-37143.md), [[Perf] Set Flashinfer sparse MLA as default backend for FP8 kv cache](../sources/prs/vllm/PR-37252.md), [[Bugfix] Disable monolithic TRTLLM MoE for Renormalize routing (#37591)](../sources/prs/vllm/PR-37605.md), [[Perf] Use torch compile to fuse pack topk in trtllm moe](../sources/prs/vllm/PR-37695.md), [[Bug] Fix fp8 deepgemm batch invariant](../sources/prs/vllm/PR-37718.md), [[Kernel] Optimize SM120 CUTLASS blockwise FP8 GEMM](../sources/prs/vllm/PR-37970.md), [[Perf] FP8 FlashInfer Attn for ViT](../sources/prs/vllm/PR-38065.md), [[Bugfix] Fix DeepGemm E8M0 accuracy degradation for Qwen3.5 FP8 on Blackwell](../sources/prs/vllm/PR-38083.md), [[Kernel] Add swapAB support for SM120 CUTLASS blockwise FP8 GEMM ](../sources/prs/vllm/PR-38325.md), [[MoE] Add RoutingMethodType.Simulated to TRT-LLM FP8/NVFP4 kernel allowlists](../sources/prs/vllm/PR-38329.md), [[NVIDIA] Bugfix NVFP4 DGX Spark and RTX50](../sources/prs/vllm/PR-38423.md), [[QeRL] Fix online quantized reloading](../sources/prs/vllm/PR-38442.md), [[Kernels][MoE] Fix legacy_routing to use bitmatrix-based routing path](../sources/prs/vllm/PR-38504.md), [[XPU] add xpu backend implementation of mxfp8 quant](../sources/prs/vllm/PR-38682.md), [[Quant] add CompressedTensorsW8A8Mxfp8 for linear and MoE layers](../sources/prs/vllm/PR-38815.md), [[Bugfix] Re-enable Renormalize routing for TRT-LLM MoE experts](../sources/prs/vllm/PR-38859.md), [[Bugfix] Fix broken explicit unquantized kv cache dtype support](../sources/prs/vllm/PR-38922.md), [[MoE Refactor] Split up compressed_tensors_moe.py](../sources/prs/vllm/PR-38960.md), [[Bug] Fix routing bias dtype for trtllm per-block fp8 moe](../sources/prs/vllm/PR-38989.md), [[Perf] Change Trtllm fp8 MoE to use Shuffled Weights and BlockMajorK Layout](../sources/prs/vllm/PR-38993.md), [[Bug] Fix Trtllm Fp8 MoE Weight Shuffle Memory Fragamentation](../sources/prs/vllm/PR-39054.md), [perf(moe): add tuned fused_moe config for RTX PRO 6000 Blackwell Server Edition](../sources/prs/vllm/PR-39183.md), [[Refactor] Move MXFP8 GEMM management into MxFp8LinearKernel](../sources/prs/vllm/PR-39205.md), [[Perf] Fuse Zero Initializer for FP8 DeepGemm Block Quant Kernel](../sources/prs/vllm/PR-39547.md), [add warning when FP8 KV cache misses prefill query quantization](../sources/prs/vllm/PR-39752.md), [[Perf] Batch invariance with Cutlass fp8 support, 28.9% E2E latency improvement](../sources/prs/vllm/PR-40408.md), [[MoE] Move cutlass moe to fused_moe/experts/](../sources/prs/vllm/PR-40574.md), [[Kernel][Helion] Optimize Helion config parsing latency](../sources/prs/vllm/PR-40850.md), [[DSV4] Add BF16 and MXFP8 A2A support for flashinfer a2a one sided](../sources/prs/vllm/PR-40960.md), [Faster per-token fp8 group quant packed kernel for blackwell](../sources/prs/vllm/PR-41326.md), [[Quantization] Rework quantization_config to use QuantKey and allow for activation override](../sources/prs/vllm/PR-41566.md), [[CPU] Add MXFP4 W4A16 MoE support](../sources/prs/vllm/PR-41922.md), [[MoE] Move various experts classes to fused_moe/experts/](../sources/prs/vllm/PR-41979.md), [[Bugfix] Add swiglu limits to deepgemm fp8 methods](../sources/prs/vllm/PR-41986.md), [[feat] Add FP8 per-tensor Q scale support to Triton attention backend](../sources/prs/vllm/PR-42080.md), [[Perf] Use 2D-grid to eliminate divmod in W8W8 group quant](../sources/prs/vllm/PR-42153.md), [[Perf] Wire silu_and_mul_per_block_quant into TritonFP8MoE (MiniMax-M2) ](../sources/prs/vllm/PR-42497.md), [[6/n] Migrate activation kernels, gptq, gguf, non cutlass w8a8 to libtorch stable ABI (continued)](../sources/prs/vllm/PR-42663.md), [[Perf] Add do_not_specialize in fused FP8 RoPE kernel](../sources/prs/vllm/PR-42849.md), [add cutedsl dsv4 indexer fp8 kernel](../sources/prs/vllm/PR-42899.md), [[Model Refactoring] Move deepseek_v4_ops to models/deepseek_v4 [3/N]](../sources/prs/vllm/PR-43073.md), [NVFP4 and Block-Scaled Narrow Precision](../wiki/hardware/nvfp4.md), [DeepGEMM — FP8 GEMM with Fine-Grained Scaling](../wiki/kernels/deepgemm.md), [FlashMLA — Multi-head Latent Attention](../wiki/kernels/flashmla.md), [FP8 Block-Scale GEMM](../wiki/kernels/fp8-block-scale-gemm.md), [Fused MoE — FP8 Block-Scale Routing + Dual GEMM](../wiki/kernels/fused-moe.md), [Grouped GEMM for MoE](../wiki/kernels/grouped-gemm.md), [Sparse MLA (DeepSeek V3.2)](../wiki/kernels/sparse-mla.md), [Fine-Grained FP8/FP4 Quantization](../wiki/techniques/fine-grained-quantization.md) | +| `2sm-cooperative` | [Microbenchmarking NVIDIA's Blackwell Architecture](../sources/blogs/blackwell-microbenchmarking.md), [Colfax CUTLASS Tutorial: GEMM Kernels Using Tensor Memory for Blackwell](../sources/blogs/colfax-cutlass-blackwell.md), [FlashAttention-4 Blog](../sources/blogs/flash-attention-4.md), [Writing High-Performance Matrix Multiplication Kernels for Blackwell with JAX Pallas](../sources/blogs/jax-pallas-blackwell-matmul.md), [Modular: Matrix Multiplication on Blackwell](../sources/blogs/modular-blackwell-matmul.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [CUTLASS 4.5.0 Cluster Launch Control Documentation](../sources/docs/cutlass-clc-documentation.md), [FlashAttention-4: Algorithm and Kernel Co-design for Blackwell GPUs](../sources/docs/flash-attention-4.md), [NVIDIA Blackwell Tuning Guide](../sources/docs/nvidia-blackwell-tuning-guide.md), [Triton v3.6.0 — Incremental Blackwell Changes](../sources/docs/triton-3.6-blackwell.md), [[TRTLLM-9831][perf] Enable 2CTA with autotune for CuteDSL MoE and Grouped GEMM optimizations](../sources/prs/TensorRT-LLM/PR-10201.md), [fix: examples/cute/tutorial/blackwell/04_mma_tma_2sm_sm100.cu GridDim miscalculated](../sources/prs/cutlass/PR-2492.md), [Feat([FA4][CUTE DSL]) Add head_dim=256 support (forward + backward)](../sources/prs/flash-attention/PR-2412.md), [[hd256] Improve forward kernel with exp2 FMA emulation (3% to 9% performance gain)](../sources/prs/flash-attention/PR-2488.md), [[hd256] Add TMA paged KV support to SM100 2CTA forward kernel](../sources/prs/flash-attention/PR-2489.md), [[FA4][hd256] Backward TMA bulk-store epilogue + LSE/dpsum coalesce](../sources/prs/flash-attention/PR-2497.md), [[Feature] 2-SM support for TMA, TMEM and TCGEN5MMA on Blackwell](../sources/prs/tilelang/PR-1882.md), [Two-SM Cooperative MMA](../wiki/hardware/2sm-cooperative.md), [FlashAttention-4](../wiki/kernels/flash-attention-4.md), [Not Reaching the Relevant Compute Ceiling](../wiki/patterns/compute-bound.md) | +| `block-scale` | [Twelve Attempts at an FP4 Kernel](../sources/blogs/amandeep-nvfp4-attempts.md), [DeepGEMM — Pinned Upstream Project Summary](../sources/blogs/deepgemm.md), [NVFP4 Format Details](../sources/blogs/nvfp4-format-details.md), [NVFP4 GEMV and Improved NVFP4 GEMV](../sources/blogs/simon-nvfp4-gemv.md), [TFLOPS Gap: Why FP4 MoE Kernel Engineering Matters on Blackwell](../sources/blogs/tflops-gap-fp4-moe.md), [Blackwell NVFP4 Kernel Hackathon Journey](../sources/blogs/yue-nvfp4-hackathon.md), [FlashInfer MLSys 2026 Track A: FP8 Block-Scale MoE](../sources/contests/flashinfer-mlsys26/track-a-fused-moe.md), [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [GPU Mode NVFP4 Hackathon - Problem 1: Batched GEMV](../sources/contests/gpu-mode-nvfp4/problem-1-gemv.md), [GPU Mode NVFP4 Hackathon - Problem 2: NVFP4 GEMM](../sources/contests/gpu-mode-nvfp4/problem-2-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 3: Gated Dual GEMM](../sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 4: Grouped GEMM](../sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [DeepSeek-V3 Technical Report: FP8 Training](../sources/docs/deepseek-v3-fp8.md), [NVIDIA Blackwell Tuning Guide](../sources/docs/nvidia-blackwell-tuning-guide.md), [NVIDIA CUDA Toolkit 13.x for Blackwell](../sources/docs/nvidia-cuda-13.md), [NVIDIA CUTLASS 4.5.0 Blackwell Sources](../sources/docs/nvidia-cutlass-blackwell.md), [PTX ISA 9.0 SM100 Instruction Reference](../sources/docs/nvidia-ptx-isa-sm100.md), [Transformer Engine 2.13: NVFP4](../sources/docs/nvidia-transformer-engine-2.13-nvfp4.md), [Triton v3.6.0 — Incremental Blackwell Changes](../sources/docs/triton-3.6-blackwell.md), [[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes](../sources/prs/DeepGEMM/PR-304.md), [[TRTLLM-9457][feat] Add cute dsl fp8 gemm for Blackwell](../sources/prs/TensorRT-LLM/PR-10130.md), [Blockwise and Groupwise GEMM for Blackwell and Improvements for Hopper](../sources/prs/cutlass/PR-2139.md), [[CuTeDSL] Fix: SM100 block-scale gemm overlapping accumulator](../sources/prs/cutlass/PR-2995.md), [perf: Enable SplitK and fix tile-scheduling for moe fp4 fused moe](../sources/prs/flashinfer/PR-1548.md), [feat: enable deepgemm jit for fp8 block-scale on SM90](../sources/prs/flashinfer/PR-1969.md), [feat: Support unpadded output hidden size for trtllm_fp4_block_scale_moe](../sources/prs/flashinfer/PR-2217.md), [fix: trtllm_mxint4_block_scale_moe unit test to index output list](../sources/prs/flashinfer/PR-2627.md), [int16 Block-Scaled State and Stochastic Rounding for SSU (mamba)](../sources/prs/flashinfer/PR-2645.md), [Support in-place update for `trtllm_fp8_block_scale_moe`](../sources/prs/flashinfer/PR-2739.md), [fix: int32 overflow in `trtllm_fp4_block_scale_moe` causing "Unsupported hidden state scale shape" for EP32+ configs](../sources/prs/flashinfer/PR-2853.md), [fix: snap weight_scale_vec_size to handle block_scale_interleave padding for SM120](../sources/prs/flashinfer/PR-2898.md), [fix: Fix autotuner crash on meta-device tensor in trtllm_fp4_block_scale_routed_moe](../sources/prs/flashinfer/PR-2916.md), [Only swizzle on v block scale; rename kv_block_scales to kv_cache_sf](../sources/prs/flashinfer/PR-2954.md), [feat(trace): embed runnable init() in every TraceTemplate](../sources/prs/flashinfer/PR-3221.md), [[Perf] Eliminate the slice op for Flashinfer `trtllm_fp4_block_scale_moe`](../sources/prs/sglang/PR-15731.md), [integrate blockwise fp8 kernel](../sources/prs/sglang/PR-3529.md), [[Feature] Block-scaled GEMM support for MXFP8 on Blackwell](../sources/prs/tilelang/PR-1945.md), [[NVIDIA] Support nvfp4 tcgen05 gemm](../sources/prs/vllm/PR-13571.md), [[FEAT] [ROCm]: Add AITER Block-Scaled GEMM Feature](../sources/prs/vllm/PR-14968.md), [[Kernel][B200] mxfp4 fused cutlass moe](../sources/prs/vllm/PR-23696.md), [[Bugfix] Handle num_expert_group=None in flashinfer block-scale FP8 MoE](../sources/prs/vllm/PR-34494.md), [NVFP4 and Block-Scaled Narrow Precision](../wiki/hardware/nvfp4.md), [DeepGEMM — FP8 GEMM with Fine-Grained Scaling](../wiki/kernels/deepgemm.md), [FP8 Fine-Grained-Scale GEMM](../wiki/kernels/fp8-block-scale-gemm.md), [FlashInfer Track A FP8 Block-Scale MoE](../wiki/kernels/fused-moe.md), [GPU Mode NVFP4 Gated Dual GEMM](../wiki/kernels/gated-dual-gemm.md), [NVFP4 GEMM — GPU Mode Problem 2 Contract](../wiki/kernels/nvfp4-gemm.md), [NVFP4 Batched GEMV](../wiki/kernels/nvfp4-gemv.md), [Fine-Grained FP8/FP4 Quantization](../wiki/techniques/fine-grained-quantization.md) | +| `clc` | [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [CUTLASS 4.5.0 Cluster Launch Control Documentation](../sources/docs/cutlass-clc-documentation.md), [NVIDIA Blackwell Tuning Guide](../sources/docs/nvidia-blackwell-tuning-guide.md), [NVIDIA CUDA Toolkit 13.x for Blackwell](../sources/docs/nvidia-cuda-13.md), [NVIDIA CUTLASS 4.5.0 Blackwell Sources](../sources/docs/nvidia-cutlass-blackwell.md), [PTX ISA 9.0 SM100 Instruction Reference](../sources/docs/nvidia-ptx-isa-sm100.md), [Blockwise Improvement and Programmatic Dependent Launch](../sources/prs/cutlass/PR-2161.md), [Support for GEMM-K=0 for Blackwell Grouped GEMMs](../sources/prs/cutlass/PR-2746.md), [[Cute-DSL] Add option for issue_clc_query without multicast](../sources/prs/cutlass/PR-3021.md), [[Ai-assisted] CLC work stealing](../sources/prs/flash-attention/PR-2218.md), [Add CLC scheduler heuristic](../sources/prs/flash-attention/PR-2455.md), [[V1][P/D]P2pNcclConnector supports flashinfer](../sources/prs/vllm/PR-23536.md), [Cluster Launch Control (CLC)](../wiki/hardware/clc.md), [PTX Instructions for SM100](../wiki/languages/ptx-sm100.md), [Low SM Utilization](../wiki/patterns/low-sm-utilization.md), [MoE Expert Load Imbalance](../wiki/patterns/moe-load-imbalance.md), [Tail Effect — Last-Wave Underutilization](../wiki/patterns/tail-effect.md), [Persistent Kernels with CLC](../wiki/techniques/persistent-kernels.md), [Tile Scheduling Strategies](../wiki/techniques/tile-scheduling.md) | +| `cluster` | [Microbenchmarking NVIDIA's Blackwell Architecture](../sources/blogs/blackwell-microbenchmarking.md), [Qwen3-Next: Hybrid GDN+MoE Architecture on NVIDIA Blackwell](../sources/blogs/qwen3-next-architecture.md), [CUTLASS 4.5.0 Cluster Launch Control Documentation](../sources/docs/cutlass-clc-documentation.md), [Two-SM Cooperative MMA](../wiki/hardware/2sm-cooperative.md) | +| `fp4` | [Twelve Attempts at an FP4 Kernel](../sources/blogs/amandeep-nvfp4-attempts.md), [Microbenchmarking NVIDIA's Blackwell Architecture](../sources/blogs/blackwell-microbenchmarking.md), [Anatomy of a Reward Hack](../sources/blogs/gpu-mode-reward-hack.md), [NVFP4 Format Details](../sources/blogs/nvfp4-format-details.md), [NVFP4 GEMV and Improved NVFP4 GEMV](../sources/blogs/simon-nvfp4-gemv.md), [TFLOPS Gap: Why FP4 MoE Kernel Engineering Matters on Blackwell](../sources/blogs/tflops-gap-fp4-moe.md), [Tilus: A Tile-Level GPGPU Programming Language for Low-Precision Computation](../sources/blogs/tilus-nvidia.md), [Blackwell NVFP4 Kernel Hackathon Journey](../sources/blogs/yue-nvfp4-hackathon.md), [GPU Mode NVFP4 Hackathon - Problem 1: Batched GEMV](../sources/contests/gpu-mode-nvfp4/problem-1-gemv.md), [GPU Mode NVFP4 Hackathon - Problem 2: NVFP4 GEMM](../sources/contests/gpu-mode-nvfp4/problem-2-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 3: Gated Dual GEMM](../sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 4: Grouped GEMM](../sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [NVIDIA Blackwell Tuning Guide](../sources/docs/nvidia-blackwell-tuning-guide.md), [NVIDIA CUDA Toolkit 13.x for Blackwell](../sources/docs/nvidia-cuda-13.md), [PTX ISA 9.0 SM100 Instruction Reference](../sources/docs/nvidia-ptx-isa-sm100.md), [Transformer Engine 2.13: NVFP4](../sources/docs/nvidia-transformer-engine-2.13-nvfp4.md), [[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes](../sources/prs/DeepGEMM/PR-304.md), [Sync nv_dev with upstream #316 (Mega MoE optimizations & benchmarks)](../sources/prs/DeepGEMM/PR-328.md), [[TRTLLM-9661][chore] Further reduce tuning time for cuteDSL nvFP4 dense gemm.](../sources/prs/TensorRT-LLM/PR-10339.md), [[None][feat] fuse shared to sparse experts in TRT-LLM Gen MoE](../sources/prs/TensorRT-LLM/PR-11143.md), [[https://nvbugs/5799917][fix] Recover from CUTLASS MoE doActivation perf regression for MXFP4/NVFP4 dtype](../sources/prs/TensorRT-LLM/PR-11165.md), [[None][feat] Optimize super-v3 nvfp4 for better perf](../sources/prs/TensorRT-LLM/PR-11273.md), [[None][feat] Optimize by fuse nvfp4_quant to layernorm_gated for mamba2_mixer](../sources/prs/TensorRT-LLM/PR-11473.md), [[None][feat] Add support for expert_number<=2048 and K<=32](../sources/prs/TensorRT-LLM/PR-11510.md), [[TRTLLM-10004][feat] Enable GEMM -> AR with GEMM output in registered buffers](../sources/prs/TensorRT-LLM/PR-11589.md), [[https://nvbugs/5799917][fix] Recover from CUTLASS MoE doActivation perf regression for MXFP4/NVFP4 dtype](../sources/prs/TensorRT-LLM/PR-11733.md), [[None][fix] Fix SM120 issue for rms_norm with nvfp4_quant_fusion](../sources/prs/TensorRT-LLM/PR-11774.md), [[None][feat] Support update weight for nvfp4](../sources/prs/TensorRT-LLM/PR-12320.md), [[None][feat] Update rms_norm + fp4_qaunt kernel supporting more dim](../sources/prs/TensorRT-LLM/PR-13033.md), [[None][feat] Add FP4 residual quantization kernel without channel reo…](../sources/prs/TensorRT-LLM/PR-13117.md), [[None][feat] Integrate FP4 indexer for DSA on Blackwell](../sources/prs/TensorRT-LLM/PR-13340.md), [[TRTLLM-11127][feat] add W4A8_MXFP4_FP8 MoE unit test support](../sources/prs/TensorRT-LLM/PR-13401.md), [[TRTLLM-12316][feat] Integrate FP4 indexer for DSv4](../sources/prs/TensorRT-LLM/PR-13575.md), [[TRTLLM-12503][feat] Parallel VAE independent scaling and fix arg passing](../sources/prs/TensorRT-LLM/PR-13873.md), [[TRTLLM-35237][feat] Add cute dsl FP4 paged MQA logits decode kernel](../sources/prs/TensorRT-LLM/PR-13929.md), [[https://nvbugs/6025177][test] rcca tests using kimi k2.5 fp4](../sources/prs/TensorRT-LLM/PR-14172.md), [[None][fix] Add SPDX Apache-2.0 headers to auto_deploy test files](../sources/prs/TensorRT-LLM/PR-14193.md), [feat: Add w4a8_mxfp4_fp8 quantization recipe.](../sources/prs/TensorRT-LLM/PR-4867.md), [[OMNIML-2336][feat] Add NVFP4 x FP8](../sources/prs/TensorRT-LLM/PR-6809.md), [[TRTLLM-8535][feat] Support DeepSeek V3.2 with FP8 + BF16 KV cache/NVFP4 + BF16 KV cache](../sources/prs/TensorRT-LLM/PR-8405.md), [[None][feat] Enable nvfp4 cuda core for sm120](../sources/prs/TensorRT-LLM/PR-8620.md), [[None][feat] Update TRTLLM MoE cubins; reduce mxfp4 weight padding requirement; tighten TMA bound](../sources/prs/TensorRT-LLM/PR-9025.md), [[None][feat] add fp4 gemm + allreduce](../sources/prs/TensorRT-LLM/PR-9729.md), [[None][feat] Port fp4 quantization kernel optimization from FlashInfer](../sources/prs/TensorRT-LLM/PR-9854.md), [bugfix: Fix test and output shape of fp4 quantize](../sources/prs/flashinfer/PR-1114.md), [Add fp4 quantization swizzling tests](../sources/prs/flashinfer/PR-1157.md), [Expose fp4 blockscale swizzling kernel](../sources/prs/flashinfer/PR-1176.md), [Feature/sm100 low latency nvfp4 kernels](../sources/prs/flashinfer/PR-1214.md), [feat: Support MXFP8 x MXFP4 CUTLASS grouped GEMM](../sources/prs/flashinfer/PR-1241.md), [Reduce the JIT compilation time of gen_gemm_sm100_module](../sources/prs/flashinfer/PR-1251.md), [add mm_fp4 use cudnn backend](../sources/prs/flashinfer/PR-1288.md), [Update cutlass fp4 moe kernels](../sources/prs/flashinfer/PR-1294.md), [add cutlass backend for mm_fp4](../sources/prs/flashinfer/PR-1296.md), [Refactor Fused Moe Module](../sources/prs/flashinfer/PR-1309.md), [Support loading autotuned results from json for cutlass fp4 moe backends](../sources/prs/flashinfer/PR-1310.md), [feat: support output nvfp4 in trtllm-gen function call.](../sources/prs/flashinfer/PR-1318.md), [feat: masked layout fp4 gemm using cute-dsl](../sources/prs/flashinfer/PR-1331.md), [add torch float4_e2m1fn_x2 check for cudnn fp4 backend](../sources/prs/flashinfer/PR-1333.md), [[Fix] remove torch 2.8 requirement for FP4 GEMM](../sources/prs/flashinfer/PR-1334.md), [feature: add fp4 mm using trtllm backend](../sources/prs/flashinfer/PR-1355.md), [hotfix: update mxfp4 groupwise-scaled gemm unittests](../sources/prs/flashinfer/PR-1359.md), [support trtllm-gen prefill fp4 output](../sources/prs/flashinfer/PR-1360.md), [Update autotune results for the nvfp4 cutlass moe backends for v0.2.9](../sources/prs/flashinfer/PR-1361.md), [Support scale factor start index for fp4 mha prefill/decode](../sources/prs/flashinfer/PR-1363.md), [bugfix: fixed cutlass fused moe usage of FP4QuantizationSFLayout::SWIZZLED](../sources/prs/flashinfer/PR-1371.md), [bugfix: Add guard for fp4/fp8 related include headers](../sources/prs/flashinfer/PR-1376.md), [GPT-OSS Support: Add Blackwell MoE mxfp4 implementation from TRTLLM and Attention Sink](../sources/prs/flashinfer/PR-1389.md), [gpt-oss: Add MXFP8 x MXFP4 CUTLASS MOE for SM100 and BF16 x MXFP4 CUTLASS for SM90 + SwigluBias Activation](../sources/prs/flashinfer/PR-1396.md), [Add Mxfp4 trtllm-gen moe unit tests](../sources/prs/flashinfer/PR-1399.md), [feature: enable cublas for fp4 gemm when cudnn == 9.11.1 or >= 9.13](../sources/prs/flashinfer/PR-1405.md), [Faster weight processing (moe nvfp4)](../sources/prs/flashinfer/PR-1412.md), [Remove getEnvEnablePDL in favor of enable_pdl parameter](../sources/prs/flashinfer/PR-1446.md), [Fix TRTLLM NVFP4-out attention kernel scale factor dim issue](../sources/prs/flashinfer/PR-1460.md), [tuner: Trtllm-gen Fp4 MoE Autotunner](../sources/prs/flashinfer/PR-1475.md), [fix missing enable_pdl argument in trtllm-gen fp4 moe](../sources/prs/flashinfer/PR-1480.md), [Add python API for masked grouped gemm](../sources/prs/flashinfer/PR-1481.md), [fix: update masked moe gemm fp4 tensor reshape](../sources/prs/flashinfer/PR-1495.md), [feat: scaling at fp4 gemm epilogue](../sources/prs/flashinfer/PR-1498.md), [refactor fp4 masked gemm cute-dsl implementation and add manual cache](../sources/prs/flashinfer/PR-1521.md), [Add GeGLU support to trtllm-gen NVFP4 Fused MoE Kernel](../sources/prs/flashinfer/PR-1525.md), [perf: Enable SplitK and fix tile-scheduling for moe fp4 fused moe](../sources/prs/flashinfer/PR-1548.md), [fix: separate out fp4 lib into sm90 and sm100 versions, add oob checking in fused moe](../sources/prs/flashinfer/PR-1565.md), [update trtllm-gen fp4 autotuner and routing](../sources/prs/flashinfer/PR-1573.md), [bugfix: Fix test_fp4_quantize test bug](../sources/prs/flashinfer/PR-1585.md), [feat: initial support for SM103, SM110, SM120, SM121](../sources/prs/flashinfer/PR-1608.md), [feat: cutlass fp4 gemm bringup for SM120 & SM121](../sources/prs/flashinfer/PR-1609.md), [bugfix: fix fp4 quantization with 8x4 scale factor layout](../sources/prs/flashinfer/PR-1611.md), [feat: add support of fp4_batched_quantize](../sources/prs/flashinfer/PR-1633.md), [Added mx_fp4 support using the cudnn backend](../sources/prs/flashinfer/PR-1644.md), [[Hotfix] `test_fp4_quantize.py` failure on sm103](../sources/prs/flashinfer/PR-1666.md), [feat: Benchmark mm_fp4 mxfp4 support and gemm autotune support. Restore mm_fp4 API behavior](../sources/prs/flashinfer/PR-1706.md), [test: skip the unsupported test cases for sm120/121](../sources/prs/flashinfer/PR-1710.md), [Added xfail for mx_fp4 matmul on SM120](../sources/prs/flashinfer/PR-1766.md), [Masked batch nvfp4 quantization](../sources/prs/flashinfer/PR-1774.md), [Support checks PoC](../sources/prs/flashinfer/PR-1809.md), [fix: fp4 moe on sm120](../sources/prs/flashinfer/PR-1817.md), [[Quantization] Add per-expert global scaling factor for fp4 batched quantize](../sources/prs/flashinfer/PR-1835.md), [raise error for group_gemm_fp8_nt_groupwise then num_groups > 1 on sm120/121](../sources/prs/flashinfer/PR-1862.md), [feat: Add FP4 TRTLLM-Gen throughput MOE batched gemms](../sources/prs/flashinfer/PR-1882.md), [silu_and_mul nvfp4 quanization fusion rework](../sources/prs/flashinfer/PR-1927.md), [fix: Add cutlass as an mm_fp4 backend in compute capability 12.0 in benchmark code](../sources/prs/flashinfer/PR-1959.md), [feat: Add backend='auto' to mm_fp4 and enable autotune for backend='cudnn'](../sources/prs/flashinfer/PR-1979.md), [Feature: Support non-gated activation in cutlass fused MoE nvfp4](../sources/prs/flashinfer/PR-2011.md), [fix: Enable SM121 for mm_fp4](../sources/prs/flashinfer/PR-2012.md), [[DSV3] Optimized Router Gemm](../sources/prs/flashinfer/PR-2019.md), [update trtllm cutlass moe ](../sources/prs/flashinfer/PR-2020.md), [perf: Speed up fp4 quantization for small batch with swizzling for cutlass MoE](../sources/prs/flashinfer/PR-2025.md), [[BUG] Fix trtllm-gen fp4 moe renormalize routing](../sources/prs/flashinfer/PR-2049.md), [Patch sm103 for 3xfp4 moe generation](../sources/prs/flashinfer/PR-2082.md), [perf: enable pdl for cutlass fp4 gemm](../sources/prs/flashinfer/PR-2095.md), [feat: MxInt4 x Bf16 TRT-LLM Gen MoE support](../sources/prs/flashinfer/PR-2159.md), [Add data type check for deepseek fp4 moe](../sources/prs/flashinfer/PR-2165.md), [feat: Support unpadded output hidden size for trtllm_fp4_block_scale_moe](../sources/prs/flashinfer/PR-2217.md), [feat: Fused RMSNorm + FP4 Quantization Kernels in CuTe-DSL](../sources/prs/flashinfer/PR-2233.md), [fix: Add global scale support and optional output allocation for RMSNorm+FP4Quant fusion kernels](../sources/prs/flashinfer/PR-2260.md), [[performance]optimize for nvfp4](../sources/prs/flashinfer/PR-2268.md), [[WIP] Refactor: simplify torch -> cute-dsl boilerplate and enable tvm-ffi for cute-dsl kernels](../sources/prs/flashinfer/PR-2279.md), [[Perf][Feature] Add SM103-specific schedulers for NVFP4 CUTLASS kernels](../sources/prs/flashinfer/PR-2303.md), [feat: Support Fused MoE non gated Relu2 NVFP4 & FP8 and support Nemotron](../sources/prs/flashinfer/PR-2304.md), [Optimize quantization function in large problem size](../sources/prs/flashinfer/PR-2343.md), [fix: In-place Residual Update for add_rmsnorm_fp4quant](../sources/prs/flashinfer/PR-2385.md), [feat: Add output_both_sf_layouts option to add_rmsnorm_fp4quant API](../sources/prs/flashinfer/PR-2395.md), [feat: cuteDSL fp4 moe for better DSR1 performance.](../sources/prs/flashinfer/PR-2398.md), [perf: mm_fp4 heuristic prioritizes CUTLASS over cuDNN on SM103](../sources/prs/flashinfer/PR-2404.md), [refactor: simplify fp4 rmsnorm](../sources/prs/flashinfer/PR-2421.md), [refactor: refactoring cuda code to cute-dsl (part 1)](../sources/prs/flashinfer/PR-2428.md), [Add cute-dsl backends to mxfp[8,4]_quantization for future refactor](../sources/prs/flashinfer/PR-2443.md), [perf: add fp4 GEMM tile configs and streamK scheduler for SM120](../sources/prs/flashinfer/PR-2460.md), [feat: Support Fused MoE non gated Relu2 NVFP4 & FP8 and support Nemotron, fixed](../sources/prs/flashinfer/PR-2462.md), [Support NVFP4 KV cache decode on SM120](../sources/prs/flashinfer/PR-2520.md), [feat: cute dsl mmfp4 for blackwell](../sources/prs/flashinfer/PR-2540.md), [[Bugfix][comm] Fix FP4 one-shot launch config instability in trtllm_allreduce_fusion](../sources/prs/flashinfer/PR-2557.md), [[Bug] Fix spark unit test failures for test_add_rmsnorm_fp4_quant_cute_dsl](../sources/prs/flashinfer/PR-2573.md), [tests: add bias testing to nvfp4 moe](../sources/prs/flashinfer/PR-2585.md), [fix: cute dsl nvfp4 moe routing index error](../sources/prs/flashinfer/PR-2629.md), [fix: add SM121 support to SM120 version guards](../sources/prs/flashinfer/PR-2631.md), [benchmark: Add MXFP4/MXFP8 quantization mode support to FP4 MoE benchmark](../sources/prs/flashinfer/PR-2635.md), [Enable sm120f compilation](../sources/prs/flashinfer/PR-2650.md), [[feat] trtllm-gen mxfp8 gemm](../sources/prs/flashinfer/PR-2653.md), [feat: support mxfp4 & mxfp8 entrypoint for blackwell cutedsl dense gemm](../sources/prs/flashinfer/PR-2660.md), [perf: Update trtllm-gen batched GEMM kernels - faster, more NVFP4 tile dims, MXFP8 with relu2 act](../sources/prs/flashinfer/PR-2667.md), [Add NVFP4 KV cache quantization support for SM100](../sources/prs/flashinfer/PR-2702.md), [fix: Add SM120 (RTX Blackwell desktop) support for NVFP4 MoE kernels](../sources/prs/flashinfer/PR-2725.md), [Support for MXFP4 and NVFP4 group GEMMs on GeForce and Spark](../sources/prs/flashinfer/PR-2738.md), [feat: Add FP4 KV cache quant/dequant kernels ](../sources/prs/flashinfer/PR-2757.md), [feat: Add CuTe-DSL backend for NVFP4 quantization](../sources/prs/flashinfer/PR-2838.md), [fix: int32 overflow in `trtllm_fp4_block_scale_moe` causing "Unsupported hidden state scale shape" for EP32+ configs](../sources/prs/flashinfer/PR-2853.md), [fix: snap weight_scale_vec_size to handle block_scale_interleave padding for SM120](../sources/prs/flashinfer/PR-2898.md), [perf: Optimize CuTe-DSL fp4 and fp8 quantization kernels](../sources/prs/flashinfer/PR-2904.md), [fix: Fix autotuner crash on meta-device tensor in trtllm_fp4_block_scale_routed_moe](../sources/prs/flashinfer/PR-2916.md), [CuTe DSL FP4 GEMM Heuristic](../sources/prs/flashinfer/PR-2940.md), [Only swizzle on v block scale; rename kv_block_scales to kv_cache_sf](../sources/prs/flashinfer/PR-2954.md), [[Fmha] support nvfp4 output keepsMmaAb generation kernels](../sources/prs/flashinfer/PR-2988.md), [ Fix MXFP4/MXFP8 failures in SM120 FAST_BUILD and expand all_tiles[] ](../sources/prs/flashinfer/PR-2994.md), [feat: add PDL support to rmsnorm_fp4quant and add_rmsnorm_fp4quant CuTe DSL kernels](../sources/prs/flashinfer/PR-3008.md), [perf: Port TRT-LLM SM120/SM121 FP4 CUTLASS GEMM optimizations. Add PDL](../sources/prs/flashinfer/PR-3026.md), [[feat] Trtllm-gen Per-token Nvfp4 MoE](../sources/prs/flashinfer/PR-3027.md), [feat: Add backend="b12x" for mm_fp4 on SM120](../sources/prs/flashinfer/PR-3051.md), [feat: Add b12x CuTe DSL fused MoE for SM120](../sources/prs/flashinfer/PR-3066.md), [Support NVFP4 KV for prefill and batch attention kernels](../sources/prs/flashinfer/PR-3097.md), [Integrate CUTLASS Small Tile N Blockscaled GEMMs/Grouped GEMMs for SM120 and SM121](../sources/prs/flashinfer/PR-3152.md), [feat(trace): embed runnable init() in every TraceTemplate](../sources/prs/flashinfer/PR-3221.md), [perf: optimize per-token nvfp4 quantization kernel.](../sources/prs/flashinfer/PR-3237.md), [feat(moe): add SM120 W4A16 b12x kernels](../sources/prs/flashinfer/PR-3271.md), [feat: Add FP4 (E2M1) KV Cache Support with Quantization Utilities for MLA](../sources/prs/sglang/PR-10078.md), [Optimize nvfp4 block scaled gemm kernel when M is small.](../sources/prs/sglang/PR-10101.md), [Fix chunked prefix cache for nvfp4](../sources/prs/sglang/PR-10180.md), [Fix correction bias undefined behavior for nvfp4 models](../sources/prs/sglang/PR-10426.md), [feat: add dsv3 fp4 cutlass moe etp ut](../sources/prs/sglang/PR-10433.md), [Fix bias handling in TritonMoeQuantInfo within quantization/mxfp4.py](../sources/prs/sglang/PR-10579.md), [Fix MTP MoE weight loading with NVFP4 target model.](../sources/prs/sglang/PR-10758.md), [[NVIDIA] Add new SMs support for Spark & Thor](../sources/prs/sglang/PR-11287.md), [Support running FP4 Deepseek on SM120.](../sources/prs/sglang/PR-11708.md), [support cutlass fp4 kernel in sm120](../sources/prs/sglang/PR-11737.md), [Use cutlass fp4 gemm by default](../sources/prs/sglang/PR-11813.md), [Support nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8/NVFP4](../sources/prs/sglang/PR-11866.md), [Feature/nano v2 offline modelopt fp8 and nvfp4](../sources/prs/sglang/PR-12018.md), [Replace [silu_and_mul_]scaled_fp4_group_quant by Flashinfer equivalent](../sources/prs/sglang/PR-12376.md), [Use sgl fp4 quant kernel by default](../sources/prs/sglang/PR-12482.md), [[NVIDIA] Fix CUDA arch requirement in nvfp4 cast](../sources/prs/sglang/PR-12581.md), [feat: Add FP4 (E2M1) KV Cache Support for MHA](../sources/prs/sglang/PR-12612.md), [[NVIDIA] Fix wrong symmetric sizes for fp4 cases](../sources/prs/sglang/PR-12640.md), [[Bugfix] Fix illegal memory access](../sources/prs/sglang/PR-12758.md), [ignore the deepgemm check when the model weight with nvfp4 and moe ba…](../sources/prs/sglang/PR-12782.md), [support mtp with deepseek r1 nvfp4 model](../sources/prs/sglang/PR-13115.md), [Fix nan in global scaling factor for large scale nvfp4 EP](../sources/prs/sglang/PR-13162.md), [Fix EPLB + FP4 Quantization Compatibility Issue](../sources/prs/sglang/PR-13715.md), [[Feat][NVFP4] Enable NVFP4 MoE for Qwen series models (eg. Qwen3-Next) #13761](../sources/prs/sglang/PR-13761.md), [Support fp4 fp8 non gated moe](../sources/prs/sglang/PR-13794.md), [Fix flashinfer cutlass MoE output shape for non-FP4-packed inputs](../sources/prs/sglang/PR-14028.md), [[FIX] trtllm-moe-fp4-renorm for Qwen series models](../sources/prs/sglang/PR-14350.md), [[CPU] Implement MXFP4 Gemm kernels for intel AMX to support GPT OSS series.](../sources/prs/sglang/PR-14385.md), [Mistral Large 3 NVFP4 support](../sources/prs/sglang/PR-14485.md), [[AMD] Support fused_rms_mxfp4_quant in the prefill stage for DeepSeek-R1-MXFP4](../sources/prs/sglang/PR-14975.md), [Mistral Large 3 NVFP4 TRTLLM MoE support](../sources/prs/sglang/PR-15049.md), [[NVIDIA] Fixes for NVFP4 all-gather with spec decoding](../sources/prs/sglang/PR-15280.md), [Fix the accuracy issue when running mxfp4 dsv3 model and enable ep](../sources/prs/sglang/PR-15304.md), [Update flashinfer to 0.6.1](../sources/prs/sglang/PR-15551.md), [[Perf] Eliminate the slice op for Flashinfer `trtllm_fp4_block_scale_moe`](../sources/prs/sglang/PR-15731.md), [Tiny fix cannot launch nvfp4 checkpoint with bf16 kv cache](../sources/prs/sglang/PR-15986.md), [[Performance] Force split_k=1 for MXFP4 Triton kernels on Hopper](../sources/prs/sglang/PR-16014.md), [Inclusion of nvfp4 blockscale in EPLB Rebalance](../sources/prs/sglang/PR-17158.md), [[Fix] GLM 4.7 + NVFP4 + MTP](../sources/prs/sglang/PR-17166.md), [[FIX] Always support TP > 4 for FP4 Gemm](../sources/prs/sglang/PR-17300.md), [[feat] Support nvfp4 quantized model of Qwen3-Next](../sources/prs/sglang/PR-17627.md), [fix(quantization): add sgl_kernel fallback for FP4 quantize on Blackwell GPUs](../sources/prs/sglang/PR-17816.md), [[Bugfix] Fix Mistral Large 3 NVFP4 TRTLLM MoE](../sources/prs/sglang/PR-18065.md), [Fix nvfp4 weight update](../sources/prs/sglang/PR-18085.md), [[ModelOpt] Fix broken Qwen3-235B-A22B-Instruct-2507-NVFP4 launch](../sources/prs/sglang/PR-18189.md), [[ModelOPT] Support Qwen 3 Next Coder NVFP4](../sources/prs/sglang/PR-18224.md), [[Kimi-K2.5] Fix NVFP4 Kimi-K2.5 weight mapping and exclude list](../sources/prs/sglang/PR-18370.md), [Nsa trtllm mla sparse fp8 support with Deepseek v3.2 NVFP4](../sources/prs/sglang/PR-18389.md), [[Perf] ~9.5x faster Blackwell MXFP4 MoE weight loading](../sources/prs/sglang/PR-18858.md), [[Qwen3.5] Enable nvfp4 checkpoint](../sources/prs/sglang/PR-18937.md), [feat: Support MXFP4 quantized dense models on AMD CDNA2/CDNA3 GPUs](../sources/prs/sglang/PR-19143.md), [Adjust padding size to improve triton_kernels moe performance](../sources/prs/sglang/PR-19174.md), [Fix nightly Mistral-Large-3 NVFP4 accuracy threshold](../sources/prs/sglang/PR-19402.md), [[AMD] Fix weight load shape mismatch for amd dsr1 0528 mxfp4](../sources/prs/sglang/PR-19425.md), [[Kernel Slimming] Migrate NVFP4 kernels to JIT](../sources/prs/sglang/PR-19437.md), [[Feature] NVFP4 Marlin fallback for non-Blackwell GPUs (SM75+)](../sources/prs/sglang/PR-19652.md), [Support `triton_kernels` for GPT-OSS on SM120](../sources/prs/sglang/PR-19718.md), [[AMD] Fix FP8 assertion failure in aiter MLA decode by falling back to self.k_scale](../sources/prs/sglang/PR-19935.md), [[JIT Kernel] Reland NVFP4 kernels to JIT](../sources/prs/sglang/PR-20012.md), [Fix SM120 `triton_kernels` MXFP4 `block_k` for GPT-OSS](../sources/prs/sglang/PR-20040.md), [MiniMax-M2.5 - Support dp attention, dp reduce scatter, FP4 all gather, AR fusion in prepare_attn](../sources/prs/sglang/PR-20067.md), [[V32/GLM5] Change default setting of V32 nvfp4 on TP4](../sources/prs/sglang/PR-20086.md), [[diffusion] Support nvfp4 for Flux.2](../sources/prs/sglang/PR-20137.md), [[4/n jit_kernel restruct] speed up CI tests and add benchmark workflow](../sources/prs/sglang/PR-20268.md), [[Benchmark] use flashinfer bench_gpu_time instead of triton do_bench](../sources/prs/sglang/PR-20305.md), [[Model] Support Nemotron 3 Super NVFP4](../sources/prs/sglang/PR-20407.md), [[JIT Kernel] Fix NVFP4 multi-arch compilation failure](../sources/prs/sglang/PR-20874.md), [Add SGLang CUDA crash API logging inspired by FlashInfer](../sources/prs/sglang/PR-20910.md), [[Chore] Clean up JIT compilation flags](../sources/prs/sglang/PR-21022.md), [[AMD]: Support MLA with nhead<16 and FP8 KV cache for TP=8 (Kimi K2.5…](../sources/prs/sglang/PR-21213.md), [Refactor JIT kernel CI to use run_suite.py registration system](../sources/prs/sglang/PR-21239.md), [[NVIDIA] Enable FP4 flashinfer trtllm routed moe](../sources/prs/sglang/PR-21240.md), [CUTLASS NVFP4 GEMM improvement of SM120](../sources/prs/sglang/PR-21314.md), [[Kernel] Support FlashInfer TRTLLM-Gen fused MoE for non-gated FP4 & FP8 (Nemotron)](../sources/prs/sglang/PR-21321.md), [[misc] clean up kernel API](../sources/prs/sglang/PR-21325.md), [Add dedicated FlashInferCuteDslMoE layer for standard-path FP4 MoE](../sources/prs/sglang/PR-21339.md), [Migrate all callers from /get_server_info to /server_info](../sources/prs/sglang/PR-21463.md), [Harden FlashInfer FP4 imports in standard dispatcher](../sources/prs/sglang/PR-21776.md), [[Diffusion] Fix weight scale swizzle and add large-M kernel config for FLUX.2-dev-NVFP4](../sources/prs/sglang/PR-22064.md), [[nvidia] Gemma4 nvfp4 fix](../sources/prs/sglang/PR-22079.md), [[diffusion] Default NVFP4 to CUTLASS and add all-model shape benchmarks](../sources/prs/sglang/PR-22091.md), [[Diffusion] Add diffusion NVFP4 scaled-mm correctness test](../sources/prs/sglang/PR-22127.md), [[RL] Refactor NVFP4 shuffling/swizzling to in-place replacement](../sources/prs/sglang/PR-22204.md), [:sparkles: [diffusion][npu][quant] Add MXFP4 quantization support for Wan2.2 Diffusion on Ascend NPU](../sources/prs/sglang/PR-22338.md), [GLM-5/5.1 MXFP4 Checkpoint Inference Compatibility Fix](../sources/prs/sglang/PR-22543.md), [[Diffusion] Add FLUX.1-dev ModelOpt NVFP4 support](../sources/prs/sglang/PR-22574.md), [reland [Diffusion] Add FLUX.1-dev ModelOpt NVFP4 support](../sources/prs/sglang/PR-22672.md), [[Diffusion] Add Wan2.2 ModelOpt NVFP4 support](../sources/prs/sglang/PR-22681.md), [[FlashInfer v0.6.11] [RL] Support FlashInfer per-token NVFP4 MoE](../sources/prs/sglang/PR-22918.md), [Reland Cute-DSL FP4 dense GEMM](../sources/prs/sglang/PR-23590.md), [Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-23686.md), [Use Cute-DSL NVFP4 quantization kernels](../sources/prs/sglang/PR-23745.md), [[MoE] Unify DeepEPMoE+MoriEPMoE through AITER MoeRunner pre/post-permute](../sources/prs/sglang/PR-23760.md), [Port MXFP4 Marlin MoE support to JIT kernel path](../sources/prs/sglang/PR-24490.md), [Add FlashInfer SM90 cutlass MXFP4 MoE backend (W4A16) for GPT-OSS + DeepSeek-V4](../sources/prs/sglang/PR-24816.md), [[rebase]Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-24986.md), [perf(nvfp4): free unused source scales after weight processing](../sources/prs/sglang/PR-25107.md), [[Intel GPU] Enable DeepSeek V4 Inference on XPU](../sources/prs/sglang/PR-25336.md), [[AMD] Enable shared-experts fusion with new KIMI-K2.5-MXFP4 model.](../sources/prs/sglang/PR-25390.md), [[codex] Update Wan2.2 ModelOpt CI checkpoints](../sources/prs/sglang/PR-25483.md), [Add no_combine support to cutlass_moe_fp4](../sources/prs/sglang/PR-25688.md), [[Test] Stage-a sanity kits; consolidate core/ + models_e2e/ tests](../sources/prs/sglang/PR-25831.md), [Support FP4 gemm (1/2)](../sources/prs/sglang/PR-3899.md), [[Build] Fix cuda12.8 build error in nvfp4_scaled_mm_kernels.cu](../sources/prs/sglang/PR-4953.md), [[1/2] Add Kernel support for Cutlass based Fused FP4 MoE](../sources/prs/sglang/PR-6093.md), [[DeepseekR1-FP4] Add Support for nvidia/DeepSeekR1-FP4 model](../sources/prs/sglang/PR-6853.md), [Fix Deepseek R1 0528 FP4 tensor name mismatch issue during weights loading.](../sources/prs/sglang/PR-7164.md), [Support NVFP4 quantized dense models on AMD CDNA2/CDNA3 GPUs](../sources/prs/sglang/PR-7302.md), [FlashInfer NVFP4 MoE with EP & 2-stream shared expert](../sources/prs/sglang/PR-7327.md), [Fix MTP with Deepseek R1 Fp4](../sources/prs/sglang/PR-7376.md), [Add fp4 quantize before all-gather for Flashinfer cutlass MoE DP (max throughput)](../sources/prs/sglang/PR-7667.md), [Qwen FP8/NVFP4 ModelOPT Quantization support](../sources/prs/sglang/PR-7912.md), [[Fix][Ready]Fix register spilling in cutlass nvfp4 gemm kernel on Blackwell](../sources/prs/sglang/PR-8127.md), [[fix] fix modelopt fp4 on b200](../sources/prs/sglang/PR-8195.md), [[NVIDIA] Add Low Latency NVFP4 decode kernels from Flashinfer](../sources/prs/sglang/PR-8552.md), [[Perf] Auto enable best flashinfer mxfp4 kernel in b200](../sources/prs/sglang/PR-8898.md), [Fix hopper launch gpt-oss model illegal memory](../sources/prs/sglang/PR-8908.md), [Faster weight processing (trtllm-gen moe nvfp4)](../sources/prs/sglang/PR-9162.md), [[NVIDIA] [3/N] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked ](../sources/prs/sglang/PR-9199.md), [[NVIDA] [1/N] Nvfp4 Masked Gemm: Add quant op for the flashinfer grouped gemm](../sources/prs/sglang/PR-9200.md), [Fix FP4 inference corruption issue in glm4.5-air model](../sources/prs/sglang/PR-9346.md), [[fix] Fix mxfp4 triton MoE tp bug](../sources/prs/sglang/PR-9473.md), [[NVIDIA] [2/N] Optimize `silu_and_mul_scaled_fp4_grouped_quant` perf](../sources/prs/sglang/PR-9556.md), [move is_sm90_supported/is_sm100_supported to python/sglang/srt/utils.py](../sources/prs/sglang/PR-9679.md), [[ModelOpt] Fix Weight Loading for DSR1-FP4 Quantization](../sources/prs/sglang/PR-9712.md), [Make fp4_quantize kernels work on sm103](../sources/prs/sglang/PR-9807.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [[TMA] Support FP4 TensorMap TMA copies](../sources/prs/tilelang/PR-2107.md), [[TIR][IR] Update to use tirx](../sources/prs/tilelang/PR-2216.md), [[NVIDIA] Support nvfp4 quantization](../sources/prs/vllm/PR-12784.md), [[NVIDIA] Support nvfp4 tcgen05 gemm](../sources/prs/vllm/PR-13571.md), [[Hardware/NVIDIA/Kernel] [Functional Enablement] [1/N] Enable nvidia/DeepSeek-R1-FP4 Model](../sources/prs/vllm/PR-16362.md), [[Kernel] fp4 marlin kernel](../sources/prs/vllm/PR-17687.md), [[Misc] Add compressed-tensors NVFP4A16 emulation support](../sources/prs/vllm/PR-17914.md), [Use NVFP4 Marlin for CompressedTensorsW4A16Fp4](../sources/prs/vllm/PR-18000.md), [[Quantization] Add compressed-tensors NVFP4 support](../sources/prs/vllm/PR-18312.md), [[V1] Support `LLM.apply_model`](../sources/prs/vllm/PR-18465.md), [[Hardware][NVIDIA] FP4 MoE kernel optimization](../sources/prs/vllm/PR-19110.md), [[Hardware][NVIDIA][kernel] Fp4 MOE quant kernel optimization](../sources/prs/vllm/PR-19500.md), [[Quantization] Add compressed-tensors emulations support for NVFP4](../sources/prs/vllm/PR-19879.md), [[Quantization] Add compressed-tensors NVFP4 MoE Support](../sources/prs/vllm/PR-19990.md), [Add ModelOpt Qwen3 nvfp4 support](../sources/prs/vllm/PR-20101.md), [[Bugfix] Fix some narrowing conversion warnings](../sources/prs/vllm/PR-20141.md), [[Kernel][Bugfix] Fixup some warnings in nvfp4_blockwise_moe when CUDA < 12.8](../sources/prs/vllm/PR-20324.md), [Support Llama 4 for cutlass_moe_fp4](../sources/prs/vllm/PR-20453.md), [[Perf] Reuse workspace for FP8+FP4 Marlin MoE](../sources/prs/vllm/PR-20500.md), [Support mnnvl all2allv from Flashinfer](../sources/prs/vllm/PR-21003.md), [[Feature][OCP MX] Support mxfp6 and mixed mxfp6-mxfp4](../sources/prs/vllm/PR-21166.md), [Support CUTLASS NVFP4 (w4a4) for Blackwell Geforce GPUs (SM120)](../sources/prs/vllm/PR-21309.md), [Support Tensorrt-LLM MoE fp4 for low-latency](../sources/prs/vllm/PR-21331.md), [Update flashinfer CUTLASS NVFP4 MoE Kernel to use per expert global scaling factor](../sources/prs/vllm/PR-21408.md), [[Bug] Fix Compressed Tensor NVFP4 `cutlass_fp4_group_mm` illegal memory access](../sources/prs/vllm/PR-21465.md), [[NVIDIA] Fix Llama4 Scout FP4 functionality issues](../sources/prs/vllm/PR-21499.md), [[Feature] Add Flashinfer MoE Support for Compressed Tensor NVFP4](../sources/prs/vllm/PR-21639.md), [[gpt-oss] flashinfer mxfp4](../sources/prs/vllm/PR-22339.md), [[gpt-oss] triton kernel mxfp4](../sources/prs/vllm/PR-22421.md), [Fix Llama4 FlashInfer FP4 MoE issues](../sources/prs/vllm/PR-22511.md), [Quantization: support FP4 quantized models on AMD CDNA2/CDNA3 GPUs](../sources/prs/vllm/PR-22527.md), [Fix torch version check for SM100 mxfp4 ](../sources/prs/vllm/PR-22535.md), [[Quantization] Expand compressed-tensors MoE matching logic to support NFP4 + FP8 MoEs](../sources/prs/vllm/PR-22674.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv NVFP4-out Attention Kernel](../sources/prs/vllm/PR-22703.md), [Use Blackwell FlashInfer MXFP4 MoE by default if available ](../sources/prs/vllm/PR-23008.md), [Add routed_scaling_factor to MoE grouped topk](../sources/prs/vllm/PR-23123.md), [Fix nvfp4 swizzling](../sources/prs/vllm/PR-23140.md), [[Kernels] Overlap shared experts with send/recv](../sources/prs/vllm/PR-23273.md), [fix incompatibililty with non cuda platform for nvfp4](../sources/prs/vllm/PR-23478.md), [Update Flashinfer to 0.2.14.post1](../sources/prs/vllm/PR-23537.md), [DP/EP Support for gpt-oss with deepep-ht comm kernel on SM100](../sources/prs/vllm/PR-23608.md), [[Bugfix] Fix Marlin NVFP4 for modelopt](../sources/prs/vllm/PR-23659.md), [[NVIDIA] Support SiluMul + NVFP4 quant fusion](../sources/prs/vllm/PR-23671.md), [[Kernel][B200] mxfp4 fused cutlass moe](../sources/prs/vllm/PR-23696.md), [[Bugfix][Misc] Fix silu_and_mul_nvfp4_quant issue and extract common utils for nvfp4 kernel source files](../sources/prs/vllm/PR-23727.md), [[Model][gpt-oss] Support DP+EP for GPT-OSS with FlashInfer trtllm-gen MoE](../sources/prs/vllm/PR-23819.md), [[BUGFIX ] fix undefined silu_and_mul_nvfp4_quant](../sources/prs/vllm/PR-23929.md), [[Model] Add LongCat-Flash ](../sources/prs/vllm/PR-23991.md), [[Transform] [Quantization] Add QuTLASS support to vLLM](../sources/prs/vllm/PR-24440.md), [[Kernel][Quantization] add w4a8 support for marlin kernel](../sources/prs/vllm/PR-24722.md), [[Bugfix] Fix accuracy issue for silu_mul + nvfp4 quant fusion kernel](../sources/prs/vllm/PR-24833.md), [[Compile] Fix Compile Warning for Ignoring `MIN_BLOCK_PER_SM`](../sources/prs/vllm/PR-25193.md), [[ROCm] Small functional changes for gptoss](../sources/prs/vllm/PR-25201.md), [Enable Fbgemm NVFP4 on Dense models](../sources/prs/vllm/PR-25609.md), [[Bugfix] Enable padded FP4 quantization](../sources/prs/vllm/PR-25947.md), [[Quantization/NVFP4] Speed up TRTLLM NVFP4 MOE weight loading and fix K/V scale loading for MLA Attn](../sources/prs/vllm/PR-25968.md), [[Bugfix] Allow skipping MoE in NVFP4 (fix for MTP)](../sources/prs/vllm/PR-25987.md), [[MoE] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked](../sources/prs/vllm/PR-25990.md), [[NVIDIA] Add support for cudnn fp4 gemm via flashinfer](../sources/prs/vllm/PR-26107.md), [[ModelOpt] Load w13/w2_input_scale for all experts, nvfp4](../sources/prs/vllm/PR-26135.md), [[ROCM] MoE fp4 CK kernel](../sources/prs/vllm/PR-26545.md), [support flashinfer_fp4 moe for 5090 gpu](../sources/prs/vllm/PR-26669.md), [[NVIDIA] [Perf] Update to leverage flashinfer trtllm FP4 MOE throughput kernel](../sources/prs/vllm/PR-26714.md), [[Bugfix] Fix gpt-oss w4a8 DP/EP on B200](../sources/prs/vllm/PR-26729.md), [Flashinfer_CUTLASS_MOE fuses quantization for TP](../sources/prs/vllm/PR-27223.md), [[Attention] Use sparse prefill kernel for fp8 kv-cache in DeepSeek-v3.2](../sources/prs/vllm/PR-27532.md), [[Bugfix] Fix GPT-OSS on AMD after #28603](../sources/prs/vllm/PR-28816.md), [Add TRTLLM MoE NVFP4 kernel to CompressedTensorsW4A4MoeMethod](../sources/prs/vllm/PR-28892.md), [[Feat] Support non-gated activations in NVFP4 modelopt path](../sources/prs/vllm/PR-29004.md), [[Kernel] Add NVFP4 MoE CUTLASS support for SM120](../sources/prs/vllm/PR-29242.md), [[Bugfix] Only use triton_kernels for MXFP4 on SM90 and SM100](../sources/prs/vllm/PR-29339.md), [[Bugfix] Fix mismatched nvfp4 gemm output shape](../sources/prs/vllm/PR-29742.md), [[ROCm][MXFP4] Infer w4a4 quant method in rocm aiter fused moe](../sources/prs/vllm/PR-29775.md), [[EPLB] Support EPLB w/ NVFP4](../sources/prs/vllm/PR-29804.md), [[Perf] Do FP4 quant before All gather on flashinfer trtllmgen MOE ](../sources/prs/vllm/PR-30014.md), [[ROCm][Quantization] GPT OSS Upstream MoE wmxfp4_afp8 with static scales](../sources/prs/vllm/PR-30357.md), [[Feature] Add SM103 (Blackwell Ultra) Support to vLLM](../sources/prs/vllm/PR-30484.md), [[Perf] Set split_k to 1 for triton_kernels](../sources/prs/vllm/PR-30528.md), [[Perf] Eliminate padding and slicing op for GPT-OSS with Flashinfer MXFP4 MXFP8 MoE](../sources/prs/vllm/PR-30647.md), [[SM100] Enable fp8 compute for prefill MLA](../sources/prs/vllm/PR-30746.md), [[Compressed-Tensors] Simplify NVFP4 Conditions, enable marlin support for NVFP4A16 MoEs](../sources/prs/vllm/PR-30881.md), [[Kernel][Performance] Enable smaller Scaling Factor tiling for NVFP4 small-batch decoding](../sources/prs/vllm/PR-30885.md), [[NVFP4][Perf] Tune NVFP4 input quant kernel for small batch size](../sources/prs/vllm/PR-30897.md), [ [FIX] Always support TP > 4 for FP4 Gemm](../sources/prs/vllm/PR-31099.md), [[Bugfix] Fix Broken ModelOpt NVFP4 MoE](../sources/prs/vllm/PR-31742.md), [[Perf] Fuse stride preparation for NVFP4 cutlass_moe](../sources/prs/vllm/PR-31837.md), [[5/N][Attention] Finish eliminating `vllm/attention` folder](../sources/prs/vllm/PR-32064.md), [[Perf][Kernel] Optimize FP4 quantization kernels (SM100F)](../sources/prs/vllm/PR-32520.md), [Support compress-tensors with nvfp4 or fp8 weights and modelopt with nvfp4 weights on Turing](../sources/prs/vllm/PR-33076.md), [fix: Add SM120 (RTX Blackwell) support for FlashInfer CUTLASS NVFP4 MoE kernels](../sources/prs/vllm/PR-33417.md), [[Kernel] Support Flashinfer trtllm fused MoE non gated FP8 & NVFP4](../sources/prs/vllm/PR-33506.md), [[Bugfix] Fix DSV3.2 NVFP4](../sources/prs/vllm/PR-33932.md), [[Bugfix]fix output Nan/Inf in marlin if dtype=float16](../sources/prs/vllm/PR-33972.md), [[ModelBash][DSR1 NVFp4] Avoid Bf16 Bias Cast](../sources/prs/vllm/PR-34298.md), [[Custom Ops] Add functional + out variant for scaled_fp4_quant](../sources/prs/vllm/PR-34389.md), [[BUGFIX] Fix accuracy regression for NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4 with TP>1](../sources/prs/vllm/PR-34476.md), [[Model] Add NVFP4 quantization support for Step3.5-Flash](../sources/prs/vllm/PR-34478.md), [[Bugfix] Rescale NVFP4 weight scales to fix BF16 dequant underflow](../sources/prs/vllm/PR-34577.md), [[torch.compile] Turn on silu+fp4 quant fusion by default for O1+](../sources/prs/vllm/PR-34718.md), [[Bugfix] Fix NVFP4 TRTLLM MoE non-gated support; add gsm8k for Nemotron-3-Nano FP8+NVFP4](../sources/prs/vllm/PR-34725.md), [[BugFix] Fix fp4 quant kernel on CUDA 12.8](../sources/prs/vllm/PR-35210.md), [fix(mxfp4): return is_monolithic=False when LoRA is enabled for Triton backend](../sources/prs/vllm/PR-35382.md), [[NVFP4] Support NVFP4 dense models from `modelopt` and `compressed-tensors` on AMD Instinct MI300, MI355X and Hopper through emulation](../sources/prs/vllm/PR-35733.md), [[Bugfix] Fix passing of activation_type to trtllm fused MoE NVFP4 and FP8](../sources/prs/vllm/PR-36017.md), [[Mamba] Flashinfer selective_state_update](../sources/prs/vllm/PR-36162.md), [[mla] Support fused FP8/NVFP4 output quantization in MLA attention (#35792)](../sources/prs/vllm/PR-36205.md), [[Bug][MoE] Fix TRTLLM NVFP4 Routing Kernel Precision](../sources/prs/vllm/PR-36725.md), [[Bug][MoE] Strengthen _supports_current_device() checks in the TRTLLM FP8, NVFP4, and FlashInfer CuteDSL MoE experts](../sources/prs/vllm/PR-36728.md), [[MoE Refactor] Mxfp4 oracle rebased](../sources/prs/vllm/PR-37128.md), [Fix minimax m2.5 nvfp4 kv scales weight loading](../sources/prs/vllm/PR-37214.md), [[MoE/EPLB] Fix FlashInfer nvfp4 experts + EPLB correctness](../sources/prs/vllm/PR-37217.md), [[Kernel] Add non-gated support for NVFP4 CUTLASS MoE](../sources/prs/vllm/PR-37320.md), [[Bugfix] Fix EP weight filter breaking EPLB and NVFP4 accuracy](../sources/prs/vllm/PR-37322.md), [Add nvfp4 support to reshape_and_cache_flash](../sources/prs/vllm/PR-37332.md), [[Kernel] Add MXFP4 W4A4 CUTLASS MoE kernel for SM100](../sources/prs/vllm/PR-37463.md), [[Bugfix] Remove assertion for NVFP4 scale dynamic range](../sources/prs/vllm/PR-37465.md), [[Bugfix] Fix marlin nvfp4 rescaling](../sources/prs/vllm/PR-37502.md), [[4/n] Migrate FP4/W4A8 CUTLASS kernels to torch stable ABI](../sources/prs/vllm/PR-37503.md), [[Perf] Use torch compile to fuse pack topk in trtllm moe](../sources/prs/vllm/PR-37695.md), [[Bugfix] Preserve CUDA arch suffix (a/f) for SM12x — fixes NVFP4 NaN on desktop Blackwell](../sources/prs/vllm/PR-37725.md), [[MoE] Move FlashInfer CuteDSL experts into fused_moe/experts/](../sources/prs/vllm/PR-37759.md), [[MoE Kernel] Flashinfer nvfp4 cutedsl moe kernel integration](../sources/prs/vllm/PR-38050.md), [[Bugfix] Fix DeepGemm E8M0 accuracy degradation for Qwen3.5 FP8 on Blackwell](../sources/prs/vllm/PR-38083.md), [Fix NaN from stale FP4 scale padding in create_fp4_scale_tensor](../sources/prs/vllm/PR-38148.md), [[Quantization] Add FlashInfer CuteDSL batched experts backend for NVFP4 MoE](../sources/prs/vllm/PR-38251.md), [[MoE] Add RoutingMethodType.Simulated to TRT-LLM FP8/NVFP4 kernel allowlists](../sources/prs/vllm/PR-38329.md), [[NVIDIA] Bugfix NVFP4 DGX Spark and RTX50](../sources/prs/vllm/PR-38423.md), [[Kernels][MoE] Fix legacy_routing to use bitmatrix-based routing path](../sources/prs/vllm/PR-38504.md), [[Compile] Fix nvfp4 compile warning](../sources/prs/vllm/PR-38573.md), [[Bugfix] Fix NVFP4+MTP crash: force unquantized mtp.fc for Qwen3.5](../sources/prs/vllm/PR-38832.md), [[MoE Refactor] Split up compressed_tensors_moe.py](../sources/prs/vllm/PR-38960.md), [[MoE] Move GPT OSS Triton kernel experts into fused_moe/experts/](../sources/prs/vllm/PR-39007.md), [[Refactor] Move NVFP4 GEMM management into NvFp4LinearKernel](../sources/prs/vllm/PR-39129.md), [[Feature] Batch invariant nvfp4 linear support](../sources/prs/vllm/PR-39322.md), [[Kernel] Support TRTLLM GEN NVFP4 MoE for non-512-aligned hidden dims via weight padding](../sources/prs/vllm/PR-39510.md), [[Bugfix] Reject non-nvfp4 dtypes when using the flashinfer_nvlink_one_sided all2all backend](../sources/prs/vllm/PR-39717.md), [[Bug] Fix batch invariance nvfp4 support](../sources/prs/vllm/PR-39820.md), [[Bugfix] Temporarily disable B200 fp4 MoE layer tests](../sources/prs/vllm/PR-40057.md), [Add nvfp4 kv cache support](../sources/prs/vllm/PR-40177.md), [[Bugfix] Guard mxfp4_experts_quant bindings on ENABLE_NVFP4_SM100](../sources/prs/vllm/PR-40191.md), [[MoE] Move cutlass moe to fused_moe/experts/](../sources/prs/vllm/PR-40574.md), [[DSV4] Add BF16 and MXFP8 A2A support for flashinfer a2a one sided](../sources/prs/vllm/PR-40960.md), [[Kernel][MoE] Support GELU on TRT-LLM NvFP4 fused MoE for Gemma4](../sources/prs/vllm/PR-41050.md), [[ROCm][Quantization][3/N] Refactor quark_moe w4a4 w/ oracle](../sources/prs/vllm/PR-41436.md), [[Quantization] Rework quantization_config to use QuantKey and allow for activation override](../sources/prs/vllm/PR-41566.md), [[MXFP4] Support for linear layers + compressed-tensors integration](../sources/prs/vllm/PR-41664.md), [Add NVFP4 all-gather GEMM fusion for AsyncTP](../sources/prs/vllm/PR-41882.md), [[CPU] Add MXFP4 W4A16 MoE support](../sources/prs/vllm/PR-41922.md), [[MoE] Move various experts classes to fused_moe/experts/](../sources/prs/vllm/PR-41979.md), [[Bugfix] Add swiglu limits to deepgemm fp8 methods](../sources/prs/vllm/PR-41986.md), [[Kernel] Pack topk id/weights triton kernel](../sources/prs/vllm/PR-42527.md), [[Bugfix] fix swiglu limit issue for humming backend + deepseek v4](../sources/prs/vllm/PR-42541.md), [[Perf] Padded nvfp4 quant kernel to remove additional copy, 2.4%~5.7% e2e performance improvement](../sources/prs/vllm/PR-42774.md), [[Perf] Re-enable flashinfer autotune by default and cleanup](../sources/prs/vllm/PR-42857.md), [NVFP4 and Block-Scaled Narrow Precision](../wiki/hardware/nvfp4.md), [NVFP4 GEMM — GPU Mode Problem 2 Contract](../wiki/kernels/nvfp4-gemm.md), [NVFP4 Batched GEMV](../wiki/kernels/nvfp4-gemv.md), [TensorRT-LLM Blackwell FP4 DSA Indexer](../wiki/kernels/tensorrt-llm-blackwell-indexer.md), [Fine-Grained FP8/FP4 Quantization](../wiki/techniques/fine-grained-quantization.md) | +| `fp6` | [Microbenchmarking NVIDIA's Blackwell Architecture](../sources/blogs/blackwell-microbenchmarking.md), [Tilus: A Tile-Level GPGPU Programming Language for Low-Precision Computation](../sources/blogs/tilus-nvidia.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [NVIDIA CUDA Toolkit 13.x for Blackwell](../sources/docs/nvidia-cuda-13.md), [PTX ISA 9.0 SM100 Instruction Reference](../sources/docs/nvidia-ptx-isa-sm100.md) | +| `fp8` | [Microbenchmarking NVIDIA's Blackwell Architecture](../sources/blogs/blackwell-microbenchmarking.md), [DeepGEMM — Pinned Upstream Project Summary](../sources/blogs/deepgemm.md), [FlashMLA — Multi-head Latent Attention](../sources/blogs/flashmla.md), [Tilus: A Tile-Level GPGPU Programming Language for Low-Precision Computation](../sources/blogs/tilus-nvidia.md), [DeepSeek-V3.2-Exp in vLLM: Fine-Grained Sparse Attention in Action](../sources/blogs/vllm-deepseek-v3-sparse-attention.md), [FlashInfer MLSys 2026 Track A: FP8 Block-Scale MoE](../sources/contests/flashinfer-mlsys26/track-a-fused-moe.md), [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [DeepSeek-V3 Technical Report: FP8 Training](../sources/docs/deepseek-v3-fp8.md), [NVIDIA Blackwell Tuning Guide](../sources/docs/nvidia-blackwell-tuning-guide.md), [NVIDIA CUDA Toolkit 13.x for Blackwell](../sources/docs/nvidia-cuda-13.md), [PTX ISA 9.0 SM100 Instruction Reference](../sources/docs/nvidia-ptx-isa-sm100.md), [[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes](../sources/prs/DeepGEMM/PR-304.md), [Sync nv_dev with upstream #316 (Mega MoE optimizations & benchmarks)](../sources/prs/DeepGEMM/PR-328.md), [ Solving bank conflict via padding and TMA 3D store](../sources/prs/DeepGEMM/PR-78.md), [Use 1D TMA store instead of 3D](../sources/prs/DeepGEMM/PR-83.md), [Use swizzling instead of padding](../sources/prs/DeepGEMM/PR-86.md), [Support TMA multicast on B with m_grouped_gemm_contiguous.](../sources/prs/DeepGEMM/PR-88.md), [[TRTLLM-9457][feat] Add cute dsl fp8 gemm for Blackwell](../sources/prs/TensorRT-LLM/PR-10130.md), [[None][fix] impl fused triton kernel for e8m0 resmooth to reduce memory footprint](../sources/prs/TensorRT-LLM/PR-10327.md), [[None][feat] fuse shared to sparse experts in TRT-LLM Gen MoE](../sources/prs/TensorRT-LLM/PR-11143.md), [[None][feat] Add support for expert_number<=2048 and K<=32](../sources/prs/TensorRT-LLM/PR-11510.md), [[TRTLLM-10004][feat] Enable GEMM -> AR with GEMM output in registered buffers](../sources/prs/TensorRT-LLM/PR-11589.md), [[TRTLLM-10421][perf] Add fused cat+fp8_quantize CUDA kernel for DSA indexer](../sources/prs/TensorRT-LLM/PR-11899.md), [[None][feat] GLM 5 support and DSA MTP fixes](../sources/prs/TensorRT-LLM/PR-11990.md), [[https://nvbugs/5879577][fix] Fix KeyError in DeepSeekV3Lite FP8 MTP weight loading](../sources/prs/TensorRT-LLM/PR-12530.md), [[#12784][feat] AutoDeploy: Optimize DeepSeek-R1 model performance](../sources/prs/TensorRT-LLM/PR-12946.md), [[TRTLLM-34871][feat] Add cute dsl FP8 paged MQA logits decode kernel](../sources/prs/TensorRT-LLM/PR-13219.md), [[TRTLLM-11127][feat] add W4A8_MXFP4_FP8 MoE unit test support](../sources/prs/TensorRT-LLM/PR-13401.md), [[None][feat] Fuse FP8 1x128 quantize + UE8M0 scale pack on SM100](../sources/prs/TensorRT-LLM/PR-13628.md), [[None][fix] Plumb swiglu_limit through DeepGEMM and TRTLLMGen FP8 fused MoE](../sources/prs/TensorRT-LLM/PR-13767.md), [[TRTLLM-12503][feat] Parallel VAE independent scaling and fix arg passing](../sources/prs/TensorRT-LLM/PR-13873.md), [[TRTLLM-35237][feat] Add cute dsl FP4 paged MQA logits decode kernel](../sources/prs/TensorRT-LLM/PR-13929.md), [[None][feat] Keep DSv4 o_a_proj as FP8, and port vLLM's fused_inv_rope_fp8_quant](../sources/prs/TensorRT-LLM/PR-13938.md), [[TRTLLM-12462][fix] Fix FP8 block scaling GEMM autotuner cache growth](../sources/prs/TensorRT-LLM/PR-14165.md), [[None][fix] Add SPDX Apache-2.0 headers to auto_deploy test files](../sources/prs/TensorRT-LLM/PR-14193.md), [feat: Add w4a8_mxfp4_fp8 quantization recipe.](../sources/prs/TensorRT-LLM/PR-4867.md), [[OMNIML-2336][feat] Add NVFP4 x FP8](../sources/prs/TensorRT-LLM/PR-6809.md), [[TRTLLM-8535][feat] Support DeepSeek V3.2 with FP8 + BF16 KV cache/NVFP4 + BF16 KV cache](../sources/prs/TensorRT-LLM/PR-8405.md), [[None][fix] Fix the performance issue of FP8 blockwise grouped GEMM when using attention DP](../sources/prs/TensorRT-LLM/PR-8501.md), [[None][feat] TRT-LLM Gen MoE optimize DeepSeek Fp8 activation kernel](../sources/prs/TensorRT-LLM/PR-9175.md), [[https://nvbugs/5726962][feat] Apply fusion for W4AFP8_AWQ MoE](../sources/prs/TensorRT-LLM/PR-9838.md), [Improve sm90 mixed dtype kernel](../sources/prs/cutlass/PR-1883.md), [Groupwise scaling along M for FP8 gemm](../sources/prs/cutlass/PR-2037.md), [Improvements for: Groupwise scaling along M for FP8 gemm](../sources/prs/cutlass/PR-2095.md), [Hopper Grouped GEMM support for FP8 Accum](../sources/prs/cutlass/PR-2123.md), [Blockwise and Groupwise GEMM for Blackwell and Improvements for Hopper](../sources/prs/cutlass/PR-2139.md), [hopper-blockwise-generalization-optimization](../sources/prs/cutlass/PR-2270.md), [support fp16 accmulator for sm89 fp8 mma](../sources/prs/cutlass/PR-2378.md), [Example 77 add blackwell flash-attention bwd for MLA shape](../sources/prs/cutlass/PR-2466.md), [Add Blackwell MLA forward (shape: d=192, dv=128) implementation](../sources/prs/cutlass/PR-2472.md), [[Cutlass gemm] Fix SM100 FP8 nosmem epilogue-fusion shape_div 'Divisibility Condition' for non-multiple-of-64 N tiles](../sources/prs/cutlass/PR-2946.md), [[Bug Fix]Set NumSplitsM to 1 when TileShapeM < 128 in sm90 fp8 blockwise scaling CollectiveMma](../sources/prs/cutlass/PR-2965.md), [Fp8 kernel with "in-kernel" transpose of V in producer](../sources/prs/flash-attention/PR-1100.md), [FA3 FP8 qkv descales + restore max offset for h128 causal + added sync for producer WG](../sources/prs/flash-attention/PR-1173.md), [FA3 kvcache + split kv + gqa parallelization](../sources/prs/flash-attention/PR-1236.md), [[Cute,Fwd,Sm100] fp8 e4m3 and e5m2 support](../sources/prs/flash-attention/PR-2109.md), [feat: add functional per-head FP8 quantization for FA3](../sources/prs/flashinfer/PR-1033.md), [perf: accelerate blackwell grouped gemm](../sources/prs/flashinfer/PR-1086.md), [bugfix: fix fp8 attention kernels aot compilation issue](../sources/prs/flashinfer/PR-1087.md), [Add CUTLASS fused moe kernels from TensorRT-LLM.](../sources/prs/flashinfer/PR-1113.md), [[Feature] Support PDL for batch Prefill and Decode](../sources/prs/flashinfer/PR-1117.md), [Add DeepGEMM kernels](../sources/prs/flashinfer/PR-1209.md), [Fix test_groupwise_scaled_gemm_fp8.py](../sources/prs/flashinfer/PR-1211.md), [feat: trtllm-gen fp8 moe kernels](../sources/prs/flashinfer/PR-1212.md), [Patch fp8 cubin availability](../sources/prs/flashinfer/PR-1240.md), [feat: Support MXFP8 x MXFP4 CUTLASS grouped GEMM](../sources/prs/flashinfer/PR-1241.md), [Add trtllm-gen attention mha kernel with FP8 Q/K/V and FP8 output](../sources/prs/flashinfer/PR-1242.md), [Reduce the JIT compilation time of gen_gemm_sm100_module](../sources/prs/flashinfer/PR-1251.md), [init add gemm fp8 using cudnn backend](../sources/prs/flashinfer/PR-1264.md), [feat: add masked deepgemm support and benchmarking](../sources/prs/flashinfer/PR-1266.md), [Unify groupwise fp8 GEMM test](../sources/prs/flashinfer/PR-1281.md), [Bug fix: guard fp8 e8m0 and e2m1 compile ](../sources/prs/flashinfer/PR-1287.md), [Update cutlass fp4 moe kernels](../sources/prs/flashinfer/PR-1294.md), [Make Fp8 MoE routing_bias optional](../sources/prs/flashinfer/PR-1319.md), [Add blockwise-scaled FP8 GEMM via TRTLLM-Gen.](../sources/prs/flashinfer/PR-1320.md), [feat: Fused rope fp8 quantize kernel for MLA](../sources/prs/flashinfer/PR-1339.md), [[fix] remove (view) transpose to keep consistent with majorness MN requirement.](../sources/prs/flashinfer/PR-1358.md), [bugfix: Add guard for fp4/fp8 related include headers](../sources/prs/flashinfer/PR-1376.md), [GPT-OSS Support: Add Blackwell MoE mxfp4 implementation from TRTLLM and Attention Sink](../sources/prs/flashinfer/PR-1389.md), [Adding FP8 benchmark on attention and matmul testing](../sources/prs/flashinfer/PR-1390.md), [gpt-oss: Add MXFP8 x MXFP4 CUTLASS MOE for SM100 and BF16 x MXFP4 CUTLASS for SM90 + SwigluBias Activation](../sources/prs/flashinfer/PR-1396.md), [feature: add cutlass as bmm_fp8 backend.](../sources/prs/flashinfer/PR-1397.md), [bugfix: fix perf issue by using fp8 graph that can use cublaslt](../sources/prs/flashinfer/PR-1435.md), [Add alignment in MxFP8Quantization](../sources/prs/flashinfer/PR-1445.md), [Remove getEnvEnablePDL in favor of enable_pdl parameter](../sources/prs/flashinfer/PR-1446.md), [perf: add 1x4x1 cluster shape for fp8 bmm M<16 cases](../sources/prs/flashinfer/PR-1473.md), [refactor: unify autotuner for bmm_fp8](../sources/prs/flashinfer/PR-1479.md), [feat: Support fp8 qkv, fp16/bf16 out MHA for trtllm-gen.](../sources/prs/flashinfer/PR-1490.md), [Perf: support scale_a/scale_b instead of combined scale in cutlass bmm_fp8](../sources/prs/flashinfer/PR-1491.md), [flashinfer_benchmark QoL Improvements and Attention FP8 Support](../sources/prs/flashinfer/PR-1512.md), [bugfix: Fix compile error for undefined swizzle enum.](../sources/prs/flashinfer/PR-1530.md), [feat: Add fp8-qkv, fp16/bf16 output MHA](../sources/prs/flashinfer/PR-1540.md), [bugfix: fix unittest test_fp8_quantize](../sources/prs/flashinfer/PR-1599.md), [feat: initial support for SM103, SM110, SM120, SM121](../sources/prs/flashinfer/PR-1608.md), [feat: cutlass fp8 gemm bringup for SM120 & SM121](../sources/prs/flashinfer/PR-1610.md), [test: pytest.mark.xfail on deepgemm](../sources/prs/flashinfer/PR-1636.md), [Add benchmark for MLARopeQuantize](../sources/prs/flashinfer/PR-1656.md), [perf&bugfix: skip kv-tile computation out of sliding window in FA2; fix __syncthreads in mergestate](../sources/prs/flashinfer/PR-1661.md), [TGV GEMM as a BF16 backend alternative to cuBLAS](../sources/prs/flashinfer/PR-1668.md), [test: better fp8 quantization init for fused_moe test](../sources/prs/flashinfer/PR-1674.md), [Update deepgemm backend for 103a](../sources/prs/flashinfer/PR-1694.md), [test: skip the unsupported test cases for sm120/121](../sources/prs/flashinfer/PR-1710.md), [TVM: support TVM binding for GroupedGemm](../sources/prs/flashinfer/PR-1725.md), [tests: xfail moe quantization classes mxfp8_bf16 UTs on sm103 ](../sources/prs/flashinfer/PR-1754.md), [tests: skip non SM100/103 for grouped deepgemm](../sources/prs/flashinfer/PR-1767.md), [feat: add xqa fp8 mha and fp8 kv cache](../sources/prs/flashinfer/PR-1769.md), [feat:enable fp8 blockscale moe for fused cultass for sm90](../sources/prs/flashinfer/PR-1819.md), [feat: trtrllm-gen global scaled FP8 GEMMs](../sources/prs/flashinfer/PR-1829.md), [raise error for group_gemm_fp8_nt_groupwise then num_groups > 1 on sm120/121](../sources/prs/flashinfer/PR-1862.md), [misc: fix some B200 GEMM bench](../sources/prs/flashinfer/PR-1883.md), [MLA RoPE + quantization fused kernel: shape generalization for MHA / GQA](../sources/prs/flashinfer/PR-1924.md), [Fix: Verify scales are not None for Cutlass FP8 FusedMoE](../sources/prs/flashinfer/PR-1961.md), [feat: enable deepgemm jit for fp8 block-scale on SM90](../sources/prs/flashinfer/PR-1969.md), [fix: correct PDL parameter handling in RopeQuantize kernel](../sources/prs/flashinfer/PR-1982.md), [update trtllm cutlass moe ](../sources/prs/flashinfer/PR-2020.md), [feat: suitable_auto_backends to prune auto backends, bmm_fp8 refactor, heuristic_func intake](../sources/prs/flashinfer/PR-2029.md), [Enable renormalize(naive) routing for fp8 per-tensor](../sources/prs/flashinfer/PR-2030.md), [Added an initial implementation of Q and KV Cache in fp8 and to use t…](../sources/prs/flashinfer/PR-2035.md), [feat: Add flashinfer.rope.rope_quantize_fp8_append_paged_kv_cache (fused RoPE + Q + KV cache, supports MLA/GQA/MHA) ](../sources/prs/flashinfer/PR-2037.md), [Rebase FP8 SM100 Cutlass FMHA Attention to main (original PR#1238)](../sources/prs/flashinfer/PR-2047.md), [test: Skip test_fp8_quantize.py on Hopper](../sources/prs/flashinfer/PR-2052.md), [Fix moe fp8 failure for sm121](../sources/prs/flashinfer/PR-2061.md), [perf: TRT-LLM MoE Block-FP8 activation optimization](../sources/prs/flashinfer/PR-2063.md), [enable xqa fp8 output](../sources/prs/flashinfer/PR-2081.md), [refactor: update dpsk fused_moe test [1]](../sources/prs/flashinfer/PR-2088.md), [refactor: update dpsk fused_moe test [2]](../sources/prs/flashinfer/PR-2097.md), [refactor: update fa3 codebase and fix hopper unittest [part 1]](../sources/prs/flashinfer/PR-2111.md), [fix: Fix bench_mm_fp8.py](../sources/prs/flashinfer/PR-2129.md), [make DeepGEMM swapAB available for linear gemm SM90](../sources/prs/flashinfer/PR-2131.md), [feat: TRTLLM FMHAv2 backend for ctx attention](../sources/prs/flashinfer/PR-2142.md), [Enable Hopper FA3 FP8 attention in decode.py](../sources/prs/flashinfer/PR-2148.md), [Fp8 attention are now part of cuDNN 9.17.1](../sources/prs/flashinfer/PR-2241.md), [feat: RMSNorm/Fused RMSNorm + FP8 Quantization kernels](../sources/prs/flashinfer/PR-2243.md), [fix: support int64 IdType for RoPE part argument in `rope_quantize_fp8_append_paged_kv_cache`](../sources/prs/flashinfer/PR-2255.md), [feat: Add support for bmm mxfp8](../sources/prs/flashinfer/PR-2256.md), [Fix CUTLASS FP8 gemm correctness issue on SM120/SM121 for shapes where N is not divisible by ScaleGranularityN.](../sources/prs/flashinfer/PR-2261.md), [test: use .float() in in F.cosine_similarity() in bmm_fp8 test](../sources/prs/flashinfer/PR-2266.md), [feat: Support Fused MoE non gated Relu2 NVFP4 & FP8 and support Nemotron](../sources/prs/flashinfer/PR-2304.md), [[perf] Improve gemm_fp8_nt_groupwise (cutlass backend) by 10-40% for batch sizes <= 32](../sources/prs/flashinfer/PR-2327.md), [fix: guard batchWarpReduceSum with ENABLE_FP8 to fix compilation without FP8](../sources/prs/flashinfer/PR-2328.md), [fix: Fix NaN output in mxfp8_quantize for very small input values](../sources/prs/flashinfer/PR-2441.md), [Add cute-dsl backends to mxfp[8,4]_quantization for future refactor](../sources/prs/flashinfer/PR-2443.md), [feat: Add TRTLLM fmha_v2 library for SM90 attention with Skip-Softmax ](../sources/prs/flashinfer/PR-2446.md), [feat: Support Fused MoE non gated Relu2 NVFP4 & FP8 and support Nemotron, fixed](../sources/prs/flashinfer/PR-2462.md), [feat: Add MXFP8 GEMM mm_mxfp8 (cutlass)](../sources/prs/flashinfer/PR-2464.md), [Feat: Trtllm-gen MxFP8 MoE integration](../sources/prs/flashinfer/PR-2505.md), [fix: include fp8_blockscale_gemm_90 in AOT jit-cache](../sources/prs/flashinfer/PR-2533.md), [fallback to fa2 (instead of fa3) for unsupported configuration (bf16 Q, Fp8 KV)](../sources/prs/flashinfer/PR-2536.md), [tests: bmm_fp8 for SM110](../sources/prs/flashinfer/PR-2538.md), [Add gen_gemm_sm100_module_cutlass_mxfp8 to jit-cache](../sources/prs/flashinfer/PR-2549.md), [Implement `cutlass_fused_moe` mxfp8](../sources/prs/flashinfer/PR-2581.md), [fix: add SM121 support to SM120 version guards](../sources/prs/flashinfer/PR-2631.md), [benchmark: Add MXFP4/MXFP8 quantization mode support to FP4 MoE benchmark](../sources/prs/flashinfer/PR-2635.md), [[fp8_blockwise]Fix int32 overflow in TRTLLM fused MoE activation kernel](../sources/prs/flashinfer/PR-2642.md), [[feat] trtllm-gen mxfp8 gemm](../sources/prs/flashinfer/PR-2653.md), [feat: support mxfp4 & mxfp8 entrypoint for blackwell cutedsl dense gemm](../sources/prs/flashinfer/PR-2660.md), [benchmarks: Add FP8 input / BF16 output in ragged prefill benchmark](../sources/prs/flashinfer/PR-2666.md), [perf: Update trtllm-gen batched GEMM kernels - faster, more NVFP4 tile dims, MXFP8 with relu2 act](../sources/prs/flashinfer/PR-2667.md), [feat: Add support for TRTLLM MXFP8 non-gated MoE with ReLU2](../sources/prs/flashinfer/PR-2707.md), [Support for MXFP4 and NVFP4 group GEMMs on GeForce and Spark](../sources/prs/flashinfer/PR-2738.md), [Support in-place update for `trtllm_fp8_block_scale_moe`](../sources/prs/flashinfer/PR-2739.md), [Add cute dsl mla decode op](../sources/prs/flashinfer/PR-2743.md), [[Spark unit test debugging] Fix for tests/gemm/test_groupwise_scaled_gemm_fp8.py](../sources/prs/flashinfer/PR-2751.md), [feat: FP8 output support for CUTLASS MLA paged attention](../sources/prs/flashinfer/PR-2779.md), [tests: skip sliding window + fp8 to prevent hang in fmha_v2 unit tests](../sources/prs/flashinfer/PR-2781.md), [[CuTe DSL] Add modular FMHA prefill and MLA decode attention kernels](../sources/prs/flashinfer/PR-2805.md), [Fix silent bug with FP8 per tensor non-gated MoE](../sources/prs/flashinfer/PR-2882.md), [feat: add pdl support for cute dsl mla decode kernel support](../sources/prs/flashinfer/PR-2901.md), [feat: add MXFP8 GEMM support for SM120](../sources/prs/flashinfer/PR-2902.md), [perf: Optimize CuTe-DSL fp4 and fp8 quantization kernels](../sources/prs/flashinfer/PR-2904.md), [[NVIDIA] fix(jit): enable GDC for CUTLASS fused MoE PDL — prevent random crashes on SM12x](../sources/prs/flashinfer/PR-2913.md), [feat: Add cuBLASLt backend for `mm_bf16` and enable multi-tactic autotuning for FP8/MXFP8 runners](../sources/prs/flashinfer/PR-2914.md), [test: xfail cuDNN FP8 prefill on Blackwell with CUDA <= 12.9](../sources/prs/flashinfer/PR-2963.md), [test: skip unsupported mm_mxfp8 configurations on SM12x](../sources/prs/flashinfer/PR-2974.md), [ Fix MXFP4/MXFP8 failures in SM120 FAST_BUILD and expand all_tiles[] ](../sources/prs/flashinfer/PR-2994.md), [Support Allreduce + Norm + Per-token Group Fp8 Quant Fusion](../sources/prs/flashinfer/PR-3059.md), [feat: Enable FP8 (E4M3/E5M2) in concat_mla_k for optimize long-context prefill performance and refactor type dispatch for BF16/FP16](../sources/prs/flashinfer/PR-3129.md), [Integrate CUTLASS Small Tile N Blockscaled GEMMs/Grouped GEMMs for SM120 and SM121](../sources/prs/flashinfer/PR-3152.md), [test: enable bmm_mxfp8 cutlass backend coverage on SM12x](../sources/prs/flashinfer/PR-3183.md), [feat(trace): embed runnable init() in every TraceTemplate](../sources/prs/flashinfer/PR-3221.md), [Support Kimi K2.5 H64 CuTe DSL MLA decode](../sources/prs/flashinfer/PR-3235.md), [fix(fmha_v2): fix FP8 V-scratch pipeline and varlen scheduler on SM90](../sources/prs/flashinfer/PR-3276.md), [Naive Support for Hopper FP8 Prefill Kernel with Per-Head Quantization](../sources/prs/flashinfer/PR-869.md), [perf: Fix python API overhead when CUDAGraph is not enabled](../sources/prs/flashinfer/PR-969.md), [Update CUTLASS. Refine KernelSchedule for fp8 (grouped) gemm.](../sources/prs/sglang/PR-10491.md), [support qwen3-next-fp8 deepep](../sources/prs/sglang/PR-10622.md), [Fix DSR1 accuracy for flashinfer_trtllm MoE with FP8 quantization](../sources/prs/sglang/PR-11081.md), [[sgl-kernel][1/N]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-11432.md), [[DeepseekV32] Enable flashmla_prefill kernel with fp8 kvcache](../sources/prs/sglang/PR-11655.md), [Support running FP4 Deepseek on SM120.](../sources/prs/sglang/PR-11708.md), [Change bf16 to fp8 for some gemms in attention for DeepSeek ckpt v2](../sources/prs/sglang/PR-11805.md), [Support nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8/NVFP4](../sources/prs/sglang/PR-11866.md), [Feature/nano v2 offline modelopt fp8 and nvfp4](../sources/prs/sglang/PR-12018.md), [[sgl-kernel][4/N]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-12080.md), [[hotfix] missing `w13_weight_fp8` and `w2_weight_fp8` in UE8M0 requantization](../sources/prs/sglang/PR-12259.md), [fix: llama 4 + trtllm gen + fp8 kv cache incompatibility](../sources/prs/sglang/PR-12347.md), [Enable Flashinfer TRTLLM-GEN-MoE FP8 blockwise kernel for Qwen3-Next on Blackwell](../sources/prs/sglang/PR-12543.md), [[sgl-kernel] support custom fp8 flashmla kernel](../sources/prs/sglang/PR-13087.md), [Aiter fp8 kv cache](../sources/prs/sglang/PR-13147.md), [[NVIDIA] Fix broken fp8 MoE of deepseek v3](../sources/prs/sglang/PR-13264.md), [[NVIDIA] Fix use case of SGLANG_ENABLE_FLASHINFER_GEMM](../sources/prs/sglang/PR-13274.md), [[ROCM] Optimized deepseek-r1 fp8 model with + triton_gemm_a8w8 + batch_gemm_a8w8 + fused set_mla_kv_buffer kernel](../sources/prs/sglang/PR-13617.md), [[sgl-kernel][Feat][B200][1/N]Support MXFP8 Grouped GEMM in Blackwell](../sources/prs/sglang/PR-13731.md), [Support fp4 fp8 non gated moe](../sources/prs/sglang/PR-13794.md), [Feat: GLM-4.6 supports shared experts fusion](../sources/prs/sglang/PR-13873.md), [Fix update weight error for blackwell DeepGEMM](../sources/prs/sglang/PR-13910.md), [[DeepSeek v3.2] opt Context Parallelism: support fused moe, multi batch and fp8 kvcache](../sources/prs/sglang/PR-13959.md), [Support checking fp8 params in weight_checker](../sources/prs/sglang/PR-14147.md), [Add Mistral Large 3 support.](../sources/prs/sglang/PR-14213.md), [[CPU] Implement MXFP4 Gemm kernels for intel AMX to support GPT OSS series.](../sources/prs/sglang/PR-14385.md), [Support FP8 MLA prefill and 128k context.](../sources/prs/sglang/PR-14395.md), [Add Mistral Large 3 Eagle Support](../sources/prs/sglang/PR-14466.md), [[sgl-kernel][Feat][B200][2/N] Support MXFP8 Grouped GEMM in Blackwell](../sources/prs/sglang/PR-14640.md), [[sgl-kernel] Update flashmla to include fp8 sparse_mla optimizations](../sources/prs/sglang/PR-15242.md), [Flashinfer MOE FP8 support for Mistral Large 3.](../sources/prs/sglang/PR-15422.md), [[sgl-kernel][6/7]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-15471.md), [[Perf] Add Flashinfer DeepGEMM SM90 for SwapAB Optimization](../sources/prs/sglang/PR-15514.md), [Optimize FP8 MLA KV cache writes with Triton kernel](../sources/prs/sglang/PR-15522.md), [[JIT kernel] Apply jit per_tensor_quant_fp8 kernel](../sources/prs/sglang/PR-15836.md), [Fix FP8 MoE NaN with DeepGEMM on Blackwell](../sources/prs/sglang/PR-16622.md), [[GLM 4.7] Add RTX 6000 Pro aka sm120](../sources/prs/sglang/PR-17235.md), [Disable mla persistent kernel when not using fp8 kv_cache](../sources/prs/sglang/PR-17327.md), [Add mxfp8 support for online quantization, Triton dense linear, and CUTLASS MoE](../sources/prs/sglang/PR-17449.md), [[ROCm] Optimize Deepseek R1 on MI300X](../sources/prs/sglang/PR-18242.md), [Nsa trtllm mla sparse fp8 support with Deepseek v3.2 NVFP4](../sources/prs/sglang/PR-18389.md), [[AMD] Update aiter to v0.1.10.post2](../sources/prs/sglang/PR-18423.md), [Fp8 prefill attn kernel integration](../sources/prs/sglang/PR-18528.md), [[AMD] DSR1/V3 use fp8 bmm in MLA for MI300X](../sources/prs/sglang/PR-18624.md), [[RL] Support per-layer mixed FP8/BF16 serving for FP8 checkpoints](../sources/prs/sglang/PR-18742.md), [Fix NSA FP8 KV cache path for both-trtllm MHA one-shot](../sources/prs/sglang/PR-18931.md), [[FlashInfer v0.6.4] [RL] Integrate FlashInfer mxfp8 gemm, MoE, and routed MoE](../sources/prs/sglang/PR-19537.md), [Various SM120 improvements](../sources/prs/sglang/PR-19721.md), [[AMD] Fix FP8 assertion failure in aiter MLA decode by falling back to self.k_scale](../sources/prs/sglang/PR-19935.md), [Enable modelopt quantized FLUX deployment](../sources/prs/sglang/PR-20082.md), [[AMD] Fp8 prefill integration with radix cache path for dpsk models](../sources/prs/sglang/PR-20187.md), [[Benchmark] use flashinfer bench_gpu_time instead of triton do_bench](../sources/prs/sglang/PR-20305.md), [[NVIDIA] Enable fp8 flashinfer_trtllm_routed MoE for MiniMax-M2.5](../sources/prs/sglang/PR-20394.md), [Support Triton MLA FP8 KV cache](../sources/prs/sglang/PR-20479.md), [FIX: (NSA) Compute topk_indices_offset when NSA prefill flashmla_sparse is used with FP8 KV cache](../sources/prs/sglang/PR-20606.md), [CUTLASS FP8 Blockwise GEMM improvement of SM120](../sources/prs/sglang/PR-20887.md), [Add SGLang CUDA crash API logging inspired by FlashInfer](../sources/prs/sglang/PR-20910.md), [ci: run Stage A CUDA tests as stage-a-test-small-1-gpu on 5090](../sources/prs/sglang/PR-20988.md), [[AMD]: Support MLA with nhead<16 and FP8 KV cache for TP=8 (Kimi K2.5…](../sources/prs/sglang/PR-21213.md), [Refactor JIT kernel CI to use run_suite.py registration system](../sources/prs/sglang/PR-21239.md), [[RL] Support mxfp8 DeepSeek V3](../sources/prs/sglang/PR-21280.md), [[Kernel] Support FlashInfer TRTLLM-Gen fused MoE for non-gated FP4 & FP8 (Nemotron)](../sources/prs/sglang/PR-21321.md), [[misc] clean up kernel API](../sources/prs/sglang/PR-21325.md), [Migrate all callers from /get_server_info to /server_info](../sources/prs/sglang/PR-21463.md), [[AMD] Enable FP8 KV cache and FP8 attention kernel for NSA on MI300/MI355 with TileLang backend](../sources/prs/sglang/PR-21511.md), [[FlashInver v0.6.7] Integrate flashinfer_trtllm mxfp8 gemm](../sources/prs/sglang/PR-21576.md), [[AMD] Add GLM-5-FP8 nightly performance benchmarks for MI30x and MI35x](../sources/prs/sglang/PR-21710.md), [[Misc] [MXFP8] Drop sm100 mxfp8 warning](../sources/prs/sglang/PR-21881.md), [fix pcg torch dynamo recompile in mxfp8 Triton path](../sources/prs/sglang/PR-21888.md), [Tiny fix trtllm_fp8_per_tensor_scale_moe_wrapper router_logits dtype](../sources/prs/sglang/PR-22006.md), [[AMD][HIP] NSA: bf16 passthrough from RMSNorm to eliminate FP8 dequantization](../sources/prs/sglang/PR-22258.md), [[AMD] Fix GLM-5 fp8 KV quant path dispatch on MI300](../sources/prs/sglang/PR-22314.md), [[Reland] DeepSeek-R1-0528-w4a8: DeepEP Low Latency Dispatch Adopts FP8 Communication](../sources/prs/sglang/PR-22316.md), [[Lora] Lora quat info re-factor and support deepseekv3 mla lora](../sources/prs/sglang/PR-22323.md), [[AMD] Add GLM-5.1-FP8 nightly accuracy and performance benchmarks for MI30x and MI35x](../sources/prs/sglang/PR-22336.md), [[Diffusion] modelopt diffusion fp8 support for flux1/flux2 and wan2.2](../sources/prs/sglang/PR-22365.md), [[DSA] Hopper FP8 FlashMLA KV padding](../sources/prs/sglang/PR-22372.md), [[RL] Fix weight update for mxfp8 flashinfer_cutlass gemm backend](../sources/prs/sglang/PR-22484.md), [[Diffusion] Add FLUX.1-dev ModelOpt NVFP4 support](../sources/prs/sglang/PR-22574.md), [reland [Diffusion] Add FLUX.1-dev ModelOpt NVFP4 support](../sources/prs/sglang/PR-22672.md), [Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-23686.md), [[MoE] Unify DeepEPMoE+MoriEPMoE through AITER MoeRunner pre/post-permute](../sources/prs/sglang/PR-23760.md), [Refactor device timer, clean up metrics collector, and add fwd occupancy metric](../sources/prs/sglang/PR-24197.md), [Port MXFP4 Marlin MoE support to JIT kernel path](../sources/prs/sglang/PR-24490.md), [[Codex] Opt Mistral Large performace ](../sources/prs/sglang/PR-24611.md), [[Gemma4] Optimize Gemm4 with fused Q/K/V RMSNorm + per-expert FP8 ckpt loader](../sources/prs/sglang/PR-24696.md), [Add FlashInfer SM90 cutlass MXFP4 MoE backend (W4A16) for GPT-OSS + DeepSeek-V4](../sources/prs/sglang/PR-24816.md), [[attn backend] Integrate tokenspeed_mla prefill/decode kernels (fp8 kv cache, blackwell)](../sources/prs/sglang/PR-24925.md), [Amd/deepseek v4 rebase main 0509](../sources/prs/sglang/PR-24933.md), [[rebase]Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-24986.md), [[Gemma4]: Fix FP8 Triton scale layout](../sources/prs/sglang/PR-25286.md), [[Intel GPU] Enable DeepSeek V4 Inference on XPU](../sources/prs/sglang/PR-25336.md), [[fp8] SM90 swap-AB scaled_mm dispatch (~1.16x kernel geomean, +5.8-18.5% end-to-end)](../sources/prs/sglang/PR-25532.md), [[Test] Stage-a sanity kits; consolidate core/ + models_e2e/ tests](../sources/prs/sglang/PR-25831.md), [support w8a8 fp8 kernel with CUTLASS](../sources/prs/sglang/PR-3047.md), [feat: integrate gemm_fp8 kernel into gemm](../sources/prs/sglang/PR-3056.md), [Apply sgl w8a8 fp8 kernel](../sources/prs/sglang/PR-3148.md), [add tensorrt_llm common and cutlass_extensions as 3rdparty](../sources/prs/sglang/PR-3216.md), [support blockwise fp8 matmul kernel](../sources/prs/sglang/PR-3267.md), [integrate blockwise fp8 kernel](../sources/prs/sglang/PR-3529.md), [add control for cutlass fp8 blockwise gemm](../sources/prs/sglang/PR-3727.md), [[tools] add fp8 max/min constant in utils](../sources/prs/sglang/PR-3959.md), [DeepGemm integrate to gemm](../sources/prs/sglang/PR-4165.md), [linear support deepgemm](../sources/prs/sglang/PR-4199.md), [Accelerate FP8 CUDA Kernel by 20-28%](../sources/prs/sglang/PR-4215.md), [Clean up fp8 support](../sources/prs/sglang/PR-4230.md), [fix per_token_group_quant_fp8 illegal memory when num_groups % 16 != 0](../sources/prs/sglang/PR-4231.md), [Support Blackwell Block Scale FP8 Gemm](../sources/prs/sglang/PR-4278.md), [[FIX] fix incorrect output when enable both deepgemm and torch compile](../sources/prs/sglang/PR-4359.md), [[ROCm] fix dtype](../sources/prs/sglang/PR-4510.md), [Create col-major and tma-aligned x_scale for deep_gemm.gemm_fp8_fp8_bf16_nt](../sources/prs/sglang/PR-4515.md), [Support fp8 gemm for blackwell](../sources/prs/sglang/PR-4558.md), [[quantization] fix channelwise conversion with scalar weight scale](../sources/prs/sglang/PR-4596.md), [Set deepgemm to the default value in the hopper architecture.](../sources/prs/sglang/PR-4613.md), [Add DeepSeek V3/R1 shared experts fusion](../sources/prs/sglang/PR-4918.md), [feat: add DeepGEMM build warning](../sources/prs/sglang/PR-5176.md), [[Fix] Turn off DeepGEMM by default](../sources/prs/sglang/PR-5263.md), [[1/2] Add FP8 Blockscale MoE CUTLASS kernel for Blackwell](../sources/prs/sglang/PR-5281.md), [fix: use deepgemm only on hopper](../sources/prs/sglang/PR-5310.md), [[perf] experimental enhance fp8 per-tensor quant](../sources/prs/sglang/PR-5370.md), [[perf] introduce deep gemm group_gemm_masked as gemm](../sources/prs/sglang/PR-5432.md), [[feature] enable pre compile jit deep_gemm](../sources/prs/sglang/PR-5580.md), [ DeepEP normal support deepgemm-contiguous](../sources/prs/sglang/PR-5626.md), [[perf] dsv3 bmm fallback to bf16](../sources/prs/sglang/PR-5662.md), [[2/2] Add python wrapper for CUTLASS FP8 Blockscale MoE Kernel. ](../sources/prs/sglang/PR-5694.md), [cutlass 3.9 supported to improve fp8_blockwise_gemm](../sources/prs/sglang/PR-5820.md), [chore: upgrade cutlass 3.9.2](../sources/prs/sglang/PR-6004.md), [Upgrade CUTLASS 4.0](../sources/prs/sglang/PR-6336.md), [Add fp8 fused_experts kernel for CPU in sgl-kernel and add UT](../sources/prs/sglang/PR-6404.md), [Fix bug of deepseek-v3 under DP+EP mode with large batchsize/seqlen](../sources/prs/sglang/PR-6449.md), [[Feature] Support Flashinfer fp8 blockwise GEMM kernel on Blackwell](../sources/prs/sglang/PR-6479.md), [Set `num_fused_shared_experts` as `num_shared_experts` when shared_experts fusion is not disabled](../sources/prs/sglang/PR-6736.md), [[CPU] add optimizations for INT8 and FP8 DeepSeek](../sources/prs/sglang/PR-6769.md), [feat: integrate deepgemm into EPMoE](../sources/prs/sglang/PR-6821.md), [CPU: map changes from developing branch in sgl-kernel](../sources/prs/sglang/PR-6833.md), [Use deepgemm instead of triton for fused_qkv_a_proj_with_mqa](../sources/prs/sglang/PR-6890.md), [Add a CUDA kernel for fusing mapping and weighted sum for MoE.](../sources/prs/sglang/PR-6916.md), [[Feature] Support Flashinfer fmha on Blackwell](../sources/prs/sglang/PR-6930.md), [Fuse routed scaling factor in deepseek](../sources/prs/sglang/PR-6970.md), [Update default settings for blackwell](../sources/prs/sglang/PR-7023.md), [Fix positional argument](../sources/prs/sglang/PR-7093.md), [fix amd EP MoE FP8 issue](../sources/prs/sglang/PR-7125.md), [Enable ModelOpt Llama4 fp8 checkpoint deployment in SGLang](../sources/prs/sglang/PR-7129.md), [[amd] Opt dsv3 moe](../sources/prs/sglang/PR-7160.md), [Support new DeepGEMM](../sources/prs/sglang/PR-7172.md), [Tiny let DeepGEMM scale checks cover more cases](../sources/prs/sglang/PR-7182.md), [[AMD] Fail gracefully when AITER is unavailable gfx90a GPUs](../sources/prs/sglang/PR-7187.md), [[fix] fix DeepGEMM blackwell input quant & ut & fix style and log](../sources/prs/sglang/PR-7247.md), [Add CUTLASS FP8 Blockscale MoE kernel for Hopper architecture](../sources/prs/sglang/PR-7278.md), [Fix torch compile run](../sources/prs/sglang/PR-7391.md), [[AMD][Quantization] Add `int4fp8_moe` online quantization on ROCm](../sources/prs/sglang/PR-7392.md), [feat: support DeepSeek-R1-W4AFP8 model with ep-moe mode](../sources/prs/sglang/PR-7762.md), [Qwen FP8/NVFP4 ModelOPT Quantization support](../sources/prs/sglang/PR-7912.md), [[feat] Support tp mode for DeepSeek-R1-W4AFP8](../sources/prs/sglang/PR-8118.md), [[sgl-kernel] Opt per_token_quant_fp8 with warp reduce](../sources/prs/sglang/PR-8130.md), [[1/N]Support DeepSeek-R1 w4a8 normal deepep](../sources/prs/sglang/PR-8247.md), [[2/N]Support DeepSeek-R1 w4a8 low latency deepep](../sources/prs/sglang/PR-8464.md), [TRTLLM-MLA FP8 path](../sources/prs/sglang/PR-8638.md), [feat: support cutlass_moe_fp8 kernel for fusedmoe in sm90](../sources/prs/sglang/PR-8678.md), [[Perf] Tunings for SM100 FP8 CUTLASS kernel](../sources/prs/sglang/PR-8818.md), [optimize: reduce shulffle and quantization overhead in cutlass_moe sm90](../sources/prs/sglang/PR-8962.md), [[fix]: fix cutlass moe ut and and Opt H20 cutlass groupGemm performance](../sources/prs/sglang/PR-9272.md), [[sgl-kernel] feat: Support sm120 cutlass fp8 gemm kernel](../sources/prs/sglang/PR-9403.md), [fix: blackwell dsv3 fp8 issue temporary solution](../sources/prs/sglang/PR-9530.md), [Update CUTLASS 4.2 & Enable K-Major Scale Factor for SM90 FP8 Blockwise Group GEMM](../sources/prs/sglang/PR-9559.md), [move is_sm90_supported/is_sm100_supported to python/sglang/srt/utils.py](../sources/prs/sglang/PR-9679.md), [[CPU] Add FP8 Bmm support](../sources/prs/sglang/PR-9744.md), [Make sm100 fp8 kernels available on sm103](../sources/prs/sglang/PR-9789.md), [CUTLASS fp8 blockwise gemm support of sm120](../sources/prs/sglang/PR-9969.md), [[WIP] support more dtypes for tcgen05](../sources/prs/tilelang/PR-1229.md), [[Enhancement] add more dtype and fix mma.ws for fp16 for tcgen05](../sources/prs/tilelang/PR-1327.md), [[Feature] Block-scaled GEMM support for MXFP8 on Blackwell](../sources/prs/tilelang/PR-1945.md), [[Kernel]: Cutlass 2:4 Sparsity + FP8/Int8 Quant Support](../sources/prs/vllm/PR-10995.md), [[Kernel] Update `cutlass_scaled_mm` to support 2d group (blockwise) scaling](../sources/prs/vllm/PR-11868.md), [Add: Support for Sparse24Bitmask Compressed Models](../sources/prs/vllm/PR-12097.md), [Expert Parallelism (EP) Support for DeepSeek Models](../sources/prs/vllm/PR-12583.md), [[Kernel][Quantization] Integrate block-quantized CUTLASS kernels for DeepSeekV3](../sources/prs/vllm/PR-12587.md), [[Attention] Deepseek v3 MLA support with FP8 compute](../sources/prs/vllm/PR-12601.md), [[Attention] MLA with chunked prefill](../sources/prs/vllm/PR-12639.md), [[AMD][ROCm] Enable DeepSeek model on ROCm](../sources/prs/vllm/PR-12662.md), [[Bugfix] Better FP8 supported defaults](../sources/prs/vllm/PR-12796.md), [[Kernel]Add streamK for block-quantized CUTLASS kernels](../sources/prs/vllm/PR-12978.md), [[core] Perf improvement for DSv3 on AMD GPUs](../sources/prs/vllm/PR-13718.md), [[V1] V1 Enablement Oracle ](../sources/prs/vllm/PR-13726.md), [[Bugfix][Quantization] Fix FP8 + EP](../sources/prs/vllm/PR-13784.md), [add tcgen05 support for tcgen05 fp8 gemm](../sources/prs/vllm/PR-13798.md), [[Kernel] CUTLASS grouped gemm fp8 MoE kernel](../sources/prs/vllm/PR-13972.md), [[core] moe fp8 block quant tuning support](../sources/prs/vllm/PR-14068.md), [dynamic distpatch of fp8 kernels](../sources/prs/vllm/PR-14245.md), [Add cutlass support for blackwell fp8 blockwise gemm](../sources/prs/vllm/PR-14383.md), [[BugFix] Illegal Memory Access in the blockwise cutlass fp8 GEMMs](../sources/prs/vllm/PR-14396.md), [[ROCm][Kernel] MoE weights padding](../sources/prs/vllm/PR-14454.md), [permute/unpermute kernel for moe optimization](../sources/prs/vllm/PR-14568.md), [[Attention] Flash Attention 3 - fp8](../sources/prs/vllm/PR-14570.md), [[Quantization][FP8] Adding support for fp8 gemm layer input in fp8](../sources/prs/vllm/PR-14578.md), [[Attention] MLA get rid of materialization](../sources/prs/vllm/PR-14770.md), [[FEAT][ROCm] Integrate Fused MoE Kernels from AITER](../sources/prs/vllm/PR-14967.md), [[FEAT] [ROCm]: Add AITER Block-Scaled GEMM Feature](../sources/prs/vllm/PR-14968.md), [[Quantization] Fp8 Channelwise Dynamic Per Token GroupedGEMM](../sources/prs/vllm/PR-15587.md), [Modularize fused experts and integrate PPLX kernels](../sources/prs/vllm/PR-15956.md), [Upstream Llama4 Support to Main](../sources/prs/vllm/PR-16113.md), [[Kernel] Support W8A8 channel-wise weights and per-token activations in triton fused_moe_kernel](../sources/prs/vllm/PR-16366.md), [Enable PTPC FP8 for CompressedTensorsW8A8Fp8MoEMethod (triton fused_moe)](../sources/prs/vllm/PR-16537.md), [[ROCm] Add aiter tkw1 kernel for Llama4 fp8](../sources/prs/vllm/PR-16727.md), [[Kernel] some optimizations for dense marlin and moe marlin](../sources/prs/vllm/PR-16850.md), [[Kernel] Add expert_map support to Cutlass FP8 MOE](../sources/prs/vllm/PR-16861.md), [[FEAT] [ROCm]: Add AITER CK 2 Stages MoE support](../sources/prs/vllm/PR-17110.md), [[ROCm][FP8][Kernel] FP8 quantization fused into Custom Paged Attention](../sources/prs/vllm/PR-17139.md), [[NVIDIA] Support Cutlass w8a8 FP8 for Blackwell Geforce GPUs (sm120)](../sources/prs/vllm/PR-17280.md), [[Kernel] fp4 marlin kernel](../sources/prs/vllm/PR-17687.md), [use ceil_div in cutlass block scaling shape check](../sources/prs/vllm/PR-17918.md), [[Feature] Expert Parallelism Load Balancer (EPLB)](../sources/prs/vllm/PR-18343.md), [[V1] Support `LLM.apply_model`](../sources/prs/vllm/PR-18465.md), [Sm100 blockwise fp8 swap ab](../sources/prs/vllm/PR-18564.md), [[Kernel] Integrate CUTLASS MoE kernel with PPLX](../sources/prs/vllm/PR-18762.md), [[Perf] Tunings for SM100 FP8 CUTLASS kernel](../sources/prs/vllm/PR-18778.md), [[Kernel] Enable fp8 support for pplx and BatchedTritonExperts.](../sources/prs/vllm/PR-18864.md), [[Kernel] Support deep_gemm for linear methods](../sources/prs/vllm/PR-19085.md), [[Kernels] Add activation chunking logic to FusedMoEModularKernel](../sources/prs/vllm/PR-19168.md), [[Perf] Further tunings for SM100 FP8 CUTLASS kernel](../sources/prs/vllm/PR-19566.md), [[feat]: CUTLASS block scaled group gemm for SM100](../sources/prs/vllm/PR-19757.md), [[Feature] Integrate new deepgemm](../sources/prs/vllm/PR-19820.md), [ [Feature] Integrate SM100 DeepGEMM support](../sources/prs/vllm/PR-20087.md), [[Bugfix] Fix topk_ids indices_type for CUTLASS w8a8 FP8 MoE](../sources/prs/vllm/PR-20166.md), [[V1] [ROCm] Enable EP with AITER Fused MoE](../sources/prs/vllm/PR-20270.md), [[Kernel] SM90 CUTLASS FP8 GEMM: add support for swap AB + kernel tuning](../sources/prs/vllm/PR-20396.md), [[feat]: add SM100 support for cutlass FP8 groupGEMM](../sources/prs/vllm/PR-20447.md), [Support Llama 4 for fused_marlin_moe](../sources/prs/vllm/PR-20457.md), [[Perf] Reuse workspace for FP8+FP4 Marlin MoE](../sources/prs/vllm/PR-20500.md), [[Performance] Performance improvements in non-blockwise fp8 CUTLASS MoE](../sources/prs/vllm/PR-20762.md), [[Perf] Use Triton instead of Torch for DeepGEMM Per Token Group Quant](../sources/prs/vllm/PR-20841.md), [[Perf] Add swap_ab to SM90 FP8 non-block CUTLASS moe grouped gemm](../sources/prs/vllm/PR-20911.md), [[Perf] Cuda Kernel for Per Token Group Quant](../sources/prs/vllm/PR-21083.md), [[perf] Add fused MLA QKV + strided layernorm](../sources/prs/vllm/PR-21116.md), [[Feature][OCP MX] Support mxfp6 and mixed mxfp6-mxfp4](../sources/prs/vllm/PR-21166.md), [[Bug] DeepGemm: Fix TypeError: per_block_cast_to_fp8() missing 1 required positional argument: 'use_ue8m0' for SM100](../sources/prs/vllm/PR-21187.md), [[NVIDIA] Explicitly disable shuffled weights for flashinfer blockscale moe fp8 kernels](../sources/prs/vllm/PR-21411.md), [[Bugfix][CUDA] fixes CUDA FP8 kv cache dtype supported](../sources/prs/vllm/PR-21420.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv/out Attention Kernel](../sources/prs/vllm/PR-21716.md), [[Kernel] Add support for block FP8 on SM120 (NVIDIA 5090 and RTX PRO 6000)](../sources/prs/vllm/PR-22131.md), [Fp8 paged attention update](../sources/prs/vllm/PR-22222.md), [[Bug] Fix B200 DeepGEMM E8M0 Accuracy Issue](../sources/prs/vllm/PR-22399.md), [[Quantization] Expand compressed-tensors MoE matching logic to support NFP4 + FP8 MoEs](../sources/prs/vllm/PR-22674.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv NVFP4-out Attention Kernel](../sources/prs/vllm/PR-22703.md), [fp8 kv cache support fix for torch.compile](../sources/prs/vllm/PR-22758.md), [[Kernel] Added flashinfer fp8 per-tensor gemms](../sources/prs/vllm/PR-22895.md), [[Bugfix] fix qwen3 moe fp8 accuracy issue](../sources/prs/vllm/PR-23031.md), [[Kernel] CUTLASS MoE FP8: Integrate cuda moe permute/unpermute](../sources/prs/vllm/PR-23045.md), [Add routed_scaling_factor to MoE grouped topk](../sources/prs/vllm/PR-23123.md), [[XPU][Feature] fp8 online quantization support for XPU](../sources/prs/vllm/PR-23148.md), [[kernel] Support W4A8 on Hopper](../sources/prs/vllm/PR-23198.md), [[ROCm][Aiter] Add triton fp8 bmm kernel for mla](../sources/prs/vllm/PR-23264.md), [[Perf] Small optimizations for silu_mul_fp8_quant_deep_gemm](../sources/prs/vllm/PR-23265.md), [[Kernels] Overlap shared experts with send/recv](../sources/prs/vllm/PR-23273.md), [[Perf] Use upstream CUTLASS for SM90 Block FP8 kernel](../sources/prs/vllm/PR-23280.md), [[Bug] Fix R1 Accuracy 0 Bug](../sources/prs/vllm/PR-23294.md), [DP/EP Support for gpt-oss with deepep-ht comm kernel on SM100](../sources/prs/vllm/PR-23608.md), [[Flashinfer] Support Flashinfer TRTLLM FP8-qkv BF16/FP16-out Attention Kernel](../sources/prs/vllm/PR-23647.md), [[Feature] Add Hopper DeepGEMM E8M0 for DeepSeekV3.1 scale_fmt](../sources/prs/vllm/PR-23666.md), [[Kernel][B200] mxfp4 fused cutlass moe](../sources/prs/vllm/PR-23696.md), [[Model] Add LongCat-Flash ](../sources/prs/vllm/PR-23991.md), [[Performance] Move apply_w8a8_block_fp8_linear to an op class](../sources/prs/vllm/PR-24666.md), [[NVIDIA] Blackwell Family](../sources/prs/vllm/PR-24673.md), [[Kernel][Quantization] add w4a8 support for marlin kernel](../sources/prs/vllm/PR-24722.md), [[Flashinfer][gpt-oss] Support FP8-qkv Flashinfer TRTLLM Sinks Attention](../sources/prs/vllm/PR-25674.md), [[Bugfix] Convert untraceable GroupShape to list for AMD impl](../sources/prs/vllm/PR-26535.md), [[Bugfix] Fix gpt-oss w4a8 DP/EP on B200](../sources/prs/vllm/PR-26729.md), [[Feature] Batch Invariant: Support DeepGEMM and Blackwell](../sources/prs/vllm/PR-27127.md), [[Kernels] Enable FlashInfer FP8 Blockscale on SM90 (for TEP DSR1)](../sources/prs/vllm/PR-27134.md), [[torch.compile] Enable silu_mul_fp8_quant fusion without custom ops enabled](../sources/prs/vllm/PR-27146.md), [Bugfix: Cutlass FP8 FusedMoE bad scaling factors](../sources/prs/vllm/PR-27255.md), [Feature: Support Relu2 in FusedMoE fp8 cutlass path](../sources/prs/vllm/PR-27261.md), [[Perf] SM100 - add swap AB optimization to CUTLASS FP8 GEMM](../sources/prs/vllm/PR-27284.md), [[Performance] Support FP8 flashinfer TRTLLM MOE on Qwen3 and Qwen-3next](../sources/prs/vllm/PR-27492.md), [[Attention] Use sparse prefill kernel for fp8 kv-cache in DeepSeek-v3.2](../sources/prs/vllm/PR-27532.md), [[Feature] Batch invariant torch.compile](../sources/prs/vllm/PR-27660.md), [[Performance] Fused blockwise quant RMS norm](../sources/prs/vllm/PR-27883.md), [[Performance][B200] Fix deepgemm prologue](../sources/prs/vllm/PR-27897.md), [[ROCm][MLA] enable fp8 MLA decode on ROCm](../sources/prs/vllm/PR-28032.md), [[Performance][B200] silu_mul_quant: pack scales in int32](../sources/prs/vllm/PR-28358.md), [[Performance] Reduce DeepGEMM N dim restriction from 128 to 64 multiplier ](../sources/prs/vllm/PR-28687.md), [[Perf][Kernels] Enable FlashInfer DeepGEMM swapAB on SM90 (for W8A8 Linear Op)](../sources/prs/vllm/PR-29213.md), [chore: add RTX_PRO_6000 GLM4.6-FP8 kernel tuning](../sources/prs/vllm/PR-29240.md), [[Perf] Disable DeepGEMM MoE by default when TP=8 is used](../sources/prs/vllm/PR-29346.md), [[Kernel]Support W4A8 Grouped GEMM on Hopper](../sources/prs/vllm/PR-29691.md), [[MoE-FP8-modelopt] Add FlashInfer alignment padding for intermediate dimensions](../sources/prs/vllm/PR-29748.md), [Add Mistral Large 3 and Ministral 3](../sources/prs/vllm/PR-29757.md), [[Perf] Improve fp8 quant in mla; replace ReduceSum with ReduceScatterSum](../sources/prs/vllm/PR-29795.md), [[Bugfix] Fix FP8 MoE LoRA](../sources/prs/vllm/PR-29890.md), [[Kernel][Quantization][MoE] add marlin kernel support for turing (sm75)](../sources/prs/vllm/PR-29901.md), [[ROCm] add fallback for aiter fp8 decode mla](../sources/prs/vllm/PR-30005.md), [[Quantization] Support Quark int4-fp8 w4a8 for MoE](../sources/prs/vllm/PR-30071.md), [Add llmcompressor fp8 kv-cache quant (per-tensor and per-attn_head)](../sources/prs/vllm/PR-30141.md), [[Bugfix] Fix DeepGEMM after #29546 ](../sources/prs/vllm/PR-30267.md), [[LoRA] Support Quantized Adapters](../sources/prs/vllm/PR-30286.md), [[Bugfix] Fix fp8 DeepGemm compilation issues](../sources/prs/vllm/PR-30336.md), [[ROCm][Quantization] GPT OSS Upstream MoE wmxfp4_afp8 with static scales](../sources/prs/vllm/PR-30357.md), [[Feature] Add SM103 (Blackwell Ultra) Support to vLLM](../sources/prs/vllm/PR-30484.md), [[Perf] Eliminate padding and slicing op for GPT-OSS with Flashinfer MXFP4 MXFP8 MoE](../sources/prs/vllm/PR-30647.md), [[SM100] Enable fp8 compute for prefill MLA](../sources/prs/vllm/PR-30746.md), [[Feature]: Support NVIDIA ModelOpt HF FP8 variants FP8_PER_CHANNEL_PER_TOKEN and FP8_PB_WO in vLLM](../sources/prs/vllm/PR-30957.md), [[Bugfix][Hardware][AMD] Consolidate FP8 min/max values helper function](../sources/prs/vllm/PR-31106.md), [[Bugfix][Hardware][AMD] Fix exception types in AITER MLA FP8 check](../sources/prs/vllm/PR-31177.md), [[SM100] Resubmit FMHA FP8 prefill for MLA](../sources/prs/vllm/PR-31195.md), [[Bugfix][ROCm] Fix Static Quant Issue](../sources/prs/vllm/PR-31502.md), [[Bugfix] Fix Broken ModelOpt NVFP4 MoE](../sources/prs/vllm/PR-31742.md), [[1/N][Attention] Restructure attention: move files](../sources/prs/vllm/PR-31916.md), [[5/N][Attention] Finish eliminating `vllm/attention` folder](../sources/prs/vllm/PR-32064.md), [[Perf] Create TMA-aligned input scale tensor for DeepGemm on Hopper](../sources/prs/vllm/PR-32619.md), [[Bugfix] Fix FP8 MoE EP Weight Loading for ModelOpt Llama4](../sources/prs/vllm/PR-32886.md), [Support compress-tensors with nvfp4 or fp8 weights and modelopt with nvfp4 weights on Turing](../sources/prs/vllm/PR-33076.md), [Add support for Mistral Large 3 inference with Flashinfer MoE](../sources/prs/vllm/PR-33174.md), [[Bugfix] Fix quant RMS norm fusion for quantization with TMA-aligned scales](../sources/prs/vllm/PR-33255.md), [[Bugfix] Register fp8 cutlass_group_gemm as supported for only SM90+SM100](../sources/prs/vllm/PR-33285.md), [[Kernel] Support Flashinfer trtllm fused MoE non gated FP8 & NVFP4](../sources/prs/vllm/PR-33506.md), [[Kernel] Add enable_sm120_or_later for SM121 (DGX Spark) CUTLASS support](../sources/prs/vllm/PR-33517.md), [[Perf] Disable clean_logits in deepgemm fp8_mqa_logits kernel](../sources/prs/vllm/PR-33568.md), [enable skipping of SW attention layers when using FP8 KV cache](../sources/prs/vllm/PR-33695.md), [[Kernel] Integrate SM100 MXFP8 blockscaled grouped MM and quant kernels](../sources/prs/vllm/PR-34448.md), [[Bugfix] Handle num_expert_group=None in flashinfer block-scale FP8 MoE](../sources/prs/vllm/PR-34494.md), [[Kernel] Add FP8 KV cache support to Triton MLA decode attention](../sources/prs/vllm/PR-34597.md), [[Bugfix] Fix NVFP4 TRTLLM MoE non-gated support; add gsm8k for Nemotron-3-Nano FP8+NVFP4](../sources/prs/vllm/PR-34725.md), [Integrate flashinfer mm_mxfp8 in ModelOpt MXFP8](../sources/prs/vllm/PR-35053.md), [[Feat] Add CUDA torch fallbacks for fp8_mqa_logits/fp8_paged_mqa_logits_torch function](../sources/prs/vllm/PR-35271.md), [[Attention][Perf] Optimize cp_gather_and_upconvert_fp8_kv_cache - DeepSeek-v3.2](../sources/prs/vllm/PR-35290.md), [[Quant][Feature] Support online MXFP8 quantization for MoE and dense models](../sources/prs/vllm/PR-35448.md), [[ROCm] Support MLA with nhead<16 and FP8 KV cache for TP=8 (Kimi K2.5/Linear)](../sources/prs/vllm/PR-35850.md), [[Perf] Support FP8 KV cache for Flashinfer MLA Sparse](../sources/prs/vllm/PR-35891.md), [Add support for ModelOpt MXFP8 MoE models](../sources/prs/vllm/PR-35986.md), [[Bugfix] Fix passing of activation_type to trtllm fused MoE NVFP4 and FP8](../sources/prs/vllm/PR-36017.md), [[mla] Support fused FP8/NVFP4 output quantization in MLA attention (#35792)](../sources/prs/vllm/PR-36205.md), [[Perf] Add TRTLLM FP8 MoE Modular Kernel](../sources/prs/vllm/PR-36307.md), [[XPU] Support block fp8 moe by fallback to TritonExpert on XPU](../sources/prs/vllm/PR-36458.md), [[Kernel] Fuse FP8 output quantization into merge_attn_states](../sources/prs/vllm/PR-36518.md), [[Bug][MoE] Strengthen _supports_current_device() checks in the TRTLLM FP8, NVFP4, and FlashInfer CuteDSL MoE experts](../sources/prs/vllm/PR-36728.md), [[Bugfix] Fix KV scales inconsistency in fp8 MLA & FlashInfer kv_cache_dtype "auto" leading to gibberish](../sources/prs/vllm/PR-37054.md), [[XPU] support MLA model on Intel GPU](../sources/prs/vllm/PR-37143.md), [[Perf] Set Flashinfer sparse MLA as default backend for FP8 kv cache](../sources/prs/vllm/PR-37252.md), [[Bugfix] Disable monolithic TRTLLM MoE for Renormalize routing (#37591)](../sources/prs/vllm/PR-37605.md), [[Perf] Use torch compile to fuse pack topk in trtllm moe](../sources/prs/vllm/PR-37695.md), [[Bug] Fix fp8 deepgemm batch invariant](../sources/prs/vllm/PR-37718.md), [[Kernel] Optimize SM120 CUTLASS blockwise FP8 GEMM](../sources/prs/vllm/PR-37970.md), [[Perf] FP8 FlashInfer Attn for ViT](../sources/prs/vllm/PR-38065.md), [[Bugfix] Fix DeepGemm E8M0 accuracy degradation for Qwen3.5 FP8 on Blackwell](../sources/prs/vllm/PR-38083.md), [[Kernel] Add swapAB support for SM120 CUTLASS blockwise FP8 GEMM ](../sources/prs/vllm/PR-38325.md), [[MoE] Add RoutingMethodType.Simulated to TRT-LLM FP8/NVFP4 kernel allowlists](../sources/prs/vllm/PR-38329.md), [[NVIDIA] Bugfix NVFP4 DGX Spark and RTX50](../sources/prs/vllm/PR-38423.md), [[QeRL] Fix online quantized reloading](../sources/prs/vllm/PR-38442.md), [[Kernels][MoE] Fix legacy_routing to use bitmatrix-based routing path](../sources/prs/vllm/PR-38504.md), [[XPU] add xpu backend implementation of mxfp8 quant](../sources/prs/vllm/PR-38682.md), [[Quant] add CompressedTensorsW8A8Mxfp8 for linear and MoE layers](../sources/prs/vllm/PR-38815.md), [[Bugfix] Re-enable Renormalize routing for TRT-LLM MoE experts](../sources/prs/vllm/PR-38859.md), [[Bugfix] Fix broken explicit unquantized kv cache dtype support](../sources/prs/vllm/PR-38922.md), [[MoE Refactor] Split up compressed_tensors_moe.py](../sources/prs/vllm/PR-38960.md), [[Bug] Fix routing bias dtype for trtllm per-block fp8 moe](../sources/prs/vllm/PR-38989.md), [[Perf] Change Trtllm fp8 MoE to use Shuffled Weights and BlockMajorK Layout](../sources/prs/vllm/PR-38993.md), [[Bug] Fix Trtllm Fp8 MoE Weight Shuffle Memory Fragamentation](../sources/prs/vllm/PR-39054.md), [perf(moe): add tuned fused_moe config for RTX PRO 6000 Blackwell Server Edition](../sources/prs/vllm/PR-39183.md), [[Refactor] Move MXFP8 GEMM management into MxFp8LinearKernel](../sources/prs/vllm/PR-39205.md), [[Perf] Fuse Zero Initializer for FP8 DeepGemm Block Quant Kernel](../sources/prs/vllm/PR-39547.md), [add warning when FP8 KV cache misses prefill query quantization](../sources/prs/vllm/PR-39752.md), [[Perf] Batch invariance with Cutlass fp8 support, 28.9% E2E latency improvement](../sources/prs/vllm/PR-40408.md), [[MoE] Move cutlass moe to fused_moe/experts/](../sources/prs/vllm/PR-40574.md), [[Kernel][Helion] Optimize Helion config parsing latency](../sources/prs/vllm/PR-40850.md), [[DSV4] Add BF16 and MXFP8 A2A support for flashinfer a2a one sided](../sources/prs/vllm/PR-40960.md), [Faster per-token fp8 group quant packed kernel for blackwell](../sources/prs/vllm/PR-41326.md), [[Quantization] Rework quantization_config to use QuantKey and allow for activation override](../sources/prs/vllm/PR-41566.md), [[CPU] Add MXFP4 W4A16 MoE support](../sources/prs/vllm/PR-41922.md), [[MoE] Move various experts classes to fused_moe/experts/](../sources/prs/vllm/PR-41979.md), [[Bugfix] Add swiglu limits to deepgemm fp8 methods](../sources/prs/vllm/PR-41986.md), [[feat] Add FP8 per-tensor Q scale support to Triton attention backend](../sources/prs/vllm/PR-42080.md), [[Perf] Use 2D-grid to eliminate divmod in W8W8 group quant](../sources/prs/vllm/PR-42153.md), [[Perf] Wire silu_and_mul_per_block_quant into TritonFP8MoE (MiniMax-M2) ](../sources/prs/vllm/PR-42497.md), [[6/n] Migrate activation kernels, gptq, gguf, non cutlass w8a8 to libtorch stable ABI (continued)](../sources/prs/vllm/PR-42663.md), [[Perf] Add do_not_specialize in fused FP8 RoPE kernel](../sources/prs/vllm/PR-42849.md), [add cutedsl dsv4 indexer fp8 kernel](../sources/prs/vllm/PR-42899.md), [[Model Refactoring] Move deepseek_v4_ops to models/deepseek_v4 [3/N]](../sources/prs/vllm/PR-43073.md), [DeepGEMM — FP8 GEMM with Fine-Grained Scaling](../wiki/kernels/deepgemm.md), [FlashMLA — Multi-head Latent Attention](../wiki/kernels/flashmla.md), [FP8 Fine-Grained-Scale GEMM](../wiki/kernels/fp8-block-scale-gemm.md), [FlashInfer Track A FP8 Block-Scale MoE](../wiki/kernels/fused-moe.md), [Grouped GEMM Contracts for MoE and NVFP4](../wiki/kernels/grouped-gemm.md), [DeepSeek Sparse Attention / Sparse MLA](../wiki/kernels/sparse-mla.md), [Fine-Grained FP8/FP4 Quantization](../wiki/techniques/fine-grained-quantization.md) | | `gdc` | [NVIDIA Blackwell Tuning Guide](../sources/docs/nvidia-blackwell-tuning-guide.md), [NVIDIA CUDA Toolkit 13.x for Blackwell](../sources/docs/nvidia-cuda-13.md), [Blockwise Improvement and Programmatic Dependent Launch](../sources/prs/cutlass/PR-2161.md), [[cute_dsl] add gemm + all reduce (two_shot)](../sources/prs/flashinfer/PR-1695.md), [Programmatic Dependent Launch / Grid Dependency Control](../wiki/hardware/pdl-gdc.md) | -| `mbarrier` | [tcgen05 for dummies](../sources/blogs/tcgen05-tutorial.md), [CUTLASS Cluster Launch Control (CLC) Documentation](../sources/docs/cutlass-clc-documentation.md), [PTX ISA SM100 Instructions Reference](../sources/docs/nvidia-ptx-isa-sm100.md), [Vectorize mbarrier initialization in warpspeed scan](../sources/prs/cccl/PR-8423.md), [A Blackwell-optimized version of selective_state_update (decode)](../sources/prs/flashinfer/PR-2387.md), [[Feature] Support cluster launch, query, synchronization and barrier operations](../sources/prs/tilelang/PR-1874.md), [[Feature] 2-SM support for TMA, TMEM and TCGEN5MMA on Blackwell](../sources/prs/tilelang/PR-1882.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [mbarrier (Memory Barrier Primitives)](../wiki/hardware/mbarrier.md), [tcgen05.mma — Blackwell MMA Instruction](../wiki/hardware/tcgen05-mma.md), [Tensor Memory Accelerator (TMA)](../wiki/hardware/tma.md), [PTX Instructions for SM100](../wiki/languages/ptx-sm100.md), [Pipeline Stalls](../wiki/patterns/pipeline-stalls.md), [Software Pipelining and Multi-Stage Buffering](../wiki/techniques/pipeline-stages.md) | -| `nvfp4` | [Twelve Attempts at NVFP4 Batched GEMV](../sources/blogs/amandeep-nvfp4-attempts.md), [Anatomy of a Reward Hack](../sources/blogs/gpu-mode-reward-hack.md), [NVFP4 Format Details](../sources/blogs/nvfp4-format-details.md), [NVFP4 GEMV and Improved NVFP4 GEMV](../sources/blogs/simon-nvfp4-gemv.md), [TFLOPS Gap: Why FP4 MoE Kernel Engineering Matters on Blackwell](../sources/blogs/tflops-gap-fp4-moe.md), [Tilus: A Tile-Level GPGPU Programming Language for Low-Precision Computation](../sources/blogs/tilus-nvidia.md), [Blackwell NVFP4 Kernel Hackathon Journey](../sources/blogs/yue-nvfp4-hackathon.md), [GPU Mode NVFP4 Hackathon - Problem 1: Batched GEMV](../sources/contests/gpu-mode-nvfp4/problem-1-gemv.md), [GPU Mode NVFP4 Hackathon - Problem 2: NVFP4 GEMM](../sources/contests/gpu-mode-nvfp4/problem-2-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 3: Gated Dual GEMM](../sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 4: Grouped GEMM](../sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [NVIDIA Blackwell Tuning Guide](../sources/docs/nvidia-blackwell-tuning-guide.md), [NVIDIA CUDA Toolkit 13.x for Blackwell](../sources/docs/nvidia-cuda-13.md), [NVIDIA CUTLASS 4.x Blackwell Support](../sources/docs/nvidia-cutlass-blackwell.md), [PTX ISA SM100 Instructions Reference](../sources/docs/nvidia-ptx-isa-sm100.md), [Triton 3.6.0 Release Notes — Blackwell (SM100) Lowering](../sources/docs/triton-3.6-blackwell.md), [[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes](../sources/prs/DeepGEMM/PR-304.md), [[TRTLLM-9661][chore] Further reduce tuning time for cuteDSL nvFP4 dense gemm.](../sources/prs/TensorRT-LLM/PR-10339.md), [[https://nvbugs/5799917][fix] Recover from CUTLASS MoE doActivation perf regression for MXFP4/NVFP4 dtype](../sources/prs/TensorRT-LLM/PR-11165.md), [[None][feat] Optimize super-v3 nvfp4 for better perf](../sources/prs/TensorRT-LLM/PR-11273.md), [[None][feat] Optimize by fuse nvfp4_quant to layernorm_gated for mamba2_mixer](../sources/prs/TensorRT-LLM/PR-11473.md), [[TRTLLM-10004][feat] Enable GEMM -> AR with GEMM output in registered buffers](../sources/prs/TensorRT-LLM/PR-11589.md), [[https://nvbugs/5799917][fix] Recover from CUTLASS MoE doActivation perf regression for MXFP4/NVFP4 dtype](../sources/prs/TensorRT-LLM/PR-11733.md), [[None][fix] Fix SM120 issue for rms_norm with nvfp4_quant_fusion](../sources/prs/TensorRT-LLM/PR-11774.md), [[None][feat] Support update weight for nvfp4](../sources/prs/TensorRT-LLM/PR-12320.md), [[None][fix] Add SPDX Apache-2.0 headers to auto_deploy test files](../sources/prs/TensorRT-LLM/PR-14193.md), [feat: Add w4a8_mxfp4_fp8 quantization recipe.](../sources/prs/TensorRT-LLM/PR-4867.md), [[OMNIML-2336][feat] Add NVFP4 x FP8](../sources/prs/TensorRT-LLM/PR-6809.md), [[TRTLLM-8535][feat] Support DeepSeek V3.2 with FP8 + BF16 KV cache/NVFP4 + BF16 KV cache](../sources/prs/TensorRT-LLM/PR-8405.md), [[None][feat] Enable nvfp4 cuda core for sm120](../sources/prs/TensorRT-LLM/PR-8620.md), [Blockwise and Groupwise GEMM for Blackwell and Improvements for Hopper](../sources/prs/cutlass/PR-2139.md), [Feature/sm100 low latency nvfp4 kernels](../sources/prs/flashinfer/PR-1214.md), [feat: support output nvfp4 in trtllm-gen function call.](../sources/prs/flashinfer/PR-1318.md), [Update autotune results for the nvfp4 cutlass moe backends for v0.2.9](../sources/prs/flashinfer/PR-1361.md), [Faster weight processing (moe nvfp4)](../sources/prs/flashinfer/PR-1412.md), [Fix TRTLLM NVFP4-out attention kernel scale factor dim issue](../sources/prs/flashinfer/PR-1460.md), [Add GeGLU support to trtllm-gen NVFP4 Fused MoE Kernel](../sources/prs/flashinfer/PR-1525.md), [perf: Enable SplitK and fix tile-scheduling for moe fp4 fused moe](../sources/prs/flashinfer/PR-1548.md), [Masked batch nvfp4 quantization](../sources/prs/flashinfer/PR-1774.md), [silu_and_mul nvfp4 quanization fusion rework](../sources/prs/flashinfer/PR-1927.md), [Feature: Support non-gated activation in cutlass fused MoE nvfp4](../sources/prs/flashinfer/PR-2011.md), [[performance]optimize for nvfp4](../sources/prs/flashinfer/PR-2268.md), [[Perf][Feature] Add SM103-specific schedulers for NVFP4 CUTLASS kernels](../sources/prs/flashinfer/PR-2303.md), [feat: Support Fused MoE non gated Relu2 NVFP4 & FP8 and support Nemotron](../sources/prs/flashinfer/PR-2304.md), [feat: Support Fused MoE non gated Relu2 NVFP4 & FP8 and support Nemotron, fixed](../sources/prs/flashinfer/PR-2462.md), [Support NVFP4 KV cache decode on SM120](../sources/prs/flashinfer/PR-2520.md), [tests: add bias testing to nvfp4 moe](../sources/prs/flashinfer/PR-2585.md), [fix: cute dsl nvfp4 moe routing index error](../sources/prs/flashinfer/PR-2629.md), [perf: Update trtllm-gen batched GEMM kernels - faster, more NVFP4 tile dims, MXFP8 with relu2 act](../sources/prs/flashinfer/PR-2667.md), [Add NVFP4 KV cache quantization support for SM100](../sources/prs/flashinfer/PR-2702.md), [fix: Add SM120 (RTX Blackwell desktop) support for NVFP4 MoE kernels](../sources/prs/flashinfer/PR-2725.md), [Support for MXFP4 and NVFP4 group GEMMs on GeForce and Spark](../sources/prs/flashinfer/PR-2738.md), [feat: Add CuTe-DSL backend for NVFP4 quantization](../sources/prs/flashinfer/PR-2838.md), [perf: Optimize CuTe-DSL fp4 and fp8 quantization kernels](../sources/prs/flashinfer/PR-2904.md), [[Fmha] support nvfp4 output keepsMmaAb generation kernels](../sources/prs/flashinfer/PR-2988.md), [[feat] Trtllm-gen Per-token Nvfp4 MoE](../sources/prs/flashinfer/PR-3027.md), [Support NVFP4 KV for prefill and batch attention kernels](../sources/prs/flashinfer/PR-3097.md), [Integrate CUTLASS Small Tile N Blockscaled GEMMs/Grouped GEMMs for SM120 and SM121](../sources/prs/flashinfer/PR-3152.md), [feat(trace): embed runnable init() in every TraceTemplate](../sources/prs/flashinfer/PR-3221.md), [perf: optimize per-token nvfp4 quantization kernel.](../sources/prs/flashinfer/PR-3237.md), [Optimize nvfp4 block scaled gemm kernel when M is small.](../sources/prs/sglang/PR-10101.md), [Fix chunked prefix cache for nvfp4](../sources/prs/sglang/PR-10180.md), [Fix correction bias undefined behavior for nvfp4 models](../sources/prs/sglang/PR-10426.md), [Fix MTP MoE weight loading with NVFP4 target model.](../sources/prs/sglang/PR-10758.md), [[NVIDIA] Add new SMs support for Spark & Thor](../sources/prs/sglang/PR-11287.md), [support cutlass fp4 kernel in sm120](../sources/prs/sglang/PR-11737.md), [Support nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8/NVFP4](../sources/prs/sglang/PR-11866.md), [Feature/nano v2 offline modelopt fp8 and nvfp4](../sources/prs/sglang/PR-12018.md), [[NVIDIA] Fix CUDA arch requirement in nvfp4 cast](../sources/prs/sglang/PR-12581.md), [ignore the deepgemm check when the model weight with nvfp4 and moe ba…](../sources/prs/sglang/PR-12782.md), [support mtp with deepseek r1 nvfp4 model](../sources/prs/sglang/PR-13115.md), [Fix nan in global scaling factor for large scale nvfp4 EP](../sources/prs/sglang/PR-13162.md), [[Feat][NVFP4] Enable NVFP4 MoE for Qwen series models (eg. Qwen3-Next) #13761](../sources/prs/sglang/PR-13761.md), [Mistral Large 3 NVFP4 support](../sources/prs/sglang/PR-14485.md), [Mistral Large 3 NVFP4 TRTLLM MoE support](../sources/prs/sglang/PR-15049.md), [[NVIDIA] Fixes for NVFP4 all-gather with spec decoding](../sources/prs/sglang/PR-15280.md), [Tiny fix cannot launch nvfp4 checkpoint with bf16 kv cache](../sources/prs/sglang/PR-15986.md), [Inclusion of nvfp4 blockscale in EPLB Rebalance](../sources/prs/sglang/PR-17158.md), [[Fix] GLM 4.7 + NVFP4 + MTP](../sources/prs/sglang/PR-17166.md), [[feat] Support nvfp4 quantized model of Qwen3-Next](../sources/prs/sglang/PR-17627.md), [[Bugfix] Fix Mistral Large 3 NVFP4 TRTLLM MoE](../sources/prs/sglang/PR-18065.md), [Fix nvfp4 weight update](../sources/prs/sglang/PR-18085.md), [[ModelOpt] Fix broken Qwen3-235B-A22B-Instruct-2507-NVFP4 launch](../sources/prs/sglang/PR-18189.md), [[ModelOPT] Support Qwen 3 Next Coder NVFP4](../sources/prs/sglang/PR-18224.md), [[Kimi-K2.5] Fix NVFP4 Kimi-K2.5 weight mapping and exclude list](../sources/prs/sglang/PR-18370.md), [Nsa trtllm mla sparse fp8 support with Deepseek v3.2 NVFP4](../sources/prs/sglang/PR-18389.md), [[Qwen3.5] Enable nvfp4 checkpoint](../sources/prs/sglang/PR-18937.md), [feat: Support MXFP4 quantized dense models on AMD CDNA2/CDNA3 GPUs](../sources/prs/sglang/PR-19143.md), [Fix nightly Mistral-Large-3 NVFP4 accuracy threshold](../sources/prs/sglang/PR-19402.md), [[Kernel Slimming] Migrate NVFP4 kernels to JIT](../sources/prs/sglang/PR-19437.md), [[Feature] NVFP4 Marlin fallback for non-Blackwell GPUs (SM75+)](../sources/prs/sglang/PR-19652.md), [[JIT Kernel] Reland NVFP4 kernels to JIT](../sources/prs/sglang/PR-20012.md), [[V32/GLM5] Change default setting of V32 nvfp4 on TP4](../sources/prs/sglang/PR-20086.md), [[diffusion] Support nvfp4 for Flux.2](../sources/prs/sglang/PR-20137.md), [[4/n jit_kernel restruct] speed up CI tests and add benchmark workflow](../sources/prs/sglang/PR-20268.md), [[Model] Support Nemotron 3 Super NVFP4](../sources/prs/sglang/PR-20407.md), [[JIT Kernel] Fix NVFP4 multi-arch compilation failure](../sources/prs/sglang/PR-20874.md), [Add SGLang CUDA crash API logging inspired by FlashInfer](../sources/prs/sglang/PR-20910.md), [[Chore] Clean up JIT compilation flags](../sources/prs/sglang/PR-21022.md), [Refactor JIT kernel CI to use run_suite.py registration system](../sources/prs/sglang/PR-21239.md), [CUTLASS NVFP4 GEMM improvement of SM120](../sources/prs/sglang/PR-21314.md), [[misc] clean up kernel API](../sources/prs/sglang/PR-21325.md), [[Diffusion] Fix weight scale swizzle and add large-M kernel config for FLUX.2-dev-NVFP4](../sources/prs/sglang/PR-22064.md), [[nvidia] Gemma4 nvfp4 fix](../sources/prs/sglang/PR-22079.md), [[diffusion] Default NVFP4 to CUTLASS and add all-model shape benchmarks](../sources/prs/sglang/PR-22091.md), [[Diffusion] Add diffusion NVFP4 scaled-mm correctness test](../sources/prs/sglang/PR-22127.md), [[RL] Refactor NVFP4 shuffling/swizzling to in-place replacement](../sources/prs/sglang/PR-22204.md), [[Diffusion] Add FLUX.1-dev ModelOpt NVFP4 support](../sources/prs/sglang/PR-22574.md), [reland [Diffusion] Add FLUX.1-dev ModelOpt NVFP4 support](../sources/prs/sglang/PR-22672.md), [[Diffusion] Add Wan2.2 ModelOpt NVFP4 support](../sources/prs/sglang/PR-22681.md), [[FlashInfer v0.6.11] [RL] Support FlashInfer per-token NVFP4 MoE](../sources/prs/sglang/PR-22918.md), [Reland Cute-DSL FP4 dense GEMM](../sources/prs/sglang/PR-23590.md), [Use Cute-DSL NVFP4 quantization kernels](../sources/prs/sglang/PR-23745.md), [perf(nvfp4): free unused source scales after weight processing](../sources/prs/sglang/PR-25107.md), [[codex] Update Wan2.2 ModelOpt CI checkpoints](../sources/prs/sglang/PR-25483.md), [Support FP4 gemm (1/2)](../sources/prs/sglang/PR-3899.md), [[Build] Fix cuda12.8 build error in nvfp4_scaled_mm_kernels.cu](../sources/prs/sglang/PR-4953.md), [[1/2] Add Kernel support for Cutlass based Fused FP4 MoE](../sources/prs/sglang/PR-6093.md), [Support NVFP4 quantized dense models on AMD CDNA2/CDNA3 GPUs](../sources/prs/sglang/PR-7302.md), [FlashInfer NVFP4 MoE with EP & 2-stream shared expert](../sources/prs/sglang/PR-7327.md), [Qwen FP8/NVFP4 ModelOPT Quantization support](../sources/prs/sglang/PR-7912.md), [[Fix][Ready]Fix register spilling in cutlass nvfp4 gemm kernel on Blackwell](../sources/prs/sglang/PR-8127.md), [[NVIDIA] Add Low Latency NVFP4 decode kernels from Flashinfer](../sources/prs/sglang/PR-8552.md), [Faster weight processing (trtllm-gen moe nvfp4)](../sources/prs/sglang/PR-9162.md), [[NVIDIA] [3/N] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked ](../sources/prs/sglang/PR-9199.md), [[NVIDA] [1/N] Nvfp4 Masked Gemm: Add quant op for the flashinfer grouped gemm](../sources/prs/sglang/PR-9200.md), [[NVIDIA] [2/N] Optimize `silu_and_mul_scaled_fp4_grouped_quant` perf](../sources/prs/sglang/PR-9556.md), [Make fp4_quantize kernels work on sm103](../sources/prs/sglang/PR-9807.md), [[NVIDIA] Support nvfp4 quantization](../sources/prs/vllm/PR-12784.md), [[NVIDIA] Support nvfp4 tcgen05 gemm](../sources/prs/vllm/PR-13571.md), [[Hardware/NVIDIA/Kernel] [Functional Enablement] [1/N] Enable nvidia/DeepSeek-R1-FP4 Model](../sources/prs/vllm/PR-16362.md), [[Misc] Add compressed-tensors NVFP4A16 emulation support](../sources/prs/vllm/PR-17914.md), [Use NVFP4 Marlin for CompressedTensorsW4A16Fp4](../sources/prs/vllm/PR-18000.md), [[Quantization] Add compressed-tensors NVFP4 support](../sources/prs/vllm/PR-18312.md), [[Hardware][NVIDIA] FP4 MoE kernel optimization](../sources/prs/vllm/PR-19110.md), [[Hardware][NVIDIA][kernel] Fp4 MOE quant kernel optimization](../sources/prs/vllm/PR-19500.md), [[Quantization] Add compressed-tensors emulations support for NVFP4](../sources/prs/vllm/PR-19879.md), [[Quantization] Add compressed-tensors NVFP4 MoE Support](../sources/prs/vllm/PR-19990.md), [Add ModelOpt Qwen3 nvfp4 support](../sources/prs/vllm/PR-20101.md), [[Bugfix] Fix some narrowing conversion warnings](../sources/prs/vllm/PR-20141.md), [[Kernel][Bugfix] Fixup some warnings in nvfp4_blockwise_moe when CUDA < 12.8](../sources/prs/vllm/PR-20324.md), [Support CUTLASS NVFP4 (w4a4) for Blackwell Geforce GPUs (SM120)](../sources/prs/vllm/PR-21309.md), [Support Tensorrt-LLM MoE fp4 for low-latency](../sources/prs/vllm/PR-21331.md), [Update flashinfer CUTLASS NVFP4 MoE Kernel to use per expert global scaling factor](../sources/prs/vllm/PR-21408.md), [[Bug] Fix Compressed Tensor NVFP4 `cutlass_fp4_group_mm` illegal memory access](../sources/prs/vllm/PR-21465.md), [[Feature] Add Flashinfer MoE Support for Compressed Tensor NVFP4](../sources/prs/vllm/PR-21639.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv NVFP4-out Attention Kernel](../sources/prs/vllm/PR-22703.md), [Fix nvfp4 swizzling](../sources/prs/vllm/PR-23140.md), [fix incompatibililty with non cuda platform for nvfp4](../sources/prs/vllm/PR-23478.md), [[Bugfix] Fix Marlin NVFP4 for modelopt](../sources/prs/vllm/PR-23659.md), [[NVIDIA] Support SiluMul + NVFP4 quant fusion](../sources/prs/vllm/PR-23671.md), [[Kernel][tcgen05] nvfp4 fused tcgen05 moe](../sources/prs/vllm/PR-23696.md), [[Bugfix][Misc] Fix silu_and_mul_nvfp4_quant issue and extract common utils for nvfp4 kernel source files](../sources/prs/vllm/PR-23727.md), [[BUGFIX ] fix undefined silu_and_mul_nvfp4_quant](../sources/prs/vllm/PR-23929.md), [[Transform] [Quantization] Add QuTLASS support to vLLM](../sources/prs/vllm/PR-24440.md), [[Bugfix] Fix accuracy issue for silu_mul + nvfp4 quant fusion kernel](../sources/prs/vllm/PR-24833.md), [[Compile] Fix Compile Warning for Ignoring `MIN_BLOCK_PER_SM`](../sources/prs/vllm/PR-25193.md), [Enable Fbgemm NVFP4 on Dense models](../sources/prs/vllm/PR-25609.md), [[Quantization/NVFP4] Speed up TRTLLM NVFP4 MOE weight loading and fix K/V scale loading for MLA Attn](../sources/prs/vllm/PR-25968.md), [[Bugfix] Allow skipping MoE in NVFP4 (fix for MTP)](../sources/prs/vllm/PR-25987.md), [[MoE] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked](../sources/prs/vllm/PR-25990.md), [[NVIDIA] Add support for cudnn fp4 gemm via flashinfer](../sources/prs/vllm/PR-26107.md), [[ModelOpt] Load w13/w2_input_scale for all experts, nvfp4](../sources/prs/vllm/PR-26135.md), [[Attention] Use sparse prefill kernel for fp8 kv-cache in DeepSeek-v3.2](../sources/prs/vllm/PR-27532.md), [Add TRTLLM MoE NVFP4 kernel to CompressedTensorsW4A4MoeMethod](../sources/prs/vllm/PR-28892.md), [[Feat] Support non-gated activations in NVFP4 modelopt path](../sources/prs/vllm/PR-29004.md), [[Kernel] Add NVFP4 MoE CUTLASS support for SM120](../sources/prs/vllm/PR-29242.md), [[Bugfix] Fix mismatched nvfp4 gemm output shape](../sources/prs/vllm/PR-29742.md), [[EPLB] Support EPLB w/ NVFP4](../sources/prs/vllm/PR-29804.md), [[Compressed-Tensors] Simplify NVFP4 Conditions, enable marlin support for NVFP4A16 MoEs](../sources/prs/vllm/PR-30881.md), [[Kernel][Performance] Enable smaller Scaling Factor tiling for NVFP4 small-batch decoding](../sources/prs/vllm/PR-30885.md), [[NVFP4][Perf] Tune NVFP4 input quant kernel for small batch size](../sources/prs/vllm/PR-30897.md), [ [FIX] Always support TP > 4 for FP4 Gemm](../sources/prs/vllm/PR-31099.md), [[Bugfix] Fix Broken ModelOpt NVFP4 MoE](../sources/prs/vllm/PR-31742.md), [[Perf] Fuse stride preparation for NVFP4 cutlass_moe](../sources/prs/vllm/PR-31837.md), [[Perf][Kernel] Optimize FP4 quantization kernels (SM100F)](../sources/prs/vllm/PR-32520.md), [Support compress-tensors with nvfp4 or fp8 weights and modelopt with nvfp4 weights on Turing](../sources/prs/vllm/PR-33076.md), [fix: Add SM120 (RTX Blackwell) support for FlashInfer CUTLASS NVFP4 MoE kernels](../sources/prs/vllm/PR-33417.md), [[Kernel] Support Flashinfer trtllm fused MoE non gated FP8 & NVFP4](../sources/prs/vllm/PR-33506.md), [[Bugfix] Fix DSV3.2 NVFP4](../sources/prs/vllm/PR-33932.md), [[ModelBash][DSR1 NVFp4] Avoid Bf16 Bias Cast](../sources/prs/vllm/PR-34298.md), [[Custom Ops] Add functional + out variant for scaled_fp4_quant](../sources/prs/vllm/PR-34389.md), [[BUGFIX] Fix accuracy regression for NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4 with TP>1](../sources/prs/vllm/PR-34476.md), [[Model] Add NVFP4 quantization support for Step3.5-Flash](../sources/prs/vllm/PR-34478.md), [[Bugfix] Rescale NVFP4 weight scales to fix BF16 dequant underflow](../sources/prs/vllm/PR-34577.md), [[Bugfix] Fix NVFP4 TRTLLM MoE non-gated support; add gsm8k for Nemotron-3-Nano FP8+NVFP4](../sources/prs/vllm/PR-34725.md), [[BugFix] Fix fp4 quant kernel on CUDA 12.8](../sources/prs/vllm/PR-35210.md), [[NVFP4] Support NVFP4 dense models from `modelopt` and `compressed-tensors` on AMD Instinct MI300, MI355X and Hopper through emulation](../sources/prs/vllm/PR-35733.md), [[Bugfix] Fix passing of activation_type to trtllm fused MoE NVFP4 and FP8](../sources/prs/vllm/PR-36017.md), [[Mamba] Flashinfer selective_state_update](../sources/prs/vllm/PR-36162.md), [[mla] Support fused FP8/NVFP4 output quantization in MLA attention (#35792)](../sources/prs/vllm/PR-36205.md), [[Bug][MoE] Fix TRTLLM NVFP4 Routing Kernel Precision](../sources/prs/vllm/PR-36725.md), [[Bug][MoE] Strengthen _supports_current_device() checks in the TRTLLM FP8, NVFP4, and FlashInfer CuteDSL MoE experts](../sources/prs/vllm/PR-36728.md), [[MoE Refactor] Mxfp4 oracle rebased](../sources/prs/vllm/PR-37128.md), [Fix minimax m2.5 nvfp4 kv scales weight loading](../sources/prs/vllm/PR-37214.md), [[MoE/EPLB] Fix FlashInfer nvfp4 experts + EPLB correctness](../sources/prs/vllm/PR-37217.md), [[Kernel] Add non-gated support for NVFP4 CUTLASS MoE](../sources/prs/vllm/PR-37320.md), [[Bugfix] Fix EP weight filter breaking EPLB and NVFP4 accuracy](../sources/prs/vllm/PR-37322.md), [Add nvfp4 support to reshape_and_cache_flash](../sources/prs/vllm/PR-37332.md), [[Bugfix] Remove assertion for NVFP4 scale dynamic range](../sources/prs/vllm/PR-37465.md), [[Bugfix] Fix marlin nvfp4 rescaling](../sources/prs/vllm/PR-37502.md), [[4/n] Migrate FP4/W4A8 CUTLASS kernels to torch stable ABI](../sources/prs/vllm/PR-37503.md), [[Perf] Use torch compile to fuse pack topk in trtllm moe](../sources/prs/vllm/PR-37695.md), [[Bugfix] Preserve CUDA arch suffix (a/f) for SM12x — fixes NVFP4 NaN on desktop Blackwell](../sources/prs/vllm/PR-37725.md), [[MoE] Move FlashInfer CuteDSL experts into fused_moe/experts/](../sources/prs/vllm/PR-37759.md), [[MoE Kernel] Flashinfer nvfp4 cutedsl moe kernel integration](../sources/prs/vllm/PR-38050.md), [[Bugfix] Fix DeepGemm E8M0 accuracy degradation for Qwen3.5 FP8 on Blackwell](../sources/prs/vllm/PR-38083.md), [[Quantization] Add FlashInfer CuteDSL batched experts backend for NVFP4 MoE](../sources/prs/vllm/PR-38251.md), [[MoE] Add RoutingMethodType.Simulated to TRT-LLM FP8/NVFP4 kernel allowlists](../sources/prs/vllm/PR-38329.md), [[NVIDIA] Bugfix NVFP4 DGX Spark and RTX50](../sources/prs/vllm/PR-38423.md), [[Compile] Fix nvfp4 compile warning](../sources/prs/vllm/PR-38573.md), [[Bugfix] Fix NVFP4+MTP crash: force unquantized mtp.fc for Qwen3.5](../sources/prs/vllm/PR-38832.md), [[MoE Refactor] Split up compressed_tensors_moe.py](../sources/prs/vllm/PR-38960.md), [[Refactor] Move NVFP4 GEMM management into NvFp4LinearKernel](../sources/prs/vllm/PR-39129.md), [[Feature] Batch invariant nvfp4 linear support](../sources/prs/vllm/PR-39322.md), [[Kernel] Support TRTLLM GEN NVFP4 MoE for non-512-aligned hidden dims via weight padding](../sources/prs/vllm/PR-39510.md), [[Bugfix] Reject non-nvfp4 dtypes when using the flashinfer_nvlink_one_sided all2all backend](../sources/prs/vllm/PR-39717.md), [[Bug] Fix batch invariance nvfp4 support](../sources/prs/vllm/PR-39820.md), [Add nvfp4 kv cache support](../sources/prs/vllm/PR-40177.md), [[Bugfix] Guard mxfp4_experts_quant bindings on ENABLE_NVFP4_SM100](../sources/prs/vllm/PR-40191.md), [[MoE] Move cutlass moe to fused_moe/experts/](../sources/prs/vllm/PR-40574.md), [[Kernel][MoE] Support GELU on TRT-LLM NvFP4 fused MoE for Gemma4](../sources/prs/vllm/PR-41050.md), [Add NVFP4 all-gather GEMM fusion for AsyncTP](../sources/prs/vllm/PR-41882.md), [[MoE] Move various experts classes to fused_moe/experts/](../sources/prs/vllm/PR-41979.md), [[Perf] Padded nvfp4 quant kernel to remove additional copy, 2.4%~5.7% e2e performance improvement](../sources/prs/vllm/PR-42774.md), [NVFP4 and Block-Scaled Narrow Precision](../wiki/hardware/nvfp4.md), [Gated Dual GEMM (Gate-Up + SwiGLU Fusion)](../wiki/kernels/gated-dual-gemm.md), [Grouped GEMM for MoE](../wiki/kernels/grouped-gemm.md), [NVFP4 GEMM — 4-bit Floating Point Matrix Multiply](../wiki/kernels/nvfp4-gemm.md), [NVFP4 Batched GEMV](../wiki/kernels/nvfp4-gemv.md), [PTX Instructions for SM100](../wiki/languages/ptx-sm100.md), [Fine-Grained FP8/FP4 Quantization](../wiki/techniques/fine-grained-quantization.md) | +| `ldmatrix` | [simveit load_and_store](../sources/blogs/simveit-load-and-store.md) | +| `mbarrier` | [tcgen05 for dummies](../sources/blogs/tcgen05-tutorial.md), [CUTLASS 4.5.0 Cluster Launch Control Documentation](../sources/docs/cutlass-clc-documentation.md), [CUDA 13.0.2 TMA Documentation](../sources/docs/nvidia-cuda-13-0-2-tma.md), [PTX ISA 9.0 SM100 Instruction Reference](../sources/docs/nvidia-ptx-isa-sm100.md), [Vectorize mbarrier initialization in warpspeed scan](../sources/prs/cccl/PR-8423.md), [A Blackwell-optimized version of selective_state_update (decode)](../sources/prs/flashinfer/PR-2387.md), [[Feature] Support cluster launch, query, synchronization and barrier operations](../sources/prs/tilelang/PR-1874.md), [[Feature] 2-SM support for TMA, TMEM and TCGEN5MMA on Blackwell](../sources/prs/tilelang/PR-1882.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [mbarrier (Memory Barrier Primitives)](../wiki/hardware/mbarrier.md), [tcgen05.mma — Blackwell MMA Instruction](../wiki/hardware/tcgen05-mma.md), [Tensor Memory Accelerator (TMA)](../wiki/hardware/tma.md), [PTX Instructions for SM100](../wiki/languages/ptx-sm100.md), [Pipeline Stalls](../wiki/patterns/pipeline-stalls.md), [Software Pipelining and Multi-Stage Buffering](../wiki/techniques/pipeline-stages.md) | +| `nvfp4` | [Twelve Attempts at an FP4 Kernel](../sources/blogs/amandeep-nvfp4-attempts.md), [Anatomy of a Reward Hack](../sources/blogs/gpu-mode-reward-hack.md), [NVFP4 Format Details](../sources/blogs/nvfp4-format-details.md), [NVFP4 GEMV and Improved NVFP4 GEMV](../sources/blogs/simon-nvfp4-gemv.md), [TFLOPS Gap: Why FP4 MoE Kernel Engineering Matters on Blackwell](../sources/blogs/tflops-gap-fp4-moe.md), [Tilus: A Tile-Level GPGPU Programming Language for Low-Precision Computation](../sources/blogs/tilus-nvidia.md), [Blackwell NVFP4 Kernel Hackathon Journey](../sources/blogs/yue-nvfp4-hackathon.md), [GPU Mode NVFP4 Hackathon - Problem 1: Batched GEMV](../sources/contests/gpu-mode-nvfp4/problem-1-gemv.md), [GPU Mode NVFP4 Hackathon - Problem 2: NVFP4 GEMM](../sources/contests/gpu-mode-nvfp4/problem-2-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 3: Gated Dual GEMM](../sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 4: Grouped GEMM](../sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [NVIDIA Blackwell Tuning Guide](../sources/docs/nvidia-blackwell-tuning-guide.md), [NVIDIA CUDA Toolkit 13.x for Blackwell](../sources/docs/nvidia-cuda-13.md), [NVIDIA CUTLASS 4.5.0 Blackwell Sources](../sources/docs/nvidia-cutlass-blackwell.md), [PTX ISA 9.0 SM100 Instruction Reference](../sources/docs/nvidia-ptx-isa-sm100.md), [Transformer Engine 2.13: NVFP4](../sources/docs/nvidia-transformer-engine-2.13-nvfp4.md), [[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes](../sources/prs/DeepGEMM/PR-304.md), [[TRTLLM-9661][chore] Further reduce tuning time for cuteDSL nvFP4 dense gemm.](../sources/prs/TensorRT-LLM/PR-10339.md), [[https://nvbugs/5799917][fix] Recover from CUTLASS MoE doActivation perf regression for MXFP4/NVFP4 dtype](../sources/prs/TensorRT-LLM/PR-11165.md), [[None][feat] Optimize super-v3 nvfp4 for better perf](../sources/prs/TensorRT-LLM/PR-11273.md), [[None][feat] Optimize by fuse nvfp4_quant to layernorm_gated for mamba2_mixer](../sources/prs/TensorRT-LLM/PR-11473.md), [[TRTLLM-10004][feat] Enable GEMM -> AR with GEMM output in registered buffers](../sources/prs/TensorRT-LLM/PR-11589.md), [[https://nvbugs/5799917][fix] Recover from CUTLASS MoE doActivation perf regression for MXFP4/NVFP4 dtype](../sources/prs/TensorRT-LLM/PR-11733.md), [[None][fix] Fix SM120 issue for rms_norm with nvfp4_quant_fusion](../sources/prs/TensorRT-LLM/PR-11774.md), [[None][feat] Support update weight for nvfp4](../sources/prs/TensorRT-LLM/PR-12320.md), [[None][fix] Add SPDX Apache-2.0 headers to auto_deploy test files](../sources/prs/TensorRT-LLM/PR-14193.md), [feat: Add w4a8_mxfp4_fp8 quantization recipe.](../sources/prs/TensorRT-LLM/PR-4867.md), [[OMNIML-2336][feat] Add NVFP4 x FP8](../sources/prs/TensorRT-LLM/PR-6809.md), [[TRTLLM-8535][feat] Support DeepSeek V3.2 with FP8 + BF16 KV cache/NVFP4 + BF16 KV cache](../sources/prs/TensorRT-LLM/PR-8405.md), [[None][feat] Enable nvfp4 cuda core for sm120](../sources/prs/TensorRT-LLM/PR-8620.md), [Feature/sm100 low latency nvfp4 kernels](../sources/prs/flashinfer/PR-1214.md), [feat: support output nvfp4 in trtllm-gen function call.](../sources/prs/flashinfer/PR-1318.md), [Update autotune results for the nvfp4 cutlass moe backends for v0.2.9](../sources/prs/flashinfer/PR-1361.md), [Faster weight processing (moe nvfp4)](../sources/prs/flashinfer/PR-1412.md), [Fix TRTLLM NVFP4-out attention kernel scale factor dim issue](../sources/prs/flashinfer/PR-1460.md), [Add GeGLU support to trtllm-gen NVFP4 Fused MoE Kernel](../sources/prs/flashinfer/PR-1525.md), [perf: Enable SplitK and fix tile-scheduling for moe fp4 fused moe](../sources/prs/flashinfer/PR-1548.md), [Masked batch nvfp4 quantization](../sources/prs/flashinfer/PR-1774.md), [silu_and_mul nvfp4 quanization fusion rework](../sources/prs/flashinfer/PR-1927.md), [Feature: Support non-gated activation in cutlass fused MoE nvfp4](../sources/prs/flashinfer/PR-2011.md), [[performance]optimize for nvfp4](../sources/prs/flashinfer/PR-2268.md), [[Perf][Feature] Add SM103-specific schedulers for NVFP4 CUTLASS kernels](../sources/prs/flashinfer/PR-2303.md), [feat: Support Fused MoE non gated Relu2 NVFP4 & FP8 and support Nemotron](../sources/prs/flashinfer/PR-2304.md), [feat: Support Fused MoE non gated Relu2 NVFP4 & FP8 and support Nemotron, fixed](../sources/prs/flashinfer/PR-2462.md), [Support NVFP4 KV cache decode on SM120](../sources/prs/flashinfer/PR-2520.md), [tests: add bias testing to nvfp4 moe](../sources/prs/flashinfer/PR-2585.md), [fix: cute dsl nvfp4 moe routing index error](../sources/prs/flashinfer/PR-2629.md), [perf: Update trtllm-gen batched GEMM kernels - faster, more NVFP4 tile dims, MXFP8 with relu2 act](../sources/prs/flashinfer/PR-2667.md), [Add NVFP4 KV cache quantization support for SM100](../sources/prs/flashinfer/PR-2702.md), [fix: Add SM120 (RTX Blackwell desktop) support for NVFP4 MoE kernels](../sources/prs/flashinfer/PR-2725.md), [Support for MXFP4 and NVFP4 group GEMMs on GeForce and Spark](../sources/prs/flashinfer/PR-2738.md), [feat: Add CuTe-DSL backend for NVFP4 quantization](../sources/prs/flashinfer/PR-2838.md), [perf: Optimize CuTe-DSL fp4 and fp8 quantization kernels](../sources/prs/flashinfer/PR-2904.md), [[Fmha] support nvfp4 output keepsMmaAb generation kernels](../sources/prs/flashinfer/PR-2988.md), [[feat] Trtllm-gen Per-token Nvfp4 MoE](../sources/prs/flashinfer/PR-3027.md), [Support NVFP4 KV for prefill and batch attention kernels](../sources/prs/flashinfer/PR-3097.md), [Integrate CUTLASS Small Tile N Blockscaled GEMMs/Grouped GEMMs for SM120 and SM121](../sources/prs/flashinfer/PR-3152.md), [feat(trace): embed runnable init() in every TraceTemplate](../sources/prs/flashinfer/PR-3221.md), [perf: optimize per-token nvfp4 quantization kernel.](../sources/prs/flashinfer/PR-3237.md), [Optimize nvfp4 block scaled gemm kernel when M is small.](../sources/prs/sglang/PR-10101.md), [Fix chunked prefix cache for nvfp4](../sources/prs/sglang/PR-10180.md), [Fix correction bias undefined behavior for nvfp4 models](../sources/prs/sglang/PR-10426.md), [Fix MTP MoE weight loading with NVFP4 target model.](../sources/prs/sglang/PR-10758.md), [[NVIDIA] Add new SMs support for Spark & Thor](../sources/prs/sglang/PR-11287.md), [support cutlass fp4 kernel in sm120](../sources/prs/sglang/PR-11737.md), [Support nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8/NVFP4](../sources/prs/sglang/PR-11866.md), [Feature/nano v2 offline modelopt fp8 and nvfp4](../sources/prs/sglang/PR-12018.md), [[NVIDIA] Fix CUDA arch requirement in nvfp4 cast](../sources/prs/sglang/PR-12581.md), [ignore the deepgemm check when the model weight with nvfp4 and moe ba…](../sources/prs/sglang/PR-12782.md), [support mtp with deepseek r1 nvfp4 model](../sources/prs/sglang/PR-13115.md), [Fix nan in global scaling factor for large scale nvfp4 EP](../sources/prs/sglang/PR-13162.md), [[Feat][NVFP4] Enable NVFP4 MoE for Qwen series models (eg. Qwen3-Next) #13761](../sources/prs/sglang/PR-13761.md), [Mistral Large 3 NVFP4 support](../sources/prs/sglang/PR-14485.md), [Mistral Large 3 NVFP4 TRTLLM MoE support](../sources/prs/sglang/PR-15049.md), [[NVIDIA] Fixes for NVFP4 all-gather with spec decoding](../sources/prs/sglang/PR-15280.md), [Tiny fix cannot launch nvfp4 checkpoint with bf16 kv cache](../sources/prs/sglang/PR-15986.md), [Inclusion of nvfp4 blockscale in EPLB Rebalance](../sources/prs/sglang/PR-17158.md), [[Fix] GLM 4.7 + NVFP4 + MTP](../sources/prs/sglang/PR-17166.md), [[feat] Support nvfp4 quantized model of Qwen3-Next](../sources/prs/sglang/PR-17627.md), [[Bugfix] Fix Mistral Large 3 NVFP4 TRTLLM MoE](../sources/prs/sglang/PR-18065.md), [Fix nvfp4 weight update](../sources/prs/sglang/PR-18085.md), [[ModelOpt] Fix broken Qwen3-235B-A22B-Instruct-2507-NVFP4 launch](../sources/prs/sglang/PR-18189.md), [[ModelOPT] Support Qwen 3 Next Coder NVFP4](../sources/prs/sglang/PR-18224.md), [[Kimi-K2.5] Fix NVFP4 Kimi-K2.5 weight mapping and exclude list](../sources/prs/sglang/PR-18370.md), [Nsa trtllm mla sparse fp8 support with Deepseek v3.2 NVFP4](../sources/prs/sglang/PR-18389.md), [[Qwen3.5] Enable nvfp4 checkpoint](../sources/prs/sglang/PR-18937.md), [feat: Support MXFP4 quantized dense models on AMD CDNA2/CDNA3 GPUs](../sources/prs/sglang/PR-19143.md), [Fix nightly Mistral-Large-3 NVFP4 accuracy threshold](../sources/prs/sglang/PR-19402.md), [[Kernel Slimming] Migrate NVFP4 kernels to JIT](../sources/prs/sglang/PR-19437.md), [[Feature] NVFP4 Marlin fallback for non-Blackwell GPUs (SM75+)](../sources/prs/sglang/PR-19652.md), [[JIT Kernel] Reland NVFP4 kernels to JIT](../sources/prs/sglang/PR-20012.md), [[V32/GLM5] Change default setting of V32 nvfp4 on TP4](../sources/prs/sglang/PR-20086.md), [[diffusion] Support nvfp4 for Flux.2](../sources/prs/sglang/PR-20137.md), [[4/n jit_kernel restruct] speed up CI tests and add benchmark workflow](../sources/prs/sglang/PR-20268.md), [[Model] Support Nemotron 3 Super NVFP4](../sources/prs/sglang/PR-20407.md), [[JIT Kernel] Fix NVFP4 multi-arch compilation failure](../sources/prs/sglang/PR-20874.md), [Add SGLang CUDA crash API logging inspired by FlashInfer](../sources/prs/sglang/PR-20910.md), [[Chore] Clean up JIT compilation flags](../sources/prs/sglang/PR-21022.md), [Refactor JIT kernel CI to use run_suite.py registration system](../sources/prs/sglang/PR-21239.md), [CUTLASS NVFP4 GEMM improvement of SM120](../sources/prs/sglang/PR-21314.md), [[misc] clean up kernel API](../sources/prs/sglang/PR-21325.md), [[Diffusion] Fix weight scale swizzle and add large-M kernel config for FLUX.2-dev-NVFP4](../sources/prs/sglang/PR-22064.md), [[nvidia] Gemma4 nvfp4 fix](../sources/prs/sglang/PR-22079.md), [[diffusion] Default NVFP4 to CUTLASS and add all-model shape benchmarks](../sources/prs/sglang/PR-22091.md), [[Diffusion] Add diffusion NVFP4 scaled-mm correctness test](../sources/prs/sglang/PR-22127.md), [[RL] Refactor NVFP4 shuffling/swizzling to in-place replacement](../sources/prs/sglang/PR-22204.md), [[Diffusion] Add FLUX.1-dev ModelOpt NVFP4 support](../sources/prs/sglang/PR-22574.md), [reland [Diffusion] Add FLUX.1-dev ModelOpt NVFP4 support](../sources/prs/sglang/PR-22672.md), [[Diffusion] Add Wan2.2 ModelOpt NVFP4 support](../sources/prs/sglang/PR-22681.md), [[FlashInfer v0.6.11] [RL] Support FlashInfer per-token NVFP4 MoE](../sources/prs/sglang/PR-22918.md), [Reland Cute-DSL FP4 dense GEMM](../sources/prs/sglang/PR-23590.md), [Use Cute-DSL NVFP4 quantization kernels](../sources/prs/sglang/PR-23745.md), [perf(nvfp4): free unused source scales after weight processing](../sources/prs/sglang/PR-25107.md), [[codex] Update Wan2.2 ModelOpt CI checkpoints](../sources/prs/sglang/PR-25483.md), [Support FP4 gemm (1/2)](../sources/prs/sglang/PR-3899.md), [[Build] Fix cuda12.8 build error in nvfp4_scaled_mm_kernels.cu](../sources/prs/sglang/PR-4953.md), [[1/2] Add Kernel support for Cutlass based Fused FP4 MoE](../sources/prs/sglang/PR-6093.md), [Support NVFP4 quantized dense models on AMD CDNA2/CDNA3 GPUs](../sources/prs/sglang/PR-7302.md), [FlashInfer NVFP4 MoE with EP & 2-stream shared expert](../sources/prs/sglang/PR-7327.md), [Qwen FP8/NVFP4 ModelOPT Quantization support](../sources/prs/sglang/PR-7912.md), [[Fix][Ready]Fix register spilling in cutlass nvfp4 gemm kernel on Blackwell](../sources/prs/sglang/PR-8127.md), [[NVIDIA] Add Low Latency NVFP4 decode kernels from Flashinfer](../sources/prs/sglang/PR-8552.md), [Faster weight processing (trtllm-gen moe nvfp4)](../sources/prs/sglang/PR-9162.md), [[NVIDIA] [3/N] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked ](../sources/prs/sglang/PR-9199.md), [[NVIDA] [1/N] Nvfp4 Masked Gemm: Add quant op for the flashinfer grouped gemm](../sources/prs/sglang/PR-9200.md), [[NVIDIA] [2/N] Optimize `silu_and_mul_scaled_fp4_grouped_quant` perf](../sources/prs/sglang/PR-9556.md), [Make fp4_quantize kernels work on sm103](../sources/prs/sglang/PR-9807.md), [[NVIDIA] Support nvfp4 quantization](../sources/prs/vllm/PR-12784.md), [[NVIDIA] Support nvfp4 tcgen05 gemm](../sources/prs/vllm/PR-13571.md), [[Hardware/NVIDIA/Kernel] [Functional Enablement] [1/N] Enable nvidia/DeepSeek-R1-FP4 Model](../sources/prs/vllm/PR-16362.md), [[Misc] Add compressed-tensors NVFP4A16 emulation support](../sources/prs/vllm/PR-17914.md), [Use NVFP4 Marlin for CompressedTensorsW4A16Fp4](../sources/prs/vllm/PR-18000.md), [[Quantization] Add compressed-tensors NVFP4 support](../sources/prs/vllm/PR-18312.md), [[Hardware][NVIDIA] FP4 MoE kernel optimization](../sources/prs/vllm/PR-19110.md), [[Hardware][NVIDIA][kernel] Fp4 MOE quant kernel optimization](../sources/prs/vllm/PR-19500.md), [[Quantization] Add compressed-tensors emulations support for NVFP4](../sources/prs/vllm/PR-19879.md), [[Quantization] Add compressed-tensors NVFP4 MoE Support](../sources/prs/vllm/PR-19990.md), [Add ModelOpt Qwen3 nvfp4 support](../sources/prs/vllm/PR-20101.md), [[Bugfix] Fix some narrowing conversion warnings](../sources/prs/vllm/PR-20141.md), [[Kernel][Bugfix] Fixup some warnings in nvfp4_blockwise_moe when CUDA < 12.8](../sources/prs/vllm/PR-20324.md), [Support CUTLASS NVFP4 (w4a4) for Blackwell Geforce GPUs (SM120)](../sources/prs/vllm/PR-21309.md), [Support Tensorrt-LLM MoE fp4 for low-latency](../sources/prs/vllm/PR-21331.md), [Update flashinfer CUTLASS NVFP4 MoE Kernel to use per expert global scaling factor](../sources/prs/vllm/PR-21408.md), [[Bug] Fix Compressed Tensor NVFP4 `cutlass_fp4_group_mm` illegal memory access](../sources/prs/vllm/PR-21465.md), [[Feature] Add Flashinfer MoE Support for Compressed Tensor NVFP4](../sources/prs/vllm/PR-21639.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv NVFP4-out Attention Kernel](../sources/prs/vllm/PR-22703.md), [Fix nvfp4 swizzling](../sources/prs/vllm/PR-23140.md), [fix incompatibililty with non cuda platform for nvfp4](../sources/prs/vllm/PR-23478.md), [[Bugfix] Fix Marlin NVFP4 for modelopt](../sources/prs/vllm/PR-23659.md), [[NVIDIA] Support SiluMul + NVFP4 quant fusion](../sources/prs/vllm/PR-23671.md), [[Bugfix][Misc] Fix silu_and_mul_nvfp4_quant issue and extract common utils for nvfp4 kernel source files](../sources/prs/vllm/PR-23727.md), [[BUGFIX ] fix undefined silu_and_mul_nvfp4_quant](../sources/prs/vllm/PR-23929.md), [[Transform] [Quantization] Add QuTLASS support to vLLM](../sources/prs/vllm/PR-24440.md), [[Bugfix] Fix accuracy issue for silu_mul + nvfp4 quant fusion kernel](../sources/prs/vllm/PR-24833.md), [[Compile] Fix Compile Warning for Ignoring `MIN_BLOCK_PER_SM`](../sources/prs/vllm/PR-25193.md), [Enable Fbgemm NVFP4 on Dense models](../sources/prs/vllm/PR-25609.md), [[Quantization/NVFP4] Speed up TRTLLM NVFP4 MOE weight loading and fix K/V scale loading for MLA Attn](../sources/prs/vllm/PR-25968.md), [[Bugfix] Allow skipping MoE in NVFP4 (fix for MTP)](../sources/prs/vllm/PR-25987.md), [[MoE] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked](../sources/prs/vllm/PR-25990.md), [[NVIDIA] Add support for cudnn fp4 gemm via flashinfer](../sources/prs/vllm/PR-26107.md), [[ModelOpt] Load w13/w2_input_scale for all experts, nvfp4](../sources/prs/vllm/PR-26135.md), [[Attention] Use sparse prefill kernel for fp8 kv-cache in DeepSeek-v3.2](../sources/prs/vllm/PR-27532.md), [Add TRTLLM MoE NVFP4 kernel to CompressedTensorsW4A4MoeMethod](../sources/prs/vllm/PR-28892.md), [[Feat] Support non-gated activations in NVFP4 modelopt path](../sources/prs/vllm/PR-29004.md), [[Kernel] Add NVFP4 MoE CUTLASS support for SM120](../sources/prs/vllm/PR-29242.md), [[Bugfix] Fix mismatched nvfp4 gemm output shape](../sources/prs/vllm/PR-29742.md), [[EPLB] Support EPLB w/ NVFP4](../sources/prs/vllm/PR-29804.md), [[Compressed-Tensors] Simplify NVFP4 Conditions, enable marlin support for NVFP4A16 MoEs](../sources/prs/vllm/PR-30881.md), [[Kernel][Performance] Enable smaller Scaling Factor tiling for NVFP4 small-batch decoding](../sources/prs/vllm/PR-30885.md), [[NVFP4][Perf] Tune NVFP4 input quant kernel for small batch size](../sources/prs/vllm/PR-30897.md), [ [FIX] Always support TP > 4 for FP4 Gemm](../sources/prs/vllm/PR-31099.md), [[Bugfix] Fix Broken ModelOpt NVFP4 MoE](../sources/prs/vllm/PR-31742.md), [[Perf] Fuse stride preparation for NVFP4 cutlass_moe](../sources/prs/vllm/PR-31837.md), [[Perf][Kernel] Optimize FP4 quantization kernels (SM100F)](../sources/prs/vllm/PR-32520.md), [Support compress-tensors with nvfp4 or fp8 weights and modelopt with nvfp4 weights on Turing](../sources/prs/vllm/PR-33076.md), [fix: Add SM120 (RTX Blackwell) support for FlashInfer CUTLASS NVFP4 MoE kernels](../sources/prs/vllm/PR-33417.md), [[Kernel] Support Flashinfer trtllm fused MoE non gated FP8 & NVFP4](../sources/prs/vllm/PR-33506.md), [[Bugfix] Fix DSV3.2 NVFP4](../sources/prs/vllm/PR-33932.md), [[ModelBash][DSR1 NVFp4] Avoid Bf16 Bias Cast](../sources/prs/vllm/PR-34298.md), [[Custom Ops] Add functional + out variant for scaled_fp4_quant](../sources/prs/vllm/PR-34389.md), [[BUGFIX] Fix accuracy regression for NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4 with TP>1](../sources/prs/vllm/PR-34476.md), [[Model] Add NVFP4 quantization support for Step3.5-Flash](../sources/prs/vllm/PR-34478.md), [[Bugfix] Rescale NVFP4 weight scales to fix BF16 dequant underflow](../sources/prs/vllm/PR-34577.md), [[Bugfix] Fix NVFP4 TRTLLM MoE non-gated support; add gsm8k for Nemotron-3-Nano FP8+NVFP4](../sources/prs/vllm/PR-34725.md), [[BugFix] Fix fp4 quant kernel on CUDA 12.8](../sources/prs/vllm/PR-35210.md), [[NVFP4] Support NVFP4 dense models from `modelopt` and `compressed-tensors` on AMD Instinct MI300, MI355X and Hopper through emulation](../sources/prs/vllm/PR-35733.md), [[Bugfix] Fix passing of activation_type to trtllm fused MoE NVFP4 and FP8](../sources/prs/vllm/PR-36017.md), [[Mamba] Flashinfer selective_state_update](../sources/prs/vllm/PR-36162.md), [[mla] Support fused FP8/NVFP4 output quantization in MLA attention (#35792)](../sources/prs/vllm/PR-36205.md), [[Bug][MoE] Fix TRTLLM NVFP4 Routing Kernel Precision](../sources/prs/vllm/PR-36725.md), [[Bug][MoE] Strengthen _supports_current_device() checks in the TRTLLM FP8, NVFP4, and FlashInfer CuteDSL MoE experts](../sources/prs/vllm/PR-36728.md), [[MoE Refactor] Mxfp4 oracle rebased](../sources/prs/vllm/PR-37128.md), [Fix minimax m2.5 nvfp4 kv scales weight loading](../sources/prs/vllm/PR-37214.md), [[MoE/EPLB] Fix FlashInfer nvfp4 experts + EPLB correctness](../sources/prs/vllm/PR-37217.md), [[Kernel] Add non-gated support for NVFP4 CUTLASS MoE](../sources/prs/vllm/PR-37320.md), [[Bugfix] Fix EP weight filter breaking EPLB and NVFP4 accuracy](../sources/prs/vllm/PR-37322.md), [Add nvfp4 support to reshape_and_cache_flash](../sources/prs/vllm/PR-37332.md), [[Bugfix] Remove assertion for NVFP4 scale dynamic range](../sources/prs/vllm/PR-37465.md), [[Bugfix] Fix marlin nvfp4 rescaling](../sources/prs/vllm/PR-37502.md), [[4/n] Migrate FP4/W4A8 CUTLASS kernels to torch stable ABI](../sources/prs/vllm/PR-37503.md), [[Perf] Use torch compile to fuse pack topk in trtllm moe](../sources/prs/vllm/PR-37695.md), [[Bugfix] Preserve CUDA arch suffix (a/f) for SM12x — fixes NVFP4 NaN on desktop Blackwell](../sources/prs/vllm/PR-37725.md), [[MoE] Move FlashInfer CuteDSL experts into fused_moe/experts/](../sources/prs/vllm/PR-37759.md), [[MoE Kernel] Flashinfer nvfp4 cutedsl moe kernel integration](../sources/prs/vllm/PR-38050.md), [[Bugfix] Fix DeepGemm E8M0 accuracy degradation for Qwen3.5 FP8 on Blackwell](../sources/prs/vllm/PR-38083.md), [[Quantization] Add FlashInfer CuteDSL batched experts backend for NVFP4 MoE](../sources/prs/vllm/PR-38251.md), [[MoE] Add RoutingMethodType.Simulated to TRT-LLM FP8/NVFP4 kernel allowlists](../sources/prs/vllm/PR-38329.md), [[NVIDIA] Bugfix NVFP4 DGX Spark and RTX50](../sources/prs/vllm/PR-38423.md), [[Compile] Fix nvfp4 compile warning](../sources/prs/vllm/PR-38573.md), [[Bugfix] Fix NVFP4+MTP crash: force unquantized mtp.fc for Qwen3.5](../sources/prs/vllm/PR-38832.md), [[MoE Refactor] Split up compressed_tensors_moe.py](../sources/prs/vllm/PR-38960.md), [[Refactor] Move NVFP4 GEMM management into NvFp4LinearKernel](../sources/prs/vllm/PR-39129.md), [[Feature] Batch invariant nvfp4 linear support](../sources/prs/vllm/PR-39322.md), [[Kernel] Support TRTLLM GEN NVFP4 MoE for non-512-aligned hidden dims via weight padding](../sources/prs/vllm/PR-39510.md), [[Bugfix] Reject non-nvfp4 dtypes when using the flashinfer_nvlink_one_sided all2all backend](../sources/prs/vllm/PR-39717.md), [[Bug] Fix batch invariance nvfp4 support](../sources/prs/vllm/PR-39820.md), [Add nvfp4 kv cache support](../sources/prs/vllm/PR-40177.md), [[Bugfix] Guard mxfp4_experts_quant bindings on ENABLE_NVFP4_SM100](../sources/prs/vllm/PR-40191.md), [[MoE] Move cutlass moe to fused_moe/experts/](../sources/prs/vllm/PR-40574.md), [[Kernel][MoE] Support GELU on TRT-LLM NvFP4 fused MoE for Gemma4](../sources/prs/vllm/PR-41050.md), [Add NVFP4 all-gather GEMM fusion for AsyncTP](../sources/prs/vllm/PR-41882.md), [[MoE] Move various experts classes to fused_moe/experts/](../sources/prs/vllm/PR-41979.md), [[Perf] Padded nvfp4 quant kernel to remove additional copy, 2.4%~5.7% e2e performance improvement](../sources/prs/vllm/PR-42774.md), [NVFP4 and Block-Scaled Narrow Precision](../wiki/hardware/nvfp4.md), [GPU Mode NVFP4 Gated Dual GEMM](../wiki/kernels/gated-dual-gemm.md), [Grouped GEMM Contracts for MoE and NVFP4](../wiki/kernels/grouped-gemm.md), [NVFP4 GEMM — GPU Mode Problem 2 Contract](../wiki/kernels/nvfp4-gemm.md), [NVFP4 Batched GEMV](../wiki/kernels/nvfp4-gemv.md), [PTX Instructions for SM100](../wiki/languages/ptx-sm100.md), [Fine-Grained FP8/FP4 Quantization](../wiki/techniques/fine-grained-quantization.md) | | `pdl` | [NVIDIA Blackwell Tuning Guide](../sources/docs/nvidia-blackwell-tuning-guide.md), [NVIDIA CUDA Toolkit 13.x for Blackwell](../sources/docs/nvidia-cuda-13.md), [Blockwise Improvement and Programmatic Dependent Launch](../sources/prs/cutlass/PR-2161.md), [TGV GEMM as a BF16 backend alternative to cuBLAS](../sources/prs/flashinfer/PR-1668.md), [Programmatic Dependent Launch / Grid Dependency Control](../wiki/hardware/pdl-gdc.md) | -| `tcgen05` | [Microbenchmarking NVIDIA's Blackwell Architecture](../sources/blogs/blackwell-microbenchmarking.md), [Colfax CUTLASS Tutorial: GEMM Kernels Using Tensor Memory for Blackwell](../sources/blogs/colfax-cutlass-blackwell.md), [DeepGEMM — FP8 GEMM Library](../sources/blogs/deepgemm.md), [FlashAttention-4 Blog](../sources/blogs/flash-attention-4.md), [FlashMLA — Multi-head Latent Attention](../sources/blogs/flashmla.md), [Writing High-Performance Matrix Multiplication Kernels for Blackwell with JAX Pallas](../sources/blogs/jax-pallas-blackwell-matmul.md), [Modular: Matrix Multiplication on Blackwell](../sources/blogs/modular-blackwell-matmul.md), [tcgen05 for dummies](../sources/blogs/tcgen05-tutorial.md), [FlashInfer MLSys 2026 - Track A: Fused MoE FP8](../sources/contests/flashinfer-mlsys26/track-a-fused-moe.md), [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [FlashInfer MLSys 2026 - Track C: Gated Delta Net](../sources/contests/flashinfer-mlsys26/track-c-gated-delta-net.md), [GPU Mode NVFP4 Hackathon - Problem 2: NVFP4 GEMM](../sources/contests/gpu-mode-nvfp4/problem-2-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 3: Gated Dual GEMM](../sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 4: Grouped GEMM](../sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md), [cuTile Python DSL Reference](../sources/docs/cutile-python-dsl.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [FlashAttention-4: Hardware-Friendly Attention on Blackwell](../sources/docs/flash-attention-4.md), [NVIDIA Blackwell Tuning Guide](../sources/docs/nvidia-blackwell-tuning-guide.md), [NVIDIA CUDA Toolkit 13.x for Blackwell](../sources/docs/nvidia-cuda-13.md), [NVIDIA CUTLASS 4.x Blackwell Support](../sources/docs/nvidia-cutlass-blackwell.md), [PTX ISA SM100 Instructions Reference](../sources/docs/nvidia-ptx-isa-sm100.md), [Tiled Flash Linear Attention (TFLA)](../sources/docs/tfla.md), [Triton 3.6.0 Release Notes — Blackwell (SM100) Lowering](../sources/docs/triton-3.6-blackwell.md), [[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes](../sources/prs/DeepGEMM/PR-304.md), [Blockwise and Groupwise GEMM for Blackwell and Improvements for Hopper](../sources/prs/cutlass/PR-2139.md), [Blockwise Improvement and Programmatic Dependent Launch](../sources/prs/cutlass/PR-2161.md), [Example 77 add blackwell flash-attention bwd for MLA shape](../sources/prs/cutlass/PR-2466.md), [Add Blackwell MLA forward (shape: d=192, dv=128) implementation](../sources/prs/cutlass/PR-2472.md), [Support for GEMM-K=0 for Blackwell Grouped GEMMs](../sources/prs/cutlass/PR-2746.md), [[Cutlass gemm] Fix SM100 FP8 nosmem epilogue-fusion shape_div 'Divisibility Condition' for non-multiple-of-64 N tiles](../sources/prs/cutlass/PR-2946.md), [[CuTeDSL] Fix: SM100 block-scale gemm overlapping accumulator](../sources/prs/cutlass/PR-2995.md), [perf: Enable SplitK and fix tile-scheduling for moe fp4 fused moe](../sources/prs/flashinfer/PR-1548.md), [TGV GEMM as a BF16 backend alternative to cuBLAS](../sources/prs/flashinfer/PR-1668.md), [perf: improve attention of tcgen05 flash-attention](../sources/prs/flashinfer/PR-1681.md), [[cute_dsl] add gemm + all reduce (two_shot)](../sources/prs/flashinfer/PR-1695.md), [Add head_dim=64 for tcgen05 tcgen05 flash-attention implementation](../sources/prs/flashinfer/PR-1850.md), [A Blackwell-optimized version of selective_state_update (decode)](../sources/prs/flashinfer/PR-2387.md), [integrate blockwise fp8 kernel](../sources/prs/sglang/PR-3529.md), [Add Cutlass MLA attention backend](../sources/prs/sglang/PR-5390.md), [[WIP] support more dtypes for tcgen05](../sources/prs/tilelang/PR-1229.md), [[Enhancement] add more dtype and fix mma.ws for fp16 for tcgen05](../sources/prs/tilelang/PR-1327.md), [[Feature] Support tcgen5mma lowering for `.kind::i8`](../sources/prs/tilelang/PR-1764.md), [[CUDA] Support tcgen5mma gemm ts](../sources/prs/tilelang/PR-1866.md), [[Feature] 2-SM support for TMA, TMEM and TCGEN5MMA on Blackwell](../sources/prs/tilelang/PR-1882.md), [[Feature] Block-scaled GEMM support for MXFP8 on Blackwell](../sources/prs/tilelang/PR-1945.md), [[Transform] Add InjectTcgen05Fence pass](../sources/prs/tilelang/PR-2003.md), [[codex] Split GEMM implementations by backend](../sources/prs/tilelang/PR-2153.md), [[TIR][IR] Update to use tirx](../sources/prs/tilelang/PR-2216.md), [[Python] Drop Python 3.9 support](../sources/prs/tilelang/PR-2218.md), [[NVIDIA] Support nvfp4 tcgen05 gemm](../sources/prs/vllm/PR-13571.md), [add tcgen05 support for tcgen05 fp8 gemm](../sources/prs/vllm/PR-13798.md), [[NVIDIA] Support Cutlass MLA for Blackwell GPUs](../sources/prs/vllm/PR-16032.md), [[Perf] Further tunings for SM100 FP8 CUTLASS kernel](../sources/prs/vllm/PR-19566.md), [[Bugfix] Fix default enable for CUTLASS MLA on SM100](../sources/prs/vllm/PR-22738.md), [[Kernel][tcgen05] nvfp4 fused tcgen05 moe](../sources/prs/vllm/PR-23696.md), [Two-SM Cooperative MMA](../wiki/hardware/2sm-cooperative.md), [tcgen05.mma — Blackwell MMA Instruction](../wiki/hardware/tcgen05-mma.md), [Tensor Memory (TMEM)](../wiki/hardware/tmem.md), [FlashAttention-4](../wiki/kernels/flash-attention-4.md), [FP8 Block-Scale GEMM](../wiki/kernels/fp8-block-scale-gemm.md), [Grouped GEMM for MoE](../wiki/kernels/grouped-gemm.md), [NVFP4 GEMM — 4-bit Floating Point Matrix Multiply](../wiki/kernels/nvfp4-gemm.md), [CUDA C++ for Blackwell Kernels](../wiki/languages/cuda-cpp.md), [CuTe DSL for Blackwell](../wiki/languages/cute-dsl.md), [PTX Instructions for SM100](../wiki/languages/ptx-sm100.md), [Register Accumulators to TMEM](../wiki/migration/register-to-tmem.md), [Migrating from wgmma to tcgen05](../wiki/migration/wgmma-to-tcgen05.md), [Not Reaching Peak FLOPS](../wiki/patterns/compute-bound.md), [Pipeline Stalls](../wiki/patterns/pipeline-stalls.md), [Warp Specialization on Blackwell](../wiki/techniques/warp-specialization.md) | -| `tma` | [Colfax Article Source Kernels](../sources/blogs/colfax-article-source-kernels.md), [Colfax CUTLASS Kernels](../sources/blogs/colfax-cutlass-kernels.md), [Writing High-Performance Matrix Multiplication Kernels for Blackwell with JAX Pallas](../sources/blogs/jax-pallas-blackwell-matmul.md), [Modular: Matrix Multiplication on Blackwell](../sources/blogs/modular-blackwell-matmul.md), [simveit effective_transpose](../sources/blogs/simveit-effective-transpose.md), [simveit load_and_store](../sources/blogs/simveit-load-and-store.md), [TFLOPS Gap: Why FP4 MoE Kernel Engineering Matters on Blackwell](../sources/blogs/tflops-gap-fp4-moe.md), [FlashInfer MLSys 2026 - Track A: Fused MoE FP8](../sources/contests/flashinfer-mlsys26/track-a-fused-moe.md), [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [FlashInfer MLSys 2026 - Track C: Gated Delta Net](../sources/contests/flashinfer-mlsys26/track-c-gated-delta-net.md), [GPU Mode NVFP4 Hackathon - Problem 2: NVFP4 GEMM](../sources/contests/gpu-mode-nvfp4/problem-2-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 3: Gated Dual GEMM](../sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 4: Grouped GEMM](../sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md), [cuTile Python DSL Reference](../sources/docs/cutile-python-dsl.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [NVIDIA Blackwell Tuning Guide](../sources/docs/nvidia-blackwell-tuning-guide.md), [NVIDIA CUDA Toolkit 13.x for Blackwell](../sources/docs/nvidia-cuda-13.md), [NVIDIA CUTLASS 4.x Blackwell Support](../sources/docs/nvidia-cutlass-blackwell.md), [PTX ISA SM100 Instructions Reference](../sources/docs/nvidia-ptx-isa-sm100.md), [ Solving bank conflict via padding and TMA 3D store](../sources/prs/DeepGEMM/PR-78.md), [Use 1D TMA store instead of 3D](../sources/prs/DeepGEMM/PR-83.md), [Support TMA multicast on B with m_grouped_gemm_contiguous.](../sources/prs/DeepGEMM/PR-88.md), [[TRTLLM-10022][feat] Add hopper xqa decode support for skip softmax attention](../sources/prs/TensorRT-LLM/PR-10264.md), [[https://nvbugs/5669671][fix] Support GuidedDecoder with sharded logits (pick #10698)](../sources/prs/TensorRT-LLM/PR-10742.md), [[TRTLLM-9831][perf] Use TMA.RED to improve effective memory bandwidth](../sources/prs/TensorRT-LLM/PR-10987.md), [[TRTLLM-11092][feat] add support for visual gen FA4 attention backend](../sources/prs/TensorRT-LLM/PR-11697.md), [[None][feat] Support sparse mqa/gqa attention](../sources/prs/TensorRT-LLM/PR-12470.md), [[None][feat] Trtllm-gen FMHA JIT support](../sources/prs/TensorRT-LLM/PR-12612.md), [[TRTLLM-11485][feat] Feature rework: Add SageAttention refreshed kernels (attentionOp only)](../sources/prs/TensorRT-LLM/PR-12937.md), [[None][perf] Drop cubin and Eliminate ~6s FMHA JIT recompile in eager generation by aligning kernel selection with CUDA graph warmup](../sources/prs/TensorRT-LLM/PR-13505.md), [[None][feat] Add DeepSeekV4 attention kernels](../sources/prs/TensorRT-LLM/PR-13652.md), [[None][feat] Update the logic of FMHA JIT path](../sources/prs/TensorRT-LLM/PR-14291.md), [[OMNIML-2336][feat] Add NVFP4 x FP8](../sources/prs/TensorRT-LLM/PR-6809.md), [[None][feat] GPT-OSS Sm120/Sm121 Support](../sources/prs/TensorRT-LLM/PR-7937.md), [[None][feat] Update TRTLLM MoE cubins; reduce mxfp4 weight padding requirement; tighten TMA bound](../sources/prs/TensorRT-LLM/PR-9025.md), [[https://nvbugs/5726962][feat] Apply fusion for W4AFP8_AWQ MoE](../sources/prs/TensorRT-LLM/PR-9838.md), [Improve sm90 mixed dtype kernel](../sources/prs/cutlass/PR-1883.md), [[EVT] Add support for Row/Col broadcast PtrArray](../sources/prs/cutlass/PR-2033.md), [Groupwise scaling along M for FP8 gemm](../sources/prs/cutlass/PR-2037.md), [Improvements for: Groupwise scaling along M for FP8 gemm](../sources/prs/cutlass/PR-2095.md), [Hopper Grouped GEMM support for FP8 Accum](../sources/prs/cutlass/PR-2123.md), [Flash MLA support](../sources/prs/cutlass/PR-2130.md), [Flash MLA Support - Step 2](../sources/prs/cutlass/PR-2134.md), [Fix sm100 gemm wrong static constexpr that breaks compilation on Windows](../sources/prs/cutlass/PR-2167.md), [Fix SM90 beta=1 hang and stream-K launch errors](../sources/prs/cutlass/PR-2172.md), [hopper-blockwise-generalization-optimization](../sources/prs/cutlass/PR-2270.md), [[ex77] fix mla split; add fwd lse; add bwd varlen](../sources/prs/cutlass/PR-2366.md), [Example 77 add blackwell flash-attention bwd for MLA shape](../sources/prs/cutlass/PR-2466.md), [Add Blackwell MLA forward (shape: d=192, dv=128) implementation](../sources/prs/cutlass/PR-2472.md), [fix: examples/cute/tutorial/blackwell/04_mma_tma_2sm_sm100.cu GridDim miscalculated](../sources/prs/cutlass/PR-2492.md), [Support PDL for SM90 Array TMA GEMM](../sources/prs/cutlass/PR-2719.md), [[Bug Fix]Bypass launch grids for SM120 Kernel with SM90 Mainloop & SM100 TileScheduler](../sources/prs/cutlass/PR-2865.md), [new example with TMA prefetch feature targeting for DRAM latency boun…](../sources/prs/cutlass/PR-2881.md), [[Bug Fix]Set NumSplitsM to 1 when TileShapeM < 128 in sm90 fp8 blockwise scaling CollectiveMma](../sources/prs/cutlass/PR-2965.md), [Replace std::min with cute::min in sm120 blockwise scaling device functions](../sources/prs/cutlass/PR-3055.md), [Small Tile N BlockScaled GEMM + Grouped GEMM on SM12x](../sources/prs/cutlass/PR-3176.md), [Add var-seq-len to FA3 fp16 / bf16 fwd](../sources/prs/flash-attention/PR-1072.md), [Fp8 kernel with "in-kernel" transpose of V in producer](../sources/prs/flash-attention/PR-1100.md), [FA3 FP8 qkv descales + restore max offset for h128 causal + added sync for producer WG](../sources/prs/flash-attention/PR-1173.md), [Add seqused_q in fwd / bwd and seqused_k in bwd in hopper FA.](../sources/prs/flash-attention/PR-1182.md), [Add local attention in Hopper FAv3](../sources/prs/flash-attention/PR-1233.md), [FA3 kvcache + split kv + gqa parallelization](../sources/prs/flash-attention/PR-1236.md), [Paged Attention support for FA3](../sources/prs/flash-attention/PR-1268.md), [FA3 paged attention: Readiness for Cutlass 3.6 / default value for block_table](../sources/prs/flash-attention/PR-1331.md), [Fix FA3 Varlen Performance regression](../sources/prs/flash-attention/PR-1361.md), [Support hdimQK != hdimV backward](../sources/prs/flash-attention/PR-1604.md), [Improve causal backward determinism perf with SPT schedule](../sources/prs/flash-attention/PR-1893.md), [[Cute,Sm100,Fwd] use correction warps for epi when not using TMA](../sources/prs/flash-attention/PR-2014.md), [[Cute,Fwd,Sm100] don't pass mask_fn to softmax_step generically](../sources/prs/flash-attention/PR-2026.md), [[Cute,Fwd] Extend score_mod to variable sequence length](../sources/prs/flash-attention/PR-2043.md), [Add score-mod bwd support ](../sources/prs/flash-attention/PR-2070.md), [[Cute,Fwd,Sm100] fp8 e4m3 and e5m2 support](../sources/prs/flash-attention/PR-2109.md), [[Fwd,Sm90] Add paged KV attention support (tma and cp.async)](../sources/prs/flash-attention/PR-2360.md), [[Cute,Sm100,Fwd] add MLA 64/512 with topk sparsity for MQA 128 heads](../sources/prs/flash-attention/PR-2441.md), [[hd256] Add TMA paged KV support to SM100 2CTA forward kernel](../sources/prs/flash-attention/PR-2489.md), [[FA4][hd256] Backward TMA bulk-store epilogue + LSE/dpsum coalesce](../sources/prs/flash-attention/PR-2497.md), [feat: Softmax free sampling](../sources/prs/flashinfer/PR-1035.md), [[nvidia] initial support for blackwell kernels](../sources/prs/flashinfer/PR-1039.md), [bugfix: adding lse output to blackwell fmha kernels](../sources/prs/flashinfer/PR-1071.md), [bugfix: follow user-specified sm_scale for blackwell cutlass fmha](../sources/prs/flashinfer/PR-1072.md), [bugfix: host-precomuted plan function for blackwell fmha](../sources/prs/flashinfer/PR-1106.md), [Add CUTLASS fused moe kernels from TensorRT-LLM.](../sources/prs/flashinfer/PR-1113.md), [feat: Fused temperature online softmax kernel](../sources/prs/flashinfer/PR-1153.md), [feat: logits processor fustion rule for temperature softmax](../sources/prs/flashinfer/PR-1170.md), [bugfix: softmax NaN results caused by large -inf masks](../sources/prs/flashinfer/PR-1178.md), [bugfix: fix blackwell fmha hanging issue for empty kv_len](../sources/prs/flashinfer/PR-1198.md), [feat: trtllm-gen fp8 moe kernels](../sources/prs/flashinfer/PR-1212.md), [Update cutlass fp4 moe kernels](../sources/prs/flashinfer/PR-1294.md), [Add blockwise-scaled FP8 GEMM via TRTLLM-Gen.](../sources/prs/flashinfer/PR-1320.md), [GPT-OSS Support: Add Blackwell MoE mxfp4 implementation from TRTLLM and Attention Sink](../sources/prs/flashinfer/PR-1389.md), [gpt-oss: Add MXFP8 x MXFP4 CUTLASS MOE for SM100 and BF16 x MXFP4 CUTLASS for SM90 + SwigluBias Activation](../sources/prs/flashinfer/PR-1396.md), [Remove getEnvEnablePDL in favor of enable_pdl parameter](../sources/prs/flashinfer/PR-1446.md), [perf: Enable SplitK and fix tile-scheduling for moe fp4 fused moe](../sources/prs/flashinfer/PR-1548.md), [bugfix: fix fused-temperature softmax IMA issue](../sources/prs/flashinfer/PR-1596.md), [feat: initial support for SM103, SM110, SM120, SM121](../sources/prs/flashinfer/PR-1608.md), [TGV GEMM as a BF16 backend alternative to cuBLAS](../sources/prs/flashinfer/PR-1668.md), [perf: improve attention of tcgen05 flash-attention](../sources/prs/flashinfer/PR-1681.md), [[cute_dsl] add gemm + all reduce (two_shot)](../sources/prs/flashinfer/PR-1695.md), [feat: add xqa fp8 mha and fp8 kv cache](../sources/prs/flashinfer/PR-1769.md), [feat:enable fp8 blockscale moe for fused cultass for sm90](../sources/prs/flashinfer/PR-1819.md), [feat: trtrllm-gen global scaled FP8 GEMMs](../sources/prs/flashinfer/PR-1829.md), [Add head_dim=64 for tcgen05 tcgen05 flash-attention implementation](../sources/prs/flashinfer/PR-1850.md), [Tune kernel compilation parameters for https://github.com/flashinfer-ai/flashinfer/pull/1850 ](../sources/prs/flashinfer/PR-1878.md), [Update trtllm-gen fused moe routing kernel and add more kernels](../sources/prs/flashinfer/PR-1955.md), [Feature: Add support for L40 FusedMoE in cutlass path](../sources/prs/flashinfer/PR-1973.md), [update trtllm cutlass moe ](../sources/prs/flashinfer/PR-2020.md), [perf: improve sampling/mask/softmax performance (part 1/2)](../sources/prs/flashinfer/PR-2044.md), [Rebase FP8 SM100 Cutlass FMHA Attention to main (original PR#1238)](../sources/prs/flashinfer/PR-2047.md), [update xqa license](../sources/prs/flashinfer/PR-2117.md), [feat: TRTLLM FMHAv2 backend for ctx attention](../sources/prs/flashinfer/PR-2142.md), [feat: MxInt4 x Bf16 TRT-LLM Gen MoE support](../sources/prs/flashinfer/PR-2159.md), [refactor: pull trtllm-gen batch-gemm/gemm headers from artifactory; update tma descriptor shape init](../sources/prs/flashinfer/PR-2235.md), [feat: add GDN Attention](../sources/prs/flashinfer/PR-2276.md), [A Blackwell-optimized version of selective_state_update (decode)](../sources/prs/flashinfer/PR-2387.md), [feat: update trtllm-gen MoE cubins](../sources/prs/flashinfer/PR-2416.md), [refactor: reduce hopper's gdn prefill compilation time and fix docstring.](../sources/prs/flashinfer/PR-2422.md), [feat: Add TRTLLM fmha_v2 library for SM90 attention with Skip-Softmax ](../sources/prs/flashinfer/PR-2446.md), [feat: Add TRTLLM-Gen Skip-Softmax kernels for prefill and decode](../sources/prs/flashinfer/PR-2477.md), [feat: Enable TRTLLM-Gen Skip-Softmax attention for MLA](../sources/prs/flashinfer/PR-2547.md), [Implement `cutlass_fused_moe` mxfp8](../sources/prs/flashinfer/PR-2581.md), [[feat] trtllm-gen mxfp8 gemm](../sources/prs/flashinfer/PR-2653.md), [misc: Update gemm/batched gemm cubins from trtllm-gen, gemm header refactor](../sources/prs/flashinfer/PR-2740.md), [Upgrade cutlass 4.2.1 -> 4.4.2](../sources/prs/flashinfer/PR-2798.md), [[CuTe DSL] Add modular FMHA prefill and MLA decode attention kernels](../sources/prs/flashinfer/PR-2805.md), [feat(gdn): state checkpointing in chunk_gated_delta_rule](../sources/prs/flashinfer/PR-2908.md), [Integrate CUTLASS Small Tile N Blockscaled GEMMs/Grouped GEMMs for SM120 and SM121](../sources/prs/flashinfer/PR-3152.md), [feat(trace): embed runnable init() in every TraceTemplate](../sources/prs/flashinfer/PR-3221.md), [[inductor][triton 3.3] Fix cpp_wrapper w/ TMA in triton 3.3](../sources/prs/pytorch/PR-149993.md), [[user triton] AOT inductor support for device-side TMA](../sources/prs/pytorch/PR-157241.md), [[kernel][moe] add moe topk fast](../sources/prs/sglang/PR-13969.md), [Move fa4 from sgl-kernel to jit kernel](../sources/prs/sglang/PR-17353.md), [Support skip-softmax attention](../sources/prs/sglang/PR-19089.md), [[Kernel] Fuse temperature + softmax in sampling for decode speedup](../sources/prs/sglang/PR-20501.md), [add tensorrt_llm common and cutlass_extensions as 3rdparty](../sources/prs/sglang/PR-3216.md), [support blockwise fp8 matmul kernel](../sources/prs/sglang/PR-3267.md), [integrate blockwise fp8 kernel](../sources/prs/sglang/PR-3529.md), [DeepGemm integrate to gemm](../sources/prs/sglang/PR-4165.md), [Create col-major and tma-aligned x_scale for deep_gemm.gemm_fp8_fp8_bf16_nt](../sources/prs/sglang/PR-4515.md), [Add Cutlass MLA attention backend](../sources/prs/sglang/PR-5390.md), [[perf] introduce deep gemm group_gemm_masked as gemm](../sources/prs/sglang/PR-5432.md), [[perf][sgl-kernel] extend cutlass_mla_decode to support num_head < 128](../sources/prs/sglang/PR-6929.md), [[1/n]: add cutlass W4A8 moe kernel for hopper architecture](../sources/prs/sglang/PR-7772.md), [[NVIDIA] Fix missing `get_col_major_tma_aligned_tensor` for Blackwell deepgemm in EpMoE](../sources/prs/sglang/PR-8955.md), [[Feature] Support `cp.reduce.async.bulk.tensor`](../sources/prs/tilelang/PR-1667.md), [[BugFix] Fix Hopper TMA lowering without warp specialization](../sources/prs/tilelang/PR-1840.md), [[Feature] 2-SM support for TMA, TMEM and TCGEN5MMA on Blackwell](../sources/prs/tilelang/PR-1882.md), [[Feature] Add T.copy_cluster to support TMA multicast and SM-to-SM cluster copy](../sources/prs/tilelang/PR-1908.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [[Feature] Support TMA store in T.tma_copy()](../sources/prs/tilelang/PR-1981.md), [[Backend] Refactor gemm_sp](../sources/prs/tilelang/PR-2048.md), [[Bugfix] Enable `.shared::cta` in TMA copy paths only on CUDA 12.8+](../sources/prs/tilelang/PR-2087.md), [[TMA] Support FP4 TensorMap TMA copies](../sources/prs/tilelang/PR-2107.md), [[CUDA][TMA] Add TMA tile::gather4 / tile::scatter4 support](../sources/prs/tilelang/PR-2129.md), [[Kernel] Update `cutlass_scaled_mm` to support 2d group (blockwise) scaling](../sources/prs/vllm/PR-11868.md), [Add: Support for Sparse24Bitmask Compressed Models](../sources/prs/vllm/PR-12097.md), [[NVIDIA] Support nvfp4 tcgen05 gemm](../sources/prs/vllm/PR-13571.md), [add tcgen05 support for tcgen05 fp8 gemm](../sources/prs/vllm/PR-13798.md), [[BugFix] Illegal Memory Access in the blockwise cutlass fp8 GEMMs](../sources/prs/vllm/PR-14396.md), [Modularize fused experts and integrate PPLX kernels](../sources/prs/vllm/PR-15956.md), [[NVIDIA] Support Cutlass MLA for Blackwell GPUs](../sources/prs/vllm/PR-16032.md), [Fix `numel()` downcast in vllm/csrc/moe/moe_align_sum_kernels.cu +2](../sources/prs/vllm/PR-17082.md), [[Perf] Further tunings for SM100 FP8 CUTLASS kernel](../sources/prs/vllm/PR-19566.md), [[feat]: CUTLASS block scaled group gemm for SM100](../sources/prs/vllm/PR-19757.md), [SM100 Cutlass MLA decode with unrestricted num_heads (< 128) for DeepSeek TP](../sources/prs/vllm/PR-20769.md), [[Perf] Use upstream CUTLASS for SM90 Block FP8 kernel](../sources/prs/vllm/PR-23280.md), [[Kernel][tcgen05] nvfp4 fused tcgen05 moe](../sources/prs/vllm/PR-23696.md), [[Kernel] Add topk_sigmoid kernel](../sources/prs/vllm/PR-31246.md), [Add TMA support to fused_moe_lora kernel](../sources/prs/vllm/PR-32195.md), [[Perf] Create TMA-aligned input scale tensor for DeepGemm on Hopper](../sources/prs/vllm/PR-32619.md), [[Bugfix] Fix quant RMS norm fusion for quantization with TMA-aligned scales](../sources/prs/vllm/PR-33255.md), [[Kernels][MoE] Fix legacy_routing to use bitmatrix-based routing path](../sources/prs/vllm/PR-38504.md), [[Perf][GDN] Align TMA usage with upstream FLA](../sources/prs/vllm/PR-38981.md), [fix: clamp NaN/Inf in topk_softmax to prevent duplicate expert IDs](../sources/prs/vllm/PR-39391.md), [mbarrier (Memory Barrier Primitives)](../wiki/hardware/mbarrier.md), [Tensor Memory Accelerator (TMA)](../wiki/hardware/tma.md), [FlashAttention SM100 MLA TopK Sparse Forward](../wiki/kernels/flash-attention-sm100-mla-topk.md), [CuTe DSL for Blackwell](../wiki/languages/cute-dsl.md), [PTX Instructions for SM100](../wiki/languages/ptx-sm100.md), [Pipeline Stalls](../wiki/patterns/pipeline-stalls.md), [External Source-Map Research For Kernel Edits](../wiki/techniques/external-source-map-research.md), [Software Pipelining and Multi-Stage Buffering](../wiki/techniques/pipeline-stages.md), [Shared Memory Swizzling](../wiki/techniques/swizzling.md) | -| `tmem` | [Microbenchmarking NVIDIA's Blackwell Architecture](../sources/blogs/blackwell-microbenchmarking.md), [Colfax CUTLASS Tutorial: GEMM Kernels Using Tensor Memory for Blackwell](../sources/blogs/colfax-cutlass-blackwell.md), [FlashAttention-4 Blog](../sources/blogs/flash-attention-4.md), [FlashMLA — Multi-head Latent Attention](../sources/blogs/flashmla.md), [Writing High-Performance Matrix Multiplication Kernels for Blackwell with JAX Pallas](../sources/blogs/jax-pallas-blackwell-matmul.md), [Modular: Matrix Multiplication on Blackwell](../sources/blogs/modular-blackwell-matmul.md), [tcgen05 for dummies](../sources/blogs/tcgen05-tutorial.md), [FlashInfer MLSys 2026 - Track A: Fused MoE FP8](../sources/contests/flashinfer-mlsys26/track-a-fused-moe.md), [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [FlashInfer MLSys 2026 - Track C: Gated Delta Net](../sources/contests/flashinfer-mlsys26/track-c-gated-delta-net.md), [GPU Mode NVFP4 Hackathon - Problem 2: NVFP4 GEMM](../sources/contests/gpu-mode-nvfp4/problem-2-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 3: Gated Dual GEMM](../sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 4: Grouped GEMM](../sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md), [cuTile Python DSL Reference](../sources/docs/cutile-python-dsl.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [FlashAttention-4: Hardware-Friendly Attention on Blackwell](../sources/docs/flash-attention-4.md), [NVIDIA Blackwell Tuning Guide](../sources/docs/nvidia-blackwell-tuning-guide.md), [NVIDIA CUDA Toolkit 13.x for Blackwell](../sources/docs/nvidia-cuda-13.md), [NVIDIA CUTLASS 4.x Blackwell Support](../sources/docs/nvidia-cutlass-blackwell.md), [PTX ISA SM100 Instructions Reference](../sources/docs/nvidia-ptx-isa-sm100.md), [Triton 3.6.0 Release Notes — Blackwell (SM100) Lowering](../sources/docs/triton-3.6-blackwell.md), [[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes](../sources/prs/DeepGEMM/PR-304.md), [[TRTLLM-9992][perf] Enable PDL for CuteDSL kernels and overlap MoeOutputMemset](../sources/prs/TensorRT-LLM/PR-10043.md), [Blockwise and Groupwise GEMM for Blackwell and Improvements for Hopper](../sources/prs/cutlass/PR-2139.md), [Blockwise Improvement and Programmatic Dependent Launch](../sources/prs/cutlass/PR-2161.md), [Example 77 add blackwell flash-attention bwd for MLA shape](../sources/prs/cutlass/PR-2466.md), [Add Blackwell MLA forward (shape: d=192, dv=128) implementation](../sources/prs/cutlass/PR-2472.md), [Support for GEMM-K=0 for Blackwell Grouped GEMMs](../sources/prs/cutlass/PR-2746.md), [[CuTeDSL] Fix: SM100 block-scale gemm overlapping accumulator](../sources/prs/cutlass/PR-2995.md), [TGV GEMM as a BF16 backend alternative to cuBLAS](../sources/prs/flashinfer/PR-1668.md), [perf: improve attention of tcgen05 flash-attention](../sources/prs/flashinfer/PR-1681.md), [[cute_dsl] add gemm + all reduce (two_shot)](../sources/prs/flashinfer/PR-1695.md), [Add head_dim=64 for tcgen05 tcgen05 flash-attention implementation](../sources/prs/flashinfer/PR-1850.md), [A Blackwell-optimized version of selective_state_update (decode)](../sources/prs/flashinfer/PR-2387.md), [[CuTe DSL] Add modular FMHA prefill and MLA decode attention kernels](../sources/prs/flashinfer/PR-2805.md), [Add Cutlass MLA attention backend](../sources/prs/sglang/PR-5390.md), [[Feature] 2-SM support for TMA, TMEM and TCGEN5MMA on Blackwell](../sources/prs/tilelang/PR-1882.md), [[NVIDIA] Support Cutlass MLA for Blackwell GPUs](../sources/prs/vllm/PR-16032.md), [[Bugfix] Fix default enable for CUTLASS MLA on SM100](../sources/prs/vllm/PR-22738.md), [tcgen05.mma — Blackwell MMA Instruction](../wiki/hardware/tcgen05-mma.md), [Tensor Memory (TMEM)](../wiki/hardware/tmem.md), [FlashAttention-4](../wiki/kernels/flash-attention-4.md), [Gated Dual GEMM (Gate-Up + SwiGLU Fusion)](../wiki/kernels/gated-dual-gemm.md), [NVFP4 GEMM — 4-bit Floating Point Matrix Multiply](../wiki/kernels/nvfp4-gemm.md), [CUDA C++ for Blackwell Kernels](../wiki/languages/cuda-cpp.md), [CuTe DSL for Blackwell](../wiki/languages/cute-dsl.md), [PTX Instructions for SM100](../wiki/languages/ptx-sm100.md), [Register Accumulators to TMEM](../wiki/migration/register-to-tmem.md), [Migrating from wgmma to tcgen05](../wiki/migration/wgmma-to-tcgen05.md), [Register Pressure — Low Occupancy](../wiki/patterns/register-pressure.md), [Double/Multi-Buffering Patterns](../wiki/techniques/double-buffering.md), [Epilogue Fusion](../wiki/techniques/epilogue-fusion.md), [Kernel Fusion](../wiki/techniques/kernel-fusion.md), [Ping-Pong Scheduling](../wiki/techniques/ping-pong-scheduling.md), [Warp Specialization on Blackwell](../wiki/techniques/warp-specialization.md) | -| `wgmma` | [Microbenchmarking NVIDIA's Blackwell Architecture](../sources/blogs/blackwell-microbenchmarking.md), [Colfax Article Source Kernels](../sources/blogs/colfax-article-source-kernels.md), [Colfax CUTLASS Kernels](../sources/blogs/colfax-cutlass-kernels.md), [DeepGEMM — FP8 GEMM Library](../sources/blogs/deepgemm.md), [simveit load_and_store](../sources/blogs/simveit-load-and-store.md), [Tiled Flash Linear Attention (TFLA)](../sources/docs/tfla.md), [feat: integrate gemm_fp8 kernel into gemm](../sources/prs/sglang/PR-3056.md), [integrate blockwise fp8 kernel](../sources/prs/sglang/PR-3529.md), [DeepGemm integrate to gemm](../sources/prs/sglang/PR-4165.md), [[perf] introduce deep gemm group_gemm_masked as gemm](../sources/prs/sglang/PR-5432.md), [Add swizzle layout detection and automatic merging for layout conflicts](../sources/prs/tilelang/PR-1736.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [[Backend] Refactor gemm_sp](../sources/prs/tilelang/PR-2048.md), [feat: auto-vectorize bf16/fp16 reduce with packed add2 intrinsics](../sources/prs/tilelang/PR-2112.md), [[codex] Split GEMM implementations by backend](../sources/prs/tilelang/PR-2153.md), [[Python] Drop Python 3.9 support](../sources/prs/tilelang/PR-2218.md), [FP8 Block-Scale GEMM](../wiki/kernels/fp8-block-scale-gemm.md), [Migrating from wgmma to tcgen05](../wiki/migration/wgmma-to-tcgen05.md), [External Source-Map Research For Kernel Edits](../wiki/techniques/external-source-map-research.md) | +| `stmatrix` | [simveit load_and_store](../sources/blogs/simveit-load-and-store.md) | +| `tcgen05` | [Microbenchmarking NVIDIA's Blackwell Architecture](../sources/blogs/blackwell-microbenchmarking.md), [Colfax CUTLASS Tutorial: GEMM Kernels Using Tensor Memory for Blackwell](../sources/blogs/colfax-cutlass-blackwell.md), [DeepGEMM — Pinned Upstream Project Summary](../sources/blogs/deepgemm.md), [FlashAttention-4 Blog](../sources/blogs/flash-attention-4.md), [FlashMLA — Multi-head Latent Attention](../sources/blogs/flashmla.md), [Writing High-Performance Matrix Multiplication Kernels for Blackwell with JAX Pallas](../sources/blogs/jax-pallas-blackwell-matmul.md), [Modular: Matrix Multiplication on Blackwell](../sources/blogs/modular-blackwell-matmul.md), [tcgen05 for dummies](../sources/blogs/tcgen05-tutorial.md), [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [cuTile Python DSL Reference](../sources/docs/cutile-python-dsl.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [FlashAttention-4: Algorithm and Kernel Co-design for Blackwell GPUs](../sources/docs/flash-attention-4.md), [NVIDIA Blackwell Tuning Guide](../sources/docs/nvidia-blackwell-tuning-guide.md), [NVIDIA CUDA Toolkit 13.x for Blackwell](../sources/docs/nvidia-cuda-13.md), [NVIDIA CUTLASS 4.5.0 Blackwell Sources](../sources/docs/nvidia-cutlass-blackwell.md), [PTX ISA 9.0 SM100 Instruction Reference](../sources/docs/nvidia-ptx-isa-sm100.md), [Triton v3.3.0 — Blackwell TCGen5/TMEM Boundary](../sources/docs/triton-3.3-blackwell.md), [Triton v3.6.0 — Incremental Blackwell Changes](../sources/docs/triton-3.6-blackwell.md), [[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes](../sources/prs/DeepGEMM/PR-304.md), [Blockwise and Groupwise GEMM for Blackwell and Improvements for Hopper](../sources/prs/cutlass/PR-2139.md), [Blockwise Improvement and Programmatic Dependent Launch](../sources/prs/cutlass/PR-2161.md), [Example 77 add blackwell flash-attention bwd for MLA shape](../sources/prs/cutlass/PR-2466.md), [Add Blackwell MLA forward (shape: d=192, dv=128) implementation](../sources/prs/cutlass/PR-2472.md), [Support for GEMM-K=0 for Blackwell Grouped GEMMs](../sources/prs/cutlass/PR-2746.md), [[Cutlass gemm] Fix SM100 FP8 nosmem epilogue-fusion shape_div 'Divisibility Condition' for non-multiple-of-64 N tiles](../sources/prs/cutlass/PR-2946.md), [[CuTeDSL] Fix: SM100 block-scale gemm overlapping accumulator](../sources/prs/cutlass/PR-2995.md), [perf: Enable SplitK and fix tile-scheduling for moe fp4 fused moe](../sources/prs/flashinfer/PR-1548.md), [TGV GEMM as a BF16 backend alternative to cuBLAS](../sources/prs/flashinfer/PR-1668.md), [perf: improve attention of tcgen05 flash-attention](../sources/prs/flashinfer/PR-1681.md), [[cute_dsl] add gemm + all reduce (two_shot)](../sources/prs/flashinfer/PR-1695.md), [Add head_dim=64 for tcgen05 tcgen05 flash-attention implementation](../sources/prs/flashinfer/PR-1850.md), [integrate blockwise fp8 kernel](../sources/prs/sglang/PR-3529.md), [Add Cutlass MLA attention backend](../sources/prs/sglang/PR-5390.md), [[WIP] support more dtypes for tcgen05](../sources/prs/tilelang/PR-1229.md), [[Enhancement] add more dtype and fix mma.ws for fp16 for tcgen05](../sources/prs/tilelang/PR-1327.md), [[Feature] Support tcgen5mma lowering for `.kind::i8`](../sources/prs/tilelang/PR-1764.md), [[CUDA] Support tcgen5mma gemm ts](../sources/prs/tilelang/PR-1866.md), [[Feature] 2-SM support for TMA, TMEM and TCGEN5MMA on Blackwell](../sources/prs/tilelang/PR-1882.md), [[Feature] Block-scaled GEMM support for MXFP8 on Blackwell](../sources/prs/tilelang/PR-1945.md), [[Transform] Add InjectTcgen05Fence pass](../sources/prs/tilelang/PR-2003.md), [[codex] Split GEMM implementations by backend](../sources/prs/tilelang/PR-2153.md), [[TIR][IR] Update to use tirx](../sources/prs/tilelang/PR-2216.md), [[Python] Drop Python 3.9 support](../sources/prs/tilelang/PR-2218.md), [[NVIDIA] Support nvfp4 tcgen05 gemm](../sources/prs/vllm/PR-13571.md), [add tcgen05 support for tcgen05 fp8 gemm](../sources/prs/vllm/PR-13798.md), [[NVIDIA] Support Cutlass MLA for Blackwell GPUs](../sources/prs/vllm/PR-16032.md), [[Perf] Further tunings for SM100 FP8 CUTLASS kernel](../sources/prs/vllm/PR-19566.md), [[Bugfix] Fix default enable for CUTLASS MLA on SM100](../sources/prs/vllm/PR-22738.md), [Two-SM Cooperative MMA](../wiki/hardware/2sm-cooperative.md), [mbarrier (Memory Barrier Primitives)](../wiki/hardware/mbarrier.md), [tcgen05.mma — Blackwell MMA Instruction](../wiki/hardware/tcgen05-mma.md), [Tensor Memory (TMEM)](../wiki/hardware/tmem.md), [FlashAttention-4](../wiki/kernels/flash-attention-4.md), [FP8 Fine-Grained-Scale GEMM](../wiki/kernels/fp8-block-scale-gemm.md), [NVFP4 GEMM — GPU Mode Problem 2 Contract](../wiki/kernels/nvfp4-gemm.md), [CUDA C++ for Blackwell Kernels](../wiki/languages/cuda-cpp.md), [CuTe DSL for Blackwell](../wiki/languages/cute-dsl.md), [PTX Instructions for SM100](../wiki/languages/ptx-sm100.md), [Register Accumulators to TMEM](../wiki/migration/register-to-tmem.md), [Migrating from wgmma to tcgen05](../wiki/migration/wgmma-to-tcgen05.md), [Not Reaching the Relevant Compute Ceiling](../wiki/patterns/compute-bound.md), [Pipeline Stalls](../wiki/patterns/pipeline-stalls.md), [Warp Specialization on Blackwell](../wiki/techniques/warp-specialization.md) | +| `tma` | [Colfax Article Source Kernels](../sources/blogs/colfax-article-source-kernels.md), [Colfax CUTLASS Kernels](../sources/blogs/colfax-cutlass-kernels.md), [Writing High-Performance Matrix Multiplication Kernels for Blackwell with JAX Pallas](../sources/blogs/jax-pallas-blackwell-matmul.md), [Modular: Matrix Multiplication on Blackwell](../sources/blogs/modular-blackwell-matmul.md), [simveit effective_transpose](../sources/blogs/simveit-effective-transpose.md), [TFLOPS Gap: Why FP4 MoE Kernel Engineering Matters on Blackwell](../sources/blogs/tflops-gap-fp4-moe.md), [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [cuTile Python DSL Reference](../sources/docs/cutile-python-dsl.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [NVIDIA Blackwell Tuning Guide](../sources/docs/nvidia-blackwell-tuning-guide.md), [CUDA 13.0.2 TMA Documentation](../sources/docs/nvidia-cuda-13-0-2-tma.md), [NVIDIA CUDA Toolkit 13.x for Blackwell](../sources/docs/nvidia-cuda-13.md), [NVIDIA CUTLASS 4.5.0 Blackwell Sources](../sources/docs/nvidia-cutlass-blackwell.md), [PTX ISA 9.0 SM100 Instruction Reference](../sources/docs/nvidia-ptx-isa-sm100.md), [ Solving bank conflict via padding and TMA 3D store](../sources/prs/DeepGEMM/PR-78.md), [Use 1D TMA store instead of 3D](../sources/prs/DeepGEMM/PR-83.md), [Support TMA multicast on B with m_grouped_gemm_contiguous.](../sources/prs/DeepGEMM/PR-88.md), [[TRTLLM-10022][feat] Add hopper xqa decode support for skip softmax attention](../sources/prs/TensorRT-LLM/PR-10264.md), [[https://nvbugs/5669671][fix] Support GuidedDecoder with sharded logits (pick #10698)](../sources/prs/TensorRT-LLM/PR-10742.md), [[TRTLLM-9831][perf] Use TMA.RED to improve effective memory bandwidth](../sources/prs/TensorRT-LLM/PR-10987.md), [[TRTLLM-11092][feat] add support for visual gen FA4 attention backend](../sources/prs/TensorRT-LLM/PR-11697.md), [[None][feat] Support sparse mqa/gqa attention](../sources/prs/TensorRT-LLM/PR-12470.md), [[None][feat] Trtllm-gen FMHA JIT support](../sources/prs/TensorRT-LLM/PR-12612.md), [[TRTLLM-11485][feat] Feature rework: Add SageAttention refreshed kernels (attentionOp only)](../sources/prs/TensorRT-LLM/PR-12937.md), [[None][perf] Drop cubin and Eliminate ~6s FMHA JIT recompile in eager generation by aligning kernel selection with CUDA graph warmup](../sources/prs/TensorRT-LLM/PR-13505.md), [[None][feat] Add DeepSeekV4 attention kernels](../sources/prs/TensorRT-LLM/PR-13652.md), [[None][feat] Update the logic of FMHA JIT path](../sources/prs/TensorRT-LLM/PR-14291.md), [[OMNIML-2336][feat] Add NVFP4 x FP8](../sources/prs/TensorRT-LLM/PR-6809.md), [[None][feat] GPT-OSS Sm120/Sm121 Support](../sources/prs/TensorRT-LLM/PR-7937.md), [[None][feat] Update TRTLLM MoE cubins; reduce mxfp4 weight padding requirement; tighten TMA bound](../sources/prs/TensorRT-LLM/PR-9025.md), [[https://nvbugs/5726962][feat] Apply fusion for W4AFP8_AWQ MoE](../sources/prs/TensorRT-LLM/PR-9838.md), [Improve sm90 mixed dtype kernel](../sources/prs/cutlass/PR-1883.md), [[EVT] Add support for Row/Col broadcast PtrArray](../sources/prs/cutlass/PR-2033.md), [Groupwise scaling along M for FP8 gemm](../sources/prs/cutlass/PR-2037.md), [Improvements for: Groupwise scaling along M for FP8 gemm](../sources/prs/cutlass/PR-2095.md), [Hopper Grouped GEMM support for FP8 Accum](../sources/prs/cutlass/PR-2123.md), [Flash MLA support](../sources/prs/cutlass/PR-2130.md), [Flash MLA Support - Step 2](../sources/prs/cutlass/PR-2134.md), [Fix sm100 gemm wrong static constexpr that breaks compilation on Windows](../sources/prs/cutlass/PR-2167.md), [Fix SM90 beta=1 hang and stream-K launch errors](../sources/prs/cutlass/PR-2172.md), [hopper-blockwise-generalization-optimization](../sources/prs/cutlass/PR-2270.md), [[ex77] fix mla split; add fwd lse; add bwd varlen](../sources/prs/cutlass/PR-2366.md), [Example 77 add blackwell flash-attention bwd for MLA shape](../sources/prs/cutlass/PR-2466.md), [Add Blackwell MLA forward (shape: d=192, dv=128) implementation](../sources/prs/cutlass/PR-2472.md), [fix: examples/cute/tutorial/blackwell/04_mma_tma_2sm_sm100.cu GridDim miscalculated](../sources/prs/cutlass/PR-2492.md), [Support PDL for SM90 Array TMA GEMM](../sources/prs/cutlass/PR-2719.md), [[Bug Fix]Bypass launch grids for SM120 Kernel with SM90 Mainloop & SM100 TileScheduler](../sources/prs/cutlass/PR-2865.md), [new example with TMA prefetch feature targeting for DRAM latency boun…](../sources/prs/cutlass/PR-2881.md), [[Bug Fix]Set NumSplitsM to 1 when TileShapeM < 128 in sm90 fp8 blockwise scaling CollectiveMma](../sources/prs/cutlass/PR-2965.md), [Replace std::min with cute::min in sm120 blockwise scaling device functions](../sources/prs/cutlass/PR-3055.md), [Small Tile N BlockScaled GEMM + Grouped GEMM on SM12x](../sources/prs/cutlass/PR-3176.md), [Add var-seq-len to FA3 fp16 / bf16 fwd](../sources/prs/flash-attention/PR-1072.md), [Fp8 kernel with "in-kernel" transpose of V in producer](../sources/prs/flash-attention/PR-1100.md), [FA3 FP8 qkv descales + restore max offset for h128 causal + added sync for producer WG](../sources/prs/flash-attention/PR-1173.md), [Add seqused_q in fwd / bwd and seqused_k in bwd in hopper FA.](../sources/prs/flash-attention/PR-1182.md), [Add local attention in Hopper FAv3](../sources/prs/flash-attention/PR-1233.md), [FA3 kvcache + split kv + gqa parallelization](../sources/prs/flash-attention/PR-1236.md), [Paged Attention support for FA3](../sources/prs/flash-attention/PR-1268.md), [FA3 paged attention: Readiness for Cutlass 3.6 / default value for block_table](../sources/prs/flash-attention/PR-1331.md), [Fix FA3 Varlen Performance regression](../sources/prs/flash-attention/PR-1361.md), [Support hdimQK != hdimV backward](../sources/prs/flash-attention/PR-1604.md), [Improve causal backward determinism perf with SPT schedule](../sources/prs/flash-attention/PR-1893.md), [[Cute,Sm100,Fwd] use correction warps for epi when not using TMA](../sources/prs/flash-attention/PR-2014.md), [[Cute,Fwd,Sm100] don't pass mask_fn to softmax_step generically](../sources/prs/flash-attention/PR-2026.md), [[Cute,Fwd] Extend score_mod to variable sequence length](../sources/prs/flash-attention/PR-2043.md), [Add score-mod bwd support ](../sources/prs/flash-attention/PR-2070.md), [[Cute,Fwd,Sm100] fp8 e4m3 and e5m2 support](../sources/prs/flash-attention/PR-2109.md), [[Fwd,Sm90] Add paged KV attention support (tma and cp.async)](../sources/prs/flash-attention/PR-2360.md), [[Cute,Sm100,Fwd] add MLA 64/512 with topk sparsity for MQA 128 heads](../sources/prs/flash-attention/PR-2441.md), [[hd256] Add TMA paged KV support to SM100 2CTA forward kernel](../sources/prs/flash-attention/PR-2489.md), [[FA4][hd256] Backward TMA bulk-store epilogue + LSE/dpsum coalesce](../sources/prs/flash-attention/PR-2497.md), [feat: Softmax free sampling](../sources/prs/flashinfer/PR-1035.md), [[nvidia] initial support for blackwell kernels](../sources/prs/flashinfer/PR-1039.md), [bugfix: adding lse output to blackwell fmha kernels](../sources/prs/flashinfer/PR-1071.md), [bugfix: follow user-specified sm_scale for blackwell cutlass fmha](../sources/prs/flashinfer/PR-1072.md), [bugfix: host-precomuted plan function for blackwell fmha](../sources/prs/flashinfer/PR-1106.md), [Add CUTLASS fused moe kernels from TensorRT-LLM.](../sources/prs/flashinfer/PR-1113.md), [feat: Fused temperature online softmax kernel](../sources/prs/flashinfer/PR-1153.md), [feat: logits processor fustion rule for temperature softmax](../sources/prs/flashinfer/PR-1170.md), [bugfix: softmax NaN results caused by large -inf masks](../sources/prs/flashinfer/PR-1178.md), [bugfix: fix blackwell fmha hanging issue for empty kv_len](../sources/prs/flashinfer/PR-1198.md), [feat: trtllm-gen fp8 moe kernels](../sources/prs/flashinfer/PR-1212.md), [Update cutlass fp4 moe kernels](../sources/prs/flashinfer/PR-1294.md), [Add blockwise-scaled FP8 GEMM via TRTLLM-Gen.](../sources/prs/flashinfer/PR-1320.md), [GPT-OSS Support: Add Blackwell MoE mxfp4 implementation from TRTLLM and Attention Sink](../sources/prs/flashinfer/PR-1389.md), [gpt-oss: Add MXFP8 x MXFP4 CUTLASS MOE for SM100 and BF16 x MXFP4 CUTLASS for SM90 + SwigluBias Activation](../sources/prs/flashinfer/PR-1396.md), [Remove getEnvEnablePDL in favor of enable_pdl parameter](../sources/prs/flashinfer/PR-1446.md), [perf: Enable SplitK and fix tile-scheduling for moe fp4 fused moe](../sources/prs/flashinfer/PR-1548.md), [bugfix: fix fused-temperature softmax IMA issue](../sources/prs/flashinfer/PR-1596.md), [feat: initial support for SM103, SM110, SM120, SM121](../sources/prs/flashinfer/PR-1608.md), [TGV GEMM as a BF16 backend alternative to cuBLAS](../sources/prs/flashinfer/PR-1668.md), [perf: improve attention of tcgen05 flash-attention](../sources/prs/flashinfer/PR-1681.md), [[cute_dsl] add gemm + all reduce (two_shot)](../sources/prs/flashinfer/PR-1695.md), [feat: add xqa fp8 mha and fp8 kv cache](../sources/prs/flashinfer/PR-1769.md), [feat:enable fp8 blockscale moe for fused cultass for sm90](../sources/prs/flashinfer/PR-1819.md), [feat: trtrllm-gen global scaled FP8 GEMMs](../sources/prs/flashinfer/PR-1829.md), [Add head_dim=64 for tcgen05 tcgen05 flash-attention implementation](../sources/prs/flashinfer/PR-1850.md), [Tune kernel compilation parameters for https://github.com/flashinfer-ai/flashinfer/pull/1850 ](../sources/prs/flashinfer/PR-1878.md), [Update trtllm-gen fused moe routing kernel and add more kernels](../sources/prs/flashinfer/PR-1955.md), [Feature: Add support for L40 FusedMoE in cutlass path](../sources/prs/flashinfer/PR-1973.md), [update trtllm cutlass moe ](../sources/prs/flashinfer/PR-2020.md), [perf: improve sampling/mask/softmax performance (part 1/2)](../sources/prs/flashinfer/PR-2044.md), [Rebase FP8 SM100 Cutlass FMHA Attention to main (original PR#1238)](../sources/prs/flashinfer/PR-2047.md), [update xqa license](../sources/prs/flashinfer/PR-2117.md), [feat: TRTLLM FMHAv2 backend for ctx attention](../sources/prs/flashinfer/PR-2142.md), [feat: MxInt4 x Bf16 TRT-LLM Gen MoE support](../sources/prs/flashinfer/PR-2159.md), [refactor: pull trtllm-gen batch-gemm/gemm headers from artifactory; update tma descriptor shape init](../sources/prs/flashinfer/PR-2235.md), [feat: add GDN Attention](../sources/prs/flashinfer/PR-2276.md), [A Blackwell-optimized version of selective_state_update (decode)](../sources/prs/flashinfer/PR-2387.md), [feat: update trtllm-gen MoE cubins](../sources/prs/flashinfer/PR-2416.md), [refactor: reduce hopper's gdn prefill compilation time and fix docstring.](../sources/prs/flashinfer/PR-2422.md), [feat: Add TRTLLM fmha_v2 library for SM90 attention with Skip-Softmax ](../sources/prs/flashinfer/PR-2446.md), [feat: Add TRTLLM-Gen Skip-Softmax kernels for prefill and decode](../sources/prs/flashinfer/PR-2477.md), [feat: Enable TRTLLM-Gen Skip-Softmax attention for MLA](../sources/prs/flashinfer/PR-2547.md), [Implement `cutlass_fused_moe` mxfp8](../sources/prs/flashinfer/PR-2581.md), [[feat] trtllm-gen mxfp8 gemm](../sources/prs/flashinfer/PR-2653.md), [misc: Update gemm/batched gemm cubins from trtllm-gen, gemm header refactor](../sources/prs/flashinfer/PR-2740.md), [Upgrade cutlass 4.2.1 -> 4.4.2](../sources/prs/flashinfer/PR-2798.md), [[CuTe DSL] Add modular FMHA prefill and MLA decode attention kernels](../sources/prs/flashinfer/PR-2805.md), [feat(gdn): state checkpointing in chunk_gated_delta_rule](../sources/prs/flashinfer/PR-2908.md), [Integrate CUTLASS Small Tile N Blockscaled GEMMs/Grouped GEMMs for SM120 and SM121](../sources/prs/flashinfer/PR-3152.md), [feat(trace): embed runnable init() in every TraceTemplate](../sources/prs/flashinfer/PR-3221.md), [[inductor][triton 3.3] Fix cpp_wrapper w/ TMA in triton 3.3](../sources/prs/pytorch/PR-149993.md), [[user triton] AOT inductor support for device-side TMA](../sources/prs/pytorch/PR-157241.md), [[kernel][moe] add moe topk fast](../sources/prs/sglang/PR-13969.md), [Move fa4 from sgl-kernel to jit kernel](../sources/prs/sglang/PR-17353.md), [Support skip-softmax attention](../sources/prs/sglang/PR-19089.md), [[Kernel] Fuse temperature + softmax in sampling for decode speedup](../sources/prs/sglang/PR-20501.md), [add tensorrt_llm common and cutlass_extensions as 3rdparty](../sources/prs/sglang/PR-3216.md), [support blockwise fp8 matmul kernel](../sources/prs/sglang/PR-3267.md), [integrate blockwise fp8 kernel](../sources/prs/sglang/PR-3529.md), [DeepGemm integrate to gemm](../sources/prs/sglang/PR-4165.md), [Create col-major and tma-aligned x_scale for deep_gemm.gemm_fp8_fp8_bf16_nt](../sources/prs/sglang/PR-4515.md), [Add Cutlass MLA attention backend](../sources/prs/sglang/PR-5390.md), [[perf] introduce deep gemm group_gemm_masked as gemm](../sources/prs/sglang/PR-5432.md), [[perf][sgl-kernel] extend cutlass_mla_decode to support num_head < 128](../sources/prs/sglang/PR-6929.md), [[1/n]: add cutlass W4A8 moe kernel for hopper architecture](../sources/prs/sglang/PR-7772.md), [[NVIDIA] Fix missing `get_col_major_tma_aligned_tensor` for Blackwell deepgemm in EpMoE](../sources/prs/sglang/PR-8955.md), [[Feature] Support `cp.reduce.async.bulk.tensor`](../sources/prs/tilelang/PR-1667.md), [[BugFix] Fix Hopper TMA lowering without warp specialization](../sources/prs/tilelang/PR-1840.md), [[Feature] 2-SM support for TMA, TMEM and TCGEN5MMA on Blackwell](../sources/prs/tilelang/PR-1882.md), [[Feature] Add T.copy_cluster to support TMA multicast and SM-to-SM cluster copy](../sources/prs/tilelang/PR-1908.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [[Feature] Support TMA store in T.tma_copy()](../sources/prs/tilelang/PR-1981.md), [[Backend] Refactor gemm_sp](../sources/prs/tilelang/PR-2048.md), [[Bugfix] Enable `.shared::cta` in TMA copy paths only on CUDA 12.8+](../sources/prs/tilelang/PR-2087.md), [[TMA] Support FP4 TensorMap TMA copies](../sources/prs/tilelang/PR-2107.md), [[CUDA][TMA] Add TMA tile::gather4 / tile::scatter4 support](../sources/prs/tilelang/PR-2129.md), [[Kernel] Update `cutlass_scaled_mm` to support 2d group (blockwise) scaling](../sources/prs/vllm/PR-11868.md), [Add: Support for Sparse24Bitmask Compressed Models](../sources/prs/vllm/PR-12097.md), [[NVIDIA] Support nvfp4 tcgen05 gemm](../sources/prs/vllm/PR-13571.md), [add tcgen05 support for tcgen05 fp8 gemm](../sources/prs/vllm/PR-13798.md), [[BugFix] Illegal Memory Access in the blockwise cutlass fp8 GEMMs](../sources/prs/vllm/PR-14396.md), [Modularize fused experts and integrate PPLX kernels](../sources/prs/vllm/PR-15956.md), [[NVIDIA] Support Cutlass MLA for Blackwell GPUs](../sources/prs/vllm/PR-16032.md), [Fix `numel()` downcast in vllm/csrc/moe/moe_align_sum_kernels.cu +2](../sources/prs/vllm/PR-17082.md), [[Perf] Further tunings for SM100 FP8 CUTLASS kernel](../sources/prs/vllm/PR-19566.md), [[feat]: CUTLASS block scaled group gemm for SM100](../sources/prs/vllm/PR-19757.md), [SM100 Cutlass MLA decode with unrestricted num_heads (< 128) for DeepSeek TP](../sources/prs/vllm/PR-20769.md), [[Perf] Use upstream CUTLASS for SM90 Block FP8 kernel](../sources/prs/vllm/PR-23280.md), [[Kernel] Add topk_sigmoid kernel](../sources/prs/vllm/PR-31246.md), [Add TMA support to fused_moe_lora kernel](../sources/prs/vllm/PR-32195.md), [[Perf] Create TMA-aligned input scale tensor for DeepGemm on Hopper](../sources/prs/vllm/PR-32619.md), [[Bugfix] Fix quant RMS norm fusion for quantization with TMA-aligned scales](../sources/prs/vllm/PR-33255.md), [[Kernels][MoE] Fix legacy_routing to use bitmatrix-based routing path](../sources/prs/vllm/PR-38504.md), [[Perf][GDN] Align TMA usage with upstream FLA](../sources/prs/vllm/PR-38981.md), [fix: clamp NaN/Inf in topk_softmax to prevent duplicate expert IDs](../sources/prs/vllm/PR-39391.md), [mbarrier (Memory Barrier Primitives)](../wiki/hardware/mbarrier.md), [Tensor Memory Accelerator (TMA)](../wiki/hardware/tma.md), [FlashAttention SM100 MLA TopK Sparse Forward](../wiki/kernels/flash-attention-sm100-mla-topk.md), [NVFP4 GEMM — GPU Mode Problem 2 Contract](../wiki/kernels/nvfp4-gemm.md), [CuTe DSL for Blackwell](../wiki/languages/cute-dsl.md), [PTX Instructions for SM100](../wiki/languages/ptx-sm100.md), [Pipeline Stalls](../wiki/patterns/pipeline-stalls.md), [External Source-Map Research For Kernel Edits](../wiki/techniques/external-source-map-research.md), [Software Pipelining and Multi-Stage Buffering](../wiki/techniques/pipeline-stages.md), [Shared Memory Swizzling](../wiki/techniques/swizzling.md) | +| `tmem` | [Microbenchmarking NVIDIA's Blackwell Architecture](../sources/blogs/blackwell-microbenchmarking.md), [Colfax CUTLASS Tutorial: GEMM Kernels Using Tensor Memory for Blackwell](../sources/blogs/colfax-cutlass-blackwell.md), [FlashAttention-4 Blog](../sources/blogs/flash-attention-4.md), [FlashMLA — Multi-head Latent Attention](../sources/blogs/flashmla.md), [Writing High-Performance Matrix Multiplication Kernels for Blackwell with JAX Pallas](../sources/blogs/jax-pallas-blackwell-matmul.md), [Modular: Matrix Multiplication on Blackwell](../sources/blogs/modular-blackwell-matmul.md), [tcgen05 for dummies](../sources/blogs/tcgen05-tutorial.md), [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [cuTile Python DSL Reference](../sources/docs/cutile-python-dsl.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [FlashAttention-4: Algorithm and Kernel Co-design for Blackwell GPUs](../sources/docs/flash-attention-4.md), [NVIDIA Blackwell Tuning Guide](../sources/docs/nvidia-blackwell-tuning-guide.md), [NVIDIA CUDA Toolkit 13.x for Blackwell](../sources/docs/nvidia-cuda-13.md), [NVIDIA CUTLASS 4.5.0 Blackwell Sources](../sources/docs/nvidia-cutlass-blackwell.md), [PTX ISA 9.0 SM100 Instruction Reference](../sources/docs/nvidia-ptx-isa-sm100.md), [Triton v3.3.0 — Blackwell TCGen5/TMEM Boundary](../sources/docs/triton-3.3-blackwell.md), [Triton v3.6.0 — Incremental Blackwell Changes](../sources/docs/triton-3.6-blackwell.md), [[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes](../sources/prs/DeepGEMM/PR-304.md), [[TRTLLM-9992][perf] Enable PDL for CuteDSL kernels and overlap MoeOutputMemset](../sources/prs/TensorRT-LLM/PR-10043.md), [Blockwise and Groupwise GEMM for Blackwell and Improvements for Hopper](../sources/prs/cutlass/PR-2139.md), [Blockwise Improvement and Programmatic Dependent Launch](../sources/prs/cutlass/PR-2161.md), [Example 77 add blackwell flash-attention bwd for MLA shape](../sources/prs/cutlass/PR-2466.md), [Add Blackwell MLA forward (shape: d=192, dv=128) implementation](../sources/prs/cutlass/PR-2472.md), [Support for GEMM-K=0 for Blackwell Grouped GEMMs](../sources/prs/cutlass/PR-2746.md), [[CuTeDSL] Fix: SM100 block-scale gemm overlapping accumulator](../sources/prs/cutlass/PR-2995.md), [TGV GEMM as a BF16 backend alternative to cuBLAS](../sources/prs/flashinfer/PR-1668.md), [perf: improve attention of tcgen05 flash-attention](../sources/prs/flashinfer/PR-1681.md), [[cute_dsl] add gemm + all reduce (two_shot)](../sources/prs/flashinfer/PR-1695.md), [Add head_dim=64 for tcgen05 tcgen05 flash-attention implementation](../sources/prs/flashinfer/PR-1850.md), [[CuTe DSL] Add modular FMHA prefill and MLA decode attention kernels](../sources/prs/flashinfer/PR-2805.md), [Add Cutlass MLA attention backend](../sources/prs/sglang/PR-5390.md), [[Feature] 2-SM support for TMA, TMEM and TCGEN5MMA on Blackwell](../sources/prs/tilelang/PR-1882.md), [[NVIDIA] Support Cutlass MLA for Blackwell GPUs](../sources/prs/vllm/PR-16032.md), [[Bugfix] Fix default enable for CUTLASS MLA on SM100](../sources/prs/vllm/PR-22738.md), [tcgen05.mma — Blackwell MMA Instruction](../wiki/hardware/tcgen05-mma.md), [Tensor Memory (TMEM)](../wiki/hardware/tmem.md), [FlashAttention-4](../wiki/kernels/flash-attention-4.md), [NVFP4 GEMM — GPU Mode Problem 2 Contract](../wiki/kernels/nvfp4-gemm.md), [CUDA C++ for Blackwell Kernels](../wiki/languages/cuda-cpp.md), [CuTe DSL for Blackwell](../wiki/languages/cute-dsl.md), [PTX Instructions for SM100](../wiki/languages/ptx-sm100.md), [Register Accumulators to TMEM](../wiki/migration/register-to-tmem.md), [Migrating from wgmma to tcgen05](../wiki/migration/wgmma-to-tcgen05.md), [Register Pressure and Residency](../wiki/patterns/register-pressure.md), [Double/Multi-Buffering Patterns](../wiki/techniques/double-buffering.md), [Epilogue Fusion](../wiki/techniques/epilogue-fusion.md), [Kernel Fusion](../wiki/techniques/kernel-fusion.md), [Ping-Pong Scheduling](../wiki/techniques/ping-pong-scheduling.md), [Warp Specialization on Blackwell](../wiki/techniques/warp-specialization.md) | +| `wgmma` | [Microbenchmarking NVIDIA's Blackwell Architecture](../sources/blogs/blackwell-microbenchmarking.md), [Colfax Article Source Kernels](../sources/blogs/colfax-article-source-kernels.md), [Colfax CUTLASS Kernels](../sources/blogs/colfax-cutlass-kernels.md), [DeepGEMM — Pinned Upstream Project Summary](../sources/blogs/deepgemm.md), [feat: integrate gemm_fp8 kernel into gemm](../sources/prs/sglang/PR-3056.md), [integrate blockwise fp8 kernel](../sources/prs/sglang/PR-3529.md), [DeepGemm integrate to gemm](../sources/prs/sglang/PR-4165.md), [[perf] introduce deep gemm group_gemm_masked as gemm](../sources/prs/sglang/PR-5432.md), [Add swizzle layout detection and automatic merging for layout conflicts](../sources/prs/tilelang/PR-1736.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [[Backend] Refactor gemm_sp](../sources/prs/tilelang/PR-2048.md), [feat: auto-vectorize bf16/fp16 reduce with packed add2 intrinsics](../sources/prs/tilelang/PR-2112.md), [[codex] Split GEMM implementations by backend](../sources/prs/tilelang/PR-2153.md), [[Python] Drop Python 3.9 support](../sources/prs/tilelang/PR-2218.md), [FP8 Fine-Grained-Scale GEMM](../wiki/kernels/fp8-block-scale-gemm.md), [Migrating from wgmma to tcgen05](../wiki/migration/wgmma-to-tcgen05.md), [External Source-Map Research For Kernel Edits](../wiki/techniques/external-source-map-research.md) | diff --git a/queries/by-kernel-type.md b/queries/by-kernel-type.md index a2d3a0295..b03a4b030 100644 --- a/queries/by-kernel-type.md +++ b/queries/by-kernel-type.md @@ -4,20 +4,21 @@ | Kernel Type | Pages | |-------------|-------| -| `attention` | [FlashAttention-4 Blog](../sources/blogs/flash-attention-4.md), [FlashMLA — Multi-head Latent Attention](../sources/blogs/flashmla.md), [Gated Delta Networks](../sources/blogs/gated-delta-net.md), [K-Search: LLM Kernel Generation via Co-Evolving Intrinsic World Model](../sources/blogs/k-search-kernel-generation.md), [Native Sparse Attention (NSA)](../sources/blogs/nsa.md), [Qwen3-Next: Hybrid GDN+MoE Architecture on NVIDIA Blackwell](../sources/blogs/qwen3-next-architecture.md), [DeepSeek-V3.2-Exp in vLLM: Fine-Grained Sparse Attention in Action](../sources/blogs/vllm-deepseek-v3-sparse-attention.md), [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [FlashAttention-4: Hardware-Friendly Attention on Blackwell](../sources/docs/flash-attention-4.md), [Sync nv_dev with upstream #316 (Mega MoE optimizations & benchmarks)](../sources/prs/DeepGEMM/PR-328.md), [[TRTLLM-9457][feat] Add cute dsl fp8 gemm for Blackwell](../sources/prs/TensorRT-LLM/PR-10130.md), [[TRTLLM-9798][feat] Change to use new DeepGEMM MQA sm100 kernel for MTP-3](../sources/prs/TensorRT-LLM/PR-10226.md), [[TRTLLM-10022][feat] Add hopper xqa decode support for skip softmax attention](../sources/prs/TensorRT-LLM/PR-10264.md), [[None][feat] Remove non flash attetnion style fmha_v2 kernel for hopper](../sources/prs/TensorRT-LLM/PR-11381.md), [[TRTLLM-11092][feat] add support for visual gen FA4 attention backend](../sources/prs/TensorRT-LLM/PR-11697.md), [[TRTLLM-11119][feat] Blackwell SageAttention, Integrate into AttentionOp API](../sources/prs/TensorRT-LLM/PR-11718.md), [[None][feat] Add fused DiT QK Norm + RoPE CUDA kernel for FLUX](../sources/prs/TensorRT-LLM/PR-11869.md), [[TRTLLM-10421][perf] Add fused cat+fp8_quantize CUDA kernel for DSA indexer](../sources/prs/TensorRT-LLM/PR-11899.md), [[None][feat] GLM 5 support and DSA MTP fixes](../sources/prs/TensorRT-LLM/PR-11990.md), [[#11694][feat] AutoDeploy: Improve the piecewise CG memory usage](../sources/prs/TensorRT-LLM/PR-11993.md), [[TRTLLM-11285][feat] Fuse indexer wk + weights_proj into single GEMM in TF32 for DS-V3.2](../sources/prs/TensorRT-LLM/PR-12055.md), [[TRTLLM-11540][feat] Add EAGLE3 dynamic tree speculative decoding support](../sources/prs/TensorRT-LLM/PR-12062.md), [[TRTLLM-11289][feat] Integrate CuteDSL's bf16 dense GEMMs](../sources/prs/TensorRT-LLM/PR-12074.md), [[TRTLLM-10407][perf] Enable CuteDSL indexer_top_k in model](../sources/prs/TensorRT-LLM/PR-12236.md), [[None][feat] Support update weight for nvfp4](../sources/prs/TensorRT-LLM/PR-12320.md), [[https://nvbugs/5983390][perf] Kernel fusions in _gather_k_cache_for_chunk of Indexer in DSA](../sources/prs/TensorRT-LLM/PR-12322.md), [[None][feat] Temporally-Correlated Heuristic-guided Indexer TopK for Sparse Attention](../sources/prs/TensorRT-LLM/PR-12385.md), [[https://nvbugs/5983390][fix] Remove redundant D2H sync to optimize perf](../sources/prs/TensorRT-LLM/PR-12445.md), [[None][feat] Support sparse mqa/gqa attention](../sources/prs/TensorRT-LLM/PR-12470.md), [[https://nvbugs/5983390][perf] Split MLA DSA custom op for piecewise CUDA graph capture](../sources/prs/TensorRT-LLM/PR-12503.md), [[https://nvbugs/5983390][perf] Multiple host perf optimizations for DSA part](../sources/prs/TensorRT-LLM/PR-12581.md), [[None][feat] Add triton paged attention for AutoDeploy](../sources/prs/TensorRT-LLM/PR-12642.md), [[None][feat] Optimize mamba SSD prefill and extend flashinfer dispatch](../sources/prs/TensorRT-LLM/PR-12731.md), [[TRTLLM-11485][feat] Feature rework: Add SageAttention refreshed kernels (attentionOp only)](../sources/prs/TensorRT-LLM/PR-12937.md), [[#12716][feat] Fused cross-head QK Norm + RoPE kernel for WAN](../sources/prs/TensorRT-LLM/PR-13052.md), [[TRTLLM-34871][feat] Add cute dsl FP8 paged MQA logits decode kernel](../sources/prs/TensorRT-LLM/PR-13219.md), [[None][feat] Integrate FP4 indexer for DSA on Blackwell](../sources/prs/TensorRT-LLM/PR-13340.md), [[TRTLLM-11285][perf] Force enable TF32 tensor cores for DSA indexer fused GEMM](../sources/prs/TensorRT-LLM/PR-13452.md), [[None][perf] Scheme X L2-aware dispatcher and PDL launchers for sparse-attention GVR Top-K](../sources/prs/TensorRT-LLM/PR-13477.md), [[None][perf] Drop cubin and Eliminate ~6s FMHA JIT recompile in eager generation by aligning kernel selection with CUDA graph warmup](../sources/prs/TensorRT-LLM/PR-13505.md), [[TRTLLM-12128][feat] enable SageAttention for Wan/FLUX (new commits)](../sources/prs/TensorRT-LLM/PR-13570.md), [[TRTLLM-12316][feat] Integrate FP4 indexer for DSv4](../sources/prs/TensorRT-LLM/PR-13575.md), [[#13580][fix] AutoDeploy: Support Gemma3n/4 E2B variants](../sources/prs/TensorRT-LLM/PR-13630.md), [[None][feat] Add DeepSeekV4 attention kernels](../sources/prs/TensorRT-LLM/PR-13652.md), [[None][perf] Optimize DeepSeek-V4 compressor BF16 input](../sources/prs/TensorRT-LLM/PR-13761.md), [[None][fix] Use compressed lengths for DeepSeek-V4 indexer](../sources/prs/TensorRT-LLM/PR-13802.md), [[None][feat] Update FMHA cubins for head_dim 80](../sources/prs/TensorRT-LLM/PR-13808.md), [[TRTLLM-12503][feat] Parallel VAE independent scaling and fix arg passing](../sources/prs/TensorRT-LLM/PR-13873.md), [[TRTLLM-35237][feat] Add cute dsl FP4 paged MQA logits decode kernel](../sources/prs/TensorRT-LLM/PR-13929.md), [[None][feat] Keep DSv4 o_a_proj as FP8, and port vLLM's fused_inv_rope_fp8_quant](../sources/prs/TensorRT-LLM/PR-13938.md), [[None][perf] Add CUDA q_b norm for DeepSeek V4](../sources/prs/TensorRT-LLM/PR-13975.md), [[None][feat] Enable 2 DSv4 perf optimizations by default](../sources/prs/TensorRT-LLM/PR-14120.md), [[None][feat] Add chunked prefill support for Gemma4 (text + vision multimodal)](../sources/prs/TensorRT-LLM/PR-14134.md), [[None][fix] Add SPDX Apache-2.0 headers to auto_deploy test files](../sources/prs/TensorRT-LLM/PR-14193.md), [[None][feat] DSv4: enable GVR Heuristic Top-K for compress_ratio=4](../sources/prs/TensorRT-LLM/PR-14219.md), [[None][refactor] clean up AttentionForwardArgs](../sources/prs/TensorRT-LLM/PR-14244.md), [[None][fix] Handle unset attention_dp_relax in ADP routers](../sources/prs/TensorRT-LLM/PR-14276.md), [[None][feat] GPT-OSS Sm120/Sm121 Support](../sources/prs/TensorRT-LLM/PR-7937.md), [[TRTLLM-8535][feat] Support DeepSeek V3.2 with FP8 + BF16 KV cache/NVFP4 + BF16 KV cache](../sources/prs/TensorRT-LLM/PR-8405.md), [[None][fix] Fix the performance issue of FP8 blockwise grouped GEMM when using attention DP](../sources/prs/TensorRT-LLM/PR-8501.md), [Example 77 add blackwell flash-attention bwd for MLA shape](../sources/prs/cutlass/PR-2466.md), [Add Blackwell MLA forward (shape: d=192, dv=128) implementation](../sources/prs/cutlass/PR-2472.md), [Fp8 kernel with "in-kernel" transpose of V in producer](../sources/prs/flash-attention/PR-1100.md), [FA3 FP8 qkv descales + restore max offset for h128 causal + added sync for producer WG](../sources/prs/flash-attention/PR-1173.md), [Add seqused_q in fwd / bwd and seqused_k in bwd in hopper FA.](../sources/prs/flash-attention/PR-1182.md), [Add local attention in Hopper FAv3](../sources/prs/flash-attention/PR-1233.md), [FA3 kvcache + split kv + gqa parallelization](../sources/prs/flash-attention/PR-1236.md), [Paged Attention support for FA3](../sources/prs/flash-attention/PR-1268.md), [FA3 paged attention: Readiness for Cutlass 3.6 / default value for block_table](../sources/prs/flash-attention/PR-1331.md), [Blackwell FlashAttention-BWD (v1.0)](../sources/prs/flash-attention/PR-1945.md), [[Cute,Fwd,Sm100] Support paged attention](../sources/prs/flash-attention/PR-1999.md), [[Cute,Fwd,Sm100] fp8 e4m3 and e5m2 support](../sources/prs/flash-attention/PR-2109.md), [Add SM120 varlen attention support](../sources/prs/flash-attention/PR-2333.md), [[Fwd,Sm90] Add paged KV attention support (tma and cp.async)](../sources/prs/flash-attention/PR-2360.md), [feat: update decode attention APIs](../sources/prs/flashinfer/PR-1007.md), [misc: fix instrument code for mla profiler](../sources/prs/flashinfer/PR-1014.md), [add multi-item scoring](../sources/prs/flashinfer/PR-1015.md), [fix: add zero init for KV tiled copy](../sources/prs/flashinfer/PR-1029.md), [feat: add functional per-head FP8 quantization for FA3](../sources/prs/flashinfer/PR-1033.md), [[nvidia] initial support for blackwell kernels](../sources/prs/flashinfer/PR-1039.md), [[nvidia] Add Blackwell FMHA decode kernel from TRT-LLM](../sources/prs/flashinfer/PR-1051.md), [Fix KV chunking for POD. ](../sources/prs/flashinfer/PR-1054.md), [bugfix: temporally disable split-kv in blackwell mla](../sources/prs/flashinfer/PR-1055.md), [Parameterize prefix mask call (needed by POD-Attention)](../sources/prs/flashinfer/PR-1059.md), [bugfix: adding lse output to blackwell fmha kernels](../sources/prs/flashinfer/PR-1071.md), [bugfix: follow user-specified sm_scale for blackwell cutlass fmha](../sources/prs/flashinfer/PR-1072.md), [bugfix: fix fp8 attention kernels aot compilation issue](../sources/prs/flashinfer/PR-1087.md), [bugfix: host-precomuted plan function for blackwell fmha](../sources/prs/flashinfer/PR-1106.md), [hotfix: fix the blackwell fmha stream](../sources/prs/flashinfer/PR-1116.md), [[Feature] Support PDL for batch Prefill and Decode](../sources/prs/flashinfer/PR-1117.md), [[feat] add unified batch attention w/ correctness tests.](../sources/prs/flashinfer/PR-1137.md), [Fix FA2 and FA3 multi-item scoring and cuda illegal memory access error](../sources/prs/flashinfer/PR-1140.md), [[feat] support block sparse attention w/ variable block sizes and head-wise sparse patterns](../sources/prs/flashinfer/PR-1177.md), [update trtllm-gen decode attention kernel launcher](../sources/prs/flashinfer/PR-1189.md), [bugfix: fix blackwell fmha hanging issue for empty kv_len](../sources/prs/flashinfer/PR-1198.md), [[feat] optimize persistent batch attention perf.](../sources/prs/flashinfer/PR-1200.md), [[fix] fix BatchAttention CTA_TILE_KV mask issue](../sources/prs/flashinfer/PR-1206.md), [feat: add trtllm-gen mla cubin](../sources/prs/flashinfer/PR-1222.md), [add trtllm-gen context attention](../sources/prs/flashinfer/PR-1239.md), [Add trtllm-gen attention mha kernel with FP8 Q/K/V and FP8 output](../sources/prs/flashinfer/PR-1242.md), [Bug fix: fix duplicate launch in POD](../sources/prs/flashinfer/PR-1267.md), [refactor: refactor trtllm-gen attention kernel integration code](../sources/prs/flashinfer/PR-1289.md), [[fix] fix integer overflow in FA2 customized_mask & add buffer overflow warning.](../sources/prs/flashinfer/PR-1290.md), [feat: Add k_scale and v_scale to persistent attention ](../sources/prs/flashinfer/PR-1322.md), [feat: Support logits_soft_cap for Persistent attn; fix kv split limit](../sources/prs/flashinfer/PR-1324.md), [Support passing kv_data_type to MultiLevelCascadeAttentionWrapper.plan()](../sources/prs/flashinfer/PR-1350.md), [Allow BatchPrefillPagedWrapper to call cudnn API](../sources/prs/flashinfer/PR-1384.md), [GPT-OSS Support: Add Blackwell MoE mxfp4 implementation from TRTLLM and Attention Sink](../sources/prs/flashinfer/PR-1389.md), [Adding FP8 benchmark on attention and matmul testing](../sources/prs/flashinfer/PR-1390.md), [benchmark: trtllm-gen mha with sink, add benchmark args](../sources/prs/flashinfer/PR-1415.md), [refactor: Sink attention AoT](../sources/prs/flashinfer/PR-1427.md), [feat: enable trtllm-gen attn speculative decoding verify by decode](../sources/prs/flashinfer/PR-1453.md), [Fix TRTLLM NVFP4-out attention kernel scale factor dim issue](../sources/prs/flashinfer/PR-1460.md), [feat: add pdl for trtllm-gen attn](../sources/prs/flashinfer/PR-1484.md), [feat: Support fp8 qkv, fp16/bf16 out MHA for trtllm-gen.](../sources/prs/flashinfer/PR-1490.md), [feat: integrate xqa attention backend](../sources/prs/flashinfer/PR-1503.md), [flashinfer_benchmark QoL Improvements and Attention FP8 Support](../sources/prs/flashinfer/PR-1512.md), [backend: Refactor trtllm-gen fmha metainfo loading](../sources/prs/flashinfer/PR-1518.md), [bugfix: Fix Persistent kernel precision for masked output ](../sources/prs/flashinfer/PR-1533.md), [feat: Integrate TRTLLM varlen kernel for deepseek R1 prefill ](../sources/prs/flashinfer/PR-1537.md), [feat: Add fp8-qkv, fp16/bf16 output MHA](../sources/prs/flashinfer/PR-1540.md), [bugfix: fix persistent attention kernel correctness on blackwell](../sources/prs/flashinfer/PR-1559.md), [Backend: downgrade trtllm-gen kernel to cuda-12](../sources/prs/flashinfer/PR-1567.md), [feat: Support for inferring out_dtype from out.dtype for TRTLLM attention kernel](../sources/prs/flashinfer/PR-1578.md), [fix: semaphoress must be at the fixed range in workspace buffer on trtllm_gen attention](../sources/prs/flashinfer/PR-1584.md), [fix: Improve TRTLLM attention kernel out_dtype unit test](../sources/prs/flashinfer/PR-1590.md), [feat: initial support for SM103, SM110, SM120, SM121](../sources/prs/flashinfer/PR-1608.md), [bugfix: fix merge_attention_state in BatchAttention w/ gqa-group-size in Qwen family](../sources/prs/flashinfer/PR-1614.md), [bugfix: collect all modules to aot](../sources/prs/flashinfer/PR-1622.md), [fix: pass workspace for trtllm-gen attention](../sources/prs/flashinfer/PR-1635.md), [fix: zero-init workspace buffer for trtllm-gen fmha](../sources/prs/flashinfer/PR-1643.md), [perf&bugfix: skip kv-tile computation out of sliding window in FA2; fix __syncthreads in mergestate](../sources/prs/flashinfer/PR-1661.md), [Refactor Blackwell unit test scripts](../sources/prs/flashinfer/PR-1667.md), [feat: Add `variant.OutputTransform()` to decode kernels](../sources/prs/flashinfer/PR-1670.md), [feat: Batch-size invariant FA2 Prefill & Decode](../sources/prs/flashinfer/PR-1675.md), [[misc] add a wrapper class for attention sink jit args](../sources/prs/flashinfer/PR-1679.md), [perf: improve attention of tcgen05 flash-attention](../sources/prs/flashinfer/PR-1681.md), [perf: Port the separate reduce kernel mode from trtllm.](../sources/prs/flashinfer/PR-1685.md), [bugfix: increase workspace to make trtllm gen attention unit test pass](../sources/prs/flashinfer/PR-1707.md), [test: skip the unsupported test cases for sm120/121](../sources/prs/flashinfer/PR-1710.md), [tests: xfail attention sink UT for sliding window + non causal case](../sources/prs/flashinfer/PR-1752.md), [Fix tests/test_trtllm_gen_attention.py::test_trtllm_batch_prefill, ::test_trtllm_batch_decode mismatch error](../sources/prs/flashinfer/PR-1755.md), [feat: add xqa fp8 mha and fp8 kv cache](../sources/prs/flashinfer/PR-1769.md), [Bugfix: Fix data hazard in persistent reduce](../sources/prs/flashinfer/PR-1826.md), [Add head_dim=64 for tcgen05 tcgen05 flash-attention implementation](../sources/prs/flashinfer/PR-1850.md), [Bugfix: fix o_strides in persistent kernel ](../sources/prs/flashinfer/PR-1865.md), [Tune kernel compilation parameters for https://github.com/flashinfer-ai/flashinfer/pull/1850 ](../sources/prs/flashinfer/PR-1878.md), [fix: Fix trtllm-gen prefill IMA when batch_size==1](../sources/prs/flashinfer/PR-1912.md), [MLA RoPE + quantization fused kernel: shape generalization for MHA / GQA](../sources/prs/flashinfer/PR-1924.md), [Add realistic bench for persistent kernel ](../sources/prs/flashinfer/PR-1942.md), [fix: Make attention microbenchmark correctly use page table](../sources/prs/flashinfer/PR-1976.md), [fix: Skipping attention sink Blackwell test outside of Blackwell](../sources/prs/flashinfer/PR-1978.md), [fix: correct PDL parameter handling in RopeQuantize kernel](../sources/prs/flashinfer/PR-1982.md), [minor fix for xqa](../sources/prs/flashinfer/PR-1994.md), [unittest: Add head dim 256 test cases and mark as xfail](../sources/prs/flashinfer/PR-1999.md), [feat: add xqa backend and completes NHD/HND coverage for trtllm-gen/xqa backend](../sources/prs/flashinfer/PR-2001.md), [Fix trtllm-gen attention illegal memory access](../sources/prs/flashinfer/PR-2002.md), [use scalar for kv_scale in xqa](../sources/prs/flashinfer/PR-2033.md), [Added an initial implementation of Q and KV Cache in fp8 and to use t…](../sources/prs/flashinfer/PR-2035.md), [feat: Add flashinfer.rope.rope_quantize_fp8_append_paged_kv_cache (fused RoPE + Q + KV cache, supports MLA/GQA/MHA) ](../sources/prs/flashinfer/PR-2037.md), [Rebase FP8 SM100 Cutlass FMHA Attention to main (original PR#1238)](../sources/prs/flashinfer/PR-2047.md), [feat: add xqa mla backend](../sources/prs/flashinfer/PR-2053.md), [misc: Add XQA decode to microbenchmark for sm90 and sm120](../sources/prs/flashinfer/PR-2055.md), [Fix: several bugs/issues with trtllm-gen attention kernels. ](../sources/prs/flashinfer/PR-2062.md), [unittest: improve the efficiency of xqa unittests](../sources/prs/flashinfer/PR-2075.md), [fix: fix test_trtllm_gen_attention when max_seq_len < page_size](../sources/prs/flashinfer/PR-2076.md), [[Feature] Support batch prefill for POD Attention](../sources/prs/flashinfer/PR-2079.md), [enable xqa fp8 output](../sources/prs/flashinfer/PR-2081.md), [[API change] Allow using torch.Tensor for scales for trtllm-gen attention](../sources/prs/flashinfer/PR-2084.md), [[DSR1] Added MLA test](../sources/prs/flashinfer/PR-2100.md), [enable xqa speculative decoding](../sources/prs/flashinfer/PR-2105.md), [feat: support more head dim in RoPE kernel](../sources/prs/flashinfer/PR-2109.md), [add tensor scale input for xqa](../sources/prs/flashinfer/PR-2110.md), [refactor: update fa3 codebase and fix hopper unittest [part 1]](../sources/prs/flashinfer/PR-2111.md), [feature: make the LSE returned by MLA support base 2 or e #2113](../sources/prs/flashinfer/PR-2114.md), [perf: bunch of features and optimizations for top-k (sampling + sparse attention)](../sources/prs/flashinfer/PR-2119.md), [feat: support variable sequence length in decode kernel of trtllm-gen attention](../sources/prs/flashinfer/PR-2125.md), [fix flaky xqa test](../sources/prs/flashinfer/PR-2126.md), [fix(trtllm): reset negative strideBatch to 0 for ragged KV layout to …](../sources/prs/flashinfer/PR-2134.md), [fix: some bugs of headDim 256 trtllm-gen fmha kernels. ](../sources/prs/flashinfer/PR-2137.md), [feat: add trtllm-gen per-tensor sparseMla kernels.](../sources/prs/flashinfer/PR-2138.md), [feat: TRTLLM FMHAv2 backend for ctx attention](../sources/prs/flashinfer/PR-2142.md), [Enable Hopper FA3 FP8 attention in decode.py](../sources/prs/flashinfer/PR-2148.md), [refactor: Move mla code from decode.py to mla.py and add to documentation](../sources/prs/flashinfer/PR-2163.md), [fix: compile flags for trtllm fmha_v2 ](../sources/prs/flashinfer/PR-2175.md), [Permute page table in benchmarking](../sources/prs/flashinfer/PR-2194.md), [Fp8 attention are now part of cuDNN 9.17.1](../sources/prs/flashinfer/PR-2241.md), [feat: support non-contiguous query for trtllm-gen attention backend](../sources/prs/flashinfer/PR-2254.md), [fix: support int64 IdType for RoPE part argument in `rope_quantize_fp8_append_paged_kv_cache`](../sources/prs/flashinfer/PR-2255.md), [[TRTLLM-Gen Fmha] add optimized trtllm-gen decode kernels for high throughput + speculative decoding](../sources/prs/flashinfer/PR-2265.md), [feat: add GDN Attention](../sources/prs/flashinfer/PR-2276.md), [fix: Decode benchmark's fa2_tc uses backend=fa2 in wrapper](../sources/prs/flashinfer/PR-2302.md), [Added the cudnn backend Ragged KV Cache wrapper](../sources/prs/flashinfer/PR-2352.md), [feat: Add TRTLLM fmha_v2 library for SM90 attention with Skip-Softmax ](../sources/prs/flashinfer/PR-2446.md), [feat: Add TRTLLM-Gen Skip-Softmax kernels for prefill and decode](../sources/prs/flashinfer/PR-2477.md), [fix: Fix memory bandwidth calculation in MLA benchmarks](../sources/prs/flashinfer/PR-2479.md), [Support NVFP4 KV cache decode on SM120](../sources/prs/flashinfer/PR-2520.md), [pick fa2 for BatchDecodeWithPagedKVCacheWrapper auto backend](../sources/prs/flashinfer/PR-2530.md), [feat: Enable TRTLLM-Gen Skip-Softmax attention for MLA](../sources/prs/flashinfer/PR-2547.md), [fix: allow fmha_v2_prefill_deepseek on SM121 (DGX Spark)](../sources/prs/flashinfer/PR-2559.md), [fix: guard CUTLASS FMHA against SM12x and fix fmha_v2 SM121a check](../sources/prs/flashinfer/PR-2560.md), [feat: add is_sm12x_supported() helper for SM12x family detection](../sources/prs/flashinfer/PR-2574.md), [support qk_nope_head_dim for 192 check for GLM-5](../sources/prs/flashinfer/PR-2607.md), [benchmark: Enable speculative decode microbenchmarking for paged decode](../sources/prs/flashinfer/PR-2628.md), [fix: add SM121 support to SM120 version guards](../sources/prs/flashinfer/PR-2631.md), [[feat] trtllm-gen mxfp8 gemm](../sources/prs/flashinfer/PR-2653.md), [benchmarks: Add FP8 input / BF16 output in ragged prefill benchmark](../sources/prs/flashinfer/PR-2666.md), [feat: add support for more MLA head dimensions](../sources/prs/flashinfer/PR-2677.md), [Add NVFP4 KV cache quantization support for SM100](../sources/prs/flashinfer/PR-2702.md), [feat: Add DiT-oriented kernels where Qk (Bmm1) type can be reinterpreted into Int8 or BFloat16](../sources/prs/flashinfer/PR-2711.md), [Add cute dsl mla decode op](../sources/prs/flashinfer/PR-2743.md), [[Spark unit test debugging] Fix for tests/attention/test_trtllm_gen_mla.py](../sources/prs/flashinfer/PR-2750.md), [feat: Expose TRT-LLM FMHA style paged KV Cache and page table layout](../sources/prs/flashinfer/PR-2770.md), [feat: FP8 output support for CUTLASS MLA paged attention](../sources/prs/flashinfer/PR-2779.md), [tests: skip sliding window + fp8 to prevent hang in fmha_v2 unit tests](../sources/prs/flashinfer/PR-2781.md), [feat: Support padding tokens with seqlen=0 for rope+quant+kv cache update fusion kernel](../sources/prs/flashinfer/PR-2792.md), [[fmha-v2] Support HND and NHD paged KV cache layouts with conditional stride handling](../sources/prs/flashinfer/PR-2799.md), [[fix] Bugfix 1367: fix VariableBlockSparseAttention buffer overflow by dynamically resizing kv_lens_buffer](../sources/prs/flashinfer/PR-2802.md), [[CuTe DSL] Add modular FMHA prefill and MLA decode attention kernels](../sources/prs/flashinfer/PR-2805.md), [[Spark unit test] Adjust tolerance for test_xqa, test_logits_processor](../sources/prs/flashinfer/PR-2828.md), [[Perf] Add FMHAv2 to flashinfer_benchmark.py and eliminate unnecessary H2D](../sources/prs/flashinfer/PR-2841.md), [[fix] bugfix 2856: Fix pre-allocated out shape check in trtllm_batch_decode_with_kv_cache_mla for q_len_per_req > 1](../sources/prs/flashinfer/PR-2876.md), [feat: add pdl support for cute dsl mla decode kernel support](../sources/prs/flashinfer/PR-2901.md), [Only swizzle on v block scale; rename kv_block_scales to kv_cache_sf](../sources/prs/flashinfer/PR-2954.md), [test: xfail cuDNN FP8 prefill on Blackwell with CUDA <= 12.9](../sources/prs/flashinfer/PR-2963.md), [[Fmha] support nvfp4 output keepsMmaAb generation kernels](../sources/prs/flashinfer/PR-2988.md), [Support lse in trtllm paged attn kernels](../sources/prs/flashinfer/PR-3058.md), [Support NVFP4 KV for prefill and batch attention kernels](../sources/prs/flashinfer/PR-3097.md), [cute-dsl fmha prefill (cubin integration): remove front-padding, add attention_sink, and pdl support](../sources/prs/flashinfer/PR-3181.md), [feat(trace): embed runnable init() in every TraceTemplate](../sources/prs/flashinfer/PR-3221.md), [Support Kimi K2.5 H64 CuTe DSL MLA decode](../sources/prs/flashinfer/PR-3235.md), [Add dynamic tokens-per-page TRTLLM-GEN GQA kernels](../sources/prs/flashinfer/PR-3259.md), [fix(fmha_v2): fix FP8 V-scratch pipeline and varlen scheduler on SM90](../sources/prs/flashinfer/PR-3276.md), [perf: fix the iteration bound of SWA in FA2 prefill template](../sources/prs/flashinfer/PR-714.md), [Align KV chunk size binary search with actual KV chunk splitting.](../sources/prs/flashinfer/PR-728.md), [feat: support deepseek prefill attention shape](../sources/prs/flashinfer/PR-765.md), [perf: refactor fa2 prefill template](../sources/prs/flashinfer/PR-776.md), [bugfix: fix batch prefill attention kernel unittests](../sources/prs/flashinfer/PR-781.md), [bugfix: MLA decode should multiply sm_scale by math::log2e](../sources/prs/flashinfer/PR-787.md), [fix rope logic in mla decoding](../sources/prs/flashinfer/PR-793.md), [feat: support f32 attention output in FA2 template](../sources/prs/flashinfer/PR-799.md), [feat: apply sm_scale at logits instead of q in FA2 template](../sources/prs/flashinfer/PR-801.md), [perf: memory efficient deepseek mla fused page-attention kernel](../sources/prs/flashinfer/PR-804.md), [bugfix: mla page-attention kernel for different page sizes](../sources/prs/flashinfer/PR-810.md), [feat: unlocking MLA for A100](../sources/prs/flashinfer/PR-812.md), [feat: unlock MLA attention for sm89 (L40/L40s/4090)](../sources/prs/flashinfer/PR-814.md), [bugfix: bugfix on sm89 MLA](../sources/prs/flashinfer/PR-821.md), [perf: MLA decode kernel implemented by CuTe targeted to SM80](../sources/prs/flashinfer/PR-844.md), [Add POD-Attention to FlashInfer](../sources/prs/flashinfer/PR-858.md), [perf: dynamic split-k for MLA](../sources/prs/flashinfer/PR-863.md), [bugfix: fix the behavior of MLA kernel when kv-length is 0](../sources/prs/flashinfer/PR-868.md), [Naive Support for Hopper FP8 Prefill Kernel with Per-Head Quantization](../sources/prs/flashinfer/PR-869.md), [perf: FlashAttention-3 style MLA PageAttention](../sources/prs/flashinfer/PR-887.md), [perf: fix MLA split-k performance bug](../sources/prs/flashinfer/PR-898.md), [perf: tweak the pipeline design of mla kernel](../sources/prs/flashinfer/PR-901.md), [feat: flashinfer intra-kernel profiler](../sources/prs/flashinfer/PR-913.md), [bugfix: fix potential issues of FA3 template loading nans for PageAttention](../sources/prs/flashinfer/PR-945.md), [perf: Use 2WG pipeline design for MLA implementation on Hopper](../sources/prs/flashinfer/PR-952.md), [[TVM] Added tvm binding for sampling kernel](../sources/prs/flashinfer/PR-958.md), [perf: prefetch page indices for mla kernel](../sources/prs/flashinfer/PR-991.md), [3rdparty: upgrade cutlass to 3.9](../sources/prs/flashinfer/PR-997.md), [[FlexAttention] Remove Old Constraint on lastdim strides](../sources/prs/pytorch/PR-153104.md), [[FlexAttention] explicilty create grad_q w/ strides](../sources/prs/pytorch/PR-153641.md), [[SDPA] [MPS] Fixes regression in 2.8.0 for scaled_dot_product_attention using mps](../sources/prs/pytorch/PR-164364.md), [[Flex attention] Fix flex attention head broadcast](../sources/prs/pytorch/PR-164368.md), [[flex_attention] adds support for low precision K/V inputs in compiled mode with GPU](../sources/prs/pytorch/PR-170486.md), [[cherry-pick] Fix vllm issue for flex (#170499)](../sources/prs/pytorch/PR-170555.md), [[RELEASE 2.11] Release only changes](../sources/prs/pytorch/PR-175091.md), [[MPS] Fix 2-pass SDPA memory corruption by forcing float accumulators](../sources/prs/pytorch/PR-175580.md), [[release 2.12] Apply Release only changes to 2.12 branch](../sources/prs/pytorch/PR-180470.md), [Disable kernel cutlass_mla_decode on SM103](../sources/prs/sglang/PR-10058.md), [feat: Add FP4 (E2M1) KV Cache Support with Quantization Utilities for MLA](../sources/prs/sglang/PR-10078.md), [[Feature] Add MLAProcess for DeepSeek MLA on NPU](../sources/prs/sglang/PR-10130.md), [Fix chunked prefix cache for nvfp4](../sources/prs/sglang/PR-10180.md), [support qwen3_next blackwell](../sources/prs/sglang/PR-10403.md), [Fix cutlass moe accuracy drop caused by attention UB from DP padding mode](../sources/prs/sglang/PR-10414.md), [Enable trtllm mla prefix extend](../sources/prs/sglang/PR-10526.md), [Fuse quantize and rope in trtllm_mla MTP](../sources/prs/sglang/PR-10779.md), [[2/2] Support MHA prefill with FlashAttention 4.](../sources/prs/sglang/PR-10937.md), [[NVIDIA] FA3/FA4 Fix ](../sources/prs/sglang/PR-11606.md), [[DeepseekV32] Enable flashmla_prefill kernel with fp8 kvcache](../sources/prs/sglang/PR-11655.md), [Use trtllm_mla decode kernel for draft extend in speculative decoding](../sources/prs/sglang/PR-11664.md), [Support running FP4 Deepseek on SM120.](../sources/prs/sglang/PR-11708.md), [Change bf16 to fp8 for some gemms in attention for DeepSeek ckpt v2](../sources/prs/sglang/PR-11805.md), [Support nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8/NVFP4](../sources/prs/sglang/PR-11866.md), [DeepSeek-V3.2: Add Adaptive MHA Attention Pathway for Short-Sequence Prefill](../sources/prs/sglang/PR-11892.md), [Feature/nano v2 offline modelopt fp8 and nvfp4](../sources/prs/sglang/PR-12018.md), [(1/n)support context parallel with deepseekv3.2-DSA](../sources/prs/sglang/PR-12065.md), [[Ascend] qwen optimization](../sources/prs/sglang/PR-12078.md), [[Ascend][feature] support L1+ L2 radixcache on ascend](../sources/prs/sglang/PR-12214.md), [[DeepseekV32]: use `_concat_mla_absorb_q_general` to replace `torch.cat`](../sources/prs/sglang/PR-12215.md), [[Deepseek V3.2] Enable flashmla_auto with MTP](../sources/prs/sglang/PR-12294.md), [fix seqlen bug for trtllm_mla's draft_extend](../sources/prs/sglang/PR-12295.md), [perf: trtllm mla performance minor improvements](../sources/prs/sglang/PR-12435.md), [[Ascend] Support enable-mixed-chunk in non-MLA scenarios](../sources/prs/sglang/PR-12491.md), [[DeepSeek-V3.2][NSA] Enable MHA Pathway for Short Sequence Prefill on B200 (SM100)](../sources/prs/sglang/PR-12788.md), [[Deepseek V3.2] Only skip Indexer logits computation when is_extend_without_speculative](../sources/prs/sglang/PR-12816.md), [[Deepseek V3.2] Use torch.compile to speed up torch.cat in nsa](../sources/prs/sglang/PR-13022.md), [support mtp with deepseek r1 nvfp4 model](../sources/prs/sglang/PR-13115.md), [Aiter fp8 kv cache](../sources/prs/sglang/PR-13147.md), [Support internvl on Blackwell (which doesn't support fa3): add `SingletonCache` support to Vision{Sdpa|Triton|Ascend}Attention](../sources/prs/sglang/PR-13151.md), [diffusion: enable fa4 for blackwell](../sources/prs/sglang/PR-13263.md), [[BugFix] fix prefixcache performance and accuracy on ascend](../sources/prs/sglang/PR-13573.md), [[DeepSeekV3.2] Enable pure TP & Partial DP Attention](../sources/prs/sglang/PR-13646.md), [fix trtllm mla spec](../sources/prs/sglang/PR-13738.md), [[DeepSeek v3.2] opt Context Parallelism: support fused moe, multi batch and fp8 kvcache](../sources/prs/sglang/PR-13959.md), [Use trtllm mha decode kernel for target_verify in speculative decoding](../sources/prs/sglang/PR-13976.md), [fix: Increase FlashInfer workspace size for Qwen3VL models](../sources/prs/sglang/PR-14173.md), [Add Mistral Large 3 support.](../sources/prs/sglang/PR-14213.md), [Tiny use trtllm_mha as default when possible](../sources/prs/sglang/PR-14291.md), [[Fix] add block size logic for sm120 smem size](../sources/prs/sglang/PR-14311.md), [sync attention, deepseek doc](../sources/prs/sglang/PR-14335.md), [Support FP8 MLA prefill and 128k context.](../sources/prs/sglang/PR-14395.md), [[NPU] perf update with kvcache nz & w4a8 quant](../sources/prs/sglang/PR-14423.md), [Add Mistral Large 3 Eagle Support](../sources/prs/sglang/PR-14466.md), [[NPU][eagle3] support qwen eagle3 on NPU](../sources/prs/sglang/PR-14820.md), [fix: trtllm mha attention auto-selection on sm120](../sources/prs/sglang/PR-14842.md), [Fix dsv3 dp accuracy issue when using bf16-kv](../sources/prs/sglang/PR-14897.md), [Fix accuracy issue when using a16w16 mla_decode_fwd](../sources/prs/sglang/PR-14936.md), [fix(attention): Prevent trtllm_mha auto-selection with eagle3 speculative decoding](../sources/prs/sglang/PR-15127.md), [[NVIDIA] upstream FA4](../sources/prs/sglang/PR-15182.md), [[NPU]DeepSeek-V3.2 support npu mlaprolog](../sources/prs/sglang/PR-15381.md), [[diffusion] Add Sage Attention 3 Support for sm 120 (RTX5090)](../sources/prs/sglang/PR-15382.md), [Optimize FP8 MLA KV cache writes with Triton kernel](../sources/prs/sglang/PR-15522.md), [Fix BatchMLAPagedAttentionWrapper query/qo_inptr mismatch for EAGLE](../sources/prs/sglang/PR-15601.md), [[jit-kernel] Add CuTe DSL GDN Decode Kernel](../sources/prs/sglang/PR-15631.md), [[diffusion] model: support TurboWan2.1-T2V-1.3B/14B SLA](../sources/prs/sglang/PR-15888.md), [Bugfix for ds-vl2](../sources/prs/sglang/PR-15894.md), [Support fa4 decoding](../sources/prs/sglang/PR-16034.md), [optimize get_topk_ragged by fusing get k and k_scale triton kernel](../sources/prs/sglang/PR-16043.md), [enhance accuracy for model kimi-vl-instruct-a3b](../sources/prs/sglang/PR-16076.md), [[VLM] Adopt jit qk_norm kernel in VLM](../sources/prs/sglang/PR-16171.md), [[diffusion] Fix RuntimeError in SageAttention3 on Nvidia Blackwell with Qwen-Image](../sources/prs/sglang/PR-16335.md), [[Fix]Fix FA3 Performance in Diffusion Model ](../sources/prs/sglang/PR-16382.md), [[DeepSeek v3.2] Opt MTP decode cuda batch sizes and nsa implementation](../sources/prs/sglang/PR-16961.md), [[NPU]bugfix: fix for dsv3.2 and dsvl2](../sources/prs/sglang/PR-17007.md), [Optimize GDN decode for Qwen3 Next](../sources/prs/sglang/PR-17094.md), [[diffusion] fix: fix using upstream flash_attn on blackwell](../sources/prs/sglang/PR-17111.md), [Enable XQA for SM90 and SM120](../sources/prs/sglang/PR-17115.md), [[New Model] GLM4.7-Flash](../sources/prs/sglang/PR-17247.md), [Disable mla persistent kernel when not using fp8 kv_cache](../sources/prs/sglang/PR-17327.md), [Move fa4 from sgl-kernel to jit kernel](../sources/prs/sglang/PR-17353.md), [[NPU] enhance accuracy for model kimi-vl-a3b-instruct](../sources/prs/sglang/PR-17480.md), [Kernel: optimize decoding metadata in NSA multi-spec backend with fused kernels](../sources/prs/sglang/PR-17554.md), [Feat/add fi selective state update kernel call](../sources/prs/sglang/PR-18070.md), [Support Qwen3 MoE context parallel](../sources/prs/sglang/PR-18233.md), [[ROCm] Optimize Deepseek R1 on MI300X](../sources/prs/sglang/PR-18242.md), [[AMD] Support Qwen3-Coder-Next on AMD platform](../sources/prs/sglang/PR-18355.md), [feat(gdn): add FlashInfer K-last SSM layout support for GDN prefill and decode for Hopper](../sources/prs/sglang/PR-18361.md), [Nsa trtllm mla sparse fp8 support with Deepseek v3.2 NVFP4](../sources/prs/sglang/PR-18389.md), [[AMD] Update aiter to v0.1.10.post2](../sources/prs/sglang/PR-18423.md), [feat: add FA4 SM90 paged KV decode support & update attention docs](../sources/prs/sglang/PR-18442.md), [Tilelang sparse decode fwd for dsv32 mi355](../sources/prs/sglang/PR-18488.md), [Fp8 prefill attn kernel integration](../sources/prs/sglang/PR-18528.md), [[AMD] Fix accuracy issue when running TP4 dsv3 model with mtp](../sources/prs/sglang/PR-18607.md), [[sgl-kernel] rebase FlashMLA 0217](../sources/prs/sglang/PR-18902.md), [Fix NSA FP8 KV cache path for both-trtllm MHA one-shot](../sources/prs/sglang/PR-18931.md), [Support skip-softmax attention](../sources/prs/sglang/PR-19089.md), [[DeepSeek-V3.2][JIT-kernel] Support nsa fuse store indexer k cache](../sources/prs/sglang/PR-19148.md), [[NVIDIA] Integrate FlashInfer decode kernel (Blackwell) for Qwen3.5](../sources/prs/sglang/PR-19150.md), [[AMD] Fix accuracy while using --enable-dp-attention](../sources/prs/sglang/PR-19247.md), [[Feature] add feature mla_ag_after_qlora for dsv3.2](../sources/prs/sglang/PR-19428.md), [[NPU] bugs fix for Deepseek models](../sources/prs/sglang/PR-19544.md), [[miles] fix for glm5](../sources/prs/sglang/PR-19634.md), [Fix MLA decode path returning unwritten (padded) rows](../sources/prs/sglang/PR-19902.md), [[AMD] Fix Tensor Memory Aliasing ](../sources/prs/sglang/PR-19928.md), [[AMD] Fix FP8 assertion failure in aiter MLA decode by falling back to self.k_scale](../sources/prs/sglang/PR-19935.md), [[AMD] Tilelang sparse fwd for dsv32 mi355/mi300](../sources/prs/sglang/PR-19945.md), [MiniMax-M2.5 - Support dp attention, dp reduce scatter, FP4 all gather, AR fusion in prepare_attn](../sources/prs/sglang/PR-20067.md), [[AMD] Fp8 prefill integration with radix cache path for dpsk models](../sources/prs/sglang/PR-20187.md), [[AMD] Add 4-GPU test suite for MI325 runners](../sources/prs/sglang/PR-20294.md), [[Benchmark] use flashinfer bench_gpu_time instead of triton do_bench](../sources/prs/sglang/PR-20305.md), [[AMD][Bug-fix] Fix gpu fault when run the test with dp-attention-enabled and max-concurrency is over 256](../sources/prs/sglang/PR-20399.md), [[AMD][AITER] Guard _use_mla_ps_kernel with self.use_mla in draft_extend_v2 paths](../sources/prs/sglang/PR-20409.md), [[GDN] Add benchmark for sglang gdn prefill](../sources/prs/sglang/PR-20428.md), [Support Triton MLA FP8 KV cache](../sources/prs/sglang/PR-20479.md), [[Diffusion] Clean upstream fa3 in hopper](../sources/prs/sglang/PR-20576.md), [Use Flashinfer for target_verify in GDN model for SM120](../sources/prs/sglang/PR-20604.md), [FIX: (NSA) Compute topk_indices_offset when NSA prefill flashmla_sparse is used with FP8 KV cache](../sources/prs/sglang/PR-20606.md), [Add SGLang CUDA crash API logging inspired by FlashInfer](../sources/prs/sglang/PR-20910.md), [ci: run Stage A CUDA tests as stage-a-test-small-1-gpu on 5090](../sources/prs/sglang/PR-20988.md), [perf: precompute FA3 scheduler_metadata to eliminate per-layer prepare_varlen_num_blocks](../sources/prs/sglang/PR-21104.md), [[Not-Merge][AMD] GLM-5 performance optimization](../sources/prs/sglang/PR-21166.md), [[Whisper] Enable CUDA graph support and timestamp for whisper model](../sources/prs/sglang/PR-21190.md), [[KDA] Support CuTeDSL KDA decode kernel](../sources/prs/sglang/PR-21203.md), [[AMD]: Support MLA with nhead<16 and FP8 KV cache for TP=8 (Kimi K2.5…](../sources/prs/sglang/PR-21213.md), [Refactor JIT kernel CI to use run_suite.py registration system](../sources/prs/sglang/PR-21239.md), [[misc] clean up kernel API](../sources/prs/sglang/PR-21325.md), [[GDN] Fuse GDN kkt + solve_tril into one kernel](../sources/prs/sglang/PR-21411.md), [[Bugfix] Lazy-import CuteDSL KDA kernel to fix AMD/ROCm startup crash](../sources/prs/sglang/PR-21428.md), [fix nemotron capture for non attention layers](../sources/prs/sglang/PR-21436.md), [fix: piecewise_cuda_graph get correct qo_indptr](../sources/prs/sglang/PR-21452.md), [Migrate all callers from /get_server_info to /server_info](../sources/prs/sglang/PR-21463.md), [[AMD] Enable FP8 KV cache and FP8 attention kernel for NSA on MI300/MI355 with TileLang backend](../sources/prs/sglang/PR-21511.md), [Change default mm-attention backend from triton_attn to fa4](../sources/prs/sglang/PR-21595.md), [fix: TRT-LLM MHA CUDA illegal address with EAGLE v2 + DP attention](../sources/prs/sglang/PR-21649.md), [[XPU] Enable qwen3.5 on XPU](../sources/prs/sglang/PR-21668.md), [[DSA] Support trtllm sparse mla kernel for prefill batches ](../sources/prs/sglang/PR-21783.md), [[Bugfix] Temporarily skip TRTLLM attention on (G)B300 (SM103) to avoid high-concurrency hang](../sources/prs/sglang/PR-21906.md), [[Bugfix] Fix CUDA graph replay issues in trtllm_mla draft_extend](../sources/prs/sglang/PR-21987.md), [[NPU] enable mla prepare fused kernel only when being mla attn](../sources/prs/sglang/PR-22024.md), [[MUSA][9/N] Add FA3 attention backend support through MATE (MUSA AI Tensor Engine)](../sources/prs/sglang/PR-22051.md), [[nvidia] Gemma4 nvfp4 fix](../sources/prs/sglang/PR-22079.md), [Reduce unnecessary kernels and copies in the NSA indexer](../sources/prs/sglang/PR-22232.md), [[AMD][HIP] NSA: bf16 passthrough from RMSNorm to eliminate FP8 dequantization](../sources/prs/sglang/PR-22258.md), [Lazy import flash_attention_v4 to avoid loading flash_attn.cute at startup](../sources/prs/sglang/PR-22306.md), [[DSA] Hopper FP8 FlashMLA KV padding](../sources/prs/sglang/PR-22372.md), [[AMD] Use aiter CK layernorm2d for LayerNorm to reduce NSA indexer kernel launches](../sources/prs/sglang/PR-22424.md), [[Fix] Fix several bugs on DSA models](../sources/prs/sglang/PR-22430.md), [[BugFix] Resolve adaptive speculative decoding conflicts for Qwen3.5 (hybrid GDN)](../sources/prs/sglang/PR-23331.md), [[Diffusion][NPU]Add attention backends for diffusion models for Ascend NPU](../sources/prs/sglang/PR-23482.md), [[feat] Init true on policy with qwen_dense](../sources/prs/sglang/PR-23961.md), [Enable PDL for various kernels in DSV32/GLM5](../sources/prs/sglang/PR-23965.md), [[KDA] Optimize prefill kernels with diagonal and recompute fuse](../sources/prs/sglang/PR-24271.md), [Support spec v2 for FlashMLA speculative decoding](../sources/prs/sglang/PR-24640.md), [[attn backend] Integrate tokenspeed_mla prefill/decode kernels (fp8 kv cache, blackwell)](../sources/prs/sglang/PR-24925.md), [Amd/deepseek v4 rebase main 0509](../sources/prs/sglang/PR-24933.md), [Fix AMX GQA extend attention](../sources/prs/sglang/PR-25180.md), [[NSA] Avoid repeated NSA MQA logits memory queries](../sources/prs/sglang/PR-25299.md), [[Intel GPU] Enable DeepSeek V4 Inference on XPU](../sources/prs/sglang/PR-25336.md), [Support draft extend cuda graph for tokenspeed_mla attention backend](../sources/prs/sglang/PR-25489.md), [amd/deepseek_v4 27/N [fix] Reduce Triton autotune configs for faster first-time server launch](../sources/prs/sglang/PR-25554.md), [Use triton_attn as default vision attention on B300 (SM103)](../sources/prs/sglang/PR-25570.md), [Introduce SchedulerDPAttnAdapter to own DP-attention state](../sources/prs/sglang/PR-25611.md), [Move DP-attention adapter methods to SchedulerDPAttnAdapter](../sources/prs/sglang/PR-25612.md), [[Refactor] Rename NSA → DSA: user-facing aliases, file/class/import rename](../sources/prs/sglang/PR-25821.md), [[Test] Stage-a sanity kits; consolidate core/ + models_e2e/ tests](../sources/prs/sglang/PR-25831.md), [feat: support flashinfer mla attention for deepseek v3](../sources/prs/sglang/PR-3550.md), [feat: support flashinfer mla with prefix cache](../sources/prs/sglang/PR-3643.md), [Add fast decode plan for flashinfer mla](../sources/prs/sglang/PR-3987.md), [[Revision] Add fast decode plan for flashinfer mla ](../sources/prs/sglang/PR-4012.md), [Support overlapping two batches](../sources/prs/sglang/PR-4068.md), [avoid cudaStreamSynchronize in DeepSeekV2AttentionMLA](../sources/prs/sglang/PR-4577.md), [Fix loading KV quantization scale; Enable modelopt kv cache](../sources/prs/sglang/PR-4686.md), [[Model] Adding Qwen3 and Qwen3MoE](../sources/prs/sglang/PR-4693.md), [support cmake for sgl-kernel](../sources/prs/sglang/PR-4706.md), [Support (1 <= dp < tp) in the dp attention in DeepEP](../sources/prs/sglang/PR-4770.md), [Feat/support encoder model (like bert)](../sources/prs/sglang/PR-4887.md), [Support MHA with chunked prefix cache for DeepSeek chunked prefill](../sources/prs/sglang/PR-5113.md), [Blackwell Cutlass MLA kernel](../sources/prs/sglang/PR-5142.md), [feat: use fa3 mla by default on hopper](../sources/prs/sglang/PR-5210.md), [Add Speculative Decoding Eagle3 topk > 1](../sources/prs/sglang/PR-5318.md), [fix: solve cu118 issue for cutlass mla](../sources/prs/sglang/PR-5331.md), [Add Cutlass MLA attention backend](../sources/prs/sglang/PR-5390.md), [[Feat] upgrade pytorch2.6](../sources/prs/sglang/PR-5417.md), [BLackwell cutlass mla: Add check for bad page size/block num combinations](../sources/prs/sglang/PR-5431.md), [Avoid computing lse in Ragged Prefill when there's no prefix.](../sources/prs/sglang/PR-5476.md), [[PP] Add pipeline parallelism](../sources/prs/sglang/PR-5724.md), [Fuse MLA set kv cache kernel](../sources/prs/sglang/PR-5748.md), [opt flashinfer mla cat](../sources/prs/sglang/PR-5822.md), [Cutlass MLA decode - fix dtype error](../sources/prs/sglang/PR-5868.md), [[Fix] Fix a bug for flashmla to run R1 model](../sources/prs/sglang/PR-5875.md), [Improve dp attention port assignment scheme](../sources/prs/sglang/PR-5889.md), [feat: mtp support dp-attention](../sources/prs/sglang/PR-6081.md), [Cutlass MLA: Disable split kv due to https://github.com/NVIDIA/cutlass/issues/2274](../sources/prs/sglang/PR-6101.md), [feat: add dp attention support for Qwen 2/3 MoE models, fixes #6088](../sources/prs/sglang/PR-6121.md), [Enable FlashInfer support encoder models and add head_dim padding workaround](../sources/prs/sglang/PR-6230.md), [[Feature] Comprehensive Hybrid Parallelism Support](../sources/prs/sglang/PR-6389.md), [[CPU] support the case where num_attention_heads or intermediate_size is not divisible by the TP size](../sources/prs/sglang/PR-6771.md), [[perf][sgl-kernel] extend cutlass_mla_decode to support num_head < 128](../sources/prs/sglang/PR-6929.md), [[Feature] Support Flashinfer fmha on Blackwell](../sources/prs/sglang/PR-6930.md), [Fix cutlass MLA gets almost zero accuracy](../sources/prs/sglang/PR-6998.md), [Tiny fix cutlass_mla_get_workspace_size stub incorrect signature](../sources/prs/sglang/PR-7057.md), [chore: upgrade sgl-kernel v0.1.8.post2](../sources/prs/sglang/PR-7186.md), [Fix grammar abort & Minor style fixes](../sources/prs/sglang/PR-7204.md), [feat: support compatibility between MTP and two-batch-overlap](../sources/prs/sglang/PR-7225.md), [Minor style and doc fix](../sources/prs/sglang/PR-7228.md), [Support non-contiguous query input for extend/decode attention](../sources/prs/sglang/PR-7462.md), [Add fp4 quantize before all-gather for Flashinfer cutlass MoE DP (max throughput)](../sources/prs/sglang/PR-7667.md), [TRTLLM-MLA FP8 path](../sources/prs/sglang/PR-8638.md), [feat: add trtllm-gen mha from direct call](../sources/prs/sglang/PR-8782.md), [Support DP attention with GPT-OSS](../sources/prs/sglang/PR-9359.md), [move is_sm90_supported/is_sm100_supported to python/sglang/srt/utils.py](../sources/prs/sglang/PR-9679.md), [[CPU] Add FP8 Bmm support](../sources/prs/sglang/PR-9744.md), [support using fa4 on deepseek on blackwell](../sources/prs/sglang/PR-9928.md), [[Feature] Support `cp.reduce.async.bulk.tensor`](../sources/prs/tilelang/PR-1667.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [[Kernel] Flash Attention 3 Support](../sources/prs/vllm/PR-12093.md), [[Core] Optimizing cross-attention `QKVParallelLinear` computation](../sources/prs/vllm/PR-12325.md), [[ROCm] Faster Custom Paged Attention kernels](../sources/prs/vllm/PR-12348.md), [[Attention] MLA decode optimizations](../sources/prs/vllm/PR-12528.md), [[Attention] Deepseek v3 MLA support with FP8 compute](../sources/prs/vllm/PR-12601.md), [[Attention] MLA with chunked prefill](../sources/prs/vllm/PR-12639.md), [[AMD][ROCm] Enable DeepSeek model on ROCm](../sources/prs/vllm/PR-12662.md), [[Perf] Mem align KV caches for CUDA devices (MLA perf improvement)](../sources/prs/vllm/PR-12676.md), [[VLM] Add MLA with pure RoPE support for deepseek-vl2 models](../sources/prs/vllm/PR-12729.md), [[Kernel] Make rotary_embedding ops more flexible with input shape](../sources/prs/vllm/PR-12777.md), [[Attention] Use FA3 for MLA on Hopper](../sources/prs/vllm/PR-12807.md), [Expand MLA to support most types of quantization](../sources/prs/vllm/PR-13181.md), [[Bugfix] Massage MLA's usage of flash attn for RoCM](../sources/prs/vllm/PR-13310.md), [[ROCM] fix native attention function call](../sources/prs/vllm/PR-13650.md), [[core] Perf improvement for DSv3 on AMD GPUs](../sources/prs/vllm/PR-13718.md), [[Bugfix] Support MLA for CompressedTensorsWNA16](../sources/prs/vllm/PR-13725.md), [[V1] V1 Enablement Oracle ](../sources/prs/vllm/PR-13726.md), [[Kernel] FlashMLA integration](../sources/prs/vllm/PR-13747.md), [[Attention] MLA support for V1](../sources/prs/vllm/PR-13789.md), [[ROCm] Disable chunked prefill/prefix caching when running MLA on non-cuda platforms](../sources/prs/vllm/PR-13844.md), [[Attention] Flash MLA for V1](../sources/prs/vllm/PR-13867.md), [Fix mla prefill context performance](../sources/prs/vllm/PR-13897.md), [[V1] EP/TP MoE + DP Attention](../sources/prs/vllm/PR-13931.md), [[V1] Implement sliding window attention in kv_cache_manager](../sources/prs/vllm/PR-14097.md), [[v1] Add comments to the new ragged paged attention Pallas kernel](../sources/prs/vllm/PR-14155.md), [[V1][TPU] TPU multimodal model support for ragged attention](../sources/prs/vllm/PR-14158.md), [[V1][TPU] Support V1 Sampler for ragged attention](../sources/prs/vllm/PR-14227.md), [[Hardware] Update the flash attn tag to support Blackwell](../sources/prs/vllm/PR-14244.md), [dynamic distpatch of fp8 kernels](../sources/prs/vllm/PR-14245.md), [[BugFix] MLA + V1, illegal memory access and accuracy issues](../sources/prs/vllm/PR-14253.md), [[BugFix] Fix prefix caching V0 MLA](../sources/prs/vllm/PR-14255.md), [[Attention] FlashAttn MLA](../sources/prs/vllm/PR-14258.md), [[Hardware][TPU]Enable ragged paged attention kernel and resolve recompilation issue](../sources/prs/vllm/PR-14310.md), [[Bug] Fix Attention when ignored in by quant_method](../sources/prs/vllm/PR-14313.md), [[ROCm] Enable chunked prefill/paged attention in MLA on ROCm](../sources/prs/vllm/PR-14316.md), [[Perf] Reduce MLA CPU overheads in V1](../sources/prs/vllm/PR-14384.md), [[Bugfix] DeepSeek Accuracy](../sources/prs/vllm/PR-14476.md), [[Perf] Improve MLA on V1](../sources/prs/vllm/PR-14540.md), [[Attention] Flash Attention 3 - fp8](../sources/prs/vllm/PR-14570.md), [[Kernel][CPU] CPU MLA](../sources/prs/vllm/PR-14744.md), [[Attention] MLA get rid of materialization](../sources/prs/vllm/PR-14770.md), [[Attention] Get rid of mla cache alignment](../sources/prs/vllm/PR-14842.md), [[V1][BugFix] Detect interleaved sliding window attention](../sources/prs/vllm/PR-14896.md), [[FEAT][ROCm] Integrate Paged Attention Kernel from AITER](../sources/prs/vllm/PR-15001.md), [[Bugfix] Fix incorrect qwen2.5-vl attention mask pre-computation](../sources/prs/vllm/PR-15200.md), [[Bugfix] Fix use_cascade_attention handling for Alibi-based models on vllm/v1](../sources/prs/vllm/PR-15211.md), [[Misc] Add attention mask pre-computation optimization back to Qwen2.5-VL](../sources/prs/vllm/PR-15273.md), [[BugFix] Fix nightly MLA failure (FA2 + MLA chunked prefill, i.e. V1, producing bad results)](../sources/prs/vllm/PR-15492.md), [[ROCM][KERNEL] Paged attention for V1](../sources/prs/vllm/PR-15720.md), [[TPU] Support sliding window and logit soft capping in the paged attention kernel for TPU.](../sources/prs/vllm/PR-15732.md), [[FEAT][ROCm]: Support AITER MLA](../sources/prs/vllm/PR-15893.md), [Modularize fused experts and integrate PPLX kernels](../sources/prs/vllm/PR-15956.md), [[NVIDIA] Support Cutlass MLA for Blackwell GPUs](../sources/prs/vllm/PR-16032.md), [[ROCM] Add gfx950 to the custom attention archs](../sources/prs/vllm/PR-16034.md), [Add FlexAttention to V1](../sources/prs/vllm/PR-16078.md), [Upstream Llama4 Support to Main](../sources/prs/vllm/PR-16113.md), [[Kernel] support merge_attn_states CUDA kernel, 3x speedup](../sources/prs/vllm/PR-16173.md), [[Bug] [ROCm] Fix Llama 4 Enablement Bug on ROCm: V0 ROCmFlashAttentionImpl and Triton Fused MoE bugs](../sources/prs/vllm/PR-16198.md), [[Llama4] Enable attention temperature tuning by default for long context (>32k)](../sources/prs/vllm/PR-16439.md), [Allocate kv_cache with stride order](../sources/prs/vllm/PR-16605.md), [[MLA] Simplification to batch P/D reordering](../sources/prs/vllm/PR-16673.md), [[V1] V1 FlashInfer Attention](../sources/prs/vllm/PR-16684.md), [[torch.compile][ROCm] Fuse quantization onto attention using a torch.compile pass](../sources/prs/vllm/PR-16756.md), [[Kernel] Unified Triton kernel that doesn't distinguish between prefill + decode](../sources/prs/vllm/PR-16828.md), [Update PyTorch to 2.7.0](../sources/prs/vllm/PR-16859.md), [[Attention] FA3 decode perf improvement - single mma warp group support for head dim 128](../sources/prs/vllm/PR-16864.md), [[Bugfix] Triton FA function takes no keyword arguments](../sources/prs/vllm/PR-16902.md), [[ROCm][Kernel][V1] Enable AMD Radeon GPU Custom Paged Attention on v1](../sources/prs/vllm/PR-17004.md), [[Bugfix] Add contiguous call inside rope kernel wrapper](../sources/prs/vllm/PR-17091.md), [[ROCm][FP8][Kernel] FP8 quantization fused into Custom Paged Attention](../sources/prs/vllm/PR-17139.md), [[Bugfix] gemma[2,3] interleaved attention when sliding window is disabled](../sources/prs/vllm/PR-17180.md), [[Bugfix] Get a specific type of layer from forward context](../sources/prs/vllm/PR-17222.md), [[BugFix] Fix vllm_flash_attn install issues](../sources/prs/vllm/PR-17267.md), [[BugFix] Fix cascade attention - RuntimeError: scheduler_metadata must have shape (metadata_size)](../sources/prs/vllm/PR-17283.md), [[v1] AttentionMetadata for each layer](../sources/prs/vllm/PR-17394.md), [[v1] Pass BlockTable and KVCacheSpec to AttentionMetadataBuilders](../sources/prs/vllm/PR-17483.md), [[Attention] MLA move o_proj q_proj into cuda-graph region](../sources/prs/vllm/PR-17484.md), [[BugFix] Fix mla cpu - missing 3 required positional arguments](../sources/prs/vllm/PR-17494.md), [[FEAT][ROCm]: Support AITER MLA on V1 Engine](../sources/prs/vllm/PR-17523.md), [[Attention] MLA move rotary embedding to cuda-graph region](../sources/prs/vllm/PR-17668.md), [fix amd triton mla path](../sources/prs/vllm/PR-17871.md), [[Bugfix][ROCm] Fix AITER MLA V1](../sources/prs/vllm/PR-17880.md), [[BugFix][AMD] Compatible patch for AITER lib after 04/20](../sources/prs/vllm/PR-17912.md), [[v1] Support multiple KV cache groups in GPU model runner](../sources/prs/vllm/PR-17945.md), [[BUG] [ROCm] [MLA] Fix variable name bug due to change in variable name in PR #17483](../sources/prs/vllm/PR-17961.md), [[Bugfix][ROCm] Use `chunked_prefill_paged_decode` as fallback for V1 attention on ROCm](../sources/prs/vllm/PR-18093.md), [[Hardware][AMD] integrate aiter chunked prefill into vllm](../sources/prs/vllm/PR-18596.md), [[BugFix] FA2 MLA Accuracy Issue](../sources/prs/vllm/PR-18807.md), [[P/D] Heterogeneous TP](../sources/prs/vllm/PR-18833.md), [[Kernel] Apply torch.Tag.needs_fixed_stride_order only for torch==2.6.0](../sources/prs/vllm/PR-19346.md), [[Core] Support Local Chunked Attention for Hybrid KV Cache](../sources/prs/vllm/PR-19351.md), [[Bugfix][V1] Allow manual FlashAttention for Blackwell](../sources/prs/vllm/PR-19492.md), [[Bugfix] Enable PP with AITER+V1](../sources/prs/vllm/PR-19822.md), [Enable V1 for Hybrid SSM/Attention Models](../sources/prs/vllm/PR-20016.md), [[Attention] MLA - Flashinfer Ragged Prefill](../sources/prs/vllm/PR-20034.md), [[Bugfix] Fix some narrowing conversion warnings](../sources/prs/vllm/PR-20141.md), [[Kernel] Optimize Prefill Attention in Unified Triton Attention Kernel](../sources/prs/vllm/PR-20308.md), [Update PyTorch to 2.8.0](../sources/prs/vllm/PR-20358.md), [SM100 Cutlass MLA decode with unrestricted num_heads (< 128) for DeepSeek TP](../sources/prs/vllm/PR-20769.md), [[v1][core] Support for attention free models](../sources/prs/vllm/PR-20811.md), [[Bugfix] Voxtral on Blackwell GPUs (RTX 50 series)](../sources/prs/vllm/PR-21077.md), [[Kernel] Flashinfer MLA (trtllm-gen) decode kernel integration](../sources/prs/vllm/PR-21078.md), [[v1] Add Whisper model support (encoder-decoder)](../sources/prs/vllm/PR-21088.md), [[Attention] Optimize FlashInfer MetadataBuilder Build call](../sources/prs/vllm/PR-21137.md), [[Attention][DBO] Add support for "splitting" the CommonAttentionMetadata](../sources/prs/vllm/PR-21153.md), [[Attention] Clean up iRoPE in V1](../sources/prs/vllm/PR-21188.md), [[Kernel] Enable Hybrid Model Support in Triton Unified Attention Kernel](../sources/prs/vllm/PR-21197.md), [[v1] - Mamba1 Attention Metadata](../sources/prs/vllm/PR-21249.md), [Support encoder-only models without KV-Cache](../sources/prs/vllm/PR-21270.md), [[v1][attention] Support Hybrid Allocator + FlashInfer](../sources/prs/vllm/PR-21412.md), [Updates to Flex + VLLm integration](../sources/prs/vllm/PR-21416.md), [[V1] Fix local chunked attention always disabled](../sources/prs/vllm/PR-21419.md), [[BugFix] Fix shared storage connector load kv only load attention layer](../sources/prs/vllm/PR-21428.md), [update flashinfer to v0.2.9rc1](../sources/prs/vllm/PR-21485.md), [[Attention] Support multiple attention metadata builders per kv_cache_spec + proper local attention no hybrid kv cache fix](../sources/prs/vllm/PR-21588.md), [Override attention metadata for fast prefill in some KV sharing setups](../sources/prs/vllm/PR-21590.md), [[BugFix] Fix IMA FlashMLA full cuda-graph and DP + Update FlashMLA](../sources/prs/vllm/PR-21691.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv/out Attention Kernel](../sources/prs/vllm/PR-21716.md), [[Perf] Disable chunked local attention by default with llama4](../sources/prs/vllm/PR-21761.md), [[Bugfix] Check NVIDIA artifactory is accessible before using flashinfer cubin kernels](../sources/prs/vllm/PR-21893.md), [[Qwen3] Enable dual-chunk-attention support for Qwen3 models.](../sources/prs/vllm/PR-21924.md), [[NVIDIA] Support Flashinfer TRT-LLM Prefill Attention Kernel](../sources/prs/vllm/PR-22095.md), [[fix] fix correct assertion syntax error in attention utils.](../sources/prs/vllm/PR-22154.md), [Fp8 paged attention update](../sources/prs/vllm/PR-22222.md), [Support encoder_only attention for FlexAttention](../sources/prs/vllm/PR-22273.md), [Upgrade FA3 for attention sink](../sources/prs/vllm/PR-22313.md), [[Bugfix] Add proper comparison for package versions](../sources/prs/vllm/PR-22314.md), [[ROCm] Add attention sink to use_rocm_custom_paged_attention](../sources/prs/vllm/PR-22329.md), [[BugFix] Fix triton compile error in `kernel_unified_attention_2/3d` caused by attention sinks](../sources/prs/vllm/PR-22368.md), [[bugfix] Fix Llama3/4 issues caused by FlashInfer 0.2.10](../sources/prs/vllm/PR-22426.md), [[Attention] FA3 Attention Sinks Perf Boost](../sources/prs/vllm/PR-22478.md), [[Bugfix] Fix ModernBert load & Enable sliding window attention for bidirectional attention.](../sources/prs/vllm/PR-22637.md), [Support multiple attention groups for KV sharing](../sources/prs/vllm/PR-22672.md), [Force TRTLLM attention for gpt-oss on SM100](../sources/prs/vllm/PR-22678.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv NVFP4-out Attention Kernel](../sources/prs/vllm/PR-22703.md), [[Bugfix] Fix default enable for CUTLASS MLA on SM100](../sources/prs/vllm/PR-22738.md), [fp8 kv cache support fix for torch.compile](../sources/prs/vllm/PR-22758.md), [[Bugfix gpt-oss] Fix float32 convert for flashinfer sink support](../sources/prs/vllm/PR-23016.md), [[V1] address post issues related to #20059 (part 1); cascade attention reenable by default](../sources/prs/vllm/PR-23046.md), [[Misc] Add @tdoublep as a maintainer of hybrid model and Triton-attention related code](../sources/prs/vllm/PR-23122.md), [[Log] Warning Once for Cutlass MLA ](../sources/prs/vllm/PR-23137.md), [Optimize input preparation for FlashInfer [2/N]](../sources/prs/vllm/PR-23174.md), [[Attention] Optimize make_local_attention_virtual_batches for Flash Attention](../sources/prs/vllm/PR-23185.md), [[Misc][qwen2_5_vl][torch.compile] Enable `supports_torch_compile` on generic nn.Module and demonstrate speedup on Qwen Vision model](../sources/prs/vllm/PR-23207.md), [[Core] Always use tensor cores for Flashinfer Decode Wrapper](../sources/prs/vllm/PR-23214.md), [[ROCm][Aiter] Add triton fp8 bmm kernel for mla](../sources/prs/vllm/PR-23264.md), [[Compile] Fix Compile Warning SM100 Cutlass MLA](../sources/prs/vllm/PR-23287.md), [[Attention] Allow V1 flash_attn to support cross-attention](../sources/prs/vllm/PR-23297.md), [[Bugfix] Fixing division by zero in triton_attn if query_heads/kv_heads > 16 ](../sources/prs/vllm/PR-23424.md), [[Perf] Warmup FlashInfer attention during startup](../sources/prs/vllm/PR-23439.md), [[Attention][FA3] Update FA3 to include new swizzle optimization](../sources/prs/vllm/PR-23465.md), [fix(v1/kv_cache): resolve async KV transfer bug in cascade attention](../sources/prs/vllm/PR-23485.md), [[Misc] Simplify FlashInfer attention metadata](../sources/prs/vllm/PR-23585.md), [[Flashinfer] Support Flashinfer TRTLLM FP8-qkv BF16/FP16-out Attention Kernel](../sources/prs/vllm/PR-23647.md), [[v1] Add cross-attention KV cache support for encoder-decoder models](../sources/prs/vllm/PR-23664.md), [[Core/DBO][1/N] Add Dual-Batch Overlap mechanism to VLLM](../sources/prs/vllm/PR-23693.md), [[FlashInfer] Cache hyper params in metadata builder](../sources/prs/vllm/PR-23732.md), [[Feature] Support Decode Context Parallel (DCP) for MLA](../sources/prs/vllm/PR-23734.md), [[BugFix][FlashInfer] Fix potential race condition for paged_kv_indptr_cpu](../sources/prs/vllm/PR-23737.md), [[Kernel] cuda kernels for upcoming decode context parallel feature](../sources/prs/vllm/PR-23791.md), [[Misc] add reorder_batch AttentionMetadataBuilder](../sources/prs/vllm/PR-23798.md), [Feature/vit attention unification# 23880](../sources/prs/vllm/PR-23978.md), [[Kernel] Support decode context parallelism on Blackwell with CUTLASS MLA](../sources/prs/vllm/PR-24385.md), [[Feature] Disallow FlashMLA on Blackwell](../sources/prs/vllm/PR-24521.md), [[Bug] Fix `is_flashmla_supported` Check Error](../sources/prs/vllm/PR-24774.md), [[DCP] Support Decode Context Parallel (DCP) for GQA with FlashAttention](../sources/prs/vllm/PR-24864.md), [[Bugfix][B200] Fix `cutlass_mla` hang](../sources/prs/vllm/PR-24966.md), [[Attention][DCP] Support DCP with query length > 1 (MTP) with FA3](../sources/prs/vllm/PR-25049.md), [[Bug] Fix `returned_lse` not Defined issue](../sources/prs/vllm/PR-25106.md), [[BugFix] Fix MLA assert with CUTLASS MLA](../sources/prs/vllm/PR-25478.md), [[Bugfix] [B200] cutlass_mla - ensure kv_split == 1 for batch size > 1](../sources/prs/vllm/PR-25509.md), [[Flashinfer][gpt-oss] Support FP8-qkv Flashinfer TRTLLM Sinks Attention](../sources/prs/vllm/PR-25674.md), [[Performance] Split FlashAttn attention and cache update](../sources/prs/vllm/PR-25954.md), [[Spec Decode] Enable efficient speculative decoding with FlashInfer-MLA](../sources/prs/vllm/PR-25984.md), [Move query quantization to attention layer for Flashinfer & Triton.](../sources/prs/vllm/PR-26534.md), [[Attention] Tune CUTLASS MLA num_splits](../sources/prs/vllm/PR-26846.md), [[BUGFIX][ROCM] ViT FlashAttention on ROCm (no GFX9) and contiguous on qwen3vl ROCm TORCH_SDPA](../sources/prs/vllm/PR-27190.md), [[Bugfix] Ensure calculated KV scales are applied in attention.](../sources/prs/vllm/PR-27232.md), [Prefer FlashAttention MLA as default over FlashMLA](../sources/prs/vllm/PR-27363.md), [[Misc] Make reorder batch also separate extends](../sources/prs/vllm/PR-27367.md), [[Attention] Use sparse prefill kernel for fp8 kv-cache in DeepSeek-v3.2](../sources/prs/vllm/PR-27532.md), [[AMD] Use Decoupled Kernel Block Size to Support AITER MLA block_size=1](../sources/prs/vllm/PR-27715.md), [[Bug] Batch invariant: Fix flash attn MLA `RuntimeError: scheduler_metadata must have shape (metadata_size)`](../sources/prs/vllm/PR-27884.md), [Update Flashinfer from `v0.4.1` to `v0.5.2`](../sources/prs/vllm/PR-27952.md), [[FlashInfer] Avoid FlashInfer block_size 16 + head_size 256 on blackwell](../sources/prs/vllm/PR-27994.md), [[ROCm][MLA] enable fp8 MLA decode on ROCm](../sources/prs/vllm/PR-28032.md), [[Mamba] - Consolidate Mambas Attention Logic](../sources/prs/vllm/PR-28133.md), [fix cross attention](../sources/prs/vllm/PR-28346.md), [[ROCm] Support for Whisper v1 with Aiter Unified Attention and Aiter Flash Attention](../sources/prs/vllm/PR-28376.md), [[Bugfix] Fix SM100 gpt-oss regression due to faulty attn sink support](../sources/prs/vllm/PR-28561.md), [[Attention][Bugfix] Fix FA sink support](../sources/prs/vllm/PR-28660.md), [[Feature] Prefill Context Parallel (PCP) basic support](../sources/prs/vllm/PR-28718.md), [[Model] Add support for openPangu moe model](../sources/prs/vllm/PR-28775.md), [bugfix: correct attn output with base 2 or e](../sources/prs/vllm/PR-28840.md), [[BugFix] Fix async-scheduling + FlashAttn MLA](../sources/prs/vllm/PR-28990.md), [[Feature] Batch invariant: Enable `TRITON_MLA` without prefix-caching](../sources/prs/vllm/PR-29125.md), [[Attention] Cache attention metadata builds across hybrid KV-cache groups](../sources/prs/vllm/PR-29627.md), [[Attention] Make `split_decodes_and_prefills(..., require_uniform=True)` support padding](../sources/prs/vllm/PR-29644.md), [[perf] Use direct copy (broadcast) instead of cat for k_nope/k_pe in MLA prefill](../sources/prs/vllm/PR-29710.md), [[Perf] Improve fp8 quant in mla; replace ReduceSum with ReduceScatterSum](../sources/prs/vllm/PR-29795.md), [[SpecDecode] Simplified alternative padded-speculation acceptance rate fix](../sources/prs/vllm/PR-29845.md), [[Quantization] fix: overflow with static per-tensor scaling](../sources/prs/vllm/PR-29867.md), [[BugFix] Fix DBO assert `assert B_block_table == B_q`](../sources/prs/vllm/PR-29933.md), [Add llmcompressor fp8 kv-cache quant (per-tensor and per-attn_head)](../sources/prs/vllm/PR-30141.md), [[ROCm][Bugfix] Add MLACommonMetadata to allowed attention types for speculative decoding](../sources/prs/vllm/PR-30430.md), [[Feature] Add SM103 (Blackwell Ultra) Support to vLLM](../sources/prs/vllm/PR-30484.md), [[Bugfix] Pass FA version in `MultiHeadAttention`](../sources/prs/vllm/PR-30575.md), [Triton Attention: Support cross-layers blocks](../sources/prs/vllm/PR-30687.md), [OffloadingConnector: Support kernel_block_size != block_size](../sources/prs/vllm/PR-30692.md), [[Misc][LLaMa4] Compile LLaMa Vision Encoder](../sources/prs/vllm/PR-30709.md), [Update note comment for flashinfer attention warmup](../sources/prs/vllm/PR-30711.md), [[Bugfix] Fix broken ViT attention selection for Blackwell device](../sources/prs/vllm/PR-30731.md), [[SM100] Enable fp8 compute for prefill MLA](../sources/prs/vllm/PR-30746.md), [[Kernels][FI] Skip trtllm attention when num_kv_heads=1](../sources/prs/vllm/PR-30842.md), [[Bugfix] [Kernel] Triton attention kernels: mask out V blocks that fall outside sliding window](../sources/prs/vllm/PR-30887.md), [[Bugfix] Fix incorrect tiles creation for mm prefix triton attention](../sources/prs/vllm/PR-30974.md), [[Misc] Fix grammar errors in comments and messages](../sources/prs/vllm/PR-31115.md), [[perf] Integrate flashinfer concat_mla_k](../sources/prs/vllm/PR-31171.md), [[SM100] Resubmit FMHA FP8 prefill for MLA](../sources/prs/vllm/PR-31195.md), [[Bugfix][Hardware][AMD] Fix last_page_len calculation in AITER MLA decode](../sources/prs/vllm/PR-31282.md), [fix(rocm): add early return in get_flash_attn_version for ROCm](../sources/prs/vllm/PR-31286.md), [[Bugfix][ROCm]Fix Qwen3-Next-80B-A3B-Thinking inference and optimize non-standard block size (544) support under rocm_atten](../sources/prs/vllm/PR-31380.md), [fixed mypy warnings for files vllm/v1/attention with TEMPORARY workaround](../sources/prs/vllm/PR-31465.md), [[1/N][Attention] Restructure attention: move files](../sources/prs/vllm/PR-31916.md), [[MISC] Add strict contiguity check for FlashInfer attention tensors](../sources/prs/vllm/PR-32008.md), [[4/N][Attention] Move MLA common to model_executor](../sources/prs/vllm/PR-32060.md), [[5/N][Attention] Finish eliminating `vllm/attention` folder](../sources/prs/vllm/PR-32064.md), [fix: Add glm4_moe_lite to MLA detection](../sources/prs/vllm/PR-32614.md), [[Bugfix][Attention] Explicitly report support for kv_cache_dtype bfloat16](../sources/prs/vllm/PR-32795.md), [[Spec Decode] Unified Parallel Drafting](../sources/prs/vllm/PR-32887.md), [[ROCm][perf] Shuffle KV cache to use paged_attention_common](../sources/prs/vllm/PR-32914.md), [[Attention] FA4 integration](../sources/prs/vllm/PR-32974.md), [[Attention] Use `has_flashinfer` helper](../sources/prs/vllm/PR-33177.md), [[Bugfix] Disable TRTLLM attention when KV transfer is enabled](../sources/prs/vllm/PR-33192.md), [[PERF] Change GDN Attention State Layout from [N, HV, K, V] to [N, HV, V, K]](../sources/prs/vllm/PR-33291.md), [Triton MLA perf fixes](../sources/prs/vllm/PR-33529.md), [[Perf] Disable clean_logits in deepgemm fp8_mqa_logits kernel](../sources/prs/vllm/PR-33568.md), [[Bugfix] Fix sparse MLA metadata building](../sources/prs/vllm/PR-33579.md), [[Bugfix] fix DeepSeek R1 with CUTLASS MLA Broken on B200](../sources/prs/vllm/PR-33637.md), [enable skipping of SW attention layers when using FP8 KV cache](../sources/prs/vllm/PR-33695.md), [[Bugfix] Fix DSV3.2 NVFP4](../sources/prs/vllm/PR-33932.md), [Reapply [Attention][FA3] Update FA3 to include new swizzle optimization](../sources/prs/vllm/PR-34043.md), [[Bugfix] Relax TRTLLM KV cache contiguity assertion for cross-layer layout](../sources/prs/vllm/PR-34158.md), [[Bugfix] Fix DP Attention Padding in Dummy Run](../sources/prs/vllm/PR-34187.md), [[CPU][Perf] Accelerate Attention head for s390x using vector intrinsics](../sources/prs/vllm/PR-34434.md), [[BugFix] Add support for MTP num_speculative_tokens > 1 with sparse MLA](../sources/prs/vllm/PR-34552.md), [[Kernel] Add FP8 KV cache support to Triton MLA decode attention](../sources/prs/vllm/PR-34597.md), [[Update] Use FlashInfer fast_decode_plan directly instead of replication](../sources/prs/vllm/PR-34687.md), [[Bugfix] Fix MLA attention crash with AWQ/GPTQ quantized models](../sources/prs/vllm/PR-34695.md), [[Attention] Use FA4 for MLA prefill](../sources/prs/vllm/PR-34732.md), [[Bugfix] Fix GDN attention crash with mixed decode/spec-decode batches](../sources/prs/vllm/PR-34871.md), [[Model Bash][DSR1] Add selective dynamic shape marking for CustomOp](../sources/prs/vllm/PR-34900.md), [[Attention][Perf][Kernel] Replace torch.cat with vectorized CUDA kernel MLA query concat - DeepSeek-V3.2](../sources/prs/vllm/PR-34917.md), [[Model Runner V2] Support attention group](../sources/prs/vllm/PR-35036.md), [[Bug][DSV3.2] Always prepare metadata for DeepGEMM Sparse Attention](../sources/prs/vllm/PR-35075.md), [[Linear Attention] fix bug for linear attention + prefix caching + reset_prefix_cache](../sources/prs/vllm/PR-35157.md), [[BUGFIX][Mamba][Qwen3.5] Zero freed SSM cache blocks on GPU](../sources/prs/vllm/PR-35219.md), [[Feat] Add CUDA torch fallbacks for fp8_mqa_logits/fp8_paged_mqa_logits_torch function](../sources/prs/vllm/PR-35271.md), [[Attention][Perf] Optimize cp_gather_and_upconvert_fp8_kv_cache - DeepSeek-v3.2](../sources/prs/vllm/PR-35290.md), [[Performance] Extract KV cache update op from flashinfer forward](../sources/prs/vllm/PR-35422.md), [Fix routed experts capture for hybrid models (Mamba + Attention)](../sources/prs/vllm/PR-35744.md), [[ROCm] Support MLA with nhead<16 and FP8 KV cache for TP=8 (Kimi K2.5/Linear)](../sources/prs/vllm/PR-35850.md), [[Perf] Support FP8 KV cache for Flashinfer MLA Sparse](../sources/prs/vllm/PR-35891.md), [[BugFix] Fallback from FA4->FA2 for Batch Invariance](../sources/prs/vllm/PR-36059.md), [Add 320 dimension size support to MLA](../sources/prs/vllm/PR-36161.md), [[Bugfix][MLA] Add logits size budget to sparse indexer prefill chunking](../sources/prs/vllm/PR-36178.md), [[mla] Support fused FP8/NVFP4 output quantization in MLA attention (#35792)](../sources/prs/vllm/PR-36205.md), [mla: don't update kv cache on dummy forwards](../sources/prs/vllm/PR-36282.md), [Disable cascade attention by default](../sources/prs/vllm/PR-36318.md), [feat(attention): extract KV-cache update from FlashAttentionDiffKV ba…](../sources/prs/vllm/PR-36466.md), [[Kernel] Fuse FP8 output quantization into merge_attn_states](../sources/prs/vllm/PR-36518.md), [[Bugfix][Sparse MLA] report indexer CG support properly](../sources/prs/vllm/PR-36519.md), [[ROCm] Utilize persistent MLA kernel from AITER](../sources/prs/vllm/PR-36574.md), [[Misc][Attention] Clean up unused method in `CPU_ATTN`](../sources/prs/vllm/PR-36673.md), [fix(kv-cache): increase hybrid attention grouping threshold from 1.25 to 1.5](../sources/prs/vllm/PR-36684.md), [[ROCm] Attention selector reordering](../sources/prs/vllm/PR-36702.md), [[DSV3.2][MTP] Optimize Indexer MTP handling](../sources/prs/vllm/PR-36723.md), [[ROCm] Fix KV copy methods and auto-select attention backend for ROCm](../sources/prs/vllm/PR-36845.md), [[ROCm] Validate block_size for explicitly selected attention backends](../sources/prs/vllm/PR-36846.md), [[Feat][Spec Decode] DFlash](../sources/prs/vllm/PR-36847.md), [[Feat][Bugfix] Enable additional dimension for Flashinfer MLA and fix routing dtype](../sources/prs/vllm/PR-36931.md), [[MTP][Sparse MLA] Take advantage of native MTP support in indexer when possible](../sources/prs/vllm/PR-36982.md), [[Bugfix] Fix KV scales inconsistency in fp8 MLA & FlashInfer kv_cache_dtype "auto" leading to gibberish](../sources/prs/vllm/PR-37054.md), [[Bugfix] Disable cross-layer KV cache for MLA attention backends](../sources/prs/vllm/PR-37090.md), [[Benchmark] Improvements to attention benchmark script](../sources/prs/vllm/PR-37115.md), [[XPU] support MLA model on Intel GPU](../sources/prs/vllm/PR-37143.md), [[Misc] Add `float16` to `CacheDType`](../sources/prs/vllm/PR-37199.md), [[ROCM][Bugfix] Use correct stride in cp_mha_gather_cache_kernel for hybrid model (#37228)](../sources/prs/vllm/PR-37228.md), [[Perf] Set Flashinfer sparse MLA as default backend for FP8 kv cache](../sources/prs/vllm/PR-37252.md), [[Attention] Support distinguishing between short extends and decodes](../sources/prs/vllm/PR-37303.md), [Add nvfp4 support to reshape_and_cache_flash](../sources/prs/vllm/PR-37332.md), [[Model Runner V2] fix draft attention metadata generation](../sources/prs/vllm/PR-37364.md), [[torch.compile] Refactor Attention Quant Fusion Pass and Remove Boilerplate](../sources/prs/vllm/PR-37373.md), [[Perf][Kernel] Persistent TopK scheduler: unified CUDAGraph-safe kernel with dynamic per-row dispatch - DeepSeek-V3.2 DSA decode](../sources/prs/vllm/PR-37421.md), [[BugFix] Allow qk_nope_head_dim=192 in FlashInfer MLA backend checks](../sources/prs/vllm/PR-37475.md), [[Performance] Remove unnecessary zero-fill of MLA decode output tensor in Aiter backend](../sources/prs/vllm/PR-37539.md), [[Bugfix][ROCm] Fix lru_cache on paged_mqa_logits_module](../sources/prs/vllm/PR-37547.md), [[Bugfix] Disable --calculate-kv-scales for hybrid GDN/Mamba+Attention…](../sources/prs/vllm/PR-37565.md), [[ROCm][Bugfix] fix cache block size mismatch for aiter unified attention](../sources/prs/vllm/PR-37606.md), [[NIXL][BUG] Fix Triton heterogeneous TP](../sources/prs/vllm/PR-37940.md), [[Perf] FP8 FlashInfer Attn for ViT](../sources/prs/vllm/PR-38065.md), [[Model Runner V2] Rebuild attention metadata before eagle decode full…](../sources/prs/vllm/PR-38311.md), [[GDN] Eliminate GPU->CPU sync in prepare_chunk_indices during prefill](../sources/prs/vllm/PR-38361.md), [[Attention Backend] TurboQuant: 2-bit KV cache compression with 4x capacity](../sources/prs/vllm/PR-38479.md), [[Bugfix][MLA] Change default SM100 MLA prefill backend back to TRT-LLM](../sources/prs/vllm/PR-38562.md), [[ROCm] Fix aiter persistent mode mla with q/o nhead<16 for kimi-k2.5 tp8](../sources/prs/vllm/PR-38615.md), [Fix MLA runs when use_inductor_graph_partition=True](../sources/prs/vllm/PR-38631.md), [[FA4] Update flash-attention to latest upstream FA4](../sources/prs/vllm/PR-38690.md), [[Bugfix] Restrict TRTLLM attention to SM100, fixing GB300 (SM103) hang](../sources/prs/vllm/PR-38730.md), [[Bugfix] Fix test mocks after SM100 restriction in #38730](../sources/prs/vllm/PR-38791.md), [[Perf] Reduce H2D pageable memory copies](../sources/prs/vllm/PR-38794.md), [[FlashAttention] Symlink FA4 instead of copying when using `VLLM_FLASH_ATTN_SRC_DIR`](../sources/prs/vllm/PR-38814.md), [[Attention][MLA] Re-enable FA4 as default MLA prefill backend](../sources/prs/vllm/PR-38819.md), [[Attention] relax the head dim 512 and paged kv for sm90+FA4](../sources/prs/vllm/PR-38835.md), [[Refactor] Improve indexer decode path metadata preparation](../sources/prs/vllm/PR-38865.md), [[Bugfix] Fix broken explicit unquantized kv cache dtype support](../sources/prs/vllm/PR-38922.md), [[Quantization] - Layerwise reloading of Attention/KV quantized models](../sources/prs/vllm/PR-38995.md), [[Bugfix] Fix FlashInfer crash with kv_cache_dtype_skip_layers](../sources/prs/vllm/PR-39002.md), [[XPU] Quick fix for TritonMLA to remove cuda hardcode](../sources/prs/vllm/PR-39088.md), [[ROCm] Align AiterFlashAttentionImpl attn_type check with backend](../sources/prs/vllm/PR-39119.md), [[Bug] Fix rocm sparse attn indexer issue](../sources/prs/vllm/PR-39225.md), [[Model Runner V2] Fix flex attention kv blocks calculation issue](../sources/prs/vllm/PR-39353.md), [[MLA] Optimize mla indexer prepare uniform decode for MTP > 1](../sources/prs/vllm/PR-39458.md), [[Bugfix] Fix tensor shape mismatch in sparse attention with speculative decoding](../sources/prs/vllm/PR-39542.md), [[XPU] properly handle q_descale on XPU as quant query input not supported](../sources/prs/vllm/PR-39676.md), [[Bugfix] Fix mismatch between global and local attention heads in tensor-parallel mode for param2moe model](../sources/prs/vllm/PR-39707.md), [add warning when FP8 KV cache misses prefill query quantization](../sources/prs/vllm/PR-39752.md), [[Bugfix] add support for 'num_attention_groups' in ModelArchConfigConvertorBase for Step3p5](../sources/prs/vllm/PR-39796.md), [[Attention] use diff kv backend for mimo v2 flash](../sources/prs/vllm/PR-40045.md), [Add nvfp4 kv cache support](../sources/prs/vllm/PR-40177.md), [[Attention][TurboQuant] Share dequant buffers, eliminate float16_copy](../sources/prs/vllm/PR-40941.md), [[Bugfix][Hybrid][NemotronH] Fix mamba_cache_mode=all + speculative decoding crash](../sources/prs/vllm/PR-41233.md), [[DSv4] Improved fused Indexer Q quant kernel](../sources/prs/vllm/PR-41428.md), [fix: remove unused norm for dpskv4](../sources/prs/vllm/PR-41710.md), [[MLA Attention Backend] Add TOKENSPEED_MLA backend for DSR1/Kimi K25 prefill + decode on Blackwell](../sources/prs/vllm/PR-41778.md), [[feat] Add FP8 per-tensor Q scale support to Triton attention backend](../sources/prs/vllm/PR-42080.md), [[Bugfix] Fix TRTLLM ragged MLA prefill workspace warmup](../sources/prs/vllm/PR-42112.md), [[DSv4] Improved dequant gather K cache kernel](../sources/prs/vllm/PR-42236.md), [[Bugfix] mamba: run single-token extends as decodes](../sources/prs/vllm/PR-42430.md), [[6/n] Migrate activation kernels, gptq, gguf, non cutlass w8a8 to libtorch stable ABI (continued)](../sources/prs/vllm/PR-42663.md), [[CPU] Add fused GDN support for AMX CPU platform](../sources/prs/vllm/PR-42707.md), [[CPU] Specify required KV cache layout for CPU attention backend](../sources/prs/vllm/PR-42740.md), [[Refactor] Remove dead cuda kernels](../sources/prs/vllm/PR-42767.md), [[ROCm] [Bugfix] Fix DeepSeek V4 Functionality and Accuracy](../sources/prs/vllm/PR-42810.md), [[Perf] Add do_not_specialize in fused FP8 RoPE kernel](../sources/prs/vllm/PR-42849.md), [[Perf][MLA] Enable FULL cudagraph capture for TRITON_MLA decode](../sources/prs/vllm/PR-42885.md), [add cutedsl dsv4 indexer fp8 kernel](../sources/prs/vllm/PR-42899.md), [[XPU] update xpu graph usage](../sources/prs/vllm/PR-43043.md), [[Model Refactoring] Move deepseek_v4_ops to models/deepseek_v4 [3/N]](../sources/prs/vllm/PR-43073.md), [FlashAttention-4](../wiki/kernels/flash-attention-4.md), [FlashAttention SM100 MLA TopK Sparse Forward](../wiki/kernels/flash-attention-sm100-mla-topk.md), [FlashMLA — Multi-head Latent Attention](../wiki/kernels/flashmla.md), [Gated Delta Net — Linear Attention](../wiki/kernels/gated-delta-net.md), [Native Sparse Attention (NSA)](../wiki/kernels/nsa.md), [Sparse MLA (DeepSeek V3.2)](../wiki/kernels/sparse-mla.md), [TensorRT-LLM Blackwell FP4 DSA Indexer](../wiki/kernels/tensorrt-llm-blackwell-indexer.md) | -| `batched-gemv` | [Twelve Attempts at NVFP4 Batched GEMV](../sources/blogs/amandeep-nvfp4-attempts.md), [NVFP4 GEMV and Improved NVFP4 GEMV](../sources/blogs/simon-nvfp4-gemv.md), [Blackwell NVFP4 Kernel Hackathon Journey](../sources/blogs/yue-nvfp4-hackathon.md), [GPU Mode NVFP4 Hackathon - Problem 1: Batched GEMV](../sources/contests/gpu-mode-nvfp4/problem-1-gemv.md), [NVFP4 Batched GEMV](../wiki/kernels/nvfp4-gemv.md) | -| `decode` | [FlashMLA — Multi-head Latent Attention](../sources/blogs/flashmla.md), [DeepSeek-V3.2-Exp in vLLM: Fine-Grained Sparse Attention in Action](../sources/blogs/vllm-deepseek-v3-sparse-attention.md), [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [FlashInfer MLSys 2026 - Track C: Gated Delta Net](../sources/contests/flashinfer-mlsys26/track-c-gated-delta-net.md), [[TRTLLM-10022][feat] Add hopper xqa decode support for skip softmax attention](../sources/prs/TensorRT-LLM/PR-10264.md), [[https://nvbugs/5669671][fix] Support GuidedDecoder with sharded logits (pick #10698)](../sources/prs/TensorRT-LLM/PR-10742.md), [[TRTLLM-10407][feat] Integrate CuTE DSL top-k kernel for Blackwell](../sources/prs/TensorRT-LLM/PR-11900.md), [[TRTLLM-11540][feat] Add EAGLE3 dynamic tree speculative decoding support](../sources/prs/TensorRT-LLM/PR-12062.md), [[None][feat] Temporally-Correlated Heuristic-guided Indexer TopK for Sparse Attention](../sources/prs/TensorRT-LLM/PR-12385.md), [[None][feat] Support sparse mqa/gqa attention](../sources/prs/TensorRT-LLM/PR-12470.md), [[None][feat] Add PDL support to CuTE DSL top-k kernels](../sources/prs/TensorRT-LLM/PR-12506.md), [[None][feat] Optimize causal_conv1d prefill and decode kernels](../sources/prs/TensorRT-LLM/PR-13103.md), [[TRTLLM-34871][feat] Add cute dsl FP8 paged MQA logits decode kernel](../sources/prs/TensorRT-LLM/PR-13219.md), [[None][perf] Scheme X L2-aware dispatcher and PDL launchers for sparse-attention GVR Top-K](../sources/prs/TensorRT-LLM/PR-13477.md), [[None][perf] Drop cubin and Eliminate ~6s FMHA JIT recompile in eager generation by aligning kernel selection with CUDA graph warmup](../sources/prs/TensorRT-LLM/PR-13505.md), [[TRTLLM-35237][feat] Add cute dsl FP4 paged MQA logits decode kernel](../sources/prs/TensorRT-LLM/PR-13929.md), [[None][feat] DSv4: enable GVR Heuristic Top-K for compress_ratio=4](../sources/prs/TensorRT-LLM/PR-14219.md), [feat: update decode attention APIs](../sources/prs/flashinfer/PR-1007.md), [bugfix: import wrapper of mla decode](../sources/prs/flashinfer/PR-1013.md), [add multi-item scoring](../sources/prs/flashinfer/PR-1015.md), [feat: add functional per-head FP8 quantization for FA3](../sources/prs/flashinfer/PR-1033.md), [[nvidia] Add Blackwell FMHA decode kernel from TRT-LLM](../sources/prs/flashinfer/PR-1051.md), [[Feature] Support PDL for batch Prefill and Decode](../sources/prs/flashinfer/PR-1117.md), [update trtllm-gen decode attention kernel launcher](../sources/prs/flashinfer/PR-1189.md), [Enable cudnn decode and add tests for the cudnn decode kernel](../sources/prs/flashinfer/PR-1221.md), [feat: add trtllm-gen mla cubin](../sources/prs/flashinfer/PR-1222.md), [add trtllm-gen context attention](../sources/prs/flashinfer/PR-1239.md), [Add trtllm-gen attention mha kernel with FP8 Q/K/V and FP8 output](../sources/prs/flashinfer/PR-1242.md), [feat: enable trtllm-gen mla MTP](../sources/prs/flashinfer/PR-1258.md), [refactor: refactor trtllm-gen attention kernel integration code](../sources/prs/flashinfer/PR-1289.md), [refactor: Improved metainfo for trtllm-gen fmha](../sources/prs/flashinfer/PR-1292.md), [test qkvo quantization not equal to 1.](../sources/prs/flashinfer/PR-1314.md), [feat: support output nvfp4 in trtllm-gen function call.](../sources/prs/flashinfer/PR-1318.md), [fix: fix trtllm-gen mla error on new interface](../sources/prs/flashinfer/PR-1348.md), [support trtllm-gen prefill fp4 output](../sources/prs/flashinfer/PR-1360.md), [Support scale factor start index for fp4 mha prefill/decode](../sources/prs/flashinfer/PR-1363.md), [GPT-OSS Support: Add Blackwell MoE mxfp4 implementation from TRTLLM and Attention Sink](../sources/prs/flashinfer/PR-1389.md), [Add Mxfp4 trtllm-gen moe unit tests](../sources/prs/flashinfer/PR-1399.md), [benchmark: trtllm-gen mha with sink, add benchmark args](../sources/prs/flashinfer/PR-1415.md), [fix: remote redundant zero_init from trtllm-gen attn](../sources/prs/flashinfer/PR-1444.md), [feat: enable trtllm-gen attn speculative decoding verify by decode](../sources/prs/flashinfer/PR-1453.md), [Fix TRTLLM NVFP4-out attention kernel scale factor dim issue](../sources/prs/flashinfer/PR-1460.md), [feat: add pdl for trtllm-gen attn](../sources/prs/flashinfer/PR-1484.md), [feat: Support fp8 qkv, fp16/bf16 out MHA for trtllm-gen.](../sources/prs/flashinfer/PR-1490.md), [backend: Refactor trtllm-gen fmha metainfo loading](../sources/prs/flashinfer/PR-1518.md), [feat: Add fp8-qkv, fp16/bf16 output MHA](../sources/prs/flashinfer/PR-1540.md), [Backend: downgrade trtllm-gen kernel to cuda-12](../sources/prs/flashinfer/PR-1567.md), [feat: Support for inferring out_dtype from out.dtype for TRTLLM attention kernel](../sources/prs/flashinfer/PR-1578.md), [fix: Improve TRTLLM attention kernel out_dtype unit test](../sources/prs/flashinfer/PR-1590.md), [bugfix: collect all modules to aot](../sources/prs/flashinfer/PR-1622.md), [fix: pass workspace for trtllm-gen attention](../sources/prs/flashinfer/PR-1635.md), [perf&bugfix: skip kv-tile computation out of sliding window in FA2; fix __syncthreads in mergestate](../sources/prs/flashinfer/PR-1661.md), [TGV GEMM as a BF16 backend alternative to cuBLAS](../sources/prs/flashinfer/PR-1668.md), [feat: Add `variant.OutputTransform()` to decode kernels](../sources/prs/flashinfer/PR-1670.md), [feat: Batch-size invariant FA2 Prefill & Decode](../sources/prs/flashinfer/PR-1675.md), [feat: port fast_decode_plan from sgl](../sources/prs/flashinfer/PR-1745.md), [Fix tests/test_trtllm_gen_attention.py::test_trtllm_batch_prefill, ::test_trtllm_batch_decode mismatch error](../sources/prs/flashinfer/PR-1755.md), [fix: should pass global_override_indptr_cpu in fast_decode_plan param list](../sources/prs/flashinfer/PR-1757.md), [Waive / disable test_mla_decode_kernel.py::test_mla_decode_kernel for not sm80 ](../sources/prs/flashinfer/PR-1771.md), [feat: add xqa backend and completes NHD/HND coverage for trtllm-gen/xqa backend](../sources/prs/flashinfer/PR-2001.md), [Fix trtllm-gen attention illegal memory access](../sources/prs/flashinfer/PR-2002.md), [use scalar for kv_scale in xqa](../sources/prs/flashinfer/PR-2033.md), [feat: add xqa mla backend](../sources/prs/flashinfer/PR-2053.md), [misc: Add XQA decode to microbenchmark for sm90 and sm120](../sources/prs/flashinfer/PR-2055.md), [unittest: improve the efficiency of xqa unittests](../sources/prs/flashinfer/PR-2075.md), [[Feature] Support batch prefill for POD Attention](../sources/prs/flashinfer/PR-2079.md), [enable xqa fp8 output](../sources/prs/flashinfer/PR-2081.md), [[API change] Allow using torch.Tensor for scales for trtllm-gen attention](../sources/prs/flashinfer/PR-2084.md), [enable xqa speculative decoding](../sources/prs/flashinfer/PR-2105.md), [add tensor scale input for xqa](../sources/prs/flashinfer/PR-2110.md), [feat: support variable sequence length in decode kernel of trtllm-gen attention](../sources/prs/flashinfer/PR-2125.md), [feat: add trtllm-gen per-tensor sparseMla kernels.](../sources/prs/flashinfer/PR-2138.md), [Enable Hopper FA3 FP8 attention in decode.py](../sources/prs/flashinfer/PR-2148.md), [refactor: Move mla code from decode.py to mla.py and add to documentation](../sources/prs/flashinfer/PR-2163.md), [[TRTLLM-Gen Fmha] add optimized trtllm-gen decode kernels for high throughput + speculative decoding](../sources/prs/flashinfer/PR-2265.md), [fix: Decode benchmark's fa2_tc uses backend=fa2 in wrapper](../sources/prs/flashinfer/PR-2302.md), [feat: [Qwen3-Next] Add Cute DSL GDN decode kernel and tests](../sources/prs/flashinfer/PR-2370.md), [fix: ensure each CTA processes full numHeadsQPerKv for trtllm decode kernel](../sources/prs/flashinfer/PR-2380.md), [A Blackwell-optimized version of selective_state_update (decode)](../sources/prs/flashinfer/PR-2387.md), [perf: improve gdn decode cute-dsl kernels](../sources/prs/flashinfer/PR-2405.md), [feat: Add TRTLLM-Gen Skip-Softmax kernels for prefill and decode](../sources/prs/flashinfer/PR-2477.md), [Ameyn/gdn decode cutedsl kernel](../sources/prs/flashinfer/PR-2498.md), [Support NVFP4 KV cache decode on SM120](../sources/prs/flashinfer/PR-2520.md), [Feat/gdn decode pooled](../sources/prs/flashinfer/PR-2521.md), [pick fa2 for BatchDecodeWithPagedKVCacheWrapper auto backend](../sources/prs/flashinfer/PR-2530.md), [Perf: Optimize GDN decode pretranspose kernel for all batch sizes](../sources/prs/flashinfer/PR-2588.md), [Ameyn/gdn bf16 tolerance parallel reduction](../sources/prs/flashinfer/PR-2610.md), [perf(gdn): optimize MTP kernel with ILP rows and SMEM v caching](../sources/prs/flashinfer/PR-2618.md), [feat: add pool+indices support to gated_delta_rule_decode_pretranspose (bf16 path) ](../sources/prs/flashinfer/PR-2619.md), [benchmark: Enable speculative decode microbenchmarking for paged decode](../sources/prs/flashinfer/PR-2628.md), [fix: add SM121 support to SM120 version guards](../sources/prs/flashinfer/PR-2631.md), [[feat] trtllm-gen mxfp8 gemm](../sources/prs/flashinfer/PR-2653.md), [feat(gdn): add BF16 state kernel with MTP support beyond T>4 with intermediate caching.](../sources/prs/flashinfer/PR-2679.md), [Add NVFP4 KV cache quantization support for SM100](../sources/prs/flashinfer/PR-2702.md), [[gdn] support non-contiguous state for decoding](../sources/prs/flashinfer/PR-2727.md), [Add cute dsl mla decode op](../sources/prs/flashinfer/PR-2743.md), [feat: Expose TRT-LLM FMHA style paged KV Cache and page table layout](../sources/prs/flashinfer/PR-2770.md), [[fix] Bugfix 1367: fix VariableBlockSparseAttention buffer overflow by dynamically resizing kv_lens_buffer](../sources/prs/flashinfer/PR-2802.md), [[CuTe DSL] Add modular FMHA prefill and MLA decode attention kernels](../sources/prs/flashinfer/PR-2805.md), [feat(gdn): add padding index guard for bf16 decode kernel](../sources/prs/flashinfer/PR-2810.md), [[Fmha] Sparse MLA decode kernel selection heuristics](../sources/prs/flashinfer/PR-2836.md), [perf: Optimize GDN MTP decode kernel (v15) — eliminate ilp=1 fallback…](../sources/prs/flashinfer/PR-2842.md), [read real strides for kv and block scale](../sources/prs/flashinfer/PR-2844.md), [[fix] bugfix 2856: Fix pre-allocated out shape check in trtllm_batch_decode_with_kv_cache_mla for q_len_per_req > 1](../sources/prs/flashinfer/PR-2876.md), [feat: add pdl support for cute dsl mla decode kernel support](../sources/prs/flashinfer/PR-2901.md), [Only swizzle on v block scale; rename kv_block_scales to kv_cache_sf](../sources/prs/flashinfer/PR-2954.md), [perf: Optimize CUTLASS MoE helper kernels for small-batch decode workloads](../sources/prs/flashinfer/PR-3014.md), [Support lse in trtllm paged attn kernels](../sources/prs/flashinfer/PR-3058.md), [Support NVFP4 KV for prefill and batch attention kernels](../sources/prs/flashinfer/PR-3097.md), [feat(trace): embed runnable init() in every TraceTemplate](../sources/prs/flashinfer/PR-3221.md), [Support Kimi K2.5 H64 CuTe DSL MLA decode](../sources/prs/flashinfer/PR-3235.md), [Ameyn/gdn bf16 dispatcher and 4d pool](../sources/prs/flashinfer/PR-3268.md), [feat: support deepseek prefill attention shape](../sources/prs/flashinfer/PR-765.md), [bugfix: MLA decode should multiply sm_scale by math::log2e](../sources/prs/flashinfer/PR-787.md), [fix rope logic in mla decoding](../sources/prs/flashinfer/PR-793.md), [feat: apply sm_scale at logits instead of q in FA2 template](../sources/prs/flashinfer/PR-801.md), [perf: MLA decode kernel implemented by CuTe targeted to SM80](../sources/prs/flashinfer/PR-844.md), [Add POD-Attention to FlashInfer](../sources/prs/flashinfer/PR-858.md), [perf: reduce torch.library dispatch overhead](../sources/prs/flashinfer/PR-968.md), [perf: Fix python API overhead when CUDAGraph is not enabled](../sources/prs/flashinfer/PR-969.md), [Disable kernel cutlass_mla_decode on SM103](../sources/prs/sglang/PR-10058.md), [Use trtllm_mla decode kernel for draft extend in speculative decoding](../sources/prs/sglang/PR-11664.md), [Use trtllm mha decode kernel for target_verify in speculative decoding](../sources/prs/sglang/PR-13976.md), [Fix accuracy issue when using a16w16 mla_decode_fwd](../sources/prs/sglang/PR-14936.md), [[jit-kernel] Add CuTe DSL GDN Decode Kernel](../sources/prs/sglang/PR-15631.md), [[DeepSeek v3.2] Opt MTP decode cuda batch sizes and nsa implementation](../sources/prs/sglang/PR-16961.md), [Optimize GDN decode for Qwen3 Next](../sources/prs/sglang/PR-17094.md), [feat(gdn): add FlashInfer K-last SSM layout support for GDN prefill and decode for Hopper](../sources/prs/sglang/PR-18361.md), [[AMD] Update aiter to v0.1.10.post2](../sources/prs/sglang/PR-18423.md), [feat: add FA4 SM90 paged KV decode support & update attention docs](../sources/prs/sglang/PR-18442.md), [Tilelang sparse decode fwd for dsv32 mi355](../sources/prs/sglang/PR-18488.md), [[NVIDIA] Integrate FlashInfer decode kernel (Blackwell) for Qwen3.5](../sources/prs/sglang/PR-19150.md), [Fix MLA decode path returning unwritten (padded) rows](../sources/prs/sglang/PR-19902.md), [[AMD] Fix FP8 assertion failure in aiter MLA decode by falling back to self.k_scale](../sources/prs/sglang/PR-19935.md), [Support Triton MLA FP8 KV cache](../sources/prs/sglang/PR-20479.md), [[Kernel] Fuse temperature + softmax in sampling for decode speedup](../sources/prs/sglang/PR-20501.md), [[KDA] Support CuTeDSL KDA decode kernel](../sources/prs/sglang/PR-21203.md), [ [GDN] Remove FlashInfer GDN decode + no_buffer guard and default to FlashInfer on SM100+ ](../sources/prs/sglang/PR-21861.md), [[attn backend] Integrate tokenspeed_mla prefill/decode kernels (fp8 kv cache, blackwell)](../sources/prs/sglang/PR-24925.md), [amd/deepseek_v4 27/N [fix] Reduce Triton autotune configs for faster first-time server launch](../sources/prs/sglang/PR-25554.md), [Introduce SchedulerDPAttnAdapter to own DP-attention state](../sources/prs/sglang/PR-25611.md), [Move DP-attention adapter methods to SchedulerDPAttnAdapter](../sources/prs/sglang/PR-25612.md), [[Refactor] Rename NSA → DSA: user-facing aliases, file/class/import rename](../sources/prs/sglang/PR-25821.md), [[Test] Stage-a sanity kits; consolidate core/ + models_e2e/ tests](../sources/prs/sglang/PR-25831.md), [Add fast decode plan for flashinfer mla](../sources/prs/sglang/PR-3987.md), [[Revision] Add fast decode plan for flashinfer mla ](../sources/prs/sglang/PR-4012.md), [support cmake for sgl-kernel](../sources/prs/sglang/PR-4706.md), [Add optimized native kernels in sgl-kernel](../sources/prs/sglang/PR-5150.md), [Add Cutlass MLA attention backend](../sources/prs/sglang/PR-5390.md), [[PD] Fix dynamic port support and MLA buffer for Mooncake](../sources/prs/sglang/PR-5415.md), [[perf] introduce deep gemm group_gemm_masked as gemm](../sources/prs/sglang/PR-5432.md), [Cutlass MLA decode - fix dtype error](../sources/prs/sglang/PR-5868.md), [[Feature] Comprehensive Hybrid Parallelism Support](../sources/prs/sglang/PR-6389.md), [CPU: map changes from developing branch in sgl-kernel](../sources/prs/sglang/PR-6833.md), [[perf][sgl-kernel] extend cutlass_mla_decode to support num_head < 128](../sources/prs/sglang/PR-6929.md), [Fix grammar abort & Minor style fixes](../sources/prs/sglang/PR-7204.md), [Support non-contiguous query input for extend/decode attention](../sources/prs/sglang/PR-7462.md), [[NVIDIA] Add Low Latency NVFP4 decode kernels from Flashinfer](../sources/prs/sglang/PR-8552.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [[CUDA] Support int4 `T.gemm`](../sources/prs/tilelang/PR-2063.md), [[Attention] MLA decode optimizations](../sources/prs/vllm/PR-12528.md), [[Perf] Mem align KV caches for CUDA devices (MLA perf improvement)](../sources/prs/vllm/PR-12676.md), [[core] Perf improvement for DSv3 on AMD GPUs](../sources/prs/vllm/PR-13718.md), [[V1] V1 Enablement Oracle ](../sources/prs/vllm/PR-13726.md), [[Kernel][CPU] CPU MLA](../sources/prs/vllm/PR-14744.md), [[FEAT][ROCm] Integrate Fused MoE Kernels from AITER](../sources/prs/vllm/PR-14967.md), [[Bugfix] Fix incorrect qwen2.5-vl attention mask pre-computation](../sources/prs/vllm/PR-15200.md), [[ROCM][KERNEL] Paged attention for V1](../sources/prs/vllm/PR-15720.md), [[NVIDIA] Support Cutlass MLA for Blackwell GPUs](../sources/prs/vllm/PR-16032.md), [Upstream Llama4 Support to Main](../sources/prs/vllm/PR-16113.md), [[Kernel] Unified Triton kernel that doesn't distinguish between prefill + decode](../sources/prs/vllm/PR-16828.md), [[Attention] FA3 decode perf improvement - single mma warp group support for head dim 128](../sources/prs/vllm/PR-16864.md), [[ROCm][Kernel][V1] Enable AMD Radeon GPU Custom Paged Attention on v1](../sources/prs/vllm/PR-17004.md), [[Misc][ROCm] Exclude `cutlass_mla_decode` for ROCm build](../sources/prs/vllm/PR-17289.md), [[v1] AttentionMetadata for each layer](../sources/prs/vllm/PR-17394.md), [[Bugfix][ROCm] Use `chunked_prefill_paged_decode` as fallback for V1 attention on ROCm](../sources/prs/vllm/PR-18093.md), [SM100 Cutlass MLA decode with unrestricted num_heads (< 128) for DeepSeek TP](../sources/prs/vllm/PR-20769.md), [[Kernel] Flashinfer MLA (trtllm-gen) decode kernel integration](../sources/prs/vllm/PR-21078.md), [[v1] Add Whisper model support (encoder-decoder)](../sources/prs/vllm/PR-21088.md), [Support encoder-only models without KV-Cache](../sources/prs/vllm/PR-21270.md), [[v1][attention] Support Hybrid Allocator + FlashInfer](../sources/prs/vllm/PR-21412.md), [[Attention] Support multiple attention metadata builders per kv_cache_spec + proper local attention no hybrid kv cache fix](../sources/prs/vllm/PR-21588.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv/out Attention Kernel](../sources/prs/vllm/PR-21716.md), [[NVIDIA] Support Flashinfer TRT-LLM Prefill Attention Kernel](../sources/prs/vllm/PR-22095.md), [[Bugfix] Add proper comparison for package versions](../sources/prs/vllm/PR-22314.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv NVFP4-out Attention Kernel](../sources/prs/vllm/PR-22703.md), [[Bugfix] Fix default enable for CUTLASS MLA on SM100](../sources/prs/vllm/PR-22738.md), [[Core] Always use tensor cores for Flashinfer Decode Wrapper](../sources/prs/vllm/PR-23214.md), [[Flashinfer] Support Flashinfer TRTLLM FP8-qkv BF16/FP16-out Attention Kernel](../sources/prs/vllm/PR-23647.md), [[v1] Add cross-attention KV cache support for encoder-decoder models](../sources/prs/vllm/PR-23664.md), [[Core/DBO][1/N] Add Dual-Batch Overlap mechanism to VLLM](../sources/prs/vllm/PR-23693.md), [[Feature] Support Decode Context Parallel (DCP) for MLA](../sources/prs/vllm/PR-23734.md), [[Kernel] cuda kernels for upcoming decode context parallel feature](../sources/prs/vllm/PR-23791.md), [[Model] Add LongCat-Flash ](../sources/prs/vllm/PR-23991.md), [[Kernel] Support decode context parallelism on Blackwell with CUTLASS MLA](../sources/prs/vllm/PR-24385.md), [[DCP] Support Decode Context Parallel (DCP) for GQA with FlashAttention](../sources/prs/vllm/PR-24864.md), [[Attention][DCP] Support DCP with query length > 1 (MTP) with FA3](../sources/prs/vllm/PR-25049.md), [[Performance] Split FlashAttn attention and cache update](../sources/prs/vllm/PR-25954.md), [[Spec Decode] Enable efficient speculative decoding with FlashInfer-MLA](../sources/prs/vllm/PR-25984.md), [[ROCm][MLA] enable fp8 MLA decode on ROCm](../sources/prs/vllm/PR-28032.md), [[Attention] Make `split_decodes_and_prefills(..., require_uniform=True)` support padding](../sources/prs/vllm/PR-29644.md), [Add Mistral Large 3 and Ministral 3](../sources/prs/vllm/PR-29757.md), [[SpecDecode] Simplified alternative padded-speculation acceptance rate fix](../sources/prs/vllm/PR-29845.md), [[BugFix] Fix DBO assert `assert B_block_table == B_q`](../sources/prs/vllm/PR-29933.md), [[ROCm] add fallback for aiter fp8 decode mla](../sources/prs/vllm/PR-30005.md), [[ROCm][Bugfix] Add MLACommonMetadata to allowed attention types for speculative decoding](../sources/prs/vllm/PR-30430.md), [[Feature] Add SM103 (Blackwell Ultra) Support to vLLM](../sources/prs/vllm/PR-30484.md), [[Bugfix][Hardware][AMD] Fix last_page_len calculation in AITER MLA decode](../sources/prs/vllm/PR-31282.md), [[Bugfix][ROCm]Fix Qwen3-Next-80B-A3B-Thinking inference and optimize non-standard block size (544) support under rocm_atten](../sources/prs/vllm/PR-31380.md), [[1/N][Attention] Restructure attention: move files](../sources/prs/vllm/PR-31916.md), [[4/N][Attention] Move MLA common to model_executor](../sources/prs/vllm/PR-32060.md), [[Spec Decode] Unified Parallel Drafting](../sources/prs/vllm/PR-32887.md), [Triton MLA perf fixes](../sources/prs/vllm/PR-33529.md), [fix(cpu): fix mla_decode compilation on x86 without AVX512](../sources/prs/vllm/PR-34052.md), [[BugFix] Add support for MTP num_speculative_tokens > 1 with sparse MLA](../sources/prs/vllm/PR-34552.md), [[Kernel] Add FP8 KV cache support to Triton MLA decode attention](../sources/prs/vllm/PR-34597.md), [[Update] Use FlashInfer fast_decode_plan directly instead of replication](../sources/prs/vllm/PR-34687.md), [[Bugfix] Fix GDN attention crash with mixed decode/spec-decode batches](../sources/prs/vllm/PR-34871.md), [[Model Runner V2] Support attention group](../sources/prs/vllm/PR-35036.md), [Kimi k2.5 MLA based eagle3](../sources/prs/vllm/PR-36361.md), [[ROCm][Perf] Allow MTP lens > 1 in Sparse MLA](../sources/prs/vllm/PR-36681.md), [[ROCm] Attention selector reordering](../sources/prs/vllm/PR-36702.md), [[ROCm] Fix KV copy methods and auto-select attention backend for ROCm](../sources/prs/vllm/PR-36845.md), [[Feat][Spec Decode] DFlash](../sources/prs/vllm/PR-36847.md), [[Benchmark] Improvements to attention benchmark script](../sources/prs/vllm/PR-37115.md), [[Attention] Support distinguishing between short extends and decodes](../sources/prs/vllm/PR-37303.md), [[Model Runner V2] fix draft attention metadata generation](../sources/prs/vllm/PR-37364.md), [[Perf][Kernel] Persistent TopK scheduler: unified CUDAGraph-safe kernel with dynamic per-row dispatch - DeepSeek-V3.2 DSA decode](../sources/prs/vllm/PR-37421.md), [[Performance] Remove unnecessary zero-fill of MLA decode output tensor in Aiter backend](../sources/prs/vllm/PR-37539.md), [[Feature] Support per-draft-model MoE backend via `--speculative-config`](../sources/prs/vllm/PR-37880.md), [[ROCm][perf] fix Aiter sparse MLA with MTP>1](../sources/prs/vllm/PR-37887.md), [[Model Runner V2] Rebuild attention metadata before eagle decode full…](../sources/prs/vllm/PR-38311.md), [[Attention Backend] TurboQuant: 2-bit KV cache compression with 4x capacity](../sources/prs/vllm/PR-38479.md), [[XPU] Fix spec-decode UTs under tests/v1/spec_decode](../sources/prs/vllm/PR-38491.md), [[Refactor] Improve indexer decode path metadata preparation](../sources/prs/vllm/PR-38865.md), [[Bugfix][MoE] Fix 6-8% decode regression: prefer multi-stream shared expert overlap](../sources/prs/vllm/PR-38990.md), [[MLA] Optimize mla indexer prepare uniform decode for MTP > 1](../sources/prs/vllm/PR-39458.md), [[Attention][TurboQuant] Share dequant buffers, eliminate float16_copy](../sources/prs/vllm/PR-40941.md), [[Spec Decode] Add Gemma4 MTP speculative decoding support](../sources/prs/vllm/PR-41745.md), [[MLA Attention Backend] Add TOKENSPEED_MLA backend for DSR1/Kimi K25 prefill + decode on Blackwell](../sources/prs/vllm/PR-41778.md), [[Bugfix] mamba: run single-token extends as decodes](../sources/prs/vllm/PR-42430.md), [[Perf][MLA] Enable FULL cudagraph capture for TRITON_MLA decode](../sources/prs/vllm/PR-42885.md), [FlashMLA — Multi-head Latent Attention](../wiki/kernels/flashmla.md), [Gated Delta Net — Linear Attention](../wiki/kernels/gated-delta-net.md), [Sparse MLA (DeepSeek V3.2)](../wiki/kernels/sparse-mla.md) | -| `flash-attention` | [FlashAttention-4 Blog](../sources/blogs/flash-attention-4.md), [DeepSeek-V3.2-Exp in vLLM: Fine-Grained Sparse Attention in Action](../sources/blogs/vllm-deepseek-v3-sparse-attention.md), [FlashAttention-4: Hardware-Friendly Attention on Blackwell](../sources/docs/flash-attention-4.md), [[None][feat] Remove non flash attetnion style fmha_v2 kernel for hopper](../sources/prs/TensorRT-LLM/PR-11381.md), [[TRTLLM-11092][feat] add support for visual gen FA4 attention backend](../sources/prs/TensorRT-LLM/PR-11697.md), [[TRTLLM-11119][feat] Blackwell SageAttention, Integrate into AttentionOp API](../sources/prs/TensorRT-LLM/PR-11718.md), [[TRTLLM-11540][feat] Add EAGLE3 dynamic tree speculative decoding support](../sources/prs/TensorRT-LLM/PR-12062.md), [[None][feat] Support sparse mqa/gqa attention](../sources/prs/TensorRT-LLM/PR-12470.md), [[None][feat] Trtllm-gen FMHA JIT support](../sources/prs/TensorRT-LLM/PR-12612.md), [[None][feat] Optimize mamba SSD prefill and extend flashinfer dispatch](../sources/prs/TensorRT-LLM/PR-12731.md), [[TRTLLM-11485][feat] Feature rework: Add SageAttention refreshed kernels (attentionOp only)](../sources/prs/TensorRT-LLM/PR-12937.md), [[None][perf] Drop cubin and Eliminate ~6s FMHA JIT recompile in eager generation by aligning kernel selection with CUDA graph warmup](../sources/prs/TensorRT-LLM/PR-13505.md), [[None][feat] Add DeepSeekV4 attention kernels](../sources/prs/TensorRT-LLM/PR-13652.md), [[None][feat] Update FMHA cubins for head_dim 80](../sources/prs/TensorRT-LLM/PR-13808.md), [[TRTLLM-35237][feat] Add cute dsl FP4 paged MQA logits decode kernel](../sources/prs/TensorRT-LLM/PR-13929.md), [[None][feat] Update the logic of FMHA JIT path](../sources/prs/TensorRT-LLM/PR-14291.md), [[None][feat] GPT-OSS Sm120/Sm121 Support](../sources/prs/TensorRT-LLM/PR-7937.md), [Correct divmod order in example 77 (blackwell fmha)](../sources/prs/cutlass/PR-2291.md), [Handle get_masked_trip_count for small length in fmha example](../sources/prs/cutlass/PR-2292.md), [[ex77] fix mla split; add fwd lse; add bwd varlen](../sources/prs/cutlass/PR-2366.md), [Example 77 add blackwell flash-attention bwd for MLA shape](../sources/prs/cutlass/PR-2466.md), [fix gqa issue for blackwell fmha.py](../sources/prs/cutlass/PR-2599.md), [Fp8 kernel with "in-kernel" transpose of V in producer](../sources/prs/flash-attention/PR-1100.md), [FA3 FP8 qkv descales + restore max offset for h128 causal + added sync for producer WG](../sources/prs/flash-attention/PR-1173.md), [FA3 kvcache + split kv + gqa parallelization](../sources/prs/flash-attention/PR-1236.md), [[Cute,Fwd,Sm100] fp8 e4m3 and e5m2 support](../sources/prs/flash-attention/PR-2109.md), [Feat([FA4][CUTE DSL]) Add head_dim=256 support (forward + backward)](../sources/prs/flash-attention/PR-2412.md), [[hd256] Improve forward kernel with exp2 FMA emulation (3% to 9% performance gain)](../sources/prs/flash-attention/PR-2488.md), [[hd256] Add TMA paged KV support to SM100 2CTA forward kernel](../sources/prs/flash-attention/PR-2489.md), [[FA4][hd256] Backward TMA bulk-store epilogue + LSE/dpsum coalesce](../sources/prs/flash-attention/PR-2497.md), [[nvidia] initial support for blackwell kernels](../sources/prs/flashinfer/PR-1039.md), [[nvidia] Add Blackwell FMHA decode kernel from TRT-LLM](../sources/prs/flashinfer/PR-1051.md), [bugfix: adding lse output to blackwell fmha kernels](../sources/prs/flashinfer/PR-1071.md), [bugfix: follow user-specified sm_scale for blackwell cutlass fmha](../sources/prs/flashinfer/PR-1072.md), [bugfix: host-precomuted plan function for blackwell fmha](../sources/prs/flashinfer/PR-1106.md), [hotfix: fix the blackwell fmha stream](../sources/prs/flashinfer/PR-1116.md), [Add more logging to TRTLLM-GEN debug trace (NFC)](../sources/prs/flashinfer/PR-1158.md), [bugfix: fix invalid blackwell fmha unittests](../sources/prs/flashinfer/PR-1181.md), [update trtllm-gen decode attention kernel launcher](../sources/prs/flashinfer/PR-1189.md), [bugfix: fix blackwell fmha hanging issue for empty kv_len](../sources/prs/flashinfer/PR-1198.md), [feat: add trtllm-gen mla cubin](../sources/prs/flashinfer/PR-1222.md), [add trtllm-gen context attention](../sources/prs/flashinfer/PR-1239.md), [Add trtllm-gen attention mha kernel with FP8 Q/K/V and FP8 output](../sources/prs/flashinfer/PR-1242.md), [feat: enable trtllm-gen mla MTP](../sources/prs/flashinfer/PR-1258.md), [fix: update trtllm-gen fmha benchmark](../sources/prs/flashinfer/PR-1280.md), [fix multiCtasKvScratchPtr misalignment issue (new one)](../sources/prs/flashinfer/PR-1286.md), [refactor: refactor trtllm-gen attention kernel integration code](../sources/prs/flashinfer/PR-1289.md), [refactor: Improved metainfo for trtllm-gen fmha](../sources/prs/flashinfer/PR-1292.md), [Fix the bug of the kernel-selection heuristic in trtllm-gen](../sources/prs/flashinfer/PR-1307.md), [feat: support output nvfp4 in trtllm-gen function call.](../sources/prs/flashinfer/PR-1318.md), [fix: fix trtllm-gen mla error on new interface](../sources/prs/flashinfer/PR-1348.md), [support trtllm-gen prefill fp4 output](../sources/prs/flashinfer/PR-1360.md), [Support scale factor start index for fp4 mha prefill/decode](../sources/prs/flashinfer/PR-1363.md), [GPT-OSS Support: Add Blackwell MoE mxfp4 implementation from TRTLLM and Attention Sink](../sources/prs/flashinfer/PR-1389.md), [benchmark: trtllm-gen mha with sink, add benchmark args](../sources/prs/flashinfer/PR-1415.md), [fix: remote redundant zero_init from trtllm-gen attn](../sources/prs/flashinfer/PR-1444.md), [Remove getEnvEnablePDL in favor of enable_pdl parameter](../sources/prs/flashinfer/PR-1446.md), [feat: add pdl for trtllm-gen attn](../sources/prs/flashinfer/PR-1484.md), [feat: Support fp8 qkv, fp16/bf16 out MHA for trtllm-gen.](../sources/prs/flashinfer/PR-1490.md), [backend: Refactor trtllm-gen fmha metainfo loading](../sources/prs/flashinfer/PR-1518.md), [feat: Integrate TRTLLM varlen kernel for deepseek R1 prefill ](../sources/prs/flashinfer/PR-1537.md), [Backend: downgrade trtllm-gen kernel to cuda-12](../sources/prs/flashinfer/PR-1567.md), [fix: semaphoress must be at the fixed range in workspace buffer on trtllm_gen attention](../sources/prs/flashinfer/PR-1584.md), [feat: initial support for SM103, SM110, SM120, SM121](../sources/prs/flashinfer/PR-1608.md), [bugfix: trtllm-gen fmha sm101 and sm100 compatibility](../sources/prs/flashinfer/PR-1631.md), [fix: zero-init workspace buffer for trtllm-gen fmha](../sources/prs/flashinfer/PR-1643.md), [perf: improve attention of tcgen05 flash-attention](../sources/prs/flashinfer/PR-1681.md), [perf: Port the separate reduce kernel mode from trtllm.](../sources/prs/flashinfer/PR-1685.md), [Add head_dim=64 for tcgen05 tcgen05 flash-attention implementation](../sources/prs/flashinfer/PR-1850.md), [Tune kernel compilation parameters for https://github.com/flashinfer-ai/flashinfer/pull/1850 ](../sources/prs/flashinfer/PR-1878.md), [fix: Fix trtllm-gen prefill IMA when batch_size==1](../sources/prs/flashinfer/PR-1912.md), [feat: add xqa backend and completes NHD/HND coverage for trtllm-gen/xqa backend](../sources/prs/flashinfer/PR-2001.md), [Rebase FP8 SM100 Cutlass FMHA Attention to main (original PR#1238)](../sources/prs/flashinfer/PR-2047.md), [Fix: several bugs/issues with trtllm-gen attention kernels. ](../sources/prs/flashinfer/PR-2062.md), [[API change] Allow using torch.Tensor for scales for trtllm-gen attention](../sources/prs/flashinfer/PR-2084.md), [feat: support variable sequence length in decode kernel of trtllm-gen attention](../sources/prs/flashinfer/PR-2125.md), [fix(trtllm): reset negative strideBatch to 0 for ragged KV layout to …](../sources/prs/flashinfer/PR-2134.md), [fix: some bugs of headDim 256 trtllm-gen fmha kernels. ](../sources/prs/flashinfer/PR-2137.md), [feat: add trtllm-gen per-tensor sparseMla kernels.](../sources/prs/flashinfer/PR-2138.md), [feat: TRTLLM FMHAv2 backend for ctx attention](../sources/prs/flashinfer/PR-2142.md), [fix: compile flags for trtllm fmha_v2 ](../sources/prs/flashinfer/PR-2175.md), [feat: support non-contiguous query for trtllm-gen attention backend](../sources/prs/flashinfer/PR-2254.md), [[TRTLLM-Gen Fmha] add optimized trtllm-gen decode kernels for high throughput + speculative decoding](../sources/prs/flashinfer/PR-2265.md), [fix: ensure each CTA processes full numHeadsQPerKv for trtllm decode kernel](../sources/prs/flashinfer/PR-2380.md), [feat: Add TRTLLM fmha_v2 library for SM90 attention with Skip-Softmax ](../sources/prs/flashinfer/PR-2446.md), [feat: Add TRTLLM-Gen Skip-Softmax kernels for prefill and decode](../sources/prs/flashinfer/PR-2477.md), [feat: Enable TRTLLM-Gen Skip-Softmax attention for MLA](../sources/prs/flashinfer/PR-2547.md), [fix: allow fmha_v2_prefill_deepseek on SM121 (DGX Spark)](../sources/prs/flashinfer/PR-2559.md), [fix: guard CUTLASS FMHA against SM12x and fix fmha_v2 SM121a check](../sources/prs/flashinfer/PR-2560.md), [feat: add is_sm12x_supported() helper for SM12x family detection](../sources/prs/flashinfer/PR-2574.md), [fix: add SM121 support to SM120 version guards](../sources/prs/flashinfer/PR-2631.md), [feat: add support for more MLA head dimensions](../sources/prs/flashinfer/PR-2677.md), [Add NVFP4 KV cache quantization support for SM100](../sources/prs/flashinfer/PR-2702.md), [feat: Add DiT-oriented kernels where Qk (Bmm1) type can be reinterpreted into Int8 or BFloat16](../sources/prs/flashinfer/PR-2711.md), [feat: Expose TRT-LLM FMHA style paged KV Cache and page table layout](../sources/prs/flashinfer/PR-2770.md), [tests: skip sliding window + fp8 to prevent hang in fmha_v2 unit tests](../sources/prs/flashinfer/PR-2781.md), [[fmha-v2] Support HND and NHD paged KV cache layouts with conditional stride handling](../sources/prs/flashinfer/PR-2799.md), [[CuTe DSL] Add modular FMHA prefill and MLA decode attention kernels](../sources/prs/flashinfer/PR-2805.md), [[Fmha] Sparse MLA decode kernel selection heuristics](../sources/prs/flashinfer/PR-2836.md), [[Perf] Add FMHAv2 to flashinfer_benchmark.py and eliminate unnecessary H2D](../sources/prs/flashinfer/PR-2841.md), [read real strides for kv and block scale](../sources/prs/flashinfer/PR-2844.md), [Only swizzle on v block scale; rename kv_block_scales to kv_cache_sf](../sources/prs/flashinfer/PR-2954.md), [[Fmha] support nvfp4 output keepsMmaAb generation kernels](../sources/prs/flashinfer/PR-2988.md), [Support lse in trtllm paged attn kernels](../sources/prs/flashinfer/PR-3058.md), [cute-dsl fmha prefill (cubin integration): remove front-padding, add attention_sink, and pdl support](../sources/prs/flashinfer/PR-3181.md), [Add dynamic tokens-per-page TRTLLM-GEN GQA kernels](../sources/prs/flashinfer/PR-3259.md), [fix(fmha_v2): fix FP8 V-scratch pipeline and varlen scheduler on SM90](../sources/prs/flashinfer/PR-3276.md), [Update trtllm FMHA cubins](../sources/prs/flashinfer/PR-3317.md), [[flex_attention] adds support for low precision K/V inputs in compiled mode with GPU](../sources/prs/pytorch/PR-170486.md), [[RELEASE 2.11] Release only changes](../sources/prs/pytorch/PR-175091.md), [[release 2.12] Apply Release only changes to 2.12 branch](../sources/prs/pytorch/PR-180470.md), [[NVIDIA] FA3/FA4 Fix ](../sources/prs/sglang/PR-11606.md), [Tiny use trtllm_mha as default when possible](../sources/prs/sglang/PR-14291.md), [[NVIDIA] upstream FA4](../sources/prs/sglang/PR-15182.md), [Move fa4 from sgl-kernel to jit kernel](../sources/prs/sglang/PR-17353.md), [feat: add FA4 SM90 paged KV decode support & update attention docs](../sources/prs/sglang/PR-18442.md), [Add SGLang CUDA crash API logging inspired by FlashInfer](../sources/prs/sglang/PR-20910.md), [Refactor JIT kernel CI to use run_suite.py registration system](../sources/prs/sglang/PR-21239.md), [[misc] clean up kernel API](../sources/prs/sglang/PR-21325.md), [Lazy import flash_attention_v4 to avoid loading flash_attn.cute at startup](../sources/prs/sglang/PR-22306.md), [[perf][sgl-kernel] extend cutlass_mla_decode to support num_head < 128](../sources/prs/sglang/PR-6929.md), [[Feature] Support Flashinfer fmha on Blackwell](../sources/prs/sglang/PR-6930.md), [[Feature] Support `cp.reduce.async.bulk.tensor`](../sources/prs/tilelang/PR-1667.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [SM100 Cutlass MLA decode with unrestricted num_heads (< 128) for DeepSeek TP](../sources/prs/vllm/PR-20769.md), [[SM100] Resubmit FMHA FP8 prefill for MLA](../sources/prs/vllm/PR-31195.md), [[FA4] Update flash-attention to latest upstream FA4](../sources/prs/vllm/PR-38690.md), [FlashAttention-4](../wiki/kernels/flash-attention-4.md), [FlashAttention SM100 MLA TopK Sparse Forward](../wiki/kernels/flash-attention-sm100-mla-topk.md) | -| `fused-kernel` | [FlashInfer MLSys 2026 - Track A: Fused MoE FP8](../sources/contests/flashinfer-mlsys26/track-a-fused-moe.md), [GPU Mode NVFP4 Hackathon - Problem 3: Gated Dual GEMM](../sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md), [[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes](../sources/prs/DeepGEMM/PR-304.md), [perf: Enable SplitK and fix tile-scheduling for moe fp4 fused moe](../sources/prs/flashinfer/PR-1548.md), [[cute_dsl] add gemm + all reduce (two_shot)](../sources/prs/flashinfer/PR-1695.md), [A Blackwell-optimized version of selective_state_update (decode)](../sources/prs/flashinfer/PR-2387.md), [[Kernel][tcgen05] nvfp4 fused tcgen05 moe](../sources/prs/vllm/PR-23696.md), [Fused MoE — FP8 Block-Scale Routing + Dual GEMM](../wiki/kernels/fused-moe.md), [Gated Dual GEMM (Gate-Up + SwiGLU Fusion)](../wiki/kernels/gated-dual-gemm.md) | -| `gated-delta-net` | [Gated Delta Networks](../sources/blogs/gated-delta-net.md), [Qwen3-Next: Hybrid GDN+MoE Architecture on NVIDIA Blackwell](../sources/blogs/qwen3-next-architecture.md), [FlashInfer MLSys 2026 - Track C: Gated Delta Net](../sources/contests/flashinfer-mlsys26/track-c-gated-delta-net.md), [Tiled Flash Linear Attention (TFLA)](../sources/docs/tfla.md), [[#11694][feat] AutoDeploy: Improve the piecewise CG memory usage](../sources/prs/TensorRT-LLM/PR-11993.md), [[None][fix] Add SPDX Apache-2.0 headers to auto_deploy test files](../sources/prs/TensorRT-LLM/PR-14193.md), [[None][fix] Update the OSS headers in derived FLA ops and AD modeling code](../sources/prs/TensorRT-LLM/PR-14281.md), [feat: add pool+indices support to gated_delta_rule_decode_pretranspose (bf16 path) ](../sources/prs/flashinfer/PR-2619.md), [feat(gdn): state checkpointing in chunk_gated_delta_rule](../sources/prs/flashinfer/PR-2908.md), [[feat] Add blackwell GDN prefill kernel](../sources/prs/flashinfer/PR-3001.md), [[XPU] Enable qwen3.5 on XPU](../sources/prs/sglang/PR-21668.md), [Gated Delta Net — Linear Attention](../wiki/kernels/gated-delta-net.md) | -| `gated-dual-gemm` | [GPU Mode NVFP4 Hackathon - Problem 3: Gated Dual GEMM](../sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md), [Fused MoE — FP8 Block-Scale Routing + Dual GEMM](../wiki/kernels/fused-moe.md), [Gated Dual GEMM (Gate-Up + SwiGLU Fusion)](../wiki/kernels/gated-dual-gemm.md) | -| `gemm` | [Microbenchmarking NVIDIA's Blackwell Architecture](../sources/blogs/blackwell-microbenchmarking.md), [Colfax Article Source Kernels](../sources/blogs/colfax-article-source-kernels.md), [Colfax CUTLASS Kernels](../sources/blogs/colfax-cutlass-kernels.md), [DeepGEMM — FP8 GEMM Library](../sources/blogs/deepgemm.md), [Anatomy of a Reward Hack](../sources/blogs/gpu-mode-reward-hack.md), [Writing High-Performance Matrix Multiplication Kernels for Blackwell with JAX Pallas](../sources/blogs/jax-pallas-blackwell-matmul.md), [K-Search: LLM Kernel Generation via Co-Evolving Intrinsic World Model](../sources/blogs/k-search-kernel-generation.md), [Modular: Matrix Multiplication on Blackwell](../sources/blogs/modular-blackwell-matmul.md), [NVIDIA Developer Code Samples](../sources/blogs/nvidia-code-samples.md), [simveit effective_transpose](../sources/blogs/simveit-effective-transpose.md), [simveit load_and_store](../sources/blogs/simveit-load-and-store.md), [TFLOPS Gap: Why FP4 MoE Kernel Engineering Matters on Blackwell](../sources/blogs/tflops-gap-fp4-moe.md), [Tilus: A Tile-Level GPGPU Programming Language for Low-Precision Computation](../sources/blogs/tilus-nvidia.md), [FlashInfer MLSys 2026 - Track A: Fused MoE FP8](../sources/contests/flashinfer-mlsys26/track-a-fused-moe.md), [GPU Mode NVFP4 Hackathon - Problem 2: NVFP4 GEMM](../sources/contests/gpu-mode-nvfp4/problem-2-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 3: Gated Dual GEMM](../sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 4: Grouped GEMM](../sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md), [cuTile Python DSL Reference](../sources/docs/cutile-python-dsl.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [CUTLASS Cluster Launch Control (CLC) Documentation](../sources/docs/cutlass-clc-documentation.md), [Fix performance issue of m-grouped contiguous GEMMs.](../sources/prs/DeepGEMM/PR-168.md), [Fix multicast bug and optimize masked GEMM](../sources/prs/DeepGEMM/PR-193.md), [fix: use SM90ArchSpec instead of SM100ArchSpec in sm90_bf16_k_grouped_gemm](../sources/prs/DeepGEMM/PR-270.md), [[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes](../sources/prs/DeepGEMM/PR-304.md), [Sync nv_dev with upstream #316 (Mega MoE optimizations & benchmarks)](../sources/prs/DeepGEMM/PR-328.md), [Performance: reducing the percentage of FFMA interleaving yields a sight performance gain, roughly 0.5%](../sources/prs/DeepGEMM/PR-42.md), [ Solving bank conflict via padding and TMA 3D store](../sources/prs/DeepGEMM/PR-78.md), [Use 1D TMA store instead of 3D](../sources/prs/DeepGEMM/PR-83.md), [Use swizzling instead of padding](../sources/prs/DeepGEMM/PR-86.md), [Support TMA multicast on B with m_grouped_gemm_contiguous.](../sources/prs/DeepGEMM/PR-88.md), [[None][perf] Add more optimization options for MOE CuteDSL finalized kernel](../sources/prs/TensorRT-LLM/PR-10042.md), [[TRTLLM-9992][perf] Enable PDL for CuteDSL kernels and overlap MoeOutputMemset](../sources/prs/TensorRT-LLM/PR-10043.md), [[None][feat] CuteDSL MOE FC1 Enhancement](../sources/prs/TensorRT-LLM/PR-10088.md), [[TRTLLM-9457][feat] Add cute dsl fp8 gemm for Blackwell](../sources/prs/TensorRT-LLM/PR-10130.md), [[None][feat] sm100 weight-only kernel](../sources/prs/TensorRT-LLM/PR-10190.md), [[TRTLLM-9831][perf] Enable 2CTA with autotune for CuteDSL MoE and Grouped GEMM optimizations](../sources/prs/TensorRT-LLM/PR-10201.md), [[TRTLLM-9798][feat] Change to use new DeepGEMM MQA sm100 kernel for MTP-3](../sources/prs/TensorRT-LLM/PR-10226.md), [[TRTLLM-9661][chore] Further reduce tuning time for cuteDSL nvFP4 dense gemm.](../sources/prs/TensorRT-LLM/PR-10339.md), [[None] [feat] Add test script and raster M for gather fc1 kernel](../sources/prs/TensorRT-LLM/PR-10429.md), [[TRTLLM-10276][feat] Integrate cutedsl argmax kernel](../sources/prs/TensorRT-LLM/PR-10476.md), [[None] [feat] Add densegemm backend for MoE](../sources/prs/TensorRT-LLM/PR-10479.md), [[TRTLLM-9831][perf] Use TMA.RED to improve effective memory bandwidth](../sources/prs/TensorRT-LLM/PR-10987.md), [[https://nvbugs/5799917][fix] Recover from CUTLASS MoE doActivation perf regression for MXFP4/NVFP4 dtype](../sources/prs/TensorRT-LLM/PR-11165.md), [[https://nvbugs/5854860][fix] Fix cutedsl argmax on sm120](../sources/prs/TensorRT-LLM/PR-11181.md), [[None][fix] Fix int4 awq for sm120/121](../sources/prs/TensorRT-LLM/PR-11561.md), [[TRTLLM-10004][feat] Enable GEMM -> AR with GEMM output in registered buffers](../sources/prs/TensorRT-LLM/PR-11589.md), [[https://nvbugs/5799917][fix] Recover from CUTLASS MoE doActivation perf regression for MXFP4/NVFP4 dtype](../sources/prs/TensorRT-LLM/PR-11733.md), [[TRTLLM-10990][feat] Fuse SwiGLU and quant into shared expert](../sources/prs/TensorRT-LLM/PR-11897.md), [[#11694][feat] AutoDeploy: Improve the piecewise CG memory usage](../sources/prs/TensorRT-LLM/PR-11993.md), [[TRTLLM-11285][feat] Fuse indexer wk + weights_proj into single GEMM in TF32 for DS-V3.2](../sources/prs/TensorRT-LLM/PR-12055.md), [[TRTLLM-11289][feat] Integrate CuteDSL's bf16 dense GEMMs](../sources/prs/TensorRT-LLM/PR-12074.md), [[None][feat] CuteDSL MOE: Add raster along M/N support for blockscaled contiguous backbone kernel](../sources/prs/TensorRT-LLM/PR-12079.md), [[None][feat] Add DWDP (Distributed Weight Data Parallelism) support for MoE inference](../sources/prs/TensorRT-LLM/PR-12136.md), [[None][feat] Minimax RMS norm optimization](../sources/prs/TensorRT-LLM/PR-12163.md), [[None][feat] Support update weight for nvfp4](../sources/prs/TensorRT-LLM/PR-12320.md), [[TRTLLM-10407][perf] Add cute dsl single pass multi cta cluster topk](../sources/prs/TensorRT-LLM/PR-12354.md), [[None][feat] Add Mamba2 MTP SSM cache CUDA kernel for tree-based speculative decoding](../sources/prs/TensorRT-LLM/PR-12537.md), [[TRTLLM-11585][feat] Add CUTEDSL moe backend for nemotron-h](../sources/prs/TensorRT-LLM/PR-12884.md), [[#12784][feat] AutoDeploy: Optimize DeepSeek-R1 model performance](../sources/prs/TensorRT-LLM/PR-12946.md), [[None][fix] Propagate init_load_balancer to DeepGemmFusedMoE in create_moe_backend](../sources/prs/TensorRT-LLM/PR-13207.md), [[None][feat] Integrate FP4 indexer for DSA on Blackwell](../sources/prs/TensorRT-LLM/PR-13340.md), [[TRTLLM-11285][perf] Force enable TF32 tensor cores for DSA indexer fused GEMM](../sources/prs/TensorRT-LLM/PR-13452.md), [[None][feat] Fuse FP8 1x128 quantize + UE8M0 scale pack on SM100](../sources/prs/TensorRT-LLM/PR-13628.md), [[#13580][fix] AutoDeploy: Support Gemma3n/4 E2B variants](../sources/prs/TensorRT-LLM/PR-13630.md), [[https://nvbugs/6108841][fix] add hidden_dim=6144 router GEMM instantiation for GLM-5](../sources/prs/TensorRT-LLM/PR-13740.md), [[None][fix] Plumb swiglu_limit through DeepGEMM and TRTLLMGen FP8 fused MoE](../sources/prs/TensorRT-LLM/PR-13767.md), [[None][feat] Indexer topk opt](../sources/prs/TensorRT-LLM/PR-13811.md), [[None][perf] FC2 DenseGEMM autotune: split-K, swap_ab, fine-grained tuning buckets](../sources/prs/TensorRT-LLM/PR-13833.md), [[None][perf] mHC fused_hc kernel optimizations + DS-V4 entry-boundary RMSNorm fold-in](../sources/prs/TensorRT-LLM/PR-13892.md), [[None][feat] Add chunked prefill support for Gemma4 (text + vision multimodal)](../sources/prs/TensorRT-LLM/PR-14134.md), [[TRTLLM-12462][fix] Fix FP8 block scaling GEMM autotuner cache growth](../sources/prs/TensorRT-LLM/PR-14165.md), [[None][fix] Add SPDX Apache-2.0 headers to auto_deploy test files](../sources/prs/TensorRT-LLM/PR-14193.md), [[None][fix] Avoid dp_size x ep_size double-count in MegaMoEDeepGemm SymmBuffer](../sources/prs/TensorRT-LLM/PR-14213.md), [[None][fix] Update the OSS headers in derived FLA ops and AD modeling code](../sources/prs/TensorRT-LLM/PR-14281.md), [feat: Add w4a8_mxfp4_fp8 quantization recipe.](../sources/prs/TensorRT-LLM/PR-4867.md), [[OMNIML-2336][feat] Add NVFP4 x FP8](../sources/prs/TensorRT-LLM/PR-6809.md), [[None][feat] GPT-OSS Sm120/Sm121 Support](../sources/prs/TensorRT-LLM/PR-7937.md), [[None][fix] Fix the performance issue of FP8 blockwise grouped GEMM when using attention DP](../sources/prs/TensorRT-LLM/PR-8501.md), [[None][feat] Enable nvfp4 cuda core for sm120](../sources/prs/TensorRT-LLM/PR-8620.md), [[None][feat] Update TRTLLM MoE cubins; reduce mxfp4 weight padding requirement; tighten TMA bound](../sources/prs/TensorRT-LLM/PR-9025.md), [[TRTLLM-8958][feat] and [TRTLLM-8960]: create ConfigurableMoE and support TRTLLMGenFusedMoE as backend](../sources/prs/TensorRT-LLM/PR-9486.md), [[TRTLLM-9685] [feat] Add gather fc1 kernel by cuteDSL](../sources/prs/TensorRT-LLM/PR-9618.md), [[None][feat] add fp4 gemm + allreduce](../sources/prs/TensorRT-LLM/PR-9729.md), [[https://nvbugs/5726962][feat] Apply fusion for W4AFP8_AWQ MoE](../sources/prs/TensorRT-LLM/PR-9838.md), [[TRTLLM-9493][feat] Add helixPostProcessNative kernel for cp_dim=2](../sources/prs/TensorRT-LLM/PR-9924.md), [Experimental Python cooperative algorithms](../sources/prs/cccl/PR-1973.md), [fix thread-reduce performance regression](../sources/prs/cccl/PR-2944.md), [Fix scan / sm90 perf regression ](../sources/prs/cccl/PR-3236.md), [Fix the vectorized loading of BlockLoad](../sources/prs/cccl/PR-3517.md), [Add b200 tunings for scan.exclusive.sum](../sources/prs/cccl/PR-3559.md), [Fix SM100 histogram tunings](../sources/prs/cccl/PR-3691.md), [Split Optimize Warp Reduce PR - CUB part](../sources/prs/cccl/PR-4716.md), [Add nondeterministic reduce that uses atomics](../sources/prs/cccl/PR-4961.md), [CUB - Add internal integer utils and tests (Split `WarpReduce` PR)](../sources/prs/cccl/PR-5314.md), [Combine `block_reduce_warp_reduction_nondeterministic.cuh` specialization with original deterministic one ](../sources/prs/cccl/PR-5408.md), [Add dynamic CUB dispatch for segmented_sort](../sources/prs/cccl/PR-6069.md), [[CUB] Use `BlockLoadToShared` in `DeviceMerge`](../sources/prs/cccl/PR-6077.md), [Fix debug section around line 390 of dispatch_topk](../sources/prs/cccl/PR-6152.md), [Split fixed-size segmented reduce dispatch header](../sources/prs/cccl/PR-6597.md), [Integrate decoupled lookahead warpspeed scan](../sources/prs/cccl/PR-6811.md), [Use integer promotion for `warp_reduce`](../sources/prs/cccl/PR-6819.md), [Implement new tuning API arch dispatching](../sources/prs/cccl/PR-7093.md), [Two-phase reduction for fixed size segmented reduction for very large segment sizes](../sources/prs/cccl/PR-7114.md), [Implement the new tuning API for deterministic (rfa) reduce dispatch](../sources/prs/cccl/PR-7346.md), [Radix-selection based `BlockTopK` specialization](../sources/prs/cccl/PR-7384.md), [Implement the new tuning API for `DeviceRleDispatch`](../sources/prs/cccl/PR-7669.md), [Optimize non fixed size segmented reduce for small segments using max_segment_size](../sources/prs/cccl/PR-7718.md), [Add env SegmentedReduce (non fixed-size overloads)](../sources/prs/cccl/PR-7795.md), [Forward policy hub from `dispatch_streaming_arg_reduce_t` to `reduce::dispatch`](../sources/prs/cccl/PR-7805.md), [Implement the new tuning API for `detail::reduce::dispatch_streaming_arg_reduce_t`](../sources/prs/cccl/PR-7807.md), [Use the new tuning API internally for `detail::transform::dispatch`](../sources/prs/cccl/PR-7810.md), [[Backport branch/3.3.x] Forward policy hub from `dispatch_streaming_arg_reduce_t` to `reduce::dispatch`](../sources/prs/cccl/PR-7814.md), [Optimized Device-to-Device Tensor Copy (`cudax`)](../sources/prs/cccl/PR-7823.md), [Implement the new tuning API for `DispatchSegmentedRadixSort`](../sources/prs/cccl/PR-7844.md), [Implement the new tuning API for `DispatchSegmentedSort`](../sources/prs/cccl/PR-7874.md), [Implement the new tuning API for `DispatchTopK`](../sources/prs/cccl/PR-7928.md), [[cuda.compute]: Fix faulty pointer arithmetic calculation in CUB dispatch](../sources/prs/cccl/PR-7940.md), [Reduce usage of `cub::DispatchReduce`](../sources/prs/cccl/PR-7944.md), [Use the new tuning API for `detail::radix_sort::dispatch`](../sources/prs/cccl/PR-7949.md), [Adds support for non-fundamental types via decomposer to `DeviceTopK` ](../sources/prs/cccl/PR-8040.md), [Optimized Device-to-Device Tensor Copy (cudax) - Transpose Case](../sources/prs/cccl/PR-8125.md), [Avoid passing uninitialized values to scan_op](../sources/prs/cccl/PR-8184.md), [[STF] Move unstable_unique from STF to generic cudax utility](../sources/prs/cccl/PR-8190.md), [Expose `max_segment_size` guarantee in cuda.compute](../sources/prs/cccl/PR-8284.md), [Port `thrust::min|max_element` to CUB](../sources/prs/cccl/PR-8291.md), [Implement the new tuning API for `DispatchSelectIf`](../sources/prs/cccl/PR-8311.md), [simplify dispatch segmented reduce to use latest dispatch and new tunings API](../sources/prs/cccl/PR-8332.md), [Apply some random warpspeed tunings](../sources/prs/cccl/PR-8352.md), [[cub]: implement utilities for policy selection](../sources/prs/cccl/PR-8355.md), [Replace `detail::merge::dispatch` by CUB's public API](../sources/prs/cccl/PR-8381.md), [[CUB] Replace `Shuffle(Up|Down|Index)` with cuda::device::warp_shuffle - RadixSort only](../sources/prs/cccl/PR-8395.md), [[thrust] Single-pass `is_partitioned` via adjacent zip_iterator](../sources/prs/cccl/PR-8427.md), [Replace `detail::merge_sort::dispatch` by CUB's public API](../sources/prs/cccl/PR-8473.md), [Replace `detail::scan::dispatch` by CUB's public API](../sources/prs/cccl/PR-8495.md), [Implement the new tuning API for `detail::batched_topk::dispatch_batched_topk`](../sources/prs/cccl/PR-8538.md), [Replace `detail::for_each::dispatch` by CUB's public API](../sources/prs/cccl/PR-8565.md), [Replace `detail::segmented_reduce::dispatch` by the public API](../sources/prs/cccl/PR-8695.md), [Use the new tuning API internally for `detail::topk::dispatch`](../sources/prs/cccl/PR-8742.md), [Use the new tuning API internally for `detail::reduce_by_key::dispatch`](../sources/prs/cccl/PR-8756.md), [Use the new tuning API internally for `detail::reduce[_nd]::dispatch[_nd]`](../sources/prs/cccl/PR-8826.md), [Fix Warpspeed scan shifted output store](../sources/prs/cccl/PR-8839.md), [[cub] Simplify arch dispatch](../sources/prs/cccl/PR-8861.md), [Use the new tuning API internally for `detail::select::dispatch` and `DeviceSelect`](../sources/prs/cccl/PR-8880.md), [[STF] Add per-handle exec_place stream resources](../sources/prs/cccl/PR-8905.md), [Use the new tuning API internally for `detail::select|three_way_partition::dispatch` and `DevicePartition`](../sources/prs/cccl/PR-8925.md), [Use the new tuning API internally for `detail::segmented_radix_sort::dispatch`](../sources/prs/cccl/PR-8927.md), [Fix segmented radix sort benchmark segment size type](../sources/prs/cccl/PR-9039.md), [[libcu++] Fix default make_shared_resource construction](../sources/prs/cccl/PR-9044.md), [Vectorize contiguous iterators in `cub::BlockLoad`/`Store`](../sources/prs/cccl/PR-9056.md), [Improve sm90 mixed dtype kernel](../sources/prs/cutlass/PR-1883.md), [Groupwise scaling along M for FP8 gemm](../sources/prs/cutlass/PR-2037.md), [Improvements for: Groupwise scaling along M for FP8 gemm](../sources/prs/cutlass/PR-2095.md), [Hopper Grouped GEMM support for FP8 Accum](../sources/prs/cutlass/PR-2123.md), [Blockwise and Groupwise GEMM for Blackwell and Improvements for Hopper](../sources/prs/cutlass/PR-2139.md), [Blockwise Improvement and Programmatic Dependent Launch](../sources/prs/cutlass/PR-2161.md), [Fix sm100 gemm wrong static constexpr that breaks compilation on Windows](../sources/prs/cutlass/PR-2167.md), [Fix SM90 beta=1 hang and stream-K launch errors](../sources/prs/cutlass/PR-2172.md), [Use cudaMemcpyAsync in gemm grouped with kRequiresPrecomputation sche…](../sources/prs/cutlass/PR-2256.md), [war to fix blackwell grouped groupwise hang](../sources/prs/cutlass/PR-2267.md), [hopper-blockwise-generalization-optimization](../sources/prs/cutlass/PR-2270.md), [support fp16 accmulator for sm89 fp8 mma](../sources/prs/cutlass/PR-2378.md), [DistGEMM bug fixes](../sources/prs/cutlass/PR-2713.md), [Support PDL for SM90 Array TMA GEMM](../sources/prs/cutlass/PR-2719.md), [Support for GEMM-K=0 for Blackwell Grouped GEMMs](../sources/prs/cutlass/PR-2746.md), [Add tutorial fp16_gemm_1](../sources/prs/cutlass/PR-2750.md), [Blockscaled Ragged Contiguous Grouped Gemm for MoEs](../sources/prs/cutlass/PR-2790.md), [[Bug Fix]Bypass launch grids for SM120 Kernel with SM90 Mainloop & SM100 TileScheduler](../sources/prs/cutlass/PR-2865.md), [[cute] Add constexpr specifier to make_tiled_copy](../sources/prs/cutlass/PR-2875.md), [new example with TMA prefetch feature targeting for DRAM latency boun…](../sources/prs/cutlass/PR-2881.md), [Fix incorrect tensor layout strides in Blackwell MMA tutorial comments](../sources/prs/cutlass/PR-2921.md), [[Cutlass gemm] Fix SM100 FP8 nosmem epilogue-fusion shape_div 'Divisibility Condition' for non-multiple-of-64 N tiles](../sources/prs/cutlass/PR-2946.md), [[Bug Fix]Set NumSplitsM to 1 when TileShapeM < 128 in sm90 fp8 blockwise scaling CollectiveMma](../sources/prs/cutlass/PR-2965.md), [[CuTeDSL] Fix: SM100 block-scale gemm overlapping accumulator](../sources/prs/cutlass/PR-2995.md), [Replace std::min with cute::min in sm120 blockwise scaling device functions](../sources/prs/cutlass/PR-3055.md), [[Hopper CuTeDSL] Add grouped GEMM kernel example](../sources/prs/cutlass/PR-3091.md), [Support for Group GEMM in CUTLASS Profiler for GeForce and Spark](../sources/prs/cutlass/PR-3092.md), [[CLI] add cutedsl fp16 gemm tutorial from 2 to 6](../sources/prs/cutlass/PR-3106.md), [Update blackwell tutorial to be compatible with 4.5-dev version](../sources/prs/cutlass/PR-3130.md), [Small Tile N BlockScaled GEMM + Grouped GEMM on SM12x](../sources/prs/cutlass/PR-3176.md), [feat: Adding varlen support to cute-dsl sm80 bwd](../sources/prs/flash-attention/PR-1934.md), [[Cute] Block sparse support Sm100](../sources/prs/flash-attention/PR-1985.md), [[Cute,Fwd,Sm100] Support `q_stage=1` for inference](../sources/prs/flash-attention/PR-1993.md), [Add blocksparse support for bwd on blackwell](../sources/prs/flash-attention/PR-2085.md), [Fix IMA in fwd on m boundary](../sources/prs/flash-attention/PR-2091.md), [Add pack-gqa fwd support for sparse impl w/ broadcasted H dim](../sources/prs/flash-attention/PR-2098.md), [[Cute,Fwd,Sm100] distributed offset calculation for paged KV](../sources/prs/flash-attention/PR-2104.md), [[NVIDIA] Enable Jetson Thor FA4](../sources/prs/flash-attention/PR-2108.md), [[CUTE][SM90]Enable pack-gqa with broadcasted maskmods](../sources/prs/flash-attention/PR-2145.md), [[Cute][Flex]Add pack-gqa divmod](../sources/prs/flash-attention/PR-2180.md), [[Cute,Fwd,Sm100] support irregular qhead / kvhead ratios](../sources/prs/flash-attention/PR-2186.md), [[Cute,Flex,Fwd] Allow vectorized score_mod definitions](../sources/prs/flash-attention/PR-2236.md), [[Bwd,Sm120] Add SM120 backward pass support](../sources/prs/flash-attention/PR-2330.md), [Fix ZeroDivisionError in num_splits_heuristic for empty Q workloads](../sources/prs/flash-attention/PR-2515.md), [feat: ragged tensor padding kernel for blackwell kernel alignment](../sources/prs/flashinfer/PR-1025.md), [fix: top_k_mask_logits hangs on -inf inputs](../sources/prs/flashinfer/PR-1050.md), [perf: accelerate blackwell grouped gemm](../sources/prs/flashinfer/PR-1086.md), [comm: refactor and initialize `flashinfer.comm` module](../sources/prs/flashinfer/PR-1089.md), [bugfix: host-precomuted plan function for blackwell fmha](../sources/prs/flashinfer/PR-1106.md), [Add CUTLASS fused moe kernels from TensorRT-LLM.](../sources/prs/flashinfer/PR-1113.md), [bugfix: Fix test and output shape of fp4 quantize](../sources/prs/flashinfer/PR-1114.md), [Fix pointer dtype bug in rope](../sources/prs/flashinfer/PR-1129.md), [fix: negative zero by type trait --> binary value](../sources/prs/flashinfer/PR-1136.md), [feat: nvshmem python bindings](../sources/prs/flashinfer/PR-1160.md), [Add DeepGEMM kernels](../sources/prs/flashinfer/PR-1209.md), [Fix test_groupwise_scaled_gemm_fp8.py](../sources/prs/flashinfer/PR-1211.md), [feat: trtllm-gen fp8 moe kernels](../sources/prs/flashinfer/PR-1212.md), [[comm] TRT-LLM's Multi-Node NVLink All-Reduce Kernel](../sources/prs/flashinfer/PR-1213.md), [Feature/sm100 low latency nvfp4 kernels](../sources/prs/flashinfer/PR-1214.md), [Fix missing hash in the cudnn cubin path](../sources/prs/flashinfer/PR-1227.md), [bugfix: support uint8_t for vec_t class template](../sources/prs/flashinfer/PR-1234.md), [Patch fp8 cubin availability](../sources/prs/flashinfer/PR-1240.md), [feat: Support MXFP8 x MXFP4 CUTLASS grouped GEMM](../sources/prs/flashinfer/PR-1241.md), [Reduce the JIT compilation time of gen_gemm_sm100_module](../sources/prs/flashinfer/PR-1251.md), [init add gemm fp8 using cudnn backend](../sources/prs/flashinfer/PR-1264.md), [Made AR output optional + esthetic changes](../sources/prs/flashinfer/PR-1265.md), [feat: add masked deepgemm support and benchmarking](../sources/prs/flashinfer/PR-1266.md), [Add shuffle matrix flag](../sources/prs/flashinfer/PR-1272.md), [hotfix: fix deepgemm artifactory hash](../sources/prs/flashinfer/PR-1278.md), [Unify groupwise fp8 GEMM test](../sources/prs/flashinfer/PR-1281.md), [add mm_fp4 use cudnn backend](../sources/prs/flashinfer/PR-1288.md), [Remove FAST_BUILD FLAG for MOE](../sources/prs/flashinfer/PR-1291.md), [Update cutlass fp4 moe kernels](../sources/prs/flashinfer/PR-1294.md), [add cutlass backend for mm_fp4](../sources/prs/flashinfer/PR-1296.md), [feat: Add weight layout option for trtllm-gen fused moe](../sources/prs/flashinfer/PR-1297.md), [[Feature] SM level profiler ](../sources/prs/flashinfer/PR-1305.md), [Refactor Fused Moe Module](../sources/prs/flashinfer/PR-1309.md), [Add blockwise-scaled FP8 GEMM via TRTLLM-Gen.](../sources/prs/flashinfer/PR-1320.md), [Optimizations for TRTLLM MNNVL Allreduce](../sources/prs/flashinfer/PR-1321.md), [refactor: Improved metainfo for trtllm-gen kernels](../sources/prs/flashinfer/PR-1328.md), [feat: masked layout fp4 gemm using cute-dsl](../sources/prs/flashinfer/PR-1331.md), [add torch float4_e2m1fn_x2 check for cudnn fp4 backend](../sources/prs/flashinfer/PR-1333.md), [[Fix] remove torch 2.8 requirement for FP4 GEMM](../sources/prs/flashinfer/PR-1334.md), [Fix bench deepgemm setting](../sources/prs/flashinfer/PR-1344.md), [feature: add fp4 mm using trtllm backend](../sources/prs/flashinfer/PR-1355.md), [[fix] remove (view) transpose to keep consistent with majorness MN requirement.](../sources/prs/flashinfer/PR-1358.md), [hotfix: update mxfp4 groupwise-scaled gemm unittests](../sources/prs/flashinfer/PR-1359.md), [refactor: download trtllm gemm metadata from server](../sources/prs/flashinfer/PR-1378.md), [GPT-OSS Support: Add Blackwell MoE mxfp4 implementation from TRTLLM and Attention Sink](../sources/prs/flashinfer/PR-1389.md), [Adding FP8 benchmark on attention and matmul testing](../sources/prs/flashinfer/PR-1390.md), [gpt-oss: Add MXFP8 x MXFP4 CUTLASS MOE for SM100 and BF16 x MXFP4 CUTLASS for SM90 + SwigluBias Activation](../sources/prs/flashinfer/PR-1396.md), [feature: add cutlass as bmm_fp8 backend.](../sources/prs/flashinfer/PR-1397.md), [fix shared memory alignment conflict in sampling.cuh](../sources/prs/flashinfer/PR-1402.md), [feature: enable cublas for fp4 gemm when cudnn == 9.11.1 or >= 9.13](../sources/prs/flashinfer/PR-1405.md), [Fixes for Blackwell Tests](../sources/prs/flashinfer/PR-1434.md), [bugfix: fix perf issue by using fp8 graph that can use cublaslt](../sources/prs/flashinfer/PR-1435.md), [Remove getEnvEnablePDL in favor of enable_pdl parameter](../sources/prs/flashinfer/PR-1446.md), [perf: add 1x4x1 cluster shape for fp8 bmm M<16 cases](../sources/prs/flashinfer/PR-1473.md), [tuner: Trtllm-gen Fp4 MoE Autotunner](../sources/prs/flashinfer/PR-1475.md), [refactor: unify autotuner for bmm_fp8](../sources/prs/flashinfer/PR-1479.md), [Add python API for masked grouped gemm](../sources/prs/flashinfer/PR-1481.md), [perf: add fast path to TopPRenormProbKernel for top_p >= 1.0, significantly boosting SGLang workloads](../sources/prs/flashinfer/PR-1483.md), [fix: update cutedsl masked moe gemm](../sources/prs/flashinfer/PR-1488.md), [Perf: support scale_a/scale_b instead of combined scale in cutlass bmm_fp8](../sources/prs/flashinfer/PR-1491.md), [fix: update masked moe gemm fp4 tensor reshape](../sources/prs/flashinfer/PR-1495.md), [feat: scaling at fp4 gemm epilogue](../sources/prs/flashinfer/PR-1498.md), [fix: Replace cub Max/Min with cuda::maximum/minimum for cuda 13 compatibility](../sources/prs/flashinfer/PR-1500.md), [Add benchmark for cutedsl gemm](../sources/prs/flashinfer/PR-1502.md), [bugfix: Fix stream handling in cutedsl gemm](../sources/prs/flashinfer/PR-1509.md), [flashinfer_benchmark QoL Improvements and Attention FP8 Support](../sources/prs/flashinfer/PR-1512.md), [refactor fp4 masked gemm cute-dsl implementation and add manual cache](../sources/prs/flashinfer/PR-1521.md), [Fix linking errors with CUDA 13](../sources/prs/flashinfer/PR-1523.md), [Add GeGLU support to trtllm-gen NVFP4 Fused MoE Kernel](../sources/prs/flashinfer/PR-1525.md), [Remove cuda-python from dependency and check at runtime](../sources/prs/flashinfer/PR-1534.md), [Add sm check for sm100 only cutlass/trtllm kernel](../sources/prs/flashinfer/PR-1535.md), [perf: Enable SplitK and fix tile-scheduling for moe fp4 fused moe](../sources/prs/flashinfer/PR-1548.md), [bugfix: update trtllm-gen gemm kernel names](../sources/prs/flashinfer/PR-1577.md), [bugfix: Fix arg passing to TORCH_CHECK and TORCH_WARN macros](../sources/prs/flashinfer/PR-1582.md), [fix: limit the number of nvcc threads for each kernel](../sources/prs/flashinfer/PR-1589.md), [bugfix: fix the register overflow issue for topk renorm kernels on blackwell](../sources/prs/flashinfer/PR-1597.md), [feat: Enable MnnvlMemory (for alltoallv) on B200](../sources/prs/flashinfer/PR-1601.md), [feat: initial support for SM103, SM110, SM120, SM121](../sources/prs/flashinfer/PR-1608.md), [feat: cutlass fp4 gemm bringup for SM120 & SM121](../sources/prs/flashinfer/PR-1609.md), [feat: cutlass fp8 gemm bringup for SM120 & SM121](../sources/prs/flashinfer/PR-1610.md), [perf: Fix the tactic sorting in TrtllmGenBatchedGemmRunner::getValidConfigIndices](../sources/prs/flashinfer/PR-1615.md), [bugfix: collect all modules to aot](../sources/prs/flashinfer/PR-1622.md), [test: pytest.mark.xfail on deepgemm](../sources/prs/flashinfer/PR-1636.md), [Added mx_fp4 support using the cudnn backend](../sources/prs/flashinfer/PR-1644.md), [Refactor Blackwell unit test scripts](../sources/prs/flashinfer/PR-1667.md), [TGV GEMM as a BF16 backend alternative to cuBLAS](../sources/prs/flashinfer/PR-1668.md), [Support output signals for overlapping for cutedsl gemm](../sources/prs/flashinfer/PR-1677.md), [Update TGV GEMM default kernel and TGV code cleanup.](../sources/prs/flashinfer/PR-1682.md), [Update deepgemm backend for 103a](../sources/prs/flashinfer/PR-1694.md), [[cute_dsl] add gemm + all reduce (two_shot)](../sources/prs/flashinfer/PR-1695.md), [hotfix: Hotfix for `test_pod_kernels.py` on B300](../sources/prs/flashinfer/PR-1698.md), [feat: Benchmark mm_fp4 mxfp4 support and gemm autotune support. Restore mm_fp4 API behavior](../sources/prs/flashinfer/PR-1706.md), [test: skip the unsupported test cases for sm120/121](../sources/prs/flashinfer/PR-1710.md), [TVM: support TVM binding for GroupedGemm](../sources/prs/flashinfer/PR-1725.md), [fix: put sampling kernel launch into macro](../sources/prs/flashinfer/PR-1727.md), [fix: fix cannot import name 'cuda' from 'cuda' in CUDA13](../sources/prs/flashinfer/PR-1764.md), [Added xfail for mx_fp4 matmul on SM120](../sources/prs/flashinfer/PR-1766.md), [tests: skip non SM100/103 for grouped deepgemm](../sources/prs/flashinfer/PR-1767.md), [Support checks PoC](../sources/prs/flashinfer/PR-1809.md), [tests: Update support for tgv_gemm to SM100 only and add to ut](../sources/prs/flashinfer/PR-1810.md), [tests: upgrade cutlass, fix import and skip non-SM100 cutedsl two shot allreduce](../sources/prs/flashinfer/PR-1812.md), [feat:enable fp8 blockscale moe for fused cultass for sm90](../sources/prs/flashinfer/PR-1819.md), [feat: trtrllm-gen global scaled FP8 GEMMs](../sources/prs/flashinfer/PR-1829.md), [raise error for group_gemm_fp8_nt_groupwise then num_groups > 1 on sm120/121](../sources/prs/flashinfer/PR-1862.md), [feat: Add FP4 TRTLLM-Gen throughput MOE batched gemms](../sources/prs/flashinfer/PR-1882.md), [misc: fix some B200 GEMM bench](../sources/prs/flashinfer/PR-1883.md), [Add layernorm op for inputs of mixed dtype](../sources/prs/flashinfer/PR-1926.md), [Feature: Support Relu2 activation in fused MoE](../sources/prs/flashinfer/PR-1954.md), [Update trtllm-gen fused moe routing kernel and add more kernels](../sources/prs/flashinfer/PR-1955.md), [fix: ensure SM120/121 SFA/SFB contiguity](../sources/prs/flashinfer/PR-1963.md), [feat: enable deepgemm jit for fp8 block-scale on SM90](../sources/prs/flashinfer/PR-1969.md), [Feature: Add support for L40 FusedMoE in cutlass path](../sources/prs/flashinfer/PR-1973.md), [feat: Add backend='auto' to mm_fp4 and enable autotune for backend='cudnn'](../sources/prs/flashinfer/PR-1979.md), [fix: Enable SM121 for mm_fp4](../sources/prs/flashinfer/PR-2012.md), [[feat] Refactor trtllmgen MOE and add Bf16 trtllmgen moe](../sources/prs/flashinfer/PR-2014.md), [[DSV3] Optimized Router Gemm](../sources/prs/flashinfer/PR-2019.md), [update trtllm cutlass moe ](../sources/prs/flashinfer/PR-2020.md), [[NVIDIA] Thor & Spark Support](../sources/prs/flashinfer/PR-2028.md), [feat: suitable_auto_backends to prune auto backends, bmm_fp8 refactor, heuristic_func intake](../sources/prs/flashinfer/PR-2029.md), [perf: Optimize helper max/minmax function in sampling.cuh](../sources/prs/flashinfer/PR-2058.md), [feat: BF16 GEMM using CUTLASS backend for SM100](../sources/prs/flashinfer/PR-2070.md), [refactor: pass hopper deepgemm include directory through python](../sources/prs/flashinfer/PR-2090.md), [perf: enable pdl for cutlass fp4 gemm](../sources/prs/flashinfer/PR-2095.md), [Refactor trtllm_mnnvl_allreduce](../sources/prs/flashinfer/PR-2118.md), [make DeepGEMM swapAB available for linear gemm SM90](../sources/prs/flashinfer/PR-2131.md), [feat: TRTLLM FMHAv2 backend for ctx attention](../sources/prs/flashinfer/PR-2142.md), [enable sm103 moe dsl backend](../sources/prs/flashinfer/PR-2149.md), [fix xqa mha_sm90.cu](../sources/prs/flashinfer/PR-2157.md), [feat: MxInt4 x Bf16 TRT-LLM Gen MoE support](../sources/prs/flashinfer/PR-2159.md), [Fix gemm allreduce two shot](../sources/prs/flashinfer/PR-2171.md), [Fix for moe on sm110](../sources/prs/flashinfer/PR-2190.md), [feat: unit-test and api change, w4a8 grouped-gemm fused MoE for SM90](../sources/prs/flashinfer/PR-2193.md), [Move the run function definition out of BatchedGemmInterface](../sources/prs/flashinfer/PR-2211.md), [misc: support checks for gemm](../sources/prs/flashinfer/PR-2214.md), [refactor: pull trtllm-gen batch-gemm/gemm headers from artifactory; update tma descriptor shape init](../sources/prs/flashinfer/PR-2235.md), [Remove cudaStreamSynchronize from gemm_groupwise_sm120.cuh for CUDA graph compatibility](../sources/prs/flashinfer/PR-2244.md), [feat: Add support for bmm mxfp8](../sources/prs/flashinfer/PR-2256.md), [Fix CUTLASS FP8 gemm correctness issue on SM120/SM121 for shapes where N is not divisible by ScaleGranularityN.](../sources/prs/flashinfer/PR-2261.md), [test: use .float() in in F.cosine_similarity() in bmm_fp8 test](../sources/prs/flashinfer/PR-2266.md), [Tiny fix bench tgv gemm](../sources/prs/flashinfer/PR-2277.md), [feat: IdType indices in sampling kernels](../sources/prs/flashinfer/PR-2281.md), [Selective State Update kernel (mamba)](../sources/prs/flashinfer/PR-2301.md), [[Perf][Feature] Add SM103-specific schedulers for NVFP4 CUTLASS kernels](../sources/prs/flashinfer/PR-2303.md), [feat: Support Fused MoE non gated Relu2 NVFP4 & FP8 and support Nemotron](../sources/prs/flashinfer/PR-2304.md), [Fix: FilteredTopKUnifiedKernel read value out of length](../sources/prs/flashinfer/PR-2308.md), [[ML3] Optimized Router Gemm](../sources/prs/flashinfer/PR-2323.md), [bugfix: fix multi-cta top-k implementation when k value is different for different row](../sources/prs/flashinfer/PR-2325.md), [[perf] Improve gemm_fp8_nt_groupwise (cutlass backend) by 10-40% for batch sizes <= 32](../sources/prs/flashinfer/PR-2327.md), [Enable fp16/bf16/f32 support for selective_state_update (mamba)](../sources/prs/flashinfer/PR-2366.md), [feat: BF16 GEMM using cuDNN backend](../sources/prs/flashinfer/PR-2376.md), [bugfix: hotfix of PR 2366 (mamba kernel)](../sources/prs/flashinfer/PR-2378.md), [feat: cuteDSL fp4 moe for better DSR1 performance.](../sources/prs/flashinfer/PR-2398.md), [perf: mm_fp4 heuristic prioritizes CUTLASS over cuDNN on SM103](../sources/prs/flashinfer/PR-2404.md), [feat: update trtllm-gen MoE cubins](../sources/prs/flashinfer/PR-2416.md), [fix: Sampling: CUDA Graph fix](../sources/prs/flashinfer/PR-2432.md), [MTP for mamba ](../sources/prs/flashinfer/PR-2444.md), [fix: fix illegal memory access for NaN input in sampling kernels](../sources/prs/flashinfer/PR-2456.md), [perf: add fp4 GEMM tile configs and streamK scheduler for SM120](../sources/prs/flashinfer/PR-2460.md), [feat: Support Fused MoE non gated Relu2 NVFP4 & FP8 and support Nemotron, fixed](../sources/prs/flashinfer/PR-2462.md), [feat: Add MXFP8 GEMM mm_mxfp8 (cutlass)](../sources/prs/flashinfer/PR-2464.md), [fix: add support check for gemm config for cutlass moe](../sources/prs/flashinfer/PR-2495.md), [refactor: Port upstream CUTLASS fixes and refactor grouped_gemm_nt_masked GEMM module location](../sources/prs/flashinfer/PR-2503.md), [Feat: Trtllm-gen MxFP8 MoE integration](../sources/prs/flashinfer/PR-2505.md), [feat: BF16 GEMM benchmarking support](../sources/prs/flashinfer/PR-2525.md), [fix: include fp8_blockscale_gemm_90 in AOT jit-cache](../sources/prs/flashinfer/PR-2533.md), [tests: bmm_fp8 for SM110](../sources/prs/flashinfer/PR-2538.md), [feat: cute dsl mmfp4 for blackwell](../sources/prs/flashinfer/PR-2540.md), [misc: point triton blackwell-ptxas to local cuda ptxas](../sources/prs/flashinfer/PR-2543.md), [Add gen_gemm_sm100_module_cutlass_mxfp8 to jit-cache](../sources/prs/flashinfer/PR-2549.md), [Add support for the combinations of allreduce, allgather, and reducescatter](../sources/prs/flashinfer/PR-2563.md), [feat: add is_sm12x_supported() helper for SM12x family detection](../sources/prs/flashinfer/PR-2574.md), [Implement `cutlass_fused_moe` mxfp8](../sources/prs/flashinfer/PR-2581.md), [feat: trtllm tinygemm2 in flashinfer as bf16 routergemm](../sources/prs/flashinfer/PR-2587.md), [Mamba SSU: better automatic kernel selection + algorithm selection optionally exposed to the user.](../sources/prs/flashinfer/PR-2591.md), [[bugfix] Fix FilteredTopK overflow correctness](../sources/prs/flashinfer/PR-2605.md), [fix: add SM121 support to SM120 version guards](../sources/prs/flashinfer/PR-2631.md), [feat: FP32 dtype output for BF16 matmuls (CUTLASS & cuDNN)](../sources/prs/flashinfer/PR-2644.md), [[feat] trtllm-gen mxfp8 gemm](../sources/prs/flashinfer/PR-2653.md), [fix: Add fused MOE and GEMM AOT modules for SM121](../sources/prs/flashinfer/PR-2654.md), [feat: support mxfp4 & mxfp8 entrypoint for blackwell cutedsl dense gemm](../sources/prs/flashinfer/PR-2660.md), [feat: implement deterministic topk](../sources/prs/flashinfer/PR-2661.md), [perf: Update trtllm-gen batched GEMM kernels - faster, more NVFP4 tile dims, MXFP8 with relu2 act](../sources/prs/flashinfer/PR-2667.md), [fix: reduce smem allocation for tinygemm2 kernel in SM120](../sources/prs/flashinfer/PR-2670.md), [Add varlen and speculative decoding support to selective state update](../sources/prs/flashinfer/PR-2700.md), [fix(jit): GEMM kernels produce NaN under concurrency — missing GDC flags cause PDL synchronization barriers to compile as no-ops](../sources/prs/flashinfer/PR-2716.md), [Support for MXFP4 and NVFP4 group GEMMs on GeForce and Spark](../sources/prs/flashinfer/PR-2738.md), [misc: Update gemm/batched gemm cubins from trtllm-gen, gemm header refactor](../sources/prs/flashinfer/PR-2740.md), [[Spark unit test debugging] Fix for tests/gemm/test_groupwise_scaled_gemm_fp8.py](../sources/prs/flashinfer/PR-2751.md), [[feat] Add air top-p algorithm](../sources/prs/flashinfer/PR-2752.md), [fix(jit): enable GDC for CUTLASS GEMM PDL — SM100 flag only](../sources/prs/flashinfer/PR-2780.md), [Upgrade cutlass 4.2.1 -> 4.4.2](../sources/prs/flashinfer/PR-2798.md), [CuteDSL MoE fix redundant output buffer zeroing](../sources/prs/flashinfer/PR-2811.md), [Add support for Relu2 in BF16 fused MoE](../sources/prs/flashinfer/PR-2864.md), [Mamba SSU: horizontal MTP kernel (+ DSTATE=96 support)](../sources/prs/flashinfer/PR-2865.md), [fix: fix cute dsl swap_ab tactic failure](../sources/prs/flashinfer/PR-2870.md), [feat: add MXFP8 GEMM support for SM120](../sources/prs/flashinfer/PR-2902.md), [[NVIDIA] fix(jit): enable GDC for CUTLASS fused MoE PDL — prevent random crashes on SM12x](../sources/prs/flashinfer/PR-2913.md), [feat: Add cuBLASLt backend for `mm_bf16` and enable multi-tactic autotuning for FP8/MXFP8 runners](../sources/prs/flashinfer/PR-2914.md), [feat: add Relu2 (squared ReLU) activation support in CUTLASS MoE backend](../sources/prs/flashinfer/PR-2926.md), [feat: SM121 (GB10) tile filtering and autotuner robustness](../sources/prs/flashinfer/PR-2927.md), [CuTe DSL FP4 GEMM Heuristic](../sources/prs/flashinfer/PR-2940.md), [feat: Add CuTe DSL grouped-gemm + combine fusion support](../sources/prs/flashinfer/PR-2944.md), [fix: use float instead of double in sampling binary search to avoid FP64 bottleneck on SM103](../sources/prs/flashinfer/PR-2945.md), [Update NVSHMEM interface to use NVSHMEM4Py instead of custom bindings](../sources/prs/flashinfer/PR-2960.md), [Improved `simple` mamba SSU kernel ](../sources/prs/flashinfer/PR-2962.md), [test: skip unsupported mm_mxfp8 configurations on SM12x](../sources/prs/flashinfer/PR-2974.md), [fix: tinygemm2 hang issue due to barrier sync](../sources/prs/flashinfer/PR-2996.md), [[chore] Install nvidia-cutlass-dsl[cu13] for cu130+](../sources/prs/flashinfer/PR-3017.md), [perf: Port TRT-LLM SM120/SM121 FP4 CUTLASS GEMM optimizations. Add PDL](../sources/prs/flashinfer/PR-3026.md), [[feat] Trtllm-gen Per-token Nvfp4 MoE](../sources/prs/flashinfer/PR-3027.md), [fused_moe: pre-filter SM89 tactics with zero occupancy on SM120 Blackwell (fix review feedback on #2764)](../sources/prs/flashinfer/PR-3032.md), [feat: Add backend="b12x" for mm_fp4 on SM120](../sources/prs/flashinfer/PR-3051.md), [perf: Add no-bias path for tinygemm_bf16](../sources/prs/flashinfer/PR-3151.md), [Integrate CUTLASS Small Tile N Blockscaled GEMMs/Grouped GEMMs for SM120 and SM121](../sources/prs/flashinfer/PR-3152.md), [Fix/3170 dense blockscaled sm12x](../sources/prs/flashinfer/PR-3180.md), [test: enable bmm_mxfp8 cutlass backend coverage on SM12x](../sources/prs/flashinfer/PR-3183.md), [feat: enable glm5 router gemm](../sources/prs/flashinfer/PR-3185.md), [Include TinyGEMM into BF16 autotuner](../sources/prs/flashinfer/PR-3203.md), [feat(trace): embed runnable init() in every TraceTemplate](../sources/prs/flashinfer/PR-3221.md), [[Bugfix] Fix fused MoE autotuning correctness issues by filtering clusterDimZ](../sources/prs/flashinfer/PR-3227.md), [Update moe gemm](../sources/prs/flashinfer/PR-3239.md), [[feat] Add gemma RMS AR fusion](../sources/prs/flashinfer/PR-3322.md), [Change `apply_rope_with_cos_sin_cache` to accept `cos_sin_cache`](../sources/prs/flashinfer/PR-754.md), [bugfix: Ensure Loop Termination by Enforcing IEEE-754 Compliance in Sampling Kernels](../sources/prs/flashinfer/PR-774.md), [bugfix: fix the signature of `CutlassSegmentGEMMSM90`](../sources/prs/flashinfer/PR-827.md), [feat: experimenta support of PDL](../sources/prs/flashinfer/PR-930.md), [perf: reduce torch.library dispatch overhead](../sources/prs/flashinfer/PR-968.md), [perf: Fix python API overhead when CUDAGraph is not enabled](../sources/prs/flashinfer/PR-969.md), [perf: dual pivot top-p/top-k renorm](../sources/prs/flashinfer/PR-974.md), [SM-constraint-GEMM by triton persistent kernel](../sources/prs/flashinfer/PR-982.md), [Triton `rms_norm` kernels](../sources/prs/flashinfer/PR-983.md), [feat: SM-constraint Communication Kernels](../sources/prs/flashinfer/PR-994.md), [Update torch-xpu-ops commit pin](../sources/prs/pytorch/PR-144209.md), [[inductor][cpu] Fix bmm b_index for dynamic expressions in inductor autotuner](../sources/prs/pytorch/PR-144248.md), [Fix PythonMod printing](../sources/prs/pytorch/PR-144335.md), [ROCm SDPA: Ensure attn_mask has the same dtype with q](../sources/prs/pytorch/PR-144398.md), [[inductor] Fix profiler tests with latest Triton](../sources/prs/pytorch/PR-149059.md), [Remove runtime dependency on packaging](../sources/prs/pytorch/PR-149125.md), [op should NOT be static in aoti_torch_call_dispatcher](../sources/prs/pytorch/PR-149644.md), [Add release branch push triggers to inductor-rocm-mi300.yml](../sources/prs/pytorch/PR-149871.md), [[inductor] Fix inductor windows linker error](../sources/prs/pytorch/PR-150447.md), [[Windows][inductor] fix blank space break windows file path](../sources/prs/pytorch/PR-150448.md), [[CUDA][avgpool2d] Fix backward launch bounds again for `sm100`, `sm120`](../sources/prs/pytorch/PR-150676.md), [[dynamo][super variable] Fix bug to use correct source](../sources/prs/pytorch/PR-152774.md), [[ATen][CUDA] Optimize 128 bit vectorization](../sources/prs/pytorch/PR-152967.md), [Mark auto_functionalized HOPs as cacheable (#151194)](../sources/prs/pytorch/PR-153304.md), [Fix uint view copy (#151598)](../sources/prs/pytorch/PR-154121.md), [Fix macOS build with `USE_MPS=OFF`](../sources/prs/pytorch/PR-156932.md), [[PowerPC] Fixed build issue for vsx vec256 complexfloat and scaled_mm_out_cpu ](../sources/prs/pytorch/PR-157422.md), [[release] Triton pin update to 3.4](../sources/prs/pytorch/PR-157752.md), [[MPS] Switch Cholesky decomp to column wise](../sources/prs/pytorch/PR-158237.md), [[cherry-pick][inductor][triton] Update HAS_WARP_SPEC to check triton.Config params. Update Triton Hash to top of release/3.4.x stack](../sources/prs/pytorch/PR-158646.md), [CUDA 13.0 Windows Nvidia Driver Update to 580.88](../sources/prs/pytorch/PR-162501.md), [[Cherry Pick][Graph Partition] allow sharing default device context](../sources/prs/pytorch/PR-163097.md), [[Release 2.9] [cuDNN][SDPA][submodule] Roll-back cuDNN frontend upgrade, update Met…](../sources/prs/pytorch/PR-163265.md), [[Graph Partition] improve custom op output alias](../sources/prs/pytorch/PR-163380.md), [[Inductor][Intel GPU] Save `threads_per_warp` from tirton compiled kernel for launching kernel correctly in cpp wrapper.](../sources/prs/pytorch/PR-163388.md), [[graph partition] Add way to register custom rule (#163310)](../sources/prs/pytorch/PR-163395.md), [[2.9 cherry pick][triton] update 3.5 pin to bbb06c0334a6772b92d24bde54956e675c8c6604 (#163382)](../sources/prs/pytorch/PR-163583.md), [CUDA 13.0 Warning update for supported architectures](../sources/prs/pytorch/PR-163633.md), [[Cherry-Pick] [CD] CUDA 13 specific followup changes. Remove sm50-70 From CUDA 12.6 and CUDA 12.8 builds (#162455)](../sources/prs/pytorch/PR-163764.md), [[CD] CUDA 13.0 fix preload logic to include nvidia/cu13/lib/](../sources/prs/pytorch/PR-163766.md), [fix pickling for BitwiseFn](../sources/prs/pytorch/PR-163861.md), [Move inductor jobs 3.9->3.10](../sources/prs/pytorch/PR-163954.md), [[cuDNN][SDPA] Disable dropout for cuDNN SDPA on 9.11 - 9.13](../sources/prs/pytorch/PR-164026.md), [[AARCH64][CD][CUDA13][Triton][PTXAS] Turn on BUILD_BUNDLE_PTXAS=1 ](../sources/prs/pytorch/PR-164236.md), [CUDA 13.0 builds fix on Amazon Linux 2023](../sources/prs/pytorch/PR-164893.md), [[inductor] don't try to reorder loops for template](../sources/prs/pytorch/PR-166910.md), [[Dynamo] Don't guard data ptrs by default with mark_static_address](../sources/prs/pytorch/PR-166913.md), [[Inductor] No longer throw error in bmm out_dtype lowering due to tem…](../sources/prs/pytorch/PR-166922.md), [[Graph Partition] move custom rules to inductor config (#166458)](../sources/prs/pytorch/PR-166967.md), [[Graph Partition] fix graph partition input signature for fallback kernels](../sources/prs/pytorch/PR-166985.md), [[GraphPartition] cache get_free_symbol_uses (#166338)](../sources/prs/pytorch/PR-166994.md), [[Minor][Inductor] move some combo kernel log from warning to debug](../sources/prs/pytorch/PR-167020.md), [[cuDNN][SDPA] Check-in test for #166211](../sources/prs/pytorch/PR-167121.md), [[cuDNN][SDPA][Convolution] Expose cuDNN runtime version in CUDA hooks](../sources/prs/pytorch/PR-167327.md), [[ROCm] Enable shared memory based pruning for Triton configs](../sources/prs/pytorch/PR-170190.md), [[Inductor] ExternKernelBenchmarkRequest best attempt](../sources/prs/pytorch/PR-170246.md), [[inductor] Fix cudagraph skip for index_put_ with boolean indices, gr…](../sources/prs/pytorch/PR-170884.md), [[Inductor] Fix constants handling for Triton constexpr (triton#8248)](../sources/prs/pytorch/PR-171129.md), [[ROCm] Make grouped GEMM CK opt‑in via env and default to fallback path](../sources/prs/pytorch/PR-171140.md), [Avoid closing random file handles in Inductor](../sources/prs/pytorch/PR-171150.md), [[cherry-pick][CUDA] Upgrade cuDNN to 9.15.1 for CUDA 13 builds ](../sources/prs/pytorch/PR-171189.md), [[xpu][fix][inductor] fallback bfloat16 atomics to eager](../sources/prs/pytorch/PR-171247.md), [[cherry-pick][cuDNN][SDPA] cuDNN SDPA off-by-default for cuDNN versions < 12.9 (#171627)](../sources/prs/pytorch/PR-171895.md), [Skip modded_nanogpt model in TorchInductor benchmark](../sources/prs/pytorch/PR-172141.md), [[Graph Partition] Improve support for mutation ops](../sources/prs/pytorch/PR-172577.md), [Update inductor expected accuracy files](../sources/prs/pytorch/PR-175096.md), [[benchmark] Skip pytorch_CycleGAN_and_pix2pix from inductor benchmarks](../sources/prs/pytorch/PR-175299.md), [[release-only] Remove +ptx from cuda 13.0 builds](../sources/prs/pytorch/PR-175567.md), [[inductor] avoid multi-stage for mix-order-red by default (#176228)](../sources/prs/pytorch/PR-176495.md), [[inductor] Fix Identity comparability and evalf recursion](../sources/prs/pytorch/PR-176783.md), [[Inductor] Don't unfuse addmm for bf16/fp16 to avoid precision loss](../sources/prs/pytorch/PR-177144.md), [[Inductor][MPS] Fix half-precision type mismatches in Metal shader codegen (#176436)](../sources/prs/pytorch/PR-177193.md), [[MPS] fix compiling of SDPA producing nan results](../sources/prs/pytorch/PR-178009.md), [Optimize nvfp4 block scaled gemm kernel when M is small.](../sources/prs/sglang/PR-10101.md), [fix: resolve gb200 image link](../sources/prs/sglang/PR-10343.md), [Update CUTLASS. Refine KernelSchedule for fp8 (grouped) gemm.](../sources/prs/sglang/PR-10491.md), [Cache the result of `is_blackwell` platform check](../sources/prs/sglang/PR-10498.md), [Unify SGL Kernel Releases](../sources/prs/sglang/PR-10701.md), [Optimize cutlass int8 gemm kernel for large M on SM89 Ada GPU](../sources/prs/sglang/PR-10714.md), [chore: upgrade sgl-kernel 0.3.13](../sources/prs/sglang/PR-11056.md), [[NVIDIA] Add new SMs support for Spark & Thor](../sources/prs/sglang/PR-11287.md), [[sgl-kernel][1/N]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-11432.md), [Improve Kernel Build Time](../sources/prs/sglang/PR-11508.md), [support cutlass fp4 kernel in sm120](../sources/prs/sglang/PR-11737.md), [Change bf16 to fp8 for some gemms in attention for DeepSeek ckpt v2](../sources/prs/sglang/PR-11805.md), [Use cutlass fp4 gemm by default](../sources/prs/sglang/PR-11813.md), [chore: upgrade flashinfer 0.4.1](../sources/prs/sglang/PR-11933.md), [[sgl-kernel][4/N]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-12080.md), [[hotfix] missing `w13_weight_fp8` and `w2_weight_fp8` in UE8M0 requantization](../sources/prs/sglang/PR-12259.md), [fix: Llama 4 BF16 load on Blackwell](../sources/prs/sglang/PR-12308.md), [[sgl-kernel] clean up fa fetch in CMakeLists.txt](../sources/prs/sglang/PR-12392.md), [chore: upgrade flashinfer 0.5.0](../sources/prs/sglang/PR-12523.md), [[NVIDIA] Fix CUDA arch requirement in nvfp4 cast](../sources/prs/sglang/PR-12581.md), [[sgl-kernel][5/N]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-12666.md), [ignore the deepgemm check when the model weight with nvfp4 and moe ba…](../sources/prs/sglang/PR-12782.md), [[NVIDIA] Fix use case of SGLANG_ENABLE_FLASHINFER_GEMM](../sources/prs/sglang/PR-13274.md), [Support weight update for blackwell DeepGEMM](../sources/prs/sglang/PR-13324.md), [[ROCM] Optimized deepseek-r1 fp8 model with + triton_gemm_a8w8 + batch_gemm_a8w8 + fused set_mla_kv_buffer kernel](../sources/prs/sglang/PR-13617.md), [[sgl-kernel][Feat][B200][1/N]Support MXFP8 Grouped GEMM in Blackwell](../sources/prs/sglang/PR-13731.md), [[chore]Upgrade flashinfer to 0.5.3](../sources/prs/sglang/PR-13751.md), [update flashinfer_cubin==0.5.3](../sources/prs/sglang/PR-13848.md), [Fix update weight error for blackwell DeepGEMM](../sources/prs/sglang/PR-13910.md), [Add new moe wna16 marlin gemm](../sources/prs/sglang/PR-14122.md), [Apply new moe wna16 marlin gemm](../sources/prs/sglang/PR-14125.md), [[CPU] Implement MXFP4 Gemm kernels for intel AMX to support GPT OSS series.](../sources/prs/sglang/PR-14385.md), [Add CUDA kernel size analysis tool for sgl-kernel optimization](../sources/prs/sglang/PR-14544.md), [[sgl-kernel][Feat][B200][2/N] Support MXFP8 Grouped GEMM in Blackwell](../sources/prs/sglang/PR-14640.md), [Add cache for flashinfer installation](../sources/prs/sglang/PR-15153.md), [[Tiny]Add warning for deepgemm on Blackwell](../sources/prs/sglang/PR-15352.md), [[sgl-kernel][6/7]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-15471.md), [[Perf] Add Flashinfer DeepGEMM SM90 for SwapAB Optimization](../sources/prs/sglang/PR-15514.md), [[sgl-kernel] Streamline kernel size report (Top 20 only) and clean up](../sources/prs/sglang/PR-15552.md), [[JIT kernel] Apply jit per_tensor_quant_fp8 kernel](../sources/prs/sglang/PR-15836.md), [[fix]deepgemm precompile when warmup](../sources/prs/sglang/PR-15891.md), [[Feature] add aligned_vector type for JIT kernel](../sources/prs/sglang/PR-16162.md), [Fix sgl-kernel jobs to skip when target_stage is specified](../sources/prs/sglang/PR-16308.md), [[Fix]Pin mooncake version to 0.3.7.post2 in grace blackwell](../sources/prs/sglang/PR-16502.md), [Fix FP8 MoE NaN with DeepGEMM on Blackwell](../sources/prs/sglang/PR-16622.md), [Support mxint4 flashinfer_trtllm moe gemm](../sources/prs/sglang/PR-16892.md), [[MUSA][2/N] sgl-kernel build](../sources/prs/sglang/PR-17053.md), [[FIX] Always support TP > 4 for FP4 Gemm](../sources/prs/sglang/PR-17300.md), [Upgrade transformers==5.3.0](../sources/prs/sglang/PR-17784.md), [Feature/support longcat flash lite](../sources/prs/sglang/PR-17838.md), [Skipped warning on sm100](../sources/prs/sglang/PR-18000.md), [[Diffsuion & JIT_kernel] QKNorm cross heads kernel](../sources/prs/sglang/PR-18073.md), [[Blackwell] Make mxint4 flashinfer_trtllm moe gemm set by default on blackwell](../sources/prs/sglang/PR-18136.md), [[FIX] Correct JIT kernel compilation on newer GPUs with outdated driver metadata.](../sources/prs/sglang/PR-18496.md), [use flashinfer.sampling](../sources/prs/sglang/PR-18696.md), [fix: update Blackwell log/error messages to include SM12x](../sources/prs/sglang/PR-18751.md), [fix: add SM110 (Jetson AGX Thor) to Blackwell capability check](../sources/prs/sglang/PR-18787.md), [Migrate renorm kernels from sgl-kernel to FlashInfer JIT](../sources/prs/sglang/PR-18854.md), [Add claude skills for sgl-kernel and jit-kernel](../sources/prs/sglang/PR-18855.md), [Migrate norm kernels to FlashInfer JIT implementation](../sources/prs/sglang/PR-18871.md), [Use single mma warp group for short q_len in FA to optimize decoding performance](../sources/prs/sglang/PR-18985.md), [[Kernel Slimming] Migrate NVFP4 kernels to JIT](../sources/prs/sglang/PR-19437.md), [[FlashInfer v0.6.4] [RL] Integrate FlashInfer mxfp8 gemm, MoE, and routed MoE](../sources/prs/sglang/PR-19537.md), [[Feature] NVFP4 Marlin fallback for non-Blackwell GPUs (SM75+)](../sources/prs/sglang/PR-19652.md), [[JIT Kernel][Feature] Support JIT custom all reduce (rewrite as v2)](../sources/prs/sglang/PR-19880.md), [[JIT Kernel] Reland NVFP4 kernels to JIT](../sources/prs/sglang/PR-20012.md), [[Benchmark] use flashinfer bench_gpu_time instead of triton do_bench](../sources/prs/sglang/PR-20305.md), [fix ci by removing nvidia-cutlass-dsl-libs-base and force reinstall n…](../sources/prs/sglang/PR-20380.md), [Fix(jit): support rmsnorm for hidden_size in {64, 128, 256}](../sources/prs/sglang/PR-20661.md), [Use FlashInfer tinygemm for GPT-OSS MoE router on SM90+](../sources/prs/sglang/PR-20755.md), [fix: guard configure_deep_gemm_num_sms when JIT disabled](../sources/prs/sglang/PR-20868.md), [CUTLASS FP8 Blockwise GEMM improvement of SM120](../sources/prs/sglang/PR-20887.md), [[Tiny Fix] Fix IS_BLACKWELL env var empty string warning in rerun-ut workflow](../sources/prs/sglang/PR-20957.md), [fix: wrap _import_static_state in inference_mode to fix resume on Blackwell](../sources/prs/sglang/PR-21035.md), [ci: remove IS_BLACKWELL env var; auto-detect Blackwell](../sources/prs/sglang/PR-21118.md), [[NPU] bugfix for import sgl-kernel error](../sources/prs/sglang/PR-21200.md), [Split pr-test.yml: extract sgl-kernel, jit-kernel, and multimodal-gen tests into separate workflow files](../sources/prs/sglang/PR-21219.md), [Refactor JIT kernel CI to use run_suite.py registration system](../sources/prs/sglang/PR-21239.md), [[MUSA] apply_vocab_mask support musa device](../sources/prs/sglang/PR-21296.md), [CUTLASS NVFP4 GEMM improvement of SM120](../sources/prs/sglang/PR-21314.md), [[FlashInver v0.6.7] Integrate flashinfer_trtllm mxfp8 gemm](../sources/prs/sglang/PR-21576.md), [[AMD] Use tgemm.mm for MoEGate router gemm in deepseek_v2.py](../sources/prs/sglang/PR-21657.md), [Remove flashinfer wheel cache cleanup that deletes other versions](../sources/prs/sglang/PR-21711.md), [[DSA] Set trtllm kernels as default for Blackwell](../sources/prs/sglang/PR-21914.md), [[Diffusion] Fix weight scale swizzle and add large-M kernel config for FLUX.2-dev-NVFP4](../sources/prs/sglang/PR-22064.md), [[nvidia] Gemma4 nvfp4 fix](../sources/prs/sglang/PR-22079.md), [[Hotfix] Fix router gemm on sm103](../sources/prs/sglang/PR-22134.md), [[hisparse]: Adding ci for hisparse kvcache-swap-in jit-kernel](../sources/prs/sglang/PR-22155.md), [[HiSparse]: Add benchmark for hisparse kernel](../sources/prs/sglang/PR-22187.md), [[Docker] Fix Trivy CVEs, cubin download 403s, and kernels command order](../sources/prs/sglang/PR-22322.md), [Upgrade sglang-torch-profiler-analysis SKILLS](../sources/prs/sglang/PR-22440.md), [[RL] Fix weight update for mxfp8 flashinfer_cutlass gemm backend](../sources/prs/sglang/PR-22484.md), [[CI/Docker] Clean up redundant flashinfer cubin downloads](../sources/prs/sglang/PR-22491.md), [[Docker] Remove flashinfer cache copy](../sources/prs/sglang/PR-22653.md), [[Fix/Kernel] Add JIT rmsnorm_hf kernel to fix transformers backend MMLU accuracy regression ](../sources/prs/sglang/PR-22931.md), [Reland Cute-DSL FP4 dense GEMM](../sources/prs/sglang/PR-23590.md), [feat: port SGLANG_JIT_DEEPGEMM_FAST_WARMUP to deepseek_v4 branch](../sources/prs/sglang/PR-23756.md), [Enable PDL for various kernels in DSV32/GLM5](../sources/prs/sglang/PR-23965.md), [[VLM] Optimize Gemma4 VLM with PCG and fuse RMSNorm + residual add + scalar](../sources/prs/sglang/PR-24048.md), [Port MXFP4 Marlin MoE support to JIT kernel path](../sources/prs/sglang/PR-24490.md), [[Gemma4] Optimize Gemm4 with fused Q/K/V RMSNorm + per-expert FP8 ckpt loader](../sources/prs/sglang/PR-24696.md), [[codex] Optimize hidden-size 512 RMSNorm dispatch](../sources/prs/sglang/PR-24710.md), [[rebase]Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-24986.md), [Support Gemma4 Pipeline Parallelism](../sources/prs/sglang/PR-25284.md), [[Gemma4]: Fix FP8 Triton scale layout](../sources/prs/sglang/PR-25286.md), [[Intel GPU] Enable DeepSeek V4 Inference on XPU](../sources/prs/sglang/PR-25336.md), [[fp8] SM90 swap-AB scaled_mm dispatch (~1.16x kernel geomean, +5.8-18.5% end-to-end)](../sources/prs/sglang/PR-25532.md), [Use DeepGEMM BF16 for unquantized DeepEP LL MoE](../sources/prs/sglang/PR-25540.md), [fix (jit kernel): elementwise activation C++ error](../sources/prs/sglang/PR-25695.md), [[Codex] Remove stale DeepSeek V4 JIT kernels](../sources/prs/sglang/PR-25764.md), [[Test] Stage-a sanity kits; consolidate core/ + models_e2e/ tests](../sources/prs/sglang/PR-25831.md), [Support cutlass Int8 gemm](../sources/prs/sglang/PR-2752.md), [upgrade cutlass v3.7.0](../sources/prs/sglang/PR-2967.md), [feat: add flashinfer as 3rdparty and use rmsnorm as example](../sources/prs/sglang/PR-3033.md), [Support sm90 Int8 gemm](../sources/prs/sglang/PR-3035.md), [Allow local cutlass directory to be used in sgl-kernel build](../sources/prs/sglang/PR-3037.md), [support w8a8 fp8 kernel with CUTLASS](../sources/prs/sglang/PR-3047.md), [sync the upstream updates of flashinfer](../sources/prs/sglang/PR-3051.md), [feat: integrate gemm_fp8 kernel into gemm](../sources/prs/sglang/PR-3056.md), [add tensorrt_llm common and cutlass_extensions as 3rdparty](../sources/prs/sglang/PR-3216.md), [support blockwise fp8 matmul kernel](../sources/prs/sglang/PR-3267.md), [fix undefined symbol cudaGetDriverEntryPointByVersion](../sources/prs/sglang/PR-3372.md), [integrate blockwise fp8 kernel](../sources/prs/sglang/PR-3529.md), [update flashinfer-python](../sources/prs/sglang/PR-3557.md), [[Feature] Apply Cublas Grouped Gemm kernel](../sources/prs/sglang/PR-3629.md), [add control for cutlass fp8 blockwise gemm](../sources/prs/sglang/PR-3727.md), [Support FP4 gemm (1/2)](../sources/prs/sglang/PR-3899.md), [upgrade flashinfer v0.2.2.post1](../sources/prs/sglang/PR-3934.md), [Support overlapping two batches](../sources/prs/sglang/PR-4068.md), [DeepGemm integrate to gemm](../sources/prs/sglang/PR-4165.md), [linear support deepgemm](../sources/prs/sglang/PR-4199.md), [Accelerate FP8 CUDA Kernel by 20-28%](../sources/prs/sglang/PR-4215.md), [fix per_token_group_quant_fp8 illegal memory when num_groups % 16 != 0](../sources/prs/sglang/PR-4231.md), [add THIRDPARTYNOTICES for DeepGEMM](../sources/prs/sglang/PR-4272.md), [Support Blackwell Block Scale FP8 Gemm](../sources/prs/sglang/PR-4278.md), [update deepgemm](../sources/prs/sglang/PR-4284.md), [upgrade flashinfer 0.2.3](../sources/prs/sglang/PR-4317.md), [[FIX] fix incorrect output when enable both deepgemm and torch compile](../sources/prs/sglang/PR-4359.md), [Create col-major and tma-aligned x_scale for deep_gemm.gemm_fp8_fp8_bf16_nt](../sources/prs/sglang/PR-4515.md), [Support fp8 gemm for blackwell](../sources/prs/sglang/PR-4558.md), [Set deepgemm to the default value in the hopper architecture.](../sources/prs/sglang/PR-4613.md), [Fix loading KV quantization scale; Enable modelopt kv cache](../sources/prs/sglang/PR-4686.md), [support cmake for sgl-kernel](../sources/prs/sglang/PR-4706.md), [[Feat] support deepgemm for cmake](../sources/prs/sglang/PR-4864.md), [[Build] Fix cuda12.8 build error in nvfp4_scaled_mm_kernels.cu](../sources/prs/sglang/PR-4953.md), [update cutlass tag](../sources/prs/sglang/PR-5011.md), [fix deepgemm as well](../sources/prs/sglang/PR-5030.md), [support sgl-kernel on blackwell](../sources/prs/sglang/PR-5074.md), [Add optimized native kernels in sgl-kernel](../sources/prs/sglang/PR-5150.md), [feat: add DeepGEMM build warning](../sources/prs/sglang/PR-5176.md), [[Fix] Turn off DeepGEMM by default](../sources/prs/sglang/PR-5263.md), [fix: use deepgemm only on hopper](../sources/prs/sglang/PR-5310.md), [fix: determine if flashinfer is installed](../sources/prs/sglang/PR-5336.md), [Fix DeepGEMM masked cannot be run on groups not being multiple or 4](../sources/prs/sglang/PR-5340.md), [chore: upgrade DeepGEMM](../sources/prs/sglang/PR-5395.md), [[perf] introduce deep gemm group_gemm_masked as gemm](../sources/prs/sglang/PR-5432.md), [Fix sampler nan check when calling top_k_top_p_sampling_from_probs](../sources/prs/sglang/PR-5546.md), [feat: use flashinfer jit package](../sources/prs/sglang/PR-5547.md), [[feature] enable pre compile jit deep_gemm](../sources/prs/sglang/PR-5580.md), [[fix] force use deepgemm in compile_deep_gemm](../sources/prs/sglang/PR-5618.md), [ DeepEP normal support deepgemm-contiguous](../sources/prs/sglang/PR-5626.md), [Turn on DeepGemm By Default and Update Doc](../sources/prs/sglang/PR-5628.md), [cutlass 3.9 supported to improve fp8_blockwise_gemm](../sources/prs/sglang/PR-5820.md), [Add sm_120 for blackwell](../sources/prs/sglang/PR-5903.md), [[Feat] Enable PDL automatically on Hopper architecture](../sources/prs/sglang/PR-5981.md), [chore: upgrade cutlass 3.9.2](../sources/prs/sglang/PR-6004.md), [chore: upgrade deepgemm](../sources/prs/sglang/PR-6073.md), [[1/2] Add Kernel support for Cutlass based Fused FP4 MoE](../sources/prs/sglang/PR-6093.md), [[Fix] Improve dependencies for Blackwell image](../sources/prs/sglang/PR-6334.md), [Upgrade CUTLASS 4.0](../sources/prs/sglang/PR-6336.md), [Add fp8 fused_experts kernel for CPU in sgl-kernel and add UT](../sources/prs/sglang/PR-6404.md), [Fix bug of deepseek-v3 under DP+EP mode with large batchsize/seqlen](../sources/prs/sglang/PR-6449.md), [[Feature] Support Flashinfer fp8 blockwise GEMM kernel on Blackwell](../sources/prs/sglang/PR-6479.md), [[CPU] [BF16] Call fused_experts_cpu, weight_packed_linear and bmm_cpu kernel in DeepSeek model](../sources/prs/sglang/PR-6641.md), [Correctly abort the failed grammar requests & Improve the handling of abort](../sources/prs/sglang/PR-6803.md), [feat: integrate deepgemm into EPMoE](../sources/prs/sglang/PR-6821.md), [CPU: map changes from developing branch in sgl-kernel](../sources/prs/sglang/PR-6833.md), [Fix AWQ Dequant and Weight Loading of deepseek v2](../sources/prs/sglang/PR-6842.md), [Use deepgemm instead of triton for fused_qkv_a_proj_with_mqa](../sources/prs/sglang/PR-6890.md), [[sgl-kernel] update deepgemm](../sources/prs/sglang/PR-6942.md), [Fix torchvision version for Blackwell](../sources/prs/sglang/PR-7015.md), [Clean up server_args.py](../sources/prs/sglang/PR-7037.md), [feat: update blackwell setup](../sources/prs/sglang/PR-7119.md), [Support new DeepGEMM](../sources/prs/sglang/PR-7172.md), [Tiny let DeepGEMM scale checks cover more cases](../sources/prs/sglang/PR-7182.md), [chore: upgrade sgl-kernel v0.1.8.post2](../sources/prs/sglang/PR-7186.md), [Fix a minor bug related to DeepGEMM upgrade](../sources/prs/sglang/PR-7191.md), [Fix error when disabling new DeepGEMM](../sources/prs/sglang/PR-7198.md), [[fix] fix DeepGEMM blackwell input quant & ut & fix style and log](../sources/prs/sglang/PR-7247.md), [Add CUTLASS FP8 Blockscale MoE kernel for Hopper architecture](../sources/prs/sglang/PR-7278.md), [Let EP prefill support new DeepGEMM](../sources/prs/sglang/PR-7310.md), [Kernels for efficient KV cache IO](../sources/prs/sglang/PR-7313.md), [fix: resolve blackwell deepep image issue](../sources/prs/sglang/PR-7331.md), [Quick fix for DeepGemm requant to also cover MTP.](../sources/prs/sglang/PR-7378.md), [[CMake] Fix sgl-kernel CMakeLists for Blackwell](../sources/prs/sglang/PR-7543.md), [Add dsv3 router gemm kernel](../sources/prs/sglang/PR-7627.md), [Add dsv3 fused a gemm to sgl-kernel](../sources/prs/sglang/PR-7630.md), [[Feature] Layer-wise Prefill](../sources/prs/sglang/PR-7634.md), [Apply dsv3_fused_a_gemm kernel](../sources/prs/sglang/PR-7635.md), [[Feature] CUDA Green Context Support](../sources/prs/sglang/PR-7649.md), [chore: upgrade flashinfer v0.2.7 jit](../sources/prs/sglang/PR-7663.md), [[1/n]: add cutlass W4A8 moe kernel for hopper architecture](../sources/prs/sglang/PR-7772.md), [[Fix][Ready]Fix register spilling in cutlass nvfp4 gemm kernel on Blackwell](../sources/prs/sglang/PR-8127.md), [[sgl-kernel] Opt per_token_quant_fp8 with warp reduce](../sources/prs/sglang/PR-8130.md), [[Perf] Tunings for SM100 FP8 CUTLASS kernel](../sources/prs/sglang/PR-8818.md), [chore: support blackwell cu129 image](../sources/prs/sglang/PR-8928.md), [[NVIDIA] Fix missing `get_col_major_tma_aligned_tensor` for Blackwell deepgemm in EpMoE](../sources/prs/sglang/PR-8955.md), [[fix] fix enable_pdl for blackwell](../sources/prs/sglang/PR-9011.md), [[sgl-kernel] Support FlashInfer top_k_top_p_sampling_from_logits](../sources/prs/sglang/PR-9060.md), [[NVIDIA] [3/N] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked ](../sources/prs/sglang/PR-9199.md), [[NVIDA] [1/N] Nvfp4 Masked Gemm: Add quant op for the flashinfer grouped gemm](../sources/prs/sglang/PR-9200.md), [[fix]: fix cutlass moe ut and and Opt H20 cutlass groupGemm performance](../sources/prs/sglang/PR-9272.md), [Fix FP4 inference corruption issue in glm4.5-air model](../sources/prs/sglang/PR-9346.md), [[sgl-kernel] feat: Support sm120 cutlass fp8 gemm kernel](../sources/prs/sglang/PR-9403.md), [fix: blackwell dsv3 fp8 issue temporary solution](../sources/prs/sglang/PR-9530.md), [[NVIDIA] [2/N] Optimize `silu_and_mul_scaled_fp4_grouped_quant` perf](../sources/prs/sglang/PR-9556.md), [Update CUTLASS 4.2 & Enable K-Major Scale Factor for SM90 FP8 Blockwise Group GEMM](../sources/prs/sglang/PR-9559.md), [Single Batch Overlap for MoE Models](../sources/prs/sglang/PR-9660.md), [[CPU] Add FP8 Bmm support](../sources/prs/sglang/PR-9744.md), [Make sm100 fp8 kernels available on sm103](../sources/prs/sglang/PR-9789.md), [Make fp4_quantize kernels work on sm103](../sources/prs/sglang/PR-9807.md), [perf: Avoid unnecessary data type conversions for DeepSeek-V3 on Blackwell](../sources/prs/sglang/PR-9834.md), [[Fix] DeepSeek EP accuracy issue on B200 GPUs](../sources/prs/sglang/PR-9946.md), [CUTLASS fp8 blockwise gemm support of sm120](../sources/prs/sglang/PR-9969.md), [[WIP] support more dtypes for tcgen05](../sources/prs/tilelang/PR-1229.md), [[Enhancement] add more dtype and fix mma.ws for fp16 for tcgen05](../sources/prs/tilelang/PR-1327.md), [[Feat] profiler support cudagraph backend](../sources/prs/tilelang/PR-1658.md), [Add swizzle layout detection and automatic merging for layout conflicts](../sources/prs/tilelang/PR-1736.md), [[Feature] Support tcgen5mma lowering for `.kind::i8`](../sources/prs/tilelang/PR-1764.md), [fix(intrinsics): add missing _legalize_to_buffer_region in SM70 emitter](../sources/prs/tilelang/PR-1786.md), [[CUDA] Support tcgen5mma gemm ts](../sources/prs/tilelang/PR-1866.md), [[Feature] Support cluster launch, query, synchronization and barrier operations](../sources/prs/tilelang/PR-1874.md), [[Feature] 2-SM support for TMA, TMEM and TCGEN5MMA on Blackwell](../sources/prs/tilelang/PR-1882.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [[Feature] Block-scaled GEMM support for MXFP8 on Blackwell](../sources/prs/tilelang/PR-1945.md), [[Bugfix] Fix CuTeDSL autotune cache invalid ELF header (#1967)](../sources/prs/tilelang/PR-1972.md), [[Transform] Add InjectTcgen05Fence pass](../sources/prs/tilelang/PR-2003.md), [[Backend] Refactor gemm_sp](../sources/prs/tilelang/PR-2048.md), [[CUDA] Support int4 `T.gemm`](../sources/prs/tilelang/PR-2063.md), [[CUDA] Improve int4 GEMM lowering and packed codegen support](../sources/prs/tilelang/PR-2073.md), [feat: auto-vectorize bf16/fp16 reduce with packed add2 intrinsics](../sources/prs/tilelang/PR-2112.md), [[CUDA][TMA] Add TMA tile::gather4 / tile::scatter4 support](../sources/prs/tilelang/PR-2129.md), [[codex] Split GEMM implementations by backend](../sources/prs/tilelang/PR-2153.md), [[CUDA] Add native SM75 MMA GEMM support for FP16, INT8 and INT4](../sources/prs/tilelang/PR-2198.md), [[TIR][IR] Update to use tirx](../sources/prs/tilelang/PR-2216.md), [[Python] Drop Python 3.9 support](../sources/prs/tilelang/PR-2218.md), [[Kernel] Update `cutlass_scaled_mm` to support 2d group (blockwise) scaling](../sources/prs/vllm/PR-11868.md), [[Build] Only build 9.0a for scaled_mm and sparse kernels](../sources/prs/vllm/PR-12339.md), [[Core][AMD] Migrate fully transparent sleep mode to ROCm platform](../sources/prs/vllm/PR-12695.md), [[Misc][Kernel]: Add GPTQAllSpark Quantization](../sources/prs/vllm/PR-12931.md), [[Kernel]Add streamK for block-quantized CUTLASS kernels](../sources/prs/vllm/PR-12978.md), [[NVIDIA] Support nvfp4 tcgen05 gemm](../sources/prs/vllm/PR-13571.md), [add tcgen05 support for tcgen05 fp8 gemm](../sources/prs/vllm/PR-13798.md), [[Kernel] CUTLASS grouped gemm fp8 MoE kernel](../sources/prs/vllm/PR-13972.md), [[Kernel] optimize performance of gptq marlin kernel when n is small](../sources/prs/vllm/PR-14138.md), [dynamic distpatch of fp8 kernels](../sources/prs/vllm/PR-14245.md), [fix minor miscalled method](../sources/prs/vllm/PR-14327.md), [Add cutlass support for blackwell fp8 blockwise gemm](../sources/prs/vllm/PR-14383.md), [[BugFix] Illegal Memory Access in the blockwise cutlass fp8 GEMMs](../sources/prs/vllm/PR-14396.md), [permute/unpermute kernel for moe optimization](../sources/prs/vllm/PR-14568.md), [[BugFix/Build] Fix sparse kernels not getting built on hopper](../sources/prs/vllm/PR-14572.md), [[Quantization][FP8] Adding support for fp8 gemm layer input in fp8](../sources/prs/vllm/PR-14578.md), [[Kernel] allow non-contiguous input for marlin kernel](../sources/prs/vllm/PR-14658.md), [[Model] Add support for Gemma 3](../sources/prs/vllm/PR-14660.md), [[Bugfix][Kernel][CPU] Fix num_tokens in CPU rotary embedding kernel](../sources/prs/vllm/PR-14667.md), [[FEAT] [ROCm]: Add AITER Block-Scaled GEMM Feature](../sources/prs/vllm/PR-14968.md), [[FEAT] [ROCm] Add AITER int8 scaled gemm kernel](../sources/prs/vllm/PR-15433.md), [[Quantization] Fp8 Channelwise Dynamic Per Token GroupedGEMM](../sources/prs/vllm/PR-15587.md), [Modularize fused experts and integrate PPLX kernels](../sources/prs/vllm/PR-15956.md), [[Perf]Optimize rotary_emb implementation to use Triton operator for improved inference performance](../sources/prs/vllm/PR-16457.md), [[Kernel] some optimizations for dense marlin and moe marlin](../sources/prs/vllm/PR-16850.md), [[Bugfix] gemma[2,3] interleaved attention when sliding window is disabled](../sources/prs/vllm/PR-17180.md), [[Kernel] fp4 marlin kernel](../sources/prs/vllm/PR-17687.md), [[Kernel] Have rotary embeddings support tensors](../sources/prs/vllm/PR-18046.md), [[Kernel] Integrate CUTLASS MoE kernel with PPLX](../sources/prs/vllm/PR-18762.md), [[Kernel] Enable fp8 support for pplx and BatchedTritonExperts.](../sources/prs/vllm/PR-18864.md), [[Kernel] Support deep_gemm for linear methods](../sources/prs/vllm/PR-19085.md), [[V1] Use FlashInfer by default on Blackwell GPUs](../sources/prs/vllm/PR-19118.md), [[Kernels] Add activation chunking logic to FusedMoEModularKernel](../sources/prs/vllm/PR-19168.md), [[Bugfix] Don't attempt to use triton if no driver is active](../sources/prs/vllm/PR-19561.md), [[Perf] Further tunings for SM100 FP8 CUTLASS kernel](../sources/prs/vllm/PR-19566.md), [[feat]: CUTLASS block scaled group gemm for SM100](../sources/prs/vllm/PR-19757.md), [Fix FA2 fallback for Blackwell V1](../sources/prs/vllm/PR-19781.md), [[Feature] Integrate new deepgemm](../sources/prs/vllm/PR-19820.md), [ [Feature] Integrate SM100 DeepGEMM support](../sources/prs/vllm/PR-20087.md), [[Nixl] Heterogeneous TP support FlashInfer](../sources/prs/vllm/PR-20189.md), [[Misc] DP : Add ExpertTokensMetadata](../sources/prs/vllm/PR-20332.md), [[Kernel] SM90 CUTLASS FP8 GEMM: add support for swap AB + kernel tuning](../sources/prs/vllm/PR-20396.md), [[feat]: add SM100 support for cutlass FP8 groupGEMM](../sources/prs/vllm/PR-20447.md), [[feat] enable SM100 CUTLASS block scaled group gemm for smaller batch sizes](../sources/prs/vllm/PR-20640.md), [[Performance] Performance improvements in non-blockwise fp8 CUTLASS MoE](../sources/prs/vllm/PR-20762.md), [[fix]: disable cutlass block scaled group gemm for EP](../sources/prs/vllm/PR-20781.md), [[Bug] Fix DeepGemm for EP low latency case](../sources/prs/vllm/PR-20833.md), [[Perf] Use Triton instead of Torch for DeepGEMM Per Token Group Quant](../sources/prs/vllm/PR-20841.md), [[Kernel] DeepGemm MoE : Integrate triton permute / unpermute kernels ](../sources/prs/vllm/PR-20903.md), [[Perf] Add swap_ab to SM90 FP8 non-block CUTLASS moe grouped gemm](../sources/prs/vllm/PR-20911.md), [[Model] Pooling models default to using chunked prefill & prefix caching if supported.](../sources/prs/vllm/PR-20930.md), [[Bugfix] Switch bailout logic for kv-cache-dtype with SM100 Flashinfer](../sources/prs/vllm/PR-20934.md), [[Bugfix] Fix Mistral3 support on SM100/SM120](../sources/prs/vllm/PR-20998.md), [[Perf] Use FlashInfer RoPE for RotaryEmbedding.forward_cuda when available](../sources/prs/vllm/PR-21126.md), [[Bug] DeepGemm: Fix TypeError: per_block_cast_to_fp8() missing 1 required positional argument: 'use_ue8m0' for SM100](../sources/prs/vllm/PR-21187.md), [Override attention metadata for fast prefill in some KV sharing setups](../sources/prs/vllm/PR-21590.md), [update flashinfer to v0.2.9rc2](../sources/prs/vllm/PR-21701.md), [[Logs] Change flashinfer sampler logs to once](../sources/prs/vllm/PR-21759.md), [[bugfix] fix blackwell deepep installation](../sources/prs/vllm/PR-22255.md), [[Bugfix] Fix 3D input passed into cutlass_scaled_mm](../sources/prs/vllm/PR-22278.md), [Update `flashinfer-python==0.2.10`](../sources/prs/vllm/PR-22389.md), [[Bug] Fix B200 DeepGEMM E8M0 Accuracy Issue](../sources/prs/vllm/PR-22399.md), [Upgrade FlashInfer to v0.2.11](../sources/prs/vllm/PR-22613.md), [[Bugfix] Fix default enable for CUTLASS MLA on SM100](../sources/prs/vllm/PR-22738.md), [[Kernel] Added flashinfer fp8 per-tensor gemms](../sources/prs/vllm/PR-22895.md), [[Kernel] CUTLASS MoE FP8: Integrate cuda moe permute/unpermute](../sources/prs/vllm/PR-23045.md), [[Perf] Small optimizations for silu_mul_fp8_quant_deep_gemm](../sources/prs/vllm/PR-23265.md), [[Perf] Use upstream CUTLASS for SM90 Block FP8 kernel](../sources/prs/vllm/PR-23280.md), [[Feature] Add Hopper DeepGEMM E8M0 for DeepSeekV3.1 scale_fmt](../sources/prs/vllm/PR-23666.md), [[Performance] Move apply_w8a8_block_fp8_linear to an op class](../sources/prs/vllm/PR-24666.md), [[Kernel][Quantization] add w4a8 support for marlin kernel](../sources/prs/vllm/PR-24722.md), [Enable Fbgemm NVFP4 on Dense models](../sources/prs/vllm/PR-25609.md), [Update launch_bounds_utils.h for correct compile on Multiple Cuda Arch - PTXAS out of range Warning](../sources/prs/vllm/PR-25843.md), [[MoE] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked](../sources/prs/vllm/PR-25990.md), [[NVIDIA] Add support for cudnn fp4 gemm via flashinfer](../sources/prs/vllm/PR-26107.md), [[Bug] Fix Shape Validation for Fallback while Enabling E8M0 for DeepGEMM](../sources/prs/vllm/PR-26322.md), [Disable FlashInfer sampler by default](../sources/prs/vllm/PR-26859.md), [[Feature] Batch Invariant: Support DeepGEMM and Blackwell](../sources/prs/vllm/PR-27127.md), [[Feature] Batch Invariant for R1 TP 8 on Blackwell](../sources/prs/vllm/PR-27229.md), [[Perf] SM100 - add swap AB optimization to CUTLASS FP8 GEMM](../sources/prs/vllm/PR-27284.md), [[Attention] Use sparse prefill kernel for fp8 kv-cache in DeepSeek-v3.2](../sources/prs/vllm/PR-27532.md), [[Feature] Extend batch invariant torch.compile to B200](../sources/prs/vllm/PR-27856.md), [[Performance] Fused blockwise quant RMS norm](../sources/prs/vllm/PR-27883.md), [[Performance][B200] Fix deepgemm prologue](../sources/prs/vllm/PR-27897.md), [[Kernel] Optimize rms_norm kernel](../sources/prs/vllm/PR-27931.md), [[flashinfer][fix] do not check nvcc availability when using pre-downloaded cubins](../sources/prs/vllm/PR-27990.md), [[Performance][B200] silu_mul_quant: pack scales in int32](../sources/prs/vllm/PR-28358.md), [[Bugfix][Nixl] Fix kernel physical<>logical block_size issue ](../sources/prs/vllm/PR-28677.md), [[Performance] Reduce DeepGEMM N dim restriction from 128 to 64 multiplier ](../sources/prs/vllm/PR-28687.md), [[Perf][Kernels] Enable FlashInfer DeepGEMM swapAB on SM90 (for W8A8 Linear Op)](../sources/prs/vllm/PR-29213.md), [[Perf] Disable DeepGEMM MoE by default when TP=8 is used](../sources/prs/vllm/PR-29346.md), [[Bugfix] Defunctionalize TRTLLM AR+Norm op for avoiding extra clone kernel before it](../sources/prs/vllm/PR-29631.md), [[Kernel]Support W4A8 Grouped GEMM on Hopper](../sources/prs/vllm/PR-29691.md), [[Bugfix] Fix mismatched nvfp4 gemm output shape](../sources/prs/vllm/PR-29742.md), [[Bugfix] Fix flashinfer ar+norm kernel not available issue](../sources/prs/vllm/PR-29960.md), [[Bugfix] Fix DeepGEMM after #29546 ](../sources/prs/vllm/PR-30267.md), [[Bugfix] Fix fp8 DeepGemm compilation issues](../sources/prs/vllm/PR-30336.md), [[BugFix] Fix `AttributeError: 'MergedColumnParallelLinear' object has no attribute 'weight_scale'`](../sources/prs/vllm/PR-30399.md), [[Feature] Add SM103 (Blackwell Ultra) Support to vLLM](../sources/prs/vllm/PR-30484.md), [[Perf] enable flashinfer rotary_embedding custom ops in DeepSeek rotary](../sources/prs/vllm/PR-30729.md), [ [FIX] Always support TP > 4 for FP4 Gemm](../sources/prs/vllm/PR-31099.md), [[Bugfix][Hardware][AMD] Consolidate FP8 min/max values helper function](../sources/prs/vllm/PR-31106.md), [[BugFix] add select_gemm_impl on CompressedTensorsWNA16MoEMethod to support LoRA](../sources/prs/vllm/PR-31453.md), [[FIX] Add NO_MUL activation support for modular kernel path](../sources/prs/vllm/PR-31528.md), [[Perf] Add opt-in SM100 Oink RMSNorm custom-op path](../sources/prs/vllm/PR-31828.md), [[1/N][Attention] Restructure attention: move files](../sources/prs/vllm/PR-31916.md), [[5/N][Attention] Finish eliminating `vllm/attention` folder](../sources/prs/vllm/PR-32064.md), [[BugFix] Fix DeepSeek-V3.1 + DeepGEMM incompatible scale shapes](../sources/prs/vllm/PR-32361.md), [[Perf] Create TMA-aligned input scale tensor for DeepGemm on Hopper](../sources/prs/vllm/PR-32619.md), [[Performance] Tune Mamba selective scan kernel for B200](../sources/prs/vllm/PR-32873.md), [[Feature] Support CPU Offloading without Pytorch Pinned Memory that leads to doubled allocation](../sources/prs/vllm/PR-32993.md), [[Kernel] Apply 256bit LDG/STG To Activation Kernels](../sources/prs/vllm/PR-33022.md), [[Bugfix] Fix quant RMS norm fusion for quantization with TMA-aligned scales](../sources/prs/vllm/PR-33255.md), [[Bugfix] Register fp8 cutlass_group_gemm as supported for only SM90+SM100](../sources/prs/vllm/PR-33285.md), [[Feature][Core] Support Fabric detection to adapt the MNNVL protocol for the GB series](../sources/prs/vllm/PR-33540.md), [[Perf] Disable clean_logits in deepgemm fp8_mqa_logits kernel](../sources/prs/vllm/PR-33568.md), [[ModelBash][DSV3] Add TRTLLM DSV3 Router GEMM kernel (6% B1 Speedup)](../sources/prs/vllm/PR-34302.md), [[Bugfix] Enforce DeepGEMM when using sparse_attn_indexer on CUDA](../sources/prs/vllm/PR-34374.md), [[Model Bash] DeepSeek R1 BF16 Min Latency QKV A GEMM (0.5% E2E Speedup)](../sources/prs/vllm/PR-34758.md), [[Bugfix] Gate 256-bit instructions to CUDA 12.9+](../sources/prs/vllm/PR-34791.md), [[Perf] Enable FlashInfer DeepGEMM swapAB on SM90 by default](../sources/prs/vllm/PR-34924.md), [[Bug][DSV3.2] Always prepare metadata for DeepGEMM Sparse Attention](../sources/prs/vllm/PR-35075.md), [[Performance] Cublas Bf16 Gate with Fp32 Output](../sources/prs/vllm/PR-35121.md), [[Bugfix] Fix DSV3 kernels breaking _C and _moe_C on unsupported arches](../sources/prs/vllm/PR-35123.md), [[Feat] Add CUDA torch fallbacks for fp8_mqa_logits/fp8_paged_mqa_logits_torch function](../sources/prs/vllm/PR-35271.md), [[MoE][Perf] Wrap DSV3 QKVAProj GEMM in custom op for torch.compile](../sources/prs/vllm/PR-35751.md), [[Mamba] Add stochastic rounding support](../sources/prs/vllm/PR-35753.md), [[MoE] Move PF Methods to Folder](../sources/prs/vllm/PR-35927.md), [[Kernel] Add FlashInfer MoE A2A Kernel](../sources/prs/vllm/PR-36022.md), [[Bugfix] Warm up Triton autotuner for GDN layers during V1 profiling](../sources/prs/vllm/PR-36599.md), [[GDN] add a config for gdn kernel selection](../sources/prs/vllm/PR-36647.md), [[Bug] Fix FlashInfer MNNVL socket collisions under concurrent vLLM jobs](../sources/prs/vllm/PR-36674.md), [Update Flashinfer to 0.6.6](../sources/prs/vllm/PR-36768.md), [[Bugfix] Fix FlashInfer GDN warmup ValueError on SM90 GPUs](../sources/prs/vllm/PR-36876.md), [[Kernel] Add gpt-oss Router GEMM kernel](../sources/prs/vllm/PR-37205.md), [[UX] Add flashinfer-cubin as CUDA default dep](../sources/prs/vllm/PR-37233.md), [refactor: abstract deepgemm support into platform](../sources/prs/vllm/PR-37519.md), [[Bug] Fix fp8 deepgemm batch invariant](../sources/prs/vllm/PR-37718.md), [[Perf] triton bilinear_pos_embed kernel for ViT](../sources/prs/vllm/PR-37948.md), [[Kernel] Optimize SM120 CUTLASS blockwise FP8 GEMM](../sources/prs/vllm/PR-37970.md), [[Bugfix] Fix DeepGemm E8M0 accuracy degradation for Qwen3.5 FP8 on Blackwell](../sources/prs/vllm/PR-38083.md), [[Kernel] Add swapAB support for SM120 CUTLASS blockwise FP8 GEMM ](../sources/prs/vllm/PR-38325.md), [[CI Bugfix] Pre-download missing FlashInfer headers in Docker build](../sources/prs/vllm/PR-38391.md), [[Bugfix] Enable batch-invariant Triton matmul on all Ampere GPUs (SM 8x) ](../sources/prs/vllm/PR-38427.md), [[Perf] Batch KV cache swap copies via cuMemcpyBatchAsync](../sources/prs/vllm/PR-38460.md), [[Gemma4] Enable Fast Prefill Optimization](../sources/prs/vllm/PR-38879.md), [[Perf] Change Trtllm fp8 MoE to use Shuffled Weights and BlockMajorK Layout](../sources/prs/vllm/PR-38993.md), [[Gemma4] Support quantized MoE ](../sources/prs/vllm/PR-39045.md), [[Refactor] Move NVFP4 GEMM management into NvFp4LinearKernel](../sources/prs/vllm/PR-39129.md), [[Refactor] Move MXFP8 GEMM management into MxFp8LinearKernel](../sources/prs/vllm/PR-39205.md), [Use CU_MEMCPY_SRC_ACCESS_ORDER_ANY for batch KV cache swaps](../sources/prs/vllm/PR-39306.md), [Fix NUMA binding on non-CDMM Grace-Blackwell systems](../sources/prs/vllm/PR-39361.md), [[Perf] Fuse Zero Initializer for FP8 DeepGemm Block Quant Kernel](../sources/prs/vllm/PR-39547.md), [[Bugfix][NIXL] Fix `_logical_to_kernel_block_ids` conversion for non-mamba models](../sources/prs/vllm/PR-39724.md), [[Bugfix] Add Marlin kernel in block scaled mm kernel selection.](../sources/prs/vllm/PR-40105.md), [[MoE] Move cutlass moe to fused_moe/experts/](../sources/prs/vllm/PR-40574.md), [[Kernel][MoE] Support GELU on TRT-LLM NvFP4 fused MoE for Gemma4](../sources/prs/vllm/PR-41050.md), [[DSV4] Fuse norm and router for low latency scenario](../sources/prs/vllm/PR-41263.md), [[Spec Decode] Add Gemma4 MTP speculative decoding support](../sources/prs/vllm/PR-41745.md), [Add NVFP4 all-gather GEMM fusion for AsyncTP](../sources/prs/vllm/PR-41882.md), [[CPU] Add MXFP4 W4A16 MoE support](../sources/prs/vllm/PR-41922.md), [[MoE] Move various experts classes to fused_moe/experts/](../sources/prs/vllm/PR-41979.md), [[Bugfix] Add swiglu limits to deepgemm fp8 methods](../sources/prs/vllm/PR-41986.md), [[6/n] Migrate activation kernels, gptq, gguf, non cutlass w8a8 to libtorch stable ABI (continued)](../sources/prs/vllm/PR-42663.md), [[Kernel] (1/N) Machete - Hopper Optimized Mixed Precision Linear Kernel ](../sources/prs/vllm/PR-7174.md), [[Kernel] (2/N) Machete - Integrate into CompressedTensorsWNA16 and GPTQMarlin](../sources/prs/vllm/PR-7701.md), [DeepGEMM — FP8 GEMM with Fine-Grained Scaling](../wiki/kernels/deepgemm.md), [FP8 Block-Scale GEMM](../wiki/kernels/fp8-block-scale-gemm.md), [Gated Dual GEMM (Gate-Up + SwiGLU Fusion)](../wiki/kernels/gated-dual-gemm.md), [Grouped GEMM for MoE](../wiki/kernels/grouped-gemm.md), [NVFP4 GEMM — 4-bit Floating Point Matrix Multiply](../wiki/kernels/nvfp4-gemm.md), [TensorRT-LLM Blackwell FP4 DSA Indexer](../wiki/kernels/tensorrt-llm-blackwell-indexer.md) | -| `gemv` | [Twelve Attempts at NVFP4 Batched GEMV](../sources/blogs/amandeep-nvfp4-attempts.md), [NVFP4 GEMV and Improved NVFP4 GEMV](../sources/blogs/simon-nvfp4-gemv.md), [Blackwell NVFP4 Kernel Hackathon Journey](../sources/blogs/yue-nvfp4-hackathon.md), [GPU Mode NVFP4 Hackathon - Problem 1: Batched GEMV](../sources/contests/gpu-mode-nvfp4/problem-1-gemv.md), [[None][feat] sm100 weight-only kernel](../sources/prs/TensorRT-LLM/PR-10190.md), [[None][feat] Enable nvfp4 cuda core for sm120](../sources/prs/TensorRT-LLM/PR-8620.md), [NVFP4 Batched GEMV](../wiki/kernels/nvfp4-gemv.md) | -| `grouped-gemm` | [Anatomy of a Reward Hack](../sources/blogs/gpu-mode-reward-hack.md), [TFLOPS Gap: Why FP4 MoE Kernel Engineering Matters on Blackwell](../sources/blogs/tflops-gap-fp4-moe.md), [FlashInfer MLSys 2026 - Track A: Fused MoE FP8](../sources/contests/flashinfer-mlsys26/track-a-fused-moe.md), [GPU Mode NVFP4 Hackathon - Problem 4: Grouped GEMM](../sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [fix: use SM90ArchSpec instead of SM100ArchSpec in sm90_bf16_k_grouped_gemm](../sources/prs/DeepGEMM/PR-270.md), [[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes](../sources/prs/DeepGEMM/PR-304.md), [ Solving bank conflict via padding and TMA 3D store](../sources/prs/DeepGEMM/PR-78.md), [Use 1D TMA store instead of 3D](../sources/prs/DeepGEMM/PR-83.md), [Use swizzling instead of padding](../sources/prs/DeepGEMM/PR-86.md), [Support TMA multicast on B with m_grouped_gemm_contiguous.](../sources/prs/DeepGEMM/PR-88.md), [[None][perf] Add more optimization options for MOE CuteDSL finalized kernel](../sources/prs/TensorRT-LLM/PR-10042.md), [[TRTLLM-9992][perf] Enable PDL for CuteDSL kernels and overlap MoeOutputMemset](../sources/prs/TensorRT-LLM/PR-10043.md), [[None][feat] CuteDSL MOE FC1 Enhancement](../sources/prs/TensorRT-LLM/PR-10088.md), [[TRTLLM-9831][perf] Enable 2CTA with autotune for CuteDSL MoE and Grouped GEMM optimizations](../sources/prs/TensorRT-LLM/PR-10201.md), [[None] [feat] Add test script and raster M for gather fc1 kernel](../sources/prs/TensorRT-LLM/PR-10429.md), [[TRTLLM-9831][perf] Use TMA.RED to improve effective memory bandwidth](../sources/prs/TensorRT-LLM/PR-10987.md), [[None][feat] CuteDSL MOE: Add raster along M/N support for blockscaled contiguous backbone kernel](../sources/prs/TensorRT-LLM/PR-12079.md), [[None][feat] Add DWDP (Distributed Weight Data Parallelism) support for MoE inference](../sources/prs/TensorRT-LLM/PR-12136.md), [[TRTLLM-11585][feat] Add CUTEDSL moe backend for nemotron-h](../sources/prs/TensorRT-LLM/PR-12884.md), [[TRTLLM-9685] [feat] Add gather fc1 kernel by cuteDSL](../sources/prs/TensorRT-LLM/PR-9618.md), [Hopper Grouped GEMM support for FP8 Accum](../sources/prs/cutlass/PR-2123.md), [Blockwise and Groupwise GEMM for Blackwell and Improvements for Hopper](../sources/prs/cutlass/PR-2139.md), [Support for GEMM-K=0 for Blackwell Grouped GEMMs](../sources/prs/cutlass/PR-2746.md), [[Hopper CuTeDSL] Add grouped GEMM kernel example](../sources/prs/cutlass/PR-3091.md), [Support for Group GEMM in CUTLASS Profiler for GeForce and Spark](../sources/prs/cutlass/PR-3092.md), [perf: accelerate blackwell grouped gemm](../sources/prs/flashinfer/PR-1086.md), [feat: Support MXFP8 x MXFP4 CUTLASS grouped GEMM](../sources/prs/flashinfer/PR-1241.md), [perf: Enable SplitK and fix tile-scheduling for moe fp4 fused moe](../sources/prs/flashinfer/PR-1548.md), [TVM: support TVM binding for GroupedGemm](../sources/prs/flashinfer/PR-1725.md), [feat: unit-test and api change, w4a8 grouped-gemm fused MoE for SM90](../sources/prs/flashinfer/PR-2193.md), [feat: cuteDSL fp4 moe for better DSR1 performance.](../sources/prs/flashinfer/PR-2398.md), [refactor: Port upstream CUTLASS fixes and refactor grouped_gemm_nt_masked GEMM module location](../sources/prs/flashinfer/PR-2503.md), [Support for MXFP4 and NVFP4 group GEMMs on GeForce and Spark](../sources/prs/flashinfer/PR-2738.md), [CuteDSL MoE fix redundant output buffer zeroing](../sources/prs/flashinfer/PR-2811.md), [feat: Add CuTe DSL grouped-gemm + combine fusion support](../sources/prs/flashinfer/PR-2944.md), [[sgl-kernel][1/N]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-11432.md), [[sgl-kernel][6/7]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-15471.md), [[Feature] Apply Cublas Grouped Gemm kernel](../sources/prs/sglang/PR-3629.md), [DeepGemm integrate to gemm](../sources/prs/sglang/PR-4165.md), [support cmake for sgl-kernel](../sources/prs/sglang/PR-4706.md), [[perf] introduce deep gemm group_gemm_masked as gemm](../sources/prs/sglang/PR-5432.md), [[NVIDIA] [3/N] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked ](../sources/prs/sglang/PR-9199.md), [[Kernel] CUTLASS grouped gemm fp8 MoE kernel](../sources/prs/vllm/PR-13972.md), [permute/unpermute kernel for moe optimization](../sources/prs/vllm/PR-14568.md), [[Kernel] Integrate CUTLASS MoE kernel with PPLX](../sources/prs/vllm/PR-18762.md), [[feat]: CUTLASS block scaled group gemm for SM100](../sources/prs/vllm/PR-19757.md), [[Performance] Performance improvements in non-blockwise fp8 CUTLASS MoE](../sources/prs/vllm/PR-20762.md), [[Kernel] CUTLASS MoE FP8: Integrate cuda moe permute/unpermute](../sources/prs/vllm/PR-23045.md), [[Kernel][tcgen05] nvfp4 fused tcgen05 moe](../sources/prs/vllm/PR-23696.md), [[MoE] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked](../sources/prs/vllm/PR-25990.md), [[MoE] Move cutlass moe to fused_moe/experts/](../sources/prs/vllm/PR-40574.md), [DeepGEMM — FP8 GEMM with Fine-Grained Scaling](../wiki/kernels/deepgemm.md), [Fused MoE — FP8 Block-Scale Routing + Dual GEMM](../wiki/kernels/fused-moe.md), [Grouped GEMM for MoE](../wiki/kernels/grouped-gemm.md) | -| `linear-attention` | [Gated Delta Networks](../sources/blogs/gated-delta-net.md), [Qwen3-Next: Hybrid GDN+MoE Architecture on NVIDIA Blackwell](../sources/blogs/qwen3-next-architecture.md), [FlashInfer MLSys 2026 - Track C: Gated Delta Net](../sources/contests/flashinfer-mlsys26/track-c-gated-delta-net.md), [Tiled Flash Linear Attention (TFLA)](../sources/docs/tfla.md), [Gated Delta Net — Linear Attention](../wiki/kernels/gated-delta-net.md) | -| `mla` | [FlashMLA — Multi-head Latent Attention](../sources/blogs/flashmla.md), [K-Search: LLM Kernel Generation via Co-Evolving Intrinsic World Model](../sources/blogs/k-search-kernel-generation.md), [DeepSeek-V3.2-Exp in vLLM: Fine-Grained Sparse Attention in Action](../sources/blogs/vllm-deepseek-v3-sparse-attention.md), [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [[#11694][feat] AutoDeploy: Improve the piecewise CG memory usage](../sources/prs/TensorRT-LLM/PR-11993.md), [[TRTLLM-11285][feat] Fuse indexer wk + weights_proj into single GEMM in TF32 for DS-V3.2](../sources/prs/TensorRT-LLM/PR-12055.md), [[https://nvbugs/5983390][perf] Split MLA DSA custom op for piecewise CUDA graph capture](../sources/prs/TensorRT-LLM/PR-12503.md), [[#12634][feat] AutoDeploy: Support rank 256 MLA in flashinfer_mla](../sources/prs/TensorRT-LLM/PR-12519.md), [[#13580][fix] AutoDeploy: Support Gemma3n/4 E2B variants](../sources/prs/TensorRT-LLM/PR-13630.md), [[None][feat] Add DeepSeekV4 attention kernels](../sources/prs/TensorRT-LLM/PR-13652.md), [[None][fix] Add SPDX Apache-2.0 headers to auto_deploy test files](../sources/prs/TensorRT-LLM/PR-14193.md), [[TRTLLM-8535][feat] Support DeepSeek V3.2 with FP8 + BF16 KV cache/NVFP4 + BF16 KV cache](../sources/prs/TensorRT-LLM/PR-8405.md), [Flash MLA support](../sources/prs/cutlass/PR-2130.md), [Flash MLA Support - Step 2](../sources/prs/cutlass/PR-2134.md), [[ex77] fix mla split; add fwd lse; add bwd varlen](../sources/prs/cutlass/PR-2366.md), [Example 77 add blackwell flash-attention bwd for MLA shape](../sources/prs/cutlass/PR-2466.md), [Add Blackwell MLA forward (shape: d=192, dv=128) implementation](../sources/prs/cutlass/PR-2472.md), [[Cute,Sm100,Fwd] add MLA 64/512 with topk sparsity for MQA 128 heads](../sources/prs/flash-attention/PR-2441.md), [bugfix: import wrapper of mla decode](../sources/prs/flashinfer/PR-1013.md), [misc: fix instrument code for mla profiler](../sources/prs/flashinfer/PR-1014.md), [[nvidia] initial support for blackwell kernels](../sources/prs/flashinfer/PR-1039.md), [bugfix: temporally disable split-kv in blackwell mla](../sources/prs/flashinfer/PR-1055.md), [[Feature] Support PDL for batch Prefill and Decode](../sources/prs/flashinfer/PR-1117.md), [feat: add trtllm-gen mla cubin](../sources/prs/flashinfer/PR-1222.md), [feat: enable trtllm-gen mla MTP](../sources/prs/flashinfer/PR-1258.md), [refactor: refactor trtllm-gen attention kernel integration code](../sources/prs/flashinfer/PR-1289.md), [Refactor Fused Moe Module](../sources/prs/flashinfer/PR-1309.md), [minor: add trtllm_gen_mla benchmark](../sources/prs/flashinfer/PR-1316.md), [feat: Fused rope fp8 quantize kernel for MLA](../sources/prs/flashinfer/PR-1339.md), [fix: fix trtllm-gen mla error on new interface](../sources/prs/flashinfer/PR-1348.md), [feat: enable trtllm-gen attn speculative decoding verify by decode](../sources/prs/flashinfer/PR-1453.md), [feat: add pdl for trtllm-gen attn](../sources/prs/flashinfer/PR-1484.md), [feat: Support fp8 qkv, fp16/bf16 out MHA for trtllm-gen.](../sources/prs/flashinfer/PR-1490.md), [feat: Add fp8-qkv, fp16/bf16 output MHA](../sources/prs/flashinfer/PR-1540.md), [feat: initial support for SM103, SM110, SM120, SM121](../sources/prs/flashinfer/PR-1608.md), [bugfix: Fix FLOPS calculation for bench_trtllm_gen_mla.py](../sources/prs/flashinfer/PR-1640.md), [fix: zero-init workspace buffer for trtllm-gen fmha](../sources/prs/flashinfer/PR-1643.md), [Add benchmark for MLARopeQuantize](../sources/prs/flashinfer/PR-1656.md), [test: skip the unsupported test cases for sm120/121](../sources/prs/flashinfer/PR-1710.md), [feat: add xqa fp8 mha and fp8 kv cache](../sources/prs/flashinfer/PR-1769.md), [Waive / disable test_mla_decode_kernel.py::test_mla_decode_kernel for not sm80 ](../sources/prs/flashinfer/PR-1771.md), [MLA RoPE + quantization fused kernel: shape generalization for MHA / GQA](../sources/prs/flashinfer/PR-1924.md), [minor fix for xqa](../sources/prs/flashinfer/PR-1994.md), [feat: add xqa backend and completes NHD/HND coverage for trtllm-gen/xqa backend](../sources/prs/flashinfer/PR-2001.md), [use scalar for kv_scale in xqa](../sources/prs/flashinfer/PR-2033.md), [feat: Add flashinfer.rope.rope_quantize_fp8_append_paged_kv_cache (fused RoPE + Q + KV cache, supports MLA/GQA/MHA) ](../sources/prs/flashinfer/PR-2037.md), [feat: add xqa mla backend](../sources/prs/flashinfer/PR-2053.md), [[API change] Allow using torch.Tensor for scales for trtllm-gen attention](../sources/prs/flashinfer/PR-2084.md), [[DSR1] Added MLA test](../sources/prs/flashinfer/PR-2100.md), [add tensor scale input for xqa](../sources/prs/flashinfer/PR-2110.md), [feature: make the LSE returned by MLA support base 2 or e #2113](../sources/prs/flashinfer/PR-2114.md), [update xqa license](../sources/prs/flashinfer/PR-2117.md), [feat: support variable sequence length in decode kernel of trtllm-gen attention](../sources/prs/flashinfer/PR-2125.md), [fix flaky xqa test](../sources/prs/flashinfer/PR-2126.md), [feat: add trtllm-gen per-tensor sparseMla kernels.](../sources/prs/flashinfer/PR-2138.md), [refactor: Move mla code from decode.py to mla.py and add to documentation](../sources/prs/flashinfer/PR-2163.md), [[feat] Integrate SGLang concat_mla_k kernel into flashinfer](../sources/prs/flashinfer/PR-2237.md), [Support both 3D and 4D kv_cache shapes in MLA APIs](../sources/prs/flashinfer/PR-2334.md), [feat: Add TRTLLM-Gen Skip-Softmax kernels for prefill and decode](../sources/prs/flashinfer/PR-2477.md), [fix: Fix memory bandwidth calculation in MLA benchmarks](../sources/prs/flashinfer/PR-2479.md), [feat: Enable TRTLLM-Gen Skip-Softmax attention for MLA](../sources/prs/flashinfer/PR-2547.md), [support qk_nope_head_dim for 192 check for GLM-5](../sources/prs/flashinfer/PR-2607.md), [fix: add SM121 support to SM120 version guards](../sources/prs/flashinfer/PR-2631.md), [feat: add support for more MLA head dimensions](../sources/prs/flashinfer/PR-2677.md), [Add NVFP4 KV cache quantization support for SM100](../sources/prs/flashinfer/PR-2702.md), [Add cute dsl mla decode op](../sources/prs/flashinfer/PR-2743.md), [[Spark unit test debugging] Fix for tests/attention/test_trtllm_gen_mla.py](../sources/prs/flashinfer/PR-2750.md), [feat: Expose TRT-LLM FMHA style paged KV Cache and page table layout](../sources/prs/flashinfer/PR-2770.md), [feat: FP8 output support for CUTLASS MLA paged attention](../sources/prs/flashinfer/PR-2779.md), [[CuTe DSL] Add modular FMHA prefill and MLA decode attention kernels](../sources/prs/flashinfer/PR-2805.md), [[Fmha] Sparse MLA decode kernel selection heuristics](../sources/prs/flashinfer/PR-2836.md), [[fix] bugfix 2856: Fix pre-allocated out shape check in trtllm_batch_decode_with_kv_cache_mla for q_len_per_req > 1](../sources/prs/flashinfer/PR-2876.md), [feat: add pdl support for cute dsl mla decode kernel support](../sources/prs/flashinfer/PR-2901.md), [Support lse in trtllm paged attn kernels](../sources/prs/flashinfer/PR-3058.md), [feat: Enable FP8 (E4M3/E5M2) in concat_mla_k for optimize long-context prefill performance and refactor type dispatch for BF16/FP16](../sources/prs/flashinfer/PR-3129.md), [feat(trace): embed runnable init() in every TraceTemplate](../sources/prs/flashinfer/PR-3221.md), [Support Kimi K2.5 H64 CuTe DSL MLA decode](../sources/prs/flashinfer/PR-3235.md), [bugfix: MLA decode should multiply sm_scale by math::log2e](../sources/prs/flashinfer/PR-787.md), [fix rope logic in mla decoding](../sources/prs/flashinfer/PR-793.md), [perf: memory efficient deepseek mla fused page-attention kernel](../sources/prs/flashinfer/PR-804.md), [bugfix: mla page-attention kernel for different page sizes](../sources/prs/flashinfer/PR-810.md), [feat: unlocking MLA for A100](../sources/prs/flashinfer/PR-812.md), [feat: unlock MLA attention for sm89 (L40/L40s/4090)](../sources/prs/flashinfer/PR-814.md), [bugfix: fix the behavior of mla plan function when provided with host tensors](../sources/prs/flashinfer/PR-816.md), [bugfix: bugfix on sm89 MLA](../sources/prs/flashinfer/PR-821.md), [perf: MLA decode kernel implemented by CuTe targeted to SM80](../sources/prs/flashinfer/PR-844.md), [misc: Remove duplicate param set in MLA kernel](../sources/prs/flashinfer/PR-850.md), [unittest: add MLA test cases where kv_len is evenly divided by page_size.](../sources/prs/flashinfer/PR-861.md), [perf: dynamic split-k for MLA](../sources/prs/flashinfer/PR-863.md), [bugfix: fix the behavior of MLA kernel when kv-length is 0](../sources/prs/flashinfer/PR-868.md), [perf: FlashAttention-3 style MLA PageAttention](../sources/prs/flashinfer/PR-887.md), [feat - support mla kvcache store](../sources/prs/flashinfer/PR-888.md), [perf: fix MLA split-k performance bug](../sources/prs/flashinfer/PR-898.md), [perf: tweak the pipeline design of mla kernel](../sources/prs/flashinfer/PR-901.md), [feat: flashinfer intra-kernel profiler](../sources/prs/flashinfer/PR-913.md), [bugfix: fix potential issues of FA3 template loading nans for PageAttention](../sources/prs/flashinfer/PR-945.md), [perf: Use 2WG pipeline design for MLA implementation on Hopper](../sources/prs/flashinfer/PR-952.md), [perf: reduce torch.library dispatch overhead](../sources/prs/flashinfer/PR-968.md), [perf: Fix python API overhead when CUDAGraph is not enabled](../sources/prs/flashinfer/PR-969.md), [perf: prefetch page indices for mla kernel](../sources/prs/flashinfer/PR-991.md), [3rdparty: upgrade cutlass to 3.9](../sources/prs/flashinfer/PR-997.md), [Disable kernel cutlass_mla_decode on SM103](../sources/prs/sglang/PR-10058.md), [feat: Add FP4 (E2M1) KV Cache Support with Quantization Utilities for MLA](../sources/prs/sglang/PR-10078.md), [[Feature] Add MLAProcess for DeepSeek MLA on NPU](../sources/prs/sglang/PR-10130.md), [Fix chunked prefix cache for nvfp4](../sources/prs/sglang/PR-10180.md), [Enable trtllm mla prefix extend](../sources/prs/sglang/PR-10526.md), [[sgl-kernel] Optimize concat_mla_k kernel](../sources/prs/sglang/PR-10543.md), [Fuse quantize and rope in trtllm_mla MTP](../sources/prs/sglang/PR-10779.md), [disable sm100 for FlashMLA and fast-hadamard-transform in cuda12.6.1](../sources/prs/sglang/PR-11274.md), [[DeepseekV32] Enable flashmla_prefill kernel with fp8 kvcache](../sources/prs/sglang/PR-11655.md), [Use trtllm_mla decode kernel for draft extend in speculative decoding](../sources/prs/sglang/PR-11664.md), [Support running FP4 Deepseek on SM120.](../sources/prs/sglang/PR-11708.md), [[sgl-kernel] support flashmla libtorch](../sources/prs/sglang/PR-11717.md), [Fixed aarch64 flash-mla](../sources/prs/sglang/PR-12009.md), [[DeepseekV32]: use `_concat_mla_absorb_q_general` to replace `torch.cat`](../sources/prs/sglang/PR-12215.md), [[Deepseek V3.2] Enable flashmla_auto with MTP](../sources/prs/sglang/PR-12294.md), [fix seqlen bug for trtllm_mla's draft_extend](../sources/prs/sglang/PR-12295.md), [perf: trtllm mla performance minor improvements](../sources/prs/sglang/PR-12435.md), [[Fix] `concat_mla_absorb_q_kernel` fails for long inputs](../sources/prs/sglang/PR-12453.md), [[Ascend] Support enable-mixed-chunk in non-MLA scenarios](../sources/prs/sglang/PR-12491.md), [[sgl-kernel] support custom fp8 flashmla kernel](../sources/prs/sglang/PR-13087.md), [support mtp with deepseek r1 nvfp4 model](../sources/prs/sglang/PR-13115.md), [Fix target MLA with eagle3 support for PD disaggregation](../sources/prs/sglang/PR-13555.md), [[BugFix] fix prefixcache performance and accuracy on ascend](../sources/prs/sglang/PR-13573.md), [[ROCM] Optimized deepseek-r1 fp8 model with + triton_gemm_a8w8 + batch_gemm_a8w8 + fused set_mla_kv_buffer kernel](../sources/prs/sglang/PR-13617.md), [fix trtllm mla spec](../sources/prs/sglang/PR-13738.md), [[DeepSeek v3.2] opt Context Parallelism: support fused moe, multi batch and fp8 kvcache](../sources/prs/sglang/PR-13959.md), [Add Mistral Large 3 support.](../sources/prs/sglang/PR-14213.md), [[bug fix] fix ima with get_mla_kv_buffer_kernel overflow](../sources/prs/sglang/PR-14224.md), [Support FP8 MLA prefill and 128k context.](../sources/prs/sglang/PR-14395.md), [[NPU] perf update with kvcache nz & w4a8 quant](../sources/prs/sglang/PR-14423.md), [Add Mistral Large 3 Eagle Support](../sources/prs/sglang/PR-14466.md), [Fix accuracy issue when using a16w16 mla_decode_fwd](../sources/prs/sglang/PR-14936.md), [[sgl-kernel] Update flashmla to include fp8 sparse_mla optimizations](../sources/prs/sglang/PR-15242.md), [[NPU]DeepSeek-V3.2 support npu mlaprolog](../sources/prs/sglang/PR-15381.md), [Optimize FP8 MLA KV cache writes with Triton kernel](../sources/prs/sglang/PR-15522.md), [Fix BatchMLAPagedAttentionWrapper query/qo_inptr mismatch for EAGLE](../sources/prs/sglang/PR-15601.md), [Bugfix for ds-vl2](../sources/prs/sglang/PR-15894.md), [enhance accuracy for model kimi-vl-instruct-a3b](../sources/prs/sglang/PR-16076.md), [[NPU]bugfix: fix for dsv3.2 and dsvl2](../sources/prs/sglang/PR-17007.md), [Disable mla persistent kernel when not using fp8 kv_cache](../sources/prs/sglang/PR-17327.md), [[NPU] enhance accuracy for model kimi-vl-a3b-instruct](../sources/prs/sglang/PR-17480.md), [Make flashMLA work on: Cu13, B300](../sources/prs/sglang/PR-17600.md), [[Move sgl-kernel Kernel to JIT] Add JIT concat MLA kernels](../sources/prs/sglang/PR-17889.md), [[Hicache & JIT_kernel] Support page first layout & mla jit kernel](../sources/prs/sglang/PR-18311.md), [Nsa trtllm mla sparse fp8 support with Deepseek v3.2 NVFP4](../sources/prs/sglang/PR-18389.md), [[AMD] Update aiter to v0.1.10.post2](../sources/prs/sglang/PR-18423.md), [[AMD] DSR1/V3 use fp8 bmm in MLA for MI300X](../sources/prs/sglang/PR-18624.md), [[sgl-kernel] rebase FlashMLA 0217](../sources/prs/sglang/PR-18902.md), [Support skip-softmax attention](../sources/prs/sglang/PR-19089.md), [[Feature] add feature mla_ag_after_qlora for dsv3.2](../sources/prs/sglang/PR-19428.md), [[NPU] bugs fix for Deepseek models](../sources/prs/sglang/PR-19544.md), [Fix MLA decode path returning unwritten (padded) rows](../sources/prs/sglang/PR-19902.md), [[AMD] Fix FP8 assertion failure in aiter MLA decode by falling back to self.k_scale](../sources/prs/sglang/PR-19935.md), [Fix streaming session with paged KV cache (SWA/MLA)](../sources/prs/sglang/PR-20070.md), [[Benchmark] use flashinfer bench_gpu_time instead of triton do_bench](../sources/prs/sglang/PR-20305.md), [[AMD][AITER] Guard _use_mla_ps_kernel with self.use_mla in draft_extend_v2 paths](../sources/prs/sglang/PR-20409.md), [Support Triton MLA FP8 KV cache](../sources/prs/sglang/PR-20479.md), [FIX: (NSA) Compute topk_indices_offset when NSA prefill flashmla_sparse is used with FP8 KV cache](../sources/prs/sglang/PR-20606.md), [[AMD]: Support MLA with nhead<16 and FP8 KV cache for TP=8 (Kimi K2.5…](../sources/prs/sglang/PR-21213.md), [Refactor JIT kernel CI to use run_suite.py registration system](../sources/prs/sglang/PR-21239.md), [Migrate all callers from /get_server_info to /server_info](../sources/prs/sglang/PR-21463.md), [[AMD] Enable FP8 KV cache and FP8 attention kernel for NSA on MI300/MI355 with TileLang backend](../sources/prs/sglang/PR-21511.md), [test: point DSV3 int8 MLA CI models to lmsys Hugging Face org](../sources/prs/sglang/PR-21561.md), [[DSA] Support trtllm sparse mla kernel for prefill batches ](../sources/prs/sglang/PR-21783.md), [[Bugfix] Fix CUDA graph replay issues in trtllm_mla draft_extend](../sources/prs/sglang/PR-21987.md), [[NPU] enable mla prepare fused kernel only when being mla attn](../sources/prs/sglang/PR-22024.md), [[Disagg][NIXL] Fix heterogeneous TP KV transfer for non-MLA models (same logic with mooncake, Step 1/2 for Qwen3.5 support)](../sources/prs/sglang/PR-22145.md), [[Lora] Lora quat info re-factor and support deepseekv3 mla lora](../sources/prs/sglang/PR-22323.md), [[DSA] Hopper FP8 FlashMLA KV padding](../sources/prs/sglang/PR-22372.md), [[Fix] Fix accuracy bug in Flashmla sparse MLA kernel](../sources/prs/sglang/PR-22723.md), [Support spec v2 for FlashMLA speculative decoding](../sources/prs/sglang/PR-24640.md), [[attn backend] Integrate tokenspeed_mla prefill/decode kernels (fp8 kv cache, blackwell)](../sources/prs/sglang/PR-24925.md), [Amd/deepseek v4 rebase main 0509](../sources/prs/sglang/PR-24933.md), [[Intel GPU] Enable DeepSeek V4 Inference on XPU](../sources/prs/sglang/PR-25336.md), [Support draft extend cuda graph for tokenspeed_mla attention backend](../sources/prs/sglang/PR-25489.md), [amd/deepseek_v4 27/N [fix] Reduce Triton autotune configs for faster first-time server launch](../sources/prs/sglang/PR-25554.md), [[Refactor] Rename NSA → DSA: user-facing aliases, file/class/import rename](../sources/prs/sglang/PR-25821.md), [feat: integrate gemm_fp8 kernel into gemm](../sources/prs/sglang/PR-3056.md), [feat: support flashinfer mla attention for deepseek v3](../sources/prs/sglang/PR-3550.md), [feat: support flashinfer mla with prefix cache](../sources/prs/sglang/PR-3643.md), [Add fast decode plan for flashinfer mla](../sources/prs/sglang/PR-3987.md), [Hierarchical Caching supports MLA](../sources/prs/sglang/PR-4009.md), [[Revision] Add fast decode plan for flashinfer mla ](../sources/prs/sglang/PR-4012.md), [[Fix] Fix raw_bs bug when using flashinfer mla and eagle](../sources/prs/sglang/PR-4557.md), [avoid cudaStreamSynchronize in DeepSeekV2AttentionMLA](../sources/prs/sglang/PR-4577.md), [Blackwell Cutlass MLA kernel](../sources/prs/sglang/PR-5142.md), [feat: use fa3 mla by default on hopper](../sources/prs/sglang/PR-5210.md), [fix: solve cu118 issue for cutlass mla](../sources/prs/sglang/PR-5331.md), [[PD Bug] fix MLA get_contiguous_buf_infos error](../sources/prs/sglang/PR-5384.md), [Add Cutlass MLA attention backend](../sources/prs/sglang/PR-5390.md), [[PD] Fix dynamic port support and MLA buffer for Mooncake](../sources/prs/sglang/PR-5415.md), [BLackwell cutlass mla: Add check for bad page size/block num combinations](../sources/prs/sglang/PR-5431.md), [[perf] introduce deep gemm group_gemm_masked as gemm](../sources/prs/sglang/PR-5432.md), [Avoid computing lse in Ragged Prefill when there's no prefix.](../sources/prs/sglang/PR-5476.md), [Fuse MLA set kv cache kernel](../sources/prs/sglang/PR-5748.md), [opt flashinfer mla cat](../sources/prs/sglang/PR-5822.md), [Cutlass MLA decode - fix dtype error](../sources/prs/sglang/PR-5868.md), [[Fix] Fix a bug for flashmla to run R1 model](../sources/prs/sglang/PR-5875.md), [KV‑Cache (MHA, MLA): add missing start_layer / end_layer fields to MHATokenToKVPoolHost and MLATokenToKVPoolHost](../sources/prs/sglang/PR-6016.md), [feat: mtp support dp-attention](../sources/prs/sglang/PR-6081.md), [Cutlass MLA: Disable split kv due to https://github.com/NVIDIA/cutlass/issues/2274](../sources/prs/sglang/PR-6101.md), [fix: fix MLA for ShardedModelLoader/RemoteModelLoader](../sources/prs/sglang/PR-6287.md), [[PD] Add different TP sizes support for no-MLA models](../sources/prs/sglang/PR-6793.md), [CPU: map changes from developing branch in sgl-kernel](../sources/prs/sglang/PR-6833.md), [[perf][sgl-kernel] extend cutlass_mla_decode to support num_head < 128](../sources/prs/sglang/PR-6929.md), [[Feature] Support Flashinfer fmha on Blackwell](../sources/prs/sglang/PR-6930.md), [Fix cutlass MLA gets almost zero accuracy](../sources/prs/sglang/PR-6998.md), [Tiny fix cutlass_mla_get_workspace_size stub incorrect signature](../sources/prs/sglang/PR-7057.md), [chore: upgrade sgl-kernel v0.1.8.post2](../sources/prs/sglang/PR-7186.md), [Fix grammar abort & Minor style fixes](../sources/prs/sglang/PR-7204.md), [Minor style and doc fix](../sources/prs/sglang/PR-7228.md), [Fix CPU offloading for MLA memory pool](../sources/prs/sglang/PR-7409.md), [TRTLLM-MLA FP8 path](../sources/prs/sglang/PR-8638.md), [fix mooncake store mla zero copy meta](../sources/prs/sglang/PR-9678.md), [move is_sm90_supported/is_sm100_supported to python/sglang/srt/utils.py](../sources/prs/sglang/PR-9679.md), [[CPU] Add FP8 Bmm support](../sources/prs/sglang/PR-9744.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [[CUDA] Support int4 `T.gemm`](../sources/prs/tilelang/PR-2063.md), [[Perf] Enable fast math in sparse MLA example](../sources/prs/tilelang/PR-2219.md), [[Attention] MLA decode optimizations](../sources/prs/vllm/PR-12528.md), [[Attention] Deepseek v3 MLA support with FP8 compute](../sources/prs/vllm/PR-12601.md), [[Attention] MLA with chunked prefill](../sources/prs/vllm/PR-12639.md), [Disable chunked prefill and/or prefix caching when MLA is enabled ](../sources/prs/vllm/PR-12642.md), [[AMD][ROCm] Enable DeepSeek model on ROCm](../sources/prs/vllm/PR-12662.md), [[Perf] Mem align KV caches for CUDA devices (MLA perf improvement)](../sources/prs/vllm/PR-12676.md), [Squelch MLA warning for Compressed-Tensors Models](../sources/prs/vllm/PR-12704.md), [[VLM] Add MLA with pure RoPE support for deepseek-vl2 models](../sources/prs/vllm/PR-12729.md), [[Kernel] Make rotary_embedding ops more flexible with input shape](../sources/prs/vllm/PR-12777.md), [[Attention] Use FA3 for MLA on Hopper](../sources/prs/vllm/PR-12807.md), [Expand MLA to support most types of quantization](../sources/prs/vllm/PR-13181.md), [[Bugfix] Massage MLA's usage of flash attn for RoCM](../sources/prs/vllm/PR-13310.md), [[Bugfix] Fix max_num_batched_tokens for MLA](../sources/prs/vllm/PR-13620.md), [[core] Perf improvement for DSv3 on AMD GPUs](../sources/prs/vllm/PR-13718.md), [[Bugfix] Support MLA for CompressedTensorsWNA16](../sources/prs/vllm/PR-13725.md), [[Kernel] FlashMLA integration](../sources/prs/vllm/PR-13747.md), [[Attention] MLA support for V1](../sources/prs/vllm/PR-13789.md), [[ROCm] Disable chunked prefill/prefix caching when running MLA on non-cuda platforms](../sources/prs/vllm/PR-13844.md), [[Attention] Flash MLA for V1](../sources/prs/vllm/PR-13867.md), [Fix mla prefill context performance](../sources/prs/vllm/PR-13897.md), [dynamic distpatch of fp8 kernels](../sources/prs/vllm/PR-14245.md), [[BugFix] MLA + V1, illegal memory access and accuracy issues](../sources/prs/vllm/PR-14253.md), [[BugFix] Fix prefix caching V0 MLA](../sources/prs/vllm/PR-14255.md), [[Attention] FlashAttn MLA](../sources/prs/vllm/PR-14258.md), [[ROCm] Enable chunked prefill/paged attention in MLA on ROCm](../sources/prs/vllm/PR-14316.md), [[Perf] Reduce MLA CPU overheads in V1](../sources/prs/vllm/PR-14384.md), [[Bugfix] DeepSeek Accuracy](../sources/prs/vllm/PR-14476.md), [[Perf] Improve MLA on V1](../sources/prs/vllm/PR-14540.md), [[BugFix][TritonMLA] Process weights after model loading for GGUF](../sources/prs/vllm/PR-14555.md), [[Attention] Flash Attention 3 - fp8](../sources/prs/vllm/PR-14570.md), [[Kernel][CPU] CPU MLA](../sources/prs/vllm/PR-14744.md), [[Attention] MLA get rid of materialization](../sources/prs/vllm/PR-14770.md), [[Attention] Get rid of mla cache alignment](../sources/prs/vllm/PR-14842.md), [[V1] Default MLA to V1](../sources/prs/vllm/PR-14921.md), [[BugFix] Fix nightly MLA failure (FA2 + MLA chunked prefill, i.e. V1, producing bad results)](../sources/prs/vllm/PR-15492.md), [[Bugfix] Fix cache block size calculation for CPU MLA](../sources/prs/vllm/PR-15848.md), [[FEAT][ROCm]: Support AITER MLA](../sources/prs/vllm/PR-15893.md), [Modularize fused experts and integrate PPLX kernels](../sources/prs/vllm/PR-15956.md), [[NVIDIA] Support Cutlass MLA for Blackwell GPUs](../sources/prs/vllm/PR-16032.md), [[Kernel] support merge_attn_states CUDA kernel, 3x speedup](../sources/prs/vllm/PR-16173.md), [[MLA] Simplification to batch P/D reordering](../sources/prs/vllm/PR-16673.md), [[V1] V1 FlashInfer Attention](../sources/prs/vllm/PR-16684.md), [[torch.compile][ROCm] Fuse quantization onto attention using a torch.compile pass](../sources/prs/vllm/PR-16756.md), [[Bugfix] Triton FA function takes no keyword arguments](../sources/prs/vllm/PR-16902.md), [[Bugfix] Add contiguous call inside rope kernel wrapper](../sources/prs/vllm/PR-17091.md), [[BugFix] Fix vllm_flash_attn install issues](../sources/prs/vllm/PR-17267.md), [[Misc][ROCm] Exclude `cutlass_mla_decode` for ROCm build](../sources/prs/vllm/PR-17289.md), [[v1] AttentionMetadata for each layer](../sources/prs/vllm/PR-17394.md), [[v1] Pass BlockTable and KVCacheSpec to AttentionMetadataBuilders](../sources/prs/vllm/PR-17483.md), [[Attention] MLA move o_proj q_proj into cuda-graph region](../sources/prs/vllm/PR-17484.md), [[BugFix] Fix mla cpu - missing 3 required positional arguments](../sources/prs/vllm/PR-17494.md), [[FEAT][ROCm]: Support AITER MLA on V1 Engine](../sources/prs/vllm/PR-17523.md), [[Attention] MLA move rotary embedding to cuda-graph region](../sources/prs/vllm/PR-17668.md), [fix amd triton mla path](../sources/prs/vllm/PR-17871.md), [[Bugfix][ROCm] Fix AITER MLA V1](../sources/prs/vllm/PR-17880.md), [[BugFix][AMD] Compatible patch for AITER lib after 04/20](../sources/prs/vllm/PR-17912.md), [[v1] Support multiple KV cache groups in GPU model runner](../sources/prs/vllm/PR-17945.md), [[BUG] [ROCm] [MLA] Fix variable name bug due to change in variable name in PR #17483](../sources/prs/vllm/PR-17961.md), [[BugFix] FA2 MLA Accuracy Issue](../sources/prs/vllm/PR-18807.md), [[Kernel] Apply torch.Tag.needs_fixed_stride_order only for torch==2.6.0](../sources/prs/vllm/PR-19346.md), [[Bugfix] Enable PP with AITER+V1](../sources/prs/vllm/PR-19822.md), [[Attention] MLA - Flashinfer Ragged Prefill](../sources/prs/vllm/PR-20034.md), [[Bugfix] Fix some narrowing conversion warnings](../sources/prs/vllm/PR-20141.md), [SM100 Cutlass MLA decode with unrestricted num_heads (< 128) for DeepSeek TP](../sources/prs/vllm/PR-20769.md), [[Kernel] Flashinfer MLA (trtllm-gen) decode kernel integration](../sources/prs/vllm/PR-21078.md), [[perf] Add fused MLA QKV + strided layernorm](../sources/prs/vllm/PR-21116.md), [[v1][attention] Support Hybrid Allocator + FlashInfer](../sources/prs/vllm/PR-21412.md), [[BugFix] Fix IMA FlashMLA full cuda-graph and DP + Update FlashMLA](../sources/prs/vllm/PR-21691.md), [[Bugfix] Check NVIDIA artifactory is accessible before using flashinfer cubin kernels](../sources/prs/vllm/PR-21893.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv NVFP4-out Attention Kernel](../sources/prs/vllm/PR-22703.md), [[Bugfix] Fix default enable for CUTLASS MLA on SM100](../sources/prs/vllm/PR-22738.md), [[Log] Warning Once for Cutlass MLA ](../sources/prs/vllm/PR-23137.md), [[ROCm][Aiter] Add triton fp8 bmm kernel for mla](../sources/prs/vllm/PR-23264.md), [[Compile] Fix Compile Warning SM100 Cutlass MLA](../sources/prs/vllm/PR-23287.md), [[Attention][FA3] Update FA3 to include new swizzle optimization](../sources/prs/vllm/PR-23465.md), [[Feature] Support Decode Context Parallel (DCP) for MLA](../sources/prs/vllm/PR-23734.md), [[Model] Add LongCat-Flash ](../sources/prs/vllm/PR-23991.md), [[Kernel] Support decode context parallelism on Blackwell with CUTLASS MLA](../sources/prs/vllm/PR-24385.md), [[Feature] Disallow FlashMLA on Blackwell](../sources/prs/vllm/PR-24521.md), [[Bug] Fix `is_flashmla_supported` Check Error](../sources/prs/vllm/PR-24774.md), [[Bugfix][B200] Fix `cutlass_mla` hang](../sources/prs/vllm/PR-24966.md), [[Attention][DCP] Support DCP with query length > 1 (MTP) with FA3](../sources/prs/vllm/PR-25049.md), [[Bug] Fix `returned_lse` not Defined issue](../sources/prs/vllm/PR-25106.md), [[BugFix] Fix MLA assert with CUTLASS MLA](../sources/prs/vllm/PR-25478.md), [[Bugfix] [B200] cutlass_mla - ensure kv_split == 1 for batch size > 1](../sources/prs/vllm/PR-25509.md), [Fuse RoPE and MLA KV-cache write](../sources/prs/vllm/PR-25774.md), [[Quantization/NVFP4] Speed up TRTLLM NVFP4 MOE weight loading and fix K/V scale loading for MLA Attn](../sources/prs/vllm/PR-25968.md), [[Spec Decode] Enable efficient speculative decoding with FlashInfer-MLA](../sources/prs/vllm/PR-25984.md), [[Attention] Tune CUTLASS MLA num_splits](../sources/prs/vllm/PR-26846.md), [Prefer FlashAttention MLA as default over FlashMLA](../sources/prs/vllm/PR-27363.md), [[Attention] Use sparse prefill kernel for fp8 kv-cache in DeepSeek-v3.2](../sources/prs/vllm/PR-27532.md), [[AMD] Use Decoupled Kernel Block Size to Support AITER MLA block_size=1](../sources/prs/vllm/PR-27715.md), [[Bug] Batch invariant: Fix flash attn MLA `RuntimeError: scheduler_metadata must have shape (metadata_size)`](../sources/prs/vllm/PR-27884.md), [[ROCm][MLA] enable fp8 MLA decode on ROCm](../sources/prs/vllm/PR-28032.md), [[Feature] Prefill Context Parallel (PCP) basic support](../sources/prs/vllm/PR-28718.md), [bugfix: correct attn output with base 2 or e](../sources/prs/vllm/PR-28840.md), [[BugFix] Fix async-scheduling + FlashAttn MLA](../sources/prs/vllm/PR-28990.md), [[DeepSeek + LMCache Multiprocess] handle MLA for deepseek model + LMCache Multiprocess connector](../sources/prs/vllm/PR-29039.md), [[Feature] Batch invariant: Enable `TRITON_MLA` without prefix-caching](../sources/prs/vllm/PR-29125.md), [[perf] Use direct copy (broadcast) instead of cat for k_nope/k_pe in MLA prefill](../sources/prs/vllm/PR-29710.md), [Add Mistral Large 3 and Ministral 3](../sources/prs/vllm/PR-29757.md), [[Perf] Improve fp8 quant in mla; replace ReduceSum with ReduceScatterSum](../sources/prs/vllm/PR-29795.md), [[SpecDecode] Simplified alternative padded-speculation acceptance rate fix](../sources/prs/vllm/PR-29845.md), [[Quantization] fix: overflow with static per-tensor scaling](../sources/prs/vllm/PR-29867.md), [[ROCm] add fallback for aiter fp8 decode mla](../sources/prs/vllm/PR-30005.md), [[ROCm][Bugfix] Add MLACommonMetadata to allowed attention types for speculative decoding](../sources/prs/vllm/PR-30430.md), [[Feature] Add SM103 (Blackwell Ultra) Support to vLLM](../sources/prs/vllm/PR-30484.md), [[SM100] Enable fp8 compute for prefill MLA](../sources/prs/vllm/PR-30746.md), [[perf] Integrate flashinfer concat_mla_k](../sources/prs/vllm/PR-31171.md), [[Bugfix][Hardware][AMD] Fix exception types in AITER MLA FP8 check](../sources/prs/vllm/PR-31177.md), [[SM100] Resubmit FMHA FP8 prefill for MLA](../sources/prs/vllm/PR-31195.md), [[Bugfix][Hardware][AMD] Fix last_page_len calculation in AITER MLA decode](../sources/prs/vllm/PR-31282.md), [fixed mypy warnings for files vllm/v1/attention with TEMPORARY workaround](../sources/prs/vllm/PR-31465.md), [[1/N][Attention] Restructure attention: move files](../sources/prs/vllm/PR-31916.md), [[4/N][Attention] Move MLA common to model_executor](../sources/prs/vllm/PR-32060.md), [[5/N][Attention] Finish eliminating `vllm/attention` folder](../sources/prs/vllm/PR-32064.md), [fix: Add glm4_moe_lite to MLA detection](../sources/prs/vllm/PR-32614.md), [[Bugfix][Attention] Explicitly report support for kv_cache_dtype bfloat16](../sources/prs/vllm/PR-32795.md), [[Attention] FA4 integration](../sources/prs/vllm/PR-32974.md), [[Attention] Use `has_flashinfer` helper](../sources/prs/vllm/PR-33177.md), [Triton MLA perf fixes](../sources/prs/vllm/PR-33529.md), [[Bugfix] Fix sparse MLA metadata building](../sources/prs/vllm/PR-33579.md), [[Bugfix] fix DeepSeek R1 with CUTLASS MLA Broken on B200](../sources/prs/vllm/PR-33637.md), [[Bugfix] Fix DSV3.2 NVFP4](../sources/prs/vllm/PR-33932.md), [Reapply [Attention][FA3] Update FA3 to include new swizzle optimization](../sources/prs/vllm/PR-34043.md), [fix(cpu): fix mla_decode compilation on x86 without AVX512](../sources/prs/vllm/PR-34052.md), [[BugFix] Add support for MTP num_speculative_tokens > 1 with sparse MLA](../sources/prs/vllm/PR-34552.md), [[Kernel] Add FP8 KV cache support to Triton MLA decode attention](../sources/prs/vllm/PR-34597.md), [[Bugfix] Fix MLA attention crash with AWQ/GPTQ quantized models](../sources/prs/vllm/PR-34695.md), [[Attention] Use FA4 for MLA prefill](../sources/prs/vllm/PR-34732.md), [[Model Bash] DeepSeek R1 BF16 Min Latency QKV A GEMM (0.5% E2E Speedup)](../sources/prs/vllm/PR-34758.md), [[Model Bash][DSR1] Add selective dynamic shape marking for CustomOp](../sources/prs/vllm/PR-34900.md), [[Attention][Perf][Kernel] Replace torch.cat with vectorized CUDA kernel MLA query concat - DeepSeek-V3.2](../sources/prs/vllm/PR-34917.md), [[Bug][DSV3.2] Always prepare metadata for DeepGEMM Sparse Attention](../sources/prs/vllm/PR-35075.md), [[Feat] Add CUDA torch fallbacks for fp8_mqa_logits/fp8_paged_mqa_logits_torch function](../sources/prs/vllm/PR-35271.md), [[Bugfix] Fix KV Scale loading for MLA Models](../sources/prs/vllm/PR-35430.md), [[ROCm] Support MLA with nhead<16 and FP8 KV cache for TP=8 (Kimi K2.5/Linear)](../sources/prs/vllm/PR-35850.md), [[Perf] Support FP8 KV cache for Flashinfer MLA Sparse](../sources/prs/vllm/PR-35891.md), [[LMCache] Pass TP size in lookup for MLA multi-reader locking](../sources/prs/vllm/PR-36129.md), [Add 320 dimension size support to MLA](../sources/prs/vllm/PR-36161.md), [[Bugfix][MLA] Add logits size budget to sparse indexer prefill chunking](../sources/prs/vllm/PR-36178.md), [[mla] Support fused FP8/NVFP4 output quantization in MLA attention (#35792)](../sources/prs/vllm/PR-36205.md), [mla: don't update kv cache on dummy forwards](../sources/prs/vllm/PR-36282.md), [Kimi k2.5 MLA based eagle3](../sources/prs/vllm/PR-36361.md), [[Bugfix][Sparse MLA] report indexer CG support properly](../sources/prs/vllm/PR-36519.md), [[ROCm] Utilize persistent MLA kernel from AITER](../sources/prs/vllm/PR-36574.md), [[ROCm][Perf] Allow MTP lens > 1 in Sparse MLA](../sources/prs/vllm/PR-36681.md), [[DSV3.2][MTP] Optimize Indexer MTP handling](../sources/prs/vllm/PR-36723.md), [[Feat][Bugfix] Enable additional dimension for Flashinfer MLA and fix routing dtype](../sources/prs/vllm/PR-36931.md), [[MTP][Sparse MLA] Take advantage of native MTP support in indexer when possible](../sources/prs/vllm/PR-36982.md), [[Bugfix] Fix KV scales inconsistency in fp8 MLA & FlashInfer kv_cache_dtype "auto" leading to gibberish](../sources/prs/vllm/PR-37054.md), [[Bugfix] Disable cross-layer KV cache for MLA attention backends](../sources/prs/vllm/PR-37090.md), [[Benchmark] Improvements to attention benchmark script](../sources/prs/vllm/PR-37115.md), [[XPU] support MLA model on Intel GPU](../sources/prs/vllm/PR-37143.md), [[Misc] Add `float16` to `CacheDType`](../sources/prs/vllm/PR-37199.md), [[Perf] Set Flashinfer sparse MLA as default backend for FP8 kv cache](../sources/prs/vllm/PR-37252.md), [[Perf][Kernel] Persistent TopK scheduler: unified CUDAGraph-safe kernel with dynamic per-row dispatch - DeepSeek-V3.2 DSA decode](../sources/prs/vllm/PR-37421.md), [[BugFix] Allow qk_nope_head_dim=192 in FlashInfer MLA backend checks](../sources/prs/vllm/PR-37475.md), [Fix KV Offloading + MLA AssertionError by using num_kv_heads=1 in cpu…](../sources/prs/vllm/PR-37536.md), [[Performance] Remove unnecessary zero-fill of MLA decode output tensor in Aiter backend](../sources/prs/vllm/PR-37539.md), [[Bugfix][ROCm] Fix lru_cache on paged_mqa_logits_module](../sources/prs/vllm/PR-37547.md), [[Test] Only Run MLA model when user explicitly set for batch invariance](../sources/prs/vllm/PR-37719.md), [[ROCm][perf] fix Aiter sparse MLA with MTP>1](../sources/prs/vllm/PR-37887.md), [[Bugfix][MLA] Change default SM100 MLA prefill backend back to TRT-LLM](../sources/prs/vllm/PR-38562.md), [[ROCm] Fix aiter persistent mode mla with q/o nhead<16 for kimi-k2.5 tp8](../sources/prs/vllm/PR-38615.md), [Fix MLA runs when use_inductor_graph_partition=True](../sources/prs/vllm/PR-38631.md), [[LMCache][MP] optimize save when mla enabled](../sources/prs/vllm/PR-38810.md), [[Attention][MLA] Re-enable FA4 as default MLA prefill backend](../sources/prs/vllm/PR-38819.md), [[Refactor] Improve indexer decode path metadata preparation](../sources/prs/vllm/PR-38865.md), [[XPU] Quick fix for TritonMLA to remove cuda hardcode](../sources/prs/vllm/PR-39088.md), [[Bug] Fix rocm sparse attn indexer issue](../sources/prs/vllm/PR-39225.md), [[MLA] Optimize mla indexer prepare uniform decode for MTP > 1](../sources/prs/vllm/PR-39458.md), [[Mooncake] Fix mixed MLA+Eagle block-size validation](../sources/prs/vllm/PR-39596.md), [add warning when FP8 KV cache misses prefill query quantization](../sources/prs/vllm/PR-39752.md), [[Performance][DSR1]: Fused RoPE+KVCache+q_concat for MLA](../sources/prs/vllm/PR-40392.md), [[MLA Attention Backend] Add TOKENSPEED_MLA backend for DSR1/Kimi K25 prefill + decode on Blackwell](../sources/prs/vllm/PR-41778.md), [[Bugfix] Fix TRTLLM ragged MLA prefill workspace warmup](../sources/prs/vllm/PR-42112.md), [[ROCm] [Bugfix] Fix DeepSeek V4 Functionality and Accuracy](../sources/prs/vllm/PR-42810.md), [[Perf][MLA] Enable FULL cudagraph capture for TRITON_MLA decode](../sources/prs/vllm/PR-42885.md), [[Model Refactoring] Move deepseek_v4_ops to models/deepseek_v4 [3/N]](../sources/prs/vllm/PR-43073.md), [FlashAttention SM100 MLA TopK Sparse Forward](../wiki/kernels/flash-attention-sm100-mla-topk.md), [FlashMLA — Multi-head Latent Attention](../wiki/kernels/flashmla.md), [Sparse MLA (DeepSeek V3.2)](../wiki/kernels/sparse-mla.md) | -| `moe` | [K-Search: LLM Kernel Generation via Co-Evolving Intrinsic World Model](../sources/blogs/k-search-kernel-generation.md), [Qwen3-Next: Hybrid GDN+MoE Architecture on NVIDIA Blackwell](../sources/blogs/qwen3-next-architecture.md), [TFLOPS Gap: Why FP4 MoE Kernel Engineering Matters on Blackwell](../sources/blogs/tflops-gap-fp4-moe.md), [FlashInfer MLSys 2026 - Track A: Fused MoE FP8](../sources/contests/flashinfer-mlsys26/track-a-fused-moe.md), [GPU Mode NVFP4 Hackathon - Problem 4: Grouped GEMM](../sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes](../sources/prs/DeepGEMM/PR-304.md), [Sync nv_dev with upstream #316 (Mega MoE optimizations & benchmarks)](../sources/prs/DeepGEMM/PR-328.md), [[None][perf] Add more optimization options for MOE CuteDSL finalized kernel](../sources/prs/TensorRT-LLM/PR-10042.md), [[TRTLLM-9992][perf] Enable PDL for CuteDSL kernels and overlap MoeOutputMemset](../sources/prs/TensorRT-LLM/PR-10043.md), [[None][feat] CuteDSL MOE FC1 Enhancement](../sources/prs/TensorRT-LLM/PR-10088.md), [[TRTLLM-9831][perf] Enable 2CTA with autotune for CuteDSL MoE and Grouped GEMM optimizations](../sources/prs/TensorRT-LLM/PR-10201.md), [[TRTLLM-10147][perf] Balanced random MoE workload generator for CuteDSL kernel UT, autotuner and layerwise benchmark](../sources/prs/TensorRT-LLM/PR-10279.md), [[None] [feat] Add densegemm backend for MoE](../sources/prs/TensorRT-LLM/PR-10479.md), [[None][feat] MiniMax M2 support](../sources/prs/TensorRT-LLM/PR-10532.md), [[None][feat] fuse shared to sparse experts in TRT-LLM Gen MoE](../sources/prs/TensorRT-LLM/PR-11143.md), [[https://nvbugs/5799917][fix] Recover from CUTLASS MoE doActivation perf regression for MXFP4/NVFP4 dtype](../sources/prs/TensorRT-LLM/PR-11165.md), [[None][feat] TRT-LLM Gen MoE finalize kernel optimization](../sources/prs/TensorRT-LLM/PR-11501.md), [[None][feat] Add support for expert_number<=2048 and K<=32](../sources/prs/TensorRT-LLM/PR-11510.md), [[None][fix] Fix int4 awq for sm120/121](../sources/prs/TensorRT-LLM/PR-11561.md), [[TRTLLM-10004][feat] Enable GEMM -> AR with GEMM output in registered buffers](../sources/prs/TensorRT-LLM/PR-11589.md), [[https://nvbugs/5799917][fix] Recover from CUTLASS MoE doActivation perf regression for MXFP4/NVFP4 dtype](../sources/prs/TensorRT-LLM/PR-11733.md), [[https://nvbugs/5885070][fix] fix deepeplowlatency with cutedsl moe backend](../sources/prs/TensorRT-LLM/PR-11769.md), [[TRTLLM-10990][feat] Fuse SwiGLU and quant into shared expert](../sources/prs/TensorRT-LLM/PR-11897.md), [[None][feat] GLM 5 support and DSA MTP fixes](../sources/prs/TensorRT-LLM/PR-11990.md), [[#11694][feat] AutoDeploy: Improve the piecewise CG memory usage](../sources/prs/TensorRT-LLM/PR-11993.md), [[https://nvbugs/5955188][fix] Fix harmony parsers and WAR routing PDL for agentic coding use cases](../sources/prs/TensorRT-LLM/PR-12046.md), [[None][feat] CuteDSL MOE: Add raster along M/N support for blockscaled contiguous backbone kernel](../sources/prs/TensorRT-LLM/PR-12079.md), [[None][feat] Add DWDP (Distributed Weight Data Parallelism) support for MoE inference](../sources/prs/TensorRT-LLM/PR-12136.md), [[None][feat] Add fused allreduce+RMSNorm op and optional residual in …](../sources/prs/TensorRT-LLM/PR-12201.md), [[None][feat] Support update weight for nvfp4](../sources/prs/TensorRT-LLM/PR-12320.md), [[None][perf] add Dynamic SMEM block routing in MOE](../sources/prs/TensorRT-LLM/PR-12456.md), [[None][feat] Optimize mamba SSD prefill and extend flashinfer dispatch](../sources/prs/TensorRT-LLM/PR-12731.md), [[None][feat] Add bf16 trtllm-gen moe support through flashinfer.](../sources/prs/TensorRT-LLM/PR-12738.md), [[TRTLLM-11797][feat] Add cutedsl moe backend supporting for qwen3.5.](../sources/prs/TensorRT-LLM/PR-12799.md), [[TRTLLM-11585][feat] Add CUTEDSL moe backend for nemotron-h](../sources/prs/TensorRT-LLM/PR-12884.md), [[#12784][feat] AutoDeploy: Optimize DeepSeek-R1 model performance](../sources/prs/TensorRT-LLM/PR-12946.md), [[None][fix] Propagate init_load_balancer to DeepGemmFusedMoE in create_moe_backend](../sources/prs/TensorRT-LLM/PR-13207.md), [[TRTLLM-11127][feat] add W4A8_MXFP4_FP8 MoE unit test support](../sources/prs/TensorRT-LLM/PR-13401.md), [[None][perf] Extend customMoeRouting kernel to support Qwen3.5](../sources/prs/TensorRT-LLM/PR-13433.md), [[None][feat] Enable EPLB for DeepSeek-V4](../sources/prs/TensorRT-LLM/PR-13595.md), [[None][feat] Add bf16 trtllm moe through flashinfer.](../sources/prs/TensorRT-LLM/PR-13689.md), [[None][fix] Plumb swiglu_limit through DeepGEMM and TRTLLMGen FP8 fused MoE](../sources/prs/TensorRT-LLM/PR-13767.md), [[None][perf] FC2 DenseGEMM autotune: split-K, swap_ab, fine-grained tuning buckets](../sources/prs/TensorRT-LLM/PR-13833.md), [[None][feat] enable TRTLLM-Gen internal routing](../sources/prs/TensorRT-LLM/PR-13997.md), [[https://nvbugs/6152892][fix] Fix Triton MOE memory free when no swizzling enabled](../sources/prs/TensorRT-LLM/PR-14069.md), [[None][fix] Add SPDX Apache-2.0 headers to auto_deploy test files](../sources/prs/TensorRT-LLM/PR-14193.md), [[None][fix] Avoid dp_size x ep_size double-count in MegaMoEDeepGemm SymmBuffer](../sources/prs/TensorRT-LLM/PR-14213.md), [[None][fix] Update the OSS headers in derived FLA ops and AD modeling code](../sources/prs/TensorRT-LLM/PR-14281.md), [[https://nvbugs/6095421][fix] Update resolve_moe_backend](../sources/prs/TensorRT-LLM/PR-14282.md), [[None][chore] Update Claude Code agents and skills](../sources/prs/TensorRT-LLM/PR-14344.md), [[None][chore] Fix kernel launch param and add TRTLLM MoE backend test](../sources/prs/TensorRT-LLM/PR-7524.md), [[None][fix] Fix and add test for TRTLLM MoE backend](../sources/prs/TensorRT-LLM/PR-7755.md), [[TRTLLM-8637][feat] Optimize the routing kernel for DeepseekV3 (MoE CUTLASS backend); Add support for 384 experts (MoE TRTLLM backend)](../sources/prs/TensorRT-LLM/PR-7761.md), [[None][feat] GPT-OSS Sm120/Sm121 Support](../sources/prs/TensorRT-LLM/PR-7937.md), [[None][fix] Fix the performance issue of FP8 blockwise grouped GEMM when using attention DP](../sources/prs/TensorRT-LLM/PR-8501.md), [[TRTLLM-8827] [feat] Enable low precision alltoall for Cutlass and TRTLLMGen backends](../sources/prs/TensorRT-LLM/PR-8675.md), [[None][feat] Enable EPLB for trtllm-gen and cutlass backend](../sources/prs/TensorRT-LLM/PR-8886.md), [[None][feat] Update TRTLLM MoE cubins; reduce mxfp4 weight padding requirement; tighten TMA bound](../sources/prs/TensorRT-LLM/PR-9025.md), [[None][fix] support topk autotuner input for expert slot per group larger than 32](../sources/prs/TensorRT-LLM/PR-9087.md), [[None][feat] TRT-LLM Gen MoE optimize DeepSeek Fp8 activation kernel](../sources/prs/TensorRT-LLM/PR-9175.md), [[TRTLLM-8958][feat] and [TRTLLM-8960]: create ConfigurableMoE and support TRTLLMGenFusedMoE as backend](../sources/prs/TensorRT-LLM/PR-9486.md), [[TRTLLM-9685] [feat] Add gather fc1 kernel by cuteDSL](../sources/prs/TensorRT-LLM/PR-9618.md), [[https://nvbugs/5726962][feat] Apply fusion for W4AFP8_AWQ MoE](../sources/prs/TensorRT-LLM/PR-9838.md), [[None][feat] Fused kernels (qknormrope + moe routing) and two-model MTP support for glm4moe](../sources/prs/TensorRT-LLM/PR-9852.md), [Blockscaled Ragged Contiguous Grouped Gemm for MoEs](../sources/prs/cutlass/PR-2790.md), [feat: add trtllm all-reduce (non-MoE)](../sources/prs/flashinfer/PR-1096.md), [feat: add trtllm moe_allreduce_fusion](../sources/prs/flashinfer/PR-1108.md), [Add CUTLASS fused moe kernels from TensorRT-LLM.](../sources/prs/flashinfer/PR-1113.md), [bugfix: Fix test and output shape of fp4 quantize](../sources/prs/flashinfer/PR-1114.md), [feat: add trtllm all-reduce fusion](../sources/prs/flashinfer/PR-1131.md), [MNNVL MoE All-to-All Support](../sources/prs/flashinfer/PR-1134.md), [feat: add finalize_moe_allreduce from trtllm](../sources/prs/flashinfer/PR-1159.md), [feat: update non-fused moe](../sources/prs/flashinfer/PR-1161.md), [feat: enable and update all-reduce fused quantization](../sources/prs/flashinfer/PR-1164.md), [feat: trtllm-gen fp8 moe kernels](../sources/prs/flashinfer/PR-1212.md), [Feature/sm100 low latency nvfp4 kernels](../sources/prs/flashinfer/PR-1214.md), [Patch fp8 cubin availability](../sources/prs/flashinfer/PR-1240.md), [Remove sm100+ requirment for trtllm allreduce kernels](../sources/prs/flashinfer/PR-1249.md), [Add shuffle matrix flag](../sources/prs/flashinfer/PR-1272.md), [Bug fix: guard fp8 e8m0 and e2m1 compile ](../sources/prs/flashinfer/PR-1287.md), [Remove FAST_BUILD FLAG for MOE](../sources/prs/flashinfer/PR-1291.md), [Update cutlass fp4 moe kernels](../sources/prs/flashinfer/PR-1294.md), [feat: Add weight layout option for trtllm-gen fused moe](../sources/prs/flashinfer/PR-1297.md), [perfix: use lightweight API to query device property](../sources/prs/flashinfer/PR-1298.md), [Refactor Fused Moe Module](../sources/prs/flashinfer/PR-1309.md), [Support loading autotuned results from json for cutlass fp4 moe backends](../sources/prs/flashinfer/PR-1310.md), [Make Fp8 MoE routing_bias optional](../sources/prs/flashinfer/PR-1319.md), [refactor: Improved metainfo for trtllm-gen kernels](../sources/prs/flashinfer/PR-1328.md), [feature: add fp4 mm using trtllm backend](../sources/prs/flashinfer/PR-1355.md), [Update autotune results for the nvfp4 cutlass moe backends for v0.2.9](../sources/prs/flashinfer/PR-1361.md), [bugfix: fixed cutlass fused moe usage of FP4QuantizationSFLayout::SWIZZLED](../sources/prs/flashinfer/PR-1371.md), [GPT-OSS Support: Add Blackwell MoE mxfp4 implementation from TRTLLM and Attention Sink](../sources/prs/flashinfer/PR-1389.md), [gpt-oss: Add MXFP8 x MXFP4 CUTLASS MOE for SM100 and BF16 x MXFP4 CUTLASS for SM90 + SwigluBias Activation](../sources/prs/flashinfer/PR-1396.md), [Fix trtllm moe launcher local_num_experts](../sources/prs/flashinfer/PR-1398.md), [Add Mxfp4 trtllm-gen moe unit tests](../sources/prs/flashinfer/PR-1399.md), [[bugfix] Fix compilation failure when compiling csrc/trtllm_moe_allreduce_fusion.cu](../sources/prs/flashinfer/PR-1410.md), [Faster weight processing (moe nvfp4)](../sources/prs/flashinfer/PR-1412.md), [Fix redundant kernels in moe](../sources/prs/flashinfer/PR-1428.md), [Remove getEnvEnablePDL in favor of enable_pdl parameter](../sources/prs/flashinfer/PR-1446.md), [bugfix: Verify num_experts greater or equal to local_experts + offset](../sources/prs/flashinfer/PR-1469.md), [feat: Enable multiple fused-moe backends](../sources/prs/flashinfer/PR-1472.md), [tuner: Trtllm-gen Fp4 MoE Autotunner](../sources/prs/flashinfer/PR-1475.md), [fix missing enable_pdl argument in trtllm-gen fp4 moe](../sources/prs/flashinfer/PR-1480.md), [fix: update cutedsl masked moe gemm](../sources/prs/flashinfer/PR-1488.md), [feat: Support fp8 qkv, fp16/bf16 out MHA for trtllm-gen.](../sources/prs/flashinfer/PR-1490.md), [fix: update masked moe gemm fp4 tensor reshape](../sources/prs/flashinfer/PR-1495.md), [Support cuda<12.8 built for trtllm_allreduce_fusion.](../sources/prs/flashinfer/PR-1508.md), [flashinfer_benchmark QoL Improvements and Attention FP8 Support](../sources/prs/flashinfer/PR-1512.md), [Add GeGLU support to trtllm-gen NVFP4 Fused MoE Kernel](../sources/prs/flashinfer/PR-1525.md), [bugfix: Fix compile error for undefined swizzle enum.](../sources/prs/flashinfer/PR-1530.md), [feat: Add fp8-qkv, fp16/bf16 output MHA](../sources/prs/flashinfer/PR-1540.md), [perf: replace cudaGetDeviceProperties with cudaDeviceGetAttribute](../sources/prs/flashinfer/PR-1547.md), [perf: Enable SplitK and fix tile-scheduling for moe fp4 fused moe](../sources/prs/flashinfer/PR-1548.md), [Add mnnvl_moe_alltoallv_prepare_without_allgather](../sources/prs/flashinfer/PR-1550.md), [fix: separate out fp4 lib into sm90 and sm100 versions, add oob checking in fused moe](../sources/prs/flashinfer/PR-1565.md), [bugfix: fix cuda version guard macros](../sources/prs/flashinfer/PR-1571.md), [update trtllm-gen fp4 autotuner and routing](../sources/prs/flashinfer/PR-1573.md), [refactor: Expose calculate_tile_tokens_dim function](../sources/prs/flashinfer/PR-1581.md), [bugfix: Fix arg passing to TORCH_CHECK and TORCH_WARN macros](../sources/prs/flashinfer/PR-1582.md), [feat: initial support for SM103, SM110, SM120, SM121](../sources/prs/flashinfer/PR-1608.md), [bugfix: collect all modules to aot](../sources/prs/flashinfer/PR-1622.md), [test: update fused_moe test to random scale factor](../sources/prs/flashinfer/PR-1665.md), [Refactor Blackwell unit test scripts](../sources/prs/flashinfer/PR-1667.md), [test: better fp8 quantization init for fused_moe test](../sources/prs/flashinfer/PR-1674.md), [Support Kimi-K2 for TRT: templatize number of experts](../sources/prs/flashinfer/PR-1696.md), [test: skip the unsupported test cases for sm120/121](../sources/prs/flashinfer/PR-1710.md), [perf: Add tuning config for cutlass moe for a hardware](../sources/prs/flashinfer/PR-1716.md), [Fix DeepSeek quality for TRTLLM fused MoE routing](../sources/prs/flashinfer/PR-1723.md), [bugfix: partially fix tests/test_trtllm_gen_fused_moe.py unit test failure](../sources/prs/flashinfer/PR-1724.md), [tests: xfail moe quantization classes mxfp8_bf16 UTs on sm103 ](../sources/prs/flashinfer/PR-1754.md), [add test case for trtllm gen fused moe with kimi k2 problem sizes](../sources/prs/flashinfer/PR-1768.md), [fix: fp4 moe on sm120](../sources/prs/flashinfer/PR-1817.md), [feat:enable fp8 blockscale moe for fused cultass for sm90](../sources/prs/flashinfer/PR-1819.md), [feat: trtrllm-gen global scaled FP8 GEMMs](../sources/prs/flashinfer/PR-1829.md), [Update the routing for TRTLLMGEN to support kimi k2 and qwen](../sources/prs/flashinfer/PR-1831.md), [feat: Add FP4 TRTLLM-Gen throughput MOE batched gemms](../sources/prs/flashinfer/PR-1882.md), [Feature: Support Relu2 activation in fused MoE](../sources/prs/flashinfer/PR-1954.md), [Update trtllm-gen fused moe routing kernel and add more kernels](../sources/prs/flashinfer/PR-1955.md), [Fix: Verify scales are not None for Cutlass FP8 FusedMoE](../sources/prs/flashinfer/PR-1961.md), [Feature: Add support for L40 FusedMoE in cutlass path](../sources/prs/flashinfer/PR-1973.md), [feat: autotune tile_tokens_dim in trtllm-gen MOE](../sources/prs/flashinfer/PR-1980.md), [Bugfix: Change get() -> GetDLTensorPtr() in cutlass FusedMoE validations](../sources/prs/flashinfer/PR-1995.md), [Feature: Support non-gated activation in cutlass fused MoE nvfp4](../sources/prs/flashinfer/PR-2011.md), [[feat] Refactor trtllmgen MOE and add Bf16 trtllmgen moe](../sources/prs/flashinfer/PR-2014.md), [update trtllm cutlass moe ](../sources/prs/flashinfer/PR-2020.md), [perf: Speed up fp4 quantization for small batch with swizzling for cutlass MoE](../sources/prs/flashinfer/PR-2025.md), [Enable renormalize(naive) routing for fp8 per-tensor](../sources/prs/flashinfer/PR-2030.md), [Fix dtype of output scales from mnnvl_moe_alltoallv_prepare_without_allgather](../sources/prs/flashinfer/PR-2048.md), [[BUG] Fix trtllm-gen fp4 moe renormalize routing](../sources/prs/flashinfer/PR-2049.md), [Add support for topkPacked input in block-level renormalize](../sources/prs/flashinfer/PR-2051.md), [Fix moe fp8 failure for sm121](../sources/prs/flashinfer/PR-2061.md), [perf: TRT-LLM MoE Block-FP8 activation optimization](../sources/prs/flashinfer/PR-2063.md), [[Test] Optimize test_trtllm_gen_fused_moe.py](../sources/prs/flashinfer/PR-2072.md), [Patch sm103 for 3xfp4 moe generation](../sources/prs/flashinfer/PR-2082.md), [refactor: update dpsk fused_moe test [1]](../sources/prs/flashinfer/PR-2088.md), [refactor: pass hopper deepgemm include directory through python](../sources/prs/flashinfer/PR-2090.md), [perf: TRT-LLM Gen finalize kernel optimization](../sources/prs/flashinfer/PR-2092.md), [refactor: update dpsk fused_moe test [2]](../sources/prs/flashinfer/PR-2097.md), [Port TRT-LLM communication kernels to flashinfer](../sources/prs/flashinfer/PR-2102.md), [enable sm103 moe dsl backend](../sources/prs/flashinfer/PR-2149.md), [feat: MxInt4 x Bf16 TRT-LLM Gen MoE support](../sources/prs/flashinfer/PR-2159.md), [Add data type check for deepseek fp4 moe](../sources/prs/flashinfer/PR-2165.md), [Rename noauxtc to fused_topk_deepseek](../sources/prs/flashinfer/PR-2181.md), [Fix for moe on sm110](../sources/prs/flashinfer/PR-2190.md), [feat: unit-test and api change, w4a8 grouped-gemm fused MoE for SM90](../sources/prs/flashinfer/PR-2193.md), [feat: Support unpadded output hidden size for trtllm_fp4_block_scale_moe](../sources/prs/flashinfer/PR-2217.md), [fix: add DeepSeek routing for Bf16xBf16 and MxIntxBf16 TRT-LLM Gen MoE](../sources/prs/flashinfer/PR-2234.md), [refactor: pull trtllm-gen batch-gemm/gemm headers from artifactory; update tma descriptor shape init](../sources/prs/flashinfer/PR-2235.md), [test: Fix MNNVL tests to skip when container lacks SYS_PTRACE capability](../sources/prs/flashinfer/PR-2245.md), [feat: Support numLocalTokens=0 for moe All-to-all](../sources/prs/flashinfer/PR-2247.md), [feat: Support Fused MoE non gated Relu2 NVFP4 & FP8 and support Nemotron](../sources/prs/flashinfer/PR-2304.md), [feat: expose swizzled_input_sf parameter for CUTLASS fused MOE](../sources/prs/flashinfer/PR-2330.md), [Optimize quantization function in large problem size](../sources/prs/flashinfer/PR-2343.md), [feat: cuteDSL fp4 moe for better DSR1 performance.](../sources/prs/flashinfer/PR-2398.md), [feat: update trtllm-gen MoE cubins](../sources/prs/flashinfer/PR-2416.md), [bugfix: fix stub generation directory in fused_moe module](../sources/prs/flashinfer/PR-2445.md), [feat: Support Fused MoE non gated Relu2 NVFP4 & FP8 and support Nemotron, fixed](../sources/prs/flashinfer/PR-2462.md), [fix: blockscale moe routine supports non-DS routing](../sources/prs/flashinfer/PR-2476.md), [fix: add support check for gemm config for cutlass moe](../sources/prs/flashinfer/PR-2495.md), [Feat: Trtllm-gen MxFP8 MoE integration](../sources/prs/flashinfer/PR-2505.md), [fix: W4A8 autotune crash in cutlass_fused_moe profiler workspace](../sources/prs/flashinfer/PR-2564.md), [Implement `cutlass_fused_moe` mxfp8](../sources/prs/flashinfer/PR-2581.md), [tests: add bias testing to nvfp4 moe](../sources/prs/flashinfer/PR-2585.md), [fix: trtllm_mxint4_block_scale_moe unit test to index output list](../sources/prs/flashinfer/PR-2627.md), [fix: cute dsl nvfp4 moe routing index error](../sources/prs/flashinfer/PR-2629.md), [fix: add SM121 support to SM120 version guards](../sources/prs/flashinfer/PR-2631.md), [benchmark: Add MXFP4/MXFP8 quantization mode support to FP4 MoE benchmark](../sources/prs/flashinfer/PR-2635.md), [[fp8_blockwise]Fix int32 overflow in TRTLLM fused MoE activation kernel](../sources/prs/flashinfer/PR-2642.md), [[feat] trtllm-gen mxfp8 gemm](../sources/prs/flashinfer/PR-2653.md), [fix: Add fused MOE and GEMM AOT modules for SM121](../sources/prs/flashinfer/PR-2654.md), [feat: Add support for TRTLLM MXFP8 non-gated MoE with ReLU2](../sources/prs/flashinfer/PR-2707.md), [fix: Add SM120 (RTX Blackwell desktop) support for NVFP4 MoE kernels](../sources/prs/flashinfer/PR-2725.md), [Support in-place update for `trtllm_fp8_block_scale_moe`](../sources/prs/flashinfer/PR-2739.md), [misc: Update gemm/batched gemm cubins from trtllm-gen, gemm header refactor](../sources/prs/flashinfer/PR-2740.md), [[feat] Add 2048 experts and 32 Top K ](../sources/prs/flashinfer/PR-2744.md), [Upgrade cutlass 4.2.1 -> 4.4.2](../sources/prs/flashinfer/PR-2798.md), [CuteDSL MoE fix redundant output buffer zeroing](../sources/prs/flashinfer/PR-2811.md), [fix: Autotuner _find_nearest_profile non-power-of-2 num_tokens, create launchers for all supported tileN in trtllm fused MoE](../sources/prs/flashinfer/PR-2821.md), [fix: int32 overflow in `trtllm_fp4_block_scale_moe` causing "Unsupported hidden state scale shape" for EP32+ configs](../sources/prs/flashinfer/PR-2853.md), [Add support for Relu2 in BF16 fused MoE](../sources/prs/flashinfer/PR-2864.md), [fix: add cute dsl moe utils to AOT](../sources/prs/flashinfer/PR-2872.md), [Fix silent bug with FP8 per tensor non-gated MoE](../sources/prs/flashinfer/PR-2882.md), [fix: snap weight_scale_vec_size to handle block_scale_interleave padding for SM120](../sources/prs/flashinfer/PR-2898.md), [[NVIDIA] fix(jit): enable GDC for CUTLASS fused MoE PDL — prevent random crashes on SM12x](../sources/prs/flashinfer/PR-2913.md), [fix: Fix autotuner crash on meta-device tensor in trtllm_fp4_block_scale_routed_moe](../sources/prs/flashinfer/PR-2916.md), [feat: add Relu2 (squared ReLU) activation support in CUTLASS MoE backend](../sources/prs/flashinfer/PR-2926.md), [[Perf] Refactor MoE autotuning to set valid topk ids in routed MoE tuning](../sources/prs/flashinfer/PR-2942.md), [Fused moe all-reduce routed scaling factor + quant support](../sources/prs/flashinfer/PR-2966.md), [feat(comm): add MOE Finalize/Reduction patterns to unified allreduce_fusion API](../sources/prs/flashinfer/PR-2982.md), [fix: restore SM120 CUTLASS MoE tile candidate removed by #2927 (test_trtllm_cutlass_fused_moe.py)](../sources/prs/flashinfer/PR-2984.md), [perf: Optimize CUTLASS MoE helper kernels for small-batch decode workloads](../sources/prs/flashinfer/PR-3014.md), [fix: extend moe alltoall top-k specializations](../sources/prs/flashinfer/PR-3021.md), [[feat] Add routing_replay_out support to MoE kernels and Python API](../sources/prs/flashinfer/PR-3024.md), [Prevent MoE autotuner buffer overflow on large token buckets](../sources/prs/flashinfer/PR-3025.md), [[feat] Trtllm-gen Per-token Nvfp4 MoE](../sources/prs/flashinfer/PR-3027.md), [fused_moe: pre-filter SM89 tactics with zero occupancy on SM120 Blackwell (fix review feedback on #2764)](../sources/prs/flashinfer/PR-3032.md), [feat: Add b12x CuTe DSL fused MoE for SM120](../sources/prs/flashinfer/PR-3066.md), [Integrate CUTLASS Small Tile N Blockscaled GEMMs/Grouped GEMMs for SM120 and SM121](../sources/prs/flashinfer/PR-3152.md), [fix(sm12x): fix micro-kernel workspace sizing when routed_rows > num_local_experts](../sources/prs/flashinfer/PR-3191.md), [fix(cute_dsl/moe): make autotuner bucket configuration adapt to runtime input](../sources/prs/flashinfer/PR-3216.md), [feat(trace): embed runnable init() in every TraceTemplate](../sources/prs/flashinfer/PR-3221.md), [[Bugfix] Fix fused MoE autotuning correctness issues by filtering clusterDimZ](../sources/prs/flashinfer/PR-3227.md), [Update moe gemm](../sources/prs/flashinfer/PR-3239.md), [fix(cute_dsl/moe): unbias autotuner profiling for tile_size enumeration](../sources/prs/flashinfer/PR-3252.md), [feat(moe): add SM120 W4A16 b12x kernels](../sources/prs/flashinfer/PR-3271.md), [feat(cute_dsl/moe): deterministic balanced autotune profile inputs](../sources/prs/flashinfer/PR-3286.md), [Ep api design - Build Infra dependencies](../sources/prs/flashinfer/PR-3315.md), [[feat] Add gemma RMS AR fusion](../sources/prs/flashinfer/PR-3322.md), [feat(cute_dsl/moe): add `moe_output_memset_inplace` dense memset wrapper](../sources/prs/flashinfer/PR-3328.md), [Add support for bf16 x bf16 cutlass fused MoE](../sources/prs/sglang/PR-10275.md), [Fix cutlass moe accuracy drop caused by attention UB from DP padding mode](../sources/prs/sglang/PR-10414.md), [Support single batch overlap](../sources/prs/sglang/PR-10422.md), [Fix correction bias undefined behavior for nvfp4 models](../sources/prs/sglang/PR-10426.md), [feat: add dsv3 fp4 cutlass moe etp ut](../sources/prs/sglang/PR-10433.md), [Update CUTLASS. Refine KernelSchedule for fp8 (grouped) gemm.](../sources/prs/sglang/PR-10491.md), [Fix bias handling in TritonMoeQuantInfo within quantization/mxfp4.py](../sources/prs/sglang/PR-10579.md), [support qwen3-next-fp8 deepep](../sources/prs/sglang/PR-10622.md), [Fix MTP MoE weight loading with NVFP4 target model.](../sources/prs/sglang/PR-10758.md), [Quick Fix: fix Qwen3-VL launch failure caused by MRotaryEmbedding arg](../sources/prs/sglang/PR-10985.md), [Fix DSR1 accuracy for flashinfer_trtllm MoE with FP8 quantization](../sources/prs/sglang/PR-11081.md), [[AMD] Clean up vllm dependencies in moe_runner/triton.py](../sources/prs/sglang/PR-11349.md), [[sgl-kernel][1/N]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-11432.md), [Support shared experts overlap in cutlass moe](../sources/prs/sglang/PR-11611.md), [Support running FP4 Deepseek on SM120.](../sources/prs/sglang/PR-11708.md), [support cutlass fp4 kernel in sm120](../sources/prs/sglang/PR-11737.md), [[Ascend] qwen optimization](../sources/prs/sglang/PR-12078.md), [[hotfix] missing `w13_weight_fp8` and `w2_weight_fp8` in UE8M0 requantization](../sources/prs/sglang/PR-12259.md), [Replace [silu_and_mul_]scaled_fp4_group_quant by Flashinfer equivalent](../sources/prs/sglang/PR-12376.md), [Enable Flashinfer TRTLLM-GEN-MoE FP8 blockwise kernel for Qwen3-Next on Blackwell](../sources/prs/sglang/PR-12543.md), [[CPU] Fix MoE layer support for DeepSeek-OCR models](../sources/prs/sglang/PR-12555.md), [[sgl-kernel][5/N]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-12666.md), [[Bugfix] Fix illegal memory access](../sources/prs/sglang/PR-12758.md), [[Ascend] support Kimi-K2-Thinking](../sources/prs/sglang/PR-12759.md), [ignore the deepgemm check when the model weight with nvfp4 and moe ba…](../sources/prs/sglang/PR-12782.md), [Apply moe_reduce_sum kernel for fused_marlin_moe](../sources/prs/sglang/PR-12888.md), [Support moe topk sigmoid kernel](../sources/prs/sglang/PR-13049.md), [support mtp with deepseek r1 nvfp4 model](../sources/prs/sglang/PR-13115.md), [[NPU]Optimization of `forward_npu` for `UnquantizedFusedMoEMethod`](../sources/prs/sglang/PR-13158.md), [Fix nan in global scaling factor for large scale nvfp4 EP](../sources/prs/sglang/PR-13162.md), [[NVIDIA] Fix broken fp8 MoE of deepseek v3](../sources/prs/sglang/PR-13264.md), [Flashinfer TRTLLM-GEN-MoE + Qwen3](../sources/prs/sglang/PR-13489.md), [Fix EPLB + FP4 Quantization Compatibility Issue](../sources/prs/sglang/PR-13715.md), [[bugfix] fix TBO crashes when attn_tp_size > 1](../sources/prs/sglang/PR-13730.md), [[sgl-kernel][Feat][B200][1/N]Support MXFP8 Grouped GEMM in Blackwell](../sources/prs/sglang/PR-13731.md), [[Feat][NVFP4] Enable NVFP4 MoE for Qwen series models (eg. Qwen3-Next) #13761](../sources/prs/sglang/PR-13761.md), [Support fp4 fp8 non gated moe](../sources/prs/sglang/PR-13794.md), [[NVIDIA] Enable TRTLLM BF16 MoE on Blackwell GPUs](../sources/prs/sglang/PR-13798.md), [[BugFix] fix outplace_fused_experts missing is_gated](../sources/prs/sglang/PR-13864.md), [Feat: GLM-4.6 supports shared experts fusion](../sources/prs/sglang/PR-13873.md), [[DeepSeek v3.2] opt Context Parallelism: support fused moe, multi batch and fp8 kvcache](../sources/prs/sglang/PR-13959.md), [[kernel][moe] add moe topk fast](../sources/prs/sglang/PR-13969.md), [Support KTransformers for Qwen3-VL moe](../sources/prs/sglang/PR-13983.md), [Fix flashinfer cutlass MoE output shape for non-FP4-packed inputs](../sources/prs/sglang/PR-14028.md), [[LoRA][III] Add LoRA support for MoE layers and enable TP](../sources/prs/sglang/PR-14105.md), [Add new moe wna16 marlin gemm](../sources/prs/sglang/PR-14122.md), [Apply new moe wna16 marlin gemm](../sources/prs/sglang/PR-14125.md), [Opt moe align block size kernel](../sources/prs/sglang/PR-14133.md), [Apply new moe align block size kernel](../sources/prs/sglang/PR-14134.md), [Add Mistral Large 3 support.](../sources/prs/sglang/PR-14213.md), [[FIX] trtllm-moe-fp4-renorm for Qwen series models](../sources/prs/sglang/PR-14350.md), [[NPU] perf update with kvcache nz & w4a8 quant](../sources/prs/sglang/PR-14423.md), [Mistral Large 3 NVFP4 support](../sources/prs/sglang/PR-14485.md), [Apply back moe_sum_reduce for fused_marlin_moe](../sources/prs/sglang/PR-14829.md), [add transformers version validation for glm-4.6v moe models](../sources/prs/sglang/PR-14998.md), [Mistral Large 3 NVFP4 TRTLLM MoE support](../sources/prs/sglang/PR-15049.md), [[sgl-kernel][1/2] Fused qk_norm_rope for GLM4.6](../sources/prs/sglang/PR-15141.md), [[NVIDIA] Fixes for NVFP4 all-gather with spec decoding](../sources/prs/sglang/PR-15280.md), [[Fix] A followup fix for TRTLLM BF16 MoE](../sources/prs/sglang/PR-15303.md), [Fix the accuracy issue when running mxfp4 dsv3 model and enable ep](../sources/prs/sglang/PR-15304.md), [Fix warp illegal instruction in kimi k2 thinking PCG](../sources/prs/sglang/PR-15306.md), [[distributed] Clean up MoE groups in destroy_model_parallel](../sources/prs/sglang/PR-15345.md), [[NPU]mindspore model support moe](../sources/prs/sglang/PR-15363.md), [Super tiny add moe_ep_rank to prometheus labels](../sources/prs/sglang/PR-15407.md), [Flashinfer MOE FP8 support for Mistral Large 3.](../sources/prs/sglang/PR-15422.md), [Optimize Bailing-MoE with FlashInfer Fused All-Reduce](../sources/prs/sglang/PR-15526.md), [MoE: Skip SiLU/GELU activation for masked experts](../sources/prs/sglang/PR-15539.md), [Update flashinfer to 0.6.1](../sources/prs/sglang/PR-15551.md), [Add SwapAB Optimization for triton fused_moe_kernel on SM90.](../sources/prs/sglang/PR-15712.md), [[Perf] Eliminate the slice op for Flashinfer `trtllm_fp4_block_scale_moe`](../sources/prs/sglang/PR-15731.md), [Fix GLM-4.7 MoE Detector complex JSON Schema type parsing](../sources/prs/sglang/PR-15753.md), [Fix: Handle empty func_name and None values in GLM MoE detectors](../sources/prs/sglang/PR-15754.md), [[Feature] JIT Fused QK norm + qk norm clean up](../sources/prs/sglang/PR-15835.md), [[JIT kernel] Apply jit per_tensor_quant_fp8 kernel](../sources/prs/sglang/PR-15836.md), [[NPU] NZ for non-quantized MOE, Qwen3 MOE double memory consumption fix](../sources/prs/sglang/PR-15904.md), [ Add tuned triton==3.5.1 h200 tp2, tp4 for qwen 3 next](../sources/prs/sglang/PR-15948.md), [enhance accuracy for model kimi-vl-instruct-a3b](../sources/prs/sglang/PR-16076.md), [fix layer intermediate size](../sources/prs/sglang/PR-16084.md), [[NemotronH] Add latent MoE support](../sources/prs/sglang/PR-16227.md), [Fix FP8 MoE NaN with DeepGEMM on Blackwell](../sources/prs/sglang/PR-16622.md), [[Rework] Add SwapAB Optimization for triton fused_moe_kernel on SM90.](../sources/prs/sglang/PR-16723.md), [[AMD] Support redundant expert with a2a moe in gfx95x.](../sources/prs/sglang/PR-16791.md), [[Fix] `flashinfer_trtllm` `intermediate_size` assertion with Qwen3 + TP=8](../sources/prs/sglang/PR-16824.md), [Support mxint4 flashinfer_trtllm moe gemm](../sources/prs/sglang/PR-16892.md), [Inclusion of nvfp4 blockscale in EPLB Rebalance](../sources/prs/sglang/PR-17158.md), [[Fix] GLM 4.7 + NVFP4 + MTP](../sources/prs/sglang/PR-17166.md), [[GLM 4.7] Add RTX 6000 Pro aka sm120](../sources/prs/sglang/PR-17235.md), [[New Model] GLM4.7-Flash](../sources/prs/sglang/PR-17247.md), [Add mxfp8 support for online quantization, Triton dense linear, and CUTLASS MoE](../sources/prs/sglang/PR-17449.md), [Upgrade transformers==5.3.0](../sources/prs/sglang/PR-17784.md), [[Bugfix] Fix Mistral Large 3 NVFP4 TRTLLM MoE](../sources/prs/sglang/PR-18065.md), [Fix nvfp4 weight update](../sources/prs/sglang/PR-18085.md), [[Blackwell] Make mxint4 flashinfer_trtllm moe gemm set by default on blackwell](../sources/prs/sglang/PR-18136.md), [[ModelOpt] Fix broken Qwen3-235B-A22B-Instruct-2507-NVFP4 launch](../sources/prs/sglang/PR-18189.md), [Support Qwen3 MoE context parallel](../sources/prs/sglang/PR-18233.md), [[sglang-miles] True on-policy training support for FSDP2](../sources/prs/sglang/PR-18639.md), [[Perf] ~9.5x faster Blackwell MXFP4 MoE weight loading](../sources/prs/sglang/PR-18858.md), [[Sarvam] Add inference support for Sarvam MoE LLMs](../sources/prs/sglang/PR-18938.md), [[jit_kernel] Add fused_qknorm_rope JIT kernel](../sources/prs/sglang/PR-19059.md), [Adjust padding size to improve triton_kernels moe performance](../sources/prs/sglang/PR-19174.md), [Fix/nemotron mtp quantaized](../sources/prs/sglang/PR-19433.md), [[Kernel Slimming] Migrate NVFP4 kernels to JIT](../sources/prs/sglang/PR-19437.md), [[FlashInfer v0.6.4] [RL] Integrate FlashInfer mxfp8 gemm, MoE, and routed MoE](../sources/prs/sglang/PR-19537.md), [[Feature] NVFP4 Marlin fallback for non-Blackwell GPUs (SM75+)](../sources/prs/sglang/PR-19652.md), [Use TRTLLM allreduce fusion for Qwen 3.5](../sources/prs/sglang/PR-19889.md), [[JIT Kernel] Reland NVFP4 kernels to JIT](../sources/prs/sglang/PR-20012.md), [[4/n jit_kernel restruct] speed up CI tests and add benchmark workflow](../sources/prs/sglang/PR-20268.md), [[Benchmark] use flashinfer bench_gpu_time instead of triton do_bench](../sources/prs/sglang/PR-20305.md), [[NVIDIA] Enable fp8 flashinfer_trtllm_routed MoE for MiniMax-M2.5](../sources/prs/sglang/PR-20394.md), [Add Mistral Small 4 (Pixtral) support](../sources/prs/sglang/PR-20708.md), [Use FlashInfer tinygemm for GPT-OSS MoE router on SM90+](../sources/prs/sglang/PR-20755.md), [Add SGLang CUDA crash API logging inspired by FlashInfer](../sources/prs/sglang/PR-20910.md), [[refactor] Clean up duplicate flashinfer trtllm moe code](../sources/prs/sglang/PR-21233.md), [Refactor JIT kernel CI to use run_suite.py registration system](../sources/prs/sglang/PR-21239.md), [[NVIDIA] Enable FP4 flashinfer trtllm routed moe](../sources/prs/sglang/PR-21240.md), [P2P Weight Update features for miles ](../sources/prs/sglang/PR-21278.md), [[RL] Support mxfp8 DeepSeek V3](../sources/prs/sglang/PR-21280.md), [[Kernel] Support FlashInfer TRTLLM-Gen fused MoE for non-gated FP4 & FP8 (Nemotron)](../sources/prs/sglang/PR-21321.md), [[misc] clean up kernel API](../sources/prs/sglang/PR-21325.md), [Add dedicated FlashInferCuteDslMoE layer for standard-path FP4 MoE](../sources/prs/sglang/PR-21339.md), [[jit_kernel] Optimize fused_qknorm_rope: deduplicate sincosf for interleave RoPE ](../sources/prs/sglang/PR-21654.md), [[AMD] Use tgemm.mm for MoEGate router gemm in deepseek_v2.py](../sources/prs/sglang/PR-21657.md), [Harden FlashInfer FP4 imports in standard dispatcher](../sources/prs/sglang/PR-21776.md), [[Fix] Fall back to triton MOE for GPT-OSS on Blackwell with driver >= 595](../sources/prs/sglang/PR-21780.md), [Remove redundant test_moe_eval_accuracy_large](../sources/prs/sglang/PR-21787.md), [Tiny fix trtllm_fp8_per_tensor_scale_moe_wrapper router_logits dtype](../sources/prs/sglang/PR-22006.md), [[RL] Refactor NVFP4 shuffling/swizzling to in-place replacement](../sources/prs/sglang/PR-22204.md), [[Reland] DeepSeek-R1-0528-w4a8: DeepEP Low Latency Dispatch Adopts FP8 Communication](../sources/prs/sglang/PR-22316.md), [[Lora] Lora quat info re-factor and support deepseekv3 mla lora](../sources/prs/sglang/PR-22323.md), [[Lora] Lora kimi support](../sources/prs/sglang/PR-22381.md), [feat: Support flashinfer_cutedsl MoE runner with flashinfer alltoall backend](../sources/prs/sglang/PR-22669.md), [[Step3p5] Optimize allreduce in MoE layers ](../sources/prs/sglang/PR-22773.md), [Dual MoE CUDA graph capture for lora/nolora batches](../sources/prs/sglang/PR-22809.md), [[Refactor] Refactor DeepEP dispatcher](../sources/prs/sglang/PR-22822.md), [[FlashInfer v0.6.11] [RL] Support FlashInfer per-token NVFP4 MoE](../sources/prs/sglang/PR-22918.md), [Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-23686.md), [Use Cute-DSL NVFP4 quantization kernels](../sources/prs/sglang/PR-23745.md), [[MoE] Unify DeepEPMoE+MoriEPMoE through AITER MoeRunner pre/post-permute](../sources/prs/sglang/PR-23760.md), [[feat] Init true on policy with qwen_dense](../sources/prs/sglang/PR-23961.md), [Port MXFP4 Marlin MoE support to JIT kernel path](../sources/prs/sglang/PR-24490.md), [Fix performance regression on Deepseek V3 on `moe-runner-backend=triton` on SM90](../sources/prs/sglang/PR-24562.md), [[Codex] Opt Mistral Large performace ](../sources/prs/sglang/PR-24611.md), [Add FlashInfer SM90 cutlass MXFP4 MoE backend (W4A16) for GPT-OSS + DeepSeek-V4](../sources/prs/sglang/PR-24816.md), [Amd/deepseek v4 rebase main 0509](../sources/prs/sglang/PR-24933.md), [[rebase]Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-24986.md), [Fix EPLB mapping for TopK paths](../sources/prs/sglang/PR-25285.md), [[Intel GPU] Enable DeepSeek V4 Inference on XPU](../sources/prs/sglang/PR-25336.md), [[AMD] test(sgl-kernel): seed RNG on ROCm in test_moe_topk_sigmoid to fix tie-break flake](../sources/prs/sglang/PR-25356.md), [Update logging for inplace setting in MoE layer](../sources/prs/sglang/PR-25499.md), [[misc] Throw error when single batch overlap is enabled on Hopper ](../sources/prs/sglang/PR-25509.md), [Fix logging for inplace setting in the flashInfer-trtllm backend](../sources/prs/sglang/PR-25522.md), [[Bug Fix] Align glm4_moe_nextn NPU MTP loading with qwen3 MTP](../sources/prs/sglang/PR-25524.md), [[MoE Refactor] Migrate flashinfer_cutedsl + DeepEP to MoeRunner](../sources/prs/sglang/PR-25525.md), [Use DeepGEMM BF16 for unquantized DeepEP LL MoE](../sources/prs/sglang/PR-25540.md), [Add DeepSeekV4 fused MoE Triton autotune support](../sources/prs/sglang/PR-25569.md), [[Benchmark] Add SGLANG_SIMULATE_UNIFORM_EXPERTS for balanced expert routing with dummy weights](../sources/prs/sglang/PR-25571.md), [[SP] Fix runtime_max_tokens_per_rank for sequence parallelism](../sources/prs/sglang/PR-25685.md), [Add no_combine support to cutlass_moe_fp4](../sources/prs/sglang/PR-25688.md), [[Refactor] Pass PP start_layer via model constructor instead of forward_batch.token_to_kv_pool](../sources/prs/sglang/PR-25825.md), [[Test] Stage-a sanity kits; consolidate core/ + models_e2e/ tests](../sources/prs/sglang/PR-25831.md), [feat: integrate gemm_fp8 kernel into gemm](../sources/prs/sglang/PR-3056.md), [add tensorrt_llm common and cutlass_extensions as 3rdparty](../sources/prs/sglang/PR-3216.md), [integrate blockwise fp8 kernel](../sources/prs/sglang/PR-3529.md), [Feature DeepSeek V3/R1 INT8 Quantization (block-wise)](../sources/prs/sglang/PR-3730.md), [[Feature] DeepSeek V3/R1 INT8 Quantization (channel-wise) ](../sources/prs/sglang/PR-3888.md), [Accelerate FP8 CUDA Kernel by 20-28%](../sources/prs/sglang/PR-4215.md), [[Feature] Integrate DeepEP into SGLang](../sources/prs/sglang/PR-4232.md), [Add deepseek style fused moe group gate selection kernel](../sources/prs/sglang/PR-4530.md), [Optimize Permute Kernel in DeepEP](../sources/prs/sglang/PR-4643.md), [Fix loading KV quantization scale; Enable modelopt kv cache](../sources/prs/sglang/PR-4686.md), [[Model] Adding Qwen3 and Qwen3MoE](../sources/prs/sglang/PR-4693.md), [support cmake for sgl-kernel](../sources/prs/sglang/PR-4706.md), [[Feature] Support DeepEP Low Latency](../sources/prs/sglang/PR-4767.md), [Support (1 <= dp < tp) in the dp attention in DeepEP](../sources/prs/sglang/PR-4770.md), [Introduce moe_dense_tp_size to fix dense layer errors in DeepSeek V3 + 4x8xH100](../sources/prs/sglang/PR-4836.md), [Add DeepSeek V3/R1 shared experts fusion](../sources/prs/sglang/PR-4918.md), [[Fix] DeepEP Compatibility with Low Latency](../sources/prs/sglang/PR-5068.md), [reduce moe_align_block_size_kernel small batch mode overhead](../sources/prs/sglang/PR-5086.md), [Add optimized native kernels in sgl-kernel](../sources/prs/sglang/PR-5150.md), [[1/2] Add FP8 Blockscale MoE CUTLASS kernel for Blackwell](../sources/prs/sglang/PR-5281.md), [Fix DeepGEMM masked cannot be run on groups not being multiple or 4](../sources/prs/sglang/PR-5340.md), [apply fused moe gate in ds v3/r1](../sources/prs/sglang/PR-5371.md), [Add Cutlass MLA attention backend](../sources/prs/sglang/PR-5390.md), [[perf] introduce deep gemm group_gemm_masked as gemm](../sources/prs/sglang/PR-5432.md), [ DeepEP normal support deepgemm-contiguous](../sources/prs/sglang/PR-5626.md), [[2/2] Add python wrapper for CUTLASS FP8 Blockscale MoE Kernel. ](../sources/prs/sglang/PR-5694.md), [[qwen3] support qwen3 ep moe](../sources/prs/sglang/PR-5917.md), [Support tuning moe for llama 4 model](../sources/prs/sglang/PR-6042.md), [[1/2] Add Kernel support for Cutlass based Fused FP4 MoE](../sources/prs/sglang/PR-6093.md), [feat: add dp attention support for Qwen 2/3 MoE models, fixes #6088](../sources/prs/sglang/PR-6121.md), [Reduce MoE memory usage](../sources/prs/sglang/PR-6147.md), [enable auto-round quantization model](../sources/prs/sglang/PR-6226.md), [fix: enable multi-GPU Triton fused MoE tuning](../sources/prs/sglang/PR-6295.md), [reduce torch.zeros overhead in moe align block size kernel](../sources/prs/sglang/PR-6369.md), [Add fp8 fused_experts kernel for CPU in sgl-kernel and add UT](../sources/prs/sglang/PR-6404.md), [Fix topk inference performance reduce](../sources/prs/sglang/PR-6474.md), [qwen3moe support two batch overlap](../sources/prs/sglang/PR-6598.md), [Refine pre_reorder_triton_kernel slightly to improve performance](../sources/prs/sglang/PR-6627.md), [[CPU] [BF16] Call fused_experts_cpu, weight_packed_linear and bmm_cpu kernel in DeepSeek model](../sources/prs/sglang/PR-6641.md), [Fix DeepEP error in Qwen 3 MoE models](../sources/prs/sglang/PR-6673.md), [[EP] Add cuda kernel for moe_ep_pre_reorder](../sources/prs/sglang/PR-6699.md), [Fix PP for Qwen3 MoE](../sources/prs/sglang/PR-6709.md), [Set `num_fused_shared_experts` as `num_shared_experts` when shared_experts fusion is not disabled](../sources/prs/sglang/PR-6736.md), [[CPU] add optimizations for INT8 and FP8 DeepSeek](../sources/prs/sglang/PR-6769.md), [[CPU] support the case where num_attention_heads or intermediate_size is not divisible by the TP size](../sources/prs/sglang/PR-6771.md), [Support token-level quantization for EP MoE](../sources/prs/sglang/PR-6782.md), [feat: integrate deepgemm into EPMoE](../sources/prs/sglang/PR-6821.md), [CPU: map changes from developing branch in sgl-kernel](../sources/prs/sglang/PR-6833.md), [[EP] Add cuda kernel for moe_ep_post_reorder](../sources/prs/sglang/PR-6837.md), [[DeepseekR1-FP4] Add Support for nvidia/DeepSeekR1-FP4 model](../sources/prs/sglang/PR-6853.md), [fix ep_moe_reorder kernel bugs](../sources/prs/sglang/PR-6858.md), [Add a CUDA kernel for fusing mapping and weighted sum for MoE.](../sources/prs/sglang/PR-6916.md), [[sgl-kernel] Add cuda kernel for moe_ep_silu_and_mul](../sources/prs/sglang/PR-6919.md), [chore: upgrade flashinfer v0.2.6.post1 jit](../sources/prs/sglang/PR-6958.md), [Fuse routed scaling factor in deepseek](../sources/prs/sglang/PR-6970.md), [Update default settings for blackwell](../sources/prs/sglang/PR-7023.md), [Fix positional argument](../sources/prs/sglang/PR-7093.md), [fix amd EP MoE FP8 issue](../sources/prs/sglang/PR-7125.md), [Enable ModelOpt Llama4 fp8 checkpoint deployment in SGLang](../sources/prs/sglang/PR-7129.md), [[amd] Opt dsv3 moe](../sources/prs/sglang/PR-7160.md), [Support new DeepGEMM](../sources/prs/sglang/PR-7172.md), [Fix error when disabling new DeepGEMM](../sources/prs/sglang/PR-7198.md), [[fix] fix DeepGEMM blackwell input quant & ut & fix style and log](../sources/prs/sglang/PR-7247.md), [[AMD] add aiter fused moe in DeepEP path](../sources/prs/sglang/PR-7268.md), [Add CUTLASS FP8 Blockscale MoE kernel for Hopper architecture](../sources/prs/sglang/PR-7278.md), [Let EP prefill support new DeepGEMM](../sources/prs/sglang/PR-7310.md), [FlashInfer NVFP4 MoE with EP & 2-stream shared expert](../sources/prs/sglang/PR-7327.md), [Fix MTP with Deepseek R1 Fp4](../sources/prs/sglang/PR-7376.md), [Fix torch compile run](../sources/prs/sglang/PR-7391.md), [[AMD][Quantization] Add `int4fp8_moe` online quantization on ROCm](../sources/prs/sglang/PR-7392.md), [Fuse sorted_token_ids padding to moe_align_block_size kernel](../sources/prs/sglang/PR-7437.md), [fix: fix apply_shuffle_mul_sum](../sources/prs/sglang/PR-7444.md), [Add Tencent HunYuanMoEV1 model support](../sources/prs/sglang/PR-7549.md), [[Feature] Layer-wise Prefill](../sources/prs/sglang/PR-7634.md), [Add fp4 quantize before all-gather for Flashinfer cutlass MoE DP (max throughput)](../sources/prs/sglang/PR-7667.md), [Integrate triton moe kernel](../sources/prs/sglang/PR-7689.md), [feat: support DeepSeek-R1-W4AFP8 model with ep-moe mode](../sources/prs/sglang/PR-7762.md), [[1/n]: add cutlass W4A8 moe kernel for hopper architecture](../sources/prs/sglang/PR-7772.md), [[kernel] opt moe align block kernel by block/warp scan algorithm](../sources/prs/sglang/PR-7884.md), [[feat] Support tp mode for DeepSeek-R1-W4AFP8](../sources/prs/sglang/PR-8118.md), [[1/N]Support DeepSeek-R1 w4a8 normal deepep](../sources/prs/sglang/PR-8247.md), [Support triton kernels v3.4.0 for fused_moe](../sources/prs/sglang/PR-8258.md), [[2/N]Support DeepSeek-R1 w4a8 low latency deepep](../sources/prs/sglang/PR-8464.md), [Update cutlass_moe.py](../sources/prs/sglang/PR-8535.md), [Update cutlass_moe.py](../sources/prs/sglang/PR-8545.md), [[NVIDIA] Add Low Latency NVFP4 decode kernels from Flashinfer](../sources/prs/sglang/PR-8552.md), [feat: support cutlass_moe_fp8 kernel for fusedmoe in sm90](../sources/prs/sglang/PR-8678.md), [fuse allreduce and residual_rmsnorm](../sources/prs/sglang/PR-8731.md), [[Perf] Auto enable best flashinfer mxfp4 kernel in b200](../sources/prs/sglang/PR-8898.md), [[NVIDIA] Fix missing `get_col_major_tma_aligned_tensor` for Blackwell deepgemm in EpMoE](../sources/prs/sglang/PR-8955.md), [optimize: reduce shulffle and quantization overhead in cutlass_moe sm90](../sources/prs/sglang/PR-8962.md), [Faster weight processing (trtllm-gen moe nvfp4)](../sources/prs/sglang/PR-9162.md), [[NVIDIA] [3/N] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked ](../sources/prs/sglang/PR-9199.md), [[fix]: fix cutlass moe ut and and Opt H20 cutlass groupGemm performance](../sources/prs/sglang/PR-9272.md), [[fix] Fix mxfp4 triton MoE tp bug](../sources/prs/sglang/PR-9473.md), [Optimize moe_sum_reduce_kernel](../sources/prs/sglang/PR-9477.md), [Update CUTLASS 4.2 & Enable K-Major Scale Factor for SM90 FP8 Blockwise Group GEMM](../sources/prs/sglang/PR-9559.md), [Single Batch Overlap for MoE Models](../sources/prs/sglang/PR-9660.md), [move is_sm90_supported/is_sm100_supported to python/sglang/srt/utils.py](../sources/prs/sglang/PR-9679.md), [[CPU] Add FP8 Bmm support](../sources/prs/sglang/PR-9744.md), [Make sm100 fp8 kernels available on sm103](../sources/prs/sglang/PR-9789.md), [[Model] Support Meituan LongCat-Flash && LongCat-Flash-MTP](../sources/prs/sglang/PR-9824.md), [[ROCm][MoE] moe tuning support for rocm](../sources/prs/vllm/PR-12049.md), [[Kernel] add triton fused moe kernel for gptq/awq](../sources/prs/vllm/PR-12185.md), [[Hardware][Gaudi][Feature] Enable Dynamic MoE for Mixtral](../sources/prs/vllm/PR-12303.md), [[Misc][MoE] add Deepseek-V3 moe tuning support](../sources/prs/vllm/PR-12558.md), [[Kernel] port sgl moe_align_block_size kernels](../sources/prs/vllm/PR-12574.md), [Expert Parallelism (EP) Support for DeepSeek Models](../sources/prs/vllm/PR-12583.md), [Apply torch.compile to fused_moe/grouped_topk](../sources/prs/vllm/PR-12637.md), [[Misc] Update w2 scale loading for GPTQMarlinMoE](../sources/prs/vllm/PR-12757.md), [Optimize moe_align_block_size for deepseek_v3](../sources/prs/vllm/PR-12850.md), [[Model] Deepseek GGUF support ](../sources/prs/vllm/PR-13167.md), [[Quant][Perf] Use moe_wna16 kernel by default for MoEs with many experts](../sources/prs/vllm/PR-13236.md), [[Kernel] moe wna16 cuda kernel](../sources/prs/vllm/PR-13321.md), [[ROCm][MoE] mi300 mixtral8x7B perf for specific BS](../sources/prs/vllm/PR-13577.md), [[Kernel] Optimize moe intermediate_cache usage](../sources/prs/vllm/PR-13625.md), [[BugFix] Illegal memory access for MoE On H20](../sources/prs/vllm/PR-13693.md), [[core] Perf improvement for DSv3 on AMD GPUs](../sources/prs/vllm/PR-13718.md), [Fix CompressedTensorsWNA16MoE with grouped scales](../sources/prs/vllm/PR-13769.md), [Fix precommit fail in fused_moe intermediate_cache2 chunking](../sources/prs/vllm/PR-13772.md), [[Bugfix][Quantization] Fix FP8 + EP](../sources/prs/vllm/PR-13784.md), [[V1] EP/TP MoE + DP Attention](../sources/prs/vllm/PR-13931.md), [[Kernel] CUTLASS grouped gemm fp8 MoE kernel](../sources/prs/vllm/PR-13972.md), [[Misc] Print FusedMoE detail info](../sources/prs/vllm/PR-13974.md), [[core] moe fp8 block quant tuning support](../sources/prs/vllm/PR-14068.md), [dynamic distpatch of fp8 kernels](../sources/prs/vllm/PR-14245.md), [[Misc] Add Qwen2MoeForCausalLM moe tuning support ](../sources/prs/vllm/PR-14276.md), [[Kernel] moe wna16 marlin kernel](../sources/prs/vllm/PR-14447.md), [[ROCm][Kernel] MoE weights padding](../sources/prs/vllm/PR-14454.md), [permute/unpermute kernel for moe optimization](../sources/prs/vllm/PR-14568.md), [[Kernel] GGUF MoE kernel](../sources/prs/vllm/PR-14613.md), [[Bugfix][IPEX] Add `VLLM_CPU_MOE_PREPACK` to allow disabling MoE prepack when CPU does not support it](../sources/prs/vllm/PR-14681.md), [[FEAT][ROCm] Integrate Fused MoE Kernels from AITER](../sources/prs/vllm/PR-14967.md), [[Model] Add Qwen3 and Qwen3MoE](../sources/prs/vllm/PR-15289.md), [[Kernel] Fix conflicting macro names for gguf kernels](../sources/prs/vllm/PR-15456.md), [Use Cache Hinting for fused_moe kernel](../sources/prs/vllm/PR-15511.md), [[moe][quant] add weight name case for offset](../sources/prs/vllm/PR-15515.md), [[Quantization] Fp8 Channelwise Dynamic Per Token GroupedGEMM](../sources/prs/vllm/PR-15587.md), [[V1] TPU - Fix fused MOE](../sources/prs/vllm/PR-15834.md), [[Hardware][Gaudi][BugFix] fix arguments of hpu fused moe](../sources/prs/vllm/PR-15945.md), [Modularize fused experts and integrate PPLX kernels](../sources/prs/vllm/PR-15956.md), [[NVIDIA] Support Cutlass MLA for Blackwell GPUs](../sources/prs/vllm/PR-16032.md), [[Kernel] Use moe_wna16 kernel for compressed tensors wna16 moe models](../sources/prs/vllm/PR-16038.md), [[Kernel][Bugfix] Re-fuse triton moe weight application](../sources/prs/vllm/PR-16071.md), [Upstream Llama4 Support to Main](../sources/prs/vllm/PR-16113.md), [[Bug] [ROCm] Fix Llama 4 Enablement Bug on ROCm: V0 ROCmFlashAttentionImpl and Triton Fused MoE bugs](../sources/prs/vllm/PR-16198.md), [[Model] use AutoWeightsLoader for phimoe,qwen2_moe,qwen3_moe](../sources/prs/vllm/PR-16203.md), [[Hardware][AMD] Improve OAM device ID + llama4 Maverick MOE tuning](../sources/prs/vllm/PR-16263.md), [[Hardware/NVIDIA/Kernel] [Functional Enablement] [1/N] Enable nvidia/DeepSeek-R1-FP4 Model](../sources/prs/vllm/PR-16362.md), [[Kernel] Support W8A8 channel-wise weights and per-token activations in triton fused_moe_kernel](../sources/prs/vllm/PR-16366.md), [Enable PTPC FP8 for CompressedTensorsW8A8Fp8MoEMethod (triton fused_moe)](../sources/prs/vllm/PR-16537.md), [[ROCM] enable aiter fused moe kernel for llama4 bf16 checkpoints](../sources/prs/vllm/PR-16674.md), [[ROCm] Add aiter tkw1 kernel for Llama4 fp8](../sources/prs/vllm/PR-16727.md), [Support W8A8 INT8 MoE for compressed-tensors](../sources/prs/vllm/PR-16745.md), [[FEAT] [ROCm]: AITER Fused MOE V1 Support](../sources/prs/vllm/PR-16752.md), [[misc] ignore marlin_moe_wna16 local gen codes](../sources/prs/vllm/PR-16760.md), [[Kernel] GGUF MoeVec kernel](../sources/prs/vllm/PR-16780.md), [[BugFix] Accuracy fix for llama4 int4 - improperly casted scales](../sources/prs/vllm/PR-16801.md), [[Kernel] some optimizations for dense marlin and moe marlin](../sources/prs/vllm/PR-16850.md), [[Bugfix] Fix moe weight losing all extra attrs after `process_weights_after_loading`.](../sources/prs/vllm/PR-16854.md), [[Kernel] Add expert_map support to Cutlass FP8 MOE](../sources/prs/vllm/PR-16861.md), [Update Qwen1.5-MoE-W4A16-compressed-tensors.yaml](../sources/prs/vllm/PR-16946.md), [Fix `numel()` downcast in vllm/csrc/moe/moe_align_sum_kernels.cu +2](../sources/prs/vllm/PR-17082.md), [[FEAT] [ROCm]: Add AITER CK 2 Stages MoE support](../sources/prs/vllm/PR-17110.md), [[FEAT][ROCm]: Support AITER MLA on V1 Engine](../sources/prs/vllm/PR-17523.md), [[Kernel] fp4 marlin kernel](../sources/prs/vllm/PR-17687.md), [[BugFix][AMD] Compatible patch for AITER lib after 04/20](../sources/prs/vllm/PR-17912.md), [Fix Broken macro for cutlass moe](../sources/prs/vllm/PR-18049.md), [[Model]: Fused MoE for nomic-embed-text-v2-moe](../sources/prs/vllm/PR-18321.md), [[Feature] Expert Parallelism Load Balancer (EPLB)](../sources/prs/vllm/PR-18343.md), [[Bug] Fix moe_sum signature](../sources/prs/vllm/PR-18440.md), [[V1] Support `LLM.apply_model`](../sources/prs/vllm/PR-18465.md), [[Kernel] Integrate CUTLASS MoE kernel with PPLX](../sources/prs/vllm/PR-18762.md), [[Kernel] Enable fp8 support for pplx and BatchedTritonExperts.](../sources/prs/vllm/PR-18864.md), [[ROCm] [AITER] [Bugfix] Patch for AITER commit `648764942e552a8bb5fe16026703716a81f05374`](../sources/prs/vllm/PR-18990.md), [[Hardware][NVIDIA] FP4 MoE kernel optimization](../sources/prs/vllm/PR-19110.md), [[Kernels] Add activation chunking logic to FusedMoEModularKernel](../sources/prs/vllm/PR-19168.md), [[Kernel] Apply torch.Tag.needs_fixed_stride_order only for torch==2.6.0](../sources/prs/vllm/PR-19346.md), [[Hardware][NVIDIA][kernel] Fp4 MOE quant kernel optimization](../sources/prs/vllm/PR-19500.md), [Only build CUTLASS MoE kernels on Hopper](../sources/prs/vllm/PR-19648.md), [[Kernels] Use empty for modular MoE workspaces](../sources/prs/vllm/PR-19667.md), [[feat]: CUTLASS block scaled group gemm for SM100](../sources/prs/vllm/PR-19757.md), [[Feature] Integrate new deepgemm](../sources/prs/vllm/PR-19820.md), [[Quantization] Add compressed-tensors NVFP4 MoE Support](../sources/prs/vllm/PR-19990.md), [Enable V1 for Hybrid SSM/Attention Models](../sources/prs/vllm/PR-20016.md), [[Bugfix] Build moe_data for both sm100 and sm90](../sources/prs/vllm/PR-20086.md), [ [Feature] Integrate SM100 DeepGEMM support](../sources/prs/vllm/PR-20087.md), [Add ModelOpt Qwen3 nvfp4 support](../sources/prs/vllm/PR-20101.md), [[Bugfix] Mark 'hidden_states' as mutable in moe_forward registration.](../sources/prs/vllm/PR-20152.md), [[Bugfix] Fix topk_ids indices_type for CUTLASS w8a8 FP8 MoE](../sources/prs/vllm/PR-20166.md), [[Bugfix] Fix Maverick correctness by filling zero to cache space in cutlass_moe](../sources/prs/vllm/PR-20167.md), [[V1] [ROCm] Enable EP with AITER Fused MoE](../sources/prs/vllm/PR-20270.md), [[Kernel][Bugfix] Fixup some warnings in nvfp4_blockwise_moe when CUDA < 12.8](../sources/prs/vllm/PR-20324.md), [[Misc] DP : Add ExpertTokensMetadata](../sources/prs/vllm/PR-20332.md), [[feat]: add SM100 support for cutlass FP8 groupGEMM](../sources/prs/vllm/PR-20447.md), [Support Llama 4 for cutlass_moe_fp4](../sources/prs/vllm/PR-20453.md), [Support Llama 4 for fused_marlin_moe](../sources/prs/vllm/PR-20457.md), [[Perf] Reuse workspace for FP8+FP4 Marlin MoE](../sources/prs/vllm/PR-20500.md), [[Bugfix] Fix missing per_act_token parameter in compressed_tensors_moe](../sources/prs/vllm/PR-20509.md), [[feat] enable SM100 CUTLASS block scaled group gemm for smaller batch sizes](../sources/prs/vllm/PR-20640.md), [GLM-4.5 Model Support](../sources/prs/vllm/PR-20736.md), [[Performance] Performance improvements in non-blockwise fp8 CUTLASS MoE](../sources/prs/vllm/PR-20762.md), [[fix]: disable cutlass block scaled group gemm for EP](../sources/prs/vllm/PR-20781.md), [[Feature][EPLB] Add eplb support for Qwen3](../sources/prs/vllm/PR-20815.md), [[Bugfix] Fix a couple PPLX+CUTLASS MoE bugs](../sources/prs/vllm/PR-20825.md), [[Bug] Fix DeepGemm for EP low latency case](../sources/prs/vllm/PR-20833.md), [[Perf] Use Triton instead of Torch for DeepGEMM Per Token Group Quant](../sources/prs/vllm/PR-20841.md), [[Kernel] DeepGemm MoE : Integrate triton permute / unpermute kernels ](../sources/prs/vllm/PR-20903.md), [[Perf] Add swap_ab to SM90 FP8 non-block CUTLASS moe grouped gemm](../sources/prs/vllm/PR-20911.md), [[Misc] Qwen MoE model supports LoRA](../sources/prs/vllm/PR-20932.md), [Support mnnvl all2allv from Flashinfer](../sources/prs/vllm/PR-21003.md), [[Bugfix] Allocate less memory in non-batched CUTLASS MoE](../sources/prs/vllm/PR-21121.md), [[Feature][OCP MX] Support mxfp6 and mixed mxfp6-mxfp4](../sources/prs/vllm/PR-21166.md), [[Feature][Kernel]FusedMoE LoRA](../sources/prs/vllm/PR-21229.md), [[v1] - Mamba1 Attention Metadata](../sources/prs/vllm/PR-21249.md), [Support CUTLASS NVFP4 (w4a4) for Blackwell Geforce GPUs (SM120)](../sources/prs/vllm/PR-21309.md), [Support Tensorrt-LLM MoE fp4 for low-latency](../sources/prs/vllm/PR-21331.md), [[TPU][Bugfix] fix moe layer](../sources/prs/vllm/PR-21340.md), [[Quantization] Enable BNB support for more MoE models](../sources/prs/vllm/PR-21370.md), [Update flashinfer CUTLASS NVFP4 MoE Kernel to use per expert global scaling factor](../sources/prs/vllm/PR-21408.md), [[NVIDIA] Explicitly disable shuffled weights for flashinfer blockscale moe fp8 kernels](../sources/prs/vllm/PR-21411.md), [[Bug] Fix Compressed Tensor NVFP4 `cutlass_fp4_group_mm` illegal memory access](../sources/prs/vllm/PR-21465.md), [[MoE] More balanced expert sharding](../sources/prs/vllm/PR-21497.md), [[NVIDIA] Fix Llama4 Scout FP4 functionality issues](../sources/prs/vllm/PR-21499.md), [Enable 4bit bnb prequant MOE](../sources/prs/vllm/PR-21548.md), [[Feature] Add Flashinfer MoE Support for Compressed Tensor NVFP4](../sources/prs/vllm/PR-21639.md), [[xpu]support moe models on XPU platform](../sources/prs/vllm/PR-21643.md), [support `torch.compile` for bailing moe](../sources/prs/vllm/PR-21664.md), [feat: Add Support GPTQ Quantization MOE on ROCM vllm serve](../sources/prs/vllm/PR-21733.md), [[Qwen3] Enable dual-chunk-attention support for Qwen3 models.](../sources/prs/vllm/PR-21924.md), [Fix Flashinfer CUTLASS MOE Allgather](../sources/prs/vllm/PR-21963.md), [[BUGFIX] KeyError 'layers.14.mlp.gate.g_idx' for Qwen3-MoE with GPTQ on ROCm](../sources/prs/vllm/PR-22017.md), [[EPLB] Support ernie4.5-moe](../sources/prs/vllm/PR-22100.md), [[Bugfix] Fix MoE BNB version](../sources/prs/vllm/PR-22260.md), [[gpt-oss] flashinfer mxfp4](../sources/prs/vllm/PR-22339.md), [[gpt-oss] triton kernel mxfp4](../sources/prs/vllm/PR-22421.md), [Fix Llama4 FlashInfer FP4 MoE issues](../sources/prs/vllm/PR-22511.md), [[Model] Add Ernie4.5 VL Model Support](../sources/prs/vllm/PR-22514.md), [Fix torch version check for SM100 mxfp4 ](../sources/prs/vllm/PR-22535.md), [[Quantization] Expand compressed-tensors MoE matching logic to support NFP4 + FP8 MoEs](../sources/prs/vllm/PR-22674.md), [Fix GGUF loader for Qwen3 MoE.](../sources/prs/vllm/PR-22785.md), [[FIXBUG] Add return_success parameter to moe_wna16_weight_loader function](../sources/prs/vllm/PR-22797.md), [[Model] Modify the gate implementation of glm4_moe](../sources/prs/vllm/PR-22832.md), [[XPU] support data parallel for MoE models on XPU](../sources/prs/vllm/PR-22887.md), [[Bugfix] Fix DeepSeek MTP](../sources/prs/vllm/PR-22934.md), [[Fix] enable swap_ab for pplx problem size computation](../sources/prs/vllm/PR-22991.md), [Use Blackwell FlashInfer MXFP4 MoE by default if available ](../sources/prs/vllm/PR-23008.md), [[Bugfix] fix qwen3 moe fp8 accuracy issue](../sources/prs/vllm/PR-23031.md), [[Kernel] CUTLASS MoE FP8: Integrate cuda moe permute/unpermute](../sources/prs/vllm/PR-23045.md), [Add routed_scaling_factor to MoE grouped topk](../sources/prs/vllm/PR-23123.md), [[Bugfix] Fix accuracy issue when using flashinfer cutlass moe, TP=1 and modelopt.](../sources/prs/vllm/PR-23125.md), [[CPU] add cpu fused moe pytorch native implementation](../sources/prs/vllm/PR-23146.md), [[Perf] Small optimizations for silu_mul_fp8_quant_deep_gemm](../sources/prs/vllm/PR-23265.md), [[Kernels] Overlap shared experts with send/recv](../sources/prs/vllm/PR-23273.md), [[Kernel] Add fused grouped_topk kernel for MoE](../sources/prs/vllm/PR-23274.md), [[Bugfix] Fix Qwen3 MoE GPTQ inference](../sources/prs/vllm/PR-23490.md), [DP/EP Support for gpt-oss with deepep-ht comm kernel on SM100](../sources/prs/vllm/PR-23608.md), [[Feature] Add Hopper DeepGEMM E8M0 for DeepSeekV3.1 scale_fmt](../sources/prs/vllm/PR-23666.md), [[Core/DBO][1/N] Add Dual-Batch Overlap mechanism to VLLM](../sources/prs/vllm/PR-23693.md), [[Kernel][tcgen05] nvfp4 fused tcgen05 moe](../sources/prs/vllm/PR-23696.md), [[Bugfix][Misc] Fix silu_and_mul_nvfp4_quant issue and extract common utils for nvfp4 kernel source files](../sources/prs/vllm/PR-23727.md), [[Feat][EPLB] A novel static EPLB placement strategy for MoE models.](../sources/prs/vllm/PR-23745.md), [[fix]: add Arm 4bit fused moe support](../sources/prs/vllm/PR-23809.md), [[Model][gpt-oss] Support DP+EP for GPT-OSS with FlashInfer trtllm-gen MoE](../sources/prs/vllm/PR-23819.md), [[Model] Add LongCat-Flash ](../sources/prs/vllm/PR-23991.md), [[BUGFIX] GPTQ quantization compatibility for Qwen3 MOE models (AutoGPTQ and AutoRound-GPTQ)](../sources/prs/vllm/PR-23994.md), [[PERF] Allreduce fusion. Support torch native matching. Tuning of the thresholds](../sources/prs/vllm/PR-24248.md), [[Kernel][Quantization] add w4a8 support for marlin kernel](../sources/prs/vllm/PR-24722.md), [[Model] Support Qwen3-VL Model Series](../sources/prs/vllm/PR-24727.md), [feat: BF16 FlashInfer Fused Cutlass MOE for Hopper and Blackwell Expert Parallel](../sources/prs/vllm/PR-25503.md), [[Quantization/NVFP4] Speed up TRTLLM NVFP4 MOE weight loading and fix K/V scale loading for MLA Attn](../sources/prs/vllm/PR-25968.md), [[Bugfix] Allow skipping MoE in NVFP4 (fix for MTP)](../sources/prs/vllm/PR-25987.md), [[MoE] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked](../sources/prs/vllm/PR-25990.md), [Fix undefined symbol: cutlass_moe_mm_sm100](../sources/prs/vllm/PR-26098.md), [[ModelOpt] Load w13/w2_input_scale for all experts, nvfp4](../sources/prs/vllm/PR-26135.md), [[Bug] Fix Shape Validation for Fallback while Enabling E8M0 for DeepGEMM](../sources/prs/vllm/PR-26322.md), [[Performance] Dual stream execution of "shared_experts" and "selected_experts" inside FusedMoE](../sources/prs/vllm/PR-26440.md), [[ROCM] MoE fp4 CK kernel](../sources/prs/vllm/PR-26545.md), [support flashinfer_fp4 moe for 5090 gpu](../sources/prs/vllm/PR-26669.md), [[NVIDIA] [Perf] Update to leverage flashinfer trtllm FP4 MOE throughput kernel](../sources/prs/vllm/PR-26714.md), [[Bugfix] Fix gpt-oss w4a8 DP/EP on B200](../sources/prs/vllm/PR-26729.md), [[Kernels] Enable FlashInfer FP8 Blockscale on SM90 (for TEP DSR1)](../sources/prs/vllm/PR-27134.md), [[ROCM] Enable CompressedTensorsWNA16](../sources/prs/vllm/PR-27187.md), [Flashinfer_CUTLASS_MOE fuses quantization for TP](../sources/prs/vllm/PR-27223.md), [Bugfix: Cutlass FP8 FusedMoE bad scaling factors](../sources/prs/vllm/PR-27255.md), [Feature: Support Relu2 in FusedMoE fp8 cutlass path](../sources/prs/vllm/PR-27261.md), [[Performance] Support FP8 flashinfer TRTLLM MOE on Qwen3 and Qwen-3next](../sources/prs/vllm/PR-27492.md), [[Attention] Use sparse prefill kernel for fp8 kv-cache in DeepSeek-v3.2](../sources/prs/vllm/PR-27532.md), [[Performance][B200] Fix deepgemm prologue](../sources/prs/vllm/PR-27897.md), [[Model] Consolidate Deepseek-MoE implementation with DeepSeek-v2](../sources/prs/vllm/PR-28101.md), [[Perf][DeepSeek] Add sigmoid+bias fusion to fused_grouped_topk from TRTLLM](../sources/prs/vllm/PR-28124.md), [[flashinfer] fix FI all2all with FI cutlass moe](../sources/prs/vllm/PR-28166.md), [[Feature] Support recording expert indices for rollout router replay](../sources/prs/vllm/PR-28284.md), [[Performance][B200] silu_mul_quant: pack scales in int32](../sources/prs/vllm/PR-28358.md), [[Bugfix][EPLB] Disabled shared expert overlap when EPLB is enabled](../sources/prs/vllm/PR-28377.md), [[Feature] Prefill Context Parallel (PCP) basic support](../sources/prs/vllm/PR-28718.md), [[Model] Add support for openPangu moe model](../sources/prs/vllm/PR-28775.md), [[Bugfix] Fix GPT-OSS AR+NORM fusion](../sources/prs/vllm/PR-28841.md), [[Bugfix] Make compressed-tensors MoEs respect ignored layers](../sources/prs/vllm/PR-28878.md), [Add TRTLLM MoE NVFP4 kernel to CompressedTensorsW4A4MoeMethod](../sources/prs/vllm/PR-28892.md), [[NVIDIA] Guard SM100 CUTLASS MoE macro to SM100 builds v2](../sources/prs/vllm/PR-28938.md), [[Feat] Support non-gated activations in NVFP4 modelopt path](../sources/prs/vllm/PR-29004.md), [[LoRA] Optimize 3D MoE logic](../sources/prs/vllm/PR-29222.md), [chore: add RTX_PRO_6000 GLM4.6-FP8 kernel tuning](../sources/prs/vllm/PR-29240.md), [[Kernel] Add NVFP4 MoE CUTLASS support for SM120](../sources/prs/vllm/PR-29242.md), [Lora MoE Align Improvements](../sources/prs/vllm/PR-29257.md), [[Perf] Disable DeepGEMM MoE by default when TP=8 is used](../sources/prs/vllm/PR-29346.md), [Add unpermute-aware fused MoE path and small-batch fallback](../sources/prs/vllm/PR-29354.md), [[Bugfix] Fix grouped_topk pytorch impl when num_experts can't be grouped properly](../sources/prs/vllm/PR-29439.md), [[Kernel][MoE] optimize `moe_align_block_size`](../sources/prs/vllm/PR-29642.md), [[Kernel]Support W4A8 Grouped GEMM on Hopper](../sources/prs/vllm/PR-29691.md), [[MoE-FP8-modelopt] Add FlashInfer alignment padding for intermediate dimensions](../sources/prs/vllm/PR-29748.md), [Add Mistral Large 3 and Ministral 3](../sources/prs/vllm/PR-29757.md), [[ROCm] [Fused Moe EP] Use binary expert mask for aiter fused moe kernel](../sources/prs/vllm/PR-29773.md), [[ROCm][MXFP4] Infer w4a4 quant method in rocm aiter fused moe](../sources/prs/vllm/PR-29775.md), [[EPLB] Support EPLB w/ NVFP4](../sources/prs/vllm/PR-29804.md), [[Bugfix][Model] Support LoRA on Qwen3 Output Embedding](../sources/prs/vllm/PR-29816.md), [[Bugfix] Fix FP8 MoE LoRA](../sources/prs/vllm/PR-29890.md), [[Kernel][Quantization][MoE] add marlin kernel support for turing (sm75)](../sources/prs/vllm/PR-29901.md), [[moe] Use enable_chunking func (to support disabling chunking)](../sources/prs/vllm/PR-29935.md), [[moe] Allow disabling DP chunking](../sources/prs/vllm/PR-29936.md), [[Perf] Do FP4 quant before All gather on flashinfer trtllmgen MOE ](../sources/prs/vllm/PR-30014.md), [[Quantization] Support Quark int4-fp8 w4a8 for MoE](../sources/prs/vllm/PR-30071.md), [[Model][Quantization] Restore MoE + GGUF models support (incl. Qwen3 MoE) by allowing Sideload Parameters](../sources/prs/vllm/PR-30116.md), [[Model][Quantization] Override HF defaults to GGUF ones (incl. Qwen3 MoE)](../sources/prs/vllm/PR-30118.md), [Add latent MoE support](../sources/prs/vllm/PR-30203.md), [[Bugfix]: Fix glm46 awq marlin moe wna16 compatibility](../sources/prs/vllm/PR-30210.md), [[LoRA] Reduce the loading time of MoE LoRA](../sources/prs/vllm/PR-30243.md), [gptq marlin quantization support for fused moe with lora](../sources/prs/vllm/PR-30254.md), [[LoRA] Support Quantized Adapters](../sources/prs/vllm/PR-30286.md), [[Model][Quantization] Fix / Add GGUF support for Qwen2 MoE models](../sources/prs/vllm/PR-30307.md), [[bugfix][quantization] fix quark qwen3 kv_cache quantization](../sources/prs/vllm/PR-30308.md), [[fix] fix SM check for Flashinfer TRTLLM MOE](../sources/prs/vllm/PR-30314.md), [[ROCm][Quantization] GPT OSS Upstream MoE wmxfp4_afp8 with static scales](../sources/prs/vllm/PR-30357.md), [[Feature] Add SM103 (Blackwell Ultra) Support to vLLM](../sources/prs/vllm/PR-30484.md), [[Bugfix] Fix Triton FusedMoE LoRA](../sources/prs/vllm/PR-30585.md), [[Perf] Eliminate padding and slicing op for GPT-OSS with Flashinfer MXFP4 MXFP8 MoE](../sources/prs/vllm/PR-30647.md), [fused_moe_lora PDL improvements](../sources/prs/vllm/PR-30716.md), [[SM100] Enable fp8 compute for prefill MLA](../sources/prs/vllm/PR-30746.md), [Add support for LoRA adapters in Nemotron-H models](../sources/prs/vllm/PR-30802.md), [[Compressed-Tensors] Simplify NVFP4 Conditions, enable marlin support for NVFP4A16 MoEs](../sources/prs/vllm/PR-30881.md), [[Mics] add pcp basic support to MoE model](../sources/prs/vllm/PR-31003.md), [[Bugfix] Fix GLM-4 MoE router logits dtype for data parallel chunking](../sources/prs/vllm/PR-31055.md), [[BugFix] LoRA: Support loading base_layer of experts](../sources/prs/vllm/PR-31104.md), [[Bugfix] Fix MoE LoRA bin/pt loading](../sources/prs/vllm/PR-31161.md), [[Kernel] Add topk_sigmoid kernel](../sources/prs/vllm/PR-31246.md), [pin lora_b moe weights on cpu](../sources/prs/vllm/PR-31317.md), [[Misc] Fix Qwen2-MoE shared_expert_gate](../sources/prs/vllm/PR-31339.md), [[BugFix] add select_gemm_impl on CompressedTensorsWNA16MoEMethod to support LoRA](../sources/prs/vllm/PR-31453.md), [[Bugfix][ROCm] Fix Static Quant Issue](../sources/prs/vllm/PR-31502.md), [[ROCm][Bugfix] Fix accuracy issue on fmoe when `VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS` enabled](../sources/prs/vllm/PR-31523.md), [[FIX] Add NO_MUL activation support for modular kernel path](../sources/prs/vllm/PR-31528.md), [Use the same memory for workspace13 and fused_output.](../sources/prs/vllm/PR-31531.md), [[Fix] Align fused moe lora_b shape with peft](../sources/prs/vllm/PR-31534.md), [[Bugfix] Fix Broken ModelOpt NVFP4 MoE](../sources/prs/vllm/PR-31742.md), [[LoRA]Disable linear LoRA kernel PDL](../sources/prs/vllm/PR-31777.md), [[Perf] Fuse stride preparation for NVFP4 cutlass_moe](../sources/prs/vllm/PR-31837.md), [[1/N][Attention] Restructure attention: move files](../sources/prs/vllm/PR-31916.md), [[5/N][Attention] Finish eliminating `vllm/attention` folder](../sources/prs/vllm/PR-32064.md), [Add TMA support to fused_moe_lora kernel](../sources/prs/vllm/PR-32195.md), [[Hardware][SM100] Add TRTLLM Kernel for INT4 W4A16 Kernel.](../sources/prs/vllm/PR-32437.md), [[Perf][Kernel] Optimize FP4 quantization kernels (SM100F)](../sources/prs/vllm/PR-32520.md), [fix: Add glm4_moe_lite to MLA detection](../sources/prs/vllm/PR-32614.md), [[Bugfix] Fix FP8 MoE EP Weight Loading for ModelOpt Llama4](../sources/prs/vllm/PR-32886.md), [[NVIDIA] [feat] Integrate flashinfer Trtllmgen bf16 moe](../sources/prs/vllm/PR-32954.md), [Add support for Mistral Large 3 inference with Flashinfer MoE](../sources/prs/vllm/PR-33174.md), [fix: Add SM120 (RTX Blackwell) support for FlashInfer CUTLASS NVFP4 MoE kernels](../sources/prs/vllm/PR-33417.md), [[Kernel] Support Flashinfer trtllm fused MoE non gated FP8 & NVFP4](../sources/prs/vllm/PR-33506.md), [Adding support to Sarvam's MoE models](../sources/prs/vllm/PR-33942.md), [[Bugfix]fix output Nan/Inf in marlin if dtype=float16](../sources/prs/vllm/PR-33972.md), [[Kernel] Optimize grouped topk kernel](../sources/prs/vllm/PR-34206.md), [[ModelBash][DSR1 NVFp4] Avoid Bf16 Bias Cast](../sources/prs/vllm/PR-34298.md), [[ModelBash][DSV3] Add TRTLLM DSV3 Router GEMM kernel (6% B1 Speedup)](../sources/prs/vllm/PR-34302.md), [[Kernel] Integrate SM100 MXFP8 blockscaled grouped MM and quant kernels](../sources/prs/vllm/PR-34448.md), [[Model] Add NVFP4 quantization support for Step3.5-Flash](../sources/prs/vllm/PR-34478.md), [[Bugfix] Handle num_expert_group=None in flashinfer block-scale FP8 MoE](../sources/prs/vllm/PR-34494.md), [[Quantization] add humming quantization kernel](../sources/prs/vllm/PR-34556.md), [[Bugfix] Fix NVFP4 TRTLLM MoE non-gated support; add gsm8k for Nemotron-3-Nano FP8+NVFP4](../sources/prs/vllm/PR-34725.md), [[Performance] Cublas Bf16 Gate with Fp32 Output](../sources/prs/vllm/PR-35121.md), [[Bugfix] Fix DSV3 kernels breaking _C and _moe_C on unsupported arches](../sources/prs/vllm/PR-35123.md), [[Bugfix] Fix expert_ids padding values in moe_align_block_size kernel](../sources/prs/vllm/PR-35161.md), [[Quant][Feature] Support online MXFP8 quantization for MoE and dense models](../sources/prs/vllm/PR-35448.md), [[MoE][Perf] Wrap DSV3 QKVAProj GEMM in custom op for torch.compile](../sources/prs/vllm/PR-35751.md), [[MoE] Move PF Methods to Folder](../sources/prs/vllm/PR-35927.md), [Add support for ModelOpt MXFP8 MoE models](../sources/prs/vllm/PR-35986.md), [[Bugfix] Fix passing of activation_type to trtllm fused MoE NVFP4 and FP8](../sources/prs/vllm/PR-36017.md), [[Kernel] Add FlashInfer MoE A2A Kernel](../sources/prs/vllm/PR-36022.md), [[Bugfix] Disable FlashInfer TRTLLM BF16 path for non-gated MoE](../sources/prs/vllm/PR-36146.md), [[Perf] Add TRTLLM FP8 MoE Modular Kernel](../sources/prs/vllm/PR-36307.md), [[XPU] Support block fp8 moe by fallback to TritonExpert on XPU](../sources/prs/vllm/PR-36458.md), [[Bug][MoE] Fix TRTLLM NVFP4 Routing Kernel Precision](../sources/prs/vllm/PR-36725.md), [[Bug][MoE] Strengthen _supports_current_device() checks in the TRTLLM FP8, NVFP4, and FlashInfer CuteDSL MoE experts](../sources/prs/vllm/PR-36728.md), [[MoE Refactor] Mxfp4 oracle rebased](../sources/prs/vllm/PR-37128.md), [[Kernel] Add gpt-oss Router GEMM kernel](../sources/prs/vllm/PR-37205.md), [[MoE/EPLB] Fix FlashInfer nvfp4 experts + EPLB correctness](../sources/prs/vllm/PR-37217.md), [[Kernel] Add non-gated support for NVFP4 CUTLASS MoE](../sources/prs/vllm/PR-37320.md), [[Kernel] Add MXFP4 W4A4 CUTLASS MoE kernel for SM100](../sources/prs/vllm/PR-37463.md), [[4/n] Migrate FP4/W4A8 CUTLASS kernels to torch stable ABI](../sources/prs/vllm/PR-37503.md), [[Bugfix] Disable monolithic TRTLLM MoE for Renormalize routing (#37591)](../sources/prs/vllm/PR-37605.md), [[Perf] Use torch compile to fuse pack topk in trtllm moe](../sources/prs/vllm/PR-37695.md), [[MoE] Move FlashInfer CuteDSL experts into fused_moe/experts/](../sources/prs/vllm/PR-37759.md), [[Feature] Support per-draft-model MoE backend via `--speculative-config`](../sources/prs/vllm/PR-37880.md), [[MoE Kernel] Flashinfer nvfp4 cutedsl moe kernel integration](../sources/prs/vllm/PR-38050.md), [[Quantization] Add FlashInfer CuteDSL batched experts backend for NVFP4 MoE](../sources/prs/vllm/PR-38251.md), [[MoE] Add RoutingMethodType.Simulated to TRT-LLM FP8/NVFP4 kernel allowlists](../sources/prs/vllm/PR-38329.md), [[NVIDIA] Bugfix NVFP4 DGX Spark and RTX50](../sources/prs/vllm/PR-38423.md), [[Kernels][MoE] Fix legacy_routing to use bitmatrix-based routing path](../sources/prs/vllm/PR-38504.md), [[Quant] add CompressedTensorsW8A8Mxfp8 for linear and MoE layers](../sources/prs/vllm/PR-38815.md), [[Bugfix] Re-enable Renormalize routing for TRT-LLM MoE experts](../sources/prs/vllm/PR-38859.md), [[MoE Refactor] Split up compressed_tensors_moe.py](../sources/prs/vllm/PR-38960.md), [[Bug] Fix routing bias dtype for trtllm per-block fp8 moe](../sources/prs/vllm/PR-38989.md), [[Bugfix][MoE] Fix 6-8% decode regression: prefer multi-stream shared expert overlap](../sources/prs/vllm/PR-38990.md), [[Perf] Change Trtllm fp8 MoE to use Shuffled Weights and BlockMajorK Layout](../sources/prs/vllm/PR-38993.md), [[MoE] Move GPT OSS Triton kernel experts into fused_moe/experts/](../sources/prs/vllm/PR-39007.md), [[Gemma4] Support quantized MoE ](../sources/prs/vllm/PR-39045.md), [[Bug] Fix Trtllm Fp8 MoE Weight Shuffle Memory Fragamentation](../sources/prs/vllm/PR-39054.md), [[XPU] Quick fix for TritonMLA to remove cuda hardcode](../sources/prs/vllm/PR-39088.md), [perf(moe): add tuned fused_moe config for RTX PRO 6000 Blackwell Server Edition](../sources/prs/vllm/PR-39183.md), [[Bugfix] FlashInfer MXINT4 MoE crashes, missing do_finalize](../sources/prs/vllm/PR-39315.md), [fix: clamp NaN/Inf in topk_softmax to prevent duplicate expert IDs](../sources/prs/vllm/PR-39391.md), [[Kernel] Support TRTLLM GEN NVFP4 MoE for non-512-aligned hidden dims via weight padding](../sources/prs/vllm/PR-39510.md), [[Bugfix] [Tests] Enforce `out` tensor device in `kernel/moe/test_cutedsl_moe.py`](../sources/prs/vllm/PR-39644.md), [[Bugfix] Fix mismatch between global and local attention heads in tensor-parallel mode for param2moe model](../sources/prs/vllm/PR-39707.md), [[Bugfix] Reject non-nvfp4 dtypes when using the flashinfer_nvlink_one_sided all2all backend](../sources/prs/vllm/PR-39717.md), [[Bugfix] Disable FlashInfer CUTLASS MoE on SM121 (DGX Spark)](../sources/prs/vllm/PR-39825.md), [[Core] Replace routing replay with device cache and async D2H pipeline](../sources/prs/vllm/PR-39917.md), [[Bugfix] Temporarily disable B200 fp4 MoE layer tests](../sources/prs/vllm/PR-40057.md), [[Bugfix] moe lora align kernel grid](../sources/prs/vllm/PR-40131.md), [Fix MoE backend selection for LoRA (unquantized MoE)](../sources/prs/vllm/PR-40273.md), [[MoE] Move cutlass moe to fused_moe/experts/](../sources/prs/vllm/PR-40574.md), [[DSV4] Add silu clamp limit to shared expert](../sources/prs/vllm/PR-40950.md), [[DSV4] Add BF16 and MXFP8 A2A support for flashinfer a2a one sided](../sources/prs/vllm/PR-40960.md), [[Kernel][MoE] Support GELU on TRT-LLM NvFP4 fused MoE for Gemma4](../sources/prs/vllm/PR-41050.md), [[DSV4] Fuse norm and router for low latency scenario](../sources/prs/vllm/PR-41263.md), [[ROCm][Quantization][3/N] Refactor quark_moe w4a4 w/ oracle](../sources/prs/vllm/PR-41436.md), [[Quantization] Rework quantization_config to use QuantKey and allow for activation override](../sources/prs/vllm/PR-41566.md), [[MXFP4] Support for linear layers + compressed-tensors integration](../sources/prs/vllm/PR-41664.md), [[CPU] Add MXFP4 W4A16 MoE support](../sources/prs/vllm/PR-41922.md), [[MoE] Move various experts classes to fused_moe/experts/](../sources/prs/vllm/PR-41979.md), [[Bugfix] Add swiglu limits to deepgemm fp8 methods](../sources/prs/vllm/PR-41986.md), [[LoRA] Support 2D and 3D MoE LoRA adapter at the same time](../sources/prs/vllm/PR-42242.md), [Refactor AWQ Marlin MoE onto modular WNA16 oracle](../sources/prs/vllm/PR-42483.md), [[Perf] Wire silu_and_mul_per_block_quant into TritonFP8MoE (MiniMax-M2) ](../sources/prs/vllm/PR-42497.md), [[Kernel] Pack topk id/weights triton kernel](../sources/prs/vllm/PR-42527.md), [[Bugfix] fix swiglu limit issue for humming backend + deepseek v4](../sources/prs/vllm/PR-42541.md), [[6/n] Migrate activation kernels, gptq, gguf, non cutlass w8a8 to libtorch stable ABI (continued)](../sources/prs/vllm/PR-42663.md), [Fix Weight loading for Qwen3.5-MTP and Qwen3-VL using runai_streamer](../sources/prs/vllm/PR-42716.md), [[Refactor] Remove dead cuda kernels](../sources/prs/vllm/PR-42767.md), [[Perf] Re-enable flashinfer autotune by default and cleanup](../sources/prs/vllm/PR-42857.md), [[Model Refactoring] Migrate DeepSeek V4 to vllm/models/ [1/N] ](../sources/prs/vllm/PR-43004.md), [[Kernel] (2/N) Machete - Integrate into CompressedTensorsWNA16 and GPTQMarlin](../sources/prs/vllm/PR-7701.md), [Fused MoE — FP8 Block-Scale Routing + Dual GEMM](../wiki/kernels/fused-moe.md), [Grouped GEMM for MoE](../wiki/kernels/grouped-gemm.md) | -| `prefill` | [FlashMLA — Multi-head Latent Attention](../sources/blogs/flashmla.md), [DeepSeek-V3.2-Exp in vLLM: Fine-Grained Sparse Attention in Action](../sources/blogs/vllm-deepseek-v3-sparse-attention.md), [FlashInfer MLSys 2026 - Track C: Gated Delta Net](../sources/contests/flashinfer-mlsys26/track-c-gated-delta-net.md), [[None][feat] Optimize mamba SSD prefill and extend flashinfer dispatch](../sources/prs/TensorRT-LLM/PR-12731.md), [[None][feat] Optimize causal_conv1d prefill and decode kernels](../sources/prs/TensorRT-LLM/PR-13103.md), [[None][feat] Add chunked prefill support for Gemma4 (text + vision multimodal)](../sources/prs/TensorRT-LLM/PR-14134.md), [Add Blackwell MLA forward (shape: d=192, dv=128) implementation](../sources/prs/cutlass/PR-2472.md), [feat: update decode attention APIs](../sources/prs/flashinfer/PR-1007.md), [add multi-item scoring](../sources/prs/flashinfer/PR-1015.md), [feat: add functional per-head FP8 quantization for FA3](../sources/prs/flashinfer/PR-1033.md), [[nvidia] initial support for blackwell kernels](../sources/prs/flashinfer/PR-1039.md), [Parameterize prefix mask call (needed by POD-Attention)](../sources/prs/flashinfer/PR-1059.md), [bugfix: follow user-specified sm_scale for blackwell cutlass fmha](../sources/prs/flashinfer/PR-1072.md), [bugfix: fix fp8 attention kernels aot compilation issue](../sources/prs/flashinfer/PR-1087.md), [bugfix: host-precomuted plan function for blackwell fmha](../sources/prs/flashinfer/PR-1106.md), [[Feature] Support PDL for batch Prefill and Decode](../sources/prs/flashinfer/PR-1117.md), [[feat] add unified batch attention w/ correctness tests.](../sources/prs/flashinfer/PR-1137.md), [Fix FA2 and FA3 multi-item scoring and cuda illegal memory access error](../sources/prs/flashinfer/PR-1140.md), [Fix the issue with auxillary kernel launch and grid dim calculation](../sources/prs/flashinfer/PR-1208.md), [feat: Add non-causal cudnn prefill kernels](../sources/prs/flashinfer/PR-1230.md), [add trtllm-gen context attention](../sources/prs/flashinfer/PR-1239.md), [refactor: refactor trtllm-gen attention kernel integration code](../sources/prs/flashinfer/PR-1289.md), [Allow cudnn prefill kernels to be called natively](../sources/prs/flashinfer/PR-1317.md), [feat: Support logits_soft_cap for Persistent attn; fix kv split limit](../sources/prs/flashinfer/PR-1324.md), [support trtllm-gen prefill fp4 output](../sources/prs/flashinfer/PR-1360.md), [Support scale factor start index for fp4 mha prefill/decode](../sources/prs/flashinfer/PR-1363.md), [Allow BatchPrefillPagedWrapper to call cudnn API](../sources/prs/flashinfer/PR-1384.md), [GPT-OSS Support: Add Blackwell MoE mxfp4 implementation from TRTLLM and Attention Sink](../sources/prs/flashinfer/PR-1389.md), [Fix TRTLLM NVFP4-out attention kernel scale factor dim issue](../sources/prs/flashinfer/PR-1460.md), [feat: add pdl for trtllm-gen attn](../sources/prs/flashinfer/PR-1484.md), [feat: Support fp8 qkv, fp16/bf16 out MHA for trtllm-gen.](../sources/prs/flashinfer/PR-1490.md), [backend: Refactor trtllm-gen fmha metainfo loading](../sources/prs/flashinfer/PR-1518.md), [bugfix: Fix Persistent kernel precision for masked output ](../sources/prs/flashinfer/PR-1533.md), [feat: Integrate TRTLLM varlen kernel for deepseek R1 prefill ](../sources/prs/flashinfer/PR-1537.md), [feat: Add fp8-qkv, fp16/bf16 output MHA](../sources/prs/flashinfer/PR-1540.md), [Backend: downgrade trtllm-gen kernel to cuda-12](../sources/prs/flashinfer/PR-1567.md), [feat: Support for inferring out_dtype from out.dtype for TRTLLM attention kernel](../sources/prs/flashinfer/PR-1578.md), [fix: Improve TRTLLM attention kernel out_dtype unit test](../sources/prs/flashinfer/PR-1590.md), [feat: initial support for SM103, SM110, SM120, SM121](../sources/prs/flashinfer/PR-1608.md), [bugfix: collect all modules to aot](../sources/prs/flashinfer/PR-1622.md), [fix: pass workspace for trtllm-gen attention](../sources/prs/flashinfer/PR-1635.md), [perf&bugfix: skip kv-tile computation out of sliding window in FA2; fix __syncthreads in mergestate](../sources/prs/flashinfer/PR-1661.md), [feat: Batch-size invariant FA2 Prefill & Decode](../sources/prs/flashinfer/PR-1675.md), [perf: improve attention of tcgen05 flash-attention](../sources/prs/flashinfer/PR-1681.md), [test: skip the unsupported test cases for sm120/121](../sources/prs/flashinfer/PR-1710.md), [Fix tests/test_trtllm_gen_attention.py::test_trtllm_batch_prefill, ::test_trtllm_batch_decode mismatch error](../sources/prs/flashinfer/PR-1755.md), [Add head_dim=64 for tcgen05 tcgen05 flash-attention implementation](../sources/prs/flashinfer/PR-1850.md), [fix: Fix trtllm-gen prefill IMA when batch_size==1](../sources/prs/flashinfer/PR-1912.md), [feat: add xqa backend and completes NHD/HND coverage for trtllm-gen/xqa backend](../sources/prs/flashinfer/PR-2001.md), [Added an initial implementation of Q and KV Cache in fp8 and to use t…](../sources/prs/flashinfer/PR-2035.md), [Rebase FP8 SM100 Cutlass FMHA Attention to main (original PR#1238)](../sources/prs/flashinfer/PR-2047.md), [[Feature] Support batch prefill for POD Attention](../sources/prs/flashinfer/PR-2079.md), [[API change] Allow using torch.Tensor for scales for trtllm-gen attention](../sources/prs/flashinfer/PR-2084.md), [refactor: update fa3 codebase and fix hopper unittest [part 1]](../sources/prs/flashinfer/PR-2111.md), [feat: TRTLLM FMHAv2 backend for ctx attention](../sources/prs/flashinfer/PR-2142.md), [Enable Hopper FA3 FP8 attention in decode.py](../sources/prs/flashinfer/PR-2148.md), [fix: compile flags for trtllm fmha_v2 ](../sources/prs/flashinfer/PR-2175.md), [Fp8 attention are now part of cuDNN 9.17.1](../sources/prs/flashinfer/PR-2241.md), [feat: add GDN Attention](../sources/prs/flashinfer/PR-2276.md), [Added the cudnn backend Ragged KV Cache wrapper](../sources/prs/flashinfer/PR-2352.md), [feat: [Qwen3-Next] Add Cute DSL GDN decode kernel and tests](../sources/prs/flashinfer/PR-2370.md), [Remove cudaMalloc/Free in GDN prefill kernel](../sources/prs/flashinfer/PR-2415.md), [refactor: reduce hopper's gdn prefill compilation time and fix docstring.](../sources/prs/flashinfer/PR-2422.md), [feat: Add TRTLLM fmha_v2 library for SM90 attention with Skip-Softmax ](../sources/prs/flashinfer/PR-2446.md), [feat: Add TRTLLM-Gen Skip-Softmax kernels for prefill and decode](../sources/prs/flashinfer/PR-2477.md), [Ameyn/gdn decode cutedsl kernel](../sources/prs/flashinfer/PR-2498.md), [perf: cache cudaGetDeviceProperties in gdn_prefill to avoid per-call overhead](../sources/prs/flashinfer/PR-2509.md), [feat: Enable TRTLLM-Gen Skip-Softmax attention for MLA](../sources/prs/flashinfer/PR-2547.md), [fix: allow fmha_v2_prefill_deepseek on SM121 (DGX Spark)](../sources/prs/flashinfer/PR-2559.md), [fix: guard CUTLASS FMHA against SM12x and fix fmha_v2 SM121a check](../sources/prs/flashinfer/PR-2560.md), [feat: add is_sm12x_supported() helper for SM12x family detection](../sources/prs/flashinfer/PR-2574.md), [benchmarks: Add FP8 input / BF16 output in ragged prefill benchmark](../sources/prs/flashinfer/PR-2666.md), [feat: add support for more MLA head dimensions](../sources/prs/flashinfer/PR-2677.md), [Add NVFP4 KV cache quantization support for SM100](../sources/prs/flashinfer/PR-2702.md), [feat: Add DiT-oriented kernels where Qk (Bmm1) type can be reinterpreted into Int8 or BFloat16](../sources/prs/flashinfer/PR-2711.md), [feat: Expose TRT-LLM FMHA style paged KV Cache and page table layout](../sources/prs/flashinfer/PR-2770.md), [tests: skip sliding window + fp8 to prevent hang in fmha_v2 unit tests](../sources/prs/flashinfer/PR-2781.md), [[fmha-v2] Support HND and NHD paged KV cache layouts with conditional stride handling](../sources/prs/flashinfer/PR-2799.md), [[fix] Bugfix 1367: fix VariableBlockSparseAttention buffer overflow by dynamically resizing kv_lens_buffer](../sources/prs/flashinfer/PR-2802.md), [[CuTe DSL] Add modular FMHA prefill and MLA decode attention kernels](../sources/prs/flashinfer/PR-2805.md), [[Perf] Add FMHAv2 to flashinfer_benchmark.py and eliminate unnecessary H2D](../sources/prs/flashinfer/PR-2841.md), [read real strides for kv and block scale](../sources/prs/flashinfer/PR-2844.md), [feat(gdn): state checkpointing in chunk_gated_delta_rule](../sources/prs/flashinfer/PR-2908.md), [Only swizzle on v block scale; rename kv_block_scales to kv_cache_sf](../sources/prs/flashinfer/PR-2954.md), [test: xfail cuDNN FP8 prefill on Blackwell with CUDA <= 12.9](../sources/prs/flashinfer/PR-2963.md), [[feat] Add blackwell GDN prefill kernel](../sources/prs/flashinfer/PR-3001.md), [Support lse in trtllm paged attn kernels](../sources/prs/flashinfer/PR-3058.md), [Support NVFP4 KV for prefill and batch attention kernels](../sources/prs/flashinfer/PR-3097.md), [feat: Enable FP8 (E4M3/E5M2) in concat_mla_k for optimize long-context prefill performance and refactor type dispatch for BF16/FP16](../sources/prs/flashinfer/PR-3129.md), [cute-dsl fmha prefill (cubin integration): remove front-padding, add attention_sink, and pdl support](../sources/prs/flashinfer/PR-3181.md), [feat(trace): embed runnable init() in every TraceTemplate](../sources/prs/flashinfer/PR-3221.md), [fix(fmha_v2): fix FP8 V-scratch pipeline and varlen scheduler on SM90](../sources/prs/flashinfer/PR-3276.md), [perf: fix the iteration bound of SWA in FA2 prefill template](../sources/prs/flashinfer/PR-714.md), [feat: support deepseek prefill attention shape](../sources/prs/flashinfer/PR-765.md), [perf: refactor fa2 prefill template](../sources/prs/flashinfer/PR-776.md), [feat: Separate QK/VO head dim dispatch for sm90 AOT](../sources/prs/flashinfer/PR-778.md), [bugfix: fix batch prefill attention kernel unittests](../sources/prs/flashinfer/PR-781.md), [bugfix: drop CTA_TILE_Q=32](../sources/prs/flashinfer/PR-785.md), [feat: support f32 attention output in FA2 template](../sources/prs/flashinfer/PR-799.md), [feat: apply sm_scale at logits instead of q in FA2 template](../sources/prs/flashinfer/PR-801.md), [perf: memory efficient deepseek mla fused page-attention kernel](../sources/prs/flashinfer/PR-804.md), [Add POD-Attention to FlashInfer](../sources/prs/flashinfer/PR-858.md), [Naive Support for Hopper FP8 Prefill Kernel with Per-Head Quantization](../sources/prs/flashinfer/PR-869.md), [perf: reduce torch.library dispatch overhead](../sources/prs/flashinfer/PR-968.md), [perf: Fix python API overhead when CUDAGraph is not enabled](../sources/prs/flashinfer/PR-969.md), [[2/2] Support MHA prefill with FlashAttention 4.](../sources/prs/sglang/PR-10937.md), [[DeepseekV32] Enable flashmla_prefill kernel with fp8 kvcache](../sources/prs/sglang/PR-11655.md), [DeepSeek-V3.2: Add Adaptive MHA Attention Pathway for Short-Sequence Prefill](../sources/prs/sglang/PR-11892.md), [[DeepSeek-V3.2][NSA] Enable MHA Pathway for Short Sequence Prefill on B200 (SM100)](../sources/prs/sglang/PR-12788.md), [[DeepSeek v3.2] opt Context Parallelism: support fused moe, multi batch and fp8 kvcache](../sources/prs/sglang/PR-13959.md), [Support FP8 MLA prefill and 128k context.](../sources/prs/sglang/PR-14395.md), [[AMD] Support fused_rms_mxfp4_quant in the prefill stage for DeepSeek-R1-MXFP4](../sources/prs/sglang/PR-14975.md), [Support Qwen3 MoE context parallel](../sources/prs/sglang/PR-18233.md), [feat(gdn): add FlashInfer K-last SSM layout support for GDN prefill and decode for Hopper](../sources/prs/sglang/PR-18361.md), [Fp8 prefill attn kernel integration](../sources/prs/sglang/PR-18528.md), [[AMD] Fp8 prefill integration with radix cache path for dpsk models](../sources/prs/sglang/PR-20187.md), [[GDN] Add benchmark for sglang gdn prefill](../sources/prs/sglang/PR-20428.md), [FIX: (NSA) Compute topk_indices_offset when NSA prefill flashmla_sparse is used with FP8 KV cache](../sources/prs/sglang/PR-20606.md), [[DSA] Support trtllm sparse mla kernel for prefill batches ](../sources/prs/sglang/PR-21783.md), [[KDA] Optimize prefill kernels with diagonal and recompute fuse](../sources/prs/sglang/PR-24271.md), [[attn backend] Integrate tokenspeed_mla prefill/decode kernels (fp8 kv cache, blackwell)](../sources/prs/sglang/PR-24925.md), [Introduce SchedulerDPAttnAdapter to own DP-attention state](../sources/prs/sglang/PR-25611.md), [Move DP-attention adapter methods to SchedulerDPAttnAdapter](../sources/prs/sglang/PR-25612.md), [[Refactor] Rename NSA → DSA: user-facing aliases, file/class/import rename](../sources/prs/sglang/PR-25821.md), [Support MHA with chunked prefix cache for DeepSeek chunked prefill](../sources/prs/sglang/PR-5113.md), [[PD] Fix dynamic port support and MLA buffer for Mooncake](../sources/prs/sglang/PR-5415.md), [Avoid computing lse in Ragged Prefill when there's no prefix.](../sources/prs/sglang/PR-5476.md), [[Feature] Comprehensive Hybrid Parallelism Support](../sources/prs/sglang/PR-6389.md), [[PD] Add different TP sizes support for no-MLA models](../sources/prs/sglang/PR-6793.md), [Let EP prefill support new DeepGEMM](../sources/prs/sglang/PR-7310.md), [[Feature] Layer-wise Prefill](../sources/prs/sglang/PR-7634.md), [[Attention] MLA with chunked prefill](../sources/prs/vllm/PR-12639.md), [Disable chunked prefill and/or prefix caching when MLA is enabled ](../sources/prs/vllm/PR-12642.md), [[V1] V1 Enablement Oracle ](../sources/prs/vllm/PR-13726.md), [[ROCm] Disable chunked prefill/prefix caching when running MLA on non-cuda platforms](../sources/prs/vllm/PR-13844.md), [Fix mla prefill context performance](../sources/prs/vllm/PR-13897.md), [[ROCm] Enable chunked prefill/paged attention in MLA on ROCm](../sources/prs/vllm/PR-14316.md), [[BugFix] Fix nightly MLA failure (FA2 + MLA chunked prefill, i.e. V1, producing bad results)](../sources/prs/vllm/PR-15492.md), [[ROCM][KERNEL] Paged attention for V1](../sources/prs/vllm/PR-15720.md), [[Kernel] Unified Triton kernel that doesn't distinguish between prefill + decode](../sources/prs/vllm/PR-16828.md), [[ROCm][Kernel][V1] Enable AMD Radeon GPU Custom Paged Attention on v1](../sources/prs/vllm/PR-17004.md), [[Bugfix][ROCm] Use `chunked_prefill_paged_decode` as fallback for V1 attention on ROCm](../sources/prs/vllm/PR-18093.md), [[Hardware][AMD] integrate aiter chunked prefill into vllm](../sources/prs/vllm/PR-18596.md), [[Attention] MLA - Flashinfer Ragged Prefill](../sources/prs/vllm/PR-20034.md), [[Kernel] Optimize Prefill Attention in Unified Triton Attention Kernel](../sources/prs/vllm/PR-20308.md), [[Model] Pooling models default to using chunked prefill & prefix caching if supported.](../sources/prs/vllm/PR-20930.md), [Override attention metadata for fast prefill in some KV sharing setups](../sources/prs/vllm/PR-21590.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv/out Attention Kernel](../sources/prs/vllm/PR-21716.md), [[NVIDIA] Support Flashinfer TRT-LLM Prefill Attention Kernel](../sources/prs/vllm/PR-22095.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv NVFP4-out Attention Kernel](../sources/prs/vllm/PR-22703.md), [[Flashinfer] Support Flashinfer TRTLLM FP8-qkv BF16/FP16-out Attention Kernel](../sources/prs/vllm/PR-23647.md), [[Attention] Use sparse prefill kernel for fp8 kv-cache in DeepSeek-v3.2](../sources/prs/vllm/PR-27532.md), [[Feature] Prefill Context Parallel (PCP) basic support](../sources/prs/vllm/PR-28718.md), [[Attention] Make `split_decodes_and_prefills(..., require_uniform=True)` support padding](../sources/prs/vllm/PR-29644.md), [[perf] Use direct copy (broadcast) instead of cat for k_nope/k_pe in MLA prefill](../sources/prs/vllm/PR-29710.md), [[SM100] Enable fp8 compute for prefill MLA](../sources/prs/vllm/PR-30746.md), [[SM100] Resubmit FMHA FP8 prefill for MLA](../sources/prs/vllm/PR-31195.md), [[Bugfix][ROCm]Fix Qwen3-Next-80B-A3B-Thinking inference and optimize non-standard block size (544) support under rocm_atten](../sources/prs/vllm/PR-31380.md), [[1/N][Attention] Restructure attention: move files](../sources/prs/vllm/PR-31916.md), [[Kernel] use flashinfer for gdn prefill](../sources/prs/vllm/PR-32846.md), [[Attention] Use FA4 for MLA prefill](../sources/prs/vllm/PR-34732.md), [[Bugfix][MLA] Add logits size budget to sparse indexer prefill chunking](../sources/prs/vllm/PR-36178.md), [[ROCm] Attention selector reordering](../sources/prs/vllm/PR-36702.md), [[Attention] Support distinguishing between short extends and decodes](../sources/prs/vllm/PR-37303.md), [[GDN] Eliminate GPU->CPU sync in prepare_chunk_indices during prefill](../sources/prs/vllm/PR-38361.md), [[Bugfix][MLA] Change default SM100 MLA prefill backend back to TRT-LLM](../sources/prs/vllm/PR-38562.md), [[Attention][MLA] Re-enable FA4 as default MLA prefill backend](../sources/prs/vllm/PR-38819.md), [[Gemma4] Enable Fast Prefill Optimization](../sources/prs/vllm/PR-38879.md), [add warning when FP8 KV cache misses prefill query quantization](../sources/prs/vllm/PR-39752.md), [[GDN] Enable FI Blackwell GDN prefill kernel](../sources/prs/vllm/PR-40717.md), [[MLA Attention Backend] Add TOKENSPEED_MLA backend for DSR1/Kimi K25 prefill + decode on Blackwell](../sources/prs/vllm/PR-41778.md), [[Bugfix] Fix TRTLLM ragged MLA prefill workspace warmup](../sources/prs/vllm/PR-42112.md), [FlashMLA — Multi-head Latent Attention](../wiki/kernels/flashmla.md), [Gated Delta Net — Linear Attention](../wiki/kernels/gated-delta-net.md), [Sparse MLA (DeepSeek V3.2)](../wiki/kernels/sparse-mla.md) | +| `attention` | [FlashAttention-4 Blog](../sources/blogs/flash-attention-4.md), [FlashMLA — Multi-head Latent Attention](../sources/blogs/flashmla.md), [Gated Delta Networks](../sources/blogs/gated-delta-net.md), [K-Search: LLM Kernel Generation via Co-Evolving Intrinsic World Model](../sources/blogs/k-search-kernel-generation.md), [Qwen3-Next: Hybrid GDN+MoE Architecture on NVIDIA Blackwell](../sources/blogs/qwen3-next-architecture.md), [DeepSeek-V3.2-Exp in vLLM: Fine-Grained Sparse Attention in Action](../sources/blogs/vllm-deepseek-v3-sparse-attention.md), [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [DeepSeek-V2: A Strong, Economical, and Efficient Mixture-of-Experts Language Model](../sources/docs/deepseek-v2-mla.md), [FlashAttention-4: Algorithm and Kernel Co-design for Blackwell GPUs](../sources/docs/flash-attention-4.md), [Native Sparse Attention: Hardware-Aligned and Natively Trainable Sparse Attention](../sources/docs/nsa.md), [Sync nv_dev with upstream #316 (Mega MoE optimizations & benchmarks)](../sources/prs/DeepGEMM/PR-328.md), [[TRTLLM-9457][feat] Add cute dsl fp8 gemm for Blackwell](../sources/prs/TensorRT-LLM/PR-10130.md), [[TRTLLM-9798][feat] Change to use new DeepGEMM MQA sm100 kernel for MTP-3](../sources/prs/TensorRT-LLM/PR-10226.md), [[TRTLLM-10022][feat] Add hopper xqa decode support for skip softmax attention](../sources/prs/TensorRT-LLM/PR-10264.md), [[None][feat] Remove non flash attetnion style fmha_v2 kernel for hopper](../sources/prs/TensorRT-LLM/PR-11381.md), [[TRTLLM-11092][feat] add support for visual gen FA4 attention backend](../sources/prs/TensorRT-LLM/PR-11697.md), [[TRTLLM-11119][feat] Blackwell SageAttention, Integrate into AttentionOp API](../sources/prs/TensorRT-LLM/PR-11718.md), [[None][feat] Add fused DiT QK Norm + RoPE CUDA kernel for FLUX](../sources/prs/TensorRT-LLM/PR-11869.md), [[TRTLLM-10421][perf] Add fused cat+fp8_quantize CUDA kernel for DSA indexer](../sources/prs/TensorRT-LLM/PR-11899.md), [[None][feat] GLM 5 support and DSA MTP fixes](../sources/prs/TensorRT-LLM/PR-11990.md), [[#11694][feat] AutoDeploy: Improve the piecewise CG memory usage](../sources/prs/TensorRT-LLM/PR-11993.md), [[TRTLLM-11285][feat] Fuse indexer wk + weights_proj into single GEMM in TF32 for DS-V3.2](../sources/prs/TensorRT-LLM/PR-12055.md), [[TRTLLM-11540][feat] Add EAGLE3 dynamic tree speculative decoding support](../sources/prs/TensorRT-LLM/PR-12062.md), [[TRTLLM-11289][feat] Integrate CuteDSL's bf16 dense GEMMs](../sources/prs/TensorRT-LLM/PR-12074.md), [[TRTLLM-10407][perf] Enable CuteDSL indexer_top_k in model](../sources/prs/TensorRT-LLM/PR-12236.md), [[None][feat] Support update weight for nvfp4](../sources/prs/TensorRT-LLM/PR-12320.md), [[https://nvbugs/5983390][perf] Kernel fusions in _gather_k_cache_for_chunk of Indexer in DSA](../sources/prs/TensorRT-LLM/PR-12322.md), [[None][feat] Temporally-Correlated Heuristic-guided Indexer TopK for Sparse Attention](../sources/prs/TensorRT-LLM/PR-12385.md), [[https://nvbugs/5983390][fix] Remove redundant D2H sync to optimize perf](../sources/prs/TensorRT-LLM/PR-12445.md), [[None][feat] Support sparse mqa/gqa attention](../sources/prs/TensorRT-LLM/PR-12470.md), [[https://nvbugs/5983390][perf] Split MLA DSA custom op for piecewise CUDA graph capture](../sources/prs/TensorRT-LLM/PR-12503.md), [[https://nvbugs/5983390][perf] Multiple host perf optimizations for DSA part](../sources/prs/TensorRT-LLM/PR-12581.md), [[None][feat] Add triton paged attention for AutoDeploy](../sources/prs/TensorRT-LLM/PR-12642.md), [[None][feat] Optimize mamba SSD prefill and extend flashinfer dispatch](../sources/prs/TensorRT-LLM/PR-12731.md), [[TRTLLM-11485][feat] Feature rework: Add SageAttention refreshed kernels (attentionOp only)](../sources/prs/TensorRT-LLM/PR-12937.md), [[#12716][feat] Fused cross-head QK Norm + RoPE kernel for WAN](../sources/prs/TensorRT-LLM/PR-13052.md), [[TRTLLM-34871][feat] Add cute dsl FP8 paged MQA logits decode kernel](../sources/prs/TensorRT-LLM/PR-13219.md), [[None][feat] Integrate FP4 indexer for DSA on Blackwell](../sources/prs/TensorRT-LLM/PR-13340.md), [[TRTLLM-11285][perf] Force enable TF32 tensor cores for DSA indexer fused GEMM](../sources/prs/TensorRT-LLM/PR-13452.md), [[None][perf] Scheme X L2-aware dispatcher and PDL launchers for sparse-attention GVR Top-K](../sources/prs/TensorRT-LLM/PR-13477.md), [[None][perf] Drop cubin and Eliminate ~6s FMHA JIT recompile in eager generation by aligning kernel selection with CUDA graph warmup](../sources/prs/TensorRT-LLM/PR-13505.md), [[TRTLLM-12128][feat] enable SageAttention for Wan/FLUX (new commits)](../sources/prs/TensorRT-LLM/PR-13570.md), [[TRTLLM-12316][feat] Integrate FP4 indexer for DSv4](../sources/prs/TensorRT-LLM/PR-13575.md), [[#13580][fix] AutoDeploy: Support Gemma3n/4 E2B variants](../sources/prs/TensorRT-LLM/PR-13630.md), [[None][feat] Add DeepSeekV4 attention kernels](../sources/prs/TensorRT-LLM/PR-13652.md), [[None][perf] Optimize DeepSeek-V4 compressor BF16 input](../sources/prs/TensorRT-LLM/PR-13761.md), [[None][fix] Use compressed lengths for DeepSeek-V4 indexer](../sources/prs/TensorRT-LLM/PR-13802.md), [[None][feat] Update FMHA cubins for head_dim 80](../sources/prs/TensorRT-LLM/PR-13808.md), [[TRTLLM-12503][feat] Parallel VAE independent scaling and fix arg passing](../sources/prs/TensorRT-LLM/PR-13873.md), [[TRTLLM-35237][feat] Add cute dsl FP4 paged MQA logits decode kernel](../sources/prs/TensorRT-LLM/PR-13929.md), [[None][feat] Keep DSv4 o_a_proj as FP8, and port vLLM's fused_inv_rope_fp8_quant](../sources/prs/TensorRT-LLM/PR-13938.md), [[None][perf] Add CUDA q_b norm for DeepSeek V4](../sources/prs/TensorRT-LLM/PR-13975.md), [[None][feat] Enable 2 DSv4 perf optimizations by default](../sources/prs/TensorRT-LLM/PR-14120.md), [[None][feat] Add chunked prefill support for Gemma4 (text + vision multimodal)](../sources/prs/TensorRT-LLM/PR-14134.md), [[None][fix] Add SPDX Apache-2.0 headers to auto_deploy test files](../sources/prs/TensorRT-LLM/PR-14193.md), [[None][feat] DSv4: enable GVR Heuristic Top-K for compress_ratio=4](../sources/prs/TensorRT-LLM/PR-14219.md), [[None][refactor] clean up AttentionForwardArgs](../sources/prs/TensorRT-LLM/PR-14244.md), [[None][fix] Handle unset attention_dp_relax in ADP routers](../sources/prs/TensorRT-LLM/PR-14276.md), [[None][feat] GPT-OSS Sm120/Sm121 Support](../sources/prs/TensorRT-LLM/PR-7937.md), [[TRTLLM-8535][feat] Support DeepSeek V3.2 with FP8 + BF16 KV cache/NVFP4 + BF16 KV cache](../sources/prs/TensorRT-LLM/PR-8405.md), [[None][fix] Fix the performance issue of FP8 blockwise grouped GEMM when using attention DP](../sources/prs/TensorRT-LLM/PR-8501.md), [Example 77 add blackwell flash-attention bwd for MLA shape](../sources/prs/cutlass/PR-2466.md), [Add Blackwell MLA forward (shape: d=192, dv=128) implementation](../sources/prs/cutlass/PR-2472.md), [Fp8 kernel with "in-kernel" transpose of V in producer](../sources/prs/flash-attention/PR-1100.md), [FA3 FP8 qkv descales + restore max offset for h128 causal + added sync for producer WG](../sources/prs/flash-attention/PR-1173.md), [Add seqused_q in fwd / bwd and seqused_k in bwd in hopper FA.](../sources/prs/flash-attention/PR-1182.md), [Add local attention in Hopper FAv3](../sources/prs/flash-attention/PR-1233.md), [FA3 kvcache + split kv + gqa parallelization](../sources/prs/flash-attention/PR-1236.md), [Paged Attention support for FA3](../sources/prs/flash-attention/PR-1268.md), [FA3 paged attention: Readiness for Cutlass 3.6 / default value for block_table](../sources/prs/flash-attention/PR-1331.md), [Blackwell FlashAttention-BWD (v1.0)](../sources/prs/flash-attention/PR-1945.md), [[Cute,Fwd,Sm100] Support paged attention](../sources/prs/flash-attention/PR-1999.md), [[Cute,Fwd,Sm100] fp8 e4m3 and e5m2 support](../sources/prs/flash-attention/PR-2109.md), [Add SM120 varlen attention support](../sources/prs/flash-attention/PR-2333.md), [[Fwd,Sm90] Add paged KV attention support (tma and cp.async)](../sources/prs/flash-attention/PR-2360.md), [feat: update decode attention APIs](../sources/prs/flashinfer/PR-1007.md), [misc: fix instrument code for mla profiler](../sources/prs/flashinfer/PR-1014.md), [add multi-item scoring](../sources/prs/flashinfer/PR-1015.md), [fix: add zero init for KV tiled copy](../sources/prs/flashinfer/PR-1029.md), [feat: add functional per-head FP8 quantization for FA3](../sources/prs/flashinfer/PR-1033.md), [[nvidia] initial support for blackwell kernels](../sources/prs/flashinfer/PR-1039.md), [[nvidia] Add Blackwell FMHA decode kernel from TRT-LLM](../sources/prs/flashinfer/PR-1051.md), [Fix KV chunking for POD. ](../sources/prs/flashinfer/PR-1054.md), [bugfix: temporally disable split-kv in blackwell mla](../sources/prs/flashinfer/PR-1055.md), [Parameterize prefix mask call (needed by POD-Attention)](../sources/prs/flashinfer/PR-1059.md), [bugfix: adding lse output to blackwell fmha kernels](../sources/prs/flashinfer/PR-1071.md), [bugfix: follow user-specified sm_scale for blackwell cutlass fmha](../sources/prs/flashinfer/PR-1072.md), [bugfix: fix fp8 attention kernels aot compilation issue](../sources/prs/flashinfer/PR-1087.md), [bugfix: host-precomuted plan function for blackwell fmha](../sources/prs/flashinfer/PR-1106.md), [hotfix: fix the blackwell fmha stream](../sources/prs/flashinfer/PR-1116.md), [[Feature] Support PDL for batch Prefill and Decode](../sources/prs/flashinfer/PR-1117.md), [[feat] add unified batch attention w/ correctness tests.](../sources/prs/flashinfer/PR-1137.md), [Fix FA2 and FA3 multi-item scoring and cuda illegal memory access error](../sources/prs/flashinfer/PR-1140.md), [[feat] support block sparse attention w/ variable block sizes and head-wise sparse patterns](../sources/prs/flashinfer/PR-1177.md), [update trtllm-gen decode attention kernel launcher](../sources/prs/flashinfer/PR-1189.md), [bugfix: fix blackwell fmha hanging issue for empty kv_len](../sources/prs/flashinfer/PR-1198.md), [[feat] optimize persistent batch attention perf.](../sources/prs/flashinfer/PR-1200.md), [[fix] fix BatchAttention CTA_TILE_KV mask issue](../sources/prs/flashinfer/PR-1206.md), [feat: add trtllm-gen mla cubin](../sources/prs/flashinfer/PR-1222.md), [add trtllm-gen context attention](../sources/prs/flashinfer/PR-1239.md), [Add trtllm-gen attention mha kernel with FP8 Q/K/V and FP8 output](../sources/prs/flashinfer/PR-1242.md), [Bug fix: fix duplicate launch in POD](../sources/prs/flashinfer/PR-1267.md), [refactor: refactor trtllm-gen attention kernel integration code](../sources/prs/flashinfer/PR-1289.md), [[fix] fix integer overflow in FA2 customized_mask & add buffer overflow warning.](../sources/prs/flashinfer/PR-1290.md), [feat: Add k_scale and v_scale to persistent attention ](../sources/prs/flashinfer/PR-1322.md), [feat: Support logits_soft_cap for Persistent attn; fix kv split limit](../sources/prs/flashinfer/PR-1324.md), [Support passing kv_data_type to MultiLevelCascadeAttentionWrapper.plan()](../sources/prs/flashinfer/PR-1350.md), [Allow BatchPrefillPagedWrapper to call cudnn API](../sources/prs/flashinfer/PR-1384.md), [GPT-OSS Support: Add Blackwell MoE mxfp4 implementation from TRTLLM and Attention Sink](../sources/prs/flashinfer/PR-1389.md), [Adding FP8 benchmark on attention and matmul testing](../sources/prs/flashinfer/PR-1390.md), [benchmark: trtllm-gen mha with sink, add benchmark args](../sources/prs/flashinfer/PR-1415.md), [refactor: Sink attention AoT](../sources/prs/flashinfer/PR-1427.md), [feat: enable trtllm-gen attn speculative decoding verify by decode](../sources/prs/flashinfer/PR-1453.md), [Fix TRTLLM NVFP4-out attention kernel scale factor dim issue](../sources/prs/flashinfer/PR-1460.md), [feat: add pdl for trtllm-gen attn](../sources/prs/flashinfer/PR-1484.md), [feat: Support fp8 qkv, fp16/bf16 out MHA for trtllm-gen.](../sources/prs/flashinfer/PR-1490.md), [feat: integrate xqa attention backend](../sources/prs/flashinfer/PR-1503.md), [flashinfer_benchmark QoL Improvements and Attention FP8 Support](../sources/prs/flashinfer/PR-1512.md), [backend: Refactor trtllm-gen fmha metainfo loading](../sources/prs/flashinfer/PR-1518.md), [bugfix: Fix Persistent kernel precision for masked output ](../sources/prs/flashinfer/PR-1533.md), [feat: Integrate TRTLLM varlen kernel for deepseek R1 prefill ](../sources/prs/flashinfer/PR-1537.md), [feat: Add fp8-qkv, fp16/bf16 output MHA](../sources/prs/flashinfer/PR-1540.md), [bugfix: fix persistent attention kernel correctness on blackwell](../sources/prs/flashinfer/PR-1559.md), [Backend: downgrade trtllm-gen kernel to cuda-12](../sources/prs/flashinfer/PR-1567.md), [feat: Support for inferring out_dtype from out.dtype for TRTLLM attention kernel](../sources/prs/flashinfer/PR-1578.md), [fix: semaphoress must be at the fixed range in workspace buffer on trtllm_gen attention](../sources/prs/flashinfer/PR-1584.md), [fix: Improve TRTLLM attention kernel out_dtype unit test](../sources/prs/flashinfer/PR-1590.md), [feat: initial support for SM103, SM110, SM120, SM121](../sources/prs/flashinfer/PR-1608.md), [bugfix: fix merge_attention_state in BatchAttention w/ gqa-group-size in Qwen family](../sources/prs/flashinfer/PR-1614.md), [bugfix: collect all modules to aot](../sources/prs/flashinfer/PR-1622.md), [fix: pass workspace for trtllm-gen attention](../sources/prs/flashinfer/PR-1635.md), [fix: zero-init workspace buffer for trtllm-gen fmha](../sources/prs/flashinfer/PR-1643.md), [perf&bugfix: skip kv-tile computation out of sliding window in FA2; fix __syncthreads in mergestate](../sources/prs/flashinfer/PR-1661.md), [Refactor Blackwell unit test scripts](../sources/prs/flashinfer/PR-1667.md), [feat: Add `variant.OutputTransform()` to decode kernels](../sources/prs/flashinfer/PR-1670.md), [feat: Batch-size invariant FA2 Prefill & Decode](../sources/prs/flashinfer/PR-1675.md), [[misc] add a wrapper class for attention sink jit args](../sources/prs/flashinfer/PR-1679.md), [perf: improve attention of tcgen05 flash-attention](../sources/prs/flashinfer/PR-1681.md), [perf: Port the separate reduce kernel mode from trtllm.](../sources/prs/flashinfer/PR-1685.md), [bugfix: increase workspace to make trtllm gen attention unit test pass](../sources/prs/flashinfer/PR-1707.md), [test: skip the unsupported test cases for sm120/121](../sources/prs/flashinfer/PR-1710.md), [tests: xfail attention sink UT for sliding window + non causal case](../sources/prs/flashinfer/PR-1752.md), [Fix tests/test_trtllm_gen_attention.py::test_trtllm_batch_prefill, ::test_trtllm_batch_decode mismatch error](../sources/prs/flashinfer/PR-1755.md), [feat: add xqa fp8 mha and fp8 kv cache](../sources/prs/flashinfer/PR-1769.md), [Bugfix: Fix data hazard in persistent reduce](../sources/prs/flashinfer/PR-1826.md), [Add head_dim=64 for tcgen05 tcgen05 flash-attention implementation](../sources/prs/flashinfer/PR-1850.md), [Bugfix: fix o_strides in persistent kernel ](../sources/prs/flashinfer/PR-1865.md), [Tune kernel compilation parameters for https://github.com/flashinfer-ai/flashinfer/pull/1850 ](../sources/prs/flashinfer/PR-1878.md), [fix: Fix trtllm-gen prefill IMA when batch_size==1](../sources/prs/flashinfer/PR-1912.md), [MLA RoPE + quantization fused kernel: shape generalization for MHA / GQA](../sources/prs/flashinfer/PR-1924.md), [Add realistic bench for persistent kernel ](../sources/prs/flashinfer/PR-1942.md), [fix: Make attention microbenchmark correctly use page table](../sources/prs/flashinfer/PR-1976.md), [fix: Skipping attention sink Blackwell test outside of Blackwell](../sources/prs/flashinfer/PR-1978.md), [fix: correct PDL parameter handling in RopeQuantize kernel](../sources/prs/flashinfer/PR-1982.md), [minor fix for xqa](../sources/prs/flashinfer/PR-1994.md), [unittest: Add head dim 256 test cases and mark as xfail](../sources/prs/flashinfer/PR-1999.md), [feat: add xqa backend and completes NHD/HND coverage for trtllm-gen/xqa backend](../sources/prs/flashinfer/PR-2001.md), [Fix trtllm-gen attention illegal memory access](../sources/prs/flashinfer/PR-2002.md), [use scalar for kv_scale in xqa](../sources/prs/flashinfer/PR-2033.md), [Added an initial implementation of Q and KV Cache in fp8 and to use t…](../sources/prs/flashinfer/PR-2035.md), [feat: Add flashinfer.rope.rope_quantize_fp8_append_paged_kv_cache (fused RoPE + Q + KV cache, supports MLA/GQA/MHA) ](../sources/prs/flashinfer/PR-2037.md), [Rebase FP8 SM100 Cutlass FMHA Attention to main (original PR#1238)](../sources/prs/flashinfer/PR-2047.md), [feat: add xqa mla backend](../sources/prs/flashinfer/PR-2053.md), [misc: Add XQA decode to microbenchmark for sm90 and sm120](../sources/prs/flashinfer/PR-2055.md), [Fix: several bugs/issues with trtllm-gen attention kernels. ](../sources/prs/flashinfer/PR-2062.md), [unittest: improve the efficiency of xqa unittests](../sources/prs/flashinfer/PR-2075.md), [fix: fix test_trtllm_gen_attention when max_seq_len < page_size](../sources/prs/flashinfer/PR-2076.md), [[Feature] Support batch prefill for POD Attention](../sources/prs/flashinfer/PR-2079.md), [enable xqa fp8 output](../sources/prs/flashinfer/PR-2081.md), [[API change] Allow using torch.Tensor for scales for trtllm-gen attention](../sources/prs/flashinfer/PR-2084.md), [[DSR1] Added MLA test](../sources/prs/flashinfer/PR-2100.md), [enable xqa speculative decoding](../sources/prs/flashinfer/PR-2105.md), [feat: support more head dim in RoPE kernel](../sources/prs/flashinfer/PR-2109.md), [add tensor scale input for xqa](../sources/prs/flashinfer/PR-2110.md), [refactor: update fa3 codebase and fix hopper unittest [part 1]](../sources/prs/flashinfer/PR-2111.md), [feature: make the LSE returned by MLA support base 2 or e #2113](../sources/prs/flashinfer/PR-2114.md), [perf: bunch of features and optimizations for top-k (sampling + sparse attention)](../sources/prs/flashinfer/PR-2119.md), [feat: support variable sequence length in decode kernel of trtllm-gen attention](../sources/prs/flashinfer/PR-2125.md), [fix flaky xqa test](../sources/prs/flashinfer/PR-2126.md), [fix(trtllm): reset negative strideBatch to 0 for ragged KV layout to …](../sources/prs/flashinfer/PR-2134.md), [fix: some bugs of headDim 256 trtllm-gen fmha kernels. ](../sources/prs/flashinfer/PR-2137.md), [feat: add trtllm-gen per-tensor sparseMla kernels.](../sources/prs/flashinfer/PR-2138.md), [feat: TRTLLM FMHAv2 backend for ctx attention](../sources/prs/flashinfer/PR-2142.md), [Enable Hopper FA3 FP8 attention in decode.py](../sources/prs/flashinfer/PR-2148.md), [refactor: Move mla code from decode.py to mla.py and add to documentation](../sources/prs/flashinfer/PR-2163.md), [fix: compile flags for trtllm fmha_v2 ](../sources/prs/flashinfer/PR-2175.md), [Permute page table in benchmarking](../sources/prs/flashinfer/PR-2194.md), [Fp8 attention are now part of cuDNN 9.17.1](../sources/prs/flashinfer/PR-2241.md), [feat: support non-contiguous query for trtllm-gen attention backend](../sources/prs/flashinfer/PR-2254.md), [fix: support int64 IdType for RoPE part argument in `rope_quantize_fp8_append_paged_kv_cache`](../sources/prs/flashinfer/PR-2255.md), [[TRTLLM-Gen Fmha] add optimized trtllm-gen decode kernels for high throughput + speculative decoding](../sources/prs/flashinfer/PR-2265.md), [feat: add GDN Attention](../sources/prs/flashinfer/PR-2276.md), [fix: Decode benchmark's fa2_tc uses backend=fa2 in wrapper](../sources/prs/flashinfer/PR-2302.md), [Added the cudnn backend Ragged KV Cache wrapper](../sources/prs/flashinfer/PR-2352.md), [feat: Add TRTLLM fmha_v2 library for SM90 attention with Skip-Softmax ](../sources/prs/flashinfer/PR-2446.md), [feat: Add TRTLLM-Gen Skip-Softmax kernels for prefill and decode](../sources/prs/flashinfer/PR-2477.md), [fix: Fix memory bandwidth calculation in MLA benchmarks](../sources/prs/flashinfer/PR-2479.md), [Support NVFP4 KV cache decode on SM120](../sources/prs/flashinfer/PR-2520.md), [pick fa2 for BatchDecodeWithPagedKVCacheWrapper auto backend](../sources/prs/flashinfer/PR-2530.md), [feat: Enable TRTLLM-Gen Skip-Softmax attention for MLA](../sources/prs/flashinfer/PR-2547.md), [fix: allow fmha_v2_prefill_deepseek on SM121 (DGX Spark)](../sources/prs/flashinfer/PR-2559.md), [fix: guard CUTLASS FMHA against SM12x and fix fmha_v2 SM121a check](../sources/prs/flashinfer/PR-2560.md), [feat: add is_sm12x_supported() helper for SM12x family detection](../sources/prs/flashinfer/PR-2574.md), [support qk_nope_head_dim for 192 check for GLM-5](../sources/prs/flashinfer/PR-2607.md), [benchmark: Enable speculative decode microbenchmarking for paged decode](../sources/prs/flashinfer/PR-2628.md), [fix: add SM121 support to SM120 version guards](../sources/prs/flashinfer/PR-2631.md), [[feat] trtllm-gen mxfp8 gemm](../sources/prs/flashinfer/PR-2653.md), [benchmarks: Add FP8 input / BF16 output in ragged prefill benchmark](../sources/prs/flashinfer/PR-2666.md), [feat: add support for more MLA head dimensions](../sources/prs/flashinfer/PR-2677.md), [Add NVFP4 KV cache quantization support for SM100](../sources/prs/flashinfer/PR-2702.md), [feat: Add DiT-oriented kernels where Qk (Bmm1) type can be reinterpreted into Int8 or BFloat16](../sources/prs/flashinfer/PR-2711.md), [Add cute dsl mla decode op](../sources/prs/flashinfer/PR-2743.md), [[Spark unit test debugging] Fix for tests/attention/test_trtllm_gen_mla.py](../sources/prs/flashinfer/PR-2750.md), [feat: Expose TRT-LLM FMHA style paged KV Cache and page table layout](../sources/prs/flashinfer/PR-2770.md), [feat: FP8 output support for CUTLASS MLA paged attention](../sources/prs/flashinfer/PR-2779.md), [tests: skip sliding window + fp8 to prevent hang in fmha_v2 unit tests](../sources/prs/flashinfer/PR-2781.md), [feat: Support padding tokens with seqlen=0 for rope+quant+kv cache update fusion kernel](../sources/prs/flashinfer/PR-2792.md), [[fmha-v2] Support HND and NHD paged KV cache layouts with conditional stride handling](../sources/prs/flashinfer/PR-2799.md), [[fix] Bugfix 1367: fix VariableBlockSparseAttention buffer overflow by dynamically resizing kv_lens_buffer](../sources/prs/flashinfer/PR-2802.md), [[CuTe DSL] Add modular FMHA prefill and MLA decode attention kernels](../sources/prs/flashinfer/PR-2805.md), [[Spark unit test] Adjust tolerance for test_xqa, test_logits_processor](../sources/prs/flashinfer/PR-2828.md), [[Perf] Add FMHAv2 to flashinfer_benchmark.py and eliminate unnecessary H2D](../sources/prs/flashinfer/PR-2841.md), [[fix] bugfix 2856: Fix pre-allocated out shape check in trtllm_batch_decode_with_kv_cache_mla for q_len_per_req > 1](../sources/prs/flashinfer/PR-2876.md), [feat: add pdl support for cute dsl mla decode kernel support](../sources/prs/flashinfer/PR-2901.md), [Only swizzle on v block scale; rename kv_block_scales to kv_cache_sf](../sources/prs/flashinfer/PR-2954.md), [test: xfail cuDNN FP8 prefill on Blackwell with CUDA <= 12.9](../sources/prs/flashinfer/PR-2963.md), [[Fmha] support nvfp4 output keepsMmaAb generation kernels](../sources/prs/flashinfer/PR-2988.md), [Support lse in trtllm paged attn kernels](../sources/prs/flashinfer/PR-3058.md), [Support NVFP4 KV for prefill and batch attention kernels](../sources/prs/flashinfer/PR-3097.md), [cute-dsl fmha prefill (cubin integration): remove front-padding, add attention_sink, and pdl support](../sources/prs/flashinfer/PR-3181.md), [feat(trace): embed runnable init() in every TraceTemplate](../sources/prs/flashinfer/PR-3221.md), [Support Kimi K2.5 H64 CuTe DSL MLA decode](../sources/prs/flashinfer/PR-3235.md), [Add dynamic tokens-per-page TRTLLM-GEN GQA kernels](../sources/prs/flashinfer/PR-3259.md), [fix(fmha_v2): fix FP8 V-scratch pipeline and varlen scheduler on SM90](../sources/prs/flashinfer/PR-3276.md), [perf: fix the iteration bound of SWA in FA2 prefill template](../sources/prs/flashinfer/PR-714.md), [Align KV chunk size binary search with actual KV chunk splitting.](../sources/prs/flashinfer/PR-728.md), [feat: support deepseek prefill attention shape](../sources/prs/flashinfer/PR-765.md), [perf: refactor fa2 prefill template](../sources/prs/flashinfer/PR-776.md), [bugfix: fix batch prefill attention kernel unittests](../sources/prs/flashinfer/PR-781.md), [bugfix: MLA decode should multiply sm_scale by math::log2e](../sources/prs/flashinfer/PR-787.md), [fix rope logic in mla decoding](../sources/prs/flashinfer/PR-793.md), [feat: support f32 attention output in FA2 template](../sources/prs/flashinfer/PR-799.md), [feat: apply sm_scale at logits instead of q in FA2 template](../sources/prs/flashinfer/PR-801.md), [perf: memory efficient deepseek mla fused page-attention kernel](../sources/prs/flashinfer/PR-804.md), [bugfix: mla page-attention kernel for different page sizes](../sources/prs/flashinfer/PR-810.md), [feat: unlocking MLA for A100](../sources/prs/flashinfer/PR-812.md), [feat: unlock MLA attention for sm89 (L40/L40s/4090)](../sources/prs/flashinfer/PR-814.md), [bugfix: bugfix on sm89 MLA](../sources/prs/flashinfer/PR-821.md), [perf: MLA decode kernel implemented by CuTe targeted to SM80](../sources/prs/flashinfer/PR-844.md), [Add POD-Attention to FlashInfer](../sources/prs/flashinfer/PR-858.md), [perf: dynamic split-k for MLA](../sources/prs/flashinfer/PR-863.md), [bugfix: fix the behavior of MLA kernel when kv-length is 0](../sources/prs/flashinfer/PR-868.md), [Naive Support for Hopper FP8 Prefill Kernel with Per-Head Quantization](../sources/prs/flashinfer/PR-869.md), [perf: FlashAttention-3 style MLA PageAttention](../sources/prs/flashinfer/PR-887.md), [perf: fix MLA split-k performance bug](../sources/prs/flashinfer/PR-898.md), [perf: tweak the pipeline design of mla kernel](../sources/prs/flashinfer/PR-901.md), [feat: flashinfer intra-kernel profiler](../sources/prs/flashinfer/PR-913.md), [bugfix: fix potential issues of FA3 template loading nans for PageAttention](../sources/prs/flashinfer/PR-945.md), [perf: Use 2WG pipeline design for MLA implementation on Hopper](../sources/prs/flashinfer/PR-952.md), [[TVM] Added tvm binding for sampling kernel](../sources/prs/flashinfer/PR-958.md), [perf: prefetch page indices for mla kernel](../sources/prs/flashinfer/PR-991.md), [3rdparty: upgrade cutlass to 3.9](../sources/prs/flashinfer/PR-997.md), [[FlexAttention] Remove Old Constraint on lastdim strides](../sources/prs/pytorch/PR-153104.md), [[FlexAttention] explicilty create grad_q w/ strides](../sources/prs/pytorch/PR-153641.md), [[SDPA] [MPS] Fixes regression in 2.8.0 for scaled_dot_product_attention using mps](../sources/prs/pytorch/PR-164364.md), [[Flex attention] Fix flex attention head broadcast](../sources/prs/pytorch/PR-164368.md), [[flex_attention] adds support for low precision K/V inputs in compiled mode with GPU](../sources/prs/pytorch/PR-170486.md), [[cherry-pick] Fix vllm issue for flex (#170499)](../sources/prs/pytorch/PR-170555.md), [[RELEASE 2.11] Release only changes](../sources/prs/pytorch/PR-175091.md), [[MPS] Fix 2-pass SDPA memory corruption by forcing float accumulators](../sources/prs/pytorch/PR-175580.md), [[release 2.12] Apply Release only changes to 2.12 branch](../sources/prs/pytorch/PR-180470.md), [Disable kernel cutlass_mla_decode on SM103](../sources/prs/sglang/PR-10058.md), [feat: Add FP4 (E2M1) KV Cache Support with Quantization Utilities for MLA](../sources/prs/sglang/PR-10078.md), [[Feature] Add MLAProcess for DeepSeek MLA on NPU](../sources/prs/sglang/PR-10130.md), [Fix chunked prefix cache for nvfp4](../sources/prs/sglang/PR-10180.md), [support qwen3_next blackwell](../sources/prs/sglang/PR-10403.md), [Fix cutlass moe accuracy drop caused by attention UB from DP padding mode](../sources/prs/sglang/PR-10414.md), [Enable trtllm mla prefix extend](../sources/prs/sglang/PR-10526.md), [Fuse quantize and rope in trtllm_mla MTP](../sources/prs/sglang/PR-10779.md), [[2/2] Support MHA prefill with FlashAttention 4.](../sources/prs/sglang/PR-10937.md), [[NVIDIA] FA3/FA4 Fix ](../sources/prs/sglang/PR-11606.md), [[DeepseekV32] Enable flashmla_prefill kernel with fp8 kvcache](../sources/prs/sglang/PR-11655.md), [Use trtllm_mla decode kernel for draft extend in speculative decoding](../sources/prs/sglang/PR-11664.md), [Support running FP4 Deepseek on SM120.](../sources/prs/sglang/PR-11708.md), [Change bf16 to fp8 for some gemms in attention for DeepSeek ckpt v2](../sources/prs/sglang/PR-11805.md), [Support nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8/NVFP4](../sources/prs/sglang/PR-11866.md), [DeepSeek-V3.2: Add Adaptive MHA Attention Pathway for Short-Sequence Prefill](../sources/prs/sglang/PR-11892.md), [Feature/nano v2 offline modelopt fp8 and nvfp4](../sources/prs/sglang/PR-12018.md), [(1/n)support context parallel with deepseekv3.2-DSA](../sources/prs/sglang/PR-12065.md), [[Ascend] qwen optimization](../sources/prs/sglang/PR-12078.md), [[Ascend][feature] support L1+ L2 radixcache on ascend](../sources/prs/sglang/PR-12214.md), [[DeepseekV32]: use `_concat_mla_absorb_q_general` to replace `torch.cat`](../sources/prs/sglang/PR-12215.md), [[Deepseek V3.2] Enable flashmla_auto with MTP](../sources/prs/sglang/PR-12294.md), [fix seqlen bug for trtllm_mla's draft_extend](../sources/prs/sglang/PR-12295.md), [perf: trtllm mla performance minor improvements](../sources/prs/sglang/PR-12435.md), [[Ascend] Support enable-mixed-chunk in non-MLA scenarios](../sources/prs/sglang/PR-12491.md), [[DeepSeek-V3.2][NSA] Enable MHA Pathway for Short Sequence Prefill on B200 (SM100)](../sources/prs/sglang/PR-12788.md), [[Deepseek V3.2] Only skip Indexer logits computation when is_extend_without_speculative](../sources/prs/sglang/PR-12816.md), [[Deepseek V3.2] Use torch.compile to speed up torch.cat in nsa](../sources/prs/sglang/PR-13022.md), [support mtp with deepseek r1 nvfp4 model](../sources/prs/sglang/PR-13115.md), [Aiter fp8 kv cache](../sources/prs/sglang/PR-13147.md), [Support internvl on Blackwell (which doesn't support fa3): add `SingletonCache` support to Vision{Sdpa|Triton|Ascend}Attention](../sources/prs/sglang/PR-13151.md), [diffusion: enable fa4 for blackwell](../sources/prs/sglang/PR-13263.md), [[BugFix] fix prefixcache performance and accuracy on ascend](../sources/prs/sglang/PR-13573.md), [[DeepSeekV3.2] Enable pure TP & Partial DP Attention](../sources/prs/sglang/PR-13646.md), [fix trtllm mla spec](../sources/prs/sglang/PR-13738.md), [[DeepSeek v3.2] opt Context Parallelism: support fused moe, multi batch and fp8 kvcache](../sources/prs/sglang/PR-13959.md), [Use trtllm mha decode kernel for target_verify in speculative decoding](../sources/prs/sglang/PR-13976.md), [fix: Increase FlashInfer workspace size for Qwen3VL models](../sources/prs/sglang/PR-14173.md), [Add Mistral Large 3 support.](../sources/prs/sglang/PR-14213.md), [Tiny use trtllm_mha as default when possible](../sources/prs/sglang/PR-14291.md), [[Fix] add block size logic for sm120 smem size](../sources/prs/sglang/PR-14311.md), [sync attention, deepseek doc](../sources/prs/sglang/PR-14335.md), [Support FP8 MLA prefill and 128k context.](../sources/prs/sglang/PR-14395.md), [[NPU] perf update with kvcache nz & w4a8 quant](../sources/prs/sglang/PR-14423.md), [Add Mistral Large 3 Eagle Support](../sources/prs/sglang/PR-14466.md), [[NPU][eagle3] support qwen eagle3 on NPU](../sources/prs/sglang/PR-14820.md), [fix: trtllm mha attention auto-selection on sm120](../sources/prs/sglang/PR-14842.md), [Fix dsv3 dp accuracy issue when using bf16-kv](../sources/prs/sglang/PR-14897.md), [Fix accuracy issue when using a16w16 mla_decode_fwd](../sources/prs/sglang/PR-14936.md), [fix(attention): Prevent trtllm_mha auto-selection with eagle3 speculative decoding](../sources/prs/sglang/PR-15127.md), [[NVIDIA] upstream FA4](../sources/prs/sglang/PR-15182.md), [[NPU]DeepSeek-V3.2 support npu mlaprolog](../sources/prs/sglang/PR-15381.md), [[diffusion] Add Sage Attention 3 Support for sm 120 (RTX5090)](../sources/prs/sglang/PR-15382.md), [Optimize FP8 MLA KV cache writes with Triton kernel](../sources/prs/sglang/PR-15522.md), [Fix BatchMLAPagedAttentionWrapper query/qo_inptr mismatch for EAGLE](../sources/prs/sglang/PR-15601.md), [[jit-kernel] Add CuTe DSL GDN Decode Kernel](../sources/prs/sglang/PR-15631.md), [[diffusion] model: support TurboWan2.1-T2V-1.3B/14B SLA](../sources/prs/sglang/PR-15888.md), [Bugfix for ds-vl2](../sources/prs/sglang/PR-15894.md), [Support fa4 decoding](../sources/prs/sglang/PR-16034.md), [optimize get_topk_ragged by fusing get k and k_scale triton kernel](../sources/prs/sglang/PR-16043.md), [enhance accuracy for model kimi-vl-instruct-a3b](../sources/prs/sglang/PR-16076.md), [[VLM] Adopt jit qk_norm kernel in VLM](../sources/prs/sglang/PR-16171.md), [[diffusion] Fix RuntimeError in SageAttention3 on Nvidia Blackwell with Qwen-Image](../sources/prs/sglang/PR-16335.md), [[Fix]Fix FA3 Performance in Diffusion Model ](../sources/prs/sglang/PR-16382.md), [[DeepSeek v3.2] Opt MTP decode cuda batch sizes and nsa implementation](../sources/prs/sglang/PR-16961.md), [[NPU]bugfix: fix for dsv3.2 and dsvl2](../sources/prs/sglang/PR-17007.md), [Optimize GDN decode for Qwen3 Next](../sources/prs/sglang/PR-17094.md), [[diffusion] fix: fix using upstream flash_attn on blackwell](../sources/prs/sglang/PR-17111.md), [Enable XQA for SM90 and SM120](../sources/prs/sglang/PR-17115.md), [[New Model] GLM4.7-Flash](../sources/prs/sglang/PR-17247.md), [Disable mla persistent kernel when not using fp8 kv_cache](../sources/prs/sglang/PR-17327.md), [Move fa4 from sgl-kernel to jit kernel](../sources/prs/sglang/PR-17353.md), [[NPU] enhance accuracy for model kimi-vl-a3b-instruct](../sources/prs/sglang/PR-17480.md), [Kernel: optimize decoding metadata in NSA multi-spec backend with fused kernels](../sources/prs/sglang/PR-17554.md), [Feat/add fi selective state update kernel call](../sources/prs/sglang/PR-18070.md), [Support Qwen3 MoE context parallel](../sources/prs/sglang/PR-18233.md), [[ROCm] Optimize Deepseek R1 on MI300X](../sources/prs/sglang/PR-18242.md), [[AMD] Support Qwen3-Coder-Next on AMD platform](../sources/prs/sglang/PR-18355.md), [feat(gdn): add FlashInfer K-last SSM layout support for GDN prefill and decode for Hopper](../sources/prs/sglang/PR-18361.md), [Nsa trtllm mla sparse fp8 support with Deepseek v3.2 NVFP4](../sources/prs/sglang/PR-18389.md), [[AMD] Update aiter to v0.1.10.post2](../sources/prs/sglang/PR-18423.md), [feat: add FA4 SM90 paged KV decode support & update attention docs](../sources/prs/sglang/PR-18442.md), [Tilelang sparse decode fwd for dsv32 mi355](../sources/prs/sglang/PR-18488.md), [Fp8 prefill attn kernel integration](../sources/prs/sglang/PR-18528.md), [[AMD] Fix accuracy issue when running TP4 dsv3 model with mtp](../sources/prs/sglang/PR-18607.md), [[sgl-kernel] rebase FlashMLA 0217](../sources/prs/sglang/PR-18902.md), [Fix NSA FP8 KV cache path for both-trtllm MHA one-shot](../sources/prs/sglang/PR-18931.md), [Support skip-softmax attention](../sources/prs/sglang/PR-19089.md), [[DeepSeek-V3.2][JIT-kernel] Support nsa fuse store indexer k cache](../sources/prs/sglang/PR-19148.md), [[NVIDIA] Integrate FlashInfer decode kernel (Blackwell) for Qwen3.5](../sources/prs/sglang/PR-19150.md), [[AMD] Fix accuracy while using --enable-dp-attention](../sources/prs/sglang/PR-19247.md), [[Feature] add feature mla_ag_after_qlora for dsv3.2](../sources/prs/sglang/PR-19428.md), [[NPU] bugs fix for Deepseek models](../sources/prs/sglang/PR-19544.md), [[miles] fix for glm5](../sources/prs/sglang/PR-19634.md), [Fix MLA decode path returning unwritten (padded) rows](../sources/prs/sglang/PR-19902.md), [[AMD] Fix Tensor Memory Aliasing ](../sources/prs/sglang/PR-19928.md), [[AMD] Fix FP8 assertion failure in aiter MLA decode by falling back to self.k_scale](../sources/prs/sglang/PR-19935.md), [[AMD] Tilelang sparse fwd for dsv32 mi355/mi300](../sources/prs/sglang/PR-19945.md), [MiniMax-M2.5 - Support dp attention, dp reduce scatter, FP4 all gather, AR fusion in prepare_attn](../sources/prs/sglang/PR-20067.md), [[AMD] Fp8 prefill integration with radix cache path for dpsk models](../sources/prs/sglang/PR-20187.md), [[AMD] Add 4-GPU test suite for MI325 runners](../sources/prs/sglang/PR-20294.md), [[Benchmark] use flashinfer bench_gpu_time instead of triton do_bench](../sources/prs/sglang/PR-20305.md), [[AMD][Bug-fix] Fix gpu fault when run the test with dp-attention-enabled and max-concurrency is over 256](../sources/prs/sglang/PR-20399.md), [[AMD][AITER] Guard _use_mla_ps_kernel with self.use_mla in draft_extend_v2 paths](../sources/prs/sglang/PR-20409.md), [[GDN] Add benchmark for sglang gdn prefill](../sources/prs/sglang/PR-20428.md), [Support Triton MLA FP8 KV cache](../sources/prs/sglang/PR-20479.md), [[Diffusion] Clean upstream fa3 in hopper](../sources/prs/sglang/PR-20576.md), [Use Flashinfer for target_verify in GDN model for SM120](../sources/prs/sglang/PR-20604.md), [FIX: (NSA) Compute topk_indices_offset when NSA prefill flashmla_sparse is used with FP8 KV cache](../sources/prs/sglang/PR-20606.md), [Add SGLang CUDA crash API logging inspired by FlashInfer](../sources/prs/sglang/PR-20910.md), [ci: run Stage A CUDA tests as stage-a-test-small-1-gpu on 5090](../sources/prs/sglang/PR-20988.md), [perf: precompute FA3 scheduler_metadata to eliminate per-layer prepare_varlen_num_blocks](../sources/prs/sglang/PR-21104.md), [[Not-Merge][AMD] GLM-5 performance optimization](../sources/prs/sglang/PR-21166.md), [[Whisper] Enable CUDA graph support and timestamp for whisper model](../sources/prs/sglang/PR-21190.md), [[KDA] Support CuTeDSL KDA decode kernel](../sources/prs/sglang/PR-21203.md), [[AMD]: Support MLA with nhead<16 and FP8 KV cache for TP=8 (Kimi K2.5…](../sources/prs/sglang/PR-21213.md), [Refactor JIT kernel CI to use run_suite.py registration system](../sources/prs/sglang/PR-21239.md), [[misc] clean up kernel API](../sources/prs/sglang/PR-21325.md), [[GDN] Fuse GDN kkt + solve_tril into one kernel](../sources/prs/sglang/PR-21411.md), [[Bugfix] Lazy-import CuteDSL KDA kernel to fix AMD/ROCm startup crash](../sources/prs/sglang/PR-21428.md), [fix nemotron capture for non attention layers](../sources/prs/sglang/PR-21436.md), [fix: piecewise_cuda_graph get correct qo_indptr](../sources/prs/sglang/PR-21452.md), [Migrate all callers from /get_server_info to /server_info](../sources/prs/sglang/PR-21463.md), [[AMD] Enable FP8 KV cache and FP8 attention kernel for NSA on MI300/MI355 with TileLang backend](../sources/prs/sglang/PR-21511.md), [Change default mm-attention backend from triton_attn to fa4](../sources/prs/sglang/PR-21595.md), [fix: TRT-LLM MHA CUDA illegal address with EAGLE v2 + DP attention](../sources/prs/sglang/PR-21649.md), [[XPU] Enable qwen3.5 on XPU](../sources/prs/sglang/PR-21668.md), [[DSA] Support trtllm sparse mla kernel for prefill batches ](../sources/prs/sglang/PR-21783.md), [[Bugfix] Temporarily skip TRTLLM attention on (G)B300 (SM103) to avoid high-concurrency hang](../sources/prs/sglang/PR-21906.md), [[Bugfix] Fix CUDA graph replay issues in trtllm_mla draft_extend](../sources/prs/sglang/PR-21987.md), [[NPU] enable mla prepare fused kernel only when being mla attn](../sources/prs/sglang/PR-22024.md), [[MUSA][9/N] Add FA3 attention backend support through MATE (MUSA AI Tensor Engine)](../sources/prs/sglang/PR-22051.md), [[nvidia] Gemma4 nvfp4 fix](../sources/prs/sglang/PR-22079.md), [Reduce unnecessary kernels and copies in the NSA indexer](../sources/prs/sglang/PR-22232.md), [[AMD][HIP] NSA: bf16 passthrough from RMSNorm to eliminate FP8 dequantization](../sources/prs/sglang/PR-22258.md), [Lazy import flash_attention_v4 to avoid loading flash_attn.cute at startup](../sources/prs/sglang/PR-22306.md), [[DSA] Hopper FP8 FlashMLA KV padding](../sources/prs/sglang/PR-22372.md), [[AMD] Use aiter CK layernorm2d for LayerNorm to reduce NSA indexer kernel launches](../sources/prs/sglang/PR-22424.md), [[Fix] Fix several bugs on DSA models](../sources/prs/sglang/PR-22430.md), [[BugFix] Resolve adaptive speculative decoding conflicts for Qwen3.5 (hybrid GDN)](../sources/prs/sglang/PR-23331.md), [[Diffusion][NPU]Add attention backends for diffusion models for Ascend NPU](../sources/prs/sglang/PR-23482.md), [[feat] Init true on policy with qwen_dense](../sources/prs/sglang/PR-23961.md), [Enable PDL for various kernels in DSV32/GLM5](../sources/prs/sglang/PR-23965.md), [[KDA] Optimize prefill kernels with diagonal and recompute fuse](../sources/prs/sglang/PR-24271.md), [Support spec v2 for FlashMLA speculative decoding](../sources/prs/sglang/PR-24640.md), [[attn backend] Integrate tokenspeed_mla prefill/decode kernels (fp8 kv cache, blackwell)](../sources/prs/sglang/PR-24925.md), [Amd/deepseek v4 rebase main 0509](../sources/prs/sglang/PR-24933.md), [Fix AMX GQA extend attention](../sources/prs/sglang/PR-25180.md), [[NSA] Avoid repeated NSA MQA logits memory queries](../sources/prs/sglang/PR-25299.md), [[Intel GPU] Enable DeepSeek V4 Inference on XPU](../sources/prs/sglang/PR-25336.md), [Support draft extend cuda graph for tokenspeed_mla attention backend](../sources/prs/sglang/PR-25489.md), [amd/deepseek_v4 27/N [fix] Reduce Triton autotune configs for faster first-time server launch](../sources/prs/sglang/PR-25554.md), [Use triton_attn as default vision attention on B300 (SM103)](../sources/prs/sglang/PR-25570.md), [Introduce SchedulerDPAttnAdapter to own DP-attention state](../sources/prs/sglang/PR-25611.md), [Move DP-attention adapter methods to SchedulerDPAttnAdapter](../sources/prs/sglang/PR-25612.md), [[Refactor] Rename NSA → DSA: user-facing aliases, file/class/import rename](../sources/prs/sglang/PR-25821.md), [[Test] Stage-a sanity kits; consolidate core/ + models_e2e/ tests](../sources/prs/sglang/PR-25831.md), [feat: support flashinfer mla attention for deepseek v3](../sources/prs/sglang/PR-3550.md), [feat: support flashinfer mla with prefix cache](../sources/prs/sglang/PR-3643.md), [Add fast decode plan for flashinfer mla](../sources/prs/sglang/PR-3987.md), [[Revision] Add fast decode plan for flashinfer mla ](../sources/prs/sglang/PR-4012.md), [Support overlapping two batches](../sources/prs/sglang/PR-4068.md), [avoid cudaStreamSynchronize in DeepSeekV2AttentionMLA](../sources/prs/sglang/PR-4577.md), [Fix loading KV quantization scale; Enable modelopt kv cache](../sources/prs/sglang/PR-4686.md), [[Model] Adding Qwen3 and Qwen3MoE](../sources/prs/sglang/PR-4693.md), [support cmake for sgl-kernel](../sources/prs/sglang/PR-4706.md), [Support (1 <= dp < tp) in the dp attention in DeepEP](../sources/prs/sglang/PR-4770.md), [Feat/support encoder model (like bert)](../sources/prs/sglang/PR-4887.md), [Support MHA with chunked prefix cache for DeepSeek chunked prefill](../sources/prs/sglang/PR-5113.md), [Blackwell Cutlass MLA kernel](../sources/prs/sglang/PR-5142.md), [feat: use fa3 mla by default on hopper](../sources/prs/sglang/PR-5210.md), [Add Speculative Decoding Eagle3 topk > 1](../sources/prs/sglang/PR-5318.md), [fix: solve cu118 issue for cutlass mla](../sources/prs/sglang/PR-5331.md), [Add Cutlass MLA attention backend](../sources/prs/sglang/PR-5390.md), [[Feat] upgrade pytorch2.6](../sources/prs/sglang/PR-5417.md), [BLackwell cutlass mla: Add check for bad page size/block num combinations](../sources/prs/sglang/PR-5431.md), [Avoid computing lse in Ragged Prefill when there's no prefix.](../sources/prs/sglang/PR-5476.md), [[PP] Add pipeline parallelism](../sources/prs/sglang/PR-5724.md), [Fuse MLA set kv cache kernel](../sources/prs/sglang/PR-5748.md), [opt flashinfer mla cat](../sources/prs/sglang/PR-5822.md), [Cutlass MLA decode - fix dtype error](../sources/prs/sglang/PR-5868.md), [[Fix] Fix a bug for flashmla to run R1 model](../sources/prs/sglang/PR-5875.md), [Improve dp attention port assignment scheme](../sources/prs/sglang/PR-5889.md), [feat: mtp support dp-attention](../sources/prs/sglang/PR-6081.md), [Cutlass MLA: Disable split kv due to https://github.com/NVIDIA/cutlass/issues/2274](../sources/prs/sglang/PR-6101.md), [feat: add dp attention support for Qwen 2/3 MoE models, fixes #6088](../sources/prs/sglang/PR-6121.md), [Enable FlashInfer support encoder models and add head_dim padding workaround](../sources/prs/sglang/PR-6230.md), [[Feature] Comprehensive Hybrid Parallelism Support](../sources/prs/sglang/PR-6389.md), [[CPU] support the case where num_attention_heads or intermediate_size is not divisible by the TP size](../sources/prs/sglang/PR-6771.md), [[perf][sgl-kernel] extend cutlass_mla_decode to support num_head < 128](../sources/prs/sglang/PR-6929.md), [[Feature] Support Flashinfer fmha on Blackwell](../sources/prs/sglang/PR-6930.md), [Fix cutlass MLA gets almost zero accuracy](../sources/prs/sglang/PR-6998.md), [Tiny fix cutlass_mla_get_workspace_size stub incorrect signature](../sources/prs/sglang/PR-7057.md), [chore: upgrade sgl-kernel v0.1.8.post2](../sources/prs/sglang/PR-7186.md), [Fix grammar abort & Minor style fixes](../sources/prs/sglang/PR-7204.md), [feat: support compatibility between MTP and two-batch-overlap](../sources/prs/sglang/PR-7225.md), [Minor style and doc fix](../sources/prs/sglang/PR-7228.md), [Support non-contiguous query input for extend/decode attention](../sources/prs/sglang/PR-7462.md), [Add fp4 quantize before all-gather for Flashinfer cutlass MoE DP (max throughput)](../sources/prs/sglang/PR-7667.md), [TRTLLM-MLA FP8 path](../sources/prs/sglang/PR-8638.md), [feat: add trtllm-gen mha from direct call](../sources/prs/sglang/PR-8782.md), [Support DP attention with GPT-OSS](../sources/prs/sglang/PR-9359.md), [move is_sm90_supported/is_sm100_supported to python/sglang/srt/utils.py](../sources/prs/sglang/PR-9679.md), [[CPU] Add FP8 Bmm support](../sources/prs/sglang/PR-9744.md), [support using fa4 on deepseek on blackwell](../sources/prs/sglang/PR-9928.md), [[Feature] Support `cp.reduce.async.bulk.tensor`](../sources/prs/tilelang/PR-1667.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [[Kernel] Flash Attention 3 Support](../sources/prs/vllm/PR-12093.md), [[Core] Optimizing cross-attention `QKVParallelLinear` computation](../sources/prs/vllm/PR-12325.md), [[ROCm] Faster Custom Paged Attention kernels](../sources/prs/vllm/PR-12348.md), [[Attention] MLA decode optimizations](../sources/prs/vllm/PR-12528.md), [[Attention] Deepseek v3 MLA support with FP8 compute](../sources/prs/vllm/PR-12601.md), [[Attention] MLA with chunked prefill](../sources/prs/vllm/PR-12639.md), [[AMD][ROCm] Enable DeepSeek model on ROCm](../sources/prs/vllm/PR-12662.md), [[Perf] Mem align KV caches for CUDA devices (MLA perf improvement)](../sources/prs/vllm/PR-12676.md), [[VLM] Add MLA with pure RoPE support for deepseek-vl2 models](../sources/prs/vllm/PR-12729.md), [[Kernel] Make rotary_embedding ops more flexible with input shape](../sources/prs/vllm/PR-12777.md), [[Attention] Use FA3 for MLA on Hopper](../sources/prs/vllm/PR-12807.md), [Expand MLA to support most types of quantization](../sources/prs/vllm/PR-13181.md), [[Bugfix] Massage MLA's usage of flash attn for RoCM](../sources/prs/vllm/PR-13310.md), [[ROCM] fix native attention function call](../sources/prs/vllm/PR-13650.md), [[core] Perf improvement for DSv3 on AMD GPUs](../sources/prs/vllm/PR-13718.md), [[Bugfix] Support MLA for CompressedTensorsWNA16](../sources/prs/vllm/PR-13725.md), [[V1] V1 Enablement Oracle ](../sources/prs/vllm/PR-13726.md), [[Kernel] FlashMLA integration](../sources/prs/vllm/PR-13747.md), [[Attention] MLA support for V1](../sources/prs/vllm/PR-13789.md), [[ROCm] Disable chunked prefill/prefix caching when running MLA on non-cuda platforms](../sources/prs/vllm/PR-13844.md), [[Attention] Flash MLA for V1](../sources/prs/vllm/PR-13867.md), [Fix mla prefill context performance](../sources/prs/vllm/PR-13897.md), [[V1] EP/TP MoE + DP Attention](../sources/prs/vllm/PR-13931.md), [[V1] Implement sliding window attention in kv_cache_manager](../sources/prs/vllm/PR-14097.md), [[v1] Add comments to the new ragged paged attention Pallas kernel](../sources/prs/vllm/PR-14155.md), [[V1][TPU] TPU multimodal model support for ragged attention](../sources/prs/vllm/PR-14158.md), [[V1][TPU] Support V1 Sampler for ragged attention](../sources/prs/vllm/PR-14227.md), [[Hardware] Update the flash attn tag to support Blackwell](../sources/prs/vllm/PR-14244.md), [dynamic distpatch of fp8 kernels](../sources/prs/vllm/PR-14245.md), [[BugFix] MLA + V1, illegal memory access and accuracy issues](../sources/prs/vllm/PR-14253.md), [[BugFix] Fix prefix caching V0 MLA](../sources/prs/vllm/PR-14255.md), [[Attention] FlashAttn MLA](../sources/prs/vllm/PR-14258.md), [[Hardware][TPU]Enable ragged paged attention kernel and resolve recompilation issue](../sources/prs/vllm/PR-14310.md), [[Bug] Fix Attention when ignored in by quant_method](../sources/prs/vllm/PR-14313.md), [[ROCm] Enable chunked prefill/paged attention in MLA on ROCm](../sources/prs/vllm/PR-14316.md), [[Perf] Reduce MLA CPU overheads in V1](../sources/prs/vllm/PR-14384.md), [[Bugfix] DeepSeek Accuracy](../sources/prs/vllm/PR-14476.md), [[Perf] Improve MLA on V1](../sources/prs/vllm/PR-14540.md), [[Attention] Flash Attention 3 - fp8](../sources/prs/vllm/PR-14570.md), [[Kernel][CPU] CPU MLA](../sources/prs/vllm/PR-14744.md), [[Attention] MLA get rid of materialization](../sources/prs/vllm/PR-14770.md), [[Attention] Get rid of mla cache alignment](../sources/prs/vllm/PR-14842.md), [[V1][BugFix] Detect interleaved sliding window attention](../sources/prs/vllm/PR-14896.md), [[FEAT][ROCm] Integrate Paged Attention Kernel from AITER](../sources/prs/vllm/PR-15001.md), [[Bugfix] Fix incorrect qwen2.5-vl attention mask pre-computation](../sources/prs/vllm/PR-15200.md), [[Bugfix] Fix use_cascade_attention handling for Alibi-based models on vllm/v1](../sources/prs/vllm/PR-15211.md), [[Misc] Add attention mask pre-computation optimization back to Qwen2.5-VL](../sources/prs/vllm/PR-15273.md), [[BugFix] Fix nightly MLA failure (FA2 + MLA chunked prefill, i.e. V1, producing bad results)](../sources/prs/vllm/PR-15492.md), [[ROCM][KERNEL] Paged attention for V1](../sources/prs/vllm/PR-15720.md), [[TPU] Support sliding window and logit soft capping in the paged attention kernel for TPU.](../sources/prs/vllm/PR-15732.md), [[FEAT][ROCm]: Support AITER MLA](../sources/prs/vllm/PR-15893.md), [Modularize fused experts and integrate PPLX kernels](../sources/prs/vllm/PR-15956.md), [[NVIDIA] Support Cutlass MLA for Blackwell GPUs](../sources/prs/vllm/PR-16032.md), [[ROCM] Add gfx950 to the custom attention archs](../sources/prs/vllm/PR-16034.md), [Add FlexAttention to V1](../sources/prs/vllm/PR-16078.md), [Upstream Llama4 Support to Main](../sources/prs/vllm/PR-16113.md), [[Kernel] support merge_attn_states CUDA kernel, 3x speedup](../sources/prs/vllm/PR-16173.md), [[Bug] [ROCm] Fix Llama 4 Enablement Bug on ROCm: V0 ROCmFlashAttentionImpl and Triton Fused MoE bugs](../sources/prs/vllm/PR-16198.md), [[Llama4] Enable attention temperature tuning by default for long context (>32k)](../sources/prs/vllm/PR-16439.md), [Allocate kv_cache with stride order](../sources/prs/vllm/PR-16605.md), [[MLA] Simplification to batch P/D reordering](../sources/prs/vllm/PR-16673.md), [[V1] V1 FlashInfer Attention](../sources/prs/vllm/PR-16684.md), [[torch.compile][ROCm] Fuse quantization onto attention using a torch.compile pass](../sources/prs/vllm/PR-16756.md), [[Kernel] Unified Triton kernel that doesn't distinguish between prefill + decode](../sources/prs/vllm/PR-16828.md), [Update PyTorch to 2.7.0](../sources/prs/vllm/PR-16859.md), [[Attention] FA3 decode perf improvement - single mma warp group support for head dim 128](../sources/prs/vllm/PR-16864.md), [[Bugfix] Triton FA function takes no keyword arguments](../sources/prs/vllm/PR-16902.md), [[ROCm][Kernel][V1] Enable AMD Radeon GPU Custom Paged Attention on v1](../sources/prs/vllm/PR-17004.md), [[Bugfix] Add contiguous call inside rope kernel wrapper](../sources/prs/vllm/PR-17091.md), [[ROCm][FP8][Kernel] FP8 quantization fused into Custom Paged Attention](../sources/prs/vllm/PR-17139.md), [[Bugfix] gemma[2,3] interleaved attention when sliding window is disabled](../sources/prs/vllm/PR-17180.md), [[Bugfix] Get a specific type of layer from forward context](../sources/prs/vllm/PR-17222.md), [[BugFix] Fix vllm_flash_attn install issues](../sources/prs/vllm/PR-17267.md), [[BugFix] Fix cascade attention - RuntimeError: scheduler_metadata must have shape (metadata_size)](../sources/prs/vllm/PR-17283.md), [[v1] AttentionMetadata for each layer](../sources/prs/vllm/PR-17394.md), [[v1] Pass BlockTable and KVCacheSpec to AttentionMetadataBuilders](../sources/prs/vllm/PR-17483.md), [[Attention] MLA move o_proj q_proj into cuda-graph region](../sources/prs/vllm/PR-17484.md), [[BugFix] Fix mla cpu - missing 3 required positional arguments](../sources/prs/vllm/PR-17494.md), [[FEAT][ROCm]: Support AITER MLA on V1 Engine](../sources/prs/vllm/PR-17523.md), [[Attention] MLA move rotary embedding to cuda-graph region](../sources/prs/vllm/PR-17668.md), [fix amd triton mla path](../sources/prs/vllm/PR-17871.md), [[Bugfix][ROCm] Fix AITER MLA V1](../sources/prs/vllm/PR-17880.md), [[BugFix][AMD] Compatible patch for AITER lib after 04/20](../sources/prs/vllm/PR-17912.md), [[v1] Support multiple KV cache groups in GPU model runner](../sources/prs/vllm/PR-17945.md), [[BUG] [ROCm] [MLA] Fix variable name bug due to change in variable name in PR #17483](../sources/prs/vllm/PR-17961.md), [[Bugfix][ROCm] Use `chunked_prefill_paged_decode` as fallback for V1 attention on ROCm](../sources/prs/vllm/PR-18093.md), [[Hardware][AMD] integrate aiter chunked prefill into vllm](../sources/prs/vllm/PR-18596.md), [[BugFix] FA2 MLA Accuracy Issue](../sources/prs/vllm/PR-18807.md), [[P/D] Heterogeneous TP](../sources/prs/vllm/PR-18833.md), [[Kernel] Apply torch.Tag.needs_fixed_stride_order only for torch==2.6.0](../sources/prs/vllm/PR-19346.md), [[Core] Support Local Chunked Attention for Hybrid KV Cache](../sources/prs/vllm/PR-19351.md), [[Bugfix][V1] Allow manual FlashAttention for Blackwell](../sources/prs/vllm/PR-19492.md), [[Bugfix] Enable PP with AITER+V1](../sources/prs/vllm/PR-19822.md), [Enable V1 for Hybrid SSM/Attention Models](../sources/prs/vllm/PR-20016.md), [[Attention] MLA - Flashinfer Ragged Prefill](../sources/prs/vllm/PR-20034.md), [[Bugfix] Fix some narrowing conversion warnings](../sources/prs/vllm/PR-20141.md), [[Kernel] Optimize Prefill Attention in Unified Triton Attention Kernel](../sources/prs/vllm/PR-20308.md), [Update PyTorch to 2.8.0](../sources/prs/vllm/PR-20358.md), [SM100 Cutlass MLA decode with unrestricted num_heads (< 128) for DeepSeek TP](../sources/prs/vllm/PR-20769.md), [[v1][core] Support for attention free models](../sources/prs/vllm/PR-20811.md), [[Bugfix] Voxtral on Blackwell GPUs (RTX 50 series)](../sources/prs/vllm/PR-21077.md), [[Kernel] Flashinfer MLA (trtllm-gen) decode kernel integration](../sources/prs/vllm/PR-21078.md), [[v1] Add Whisper model support (encoder-decoder)](../sources/prs/vllm/PR-21088.md), [[Attention] Optimize FlashInfer MetadataBuilder Build call](../sources/prs/vllm/PR-21137.md), [[Attention][DBO] Add support for "splitting" the CommonAttentionMetadata](../sources/prs/vllm/PR-21153.md), [[Attention] Clean up iRoPE in V1](../sources/prs/vllm/PR-21188.md), [[Kernel] Enable Hybrid Model Support in Triton Unified Attention Kernel](../sources/prs/vllm/PR-21197.md), [[v1] - Mamba1 Attention Metadata](../sources/prs/vllm/PR-21249.md), [Support encoder-only models without KV-Cache](../sources/prs/vllm/PR-21270.md), [[v1][attention] Support Hybrid Allocator + FlashInfer](../sources/prs/vllm/PR-21412.md), [Updates to Flex + VLLm integration](../sources/prs/vllm/PR-21416.md), [[V1] Fix local chunked attention always disabled](../sources/prs/vllm/PR-21419.md), [[BugFix] Fix shared storage connector load kv only load attention layer](../sources/prs/vllm/PR-21428.md), [update flashinfer to v0.2.9rc1](../sources/prs/vllm/PR-21485.md), [[Attention] Support multiple attention metadata builders per kv_cache_spec + proper local attention no hybrid kv cache fix](../sources/prs/vllm/PR-21588.md), [Override attention metadata for fast prefill in some KV sharing setups](../sources/prs/vllm/PR-21590.md), [[BugFix] Fix IMA FlashMLA full cuda-graph and DP + Update FlashMLA](../sources/prs/vllm/PR-21691.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv/out Attention Kernel](../sources/prs/vllm/PR-21716.md), [[Perf] Disable chunked local attention by default with llama4](../sources/prs/vllm/PR-21761.md), [[Bugfix] Check NVIDIA artifactory is accessible before using flashinfer cubin kernels](../sources/prs/vllm/PR-21893.md), [[Qwen3] Enable dual-chunk-attention support for Qwen3 models.](../sources/prs/vllm/PR-21924.md), [[NVIDIA] Support Flashinfer TRT-LLM Prefill Attention Kernel](../sources/prs/vllm/PR-22095.md), [[fix] fix correct assertion syntax error in attention utils.](../sources/prs/vllm/PR-22154.md), [Fp8 paged attention update](../sources/prs/vllm/PR-22222.md), [Support encoder_only attention for FlexAttention](../sources/prs/vllm/PR-22273.md), [Upgrade FA3 for attention sink](../sources/prs/vllm/PR-22313.md), [[Bugfix] Add proper comparison for package versions](../sources/prs/vllm/PR-22314.md), [[ROCm] Add attention sink to use_rocm_custom_paged_attention](../sources/prs/vllm/PR-22329.md), [[BugFix] Fix triton compile error in `kernel_unified_attention_2/3d` caused by attention sinks](../sources/prs/vllm/PR-22368.md), [[bugfix] Fix Llama3/4 issues caused by FlashInfer 0.2.10](../sources/prs/vllm/PR-22426.md), [[Attention] FA3 Attention Sinks Perf Boost](../sources/prs/vllm/PR-22478.md), [[Bugfix] Fix ModernBert load & Enable sliding window attention for bidirectional attention.](../sources/prs/vllm/PR-22637.md), [Support multiple attention groups for KV sharing](../sources/prs/vllm/PR-22672.md), [Force TRTLLM attention for gpt-oss on SM100](../sources/prs/vllm/PR-22678.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv NVFP4-out Attention Kernel](../sources/prs/vllm/PR-22703.md), [[Bugfix] Fix default enable for CUTLASS MLA on SM100](../sources/prs/vllm/PR-22738.md), [fp8 kv cache support fix for torch.compile](../sources/prs/vllm/PR-22758.md), [[Bugfix gpt-oss] Fix float32 convert for flashinfer sink support](../sources/prs/vllm/PR-23016.md), [[V1] address post issues related to #20059 (part 1); cascade attention reenable by default](../sources/prs/vllm/PR-23046.md), [[Misc] Add @tdoublep as a maintainer of hybrid model and Triton-attention related code](../sources/prs/vllm/PR-23122.md), [[Log] Warning Once for Cutlass MLA ](../sources/prs/vllm/PR-23137.md), [Optimize input preparation for FlashInfer [2/N]](../sources/prs/vllm/PR-23174.md), [[Attention] Optimize make_local_attention_virtual_batches for Flash Attention](../sources/prs/vllm/PR-23185.md), [[Misc][qwen2_5_vl][torch.compile] Enable `supports_torch_compile` on generic nn.Module and demonstrate speedup on Qwen Vision model](../sources/prs/vllm/PR-23207.md), [[Core] Always use tensor cores for Flashinfer Decode Wrapper](../sources/prs/vllm/PR-23214.md), [[ROCm][Aiter] Add triton fp8 bmm kernel for mla](../sources/prs/vllm/PR-23264.md), [[Compile] Fix Compile Warning SM100 Cutlass MLA](../sources/prs/vllm/PR-23287.md), [[Attention] Allow V1 flash_attn to support cross-attention](../sources/prs/vllm/PR-23297.md), [[Bugfix] Fixing division by zero in triton_attn if query_heads/kv_heads > 16 ](../sources/prs/vllm/PR-23424.md), [[Perf] Warmup FlashInfer attention during startup](../sources/prs/vllm/PR-23439.md), [[Attention][FA3] Update FA3 to include new swizzle optimization](../sources/prs/vllm/PR-23465.md), [fix(v1/kv_cache): resolve async KV transfer bug in cascade attention](../sources/prs/vllm/PR-23485.md), [[Misc] Simplify FlashInfer attention metadata](../sources/prs/vllm/PR-23585.md), [[Flashinfer] Support Flashinfer TRTLLM FP8-qkv BF16/FP16-out Attention Kernel](../sources/prs/vllm/PR-23647.md), [[v1] Add cross-attention KV cache support for encoder-decoder models](../sources/prs/vllm/PR-23664.md), [[Core/DBO][1/N] Add Dual-Batch Overlap mechanism to VLLM](../sources/prs/vllm/PR-23693.md), [[FlashInfer] Cache hyper params in metadata builder](../sources/prs/vllm/PR-23732.md), [[Feature] Support Decode Context Parallel (DCP) for MLA](../sources/prs/vllm/PR-23734.md), [[BugFix][FlashInfer] Fix potential race condition for paged_kv_indptr_cpu](../sources/prs/vllm/PR-23737.md), [[Kernel] cuda kernels for upcoming decode context parallel feature](../sources/prs/vllm/PR-23791.md), [[Misc] add reorder_batch AttentionMetadataBuilder](../sources/prs/vllm/PR-23798.md), [Feature/vit attention unification# 23880](../sources/prs/vllm/PR-23978.md), [[Kernel] Support decode context parallelism on Blackwell with CUTLASS MLA](../sources/prs/vllm/PR-24385.md), [[Feature] Disallow FlashMLA on Blackwell](../sources/prs/vllm/PR-24521.md), [[Bug] Fix `is_flashmla_supported` Check Error](../sources/prs/vllm/PR-24774.md), [[DCP] Support Decode Context Parallel (DCP) for GQA with FlashAttention](../sources/prs/vllm/PR-24864.md), [[Bugfix][B200] Fix `cutlass_mla` hang](../sources/prs/vllm/PR-24966.md), [[Attention][DCP] Support DCP with query length > 1 (MTP) with FA3](../sources/prs/vllm/PR-25049.md), [[Bug] Fix `returned_lse` not Defined issue](../sources/prs/vllm/PR-25106.md), [[BugFix] Fix MLA assert with CUTLASS MLA](../sources/prs/vllm/PR-25478.md), [[Bugfix] [B200] cutlass_mla - ensure kv_split == 1 for batch size > 1](../sources/prs/vllm/PR-25509.md), [[Flashinfer][gpt-oss] Support FP8-qkv Flashinfer TRTLLM Sinks Attention](../sources/prs/vllm/PR-25674.md), [[Performance] Split FlashAttn attention and cache update](../sources/prs/vllm/PR-25954.md), [[Spec Decode] Enable efficient speculative decoding with FlashInfer-MLA](../sources/prs/vllm/PR-25984.md), [Move query quantization to attention layer for Flashinfer & Triton.](../sources/prs/vllm/PR-26534.md), [[Attention] Tune CUTLASS MLA num_splits](../sources/prs/vllm/PR-26846.md), [[BUGFIX][ROCM] ViT FlashAttention on ROCm (no GFX9) and contiguous on qwen3vl ROCm TORCH_SDPA](../sources/prs/vllm/PR-27190.md), [[Bugfix] Ensure calculated KV scales are applied in attention.](../sources/prs/vllm/PR-27232.md), [Prefer FlashAttention MLA as default over FlashMLA](../sources/prs/vllm/PR-27363.md), [[Misc] Make reorder batch also separate extends](../sources/prs/vllm/PR-27367.md), [[Attention] Use sparse prefill kernel for fp8 kv-cache in DeepSeek-v3.2](../sources/prs/vllm/PR-27532.md), [[AMD] Use Decoupled Kernel Block Size to Support AITER MLA block_size=1](../sources/prs/vllm/PR-27715.md), [[Bug] Batch invariant: Fix flash attn MLA `RuntimeError: scheduler_metadata must have shape (metadata_size)`](../sources/prs/vllm/PR-27884.md), [Update Flashinfer from `v0.4.1` to `v0.5.2`](../sources/prs/vllm/PR-27952.md), [[FlashInfer] Avoid FlashInfer block_size 16 + head_size 256 on blackwell](../sources/prs/vllm/PR-27994.md), [[ROCm][MLA] enable fp8 MLA decode on ROCm](../sources/prs/vllm/PR-28032.md), [[Mamba] - Consolidate Mambas Attention Logic](../sources/prs/vllm/PR-28133.md), [fix cross attention](../sources/prs/vllm/PR-28346.md), [[ROCm] Support for Whisper v1 with Aiter Unified Attention and Aiter Flash Attention](../sources/prs/vllm/PR-28376.md), [[Bugfix] Fix SM100 gpt-oss regression due to faulty attn sink support](../sources/prs/vllm/PR-28561.md), [[Attention][Bugfix] Fix FA sink support](../sources/prs/vllm/PR-28660.md), [[Feature] Prefill Context Parallel (PCP) basic support](../sources/prs/vllm/PR-28718.md), [[Model] Add support for openPangu moe model](../sources/prs/vllm/PR-28775.md), [bugfix: correct attn output with base 2 or e](../sources/prs/vllm/PR-28840.md), [[BugFix] Fix async-scheduling + FlashAttn MLA](../sources/prs/vllm/PR-28990.md), [[Feature] Batch invariant: Enable `TRITON_MLA` without prefix-caching](../sources/prs/vllm/PR-29125.md), [[Attention] Cache attention metadata builds across hybrid KV-cache groups](../sources/prs/vllm/PR-29627.md), [[Attention] Make `split_decodes_and_prefills(..., require_uniform=True)` support padding](../sources/prs/vllm/PR-29644.md), [[perf] Use direct copy (broadcast) instead of cat for k_nope/k_pe in MLA prefill](../sources/prs/vllm/PR-29710.md), [[Perf] Improve fp8 quant in mla; replace ReduceSum with ReduceScatterSum](../sources/prs/vllm/PR-29795.md), [[SpecDecode] Simplified alternative padded-speculation acceptance rate fix](../sources/prs/vllm/PR-29845.md), [[Quantization] fix: overflow with static per-tensor scaling](../sources/prs/vllm/PR-29867.md), [[BugFix] Fix DBO assert `assert B_block_table == B_q`](../sources/prs/vllm/PR-29933.md), [Add llmcompressor fp8 kv-cache quant (per-tensor and per-attn_head)](../sources/prs/vllm/PR-30141.md), [[ROCm][Bugfix] Add MLACommonMetadata to allowed attention types for speculative decoding](../sources/prs/vllm/PR-30430.md), [[Feature] Add SM103 (Blackwell Ultra) Support to vLLM](../sources/prs/vllm/PR-30484.md), [[Bugfix] Pass FA version in `MultiHeadAttention`](../sources/prs/vllm/PR-30575.md), [Triton Attention: Support cross-layers blocks](../sources/prs/vllm/PR-30687.md), [OffloadingConnector: Support kernel_block_size != block_size](../sources/prs/vllm/PR-30692.md), [[Misc][LLaMa4] Compile LLaMa Vision Encoder](../sources/prs/vllm/PR-30709.md), [Update note comment for flashinfer attention warmup](../sources/prs/vllm/PR-30711.md), [[Bugfix] Fix broken ViT attention selection for Blackwell device](../sources/prs/vllm/PR-30731.md), [[SM100] Enable fp8 compute for prefill MLA](../sources/prs/vllm/PR-30746.md), [[Kernels][FI] Skip trtllm attention when num_kv_heads=1](../sources/prs/vllm/PR-30842.md), [[Bugfix] [Kernel] Triton attention kernels: mask out V blocks that fall outside sliding window](../sources/prs/vllm/PR-30887.md), [[Bugfix] Fix incorrect tiles creation for mm prefix triton attention](../sources/prs/vllm/PR-30974.md), [[Misc] Fix grammar errors in comments and messages](../sources/prs/vllm/PR-31115.md), [[perf] Integrate flashinfer concat_mla_k](../sources/prs/vllm/PR-31171.md), [[SM100] Resubmit FMHA FP8 prefill for MLA](../sources/prs/vllm/PR-31195.md), [[Bugfix][Hardware][AMD] Fix last_page_len calculation in AITER MLA decode](../sources/prs/vllm/PR-31282.md), [fix(rocm): add early return in get_flash_attn_version for ROCm](../sources/prs/vllm/PR-31286.md), [[Bugfix][ROCm]Fix Qwen3-Next-80B-A3B-Thinking inference and optimize non-standard block size (544) support under rocm_atten](../sources/prs/vllm/PR-31380.md), [fixed mypy warnings for files vllm/v1/attention with TEMPORARY workaround](../sources/prs/vllm/PR-31465.md), [[1/N][Attention] Restructure attention: move files](../sources/prs/vllm/PR-31916.md), [[MISC] Add strict contiguity check for FlashInfer attention tensors](../sources/prs/vllm/PR-32008.md), [[4/N][Attention] Move MLA common to model_executor](../sources/prs/vllm/PR-32060.md), [[5/N][Attention] Finish eliminating `vllm/attention` folder](../sources/prs/vllm/PR-32064.md), [fix: Add glm4_moe_lite to MLA detection](../sources/prs/vllm/PR-32614.md), [[Bugfix][Attention] Explicitly report support for kv_cache_dtype bfloat16](../sources/prs/vllm/PR-32795.md), [[Spec Decode] Unified Parallel Drafting](../sources/prs/vllm/PR-32887.md), [[ROCm][perf] Shuffle KV cache to use paged_attention_common](../sources/prs/vllm/PR-32914.md), [[Attention] FA4 integration](../sources/prs/vllm/PR-32974.md), [[Attention] Use `has_flashinfer` helper](../sources/prs/vllm/PR-33177.md), [[Bugfix] Disable TRTLLM attention when KV transfer is enabled](../sources/prs/vllm/PR-33192.md), [[PERF] Change GDN Attention State Layout from [N, HV, K, V] to [N, HV, V, K]](../sources/prs/vllm/PR-33291.md), [Triton MLA perf fixes](../sources/prs/vllm/PR-33529.md), [[Perf] Disable clean_logits in deepgemm fp8_mqa_logits kernel](../sources/prs/vllm/PR-33568.md), [[Bugfix] Fix sparse MLA metadata building](../sources/prs/vllm/PR-33579.md), [[Bugfix] fix DeepSeek R1 with CUTLASS MLA Broken on B200](../sources/prs/vllm/PR-33637.md), [enable skipping of SW attention layers when using FP8 KV cache](../sources/prs/vllm/PR-33695.md), [[Bugfix] Fix DSV3.2 NVFP4](../sources/prs/vllm/PR-33932.md), [Reapply [Attention][FA3] Update FA3 to include new swizzle optimization](../sources/prs/vllm/PR-34043.md), [[Bugfix] Relax TRTLLM KV cache contiguity assertion for cross-layer layout](../sources/prs/vllm/PR-34158.md), [[Bugfix] Fix DP Attention Padding in Dummy Run](../sources/prs/vllm/PR-34187.md), [[CPU][Perf] Accelerate Attention head for s390x using vector intrinsics](../sources/prs/vllm/PR-34434.md), [[BugFix] Add support for MTP num_speculative_tokens > 1 with sparse MLA](../sources/prs/vllm/PR-34552.md), [[Kernel] Add FP8 KV cache support to Triton MLA decode attention](../sources/prs/vllm/PR-34597.md), [[Update] Use FlashInfer fast_decode_plan directly instead of replication](../sources/prs/vllm/PR-34687.md), [[Bugfix] Fix MLA attention crash with AWQ/GPTQ quantized models](../sources/prs/vllm/PR-34695.md), [[Attention] Use FA4 for MLA prefill](../sources/prs/vllm/PR-34732.md), [[Bugfix] Fix GDN attention crash with mixed decode/spec-decode batches](../sources/prs/vllm/PR-34871.md), [[Model Bash][DSR1] Add selective dynamic shape marking for CustomOp](../sources/prs/vllm/PR-34900.md), [[Attention][Perf][Kernel] Replace torch.cat with vectorized CUDA kernel MLA query concat - DeepSeek-V3.2](../sources/prs/vllm/PR-34917.md), [[Model Runner V2] Support attention group](../sources/prs/vllm/PR-35036.md), [[Bug][DSV3.2] Always prepare metadata for DeepGEMM Sparse Attention](../sources/prs/vllm/PR-35075.md), [[Linear Attention] fix bug for linear attention + prefix caching + reset_prefix_cache](../sources/prs/vllm/PR-35157.md), [[BUGFIX][Mamba][Qwen3.5] Zero freed SSM cache blocks on GPU](../sources/prs/vllm/PR-35219.md), [[Feat] Add CUDA torch fallbacks for fp8_mqa_logits/fp8_paged_mqa_logits_torch function](../sources/prs/vllm/PR-35271.md), [[Attention][Perf] Optimize cp_gather_and_upconvert_fp8_kv_cache - DeepSeek-v3.2](../sources/prs/vllm/PR-35290.md), [[Performance] Extract KV cache update op from flashinfer forward](../sources/prs/vllm/PR-35422.md), [Fix routed experts capture for hybrid models (Mamba + Attention)](../sources/prs/vllm/PR-35744.md), [[ROCm] Support MLA with nhead<16 and FP8 KV cache for TP=8 (Kimi K2.5/Linear)](../sources/prs/vllm/PR-35850.md), [[Perf] Support FP8 KV cache for Flashinfer MLA Sparse](../sources/prs/vllm/PR-35891.md), [[BugFix] Fallback from FA4->FA2 for Batch Invariance](../sources/prs/vllm/PR-36059.md), [Add 320 dimension size support to MLA](../sources/prs/vllm/PR-36161.md), [[Bugfix][MLA] Add logits size budget to sparse indexer prefill chunking](../sources/prs/vllm/PR-36178.md), [[mla] Support fused FP8/NVFP4 output quantization in MLA attention (#35792)](../sources/prs/vllm/PR-36205.md), [mla: don't update kv cache on dummy forwards](../sources/prs/vllm/PR-36282.md), [Disable cascade attention by default](../sources/prs/vllm/PR-36318.md), [feat(attention): extract KV-cache update from FlashAttentionDiffKV ba…](../sources/prs/vllm/PR-36466.md), [[Kernel] Fuse FP8 output quantization into merge_attn_states](../sources/prs/vllm/PR-36518.md), [[Bugfix][Sparse MLA] report indexer CG support properly](../sources/prs/vllm/PR-36519.md), [[ROCm] Utilize persistent MLA kernel from AITER](../sources/prs/vllm/PR-36574.md), [[Misc][Attention] Clean up unused method in `CPU_ATTN`](../sources/prs/vllm/PR-36673.md), [fix(kv-cache): increase hybrid attention grouping threshold from 1.25 to 1.5](../sources/prs/vllm/PR-36684.md), [[ROCm] Attention selector reordering](../sources/prs/vllm/PR-36702.md), [[DSV3.2][MTP] Optimize Indexer MTP handling](../sources/prs/vllm/PR-36723.md), [[ROCm] Fix KV copy methods and auto-select attention backend for ROCm](../sources/prs/vllm/PR-36845.md), [[ROCm] Validate block_size for explicitly selected attention backends](../sources/prs/vllm/PR-36846.md), [[Feat][Spec Decode] DFlash](../sources/prs/vllm/PR-36847.md), [[Feat][Bugfix] Enable additional dimension for Flashinfer MLA and fix routing dtype](../sources/prs/vllm/PR-36931.md), [[MTP][Sparse MLA] Take advantage of native MTP support in indexer when possible](../sources/prs/vllm/PR-36982.md), [[Bugfix] Fix KV scales inconsistency in fp8 MLA & FlashInfer kv_cache_dtype "auto" leading to gibberish](../sources/prs/vllm/PR-37054.md), [[Bugfix] Disable cross-layer KV cache for MLA attention backends](../sources/prs/vllm/PR-37090.md), [[Benchmark] Improvements to attention benchmark script](../sources/prs/vllm/PR-37115.md), [[XPU] support MLA model on Intel GPU](../sources/prs/vllm/PR-37143.md), [[Misc] Add `float16` to `CacheDType`](../sources/prs/vllm/PR-37199.md), [[ROCM][Bugfix] Use correct stride in cp_mha_gather_cache_kernel for hybrid model (#37228)](../sources/prs/vllm/PR-37228.md), [[Perf] Set Flashinfer sparse MLA as default backend for FP8 kv cache](../sources/prs/vllm/PR-37252.md), [[Attention] Support distinguishing between short extends and decodes](../sources/prs/vllm/PR-37303.md), [Add nvfp4 support to reshape_and_cache_flash](../sources/prs/vllm/PR-37332.md), [[Model Runner V2] fix draft attention metadata generation](../sources/prs/vllm/PR-37364.md), [[torch.compile] Refactor Attention Quant Fusion Pass and Remove Boilerplate](../sources/prs/vllm/PR-37373.md), [[Perf][Kernel] Persistent TopK scheduler: unified CUDAGraph-safe kernel with dynamic per-row dispatch - DeepSeek-V3.2 DSA decode](../sources/prs/vllm/PR-37421.md), [[BugFix] Allow qk_nope_head_dim=192 in FlashInfer MLA backend checks](../sources/prs/vllm/PR-37475.md), [[Performance] Remove unnecessary zero-fill of MLA decode output tensor in Aiter backend](../sources/prs/vllm/PR-37539.md), [[Bugfix][ROCm] Fix lru_cache on paged_mqa_logits_module](../sources/prs/vllm/PR-37547.md), [[Bugfix] Disable --calculate-kv-scales for hybrid GDN/Mamba+Attention…](../sources/prs/vllm/PR-37565.md), [[ROCm][Bugfix] fix cache block size mismatch for aiter unified attention](../sources/prs/vllm/PR-37606.md), [[NIXL][BUG] Fix Triton heterogeneous TP](../sources/prs/vllm/PR-37940.md), [[Perf] FP8 FlashInfer Attn for ViT](../sources/prs/vllm/PR-38065.md), [[Model Runner V2] Rebuild attention metadata before eagle decode full…](../sources/prs/vllm/PR-38311.md), [[GDN] Eliminate GPU->CPU sync in prepare_chunk_indices during prefill](../sources/prs/vllm/PR-38361.md), [[Attention Backend] TurboQuant: 2-bit KV cache compression with 4x capacity](../sources/prs/vllm/PR-38479.md), [[Bugfix][MLA] Change default SM100 MLA prefill backend back to TRT-LLM](../sources/prs/vllm/PR-38562.md), [[ROCm] Fix aiter persistent mode mla with q/o nhead<16 for kimi-k2.5 tp8](../sources/prs/vllm/PR-38615.md), [Fix MLA runs when use_inductor_graph_partition=True](../sources/prs/vllm/PR-38631.md), [[FA4] Update flash-attention to latest upstream FA4](../sources/prs/vllm/PR-38690.md), [[Bugfix] Restrict TRTLLM attention to SM100, fixing GB300 (SM103) hang](../sources/prs/vllm/PR-38730.md), [[Bugfix] Fix test mocks after SM100 restriction in #38730](../sources/prs/vllm/PR-38791.md), [[Perf] Reduce H2D pageable memory copies](../sources/prs/vllm/PR-38794.md), [[FlashAttention] Symlink FA4 instead of copying when using `VLLM_FLASH_ATTN_SRC_DIR`](../sources/prs/vllm/PR-38814.md), [[Attention][MLA] Re-enable FA4 as default MLA prefill backend](../sources/prs/vllm/PR-38819.md), [[Attention] relax the head dim 512 and paged kv for sm90+FA4](../sources/prs/vllm/PR-38835.md), [[Refactor] Improve indexer decode path metadata preparation](../sources/prs/vllm/PR-38865.md), [[Bugfix] Fix broken explicit unquantized kv cache dtype support](../sources/prs/vllm/PR-38922.md), [[Quantization] - Layerwise reloading of Attention/KV quantized models](../sources/prs/vllm/PR-38995.md), [[Bugfix] Fix FlashInfer crash with kv_cache_dtype_skip_layers](../sources/prs/vllm/PR-39002.md), [[XPU] Quick fix for TritonMLA to remove cuda hardcode](../sources/prs/vllm/PR-39088.md), [[ROCm] Align AiterFlashAttentionImpl attn_type check with backend](../sources/prs/vllm/PR-39119.md), [[Bug] Fix rocm sparse attn indexer issue](../sources/prs/vllm/PR-39225.md), [[Model Runner V2] Fix flex attention kv blocks calculation issue](../sources/prs/vllm/PR-39353.md), [[MLA] Optimize mla indexer prepare uniform decode for MTP > 1](../sources/prs/vllm/PR-39458.md), [[Bugfix] Fix tensor shape mismatch in sparse attention with speculative decoding](../sources/prs/vllm/PR-39542.md), [[XPU] properly handle q_descale on XPU as quant query input not supported](../sources/prs/vllm/PR-39676.md), [[Bugfix] Fix mismatch between global and local attention heads in tensor-parallel mode for param2moe model](../sources/prs/vllm/PR-39707.md), [add warning when FP8 KV cache misses prefill query quantization](../sources/prs/vllm/PR-39752.md), [[Bugfix] add support for 'num_attention_groups' in ModelArchConfigConvertorBase for Step3p5](../sources/prs/vllm/PR-39796.md), [[Attention] use diff kv backend for mimo v2 flash](../sources/prs/vllm/PR-40045.md), [Add nvfp4 kv cache support](../sources/prs/vllm/PR-40177.md), [[Attention][TurboQuant] Share dequant buffers, eliminate float16_copy](../sources/prs/vllm/PR-40941.md), [[Bugfix][Hybrid][NemotronH] Fix mamba_cache_mode=all + speculative decoding crash](../sources/prs/vllm/PR-41233.md), [[DSv4] Improved fused Indexer Q quant kernel](../sources/prs/vllm/PR-41428.md), [fix: remove unused norm for dpskv4](../sources/prs/vllm/PR-41710.md), [[MLA Attention Backend] Add TOKENSPEED_MLA backend for DSR1/Kimi K25 prefill + decode on Blackwell](../sources/prs/vllm/PR-41778.md), [[feat] Add FP8 per-tensor Q scale support to Triton attention backend](../sources/prs/vllm/PR-42080.md), [[Bugfix] Fix TRTLLM ragged MLA prefill workspace warmup](../sources/prs/vllm/PR-42112.md), [[DSv4] Improved dequant gather K cache kernel](../sources/prs/vllm/PR-42236.md), [[Bugfix] mamba: run single-token extends as decodes](../sources/prs/vllm/PR-42430.md), [[6/n] Migrate activation kernels, gptq, gguf, non cutlass w8a8 to libtorch stable ABI (continued)](../sources/prs/vllm/PR-42663.md), [[CPU] Add fused GDN support for AMX CPU platform](../sources/prs/vllm/PR-42707.md), [[CPU] Specify required KV cache layout for CPU attention backend](../sources/prs/vllm/PR-42740.md), [[Refactor] Remove dead cuda kernels](../sources/prs/vllm/PR-42767.md), [[ROCm] [Bugfix] Fix DeepSeek V4 Functionality and Accuracy](../sources/prs/vllm/PR-42810.md), [[Perf] Add do_not_specialize in fused FP8 RoPE kernel](../sources/prs/vllm/PR-42849.md), [[Perf][MLA] Enable FULL cudagraph capture for TRITON_MLA decode](../sources/prs/vllm/PR-42885.md), [add cutedsl dsv4 indexer fp8 kernel](../sources/prs/vllm/PR-42899.md), [[XPU] update xpu graph usage](../sources/prs/vllm/PR-43043.md), [[Model Refactoring] Move deepseek_v4_ops to models/deepseek_v4 [3/N]](../sources/prs/vllm/PR-43073.md), [FlashAttention-4](../wiki/kernels/flash-attention-4.md), [FlashAttention SM100 MLA TopK Sparse Forward](../wiki/kernels/flash-attention-sm100-mla-topk.md), [FlashMLA — Multi-head Latent Attention](../wiki/kernels/flashmla.md), [Gated Delta Net — Linear Attention](../wiki/kernels/gated-delta-net.md), [Native Sparse Attention (NSA)](../wiki/kernels/nsa.md), [DeepSeek Sparse Attention / Sparse MLA](../wiki/kernels/sparse-mla.md), [TensorRT-LLM Blackwell FP4 DSA Indexer](../wiki/kernels/tensorrt-llm-blackwell-indexer.md) | +| `batched-gemv` | [Twelve Attempts at an FP4 Kernel](../sources/blogs/amandeep-nvfp4-attempts.md), [NVFP4 GEMV and Improved NVFP4 GEMV](../sources/blogs/simon-nvfp4-gemv.md), [Blackwell NVFP4 Kernel Hackathon Journey](../sources/blogs/yue-nvfp4-hackathon.md), [GPU Mode NVFP4 Hackathon - Problem 1: Batched GEMV](../sources/contests/gpu-mode-nvfp4/problem-1-gemv.md), [NVFP4 Batched GEMV](../wiki/kernels/nvfp4-gemv.md) | +| `decode` | [FlashMLA — Multi-head Latent Attention](../sources/blogs/flashmla.md), [DeepSeek-V3.2-Exp in vLLM: Fine-Grained Sparse Attention in Action](../sources/blogs/vllm-deepseek-v3-sparse-attention.md), [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [FlashInfer MLSys 2026 Track C: Gated Delta Net](../sources/contests/flashinfer-mlsys26/track-c-gated-delta-net.md), [[TRTLLM-10022][feat] Add hopper xqa decode support for skip softmax attention](../sources/prs/TensorRT-LLM/PR-10264.md), [[https://nvbugs/5669671][fix] Support GuidedDecoder with sharded logits (pick #10698)](../sources/prs/TensorRT-LLM/PR-10742.md), [[TRTLLM-10407][feat] Integrate CuTE DSL top-k kernel for Blackwell](../sources/prs/TensorRT-LLM/PR-11900.md), [[TRTLLM-11540][feat] Add EAGLE3 dynamic tree speculative decoding support](../sources/prs/TensorRT-LLM/PR-12062.md), [[None][feat] Temporally-Correlated Heuristic-guided Indexer TopK for Sparse Attention](../sources/prs/TensorRT-LLM/PR-12385.md), [[None][feat] Support sparse mqa/gqa attention](../sources/prs/TensorRT-LLM/PR-12470.md), [[None][feat] Add PDL support to CuTE DSL top-k kernels](../sources/prs/TensorRT-LLM/PR-12506.md), [[None][feat] Optimize causal_conv1d prefill and decode kernels](../sources/prs/TensorRT-LLM/PR-13103.md), [[TRTLLM-34871][feat] Add cute dsl FP8 paged MQA logits decode kernel](../sources/prs/TensorRT-LLM/PR-13219.md), [[None][perf] Scheme X L2-aware dispatcher and PDL launchers for sparse-attention GVR Top-K](../sources/prs/TensorRT-LLM/PR-13477.md), [[None][perf] Drop cubin and Eliminate ~6s FMHA JIT recompile in eager generation by aligning kernel selection with CUDA graph warmup](../sources/prs/TensorRT-LLM/PR-13505.md), [[TRTLLM-35237][feat] Add cute dsl FP4 paged MQA logits decode kernel](../sources/prs/TensorRT-LLM/PR-13929.md), [[None][feat] DSv4: enable GVR Heuristic Top-K for compress_ratio=4](../sources/prs/TensorRT-LLM/PR-14219.md), [feat: update decode attention APIs](../sources/prs/flashinfer/PR-1007.md), [bugfix: import wrapper of mla decode](../sources/prs/flashinfer/PR-1013.md), [add multi-item scoring](../sources/prs/flashinfer/PR-1015.md), [feat: add functional per-head FP8 quantization for FA3](../sources/prs/flashinfer/PR-1033.md), [[nvidia] Add Blackwell FMHA decode kernel from TRT-LLM](../sources/prs/flashinfer/PR-1051.md), [[Feature] Support PDL for batch Prefill and Decode](../sources/prs/flashinfer/PR-1117.md), [update trtllm-gen decode attention kernel launcher](../sources/prs/flashinfer/PR-1189.md), [Enable cudnn decode and add tests for the cudnn decode kernel](../sources/prs/flashinfer/PR-1221.md), [feat: add trtllm-gen mla cubin](../sources/prs/flashinfer/PR-1222.md), [add trtllm-gen context attention](../sources/prs/flashinfer/PR-1239.md), [Add trtllm-gen attention mha kernel with FP8 Q/K/V and FP8 output](../sources/prs/flashinfer/PR-1242.md), [feat: enable trtllm-gen mla MTP](../sources/prs/flashinfer/PR-1258.md), [refactor: refactor trtllm-gen attention kernel integration code](../sources/prs/flashinfer/PR-1289.md), [refactor: Improved metainfo for trtllm-gen fmha](../sources/prs/flashinfer/PR-1292.md), [test qkvo quantization not equal to 1.](../sources/prs/flashinfer/PR-1314.md), [feat: support output nvfp4 in trtllm-gen function call.](../sources/prs/flashinfer/PR-1318.md), [fix: fix trtllm-gen mla error on new interface](../sources/prs/flashinfer/PR-1348.md), [support trtllm-gen prefill fp4 output](../sources/prs/flashinfer/PR-1360.md), [Support scale factor start index for fp4 mha prefill/decode](../sources/prs/flashinfer/PR-1363.md), [GPT-OSS Support: Add Blackwell MoE mxfp4 implementation from TRTLLM and Attention Sink](../sources/prs/flashinfer/PR-1389.md), [Add Mxfp4 trtllm-gen moe unit tests](../sources/prs/flashinfer/PR-1399.md), [benchmark: trtllm-gen mha with sink, add benchmark args](../sources/prs/flashinfer/PR-1415.md), [fix: remote redundant zero_init from trtllm-gen attn](../sources/prs/flashinfer/PR-1444.md), [feat: enable trtllm-gen attn speculative decoding verify by decode](../sources/prs/flashinfer/PR-1453.md), [Fix TRTLLM NVFP4-out attention kernel scale factor dim issue](../sources/prs/flashinfer/PR-1460.md), [feat: add pdl for trtllm-gen attn](../sources/prs/flashinfer/PR-1484.md), [feat: Support fp8 qkv, fp16/bf16 out MHA for trtllm-gen.](../sources/prs/flashinfer/PR-1490.md), [backend: Refactor trtllm-gen fmha metainfo loading](../sources/prs/flashinfer/PR-1518.md), [feat: Add fp8-qkv, fp16/bf16 output MHA](../sources/prs/flashinfer/PR-1540.md), [Backend: downgrade trtllm-gen kernel to cuda-12](../sources/prs/flashinfer/PR-1567.md), [feat: Support for inferring out_dtype from out.dtype for TRTLLM attention kernel](../sources/prs/flashinfer/PR-1578.md), [fix: Improve TRTLLM attention kernel out_dtype unit test](../sources/prs/flashinfer/PR-1590.md), [bugfix: collect all modules to aot](../sources/prs/flashinfer/PR-1622.md), [fix: pass workspace for trtllm-gen attention](../sources/prs/flashinfer/PR-1635.md), [perf&bugfix: skip kv-tile computation out of sliding window in FA2; fix __syncthreads in mergestate](../sources/prs/flashinfer/PR-1661.md), [TGV GEMM as a BF16 backend alternative to cuBLAS](../sources/prs/flashinfer/PR-1668.md), [feat: Add `variant.OutputTransform()` to decode kernels](../sources/prs/flashinfer/PR-1670.md), [feat: Batch-size invariant FA2 Prefill & Decode](../sources/prs/flashinfer/PR-1675.md), [feat: port fast_decode_plan from sgl](../sources/prs/flashinfer/PR-1745.md), [Fix tests/test_trtllm_gen_attention.py::test_trtllm_batch_prefill, ::test_trtllm_batch_decode mismatch error](../sources/prs/flashinfer/PR-1755.md), [fix: should pass global_override_indptr_cpu in fast_decode_plan param list](../sources/prs/flashinfer/PR-1757.md), [Waive / disable test_mla_decode_kernel.py::test_mla_decode_kernel for not sm80 ](../sources/prs/flashinfer/PR-1771.md), [feat: add xqa backend and completes NHD/HND coverage for trtllm-gen/xqa backend](../sources/prs/flashinfer/PR-2001.md), [Fix trtllm-gen attention illegal memory access](../sources/prs/flashinfer/PR-2002.md), [use scalar for kv_scale in xqa](../sources/prs/flashinfer/PR-2033.md), [feat: add xqa mla backend](../sources/prs/flashinfer/PR-2053.md), [misc: Add XQA decode to microbenchmark for sm90 and sm120](../sources/prs/flashinfer/PR-2055.md), [unittest: improve the efficiency of xqa unittests](../sources/prs/flashinfer/PR-2075.md), [[Feature] Support batch prefill for POD Attention](../sources/prs/flashinfer/PR-2079.md), [enable xqa fp8 output](../sources/prs/flashinfer/PR-2081.md), [[API change] Allow using torch.Tensor for scales for trtllm-gen attention](../sources/prs/flashinfer/PR-2084.md), [enable xqa speculative decoding](../sources/prs/flashinfer/PR-2105.md), [add tensor scale input for xqa](../sources/prs/flashinfer/PR-2110.md), [feat: support variable sequence length in decode kernel of trtllm-gen attention](../sources/prs/flashinfer/PR-2125.md), [feat: add trtllm-gen per-tensor sparseMla kernels.](../sources/prs/flashinfer/PR-2138.md), [Enable Hopper FA3 FP8 attention in decode.py](../sources/prs/flashinfer/PR-2148.md), [refactor: Move mla code from decode.py to mla.py and add to documentation](../sources/prs/flashinfer/PR-2163.md), [[TRTLLM-Gen Fmha] add optimized trtllm-gen decode kernels for high throughput + speculative decoding](../sources/prs/flashinfer/PR-2265.md), [fix: Decode benchmark's fa2_tc uses backend=fa2 in wrapper](../sources/prs/flashinfer/PR-2302.md), [feat: [Qwen3-Next] Add Cute DSL GDN decode kernel and tests](../sources/prs/flashinfer/PR-2370.md), [fix: ensure each CTA processes full numHeadsQPerKv for trtllm decode kernel](../sources/prs/flashinfer/PR-2380.md), [A Blackwell-optimized version of selective_state_update (decode)](../sources/prs/flashinfer/PR-2387.md), [perf: improve gdn decode cute-dsl kernels](../sources/prs/flashinfer/PR-2405.md), [feat: Add TRTLLM-Gen Skip-Softmax kernels for prefill and decode](../sources/prs/flashinfer/PR-2477.md), [Ameyn/gdn decode cutedsl kernel](../sources/prs/flashinfer/PR-2498.md), [Support NVFP4 KV cache decode on SM120](../sources/prs/flashinfer/PR-2520.md), [Feat/gdn decode pooled](../sources/prs/flashinfer/PR-2521.md), [pick fa2 for BatchDecodeWithPagedKVCacheWrapper auto backend](../sources/prs/flashinfer/PR-2530.md), [Perf: Optimize GDN decode pretranspose kernel for all batch sizes](../sources/prs/flashinfer/PR-2588.md), [Ameyn/gdn bf16 tolerance parallel reduction](../sources/prs/flashinfer/PR-2610.md), [perf(gdn): optimize MTP kernel with ILP rows and SMEM v caching](../sources/prs/flashinfer/PR-2618.md), [feat: add pool+indices support to gated_delta_rule_decode_pretranspose (bf16 path) ](../sources/prs/flashinfer/PR-2619.md), [benchmark: Enable speculative decode microbenchmarking for paged decode](../sources/prs/flashinfer/PR-2628.md), [fix: add SM121 support to SM120 version guards](../sources/prs/flashinfer/PR-2631.md), [[feat] trtllm-gen mxfp8 gemm](../sources/prs/flashinfer/PR-2653.md), [feat(gdn): add BF16 state kernel with MTP support beyond T>4 with intermediate caching.](../sources/prs/flashinfer/PR-2679.md), [Add NVFP4 KV cache quantization support for SM100](../sources/prs/flashinfer/PR-2702.md), [[gdn] support non-contiguous state for decoding](../sources/prs/flashinfer/PR-2727.md), [Add cute dsl mla decode op](../sources/prs/flashinfer/PR-2743.md), [feat: Expose TRT-LLM FMHA style paged KV Cache and page table layout](../sources/prs/flashinfer/PR-2770.md), [[fix] Bugfix 1367: fix VariableBlockSparseAttention buffer overflow by dynamically resizing kv_lens_buffer](../sources/prs/flashinfer/PR-2802.md), [[CuTe DSL] Add modular FMHA prefill and MLA decode attention kernels](../sources/prs/flashinfer/PR-2805.md), [feat(gdn): add padding index guard for bf16 decode kernel](../sources/prs/flashinfer/PR-2810.md), [[Fmha] Sparse MLA decode kernel selection heuristics](../sources/prs/flashinfer/PR-2836.md), [perf: Optimize GDN MTP decode kernel (v15) — eliminate ilp=1 fallback…](../sources/prs/flashinfer/PR-2842.md), [read real strides for kv and block scale](../sources/prs/flashinfer/PR-2844.md), [[fix] bugfix 2856: Fix pre-allocated out shape check in trtllm_batch_decode_with_kv_cache_mla for q_len_per_req > 1](../sources/prs/flashinfer/PR-2876.md), [feat: add pdl support for cute dsl mla decode kernel support](../sources/prs/flashinfer/PR-2901.md), [Only swizzle on v block scale; rename kv_block_scales to kv_cache_sf](../sources/prs/flashinfer/PR-2954.md), [perf: Optimize CUTLASS MoE helper kernels for small-batch decode workloads](../sources/prs/flashinfer/PR-3014.md), [Support lse in trtllm paged attn kernels](../sources/prs/flashinfer/PR-3058.md), [Support NVFP4 KV for prefill and batch attention kernels](../sources/prs/flashinfer/PR-3097.md), [feat(trace): embed runnable init() in every TraceTemplate](../sources/prs/flashinfer/PR-3221.md), [Support Kimi K2.5 H64 CuTe DSL MLA decode](../sources/prs/flashinfer/PR-3235.md), [Ameyn/gdn bf16 dispatcher and 4d pool](../sources/prs/flashinfer/PR-3268.md), [feat: support deepseek prefill attention shape](../sources/prs/flashinfer/PR-765.md), [bugfix: MLA decode should multiply sm_scale by math::log2e](../sources/prs/flashinfer/PR-787.md), [fix rope logic in mla decoding](../sources/prs/flashinfer/PR-793.md), [feat: apply sm_scale at logits instead of q in FA2 template](../sources/prs/flashinfer/PR-801.md), [perf: MLA decode kernel implemented by CuTe targeted to SM80](../sources/prs/flashinfer/PR-844.md), [Add POD-Attention to FlashInfer](../sources/prs/flashinfer/PR-858.md), [perf: reduce torch.library dispatch overhead](../sources/prs/flashinfer/PR-968.md), [perf: Fix python API overhead when CUDAGraph is not enabled](../sources/prs/flashinfer/PR-969.md), [Disable kernel cutlass_mla_decode on SM103](../sources/prs/sglang/PR-10058.md), [Use trtllm_mla decode kernel for draft extend in speculative decoding](../sources/prs/sglang/PR-11664.md), [Use trtllm mha decode kernel for target_verify in speculative decoding](../sources/prs/sglang/PR-13976.md), [Fix accuracy issue when using a16w16 mla_decode_fwd](../sources/prs/sglang/PR-14936.md), [[jit-kernel] Add CuTe DSL GDN Decode Kernel](../sources/prs/sglang/PR-15631.md), [[DeepSeek v3.2] Opt MTP decode cuda batch sizes and nsa implementation](../sources/prs/sglang/PR-16961.md), [Optimize GDN decode for Qwen3 Next](../sources/prs/sglang/PR-17094.md), [feat(gdn): add FlashInfer K-last SSM layout support for GDN prefill and decode for Hopper](../sources/prs/sglang/PR-18361.md), [[AMD] Update aiter to v0.1.10.post2](../sources/prs/sglang/PR-18423.md), [feat: add FA4 SM90 paged KV decode support & update attention docs](../sources/prs/sglang/PR-18442.md), [Tilelang sparse decode fwd for dsv32 mi355](../sources/prs/sglang/PR-18488.md), [[NVIDIA] Integrate FlashInfer decode kernel (Blackwell) for Qwen3.5](../sources/prs/sglang/PR-19150.md), [Fix MLA decode path returning unwritten (padded) rows](../sources/prs/sglang/PR-19902.md), [[AMD] Fix FP8 assertion failure in aiter MLA decode by falling back to self.k_scale](../sources/prs/sglang/PR-19935.md), [Support Triton MLA FP8 KV cache](../sources/prs/sglang/PR-20479.md), [[Kernel] Fuse temperature + softmax in sampling for decode speedup](../sources/prs/sglang/PR-20501.md), [[KDA] Support CuTeDSL KDA decode kernel](../sources/prs/sglang/PR-21203.md), [ [GDN] Remove FlashInfer GDN decode + no_buffer guard and default to FlashInfer on SM100+ ](../sources/prs/sglang/PR-21861.md), [[attn backend] Integrate tokenspeed_mla prefill/decode kernels (fp8 kv cache, blackwell)](../sources/prs/sglang/PR-24925.md), [amd/deepseek_v4 27/N [fix] Reduce Triton autotune configs for faster first-time server launch](../sources/prs/sglang/PR-25554.md), [Introduce SchedulerDPAttnAdapter to own DP-attention state](../sources/prs/sglang/PR-25611.md), [Move DP-attention adapter methods to SchedulerDPAttnAdapter](../sources/prs/sglang/PR-25612.md), [[Refactor] Rename NSA → DSA: user-facing aliases, file/class/import rename](../sources/prs/sglang/PR-25821.md), [[Test] Stage-a sanity kits; consolidate core/ + models_e2e/ tests](../sources/prs/sglang/PR-25831.md), [Add fast decode plan for flashinfer mla](../sources/prs/sglang/PR-3987.md), [[Revision] Add fast decode plan for flashinfer mla ](../sources/prs/sglang/PR-4012.md), [support cmake for sgl-kernel](../sources/prs/sglang/PR-4706.md), [Add optimized native kernels in sgl-kernel](../sources/prs/sglang/PR-5150.md), [Add Cutlass MLA attention backend](../sources/prs/sglang/PR-5390.md), [[PD] Fix dynamic port support and MLA buffer for Mooncake](../sources/prs/sglang/PR-5415.md), [[perf] introduce deep gemm group_gemm_masked as gemm](../sources/prs/sglang/PR-5432.md), [Cutlass MLA decode - fix dtype error](../sources/prs/sglang/PR-5868.md), [[Feature] Comprehensive Hybrid Parallelism Support](../sources/prs/sglang/PR-6389.md), [CPU: map changes from developing branch in sgl-kernel](../sources/prs/sglang/PR-6833.md), [[perf][sgl-kernel] extend cutlass_mla_decode to support num_head < 128](../sources/prs/sglang/PR-6929.md), [Fix grammar abort & Minor style fixes](../sources/prs/sglang/PR-7204.md), [Support non-contiguous query input for extend/decode attention](../sources/prs/sglang/PR-7462.md), [[NVIDIA] Add Low Latency NVFP4 decode kernels from Flashinfer](../sources/prs/sglang/PR-8552.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [[CUDA] Support int4 `T.gemm`](../sources/prs/tilelang/PR-2063.md), [[Attention] MLA decode optimizations](../sources/prs/vllm/PR-12528.md), [[Perf] Mem align KV caches for CUDA devices (MLA perf improvement)](../sources/prs/vllm/PR-12676.md), [[core] Perf improvement for DSv3 on AMD GPUs](../sources/prs/vllm/PR-13718.md), [[V1] V1 Enablement Oracle ](../sources/prs/vllm/PR-13726.md), [[Kernel][CPU] CPU MLA](../sources/prs/vllm/PR-14744.md), [[FEAT][ROCm] Integrate Fused MoE Kernels from AITER](../sources/prs/vllm/PR-14967.md), [[Bugfix] Fix incorrect qwen2.5-vl attention mask pre-computation](../sources/prs/vllm/PR-15200.md), [[ROCM][KERNEL] Paged attention for V1](../sources/prs/vllm/PR-15720.md), [[NVIDIA] Support Cutlass MLA for Blackwell GPUs](../sources/prs/vllm/PR-16032.md), [Upstream Llama4 Support to Main](../sources/prs/vllm/PR-16113.md), [[Kernel] Unified Triton kernel that doesn't distinguish between prefill + decode](../sources/prs/vllm/PR-16828.md), [[Attention] FA3 decode perf improvement - single mma warp group support for head dim 128](../sources/prs/vllm/PR-16864.md), [[ROCm][Kernel][V1] Enable AMD Radeon GPU Custom Paged Attention on v1](../sources/prs/vllm/PR-17004.md), [[Misc][ROCm] Exclude `cutlass_mla_decode` for ROCm build](../sources/prs/vllm/PR-17289.md), [[v1] AttentionMetadata for each layer](../sources/prs/vllm/PR-17394.md), [[Bugfix][ROCm] Use `chunked_prefill_paged_decode` as fallback for V1 attention on ROCm](../sources/prs/vllm/PR-18093.md), [SM100 Cutlass MLA decode with unrestricted num_heads (< 128) for DeepSeek TP](../sources/prs/vllm/PR-20769.md), [[Kernel] Flashinfer MLA (trtllm-gen) decode kernel integration](../sources/prs/vllm/PR-21078.md), [[v1] Add Whisper model support (encoder-decoder)](../sources/prs/vllm/PR-21088.md), [Support encoder-only models without KV-Cache](../sources/prs/vllm/PR-21270.md), [[v1][attention] Support Hybrid Allocator + FlashInfer](../sources/prs/vllm/PR-21412.md), [[Attention] Support multiple attention metadata builders per kv_cache_spec + proper local attention no hybrid kv cache fix](../sources/prs/vllm/PR-21588.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv/out Attention Kernel](../sources/prs/vllm/PR-21716.md), [[NVIDIA] Support Flashinfer TRT-LLM Prefill Attention Kernel](../sources/prs/vllm/PR-22095.md), [[Bugfix] Add proper comparison for package versions](../sources/prs/vllm/PR-22314.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv NVFP4-out Attention Kernel](../sources/prs/vllm/PR-22703.md), [[Bugfix] Fix default enable for CUTLASS MLA on SM100](../sources/prs/vllm/PR-22738.md), [[Core] Always use tensor cores for Flashinfer Decode Wrapper](../sources/prs/vllm/PR-23214.md), [[Flashinfer] Support Flashinfer TRTLLM FP8-qkv BF16/FP16-out Attention Kernel](../sources/prs/vllm/PR-23647.md), [[v1] Add cross-attention KV cache support for encoder-decoder models](../sources/prs/vllm/PR-23664.md), [[Core/DBO][1/N] Add Dual-Batch Overlap mechanism to VLLM](../sources/prs/vllm/PR-23693.md), [[Feature] Support Decode Context Parallel (DCP) for MLA](../sources/prs/vllm/PR-23734.md), [[Kernel] cuda kernels for upcoming decode context parallel feature](../sources/prs/vllm/PR-23791.md), [[Model] Add LongCat-Flash ](../sources/prs/vllm/PR-23991.md), [[Kernel] Support decode context parallelism on Blackwell with CUTLASS MLA](../sources/prs/vllm/PR-24385.md), [[DCP] Support Decode Context Parallel (DCP) for GQA with FlashAttention](../sources/prs/vllm/PR-24864.md), [[Attention][DCP] Support DCP with query length > 1 (MTP) with FA3](../sources/prs/vllm/PR-25049.md), [[Performance] Split FlashAttn attention and cache update](../sources/prs/vllm/PR-25954.md), [[Spec Decode] Enable efficient speculative decoding with FlashInfer-MLA](../sources/prs/vllm/PR-25984.md), [[ROCm][MLA] enable fp8 MLA decode on ROCm](../sources/prs/vllm/PR-28032.md), [[Attention] Make `split_decodes_and_prefills(..., require_uniform=True)` support padding](../sources/prs/vllm/PR-29644.md), [Add Mistral Large 3 and Ministral 3](../sources/prs/vllm/PR-29757.md), [[SpecDecode] Simplified alternative padded-speculation acceptance rate fix](../sources/prs/vllm/PR-29845.md), [[BugFix] Fix DBO assert `assert B_block_table == B_q`](../sources/prs/vllm/PR-29933.md), [[ROCm] add fallback for aiter fp8 decode mla](../sources/prs/vllm/PR-30005.md), [[ROCm][Bugfix] Add MLACommonMetadata to allowed attention types for speculative decoding](../sources/prs/vllm/PR-30430.md), [[Feature] Add SM103 (Blackwell Ultra) Support to vLLM](../sources/prs/vllm/PR-30484.md), [[Bugfix][Hardware][AMD] Fix last_page_len calculation in AITER MLA decode](../sources/prs/vllm/PR-31282.md), [[Bugfix][ROCm]Fix Qwen3-Next-80B-A3B-Thinking inference and optimize non-standard block size (544) support under rocm_atten](../sources/prs/vllm/PR-31380.md), [[1/N][Attention] Restructure attention: move files](../sources/prs/vllm/PR-31916.md), [[4/N][Attention] Move MLA common to model_executor](../sources/prs/vllm/PR-32060.md), [[Spec Decode] Unified Parallel Drafting](../sources/prs/vllm/PR-32887.md), [Triton MLA perf fixes](../sources/prs/vllm/PR-33529.md), [fix(cpu): fix mla_decode compilation on x86 without AVX512](../sources/prs/vllm/PR-34052.md), [[BugFix] Add support for MTP num_speculative_tokens > 1 with sparse MLA](../sources/prs/vllm/PR-34552.md), [[Kernel] Add FP8 KV cache support to Triton MLA decode attention](../sources/prs/vllm/PR-34597.md), [[Update] Use FlashInfer fast_decode_plan directly instead of replication](../sources/prs/vllm/PR-34687.md), [[Bugfix] Fix GDN attention crash with mixed decode/spec-decode batches](../sources/prs/vllm/PR-34871.md), [[Model Runner V2] Support attention group](../sources/prs/vllm/PR-35036.md), [Kimi k2.5 MLA based eagle3](../sources/prs/vllm/PR-36361.md), [[ROCm][Perf] Allow MTP lens > 1 in Sparse MLA](../sources/prs/vllm/PR-36681.md), [[ROCm] Attention selector reordering](../sources/prs/vllm/PR-36702.md), [[ROCm] Fix KV copy methods and auto-select attention backend for ROCm](../sources/prs/vllm/PR-36845.md), [[Feat][Spec Decode] DFlash](../sources/prs/vllm/PR-36847.md), [[Benchmark] Improvements to attention benchmark script](../sources/prs/vllm/PR-37115.md), [[Attention] Support distinguishing between short extends and decodes](../sources/prs/vllm/PR-37303.md), [[Model Runner V2] fix draft attention metadata generation](../sources/prs/vllm/PR-37364.md), [[Perf][Kernel] Persistent TopK scheduler: unified CUDAGraph-safe kernel with dynamic per-row dispatch - DeepSeek-V3.2 DSA decode](../sources/prs/vllm/PR-37421.md), [[Performance] Remove unnecessary zero-fill of MLA decode output tensor in Aiter backend](../sources/prs/vllm/PR-37539.md), [[Feature] Support per-draft-model MoE backend via `--speculative-config`](../sources/prs/vllm/PR-37880.md), [[ROCm][perf] fix Aiter sparse MLA with MTP>1](../sources/prs/vllm/PR-37887.md), [[Model Runner V2] Rebuild attention metadata before eagle decode full…](../sources/prs/vllm/PR-38311.md), [[Attention Backend] TurboQuant: 2-bit KV cache compression with 4x capacity](../sources/prs/vllm/PR-38479.md), [[XPU] Fix spec-decode UTs under tests/v1/spec_decode](../sources/prs/vllm/PR-38491.md), [[Refactor] Improve indexer decode path metadata preparation](../sources/prs/vllm/PR-38865.md), [[Bugfix][MoE] Fix 6-8% decode regression: prefer multi-stream shared expert overlap](../sources/prs/vllm/PR-38990.md), [[MLA] Optimize mla indexer prepare uniform decode for MTP > 1](../sources/prs/vllm/PR-39458.md), [[Attention][TurboQuant] Share dequant buffers, eliminate float16_copy](../sources/prs/vllm/PR-40941.md), [[Spec Decode] Add Gemma4 MTP speculative decoding support](../sources/prs/vllm/PR-41745.md), [[MLA Attention Backend] Add TOKENSPEED_MLA backend for DSR1/Kimi K25 prefill + decode on Blackwell](../sources/prs/vllm/PR-41778.md), [[Bugfix] mamba: run single-token extends as decodes](../sources/prs/vllm/PR-42430.md), [[Perf][MLA] Enable FULL cudagraph capture for TRITON_MLA decode](../sources/prs/vllm/PR-42885.md), [FlashMLA — Multi-head Latent Attention](../wiki/kernels/flashmla.md), [Gated Delta Net — Linear Attention](../wiki/kernels/gated-delta-net.md), [DeepSeek Sparse Attention / Sparse MLA](../wiki/kernels/sparse-mla.md) | +| `flash-attention` | [FlashAttention-4 Blog](../sources/blogs/flash-attention-4.md), [DeepSeek-V3.2-Exp in vLLM: Fine-Grained Sparse Attention in Action](../sources/blogs/vllm-deepseek-v3-sparse-attention.md), [FlashAttention-4: Algorithm and Kernel Co-design for Blackwell GPUs](../sources/docs/flash-attention-4.md), [[None][feat] Remove non flash attetnion style fmha_v2 kernel for hopper](../sources/prs/TensorRT-LLM/PR-11381.md), [[TRTLLM-11092][feat] add support for visual gen FA4 attention backend](../sources/prs/TensorRT-LLM/PR-11697.md), [[TRTLLM-11119][feat] Blackwell SageAttention, Integrate into AttentionOp API](../sources/prs/TensorRT-LLM/PR-11718.md), [[TRTLLM-11540][feat] Add EAGLE3 dynamic tree speculative decoding support](../sources/prs/TensorRT-LLM/PR-12062.md), [[None][feat] Support sparse mqa/gqa attention](../sources/prs/TensorRT-LLM/PR-12470.md), [[None][feat] Trtllm-gen FMHA JIT support](../sources/prs/TensorRT-LLM/PR-12612.md), [[None][feat] Optimize mamba SSD prefill and extend flashinfer dispatch](../sources/prs/TensorRT-LLM/PR-12731.md), [[TRTLLM-11485][feat] Feature rework: Add SageAttention refreshed kernels (attentionOp only)](../sources/prs/TensorRT-LLM/PR-12937.md), [[None][perf] Drop cubin and Eliminate ~6s FMHA JIT recompile in eager generation by aligning kernel selection with CUDA graph warmup](../sources/prs/TensorRT-LLM/PR-13505.md), [[None][feat] Add DeepSeekV4 attention kernels](../sources/prs/TensorRT-LLM/PR-13652.md), [[None][feat] Update FMHA cubins for head_dim 80](../sources/prs/TensorRT-LLM/PR-13808.md), [[TRTLLM-35237][feat] Add cute dsl FP4 paged MQA logits decode kernel](../sources/prs/TensorRT-LLM/PR-13929.md), [[None][feat] Update the logic of FMHA JIT path](../sources/prs/TensorRT-LLM/PR-14291.md), [[None][feat] GPT-OSS Sm120/Sm121 Support](../sources/prs/TensorRT-LLM/PR-7937.md), [Correct divmod order in example 77 (blackwell fmha)](../sources/prs/cutlass/PR-2291.md), [Handle get_masked_trip_count for small length in fmha example](../sources/prs/cutlass/PR-2292.md), [[ex77] fix mla split; add fwd lse; add bwd varlen](../sources/prs/cutlass/PR-2366.md), [Example 77 add blackwell flash-attention bwd for MLA shape](../sources/prs/cutlass/PR-2466.md), [fix gqa issue for blackwell fmha.py](../sources/prs/cutlass/PR-2599.md), [Fp8 kernel with "in-kernel" transpose of V in producer](../sources/prs/flash-attention/PR-1100.md), [FA3 FP8 qkv descales + restore max offset for h128 causal + added sync for producer WG](../sources/prs/flash-attention/PR-1173.md), [FA3 kvcache + split kv + gqa parallelization](../sources/prs/flash-attention/PR-1236.md), [[Cute,Fwd,Sm100] fp8 e4m3 and e5m2 support](../sources/prs/flash-attention/PR-2109.md), [Feat([FA4][CUTE DSL]) Add head_dim=256 support (forward + backward)](../sources/prs/flash-attention/PR-2412.md), [[hd256] Improve forward kernel with exp2 FMA emulation (3% to 9% performance gain)](../sources/prs/flash-attention/PR-2488.md), [[hd256] Add TMA paged KV support to SM100 2CTA forward kernel](../sources/prs/flash-attention/PR-2489.md), [[FA4][hd256] Backward TMA bulk-store epilogue + LSE/dpsum coalesce](../sources/prs/flash-attention/PR-2497.md), [[nvidia] initial support for blackwell kernels](../sources/prs/flashinfer/PR-1039.md), [[nvidia] Add Blackwell FMHA decode kernel from TRT-LLM](../sources/prs/flashinfer/PR-1051.md), [bugfix: adding lse output to blackwell fmha kernels](../sources/prs/flashinfer/PR-1071.md), [bugfix: follow user-specified sm_scale for blackwell cutlass fmha](../sources/prs/flashinfer/PR-1072.md), [bugfix: host-precomuted plan function for blackwell fmha](../sources/prs/flashinfer/PR-1106.md), [hotfix: fix the blackwell fmha stream](../sources/prs/flashinfer/PR-1116.md), [Add more logging to TRTLLM-GEN debug trace (NFC)](../sources/prs/flashinfer/PR-1158.md), [bugfix: fix invalid blackwell fmha unittests](../sources/prs/flashinfer/PR-1181.md), [update trtllm-gen decode attention kernel launcher](../sources/prs/flashinfer/PR-1189.md), [bugfix: fix blackwell fmha hanging issue for empty kv_len](../sources/prs/flashinfer/PR-1198.md), [feat: add trtllm-gen mla cubin](../sources/prs/flashinfer/PR-1222.md), [add trtllm-gen context attention](../sources/prs/flashinfer/PR-1239.md), [Add trtllm-gen attention mha kernel with FP8 Q/K/V and FP8 output](../sources/prs/flashinfer/PR-1242.md), [feat: enable trtllm-gen mla MTP](../sources/prs/flashinfer/PR-1258.md), [fix: update trtllm-gen fmha benchmark](../sources/prs/flashinfer/PR-1280.md), [fix multiCtasKvScratchPtr misalignment issue (new one)](../sources/prs/flashinfer/PR-1286.md), [refactor: refactor trtllm-gen attention kernel integration code](../sources/prs/flashinfer/PR-1289.md), [refactor: Improved metainfo for trtllm-gen fmha](../sources/prs/flashinfer/PR-1292.md), [Fix the bug of the kernel-selection heuristic in trtllm-gen](../sources/prs/flashinfer/PR-1307.md), [feat: support output nvfp4 in trtllm-gen function call.](../sources/prs/flashinfer/PR-1318.md), [fix: fix trtllm-gen mla error on new interface](../sources/prs/flashinfer/PR-1348.md), [support trtllm-gen prefill fp4 output](../sources/prs/flashinfer/PR-1360.md), [Support scale factor start index for fp4 mha prefill/decode](../sources/prs/flashinfer/PR-1363.md), [GPT-OSS Support: Add Blackwell MoE mxfp4 implementation from TRTLLM and Attention Sink](../sources/prs/flashinfer/PR-1389.md), [benchmark: trtllm-gen mha with sink, add benchmark args](../sources/prs/flashinfer/PR-1415.md), [fix: remote redundant zero_init from trtllm-gen attn](../sources/prs/flashinfer/PR-1444.md), [Remove getEnvEnablePDL in favor of enable_pdl parameter](../sources/prs/flashinfer/PR-1446.md), [feat: add pdl for trtllm-gen attn](../sources/prs/flashinfer/PR-1484.md), [feat: Support fp8 qkv, fp16/bf16 out MHA for trtllm-gen.](../sources/prs/flashinfer/PR-1490.md), [backend: Refactor trtllm-gen fmha metainfo loading](../sources/prs/flashinfer/PR-1518.md), [feat: Integrate TRTLLM varlen kernel for deepseek R1 prefill ](../sources/prs/flashinfer/PR-1537.md), [Backend: downgrade trtllm-gen kernel to cuda-12](../sources/prs/flashinfer/PR-1567.md), [fix: semaphoress must be at the fixed range in workspace buffer on trtllm_gen attention](../sources/prs/flashinfer/PR-1584.md), [feat: initial support for SM103, SM110, SM120, SM121](../sources/prs/flashinfer/PR-1608.md), [bugfix: trtllm-gen fmha sm101 and sm100 compatibility](../sources/prs/flashinfer/PR-1631.md), [fix: zero-init workspace buffer for trtllm-gen fmha](../sources/prs/flashinfer/PR-1643.md), [perf: improve attention of tcgen05 flash-attention](../sources/prs/flashinfer/PR-1681.md), [perf: Port the separate reduce kernel mode from trtllm.](../sources/prs/flashinfer/PR-1685.md), [Add head_dim=64 for tcgen05 tcgen05 flash-attention implementation](../sources/prs/flashinfer/PR-1850.md), [Tune kernel compilation parameters for https://github.com/flashinfer-ai/flashinfer/pull/1850 ](../sources/prs/flashinfer/PR-1878.md), [fix: Fix trtllm-gen prefill IMA when batch_size==1](../sources/prs/flashinfer/PR-1912.md), [feat: add xqa backend and completes NHD/HND coverage for trtllm-gen/xqa backend](../sources/prs/flashinfer/PR-2001.md), [Rebase FP8 SM100 Cutlass FMHA Attention to main (original PR#1238)](../sources/prs/flashinfer/PR-2047.md), [Fix: several bugs/issues with trtllm-gen attention kernels. ](../sources/prs/flashinfer/PR-2062.md), [[API change] Allow using torch.Tensor for scales for trtllm-gen attention](../sources/prs/flashinfer/PR-2084.md), [feat: support variable sequence length in decode kernel of trtllm-gen attention](../sources/prs/flashinfer/PR-2125.md), [fix(trtllm): reset negative strideBatch to 0 for ragged KV layout to …](../sources/prs/flashinfer/PR-2134.md), [fix: some bugs of headDim 256 trtllm-gen fmha kernels. ](../sources/prs/flashinfer/PR-2137.md), [feat: add trtllm-gen per-tensor sparseMla kernels.](../sources/prs/flashinfer/PR-2138.md), [feat: TRTLLM FMHAv2 backend for ctx attention](../sources/prs/flashinfer/PR-2142.md), [fix: compile flags for trtllm fmha_v2 ](../sources/prs/flashinfer/PR-2175.md), [feat: support non-contiguous query for trtllm-gen attention backend](../sources/prs/flashinfer/PR-2254.md), [[TRTLLM-Gen Fmha] add optimized trtllm-gen decode kernels for high throughput + speculative decoding](../sources/prs/flashinfer/PR-2265.md), [fix: ensure each CTA processes full numHeadsQPerKv for trtllm decode kernel](../sources/prs/flashinfer/PR-2380.md), [feat: Add TRTLLM fmha_v2 library for SM90 attention with Skip-Softmax ](../sources/prs/flashinfer/PR-2446.md), [feat: Add TRTLLM-Gen Skip-Softmax kernels for prefill and decode](../sources/prs/flashinfer/PR-2477.md), [feat: Enable TRTLLM-Gen Skip-Softmax attention for MLA](../sources/prs/flashinfer/PR-2547.md), [fix: allow fmha_v2_prefill_deepseek on SM121 (DGX Spark)](../sources/prs/flashinfer/PR-2559.md), [fix: guard CUTLASS FMHA against SM12x and fix fmha_v2 SM121a check](../sources/prs/flashinfer/PR-2560.md), [feat: add is_sm12x_supported() helper for SM12x family detection](../sources/prs/flashinfer/PR-2574.md), [fix: add SM121 support to SM120 version guards](../sources/prs/flashinfer/PR-2631.md), [feat: add support for more MLA head dimensions](../sources/prs/flashinfer/PR-2677.md), [Add NVFP4 KV cache quantization support for SM100](../sources/prs/flashinfer/PR-2702.md), [feat: Add DiT-oriented kernels where Qk (Bmm1) type can be reinterpreted into Int8 or BFloat16](../sources/prs/flashinfer/PR-2711.md), [feat: Expose TRT-LLM FMHA style paged KV Cache and page table layout](../sources/prs/flashinfer/PR-2770.md), [tests: skip sliding window + fp8 to prevent hang in fmha_v2 unit tests](../sources/prs/flashinfer/PR-2781.md), [[fmha-v2] Support HND and NHD paged KV cache layouts with conditional stride handling](../sources/prs/flashinfer/PR-2799.md), [[CuTe DSL] Add modular FMHA prefill and MLA decode attention kernels](../sources/prs/flashinfer/PR-2805.md), [[Fmha] Sparse MLA decode kernel selection heuristics](../sources/prs/flashinfer/PR-2836.md), [[Perf] Add FMHAv2 to flashinfer_benchmark.py and eliminate unnecessary H2D](../sources/prs/flashinfer/PR-2841.md), [read real strides for kv and block scale](../sources/prs/flashinfer/PR-2844.md), [Only swizzle on v block scale; rename kv_block_scales to kv_cache_sf](../sources/prs/flashinfer/PR-2954.md), [[Fmha] support nvfp4 output keepsMmaAb generation kernels](../sources/prs/flashinfer/PR-2988.md), [Support lse in trtllm paged attn kernels](../sources/prs/flashinfer/PR-3058.md), [cute-dsl fmha prefill (cubin integration): remove front-padding, add attention_sink, and pdl support](../sources/prs/flashinfer/PR-3181.md), [Add dynamic tokens-per-page TRTLLM-GEN GQA kernels](../sources/prs/flashinfer/PR-3259.md), [fix(fmha_v2): fix FP8 V-scratch pipeline and varlen scheduler on SM90](../sources/prs/flashinfer/PR-3276.md), [Update trtllm FMHA cubins](../sources/prs/flashinfer/PR-3317.md), [[flex_attention] adds support for low precision K/V inputs in compiled mode with GPU](../sources/prs/pytorch/PR-170486.md), [[RELEASE 2.11] Release only changes](../sources/prs/pytorch/PR-175091.md), [[release 2.12] Apply Release only changes to 2.12 branch](../sources/prs/pytorch/PR-180470.md), [[NVIDIA] FA3/FA4 Fix ](../sources/prs/sglang/PR-11606.md), [Tiny use trtllm_mha as default when possible](../sources/prs/sglang/PR-14291.md), [[NVIDIA] upstream FA4](../sources/prs/sglang/PR-15182.md), [Move fa4 from sgl-kernel to jit kernel](../sources/prs/sglang/PR-17353.md), [feat: add FA4 SM90 paged KV decode support & update attention docs](../sources/prs/sglang/PR-18442.md), [Add SGLang CUDA crash API logging inspired by FlashInfer](../sources/prs/sglang/PR-20910.md), [Refactor JIT kernel CI to use run_suite.py registration system](../sources/prs/sglang/PR-21239.md), [[misc] clean up kernel API](../sources/prs/sglang/PR-21325.md), [Lazy import flash_attention_v4 to avoid loading flash_attn.cute at startup](../sources/prs/sglang/PR-22306.md), [[perf][sgl-kernel] extend cutlass_mla_decode to support num_head < 128](../sources/prs/sglang/PR-6929.md), [[Feature] Support Flashinfer fmha on Blackwell](../sources/prs/sglang/PR-6930.md), [[Feature] Support `cp.reduce.async.bulk.tensor`](../sources/prs/tilelang/PR-1667.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [SM100 Cutlass MLA decode with unrestricted num_heads (< 128) for DeepSeek TP](../sources/prs/vllm/PR-20769.md), [[SM100] Resubmit FMHA FP8 prefill for MLA](../sources/prs/vllm/PR-31195.md), [[FA4] Update flash-attention to latest upstream FA4](../sources/prs/vllm/PR-38690.md), [FlashAttention-4](../wiki/kernels/flash-attention-4.md), [FlashAttention SM100 MLA TopK Sparse Forward](../wiki/kernels/flash-attention-sm100-mla-topk.md) | +| `fused-kernel` | [FlashInfer MLSys 2026 Track A: FP8 Block-Scale MoE](../sources/contests/flashinfer-mlsys26/track-a-fused-moe.md), [[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes](../sources/prs/DeepGEMM/PR-304.md), [perf: Enable SplitK and fix tile-scheduling for moe fp4 fused moe](../sources/prs/flashinfer/PR-1548.md), [[cute_dsl] add gemm + all reduce (two_shot)](../sources/prs/flashinfer/PR-1695.md), [A Blackwell-optimized version of selective_state_update (decode)](../sources/prs/flashinfer/PR-2387.md), [[Kernel][B200] mxfp4 fused cutlass moe](../sources/prs/vllm/PR-23696.md), [FlashInfer Track A FP8 Block-Scale MoE](../wiki/kernels/fused-moe.md), [GPU Mode NVFP4 Gated Dual GEMM](../wiki/kernels/gated-dual-gemm.md) | +| `gated-delta-net` | [Gated Delta Networks](../sources/blogs/gated-delta-net.md), [Qwen3-Next: Hybrid GDN+MoE Architecture on NVIDIA Blackwell](../sources/blogs/qwen3-next-architecture.md), [FlashInfer MLSys 2026 Track C: Gated Delta Net](../sources/contests/flashinfer-mlsys26/track-c-gated-delta-net.md), [[#11694][feat] AutoDeploy: Improve the piecewise CG memory usage](../sources/prs/TensorRT-LLM/PR-11993.md), [[None][fix] Add SPDX Apache-2.0 headers to auto_deploy test files](../sources/prs/TensorRT-LLM/PR-14193.md), [[None][fix] Update the OSS headers in derived FLA ops and AD modeling code](../sources/prs/TensorRT-LLM/PR-14281.md), [feat: add pool+indices support to gated_delta_rule_decode_pretranspose (bf16 path) ](../sources/prs/flashinfer/PR-2619.md), [feat(gdn): state checkpointing in chunk_gated_delta_rule](../sources/prs/flashinfer/PR-2908.md), [[feat] Add blackwell GDN prefill kernel](../sources/prs/flashinfer/PR-3001.md), [[XPU] Enable qwen3.5 on XPU](../sources/prs/sglang/PR-21668.md), [Gated Delta Net — Linear Attention](../wiki/kernels/gated-delta-net.md) | +| `gated-dual-gemm` | [GPU Mode NVFP4 Hackathon - Problem 3: Gated Dual GEMM](../sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md), [GPU Mode NVFP4 Gated Dual GEMM](../wiki/kernels/gated-dual-gemm.md) | +| `gemm` | [Microbenchmarking NVIDIA's Blackwell Architecture](../sources/blogs/blackwell-microbenchmarking.md), [Colfax Article Source Kernels](../sources/blogs/colfax-article-source-kernels.md), [Colfax CUTLASS Kernels](../sources/blogs/colfax-cutlass-kernels.md), [DeepGEMM — Pinned Upstream Project Summary](../sources/blogs/deepgemm.md), [Anatomy of a Reward Hack](../sources/blogs/gpu-mode-reward-hack.md), [Writing High-Performance Matrix Multiplication Kernels for Blackwell with JAX Pallas](../sources/blogs/jax-pallas-blackwell-matmul.md), [K-Search: LLM Kernel Generation via Co-Evolving Intrinsic World Model](../sources/blogs/k-search-kernel-generation.md), [Modular: Matrix Multiplication on Blackwell](../sources/blogs/modular-blackwell-matmul.md), [NVIDIA Developer Code Samples](../sources/blogs/nvidia-code-samples.md), [TFLOPS Gap: Why FP4 MoE Kernel Engineering Matters on Blackwell](../sources/blogs/tflops-gap-fp4-moe.md), [Tilus: A Tile-Level GPGPU Programming Language for Low-Precision Computation](../sources/blogs/tilus-nvidia.md), [GPU Mode NVFP4 Hackathon - Problem 2: NVFP4 GEMM](../sources/contests/gpu-mode-nvfp4/problem-2-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 3: Gated Dual GEMM](../sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 4: Grouped GEMM](../sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md), [cuTile Python DSL Reference](../sources/docs/cutile-python-dsl.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [CUTLASS 4.5.0 Cluster Launch Control Documentation](../sources/docs/cutlass-clc-documentation.md), [DeepSeek-V3 Technical Report: FP8 Training](../sources/docs/deepseek-v3-fp8.md), [Fix performance issue of m-grouped contiguous GEMMs.](../sources/prs/DeepGEMM/PR-168.md), [Fix multicast bug and optimize masked GEMM](../sources/prs/DeepGEMM/PR-193.md), [fix: use SM90ArchSpec instead of SM100ArchSpec in sm90_bf16_k_grouped_gemm](../sources/prs/DeepGEMM/PR-270.md), [[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes](../sources/prs/DeepGEMM/PR-304.md), [Sync nv_dev with upstream #316 (Mega MoE optimizations & benchmarks)](../sources/prs/DeepGEMM/PR-328.md), [Performance: reducing the percentage of FFMA interleaving yields a sight performance gain, roughly 0.5%](../sources/prs/DeepGEMM/PR-42.md), [ Solving bank conflict via padding and TMA 3D store](../sources/prs/DeepGEMM/PR-78.md), [Use 1D TMA store instead of 3D](../sources/prs/DeepGEMM/PR-83.md), [Use swizzling instead of padding](../sources/prs/DeepGEMM/PR-86.md), [Support TMA multicast on B with m_grouped_gemm_contiguous.](../sources/prs/DeepGEMM/PR-88.md), [[None][perf] Add more optimization options for MOE CuteDSL finalized kernel](../sources/prs/TensorRT-LLM/PR-10042.md), [[TRTLLM-9992][perf] Enable PDL for CuteDSL kernels and overlap MoeOutputMemset](../sources/prs/TensorRT-LLM/PR-10043.md), [[None][feat] CuteDSL MOE FC1 Enhancement](../sources/prs/TensorRT-LLM/PR-10088.md), [[TRTLLM-9457][feat] Add cute dsl fp8 gemm for Blackwell](../sources/prs/TensorRT-LLM/PR-10130.md), [[None][feat] sm100 weight-only kernel](../sources/prs/TensorRT-LLM/PR-10190.md), [[TRTLLM-9831][perf] Enable 2CTA with autotune for CuteDSL MoE and Grouped GEMM optimizations](../sources/prs/TensorRT-LLM/PR-10201.md), [[TRTLLM-9798][feat] Change to use new DeepGEMM MQA sm100 kernel for MTP-3](../sources/prs/TensorRT-LLM/PR-10226.md), [[TRTLLM-9661][chore] Further reduce tuning time for cuteDSL nvFP4 dense gemm.](../sources/prs/TensorRT-LLM/PR-10339.md), [[None] [feat] Add test script and raster M for gather fc1 kernel](../sources/prs/TensorRT-LLM/PR-10429.md), [[TRTLLM-10276][feat] Integrate cutedsl argmax kernel](../sources/prs/TensorRT-LLM/PR-10476.md), [[None] [feat] Add densegemm backend for MoE](../sources/prs/TensorRT-LLM/PR-10479.md), [[TRTLLM-9831][perf] Use TMA.RED to improve effective memory bandwidth](../sources/prs/TensorRT-LLM/PR-10987.md), [[https://nvbugs/5799917][fix] Recover from CUTLASS MoE doActivation perf regression for MXFP4/NVFP4 dtype](../sources/prs/TensorRT-LLM/PR-11165.md), [[https://nvbugs/5854860][fix] Fix cutedsl argmax on sm120](../sources/prs/TensorRT-LLM/PR-11181.md), [[None][fix] Fix int4 awq for sm120/121](../sources/prs/TensorRT-LLM/PR-11561.md), [[TRTLLM-10004][feat] Enable GEMM -> AR with GEMM output in registered buffers](../sources/prs/TensorRT-LLM/PR-11589.md), [[https://nvbugs/5799917][fix] Recover from CUTLASS MoE doActivation perf regression for MXFP4/NVFP4 dtype](../sources/prs/TensorRT-LLM/PR-11733.md), [[TRTLLM-10990][feat] Fuse SwiGLU and quant into shared expert](../sources/prs/TensorRT-LLM/PR-11897.md), [[#11694][feat] AutoDeploy: Improve the piecewise CG memory usage](../sources/prs/TensorRT-LLM/PR-11993.md), [[TRTLLM-11285][feat] Fuse indexer wk + weights_proj into single GEMM in TF32 for DS-V3.2](../sources/prs/TensorRT-LLM/PR-12055.md), [[TRTLLM-11289][feat] Integrate CuteDSL's bf16 dense GEMMs](../sources/prs/TensorRT-LLM/PR-12074.md), [[None][feat] CuteDSL MOE: Add raster along M/N support for blockscaled contiguous backbone kernel](../sources/prs/TensorRT-LLM/PR-12079.md), [[None][feat] Add DWDP (Distributed Weight Data Parallelism) support for MoE inference](../sources/prs/TensorRT-LLM/PR-12136.md), [[None][feat] Minimax RMS norm optimization](../sources/prs/TensorRT-LLM/PR-12163.md), [[None][feat] Support update weight for nvfp4](../sources/prs/TensorRT-LLM/PR-12320.md), [[TRTLLM-10407][perf] Add cute dsl single pass multi cta cluster topk](../sources/prs/TensorRT-LLM/PR-12354.md), [[None][feat] Add Mamba2 MTP SSM cache CUDA kernel for tree-based speculative decoding](../sources/prs/TensorRT-LLM/PR-12537.md), [[TRTLLM-11585][feat] Add CUTEDSL moe backend for nemotron-h](../sources/prs/TensorRT-LLM/PR-12884.md), [[#12784][feat] AutoDeploy: Optimize DeepSeek-R1 model performance](../sources/prs/TensorRT-LLM/PR-12946.md), [[None][fix] Propagate init_load_balancer to DeepGemmFusedMoE in create_moe_backend](../sources/prs/TensorRT-LLM/PR-13207.md), [[None][feat] Integrate FP4 indexer for DSA on Blackwell](../sources/prs/TensorRT-LLM/PR-13340.md), [[TRTLLM-11285][perf] Force enable TF32 tensor cores for DSA indexer fused GEMM](../sources/prs/TensorRT-LLM/PR-13452.md), [[None][feat] Fuse FP8 1x128 quantize + UE8M0 scale pack on SM100](../sources/prs/TensorRT-LLM/PR-13628.md), [[#13580][fix] AutoDeploy: Support Gemma3n/4 E2B variants](../sources/prs/TensorRT-LLM/PR-13630.md), [[https://nvbugs/6108841][fix] add hidden_dim=6144 router GEMM instantiation for GLM-5](../sources/prs/TensorRT-LLM/PR-13740.md), [[None][fix] Plumb swiglu_limit through DeepGEMM and TRTLLMGen FP8 fused MoE](../sources/prs/TensorRT-LLM/PR-13767.md), [[None][feat] Indexer topk opt](../sources/prs/TensorRT-LLM/PR-13811.md), [[None][perf] FC2 DenseGEMM autotune: split-K, swap_ab, fine-grained tuning buckets](../sources/prs/TensorRT-LLM/PR-13833.md), [[None][perf] mHC fused_hc kernel optimizations + DS-V4 entry-boundary RMSNorm fold-in](../sources/prs/TensorRT-LLM/PR-13892.md), [[None][feat] Add chunked prefill support for Gemma4 (text + vision multimodal)](../sources/prs/TensorRT-LLM/PR-14134.md), [[TRTLLM-12462][fix] Fix FP8 block scaling GEMM autotuner cache growth](../sources/prs/TensorRT-LLM/PR-14165.md), [[None][fix] Add SPDX Apache-2.0 headers to auto_deploy test files](../sources/prs/TensorRT-LLM/PR-14193.md), [[None][fix] Avoid dp_size x ep_size double-count in MegaMoEDeepGemm SymmBuffer](../sources/prs/TensorRT-LLM/PR-14213.md), [[None][fix] Update the OSS headers in derived FLA ops and AD modeling code](../sources/prs/TensorRT-LLM/PR-14281.md), [feat: Add w4a8_mxfp4_fp8 quantization recipe.](../sources/prs/TensorRT-LLM/PR-4867.md), [[OMNIML-2336][feat] Add NVFP4 x FP8](../sources/prs/TensorRT-LLM/PR-6809.md), [[None][feat] GPT-OSS Sm120/Sm121 Support](../sources/prs/TensorRT-LLM/PR-7937.md), [[None][fix] Fix the performance issue of FP8 blockwise grouped GEMM when using attention DP](../sources/prs/TensorRT-LLM/PR-8501.md), [[None][feat] Enable nvfp4 cuda core for sm120](../sources/prs/TensorRT-LLM/PR-8620.md), [[None][feat] Update TRTLLM MoE cubins; reduce mxfp4 weight padding requirement; tighten TMA bound](../sources/prs/TensorRT-LLM/PR-9025.md), [[TRTLLM-8958][feat] and [TRTLLM-8960]: create ConfigurableMoE and support TRTLLMGenFusedMoE as backend](../sources/prs/TensorRT-LLM/PR-9486.md), [[TRTLLM-9685] [feat] Add gather fc1 kernel by cuteDSL](../sources/prs/TensorRT-LLM/PR-9618.md), [[None][feat] add fp4 gemm + allreduce](../sources/prs/TensorRT-LLM/PR-9729.md), [[https://nvbugs/5726962][feat] Apply fusion for W4AFP8_AWQ MoE](../sources/prs/TensorRT-LLM/PR-9838.md), [[TRTLLM-9493][feat] Add helixPostProcessNative kernel for cp_dim=2](../sources/prs/TensorRT-LLM/PR-9924.md), [Experimental Python cooperative algorithms](../sources/prs/cccl/PR-1973.md), [fix thread-reduce performance regression](../sources/prs/cccl/PR-2944.md), [Fix scan / sm90 perf regression ](../sources/prs/cccl/PR-3236.md), [Fix the vectorized loading of BlockLoad](../sources/prs/cccl/PR-3517.md), [Fix SM100 histogram tunings](../sources/prs/cccl/PR-3691.md), [Split Optimize Warp Reduce PR - CUB part](../sources/prs/cccl/PR-4716.md), [Add nondeterministic reduce that uses atomics](../sources/prs/cccl/PR-4961.md), [CUB - Add internal integer utils and tests (Split `WarpReduce` PR)](../sources/prs/cccl/PR-5314.md), [Combine `block_reduce_warp_reduction_nondeterministic.cuh` specialization with original deterministic one ](../sources/prs/cccl/PR-5408.md), [Add dynamic CUB dispatch for segmented_sort](../sources/prs/cccl/PR-6069.md), [[CUB] Use `BlockLoadToShared` in `DeviceMerge`](../sources/prs/cccl/PR-6077.md), [Split fixed-size segmented reduce dispatch header](../sources/prs/cccl/PR-6597.md), [Integrate decoupled lookahead warpspeed scan](../sources/prs/cccl/PR-6811.md), [Use integer promotion for `warp_reduce`](../sources/prs/cccl/PR-6819.md), [Implement new tuning API arch dispatching](../sources/prs/cccl/PR-7093.md), [Two-phase reduction for fixed size segmented reduction for very large segment sizes](../sources/prs/cccl/PR-7114.md), [Implement the new tuning API for deterministic (rfa) reduce dispatch](../sources/prs/cccl/PR-7346.md), [Radix-selection based `BlockTopK` specialization](../sources/prs/cccl/PR-7384.md), [Implement the new tuning API for `DeviceRleDispatch`](../sources/prs/cccl/PR-7669.md), [Optimize non fixed size segmented reduce for small segments using max_segment_size](../sources/prs/cccl/PR-7718.md), [Add env SegmentedReduce (non fixed-size overloads)](../sources/prs/cccl/PR-7795.md), [Forward policy hub from `dispatch_streaming_arg_reduce_t` to `reduce::dispatch`](../sources/prs/cccl/PR-7805.md), [Implement the new tuning API for `detail::reduce::dispatch_streaming_arg_reduce_t`](../sources/prs/cccl/PR-7807.md), [Use the new tuning API internally for `detail::transform::dispatch`](../sources/prs/cccl/PR-7810.md), [[Backport branch/3.3.x] Forward policy hub from `dispatch_streaming_arg_reduce_t` to `reduce::dispatch`](../sources/prs/cccl/PR-7814.md), [Optimized Device-to-Device Tensor Copy (`cudax`)](../sources/prs/cccl/PR-7823.md), [Implement the new tuning API for `DispatchSegmentedRadixSort`](../sources/prs/cccl/PR-7844.md), [Implement the new tuning API for `DispatchSegmentedSort`](../sources/prs/cccl/PR-7874.md), [Implement the new tuning API for `DispatchTopK`](../sources/prs/cccl/PR-7928.md), [[cuda.compute]: Fix faulty pointer arithmetic calculation in CUB dispatch](../sources/prs/cccl/PR-7940.md), [Reduce usage of `cub::DispatchReduce`](../sources/prs/cccl/PR-7944.md), [Use the new tuning API for `detail::radix_sort::dispatch`](../sources/prs/cccl/PR-7949.md), [Adds support for non-fundamental types via decomposer to `DeviceTopK` ](../sources/prs/cccl/PR-8040.md), [Optimized Device-to-Device Tensor Copy (cudax) - Transpose Case](../sources/prs/cccl/PR-8125.md), [Avoid passing uninitialized values to scan_op](../sources/prs/cccl/PR-8184.md), [[STF] Move unstable_unique from STF to generic cudax utility](../sources/prs/cccl/PR-8190.md), [Expose `max_segment_size` guarantee in cuda.compute](../sources/prs/cccl/PR-8284.md), [Port `thrust::min|max_element` to CUB](../sources/prs/cccl/PR-8291.md), [Implement the new tuning API for `DispatchSelectIf`](../sources/prs/cccl/PR-8311.md), [simplify dispatch segmented reduce to use latest dispatch and new tunings API](../sources/prs/cccl/PR-8332.md), [Apply some random warpspeed tunings](../sources/prs/cccl/PR-8352.md), [[cub]: implement utilities for policy selection](../sources/prs/cccl/PR-8355.md), [Replace `detail::merge::dispatch` by CUB's public API](../sources/prs/cccl/PR-8381.md), [[CUB] Replace `Shuffle(Up|Down|Index)` with cuda::device::warp_shuffle - RadixSort only](../sources/prs/cccl/PR-8395.md), [[thrust] Single-pass `is_partitioned` via adjacent zip_iterator](../sources/prs/cccl/PR-8427.md), [Replace `detail::merge_sort::dispatch` by CUB's public API](../sources/prs/cccl/PR-8473.md), [Replace `detail::scan::dispatch` by CUB's public API](../sources/prs/cccl/PR-8495.md), [Implement the new tuning API for `detail::batched_topk::dispatch_batched_topk`](../sources/prs/cccl/PR-8538.md), [Replace `detail::for_each::dispatch` by CUB's public API](../sources/prs/cccl/PR-8565.md), [Replace `detail::segmented_reduce::dispatch` by the public API](../sources/prs/cccl/PR-8695.md), [Use the new tuning API internally for `detail::topk::dispatch`](../sources/prs/cccl/PR-8742.md), [Use the new tuning API internally for `detail::reduce_by_key::dispatch`](../sources/prs/cccl/PR-8756.md), [Use the new tuning API internally for `detail::reduce[_nd]::dispatch[_nd]`](../sources/prs/cccl/PR-8826.md), [Fix Warpspeed scan shifted output store](../sources/prs/cccl/PR-8839.md), [[cub] Simplify arch dispatch](../sources/prs/cccl/PR-8861.md), [Use the new tuning API internally for `detail::select::dispatch` and `DeviceSelect`](../sources/prs/cccl/PR-8880.md), [[STF] Add per-handle exec_place stream resources](../sources/prs/cccl/PR-8905.md), [Use the new tuning API internally for `detail::select|three_way_partition::dispatch` and `DevicePartition`](../sources/prs/cccl/PR-8925.md), [Use the new tuning API internally for `detail::segmented_radix_sort::dispatch`](../sources/prs/cccl/PR-8927.md), [Fix segmented radix sort benchmark segment size type](../sources/prs/cccl/PR-9039.md), [[libcu++] Fix default make_shared_resource construction](../sources/prs/cccl/PR-9044.md), [Vectorize contiguous iterators in `cub::BlockLoad`/`Store`](../sources/prs/cccl/PR-9056.md), [Improve sm90 mixed dtype kernel](../sources/prs/cutlass/PR-1883.md), [Groupwise scaling along M for FP8 gemm](../sources/prs/cutlass/PR-2037.md), [Improvements for: Groupwise scaling along M for FP8 gemm](../sources/prs/cutlass/PR-2095.md), [Hopper Grouped GEMM support for FP8 Accum](../sources/prs/cutlass/PR-2123.md), [Blockwise and Groupwise GEMM for Blackwell and Improvements for Hopper](../sources/prs/cutlass/PR-2139.md), [Blockwise Improvement and Programmatic Dependent Launch](../sources/prs/cutlass/PR-2161.md), [Fix sm100 gemm wrong static constexpr that breaks compilation on Windows](../sources/prs/cutlass/PR-2167.md), [Fix SM90 beta=1 hang and stream-K launch errors](../sources/prs/cutlass/PR-2172.md), [Use cudaMemcpyAsync in gemm grouped with kRequiresPrecomputation sche…](../sources/prs/cutlass/PR-2256.md), [war to fix blackwell grouped groupwise hang](../sources/prs/cutlass/PR-2267.md), [hopper-blockwise-generalization-optimization](../sources/prs/cutlass/PR-2270.md), [support fp16 accmulator for sm89 fp8 mma](../sources/prs/cutlass/PR-2378.md), [DistGEMM bug fixes](../sources/prs/cutlass/PR-2713.md), [Support PDL for SM90 Array TMA GEMM](../sources/prs/cutlass/PR-2719.md), [Support for GEMM-K=0 for Blackwell Grouped GEMMs](../sources/prs/cutlass/PR-2746.md), [Add tutorial fp16_gemm_1](../sources/prs/cutlass/PR-2750.md), [Blockscaled Ragged Contiguous Grouped Gemm for MoEs](../sources/prs/cutlass/PR-2790.md), [[Bug Fix]Bypass launch grids for SM120 Kernel with SM90 Mainloop & SM100 TileScheduler](../sources/prs/cutlass/PR-2865.md), [[cute] Add constexpr specifier to make_tiled_copy](../sources/prs/cutlass/PR-2875.md), [new example with TMA prefetch feature targeting for DRAM latency boun…](../sources/prs/cutlass/PR-2881.md), [Fix incorrect tensor layout strides in Blackwell MMA tutorial comments](../sources/prs/cutlass/PR-2921.md), [[Cutlass gemm] Fix SM100 FP8 nosmem epilogue-fusion shape_div 'Divisibility Condition' for non-multiple-of-64 N tiles](../sources/prs/cutlass/PR-2946.md), [[Bug Fix]Set NumSplitsM to 1 when TileShapeM < 128 in sm90 fp8 blockwise scaling CollectiveMma](../sources/prs/cutlass/PR-2965.md), [[CuTeDSL] Fix: SM100 block-scale gemm overlapping accumulator](../sources/prs/cutlass/PR-2995.md), [Replace std::min with cute::min in sm120 blockwise scaling device functions](../sources/prs/cutlass/PR-3055.md), [[Hopper CuTeDSL] Add grouped GEMM kernel example](../sources/prs/cutlass/PR-3091.md), [Support for Group GEMM in CUTLASS Profiler for GeForce and Spark](../sources/prs/cutlass/PR-3092.md), [[CLI] add cutedsl fp16 gemm tutorial from 2 to 6](../sources/prs/cutlass/PR-3106.md), [Update blackwell tutorial to be compatible with 4.5-dev version](../sources/prs/cutlass/PR-3130.md), [Small Tile N BlockScaled GEMM + Grouped GEMM on SM12x](../sources/prs/cutlass/PR-3176.md), [feat: Adding varlen support to cute-dsl sm80 bwd](../sources/prs/flash-attention/PR-1934.md), [[Cute] Block sparse support Sm100](../sources/prs/flash-attention/PR-1985.md), [[Cute,Fwd,Sm100] Support `q_stage=1` for inference](../sources/prs/flash-attention/PR-1993.md), [Add blocksparse support for bwd on blackwell](../sources/prs/flash-attention/PR-2085.md), [Fix IMA in fwd on m boundary](../sources/prs/flash-attention/PR-2091.md), [Add pack-gqa fwd support for sparse impl w/ broadcasted H dim](../sources/prs/flash-attention/PR-2098.md), [[Cute,Fwd,Sm100] distributed offset calculation for paged KV](../sources/prs/flash-attention/PR-2104.md), [[NVIDIA] Enable Jetson Thor FA4](../sources/prs/flash-attention/PR-2108.md), [[CUTE][SM90]Enable pack-gqa with broadcasted maskmods](../sources/prs/flash-attention/PR-2145.md), [[Cute][Flex]Add pack-gqa divmod](../sources/prs/flash-attention/PR-2180.md), [[Cute,Fwd,Sm100] support irregular qhead / kvhead ratios](../sources/prs/flash-attention/PR-2186.md), [[Cute,Flex,Fwd] Allow vectorized score_mod definitions](../sources/prs/flash-attention/PR-2236.md), [[Bwd,Sm120] Add SM120 backward pass support](../sources/prs/flash-attention/PR-2330.md), [Fix ZeroDivisionError in num_splits_heuristic for empty Q workloads](../sources/prs/flash-attention/PR-2515.md), [feat: ragged tensor padding kernel for blackwell kernel alignment](../sources/prs/flashinfer/PR-1025.md), [fix: top_k_mask_logits hangs on -inf inputs](../sources/prs/flashinfer/PR-1050.md), [perf: accelerate blackwell grouped gemm](../sources/prs/flashinfer/PR-1086.md), [comm: refactor and initialize `flashinfer.comm` module](../sources/prs/flashinfer/PR-1089.md), [bugfix: host-precomuted plan function for blackwell fmha](../sources/prs/flashinfer/PR-1106.md), [Add CUTLASS fused moe kernels from TensorRT-LLM.](../sources/prs/flashinfer/PR-1113.md), [bugfix: Fix test and output shape of fp4 quantize](../sources/prs/flashinfer/PR-1114.md), [Fix pointer dtype bug in rope](../sources/prs/flashinfer/PR-1129.md), [fix: negative zero by type trait --> binary value](../sources/prs/flashinfer/PR-1136.md), [feat: nvshmem python bindings](../sources/prs/flashinfer/PR-1160.md), [Add DeepGEMM kernels](../sources/prs/flashinfer/PR-1209.md), [Fix test_groupwise_scaled_gemm_fp8.py](../sources/prs/flashinfer/PR-1211.md), [feat: trtllm-gen fp8 moe kernels](../sources/prs/flashinfer/PR-1212.md), [[comm] TRT-LLM's Multi-Node NVLink All-Reduce Kernel](../sources/prs/flashinfer/PR-1213.md), [Feature/sm100 low latency nvfp4 kernels](../sources/prs/flashinfer/PR-1214.md), [Fix missing hash in the cudnn cubin path](../sources/prs/flashinfer/PR-1227.md), [bugfix: support uint8_t for vec_t class template](../sources/prs/flashinfer/PR-1234.md), [Patch fp8 cubin availability](../sources/prs/flashinfer/PR-1240.md), [feat: Support MXFP8 x MXFP4 CUTLASS grouped GEMM](../sources/prs/flashinfer/PR-1241.md), [Reduce the JIT compilation time of gen_gemm_sm100_module](../sources/prs/flashinfer/PR-1251.md), [init add gemm fp8 using cudnn backend](../sources/prs/flashinfer/PR-1264.md), [Made AR output optional + esthetic changes](../sources/prs/flashinfer/PR-1265.md), [feat: add masked deepgemm support and benchmarking](../sources/prs/flashinfer/PR-1266.md), [Add shuffle matrix flag](../sources/prs/flashinfer/PR-1272.md), [hotfix: fix deepgemm artifactory hash](../sources/prs/flashinfer/PR-1278.md), [Unify groupwise fp8 GEMM test](../sources/prs/flashinfer/PR-1281.md), [add mm_fp4 use cudnn backend](../sources/prs/flashinfer/PR-1288.md), [Remove FAST_BUILD FLAG for MOE](../sources/prs/flashinfer/PR-1291.md), [Update cutlass fp4 moe kernels](../sources/prs/flashinfer/PR-1294.md), [add cutlass backend for mm_fp4](../sources/prs/flashinfer/PR-1296.md), [feat: Add weight layout option for trtllm-gen fused moe](../sources/prs/flashinfer/PR-1297.md), [[Feature] SM level profiler ](../sources/prs/flashinfer/PR-1305.md), [Refactor Fused Moe Module](../sources/prs/flashinfer/PR-1309.md), [Add blockwise-scaled FP8 GEMM via TRTLLM-Gen.](../sources/prs/flashinfer/PR-1320.md), [Optimizations for TRTLLM MNNVL Allreduce](../sources/prs/flashinfer/PR-1321.md), [refactor: Improved metainfo for trtllm-gen kernels](../sources/prs/flashinfer/PR-1328.md), [feat: masked layout fp4 gemm using cute-dsl](../sources/prs/flashinfer/PR-1331.md), [add torch float4_e2m1fn_x2 check for cudnn fp4 backend](../sources/prs/flashinfer/PR-1333.md), [[Fix] remove torch 2.8 requirement for FP4 GEMM](../sources/prs/flashinfer/PR-1334.md), [Fix bench deepgemm setting](../sources/prs/flashinfer/PR-1344.md), [feature: add fp4 mm using trtllm backend](../sources/prs/flashinfer/PR-1355.md), [[fix] remove (view) transpose to keep consistent with majorness MN requirement.](../sources/prs/flashinfer/PR-1358.md), [hotfix: update mxfp4 groupwise-scaled gemm unittests](../sources/prs/flashinfer/PR-1359.md), [refactor: download trtllm gemm metadata from server](../sources/prs/flashinfer/PR-1378.md), [GPT-OSS Support: Add Blackwell MoE mxfp4 implementation from TRTLLM and Attention Sink](../sources/prs/flashinfer/PR-1389.md), [Adding FP8 benchmark on attention and matmul testing](../sources/prs/flashinfer/PR-1390.md), [gpt-oss: Add MXFP8 x MXFP4 CUTLASS MOE for SM100 and BF16 x MXFP4 CUTLASS for SM90 + SwigluBias Activation](../sources/prs/flashinfer/PR-1396.md), [feature: add cutlass as bmm_fp8 backend.](../sources/prs/flashinfer/PR-1397.md), [fix shared memory alignment conflict in sampling.cuh](../sources/prs/flashinfer/PR-1402.md), [feature: enable cublas for fp4 gemm when cudnn == 9.11.1 or >= 9.13](../sources/prs/flashinfer/PR-1405.md), [Fixes for Blackwell Tests](../sources/prs/flashinfer/PR-1434.md), [bugfix: fix perf issue by using fp8 graph that can use cublaslt](../sources/prs/flashinfer/PR-1435.md), [Remove getEnvEnablePDL in favor of enable_pdl parameter](../sources/prs/flashinfer/PR-1446.md), [perf: add 1x4x1 cluster shape for fp8 bmm M<16 cases](../sources/prs/flashinfer/PR-1473.md), [tuner: Trtllm-gen Fp4 MoE Autotunner](../sources/prs/flashinfer/PR-1475.md), [refactor: unify autotuner for bmm_fp8](../sources/prs/flashinfer/PR-1479.md), [Add python API for masked grouped gemm](../sources/prs/flashinfer/PR-1481.md), [perf: add fast path to TopPRenormProbKernel for top_p >= 1.0, significantly boosting SGLang workloads](../sources/prs/flashinfer/PR-1483.md), [fix: update cutedsl masked moe gemm](../sources/prs/flashinfer/PR-1488.md), [Perf: support scale_a/scale_b instead of combined scale in cutlass bmm_fp8](../sources/prs/flashinfer/PR-1491.md), [fix: update masked moe gemm fp4 tensor reshape](../sources/prs/flashinfer/PR-1495.md), [feat: scaling at fp4 gemm epilogue](../sources/prs/flashinfer/PR-1498.md), [fix: Replace cub Max/Min with cuda::maximum/minimum for cuda 13 compatibility](../sources/prs/flashinfer/PR-1500.md), [Add benchmark for cutedsl gemm](../sources/prs/flashinfer/PR-1502.md), [bugfix: Fix stream handling in cutedsl gemm](../sources/prs/flashinfer/PR-1509.md), [flashinfer_benchmark QoL Improvements and Attention FP8 Support](../sources/prs/flashinfer/PR-1512.md), [refactor fp4 masked gemm cute-dsl implementation and add manual cache](../sources/prs/flashinfer/PR-1521.md), [Fix linking errors with CUDA 13](../sources/prs/flashinfer/PR-1523.md), [Add GeGLU support to trtllm-gen NVFP4 Fused MoE Kernel](../sources/prs/flashinfer/PR-1525.md), [Remove cuda-python from dependency and check at runtime](../sources/prs/flashinfer/PR-1534.md), [Add sm check for sm100 only cutlass/trtllm kernel](../sources/prs/flashinfer/PR-1535.md), [perf: Enable SplitK and fix tile-scheduling for moe fp4 fused moe](../sources/prs/flashinfer/PR-1548.md), [bugfix: update trtllm-gen gemm kernel names](../sources/prs/flashinfer/PR-1577.md), [bugfix: Fix arg passing to TORCH_CHECK and TORCH_WARN macros](../sources/prs/flashinfer/PR-1582.md), [fix: limit the number of nvcc threads for each kernel](../sources/prs/flashinfer/PR-1589.md), [bugfix: fix the register overflow issue for topk renorm kernels on blackwell](../sources/prs/flashinfer/PR-1597.md), [feat: Enable MnnvlMemory (for alltoallv) on B200](../sources/prs/flashinfer/PR-1601.md), [feat: initial support for SM103, SM110, SM120, SM121](../sources/prs/flashinfer/PR-1608.md), [feat: cutlass fp4 gemm bringup for SM120 & SM121](../sources/prs/flashinfer/PR-1609.md), [feat: cutlass fp8 gemm bringup for SM120 & SM121](../sources/prs/flashinfer/PR-1610.md), [perf: Fix the tactic sorting in TrtllmGenBatchedGemmRunner::getValidConfigIndices](../sources/prs/flashinfer/PR-1615.md), [bugfix: collect all modules to aot](../sources/prs/flashinfer/PR-1622.md), [test: pytest.mark.xfail on deepgemm](../sources/prs/flashinfer/PR-1636.md), [Added mx_fp4 support using the cudnn backend](../sources/prs/flashinfer/PR-1644.md), [Refactor Blackwell unit test scripts](../sources/prs/flashinfer/PR-1667.md), [TGV GEMM as a BF16 backend alternative to cuBLAS](../sources/prs/flashinfer/PR-1668.md), [Support output signals for overlapping for cutedsl gemm](../sources/prs/flashinfer/PR-1677.md), [Update TGV GEMM default kernel and TGV code cleanup.](../sources/prs/flashinfer/PR-1682.md), [Update deepgemm backend for 103a](../sources/prs/flashinfer/PR-1694.md), [[cute_dsl] add gemm + all reduce (two_shot)](../sources/prs/flashinfer/PR-1695.md), [hotfix: Hotfix for `test_pod_kernels.py` on B300](../sources/prs/flashinfer/PR-1698.md), [feat: Benchmark mm_fp4 mxfp4 support and gemm autotune support. Restore mm_fp4 API behavior](../sources/prs/flashinfer/PR-1706.md), [test: skip the unsupported test cases for sm120/121](../sources/prs/flashinfer/PR-1710.md), [TVM: support TVM binding for GroupedGemm](../sources/prs/flashinfer/PR-1725.md), [fix: put sampling kernel launch into macro](../sources/prs/flashinfer/PR-1727.md), [fix: fix cannot import name 'cuda' from 'cuda' in CUDA13](../sources/prs/flashinfer/PR-1764.md), [Added xfail for mx_fp4 matmul on SM120](../sources/prs/flashinfer/PR-1766.md), [tests: skip non SM100/103 for grouped deepgemm](../sources/prs/flashinfer/PR-1767.md), [Support checks PoC](../sources/prs/flashinfer/PR-1809.md), [tests: Update support for tgv_gemm to SM100 only and add to ut](../sources/prs/flashinfer/PR-1810.md), [tests: upgrade cutlass, fix import and skip non-SM100 cutedsl two shot allreduce](../sources/prs/flashinfer/PR-1812.md), [feat:enable fp8 blockscale moe for fused cultass for sm90](../sources/prs/flashinfer/PR-1819.md), [feat: trtrllm-gen global scaled FP8 GEMMs](../sources/prs/flashinfer/PR-1829.md), [raise error for group_gemm_fp8_nt_groupwise then num_groups > 1 on sm120/121](../sources/prs/flashinfer/PR-1862.md), [feat: Add FP4 TRTLLM-Gen throughput MOE batched gemms](../sources/prs/flashinfer/PR-1882.md), [misc: fix some B200 GEMM bench](../sources/prs/flashinfer/PR-1883.md), [Add layernorm op for inputs of mixed dtype](../sources/prs/flashinfer/PR-1926.md), [Feature: Support Relu2 activation in fused MoE](../sources/prs/flashinfer/PR-1954.md), [Update trtllm-gen fused moe routing kernel and add more kernels](../sources/prs/flashinfer/PR-1955.md), [fix: ensure SM120/121 SFA/SFB contiguity](../sources/prs/flashinfer/PR-1963.md), [feat: enable deepgemm jit for fp8 block-scale on SM90](../sources/prs/flashinfer/PR-1969.md), [Feature: Add support for L40 FusedMoE in cutlass path](../sources/prs/flashinfer/PR-1973.md), [feat: Add backend='auto' to mm_fp4 and enable autotune for backend='cudnn'](../sources/prs/flashinfer/PR-1979.md), [fix: Enable SM121 for mm_fp4](../sources/prs/flashinfer/PR-2012.md), [[feat] Refactor trtllmgen MOE and add Bf16 trtllmgen moe](../sources/prs/flashinfer/PR-2014.md), [[DSV3] Optimized Router Gemm](../sources/prs/flashinfer/PR-2019.md), [update trtllm cutlass moe ](../sources/prs/flashinfer/PR-2020.md), [[NVIDIA] Thor & Spark Support](../sources/prs/flashinfer/PR-2028.md), [feat: suitable_auto_backends to prune auto backends, bmm_fp8 refactor, heuristic_func intake](../sources/prs/flashinfer/PR-2029.md), [perf: Optimize helper max/minmax function in sampling.cuh](../sources/prs/flashinfer/PR-2058.md), [feat: BF16 GEMM using CUTLASS backend for SM100](../sources/prs/flashinfer/PR-2070.md), [refactor: pass hopper deepgemm include directory through python](../sources/prs/flashinfer/PR-2090.md), [perf: enable pdl for cutlass fp4 gemm](../sources/prs/flashinfer/PR-2095.md), [Refactor trtllm_mnnvl_allreduce](../sources/prs/flashinfer/PR-2118.md), [make DeepGEMM swapAB available for linear gemm SM90](../sources/prs/flashinfer/PR-2131.md), [feat: TRTLLM FMHAv2 backend for ctx attention](../sources/prs/flashinfer/PR-2142.md), [enable sm103 moe dsl backend](../sources/prs/flashinfer/PR-2149.md), [fix xqa mha_sm90.cu](../sources/prs/flashinfer/PR-2157.md), [feat: MxInt4 x Bf16 TRT-LLM Gen MoE support](../sources/prs/flashinfer/PR-2159.md), [Fix gemm allreduce two shot](../sources/prs/flashinfer/PR-2171.md), [Fix for moe on sm110](../sources/prs/flashinfer/PR-2190.md), [feat: unit-test and api change, w4a8 grouped-gemm fused MoE for SM90](../sources/prs/flashinfer/PR-2193.md), [Move the run function definition out of BatchedGemmInterface](../sources/prs/flashinfer/PR-2211.md), [misc: support checks for gemm](../sources/prs/flashinfer/PR-2214.md), [refactor: pull trtllm-gen batch-gemm/gemm headers from artifactory; update tma descriptor shape init](../sources/prs/flashinfer/PR-2235.md), [Remove cudaStreamSynchronize from gemm_groupwise_sm120.cuh for CUDA graph compatibility](../sources/prs/flashinfer/PR-2244.md), [feat: Add support for bmm mxfp8](../sources/prs/flashinfer/PR-2256.md), [Fix CUTLASS FP8 gemm correctness issue on SM120/SM121 for shapes where N is not divisible by ScaleGranularityN.](../sources/prs/flashinfer/PR-2261.md), [test: use .float() in in F.cosine_similarity() in bmm_fp8 test](../sources/prs/flashinfer/PR-2266.md), [Tiny fix bench tgv gemm](../sources/prs/flashinfer/PR-2277.md), [feat: IdType indices in sampling kernels](../sources/prs/flashinfer/PR-2281.md), [Selective State Update kernel (mamba)](../sources/prs/flashinfer/PR-2301.md), [[Perf][Feature] Add SM103-specific schedulers for NVFP4 CUTLASS kernels](../sources/prs/flashinfer/PR-2303.md), [feat: Support Fused MoE non gated Relu2 NVFP4 & FP8 and support Nemotron](../sources/prs/flashinfer/PR-2304.md), [Fix: FilteredTopKUnifiedKernel read value out of length](../sources/prs/flashinfer/PR-2308.md), [[ML3] Optimized Router Gemm](../sources/prs/flashinfer/PR-2323.md), [bugfix: fix multi-cta top-k implementation when k value is different for different row](../sources/prs/flashinfer/PR-2325.md), [[perf] Improve gemm_fp8_nt_groupwise (cutlass backend) by 10-40% for batch sizes <= 32](../sources/prs/flashinfer/PR-2327.md), [Enable fp16/bf16/f32 support for selective_state_update (mamba)](../sources/prs/flashinfer/PR-2366.md), [feat: BF16 GEMM using cuDNN backend](../sources/prs/flashinfer/PR-2376.md), [bugfix: hotfix of PR 2366 (mamba kernel)](../sources/prs/flashinfer/PR-2378.md), [feat: cuteDSL fp4 moe for better DSR1 performance.](../sources/prs/flashinfer/PR-2398.md), [perf: mm_fp4 heuristic prioritizes CUTLASS over cuDNN on SM103](../sources/prs/flashinfer/PR-2404.md), [feat: update trtllm-gen MoE cubins](../sources/prs/flashinfer/PR-2416.md), [fix: Sampling: CUDA Graph fix](../sources/prs/flashinfer/PR-2432.md), [MTP for mamba ](../sources/prs/flashinfer/PR-2444.md), [fix: fix illegal memory access for NaN input in sampling kernels](../sources/prs/flashinfer/PR-2456.md), [perf: add fp4 GEMM tile configs and streamK scheduler for SM120](../sources/prs/flashinfer/PR-2460.md), [feat: Support Fused MoE non gated Relu2 NVFP4 & FP8 and support Nemotron, fixed](../sources/prs/flashinfer/PR-2462.md), [feat: Add MXFP8 GEMM mm_mxfp8 (cutlass)](../sources/prs/flashinfer/PR-2464.md), [fix: add support check for gemm config for cutlass moe](../sources/prs/flashinfer/PR-2495.md), [refactor: Port upstream CUTLASS fixes and refactor grouped_gemm_nt_masked GEMM module location](../sources/prs/flashinfer/PR-2503.md), [Feat: Trtllm-gen MxFP8 MoE integration](../sources/prs/flashinfer/PR-2505.md), [feat: BF16 GEMM benchmarking support](../sources/prs/flashinfer/PR-2525.md), [fix: include fp8_blockscale_gemm_90 in AOT jit-cache](../sources/prs/flashinfer/PR-2533.md), [tests: bmm_fp8 for SM110](../sources/prs/flashinfer/PR-2538.md), [feat: cute dsl mmfp4 for blackwell](../sources/prs/flashinfer/PR-2540.md), [misc: point triton blackwell-ptxas to local cuda ptxas](../sources/prs/flashinfer/PR-2543.md), [Add gen_gemm_sm100_module_cutlass_mxfp8 to jit-cache](../sources/prs/flashinfer/PR-2549.md), [Add support for the combinations of allreduce, allgather, and reducescatter](../sources/prs/flashinfer/PR-2563.md), [feat: add is_sm12x_supported() helper for SM12x family detection](../sources/prs/flashinfer/PR-2574.md), [Implement `cutlass_fused_moe` mxfp8](../sources/prs/flashinfer/PR-2581.md), [feat: trtllm tinygemm2 in flashinfer as bf16 routergemm](../sources/prs/flashinfer/PR-2587.md), [Mamba SSU: better automatic kernel selection + algorithm selection optionally exposed to the user.](../sources/prs/flashinfer/PR-2591.md), [[bugfix] Fix FilteredTopK overflow correctness](../sources/prs/flashinfer/PR-2605.md), [fix: add SM121 support to SM120 version guards](../sources/prs/flashinfer/PR-2631.md), [feat: FP32 dtype output for BF16 matmuls (CUTLASS & cuDNN)](../sources/prs/flashinfer/PR-2644.md), [[feat] trtllm-gen mxfp8 gemm](../sources/prs/flashinfer/PR-2653.md), [fix: Add fused MOE and GEMM AOT modules for SM121](../sources/prs/flashinfer/PR-2654.md), [feat: support mxfp4 & mxfp8 entrypoint for blackwell cutedsl dense gemm](../sources/prs/flashinfer/PR-2660.md), [feat: implement deterministic topk](../sources/prs/flashinfer/PR-2661.md), [perf: Update trtllm-gen batched GEMM kernels - faster, more NVFP4 tile dims, MXFP8 with relu2 act](../sources/prs/flashinfer/PR-2667.md), [fix: reduce smem allocation for tinygemm2 kernel in SM120](../sources/prs/flashinfer/PR-2670.md), [Add varlen and speculative decoding support to selective state update](../sources/prs/flashinfer/PR-2700.md), [fix(jit): GEMM kernels produce NaN under concurrency — missing GDC flags cause PDL synchronization barriers to compile as no-ops](../sources/prs/flashinfer/PR-2716.md), [Support for MXFP4 and NVFP4 group GEMMs on GeForce and Spark](../sources/prs/flashinfer/PR-2738.md), [misc: Update gemm/batched gemm cubins from trtllm-gen, gemm header refactor](../sources/prs/flashinfer/PR-2740.md), [[Spark unit test debugging] Fix for tests/gemm/test_groupwise_scaled_gemm_fp8.py](../sources/prs/flashinfer/PR-2751.md), [[feat] Add air top-p algorithm](../sources/prs/flashinfer/PR-2752.md), [fix(jit): enable GDC for CUTLASS GEMM PDL — SM100 flag only](../sources/prs/flashinfer/PR-2780.md), [Upgrade cutlass 4.2.1 -> 4.4.2](../sources/prs/flashinfer/PR-2798.md), [CuteDSL MoE fix redundant output buffer zeroing](../sources/prs/flashinfer/PR-2811.md), [Add support for Relu2 in BF16 fused MoE](../sources/prs/flashinfer/PR-2864.md), [Mamba SSU: horizontal MTP kernel (+ DSTATE=96 support)](../sources/prs/flashinfer/PR-2865.md), [fix: fix cute dsl swap_ab tactic failure](../sources/prs/flashinfer/PR-2870.md), [feat: add MXFP8 GEMM support for SM120](../sources/prs/flashinfer/PR-2902.md), [[NVIDIA] fix(jit): enable GDC for CUTLASS fused MoE PDL — prevent random crashes on SM12x](../sources/prs/flashinfer/PR-2913.md), [feat: Add cuBLASLt backend for `mm_bf16` and enable multi-tactic autotuning for FP8/MXFP8 runners](../sources/prs/flashinfer/PR-2914.md), [feat: add Relu2 (squared ReLU) activation support in CUTLASS MoE backend](../sources/prs/flashinfer/PR-2926.md), [feat: SM121 (GB10) tile filtering and autotuner robustness](../sources/prs/flashinfer/PR-2927.md), [CuTe DSL FP4 GEMM Heuristic](../sources/prs/flashinfer/PR-2940.md), [feat: Add CuTe DSL grouped-gemm + combine fusion support](../sources/prs/flashinfer/PR-2944.md), [fix: use float instead of double in sampling binary search to avoid FP64 bottleneck on SM103](../sources/prs/flashinfer/PR-2945.md), [Update NVSHMEM interface to use NVSHMEM4Py instead of custom bindings](../sources/prs/flashinfer/PR-2960.md), [Improved `simple` mamba SSU kernel ](../sources/prs/flashinfer/PR-2962.md), [test: skip unsupported mm_mxfp8 configurations on SM12x](../sources/prs/flashinfer/PR-2974.md), [fix: tinygemm2 hang issue due to barrier sync](../sources/prs/flashinfer/PR-2996.md), [[chore] Install nvidia-cutlass-dsl[cu13] for cu130+](../sources/prs/flashinfer/PR-3017.md), [perf: Port TRT-LLM SM120/SM121 FP4 CUTLASS GEMM optimizations. Add PDL](../sources/prs/flashinfer/PR-3026.md), [[feat] Trtllm-gen Per-token Nvfp4 MoE](../sources/prs/flashinfer/PR-3027.md), [fused_moe: pre-filter SM89 tactics with zero occupancy on SM120 Blackwell (fix review feedback on #2764)](../sources/prs/flashinfer/PR-3032.md), [feat: Add backend="b12x" for mm_fp4 on SM120](../sources/prs/flashinfer/PR-3051.md), [perf: Add no-bias path for tinygemm_bf16](../sources/prs/flashinfer/PR-3151.md), [Integrate CUTLASS Small Tile N Blockscaled GEMMs/Grouped GEMMs for SM120 and SM121](../sources/prs/flashinfer/PR-3152.md), [Fix/3170 dense blockscaled sm12x](../sources/prs/flashinfer/PR-3180.md), [test: enable bmm_mxfp8 cutlass backend coverage on SM12x](../sources/prs/flashinfer/PR-3183.md), [feat: enable glm5 router gemm](../sources/prs/flashinfer/PR-3185.md), [Include TinyGEMM into BF16 autotuner](../sources/prs/flashinfer/PR-3203.md), [feat(trace): embed runnable init() in every TraceTemplate](../sources/prs/flashinfer/PR-3221.md), [[Bugfix] Fix fused MoE autotuning correctness issues by filtering clusterDimZ](../sources/prs/flashinfer/PR-3227.md), [Update moe gemm](../sources/prs/flashinfer/PR-3239.md), [[feat] Add gemma RMS AR fusion](../sources/prs/flashinfer/PR-3322.md), [Change `apply_rope_with_cos_sin_cache` to accept `cos_sin_cache`](../sources/prs/flashinfer/PR-754.md), [bugfix: Ensure Loop Termination by Enforcing IEEE-754 Compliance in Sampling Kernels](../sources/prs/flashinfer/PR-774.md), [bugfix: fix the signature of `CutlassSegmentGEMMSM90`](../sources/prs/flashinfer/PR-827.md), [feat: experimenta support of PDL](../sources/prs/flashinfer/PR-930.md), [perf: reduce torch.library dispatch overhead](../sources/prs/flashinfer/PR-968.md), [perf: Fix python API overhead when CUDAGraph is not enabled](../sources/prs/flashinfer/PR-969.md), [perf: dual pivot top-p/top-k renorm](../sources/prs/flashinfer/PR-974.md), [SM-constraint-GEMM by triton persistent kernel](../sources/prs/flashinfer/PR-982.md), [Triton `rms_norm` kernels](../sources/prs/flashinfer/PR-983.md), [feat: SM-constraint Communication Kernels](../sources/prs/flashinfer/PR-994.md), [Update torch-xpu-ops commit pin](../sources/prs/pytorch/PR-144209.md), [[inductor][cpu] Fix bmm b_index for dynamic expressions in inductor autotuner](../sources/prs/pytorch/PR-144248.md), [Fix PythonMod printing](../sources/prs/pytorch/PR-144335.md), [ROCm SDPA: Ensure attn_mask has the same dtype with q](../sources/prs/pytorch/PR-144398.md), [[inductor] Fix profiler tests with latest Triton](../sources/prs/pytorch/PR-149059.md), [Remove runtime dependency on packaging](../sources/prs/pytorch/PR-149125.md), [op should NOT be static in aoti_torch_call_dispatcher](../sources/prs/pytorch/PR-149644.md), [Add release branch push triggers to inductor-rocm-mi300.yml](../sources/prs/pytorch/PR-149871.md), [[inductor] Fix inductor windows linker error](../sources/prs/pytorch/PR-150447.md), [[Windows][inductor] fix blank space break windows file path](../sources/prs/pytorch/PR-150448.md), [[CUDA][avgpool2d] Fix backward launch bounds again for `sm100`, `sm120`](../sources/prs/pytorch/PR-150676.md), [[dynamo][super variable] Fix bug to use correct source](../sources/prs/pytorch/PR-152774.md), [[ATen][CUDA] Optimize 128 bit vectorization](../sources/prs/pytorch/PR-152967.md), [Mark auto_functionalized HOPs as cacheable (#151194)](../sources/prs/pytorch/PR-153304.md), [Fix uint view copy (#151598)](../sources/prs/pytorch/PR-154121.md), [Fix macOS build with `USE_MPS=OFF`](../sources/prs/pytorch/PR-156932.md), [[PowerPC] Fixed build issue for vsx vec256 complexfloat and scaled_mm_out_cpu ](../sources/prs/pytorch/PR-157422.md), [[release] Triton pin update to 3.4](../sources/prs/pytorch/PR-157752.md), [[MPS] Switch Cholesky decomp to column wise](../sources/prs/pytorch/PR-158237.md), [[cherry-pick][inductor][triton] Update HAS_WARP_SPEC to check triton.Config params. Update Triton Hash to top of release/3.4.x stack](../sources/prs/pytorch/PR-158646.md), [CUDA 13.0 Windows Nvidia Driver Update to 580.88](../sources/prs/pytorch/PR-162501.md), [[Cherry Pick][Graph Partition] allow sharing default device context](../sources/prs/pytorch/PR-163097.md), [[Release 2.9] [cuDNN][SDPA][submodule] Roll-back cuDNN frontend upgrade, update Met…](../sources/prs/pytorch/PR-163265.md), [[Graph Partition] improve custom op output alias](../sources/prs/pytorch/PR-163380.md), [[Inductor][Intel GPU] Save `threads_per_warp` from tirton compiled kernel for launching kernel correctly in cpp wrapper.](../sources/prs/pytorch/PR-163388.md), [[graph partition] Add way to register custom rule (#163310)](../sources/prs/pytorch/PR-163395.md), [[2.9 cherry pick][triton] update 3.5 pin to bbb06c0334a6772b92d24bde54956e675c8c6604 (#163382)](../sources/prs/pytorch/PR-163583.md), [CUDA 13.0 Warning update for supported architectures](../sources/prs/pytorch/PR-163633.md), [[Cherry-Pick] [CD] CUDA 13 specific followup changes. Remove sm50-70 From CUDA 12.6 and CUDA 12.8 builds (#162455)](../sources/prs/pytorch/PR-163764.md), [[CD] CUDA 13.0 fix preload logic to include nvidia/cu13/lib/](../sources/prs/pytorch/PR-163766.md), [fix pickling for BitwiseFn](../sources/prs/pytorch/PR-163861.md), [Move inductor jobs 3.9->3.10](../sources/prs/pytorch/PR-163954.md), [[cuDNN][SDPA] Disable dropout for cuDNN SDPA on 9.11 - 9.13](../sources/prs/pytorch/PR-164026.md), [[AARCH64][CD][CUDA13][Triton][PTXAS] Turn on BUILD_BUNDLE_PTXAS=1 ](../sources/prs/pytorch/PR-164236.md), [CUDA 13.0 builds fix on Amazon Linux 2023](../sources/prs/pytorch/PR-164893.md), [[inductor] don't try to reorder loops for template](../sources/prs/pytorch/PR-166910.md), [[Dynamo] Don't guard data ptrs by default with mark_static_address](../sources/prs/pytorch/PR-166913.md), [[Inductor] No longer throw error in bmm out_dtype lowering due to tem…](../sources/prs/pytorch/PR-166922.md), [[Graph Partition] move custom rules to inductor config (#166458)](../sources/prs/pytorch/PR-166967.md), [[Graph Partition] fix graph partition input signature for fallback kernels](../sources/prs/pytorch/PR-166985.md), [[GraphPartition] cache get_free_symbol_uses (#166338)](../sources/prs/pytorch/PR-166994.md), [[Minor][Inductor] move some combo kernel log from warning to debug](../sources/prs/pytorch/PR-167020.md), [[cuDNN][SDPA] Check-in test for #166211](../sources/prs/pytorch/PR-167121.md), [[cuDNN][SDPA][Convolution] Expose cuDNN runtime version in CUDA hooks](../sources/prs/pytorch/PR-167327.md), [[ROCm] Enable shared memory based pruning for Triton configs](../sources/prs/pytorch/PR-170190.md), [[Inductor] ExternKernelBenchmarkRequest best attempt](../sources/prs/pytorch/PR-170246.md), [[inductor] Fix cudagraph skip for index_put_ with boolean indices, gr…](../sources/prs/pytorch/PR-170884.md), [[Inductor] Fix constants handling for Triton constexpr (triton#8248)](../sources/prs/pytorch/PR-171129.md), [[ROCm] Make grouped GEMM CK opt‑in via env and default to fallback path](../sources/prs/pytorch/PR-171140.md), [Avoid closing random file handles in Inductor](../sources/prs/pytorch/PR-171150.md), [[cherry-pick][CUDA] Upgrade cuDNN to 9.15.1 for CUDA 13 builds ](../sources/prs/pytorch/PR-171189.md), [[xpu][fix][inductor] fallback bfloat16 atomics to eager](../sources/prs/pytorch/PR-171247.md), [[cherry-pick][cuDNN][SDPA] cuDNN SDPA off-by-default for cuDNN versions < 12.9 (#171627)](../sources/prs/pytorch/PR-171895.md), [Skip modded_nanogpt model in TorchInductor benchmark](../sources/prs/pytorch/PR-172141.md), [[Graph Partition] Improve support for mutation ops](../sources/prs/pytorch/PR-172577.md), [Update inductor expected accuracy files](../sources/prs/pytorch/PR-175096.md), [[benchmark] Skip pytorch_CycleGAN_and_pix2pix from inductor benchmarks](../sources/prs/pytorch/PR-175299.md), [[release-only] Remove +ptx from cuda 13.0 builds](../sources/prs/pytorch/PR-175567.md), [[inductor] avoid multi-stage for mix-order-red by default (#176228)](../sources/prs/pytorch/PR-176495.md), [[inductor] Fix Identity comparability and evalf recursion](../sources/prs/pytorch/PR-176783.md), [[Inductor] Don't unfuse addmm for bf16/fp16 to avoid precision loss](../sources/prs/pytorch/PR-177144.md), [[Inductor][MPS] Fix half-precision type mismatches in Metal shader codegen (#176436)](../sources/prs/pytorch/PR-177193.md), [[MPS] fix compiling of SDPA producing nan results](../sources/prs/pytorch/PR-178009.md), [Optimize nvfp4 block scaled gemm kernel when M is small.](../sources/prs/sglang/PR-10101.md), [fix: resolve gb200 image link](../sources/prs/sglang/PR-10343.md), [Update CUTLASS. Refine KernelSchedule for fp8 (grouped) gemm.](../sources/prs/sglang/PR-10491.md), [Cache the result of `is_blackwell` platform check](../sources/prs/sglang/PR-10498.md), [Unify SGL Kernel Releases](../sources/prs/sglang/PR-10701.md), [Optimize cutlass int8 gemm kernel for large M on SM89 Ada GPU](../sources/prs/sglang/PR-10714.md), [chore: upgrade sgl-kernel 0.3.13](../sources/prs/sglang/PR-11056.md), [[NVIDIA] Add new SMs support for Spark & Thor](../sources/prs/sglang/PR-11287.md), [[sgl-kernel][1/N]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-11432.md), [Improve Kernel Build Time](../sources/prs/sglang/PR-11508.md), [support cutlass fp4 kernel in sm120](../sources/prs/sglang/PR-11737.md), [Change bf16 to fp8 for some gemms in attention for DeepSeek ckpt v2](../sources/prs/sglang/PR-11805.md), [Use cutlass fp4 gemm by default](../sources/prs/sglang/PR-11813.md), [chore: upgrade flashinfer 0.4.1](../sources/prs/sglang/PR-11933.md), [[sgl-kernel][4/N]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-12080.md), [[hotfix] missing `w13_weight_fp8` and `w2_weight_fp8` in UE8M0 requantization](../sources/prs/sglang/PR-12259.md), [fix: Llama 4 BF16 load on Blackwell](../sources/prs/sglang/PR-12308.md), [[sgl-kernel] clean up fa fetch in CMakeLists.txt](../sources/prs/sglang/PR-12392.md), [chore: upgrade flashinfer 0.5.0](../sources/prs/sglang/PR-12523.md), [[NVIDIA] Fix CUDA arch requirement in nvfp4 cast](../sources/prs/sglang/PR-12581.md), [[sgl-kernel][5/N]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-12666.md), [ignore the deepgemm check when the model weight with nvfp4 and moe ba…](../sources/prs/sglang/PR-12782.md), [[NVIDIA] Fix use case of SGLANG_ENABLE_FLASHINFER_GEMM](../sources/prs/sglang/PR-13274.md), [Support weight update for blackwell DeepGEMM](../sources/prs/sglang/PR-13324.md), [[ROCM] Optimized deepseek-r1 fp8 model with + triton_gemm_a8w8 + batch_gemm_a8w8 + fused set_mla_kv_buffer kernel](../sources/prs/sglang/PR-13617.md), [[sgl-kernel][Feat][B200][1/N]Support MXFP8 Grouped GEMM in Blackwell](../sources/prs/sglang/PR-13731.md), [[chore]Upgrade flashinfer to 0.5.3](../sources/prs/sglang/PR-13751.md), [update flashinfer_cubin==0.5.3](../sources/prs/sglang/PR-13848.md), [Fix update weight error for blackwell DeepGEMM](../sources/prs/sglang/PR-13910.md), [Add new moe wna16 marlin gemm](../sources/prs/sglang/PR-14122.md), [Apply new moe wna16 marlin gemm](../sources/prs/sglang/PR-14125.md), [[CPU] Implement MXFP4 Gemm kernels for intel AMX to support GPT OSS series.](../sources/prs/sglang/PR-14385.md), [Add CUDA kernel size analysis tool for sgl-kernel optimization](../sources/prs/sglang/PR-14544.md), [[sgl-kernel][Feat][B200][2/N] Support MXFP8 Grouped GEMM in Blackwell](../sources/prs/sglang/PR-14640.md), [Add cache for flashinfer installation](../sources/prs/sglang/PR-15153.md), [[Tiny]Add warning for deepgemm on Blackwell](../sources/prs/sglang/PR-15352.md), [[sgl-kernel][6/7]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-15471.md), [[Perf] Add Flashinfer DeepGEMM SM90 for SwapAB Optimization](../sources/prs/sglang/PR-15514.md), [[sgl-kernel] Streamline kernel size report (Top 20 only) and clean up](../sources/prs/sglang/PR-15552.md), [[JIT kernel] Apply jit per_tensor_quant_fp8 kernel](../sources/prs/sglang/PR-15836.md), [[fix]deepgemm precompile when warmup](../sources/prs/sglang/PR-15891.md), [[Feature] add aligned_vector type for JIT kernel](../sources/prs/sglang/PR-16162.md), [Fix sgl-kernel jobs to skip when target_stage is specified](../sources/prs/sglang/PR-16308.md), [[Fix]Pin mooncake version to 0.3.7.post2 in grace blackwell](../sources/prs/sglang/PR-16502.md), [Fix FP8 MoE NaN with DeepGEMM on Blackwell](../sources/prs/sglang/PR-16622.md), [Support mxint4 flashinfer_trtllm moe gemm](../sources/prs/sglang/PR-16892.md), [[MUSA][2/N] sgl-kernel build](../sources/prs/sglang/PR-17053.md), [[FIX] Always support TP > 4 for FP4 Gemm](../sources/prs/sglang/PR-17300.md), [Upgrade transformers==5.3.0](../sources/prs/sglang/PR-17784.md), [Feature/support longcat flash lite](../sources/prs/sglang/PR-17838.md), [Skipped warning on sm100](../sources/prs/sglang/PR-18000.md), [[Diffsuion & JIT_kernel] QKNorm cross heads kernel](../sources/prs/sglang/PR-18073.md), [[Blackwell] Make mxint4 flashinfer_trtllm moe gemm set by default on blackwell](../sources/prs/sglang/PR-18136.md), [[FIX] Correct JIT kernel compilation on newer GPUs with outdated driver metadata.](../sources/prs/sglang/PR-18496.md), [use flashinfer.sampling](../sources/prs/sglang/PR-18696.md), [fix: update Blackwell log/error messages to include SM12x](../sources/prs/sglang/PR-18751.md), [fix: add SM110 (Jetson AGX Thor) to Blackwell capability check](../sources/prs/sglang/PR-18787.md), [Migrate renorm kernels from sgl-kernel to FlashInfer JIT](../sources/prs/sglang/PR-18854.md), [Add claude skills for sgl-kernel and jit-kernel](../sources/prs/sglang/PR-18855.md), [Migrate norm kernels to FlashInfer JIT implementation](../sources/prs/sglang/PR-18871.md), [Use single mma warp group for short q_len in FA to optimize decoding performance](../sources/prs/sglang/PR-18985.md), [[Kernel Slimming] Migrate NVFP4 kernels to JIT](../sources/prs/sglang/PR-19437.md), [[FlashInfer v0.6.4] [RL] Integrate FlashInfer mxfp8 gemm, MoE, and routed MoE](../sources/prs/sglang/PR-19537.md), [[Feature] NVFP4 Marlin fallback for non-Blackwell GPUs (SM75+)](../sources/prs/sglang/PR-19652.md), [[JIT Kernel][Feature] Support JIT custom all reduce (rewrite as v2)](../sources/prs/sglang/PR-19880.md), [[JIT Kernel] Reland NVFP4 kernels to JIT](../sources/prs/sglang/PR-20012.md), [[Benchmark] use flashinfer bench_gpu_time instead of triton do_bench](../sources/prs/sglang/PR-20305.md), [fix ci by removing nvidia-cutlass-dsl-libs-base and force reinstall n…](../sources/prs/sglang/PR-20380.md), [Fix(jit): support rmsnorm for hidden_size in {64, 128, 256}](../sources/prs/sglang/PR-20661.md), [Use FlashInfer tinygemm for GPT-OSS MoE router on SM90+](../sources/prs/sglang/PR-20755.md), [fix: guard configure_deep_gemm_num_sms when JIT disabled](../sources/prs/sglang/PR-20868.md), [CUTLASS FP8 Blockwise GEMM improvement of SM120](../sources/prs/sglang/PR-20887.md), [[Tiny Fix] Fix IS_BLACKWELL env var empty string warning in rerun-ut workflow](../sources/prs/sglang/PR-20957.md), [fix: wrap _import_static_state in inference_mode to fix resume on Blackwell](../sources/prs/sglang/PR-21035.md), [ci: remove IS_BLACKWELL env var; auto-detect Blackwell](../sources/prs/sglang/PR-21118.md), [[NPU] bugfix for import sgl-kernel error](../sources/prs/sglang/PR-21200.md), [Split pr-test.yml: extract sgl-kernel, jit-kernel, and multimodal-gen tests into separate workflow files](../sources/prs/sglang/PR-21219.md), [Refactor JIT kernel CI to use run_suite.py registration system](../sources/prs/sglang/PR-21239.md), [[MUSA] apply_vocab_mask support musa device](../sources/prs/sglang/PR-21296.md), [CUTLASS NVFP4 GEMM improvement of SM120](../sources/prs/sglang/PR-21314.md), [[FlashInver v0.6.7] Integrate flashinfer_trtllm mxfp8 gemm](../sources/prs/sglang/PR-21576.md), [[AMD] Use tgemm.mm for MoEGate router gemm in deepseek_v2.py](../sources/prs/sglang/PR-21657.md), [Remove flashinfer wheel cache cleanup that deletes other versions](../sources/prs/sglang/PR-21711.md), [[DSA] Set trtllm kernels as default for Blackwell](../sources/prs/sglang/PR-21914.md), [[Diffusion] Fix weight scale swizzle and add large-M kernel config for FLUX.2-dev-NVFP4](../sources/prs/sglang/PR-22064.md), [[nvidia] Gemma4 nvfp4 fix](../sources/prs/sglang/PR-22079.md), [[Hotfix] Fix router gemm on sm103](../sources/prs/sglang/PR-22134.md), [[hisparse]: Adding ci for hisparse kvcache-swap-in jit-kernel](../sources/prs/sglang/PR-22155.md), [[HiSparse]: Add benchmark for hisparse kernel](../sources/prs/sglang/PR-22187.md), [[Docker] Fix Trivy CVEs, cubin download 403s, and kernels command order](../sources/prs/sglang/PR-22322.md), [Upgrade sglang-torch-profiler-analysis SKILLS](../sources/prs/sglang/PR-22440.md), [[RL] Fix weight update for mxfp8 flashinfer_cutlass gemm backend](../sources/prs/sglang/PR-22484.md), [[CI/Docker] Clean up redundant flashinfer cubin downloads](../sources/prs/sglang/PR-22491.md), [[Docker] Remove flashinfer cache copy](../sources/prs/sglang/PR-22653.md), [[Fix/Kernel] Add JIT rmsnorm_hf kernel to fix transformers backend MMLU accuracy regression ](../sources/prs/sglang/PR-22931.md), [Reland Cute-DSL FP4 dense GEMM](../sources/prs/sglang/PR-23590.md), [feat: port SGLANG_JIT_DEEPGEMM_FAST_WARMUP to deepseek_v4 branch](../sources/prs/sglang/PR-23756.md), [Enable PDL for various kernels in DSV32/GLM5](../sources/prs/sglang/PR-23965.md), [[VLM] Optimize Gemma4 VLM with PCG and fuse RMSNorm + residual add + scalar](../sources/prs/sglang/PR-24048.md), [Port MXFP4 Marlin MoE support to JIT kernel path](../sources/prs/sglang/PR-24490.md), [[Gemma4] Optimize Gemm4 with fused Q/K/V RMSNorm + per-expert FP8 ckpt loader](../sources/prs/sglang/PR-24696.md), [[codex] Optimize hidden-size 512 RMSNorm dispatch](../sources/prs/sglang/PR-24710.md), [[rebase]Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-24986.md), [Support Gemma4 Pipeline Parallelism](../sources/prs/sglang/PR-25284.md), [[Gemma4]: Fix FP8 Triton scale layout](../sources/prs/sglang/PR-25286.md), [[Intel GPU] Enable DeepSeek V4 Inference on XPU](../sources/prs/sglang/PR-25336.md), [[fp8] SM90 swap-AB scaled_mm dispatch (~1.16x kernel geomean, +5.8-18.5% end-to-end)](../sources/prs/sglang/PR-25532.md), [Use DeepGEMM BF16 for unquantized DeepEP LL MoE](../sources/prs/sglang/PR-25540.md), [fix (jit kernel): elementwise activation C++ error](../sources/prs/sglang/PR-25695.md), [[Codex] Remove stale DeepSeek V4 JIT kernels](../sources/prs/sglang/PR-25764.md), [[Test] Stage-a sanity kits; consolidate core/ + models_e2e/ tests](../sources/prs/sglang/PR-25831.md), [Support cutlass Int8 gemm](../sources/prs/sglang/PR-2752.md), [upgrade cutlass v3.7.0](../sources/prs/sglang/PR-2967.md), [feat: add flashinfer as 3rdparty and use rmsnorm as example](../sources/prs/sglang/PR-3033.md), [Support sm90 Int8 gemm](../sources/prs/sglang/PR-3035.md), [Allow local cutlass directory to be used in sgl-kernel build](../sources/prs/sglang/PR-3037.md), [support w8a8 fp8 kernel with CUTLASS](../sources/prs/sglang/PR-3047.md), [sync the upstream updates of flashinfer](../sources/prs/sglang/PR-3051.md), [feat: integrate gemm_fp8 kernel into gemm](../sources/prs/sglang/PR-3056.md), [add tensorrt_llm common and cutlass_extensions as 3rdparty](../sources/prs/sglang/PR-3216.md), [support blockwise fp8 matmul kernel](../sources/prs/sglang/PR-3267.md), [fix undefined symbol cudaGetDriverEntryPointByVersion](../sources/prs/sglang/PR-3372.md), [integrate blockwise fp8 kernel](../sources/prs/sglang/PR-3529.md), [update flashinfer-python](../sources/prs/sglang/PR-3557.md), [[Feature] Apply Cublas Grouped Gemm kernel](../sources/prs/sglang/PR-3629.md), [add control for cutlass fp8 blockwise gemm](../sources/prs/sglang/PR-3727.md), [Support FP4 gemm (1/2)](../sources/prs/sglang/PR-3899.md), [upgrade flashinfer v0.2.2.post1](../sources/prs/sglang/PR-3934.md), [Support overlapping two batches](../sources/prs/sglang/PR-4068.md), [DeepGemm integrate to gemm](../sources/prs/sglang/PR-4165.md), [linear support deepgemm](../sources/prs/sglang/PR-4199.md), [Accelerate FP8 CUDA Kernel by 20-28%](../sources/prs/sglang/PR-4215.md), [fix per_token_group_quant_fp8 illegal memory when num_groups % 16 != 0](../sources/prs/sglang/PR-4231.md), [add THIRDPARTYNOTICES for DeepGEMM](../sources/prs/sglang/PR-4272.md), [Support Blackwell Block Scale FP8 Gemm](../sources/prs/sglang/PR-4278.md), [update deepgemm](../sources/prs/sglang/PR-4284.md), [upgrade flashinfer 0.2.3](../sources/prs/sglang/PR-4317.md), [[FIX] fix incorrect output when enable both deepgemm and torch compile](../sources/prs/sglang/PR-4359.md), [Create col-major and tma-aligned x_scale for deep_gemm.gemm_fp8_fp8_bf16_nt](../sources/prs/sglang/PR-4515.md), [Support fp8 gemm for blackwell](../sources/prs/sglang/PR-4558.md), [Set deepgemm to the default value in the hopper architecture.](../sources/prs/sglang/PR-4613.md), [Fix loading KV quantization scale; Enable modelopt kv cache](../sources/prs/sglang/PR-4686.md), [support cmake for sgl-kernel](../sources/prs/sglang/PR-4706.md), [[Feat] support deepgemm for cmake](../sources/prs/sglang/PR-4864.md), [[Build] Fix cuda12.8 build error in nvfp4_scaled_mm_kernels.cu](../sources/prs/sglang/PR-4953.md), [update cutlass tag](../sources/prs/sglang/PR-5011.md), [fix deepgemm as well](../sources/prs/sglang/PR-5030.md), [support sgl-kernel on blackwell](../sources/prs/sglang/PR-5074.md), [Add optimized native kernels in sgl-kernel](../sources/prs/sglang/PR-5150.md), [feat: add DeepGEMM build warning](../sources/prs/sglang/PR-5176.md), [[Fix] Turn off DeepGEMM by default](../sources/prs/sglang/PR-5263.md), [fix: use deepgemm only on hopper](../sources/prs/sglang/PR-5310.md), [fix: determine if flashinfer is installed](../sources/prs/sglang/PR-5336.md), [Fix DeepGEMM masked cannot be run on groups not being multiple or 4](../sources/prs/sglang/PR-5340.md), [chore: upgrade DeepGEMM](../sources/prs/sglang/PR-5395.md), [[perf] introduce deep gemm group_gemm_masked as gemm](../sources/prs/sglang/PR-5432.md), [Fix sampler nan check when calling top_k_top_p_sampling_from_probs](../sources/prs/sglang/PR-5546.md), [feat: use flashinfer jit package](../sources/prs/sglang/PR-5547.md), [[feature] enable pre compile jit deep_gemm](../sources/prs/sglang/PR-5580.md), [[fix] force use deepgemm in compile_deep_gemm](../sources/prs/sglang/PR-5618.md), [ DeepEP normal support deepgemm-contiguous](../sources/prs/sglang/PR-5626.md), [Turn on DeepGemm By Default and Update Doc](../sources/prs/sglang/PR-5628.md), [cutlass 3.9 supported to improve fp8_blockwise_gemm](../sources/prs/sglang/PR-5820.md), [Add sm_120 for blackwell](../sources/prs/sglang/PR-5903.md), [[Feat] Enable PDL automatically on Hopper architecture](../sources/prs/sglang/PR-5981.md), [chore: upgrade cutlass 3.9.2](../sources/prs/sglang/PR-6004.md), [chore: upgrade deepgemm](../sources/prs/sglang/PR-6073.md), [[1/2] Add Kernel support for Cutlass based Fused FP4 MoE](../sources/prs/sglang/PR-6093.md), [[Fix] Improve dependencies for Blackwell image](../sources/prs/sglang/PR-6334.md), [Upgrade CUTLASS 4.0](../sources/prs/sglang/PR-6336.md), [Add fp8 fused_experts kernel for CPU in sgl-kernel and add UT](../sources/prs/sglang/PR-6404.md), [Fix bug of deepseek-v3 under DP+EP mode with large batchsize/seqlen](../sources/prs/sglang/PR-6449.md), [[Feature] Support Flashinfer fp8 blockwise GEMM kernel on Blackwell](../sources/prs/sglang/PR-6479.md), [[CPU] [BF16] Call fused_experts_cpu, weight_packed_linear and bmm_cpu kernel in DeepSeek model](../sources/prs/sglang/PR-6641.md), [Correctly abort the failed grammar requests & Improve the handling of abort](../sources/prs/sglang/PR-6803.md), [feat: integrate deepgemm into EPMoE](../sources/prs/sglang/PR-6821.md), [CPU: map changes from developing branch in sgl-kernel](../sources/prs/sglang/PR-6833.md), [Fix AWQ Dequant and Weight Loading of deepseek v2](../sources/prs/sglang/PR-6842.md), [Use deepgemm instead of triton for fused_qkv_a_proj_with_mqa](../sources/prs/sglang/PR-6890.md), [[sgl-kernel] update deepgemm](../sources/prs/sglang/PR-6942.md), [Fix torchvision version for Blackwell](../sources/prs/sglang/PR-7015.md), [Clean up server_args.py](../sources/prs/sglang/PR-7037.md), [feat: update blackwell setup](../sources/prs/sglang/PR-7119.md), [Support new DeepGEMM](../sources/prs/sglang/PR-7172.md), [Tiny let DeepGEMM scale checks cover more cases](../sources/prs/sglang/PR-7182.md), [chore: upgrade sgl-kernel v0.1.8.post2](../sources/prs/sglang/PR-7186.md), [Fix a minor bug related to DeepGEMM upgrade](../sources/prs/sglang/PR-7191.md), [Fix error when disabling new DeepGEMM](../sources/prs/sglang/PR-7198.md), [[fix] fix DeepGEMM blackwell input quant & ut & fix style and log](../sources/prs/sglang/PR-7247.md), [Add CUTLASS FP8 Blockscale MoE kernel for Hopper architecture](../sources/prs/sglang/PR-7278.md), [Let EP prefill support new DeepGEMM](../sources/prs/sglang/PR-7310.md), [Kernels for efficient KV cache IO](../sources/prs/sglang/PR-7313.md), [fix: resolve blackwell deepep image issue](../sources/prs/sglang/PR-7331.md), [Quick fix for DeepGemm requant to also cover MTP.](../sources/prs/sglang/PR-7378.md), [[CMake] Fix sgl-kernel CMakeLists for Blackwell](../sources/prs/sglang/PR-7543.md), [Add dsv3 router gemm kernel](../sources/prs/sglang/PR-7627.md), [Add dsv3 fused a gemm to sgl-kernel](../sources/prs/sglang/PR-7630.md), [[Feature] Layer-wise Prefill](../sources/prs/sglang/PR-7634.md), [Apply dsv3_fused_a_gemm kernel](../sources/prs/sglang/PR-7635.md), [[Feature] CUDA Green Context Support](../sources/prs/sglang/PR-7649.md), [chore: upgrade flashinfer v0.2.7 jit](../sources/prs/sglang/PR-7663.md), [[1/n]: add cutlass W4A8 moe kernel for hopper architecture](../sources/prs/sglang/PR-7772.md), [[Fix][Ready]Fix register spilling in cutlass nvfp4 gemm kernel on Blackwell](../sources/prs/sglang/PR-8127.md), [[sgl-kernel] Opt per_token_quant_fp8 with warp reduce](../sources/prs/sglang/PR-8130.md), [[Perf] Tunings for SM100 FP8 CUTLASS kernel](../sources/prs/sglang/PR-8818.md), [chore: support blackwell cu129 image](../sources/prs/sglang/PR-8928.md), [[NVIDIA] Fix missing `get_col_major_tma_aligned_tensor` for Blackwell deepgemm in EpMoE](../sources/prs/sglang/PR-8955.md), [[fix] fix enable_pdl for blackwell](../sources/prs/sglang/PR-9011.md), [[sgl-kernel] Support FlashInfer top_k_top_p_sampling_from_logits](../sources/prs/sglang/PR-9060.md), [[NVIDIA] [3/N] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked ](../sources/prs/sglang/PR-9199.md), [[NVIDA] [1/N] Nvfp4 Masked Gemm: Add quant op for the flashinfer grouped gemm](../sources/prs/sglang/PR-9200.md), [[fix]: fix cutlass moe ut and and Opt H20 cutlass groupGemm performance](../sources/prs/sglang/PR-9272.md), [Fix FP4 inference corruption issue in glm4.5-air model](../sources/prs/sglang/PR-9346.md), [[sgl-kernel] feat: Support sm120 cutlass fp8 gemm kernel](../sources/prs/sglang/PR-9403.md), [fix: blackwell dsv3 fp8 issue temporary solution](../sources/prs/sglang/PR-9530.md), [[NVIDIA] [2/N] Optimize `silu_and_mul_scaled_fp4_grouped_quant` perf](../sources/prs/sglang/PR-9556.md), [Update CUTLASS 4.2 & Enable K-Major Scale Factor for SM90 FP8 Blockwise Group GEMM](../sources/prs/sglang/PR-9559.md), [Single Batch Overlap for MoE Models](../sources/prs/sglang/PR-9660.md), [[CPU] Add FP8 Bmm support](../sources/prs/sglang/PR-9744.md), [Make sm100 fp8 kernels available on sm103](../sources/prs/sglang/PR-9789.md), [Make fp4_quantize kernels work on sm103](../sources/prs/sglang/PR-9807.md), [perf: Avoid unnecessary data type conversions for DeepSeek-V3 on Blackwell](../sources/prs/sglang/PR-9834.md), [[Fix] DeepSeek EP accuracy issue on B200 GPUs](../sources/prs/sglang/PR-9946.md), [CUTLASS fp8 blockwise gemm support of sm120](../sources/prs/sglang/PR-9969.md), [[WIP] support more dtypes for tcgen05](../sources/prs/tilelang/PR-1229.md), [[Enhancement] add more dtype and fix mma.ws for fp16 for tcgen05](../sources/prs/tilelang/PR-1327.md), [[Feat] profiler support cudagraph backend](../sources/prs/tilelang/PR-1658.md), [Add swizzle layout detection and automatic merging for layout conflicts](../sources/prs/tilelang/PR-1736.md), [[Feature] Support tcgen5mma lowering for `.kind::i8`](../sources/prs/tilelang/PR-1764.md), [fix(intrinsics): add missing _legalize_to_buffer_region in SM70 emitter](../sources/prs/tilelang/PR-1786.md), [[CUDA] Support tcgen5mma gemm ts](../sources/prs/tilelang/PR-1866.md), [[Feature] Support cluster launch, query, synchronization and barrier operations](../sources/prs/tilelang/PR-1874.md), [[Feature] 2-SM support for TMA, TMEM and TCGEN5MMA on Blackwell](../sources/prs/tilelang/PR-1882.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [[Feature] Block-scaled GEMM support for MXFP8 on Blackwell](../sources/prs/tilelang/PR-1945.md), [[Bugfix] Fix CuTeDSL autotune cache invalid ELF header (#1967)](../sources/prs/tilelang/PR-1972.md), [[Transform] Add InjectTcgen05Fence pass](../sources/prs/tilelang/PR-2003.md), [[Backend] Refactor gemm_sp](../sources/prs/tilelang/PR-2048.md), [[CUDA] Support int4 `T.gemm`](../sources/prs/tilelang/PR-2063.md), [[CUDA] Improve int4 GEMM lowering and packed codegen support](../sources/prs/tilelang/PR-2073.md), [feat: auto-vectorize bf16/fp16 reduce with packed add2 intrinsics](../sources/prs/tilelang/PR-2112.md), [[CUDA][TMA] Add TMA tile::gather4 / tile::scatter4 support](../sources/prs/tilelang/PR-2129.md), [[codex] Split GEMM implementations by backend](../sources/prs/tilelang/PR-2153.md), [[CUDA] Add native SM75 MMA GEMM support for FP16, INT8 and INT4](../sources/prs/tilelang/PR-2198.md), [[TIR][IR] Update to use tirx](../sources/prs/tilelang/PR-2216.md), [[Python] Drop Python 3.9 support](../sources/prs/tilelang/PR-2218.md), [[Kernel] Update `cutlass_scaled_mm` to support 2d group (blockwise) scaling](../sources/prs/vllm/PR-11868.md), [[Build] Only build 9.0a for scaled_mm and sparse kernels](../sources/prs/vllm/PR-12339.md), [[Core][AMD] Migrate fully transparent sleep mode to ROCm platform](../sources/prs/vllm/PR-12695.md), [[Misc][Kernel]: Add GPTQAllSpark Quantization](../sources/prs/vllm/PR-12931.md), [[Kernel]Add streamK for block-quantized CUTLASS kernels](../sources/prs/vllm/PR-12978.md), [[NVIDIA] Support nvfp4 tcgen05 gemm](../sources/prs/vllm/PR-13571.md), [add tcgen05 support for tcgen05 fp8 gemm](../sources/prs/vllm/PR-13798.md), [[Kernel] CUTLASS grouped gemm fp8 MoE kernel](../sources/prs/vllm/PR-13972.md), [[Kernel] optimize performance of gptq marlin kernel when n is small](../sources/prs/vllm/PR-14138.md), [dynamic distpatch of fp8 kernels](../sources/prs/vllm/PR-14245.md), [fix minor miscalled method](../sources/prs/vllm/PR-14327.md), [Add cutlass support for blackwell fp8 blockwise gemm](../sources/prs/vllm/PR-14383.md), [[BugFix] Illegal Memory Access in the blockwise cutlass fp8 GEMMs](../sources/prs/vllm/PR-14396.md), [permute/unpermute kernel for moe optimization](../sources/prs/vllm/PR-14568.md), [[BugFix/Build] Fix sparse kernels not getting built on hopper](../sources/prs/vllm/PR-14572.md), [[Quantization][FP8] Adding support for fp8 gemm layer input in fp8](../sources/prs/vllm/PR-14578.md), [[Kernel] allow non-contiguous input for marlin kernel](../sources/prs/vllm/PR-14658.md), [[Model] Add support for Gemma 3](../sources/prs/vllm/PR-14660.md), [[Bugfix][Kernel][CPU] Fix num_tokens in CPU rotary embedding kernel](../sources/prs/vllm/PR-14667.md), [[FEAT] [ROCm]: Add AITER Block-Scaled GEMM Feature](../sources/prs/vllm/PR-14968.md), [[FEAT] [ROCm] Add AITER int8 scaled gemm kernel](../sources/prs/vllm/PR-15433.md), [[Quantization] Fp8 Channelwise Dynamic Per Token GroupedGEMM](../sources/prs/vllm/PR-15587.md), [Modularize fused experts and integrate PPLX kernels](../sources/prs/vllm/PR-15956.md), [[Perf]Optimize rotary_emb implementation to use Triton operator for improved inference performance](../sources/prs/vllm/PR-16457.md), [[Kernel] some optimizations for dense marlin and moe marlin](../sources/prs/vllm/PR-16850.md), [[Bugfix] gemma[2,3] interleaved attention when sliding window is disabled](../sources/prs/vllm/PR-17180.md), [[Kernel] fp4 marlin kernel](../sources/prs/vllm/PR-17687.md), [[Kernel] Have rotary embeddings support tensors](../sources/prs/vllm/PR-18046.md), [[Kernel] Integrate CUTLASS MoE kernel with PPLX](../sources/prs/vllm/PR-18762.md), [[Kernel] Enable fp8 support for pplx and BatchedTritonExperts.](../sources/prs/vllm/PR-18864.md), [[Kernel] Support deep_gemm for linear methods](../sources/prs/vllm/PR-19085.md), [[V1] Use FlashInfer by default on Blackwell GPUs](../sources/prs/vllm/PR-19118.md), [[Kernels] Add activation chunking logic to FusedMoEModularKernel](../sources/prs/vllm/PR-19168.md), [[Bugfix] Don't attempt to use triton if no driver is active](../sources/prs/vllm/PR-19561.md), [[Perf] Further tunings for SM100 FP8 CUTLASS kernel](../sources/prs/vllm/PR-19566.md), [[feat]: CUTLASS block scaled group gemm for SM100](../sources/prs/vllm/PR-19757.md), [Fix FA2 fallback for Blackwell V1](../sources/prs/vllm/PR-19781.md), [[Feature] Integrate new deepgemm](../sources/prs/vllm/PR-19820.md), [ [Feature] Integrate SM100 DeepGEMM support](../sources/prs/vllm/PR-20087.md), [[Nixl] Heterogeneous TP support FlashInfer](../sources/prs/vllm/PR-20189.md), [[Misc] DP : Add ExpertTokensMetadata](../sources/prs/vllm/PR-20332.md), [[Kernel] SM90 CUTLASS FP8 GEMM: add support for swap AB + kernel tuning](../sources/prs/vllm/PR-20396.md), [[feat]: add SM100 support for cutlass FP8 groupGEMM](../sources/prs/vllm/PR-20447.md), [[feat] enable SM100 CUTLASS block scaled group gemm for smaller batch sizes](../sources/prs/vllm/PR-20640.md), [[Performance] Performance improvements in non-blockwise fp8 CUTLASS MoE](../sources/prs/vllm/PR-20762.md), [[fix]: disable cutlass block scaled group gemm for EP](../sources/prs/vllm/PR-20781.md), [[Bug] Fix DeepGemm for EP low latency case](../sources/prs/vllm/PR-20833.md), [[Perf] Use Triton instead of Torch for DeepGEMM Per Token Group Quant](../sources/prs/vllm/PR-20841.md), [[Kernel] DeepGemm MoE : Integrate triton permute / unpermute kernels ](../sources/prs/vllm/PR-20903.md), [[Perf] Add swap_ab to SM90 FP8 non-block CUTLASS moe grouped gemm](../sources/prs/vllm/PR-20911.md), [[Model] Pooling models default to using chunked prefill & prefix caching if supported.](../sources/prs/vllm/PR-20930.md), [[Bugfix] Switch bailout logic for kv-cache-dtype with SM100 Flashinfer](../sources/prs/vllm/PR-20934.md), [[Bugfix] Fix Mistral3 support on SM100/SM120](../sources/prs/vllm/PR-20998.md), [[Perf] Use FlashInfer RoPE for RotaryEmbedding.forward_cuda when available](../sources/prs/vllm/PR-21126.md), [[Bug] DeepGemm: Fix TypeError: per_block_cast_to_fp8() missing 1 required positional argument: 'use_ue8m0' for SM100](../sources/prs/vllm/PR-21187.md), [Override attention metadata for fast prefill in some KV sharing setups](../sources/prs/vllm/PR-21590.md), [update flashinfer to v0.2.9rc2](../sources/prs/vllm/PR-21701.md), [[Logs] Change flashinfer sampler logs to once](../sources/prs/vllm/PR-21759.md), [[bugfix] fix blackwell deepep installation](../sources/prs/vllm/PR-22255.md), [[Bugfix] Fix 3D input passed into cutlass_scaled_mm](../sources/prs/vllm/PR-22278.md), [Update `flashinfer-python==0.2.10`](../sources/prs/vllm/PR-22389.md), [[Bug] Fix B200 DeepGEMM E8M0 Accuracy Issue](../sources/prs/vllm/PR-22399.md), [Upgrade FlashInfer to v0.2.11](../sources/prs/vllm/PR-22613.md), [[Bugfix] Fix default enable for CUTLASS MLA on SM100](../sources/prs/vllm/PR-22738.md), [[Kernel] Added flashinfer fp8 per-tensor gemms](../sources/prs/vllm/PR-22895.md), [[Kernel] CUTLASS MoE FP8: Integrate cuda moe permute/unpermute](../sources/prs/vllm/PR-23045.md), [[Perf] Small optimizations for silu_mul_fp8_quant_deep_gemm](../sources/prs/vllm/PR-23265.md), [[Perf] Use upstream CUTLASS for SM90 Block FP8 kernel](../sources/prs/vllm/PR-23280.md), [[Feature] Add Hopper DeepGEMM E8M0 for DeepSeekV3.1 scale_fmt](../sources/prs/vllm/PR-23666.md), [[Performance] Move apply_w8a8_block_fp8_linear to an op class](../sources/prs/vllm/PR-24666.md), [[Kernel][Quantization] add w4a8 support for marlin kernel](../sources/prs/vllm/PR-24722.md), [Enable Fbgemm NVFP4 on Dense models](../sources/prs/vllm/PR-25609.md), [Update launch_bounds_utils.h for correct compile on Multiple Cuda Arch - PTXAS out of range Warning](../sources/prs/vllm/PR-25843.md), [[MoE] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked](../sources/prs/vllm/PR-25990.md), [[NVIDIA] Add support for cudnn fp4 gemm via flashinfer](../sources/prs/vllm/PR-26107.md), [[Bug] Fix Shape Validation for Fallback while Enabling E8M0 for DeepGEMM](../sources/prs/vllm/PR-26322.md), [Disable FlashInfer sampler by default](../sources/prs/vllm/PR-26859.md), [[Feature] Batch Invariant: Support DeepGEMM and Blackwell](../sources/prs/vllm/PR-27127.md), [[Feature] Batch Invariant for R1 TP 8 on Blackwell](../sources/prs/vllm/PR-27229.md), [[Perf] SM100 - add swap AB optimization to CUTLASS FP8 GEMM](../sources/prs/vllm/PR-27284.md), [[Attention] Use sparse prefill kernel for fp8 kv-cache in DeepSeek-v3.2](../sources/prs/vllm/PR-27532.md), [[Feature] Extend batch invariant torch.compile to B200](../sources/prs/vllm/PR-27856.md), [[Performance] Fused blockwise quant RMS norm](../sources/prs/vllm/PR-27883.md), [[Performance][B200] Fix deepgemm prologue](../sources/prs/vllm/PR-27897.md), [[Kernel] Optimize rms_norm kernel](../sources/prs/vllm/PR-27931.md), [[flashinfer][fix] do not check nvcc availability when using pre-downloaded cubins](../sources/prs/vllm/PR-27990.md), [[Performance][B200] silu_mul_quant: pack scales in int32](../sources/prs/vllm/PR-28358.md), [[Bugfix][Nixl] Fix kernel physical<>logical block_size issue ](../sources/prs/vllm/PR-28677.md), [[Performance] Reduce DeepGEMM N dim restriction from 128 to 64 multiplier ](../sources/prs/vllm/PR-28687.md), [[Perf][Kernels] Enable FlashInfer DeepGEMM swapAB on SM90 (for W8A8 Linear Op)](../sources/prs/vllm/PR-29213.md), [[Perf] Disable DeepGEMM MoE by default when TP=8 is used](../sources/prs/vllm/PR-29346.md), [[Bugfix] Defunctionalize TRTLLM AR+Norm op for avoiding extra clone kernel before it](../sources/prs/vllm/PR-29631.md), [[Kernel]Support W4A8 Grouped GEMM on Hopper](../sources/prs/vllm/PR-29691.md), [[Bugfix] Fix mismatched nvfp4 gemm output shape](../sources/prs/vllm/PR-29742.md), [[Bugfix] Fix flashinfer ar+norm kernel not available issue](../sources/prs/vllm/PR-29960.md), [[Bugfix] Fix DeepGEMM after #29546 ](../sources/prs/vllm/PR-30267.md), [[Bugfix] Fix fp8 DeepGemm compilation issues](../sources/prs/vllm/PR-30336.md), [[BugFix] Fix `AttributeError: 'MergedColumnParallelLinear' object has no attribute 'weight_scale'`](../sources/prs/vllm/PR-30399.md), [[Feature] Add SM103 (Blackwell Ultra) Support to vLLM](../sources/prs/vllm/PR-30484.md), [[Perf] enable flashinfer rotary_embedding custom ops in DeepSeek rotary](../sources/prs/vllm/PR-30729.md), [ [FIX] Always support TP > 4 for FP4 Gemm](../sources/prs/vllm/PR-31099.md), [[Bugfix][Hardware][AMD] Consolidate FP8 min/max values helper function](../sources/prs/vllm/PR-31106.md), [[BugFix] add select_gemm_impl on CompressedTensorsWNA16MoEMethod to support LoRA](../sources/prs/vllm/PR-31453.md), [[FIX] Add NO_MUL activation support for modular kernel path](../sources/prs/vllm/PR-31528.md), [[Perf] Add opt-in SM100 Oink RMSNorm custom-op path](../sources/prs/vllm/PR-31828.md), [[1/N][Attention] Restructure attention: move files](../sources/prs/vllm/PR-31916.md), [[5/N][Attention] Finish eliminating `vllm/attention` folder](../sources/prs/vllm/PR-32064.md), [[BugFix] Fix DeepSeek-V3.1 + DeepGEMM incompatible scale shapes](../sources/prs/vllm/PR-32361.md), [[Perf] Create TMA-aligned input scale tensor for DeepGemm on Hopper](../sources/prs/vllm/PR-32619.md), [[Performance] Tune Mamba selective scan kernel for B200](../sources/prs/vllm/PR-32873.md), [[Feature] Support CPU Offloading without Pytorch Pinned Memory that leads to doubled allocation](../sources/prs/vllm/PR-32993.md), [[Kernel] Apply 256bit LDG/STG To Activation Kernels](../sources/prs/vllm/PR-33022.md), [[Bugfix] Fix quant RMS norm fusion for quantization with TMA-aligned scales](../sources/prs/vllm/PR-33255.md), [[Bugfix] Register fp8 cutlass_group_gemm as supported for only SM90+SM100](../sources/prs/vllm/PR-33285.md), [[Feature][Core] Support Fabric detection to adapt the MNNVL protocol for the GB series](../sources/prs/vllm/PR-33540.md), [[Perf] Disable clean_logits in deepgemm fp8_mqa_logits kernel](../sources/prs/vllm/PR-33568.md), [[ModelBash][DSV3] Add TRTLLM DSV3 Router GEMM kernel (6% B1 Speedup)](../sources/prs/vllm/PR-34302.md), [[Bugfix] Enforce DeepGEMM when using sparse_attn_indexer on CUDA](../sources/prs/vllm/PR-34374.md), [[Model Bash] DeepSeek R1 BF16 Min Latency QKV A GEMM (0.5% E2E Speedup)](../sources/prs/vllm/PR-34758.md), [[Bugfix] Gate 256-bit instructions to CUDA 12.9+](../sources/prs/vllm/PR-34791.md), [[Perf] Enable FlashInfer DeepGEMM swapAB on SM90 by default](../sources/prs/vllm/PR-34924.md), [[Bug][DSV3.2] Always prepare metadata for DeepGEMM Sparse Attention](../sources/prs/vllm/PR-35075.md), [[Performance] Cublas Bf16 Gate with Fp32 Output](../sources/prs/vllm/PR-35121.md), [[Bugfix] Fix DSV3 kernels breaking _C and _moe_C on unsupported arches](../sources/prs/vllm/PR-35123.md), [[Feat] Add CUDA torch fallbacks for fp8_mqa_logits/fp8_paged_mqa_logits_torch function](../sources/prs/vllm/PR-35271.md), [[MoE][Perf] Wrap DSV3 QKVAProj GEMM in custom op for torch.compile](../sources/prs/vllm/PR-35751.md), [[Mamba] Add stochastic rounding support](../sources/prs/vllm/PR-35753.md), [[MoE] Move PF Methods to Folder](../sources/prs/vllm/PR-35927.md), [[Kernel] Add FlashInfer MoE A2A Kernel](../sources/prs/vllm/PR-36022.md), [[Bugfix] Warm up Triton autotuner for GDN layers during V1 profiling](../sources/prs/vllm/PR-36599.md), [[GDN] add a config for gdn kernel selection](../sources/prs/vllm/PR-36647.md), [[Bug] Fix FlashInfer MNNVL socket collisions under concurrent vLLM jobs](../sources/prs/vllm/PR-36674.md), [Update Flashinfer to 0.6.6](../sources/prs/vllm/PR-36768.md), [[Bugfix] Fix FlashInfer GDN warmup ValueError on SM90 GPUs](../sources/prs/vllm/PR-36876.md), [[Kernel] Add gpt-oss Router GEMM kernel](../sources/prs/vllm/PR-37205.md), [[UX] Add flashinfer-cubin as CUDA default dep](../sources/prs/vllm/PR-37233.md), [refactor: abstract deepgemm support into platform](../sources/prs/vllm/PR-37519.md), [[Bug] Fix fp8 deepgemm batch invariant](../sources/prs/vllm/PR-37718.md), [[Perf] triton bilinear_pos_embed kernel for ViT](../sources/prs/vllm/PR-37948.md), [[Kernel] Optimize SM120 CUTLASS blockwise FP8 GEMM](../sources/prs/vllm/PR-37970.md), [[Bugfix] Fix DeepGemm E8M0 accuracy degradation for Qwen3.5 FP8 on Blackwell](../sources/prs/vllm/PR-38083.md), [[Kernel] Add swapAB support for SM120 CUTLASS blockwise FP8 GEMM ](../sources/prs/vllm/PR-38325.md), [[CI Bugfix] Pre-download missing FlashInfer headers in Docker build](../sources/prs/vllm/PR-38391.md), [[Bugfix] Enable batch-invariant Triton matmul on all Ampere GPUs (SM 8x) ](../sources/prs/vllm/PR-38427.md), [[Perf] Batch KV cache swap copies via cuMemcpyBatchAsync](../sources/prs/vllm/PR-38460.md), [[Gemma4] Enable Fast Prefill Optimization](../sources/prs/vllm/PR-38879.md), [[Perf] Change Trtllm fp8 MoE to use Shuffled Weights and BlockMajorK Layout](../sources/prs/vllm/PR-38993.md), [[Gemma4] Support quantized MoE ](../sources/prs/vllm/PR-39045.md), [[Refactor] Move NVFP4 GEMM management into NvFp4LinearKernel](../sources/prs/vllm/PR-39129.md), [[Refactor] Move MXFP8 GEMM management into MxFp8LinearKernel](../sources/prs/vllm/PR-39205.md), [Use CU_MEMCPY_SRC_ACCESS_ORDER_ANY for batch KV cache swaps](../sources/prs/vllm/PR-39306.md), [Fix NUMA binding on non-CDMM Grace-Blackwell systems](../sources/prs/vllm/PR-39361.md), [[Perf] Fuse Zero Initializer for FP8 DeepGemm Block Quant Kernel](../sources/prs/vllm/PR-39547.md), [[Bugfix][NIXL] Fix `_logical_to_kernel_block_ids` conversion for non-mamba models](../sources/prs/vllm/PR-39724.md), [[Bugfix] Add Marlin kernel in block scaled mm kernel selection.](../sources/prs/vllm/PR-40105.md), [[MoE] Move cutlass moe to fused_moe/experts/](../sources/prs/vllm/PR-40574.md), [[Kernel][MoE] Support GELU on TRT-LLM NvFP4 fused MoE for Gemma4](../sources/prs/vllm/PR-41050.md), [[DSV4] Fuse norm and router for low latency scenario](../sources/prs/vllm/PR-41263.md), [[Spec Decode] Add Gemma4 MTP speculative decoding support](../sources/prs/vllm/PR-41745.md), [Add NVFP4 all-gather GEMM fusion for AsyncTP](../sources/prs/vllm/PR-41882.md), [[CPU] Add MXFP4 W4A16 MoE support](../sources/prs/vllm/PR-41922.md), [[MoE] Move various experts classes to fused_moe/experts/](../sources/prs/vllm/PR-41979.md), [[Bugfix] Add swiglu limits to deepgemm fp8 methods](../sources/prs/vllm/PR-41986.md), [[6/n] Migrate activation kernels, gptq, gguf, non cutlass w8a8 to libtorch stable ABI (continued)](../sources/prs/vllm/PR-42663.md), [[Kernel] (1/N) Machete - Hopper Optimized Mixed Precision Linear Kernel ](../sources/prs/vllm/PR-7174.md), [[Kernel] (2/N) Machete - Integrate into CompressedTensorsWNA16 and GPTQMarlin](../sources/prs/vllm/PR-7701.md), [DeepGEMM — FP8 GEMM with Fine-Grained Scaling](../wiki/kernels/deepgemm.md), [FP8 Fine-Grained-Scale GEMM](../wiki/kernels/fp8-block-scale-gemm.md), [GPU Mode NVFP4 Gated Dual GEMM](../wiki/kernels/gated-dual-gemm.md), [Grouped GEMM Contracts for MoE and NVFP4](../wiki/kernels/grouped-gemm.md), [NVFP4 GEMM — GPU Mode Problem 2 Contract](../wiki/kernels/nvfp4-gemm.md), [TensorRT-LLM Blackwell FP4 DSA Indexer](../wiki/kernels/tensorrt-llm-blackwell-indexer.md) | +| `gemv` | [Twelve Attempts at an FP4 Kernel](../sources/blogs/amandeep-nvfp4-attempts.md), [NVFP4 GEMV and Improved NVFP4 GEMV](../sources/blogs/simon-nvfp4-gemv.md), [Blackwell NVFP4 Kernel Hackathon Journey](../sources/blogs/yue-nvfp4-hackathon.md), [GPU Mode NVFP4 Hackathon - Problem 1: Batched GEMV](../sources/contests/gpu-mode-nvfp4/problem-1-gemv.md), [[None][feat] sm100 weight-only kernel](../sources/prs/TensorRT-LLM/PR-10190.md), [[None][feat] Enable nvfp4 cuda core for sm120](../sources/prs/TensorRT-LLM/PR-8620.md), [NVFP4 Batched GEMV](../wiki/kernels/nvfp4-gemv.md) | +| `grouped-gemm` | [Anatomy of a Reward Hack](../sources/blogs/gpu-mode-reward-hack.md), [TFLOPS Gap: Why FP4 MoE Kernel Engineering Matters on Blackwell](../sources/blogs/tflops-gap-fp4-moe.md), [FlashInfer MLSys 2026 Track A: FP8 Block-Scale MoE](../sources/contests/flashinfer-mlsys26/track-a-fused-moe.md), [GPU Mode NVFP4 Hackathon - Problem 4: Grouped GEMM](../sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [fix: use SM90ArchSpec instead of SM100ArchSpec in sm90_bf16_k_grouped_gemm](../sources/prs/DeepGEMM/PR-270.md), [[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes](../sources/prs/DeepGEMM/PR-304.md), [ Solving bank conflict via padding and TMA 3D store](../sources/prs/DeepGEMM/PR-78.md), [Use 1D TMA store instead of 3D](../sources/prs/DeepGEMM/PR-83.md), [Use swizzling instead of padding](../sources/prs/DeepGEMM/PR-86.md), [Support TMA multicast on B with m_grouped_gemm_contiguous.](../sources/prs/DeepGEMM/PR-88.md), [[None][perf] Add more optimization options for MOE CuteDSL finalized kernel](../sources/prs/TensorRT-LLM/PR-10042.md), [[TRTLLM-9992][perf] Enable PDL for CuteDSL kernels and overlap MoeOutputMemset](../sources/prs/TensorRT-LLM/PR-10043.md), [[None][feat] CuteDSL MOE FC1 Enhancement](../sources/prs/TensorRT-LLM/PR-10088.md), [[TRTLLM-9831][perf] Enable 2CTA with autotune for CuteDSL MoE and Grouped GEMM optimizations](../sources/prs/TensorRT-LLM/PR-10201.md), [[None] [feat] Add test script and raster M for gather fc1 kernel](../sources/prs/TensorRT-LLM/PR-10429.md), [[TRTLLM-9831][perf] Use TMA.RED to improve effective memory bandwidth](../sources/prs/TensorRT-LLM/PR-10987.md), [[None][feat] CuteDSL MOE: Add raster along M/N support for blockscaled contiguous backbone kernel](../sources/prs/TensorRT-LLM/PR-12079.md), [[None][feat] Add DWDP (Distributed Weight Data Parallelism) support for MoE inference](../sources/prs/TensorRT-LLM/PR-12136.md), [[TRTLLM-11585][feat] Add CUTEDSL moe backend for nemotron-h](../sources/prs/TensorRT-LLM/PR-12884.md), [[TRTLLM-9685] [feat] Add gather fc1 kernel by cuteDSL](../sources/prs/TensorRT-LLM/PR-9618.md), [Hopper Grouped GEMM support for FP8 Accum](../sources/prs/cutlass/PR-2123.md), [Blockwise and Groupwise GEMM for Blackwell and Improvements for Hopper](../sources/prs/cutlass/PR-2139.md), [Support for GEMM-K=0 for Blackwell Grouped GEMMs](../sources/prs/cutlass/PR-2746.md), [[Hopper CuTeDSL] Add grouped GEMM kernel example](../sources/prs/cutlass/PR-3091.md), [Support for Group GEMM in CUTLASS Profiler for GeForce and Spark](../sources/prs/cutlass/PR-3092.md), [perf: accelerate blackwell grouped gemm](../sources/prs/flashinfer/PR-1086.md), [feat: Support MXFP8 x MXFP4 CUTLASS grouped GEMM](../sources/prs/flashinfer/PR-1241.md), [perf: Enable SplitK and fix tile-scheduling for moe fp4 fused moe](../sources/prs/flashinfer/PR-1548.md), [TVM: support TVM binding for GroupedGemm](../sources/prs/flashinfer/PR-1725.md), [feat: unit-test and api change, w4a8 grouped-gemm fused MoE for SM90](../sources/prs/flashinfer/PR-2193.md), [feat: cuteDSL fp4 moe for better DSR1 performance.](../sources/prs/flashinfer/PR-2398.md), [refactor: Port upstream CUTLASS fixes and refactor grouped_gemm_nt_masked GEMM module location](../sources/prs/flashinfer/PR-2503.md), [Support for MXFP4 and NVFP4 group GEMMs on GeForce and Spark](../sources/prs/flashinfer/PR-2738.md), [CuteDSL MoE fix redundant output buffer zeroing](../sources/prs/flashinfer/PR-2811.md), [feat: Add CuTe DSL grouped-gemm + combine fusion support](../sources/prs/flashinfer/PR-2944.md), [[sgl-kernel][1/N]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-11432.md), [[sgl-kernel][6/7]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-15471.md), [[Feature] Apply Cublas Grouped Gemm kernel](../sources/prs/sglang/PR-3629.md), [DeepGemm integrate to gemm](../sources/prs/sglang/PR-4165.md), [support cmake for sgl-kernel](../sources/prs/sglang/PR-4706.md), [[perf] introduce deep gemm group_gemm_masked as gemm](../sources/prs/sglang/PR-5432.md), [[NVIDIA] [3/N] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked ](../sources/prs/sglang/PR-9199.md), [[Kernel] CUTLASS grouped gemm fp8 MoE kernel](../sources/prs/vllm/PR-13972.md), [permute/unpermute kernel for moe optimization](../sources/prs/vllm/PR-14568.md), [[Kernel] Integrate CUTLASS MoE kernel with PPLX](../sources/prs/vllm/PR-18762.md), [[feat]: CUTLASS block scaled group gemm for SM100](../sources/prs/vllm/PR-19757.md), [[Performance] Performance improvements in non-blockwise fp8 CUTLASS MoE](../sources/prs/vllm/PR-20762.md), [[Kernel] CUTLASS MoE FP8: Integrate cuda moe permute/unpermute](../sources/prs/vllm/PR-23045.md), [[Kernel][B200] mxfp4 fused cutlass moe](../sources/prs/vllm/PR-23696.md), [[MoE] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked](../sources/prs/vllm/PR-25990.md), [[MoE] Move cutlass moe to fused_moe/experts/](../sources/prs/vllm/PR-40574.md), [DeepGEMM — FP8 GEMM with Fine-Grained Scaling](../wiki/kernels/deepgemm.md), [FlashInfer Track A FP8 Block-Scale MoE](../wiki/kernels/fused-moe.md), [Grouped GEMM Contracts for MoE and NVFP4](../wiki/kernels/grouped-gemm.md) | +| `linear-attention` | [Gated Delta Networks](../sources/blogs/gated-delta-net.md), [Qwen3-Next: Hybrid GDN+MoE Architecture on NVIDIA Blackwell](../sources/blogs/qwen3-next-architecture.md), [FlashInfer MLSys 2026 Track C: Gated Delta Net](../sources/contests/flashinfer-mlsys26/track-c-gated-delta-net.md), [Tiled Flash Linear Attention (TFLA)](../sources/docs/tfla.md), [Gated Delta Net — Linear Attention](../wiki/kernels/gated-delta-net.md) | +| `mla` | [FlashMLA — Multi-head Latent Attention](../sources/blogs/flashmla.md), [K-Search: LLM Kernel Generation via Co-Evolving Intrinsic World Model](../sources/blogs/k-search-kernel-generation.md), [DeepSeek-V3.2-Exp in vLLM: Fine-Grained Sparse Attention in Action](../sources/blogs/vllm-deepseek-v3-sparse-attention.md), [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [DeepSeek-V2: A Strong, Economical, and Efficient Mixture-of-Experts Language Model](../sources/docs/deepseek-v2-mla.md), [[#11694][feat] AutoDeploy: Improve the piecewise CG memory usage](../sources/prs/TensorRT-LLM/PR-11993.md), [[TRTLLM-11285][feat] Fuse indexer wk + weights_proj into single GEMM in TF32 for DS-V3.2](../sources/prs/TensorRT-LLM/PR-12055.md), [[https://nvbugs/5983390][perf] Split MLA DSA custom op for piecewise CUDA graph capture](../sources/prs/TensorRT-LLM/PR-12503.md), [[#12634][feat] AutoDeploy: Support rank 256 MLA in flashinfer_mla](../sources/prs/TensorRT-LLM/PR-12519.md), [[#13580][fix] AutoDeploy: Support Gemma3n/4 E2B variants](../sources/prs/TensorRT-LLM/PR-13630.md), [[None][feat] Add DeepSeekV4 attention kernels](../sources/prs/TensorRT-LLM/PR-13652.md), [[None][fix] Add SPDX Apache-2.0 headers to auto_deploy test files](../sources/prs/TensorRT-LLM/PR-14193.md), [[TRTLLM-8535][feat] Support DeepSeek V3.2 with FP8 + BF16 KV cache/NVFP4 + BF16 KV cache](../sources/prs/TensorRT-LLM/PR-8405.md), [Flash MLA support](../sources/prs/cutlass/PR-2130.md), [Flash MLA Support - Step 2](../sources/prs/cutlass/PR-2134.md), [[ex77] fix mla split; add fwd lse; add bwd varlen](../sources/prs/cutlass/PR-2366.md), [Example 77 add blackwell flash-attention bwd for MLA shape](../sources/prs/cutlass/PR-2466.md), [Add Blackwell MLA forward (shape: d=192, dv=128) implementation](../sources/prs/cutlass/PR-2472.md), [[Cute,Sm100,Fwd] add MLA 64/512 with topk sparsity for MQA 128 heads](../sources/prs/flash-attention/PR-2441.md), [bugfix: import wrapper of mla decode](../sources/prs/flashinfer/PR-1013.md), [misc: fix instrument code for mla profiler](../sources/prs/flashinfer/PR-1014.md), [[nvidia] initial support for blackwell kernels](../sources/prs/flashinfer/PR-1039.md), [bugfix: temporally disable split-kv in blackwell mla](../sources/prs/flashinfer/PR-1055.md), [[Feature] Support PDL for batch Prefill and Decode](../sources/prs/flashinfer/PR-1117.md), [feat: add trtllm-gen mla cubin](../sources/prs/flashinfer/PR-1222.md), [feat: enable trtllm-gen mla MTP](../sources/prs/flashinfer/PR-1258.md), [refactor: refactor trtllm-gen attention kernel integration code](../sources/prs/flashinfer/PR-1289.md), [Refactor Fused Moe Module](../sources/prs/flashinfer/PR-1309.md), [minor: add trtllm_gen_mla benchmark](../sources/prs/flashinfer/PR-1316.md), [feat: Fused rope fp8 quantize kernel for MLA](../sources/prs/flashinfer/PR-1339.md), [fix: fix trtllm-gen mla error on new interface](../sources/prs/flashinfer/PR-1348.md), [feat: enable trtllm-gen attn speculative decoding verify by decode](../sources/prs/flashinfer/PR-1453.md), [feat: add pdl for trtllm-gen attn](../sources/prs/flashinfer/PR-1484.md), [feat: Support fp8 qkv, fp16/bf16 out MHA for trtllm-gen.](../sources/prs/flashinfer/PR-1490.md), [feat: Add fp8-qkv, fp16/bf16 output MHA](../sources/prs/flashinfer/PR-1540.md), [feat: initial support for SM103, SM110, SM120, SM121](../sources/prs/flashinfer/PR-1608.md), [bugfix: Fix FLOPS calculation for bench_trtllm_gen_mla.py](../sources/prs/flashinfer/PR-1640.md), [fix: zero-init workspace buffer for trtllm-gen fmha](../sources/prs/flashinfer/PR-1643.md), [Add benchmark for MLARopeQuantize](../sources/prs/flashinfer/PR-1656.md), [test: skip the unsupported test cases for sm120/121](../sources/prs/flashinfer/PR-1710.md), [feat: add xqa fp8 mha and fp8 kv cache](../sources/prs/flashinfer/PR-1769.md), [Waive / disable test_mla_decode_kernel.py::test_mla_decode_kernel for not sm80 ](../sources/prs/flashinfer/PR-1771.md), [MLA RoPE + quantization fused kernel: shape generalization for MHA / GQA](../sources/prs/flashinfer/PR-1924.md), [minor fix for xqa](../sources/prs/flashinfer/PR-1994.md), [feat: add xqa backend and completes NHD/HND coverage for trtllm-gen/xqa backend](../sources/prs/flashinfer/PR-2001.md), [use scalar for kv_scale in xqa](../sources/prs/flashinfer/PR-2033.md), [feat: Add flashinfer.rope.rope_quantize_fp8_append_paged_kv_cache (fused RoPE + Q + KV cache, supports MLA/GQA/MHA) ](../sources/prs/flashinfer/PR-2037.md), [feat: add xqa mla backend](../sources/prs/flashinfer/PR-2053.md), [[API change] Allow using torch.Tensor for scales for trtllm-gen attention](../sources/prs/flashinfer/PR-2084.md), [[DSR1] Added MLA test](../sources/prs/flashinfer/PR-2100.md), [add tensor scale input for xqa](../sources/prs/flashinfer/PR-2110.md), [feature: make the LSE returned by MLA support base 2 or e #2113](../sources/prs/flashinfer/PR-2114.md), [update xqa license](../sources/prs/flashinfer/PR-2117.md), [feat: support variable sequence length in decode kernel of trtllm-gen attention](../sources/prs/flashinfer/PR-2125.md), [fix flaky xqa test](../sources/prs/flashinfer/PR-2126.md), [feat: add trtllm-gen per-tensor sparseMla kernels.](../sources/prs/flashinfer/PR-2138.md), [refactor: Move mla code from decode.py to mla.py and add to documentation](../sources/prs/flashinfer/PR-2163.md), [[feat] Integrate SGLang concat_mla_k kernel into flashinfer](../sources/prs/flashinfer/PR-2237.md), [Support both 3D and 4D kv_cache shapes in MLA APIs](../sources/prs/flashinfer/PR-2334.md), [feat: Add TRTLLM-Gen Skip-Softmax kernels for prefill and decode](../sources/prs/flashinfer/PR-2477.md), [fix: Fix memory bandwidth calculation in MLA benchmarks](../sources/prs/flashinfer/PR-2479.md), [feat: Enable TRTLLM-Gen Skip-Softmax attention for MLA](../sources/prs/flashinfer/PR-2547.md), [support qk_nope_head_dim for 192 check for GLM-5](../sources/prs/flashinfer/PR-2607.md), [fix: add SM121 support to SM120 version guards](../sources/prs/flashinfer/PR-2631.md), [feat: add support for more MLA head dimensions](../sources/prs/flashinfer/PR-2677.md), [Add NVFP4 KV cache quantization support for SM100](../sources/prs/flashinfer/PR-2702.md), [Add cute dsl mla decode op](../sources/prs/flashinfer/PR-2743.md), [[Spark unit test debugging] Fix for tests/attention/test_trtllm_gen_mla.py](../sources/prs/flashinfer/PR-2750.md), [feat: Expose TRT-LLM FMHA style paged KV Cache and page table layout](../sources/prs/flashinfer/PR-2770.md), [feat: FP8 output support for CUTLASS MLA paged attention](../sources/prs/flashinfer/PR-2779.md), [[CuTe DSL] Add modular FMHA prefill and MLA decode attention kernels](../sources/prs/flashinfer/PR-2805.md), [[Fmha] Sparse MLA decode kernel selection heuristics](../sources/prs/flashinfer/PR-2836.md), [[fix] bugfix 2856: Fix pre-allocated out shape check in trtllm_batch_decode_with_kv_cache_mla for q_len_per_req > 1](../sources/prs/flashinfer/PR-2876.md), [feat: add pdl support for cute dsl mla decode kernel support](../sources/prs/flashinfer/PR-2901.md), [Support lse in trtllm paged attn kernels](../sources/prs/flashinfer/PR-3058.md), [feat: Enable FP8 (E4M3/E5M2) in concat_mla_k for optimize long-context prefill performance and refactor type dispatch for BF16/FP16](../sources/prs/flashinfer/PR-3129.md), [feat(trace): embed runnable init() in every TraceTemplate](../sources/prs/flashinfer/PR-3221.md), [Support Kimi K2.5 H64 CuTe DSL MLA decode](../sources/prs/flashinfer/PR-3235.md), [bugfix: MLA decode should multiply sm_scale by math::log2e](../sources/prs/flashinfer/PR-787.md), [fix rope logic in mla decoding](../sources/prs/flashinfer/PR-793.md), [perf: memory efficient deepseek mla fused page-attention kernel](../sources/prs/flashinfer/PR-804.md), [bugfix: mla page-attention kernel for different page sizes](../sources/prs/flashinfer/PR-810.md), [feat: unlocking MLA for A100](../sources/prs/flashinfer/PR-812.md), [feat: unlock MLA attention for sm89 (L40/L40s/4090)](../sources/prs/flashinfer/PR-814.md), [bugfix: fix the behavior of mla plan function when provided with host tensors](../sources/prs/flashinfer/PR-816.md), [bugfix: bugfix on sm89 MLA](../sources/prs/flashinfer/PR-821.md), [perf: MLA decode kernel implemented by CuTe targeted to SM80](../sources/prs/flashinfer/PR-844.md), [misc: Remove duplicate param set in MLA kernel](../sources/prs/flashinfer/PR-850.md), [unittest: add MLA test cases where kv_len is evenly divided by page_size.](../sources/prs/flashinfer/PR-861.md), [perf: dynamic split-k for MLA](../sources/prs/flashinfer/PR-863.md), [bugfix: fix the behavior of MLA kernel when kv-length is 0](../sources/prs/flashinfer/PR-868.md), [perf: FlashAttention-3 style MLA PageAttention](../sources/prs/flashinfer/PR-887.md), [feat - support mla kvcache store](../sources/prs/flashinfer/PR-888.md), [perf: fix MLA split-k performance bug](../sources/prs/flashinfer/PR-898.md), [perf: tweak the pipeline design of mla kernel](../sources/prs/flashinfer/PR-901.md), [feat: flashinfer intra-kernel profiler](../sources/prs/flashinfer/PR-913.md), [bugfix: fix potential issues of FA3 template loading nans for PageAttention](../sources/prs/flashinfer/PR-945.md), [perf: Use 2WG pipeline design for MLA implementation on Hopper](../sources/prs/flashinfer/PR-952.md), [perf: reduce torch.library dispatch overhead](../sources/prs/flashinfer/PR-968.md), [perf: Fix python API overhead when CUDAGraph is not enabled](../sources/prs/flashinfer/PR-969.md), [perf: prefetch page indices for mla kernel](../sources/prs/flashinfer/PR-991.md), [3rdparty: upgrade cutlass to 3.9](../sources/prs/flashinfer/PR-997.md), [Disable kernel cutlass_mla_decode on SM103](../sources/prs/sglang/PR-10058.md), [feat: Add FP4 (E2M1) KV Cache Support with Quantization Utilities for MLA](../sources/prs/sglang/PR-10078.md), [[Feature] Add MLAProcess for DeepSeek MLA on NPU](../sources/prs/sglang/PR-10130.md), [Fix chunked prefix cache for nvfp4](../sources/prs/sglang/PR-10180.md), [Enable trtllm mla prefix extend](../sources/prs/sglang/PR-10526.md), [[sgl-kernel] Optimize concat_mla_k kernel](../sources/prs/sglang/PR-10543.md), [Fuse quantize and rope in trtllm_mla MTP](../sources/prs/sglang/PR-10779.md), [disable sm100 for FlashMLA and fast-hadamard-transform in cuda12.6.1](../sources/prs/sglang/PR-11274.md), [[DeepseekV32] Enable flashmla_prefill kernel with fp8 kvcache](../sources/prs/sglang/PR-11655.md), [Use trtllm_mla decode kernel for draft extend in speculative decoding](../sources/prs/sglang/PR-11664.md), [Support running FP4 Deepseek on SM120.](../sources/prs/sglang/PR-11708.md), [[sgl-kernel] support flashmla libtorch](../sources/prs/sglang/PR-11717.md), [Fixed aarch64 flash-mla](../sources/prs/sglang/PR-12009.md), [[DeepseekV32]: use `_concat_mla_absorb_q_general` to replace `torch.cat`](../sources/prs/sglang/PR-12215.md), [[Deepseek V3.2] Enable flashmla_auto with MTP](../sources/prs/sglang/PR-12294.md), [fix seqlen bug for trtllm_mla's draft_extend](../sources/prs/sglang/PR-12295.md), [perf: trtllm mla performance minor improvements](../sources/prs/sglang/PR-12435.md), [[Fix] `concat_mla_absorb_q_kernel` fails for long inputs](../sources/prs/sglang/PR-12453.md), [[Ascend] Support enable-mixed-chunk in non-MLA scenarios](../sources/prs/sglang/PR-12491.md), [[sgl-kernel] support custom fp8 flashmla kernel](../sources/prs/sglang/PR-13087.md), [support mtp with deepseek r1 nvfp4 model](../sources/prs/sglang/PR-13115.md), [Fix target MLA with eagle3 support for PD disaggregation](../sources/prs/sglang/PR-13555.md), [[BugFix] fix prefixcache performance and accuracy on ascend](../sources/prs/sglang/PR-13573.md), [[ROCM] Optimized deepseek-r1 fp8 model with + triton_gemm_a8w8 + batch_gemm_a8w8 + fused set_mla_kv_buffer kernel](../sources/prs/sglang/PR-13617.md), [fix trtllm mla spec](../sources/prs/sglang/PR-13738.md), [[DeepSeek v3.2] opt Context Parallelism: support fused moe, multi batch and fp8 kvcache](../sources/prs/sglang/PR-13959.md), [Add Mistral Large 3 support.](../sources/prs/sglang/PR-14213.md), [[bug fix] fix ima with get_mla_kv_buffer_kernel overflow](../sources/prs/sglang/PR-14224.md), [Support FP8 MLA prefill and 128k context.](../sources/prs/sglang/PR-14395.md), [[NPU] perf update with kvcache nz & w4a8 quant](../sources/prs/sglang/PR-14423.md), [Add Mistral Large 3 Eagle Support](../sources/prs/sglang/PR-14466.md), [Fix accuracy issue when using a16w16 mla_decode_fwd](../sources/prs/sglang/PR-14936.md), [[sgl-kernel] Update flashmla to include fp8 sparse_mla optimizations](../sources/prs/sglang/PR-15242.md), [[NPU]DeepSeek-V3.2 support npu mlaprolog](../sources/prs/sglang/PR-15381.md), [Optimize FP8 MLA KV cache writes with Triton kernel](../sources/prs/sglang/PR-15522.md), [Fix BatchMLAPagedAttentionWrapper query/qo_inptr mismatch for EAGLE](../sources/prs/sglang/PR-15601.md), [Bugfix for ds-vl2](../sources/prs/sglang/PR-15894.md), [enhance accuracy for model kimi-vl-instruct-a3b](../sources/prs/sglang/PR-16076.md), [[NPU]bugfix: fix for dsv3.2 and dsvl2](../sources/prs/sglang/PR-17007.md), [Disable mla persistent kernel when not using fp8 kv_cache](../sources/prs/sglang/PR-17327.md), [[NPU] enhance accuracy for model kimi-vl-a3b-instruct](../sources/prs/sglang/PR-17480.md), [Make flashMLA work on: Cu13, B300](../sources/prs/sglang/PR-17600.md), [[Move sgl-kernel Kernel to JIT] Add JIT concat MLA kernels](../sources/prs/sglang/PR-17889.md), [[Hicache & JIT_kernel] Support page first layout & mla jit kernel](../sources/prs/sglang/PR-18311.md), [Nsa trtllm mla sparse fp8 support with Deepseek v3.2 NVFP4](../sources/prs/sglang/PR-18389.md), [[AMD] Update aiter to v0.1.10.post2](../sources/prs/sglang/PR-18423.md), [[AMD] DSR1/V3 use fp8 bmm in MLA for MI300X](../sources/prs/sglang/PR-18624.md), [[sgl-kernel] rebase FlashMLA 0217](../sources/prs/sglang/PR-18902.md), [Support skip-softmax attention](../sources/prs/sglang/PR-19089.md), [[Feature] add feature mla_ag_after_qlora for dsv3.2](../sources/prs/sglang/PR-19428.md), [[NPU] bugs fix for Deepseek models](../sources/prs/sglang/PR-19544.md), [Fix MLA decode path returning unwritten (padded) rows](../sources/prs/sglang/PR-19902.md), [[AMD] Fix FP8 assertion failure in aiter MLA decode by falling back to self.k_scale](../sources/prs/sglang/PR-19935.md), [Fix streaming session with paged KV cache (SWA/MLA)](../sources/prs/sglang/PR-20070.md), [[Benchmark] use flashinfer bench_gpu_time instead of triton do_bench](../sources/prs/sglang/PR-20305.md), [[AMD][AITER] Guard _use_mla_ps_kernel with self.use_mla in draft_extend_v2 paths](../sources/prs/sglang/PR-20409.md), [Support Triton MLA FP8 KV cache](../sources/prs/sglang/PR-20479.md), [FIX: (NSA) Compute topk_indices_offset when NSA prefill flashmla_sparse is used with FP8 KV cache](../sources/prs/sglang/PR-20606.md), [[AMD]: Support MLA with nhead<16 and FP8 KV cache for TP=8 (Kimi K2.5…](../sources/prs/sglang/PR-21213.md), [Refactor JIT kernel CI to use run_suite.py registration system](../sources/prs/sglang/PR-21239.md), [Migrate all callers from /get_server_info to /server_info](../sources/prs/sglang/PR-21463.md), [[AMD] Enable FP8 KV cache and FP8 attention kernel for NSA on MI300/MI355 with TileLang backend](../sources/prs/sglang/PR-21511.md), [test: point DSV3 int8 MLA CI models to lmsys Hugging Face org](../sources/prs/sglang/PR-21561.md), [[DSA] Support trtllm sparse mla kernel for prefill batches ](../sources/prs/sglang/PR-21783.md), [[Bugfix] Fix CUDA graph replay issues in trtllm_mla draft_extend](../sources/prs/sglang/PR-21987.md), [[NPU] enable mla prepare fused kernel only when being mla attn](../sources/prs/sglang/PR-22024.md), [[Disagg][NIXL] Fix heterogeneous TP KV transfer for non-MLA models (same logic with mooncake, Step 1/2 for Qwen3.5 support)](../sources/prs/sglang/PR-22145.md), [[Lora] Lora quat info re-factor and support deepseekv3 mla lora](../sources/prs/sglang/PR-22323.md), [[DSA] Hopper FP8 FlashMLA KV padding](../sources/prs/sglang/PR-22372.md), [[Fix] Fix accuracy bug in Flashmla sparse MLA kernel](../sources/prs/sglang/PR-22723.md), [Support spec v2 for FlashMLA speculative decoding](../sources/prs/sglang/PR-24640.md), [[attn backend] Integrate tokenspeed_mla prefill/decode kernels (fp8 kv cache, blackwell)](../sources/prs/sglang/PR-24925.md), [Amd/deepseek v4 rebase main 0509](../sources/prs/sglang/PR-24933.md), [[Intel GPU] Enable DeepSeek V4 Inference on XPU](../sources/prs/sglang/PR-25336.md), [Support draft extend cuda graph for tokenspeed_mla attention backend](../sources/prs/sglang/PR-25489.md), [amd/deepseek_v4 27/N [fix] Reduce Triton autotune configs for faster first-time server launch](../sources/prs/sglang/PR-25554.md), [[Refactor] Rename NSA → DSA: user-facing aliases, file/class/import rename](../sources/prs/sglang/PR-25821.md), [feat: integrate gemm_fp8 kernel into gemm](../sources/prs/sglang/PR-3056.md), [feat: support flashinfer mla attention for deepseek v3](../sources/prs/sglang/PR-3550.md), [feat: support flashinfer mla with prefix cache](../sources/prs/sglang/PR-3643.md), [Add fast decode plan for flashinfer mla](../sources/prs/sglang/PR-3987.md), [Hierarchical Caching supports MLA](../sources/prs/sglang/PR-4009.md), [[Revision] Add fast decode plan for flashinfer mla ](../sources/prs/sglang/PR-4012.md), [[Fix] Fix raw_bs bug when using flashinfer mla and eagle](../sources/prs/sglang/PR-4557.md), [avoid cudaStreamSynchronize in DeepSeekV2AttentionMLA](../sources/prs/sglang/PR-4577.md), [Blackwell Cutlass MLA kernel](../sources/prs/sglang/PR-5142.md), [feat: use fa3 mla by default on hopper](../sources/prs/sglang/PR-5210.md), [fix: solve cu118 issue for cutlass mla](../sources/prs/sglang/PR-5331.md), [[PD Bug] fix MLA get_contiguous_buf_infos error](../sources/prs/sglang/PR-5384.md), [Add Cutlass MLA attention backend](../sources/prs/sglang/PR-5390.md), [[PD] Fix dynamic port support and MLA buffer for Mooncake](../sources/prs/sglang/PR-5415.md), [BLackwell cutlass mla: Add check for bad page size/block num combinations](../sources/prs/sglang/PR-5431.md), [[perf] introduce deep gemm group_gemm_masked as gemm](../sources/prs/sglang/PR-5432.md), [Avoid computing lse in Ragged Prefill when there's no prefix.](../sources/prs/sglang/PR-5476.md), [Fuse MLA set kv cache kernel](../sources/prs/sglang/PR-5748.md), [opt flashinfer mla cat](../sources/prs/sglang/PR-5822.md), [Cutlass MLA decode - fix dtype error](../sources/prs/sglang/PR-5868.md), [[Fix] Fix a bug for flashmla to run R1 model](../sources/prs/sglang/PR-5875.md), [KV‑Cache (MHA, MLA): add missing start_layer / end_layer fields to MHATokenToKVPoolHost and MLATokenToKVPoolHost](../sources/prs/sglang/PR-6016.md), [feat: mtp support dp-attention](../sources/prs/sglang/PR-6081.md), [Cutlass MLA: Disable split kv due to https://github.com/NVIDIA/cutlass/issues/2274](../sources/prs/sglang/PR-6101.md), [fix: fix MLA for ShardedModelLoader/RemoteModelLoader](../sources/prs/sglang/PR-6287.md), [[PD] Add different TP sizes support for no-MLA models](../sources/prs/sglang/PR-6793.md), [CPU: map changes from developing branch in sgl-kernel](../sources/prs/sglang/PR-6833.md), [[perf][sgl-kernel] extend cutlass_mla_decode to support num_head < 128](../sources/prs/sglang/PR-6929.md), [[Feature] Support Flashinfer fmha on Blackwell](../sources/prs/sglang/PR-6930.md), [Fix cutlass MLA gets almost zero accuracy](../sources/prs/sglang/PR-6998.md), [Tiny fix cutlass_mla_get_workspace_size stub incorrect signature](../sources/prs/sglang/PR-7057.md), [chore: upgrade sgl-kernel v0.1.8.post2](../sources/prs/sglang/PR-7186.md), [Fix grammar abort & Minor style fixes](../sources/prs/sglang/PR-7204.md), [Minor style and doc fix](../sources/prs/sglang/PR-7228.md), [Fix CPU offloading for MLA memory pool](../sources/prs/sglang/PR-7409.md), [TRTLLM-MLA FP8 path](../sources/prs/sglang/PR-8638.md), [fix mooncake store mla zero copy meta](../sources/prs/sglang/PR-9678.md), [move is_sm90_supported/is_sm100_supported to python/sglang/srt/utils.py](../sources/prs/sglang/PR-9679.md), [[CPU] Add FP8 Bmm support](../sources/prs/sglang/PR-9744.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [[CUDA] Support int4 `T.gemm`](../sources/prs/tilelang/PR-2063.md), [[Perf] Enable fast math in sparse MLA example](../sources/prs/tilelang/PR-2219.md), [[Attention] MLA decode optimizations](../sources/prs/vllm/PR-12528.md), [[Attention] Deepseek v3 MLA support with FP8 compute](../sources/prs/vllm/PR-12601.md), [[Attention] MLA with chunked prefill](../sources/prs/vllm/PR-12639.md), [Disable chunked prefill and/or prefix caching when MLA is enabled ](../sources/prs/vllm/PR-12642.md), [[AMD][ROCm] Enable DeepSeek model on ROCm](../sources/prs/vllm/PR-12662.md), [[Perf] Mem align KV caches for CUDA devices (MLA perf improvement)](../sources/prs/vllm/PR-12676.md), [Squelch MLA warning for Compressed-Tensors Models](../sources/prs/vllm/PR-12704.md), [[VLM] Add MLA with pure RoPE support for deepseek-vl2 models](../sources/prs/vllm/PR-12729.md), [[Kernel] Make rotary_embedding ops more flexible with input shape](../sources/prs/vllm/PR-12777.md), [[Attention] Use FA3 for MLA on Hopper](../sources/prs/vllm/PR-12807.md), [Expand MLA to support most types of quantization](../sources/prs/vllm/PR-13181.md), [[Bugfix] Massage MLA's usage of flash attn for RoCM](../sources/prs/vllm/PR-13310.md), [[Bugfix] Fix max_num_batched_tokens for MLA](../sources/prs/vllm/PR-13620.md), [[core] Perf improvement for DSv3 on AMD GPUs](../sources/prs/vllm/PR-13718.md), [[Bugfix] Support MLA for CompressedTensorsWNA16](../sources/prs/vllm/PR-13725.md), [[Kernel] FlashMLA integration](../sources/prs/vllm/PR-13747.md), [[Attention] MLA support for V1](../sources/prs/vllm/PR-13789.md), [[ROCm] Disable chunked prefill/prefix caching when running MLA on non-cuda platforms](../sources/prs/vllm/PR-13844.md), [[Attention] Flash MLA for V1](../sources/prs/vllm/PR-13867.md), [Fix mla prefill context performance](../sources/prs/vllm/PR-13897.md), [dynamic distpatch of fp8 kernels](../sources/prs/vllm/PR-14245.md), [[BugFix] MLA + V1, illegal memory access and accuracy issues](../sources/prs/vllm/PR-14253.md), [[BugFix] Fix prefix caching V0 MLA](../sources/prs/vllm/PR-14255.md), [[Attention] FlashAttn MLA](../sources/prs/vllm/PR-14258.md), [[ROCm] Enable chunked prefill/paged attention in MLA on ROCm](../sources/prs/vllm/PR-14316.md), [[Perf] Reduce MLA CPU overheads in V1](../sources/prs/vllm/PR-14384.md), [[Bugfix] DeepSeek Accuracy](../sources/prs/vllm/PR-14476.md), [[Perf] Improve MLA on V1](../sources/prs/vllm/PR-14540.md), [[BugFix][TritonMLA] Process weights after model loading for GGUF](../sources/prs/vllm/PR-14555.md), [[Attention] Flash Attention 3 - fp8](../sources/prs/vllm/PR-14570.md), [[Kernel][CPU] CPU MLA](../sources/prs/vllm/PR-14744.md), [[Attention] MLA get rid of materialization](../sources/prs/vllm/PR-14770.md), [[Attention] Get rid of mla cache alignment](../sources/prs/vllm/PR-14842.md), [[V1] Default MLA to V1](../sources/prs/vllm/PR-14921.md), [[BugFix] Fix nightly MLA failure (FA2 + MLA chunked prefill, i.e. V1, producing bad results)](../sources/prs/vllm/PR-15492.md), [[Bugfix] Fix cache block size calculation for CPU MLA](../sources/prs/vllm/PR-15848.md), [[FEAT][ROCm]: Support AITER MLA](../sources/prs/vllm/PR-15893.md), [Modularize fused experts and integrate PPLX kernels](../sources/prs/vllm/PR-15956.md), [[NVIDIA] Support Cutlass MLA for Blackwell GPUs](../sources/prs/vllm/PR-16032.md), [[Kernel] support merge_attn_states CUDA kernel, 3x speedup](../sources/prs/vllm/PR-16173.md), [[MLA] Simplification to batch P/D reordering](../sources/prs/vllm/PR-16673.md), [[V1] V1 FlashInfer Attention](../sources/prs/vllm/PR-16684.md), [[torch.compile][ROCm] Fuse quantization onto attention using a torch.compile pass](../sources/prs/vllm/PR-16756.md), [[Bugfix] Triton FA function takes no keyword arguments](../sources/prs/vllm/PR-16902.md), [[Bugfix] Add contiguous call inside rope kernel wrapper](../sources/prs/vllm/PR-17091.md), [[BugFix] Fix vllm_flash_attn install issues](../sources/prs/vllm/PR-17267.md), [[Misc][ROCm] Exclude `cutlass_mla_decode` for ROCm build](../sources/prs/vllm/PR-17289.md), [[v1] AttentionMetadata for each layer](../sources/prs/vllm/PR-17394.md), [[v1] Pass BlockTable and KVCacheSpec to AttentionMetadataBuilders](../sources/prs/vllm/PR-17483.md), [[Attention] MLA move o_proj q_proj into cuda-graph region](../sources/prs/vllm/PR-17484.md), [[BugFix] Fix mla cpu - missing 3 required positional arguments](../sources/prs/vllm/PR-17494.md), [[FEAT][ROCm]: Support AITER MLA on V1 Engine](../sources/prs/vllm/PR-17523.md), [[Attention] MLA move rotary embedding to cuda-graph region](../sources/prs/vllm/PR-17668.md), [fix amd triton mla path](../sources/prs/vllm/PR-17871.md), [[Bugfix][ROCm] Fix AITER MLA V1](../sources/prs/vllm/PR-17880.md), [[BugFix][AMD] Compatible patch for AITER lib after 04/20](../sources/prs/vllm/PR-17912.md), [[v1] Support multiple KV cache groups in GPU model runner](../sources/prs/vllm/PR-17945.md), [[BUG] [ROCm] [MLA] Fix variable name bug due to change in variable name in PR #17483](../sources/prs/vllm/PR-17961.md), [[BugFix] FA2 MLA Accuracy Issue](../sources/prs/vllm/PR-18807.md), [[Kernel] Apply torch.Tag.needs_fixed_stride_order only for torch==2.6.0](../sources/prs/vllm/PR-19346.md), [[Bugfix] Enable PP with AITER+V1](../sources/prs/vllm/PR-19822.md), [[Attention] MLA - Flashinfer Ragged Prefill](../sources/prs/vllm/PR-20034.md), [[Bugfix] Fix some narrowing conversion warnings](../sources/prs/vllm/PR-20141.md), [SM100 Cutlass MLA decode with unrestricted num_heads (< 128) for DeepSeek TP](../sources/prs/vllm/PR-20769.md), [[Kernel] Flashinfer MLA (trtllm-gen) decode kernel integration](../sources/prs/vllm/PR-21078.md), [[perf] Add fused MLA QKV + strided layernorm](../sources/prs/vllm/PR-21116.md), [[v1][attention] Support Hybrid Allocator + FlashInfer](../sources/prs/vllm/PR-21412.md), [[BugFix] Fix IMA FlashMLA full cuda-graph and DP + Update FlashMLA](../sources/prs/vllm/PR-21691.md), [[Bugfix] Check NVIDIA artifactory is accessible before using flashinfer cubin kernels](../sources/prs/vllm/PR-21893.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv NVFP4-out Attention Kernel](../sources/prs/vllm/PR-22703.md), [[Bugfix] Fix default enable for CUTLASS MLA on SM100](../sources/prs/vllm/PR-22738.md), [[Log] Warning Once for Cutlass MLA ](../sources/prs/vllm/PR-23137.md), [[ROCm][Aiter] Add triton fp8 bmm kernel for mla](../sources/prs/vllm/PR-23264.md), [[Compile] Fix Compile Warning SM100 Cutlass MLA](../sources/prs/vllm/PR-23287.md), [[Attention][FA3] Update FA3 to include new swizzle optimization](../sources/prs/vllm/PR-23465.md), [[Feature] Support Decode Context Parallel (DCP) for MLA](../sources/prs/vllm/PR-23734.md), [[Model] Add LongCat-Flash ](../sources/prs/vllm/PR-23991.md), [[Kernel] Support decode context parallelism on Blackwell with CUTLASS MLA](../sources/prs/vllm/PR-24385.md), [[Feature] Disallow FlashMLA on Blackwell](../sources/prs/vllm/PR-24521.md), [[Bug] Fix `is_flashmla_supported` Check Error](../sources/prs/vllm/PR-24774.md), [[Bugfix][B200] Fix `cutlass_mla` hang](../sources/prs/vllm/PR-24966.md), [[Attention][DCP] Support DCP with query length > 1 (MTP) with FA3](../sources/prs/vllm/PR-25049.md), [[Bug] Fix `returned_lse` not Defined issue](../sources/prs/vllm/PR-25106.md), [[BugFix] Fix MLA assert with CUTLASS MLA](../sources/prs/vllm/PR-25478.md), [[Bugfix] [B200] cutlass_mla - ensure kv_split == 1 for batch size > 1](../sources/prs/vllm/PR-25509.md), [Fuse RoPE and MLA KV-cache write](../sources/prs/vllm/PR-25774.md), [[Quantization/NVFP4] Speed up TRTLLM NVFP4 MOE weight loading and fix K/V scale loading for MLA Attn](../sources/prs/vllm/PR-25968.md), [[Spec Decode] Enable efficient speculative decoding with FlashInfer-MLA](../sources/prs/vllm/PR-25984.md), [[Attention] Tune CUTLASS MLA num_splits](../sources/prs/vllm/PR-26846.md), [Prefer FlashAttention MLA as default over FlashMLA](../sources/prs/vllm/PR-27363.md), [[Attention] Use sparse prefill kernel for fp8 kv-cache in DeepSeek-v3.2](../sources/prs/vllm/PR-27532.md), [[AMD] Use Decoupled Kernel Block Size to Support AITER MLA block_size=1](../sources/prs/vllm/PR-27715.md), [[Bug] Batch invariant: Fix flash attn MLA `RuntimeError: scheduler_metadata must have shape (metadata_size)`](../sources/prs/vllm/PR-27884.md), [[ROCm][MLA] enable fp8 MLA decode on ROCm](../sources/prs/vllm/PR-28032.md), [[Feature] Prefill Context Parallel (PCP) basic support](../sources/prs/vllm/PR-28718.md), [bugfix: correct attn output with base 2 or e](../sources/prs/vllm/PR-28840.md), [[BugFix] Fix async-scheduling + FlashAttn MLA](../sources/prs/vllm/PR-28990.md), [[DeepSeek + LMCache Multiprocess] handle MLA for deepseek model + LMCache Multiprocess connector](../sources/prs/vllm/PR-29039.md), [[Feature] Batch invariant: Enable `TRITON_MLA` without prefix-caching](../sources/prs/vllm/PR-29125.md), [[perf] Use direct copy (broadcast) instead of cat for k_nope/k_pe in MLA prefill](../sources/prs/vllm/PR-29710.md), [Add Mistral Large 3 and Ministral 3](../sources/prs/vllm/PR-29757.md), [[Perf] Improve fp8 quant in mla; replace ReduceSum with ReduceScatterSum](../sources/prs/vllm/PR-29795.md), [[SpecDecode] Simplified alternative padded-speculation acceptance rate fix](../sources/prs/vllm/PR-29845.md), [[Quantization] fix: overflow with static per-tensor scaling](../sources/prs/vllm/PR-29867.md), [[ROCm] add fallback for aiter fp8 decode mla](../sources/prs/vllm/PR-30005.md), [[ROCm][Bugfix] Add MLACommonMetadata to allowed attention types for speculative decoding](../sources/prs/vllm/PR-30430.md), [[Feature] Add SM103 (Blackwell Ultra) Support to vLLM](../sources/prs/vllm/PR-30484.md), [[SM100] Enable fp8 compute for prefill MLA](../sources/prs/vllm/PR-30746.md), [[perf] Integrate flashinfer concat_mla_k](../sources/prs/vllm/PR-31171.md), [[Bugfix][Hardware][AMD] Fix exception types in AITER MLA FP8 check](../sources/prs/vllm/PR-31177.md), [[SM100] Resubmit FMHA FP8 prefill for MLA](../sources/prs/vllm/PR-31195.md), [[Bugfix][Hardware][AMD] Fix last_page_len calculation in AITER MLA decode](../sources/prs/vllm/PR-31282.md), [fixed mypy warnings for files vllm/v1/attention with TEMPORARY workaround](../sources/prs/vllm/PR-31465.md), [[1/N][Attention] Restructure attention: move files](../sources/prs/vllm/PR-31916.md), [[4/N][Attention] Move MLA common to model_executor](../sources/prs/vllm/PR-32060.md), [[5/N][Attention] Finish eliminating `vllm/attention` folder](../sources/prs/vllm/PR-32064.md), [fix: Add glm4_moe_lite to MLA detection](../sources/prs/vllm/PR-32614.md), [[Bugfix][Attention] Explicitly report support for kv_cache_dtype bfloat16](../sources/prs/vllm/PR-32795.md), [[Attention] FA4 integration](../sources/prs/vllm/PR-32974.md), [[Attention] Use `has_flashinfer` helper](../sources/prs/vllm/PR-33177.md), [Triton MLA perf fixes](../sources/prs/vllm/PR-33529.md), [[Bugfix] Fix sparse MLA metadata building](../sources/prs/vllm/PR-33579.md), [[Bugfix] fix DeepSeek R1 with CUTLASS MLA Broken on B200](../sources/prs/vllm/PR-33637.md), [[Bugfix] Fix DSV3.2 NVFP4](../sources/prs/vllm/PR-33932.md), [Reapply [Attention][FA3] Update FA3 to include new swizzle optimization](../sources/prs/vllm/PR-34043.md), [fix(cpu): fix mla_decode compilation on x86 without AVX512](../sources/prs/vllm/PR-34052.md), [[BugFix] Add support for MTP num_speculative_tokens > 1 with sparse MLA](../sources/prs/vllm/PR-34552.md), [[Kernel] Add FP8 KV cache support to Triton MLA decode attention](../sources/prs/vllm/PR-34597.md), [[Bugfix] Fix MLA attention crash with AWQ/GPTQ quantized models](../sources/prs/vllm/PR-34695.md), [[Attention] Use FA4 for MLA prefill](../sources/prs/vllm/PR-34732.md), [[Model Bash] DeepSeek R1 BF16 Min Latency QKV A GEMM (0.5% E2E Speedup)](../sources/prs/vllm/PR-34758.md), [[Model Bash][DSR1] Add selective dynamic shape marking for CustomOp](../sources/prs/vllm/PR-34900.md), [[Attention][Perf][Kernel] Replace torch.cat with vectorized CUDA kernel MLA query concat - DeepSeek-V3.2](../sources/prs/vllm/PR-34917.md), [[Bug][DSV3.2] Always prepare metadata for DeepGEMM Sparse Attention](../sources/prs/vllm/PR-35075.md), [[Feat] Add CUDA torch fallbacks for fp8_mqa_logits/fp8_paged_mqa_logits_torch function](../sources/prs/vllm/PR-35271.md), [[Bugfix] Fix KV Scale loading for MLA Models](../sources/prs/vllm/PR-35430.md), [[ROCm] Support MLA with nhead<16 and FP8 KV cache for TP=8 (Kimi K2.5/Linear)](../sources/prs/vllm/PR-35850.md), [[Perf] Support FP8 KV cache for Flashinfer MLA Sparse](../sources/prs/vllm/PR-35891.md), [[LMCache] Pass TP size in lookup for MLA multi-reader locking](../sources/prs/vllm/PR-36129.md), [Add 320 dimension size support to MLA](../sources/prs/vllm/PR-36161.md), [[Bugfix][MLA] Add logits size budget to sparse indexer prefill chunking](../sources/prs/vllm/PR-36178.md), [[mla] Support fused FP8/NVFP4 output quantization in MLA attention (#35792)](../sources/prs/vllm/PR-36205.md), [mla: don't update kv cache on dummy forwards](../sources/prs/vllm/PR-36282.md), [Kimi k2.5 MLA based eagle3](../sources/prs/vllm/PR-36361.md), [[Bugfix][Sparse MLA] report indexer CG support properly](../sources/prs/vllm/PR-36519.md), [[ROCm] Utilize persistent MLA kernel from AITER](../sources/prs/vllm/PR-36574.md), [[ROCm][Perf] Allow MTP lens > 1 in Sparse MLA](../sources/prs/vllm/PR-36681.md), [[DSV3.2][MTP] Optimize Indexer MTP handling](../sources/prs/vllm/PR-36723.md), [[Feat][Bugfix] Enable additional dimension for Flashinfer MLA and fix routing dtype](../sources/prs/vllm/PR-36931.md), [[MTP][Sparse MLA] Take advantage of native MTP support in indexer when possible](../sources/prs/vllm/PR-36982.md), [[Bugfix] Fix KV scales inconsistency in fp8 MLA & FlashInfer kv_cache_dtype "auto" leading to gibberish](../sources/prs/vllm/PR-37054.md), [[Bugfix] Disable cross-layer KV cache for MLA attention backends](../sources/prs/vllm/PR-37090.md), [[Benchmark] Improvements to attention benchmark script](../sources/prs/vllm/PR-37115.md), [[XPU] support MLA model on Intel GPU](../sources/prs/vllm/PR-37143.md), [[Misc] Add `float16` to `CacheDType`](../sources/prs/vllm/PR-37199.md), [[Perf] Set Flashinfer sparse MLA as default backend for FP8 kv cache](../sources/prs/vllm/PR-37252.md), [[Perf][Kernel] Persistent TopK scheduler: unified CUDAGraph-safe kernel with dynamic per-row dispatch - DeepSeek-V3.2 DSA decode](../sources/prs/vllm/PR-37421.md), [[BugFix] Allow qk_nope_head_dim=192 in FlashInfer MLA backend checks](../sources/prs/vllm/PR-37475.md), [Fix KV Offloading + MLA AssertionError by using num_kv_heads=1 in cpu…](../sources/prs/vllm/PR-37536.md), [[Performance] Remove unnecessary zero-fill of MLA decode output tensor in Aiter backend](../sources/prs/vllm/PR-37539.md), [[Bugfix][ROCm] Fix lru_cache on paged_mqa_logits_module](../sources/prs/vllm/PR-37547.md), [[Test] Only Run MLA model when user explicitly set for batch invariance](../sources/prs/vllm/PR-37719.md), [[ROCm][perf] fix Aiter sparse MLA with MTP>1](../sources/prs/vllm/PR-37887.md), [[Bugfix][MLA] Change default SM100 MLA prefill backend back to TRT-LLM](../sources/prs/vllm/PR-38562.md), [[ROCm] Fix aiter persistent mode mla with q/o nhead<16 for kimi-k2.5 tp8](../sources/prs/vllm/PR-38615.md), [Fix MLA runs when use_inductor_graph_partition=True](../sources/prs/vllm/PR-38631.md), [[LMCache][MP] optimize save when mla enabled](../sources/prs/vllm/PR-38810.md), [[Attention][MLA] Re-enable FA4 as default MLA prefill backend](../sources/prs/vllm/PR-38819.md), [[Refactor] Improve indexer decode path metadata preparation](../sources/prs/vllm/PR-38865.md), [[XPU] Quick fix for TritonMLA to remove cuda hardcode](../sources/prs/vllm/PR-39088.md), [[Bug] Fix rocm sparse attn indexer issue](../sources/prs/vllm/PR-39225.md), [[MLA] Optimize mla indexer prepare uniform decode for MTP > 1](../sources/prs/vllm/PR-39458.md), [[Mooncake] Fix mixed MLA+Eagle block-size validation](../sources/prs/vllm/PR-39596.md), [add warning when FP8 KV cache misses prefill query quantization](../sources/prs/vllm/PR-39752.md), [[Performance][DSR1]: Fused RoPE+KVCache+q_concat for MLA](../sources/prs/vllm/PR-40392.md), [[MLA Attention Backend] Add TOKENSPEED_MLA backend for DSR1/Kimi K25 prefill + decode on Blackwell](../sources/prs/vllm/PR-41778.md), [[Bugfix] Fix TRTLLM ragged MLA prefill workspace warmup](../sources/prs/vllm/PR-42112.md), [[ROCm] [Bugfix] Fix DeepSeek V4 Functionality and Accuracy](../sources/prs/vllm/PR-42810.md), [[Perf][MLA] Enable FULL cudagraph capture for TRITON_MLA decode](../sources/prs/vllm/PR-42885.md), [[Model Refactoring] Move deepseek_v4_ops to models/deepseek_v4 [3/N]](../sources/prs/vllm/PR-43073.md), [FlashAttention SM100 MLA TopK Sparse Forward](../wiki/kernels/flash-attention-sm100-mla-topk.md), [FlashMLA — Multi-head Latent Attention](../wiki/kernels/flashmla.md), [DeepSeek Sparse Attention / Sparse MLA](../wiki/kernels/sparse-mla.md) | +| `moe` | [K-Search: LLM Kernel Generation via Co-Evolving Intrinsic World Model](../sources/blogs/k-search-kernel-generation.md), [Qwen3-Next: Hybrid GDN+MoE Architecture on NVIDIA Blackwell](../sources/blogs/qwen3-next-architecture.md), [TFLOPS Gap: Why FP4 MoE Kernel Engineering Matters on Blackwell](../sources/blogs/tflops-gap-fp4-moe.md), [FlashInfer MLSys 2026 Track A: FP8 Block-Scale MoE](../sources/contests/flashinfer-mlsys26/track-a-fused-moe.md), [GPU Mode NVFP4 Hackathon - Problem 4: Grouped GEMM](../sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes](../sources/prs/DeepGEMM/PR-304.md), [Sync nv_dev with upstream #316 (Mega MoE optimizations & benchmarks)](../sources/prs/DeepGEMM/PR-328.md), [[None][perf] Add more optimization options for MOE CuteDSL finalized kernel](../sources/prs/TensorRT-LLM/PR-10042.md), [[TRTLLM-9992][perf] Enable PDL for CuteDSL kernels and overlap MoeOutputMemset](../sources/prs/TensorRT-LLM/PR-10043.md), [[None][feat] CuteDSL MOE FC1 Enhancement](../sources/prs/TensorRT-LLM/PR-10088.md), [[TRTLLM-9831][perf] Enable 2CTA with autotune for CuteDSL MoE and Grouped GEMM optimizations](../sources/prs/TensorRT-LLM/PR-10201.md), [[TRTLLM-10147][perf] Balanced random MoE workload generator for CuteDSL kernel UT, autotuner and layerwise benchmark](../sources/prs/TensorRT-LLM/PR-10279.md), [[None] [feat] Add densegemm backend for MoE](../sources/prs/TensorRT-LLM/PR-10479.md), [[None][feat] MiniMax M2 support](../sources/prs/TensorRT-LLM/PR-10532.md), [[None][feat] fuse shared to sparse experts in TRT-LLM Gen MoE](../sources/prs/TensorRT-LLM/PR-11143.md), [[https://nvbugs/5799917][fix] Recover from CUTLASS MoE doActivation perf regression for MXFP4/NVFP4 dtype](../sources/prs/TensorRT-LLM/PR-11165.md), [[None][feat] TRT-LLM Gen MoE finalize kernel optimization](../sources/prs/TensorRT-LLM/PR-11501.md), [[None][feat] Add support for expert_number<=2048 and K<=32](../sources/prs/TensorRT-LLM/PR-11510.md), [[None][fix] Fix int4 awq for sm120/121](../sources/prs/TensorRT-LLM/PR-11561.md), [[TRTLLM-10004][feat] Enable GEMM -> AR with GEMM output in registered buffers](../sources/prs/TensorRT-LLM/PR-11589.md), [[https://nvbugs/5799917][fix] Recover from CUTLASS MoE doActivation perf regression for MXFP4/NVFP4 dtype](../sources/prs/TensorRT-LLM/PR-11733.md), [[https://nvbugs/5885070][fix] fix deepeplowlatency with cutedsl moe backend](../sources/prs/TensorRT-LLM/PR-11769.md), [[TRTLLM-10990][feat] Fuse SwiGLU and quant into shared expert](../sources/prs/TensorRT-LLM/PR-11897.md), [[None][feat] GLM 5 support and DSA MTP fixes](../sources/prs/TensorRT-LLM/PR-11990.md), [[#11694][feat] AutoDeploy: Improve the piecewise CG memory usage](../sources/prs/TensorRT-LLM/PR-11993.md), [[https://nvbugs/5955188][fix] Fix harmony parsers and WAR routing PDL for agentic coding use cases](../sources/prs/TensorRT-LLM/PR-12046.md), [[None][feat] CuteDSL MOE: Add raster along M/N support for blockscaled contiguous backbone kernel](../sources/prs/TensorRT-LLM/PR-12079.md), [[None][feat] Add DWDP (Distributed Weight Data Parallelism) support for MoE inference](../sources/prs/TensorRT-LLM/PR-12136.md), [[None][feat] Add fused allreduce+RMSNorm op and optional residual in …](../sources/prs/TensorRT-LLM/PR-12201.md), [[None][feat] Support update weight for nvfp4](../sources/prs/TensorRT-LLM/PR-12320.md), [[None][perf] add Dynamic SMEM block routing in MOE](../sources/prs/TensorRT-LLM/PR-12456.md), [[None][feat] Optimize mamba SSD prefill and extend flashinfer dispatch](../sources/prs/TensorRT-LLM/PR-12731.md), [[None][feat] Add bf16 trtllm-gen moe support through flashinfer.](../sources/prs/TensorRT-LLM/PR-12738.md), [[TRTLLM-11797][feat] Add cutedsl moe backend supporting for qwen3.5.](../sources/prs/TensorRT-LLM/PR-12799.md), [[TRTLLM-11585][feat] Add CUTEDSL moe backend for nemotron-h](../sources/prs/TensorRT-LLM/PR-12884.md), [[#12784][feat] AutoDeploy: Optimize DeepSeek-R1 model performance](../sources/prs/TensorRT-LLM/PR-12946.md), [[None][fix] Propagate init_load_balancer to DeepGemmFusedMoE in create_moe_backend](../sources/prs/TensorRT-LLM/PR-13207.md), [[TRTLLM-11127][feat] add W4A8_MXFP4_FP8 MoE unit test support](../sources/prs/TensorRT-LLM/PR-13401.md), [[None][perf] Extend customMoeRouting kernel to support Qwen3.5](../sources/prs/TensorRT-LLM/PR-13433.md), [[None][feat] Enable EPLB for DeepSeek-V4](../sources/prs/TensorRT-LLM/PR-13595.md), [[None][feat] Add bf16 trtllm moe through flashinfer.](../sources/prs/TensorRT-LLM/PR-13689.md), [[None][fix] Plumb swiglu_limit through DeepGEMM and TRTLLMGen FP8 fused MoE](../sources/prs/TensorRT-LLM/PR-13767.md), [[None][perf] FC2 DenseGEMM autotune: split-K, swap_ab, fine-grained tuning buckets](../sources/prs/TensorRT-LLM/PR-13833.md), [[None][feat] enable TRTLLM-Gen internal routing](../sources/prs/TensorRT-LLM/PR-13997.md), [[https://nvbugs/6152892][fix] Fix Triton MOE memory free when no swizzling enabled](../sources/prs/TensorRT-LLM/PR-14069.md), [[None][fix] Add SPDX Apache-2.0 headers to auto_deploy test files](../sources/prs/TensorRT-LLM/PR-14193.md), [[None][fix] Avoid dp_size x ep_size double-count in MegaMoEDeepGemm SymmBuffer](../sources/prs/TensorRT-LLM/PR-14213.md), [[None][fix] Update the OSS headers in derived FLA ops and AD modeling code](../sources/prs/TensorRT-LLM/PR-14281.md), [[https://nvbugs/6095421][fix] Update resolve_moe_backend](../sources/prs/TensorRT-LLM/PR-14282.md), [[None][chore] Update Claude Code agents and skills](../sources/prs/TensorRT-LLM/PR-14344.md), [[None][chore] Fix kernel launch param and add TRTLLM MoE backend test](../sources/prs/TensorRT-LLM/PR-7524.md), [[None][fix] Fix and add test for TRTLLM MoE backend](../sources/prs/TensorRT-LLM/PR-7755.md), [[TRTLLM-8637][feat] Optimize the routing kernel for DeepseekV3 (MoE CUTLASS backend); Add support for 384 experts (MoE TRTLLM backend)](../sources/prs/TensorRT-LLM/PR-7761.md), [[None][feat] GPT-OSS Sm120/Sm121 Support](../sources/prs/TensorRT-LLM/PR-7937.md), [[None][fix] Fix the performance issue of FP8 blockwise grouped GEMM when using attention DP](../sources/prs/TensorRT-LLM/PR-8501.md), [[TRTLLM-8827] [feat] Enable low precision alltoall for Cutlass and TRTLLMGen backends](../sources/prs/TensorRT-LLM/PR-8675.md), [[None][feat] Enable EPLB for trtllm-gen and cutlass backend](../sources/prs/TensorRT-LLM/PR-8886.md), [[None][feat] Update TRTLLM MoE cubins; reduce mxfp4 weight padding requirement; tighten TMA bound](../sources/prs/TensorRT-LLM/PR-9025.md), [[None][fix] support topk autotuner input for expert slot per group larger than 32](../sources/prs/TensorRT-LLM/PR-9087.md), [[None][feat] TRT-LLM Gen MoE optimize DeepSeek Fp8 activation kernel](../sources/prs/TensorRT-LLM/PR-9175.md), [[TRTLLM-8958][feat] and [TRTLLM-8960]: create ConfigurableMoE and support TRTLLMGenFusedMoE as backend](../sources/prs/TensorRT-LLM/PR-9486.md), [[TRTLLM-9685] [feat] Add gather fc1 kernel by cuteDSL](../sources/prs/TensorRT-LLM/PR-9618.md), [[https://nvbugs/5726962][feat] Apply fusion for W4AFP8_AWQ MoE](../sources/prs/TensorRT-LLM/PR-9838.md), [[None][feat] Fused kernels (qknormrope + moe routing) and two-model MTP support for glm4moe](../sources/prs/TensorRT-LLM/PR-9852.md), [Blockscaled Ragged Contiguous Grouped Gemm for MoEs](../sources/prs/cutlass/PR-2790.md), [feat: add trtllm all-reduce (non-MoE)](../sources/prs/flashinfer/PR-1096.md), [feat: add trtllm moe_allreduce_fusion](../sources/prs/flashinfer/PR-1108.md), [Add CUTLASS fused moe kernels from TensorRT-LLM.](../sources/prs/flashinfer/PR-1113.md), [bugfix: Fix test and output shape of fp4 quantize](../sources/prs/flashinfer/PR-1114.md), [feat: add trtllm all-reduce fusion](../sources/prs/flashinfer/PR-1131.md), [MNNVL MoE All-to-All Support](../sources/prs/flashinfer/PR-1134.md), [feat: add finalize_moe_allreduce from trtllm](../sources/prs/flashinfer/PR-1159.md), [feat: update non-fused moe](../sources/prs/flashinfer/PR-1161.md), [feat: enable and update all-reduce fused quantization](../sources/prs/flashinfer/PR-1164.md), [feat: trtllm-gen fp8 moe kernels](../sources/prs/flashinfer/PR-1212.md), [Feature/sm100 low latency nvfp4 kernels](../sources/prs/flashinfer/PR-1214.md), [Patch fp8 cubin availability](../sources/prs/flashinfer/PR-1240.md), [Remove sm100+ requirment for trtllm allreduce kernels](../sources/prs/flashinfer/PR-1249.md), [Add shuffle matrix flag](../sources/prs/flashinfer/PR-1272.md), [Bug fix: guard fp8 e8m0 and e2m1 compile ](../sources/prs/flashinfer/PR-1287.md), [Remove FAST_BUILD FLAG for MOE](../sources/prs/flashinfer/PR-1291.md), [Update cutlass fp4 moe kernels](../sources/prs/flashinfer/PR-1294.md), [feat: Add weight layout option for trtllm-gen fused moe](../sources/prs/flashinfer/PR-1297.md), [perfix: use lightweight API to query device property](../sources/prs/flashinfer/PR-1298.md), [Refactor Fused Moe Module](../sources/prs/flashinfer/PR-1309.md), [Support loading autotuned results from json for cutlass fp4 moe backends](../sources/prs/flashinfer/PR-1310.md), [Make Fp8 MoE routing_bias optional](../sources/prs/flashinfer/PR-1319.md), [refactor: Improved metainfo for trtllm-gen kernels](../sources/prs/flashinfer/PR-1328.md), [feature: add fp4 mm using trtllm backend](../sources/prs/flashinfer/PR-1355.md), [Update autotune results for the nvfp4 cutlass moe backends for v0.2.9](../sources/prs/flashinfer/PR-1361.md), [bugfix: fixed cutlass fused moe usage of FP4QuantizationSFLayout::SWIZZLED](../sources/prs/flashinfer/PR-1371.md), [GPT-OSS Support: Add Blackwell MoE mxfp4 implementation from TRTLLM and Attention Sink](../sources/prs/flashinfer/PR-1389.md), [gpt-oss: Add MXFP8 x MXFP4 CUTLASS MOE for SM100 and BF16 x MXFP4 CUTLASS for SM90 + SwigluBias Activation](../sources/prs/flashinfer/PR-1396.md), [Fix trtllm moe launcher local_num_experts](../sources/prs/flashinfer/PR-1398.md), [Add Mxfp4 trtllm-gen moe unit tests](../sources/prs/flashinfer/PR-1399.md), [[bugfix] Fix compilation failure when compiling csrc/trtllm_moe_allreduce_fusion.cu](../sources/prs/flashinfer/PR-1410.md), [Faster weight processing (moe nvfp4)](../sources/prs/flashinfer/PR-1412.md), [Fix redundant kernels in moe](../sources/prs/flashinfer/PR-1428.md), [Remove getEnvEnablePDL in favor of enable_pdl parameter](../sources/prs/flashinfer/PR-1446.md), [bugfix: Verify num_experts greater or equal to local_experts + offset](../sources/prs/flashinfer/PR-1469.md), [feat: Enable multiple fused-moe backends](../sources/prs/flashinfer/PR-1472.md), [tuner: Trtllm-gen Fp4 MoE Autotunner](../sources/prs/flashinfer/PR-1475.md), [fix missing enable_pdl argument in trtllm-gen fp4 moe](../sources/prs/flashinfer/PR-1480.md), [fix: update cutedsl masked moe gemm](../sources/prs/flashinfer/PR-1488.md), [feat: Support fp8 qkv, fp16/bf16 out MHA for trtllm-gen.](../sources/prs/flashinfer/PR-1490.md), [fix: update masked moe gemm fp4 tensor reshape](../sources/prs/flashinfer/PR-1495.md), [Support cuda<12.8 built for trtllm_allreduce_fusion.](../sources/prs/flashinfer/PR-1508.md), [flashinfer_benchmark QoL Improvements and Attention FP8 Support](../sources/prs/flashinfer/PR-1512.md), [Add GeGLU support to trtllm-gen NVFP4 Fused MoE Kernel](../sources/prs/flashinfer/PR-1525.md), [bugfix: Fix compile error for undefined swizzle enum.](../sources/prs/flashinfer/PR-1530.md), [feat: Add fp8-qkv, fp16/bf16 output MHA](../sources/prs/flashinfer/PR-1540.md), [perf: replace cudaGetDeviceProperties with cudaDeviceGetAttribute](../sources/prs/flashinfer/PR-1547.md), [perf: Enable SplitK and fix tile-scheduling for moe fp4 fused moe](../sources/prs/flashinfer/PR-1548.md), [Add mnnvl_moe_alltoallv_prepare_without_allgather](../sources/prs/flashinfer/PR-1550.md), [fix: separate out fp4 lib into sm90 and sm100 versions, add oob checking in fused moe](../sources/prs/flashinfer/PR-1565.md), [bugfix: fix cuda version guard macros](../sources/prs/flashinfer/PR-1571.md), [update trtllm-gen fp4 autotuner and routing](../sources/prs/flashinfer/PR-1573.md), [refactor: Expose calculate_tile_tokens_dim function](../sources/prs/flashinfer/PR-1581.md), [bugfix: Fix arg passing to TORCH_CHECK and TORCH_WARN macros](../sources/prs/flashinfer/PR-1582.md), [feat: initial support for SM103, SM110, SM120, SM121](../sources/prs/flashinfer/PR-1608.md), [bugfix: collect all modules to aot](../sources/prs/flashinfer/PR-1622.md), [test: update fused_moe test to random scale factor](../sources/prs/flashinfer/PR-1665.md), [Refactor Blackwell unit test scripts](../sources/prs/flashinfer/PR-1667.md), [test: better fp8 quantization init for fused_moe test](../sources/prs/flashinfer/PR-1674.md), [Support Kimi-K2 for TRT: templatize number of experts](../sources/prs/flashinfer/PR-1696.md), [test: skip the unsupported test cases for sm120/121](../sources/prs/flashinfer/PR-1710.md), [perf: Add tuning config for cutlass moe for a hardware](../sources/prs/flashinfer/PR-1716.md), [Fix DeepSeek quality for TRTLLM fused MoE routing](../sources/prs/flashinfer/PR-1723.md), [bugfix: partially fix tests/test_trtllm_gen_fused_moe.py unit test failure](../sources/prs/flashinfer/PR-1724.md), [tests: xfail moe quantization classes mxfp8_bf16 UTs on sm103 ](../sources/prs/flashinfer/PR-1754.md), [add test case for trtllm gen fused moe with kimi k2 problem sizes](../sources/prs/flashinfer/PR-1768.md), [fix: fp4 moe on sm120](../sources/prs/flashinfer/PR-1817.md), [feat:enable fp8 blockscale moe for fused cultass for sm90](../sources/prs/flashinfer/PR-1819.md), [feat: trtrllm-gen global scaled FP8 GEMMs](../sources/prs/flashinfer/PR-1829.md), [Update the routing for TRTLLMGEN to support kimi k2 and qwen](../sources/prs/flashinfer/PR-1831.md), [feat: Add FP4 TRTLLM-Gen throughput MOE batched gemms](../sources/prs/flashinfer/PR-1882.md), [Feature: Support Relu2 activation in fused MoE](../sources/prs/flashinfer/PR-1954.md), [Update trtllm-gen fused moe routing kernel and add more kernels](../sources/prs/flashinfer/PR-1955.md), [Fix: Verify scales are not None for Cutlass FP8 FusedMoE](../sources/prs/flashinfer/PR-1961.md), [Feature: Add support for L40 FusedMoE in cutlass path](../sources/prs/flashinfer/PR-1973.md), [feat: autotune tile_tokens_dim in trtllm-gen MOE](../sources/prs/flashinfer/PR-1980.md), [Bugfix: Change get() -> GetDLTensorPtr() in cutlass FusedMoE validations](../sources/prs/flashinfer/PR-1995.md), [Feature: Support non-gated activation in cutlass fused MoE nvfp4](../sources/prs/flashinfer/PR-2011.md), [[feat] Refactor trtllmgen MOE and add Bf16 trtllmgen moe](../sources/prs/flashinfer/PR-2014.md), [update trtllm cutlass moe ](../sources/prs/flashinfer/PR-2020.md), [perf: Speed up fp4 quantization for small batch with swizzling for cutlass MoE](../sources/prs/flashinfer/PR-2025.md), [Enable renormalize(naive) routing for fp8 per-tensor](../sources/prs/flashinfer/PR-2030.md), [Fix dtype of output scales from mnnvl_moe_alltoallv_prepare_without_allgather](../sources/prs/flashinfer/PR-2048.md), [[BUG] Fix trtllm-gen fp4 moe renormalize routing](../sources/prs/flashinfer/PR-2049.md), [Add support for topkPacked input in block-level renormalize](../sources/prs/flashinfer/PR-2051.md), [Fix moe fp8 failure for sm121](../sources/prs/flashinfer/PR-2061.md), [perf: TRT-LLM MoE Block-FP8 activation optimization](../sources/prs/flashinfer/PR-2063.md), [[Test] Optimize test_trtllm_gen_fused_moe.py](../sources/prs/flashinfer/PR-2072.md), [Patch sm103 for 3xfp4 moe generation](../sources/prs/flashinfer/PR-2082.md), [refactor: update dpsk fused_moe test [1]](../sources/prs/flashinfer/PR-2088.md), [refactor: pass hopper deepgemm include directory through python](../sources/prs/flashinfer/PR-2090.md), [perf: TRT-LLM Gen finalize kernel optimization](../sources/prs/flashinfer/PR-2092.md), [refactor: update dpsk fused_moe test [2]](../sources/prs/flashinfer/PR-2097.md), [Port TRT-LLM communication kernels to flashinfer](../sources/prs/flashinfer/PR-2102.md), [enable sm103 moe dsl backend](../sources/prs/flashinfer/PR-2149.md), [feat: MxInt4 x Bf16 TRT-LLM Gen MoE support](../sources/prs/flashinfer/PR-2159.md), [Add data type check for deepseek fp4 moe](../sources/prs/flashinfer/PR-2165.md), [Rename noauxtc to fused_topk_deepseek](../sources/prs/flashinfer/PR-2181.md), [Fix for moe on sm110](../sources/prs/flashinfer/PR-2190.md), [feat: unit-test and api change, w4a8 grouped-gemm fused MoE for SM90](../sources/prs/flashinfer/PR-2193.md), [feat: Support unpadded output hidden size for trtllm_fp4_block_scale_moe](../sources/prs/flashinfer/PR-2217.md), [fix: add DeepSeek routing for Bf16xBf16 and MxIntxBf16 TRT-LLM Gen MoE](../sources/prs/flashinfer/PR-2234.md), [refactor: pull trtllm-gen batch-gemm/gemm headers from artifactory; update tma descriptor shape init](../sources/prs/flashinfer/PR-2235.md), [test: Fix MNNVL tests to skip when container lacks SYS_PTRACE capability](../sources/prs/flashinfer/PR-2245.md), [feat: Support numLocalTokens=0 for moe All-to-all](../sources/prs/flashinfer/PR-2247.md), [feat: Support Fused MoE non gated Relu2 NVFP4 & FP8 and support Nemotron](../sources/prs/flashinfer/PR-2304.md), [feat: expose swizzled_input_sf parameter for CUTLASS fused MOE](../sources/prs/flashinfer/PR-2330.md), [Optimize quantization function in large problem size](../sources/prs/flashinfer/PR-2343.md), [feat: cuteDSL fp4 moe for better DSR1 performance.](../sources/prs/flashinfer/PR-2398.md), [feat: update trtllm-gen MoE cubins](../sources/prs/flashinfer/PR-2416.md), [bugfix: fix stub generation directory in fused_moe module](../sources/prs/flashinfer/PR-2445.md), [feat: Support Fused MoE non gated Relu2 NVFP4 & FP8 and support Nemotron, fixed](../sources/prs/flashinfer/PR-2462.md), [fix: blockscale moe routine supports non-DS routing](../sources/prs/flashinfer/PR-2476.md), [fix: add support check for gemm config for cutlass moe](../sources/prs/flashinfer/PR-2495.md), [Feat: Trtllm-gen MxFP8 MoE integration](../sources/prs/flashinfer/PR-2505.md), [fix: W4A8 autotune crash in cutlass_fused_moe profiler workspace](../sources/prs/flashinfer/PR-2564.md), [Implement `cutlass_fused_moe` mxfp8](../sources/prs/flashinfer/PR-2581.md), [tests: add bias testing to nvfp4 moe](../sources/prs/flashinfer/PR-2585.md), [fix: trtllm_mxint4_block_scale_moe unit test to index output list](../sources/prs/flashinfer/PR-2627.md), [fix: cute dsl nvfp4 moe routing index error](../sources/prs/flashinfer/PR-2629.md), [fix: add SM121 support to SM120 version guards](../sources/prs/flashinfer/PR-2631.md), [benchmark: Add MXFP4/MXFP8 quantization mode support to FP4 MoE benchmark](../sources/prs/flashinfer/PR-2635.md), [[fp8_blockwise]Fix int32 overflow in TRTLLM fused MoE activation kernel](../sources/prs/flashinfer/PR-2642.md), [[feat] trtllm-gen mxfp8 gemm](../sources/prs/flashinfer/PR-2653.md), [fix: Add fused MOE and GEMM AOT modules for SM121](../sources/prs/flashinfer/PR-2654.md), [feat: Add support for TRTLLM MXFP8 non-gated MoE with ReLU2](../sources/prs/flashinfer/PR-2707.md), [fix: Add SM120 (RTX Blackwell desktop) support for NVFP4 MoE kernels](../sources/prs/flashinfer/PR-2725.md), [Support in-place update for `trtllm_fp8_block_scale_moe`](../sources/prs/flashinfer/PR-2739.md), [misc: Update gemm/batched gemm cubins from trtllm-gen, gemm header refactor](../sources/prs/flashinfer/PR-2740.md), [[feat] Add 2048 experts and 32 Top K ](../sources/prs/flashinfer/PR-2744.md), [Upgrade cutlass 4.2.1 -> 4.4.2](../sources/prs/flashinfer/PR-2798.md), [CuteDSL MoE fix redundant output buffer zeroing](../sources/prs/flashinfer/PR-2811.md), [fix: Autotuner _find_nearest_profile non-power-of-2 num_tokens, create launchers for all supported tileN in trtllm fused MoE](../sources/prs/flashinfer/PR-2821.md), [fix: int32 overflow in `trtllm_fp4_block_scale_moe` causing "Unsupported hidden state scale shape" for EP32+ configs](../sources/prs/flashinfer/PR-2853.md), [Add support for Relu2 in BF16 fused MoE](../sources/prs/flashinfer/PR-2864.md), [fix: add cute dsl moe utils to AOT](../sources/prs/flashinfer/PR-2872.md), [Fix silent bug with FP8 per tensor non-gated MoE](../sources/prs/flashinfer/PR-2882.md), [fix: snap weight_scale_vec_size to handle block_scale_interleave padding for SM120](../sources/prs/flashinfer/PR-2898.md), [[NVIDIA] fix(jit): enable GDC for CUTLASS fused MoE PDL — prevent random crashes on SM12x](../sources/prs/flashinfer/PR-2913.md), [fix: Fix autotuner crash on meta-device tensor in trtllm_fp4_block_scale_routed_moe](../sources/prs/flashinfer/PR-2916.md), [feat: add Relu2 (squared ReLU) activation support in CUTLASS MoE backend](../sources/prs/flashinfer/PR-2926.md), [[Perf] Refactor MoE autotuning to set valid topk ids in routed MoE tuning](../sources/prs/flashinfer/PR-2942.md), [Fused moe all-reduce routed scaling factor + quant support](../sources/prs/flashinfer/PR-2966.md), [feat(comm): add MOE Finalize/Reduction patterns to unified allreduce_fusion API](../sources/prs/flashinfer/PR-2982.md), [fix: restore SM120 CUTLASS MoE tile candidate removed by #2927 (test_trtllm_cutlass_fused_moe.py)](../sources/prs/flashinfer/PR-2984.md), [perf: Optimize CUTLASS MoE helper kernels for small-batch decode workloads](../sources/prs/flashinfer/PR-3014.md), [fix: extend moe alltoall top-k specializations](../sources/prs/flashinfer/PR-3021.md), [[feat] Add routing_replay_out support to MoE kernels and Python API](../sources/prs/flashinfer/PR-3024.md), [Prevent MoE autotuner buffer overflow on large token buckets](../sources/prs/flashinfer/PR-3025.md), [[feat] Trtllm-gen Per-token Nvfp4 MoE](../sources/prs/flashinfer/PR-3027.md), [fused_moe: pre-filter SM89 tactics with zero occupancy on SM120 Blackwell (fix review feedback on #2764)](../sources/prs/flashinfer/PR-3032.md), [feat: Add b12x CuTe DSL fused MoE for SM120](../sources/prs/flashinfer/PR-3066.md), [Integrate CUTLASS Small Tile N Blockscaled GEMMs/Grouped GEMMs for SM120 and SM121](../sources/prs/flashinfer/PR-3152.md), [fix(sm12x): fix micro-kernel workspace sizing when routed_rows > num_local_experts](../sources/prs/flashinfer/PR-3191.md), [fix(cute_dsl/moe): make autotuner bucket configuration adapt to runtime input](../sources/prs/flashinfer/PR-3216.md), [feat(trace): embed runnable init() in every TraceTemplate](../sources/prs/flashinfer/PR-3221.md), [[Bugfix] Fix fused MoE autotuning correctness issues by filtering clusterDimZ](../sources/prs/flashinfer/PR-3227.md), [Update moe gemm](../sources/prs/flashinfer/PR-3239.md), [fix(cute_dsl/moe): unbias autotuner profiling for tile_size enumeration](../sources/prs/flashinfer/PR-3252.md), [feat(moe): add SM120 W4A16 b12x kernels](../sources/prs/flashinfer/PR-3271.md), [feat(cute_dsl/moe): deterministic balanced autotune profile inputs](../sources/prs/flashinfer/PR-3286.md), [Ep api design - Build Infra dependencies](../sources/prs/flashinfer/PR-3315.md), [[feat] Add gemma RMS AR fusion](../sources/prs/flashinfer/PR-3322.md), [feat(cute_dsl/moe): add `moe_output_memset_inplace` dense memset wrapper](../sources/prs/flashinfer/PR-3328.md), [Add support for bf16 x bf16 cutlass fused MoE](../sources/prs/sglang/PR-10275.md), [Fix cutlass moe accuracy drop caused by attention UB from DP padding mode](../sources/prs/sglang/PR-10414.md), [Support single batch overlap](../sources/prs/sglang/PR-10422.md), [Fix correction bias undefined behavior for nvfp4 models](../sources/prs/sglang/PR-10426.md), [feat: add dsv3 fp4 cutlass moe etp ut](../sources/prs/sglang/PR-10433.md), [Update CUTLASS. Refine KernelSchedule for fp8 (grouped) gemm.](../sources/prs/sglang/PR-10491.md), [Fix bias handling in TritonMoeQuantInfo within quantization/mxfp4.py](../sources/prs/sglang/PR-10579.md), [support qwen3-next-fp8 deepep](../sources/prs/sglang/PR-10622.md), [Fix MTP MoE weight loading with NVFP4 target model.](../sources/prs/sglang/PR-10758.md), [Quick Fix: fix Qwen3-VL launch failure caused by MRotaryEmbedding arg](../sources/prs/sglang/PR-10985.md), [Fix DSR1 accuracy for flashinfer_trtllm MoE with FP8 quantization](../sources/prs/sglang/PR-11081.md), [[AMD] Clean up vllm dependencies in moe_runner/triton.py](../sources/prs/sglang/PR-11349.md), [[sgl-kernel][1/N]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-11432.md), [Support shared experts overlap in cutlass moe](../sources/prs/sglang/PR-11611.md), [Support running FP4 Deepseek on SM120.](../sources/prs/sglang/PR-11708.md), [support cutlass fp4 kernel in sm120](../sources/prs/sglang/PR-11737.md), [[Ascend] qwen optimization](../sources/prs/sglang/PR-12078.md), [[hotfix] missing `w13_weight_fp8` and `w2_weight_fp8` in UE8M0 requantization](../sources/prs/sglang/PR-12259.md), [Replace [silu_and_mul_]scaled_fp4_group_quant by Flashinfer equivalent](../sources/prs/sglang/PR-12376.md), [Enable Flashinfer TRTLLM-GEN-MoE FP8 blockwise kernel for Qwen3-Next on Blackwell](../sources/prs/sglang/PR-12543.md), [[CPU] Fix MoE layer support for DeepSeek-OCR models](../sources/prs/sglang/PR-12555.md), [[sgl-kernel][5/N]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-12666.md), [[Bugfix] Fix illegal memory access](../sources/prs/sglang/PR-12758.md), [[Ascend] support Kimi-K2-Thinking](../sources/prs/sglang/PR-12759.md), [ignore the deepgemm check when the model weight with nvfp4 and moe ba…](../sources/prs/sglang/PR-12782.md), [Apply moe_reduce_sum kernel for fused_marlin_moe](../sources/prs/sglang/PR-12888.md), [Support moe topk sigmoid kernel](../sources/prs/sglang/PR-13049.md), [support mtp with deepseek r1 nvfp4 model](../sources/prs/sglang/PR-13115.md), [[NPU]Optimization of `forward_npu` for `UnquantizedFusedMoEMethod`](../sources/prs/sglang/PR-13158.md), [Fix nan in global scaling factor for large scale nvfp4 EP](../sources/prs/sglang/PR-13162.md), [[NVIDIA] Fix broken fp8 MoE of deepseek v3](../sources/prs/sglang/PR-13264.md), [Flashinfer TRTLLM-GEN-MoE + Qwen3](../sources/prs/sglang/PR-13489.md), [Fix EPLB + FP4 Quantization Compatibility Issue](../sources/prs/sglang/PR-13715.md), [[bugfix] fix TBO crashes when attn_tp_size > 1](../sources/prs/sglang/PR-13730.md), [[sgl-kernel][Feat][B200][1/N]Support MXFP8 Grouped GEMM in Blackwell](../sources/prs/sglang/PR-13731.md), [[Feat][NVFP4] Enable NVFP4 MoE for Qwen series models (eg. Qwen3-Next) #13761](../sources/prs/sglang/PR-13761.md), [Support fp4 fp8 non gated moe](../sources/prs/sglang/PR-13794.md), [[NVIDIA] Enable TRTLLM BF16 MoE on Blackwell GPUs](../sources/prs/sglang/PR-13798.md), [[BugFix] fix outplace_fused_experts missing is_gated](../sources/prs/sglang/PR-13864.md), [Feat: GLM-4.6 supports shared experts fusion](../sources/prs/sglang/PR-13873.md), [[DeepSeek v3.2] opt Context Parallelism: support fused moe, multi batch and fp8 kvcache](../sources/prs/sglang/PR-13959.md), [[kernel][moe] add moe topk fast](../sources/prs/sglang/PR-13969.md), [Support KTransformers for Qwen3-VL moe](../sources/prs/sglang/PR-13983.md), [Fix flashinfer cutlass MoE output shape for non-FP4-packed inputs](../sources/prs/sglang/PR-14028.md), [[LoRA][III] Add LoRA support for MoE layers and enable TP](../sources/prs/sglang/PR-14105.md), [Add new moe wna16 marlin gemm](../sources/prs/sglang/PR-14122.md), [Apply new moe wna16 marlin gemm](../sources/prs/sglang/PR-14125.md), [Opt moe align block size kernel](../sources/prs/sglang/PR-14133.md), [Apply new moe align block size kernel](../sources/prs/sglang/PR-14134.md), [Add Mistral Large 3 support.](../sources/prs/sglang/PR-14213.md), [[FIX] trtllm-moe-fp4-renorm for Qwen series models](../sources/prs/sglang/PR-14350.md), [[NPU] perf update with kvcache nz & w4a8 quant](../sources/prs/sglang/PR-14423.md), [Mistral Large 3 NVFP4 support](../sources/prs/sglang/PR-14485.md), [Apply back moe_sum_reduce for fused_marlin_moe](../sources/prs/sglang/PR-14829.md), [add transformers version validation for glm-4.6v moe models](../sources/prs/sglang/PR-14998.md), [Mistral Large 3 NVFP4 TRTLLM MoE support](../sources/prs/sglang/PR-15049.md), [[sgl-kernel][1/2] Fused qk_norm_rope for GLM4.6](../sources/prs/sglang/PR-15141.md), [[NVIDIA] Fixes for NVFP4 all-gather with spec decoding](../sources/prs/sglang/PR-15280.md), [[Fix] A followup fix for TRTLLM BF16 MoE](../sources/prs/sglang/PR-15303.md), [Fix the accuracy issue when running mxfp4 dsv3 model and enable ep](../sources/prs/sglang/PR-15304.md), [Fix warp illegal instruction in kimi k2 thinking PCG](../sources/prs/sglang/PR-15306.md), [[distributed] Clean up MoE groups in destroy_model_parallel](../sources/prs/sglang/PR-15345.md), [[NPU]mindspore model support moe](../sources/prs/sglang/PR-15363.md), [Super tiny add moe_ep_rank to prometheus labels](../sources/prs/sglang/PR-15407.md), [Flashinfer MOE FP8 support for Mistral Large 3.](../sources/prs/sglang/PR-15422.md), [Optimize Bailing-MoE with FlashInfer Fused All-Reduce](../sources/prs/sglang/PR-15526.md), [MoE: Skip SiLU/GELU activation for masked experts](../sources/prs/sglang/PR-15539.md), [Update flashinfer to 0.6.1](../sources/prs/sglang/PR-15551.md), [Add SwapAB Optimization for triton fused_moe_kernel on SM90.](../sources/prs/sglang/PR-15712.md), [[Perf] Eliminate the slice op for Flashinfer `trtllm_fp4_block_scale_moe`](../sources/prs/sglang/PR-15731.md), [Fix GLM-4.7 MoE Detector complex JSON Schema type parsing](../sources/prs/sglang/PR-15753.md), [Fix: Handle empty func_name and None values in GLM MoE detectors](../sources/prs/sglang/PR-15754.md), [[Feature] JIT Fused QK norm + qk norm clean up](../sources/prs/sglang/PR-15835.md), [[JIT kernel] Apply jit per_tensor_quant_fp8 kernel](../sources/prs/sglang/PR-15836.md), [[NPU] NZ for non-quantized MOE, Qwen3 MOE double memory consumption fix](../sources/prs/sglang/PR-15904.md), [ Add tuned triton==3.5.1 h200 tp2, tp4 for qwen 3 next](../sources/prs/sglang/PR-15948.md), [enhance accuracy for model kimi-vl-instruct-a3b](../sources/prs/sglang/PR-16076.md), [fix layer intermediate size](../sources/prs/sglang/PR-16084.md), [[NemotronH] Add latent MoE support](../sources/prs/sglang/PR-16227.md), [Fix FP8 MoE NaN with DeepGEMM on Blackwell](../sources/prs/sglang/PR-16622.md), [[Rework] Add SwapAB Optimization for triton fused_moe_kernel on SM90.](../sources/prs/sglang/PR-16723.md), [[AMD] Support redundant expert with a2a moe in gfx95x.](../sources/prs/sglang/PR-16791.md), [[Fix] `flashinfer_trtllm` `intermediate_size` assertion with Qwen3 + TP=8](../sources/prs/sglang/PR-16824.md), [Support mxint4 flashinfer_trtllm moe gemm](../sources/prs/sglang/PR-16892.md), [Inclusion of nvfp4 blockscale in EPLB Rebalance](../sources/prs/sglang/PR-17158.md), [[Fix] GLM 4.7 + NVFP4 + MTP](../sources/prs/sglang/PR-17166.md), [[GLM 4.7] Add RTX 6000 Pro aka sm120](../sources/prs/sglang/PR-17235.md), [[New Model] GLM4.7-Flash](../sources/prs/sglang/PR-17247.md), [Add mxfp8 support for online quantization, Triton dense linear, and CUTLASS MoE](../sources/prs/sglang/PR-17449.md), [Upgrade transformers==5.3.0](../sources/prs/sglang/PR-17784.md), [[Bugfix] Fix Mistral Large 3 NVFP4 TRTLLM MoE](../sources/prs/sglang/PR-18065.md), [Fix nvfp4 weight update](../sources/prs/sglang/PR-18085.md), [[Blackwell] Make mxint4 flashinfer_trtllm moe gemm set by default on blackwell](../sources/prs/sglang/PR-18136.md), [[ModelOpt] Fix broken Qwen3-235B-A22B-Instruct-2507-NVFP4 launch](../sources/prs/sglang/PR-18189.md), [Support Qwen3 MoE context parallel](../sources/prs/sglang/PR-18233.md), [[sglang-miles] True on-policy training support for FSDP2](../sources/prs/sglang/PR-18639.md), [[Perf] ~9.5x faster Blackwell MXFP4 MoE weight loading](../sources/prs/sglang/PR-18858.md), [[Sarvam] Add inference support for Sarvam MoE LLMs](../sources/prs/sglang/PR-18938.md), [[jit_kernel] Add fused_qknorm_rope JIT kernel](../sources/prs/sglang/PR-19059.md), [Adjust padding size to improve triton_kernels moe performance](../sources/prs/sglang/PR-19174.md), [Fix/nemotron mtp quantaized](../sources/prs/sglang/PR-19433.md), [[Kernel Slimming] Migrate NVFP4 kernels to JIT](../sources/prs/sglang/PR-19437.md), [[FlashInfer v0.6.4] [RL] Integrate FlashInfer mxfp8 gemm, MoE, and routed MoE](../sources/prs/sglang/PR-19537.md), [[Feature] NVFP4 Marlin fallback for non-Blackwell GPUs (SM75+)](../sources/prs/sglang/PR-19652.md), [Use TRTLLM allreduce fusion for Qwen 3.5](../sources/prs/sglang/PR-19889.md), [[JIT Kernel] Reland NVFP4 kernels to JIT](../sources/prs/sglang/PR-20012.md), [[4/n jit_kernel restruct] speed up CI tests and add benchmark workflow](../sources/prs/sglang/PR-20268.md), [[Benchmark] use flashinfer bench_gpu_time instead of triton do_bench](../sources/prs/sglang/PR-20305.md), [[NVIDIA] Enable fp8 flashinfer_trtllm_routed MoE for MiniMax-M2.5](../sources/prs/sglang/PR-20394.md), [Add Mistral Small 4 (Pixtral) support](../sources/prs/sglang/PR-20708.md), [Use FlashInfer tinygemm for GPT-OSS MoE router on SM90+](../sources/prs/sglang/PR-20755.md), [Add SGLang CUDA crash API logging inspired by FlashInfer](../sources/prs/sglang/PR-20910.md), [[refactor] Clean up duplicate flashinfer trtllm moe code](../sources/prs/sglang/PR-21233.md), [Refactor JIT kernel CI to use run_suite.py registration system](../sources/prs/sglang/PR-21239.md), [[NVIDIA] Enable FP4 flashinfer trtllm routed moe](../sources/prs/sglang/PR-21240.md), [P2P Weight Update features for miles ](../sources/prs/sglang/PR-21278.md), [[RL] Support mxfp8 DeepSeek V3](../sources/prs/sglang/PR-21280.md), [[Kernel] Support FlashInfer TRTLLM-Gen fused MoE for non-gated FP4 & FP8 (Nemotron)](../sources/prs/sglang/PR-21321.md), [[misc] clean up kernel API](../sources/prs/sglang/PR-21325.md), [Add dedicated FlashInferCuteDslMoE layer for standard-path FP4 MoE](../sources/prs/sglang/PR-21339.md), [[jit_kernel] Optimize fused_qknorm_rope: deduplicate sincosf for interleave RoPE ](../sources/prs/sglang/PR-21654.md), [[AMD] Use tgemm.mm for MoEGate router gemm in deepseek_v2.py](../sources/prs/sglang/PR-21657.md), [Harden FlashInfer FP4 imports in standard dispatcher](../sources/prs/sglang/PR-21776.md), [[Fix] Fall back to triton MOE for GPT-OSS on Blackwell with driver >= 595](../sources/prs/sglang/PR-21780.md), [Remove redundant test_moe_eval_accuracy_large](../sources/prs/sglang/PR-21787.md), [Tiny fix trtllm_fp8_per_tensor_scale_moe_wrapper router_logits dtype](../sources/prs/sglang/PR-22006.md), [[RL] Refactor NVFP4 shuffling/swizzling to in-place replacement](../sources/prs/sglang/PR-22204.md), [[Reland] DeepSeek-R1-0528-w4a8: DeepEP Low Latency Dispatch Adopts FP8 Communication](../sources/prs/sglang/PR-22316.md), [[Lora] Lora quat info re-factor and support deepseekv3 mla lora](../sources/prs/sglang/PR-22323.md), [[Lora] Lora kimi support](../sources/prs/sglang/PR-22381.md), [feat: Support flashinfer_cutedsl MoE runner with flashinfer alltoall backend](../sources/prs/sglang/PR-22669.md), [[Step3p5] Optimize allreduce in MoE layers ](../sources/prs/sglang/PR-22773.md), [Dual MoE CUDA graph capture for lora/nolora batches](../sources/prs/sglang/PR-22809.md), [[Refactor] Refactor DeepEP dispatcher](../sources/prs/sglang/PR-22822.md), [[FlashInfer v0.6.11] [RL] Support FlashInfer per-token NVFP4 MoE](../sources/prs/sglang/PR-22918.md), [Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-23686.md), [Use Cute-DSL NVFP4 quantization kernels](../sources/prs/sglang/PR-23745.md), [[MoE] Unify DeepEPMoE+MoriEPMoE through AITER MoeRunner pre/post-permute](../sources/prs/sglang/PR-23760.md), [[feat] Init true on policy with qwen_dense](../sources/prs/sglang/PR-23961.md), [Port MXFP4 Marlin MoE support to JIT kernel path](../sources/prs/sglang/PR-24490.md), [Fix performance regression on Deepseek V3 on `moe-runner-backend=triton` on SM90](../sources/prs/sglang/PR-24562.md), [[Codex] Opt Mistral Large performace ](../sources/prs/sglang/PR-24611.md), [Add FlashInfer SM90 cutlass MXFP4 MoE backend (W4A16) for GPT-OSS + DeepSeek-V4](../sources/prs/sglang/PR-24816.md), [Amd/deepseek v4 rebase main 0509](../sources/prs/sglang/PR-24933.md), [[rebase]Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-24986.md), [Fix EPLB mapping for TopK paths](../sources/prs/sglang/PR-25285.md), [[Intel GPU] Enable DeepSeek V4 Inference on XPU](../sources/prs/sglang/PR-25336.md), [[AMD] test(sgl-kernel): seed RNG on ROCm in test_moe_topk_sigmoid to fix tie-break flake](../sources/prs/sglang/PR-25356.md), [Update logging for inplace setting in MoE layer](../sources/prs/sglang/PR-25499.md), [[misc] Throw error when single batch overlap is enabled on Hopper ](../sources/prs/sglang/PR-25509.md), [Fix logging for inplace setting in the flashInfer-trtllm backend](../sources/prs/sglang/PR-25522.md), [[Bug Fix] Align glm4_moe_nextn NPU MTP loading with qwen3 MTP](../sources/prs/sglang/PR-25524.md), [[MoE Refactor] Migrate flashinfer_cutedsl + DeepEP to MoeRunner](../sources/prs/sglang/PR-25525.md), [Use DeepGEMM BF16 for unquantized DeepEP LL MoE](../sources/prs/sglang/PR-25540.md), [Add DeepSeekV4 fused MoE Triton autotune support](../sources/prs/sglang/PR-25569.md), [[Benchmark] Add SGLANG_SIMULATE_UNIFORM_EXPERTS for balanced expert routing with dummy weights](../sources/prs/sglang/PR-25571.md), [[SP] Fix runtime_max_tokens_per_rank for sequence parallelism](../sources/prs/sglang/PR-25685.md), [Add no_combine support to cutlass_moe_fp4](../sources/prs/sglang/PR-25688.md), [[Refactor] Pass PP start_layer via model constructor instead of forward_batch.token_to_kv_pool](../sources/prs/sglang/PR-25825.md), [[Test] Stage-a sanity kits; consolidate core/ + models_e2e/ tests](../sources/prs/sglang/PR-25831.md), [feat: integrate gemm_fp8 kernel into gemm](../sources/prs/sglang/PR-3056.md), [add tensorrt_llm common and cutlass_extensions as 3rdparty](../sources/prs/sglang/PR-3216.md), [integrate blockwise fp8 kernel](../sources/prs/sglang/PR-3529.md), [Feature DeepSeek V3/R1 INT8 Quantization (block-wise)](../sources/prs/sglang/PR-3730.md), [[Feature] DeepSeek V3/R1 INT8 Quantization (channel-wise) ](../sources/prs/sglang/PR-3888.md), [Accelerate FP8 CUDA Kernel by 20-28%](../sources/prs/sglang/PR-4215.md), [[Feature] Integrate DeepEP into SGLang](../sources/prs/sglang/PR-4232.md), [Add deepseek style fused moe group gate selection kernel](../sources/prs/sglang/PR-4530.md), [Optimize Permute Kernel in DeepEP](../sources/prs/sglang/PR-4643.md), [Fix loading KV quantization scale; Enable modelopt kv cache](../sources/prs/sglang/PR-4686.md), [[Model] Adding Qwen3 and Qwen3MoE](../sources/prs/sglang/PR-4693.md), [support cmake for sgl-kernel](../sources/prs/sglang/PR-4706.md), [[Feature] Support DeepEP Low Latency](../sources/prs/sglang/PR-4767.md), [Support (1 <= dp < tp) in the dp attention in DeepEP](../sources/prs/sglang/PR-4770.md), [Introduce moe_dense_tp_size to fix dense layer errors in DeepSeek V3 + 4x8xH100](../sources/prs/sglang/PR-4836.md), [Add DeepSeek V3/R1 shared experts fusion](../sources/prs/sglang/PR-4918.md), [[Fix] DeepEP Compatibility with Low Latency](../sources/prs/sglang/PR-5068.md), [reduce moe_align_block_size_kernel small batch mode overhead](../sources/prs/sglang/PR-5086.md), [Add optimized native kernels in sgl-kernel](../sources/prs/sglang/PR-5150.md), [[1/2] Add FP8 Blockscale MoE CUTLASS kernel for Blackwell](../sources/prs/sglang/PR-5281.md), [Fix DeepGEMM masked cannot be run on groups not being multiple or 4](../sources/prs/sglang/PR-5340.md), [apply fused moe gate in ds v3/r1](../sources/prs/sglang/PR-5371.md), [Add Cutlass MLA attention backend](../sources/prs/sglang/PR-5390.md), [[perf] introduce deep gemm group_gemm_masked as gemm](../sources/prs/sglang/PR-5432.md), [ DeepEP normal support deepgemm-contiguous](../sources/prs/sglang/PR-5626.md), [[2/2] Add python wrapper for CUTLASS FP8 Blockscale MoE Kernel. ](../sources/prs/sglang/PR-5694.md), [[qwen3] support qwen3 ep moe](../sources/prs/sglang/PR-5917.md), [Support tuning moe for llama 4 model](../sources/prs/sglang/PR-6042.md), [[1/2] Add Kernel support for Cutlass based Fused FP4 MoE](../sources/prs/sglang/PR-6093.md), [feat: add dp attention support for Qwen 2/3 MoE models, fixes #6088](../sources/prs/sglang/PR-6121.md), [Reduce MoE memory usage](../sources/prs/sglang/PR-6147.md), [enable auto-round quantization model](../sources/prs/sglang/PR-6226.md), [fix: enable multi-GPU Triton fused MoE tuning](../sources/prs/sglang/PR-6295.md), [reduce torch.zeros overhead in moe align block size kernel](../sources/prs/sglang/PR-6369.md), [Add fp8 fused_experts kernel for CPU in sgl-kernel and add UT](../sources/prs/sglang/PR-6404.md), [Fix topk inference performance reduce](../sources/prs/sglang/PR-6474.md), [qwen3moe support two batch overlap](../sources/prs/sglang/PR-6598.md), [Refine pre_reorder_triton_kernel slightly to improve performance](../sources/prs/sglang/PR-6627.md), [[CPU] [BF16] Call fused_experts_cpu, weight_packed_linear and bmm_cpu kernel in DeepSeek model](../sources/prs/sglang/PR-6641.md), [Fix DeepEP error in Qwen 3 MoE models](../sources/prs/sglang/PR-6673.md), [[EP] Add cuda kernel for moe_ep_pre_reorder](../sources/prs/sglang/PR-6699.md), [Fix PP for Qwen3 MoE](../sources/prs/sglang/PR-6709.md), [Set `num_fused_shared_experts` as `num_shared_experts` when shared_experts fusion is not disabled](../sources/prs/sglang/PR-6736.md), [[CPU] add optimizations for INT8 and FP8 DeepSeek](../sources/prs/sglang/PR-6769.md), [[CPU] support the case where num_attention_heads or intermediate_size is not divisible by the TP size](../sources/prs/sglang/PR-6771.md), [Support token-level quantization for EP MoE](../sources/prs/sglang/PR-6782.md), [feat: integrate deepgemm into EPMoE](../sources/prs/sglang/PR-6821.md), [CPU: map changes from developing branch in sgl-kernel](../sources/prs/sglang/PR-6833.md), [[EP] Add cuda kernel for moe_ep_post_reorder](../sources/prs/sglang/PR-6837.md), [[DeepseekR1-FP4] Add Support for nvidia/DeepSeekR1-FP4 model](../sources/prs/sglang/PR-6853.md), [fix ep_moe_reorder kernel bugs](../sources/prs/sglang/PR-6858.md), [Add a CUDA kernel for fusing mapping and weighted sum for MoE.](../sources/prs/sglang/PR-6916.md), [[sgl-kernel] Add cuda kernel for moe_ep_silu_and_mul](../sources/prs/sglang/PR-6919.md), [chore: upgrade flashinfer v0.2.6.post1 jit](../sources/prs/sglang/PR-6958.md), [Fuse routed scaling factor in deepseek](../sources/prs/sglang/PR-6970.md), [Update default settings for blackwell](../sources/prs/sglang/PR-7023.md), [Fix positional argument](../sources/prs/sglang/PR-7093.md), [fix amd EP MoE FP8 issue](../sources/prs/sglang/PR-7125.md), [Enable ModelOpt Llama4 fp8 checkpoint deployment in SGLang](../sources/prs/sglang/PR-7129.md), [[amd] Opt dsv3 moe](../sources/prs/sglang/PR-7160.md), [Support new DeepGEMM](../sources/prs/sglang/PR-7172.md), [Fix error when disabling new DeepGEMM](../sources/prs/sglang/PR-7198.md), [[fix] fix DeepGEMM blackwell input quant & ut & fix style and log](../sources/prs/sglang/PR-7247.md), [[AMD] add aiter fused moe in DeepEP path](../sources/prs/sglang/PR-7268.md), [Add CUTLASS FP8 Blockscale MoE kernel for Hopper architecture](../sources/prs/sglang/PR-7278.md), [Let EP prefill support new DeepGEMM](../sources/prs/sglang/PR-7310.md), [FlashInfer NVFP4 MoE with EP & 2-stream shared expert](../sources/prs/sglang/PR-7327.md), [Fix MTP with Deepseek R1 Fp4](../sources/prs/sglang/PR-7376.md), [Fix torch compile run](../sources/prs/sglang/PR-7391.md), [[AMD][Quantization] Add `int4fp8_moe` online quantization on ROCm](../sources/prs/sglang/PR-7392.md), [Fuse sorted_token_ids padding to moe_align_block_size kernel](../sources/prs/sglang/PR-7437.md), [fix: fix apply_shuffle_mul_sum](../sources/prs/sglang/PR-7444.md), [Add Tencent HunYuanMoEV1 model support](../sources/prs/sglang/PR-7549.md), [[Feature] Layer-wise Prefill](../sources/prs/sglang/PR-7634.md), [Add fp4 quantize before all-gather for Flashinfer cutlass MoE DP (max throughput)](../sources/prs/sglang/PR-7667.md), [Integrate triton moe kernel](../sources/prs/sglang/PR-7689.md), [feat: support DeepSeek-R1-W4AFP8 model with ep-moe mode](../sources/prs/sglang/PR-7762.md), [[1/n]: add cutlass W4A8 moe kernel for hopper architecture](../sources/prs/sglang/PR-7772.md), [[kernel] opt moe align block kernel by block/warp scan algorithm](../sources/prs/sglang/PR-7884.md), [[feat] Support tp mode for DeepSeek-R1-W4AFP8](../sources/prs/sglang/PR-8118.md), [[1/N]Support DeepSeek-R1 w4a8 normal deepep](../sources/prs/sglang/PR-8247.md), [Support triton kernels v3.4.0 for fused_moe](../sources/prs/sglang/PR-8258.md), [[2/N]Support DeepSeek-R1 w4a8 low latency deepep](../sources/prs/sglang/PR-8464.md), [Update cutlass_moe.py](../sources/prs/sglang/PR-8535.md), [Update cutlass_moe.py](../sources/prs/sglang/PR-8545.md), [[NVIDIA] Add Low Latency NVFP4 decode kernels from Flashinfer](../sources/prs/sglang/PR-8552.md), [feat: support cutlass_moe_fp8 kernel for fusedmoe in sm90](../sources/prs/sglang/PR-8678.md), [fuse allreduce and residual_rmsnorm](../sources/prs/sglang/PR-8731.md), [[Perf] Auto enable best flashinfer mxfp4 kernel in b200](../sources/prs/sglang/PR-8898.md), [[NVIDIA] Fix missing `get_col_major_tma_aligned_tensor` for Blackwell deepgemm in EpMoE](../sources/prs/sglang/PR-8955.md), [optimize: reduce shulffle and quantization overhead in cutlass_moe sm90](../sources/prs/sglang/PR-8962.md), [Faster weight processing (trtllm-gen moe nvfp4)](../sources/prs/sglang/PR-9162.md), [[NVIDIA] [3/N] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked ](../sources/prs/sglang/PR-9199.md), [[fix]: fix cutlass moe ut and and Opt H20 cutlass groupGemm performance](../sources/prs/sglang/PR-9272.md), [[fix] Fix mxfp4 triton MoE tp bug](../sources/prs/sglang/PR-9473.md), [Optimize moe_sum_reduce_kernel](../sources/prs/sglang/PR-9477.md), [Update CUTLASS 4.2 & Enable K-Major Scale Factor for SM90 FP8 Blockwise Group GEMM](../sources/prs/sglang/PR-9559.md), [Single Batch Overlap for MoE Models](../sources/prs/sglang/PR-9660.md), [move is_sm90_supported/is_sm100_supported to python/sglang/srt/utils.py](../sources/prs/sglang/PR-9679.md), [[CPU] Add FP8 Bmm support](../sources/prs/sglang/PR-9744.md), [Make sm100 fp8 kernels available on sm103](../sources/prs/sglang/PR-9789.md), [[Model] Support Meituan LongCat-Flash && LongCat-Flash-MTP](../sources/prs/sglang/PR-9824.md), [[ROCm][MoE] moe tuning support for rocm](../sources/prs/vllm/PR-12049.md), [[Kernel] add triton fused moe kernel for gptq/awq](../sources/prs/vllm/PR-12185.md), [[Hardware][Gaudi][Feature] Enable Dynamic MoE for Mixtral](../sources/prs/vllm/PR-12303.md), [[Misc][MoE] add Deepseek-V3 moe tuning support](../sources/prs/vllm/PR-12558.md), [[Kernel] port sgl moe_align_block_size kernels](../sources/prs/vllm/PR-12574.md), [Expert Parallelism (EP) Support for DeepSeek Models](../sources/prs/vllm/PR-12583.md), [Apply torch.compile to fused_moe/grouped_topk](../sources/prs/vllm/PR-12637.md), [[Misc] Update w2 scale loading for GPTQMarlinMoE](../sources/prs/vllm/PR-12757.md), [Optimize moe_align_block_size for deepseek_v3](../sources/prs/vllm/PR-12850.md), [[Model] Deepseek GGUF support ](../sources/prs/vllm/PR-13167.md), [[Quant][Perf] Use moe_wna16 kernel by default for MoEs with many experts](../sources/prs/vllm/PR-13236.md), [[Kernel] moe wna16 cuda kernel](../sources/prs/vllm/PR-13321.md), [[ROCm][MoE] mi300 mixtral8x7B perf for specific BS](../sources/prs/vllm/PR-13577.md), [[Kernel] Optimize moe intermediate_cache usage](../sources/prs/vllm/PR-13625.md), [[BugFix] Illegal memory access for MoE On H20](../sources/prs/vllm/PR-13693.md), [[core] Perf improvement for DSv3 on AMD GPUs](../sources/prs/vllm/PR-13718.md), [Fix CompressedTensorsWNA16MoE with grouped scales](../sources/prs/vllm/PR-13769.md), [Fix precommit fail in fused_moe intermediate_cache2 chunking](../sources/prs/vllm/PR-13772.md), [[Bugfix][Quantization] Fix FP8 + EP](../sources/prs/vllm/PR-13784.md), [[V1] EP/TP MoE + DP Attention](../sources/prs/vllm/PR-13931.md), [[Kernel] CUTLASS grouped gemm fp8 MoE kernel](../sources/prs/vllm/PR-13972.md), [[Misc] Print FusedMoE detail info](../sources/prs/vllm/PR-13974.md), [[core] moe fp8 block quant tuning support](../sources/prs/vllm/PR-14068.md), [dynamic distpatch of fp8 kernels](../sources/prs/vllm/PR-14245.md), [[Misc] Add Qwen2MoeForCausalLM moe tuning support ](../sources/prs/vllm/PR-14276.md), [[Kernel] moe wna16 marlin kernel](../sources/prs/vllm/PR-14447.md), [[ROCm][Kernel] MoE weights padding](../sources/prs/vllm/PR-14454.md), [permute/unpermute kernel for moe optimization](../sources/prs/vllm/PR-14568.md), [[Kernel] GGUF MoE kernel](../sources/prs/vllm/PR-14613.md), [[Bugfix][IPEX] Add `VLLM_CPU_MOE_PREPACK` to allow disabling MoE prepack when CPU does not support it](../sources/prs/vllm/PR-14681.md), [[FEAT][ROCm] Integrate Fused MoE Kernels from AITER](../sources/prs/vllm/PR-14967.md), [[Model] Add Qwen3 and Qwen3MoE](../sources/prs/vllm/PR-15289.md), [[Kernel] Fix conflicting macro names for gguf kernels](../sources/prs/vllm/PR-15456.md), [Use Cache Hinting for fused_moe kernel](../sources/prs/vllm/PR-15511.md), [[moe][quant] add weight name case for offset](../sources/prs/vllm/PR-15515.md), [[Quantization] Fp8 Channelwise Dynamic Per Token GroupedGEMM](../sources/prs/vllm/PR-15587.md), [[V1] TPU - Fix fused MOE](../sources/prs/vllm/PR-15834.md), [[Hardware][Gaudi][BugFix] fix arguments of hpu fused moe](../sources/prs/vllm/PR-15945.md), [Modularize fused experts and integrate PPLX kernels](../sources/prs/vllm/PR-15956.md), [[Kernel] Use moe_wna16 kernel for compressed tensors wna16 moe models](../sources/prs/vllm/PR-16038.md), [[Kernel][Bugfix] Re-fuse triton moe weight application](../sources/prs/vllm/PR-16071.md), [Upstream Llama4 Support to Main](../sources/prs/vllm/PR-16113.md), [[Bug] [ROCm] Fix Llama 4 Enablement Bug on ROCm: V0 ROCmFlashAttentionImpl and Triton Fused MoE bugs](../sources/prs/vllm/PR-16198.md), [[Model] use AutoWeightsLoader for phimoe,qwen2_moe,qwen3_moe](../sources/prs/vllm/PR-16203.md), [[Hardware][AMD] Improve OAM device ID + llama4 Maverick MOE tuning](../sources/prs/vllm/PR-16263.md), [[Hardware/NVIDIA/Kernel] [Functional Enablement] [1/N] Enable nvidia/DeepSeek-R1-FP4 Model](../sources/prs/vllm/PR-16362.md), [[Kernel] Support W8A8 channel-wise weights and per-token activations in triton fused_moe_kernel](../sources/prs/vllm/PR-16366.md), [Enable PTPC FP8 for CompressedTensorsW8A8Fp8MoEMethod (triton fused_moe)](../sources/prs/vllm/PR-16537.md), [[ROCM] enable aiter fused moe kernel for llama4 bf16 checkpoints](../sources/prs/vllm/PR-16674.md), [[ROCm] Add aiter tkw1 kernel for Llama4 fp8](../sources/prs/vllm/PR-16727.md), [Support W8A8 INT8 MoE for compressed-tensors](../sources/prs/vllm/PR-16745.md), [[FEAT] [ROCm]: AITER Fused MOE V1 Support](../sources/prs/vllm/PR-16752.md), [[misc] ignore marlin_moe_wna16 local gen codes](../sources/prs/vllm/PR-16760.md), [[Kernel] GGUF MoeVec kernel](../sources/prs/vllm/PR-16780.md), [[BugFix] Accuracy fix for llama4 int4 - improperly casted scales](../sources/prs/vllm/PR-16801.md), [[Kernel] some optimizations for dense marlin and moe marlin](../sources/prs/vllm/PR-16850.md), [[Bugfix] Fix moe weight losing all extra attrs after `process_weights_after_loading`.](../sources/prs/vllm/PR-16854.md), [[Kernel] Add expert_map support to Cutlass FP8 MOE](../sources/prs/vllm/PR-16861.md), [Update Qwen1.5-MoE-W4A16-compressed-tensors.yaml](../sources/prs/vllm/PR-16946.md), [Fix `numel()` downcast in vllm/csrc/moe/moe_align_sum_kernels.cu +2](../sources/prs/vllm/PR-17082.md), [[FEAT] [ROCm]: Add AITER CK 2 Stages MoE support](../sources/prs/vllm/PR-17110.md), [[FEAT][ROCm]: Support AITER MLA on V1 Engine](../sources/prs/vllm/PR-17523.md), [[Kernel] fp4 marlin kernel](../sources/prs/vllm/PR-17687.md), [[BugFix][AMD] Compatible patch for AITER lib after 04/20](../sources/prs/vllm/PR-17912.md), [Fix Broken macro for cutlass moe](../sources/prs/vllm/PR-18049.md), [[Model]: Fused MoE for nomic-embed-text-v2-moe](../sources/prs/vllm/PR-18321.md), [[Feature] Expert Parallelism Load Balancer (EPLB)](../sources/prs/vllm/PR-18343.md), [[Bug] Fix moe_sum signature](../sources/prs/vllm/PR-18440.md), [[V1] Support `LLM.apply_model`](../sources/prs/vllm/PR-18465.md), [[Kernel] Integrate CUTLASS MoE kernel with PPLX](../sources/prs/vllm/PR-18762.md), [[Kernel] Enable fp8 support for pplx and BatchedTritonExperts.](../sources/prs/vllm/PR-18864.md), [[ROCm] [AITER] [Bugfix] Patch for AITER commit `648764942e552a8bb5fe16026703716a81f05374`](../sources/prs/vllm/PR-18990.md), [[Hardware][NVIDIA] FP4 MoE kernel optimization](../sources/prs/vllm/PR-19110.md), [[Kernels] Add activation chunking logic to FusedMoEModularKernel](../sources/prs/vllm/PR-19168.md), [[Kernel] Apply torch.Tag.needs_fixed_stride_order only for torch==2.6.0](../sources/prs/vllm/PR-19346.md), [[Hardware][NVIDIA][kernel] Fp4 MOE quant kernel optimization](../sources/prs/vllm/PR-19500.md), [Only build CUTLASS MoE kernels on Hopper](../sources/prs/vllm/PR-19648.md), [[Kernels] Use empty for modular MoE workspaces](../sources/prs/vllm/PR-19667.md), [[feat]: CUTLASS block scaled group gemm for SM100](../sources/prs/vllm/PR-19757.md), [[Feature] Integrate new deepgemm](../sources/prs/vllm/PR-19820.md), [[Quantization] Add compressed-tensors NVFP4 MoE Support](../sources/prs/vllm/PR-19990.md), [Enable V1 for Hybrid SSM/Attention Models](../sources/prs/vllm/PR-20016.md), [[Bugfix] Build moe_data for both sm100 and sm90](../sources/prs/vllm/PR-20086.md), [ [Feature] Integrate SM100 DeepGEMM support](../sources/prs/vllm/PR-20087.md), [Add ModelOpt Qwen3 nvfp4 support](../sources/prs/vllm/PR-20101.md), [[Bugfix] Mark 'hidden_states' as mutable in moe_forward registration.](../sources/prs/vllm/PR-20152.md), [[Bugfix] Fix topk_ids indices_type for CUTLASS w8a8 FP8 MoE](../sources/prs/vllm/PR-20166.md), [[Bugfix] Fix Maverick correctness by filling zero to cache space in cutlass_moe](../sources/prs/vllm/PR-20167.md), [[V1] [ROCm] Enable EP with AITER Fused MoE](../sources/prs/vllm/PR-20270.md), [[Kernel][Bugfix] Fixup some warnings in nvfp4_blockwise_moe when CUDA < 12.8](../sources/prs/vllm/PR-20324.md), [[Misc] DP : Add ExpertTokensMetadata](../sources/prs/vllm/PR-20332.md), [[feat]: add SM100 support for cutlass FP8 groupGEMM](../sources/prs/vllm/PR-20447.md), [Support Llama 4 for cutlass_moe_fp4](../sources/prs/vllm/PR-20453.md), [Support Llama 4 for fused_marlin_moe](../sources/prs/vllm/PR-20457.md), [[Perf] Reuse workspace for FP8+FP4 Marlin MoE](../sources/prs/vllm/PR-20500.md), [[Bugfix] Fix missing per_act_token parameter in compressed_tensors_moe](../sources/prs/vllm/PR-20509.md), [[feat] enable SM100 CUTLASS block scaled group gemm for smaller batch sizes](../sources/prs/vllm/PR-20640.md), [GLM-4.5 Model Support](../sources/prs/vllm/PR-20736.md), [[Performance] Performance improvements in non-blockwise fp8 CUTLASS MoE](../sources/prs/vllm/PR-20762.md), [[fix]: disable cutlass block scaled group gemm for EP](../sources/prs/vllm/PR-20781.md), [[Feature][EPLB] Add eplb support for Qwen3](../sources/prs/vllm/PR-20815.md), [[Bugfix] Fix a couple PPLX+CUTLASS MoE bugs](../sources/prs/vllm/PR-20825.md), [[Bug] Fix DeepGemm for EP low latency case](../sources/prs/vllm/PR-20833.md), [[Perf] Use Triton instead of Torch for DeepGEMM Per Token Group Quant](../sources/prs/vllm/PR-20841.md), [[Kernel] DeepGemm MoE : Integrate triton permute / unpermute kernels ](../sources/prs/vllm/PR-20903.md), [[Perf] Add swap_ab to SM90 FP8 non-block CUTLASS moe grouped gemm](../sources/prs/vllm/PR-20911.md), [[Misc] Qwen MoE model supports LoRA](../sources/prs/vllm/PR-20932.md), [Support mnnvl all2allv from Flashinfer](../sources/prs/vllm/PR-21003.md), [[Bugfix] Allocate less memory in non-batched CUTLASS MoE](../sources/prs/vllm/PR-21121.md), [[Feature][OCP MX] Support mxfp6 and mixed mxfp6-mxfp4](../sources/prs/vllm/PR-21166.md), [[Feature][Kernel]FusedMoE LoRA](../sources/prs/vllm/PR-21229.md), [[v1] - Mamba1 Attention Metadata](../sources/prs/vllm/PR-21249.md), [Support CUTLASS NVFP4 (w4a4) for Blackwell Geforce GPUs (SM120)](../sources/prs/vllm/PR-21309.md), [Support Tensorrt-LLM MoE fp4 for low-latency](../sources/prs/vllm/PR-21331.md), [[TPU][Bugfix] fix moe layer](../sources/prs/vllm/PR-21340.md), [[Quantization] Enable BNB support for more MoE models](../sources/prs/vllm/PR-21370.md), [Update flashinfer CUTLASS NVFP4 MoE Kernel to use per expert global scaling factor](../sources/prs/vllm/PR-21408.md), [[NVIDIA] Explicitly disable shuffled weights for flashinfer blockscale moe fp8 kernels](../sources/prs/vllm/PR-21411.md), [[Bug] Fix Compressed Tensor NVFP4 `cutlass_fp4_group_mm` illegal memory access](../sources/prs/vllm/PR-21465.md), [[MoE] More balanced expert sharding](../sources/prs/vllm/PR-21497.md), [[NVIDIA] Fix Llama4 Scout FP4 functionality issues](../sources/prs/vllm/PR-21499.md), [Enable 4bit bnb prequant MOE](../sources/prs/vllm/PR-21548.md), [[Feature] Add Flashinfer MoE Support for Compressed Tensor NVFP4](../sources/prs/vllm/PR-21639.md), [[xpu]support moe models on XPU platform](../sources/prs/vllm/PR-21643.md), [support `torch.compile` for bailing moe](../sources/prs/vllm/PR-21664.md), [feat: Add Support GPTQ Quantization MOE on ROCM vllm serve](../sources/prs/vllm/PR-21733.md), [[Qwen3] Enable dual-chunk-attention support for Qwen3 models.](../sources/prs/vllm/PR-21924.md), [Fix Flashinfer CUTLASS MOE Allgather](../sources/prs/vllm/PR-21963.md), [[BUGFIX] KeyError 'layers.14.mlp.gate.g_idx' for Qwen3-MoE with GPTQ on ROCm](../sources/prs/vllm/PR-22017.md), [[EPLB] Support ernie4.5-moe](../sources/prs/vllm/PR-22100.md), [[Bugfix] Fix MoE BNB version](../sources/prs/vllm/PR-22260.md), [[gpt-oss] flashinfer mxfp4](../sources/prs/vllm/PR-22339.md), [[gpt-oss] triton kernel mxfp4](../sources/prs/vllm/PR-22421.md), [Fix Llama4 FlashInfer FP4 MoE issues](../sources/prs/vllm/PR-22511.md), [[Model] Add Ernie4.5 VL Model Support](../sources/prs/vllm/PR-22514.md), [Fix torch version check for SM100 mxfp4 ](../sources/prs/vllm/PR-22535.md), [[Quantization] Expand compressed-tensors MoE matching logic to support NFP4 + FP8 MoEs](../sources/prs/vllm/PR-22674.md), [Fix GGUF loader for Qwen3 MoE.](../sources/prs/vllm/PR-22785.md), [[FIXBUG] Add return_success parameter to moe_wna16_weight_loader function](../sources/prs/vllm/PR-22797.md), [[Model] Modify the gate implementation of glm4_moe](../sources/prs/vllm/PR-22832.md), [[XPU] support data parallel for MoE models on XPU](../sources/prs/vllm/PR-22887.md), [[Bugfix] Fix DeepSeek MTP](../sources/prs/vllm/PR-22934.md), [[Fix] enable swap_ab for pplx problem size computation](../sources/prs/vllm/PR-22991.md), [Use Blackwell FlashInfer MXFP4 MoE by default if available ](../sources/prs/vllm/PR-23008.md), [[Bugfix] fix qwen3 moe fp8 accuracy issue](../sources/prs/vllm/PR-23031.md), [[Kernel] CUTLASS MoE FP8: Integrate cuda moe permute/unpermute](../sources/prs/vllm/PR-23045.md), [Add routed_scaling_factor to MoE grouped topk](../sources/prs/vllm/PR-23123.md), [[Bugfix] Fix accuracy issue when using flashinfer cutlass moe, TP=1 and modelopt.](../sources/prs/vllm/PR-23125.md), [[CPU] add cpu fused moe pytorch native implementation](../sources/prs/vllm/PR-23146.md), [[Perf] Small optimizations for silu_mul_fp8_quant_deep_gemm](../sources/prs/vllm/PR-23265.md), [[Kernels] Overlap shared experts with send/recv](../sources/prs/vllm/PR-23273.md), [[Kernel] Add fused grouped_topk kernel for MoE](../sources/prs/vllm/PR-23274.md), [[Bugfix] Fix Qwen3 MoE GPTQ inference](../sources/prs/vllm/PR-23490.md), [DP/EP Support for gpt-oss with deepep-ht comm kernel on SM100](../sources/prs/vllm/PR-23608.md), [[Feature] Add Hopper DeepGEMM E8M0 for DeepSeekV3.1 scale_fmt](../sources/prs/vllm/PR-23666.md), [[Core/DBO][1/N] Add Dual-Batch Overlap mechanism to VLLM](../sources/prs/vllm/PR-23693.md), [[Kernel][B200] mxfp4 fused cutlass moe](../sources/prs/vllm/PR-23696.md), [[Bugfix][Misc] Fix silu_and_mul_nvfp4_quant issue and extract common utils for nvfp4 kernel source files](../sources/prs/vllm/PR-23727.md), [[Feat][EPLB] A novel static EPLB placement strategy for MoE models.](../sources/prs/vllm/PR-23745.md), [[fix]: add Arm 4bit fused moe support](../sources/prs/vllm/PR-23809.md), [[Model][gpt-oss] Support DP+EP for GPT-OSS with FlashInfer trtllm-gen MoE](../sources/prs/vllm/PR-23819.md), [[Model] Add LongCat-Flash ](../sources/prs/vllm/PR-23991.md), [[BUGFIX] GPTQ quantization compatibility for Qwen3 MOE models (AutoGPTQ and AutoRound-GPTQ)](../sources/prs/vllm/PR-23994.md), [[PERF] Allreduce fusion. Support torch native matching. Tuning of the thresholds](../sources/prs/vllm/PR-24248.md), [[Kernel][Quantization] add w4a8 support for marlin kernel](../sources/prs/vllm/PR-24722.md), [[Model] Support Qwen3-VL Model Series](../sources/prs/vllm/PR-24727.md), [feat: BF16 FlashInfer Fused Cutlass MOE for Hopper and Blackwell Expert Parallel](../sources/prs/vllm/PR-25503.md), [[Quantization/NVFP4] Speed up TRTLLM NVFP4 MOE weight loading and fix K/V scale loading for MLA Attn](../sources/prs/vllm/PR-25968.md), [[Bugfix] Allow skipping MoE in NVFP4 (fix for MTP)](../sources/prs/vllm/PR-25987.md), [[MoE] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked](../sources/prs/vllm/PR-25990.md), [Fix undefined symbol: cutlass_moe_mm_sm100](../sources/prs/vllm/PR-26098.md), [[ModelOpt] Load w13/w2_input_scale for all experts, nvfp4](../sources/prs/vllm/PR-26135.md), [[Bug] Fix Shape Validation for Fallback while Enabling E8M0 for DeepGEMM](../sources/prs/vllm/PR-26322.md), [[Performance] Dual stream execution of "shared_experts" and "selected_experts" inside FusedMoE](../sources/prs/vllm/PR-26440.md), [[ROCM] MoE fp4 CK kernel](../sources/prs/vllm/PR-26545.md), [support flashinfer_fp4 moe for 5090 gpu](../sources/prs/vllm/PR-26669.md), [[NVIDIA] [Perf] Update to leverage flashinfer trtllm FP4 MOE throughput kernel](../sources/prs/vllm/PR-26714.md), [[Bugfix] Fix gpt-oss w4a8 DP/EP on B200](../sources/prs/vllm/PR-26729.md), [[Kernels] Enable FlashInfer FP8 Blockscale on SM90 (for TEP DSR1)](../sources/prs/vllm/PR-27134.md), [[ROCM] Enable CompressedTensorsWNA16](../sources/prs/vllm/PR-27187.md), [Flashinfer_CUTLASS_MOE fuses quantization for TP](../sources/prs/vllm/PR-27223.md), [Bugfix: Cutlass FP8 FusedMoE bad scaling factors](../sources/prs/vllm/PR-27255.md), [Feature: Support Relu2 in FusedMoE fp8 cutlass path](../sources/prs/vllm/PR-27261.md), [[Performance] Support FP8 flashinfer TRTLLM MOE on Qwen3 and Qwen-3next](../sources/prs/vllm/PR-27492.md), [[Attention] Use sparse prefill kernel for fp8 kv-cache in DeepSeek-v3.2](../sources/prs/vllm/PR-27532.md), [[Performance][B200] Fix deepgemm prologue](../sources/prs/vllm/PR-27897.md), [[Model] Consolidate Deepseek-MoE implementation with DeepSeek-v2](../sources/prs/vllm/PR-28101.md), [[Perf][DeepSeek] Add sigmoid+bias fusion to fused_grouped_topk from TRTLLM](../sources/prs/vllm/PR-28124.md), [[flashinfer] fix FI all2all with FI cutlass moe](../sources/prs/vllm/PR-28166.md), [[Feature] Support recording expert indices for rollout router replay](../sources/prs/vllm/PR-28284.md), [[Performance][B200] silu_mul_quant: pack scales in int32](../sources/prs/vllm/PR-28358.md), [[Bugfix][EPLB] Disabled shared expert overlap when EPLB is enabled](../sources/prs/vllm/PR-28377.md), [[Feature] Prefill Context Parallel (PCP) basic support](../sources/prs/vllm/PR-28718.md), [[Model] Add support for openPangu moe model](../sources/prs/vllm/PR-28775.md), [[Bugfix] Fix GPT-OSS AR+NORM fusion](../sources/prs/vllm/PR-28841.md), [[Bugfix] Make compressed-tensors MoEs respect ignored layers](../sources/prs/vllm/PR-28878.md), [Add TRTLLM MoE NVFP4 kernel to CompressedTensorsW4A4MoeMethod](../sources/prs/vllm/PR-28892.md), [[NVIDIA] Guard SM100 CUTLASS MoE macro to SM100 builds v2](../sources/prs/vllm/PR-28938.md), [[Feat] Support non-gated activations in NVFP4 modelopt path](../sources/prs/vllm/PR-29004.md), [[LoRA] Optimize 3D MoE logic](../sources/prs/vllm/PR-29222.md), [chore: add RTX_PRO_6000 GLM4.6-FP8 kernel tuning](../sources/prs/vllm/PR-29240.md), [[Kernel] Add NVFP4 MoE CUTLASS support for SM120](../sources/prs/vllm/PR-29242.md), [Lora MoE Align Improvements](../sources/prs/vllm/PR-29257.md), [[Perf] Disable DeepGEMM MoE by default when TP=8 is used](../sources/prs/vllm/PR-29346.md), [Add unpermute-aware fused MoE path and small-batch fallback](../sources/prs/vllm/PR-29354.md), [[Bugfix] Fix grouped_topk pytorch impl when num_experts can't be grouped properly](../sources/prs/vllm/PR-29439.md), [[Kernel][MoE] optimize `moe_align_block_size`](../sources/prs/vllm/PR-29642.md), [[Kernel]Support W4A8 Grouped GEMM on Hopper](../sources/prs/vllm/PR-29691.md), [[MoE-FP8-modelopt] Add FlashInfer alignment padding for intermediate dimensions](../sources/prs/vllm/PR-29748.md), [Add Mistral Large 3 and Ministral 3](../sources/prs/vllm/PR-29757.md), [[ROCm] [Fused Moe EP] Use binary expert mask for aiter fused moe kernel](../sources/prs/vllm/PR-29773.md), [[ROCm][MXFP4] Infer w4a4 quant method in rocm aiter fused moe](../sources/prs/vllm/PR-29775.md), [[EPLB] Support EPLB w/ NVFP4](../sources/prs/vllm/PR-29804.md), [[Bugfix][Model] Support LoRA on Qwen3 Output Embedding](../sources/prs/vllm/PR-29816.md), [[Bugfix] Fix FP8 MoE LoRA](../sources/prs/vllm/PR-29890.md), [[Kernel][Quantization][MoE] add marlin kernel support for turing (sm75)](../sources/prs/vllm/PR-29901.md), [[moe] Use enable_chunking func (to support disabling chunking)](../sources/prs/vllm/PR-29935.md), [[moe] Allow disabling DP chunking](../sources/prs/vllm/PR-29936.md), [[Perf] Do FP4 quant before All gather on flashinfer trtllmgen MOE ](../sources/prs/vllm/PR-30014.md), [[Quantization] Support Quark int4-fp8 w4a8 for MoE](../sources/prs/vllm/PR-30071.md), [[Model][Quantization] Restore MoE + GGUF models support (incl. Qwen3 MoE) by allowing Sideload Parameters](../sources/prs/vllm/PR-30116.md), [[Model][Quantization] Override HF defaults to GGUF ones (incl. Qwen3 MoE)](../sources/prs/vllm/PR-30118.md), [Add latent MoE support](../sources/prs/vllm/PR-30203.md), [[Bugfix]: Fix glm46 awq marlin moe wna16 compatibility](../sources/prs/vllm/PR-30210.md), [[LoRA] Reduce the loading time of MoE LoRA](../sources/prs/vllm/PR-30243.md), [gptq marlin quantization support for fused moe with lora](../sources/prs/vllm/PR-30254.md), [[LoRA] Support Quantized Adapters](../sources/prs/vllm/PR-30286.md), [[Model][Quantization] Fix / Add GGUF support for Qwen2 MoE models](../sources/prs/vllm/PR-30307.md), [[bugfix][quantization] fix quark qwen3 kv_cache quantization](../sources/prs/vllm/PR-30308.md), [[fix] fix SM check for Flashinfer TRTLLM MOE](../sources/prs/vllm/PR-30314.md), [[ROCm][Quantization] GPT OSS Upstream MoE wmxfp4_afp8 with static scales](../sources/prs/vllm/PR-30357.md), [[Feature] Add SM103 (Blackwell Ultra) Support to vLLM](../sources/prs/vllm/PR-30484.md), [[Bugfix] Fix Triton FusedMoE LoRA](../sources/prs/vllm/PR-30585.md), [[Perf] Eliminate padding and slicing op for GPT-OSS with Flashinfer MXFP4 MXFP8 MoE](../sources/prs/vllm/PR-30647.md), [fused_moe_lora PDL improvements](../sources/prs/vllm/PR-30716.md), [[SM100] Enable fp8 compute for prefill MLA](../sources/prs/vllm/PR-30746.md), [Add support for LoRA adapters in Nemotron-H models](../sources/prs/vllm/PR-30802.md), [[Compressed-Tensors] Simplify NVFP4 Conditions, enable marlin support for NVFP4A16 MoEs](../sources/prs/vllm/PR-30881.md), [[Mics] add pcp basic support to MoE model](../sources/prs/vllm/PR-31003.md), [[Bugfix] Fix GLM-4 MoE router logits dtype for data parallel chunking](../sources/prs/vllm/PR-31055.md), [[BugFix] LoRA: Support loading base_layer of experts](../sources/prs/vllm/PR-31104.md), [[Bugfix] Fix MoE LoRA bin/pt loading](../sources/prs/vllm/PR-31161.md), [[Kernel] Add topk_sigmoid kernel](../sources/prs/vllm/PR-31246.md), [pin lora_b moe weights on cpu](../sources/prs/vllm/PR-31317.md), [[Misc] Fix Qwen2-MoE shared_expert_gate](../sources/prs/vllm/PR-31339.md), [[BugFix] add select_gemm_impl on CompressedTensorsWNA16MoEMethod to support LoRA](../sources/prs/vllm/PR-31453.md), [[Bugfix][ROCm] Fix Static Quant Issue](../sources/prs/vllm/PR-31502.md), [[ROCm][Bugfix] Fix accuracy issue on fmoe when `VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS` enabled](../sources/prs/vllm/PR-31523.md), [[FIX] Add NO_MUL activation support for modular kernel path](../sources/prs/vllm/PR-31528.md), [Use the same memory for workspace13 and fused_output.](../sources/prs/vllm/PR-31531.md), [[Fix] Align fused moe lora_b shape with peft](../sources/prs/vllm/PR-31534.md), [[Bugfix] Fix Broken ModelOpt NVFP4 MoE](../sources/prs/vllm/PR-31742.md), [[LoRA]Disable linear LoRA kernel PDL](../sources/prs/vllm/PR-31777.md), [[Perf] Fuse stride preparation for NVFP4 cutlass_moe](../sources/prs/vllm/PR-31837.md), [[1/N][Attention] Restructure attention: move files](../sources/prs/vllm/PR-31916.md), [[5/N][Attention] Finish eliminating `vllm/attention` folder](../sources/prs/vllm/PR-32064.md), [Add TMA support to fused_moe_lora kernel](../sources/prs/vllm/PR-32195.md), [[Hardware][SM100] Add TRTLLM Kernel for INT4 W4A16 Kernel.](../sources/prs/vllm/PR-32437.md), [[Perf][Kernel] Optimize FP4 quantization kernels (SM100F)](../sources/prs/vllm/PR-32520.md), [fix: Add glm4_moe_lite to MLA detection](../sources/prs/vllm/PR-32614.md), [[Bugfix] Fix FP8 MoE EP Weight Loading for ModelOpt Llama4](../sources/prs/vllm/PR-32886.md), [[NVIDIA] [feat] Integrate flashinfer Trtllmgen bf16 moe](../sources/prs/vllm/PR-32954.md), [Add support for Mistral Large 3 inference with Flashinfer MoE](../sources/prs/vllm/PR-33174.md), [fix: Add SM120 (RTX Blackwell) support for FlashInfer CUTLASS NVFP4 MoE kernels](../sources/prs/vllm/PR-33417.md), [[Kernel] Support Flashinfer trtllm fused MoE non gated FP8 & NVFP4](../sources/prs/vllm/PR-33506.md), [Adding support to Sarvam's MoE models](../sources/prs/vllm/PR-33942.md), [[Bugfix]fix output Nan/Inf in marlin if dtype=float16](../sources/prs/vllm/PR-33972.md), [[Kernel] Optimize grouped topk kernel](../sources/prs/vllm/PR-34206.md), [[ModelBash][DSR1 NVFp4] Avoid Bf16 Bias Cast](../sources/prs/vllm/PR-34298.md), [[ModelBash][DSV3] Add TRTLLM DSV3 Router GEMM kernel (6% B1 Speedup)](../sources/prs/vllm/PR-34302.md), [[Kernel] Integrate SM100 MXFP8 blockscaled grouped MM and quant kernels](../sources/prs/vllm/PR-34448.md), [[Model] Add NVFP4 quantization support for Step3.5-Flash](../sources/prs/vllm/PR-34478.md), [[Bugfix] Handle num_expert_group=None in flashinfer block-scale FP8 MoE](../sources/prs/vllm/PR-34494.md), [[Quantization] add humming quantization kernel](../sources/prs/vllm/PR-34556.md), [[Bugfix] Fix NVFP4 TRTLLM MoE non-gated support; add gsm8k for Nemotron-3-Nano FP8+NVFP4](../sources/prs/vllm/PR-34725.md), [[Performance] Cublas Bf16 Gate with Fp32 Output](../sources/prs/vllm/PR-35121.md), [[Bugfix] Fix DSV3 kernels breaking _C and _moe_C on unsupported arches](../sources/prs/vllm/PR-35123.md), [[Bugfix] Fix expert_ids padding values in moe_align_block_size kernel](../sources/prs/vllm/PR-35161.md), [[Quant][Feature] Support online MXFP8 quantization for MoE and dense models](../sources/prs/vllm/PR-35448.md), [[MoE][Perf] Wrap DSV3 QKVAProj GEMM in custom op for torch.compile](../sources/prs/vllm/PR-35751.md), [[MoE] Move PF Methods to Folder](../sources/prs/vllm/PR-35927.md), [Add support for ModelOpt MXFP8 MoE models](../sources/prs/vllm/PR-35986.md), [[Bugfix] Fix passing of activation_type to trtllm fused MoE NVFP4 and FP8](../sources/prs/vllm/PR-36017.md), [[Kernel] Add FlashInfer MoE A2A Kernel](../sources/prs/vllm/PR-36022.md), [[Bugfix] Disable FlashInfer TRTLLM BF16 path for non-gated MoE](../sources/prs/vllm/PR-36146.md), [[Perf] Add TRTLLM FP8 MoE Modular Kernel](../sources/prs/vllm/PR-36307.md), [[XPU] Support block fp8 moe by fallback to TritonExpert on XPU](../sources/prs/vllm/PR-36458.md), [[Bug][MoE] Fix TRTLLM NVFP4 Routing Kernel Precision](../sources/prs/vllm/PR-36725.md), [[Bug][MoE] Strengthen _supports_current_device() checks in the TRTLLM FP8, NVFP4, and FlashInfer CuteDSL MoE experts](../sources/prs/vllm/PR-36728.md), [[MoE Refactor] Mxfp4 oracle rebased](../sources/prs/vllm/PR-37128.md), [[Kernel] Add gpt-oss Router GEMM kernel](../sources/prs/vllm/PR-37205.md), [[MoE/EPLB] Fix FlashInfer nvfp4 experts + EPLB correctness](../sources/prs/vllm/PR-37217.md), [[Kernel] Add non-gated support for NVFP4 CUTLASS MoE](../sources/prs/vllm/PR-37320.md), [[Kernel] Add MXFP4 W4A4 CUTLASS MoE kernel for SM100](../sources/prs/vllm/PR-37463.md), [[4/n] Migrate FP4/W4A8 CUTLASS kernels to torch stable ABI](../sources/prs/vllm/PR-37503.md), [[Bugfix] Disable monolithic TRTLLM MoE for Renormalize routing (#37591)](../sources/prs/vllm/PR-37605.md), [[Perf] Use torch compile to fuse pack topk in trtllm moe](../sources/prs/vllm/PR-37695.md), [[MoE] Move FlashInfer CuteDSL experts into fused_moe/experts/](../sources/prs/vllm/PR-37759.md), [[Feature] Support per-draft-model MoE backend via `--speculative-config`](../sources/prs/vllm/PR-37880.md), [[MoE Kernel] Flashinfer nvfp4 cutedsl moe kernel integration](../sources/prs/vllm/PR-38050.md), [[Quantization] Add FlashInfer CuteDSL batched experts backend for NVFP4 MoE](../sources/prs/vllm/PR-38251.md), [[MoE] Add RoutingMethodType.Simulated to TRT-LLM FP8/NVFP4 kernel allowlists](../sources/prs/vllm/PR-38329.md), [[NVIDIA] Bugfix NVFP4 DGX Spark and RTX50](../sources/prs/vllm/PR-38423.md), [[Kernels][MoE] Fix legacy_routing to use bitmatrix-based routing path](../sources/prs/vllm/PR-38504.md), [[Quant] add CompressedTensorsW8A8Mxfp8 for linear and MoE layers](../sources/prs/vllm/PR-38815.md), [[Bugfix] Re-enable Renormalize routing for TRT-LLM MoE experts](../sources/prs/vllm/PR-38859.md), [[MoE Refactor] Split up compressed_tensors_moe.py](../sources/prs/vllm/PR-38960.md), [[Bug] Fix routing bias dtype for trtllm per-block fp8 moe](../sources/prs/vllm/PR-38989.md), [[Bugfix][MoE] Fix 6-8% decode regression: prefer multi-stream shared expert overlap](../sources/prs/vllm/PR-38990.md), [[Perf] Change Trtllm fp8 MoE to use Shuffled Weights and BlockMajorK Layout](../sources/prs/vllm/PR-38993.md), [[MoE] Move GPT OSS Triton kernel experts into fused_moe/experts/](../sources/prs/vllm/PR-39007.md), [[Gemma4] Support quantized MoE ](../sources/prs/vllm/PR-39045.md), [[Bug] Fix Trtllm Fp8 MoE Weight Shuffle Memory Fragamentation](../sources/prs/vllm/PR-39054.md), [[XPU] Quick fix for TritonMLA to remove cuda hardcode](../sources/prs/vllm/PR-39088.md), [perf(moe): add tuned fused_moe config for RTX PRO 6000 Blackwell Server Edition](../sources/prs/vllm/PR-39183.md), [[Bugfix] FlashInfer MXINT4 MoE crashes, missing do_finalize](../sources/prs/vllm/PR-39315.md), [fix: clamp NaN/Inf in topk_softmax to prevent duplicate expert IDs](../sources/prs/vllm/PR-39391.md), [[Kernel] Support TRTLLM GEN NVFP4 MoE for non-512-aligned hidden dims via weight padding](../sources/prs/vllm/PR-39510.md), [[Bugfix] [Tests] Enforce `out` tensor device in `kernel/moe/test_cutedsl_moe.py`](../sources/prs/vllm/PR-39644.md), [[Bugfix] Fix mismatch between global and local attention heads in tensor-parallel mode for param2moe model](../sources/prs/vllm/PR-39707.md), [[Bugfix] Reject non-nvfp4 dtypes when using the flashinfer_nvlink_one_sided all2all backend](../sources/prs/vllm/PR-39717.md), [[Bugfix] Disable FlashInfer CUTLASS MoE on SM121 (DGX Spark)](../sources/prs/vllm/PR-39825.md), [[Core] Replace routing replay with device cache and async D2H pipeline](../sources/prs/vllm/PR-39917.md), [[Bugfix] Temporarily disable B200 fp4 MoE layer tests](../sources/prs/vllm/PR-40057.md), [[Bugfix] moe lora align kernel grid](../sources/prs/vllm/PR-40131.md), [Fix MoE backend selection for LoRA (unquantized MoE)](../sources/prs/vllm/PR-40273.md), [[MoE] Move cutlass moe to fused_moe/experts/](../sources/prs/vllm/PR-40574.md), [[DSV4] Add silu clamp limit to shared expert](../sources/prs/vllm/PR-40950.md), [[DSV4] Add BF16 and MXFP8 A2A support for flashinfer a2a one sided](../sources/prs/vllm/PR-40960.md), [[Kernel][MoE] Support GELU on TRT-LLM NvFP4 fused MoE for Gemma4](../sources/prs/vllm/PR-41050.md), [[DSV4] Fuse norm and router for low latency scenario](../sources/prs/vllm/PR-41263.md), [[ROCm][Quantization][3/N] Refactor quark_moe w4a4 w/ oracle](../sources/prs/vllm/PR-41436.md), [[Quantization] Rework quantization_config to use QuantKey and allow for activation override](../sources/prs/vllm/PR-41566.md), [[MXFP4] Support for linear layers + compressed-tensors integration](../sources/prs/vllm/PR-41664.md), [[CPU] Add MXFP4 W4A16 MoE support](../sources/prs/vllm/PR-41922.md), [[MoE] Move various experts classes to fused_moe/experts/](../sources/prs/vllm/PR-41979.md), [[Bugfix] Add swiglu limits to deepgemm fp8 methods](../sources/prs/vllm/PR-41986.md), [[LoRA] Support 2D and 3D MoE LoRA adapter at the same time](../sources/prs/vllm/PR-42242.md), [Refactor AWQ Marlin MoE onto modular WNA16 oracle](../sources/prs/vllm/PR-42483.md), [[Perf] Wire silu_and_mul_per_block_quant into TritonFP8MoE (MiniMax-M2) ](../sources/prs/vllm/PR-42497.md), [[Kernel] Pack topk id/weights triton kernel](../sources/prs/vllm/PR-42527.md), [[Bugfix] fix swiglu limit issue for humming backend + deepseek v4](../sources/prs/vllm/PR-42541.md), [[6/n] Migrate activation kernels, gptq, gguf, non cutlass w8a8 to libtorch stable ABI (continued)](../sources/prs/vllm/PR-42663.md), [Fix Weight loading for Qwen3.5-MTP and Qwen3-VL using runai_streamer](../sources/prs/vllm/PR-42716.md), [[Refactor] Remove dead cuda kernels](../sources/prs/vllm/PR-42767.md), [[Perf] Re-enable flashinfer autotune by default and cleanup](../sources/prs/vllm/PR-42857.md), [[Model Refactoring] Migrate DeepSeek V4 to vllm/models/ [1/N] ](../sources/prs/vllm/PR-43004.md), [[Kernel] (2/N) Machete - Integrate into CompressedTensorsWNA16 and GPTQMarlin](../sources/prs/vllm/PR-7701.md), [FlashInfer Track A FP8 Block-Scale MoE](../wiki/kernels/fused-moe.md), [Grouped GEMM Contracts for MoE and NVFP4](../wiki/kernels/grouped-gemm.md) | +| `prefill` | [FlashMLA — Multi-head Latent Attention](../sources/blogs/flashmla.md), [DeepSeek-V3.2-Exp in vLLM: Fine-Grained Sparse Attention in Action](../sources/blogs/vllm-deepseek-v3-sparse-attention.md), [FlashInfer MLSys 2026 Track C: Gated Delta Net](../sources/contests/flashinfer-mlsys26/track-c-gated-delta-net.md), [[None][feat] Optimize mamba SSD prefill and extend flashinfer dispatch](../sources/prs/TensorRT-LLM/PR-12731.md), [[None][feat] Optimize causal_conv1d prefill and decode kernels](../sources/prs/TensorRT-LLM/PR-13103.md), [[None][feat] Add chunked prefill support for Gemma4 (text + vision multimodal)](../sources/prs/TensorRT-LLM/PR-14134.md), [Add Blackwell MLA forward (shape: d=192, dv=128) implementation](../sources/prs/cutlass/PR-2472.md), [feat: update decode attention APIs](../sources/prs/flashinfer/PR-1007.md), [add multi-item scoring](../sources/prs/flashinfer/PR-1015.md), [feat: add functional per-head FP8 quantization for FA3](../sources/prs/flashinfer/PR-1033.md), [[nvidia] initial support for blackwell kernels](../sources/prs/flashinfer/PR-1039.md), [Parameterize prefix mask call (needed by POD-Attention)](../sources/prs/flashinfer/PR-1059.md), [bugfix: follow user-specified sm_scale for blackwell cutlass fmha](../sources/prs/flashinfer/PR-1072.md), [bugfix: fix fp8 attention kernels aot compilation issue](../sources/prs/flashinfer/PR-1087.md), [bugfix: host-precomuted plan function for blackwell fmha](../sources/prs/flashinfer/PR-1106.md), [[Feature] Support PDL for batch Prefill and Decode](../sources/prs/flashinfer/PR-1117.md), [[feat] add unified batch attention w/ correctness tests.](../sources/prs/flashinfer/PR-1137.md), [Fix FA2 and FA3 multi-item scoring and cuda illegal memory access error](../sources/prs/flashinfer/PR-1140.md), [Fix the issue with auxillary kernel launch and grid dim calculation](../sources/prs/flashinfer/PR-1208.md), [feat: Add non-causal cudnn prefill kernels](../sources/prs/flashinfer/PR-1230.md), [add trtllm-gen context attention](../sources/prs/flashinfer/PR-1239.md), [refactor: refactor trtllm-gen attention kernel integration code](../sources/prs/flashinfer/PR-1289.md), [Allow cudnn prefill kernels to be called natively](../sources/prs/flashinfer/PR-1317.md), [feat: Support logits_soft_cap for Persistent attn; fix kv split limit](../sources/prs/flashinfer/PR-1324.md), [support trtllm-gen prefill fp4 output](../sources/prs/flashinfer/PR-1360.md), [Support scale factor start index for fp4 mha prefill/decode](../sources/prs/flashinfer/PR-1363.md), [Allow BatchPrefillPagedWrapper to call cudnn API](../sources/prs/flashinfer/PR-1384.md), [GPT-OSS Support: Add Blackwell MoE mxfp4 implementation from TRTLLM and Attention Sink](../sources/prs/flashinfer/PR-1389.md), [Fix TRTLLM NVFP4-out attention kernel scale factor dim issue](../sources/prs/flashinfer/PR-1460.md), [feat: add pdl for trtllm-gen attn](../sources/prs/flashinfer/PR-1484.md), [feat: Support fp8 qkv, fp16/bf16 out MHA for trtllm-gen.](../sources/prs/flashinfer/PR-1490.md), [backend: Refactor trtllm-gen fmha metainfo loading](../sources/prs/flashinfer/PR-1518.md), [bugfix: Fix Persistent kernel precision for masked output ](../sources/prs/flashinfer/PR-1533.md), [feat: Integrate TRTLLM varlen kernel for deepseek R1 prefill ](../sources/prs/flashinfer/PR-1537.md), [feat: Add fp8-qkv, fp16/bf16 output MHA](../sources/prs/flashinfer/PR-1540.md), [Backend: downgrade trtllm-gen kernel to cuda-12](../sources/prs/flashinfer/PR-1567.md), [feat: Support for inferring out_dtype from out.dtype for TRTLLM attention kernel](../sources/prs/flashinfer/PR-1578.md), [fix: Improve TRTLLM attention kernel out_dtype unit test](../sources/prs/flashinfer/PR-1590.md), [feat: initial support for SM103, SM110, SM120, SM121](../sources/prs/flashinfer/PR-1608.md), [bugfix: collect all modules to aot](../sources/prs/flashinfer/PR-1622.md), [fix: pass workspace for trtllm-gen attention](../sources/prs/flashinfer/PR-1635.md), [perf&bugfix: skip kv-tile computation out of sliding window in FA2; fix __syncthreads in mergestate](../sources/prs/flashinfer/PR-1661.md), [feat: Batch-size invariant FA2 Prefill & Decode](../sources/prs/flashinfer/PR-1675.md), [perf: improve attention of tcgen05 flash-attention](../sources/prs/flashinfer/PR-1681.md), [test: skip the unsupported test cases for sm120/121](../sources/prs/flashinfer/PR-1710.md), [Fix tests/test_trtllm_gen_attention.py::test_trtllm_batch_prefill, ::test_trtllm_batch_decode mismatch error](../sources/prs/flashinfer/PR-1755.md), [Add head_dim=64 for tcgen05 tcgen05 flash-attention implementation](../sources/prs/flashinfer/PR-1850.md), [fix: Fix trtllm-gen prefill IMA when batch_size==1](../sources/prs/flashinfer/PR-1912.md), [feat: add xqa backend and completes NHD/HND coverage for trtllm-gen/xqa backend](../sources/prs/flashinfer/PR-2001.md), [Added an initial implementation of Q and KV Cache in fp8 and to use t…](../sources/prs/flashinfer/PR-2035.md), [Rebase FP8 SM100 Cutlass FMHA Attention to main (original PR#1238)](../sources/prs/flashinfer/PR-2047.md), [[Feature] Support batch prefill for POD Attention](../sources/prs/flashinfer/PR-2079.md), [[API change] Allow using torch.Tensor for scales for trtllm-gen attention](../sources/prs/flashinfer/PR-2084.md), [refactor: update fa3 codebase and fix hopper unittest [part 1]](../sources/prs/flashinfer/PR-2111.md), [feat: TRTLLM FMHAv2 backend for ctx attention](../sources/prs/flashinfer/PR-2142.md), [Enable Hopper FA3 FP8 attention in decode.py](../sources/prs/flashinfer/PR-2148.md), [fix: compile flags for trtllm fmha_v2 ](../sources/prs/flashinfer/PR-2175.md), [Fp8 attention are now part of cuDNN 9.17.1](../sources/prs/flashinfer/PR-2241.md), [feat: add GDN Attention](../sources/prs/flashinfer/PR-2276.md), [Added the cudnn backend Ragged KV Cache wrapper](../sources/prs/flashinfer/PR-2352.md), [feat: [Qwen3-Next] Add Cute DSL GDN decode kernel and tests](../sources/prs/flashinfer/PR-2370.md), [Remove cudaMalloc/Free in GDN prefill kernel](../sources/prs/flashinfer/PR-2415.md), [refactor: reduce hopper's gdn prefill compilation time and fix docstring.](../sources/prs/flashinfer/PR-2422.md), [feat: Add TRTLLM fmha_v2 library for SM90 attention with Skip-Softmax ](../sources/prs/flashinfer/PR-2446.md), [feat: Add TRTLLM-Gen Skip-Softmax kernels for prefill and decode](../sources/prs/flashinfer/PR-2477.md), [Ameyn/gdn decode cutedsl kernel](../sources/prs/flashinfer/PR-2498.md), [perf: cache cudaGetDeviceProperties in gdn_prefill to avoid per-call overhead](../sources/prs/flashinfer/PR-2509.md), [feat: Enable TRTLLM-Gen Skip-Softmax attention for MLA](../sources/prs/flashinfer/PR-2547.md), [fix: allow fmha_v2_prefill_deepseek on SM121 (DGX Spark)](../sources/prs/flashinfer/PR-2559.md), [fix: guard CUTLASS FMHA against SM12x and fix fmha_v2 SM121a check](../sources/prs/flashinfer/PR-2560.md), [feat: add is_sm12x_supported() helper for SM12x family detection](../sources/prs/flashinfer/PR-2574.md), [benchmarks: Add FP8 input / BF16 output in ragged prefill benchmark](../sources/prs/flashinfer/PR-2666.md), [feat: add support for more MLA head dimensions](../sources/prs/flashinfer/PR-2677.md), [Add NVFP4 KV cache quantization support for SM100](../sources/prs/flashinfer/PR-2702.md), [feat: Add DiT-oriented kernels where Qk (Bmm1) type can be reinterpreted into Int8 or BFloat16](../sources/prs/flashinfer/PR-2711.md), [feat: Expose TRT-LLM FMHA style paged KV Cache and page table layout](../sources/prs/flashinfer/PR-2770.md), [tests: skip sliding window + fp8 to prevent hang in fmha_v2 unit tests](../sources/prs/flashinfer/PR-2781.md), [[fmha-v2] Support HND and NHD paged KV cache layouts with conditional stride handling](../sources/prs/flashinfer/PR-2799.md), [[fix] Bugfix 1367: fix VariableBlockSparseAttention buffer overflow by dynamically resizing kv_lens_buffer](../sources/prs/flashinfer/PR-2802.md), [[CuTe DSL] Add modular FMHA prefill and MLA decode attention kernels](../sources/prs/flashinfer/PR-2805.md), [[Perf] Add FMHAv2 to flashinfer_benchmark.py and eliminate unnecessary H2D](../sources/prs/flashinfer/PR-2841.md), [read real strides for kv and block scale](../sources/prs/flashinfer/PR-2844.md), [feat(gdn): state checkpointing in chunk_gated_delta_rule](../sources/prs/flashinfer/PR-2908.md), [Only swizzle on v block scale; rename kv_block_scales to kv_cache_sf](../sources/prs/flashinfer/PR-2954.md), [test: xfail cuDNN FP8 prefill on Blackwell with CUDA <= 12.9](../sources/prs/flashinfer/PR-2963.md), [[feat] Add blackwell GDN prefill kernel](../sources/prs/flashinfer/PR-3001.md), [Support lse in trtllm paged attn kernels](../sources/prs/flashinfer/PR-3058.md), [Support NVFP4 KV for prefill and batch attention kernels](../sources/prs/flashinfer/PR-3097.md), [feat: Enable FP8 (E4M3/E5M2) in concat_mla_k for optimize long-context prefill performance and refactor type dispatch for BF16/FP16](../sources/prs/flashinfer/PR-3129.md), [cute-dsl fmha prefill (cubin integration): remove front-padding, add attention_sink, and pdl support](../sources/prs/flashinfer/PR-3181.md), [feat(trace): embed runnable init() in every TraceTemplate](../sources/prs/flashinfer/PR-3221.md), [fix(fmha_v2): fix FP8 V-scratch pipeline and varlen scheduler on SM90](../sources/prs/flashinfer/PR-3276.md), [perf: fix the iteration bound of SWA in FA2 prefill template](../sources/prs/flashinfer/PR-714.md), [feat: support deepseek prefill attention shape](../sources/prs/flashinfer/PR-765.md), [perf: refactor fa2 prefill template](../sources/prs/flashinfer/PR-776.md), [feat: Separate QK/VO head dim dispatch for sm90 AOT](../sources/prs/flashinfer/PR-778.md), [bugfix: fix batch prefill attention kernel unittests](../sources/prs/flashinfer/PR-781.md), [bugfix: drop CTA_TILE_Q=32](../sources/prs/flashinfer/PR-785.md), [feat: support f32 attention output in FA2 template](../sources/prs/flashinfer/PR-799.md), [feat: apply sm_scale at logits instead of q in FA2 template](../sources/prs/flashinfer/PR-801.md), [perf: memory efficient deepseek mla fused page-attention kernel](../sources/prs/flashinfer/PR-804.md), [Add POD-Attention to FlashInfer](../sources/prs/flashinfer/PR-858.md), [Naive Support for Hopper FP8 Prefill Kernel with Per-Head Quantization](../sources/prs/flashinfer/PR-869.md), [perf: reduce torch.library dispatch overhead](../sources/prs/flashinfer/PR-968.md), [perf: Fix python API overhead when CUDAGraph is not enabled](../sources/prs/flashinfer/PR-969.md), [[2/2] Support MHA prefill with FlashAttention 4.](../sources/prs/sglang/PR-10937.md), [[DeepseekV32] Enable flashmla_prefill kernel with fp8 kvcache](../sources/prs/sglang/PR-11655.md), [DeepSeek-V3.2: Add Adaptive MHA Attention Pathway for Short-Sequence Prefill](../sources/prs/sglang/PR-11892.md), [[DeepSeek-V3.2][NSA] Enable MHA Pathway for Short Sequence Prefill on B200 (SM100)](../sources/prs/sglang/PR-12788.md), [[DeepSeek v3.2] opt Context Parallelism: support fused moe, multi batch and fp8 kvcache](../sources/prs/sglang/PR-13959.md), [Support FP8 MLA prefill and 128k context.](../sources/prs/sglang/PR-14395.md), [[AMD] Support fused_rms_mxfp4_quant in the prefill stage for DeepSeek-R1-MXFP4](../sources/prs/sglang/PR-14975.md), [Support Qwen3 MoE context parallel](../sources/prs/sglang/PR-18233.md), [feat(gdn): add FlashInfer K-last SSM layout support for GDN prefill and decode for Hopper](../sources/prs/sglang/PR-18361.md), [Fp8 prefill attn kernel integration](../sources/prs/sglang/PR-18528.md), [[AMD] Fp8 prefill integration with radix cache path for dpsk models](../sources/prs/sglang/PR-20187.md), [[GDN] Add benchmark for sglang gdn prefill](../sources/prs/sglang/PR-20428.md), [FIX: (NSA) Compute topk_indices_offset when NSA prefill flashmla_sparse is used with FP8 KV cache](../sources/prs/sglang/PR-20606.md), [[DSA] Support trtllm sparse mla kernel for prefill batches ](../sources/prs/sglang/PR-21783.md), [[KDA] Optimize prefill kernels with diagonal and recompute fuse](../sources/prs/sglang/PR-24271.md), [[attn backend] Integrate tokenspeed_mla prefill/decode kernels (fp8 kv cache, blackwell)](../sources/prs/sglang/PR-24925.md), [Introduce SchedulerDPAttnAdapter to own DP-attention state](../sources/prs/sglang/PR-25611.md), [Move DP-attention adapter methods to SchedulerDPAttnAdapter](../sources/prs/sglang/PR-25612.md), [[Refactor] Rename NSA → DSA: user-facing aliases, file/class/import rename](../sources/prs/sglang/PR-25821.md), [Support MHA with chunked prefix cache for DeepSeek chunked prefill](../sources/prs/sglang/PR-5113.md), [[PD] Fix dynamic port support and MLA buffer for Mooncake](../sources/prs/sglang/PR-5415.md), [Avoid computing lse in Ragged Prefill when there's no prefix.](../sources/prs/sglang/PR-5476.md), [[Feature] Comprehensive Hybrid Parallelism Support](../sources/prs/sglang/PR-6389.md), [[PD] Add different TP sizes support for no-MLA models](../sources/prs/sglang/PR-6793.md), [Let EP prefill support new DeepGEMM](../sources/prs/sglang/PR-7310.md), [[Feature] Layer-wise Prefill](../sources/prs/sglang/PR-7634.md), [[Attention] MLA with chunked prefill](../sources/prs/vllm/PR-12639.md), [Disable chunked prefill and/or prefix caching when MLA is enabled ](../sources/prs/vllm/PR-12642.md), [[V1] V1 Enablement Oracle ](../sources/prs/vllm/PR-13726.md), [[ROCm] Disable chunked prefill/prefix caching when running MLA on non-cuda platforms](../sources/prs/vllm/PR-13844.md), [Fix mla prefill context performance](../sources/prs/vllm/PR-13897.md), [[ROCm] Enable chunked prefill/paged attention in MLA on ROCm](../sources/prs/vllm/PR-14316.md), [[BugFix] Fix nightly MLA failure (FA2 + MLA chunked prefill, i.e. V1, producing bad results)](../sources/prs/vllm/PR-15492.md), [[ROCM][KERNEL] Paged attention for V1](../sources/prs/vllm/PR-15720.md), [[Kernel] Unified Triton kernel that doesn't distinguish between prefill + decode](../sources/prs/vllm/PR-16828.md), [[ROCm][Kernel][V1] Enable AMD Radeon GPU Custom Paged Attention on v1](../sources/prs/vllm/PR-17004.md), [[Bugfix][ROCm] Use `chunked_prefill_paged_decode` as fallback for V1 attention on ROCm](../sources/prs/vllm/PR-18093.md), [[Hardware][AMD] integrate aiter chunked prefill into vllm](../sources/prs/vllm/PR-18596.md), [[Attention] MLA - Flashinfer Ragged Prefill](../sources/prs/vllm/PR-20034.md), [[Kernel] Optimize Prefill Attention in Unified Triton Attention Kernel](../sources/prs/vllm/PR-20308.md), [[Model] Pooling models default to using chunked prefill & prefix caching if supported.](../sources/prs/vllm/PR-20930.md), [Override attention metadata for fast prefill in some KV sharing setups](../sources/prs/vllm/PR-21590.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv/out Attention Kernel](../sources/prs/vllm/PR-21716.md), [[NVIDIA] Support Flashinfer TRT-LLM Prefill Attention Kernel](../sources/prs/vllm/PR-22095.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv NVFP4-out Attention Kernel](../sources/prs/vllm/PR-22703.md), [[Flashinfer] Support Flashinfer TRTLLM FP8-qkv BF16/FP16-out Attention Kernel](../sources/prs/vllm/PR-23647.md), [[Attention] Use sparse prefill kernel for fp8 kv-cache in DeepSeek-v3.2](../sources/prs/vllm/PR-27532.md), [[Feature] Prefill Context Parallel (PCP) basic support](../sources/prs/vllm/PR-28718.md), [[Attention] Make `split_decodes_and_prefills(..., require_uniform=True)` support padding](../sources/prs/vllm/PR-29644.md), [[perf] Use direct copy (broadcast) instead of cat for k_nope/k_pe in MLA prefill](../sources/prs/vllm/PR-29710.md), [[SM100] Enable fp8 compute for prefill MLA](../sources/prs/vllm/PR-30746.md), [[SM100] Resubmit FMHA FP8 prefill for MLA](../sources/prs/vllm/PR-31195.md), [[Bugfix][ROCm]Fix Qwen3-Next-80B-A3B-Thinking inference and optimize non-standard block size (544) support under rocm_atten](../sources/prs/vllm/PR-31380.md), [[1/N][Attention] Restructure attention: move files](../sources/prs/vllm/PR-31916.md), [[Kernel] use flashinfer for gdn prefill](../sources/prs/vllm/PR-32846.md), [[Attention] Use FA4 for MLA prefill](../sources/prs/vllm/PR-34732.md), [[Bugfix][MLA] Add logits size budget to sparse indexer prefill chunking](../sources/prs/vllm/PR-36178.md), [[ROCm] Attention selector reordering](../sources/prs/vllm/PR-36702.md), [[Attention] Support distinguishing between short extends and decodes](../sources/prs/vllm/PR-37303.md), [[GDN] Eliminate GPU->CPU sync in prepare_chunk_indices during prefill](../sources/prs/vllm/PR-38361.md), [[Bugfix][MLA] Change default SM100 MLA prefill backend back to TRT-LLM](../sources/prs/vllm/PR-38562.md), [[Attention][MLA] Re-enable FA4 as default MLA prefill backend](../sources/prs/vllm/PR-38819.md), [[Gemma4] Enable Fast Prefill Optimization](../sources/prs/vllm/PR-38879.md), [add warning when FP8 KV cache misses prefill query quantization](../sources/prs/vllm/PR-39752.md), [[GDN] Enable FI Blackwell GDN prefill kernel](../sources/prs/vllm/PR-40717.md), [[MLA Attention Backend] Add TOKENSPEED_MLA backend for DSR1/Kimi K25 prefill + decode on Blackwell](../sources/prs/vllm/PR-41778.md), [[Bugfix] Fix TRTLLM ragged MLA prefill workspace warmup](../sources/prs/vllm/PR-42112.md), [FlashMLA — Multi-head Latent Attention](../wiki/kernels/flashmla.md), [Gated Delta Net — Linear Attention](../wiki/kernels/gated-delta-net.md), [DeepSeek Sparse Attention / Sparse MLA](../wiki/kernels/sparse-mla.md) | | `quantization` | [NVFP4 Format Details](../sources/blogs/nvfp4-format-details.md), [DeepSeek-V3.2-Exp in vLLM: Fine-Grained Sparse Attention in Action](../sources/blogs/vllm-deepseek-v3-sparse-attention.md), [[TRTLLM-9457][feat] Add cute dsl fp8 gemm for Blackwell](../sources/prs/TensorRT-LLM/PR-10130.md), [[None][feat] sm100 weight-only kernel](../sources/prs/TensorRT-LLM/PR-10190.md), [[None][fix] impl fused triton kernel for e8m0 resmooth to reduce memory footprint](../sources/prs/TensorRT-LLM/PR-10327.md), [[None][feat] fuse shared to sparse experts in TRT-LLM Gen MoE](../sources/prs/TensorRT-LLM/PR-11143.md), [[None][fix] Fix int4 awq for sm120/121](../sources/prs/TensorRT-LLM/PR-11561.md), [[TRTLLM-10004][feat] Enable GEMM -> AR with GEMM output in registered buffers](../sources/prs/TensorRT-LLM/PR-11589.md), [[TRTLLM-10421][perf] Add fused cat+fp8_quantize CUDA kernel for DSA indexer](../sources/prs/TensorRT-LLM/PR-11899.md), [[None][feat] Support update weight for nvfp4](../sources/prs/TensorRT-LLM/PR-12320.md), [[None][feat] Add bf16 trtllm-gen moe support through flashinfer.](../sources/prs/TensorRT-LLM/PR-12738.md), [[TRTLLM-11585][feat] Add CUTEDSL moe backend for nemotron-h](../sources/prs/TensorRT-LLM/PR-12884.md), [[#12784][feat] AutoDeploy: Optimize DeepSeek-R1 model performance](../sources/prs/TensorRT-LLM/PR-12946.md), [[None][feat] Add FP4 residual quantization kernel without channel reo…](../sources/prs/TensorRT-LLM/PR-13117.md), [[TRTLLM-11127][feat] add W4A8_MXFP4_FP8 MoE unit test support](../sources/prs/TensorRT-LLM/PR-13401.md), [[None][feat] Fuse FP8 1x128 quantize + UE8M0 scale pack on SM100](../sources/prs/TensorRT-LLM/PR-13628.md), [[None][feat] Add bf16 trtllm moe through flashinfer.](../sources/prs/TensorRT-LLM/PR-13689.md), [[None][fix] Plumb swiglu_limit through DeepGEMM and TRTLLMGen FP8 fused MoE](../sources/prs/TensorRT-LLM/PR-13767.md), [[https://nvbugs/6163147][fix] swap layer.mlp in place for Mixtral modelopt export](../sources/prs/TensorRT-LLM/PR-14179.md), [[None][fix] Add SPDX Apache-2.0 headers to auto_deploy test files](../sources/prs/TensorRT-LLM/PR-14193.md), [[None][fix] Update the OSS headers in derived FLA ops and AD modeling code](../sources/prs/TensorRT-LLM/PR-14281.md), [feat: Add w4a8_mxfp4_fp8 quantization recipe.](../sources/prs/TensorRT-LLM/PR-4867.md), [[None][feat] Enable EPLB for trtllm-gen and cutlass backend](../sources/prs/TensorRT-LLM/PR-8886.md), [[None][feat] Port fp4 quantization kernel optimization from FlashInfer](../sources/prs/TensorRT-LLM/PR-9854.md), [add multi-item scoring](../sources/prs/flashinfer/PR-1015.md), [feat: add functional per-head FP8 quantization for FA3](../sources/prs/flashinfer/PR-1033.md), [bugfix: fix fp8 attention kernels aot compilation issue](../sources/prs/flashinfer/PR-1087.md), [Add CUTLASS fused moe kernels from TensorRT-LLM.](../sources/prs/flashinfer/PR-1113.md), [bugfix: Fix test and output shape of fp4 quantize](../sources/prs/flashinfer/PR-1114.md), [[Feature] Support PDL for batch Prefill and Decode](../sources/prs/flashinfer/PR-1117.md), [Add fp4 quantization swizzling tests](../sources/prs/flashinfer/PR-1157.md), [feat: enable and update all-reduce fused quantization](../sources/prs/flashinfer/PR-1164.md), [Expose fp4 blockscale swizzling kernel](../sources/prs/flashinfer/PR-1176.md), [Feature/sm100 low latency nvfp4 kernels](../sources/prs/flashinfer/PR-1214.md), [Bug fix: guard fp8 e8m0 and e2m1 compile ](../sources/prs/flashinfer/PR-1287.md), [add mm_fp4 use cudnn backend](../sources/prs/flashinfer/PR-1288.md), [[fix] fix integer overflow in FA2 customized_mask & add buffer overflow warning.](../sources/prs/flashinfer/PR-1290.md), [Update cutlass fp4 moe kernels](../sources/prs/flashinfer/PR-1294.md), [Refactor Fused Moe Module](../sources/prs/flashinfer/PR-1309.md), [test qkvo quantization not equal to 1.](../sources/prs/flashinfer/PR-1314.md), [feat: support output nvfp4 in trtllm-gen function call.](../sources/prs/flashinfer/PR-1318.md), [feat: Fused rope fp8 quantize kernel for MLA](../sources/prs/flashinfer/PR-1339.md), [feature: add fp4 mm using trtllm backend](../sources/prs/flashinfer/PR-1355.md), [bugfix: fixed cutlass fused moe usage of FP4QuantizationSFLayout::SWIZZLED](../sources/prs/flashinfer/PR-1371.md), [GPT-OSS Support: Add Blackwell MoE mxfp4 implementation from TRTLLM and Attention Sink](../sources/prs/flashinfer/PR-1389.md), [gpt-oss: Add MXFP8 x MXFP4 CUTLASS MOE for SM100 and BF16 x MXFP4 CUTLASS for SM90 + SwigluBias Activation](../sources/prs/flashinfer/PR-1396.md), [Add Mxfp4 trtllm-gen moe unit tests](../sources/prs/flashinfer/PR-1399.md), [Add alignment in MxFP8Quantization](../sources/prs/flashinfer/PR-1445.md), [Remove getEnvEnablePDL in favor of enable_pdl parameter](../sources/prs/flashinfer/PR-1446.md), [bugfix: Fix compile error for undefined swizzle enum.](../sources/prs/flashinfer/PR-1530.md), [fix: separate out fp4 lib into sm90 and sm100 versions, add oob checking in fused moe](../sources/prs/flashinfer/PR-1565.md), [bugfix: Fix test_fp4_quantize test bug](../sources/prs/flashinfer/PR-1585.md), [bugfix: fix unittest test_fp8_quantize](../sources/prs/flashinfer/PR-1599.md), [feat: initial support for SM103, SM110, SM120, SM121](../sources/prs/flashinfer/PR-1608.md), [feat: cutlass fp4 gemm bringup for SM120 & SM121](../sources/prs/flashinfer/PR-1609.md), [bugfix: fix fp4 quantization with 8x4 scale factor layout](../sources/prs/flashinfer/PR-1611.md), [feat: add support of fp4_batched_quantize](../sources/prs/flashinfer/PR-1633.md), [Add benchmark for MLARopeQuantize](../sources/prs/flashinfer/PR-1656.md), [[Hotfix] `test_fp4_quantize.py` failure on sm103](../sources/prs/flashinfer/PR-1666.md), [test: better fp8 quantization init for fused_moe test](../sources/prs/flashinfer/PR-1674.md), [tests: xfail moe quantization classes mxfp8_bf16 UTs on sm103 ](../sources/prs/flashinfer/PR-1754.md), [Masked batch nvfp4 quantization](../sources/prs/flashinfer/PR-1774.md), [[Quantization] Add per-expert global scaling factor for fp4 batched quantize](../sources/prs/flashinfer/PR-1835.md), [MLA RoPE + quantization fused kernel: shape generalization for MHA / GQA](../sources/prs/flashinfer/PR-1924.md), [silu_and_mul nvfp4 quanization fusion rework](../sources/prs/flashinfer/PR-1927.md), [fix: correct PDL parameter handling in RopeQuantize kernel](../sources/prs/flashinfer/PR-1982.md), [perf: Speed up fp4 quantization for small batch with swizzling for cutlass MoE](../sources/prs/flashinfer/PR-2025.md), [feat: Add flashinfer.rope.rope_quantize_fp8_append_paged_kv_cache (fused RoPE + Q + KV cache, supports MLA/GQA/MHA) ](../sources/prs/flashinfer/PR-2037.md), [test: Skip test_fp8_quantize.py on Hopper](../sources/prs/flashinfer/PR-2052.md), [refactor: update fa3 codebase and fix hopper unittest [part 1]](../sources/prs/flashinfer/PR-2111.md), [feat: MxInt4 x Bf16 TRT-LLM Gen MoE support](../sources/prs/flashinfer/PR-2159.md), [feat: Fused RMSNorm + FP4 Quantization Kernels in CuTe-DSL](../sources/prs/flashinfer/PR-2233.md), [feat: RMSNorm/Fused RMSNorm + FP8 Quantization kernels](../sources/prs/flashinfer/PR-2243.md), [fix: support int64 IdType for RoPE part argument in `rope_quantize_fp8_append_paged_kv_cache`](../sources/prs/flashinfer/PR-2255.md), [Optimize quantization function in large problem size](../sources/prs/flashinfer/PR-2343.md), [benchmarks: Add norm and quantization routines to microbenchmark harness.](../sources/prs/flashinfer/PR-2362.md), [fix: Fix NaN output in mxfp8_quantize for very small input values](../sources/prs/flashinfer/PR-2441.md), [Add cute-dsl backends to mxfp[8,4]_quantization for future refactor](../sources/prs/flashinfer/PR-2443.md), [Support NVFP4 KV cache decode on SM120](../sources/prs/flashinfer/PR-2520.md), [benchmark: Add MXFP4/MXFP8 quantization mode support to FP4 MoE benchmark](../sources/prs/flashinfer/PR-2635.md), [Enable sm120f compilation](../sources/prs/flashinfer/PR-2650.md), [[feat] trtllm-gen mxfp8 gemm](../sources/prs/flashinfer/PR-2653.md), [Add NVFP4 KV cache quantization support for SM100](../sources/prs/flashinfer/PR-2702.md), [feat: Add FP4 KV cache quant/dequant kernels ](../sources/prs/flashinfer/PR-2757.md), [feat: Add CuTe-DSL backend for NVFP4 quantization](../sources/prs/flashinfer/PR-2838.md), [fix: snap weight_scale_vec_size to handle block_scale_interleave padding for SM120](../sources/prs/flashinfer/PR-2898.md), [perf: Optimize CuTe-DSL fp4 and fp8 quantization kernels](../sources/prs/flashinfer/PR-2904.md), [Only swizzle on v block scale; rename kv_block_scales to kv_cache_sf](../sources/prs/flashinfer/PR-2954.md), [[feat] Trtllm-gen Per-token Nvfp4 MoE](../sources/prs/flashinfer/PR-3027.md), [Support NVFP4 KV for prefill and batch attention kernels](../sources/prs/flashinfer/PR-3097.md), [feat(trace): embed runnable init() in every TraceTemplate](../sources/prs/flashinfer/PR-3221.md), [perf: optimize per-token nvfp4 quantization kernel.](../sources/prs/flashinfer/PR-3237.md), [Naive Support for Hopper FP8 Prefill Kernel with Per-Head Quantization](../sources/prs/flashinfer/PR-869.md), [perf: reduce torch.library dispatch overhead](../sources/prs/flashinfer/PR-968.md), [perf: Fix python API overhead when CUDAGraph is not enabled](../sources/prs/flashinfer/PR-969.md), [Add AOTI shim for _weight_int4pack_mm_cpu_tensor (#149031)](../sources/prs/pytorch/PR-149386.md), [[RELEASE 2.10] Release only changes](../sources/prs/pytorch/PR-170112.md), [[RELEASE 2.11] Release only changes](../sources/prs/pytorch/PR-175091.md), [[release 2.12] Apply Release only changes to 2.12 branch](../sources/prs/pytorch/PR-180470.md), [feat: Add FP4 (E2M1) KV Cache Support with Quantization Utilities for MLA](../sources/prs/sglang/PR-10078.md), [Enable native ModelOpt quantization support (3/3)](../sources/prs/sglang/PR-10154.md), [Add support for bf16 x bf16 cutlass fused MoE](../sources/prs/sglang/PR-10275.md), [Support single batch overlap](../sources/prs/sglang/PR-10422.md), [Cache the result of `is_blackwell` platform check](../sources/prs/sglang/PR-10498.md), [Fix bias handling in TritonMoeQuantInfo within quantization/mxfp4.py](../sources/prs/sglang/PR-10579.md), [[Auto Sync] Update modelopt_quant.py (20250920)](../sources/prs/sglang/PR-10688.md), [Fuse quantize and rope in trtllm_mla MTP](../sources/prs/sglang/PR-10779.md), [Fix DSR1 accuracy for flashinfer_trtllm MoE with FP8 quantization](../sources/prs/sglang/PR-11081.md), [Support shared experts overlap in cutlass moe](../sources/prs/sglang/PR-11611.md), [Support running FP4 Deepseek on SM120.](../sources/prs/sglang/PR-11708.md), [Change bf16 to fp8 for some gemms in attention for DeepSeek ckpt v2](../sources/prs/sglang/PR-11805.md), [Use cutlass fp4 gemm by default](../sources/prs/sglang/PR-11813.md), [Support nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8/NVFP4](../sources/prs/sglang/PR-11866.md), [Feature/nano v2 offline modelopt fp8 and nvfp4](../sources/prs/sglang/PR-12018.md), [[Ascend] qwen optimization](../sources/prs/sglang/PR-12078.md), [[hotfix] missing `w13_weight_fp8` and `w2_weight_fp8` in UE8M0 requantization](../sources/prs/sglang/PR-12259.md), [Replace [silu_and_mul_]scaled_fp4_group_quant by Flashinfer equivalent](../sources/prs/sglang/PR-12376.md), [Use sgl fp4 quant kernel by default](../sources/prs/sglang/PR-12482.md), [Enable Flashinfer TRTLLM-GEN-MoE FP8 blockwise kernel for Qwen3-Next on Blackwell](../sources/prs/sglang/PR-12543.md), [[NVIDIA] Fix wrong symmetric sizes for fp4 cases](../sources/prs/sglang/PR-12640.md), [[Bugfix] Fix illegal memory access](../sources/prs/sglang/PR-12758.md), [[Ascend] support Kimi-K2-Thinking](../sources/prs/sglang/PR-12759.md), [Update dsv3 quantization auto setting for sm100](../sources/prs/sglang/PR-12778.md), [Aiter fp8 kv cache](../sources/prs/sglang/PR-13147.md), [[NPU]Optimization of `forward_npu` for `UnquantizedFusedMoEMethod`](../sources/prs/sglang/PR-13158.md), [[NVIDIA] Fix broken fp8 MoE of deepseek v3](../sources/prs/sglang/PR-13264.md), [[NVIDIA] Fix use case of SGLANG_ENABLE_FLASHINFER_GEMM](../sources/prs/sglang/PR-13274.md), [[ROCM] Optimized deepseek-r1 fp8 model with + triton_gemm_a8w8 + batch_gemm_a8w8 + fused set_mla_kv_buffer kernel](../sources/prs/sglang/PR-13617.md), [Fix EPLB + FP4 Quantization Compatibility Issue](../sources/prs/sglang/PR-13715.md), [Support fp4 fp8 non gated moe](../sources/prs/sglang/PR-13794.md), [[NVIDIA] Enable TRTLLM BF16 MoE on Blackwell GPUs](../sources/prs/sglang/PR-13798.md), [Fix update weight error for blackwell DeepGEMM](../sources/prs/sglang/PR-13910.md), [Fix flashinfer cutlass MoE output shape for non-FP4-packed inputs](../sources/prs/sglang/PR-14028.md), [Add Mistral Large 3 support.](../sources/prs/sglang/PR-14213.md), [[NPU] perf update with kvcache nz & w4a8 quant](../sources/prs/sglang/PR-14423.md), [Add Mistral Large 3 Eagle Support](../sources/prs/sglang/PR-14466.md), [Mistral Large 3 NVFP4 support](../sources/prs/sglang/PR-14485.md), [Mistral Large 3 NVFP4 TRTLLM MoE support](../sources/prs/sglang/PR-15049.md), [[NVIDIA] Fixes for NVFP4 all-gather with spec decoding](../sources/prs/sglang/PR-15280.md), [Fix the accuracy issue when running mxfp4 dsv3 model and enable ep](../sources/prs/sglang/PR-15304.md), [feat: support bitsandbytes quantization algorithm](../sources/prs/sglang/PR-15325.md), [[NPU]DeepSeek-V3.2 support npu mlaprolog](../sources/prs/sglang/PR-15381.md), [Flashinfer MOE FP8 support for Mistral Large 3.](../sources/prs/sglang/PR-15422.md), [[Perf] Add Flashinfer DeepGEMM SM90 for SwapAB Optimization](../sources/prs/sglang/PR-15514.md), [Update flashinfer to 0.6.1](../sources/prs/sglang/PR-15551.md), [[Perf] Eliminate the slice op for Flashinfer `trtllm_fp4_block_scale_moe`](../sources/prs/sglang/PR-15731.md), [[JIT kernel] Apply jit per_tensor_quant_fp8 kernel](../sources/prs/sglang/PR-15836.md), [[NPU] NZ for non-quantized MOE, Qwen3 MOE double memory consumption fix](../sources/prs/sglang/PR-15904.md), [[Performance] Force split_k=1 for MXFP4 Triton kernels on Hopper](../sources/prs/sglang/PR-16014.md), [Fix FP8 MoE NaN with DeepGEMM on Blackwell](../sources/prs/sglang/PR-16622.md), [Support mxint4 flashinfer_trtllm moe gemm](../sources/prs/sglang/PR-16892.md), [[FIX] Always support TP > 4 for FP4 Gemm](../sources/prs/sglang/PR-17300.md), [Add mxfp8 support for online quantization, Triton dense linear, and CUTLASS MoE](../sources/prs/sglang/PR-17449.md), [[feat] Support nvfp4 quantized model of Qwen3-Next](../sources/prs/sglang/PR-17627.md), [fix(quantization): add sgl_kernel fallback for FP4 quantize on Blackwell GPUs](../sources/prs/sglang/PR-17816.md), [[Bugfix] Fix Mistral Large 3 NVFP4 TRTLLM MoE](../sources/prs/sglang/PR-18065.md), [Fix nvfp4 weight update](../sources/prs/sglang/PR-18085.md), [[ROCm] Optimize Deepseek R1 on MI300X](../sources/prs/sglang/PR-18242.md), [[MUSA][10/N] Add GGUF support](../sources/prs/sglang/PR-18357.md), [[Kimi-K2.5] Fix NVFP4 Kimi-K2.5 weight mapping and exclude list](../sources/prs/sglang/PR-18370.md), [docs: expand and update modelopt documentation](../sources/prs/sglang/PR-18479.md), [[sglang-miles] True on-policy training support for FSDP2](../sources/prs/sglang/PR-18639.md), [[RL] Support per-layer mixed FP8/BF16 serving for FP8 checkpoints](../sources/prs/sglang/PR-18742.md), [[Perf] ~9.5x faster Blackwell MXFP4 MoE weight loading](../sources/prs/sglang/PR-18858.md), [feat: Support MXFP4 quantized dense models on AMD CDNA2/CDNA3 GPUs](../sources/prs/sglang/PR-19143.md), [Adjust padding size to improve triton_kernels moe performance](../sources/prs/sglang/PR-19174.md), [[AMD] Fix weight load shape mismatch for amd dsr1 0528 mxfp4](../sources/prs/sglang/PR-19425.md), [Fix/nemotron mtp quantaized](../sources/prs/sglang/PR-19433.md), [[Kernel Slimming] Migrate NVFP4 kernels to JIT](../sources/prs/sglang/PR-19437.md), [[FlashInfer v0.6.4] [RL] Integrate FlashInfer mxfp8 gemm, MoE, and routed MoE](../sources/prs/sglang/PR-19537.md), [[Feature] NVFP4 Marlin fallback for non-Blackwell GPUs (SM75+)](../sources/prs/sglang/PR-19652.md), [Support `triton_kernels` for GPT-OSS on SM120](../sources/prs/sglang/PR-19718.md), [Various SM120 improvements](../sources/prs/sglang/PR-19721.md), [[JIT Kernel] Reland NVFP4 kernels to JIT](../sources/prs/sglang/PR-20012.md), [Fix SM120 `triton_kernels` MXFP4 `block_k` for GPT-OSS](../sources/prs/sglang/PR-20040.md), [Enable modelopt quantized FLUX deployment](../sources/prs/sglang/PR-20082.md), [[diffusion] Support nvfp4 for Flux.2](../sources/prs/sglang/PR-20137.md), [[4/n jit_kernel restruct] speed up CI tests and add benchmark workflow](../sources/prs/sglang/PR-20268.md), [[Benchmark] use flashinfer bench_gpu_time instead of triton do_bench](../sources/prs/sglang/PR-20305.md), [[NVIDIA] Enable fp8 flashinfer_trtllm_routed MoE for MiniMax-M2.5](../sources/prs/sglang/PR-20394.md), [[Model] Support Nemotron 3 Super NVFP4](../sources/prs/sglang/PR-20407.md), [fix(docs): correct quantization documentation (#20301)](../sources/prs/sglang/PR-20619.md), [Add SGLang CUDA crash API logging inspired by FlashInfer](../sources/prs/sglang/PR-20910.md), [[refactor] Clean up duplicate flashinfer trtllm moe code](../sources/prs/sglang/PR-21233.md), [Refactor JIT kernel CI to use run_suite.py registration system](../sources/prs/sglang/PR-21239.md), [[NVIDIA] Enable FP4 flashinfer trtllm routed moe](../sources/prs/sglang/PR-21240.md), [[RL] Support mxfp8 DeepSeek V3](../sources/prs/sglang/PR-21280.md), [CUTLASS NVFP4 GEMM improvement of SM120](../sources/prs/sglang/PR-21314.md), [[Kernel] Support FlashInfer TRTLLM-Gen fused MoE for non-gated FP4 & FP8 (Nemotron)](../sources/prs/sglang/PR-21321.md), [[misc] clean up kernel API](../sources/prs/sglang/PR-21325.md), [Add dedicated FlashInferCuteDslMoE layer for standard-path FP4 MoE](../sources/prs/sglang/PR-21339.md), [[FlashInver v0.6.7] Integrate flashinfer_trtllm mxfp8 gemm](../sources/prs/sglang/PR-21576.md), [[server] Add --quantization unquant to explicitly opt out of quantization](../sources/prs/sglang/PR-21863.md), [fix pcg torch dynamo recompile in mxfp8 Triton path](../sources/prs/sglang/PR-21888.md), [[Diffusion] Fix weight scale swizzle and add large-M kernel config for FLUX.2-dev-NVFP4](../sources/prs/sglang/PR-22064.md), [[RL] Refactor NVFP4 shuffling/swizzling to in-place replacement](../sources/prs/sglang/PR-22204.md), [[AMD][HIP] NSA: bf16 passthrough from RMSNorm to eliminate FP8 dequantization](../sources/prs/sglang/PR-22258.md), [[Reland] DeepSeek-R1-0528-w4a8: DeepEP Low Latency Dispatch Adopts FP8 Communication](../sources/prs/sglang/PR-22316.md), [[Lora] Lora quat info re-factor and support deepseekv3 mla lora](../sources/prs/sglang/PR-22323.md), [:sparkles: [diffusion][npu][quant] Add MXFP4 quantization support for Wan2.2 Diffusion on Ascend NPU](../sources/prs/sglang/PR-22338.md), [[Diffusion] modelopt diffusion fp8 support for flux1/flux2 and wan2.2](../sources/prs/sglang/PR-22365.md), [[Lora] Lora kimi support](../sources/prs/sglang/PR-22381.md), [[RL] Fix weight update for mxfp8 flashinfer_cutlass gemm backend](../sources/prs/sglang/PR-22484.md), [[Diffusion] Add FLUX.1-dev ModelOpt NVFP4 support](../sources/prs/sglang/PR-22574.md), [diffusion: fix layerwise offload for ModelOpt quantized DiTs](../sources/prs/sglang/PR-22594.md), [feat: Support flashinfer_cutedsl MoE runner with flashinfer alltoall backend](../sources/prs/sglang/PR-22669.md), [reland [Diffusion] Add FLUX.1-dev ModelOpt NVFP4 support](../sources/prs/sglang/PR-22672.md), [[Diffusion] Add Wan2.2 ModelOpt NVFP4 support](../sources/prs/sglang/PR-22681.md), [[Refactor] Refactor DeepEP dispatcher](../sources/prs/sglang/PR-22822.md), [[FlashInfer v0.6.11] [RL] Support FlashInfer per-token NVFP4 MoE](../sources/prs/sglang/PR-22918.md), [Reland Cute-DSL FP4 dense GEMM](../sources/prs/sglang/PR-23590.md), [Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-23686.md), [Use Cute-DSL NVFP4 quantization kernels](../sources/prs/sglang/PR-23745.md), [[MoE] Unify DeepEPMoE+MoriEPMoE through AITER MoeRunner pre/post-permute](../sources/prs/sglang/PR-23760.md), [Refactor device timer, clean up metrics collector, and add fwd occupancy metric](../sources/prs/sglang/PR-24197.md), [Port MXFP4 Marlin MoE support to JIT kernel path](../sources/prs/sglang/PR-24490.md), [Add FlashInfer SM90 cutlass MXFP4 MoE backend (W4A16) for GPT-OSS + DeepSeek-V4](../sources/prs/sglang/PR-24816.md), [Amd/deepseek v4 rebase main 0509](../sources/prs/sglang/PR-24933.md), [[rebase]Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-24986.md), [perf(nvfp4): free unused source scales after weight processing](../sources/prs/sglang/PR-25107.md), [[Gemma4]: Fix FP8 Triton scale layout](../sources/prs/sglang/PR-25286.md), [[Intel GPU] Enable DeepSeek V4 Inference on XPU](../sources/prs/sglang/PR-25336.md), [[AMD] Enable shared-experts fusion with new KIMI-K2.5-MXFP4 model.](../sources/prs/sglang/PR-25390.md), [[codex] Update Wan2.2 ModelOpt CI checkpoints](../sources/prs/sglang/PR-25483.md), [[MoE Refactor] Migrate flashinfer_cutedsl + DeepEP to MoeRunner](../sources/prs/sglang/PR-25525.md), [Use DeepGEMM BF16 for unquantized DeepEP LL MoE](../sources/prs/sglang/PR-25540.md), [Add no_combine support to cutlass_moe_fp4](../sources/prs/sglang/PR-25688.md), [Apply sgl w8a8 fp8 kernel](../sources/prs/sglang/PR-3148.md), [add tensorrt_llm common and cutlass_extensions as 3rdparty](../sources/prs/sglang/PR-3216.md), [integrate blockwise fp8 kernel](../sources/prs/sglang/PR-3529.md), [add control for cutlass fp8 blockwise gemm](../sources/prs/sglang/PR-3727.md), [Feature DeepSeek V3/R1 INT8 Quantization (block-wise)](../sources/prs/sglang/PR-3730.md), [[Feature] DeepSeek V3/R1 INT8 Quantization (channel-wise) ](../sources/prs/sglang/PR-3888.md), [Support FP4 gemm (1/2)](../sources/prs/sglang/PR-3899.md), [Support overlapping two batches](../sources/prs/sglang/PR-4068.md), [linear support deepgemm](../sources/prs/sglang/PR-4199.md), [Clean up fp8 support](../sources/prs/sglang/PR-4230.md), [[FIX] fix incorrect output when enable both deepgemm and torch compile](../sources/prs/sglang/PR-4359.md), [[ROCm] fix dtype](../sources/prs/sglang/PR-4510.md), [Create col-major and tma-aligned x_scale for deep_gemm.gemm_fp8_fp8_bf16_nt](../sources/prs/sglang/PR-4515.md), [[quantization] fix channelwise conversion with scalar weight scale](../sources/prs/sglang/PR-4596.md), [Set deepgemm to the default value in the hopper architecture.](../sources/prs/sglang/PR-4613.md), [Fix loading KV quantization scale; Enable modelopt kv cache](../sources/prs/sglang/PR-4686.md), [Add DeepSeek V3/R1 shared experts fusion](../sources/prs/sglang/PR-4918.md), [feat: add DeepGEMM build warning](../sources/prs/sglang/PR-5176.md), [[Fix] Turn off DeepGEMM by default](../sources/prs/sglang/PR-5263.md), [fix: use deepgemm only on hopper](../sources/prs/sglang/PR-5310.md), [[perf] experimental enhance fp8 per-tensor quant](../sources/prs/sglang/PR-5370.md), [[feature] enable pre compile jit deep_gemm](../sources/prs/sglang/PR-5580.md), [ DeepEP normal support deepgemm-contiguous](../sources/prs/sglang/PR-5626.md), [Turn on DeepGemm By Default and Update Doc](../sources/prs/sglang/PR-5628.md), [[perf] dsv3 bmm fallback to bf16](../sources/prs/sglang/PR-5662.md), [[2/2] Add python wrapper for CUTLASS FP8 Blockscale MoE Kernel. ](../sources/prs/sglang/PR-5694.md), [enable auto-round quantization model](../sources/prs/sglang/PR-6226.md), [Fix bug of deepseek-v3 under DP+EP mode with large batchsize/seqlen](../sources/prs/sglang/PR-6449.md), [[Feature] Support Flashinfer fp8 blockwise GEMM kernel on Blackwell](../sources/prs/sglang/PR-6479.md), [Set `num_fused_shared_experts` as `num_shared_experts` when shared_experts fusion is not disabled](../sources/prs/sglang/PR-6736.md), [[CPU] add optimizations for INT8 and FP8 DeepSeek](../sources/prs/sglang/PR-6769.md), [Support token-level quantization for EP MoE](../sources/prs/sglang/PR-6782.md), [[DeepseekR1-FP4] Add Support for nvidia/DeepSeekR1-FP4 model](../sources/prs/sglang/PR-6853.md), [Use deepgemm instead of triton for fused_qkv_a_proj_with_mqa](../sources/prs/sglang/PR-6890.md), [[Feature] Support Flashinfer fmha on Blackwell](../sources/prs/sglang/PR-6930.md), [chore: upgrade flashinfer v0.2.6.post1 jit](../sources/prs/sglang/PR-6958.md), [Fuse routed scaling factor in deepseek](../sources/prs/sglang/PR-6970.md), [Clean up server_args.py](../sources/prs/sglang/PR-7037.md), [Enable ModelOpt Llama4 fp8 checkpoint deployment in SGLang](../sources/prs/sglang/PR-7129.md), [Enable native ModelOpt quantization support (1/3) ](../sources/prs/sglang/PR-7149.md), [[amd] Opt dsv3 moe](../sources/prs/sglang/PR-7160.md), [Support new DeepGEMM](../sources/prs/sglang/PR-7172.md), [Tiny let DeepGEMM scale checks cover more cases](../sources/prs/sglang/PR-7182.md), [chore: upgrade sgl-kernel v0.1.8.post2](../sources/prs/sglang/PR-7186.md), [[AMD] Fail gracefully when AITER is unavailable gfx90a GPUs](../sources/prs/sglang/PR-7187.md), [Fix a minor bug related to DeepGEMM upgrade](../sources/prs/sglang/PR-7191.md), [[fix] fix DeepGEMM blackwell input quant & ut & fix style and log](../sources/prs/sglang/PR-7247.md), [Support NVFP4 quantized dense models on AMD CDNA2/CDNA3 GPUs](../sources/prs/sglang/PR-7302.md), [FlashInfer NVFP4 MoE with EP & 2-stream shared expert](../sources/prs/sglang/PR-7327.md), [Fix torch compile run](../sources/prs/sglang/PR-7391.md), [[AMD][Quantization] Add `int4fp8_moe` online quantization on ROCm](../sources/prs/sglang/PR-7392.md), [Add fp4 quantize before all-gather for Flashinfer cutlass MoE DP (max throughput)](../sources/prs/sglang/PR-7667.md), [feat: support DeepSeek-R1-W4AFP8 model with ep-moe mode](../sources/prs/sglang/PR-7762.md), [Qwen FP8/NVFP4 ModelOPT Quantization support](../sources/prs/sglang/PR-7912.md), [[feat] Support tp mode for DeepSeek-R1-W4AFP8](../sources/prs/sglang/PR-8118.md), [[fix] fix modelopt fp4 on b200](../sources/prs/sglang/PR-8195.md), [[1/N]Support DeepSeek-R1 w4a8 normal deepep](../sources/prs/sglang/PR-8247.md), [Support triton kernels v3.4.0 for fused_moe](../sources/prs/sglang/PR-8258.md), [[2/N]Support DeepSeek-R1 w4a8 low latency deepep](../sources/prs/sglang/PR-8464.md), [[NVIDIA] Add Low Latency NVFP4 decode kernels from Flashinfer](../sources/prs/sglang/PR-8552.md), [feat: support cutlass_moe_fp8 kernel for fusedmoe in sm90](../sources/prs/sglang/PR-8678.md), [Fix mismatch between padded_scales shape and reshape dimensions in modelopt quantization](../sources/prs/sglang/PR-8766.md), [[Perf] Auto enable best flashinfer mxfp4 kernel in b200](../sources/prs/sglang/PR-8898.md), [Fix hopper launch gpt-oss model illegal memory](../sources/prs/sglang/PR-8908.md), [optimize: reduce shulffle and quantization overhead in cutlass_moe sm90](../sources/prs/sglang/PR-8962.md), [Faster weight processing (trtllm-gen moe nvfp4)](../sources/prs/sglang/PR-9162.md), [[NVIDIA] [3/N] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked ](../sources/prs/sglang/PR-9199.md), [[NVIDA] [1/N] Nvfp4 Masked Gemm: Add quant op for the flashinfer grouped gemm](../sources/prs/sglang/PR-9200.md), [[fix] Fix mxfp4 triton MoE tp bug](../sources/prs/sglang/PR-9473.md), [fix: blackwell dsv3 fp8 issue temporary solution](../sources/prs/sglang/PR-9530.md), [[NVIDIA] [2/N] Optimize `silu_and_mul_scaled_fp4_grouped_quant` perf](../sources/prs/sglang/PR-9556.md), [Tiny fix wrong comments](../sources/prs/sglang/PR-9589.md), [move is_sm90_supported/is_sm100_supported to python/sglang/srt/utils.py](../sources/prs/sglang/PR-9679.md), [[ModelOpt] Fix Weight Loading for DSR1-FP4 Quantization](../sources/prs/sglang/PR-9712.md), [Make fp4_quantize kernels work on sm103](../sources/prs/sglang/PR-9807.md), [[Model] Support Meituan LongCat-Flash && LongCat-Flash-MTP](../sources/prs/sglang/PR-9824.md), [[Fix] DeepSeek EP accuracy issue on B200 GPUs](../sources/prs/sglang/PR-9946.md), [Enable native ModelOpt quantization support (2/3)](../sources/prs/sglang/PR-9991.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [[TIR][IR] Update to use tirx](../sources/prs/tilelang/PR-2216.md), [[Kernel]: Cutlass 2:4 Sparsity + FP8/Int8 Quant Support](../sources/prs/vllm/PR-10995.md), [[Kernel] Update `cutlass_scaled_mm` to support 2d group (blockwise) scaling](../sources/prs/vllm/PR-11868.md), [Add: Support for Sparse24Bitmask Compressed Models](../sources/prs/vllm/PR-12097.md), [[Kernel] add triton fused moe kernel for gptq/awq](../sources/prs/vllm/PR-12185.md), [[AMD][Quantization] Add TritonScaledMMLinearKernel since int8 is broken for AMD](../sources/prs/vllm/PR-12282.md), [[Bugfix] Disable w16a16 2of4 sparse CompressedTensors24](../sources/prs/vllm/PR-12417.md), [Expert Parallelism (EP) Support for DeepSeek Models](../sources/prs/vllm/PR-12583.md), [[Kernel][Quantization] Integrate block-quantized CUTLASS kernels for DeepSeekV3](../sources/prs/vllm/PR-12587.md), [[Attention] Deepseek v3 MLA support with FP8 compute](../sources/prs/vllm/PR-12601.md), [[Attention] MLA with chunked prefill](../sources/prs/vllm/PR-12639.md), [[AMD][ROCm] Enable DeepSeek model on ROCm](../sources/prs/vllm/PR-12662.md), [[Bugfix][Kernel] Fix per-token/per-channel quantization for Hopper scaled mm](../sources/prs/vllm/PR-12696.md), [[Misc] Update w2 scale loading for GPTQMarlinMoE](../sources/prs/vllm/PR-12757.md), [[NVIDIA] Support nvfp4 quantization](../sources/prs/vllm/PR-12784.md), [[Bugfix] Better FP8 supported defaults](../sources/prs/vllm/PR-12796.md), [[Misc][Kernel]: Add GPTQAllSpark Quantization](../sources/prs/vllm/PR-12931.md), [[Kernel]Add streamK for block-quantized CUTLASS kernels](../sources/prs/vllm/PR-12978.md), [[Model] Deepseek GGUF support ](../sources/prs/vllm/PR-13167.md), [Expand MLA to support most types of quantization](../sources/prs/vllm/PR-13181.md), [[Quant][Perf] Use moe_wna16 kernel by default for MoEs with many experts](../sources/prs/vllm/PR-13236.md), [[NVIDIA] Support nvfp4 tcgen05 gemm](../sources/prs/vllm/PR-13571.md), [[V1] V1 Enablement Oracle ](../sources/prs/vllm/PR-13726.md), [Fix CompressedTensorsWNA16MoE with grouped scales](../sources/prs/vllm/PR-13769.md), [[Bugfix][Quantization] Fix FP8 + EP](../sources/prs/vllm/PR-13784.md), [[Kernel] CUTLASS grouped gemm fp8 MoE kernel](../sources/prs/vllm/PR-13972.md), [[Kernel] optimize performance of gptq marlin kernel when n is small](../sources/prs/vllm/PR-14138.md), [dynamic distpatch of fp8 kernels](../sources/prs/vllm/PR-14245.md), [[Build/BugFix] Fix hopper 12.8 build](../sources/prs/vllm/PR-14354.md), [Add cutlass support for blackwell fp8 blockwise gemm](../sources/prs/vllm/PR-14383.md), [[Kernel] moe wna16 marlin kernel](../sources/prs/vllm/PR-14447.md), [[ROCm][Kernel] MoE weights padding](../sources/prs/vllm/PR-14454.md), [permute/unpermute kernel for moe optimization](../sources/prs/vllm/PR-14568.md), [[Attention] Flash Attention 3 - fp8](../sources/prs/vllm/PR-14570.md), [[Quantization][FP8] Adding support for fp8 gemm layer input in fp8](../sources/prs/vllm/PR-14578.md), [[Kernel] GGUF MoE kernel](../sources/prs/vllm/PR-14613.md), [[Kernel] allow non-contiguous input for marlin kernel](../sources/prs/vllm/PR-14658.md), [[Attention] MLA get rid of materialization](../sources/prs/vllm/PR-14770.md), [[FEAT][ROCm] Integrate Fused MoE Kernels from AITER](../sources/prs/vllm/PR-14967.md), [[FEAT] [ROCm]: Add AITER Block-Scaled GEMM Feature](../sources/prs/vllm/PR-14968.md), [Fix non-contiguous input passed to Marlin kernel](../sources/prs/vllm/PR-15319.md), [[V1] Fully Transparent Implementation of CPU Offloading](../sources/prs/vllm/PR-15354.md), [[FEAT] [ROCm] Add AITER int8 scaled gemm kernel](../sources/prs/vllm/PR-15433.md), [[Kernel] Fix conflicting macro names for gguf kernels](../sources/prs/vllm/PR-15456.md), [[Quantization] Fp8 Channelwise Dynamic Per Token GroupedGEMM](../sources/prs/vllm/PR-15587.md), [[Bugfix] fix use_atomic_add support of marlin kernel when using v1 engine](../sources/prs/vllm/PR-15946.md), [Modularize fused experts and integrate PPLX kernels](../sources/prs/vllm/PR-15956.md), [Add support to modelopt quantization of Mixtral model](../sources/prs/vllm/PR-15961.md), [[Kernel] Use moe_wna16 kernel for compressed tensors wna16 moe models](../sources/prs/vllm/PR-16038.md), [Upstream Llama4 Support to Main](../sources/prs/vllm/PR-16113.md), [[Hardware/NVIDIA/Kernel] [Functional Enablement] [1/N] Enable nvidia/DeepSeek-R1-FP4 Model](../sources/prs/vllm/PR-16362.md), [[Kernel] Support W8A8 channel-wise weights and per-token activations in triton fused_moe_kernel](../sources/prs/vllm/PR-16366.md), [Enable PTPC FP8 for CompressedTensorsW8A8Fp8MoEMethod (triton fused_moe)](../sources/prs/vllm/PR-16537.md), [[ROCm] Add aiter tkw1 kernel for Llama4 fp8](../sources/prs/vllm/PR-16727.md), [Support W8A8 INT8 MoE for compressed-tensors](../sources/prs/vllm/PR-16745.md), [[FEAT] [ROCm]: AITER Fused MOE V1 Support](../sources/prs/vllm/PR-16752.md), [[torch.compile][ROCm] Fuse quantization onto attention using a torch.compile pass](../sources/prs/vllm/PR-16756.md), [[Kernel] GGUF MoeVec kernel](../sources/prs/vllm/PR-16780.md), [[Kernel] some optimizations for dense marlin and moe marlin](../sources/prs/vllm/PR-16850.md), [[Bugfix] Fix moe weight losing all extra attrs after `process_weights_after_loading`.](../sources/prs/vllm/PR-16854.md), [[Kernel] Add expert_map support to Cutlass FP8 MOE](../sources/prs/vllm/PR-16861.md), [[FEAT] [ROCm]: Add AITER CK 2 Stages MoE support](../sources/prs/vllm/PR-17110.md), [[ROCm][FP8][Kernel] FP8 quantization fused into Custom Paged Attention](../sources/prs/vllm/PR-17139.md), [[NVIDIA] Support Cutlass w8a8 FP8 for Blackwell Geforce GPUs (sm120)](../sources/prs/vllm/PR-17280.md), [Fix noisy warning for uncalibrated q_scale/p_scale](../sources/prs/vllm/PR-17414.md), [[Kernel] fp4 marlin kernel](../sources/prs/vllm/PR-17687.md), [[Misc] Add compressed-tensors NVFP4A16 emulation support](../sources/prs/vllm/PR-17914.md), [use ceil_div in cutlass block scaling shape check](../sources/prs/vllm/PR-17918.md), [Use NVFP4 Marlin for CompressedTensorsW4A16Fp4](../sources/prs/vllm/PR-18000.md), [Fix Broken macro for cutlass moe](../sources/prs/vllm/PR-18049.md), [[Quantization] Add compressed-tensors NVFP4 support](../sources/prs/vllm/PR-18312.md), [[Feature] Expert Parallelism Load Balancer (EPLB)](../sources/prs/vllm/PR-18343.md), [[V1] Support `LLM.apply_model`](../sources/prs/vllm/PR-18465.md), [Sm100 blockwise fp8 swap ab](../sources/prs/vllm/PR-18564.md), [[Kernel] Integrate CUTLASS MoE kernel with PPLX](../sources/prs/vllm/PR-18762.md), [[Perf] Tunings for SM100 FP8 CUTLASS kernel](../sources/prs/vllm/PR-18778.md), [[Kernel] Enable fp8 support for pplx and BatchedTritonExperts.](../sources/prs/vllm/PR-18864.md), [[Kernel] Support deep_gemm for linear methods](../sources/prs/vllm/PR-19085.md), [[Hardware][NVIDIA] FP4 MoE kernel optimization](../sources/prs/vllm/PR-19110.md), [[Kernels] Add activation chunking logic to FusedMoEModularKernel](../sources/prs/vllm/PR-19168.md), [[Hardware][NVIDIA][kernel] Fp4 MOE quant kernel optimization](../sources/prs/vllm/PR-19500.md), [[feat]: CUTLASS block scaled group gemm for SM100](../sources/prs/vllm/PR-19757.md), [[Feature] Integrate new deepgemm](../sources/prs/vllm/PR-19820.md), [[Quantization] Add compressed-tensors emulations support for NVFP4](../sources/prs/vllm/PR-19879.md), [[Quantization] Add compressed-tensors NVFP4 MoE Support](../sources/prs/vllm/PR-19990.md), [[Bugfix] Build moe_data for both sm100 and sm90](../sources/prs/vllm/PR-20086.md), [ [Feature] Integrate SM100 DeepGEMM support](../sources/prs/vllm/PR-20087.md), [[Bugfix] Fix some narrowing conversion warnings](../sources/prs/vllm/PR-20141.md), [[Bugfix] Fix topk_ids indices_type for CUTLASS w8a8 FP8 MoE](../sources/prs/vllm/PR-20166.md), [[V1] [ROCm] Enable EP with AITER Fused MoE](../sources/prs/vllm/PR-20270.md), [[Kernel][Bugfix] Fixup some warnings in nvfp4_blockwise_moe when CUDA < 12.8](../sources/prs/vllm/PR-20324.md), [[Kernel] SM90 CUTLASS FP8 GEMM: add support for swap AB + kernel tuning](../sources/prs/vllm/PR-20396.md), [[feat]: add SM100 support for cutlass FP8 groupGEMM](../sources/prs/vllm/PR-20447.md), [Support Llama 4 for cutlass_moe_fp4](../sources/prs/vllm/PR-20453.md), [Support Llama 4 for fused_marlin_moe](../sources/prs/vllm/PR-20457.md), [[Perf] Reuse workspace for FP8+FP4 Marlin MoE](../sources/prs/vllm/PR-20500.md), [[Performance] Performance improvements in non-blockwise fp8 CUTLASS MoE](../sources/prs/vllm/PR-20762.md), [[fix]: disable cutlass block scaled group gemm for EP](../sources/prs/vllm/PR-20781.md), [[Bugfix] Fix a couple PPLX+CUTLASS MoE bugs](../sources/prs/vllm/PR-20825.md), [[Perf] Use Triton instead of Torch for DeepGEMM Per Token Group Quant](../sources/prs/vllm/PR-20841.md), [[Perf] Add swap_ab to SM90 FP8 non-block CUTLASS moe grouped gemm](../sources/prs/vllm/PR-20911.md), [Support mnnvl all2allv from Flashinfer](../sources/prs/vllm/PR-21003.md), [[Perf] Cuda Kernel for Per Token Group Quant](../sources/prs/vllm/PR-21083.md), [[perf] Add fused MLA QKV + strided layernorm](../sources/prs/vllm/PR-21116.md), [[Feature][OCP MX] Support mxfp6 and mixed mxfp6-mxfp4](../sources/prs/vllm/PR-21166.md), [Support CUTLASS NVFP4 (w4a4) for Blackwell Geforce GPUs (SM120)](../sources/prs/vllm/PR-21309.md), [Support Tensorrt-LLM MoE fp4 for low-latency](../sources/prs/vllm/PR-21331.md), [[Quantization] Enable BNB support for more MoE models](../sources/prs/vllm/PR-21370.md), [Update flashinfer CUTLASS NVFP4 MoE Kernel to use per expert global scaling factor](../sources/prs/vllm/PR-21408.md), [[Bug] Fix Compressed Tensor NVFP4 `cutlass_fp4_group_mm` illegal memory access](../sources/prs/vllm/PR-21465.md), [[NVIDIA] Fix Llama4 Scout FP4 functionality issues](../sources/prs/vllm/PR-21499.md), [[Kernel] Improve machete memory bound perf](../sources/prs/vllm/PR-21556.md), [[Feature] Add Flashinfer MoE Support for Compressed Tensor NVFP4](../sources/prs/vllm/PR-21639.md), [feat: Add Support GPTQ Quantization MOE on ROCM vllm serve](../sources/prs/vllm/PR-21733.md), [[Kernel] Add support for block FP8 on SM120 (NVIDIA 5090 and RTX PRO 6000)](../sources/prs/vllm/PR-22131.md), [[Bugfix] Fix MoE BNB version](../sources/prs/vllm/PR-22260.md), [[Bugfix] Add proper comparison for package versions](../sources/prs/vllm/PR-22314.md), [[gpt-oss] flashinfer mxfp4](../sources/prs/vllm/PR-22339.md), [[Bug] Fix B200 DeepGEMM E8M0 Accuracy Issue](../sources/prs/vllm/PR-22399.md), [[gpt-oss] triton kernel mxfp4](../sources/prs/vllm/PR-22421.md), [[bugfix] Fix Llama3/4 issues caused by FlashInfer 0.2.10](../sources/prs/vllm/PR-22426.md), [[Quantization]: Support compressed-tensors mixed-precision model loading](../sources/prs/vllm/PR-22468.md), [Fix Llama4 FlashInfer FP4 MoE issues](../sources/prs/vllm/PR-22511.md), [Quantization: support FP4 quantized models on AMD CDNA2/CDNA3 GPUs](../sources/prs/vllm/PR-22527.md), [[Quantization] Expand compressed-tensors MoE matching logic to support NFP4 + FP8 MoEs](../sources/prs/vllm/PR-22674.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv NVFP4-out Attention Kernel](../sources/prs/vllm/PR-22703.md), [fp8 kv cache support fix for torch.compile](../sources/prs/vllm/PR-22758.md), [[FIXBUG] Add return_success parameter to moe_wna16_weight_loader function](../sources/prs/vllm/PR-22797.md), [[Kernel] Added flashinfer fp8 per-tensor gemms](../sources/prs/vllm/PR-22895.md), [[Fix] enable swap_ab for pplx problem size computation](../sources/prs/vllm/PR-22991.md), [Use Blackwell FlashInfer MXFP4 MoE by default if available ](../sources/prs/vllm/PR-23008.md), [[Bugfix] fix qwen3 moe fp8 accuracy issue](../sources/prs/vllm/PR-23031.md), [[Core] Support weight_loader_v2 for `UnquantizedLinearMethod`](../sources/prs/vllm/PR-23036.md), [[Kernel] CUTLASS MoE FP8: Integrate cuda moe permute/unpermute](../sources/prs/vllm/PR-23045.md), [Add routed_scaling_factor to MoE grouped topk](../sources/prs/vllm/PR-23123.md), [[Bugfix] Fix accuracy issue when using flashinfer cutlass moe, TP=1 and modelopt.](../sources/prs/vllm/PR-23125.md), [Fix nvfp4 swizzling](../sources/prs/vllm/PR-23140.md), [[XPU][Feature] fp8 online quantization support for XPU](../sources/prs/vllm/PR-23148.md), [[kernel] Support W4A8 on Hopper](../sources/prs/vllm/PR-23198.md), [[Kernels] Overlap shared experts with send/recv](../sources/prs/vllm/PR-23273.md), [[Perf] Use upstream CUTLASS for SM90 Block FP8 kernel](../sources/prs/vllm/PR-23280.md), [[Bug] Fix R1 Accuracy 0 Bug](../sources/prs/vllm/PR-23294.md), [Update Flashinfer to 0.2.14.post1](../sources/prs/vllm/PR-23537.md), [DP/EP Support for gpt-oss with deepep-ht comm kernel on SM100](../sources/prs/vllm/PR-23608.md), [[Bugfix] Fix Marlin NVFP4 for modelopt](../sources/prs/vllm/PR-23659.md), [[Compile] Fix Compile Warning for `w4a8_mm_entry.cu`](../sources/prs/vllm/PR-23660.md), [[Feature] Add Hopper DeepGEMM E8M0 for DeepSeekV3.1 scale_fmt](../sources/prs/vllm/PR-23666.md), [[NVIDIA] Support SiluMul + NVFP4 quant fusion](../sources/prs/vllm/PR-23671.md), [[Bugfix][Misc] Fix silu_and_mul_nvfp4_quant issue and extract common utils for nvfp4 kernel source files](../sources/prs/vllm/PR-23727.md), [[fix]: add Arm 4bit fused moe support](../sources/prs/vllm/PR-23809.md), [[Model][gpt-oss] Support DP+EP for GPT-OSS with FlashInfer trtllm-gen MoE](../sources/prs/vllm/PR-23819.md), [[Kernel] Faster pre-processing time for W4A8](../sources/prs/vllm/PR-23972.md), [[Model] Add LongCat-Flash ](../sources/prs/vllm/PR-23991.md), [[BUGFIX] GPTQ quantization compatibility for Qwen3 MOE models (AutoGPTQ and AutoRound-GPTQ)](../sources/prs/vllm/PR-23994.md), [[Transform] [Quantization] Add QuTLASS support to vLLM](../sources/prs/vllm/PR-24440.md), [[Performance] Move apply_w8a8_block_fp8_linear to an op class](../sources/prs/vllm/PR-24666.md), [[NVIDIA] Blackwell Family](../sources/prs/vllm/PR-24673.md), [[Kernel][Quantization] add w4a8 support for marlin kernel](../sources/prs/vllm/PR-24722.md), [[Bugfix] Fix accuracy issue for silu_mul + nvfp4 quant fusion kernel](../sources/prs/vllm/PR-24833.md), [[Compile] Fix Compile Warning for Ignoring `MIN_BLOCK_PER_SM`](../sources/prs/vllm/PR-25193.md), [[ROCm] Small functional changes for gptoss](../sources/prs/vllm/PR-25201.md), [Enable Fbgemm NVFP4 on Dense models](../sources/prs/vllm/PR-25609.md), [Fix INT8 quantization error on Blackwell GPUs (SM100+)](../sources/prs/vllm/PR-25935.md), [[Bugfix] Enable padded FP4 quantization](../sources/prs/vllm/PR-25947.md), [[Quantization/NVFP4] Speed up TRTLLM NVFP4 MOE weight loading and fix K/V scale loading for MLA Attn](../sources/prs/vllm/PR-25968.md), [[Bugfix] Allow skipping MoE in NVFP4 (fix for MTP)](../sources/prs/vllm/PR-25987.md), [[MoE] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked](../sources/prs/vllm/PR-25990.md), [Fix undefined symbol: cutlass_moe_mm_sm100](../sources/prs/vllm/PR-26098.md), [[NVIDIA] Add support for cudnn fp4 gemm via flashinfer](../sources/prs/vllm/PR-26107.md), [[ModelOpt] Load w13/w2_input_scale for all experts, nvfp4](../sources/prs/vllm/PR-26135.md), [Move query quantization to attention layer for Flashinfer & Triton.](../sources/prs/vllm/PR-26534.md), [[Bugfix] Convert untraceable GroupShape to list for AMD impl](../sources/prs/vllm/PR-26535.md), [[ROCM] MoE fp4 CK kernel](../sources/prs/vllm/PR-26545.md), [support flashinfer_fp4 moe for 5090 gpu](../sources/prs/vllm/PR-26669.md), [[NVIDIA] [Perf] Update to leverage flashinfer trtllm FP4 MOE throughput kernel](../sources/prs/vllm/PR-26714.md), [[Bugfix] Fix gpt-oss w4a8 DP/EP on B200](../sources/prs/vllm/PR-26729.md), [[Feature] Batch Invariant: Support DeepGEMM and Blackwell](../sources/prs/vllm/PR-27127.md), [[Kernels] Enable FlashInfer FP8 Blockscale on SM90 (for TEP DSR1)](../sources/prs/vllm/PR-27134.md), [[ROCM] Enable CompressedTensorsWNA16](../sources/prs/vllm/PR-27187.md), [Flashinfer_CUTLASS_MOE fuses quantization for TP](../sources/prs/vllm/PR-27223.md), [Bugfix: Cutlass FP8 FusedMoE bad scaling factors](../sources/prs/vllm/PR-27255.md), [Feature: Support Relu2 in FusedMoE fp8 cutlass path](../sources/prs/vllm/PR-27261.md), [[Perf] SM100 - add swap AB optimization to CUTLASS FP8 GEMM](../sources/prs/vllm/PR-27284.md), [[Performance] Support FP8 flashinfer TRTLLM MOE on Qwen3 and Qwen-3next](../sources/prs/vllm/PR-27492.md), [[Feature] Batch invariant torch.compile](../sources/prs/vllm/PR-27660.md), [[Performance] Fused blockwise quant RMS norm](../sources/prs/vllm/PR-27883.md), [[Performance][B200] Fix deepgemm prologue](../sources/prs/vllm/PR-27897.md), [[Performance][B200] silu_mul_quant: pack scales in int32](../sources/prs/vllm/PR-28358.md), [[Performance] Reduce DeepGEMM N dim restriction from 128 to 64 multiplier ](../sources/prs/vllm/PR-28687.md), [[Bugfix] Fix GPT-OSS on AMD after #28603](../sources/prs/vllm/PR-28816.md), [[Bugfix] Make compressed-tensors MoEs respect ignored layers](../sources/prs/vllm/PR-28878.md), [Add TRTLLM MoE NVFP4 kernel to CompressedTensorsW4A4MoeMethod](../sources/prs/vllm/PR-28892.md), [[Feat] Support non-gated activations in NVFP4 modelopt path](../sources/prs/vllm/PR-29004.md), [[Perf][Kernels] Enable FlashInfer DeepGEMM swapAB on SM90 (for W8A8 Linear Op)](../sources/prs/vllm/PR-29213.md), [[Kernel] Add NVFP4 MoE CUTLASS support for SM120](../sources/prs/vllm/PR-29242.md), [[Bugfix] Only use triton_kernels for MXFP4 on SM90 and SM100](../sources/prs/vllm/PR-29339.md), [[Perf] Disable DeepGEMM MoE by default when TP=8 is used](../sources/prs/vllm/PR-29346.md), [[Kernel]Support W4A8 Grouped GEMM on Hopper](../sources/prs/vllm/PR-29691.md), [[Quantization] Enable compressed-tensors AWQ for Turing GPU](../sources/prs/vllm/PR-29732.md), [[Bugfix] Fix mismatched nvfp4 gemm output shape](../sources/prs/vllm/PR-29742.md), [[MoE-FP8-modelopt] Add FlashInfer alignment padding for intermediate dimensions](../sources/prs/vllm/PR-29748.md), [[ROCm] [Fused Moe EP] Use binary expert mask for aiter fused moe kernel](../sources/prs/vllm/PR-29773.md), [[EPLB] Support EPLB w/ NVFP4](../sources/prs/vllm/PR-29804.md), [[Quantization] fix: overflow with static per-tensor scaling](../sources/prs/vllm/PR-29867.md), [[Bugfix] Fix FP8 MoE LoRA](../sources/prs/vllm/PR-29890.md), [[Kernel][Quantization][MoE] add marlin kernel support for turing (sm75)](../sources/prs/vllm/PR-29901.md), [[Perf] Do FP4 quant before All gather on flashinfer trtllmgen MOE ](../sources/prs/vllm/PR-30014.md), [[Quantization] Support Quark int4-fp8 w4a8 for MoE](../sources/prs/vllm/PR-30071.md), [[Model][Quantization] Restore MoE + GGUF models support (incl. Qwen3 MoE) by allowing Sideload Parameters](../sources/prs/vllm/PR-30116.md), [[Model][Quantization] Override HF defaults to GGUF ones (incl. Qwen3 MoE)](../sources/prs/vllm/PR-30118.md), [Add llmcompressor fp8 kv-cache quant (per-tensor and per-attn_head)](../sources/prs/vllm/PR-30141.md), [Nvidia ModelOpt workaround for issue 28072](../sources/prs/vllm/PR-30164.md), [[Bugfix]: Fix glm46 awq marlin moe wna16 compatibility](../sources/prs/vllm/PR-30210.md), [gptq marlin quantization support for fused moe with lora](../sources/prs/vllm/PR-30254.md), [[Bugfix] Fix DeepGEMM after #29546 ](../sources/prs/vllm/PR-30267.md), [[LoRA] Support Quantized Adapters](../sources/prs/vllm/PR-30286.md), [[Model][Quantization] Fix / Add GGUF support for Qwen2 MoE models](../sources/prs/vllm/PR-30307.md), [[bugfix][quantization] fix quark qwen3 kv_cache quantization](../sources/prs/vllm/PR-30308.md), [[fix] fix SM check for Flashinfer TRTLLM MOE](../sources/prs/vllm/PR-30314.md), [[Bugfix] Fix fp8 DeepGemm compilation issues](../sources/prs/vllm/PR-30336.md), [[ROCm][Quantization] GPT OSS Upstream MoE wmxfp4_afp8 with static scales](../sources/prs/vllm/PR-30357.md), [fix(gguf): Disable bfloat16 for GGUF on blackwell device](../sources/prs/vllm/PR-30408.md), [[Feature] Add SM103 (Blackwell Ultra) Support to vLLM](../sources/prs/vllm/PR-30484.md), [[Perf] Set split_k to 1 for triton_kernels](../sources/prs/vllm/PR-30528.md), [[Perf] Eliminate padding and slicing op for GPT-OSS with Flashinfer MXFP4 MXFP8 MoE](../sources/prs/vllm/PR-30647.md), [[SM100] Enable fp8 compute for prefill MLA](../sources/prs/vllm/PR-30746.md), [[Compressed-Tensors] Simplify NVFP4 Conditions, enable marlin support for NVFP4A16 MoEs](../sources/prs/vllm/PR-30881.md), [[Kernel][Performance] Enable smaller Scaling Factor tiling for NVFP4 small-batch decoding](../sources/prs/vllm/PR-30885.md), [[NVFP4][Perf] Tune NVFP4 input quant kernel for small batch size](../sources/prs/vllm/PR-30897.md), [[Feature]: Support NVIDIA ModelOpt HF FP8 variants FP8_PER_CHANNEL_PER_TOKEN and FP8_PB_WO in vLLM](../sources/prs/vllm/PR-30957.md), [ [FIX] Always support TP > 4 for FP4 Gemm](../sources/prs/vllm/PR-31099.md), [[Bugfix][Hardware][AMD] Consolidate FP8 min/max values helper function](../sources/prs/vllm/PR-31106.md), [[Misc] Fix grammar errors in comments and messages](../sources/prs/vllm/PR-31115.md), [[BugFix] add select_gemm_impl on CompressedTensorsWNA16MoEMethod to support LoRA](../sources/prs/vllm/PR-31453.md), [[Bugfix][ROCm] Fix Static Quant Issue](../sources/prs/vllm/PR-31502.md), [[Bugfix] Fix Broken ModelOpt NVFP4 MoE](../sources/prs/vllm/PR-31742.md), [[Perf] Fuse stride preparation for NVFP4 cutlass_moe](../sources/prs/vllm/PR-31837.md), [[1/N][Attention] Restructure attention: move files](../sources/prs/vllm/PR-31916.md), [[5/N][Attention] Finish eliminating `vllm/attention` folder](../sources/prs/vllm/PR-32064.md), [[BugFix] Fix DeepSeek-V3.1 + DeepGEMM incompatible scale shapes](../sources/prs/vllm/PR-32361.md), [[Model] Molmo2: Enable quantized weight mapping for vision backbone](../sources/prs/vllm/PR-32385.md), [[Hardware][SM100] Add TRTLLM Kernel for INT4 W4A16 Kernel.](../sources/prs/vllm/PR-32437.md), [[Perf][Kernel] Optimize FP4 quantization kernels (SM100F)](../sources/prs/vllm/PR-32520.md), [[Perf] Create TMA-aligned input scale tensor for DeepGemm on Hopper](../sources/prs/vllm/PR-32619.md), [[Bugfix][Attention] Explicitly report support for kv_cache_dtype bfloat16](../sources/prs/vllm/PR-32795.md), [[NVIDIA] [feat] Integrate flashinfer Trtllmgen bf16 moe](../sources/prs/vllm/PR-32954.md), [Support compress-tensors with nvfp4 or fp8 weights and modelopt with nvfp4 weights on Turing](../sources/prs/vllm/PR-33076.md), [Add support for Mistral Large 3 inference with Flashinfer MoE](../sources/prs/vllm/PR-33174.md), [[Bugfix] Fix quant RMS norm fusion for quantization with TMA-aligned scales](../sources/prs/vllm/PR-33255.md), [fix: Add SM120 (RTX Blackwell) support for FlashInfer CUTLASS NVFP4 MoE kernels](../sources/prs/vllm/PR-33417.md), [[Kernel] Support Flashinfer trtllm fused MoE non gated FP8 & NVFP4](../sources/prs/vllm/PR-33506.md), [[Kernel] Add enable_sm120_or_later for SM121 (DGX Spark) CUTLASS support](../sources/prs/vllm/PR-33517.md), [enable skipping of SW attention layers when using FP8 KV cache](../sources/prs/vllm/PR-33695.md), [[Bugfix]fix output Nan/Inf in marlin if dtype=float16](../sources/prs/vllm/PR-33972.md), [[ModelBash][DSR1 NVFp4] Avoid Bf16 Bias Cast](../sources/prs/vllm/PR-34298.md), [[Custom Ops] Add functional + out variant for scaled_fp4_quant](../sources/prs/vllm/PR-34389.md), [[Llama4,Quantization] Simplify and generalize logic for Q/K permutations in quantized self-attn layers ](../sources/prs/vllm/PR-34471.md), [[Model] Add NVFP4 quantization support for Step3.5-Flash](../sources/prs/vllm/PR-34478.md), [[Quantization] add humming quantization kernel](../sources/prs/vllm/PR-34556.md), [[Bugfix] Rescale NVFP4 weight scales to fix BF16 dequant underflow](../sources/prs/vllm/PR-34577.md), [[Bugfix] Fix MLA attention crash with AWQ/GPTQ quantized models](../sources/prs/vllm/PR-34695.md), [[Bugfix] Fix NVFP4 TRTLLM MoE non-gated support; add gsm8k for Nemotron-3-Nano FP8+NVFP4](../sources/prs/vllm/PR-34725.md), [add mixed precision support for modelopt](../sources/prs/vllm/PR-35047.md), [Integrate flashinfer mm_mxfp8 in ModelOpt MXFP8](../sources/prs/vllm/PR-35053.md), [[BUGFIX][Qwen3.5] Hardcode `mlp.gate` as not quantizable ](../sources/prs/vllm/PR-35156.md), [[BugFix] Fix fp4 quant kernel on CUDA 12.8](../sources/prs/vllm/PR-35210.md), [fix(mxfp4): return is_monolithic=False when LoRA is enabled for Triton backend](../sources/prs/vllm/PR-35382.md), [[Bugfix] Fix KV Scale loading for MLA Models](../sources/prs/vllm/PR-35430.md), [[Quant][Feature] Support online MXFP8 quantization for MoE and dense models](../sources/prs/vllm/PR-35448.md), [[NVFP4] Support NVFP4 dense models from `modelopt` and `compressed-tensors` on AMD Instinct MI300, MI355X and Hopper through emulation](../sources/prs/vllm/PR-35733.md), [[Bugfix] Fix score layer quantization for sequence classification models - Qwen3 (VL) Reranker](../sources/prs/vllm/PR-35849.md), [Add support for ModelOpt MXFP8 MoE models](../sources/prs/vllm/PR-35986.md), [[mla] Support fused FP8/NVFP4 output quantization in MLA attention (#35792)](../sources/prs/vllm/PR-36205.md), [docs: fix wrong cc in int8.md](../sources/prs/vllm/PR-36209.md), [[Kernel] Fuse FP8 output quantization into merge_attn_states](../sources/prs/vllm/PR-36518.md), [[MoE Refactor] Mxfp4 oracle rebased](../sources/prs/vllm/PR-37128.md), [[XPU] support MLA model on Intel GPU](../sources/prs/vllm/PR-37143.md), [[MoE/EPLB] Fix FlashInfer nvfp4 experts + EPLB correctness](../sources/prs/vllm/PR-37217.md), [[Bugfix] Expand quantization method support in perf metrics](../sources/prs/vllm/PR-37231.md), [[Kernel] Add non-gated support for NVFP4 CUTLASS MoE](../sources/prs/vllm/PR-37320.md), [Add nvfp4 support to reshape_and_cache_flash](../sources/prs/vllm/PR-37332.md), [[Kernel] Add MXFP4 W4A4 CUTLASS MoE kernel for SM100](../sources/prs/vllm/PR-37463.md), [[Bugfix] Remove assertion for NVFP4 scale dynamic range](../sources/prs/vllm/PR-37465.md), [[Bugfix] Fix marlin nvfp4 rescaling](../sources/prs/vllm/PR-37502.md), [[4/n] Migrate FP4/W4A8 CUTLASS kernels to torch stable ABI](../sources/prs/vllm/PR-37503.md), [[Bug] Fix fp8 deepgemm batch invariant](../sources/prs/vllm/PR-37718.md), [[XPU] add gptq(int4) support](../sources/prs/vllm/PR-37844.md), [[Kernel] Optimize SM120 CUTLASS blockwise FP8 GEMM](../sources/prs/vllm/PR-37970.md), [[MoE Kernel] Flashinfer nvfp4 cutedsl moe kernel integration](../sources/prs/vllm/PR-38050.md), [[Perf] FP8 FlashInfer Attn for ViT](../sources/prs/vllm/PR-38065.md), [[Bugfix] Fix DeepGemm E8M0 accuracy degradation for Qwen3.5 FP8 on Blackwell](../sources/prs/vllm/PR-38083.md), [[Quantization] Add FlashInfer CuteDSL batched experts backend for NVFP4 MoE](../sources/prs/vllm/PR-38251.md), [[Kernel] Add swapAB support for SM120 CUTLASS blockwise FP8 GEMM ](../sources/prs/vllm/PR-38325.md), [[NVIDIA] Bugfix NVFP4 DGX Spark and RTX50](../sources/prs/vllm/PR-38423.md), [[QeRL] Fix online quantized reloading](../sources/prs/vllm/PR-38442.md), [[Attention Backend] TurboQuant: 2-bit KV cache compression with 4x capacity](../sources/prs/vllm/PR-38479.md), [[Kernels][MoE] Fix legacy_routing to use bitmatrix-based routing path](../sources/prs/vllm/PR-38504.md), [[Compile] Fix nvfp4 compile warning](../sources/prs/vllm/PR-38573.md), [[Bugfix] Fix AWQ models batch invariance issues](../sources/prs/vllm/PR-38670.md), [[XPU] add xpu backend implementation of mxfp8 quant](../sources/prs/vllm/PR-38682.md), [[Quant] add CompressedTensorsW8A8Mxfp8 for linear and MoE layers](../sources/prs/vllm/PR-38815.md), [[Bugfix] Fix NVFP4+MTP crash: force unquantized mtp.fc for Qwen3.5](../sources/prs/vllm/PR-38832.md), [[Bugfix] Fix broken explicit unquantized kv cache dtype support](../sources/prs/vllm/PR-38922.md), [[MoE Refactor] Split up compressed_tensors_moe.py](../sources/prs/vllm/PR-38960.md), [[Perf] Change Trtllm fp8 MoE to use Shuffled Weights and BlockMajorK Layout](../sources/prs/vllm/PR-38993.md), [[Quantization] - Layerwise reloading of Attention/KV quantized models](../sources/prs/vllm/PR-38995.md), [[MoE] Move GPT OSS Triton kernel experts into fused_moe/experts/](../sources/prs/vllm/PR-39007.md), [[Gemma4] Support quantized MoE ](../sources/prs/vllm/PR-39045.md), [[Bug] Fix Trtllm Fp8 MoE Weight Shuffle Memory Fragamentation](../sources/prs/vllm/PR-39054.md), [[XPU] Quick fix for TritonMLA to remove cuda hardcode](../sources/prs/vllm/PR-39088.md), [[Refactor] Move NVFP4 GEMM management into NvFp4LinearKernel](../sources/prs/vllm/PR-39129.md), [[Refactor] Move MXFP8 GEMM management into MxFp8LinearKernel](../sources/prs/vllm/PR-39205.md), [[Bugfix] FlashInfer MXINT4 MoE crashes, missing do_finalize](../sources/prs/vllm/PR-39315.md), [[Feature] Batch invariant nvfp4 linear support](../sources/prs/vllm/PR-39322.md), [[Bugfix][CT] Fix KV cache scale handling](../sources/prs/vllm/PR-39418.md), [[Kernel] Support TRTLLM GEN NVFP4 MoE for non-512-aligned hidden dims via weight padding](../sources/prs/vllm/PR-39510.md), [[Perf] Fuse Zero Initializer for FP8 DeepGemm Block Quant Kernel](../sources/prs/vllm/PR-39547.md), [add warning when FP8 KV cache misses prefill query quantization](../sources/prs/vllm/PR-39752.md), [Add nvfp4 kv cache support](../sources/prs/vllm/PR-40177.md), [[Bugfix] Guard mxfp4_experts_quant bindings on ENABLE_NVFP4_SM100](../sources/prs/vllm/PR-40191.md), [Fix MoE backend selection for LoRA (unquantized MoE)](../sources/prs/vllm/PR-40273.md), [[Perf] Batch invariance with Cutlass fp8 support, 28.9% E2E latency improvement](../sources/prs/vllm/PR-40408.md), [[MoE] Move cutlass moe to fused_moe/experts/](../sources/prs/vllm/PR-40574.md), [[DSV4] Add BF16 and MXFP8 A2A support for flashinfer a2a one sided](../sources/prs/vllm/PR-40960.md), [Faster per-token fp8 group quant packed kernel for blackwell](../sources/prs/vllm/PR-41326.md), [[ROCm][Quantization][3/N] Refactor quark_moe w4a4 w/ oracle](../sources/prs/vllm/PR-41436.md), [[Quantization] Rework quantization_config to use QuantKey and allow for activation override](../sources/prs/vllm/PR-41566.md), [[MXFP4] Support for linear layers + compressed-tensors integration](../sources/prs/vllm/PR-41664.md), [[CUDA][CUTLASS] Enable cutlass scaled mm for non-compatible sizes ](../sources/prs/vllm/PR-41868.md), [[CPU] Add MXFP4 W4A16 MoE support](../sources/prs/vllm/PR-41922.md), [[MoE] Move various experts classes to fused_moe/experts/](../sources/prs/vllm/PR-41979.md), [[Bugfix] Add swiglu limits to deepgemm fp8 methods](../sources/prs/vllm/PR-41986.md), [[Perf] Use 2D-grid to eliminate divmod in W8W8 group quant](../sources/prs/vllm/PR-42153.md), [Refactor AWQ Marlin MoE onto modular WNA16 oracle](../sources/prs/vllm/PR-42483.md), [[Misc] add humming to dependencies](../sources/prs/vllm/PR-42540.md), [[Bugfix] fix swiglu limit issue for humming backend + deepseek v4](../sources/prs/vllm/PR-42541.md), [[6/n] Migrate activation kernels, gptq, gguf, non cutlass w8a8 to libtorch stable ABI (continued)](../sources/prs/vllm/PR-42663.md), [[Perf] Padded nvfp4 quant kernel to remove additional copy, 2.4%~5.7% e2e performance improvement](../sources/prs/vllm/PR-42774.md), [[Model Refactoring] Migrate DeepSeek V4 to vllm/models/ [1/N] ](../sources/prs/vllm/PR-43004.md), [[Kernel] (1/N) Machete - Hopper Optimized Mixed Precision Linear Kernel ](../sources/prs/vllm/PR-7174.md), [[Kernel] (2/N) Machete - Integrate into CompressedTensorsWNA16 and GPTQMarlin](../sources/prs/vllm/PR-7701.md), [[Bugfix] Fix Machete unittests failing with `NotImplementedError`](../sources/prs/vllm/PR-9218.md) | -| `sparse-attention` | [FlashMLA — Multi-head Latent Attention](../sources/blogs/flashmla.md), [Native Sparse Attention (NSA)](../sources/blogs/nsa.md), [Qwen3-Next: Hybrid GDN+MoE Architecture on NVIDIA Blackwell](../sources/blogs/qwen3-next-architecture.md), [DeepSeek-V3.2-Exp in vLLM: Fine-Grained Sparse Attention in Action](../sources/blogs/vllm-deepseek-v3-sparse-attention.md), [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [[None][perf] Scheme X L2-aware dispatcher and PDL launchers for sparse-attention GVR Top-K](../sources/prs/TensorRT-LLM/PR-13477.md), [[TRTLLM-8535][feat] Support DeepSeek V3.2 with FP8 + BF16 KV cache/NVFP4 + BF16 KV cache](../sources/prs/TensorRT-LLM/PR-8405.md), [Add seqused_q in fwd / bwd and seqused_k in bwd in hopper FA.](../sources/prs/flash-attention/PR-1182.md), [[feat] support block sparse attention w/ variable block sizes and head-wise sparse patterns](../sources/prs/flashinfer/PR-1177.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [FlashAttention SM100 MLA TopK Sparse Forward](../wiki/kernels/flash-attention-sm100-mla-topk.md), [FlashMLA — Multi-head Latent Attention](../wiki/kernels/flashmla.md), [Native Sparse Attention (NSA)](../wiki/kernels/nsa.md), [Sparse MLA (DeepSeek V3.2)](../wiki/kernels/sparse-mla.md) | -| `topk` | [FlashAttention SM100 MLA TopK Sparse Forward](../wiki/kernels/flash-attention-sm100-mla-topk.md), [TensorRT-LLM Blackwell FP4 DSA Indexer](../wiki/kernels/tensorrt-llm-blackwell-indexer.md) | +| `scan` | [Add b200 tunings for scan.exclusive.sum](../sources/prs/cccl/PR-3559.md) | +| `sparse-attention` | [FlashMLA — Multi-head Latent Attention](../sources/blogs/flashmla.md), [Qwen3-Next: Hybrid GDN+MoE Architecture on NVIDIA Blackwell](../sources/blogs/qwen3-next-architecture.md), [DeepSeek-V3.2-Exp in vLLM: Fine-Grained Sparse Attention in Action](../sources/blogs/vllm-deepseek-v3-sparse-attention.md), [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [Native Sparse Attention: Hardware-Aligned and Natively Trainable Sparse Attention](../sources/docs/nsa.md), [[None][perf] Scheme X L2-aware dispatcher and PDL launchers for sparse-attention GVR Top-K](../sources/prs/TensorRT-LLM/PR-13477.md), [[TRTLLM-8535][feat] Support DeepSeek V3.2 with FP8 + BF16 KV cache/NVFP4 + BF16 KV cache](../sources/prs/TensorRT-LLM/PR-8405.md), [Add seqused_q in fwd / bwd and seqused_k in bwd in hopper FA.](../sources/prs/flash-attention/PR-1182.md), [[feat] support block sparse attention w/ variable block sizes and head-wise sparse patterns](../sources/prs/flashinfer/PR-1177.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [FlashAttention SM100 MLA TopK Sparse Forward](../wiki/kernels/flash-attention-sm100-mla-topk.md), [FlashMLA — Multi-head Latent Attention](../wiki/kernels/flashmla.md), [Native Sparse Attention (NSA)](../wiki/kernels/nsa.md), [DeepSeek Sparse Attention / Sparse MLA](../wiki/kernels/sparse-mla.md) | +| `topk` | [Fix debug section around line 390 of dispatch_topk](../sources/prs/cccl/PR-6152.md), [FlashAttention SM100 MLA TopK Sparse Forward](../wiki/kernels/flash-attention-sm100-mla-topk.md), [TensorRT-LLM Blackwell FP4 DSA Indexer](../wiki/kernels/tensorrt-llm-blackwell-indexer.md) | diff --git a/queries/by-language.md b/queries/by-language.md index 7864341e7..31da8c019 100644 --- a/queries/by-language.md +++ b/queries/by-language.md @@ -4,11 +4,12 @@ | Language | Guide | Related Pages | |----------|-------|--------------| -| `cuda-cpp` | [CUDA C++ for Blackwell Kernels](../wiki/languages/cuda-cpp.md) | [Twelve Attempts at NVFP4 Batched GEMV](../sources/blogs/amandeep-nvfp4-attempts.md), [Colfax Article Source Kernels](../sources/blogs/colfax-article-source-kernels.md), [Colfax CUTLASS Kernels](../sources/blogs/colfax-cutlass-kernels.md), [Modular: Matrix Multiplication on Blackwell](../sources/blogs/modular-blackwell-matmul.md), [NVIDIA Developer Code Samples](../sources/blogs/nvidia-code-samples.md), [tcgen05 for dummies](../sources/blogs/tcgen05-tutorial.md), [Blackwell NVFP4 Kernel Hackathon Journey](../sources/blogs/yue-nvfp4-hackathon.md), [FlashInfer MLSys 2026 - Track A: Fused MoE FP8](../sources/contests/flashinfer-mlsys26/track-a-fused-moe.md), [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [FlashInfer MLSys 2026 - Track C: Gated Delta Net](../sources/contests/flashinfer-mlsys26/track-c-gated-delta-net.md), [GPU Mode NVFP4 Hackathon - Problem 1: Batched GEMV](../sources/contests/gpu-mode-nvfp4/problem-1-gemv.md), [GPU Mode NVFP4 Hackathon - Problem 2: NVFP4 GEMM](../sources/contests/gpu-mode-nvfp4/problem-2-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 3: Gated Dual GEMM](../sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 4: Grouped GEMM](../sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md), [NVIDIA Blackwell Compatibility Guide](../sources/docs/blackwell-compatibility-guide.md), [Fix performance issue of m-grouped contiguous GEMMs.](../sources/prs/DeepGEMM/PR-168.md), [Fix multicast bug and optimize masked GEMM](../sources/prs/DeepGEMM/PR-193.md), [fix: use SM90ArchSpec instead of SM100ArchSpec in sm90_bf16_k_grouped_gemm](../sources/prs/DeepGEMM/PR-270.md), [[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes](../sources/prs/DeepGEMM/PR-304.md), [Sync nv_dev with upstream #316 (Mega MoE optimizations & benchmarks)](../sources/prs/DeepGEMM/PR-328.md), [ Solving bank conflict via padding and TMA 3D store](../sources/prs/DeepGEMM/PR-78.md), [Use 1D TMA store instead of 3D](../sources/prs/DeepGEMM/PR-83.md), [Use swizzling instead of padding](../sources/prs/DeepGEMM/PR-86.md), [Support TMA multicast on B with m_grouped_gemm_contiguous.](../sources/prs/DeepGEMM/PR-88.md), [[None][feat] sm100 weight-only kernel](../sources/prs/TensorRT-LLM/PR-10190.md), [[TRTLLM-10022][feat] Add hopper xqa decode support for skip softmax attention](../sources/prs/TensorRT-LLM/PR-10264.md), [[https://nvbugs/5669671][fix] Support GuidedDecoder with sharded logits (pick #10698)](../sources/prs/TensorRT-LLM/PR-10742.md), [[None][feat] fuse shared to sparse experts in TRT-LLM Gen MoE](../sources/prs/TensorRT-LLM/PR-11143.md), [[https://nvbugs/5799917][fix] Recover from CUTLASS MoE doActivation perf regression for MXFP4/NVFP4 dtype](../sources/prs/TensorRT-LLM/PR-11165.md), [[None][feat] Optimize super-v3 nvfp4 for better perf](../sources/prs/TensorRT-LLM/PR-11273.md), [[None][feat] Optimize by fuse nvfp4_quant to layernorm_gated for mamba2_mixer](../sources/prs/TensorRT-LLM/PR-11473.md), [[None][feat] TRT-LLM Gen MoE finalize kernel optimization](../sources/prs/TensorRT-LLM/PR-11501.md), [[None][feat] Add support for expert_number<=2048 and K<=32](../sources/prs/TensorRT-LLM/PR-11510.md), [[TRTLLM-10004][feat] Enable GEMM -> AR with GEMM output in registered buffers](../sources/prs/TensorRT-LLM/PR-11589.md), [[TRTLLM-11119][feat] Blackwell SageAttention, Integrate into AttentionOp API](../sources/prs/TensorRT-LLM/PR-11718.md), [[https://nvbugs/5799917][fix] Recover from CUTLASS MoE doActivation perf regression for MXFP4/NVFP4 dtype](../sources/prs/TensorRT-LLM/PR-11733.md), [[None][feat] Add fused DiT QK Norm + RoPE CUDA kernel for FLUX](../sources/prs/TensorRT-LLM/PR-11869.md), [[TRTLLM-10421][perf] Add fused cat+fp8_quantize CUDA kernel for DSA indexer](../sources/prs/TensorRT-LLM/PR-11899.md), [[#11694][feat] AutoDeploy: Improve the piecewise CG memory usage](../sources/prs/TensorRT-LLM/PR-11993.md), [[https://nvbugs/5955188][fix] Fix harmony parsers and WAR routing PDL for agentic coding use cases](../sources/prs/TensorRT-LLM/PR-12046.md), [[TRTLLM-11540][feat] Add EAGLE3 dynamic tree speculative decoding support](../sources/prs/TensorRT-LLM/PR-12062.md), [[None][feat] Minimax RMS norm optimization](../sources/prs/TensorRT-LLM/PR-12163.md), [[None][feat] Add fused allreduce+RMSNorm op and optional residual in …](../sources/prs/TensorRT-LLM/PR-12201.md), [[None][feat] Support update weight for nvfp4](../sources/prs/TensorRT-LLM/PR-12320.md), [[None][feat] Temporally-Correlated Heuristic-guided Indexer TopK for Sparse Attention](../sources/prs/TensorRT-LLM/PR-12385.md), [[None][perf] add Dynamic SMEM block routing in MOE](../sources/prs/TensorRT-LLM/PR-12456.md), [[None][feat] Support sparse mqa/gqa attention](../sources/prs/TensorRT-LLM/PR-12470.md), [[https://nvbugs/5983390][perf] Split MLA DSA custom op for piecewise CUDA graph capture](../sources/prs/TensorRT-LLM/PR-12503.md), [[None][feat] Add Mamba2 MTP SSM cache CUDA kernel for tree-based speculative decoding](../sources/prs/TensorRT-LLM/PR-12537.md), [[https://nvbugs/5983390][perf] Multiple host perf optimizations for DSA part](../sources/prs/TensorRT-LLM/PR-12581.md), [[None][feat] Trtllm-gen FMHA JIT support](../sources/prs/TensorRT-LLM/PR-12612.md), [[None][feat] Optimize mamba SSD prefill and extend flashinfer dispatch](../sources/prs/TensorRT-LLM/PR-12731.md), [[TRTLLM-11485][feat] Feature rework: Add SageAttention refreshed kernels (attentionOp only)](../sources/prs/TensorRT-LLM/PR-12937.md), [[None][feat] Update rms_norm + fp4_qaunt kernel supporting more dim](../sources/prs/TensorRT-LLM/PR-13033.md), [[#12716][feat] Fused cross-head QK Norm + RoPE kernel for WAN](../sources/prs/TensorRT-LLM/PR-13052.md), [[None][feat] Optimize causal_conv1d prefill and decode kernels](../sources/prs/TensorRT-LLM/PR-13103.md), [[None][feat] Add FP4 residual quantization kernel without channel reo…](../sources/prs/TensorRT-LLM/PR-13117.md), [[https://nvbugs/5945047][fix] Fix cluster launch enablement for SM120 GPUs in allReduce fusion](../sources/prs/TensorRT-LLM/PR-13169.md), [[None][feat] Integrate FP4 indexer for DSA on Blackwell](../sources/prs/TensorRT-LLM/PR-13340.md), [[None][perf] Extend customMoeRouting kernel to support Qwen3.5](../sources/prs/TensorRT-LLM/PR-13433.md), [[None][perf] Scheme X L2-aware dispatcher and PDL launchers for sparse-attention GVR Top-K](../sources/prs/TensorRT-LLM/PR-13477.md), [[None][perf] Drop cubin and Eliminate ~6s FMHA JIT recompile in eager generation by aligning kernel selection with CUDA graph warmup](../sources/prs/TensorRT-LLM/PR-13505.md), [[None][feat] Fuse FP8 1x128 quantize + UE8M0 scale pack on SM100](../sources/prs/TensorRT-LLM/PR-13628.md), [[#13580][fix] AutoDeploy: Support Gemma3n/4 E2B variants](../sources/prs/TensorRT-LLM/PR-13630.md), [[None][feat] Add DeepSeekV4 attention kernels](../sources/prs/TensorRT-LLM/PR-13652.md), [[https://nvbugs/6108841][fix] add hidden_dim=6144 router GEMM instantiation for GLM-5](../sources/prs/TensorRT-LLM/PR-13740.md), [[None][perf] Optimize DeepSeek-V4 compressor BF16 input](../sources/prs/TensorRT-LLM/PR-13761.md), [[None][fix] Plumb swiglu_limit through DeepGEMM and TRTLLMGen FP8 fused MoE](../sources/prs/TensorRT-LLM/PR-13767.md), [[None][fix] Fix fused MHC for DeepSeek-V4-Pro hidden size](../sources/prs/TensorRT-LLM/PR-13771.md), [[None][feat] Indexer topk opt](../sources/prs/TensorRT-LLM/PR-13811.md), [[None][perf] mHC fused_hc kernel optimizations + DS-V4 entry-boundary RMSNorm fold-in](../sources/prs/TensorRT-LLM/PR-13892.md), [[None][perf] Add CUDA q_b norm for DeepSeek V4](../sources/prs/TensorRT-LLM/PR-13975.md), [[None][fix] Add SPDX Apache-2.0 headers to auto_deploy test files](../sources/prs/TensorRT-LLM/PR-14193.md), [[None][feat] DSv4: enable GVR Heuristic Top-K for compress_ratio=4](../sources/prs/TensorRT-LLM/PR-14219.md), [[None][feat] Update the logic of FMHA JIT path](../sources/prs/TensorRT-LLM/PR-14291.md), [[None][chore] Update Claude Code agents and skills](../sources/prs/TensorRT-LLM/PR-14344.md), [feat: Add w4a8_mxfp4_fp8 quantization recipe.](../sources/prs/TensorRT-LLM/PR-4867.md), [[OMNIML-2336][feat] Add NVFP4 x FP8](../sources/prs/TensorRT-LLM/PR-6809.md), [[None][chore] Fix kernel launch param and add TRTLLM MoE backend test](../sources/prs/TensorRT-LLM/PR-7524.md), [[None][fix] Fix and add test for TRTLLM MoE backend](../sources/prs/TensorRT-LLM/PR-7755.md), [[TRTLLM-8637][feat] Optimize the routing kernel for DeepseekV3 (MoE CUTLASS backend); Add support for 384 experts (MoE TRTLLM backend)](../sources/prs/TensorRT-LLM/PR-7761.md), [[None][feat] GPT-OSS Sm120/Sm121 Support](../sources/prs/TensorRT-LLM/PR-7937.md), [[None][fix] Fix the performance issue of FP8 blockwise grouped GEMM when using attention DP](../sources/prs/TensorRT-LLM/PR-8501.md), [[None][feat] Enable nvfp4 cuda core for sm120](../sources/prs/TensorRT-LLM/PR-8620.md), [[None][feat] Update TRTLLM MoE cubins; reduce mxfp4 weight padding requirement; tighten TMA bound](../sources/prs/TensorRT-LLM/PR-9025.md), [[None][fix] support topk autotuner input for expert slot per group larger than 32](../sources/prs/TensorRT-LLM/PR-9087.md), [[None][feat] TRT-LLM Gen MoE optimize DeepSeek Fp8 activation kernel](../sources/prs/TensorRT-LLM/PR-9175.md), [[None][feat] add fp4 gemm + allreduce](../sources/prs/TensorRT-LLM/PR-9729.md), [[https://nvbugs/5726962][feat] Apply fusion for W4AFP8_AWQ MoE](../sources/prs/TensorRT-LLM/PR-9838.md), [[None][feat] Fused kernels (qknormrope + moe routing) and two-model MTP support for glm4moe](../sources/prs/TensorRT-LLM/PR-9852.md), [[None][feat] Port fp4 quantization kernel optimization from FlashInfer](../sources/prs/TensorRT-LLM/PR-9854.md), [[None][feat] Adding torch ext API for FusedAddRMSNormQuant kernel](../sources/prs/TensorRT-LLM/PR-9905.md), [[TRTLLM-9493][feat] Add helixPostProcessNative kernel for cp_dim=2](../sources/prs/TensorRT-LLM/PR-9924.md), [Experimental Python cooperative algorithms](../sources/prs/cccl/PR-1973.md), [fix thread-reduce performance regression](../sources/prs/cccl/PR-2944.md), [Fix scan / sm90 perf regression ](../sources/prs/cccl/PR-3236.md), [Fix the vectorized loading of BlockLoad](../sources/prs/cccl/PR-3517.md), [Add b200 tunings for scan.exclusive.sum](../sources/prs/cccl/PR-3559.md), [Fix SM100 histogram tunings](../sources/prs/cccl/PR-3691.md), [Split Optimize Warp Reduce PR - CUB part](../sources/prs/cccl/PR-4716.md), [Add nondeterministic reduce that uses atomics](../sources/prs/cccl/PR-4961.md), [CUB - Add internal integer utils and tests (Split `WarpReduce` PR)](../sources/prs/cccl/PR-5314.md), [Combine `block_reduce_warp_reduction_nondeterministic.cuh` specialization with original deterministic one ](../sources/prs/cccl/PR-5408.md), [Add dynamic CUB dispatch for segmented_sort](../sources/prs/cccl/PR-6069.md), [[CUB] Use `BlockLoadToShared` in `DeviceMerge`](../sources/prs/cccl/PR-6077.md), [Fix debug section around line 390 of dispatch_topk](../sources/prs/cccl/PR-6152.md), [Split fixed-size segmented reduce dispatch header](../sources/prs/cccl/PR-6597.md), [Integrate decoupled lookahead warpspeed scan](../sources/prs/cccl/PR-6811.md), [Use integer promotion for `warp_reduce`](../sources/prs/cccl/PR-6819.md), [Implement new tuning API arch dispatching](../sources/prs/cccl/PR-7093.md), [Two-phase reduction for fixed size segmented reduction for very large segment sizes](../sources/prs/cccl/PR-7114.md), [Implement the new tuning API for deterministic (rfa) reduce dispatch](../sources/prs/cccl/PR-7346.md), [Radix-selection based `BlockTopK` specialization](../sources/prs/cccl/PR-7384.md), [Implement the new tuning API for `DeviceRleDispatch`](../sources/prs/cccl/PR-7669.md), [Optimize non fixed size segmented reduce for small segments using max_segment_size](../sources/prs/cccl/PR-7718.md), [Add env SegmentedReduce (non fixed-size overloads)](../sources/prs/cccl/PR-7795.md), [Forward policy hub from `dispatch_streaming_arg_reduce_t` to `reduce::dispatch`](../sources/prs/cccl/PR-7805.md), [Implement the new tuning API for `detail::reduce::dispatch_streaming_arg_reduce_t`](../sources/prs/cccl/PR-7807.md), [Use the new tuning API internally for `detail::transform::dispatch`](../sources/prs/cccl/PR-7810.md), [[Backport branch/3.3.x] Forward policy hub from `dispatch_streaming_arg_reduce_t` to `reduce::dispatch`](../sources/prs/cccl/PR-7814.md), [Optimized Device-to-Device Tensor Copy (`cudax`)](../sources/prs/cccl/PR-7823.md), [Implement the new tuning API for `DispatchSegmentedRadixSort`](../sources/prs/cccl/PR-7844.md), [Implement the new tuning API for `DispatchSegmentedSort`](../sources/prs/cccl/PR-7874.md), [Implement the new tuning API for `DispatchTopK`](../sources/prs/cccl/PR-7928.md), [[cuda.compute]: Fix faulty pointer arithmetic calculation in CUB dispatch](../sources/prs/cccl/PR-7940.md), [Reduce usage of `cub::DispatchReduce`](../sources/prs/cccl/PR-7944.md), [Use the new tuning API for `detail::radix_sort::dispatch`](../sources/prs/cccl/PR-7949.md), [Adds support for non-fundamental types via decomposer to `DeviceTopK` ](../sources/prs/cccl/PR-8040.md), [Optimized Device-to-Device Tensor Copy (cudax) - Transpose Case](../sources/prs/cccl/PR-8125.md), [Avoid passing uninitialized values to scan_op](../sources/prs/cccl/PR-8184.md), [[STF] Move unstable_unique from STF to generic cudax utility](../sources/prs/cccl/PR-8190.md), [Expose `max_segment_size` guarantee in cuda.compute](../sources/prs/cccl/PR-8284.md), [Port `thrust::min|max_element` to CUB](../sources/prs/cccl/PR-8291.md), [Implement the new tuning API for `DispatchSelectIf`](../sources/prs/cccl/PR-8311.md), [simplify dispatch segmented reduce to use latest dispatch and new tunings API](../sources/prs/cccl/PR-8332.md), [Apply some random warpspeed tunings](../sources/prs/cccl/PR-8352.md), [[cub]: implement utilities for policy selection](../sources/prs/cccl/PR-8355.md), [Replace `detail::merge::dispatch` by CUB's public API](../sources/prs/cccl/PR-8381.md), [[CUB] Replace `Shuffle(Up|Down|Index)` with cuda::device::warp_shuffle - RadixSort only](../sources/prs/cccl/PR-8395.md), [Vectorize mbarrier initialization in warpspeed scan](../sources/prs/cccl/PR-8423.md), [[thrust] Single-pass `is_partitioned` via adjacent zip_iterator](../sources/prs/cccl/PR-8427.md), [Replace `detail::merge_sort::dispatch` by CUB's public API](../sources/prs/cccl/PR-8473.md), [Replace `detail::scan::dispatch` by CUB's public API](../sources/prs/cccl/PR-8495.md), [Implement the new tuning API for `detail::batched_topk::dispatch_batched_topk`](../sources/prs/cccl/PR-8538.md), [Replace `detail::for_each::dispatch` by CUB's public API](../sources/prs/cccl/PR-8565.md), [Replace `detail::segmented_reduce::dispatch` by the public API](../sources/prs/cccl/PR-8695.md), [Use the new tuning API internally for `detail::topk::dispatch`](../sources/prs/cccl/PR-8742.md), [Use the new tuning API internally for `detail::reduce_by_key::dispatch`](../sources/prs/cccl/PR-8756.md), [Use the new tuning API internally for `detail::reduce[_nd]::dispatch[_nd]`](../sources/prs/cccl/PR-8826.md), [Fix Warpspeed scan shifted output store](../sources/prs/cccl/PR-8839.md), [[cub] Simplify arch dispatch](../sources/prs/cccl/PR-8861.md), [Use the new tuning API internally for `detail::select::dispatch` and `DeviceSelect`](../sources/prs/cccl/PR-8880.md), [[STF] Add per-handle exec_place stream resources](../sources/prs/cccl/PR-8905.md), [Use the new tuning API internally for `detail::select|three_way_partition::dispatch` and `DevicePartition`](../sources/prs/cccl/PR-8925.md), [Use the new tuning API internally for `detail::segmented_radix_sort::dispatch`](../sources/prs/cccl/PR-8927.md), [[libcu++] Always suppress C++ extensions warnings in prologue](../sources/prs/cccl/PR-9019.md), [Fix segmented radix sort benchmark segment size type](../sources/prs/cccl/PR-9039.md), [[libcu++] Fix default make_shared_resource construction](../sources/prs/cccl/PR-9044.md), [Vectorize contiguous iterators in `cub::BlockLoad`/`Store`](../sources/prs/cccl/PR-9056.md), [Improve sm90 mixed dtype kernel](../sources/prs/cutlass/PR-1883.md), [[EVT] Add support for Row/Col broadcast PtrArray](../sources/prs/cutlass/PR-2033.md), [Groupwise scaling along M for FP8 gemm](../sources/prs/cutlass/PR-2037.md), [Improvements for: Groupwise scaling along M for FP8 gemm](../sources/prs/cutlass/PR-2095.md), [Flash MLA support](../sources/prs/cutlass/PR-2130.md), [Flash MLA Support - Step 2](../sources/prs/cutlass/PR-2134.md), [Blockwise and Groupwise GEMM for Blackwell and Improvements for Hopper](../sources/prs/cutlass/PR-2139.md), [Blockwise Improvement and Programmatic Dependent Launch](../sources/prs/cutlass/PR-2161.md), [Fix sm100 gemm wrong static constexpr that breaks compilation on Windows](../sources/prs/cutlass/PR-2167.md), [Fix SM90 beta=1 hang and stream-K launch errors](../sources/prs/cutlass/PR-2172.md), [Set EpiTile correctly when TileN is not divisible by 32](../sources/prs/cutlass/PR-2220.md), [Use cudaMemcpyAsync in gemm grouped with kRequiresPrecomputation sche…](../sources/prs/cutlass/PR-2256.md), [war to fix blackwell grouped groupwise hang](../sources/prs/cutlass/PR-2267.md), [hopper-blockwise-generalization-optimization](../sources/prs/cutlass/PR-2270.md), [Correct divmod order in example 77 (blackwell fmha)](../sources/prs/cutlass/PR-2291.md), [Handle get_masked_trip_count for small length in fmha example](../sources/prs/cutlass/PR-2292.md), [Fix epilogue::thread::Convert cannot be used with DefaultEpilogue](../sources/prs/cutlass/PR-2333.md), [[ex77] fix mla split; add fwd lse; add bwd varlen](../sources/prs/cutlass/PR-2366.md), [support fp16 accmulator for sm89 fp8 mma](../sources/prs/cutlass/PR-2378.md), [Example 77 add blackwell flash-attention bwd for MLA shape](../sources/prs/cutlass/PR-2466.md), [Add Blackwell MLA forward (shape: d=192, dv=128) implementation](../sources/prs/cutlass/PR-2472.md), [fix: examples/cute/tutorial/blackwell/04_mma_tma_2sm_sm100.cu GridDim miscalculated](../sources/prs/cutlass/PR-2492.md), [DistGEMM bug fixes](../sources/prs/cutlass/PR-2713.md), [Support PDL for SM90 Array TMA GEMM](../sources/prs/cutlass/PR-2719.md), [Support for GEMM-K=0 for Blackwell Grouped GEMMs](../sources/prs/cutlass/PR-2746.md), [Blockscaled Ragged Contiguous Grouped Gemm for MoEs](../sources/prs/cutlass/PR-2790.md), [[Bug Fix]Bypass launch grids for SM120 Kernel with SM90 Mainloop & SM100 TileScheduler](../sources/prs/cutlass/PR-2865.md), [[cute] Add constexpr specifier to make_tiled_copy](../sources/prs/cutlass/PR-2875.md), [Fix incorrect tensor layout strides in Blackwell MMA tutorial comments](../sources/prs/cutlass/PR-2921.md), [[Cutlass gemm] Fix SM100 FP8 nosmem epilogue-fusion shape_div 'Divisibility Condition' for non-multiple-of-64 N tiles](../sources/prs/cutlass/PR-2946.md), [[Bug Fix]Set NumSplitsM to 1 when TileShapeM < 128 in sm90 fp8 blockwise scaling CollectiveMma](../sources/prs/cutlass/PR-2965.md), [Replace std::min with cute::min in sm120 blockwise scaling device functions](../sources/prs/cutlass/PR-3055.md), [Small Tile N BlockScaled GEMM + Grouped GEMM on SM12x](../sources/prs/cutlass/PR-3176.md), [Add Snake activation functor for EVT](../sources/prs/cutlass/PR-3184.md), [Fp8 kernel with "in-kernel" transpose of V in producer](../sources/prs/flash-attention/PR-1100.md), [FA3 kvcache + split kv + gqa parallelization](../sources/prs/flash-attention/PR-1236.md), [Fix FA3 Varlen Performance regression](../sources/prs/flash-attention/PR-1361.md), [Add sorting and head swizzle to varlen scheduler](../sources/prs/flash-attention/PR-1823.md), [feat: update decode attention APIs](../sources/prs/flashinfer/PR-1007.md), [misc: fix instrument code for mla profiler](../sources/prs/flashinfer/PR-1014.md), [add multi-item scoring](../sources/prs/flashinfer/PR-1015.md), [fix: add zero init for KV tiled copy](../sources/prs/flashinfer/PR-1029.md), [feat: add functional per-head FP8 quantization for FA3](../sources/prs/flashinfer/PR-1033.md), [feat: Softmax free sampling](../sources/prs/flashinfer/PR-1035.md), [[nvidia] initial support for blackwell kernels](../sources/prs/flashinfer/PR-1039.md), [fix: top_k_mask_logits hangs on -inf inputs](../sources/prs/flashinfer/PR-1050.md), [[nvidia] Add Blackwell FMHA decode kernel from TRT-LLM](../sources/prs/flashinfer/PR-1051.md), [Fix KV chunking for POD. ](../sources/prs/flashinfer/PR-1054.md), [bugfix: temporally disable split-kv in blackwell mla](../sources/prs/flashinfer/PR-1055.md), [Parameterize prefix mask call (needed by POD-Attention)](../sources/prs/flashinfer/PR-1059.md), [bugfix: adding lse output to blackwell fmha kernels](../sources/prs/flashinfer/PR-1071.md), [bugfix: follow user-specified sm_scale for blackwell cutlass fmha](../sources/prs/flashinfer/PR-1072.md), [perf: accelerate blackwell grouped gemm](../sources/prs/flashinfer/PR-1086.md), [bugfix: fix fp8 attention kernels aot compilation issue](../sources/prs/flashinfer/PR-1087.md), [comm: refactor and initialize `flashinfer.comm` module](../sources/prs/flashinfer/PR-1089.md), [feat: add trtllm all-reduce (non-MoE)](../sources/prs/flashinfer/PR-1096.md), [bugfix: host-precomuted plan function for blackwell fmha](../sources/prs/flashinfer/PR-1106.md), [feat: add trtllm moe_allreduce_fusion](../sources/prs/flashinfer/PR-1108.md), [Add CUTLASS fused moe kernels from TensorRT-LLM.](../sources/prs/flashinfer/PR-1113.md), [bugfix: Fix test and output shape of fp4 quantize](../sources/prs/flashinfer/PR-1114.md), [hotfix: fix the blackwell fmha stream](../sources/prs/flashinfer/PR-1116.md), [[Feature] Support PDL for batch Prefill and Decode](../sources/prs/flashinfer/PR-1117.md), [Fix pointer dtype bug in rope](../sources/prs/flashinfer/PR-1129.md), [feat: add trtllm all-reduce fusion](../sources/prs/flashinfer/PR-1131.md), [MNNVL MoE All-to-All Support](../sources/prs/flashinfer/PR-1134.md), [fix: negative zero by type trait --> binary value](../sources/prs/flashinfer/PR-1136.md), [[feat] add unified batch attention w/ correctness tests.](../sources/prs/flashinfer/PR-1137.md), [Fix FA2 and FA3 multi-item scoring and cuda illegal memory access error](../sources/prs/flashinfer/PR-1140.md), [feat: Fused temperature online softmax kernel](../sources/prs/flashinfer/PR-1153.md), [Add more logging to TRTLLM-GEN debug trace (NFC)](../sources/prs/flashinfer/PR-1158.md), [feat: add finalize_moe_allreduce from trtllm](../sources/prs/flashinfer/PR-1159.md), [feat: nvshmem python bindings](../sources/prs/flashinfer/PR-1160.md), [feat: update non-fused moe](../sources/prs/flashinfer/PR-1161.md), [feat: enable and update all-reduce fused quantization](../sources/prs/flashinfer/PR-1164.md), [bugfix: softmax NaN results caused by large -inf masks](../sources/prs/flashinfer/PR-1178.md), [update trtllm-gen decode attention kernel launcher](../sources/prs/flashinfer/PR-1189.md), [[feat] optimize persistent batch attention perf.](../sources/prs/flashinfer/PR-1200.md), [[fix] fix BatchAttention CTA_TILE_KV mask issue](../sources/prs/flashinfer/PR-1206.md), [Fix the issue with auxillary kernel launch and grid dim calculation](../sources/prs/flashinfer/PR-1208.md), [feat: trtllm-gen fp8 moe kernels](../sources/prs/flashinfer/PR-1212.md), [[comm] TRT-LLM's Multi-Node NVLink All-Reduce Kernel](../sources/prs/flashinfer/PR-1213.md), [Feature/sm100 low latency nvfp4 kernels](../sources/prs/flashinfer/PR-1214.md), [Enable cudnn decode and add tests for the cudnn decode kernel](../sources/prs/flashinfer/PR-1221.md), [feat: add trtllm-gen mla cubin](../sources/prs/flashinfer/PR-1222.md), [Fix missing hash in the cudnn cubin path](../sources/prs/flashinfer/PR-1227.md), [feat: Add non-causal cudnn prefill kernels](../sources/prs/flashinfer/PR-1230.md), [bugfix: support uint8_t for vec_t class template](../sources/prs/flashinfer/PR-1234.md), [add trtllm-gen context attention](../sources/prs/flashinfer/PR-1239.md), [Patch fp8 cubin availability](../sources/prs/flashinfer/PR-1240.md), [feat: Support MXFP8 x MXFP4 CUTLASS grouped GEMM](../sources/prs/flashinfer/PR-1241.md), [Add trtllm-gen attention mha kernel with FP8 Q/K/V and FP8 output](../sources/prs/flashinfer/PR-1242.md), [Remove sm100+ requirment for trtllm allreduce kernels](../sources/prs/flashinfer/PR-1249.md), [Reduce the JIT compilation time of gen_gemm_sm100_module](../sources/prs/flashinfer/PR-1251.md), [TRT-LLM's Multi-Node NVLink AR + fused RMSNorm kernel](../sources/prs/flashinfer/PR-1255.md), [feat: enable trtllm-gen mla MTP](../sources/prs/flashinfer/PR-1258.md), [Made AR output optional + esthetic changes](../sources/prs/flashinfer/PR-1265.md), [Bug fix: fix duplicate launch in POD](../sources/prs/flashinfer/PR-1267.md), [Add shuffle matrix flag](../sources/prs/flashinfer/PR-1272.md), [Convert scale_factor from scalar to Tensor in trt_allreduce_fusion](../sources/prs/flashinfer/PR-1284.md), [fix multiCtasKvScratchPtr misalignment issue (new one)](../sources/prs/flashinfer/PR-1286.md), [Bug fix: guard fp8 e8m0 and e2m1 compile ](../sources/prs/flashinfer/PR-1287.md), [refactor: refactor trtllm-gen attention kernel integration code](../sources/prs/flashinfer/PR-1289.md), [[fix] fix integer overflow in FA2 customized_mask & add buffer overflow warning.](../sources/prs/flashinfer/PR-1290.md), [refactor: Improved metainfo for trtllm-gen fmha](../sources/prs/flashinfer/PR-1292.md), [Update cutlass fp4 moe kernels](../sources/prs/flashinfer/PR-1294.md), [add cutlass backend for mm_fp4](../sources/prs/flashinfer/PR-1296.md), [feat: Add weight layout option for trtllm-gen fused moe](../sources/prs/flashinfer/PR-1297.md), [perfix: use lightweight API to query device property](../sources/prs/flashinfer/PR-1298.md), [[Feature] SM level profiler ](../sources/prs/flashinfer/PR-1305.md), [Fix the bug of the kernel-selection heuristic in trtllm-gen](../sources/prs/flashinfer/PR-1307.md), [Refactor Fused Moe Module](../sources/prs/flashinfer/PR-1309.md), [feat: support output nvfp4 in trtllm-gen function call.](../sources/prs/flashinfer/PR-1318.md), [Make Fp8 MoE routing_bias optional](../sources/prs/flashinfer/PR-1319.md), [Add blockwise-scaled FP8 GEMM via TRTLLM-Gen.](../sources/prs/flashinfer/PR-1320.md), [Optimizations for TRTLLM MNNVL Allreduce](../sources/prs/flashinfer/PR-1321.md), [feat: Add k_scale and v_scale to persistent attention ](../sources/prs/flashinfer/PR-1322.md), [feat: Support logits_soft_cap for Persistent attn; fix kv split limit](../sources/prs/flashinfer/PR-1324.md), [feat: Fused rope fp8 quantize kernel for MLA](../sources/prs/flashinfer/PR-1339.md), [fix: fix trtllm-gen mla error on new interface](../sources/prs/flashinfer/PR-1348.md), [feature: add fp4 mm using trtllm backend](../sources/prs/flashinfer/PR-1355.md), [support trtllm-gen prefill fp4 output](../sources/prs/flashinfer/PR-1360.md), [Support scale factor start index for fp4 mha prefill/decode](../sources/prs/flashinfer/PR-1363.md), [bugfix: fixed cutlass fused moe usage of FP4QuantizationSFLayout::SWIZZLED](../sources/prs/flashinfer/PR-1371.md), [bugfix: Add guard for fp4/fp8 related include headers](../sources/prs/flashinfer/PR-1376.md), [GPT-OSS Support: Add Blackwell MoE mxfp4 implementation from TRTLLM and Attention Sink](../sources/prs/flashinfer/PR-1389.md), [gpt-oss: Add MXFP8 x MXFP4 CUTLASS MOE for SM100 and BF16 x MXFP4 CUTLASS for SM90 + SwigluBias Activation](../sources/prs/flashinfer/PR-1396.md), [feature: add cutlass as bmm_fp8 backend.](../sources/prs/flashinfer/PR-1397.md), [Fix trtllm moe launcher local_num_experts](../sources/prs/flashinfer/PR-1398.md), [fix shared memory alignment conflict in sampling.cuh](../sources/prs/flashinfer/PR-1402.md), [[bugfix] Fix compilation failure when compiling csrc/trtllm_moe_allreduce_fusion.cu](../sources/prs/flashinfer/PR-1410.md), [Fixes for Blackwell Tests](../sources/prs/flashinfer/PR-1434.md), [fix: remote redundant zero_init from trtllm-gen attn](../sources/prs/flashinfer/PR-1444.md), [Add alignment in MxFP8Quantization](../sources/prs/flashinfer/PR-1445.md), [Remove getEnvEnablePDL in favor of enable_pdl parameter](../sources/prs/flashinfer/PR-1446.md), [bugfix: Verify num_experts greater or equal to local_experts + offset](../sources/prs/flashinfer/PR-1469.md), [perf: add 1x4x1 cluster shape for fp8 bmm M<16 cases](../sources/prs/flashinfer/PR-1473.md), [tuner: Trtllm-gen Fp4 MoE Autotunner](../sources/prs/flashinfer/PR-1475.md), [perf: add fast path to TopPRenormProbKernel for top_p >= 1.0, significantly boosting SGLang workloads](../sources/prs/flashinfer/PR-1483.md), [feat: add pdl for trtllm-gen attn](../sources/prs/flashinfer/PR-1484.md), [feat: Support fp8 qkv, fp16/bf16 out MHA for trtllm-gen.](../sources/prs/flashinfer/PR-1490.md), [Perf: support scale_a/scale_b instead of combined scale in cutlass bmm_fp8](../sources/prs/flashinfer/PR-1491.md), [fix: Replace cub Max/Min with cuda::maximum/minimum for cuda 13 compatibility](../sources/prs/flashinfer/PR-1500.md), [feat: integrate xqa attention backend](../sources/prs/flashinfer/PR-1503.md), [update allreduce to match trtllm](../sources/prs/flashinfer/PR-1507.md), [Support cuda<12.8 built for trtllm_allreduce_fusion.](../sources/prs/flashinfer/PR-1508.md), [backend: Refactor trtllm-gen fmha metainfo loading](../sources/prs/flashinfer/PR-1518.md), [Fix linking errors with CUDA 13](../sources/prs/flashinfer/PR-1523.md), [Add GeGLU support to trtllm-gen NVFP4 Fused MoE Kernel](../sources/prs/flashinfer/PR-1525.md), [bugfix: Fix compile error for undefined swizzle enum.](../sources/prs/flashinfer/PR-1530.md), [bugfix: Fix Persistent kernel precision for masked output ](../sources/prs/flashinfer/PR-1533.md), [Remove cuda-python from dependency and check at runtime](../sources/prs/flashinfer/PR-1534.md), [feat: Integrate TRTLLM varlen kernel for deepseek R1 prefill ](../sources/prs/flashinfer/PR-1537.md), [fix trtllm_allreduce_fusion twoshot register problem.](../sources/prs/flashinfer/PR-1545.md), [perf: replace cudaGetDeviceProperties with cudaDeviceGetAttribute](../sources/prs/flashinfer/PR-1547.md), [perf: Enable SplitK and fix tile-scheduling for moe fp4 fused moe](../sources/prs/flashinfer/PR-1548.md), [Add mnnvl_moe_alltoallv_prepare_without_allgather](../sources/prs/flashinfer/PR-1550.md), [bugfix: fix persistent attention kernel correctness on blackwell](../sources/prs/flashinfer/PR-1559.md), [fix: separate out fp4 lib into sm90 and sm100 versions, add oob checking in fused moe](../sources/prs/flashinfer/PR-1565.md), [Backend: downgrade trtllm-gen kernel to cuda-12](../sources/prs/flashinfer/PR-1567.md), [bugfix: fix cuda version guard macros](../sources/prs/flashinfer/PR-1571.md), [update trtllm-gen fp4 autotuner and routing](../sources/prs/flashinfer/PR-1573.md), [bugfix: update trtllm-gen gemm kernel names](../sources/prs/flashinfer/PR-1577.md), [bugfix: Fix arg passing to TORCH_CHECK and TORCH_WARN macros](../sources/prs/flashinfer/PR-1582.md), [fix: semaphoress must be at the fixed range in workspace buffer on trtllm_gen attention](../sources/prs/flashinfer/PR-1584.md), [bugfix: fix fused-temperature softmax IMA issue](../sources/prs/flashinfer/PR-1596.md), [bugfix: fix the register overflow issue for topk renorm kernels on blackwell](../sources/prs/flashinfer/PR-1597.md), [feat: initial support for SM103, SM110, SM120, SM121](../sources/prs/flashinfer/PR-1608.md), [feat: cutlass fp4 gemm bringup for SM120 & SM121](../sources/prs/flashinfer/PR-1609.md), [feat: cutlass fp8 gemm bringup for SM120 & SM121](../sources/prs/flashinfer/PR-1610.md), [bugfix: fix fp4 quantization with 8x4 scale factor layout](../sources/prs/flashinfer/PR-1611.md), [bugfix: fix merge_attention_state in BatchAttention w/ gqa-group-size in Qwen family](../sources/prs/flashinfer/PR-1614.md), [perf: Fix the tactic sorting in TrtllmGenBatchedGemmRunner::getValidConfigIndices](../sources/prs/flashinfer/PR-1615.md), [bugfix: collect all modules to aot](../sources/prs/flashinfer/PR-1622.md), [bugfix: trtllm-gen fmha sm101 and sm100 compatibility](../sources/prs/flashinfer/PR-1631.md), [perf&bugfix: skip kv-tile computation out of sliding window in FA2; fix __syncthreads in mergestate](../sources/prs/flashinfer/PR-1661.md), [Refactor Blackwell unit test scripts](../sources/prs/flashinfer/PR-1667.md), [TGV GEMM as a BF16 backend alternative to cuBLAS](../sources/prs/flashinfer/PR-1668.md), [feat: Add `variant.OutputTransform()` to decode kernels](../sources/prs/flashinfer/PR-1670.md), [feat: Batch-size invariant FA2 Prefill & Decode](../sources/prs/flashinfer/PR-1675.md), [perf: improve attention of tcgen05 flash-attention](../sources/prs/flashinfer/PR-1681.md), [Update TGV GEMM default kernel and TGV code cleanup.](../sources/prs/flashinfer/PR-1682.md), [perf: Port the separate reduce kernel mode from trtllm.](../sources/prs/flashinfer/PR-1685.md), [Support Kimi-K2 for TRT: templatize number of experts](../sources/prs/flashinfer/PR-1696.md), [Fix DeepSeek quality for TRTLLM fused MoE routing](../sources/prs/flashinfer/PR-1723.md), [bugfix: partially fix tests/test_trtllm_gen_fused_moe.py unit test failure](../sources/prs/flashinfer/PR-1724.md), [TVM: support TVM binding for GroupedGemm](../sources/prs/flashinfer/PR-1725.md), [fix: put sampling kernel launch into macro](../sources/prs/flashinfer/PR-1727.md), [fix: fix cannot import name 'cuda' from 'cuda' in CUDA13](../sources/prs/flashinfer/PR-1764.md), [feat: add xqa fp8 mha and fp8 kv cache](../sources/prs/flashinfer/PR-1769.md), [Masked batch nvfp4 quantization](../sources/prs/flashinfer/PR-1774.md), [feat:enable fp8 blockscale moe for fused cultass for sm90](../sources/prs/flashinfer/PR-1819.md), [Bugfix: Fix data hazard in persistent reduce](../sources/prs/flashinfer/PR-1826.md), [feat: trtrllm-gen global scaled FP8 GEMMs](../sources/prs/flashinfer/PR-1829.md), [Update the routing for TRTLLMGEN to support kimi k2 and qwen](../sources/prs/flashinfer/PR-1831.md), [[Quantization] Add per-expert global scaling factor for fp4 batched quantize](../sources/prs/flashinfer/PR-1835.md), [Add head_dim=64 for tcgen05 tcgen05 flash-attention implementation](../sources/prs/flashinfer/PR-1850.md), [Bugfix: fix o_strides in persistent kernel ](../sources/prs/flashinfer/PR-1865.md), [Tune kernel compilation parameters for https://github.com/flashinfer-ai/flashinfer/pull/1850 ](../sources/prs/flashinfer/PR-1878.md), [feat: Add FP4 TRTLLM-Gen throughput MOE batched gemms](../sources/prs/flashinfer/PR-1882.md), [MLA RoPE + quantization fused kernel: shape generalization for MHA / GQA](../sources/prs/flashinfer/PR-1924.md), [Add layernorm op for inputs of mixed dtype](../sources/prs/flashinfer/PR-1926.md), [silu_and_mul nvfp4 quanization fusion rework](../sources/prs/flashinfer/PR-1927.md), [Feature: Support Relu2 activation in fused MoE](../sources/prs/flashinfer/PR-1954.md), [Update trtllm-gen fused moe routing kernel and add more kernels](../sources/prs/flashinfer/PR-1955.md), [Fix: Verify scales are not None for Cutlass FP8 FusedMoE](../sources/prs/flashinfer/PR-1961.md), [feat: enable deepgemm jit for fp8 block-scale on SM90](../sources/prs/flashinfer/PR-1969.md), [feat: autotune tile_tokens_dim in trtllm-gen MOE](../sources/prs/flashinfer/PR-1980.md), [fix: correct PDL parameter handling in RopeQuantize kernel](../sources/prs/flashinfer/PR-1982.md), [minor fix for xqa](../sources/prs/flashinfer/PR-1994.md), [Bugfix: Change get() -> GetDLTensorPtr() in cutlass FusedMoE validations](../sources/prs/flashinfer/PR-1995.md), [feat: add xqa backend and completes NHD/HND coverage for trtllm-gen/xqa backend](../sources/prs/flashinfer/PR-2001.md), [Feature: Support non-gated activation in cutlass fused MoE nvfp4](../sources/prs/flashinfer/PR-2011.md), [[feat] Refactor trtllmgen MOE and add Bf16 trtllmgen moe](../sources/prs/flashinfer/PR-2014.md), [[DSV3] Optimized Router Gemm](../sources/prs/flashinfer/PR-2019.md), [update trtllm cutlass moe ](../sources/prs/flashinfer/PR-2020.md), [perf: Speed up fp4 quantization for small batch with swizzling for cutlass MoE](../sources/prs/flashinfer/PR-2025.md), [[NVIDIA] Thor & Spark Support](../sources/prs/flashinfer/PR-2028.md), [Enable renormalize(naive) routing for fp8 per-tensor](../sources/prs/flashinfer/PR-2030.md), [use scalar for kv_scale in xqa](../sources/prs/flashinfer/PR-2033.md), [feat: Add flashinfer.rope.rope_quantize_fp8_append_paged_kv_cache (fused RoPE + Q + KV cache, supports MLA/GQA/MHA) ](../sources/prs/flashinfer/PR-2037.md), [perf: improve sampling/mask/softmax performance (part 1/2)](../sources/prs/flashinfer/PR-2044.md), [Rebase FP8 SM100 Cutlass FMHA Attention to main (original PR#1238)](../sources/prs/flashinfer/PR-2047.md), [Fix dtype of output scales from mnnvl_moe_alltoallv_prepare_without_allgather](../sources/prs/flashinfer/PR-2048.md), [[BUG] Fix trtllm-gen fp4 moe renormalize routing](../sources/prs/flashinfer/PR-2049.md), [Add support for topkPacked input in block-level renormalize](../sources/prs/flashinfer/PR-2051.md), [feat: add xqa mla backend](../sources/prs/flashinfer/PR-2053.md), [perf: Optimize helper max/minmax function in sampling.cuh](../sources/prs/flashinfer/PR-2058.md), [Fix moe fp8 failure for sm121](../sources/prs/flashinfer/PR-2061.md), [Fix: several bugs/issues with trtllm-gen attention kernels. ](../sources/prs/flashinfer/PR-2062.md), [perf: TRT-LLM MoE Block-FP8 activation optimization](../sources/prs/flashinfer/PR-2063.md), [feat: BF16 GEMM using CUTLASS backend for SM100](../sources/prs/flashinfer/PR-2070.md), [[Feature] Support batch prefill for POD Attention](../sources/prs/flashinfer/PR-2079.md), [enable xqa fp8 output](../sources/prs/flashinfer/PR-2081.md), [[API change] Allow using torch.Tensor for scales for trtllm-gen attention](../sources/prs/flashinfer/PR-2084.md), [refactor: pass hopper deepgemm include directory through python](../sources/prs/flashinfer/PR-2090.md), [perf: TRT-LLM Gen finalize kernel optimization](../sources/prs/flashinfer/PR-2092.md), [perf: enable pdl for cutlass fp4 gemm](../sources/prs/flashinfer/PR-2095.md), [Port TRT-LLM communication kernels to flashinfer](../sources/prs/flashinfer/PR-2102.md), [enable xqa speculative decoding](../sources/prs/flashinfer/PR-2105.md), [feat: support more head dim in RoPE kernel](../sources/prs/flashinfer/PR-2109.md), [add tensor scale input for xqa](../sources/prs/flashinfer/PR-2110.md), [refactor: update fa3 codebase and fix hopper unittest [part 1]](../sources/prs/flashinfer/PR-2111.md), [feature: make the LSE returned by MLA support base 2 or e #2113](../sources/prs/flashinfer/PR-2114.md), [update xqa license](../sources/prs/flashinfer/PR-2117.md), [Refactor trtllm_mnnvl_allreduce](../sources/prs/flashinfer/PR-2118.md), [perf: bunch of features and optimizations for top-k (sampling + sparse attention)](../sources/prs/flashinfer/PR-2119.md), [feat: support variable sequence length in decode kernel of trtllm-gen attention](../sources/prs/flashinfer/PR-2125.md), [fix flaky xqa test](../sources/prs/flashinfer/PR-2126.md), [make DeepGEMM swapAB available for linear gemm SM90](../sources/prs/flashinfer/PR-2131.md), [feat: add trtllm-gen per-tensor sparseMla kernels.](../sources/prs/flashinfer/PR-2138.md), [feat: TRTLLM FMHAv2 backend for ctx attention](../sources/prs/flashinfer/PR-2142.md), [fix xqa mha_sm90.cu](../sources/prs/flashinfer/PR-2157.md), [feat: MxInt4 x Bf16 TRT-LLM Gen MoE support](../sources/prs/flashinfer/PR-2159.md), [Add data type check for deepseek fp4 moe](../sources/prs/flashinfer/PR-2165.md), [Fix for moe on sm110](../sources/prs/flashinfer/PR-2190.md), [feat: unit-test and api change, w4a8 grouped-gemm fused MoE for SM90](../sources/prs/flashinfer/PR-2193.md), [Move the run function definition out of BatchedGemmInterface](../sources/prs/flashinfer/PR-2211.md), [feat: further optimize top-k and add fused top-k page construction kernels for DSA](../sources/prs/flashinfer/PR-2215.md), [feat: Support unpadded output hidden size for trtllm_fp4_block_scale_moe](../sources/prs/flashinfer/PR-2217.md), [fix: add DeepSeek routing for Bf16xBf16 and MxIntxBf16 TRT-LLM Gen MoE](../sources/prs/flashinfer/PR-2234.md), [refactor: pull trtllm-gen batch-gemm/gemm headers from artifactory; update tma descriptor shape init](../sources/prs/flashinfer/PR-2235.md), [[feat] Integrate SGLang concat_mla_k kernel into flashinfer](../sources/prs/flashinfer/PR-2237.md), [feat: RMSNorm/Fused RMSNorm + FP8 Quantization kernels](../sources/prs/flashinfer/PR-2243.md), [Remove cudaStreamSynchronize from gemm_groupwise_sm120.cuh for CUDA graph compatibility](../sources/prs/flashinfer/PR-2244.md), [feat: Support numLocalTokens=0 for moe All-to-all](../sources/prs/flashinfer/PR-2247.md), [feat: support non-contiguous query for trtllm-gen attention backend](../sources/prs/flashinfer/PR-2254.md), [fix: support int64 IdType for RoPE part argument in `rope_quantize_fp8_append_paged_kv_cache`](../sources/prs/flashinfer/PR-2255.md), [[TRTLLM-Gen Fmha] add optimized trtllm-gen decode kernels for high throughput + speculative decoding](../sources/prs/flashinfer/PR-2265.md), [[performance]optimize for nvfp4](../sources/prs/flashinfer/PR-2268.md), [feat: add GDN Attention](../sources/prs/flashinfer/PR-2276.md), [feat: IdType indices in sampling kernels](../sources/prs/flashinfer/PR-2281.md), [Selective State Update kernel (mamba)](../sources/prs/flashinfer/PR-2301.md), [[Perf][Feature] Add SM103-specific schedulers for NVFP4 CUTLASS kernels](../sources/prs/flashinfer/PR-2303.md), [feat: Support Fused MoE non gated Relu2 NVFP4 & FP8 and support Nemotron](../sources/prs/flashinfer/PR-2304.md), [Fix: FilteredTopKUnifiedKernel read value out of length](../sources/prs/flashinfer/PR-2308.md), [[ML3] Optimized Router Gemm](../sources/prs/flashinfer/PR-2323.md), [bugfix: fix multi-cta top-k implementation when k value is different for different row](../sources/prs/flashinfer/PR-2325.md), [[perf] Improve gemm_fp8_nt_groupwise (cutlass backend) by 10-40% for batch sizes <= 32](../sources/prs/flashinfer/PR-2327.md), [fix: guard batchWarpReduceSum with ENABLE_FP8 to fix compilation without FP8](../sources/prs/flashinfer/PR-2328.md), [feat: expose swizzled_input_sf parameter for CUTLASS fused MOE](../sources/prs/flashinfer/PR-2330.md), [Optimize quantization function in large problem size](../sources/prs/flashinfer/PR-2343.md), [Enable fp16/bf16/f32 support for selective_state_update (mamba)](../sources/prs/flashinfer/PR-2366.md), [bugfix: hotfix of PR 2366 (mamba kernel)](../sources/prs/flashinfer/PR-2378.md), [fix: ensure each CTA processes full numHeadsQPerKv for trtllm decode kernel](../sources/prs/flashinfer/PR-2380.md), [A Blackwell-optimized version of selective_state_update (decode)](../sources/prs/flashinfer/PR-2387.md), [feat: cuteDSL fp4 moe for better DSR1 performance.](../sources/prs/flashinfer/PR-2398.md), [Remove cudaMalloc/Free in GDN prefill kernel](../sources/prs/flashinfer/PR-2415.md), [feat: update trtllm-gen MoE cubins](../sources/prs/flashinfer/PR-2416.md), [refactor: reduce hopper's gdn prefill compilation time and fix docstring.](../sources/prs/flashinfer/PR-2422.md), [refactor: refactoring cuda code to cute-dsl (part 1)](../sources/prs/flashinfer/PR-2428.md), [fix: Sampling: CUDA Graph fix](../sources/prs/flashinfer/PR-2432.md), [fix: Fix NaN output in mxfp8_quantize for very small input values](../sources/prs/flashinfer/PR-2441.md), [MTP for mamba ](../sources/prs/flashinfer/PR-2444.md), [bugfix: fix stub generation directory in fused_moe module](../sources/prs/flashinfer/PR-2445.md), [feat: Add TRTLLM fmha_v2 library for SM90 attention with Skip-Softmax ](../sources/prs/flashinfer/PR-2446.md), [fix: fix illegal memory access for NaN input in sampling kernels](../sources/prs/flashinfer/PR-2456.md), [feat: Support Fused MoE non gated Relu2 NVFP4 & FP8 and support Nemotron, fixed](../sources/prs/flashinfer/PR-2462.md), [feat: Add MXFP8 GEMM mm_mxfp8 (cutlass)](../sources/prs/flashinfer/PR-2464.md), [feat: Add TRTLLM-Gen Skip-Softmax kernels for prefill and decode](../sources/prs/flashinfer/PR-2477.md), [fix: add support check for gemm config for cutlass moe](../sources/prs/flashinfer/PR-2495.md), [Feat: Trtllm-gen MxFP8 MoE integration](../sources/prs/flashinfer/PR-2505.md), [perf: cache cudaGetDeviceProperties in gdn_prefill to avoid per-call overhead](../sources/prs/flashinfer/PR-2509.md), [Support NVFP4 KV cache decode on SM120](../sources/prs/flashinfer/PR-2520.md), [misc: point triton blackwell-ptxas to local cuda ptxas](../sources/prs/flashinfer/PR-2543.md), [feat: Enable TRTLLM-Gen Skip-Softmax attention for MLA](../sources/prs/flashinfer/PR-2547.md), [[Bugfix][comm] Fix FP4 one-shot launch config instability in trtllm_allreduce_fusion](../sources/prs/flashinfer/PR-2557.md), [Add support for the combinations of allreduce, allgather, and reducescatter](../sources/prs/flashinfer/PR-2563.md), [fix: W4A8 autotune crash in cutlass_fused_moe profiler workspace](../sources/prs/flashinfer/PR-2564.md), [Implement `cutlass_fused_moe` mxfp8](../sources/prs/flashinfer/PR-2581.md), [feat: trtllm tinygemm2 in flashinfer as bf16 routergemm](../sources/prs/flashinfer/PR-2587.md), [Mamba SSU: better automatic kernel selection + algorithm selection optionally exposed to the user.](../sources/prs/flashinfer/PR-2591.md), [[bugfix] Fix FilteredTopK overflow correctness](../sources/prs/flashinfer/PR-2605.md), [fix: cute dsl nvfp4 moe routing index error](../sources/prs/flashinfer/PR-2629.md), [fix: add SM121 support to SM120 version guards](../sources/prs/flashinfer/PR-2631.md), [[fp8_blockwise]Fix int32 overflow in TRTLLM fused MoE activation kernel](../sources/prs/flashinfer/PR-2642.md), [feat: FP32 dtype output for BF16 matmuls (CUTLASS & cuDNN)](../sources/prs/flashinfer/PR-2644.md), [int16 Block-Scaled State and Stochastic Rounding for SSU (mamba)](../sources/prs/flashinfer/PR-2645.md), [[feat] trtllm-gen mxfp8 gemm](../sources/prs/flashinfer/PR-2653.md), [feat: implement deterministic topk](../sources/prs/flashinfer/PR-2661.md), [perf: Update trtllm-gen batched GEMM kernels - faster, more NVFP4 tile dims, MXFP8 with relu2 act](../sources/prs/flashinfer/PR-2667.md), [fix: reduce smem allocation for tinygemm2 kernel in SM120](../sources/prs/flashinfer/PR-2670.md), [feat: add support for more MLA head dimensions](../sources/prs/flashinfer/PR-2677.md), [Add varlen and speculative decoding support to selective state update](../sources/prs/flashinfer/PR-2700.md), [Add NVFP4 KV cache quantization support for SM100](../sources/prs/flashinfer/PR-2702.md), [feat: Add support for TRTLLM MXFP8 non-gated MoE with ReLU2](../sources/prs/flashinfer/PR-2707.md), [Mamba2 SSD Combined Forward Pass (Blackwell CuTe DSL Kernel)](../sources/prs/flashinfer/PR-2709.md), [feat: Add DiT-oriented kernels where Qk (Bmm1) type can be reinterpreted into Int8 or BFloat16](../sources/prs/flashinfer/PR-2711.md), [fix: Add SM120 (RTX Blackwell desktop) support for NVFP4 MoE kernels](../sources/prs/flashinfer/PR-2725.md), [Support for MXFP4 and NVFP4 group GEMMs on GeForce and Spark](../sources/prs/flashinfer/PR-2738.md), [misc: Update gemm/batched gemm cubins from trtllm-gen, gemm header refactor](../sources/prs/flashinfer/PR-2740.md), [[feat] Add 2048 experts and 32 Top K ](../sources/prs/flashinfer/PR-2744.md), [[feat] Add air top-p algorithm](../sources/prs/flashinfer/PR-2752.md), [feat: Add FP4 KV cache quant/dequant kernels ](../sources/prs/flashinfer/PR-2757.md), [feat: Expose TRT-LLM FMHA style paged KV Cache and page table layout](../sources/prs/flashinfer/PR-2770.md), [feat: FP8 output support for CUTLASS MLA paged attention](../sources/prs/flashinfer/PR-2779.md), [feat: Support padding tokens with seqlen=0 for rope+quant+kv cache update fusion kernel](../sources/prs/flashinfer/PR-2792.md), [Upgrade cutlass 4.2.1 -> 4.4.2](../sources/prs/flashinfer/PR-2798.md), [[fmha-v2] Support HND and NHD paged KV cache layouts with conditional stride handling](../sources/prs/flashinfer/PR-2799.md), [fix: Autotuner _find_nearest_profile non-power-of-2 num_tokens, create launchers for all supported tileN in trtllm fused MoE](../sources/prs/flashinfer/PR-2821.md), [[Fmha] Sparse MLA decode kernel selection heuristics](../sources/prs/flashinfer/PR-2836.md), [[Perf] Add FMHAv2 to flashinfer_benchmark.py and eliminate unnecessary H2D](../sources/prs/flashinfer/PR-2841.md), [read real strides for kv and block scale](../sources/prs/flashinfer/PR-2844.md), [fix: int32 overflow in `trtllm_fp4_block_scale_moe` causing "Unsupported hidden state scale shape" for EP32+ configs](../sources/prs/flashinfer/PR-2853.md), [Add support for Relu2 in BF16 fused MoE](../sources/prs/flashinfer/PR-2864.md), [Mamba SSU: horizontal MTP kernel (+ DSTATE=96 support)](../sources/prs/flashinfer/PR-2865.md), [Fix silent bug with FP8 per tensor non-gated MoE](../sources/prs/flashinfer/PR-2882.md), [fix: snap weight_scale_vec_size to handle block_scale_interleave padding for SM120](../sources/prs/flashinfer/PR-2898.md), [feat: add MXFP8 GEMM support for SM120](../sources/prs/flashinfer/PR-2902.md), [feat(gdn): state checkpointing in chunk_gated_delta_rule](../sources/prs/flashinfer/PR-2908.md), [feat: Add cuBLASLt backend for `mm_bf16` and enable multi-tactic autotuning for FP8/MXFP8 runners](../sources/prs/flashinfer/PR-2914.md), [feat: add Relu2 (squared ReLU) activation support in CUTLASS MoE backend](../sources/prs/flashinfer/PR-2926.md), [fix: use float instead of double in sampling binary search to avoid FP64 bottleneck on SM103](../sources/prs/flashinfer/PR-2945.md), [Update NVSHMEM interface to use NVSHMEM4Py instead of custom bindings](../sources/prs/flashinfer/PR-2960.md), [Improved `simple` mamba SSU kernel ](../sources/prs/flashinfer/PR-2962.md), [test: xfail cuDNN FP8 prefill on Blackwell with CUDA <= 12.9](../sources/prs/flashinfer/PR-2963.md), [Add flashinfer.fused_rmsnorm_silu() with native kernel backend](../sources/prs/flashinfer/PR-2965.md), [Fused moe all-reduce routed scaling factor + quant support](../sources/prs/flashinfer/PR-2966.md), [fix: restore SM120 CUTLASS MoE tile candidate removed by #2927 (test_trtllm_cutlass_fused_moe.py)](../sources/prs/flashinfer/PR-2984.md), [[Fmha] support nvfp4 output keepsMmaAb generation kernels](../sources/prs/flashinfer/PR-2988.md), [fix: tinygemm2 hang issue due to barrier sync](../sources/prs/flashinfer/PR-2996.md), [perf: Optimize CUTLASS MoE helper kernels for small-batch decode workloads](../sources/prs/flashinfer/PR-3014.md), [fix: extend moe alltoall top-k specializations](../sources/prs/flashinfer/PR-3021.md), [[feat] Add routing_replay_out support to MoE kernels and Python API](../sources/prs/flashinfer/PR-3024.md), [perf: Port TRT-LLM SM120/SM121 FP4 CUTLASS GEMM optimizations. Add PDL](../sources/prs/flashinfer/PR-3026.md), [[feat] Trtllm-gen Per-token Nvfp4 MoE](../sources/prs/flashinfer/PR-3027.md), [fused_moe: pre-filter SM89 tactics with zero occupancy on SM120 Blackwell (fix review feedback on #2764)](../sources/prs/flashinfer/PR-3032.md), [Support lse in trtllm paged attn kernels](../sources/prs/flashinfer/PR-3058.md), [Support Allreduce + Norm + Per-token Group Fp8 Quant Fusion](../sources/prs/flashinfer/PR-3059.md), [Support NVFP4 KV for prefill and batch attention kernels](../sources/prs/flashinfer/PR-3097.md), [feat: Enable FP8 (E4M3/E5M2) in concat_mla_k for optimize long-context prefill performance and refactor type dispatch for BF16/FP16](../sources/prs/flashinfer/PR-3129.md), [perf: Add no-bias path for tinygemm_bf16](../sources/prs/flashinfer/PR-3151.md), [Integrate CUTLASS Small Tile N Blockscaled GEMMs/Grouped GEMMs for SM120 and SM121](../sources/prs/flashinfer/PR-3152.md), [feat: DiT layer norm fusions for WAN: flashinfer.diffusion_ops](../sources/prs/flashinfer/PR-3157.md), [feat: enable glm5 router gemm](../sources/prs/flashinfer/PR-3185.md), [[Bugfix] Fix fused MoE autotuning correctness issues by filtering clusterDimZ](../sources/prs/flashinfer/PR-3227.md), [perf: optimize per-token nvfp4 quantization kernel.](../sources/prs/flashinfer/PR-3237.md), [Update moe gemm](../sources/prs/flashinfer/PR-3239.md), [Add dynamic tokens-per-page TRTLLM-GEN GQA kernels](../sources/prs/flashinfer/PR-3259.md), [Update trtllm FMHA cubins](../sources/prs/flashinfer/PR-3317.md), [[feat] Add gemma RMS AR fusion](../sources/prs/flashinfer/PR-3322.md), [checkpointing_ssu kernel: fused replay + conditional state-write for Mamba2](../sources/prs/flashinfer/PR-3324.md), [feat(cute_dsl/moe): add `moe_output_memset_inplace` dense memset wrapper](../sources/prs/flashinfer/PR-3328.md), [perf: fix the iteration bound of SWA in FA2 prefill template](../sources/prs/flashinfer/PR-714.md), [bugfix: FusedAddRMSNorm kernels might require more than 48KB shared memory when d is large.](../sources/prs/flashinfer/PR-718.md), [Align KV chunk size binary search with actual KV chunk splitting.](../sources/prs/flashinfer/PR-728.md), [Change `apply_rope_with_cos_sin_cache` to accept `cos_sin_cache`](../sources/prs/flashinfer/PR-754.md), [feat: support deepseek prefill attention shape](../sources/prs/flashinfer/PR-765.md), [bugfix: Ensure Loop Termination by Enforcing IEEE-754 Compliance in Sampling Kernels](../sources/prs/flashinfer/PR-774.md), [perf: refactor fa2 prefill template](../sources/prs/flashinfer/PR-776.md), [bugfix: drop CTA_TILE_Q=32](../sources/prs/flashinfer/PR-785.md), [bugfix: MLA decode should multiply sm_scale by math::log2e](../sources/prs/flashinfer/PR-787.md), [fix rope logic in mla decoding](../sources/prs/flashinfer/PR-793.md), [feat: support f32 attention output in FA2 template](../sources/prs/flashinfer/PR-799.md), [feat: apply sm_scale at logits instead of q in FA2 template](../sources/prs/flashinfer/PR-801.md), [perf: memory efficient deepseek mla fused page-attention kernel](../sources/prs/flashinfer/PR-804.md), [bugfix: mla page-attention kernel for different page sizes](../sources/prs/flashinfer/PR-810.md), [feat: unlocking MLA for A100](../sources/prs/flashinfer/PR-812.md), [feat: unlock MLA attention for sm89 (L40/L40s/4090)](../sources/prs/flashinfer/PR-814.md), [bugfix: bugfix on sm89 MLA](../sources/prs/flashinfer/PR-821.md), [bugfix: fix the signature of `CutlassSegmentGEMMSM90`](../sources/prs/flashinfer/PR-827.md), [perf: MLA decode kernel implemented by CuTe targeted to SM80](../sources/prs/flashinfer/PR-844.md), [misc: Remove duplicate param set in MLA kernel](../sources/prs/flashinfer/PR-850.md), [Add POD-Attention to FlashInfer](../sources/prs/flashinfer/PR-858.md), [perf: dynamic split-k for MLA](../sources/prs/flashinfer/PR-863.md), [bugfix: fix the behavior of MLA kernel when kv-length is 0](../sources/prs/flashinfer/PR-868.md), [Naive Support for Hopper FP8 Prefill Kernel with Per-Head Quantization](../sources/prs/flashinfer/PR-869.md), [perf: FlashAttention-3 style MLA PageAttention](../sources/prs/flashinfer/PR-887.md), [feat - support mla kvcache store](../sources/prs/flashinfer/PR-888.md), [perf: fix MLA split-k performance bug](../sources/prs/flashinfer/PR-898.md), [perf: tweak the pipeline design of mla kernel](../sources/prs/flashinfer/PR-901.md), [feat: flashinfer intra-kernel profiler](../sources/prs/flashinfer/PR-913.md), [feat: experimenta support of PDL](../sources/prs/flashinfer/PR-930.md), [bugfix: fix potential issues of FA3 template loading nans for PageAttention](../sources/prs/flashinfer/PR-945.md), [perf: Use 2WG pipeline design for MLA implementation on Hopper](../sources/prs/flashinfer/PR-952.md), [[TVM] Added tvm binding for sampling kernel](../sources/prs/flashinfer/PR-958.md), [perf: Fix python API overhead when CUDAGraph is not enabled](../sources/prs/flashinfer/PR-969.md), [perf: dual pivot top-p/top-k renorm](../sources/prs/flashinfer/PR-974.md), [perf: prefetch page indices for mla kernel](../sources/prs/flashinfer/PR-991.md), [feat: SM-constraint Communication Kernels](../sources/prs/flashinfer/PR-994.md), [3rdparty: upgrade cutlass to 3.9](../sources/prs/flashinfer/PR-997.md), [ROCm SDPA: Ensure attn_mask has the same dtype with q](../sources/prs/pytorch/PR-144398.md), [Add release branch push triggers to inductor-rocm-mi300.yml](../sources/prs/pytorch/PR-149871.md), [[CUDA][avgpool2d] Fix backward launch bounds again for sm100, sm120](../sources/prs/pytorch/PR-150640.md), [[CUDA][avgpool2d] Fix backward launch bounds again for `sm100`, `sm120`](../sources/prs/pytorch/PR-150676.md), [[CUDA] Only use vec128 if CUDA version is newer than 12.8](../sources/prs/pytorch/PR-150705.md), [[ATen][CUDA] Optimize 128 bit vectorization](../sources/prs/pytorch/PR-152967.md), [Fix uint view copy (#151598)](../sources/prs/pytorch/PR-154121.md), [Fix macOS build with `USE_MPS=OFF`](../sources/prs/pytorch/PR-156932.md), [[user triton] AOT inductor support for device-side TMA](../sources/prs/pytorch/PR-157241.md), [[PowerPC] Fixed build issue for vsx vec256 complexfloat and scaled_mm_out_cpu ](../sources/prs/pytorch/PR-157422.md), [[cherry-pick][inductor][triton] Update HAS_WARP_SPEC to check triton.Config params. Update Triton Hash to top of release/3.4.x stack](../sources/prs/pytorch/PR-158646.md), [CUDA 13.0 Windows Nvidia Driver Update to 580.88](../sources/prs/pytorch/PR-162501.md), [fix cpp extension distributed warning spew](../sources/prs/pytorch/PR-162764.md), [[Graph Partition] improve custom op output alias](../sources/prs/pytorch/PR-163380.md), [[graph partition] Add way to register custom rule (#163310)](../sources/prs/pytorch/PR-163395.md), [CUDA 13.0 Warning update for supported architectures](../sources/prs/pytorch/PR-163633.md), [[Cherry-Pick] [CD] CUDA 13 specific followup changes. Remove sm50-70 From CUDA 12.6 and CUDA 12.8 builds (#162455)](../sources/prs/pytorch/PR-163764.md), [[CD] CUDA 13.0 fix preload logic to include nvidia/cu13/lib/](../sources/prs/pytorch/PR-163766.md), [Move inductor jobs 3.9->3.10](../sources/prs/pytorch/PR-163954.md), [[cuDNN][SDPA] Disable dropout for cuDNN SDPA on 9.11 - 9.13](../sources/prs/pytorch/PR-164026.md), [[AARCH64][CD][CUDA13][Triton][PTXAS] Turn on BUILD_BUNDLE_PTXAS=1 ](../sources/prs/pytorch/PR-164236.md), [CUDA 13.0 builds fix on Amazon Linux 2023](../sources/prs/pytorch/PR-164893.md), [[Graph Partition] move custom rules to inductor config (#166458)](../sources/prs/pytorch/PR-166967.md), [[Graph Partition] fix graph partition input signature for fallback kernels](../sources/prs/pytorch/PR-166985.md), [[cuDNN][SDPA][Convolution] Expose cuDNN runtime version in CUDA hooks](../sources/prs/pytorch/PR-167327.md), [[ROCm] Enable shared memory based pruning for Triton configs](../sources/prs/pytorch/PR-170190.md), [[inductor] Fix cudagraph skip for index_put_ with boolean indices, gr…](../sources/prs/pytorch/PR-170884.md), [[ROCm] Make grouped GEMM CK opt‑in via env and default to fallback path](../sources/prs/pytorch/PR-171140.md), [[cherry-pick][CUDA] Upgrade cuDNN to 9.15.1 for CUDA 13 builds ](../sources/prs/pytorch/PR-171189.md), [[cherry-pick][cuDNN][SDPA] cuDNN SDPA off-by-default for cuDNN versions < 12.9 (#171627)](../sources/prs/pytorch/PR-171895.md), [Skip modded_nanogpt model in TorchInductor benchmark](../sources/prs/pytorch/PR-172141.md), [[Graph Partition] Improve support for mutation ops](../sources/prs/pytorch/PR-172577.md), [Update inductor expected accuracy files](../sources/prs/pytorch/PR-175096.md), [[benchmark] Skip pytorch_CycleGAN_and_pix2pix from inductor benchmarks](../sources/prs/pytorch/PR-175299.md), [[release-only] Remove +ptx from cuda 13.0 builds](../sources/prs/pytorch/PR-175567.md), [[release 2.12] Apply Release only changes to 2.12 branch](../sources/prs/pytorch/PR-180470.md), [Disable kernel cutlass_mla_decode on SM103](../sources/prs/sglang/PR-10058.md), [Optimize nvfp4 block scaled gemm kernel when M is small.](../sources/prs/sglang/PR-10101.md), [fix: resolve gb200 image link](../sources/prs/sglang/PR-10343.md), [Fix correction bias undefined behavior for nvfp4 models](../sources/prs/sglang/PR-10426.md), [Update CUTLASS. Refine KernelSchedule for fp8 (grouped) gemm.](../sources/prs/sglang/PR-10491.md), [[sgl-kernel] Optimize concat_mla_k kernel](../sources/prs/sglang/PR-10543.md), [Optimize cutlass int8 gemm kernel for large M on SM89 Ada GPU](../sources/prs/sglang/PR-10714.md), [disable sm100 for FlashMLA and fast-hadamard-transform in cuda12.6.1](../sources/prs/sglang/PR-11274.md), [[NVIDIA] Add new SMs support for Spark & Thor](../sources/prs/sglang/PR-11287.md), [[sgl-kernel][1/N]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-11432.md), [Improve Kernel Build Time](../sources/prs/sglang/PR-11508.md), [support cutlass fp4 kernel in sm120](../sources/prs/sglang/PR-11737.md), [Fixed aarch64 flash-mla](../sources/prs/sglang/PR-12009.md), [(1/n)support context parallel with deepseekv3.2-DSA](../sources/prs/sglang/PR-12065.md), [[sgl-kernel][4/N]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-12080.md), [[sgl-kernel] clean up fa fetch in CMakeLists.txt](../sources/prs/sglang/PR-12392.md), [[Fix] `concat_mla_absorb_q_kernel` fails for long inputs](../sources/prs/sglang/PR-12453.md), [[NVIDIA] Fix CUDA arch requirement in nvfp4 cast](../sources/prs/sglang/PR-12581.md), [Support moe topk sigmoid kernel](../sources/prs/sglang/PR-13049.md), [diffusion: enable fa4 for blackwell](../sources/prs/sglang/PR-13263.md), [[sgl-kernel][Feat][B200][1/N]Support MXFP8 Grouped GEMM in Blackwell](../sources/prs/sglang/PR-13731.md), [[kernel][moe] add moe topk fast](../sources/prs/sglang/PR-13969.md), [Add new moe wna16 marlin gemm](../sources/prs/sglang/PR-14122.md), [Opt moe align block size kernel](../sources/prs/sglang/PR-14133.md), [sync attention, deepseek doc](../sources/prs/sglang/PR-14335.md), [[CPU] Implement MXFP4 Gemm kernels for intel AMX to support GPT OSS series.](../sources/prs/sglang/PR-14385.md), [Add CUDA kernel size analysis tool for sgl-kernel optimization](../sources/prs/sglang/PR-14544.md), [[sgl-kernel][Feat][B200][2/N] Support MXFP8 Grouped GEMM in Blackwell](../sources/prs/sglang/PR-14640.md), [[NPU][eagle3] support qwen eagle3 on NPU](../sources/prs/sglang/PR-14820.md), [[sgl-kernel][1/2] Fused qk_norm_rope for GLM4.6](../sources/prs/sglang/PR-15141.md), [Add cache for flashinfer installation](../sources/prs/sglang/PR-15153.md), [[sgl-kernel] Update flashmla to include fp8 sparse_mla optimizations](../sources/prs/sglang/PR-15242.md), [Fix warp illegal instruction in kimi k2 thinking PCG](../sources/prs/sglang/PR-15306.md), [[diffusion] Add Sage Attention 3 Support for sm 120 (RTX5090)](../sources/prs/sglang/PR-15382.md), [[sgl-kernel][6/7]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-15471.md), [[Feature] JIT Fused QK norm + qk norm clean up](../sources/prs/sglang/PR-15835.md), [[JIT kernel] Apply jit per_tensor_quant_fp8 kernel](../sources/prs/sglang/PR-15836.md), [[Feature] add aligned_vector type for JIT kernel](../sources/prs/sglang/PR-16162.md), [Fix sgl-kernel jobs to skip when target_stage is specified](../sources/prs/sglang/PR-16308.md), [[diffusion] Fix RuntimeError in SageAttention3 on Nvidia Blackwell with Qwen-Image](../sources/prs/sglang/PR-16335.md), [[Fix]Pin mooncake version to 0.3.7.post2 in grace blackwell](../sources/prs/sglang/PR-16502.md), [[DeepSeek v3.2] Opt MTP decode cuda batch sizes and nsa implementation](../sources/prs/sglang/PR-16961.md), [Kernel: optimize decoding metadata in NSA multi-spec backend with fused kernels](../sources/prs/sglang/PR-17554.md), [Make flashMLA work on: Cu13, B300](../sources/prs/sglang/PR-17600.md), [Feature/support longcat flash lite](../sources/prs/sglang/PR-17838.md), [[Move sgl-kernel Kernel to JIT] Add JIT concat MLA kernels](../sources/prs/sglang/PR-17889.md), [[Diffsuion & JIT_kernel] QKNorm cross heads kernel](../sources/prs/sglang/PR-18073.md), [[Hicache & JIT_kernel] Support page first layout & mla jit kernel](../sources/prs/sglang/PR-18311.md), [docs: expand and update modelopt documentation](../sources/prs/sglang/PR-18479.md), [[FIX] Correct JIT kernel compilation on newer GPUs with outdated driver metadata.](../sources/prs/sglang/PR-18496.md), [Add claude skills for sgl-kernel and jit-kernel](../sources/prs/sglang/PR-18855.md), [Use single mma warp group for short q_len in FA to optimize decoding performance](../sources/prs/sglang/PR-18985.md), [[jit_kernel] Add fused_qknorm_rope JIT kernel](../sources/prs/sglang/PR-19059.md), [[DeepSeek-V3.2][JIT-kernel] Support nsa fuse store indexer k cache](../sources/prs/sglang/PR-19148.md), [[Kernel Slimming] Migrate NVFP4 kernels to JIT](../sources/prs/sglang/PR-19437.md), [[Feature] NVFP4 Marlin fallback for non-Blackwell GPUs (SM75+)](../sources/prs/sglang/PR-19652.md), [Add compile-time 256-bit vector guard for pre-Blackwell](../sources/prs/sglang/PR-19794.md), [[JIT Kernel][Feature] Support JIT custom all reduce (rewrite as v2)](../sources/prs/sglang/PR-19880.md), [[JIT Kernel] Reland NVFP4 kernels to JIT](../sources/prs/sglang/PR-20012.md), [[diffusion] Support nvfp4 for Flux.2](../sources/prs/sglang/PR-20137.md), [fix ci by removing nvidia-cutlass-dsl-libs-base and force reinstall n…](../sources/prs/sglang/PR-20380.md), [fix(docs): correct quantization documentation (#20301)](../sources/prs/sglang/PR-20619.md), [Fix(jit): support rmsnorm for hidden_size in {64, 128, 256}](../sources/prs/sglang/PR-20661.md), [[Feature][JIT Kernel] Fused TP QK norm For Minimax](../sources/prs/sglang/PR-20673.md), [CUTLASS FP8 Blockwise GEMM improvement of SM120](../sources/prs/sglang/PR-20887.md), [Add SGLang CUDA crash API logging inspired by FlashInfer](../sources/prs/sglang/PR-20910.md), [[Tiny Fix] Fix IS_BLACKWELL env var empty string warning in rerun-ut workflow](../sources/prs/sglang/PR-20957.md), [ci: run Stage A CUDA tests as stage-a-test-small-1-gpu on 5090](../sources/prs/sglang/PR-20988.md), [[Whisper] Enable CUDA graph support and timestamp for whisper model](../sources/prs/sglang/PR-21190.md), [Split pr-test.yml: extract sgl-kernel, jit-kernel, and multimodal-gen tests into separate workflow files](../sources/prs/sglang/PR-21219.md), [CUTLASS NVFP4 GEMM improvement of SM120](../sources/prs/sglang/PR-21314.md), [[Diffusion] Add qknorm rope fuse kernel](../sources/prs/sglang/PR-21440.md), [fix: piecewise_cuda_graph get correct qo_indptr](../sources/prs/sglang/PR-21452.md), [fix: TRT-LLM MHA CUDA illegal address with EAGLE v2 + DP attention](../sources/prs/sglang/PR-21649.md), [[jit_kernel] Optimize fused_qknorm_rope: deduplicate sincosf for interleave RoPE ](../sources/prs/sglang/PR-21654.md), [Remove flashinfer wheel cache cleanup that deletes other versions](../sources/prs/sglang/PR-21711.md), [[Feature] JIT rmsnorm update (with claude)](../sources/prs/sglang/PR-21834.md), [[Bugfix] Fix CUDA graph replay issues in trtllm_mla draft_extend](../sources/prs/sglang/PR-21987.md), [[Diffusion] Fix weight scale swizzle and add large-M kernel config for FLUX.2-dev-NVFP4](../sources/prs/sglang/PR-22064.md), [[diffusion] Default NVFP4 to CUTLASS and add all-model shape benchmarks](../sources/prs/sglang/PR-22091.md), [Lazy import flash_attention_v4 to avoid loading flash_attn.cute at startup](../sources/prs/sglang/PR-22306.md), [[Docker] Fix Trivy CVEs, cubin download 403s, and kernels command order](../sources/prs/sglang/PR-22322.md), [[CI/Docker] Clean up redundant flashinfer cubin downloads](../sources/prs/sglang/PR-22491.md), [[Docker] Remove flashinfer cache copy](../sources/prs/sglang/PR-22653.md), [reland [Diffusion] Add FLUX.1-dev ModelOpt NVFP4 support](../sources/prs/sglang/PR-22672.md), [[Diffusion] Add Wan2.2 ModelOpt NVFP4 support](../sources/prs/sglang/PR-22681.md), [[Fix] Fix accuracy bug in Flashmla sparse MLA kernel](../sources/prs/sglang/PR-22723.md), [Dual MoE CUDA graph capture for lora/nolora batches](../sources/prs/sglang/PR-22809.md), [[Fix/Kernel] Add JIT rmsnorm_hf kernel to fix transformers backend MMLU accuracy regression ](../sources/prs/sglang/PR-22931.md), [Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-23686.md), [[feat] Init true on policy with qwen_dense](../sources/prs/sglang/PR-23961.md), [Enable PDL for various kernels in DSV32/GLM5](../sources/prs/sglang/PR-23965.md), [[VLM] Optimize Gemma4 VLM with PCG and fuse RMSNorm + residual add + scalar](../sources/prs/sglang/PR-24048.md), [Refactor device timer, clean up metrics collector, and add fwd occupancy metric](../sources/prs/sglang/PR-24197.md), [[codex] Optimize hidden-size 512 RMSNorm dispatch](../sources/prs/sglang/PR-24710.md), [[rebase]Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-24986.md), [Support draft extend cuda graph for tokenspeed_mla attention backend](../sources/prs/sglang/PR-25489.md), [[fp8] SM90 swap-AB scaled_mm dispatch (~1.16x kernel geomean, +5.8-18.5% end-to-end)](../sources/prs/sglang/PR-25532.md), [fix (jit kernel): elementwise activation C++ error](../sources/prs/sglang/PR-25695.md), [[Codex] Remove stale DeepSeek V4 JIT kernels](../sources/prs/sglang/PR-25764.md), [[Refactor] Rename NSA → DSA: user-facing aliases, file/class/import rename](../sources/prs/sglang/PR-25821.md), [[Test] Stage-a sanity kits; consolidate core/ + models_e2e/ tests](../sources/prs/sglang/PR-25831.md), [Support cutlass Int8 gemm](../sources/prs/sglang/PR-2752.md), [upgrade cutlass v3.7.0](../sources/prs/sglang/PR-2967.md), [feat: add flashinfer as 3rdparty and use rmsnorm as example](../sources/prs/sglang/PR-3033.md), [Support sm90 Int8 gemm](../sources/prs/sglang/PR-3035.md), [support w8a8 fp8 kernel with CUTLASS](../sources/prs/sglang/PR-3047.md), [feat: integrate gemm_fp8 kernel into gemm](../sources/prs/sglang/PR-3056.md), [add tensorrt_llm common and cutlass_extensions as 3rdparty](../sources/prs/sglang/PR-3216.md), [support blockwise fp8 matmul kernel](../sources/prs/sglang/PR-3267.md), [fix undefined symbol cudaGetDriverEntryPointByVersion](../sources/prs/sglang/PR-3372.md), [integrate blockwise fp8 kernel](../sources/prs/sglang/PR-3529.md), [[Feature] Apply Cublas Grouped Gemm kernel](../sources/prs/sglang/PR-3629.md), [Support FP4 gemm (1/2)](../sources/prs/sglang/PR-3899.md), [Add fast decode plan for flashinfer mla](../sources/prs/sglang/PR-3987.md), [[Revision] Add fast decode plan for flashinfer mla ](../sources/prs/sglang/PR-4012.md), [Support overlapping two batches](../sources/prs/sglang/PR-4068.md), [DeepGemm integrate to gemm](../sources/prs/sglang/PR-4165.md), [Accelerate FP8 CUDA Kernel by 20-28%](../sources/prs/sglang/PR-4215.md), [fix per_token_group_quant_fp8 illegal memory when num_groups % 16 != 0](../sources/prs/sglang/PR-4231.md), [add THIRDPARTYNOTICES for DeepGEMM](../sources/prs/sglang/PR-4272.md), [Support Blackwell Block Scale FP8 Gemm](../sources/prs/sglang/PR-4278.md), [update deepgemm](../sources/prs/sglang/PR-4284.md), [Add deepseek style fused moe group gate selection kernel](../sources/prs/sglang/PR-4530.md), [[Fix] Fix raw_bs bug when using flashinfer mla and eagle](../sources/prs/sglang/PR-4557.md), [Support fp8 gemm for blackwell](../sources/prs/sglang/PR-4558.md), [avoid cudaStreamSynchronize in DeepSeekV2AttentionMLA](../sources/prs/sglang/PR-4577.md), [support cmake for sgl-kernel](../sources/prs/sglang/PR-4706.md), [Support (1 <= dp < tp) in the dp attention in DeepEP](../sources/prs/sglang/PR-4770.md), [[Feat] support deepgemm for cmake](../sources/prs/sglang/PR-4864.md), [[Build] Fix cuda12.8 build error in nvfp4_scaled_mm_kernels.cu](../sources/prs/sglang/PR-4953.md), [update cutlass tag](../sources/prs/sglang/PR-5011.md), [fix deepgemm as well](../sources/prs/sglang/PR-5030.md), [support sgl-kernel on blackwell](../sources/prs/sglang/PR-5074.md), [reduce moe_align_block_size_kernel small batch mode overhead](../sources/prs/sglang/PR-5086.md), [Blackwell Cutlass MLA kernel](../sources/prs/sglang/PR-5142.md), [[1/2] Add FP8 Blockscale MoE CUTLASS kernel for Blackwell](../sources/prs/sglang/PR-5281.md), [fix: solve cu118 issue for cutlass mla](../sources/prs/sglang/PR-5331.md), [Add Cutlass MLA attention backend](../sources/prs/sglang/PR-5390.md), [chore: upgrade DeepGEMM](../sources/prs/sglang/PR-5395.md), [[perf] introduce deep gemm group_gemm_masked as gemm](../sources/prs/sglang/PR-5432.md), [feat: use flashinfer jit package](../sources/prs/sglang/PR-5547.md), [[2/2] Add python wrapper for CUTLASS FP8 Blockscale MoE Kernel. ](../sources/prs/sglang/PR-5694.md), [[PP] Add pipeline parallelism](../sources/prs/sglang/PR-5724.md), [cutlass 3.9 supported to improve fp8_blockwise_gemm](../sources/prs/sglang/PR-5820.md), [Add sm_120 for blackwell](../sources/prs/sglang/PR-5903.md), [chore: upgrade cutlass 3.9.2](../sources/prs/sglang/PR-6004.md), [chore: upgrade deepgemm](../sources/prs/sglang/PR-6073.md), [feat: mtp support dp-attention](../sources/prs/sglang/PR-6081.md), [[1/2] Add Kernel support for Cutlass based Fused FP4 MoE](../sources/prs/sglang/PR-6093.md), [Cutlass MLA: Disable split kv due to https://github.com/NVIDIA/cutlass/issues/2274](../sources/prs/sglang/PR-6101.md), [Upgrade CUTLASS 4.0](../sources/prs/sglang/PR-6336.md), [reduce torch.zeros overhead in moe align block size kernel](../sources/prs/sglang/PR-6369.md), [[Feature] Comprehensive Hybrid Parallelism Support](../sources/prs/sglang/PR-6389.md), [Fix bug of deepseek-v3 under DP+EP mode with large batchsize/seqlen](../sources/prs/sglang/PR-6449.md), [[EP] Add cuda kernel for moe_ep_pre_reorder](../sources/prs/sglang/PR-6699.md), [Set `num_fused_shared_experts` as `num_shared_experts` when shared_experts fusion is not disabled](../sources/prs/sglang/PR-6736.md), [Correctly abort the failed grammar requests & Improve the handling of abort](../sources/prs/sglang/PR-6803.md), [[EP] Add cuda kernel for moe_ep_post_reorder](../sources/prs/sglang/PR-6837.md), [Fix AWQ Dequant and Weight Loading of deepseek v2](../sources/prs/sglang/PR-6842.md), [fix ep_moe_reorder kernel bugs](../sources/prs/sglang/PR-6858.md), [Add a CUDA kernel for fusing mapping and weighted sum for MoE.](../sources/prs/sglang/PR-6916.md), [[sgl-kernel] Add cuda kernel for moe_ep_silu_and_mul](../sources/prs/sglang/PR-6919.md), [[perf][sgl-kernel] extend cutlass_mla_decode to support num_head < 128](../sources/prs/sglang/PR-6929.md), [Clean up server_args.py](../sources/prs/sglang/PR-7037.md), [Tiny fix cutlass_mla_get_workspace_size stub incorrect signature](../sources/prs/sglang/PR-7057.md), [Fix grammar abort & Minor style fixes](../sources/prs/sglang/PR-7204.md), [feat: support compatibility between MTP and two-batch-overlap](../sources/prs/sglang/PR-7225.md), [Add CUTLASS FP8 Blockscale MoE kernel for Hopper architecture](../sources/prs/sglang/PR-7278.md), [fix: resolve blackwell deepep image issue](../sources/prs/sglang/PR-7331.md), [Fuse sorted_token_ids padding to moe_align_block_size kernel](../sources/prs/sglang/PR-7437.md), [fix: fix apply_shuffle_mul_sum](../sources/prs/sglang/PR-7444.md), [[CMake] Fix sgl-kernel CMakeLists for Blackwell](../sources/prs/sglang/PR-7543.md), [Add dsv3 router gemm kernel](../sources/prs/sglang/PR-7627.md), [Add dsv3 fused a gemm to sgl-kernel](../sources/prs/sglang/PR-7630.md), [[Feature] CUDA Green Context Support](../sources/prs/sglang/PR-7649.md), [[1/n]: add cutlass W4A8 moe kernel for hopper architecture](../sources/prs/sglang/PR-7772.md), [[kernel] opt moe align block kernel by block/warp scan algorithm](../sources/prs/sglang/PR-7884.md), [[feat] Support tp mode for DeepSeek-R1-W4AFP8](../sources/prs/sglang/PR-8118.md), [[Fix][Ready]Fix register spilling in cutlass nvfp4 gemm kernel on Blackwell](../sources/prs/sglang/PR-8127.md), [[sgl-kernel] Opt per_token_quant_fp8 with warp reduce](../sources/prs/sglang/PR-8130.md), [[2/N]Support DeepSeek-R1 w4a8 low latency deepep](../sources/prs/sglang/PR-8464.md), [[Perf] Tunings for SM100 FP8 CUTLASS kernel](../sources/prs/sglang/PR-8818.md), [chore: support blackwell cu129 image](../sources/prs/sglang/PR-8928.md), [[NVIDA] [1/N] Nvfp4 Masked Gemm: Add quant op for the flashinfer grouped gemm](../sources/prs/sglang/PR-9200.md), [[fix]: fix cutlass moe ut and and Opt H20 cutlass groupGemm performance](../sources/prs/sglang/PR-9272.md), [Support trtllm_allreduce_fusion in flashinfer for cuda<12.8](../sources/prs/sglang/PR-9339.md), [[sgl-kernel] feat: Support sm120 cutlass fp8 gemm kernel](../sources/prs/sglang/PR-9403.md), [[NVIDIA] [2/N] Optimize `silu_and_mul_scaled_fp4_grouped_quant` perf](../sources/prs/sglang/PR-9556.md), [Update CUTLASS 4.2 & Enable K-Major Scale Factor for SM90 FP8 Blockwise Group GEMM](../sources/prs/sglang/PR-9559.md), [Make sm100 fp8 kernels available on sm103](../sources/prs/sglang/PR-9789.md), [Make fp4_quantize kernels work on sm103](../sources/prs/sglang/PR-9807.md), [CUTLASS fp8 blockwise gemm support of sm120](../sources/prs/sglang/PR-9969.md), [[WIP] support more dtypes for tcgen05](../sources/prs/tilelang/PR-1229.md), [[Enhancement] add more dtype and fix mma.ws for fp16 for tcgen05](../sources/prs/tilelang/PR-1327.md), [[Feat] profiler support cudagraph backend](../sources/prs/tilelang/PR-1658.md), [[Feature] Support `cp.reduce.async.bulk.tensor`](../sources/prs/tilelang/PR-1667.md), [[Feature] Support tcgen5mma lowering for `.kind::i8`](../sources/prs/tilelang/PR-1764.md), [[CUDA] Support tcgen5mma gemm ts](../sources/prs/tilelang/PR-1866.md), [[Feature] Support cluster launch, query, synchronization and barrier operations](../sources/prs/tilelang/PR-1874.md), [[Feature] 2-SM support for TMA, TMEM and TCGEN5MMA on Blackwell](../sources/prs/tilelang/PR-1882.md), [[Feature] Add T.copy_cluster to support TMA multicast and SM-to-SM cluster copy](../sources/prs/tilelang/PR-1908.md), [[Feature] Block-scaled GEMM support for MXFP8 on Blackwell](../sources/prs/tilelang/PR-1945.md), [[Feature] Support TMA store in T.tma_copy()](../sources/prs/tilelang/PR-1981.md), [[Transform] Add InjectTcgen05Fence pass](../sources/prs/tilelang/PR-2003.md), [[Backend] Refactor gemm_sp](../sources/prs/tilelang/PR-2048.md), [[CUDA] Support int4 `T.gemm`](../sources/prs/tilelang/PR-2063.md), [[CUDA] Improve int4 GEMM lowering and packed codegen support](../sources/prs/tilelang/PR-2073.md), [[Bugfix] Enable `.shared::cta` in TMA copy paths only on CUDA 12.8+](../sources/prs/tilelang/PR-2087.md), [feat: auto-vectorize bf16/fp16 reduce with packed add2 intrinsics](../sources/prs/tilelang/PR-2112.md), [[CUDA][TMA] Add TMA tile::gather4 / tile::scatter4 support](../sources/prs/tilelang/PR-2129.md), [[codex] Split GEMM implementations by backend](../sources/prs/tilelang/PR-2153.md), [[CUDA] Add native SM75 MMA GEMM support for FP16, INT8 and INT4](../sources/prs/tilelang/PR-2198.md), [[TIR][IR] Update to use tirx](../sources/prs/tilelang/PR-2216.md), [[Python] Drop Python 3.9 support](../sources/prs/tilelang/PR-2218.md), [[Kernel]: Cutlass 2:4 Sparsity + FP8/Int8 Quant Support](../sources/prs/vllm/PR-10995.md), [[Kernel] Update `cutlass_scaled_mm` to support 2d group (blockwise) scaling](../sources/prs/vllm/PR-11868.md), [[Build] Only build 9.0a for scaled_mm and sparse kernels](../sources/prs/vllm/PR-12339.md), [[ROCm] Faster Custom Paged Attention kernels](../sources/prs/vllm/PR-12348.md), [[Attention] MLA decode optimizations](../sources/prs/vllm/PR-12528.md), [[Kernel] port sgl moe_align_block_size kernels](../sources/prs/vllm/PR-12574.md), [[Kernel][Quantization] Integrate block-quantized CUTLASS kernels for DeepSeekV3](../sources/prs/vllm/PR-12587.md), [[Attention] MLA with chunked prefill](../sources/prs/vllm/PR-12639.md), [[Perf] Mem align KV caches for CUDA devices (MLA perf improvement)](../sources/prs/vllm/PR-12676.md), [[Core][AMD] Migrate fully transparent sleep mode to ROCm platform](../sources/prs/vllm/PR-12695.md), [[Bugfix][Kernel] Fix per-token/per-channel quantization for Hopper scaled mm](../sources/prs/vllm/PR-12696.md), [[Kernel] Make rotary_embedding ops more flexible with input shape](../sources/prs/vllm/PR-12777.md), [[NVIDIA] Support nvfp4 quantization](../sources/prs/vllm/PR-12784.md), [Optimize moe_align_block_size for deepseek_v3](../sources/prs/vllm/PR-12850.md), [[Misc][Kernel]: Add GPTQAllSpark Quantization](../sources/prs/vllm/PR-12931.md), [[Kernel]Add streamK for block-quantized CUTLASS kernels](../sources/prs/vllm/PR-12978.md), [[Kernel] moe wna16 cuda kernel](../sources/prs/vllm/PR-13321.md), [[NVIDIA] Support nvfp4 tcgen05 gemm](../sources/prs/vllm/PR-13571.md), [[ROCm][MoE] mi300 mixtral8x7B perf for specific BS](../sources/prs/vllm/PR-13577.md), [[Kernel] FlashMLA integration](../sources/prs/vllm/PR-13747.md), [[Attention] MLA support for V1](../sources/prs/vllm/PR-13789.md), [add tcgen05 support for tcgen05 fp8 gemm](../sources/prs/vllm/PR-13798.md), [[ROCm] Disable chunked prefill/prefix caching when running MLA on non-cuda platforms](../sources/prs/vllm/PR-13844.md), [[Attention] Flash MLA for V1](../sources/prs/vllm/PR-13867.md), [[V1] EP/TP MoE + DP Attention](../sources/prs/vllm/PR-13931.md), [[Kernel] CUTLASS grouped gemm fp8 MoE kernel](../sources/prs/vllm/PR-13972.md), [[Kernel] optimize performance of gptq marlin kernel when n is small](../sources/prs/vllm/PR-14138.md), [dynamic distpatch of fp8 kernels](../sources/prs/vllm/PR-14245.md), [[Attention] FlashAttn MLA](../sources/prs/vllm/PR-14258.md), [fix minor miscalled method](../sources/prs/vllm/PR-14327.md), [[Build/BugFix] Fix hopper 12.8 build](../sources/prs/vllm/PR-14354.md), [Add cutlass support for blackwell fp8 blockwise gemm](../sources/prs/vllm/PR-14383.md), [[BugFix] Illegal Memory Access in the blockwise cutlass fp8 GEMMs](../sources/prs/vllm/PR-14396.md), [[Kernel] moe wna16 marlin kernel](../sources/prs/vllm/PR-14447.md), [permute/unpermute kernel for moe optimization](../sources/prs/vllm/PR-14568.md), [[Attention] Flash Attention 3 - fp8](../sources/prs/vllm/PR-14570.md), [[BugFix/Build] Fix sparse kernels not getting built on hopper](../sources/prs/vllm/PR-14572.md), [[Kernel] GGUF MoE kernel](../sources/prs/vllm/PR-14613.md), [[Kernel] allow non-contiguous input for marlin kernel](../sources/prs/vllm/PR-14658.md), [[Bugfix][Kernel][CPU] Fix num_tokens in CPU rotary embedding kernel](../sources/prs/vllm/PR-14667.md), [[V1] Fully Transparent Implementation of CPU Offloading](../sources/prs/vllm/PR-15354.md), [[Kernel] Fix conflicting macro names for gguf kernels](../sources/prs/vllm/PR-15456.md), [[ROCM][KERNEL] Paged attention for V1](../sources/prs/vllm/PR-15720.md), [[Bugfix] fix use_atomic_add support of marlin kernel when using v1 engine](../sources/prs/vllm/PR-15946.md), [Modularize fused experts and integrate PPLX kernels](../sources/prs/vllm/PR-15956.md), [[NVIDIA] Support Cutlass MLA for Blackwell GPUs](../sources/prs/vllm/PR-16032.md), [[ROCM] Add gfx950 to the custom attention archs](../sources/prs/vllm/PR-16034.md), [Add FlexAttention to V1](../sources/prs/vllm/PR-16078.md), [[Kernel] support merge_attn_states CUDA kernel, 3x speedup](../sources/prs/vllm/PR-16173.md), [[Hardware/NVIDIA/Kernel] [Functional Enablement] [1/N] Enable nvidia/DeepSeek-R1-FP4 Model](../sources/prs/vllm/PR-16362.md), [Allocate kv_cache with stride order](../sources/prs/vllm/PR-16605.md), [[V1] V1 FlashInfer Attention](../sources/prs/vllm/PR-16684.md), [[misc] ignore marlin_moe_wna16 local gen codes](../sources/prs/vllm/PR-16760.md), [[Kernel] GGUF MoeVec kernel](../sources/prs/vllm/PR-16780.md), [[BugFix] Accuracy fix for llama4 int4 - improperly casted scales](../sources/prs/vllm/PR-16801.md), [[Kernel] some optimizations for dense marlin and moe marlin](../sources/prs/vllm/PR-16850.md), [Update PyTorch to 2.7.0](../sources/prs/vllm/PR-16859.md), [[Kernel] Add expert_map support to Cutlass FP8 MOE](../sources/prs/vllm/PR-16861.md), [[Attention] FA3 decode perf improvement - single mma warp group support for head dim 128](../sources/prs/vllm/PR-16864.md), [Update Qwen1.5-MoE-W4A16-compressed-tensors.yaml](../sources/prs/vllm/PR-16946.md), [[ROCm][Kernel][V1] Enable AMD Radeon GPU Custom Paged Attention on v1](../sources/prs/vllm/PR-17004.md), [Fix `numel()` downcast in vllm/csrc/moe/moe_align_sum_kernels.cu +2](../sources/prs/vllm/PR-17082.md), [[ROCm][FP8][Kernel] FP8 quantization fused into Custom Paged Attention](../sources/prs/vllm/PR-17139.md), [[NVIDIA] Support Cutlass w8a8 FP8 for Blackwell Geforce GPUs (sm120)](../sources/prs/vllm/PR-17280.md), [[Misc][ROCm] Exclude `cutlass_mla_decode` for ROCm build](../sources/prs/vllm/PR-17289.md), [[Attention] MLA move o_proj q_proj into cuda-graph region](../sources/prs/vllm/PR-17484.md), [[Attention] MLA move rotary embedding to cuda-graph region](../sources/prs/vllm/PR-17668.md), [[Kernel] fp4 marlin kernel](../sources/prs/vllm/PR-17687.md), [[Kernel] Have rotary embeddings support tensors](../sources/prs/vllm/PR-18046.md), [Fix Broken macro for cutlass moe](../sources/prs/vllm/PR-18049.md), [[Build] Supports CUDA 12.6 and 11.8 after Blackwell Update](../sources/prs/vllm/PR-18316.md), [Sm100 blockwise fp8 swap ab](../sources/prs/vllm/PR-18564.md), [[Kernel] Integrate CUTLASS MoE kernel with PPLX](../sources/prs/vllm/PR-18762.md), [[Perf] Tunings for SM100 FP8 CUTLASS kernel](../sources/prs/vllm/PR-18778.md), [[BugFix] FA2 MLA Accuracy Issue](../sources/prs/vllm/PR-18807.md), [[Hardware][NVIDIA] FP4 MoE kernel optimization](../sources/prs/vllm/PR-19110.md), [[V1] Use FlashInfer by default on Blackwell GPUs](../sources/prs/vllm/PR-19118.md), [[Bugfix][V1] Allow manual FlashAttention for Blackwell](../sources/prs/vllm/PR-19492.md), [[Hardware][NVIDIA][kernel] Fp4 MOE quant kernel optimization](../sources/prs/vllm/PR-19500.md), [[Perf] Further tunings for SM100 FP8 CUTLASS kernel](../sources/prs/vllm/PR-19566.md), [Only build CUTLASS MoE kernels on Hopper](../sources/prs/vllm/PR-19648.md), [[feat]: CUTLASS block scaled group gemm for SM100](../sources/prs/vllm/PR-19757.md), [Fix FA2 fallback for Blackwell V1](../sources/prs/vllm/PR-19781.md), [[Bugfix] Build moe_data for both sm100 and sm90](../sources/prs/vllm/PR-20086.md), [[Bugfix] Fix some narrowing conversion warnings](../sources/prs/vllm/PR-20141.md), [Replace `multiply_add` with `homogeneous_multiply_add` to Address Clang Template Parameter Issue](../sources/prs/vllm/PR-20142.md), [[Bugfix] Fix topk_ids indices_type for CUTLASS w8a8 FP8 MoE](../sources/prs/vllm/PR-20166.md), [[Kernel][Bugfix] Fixup some warnings in nvfp4_blockwise_moe when CUDA < 12.8](../sources/prs/vllm/PR-20324.md), [Update PyTorch to 2.8.0](../sources/prs/vllm/PR-20358.md), [[Kernel] SM90 CUTLASS FP8 GEMM: add support for swap AB + kernel tuning](../sources/prs/vllm/PR-20396.md), [[feat]: add SM100 support for cutlass FP8 groupGEMM](../sources/prs/vllm/PR-20447.md), [[Performance] Performance improvements in non-blockwise fp8 CUTLASS MoE](../sources/prs/vllm/PR-20762.md), [SM100 Cutlass MLA decode with unrestricted num_heads (< 128) for DeepSeek TP](../sources/prs/vllm/PR-20769.md), [[fix]: disable cutlass block scaled group gemm for EP](../sources/prs/vllm/PR-20781.md), [[Perf] Add swap_ab to SM90 FP8 non-block CUTLASS moe grouped gemm](../sources/prs/vllm/PR-20911.md), [Support mnnvl all2allv from Flashinfer](../sources/prs/vllm/PR-21003.md), [[Kernel] Flashinfer MLA (trtllm-gen) decode kernel integration](../sources/prs/vllm/PR-21078.md), [[Perf] Cuda Kernel for Per Token Group Quant](../sources/prs/vllm/PR-21083.md), [[perf] Add fused MLA QKV + strided layernorm](../sources/prs/vllm/PR-21116.md), [[Perf] Use FlashInfer RoPE for RotaryEmbedding.forward_cuda when available](../sources/prs/vllm/PR-21126.md), [[Feature][Kernel]FusedMoE LoRA](../sources/prs/vllm/PR-21229.md), [[v1] - Mamba1 Attention Metadata](../sources/prs/vllm/PR-21249.md), [Support CUTLASS NVFP4 (w4a4) for Blackwell Geforce GPUs (SM120)](../sources/prs/vllm/PR-21309.md), [[Bugfix][CUDA] fixes CUDA FP8 kv cache dtype supported](../sources/prs/vllm/PR-21420.md), [[Bug] Fix Compressed Tensor NVFP4 `cutlass_fp4_group_mm` illegal memory access](../sources/prs/vllm/PR-21465.md), [[Kernel] Improve machete memory bound perf](../sources/prs/vllm/PR-21556.md), [[BugFix] Fix IMA FlashMLA full cuda-graph and DP + Update FlashMLA](../sources/prs/vllm/PR-21691.md), [update flashinfer to v0.2.9rc2](../sources/prs/vllm/PR-21701.md), [Fix Flashinfer CUTLASS MOE Allgather](../sources/prs/vllm/PR-21963.md), [[Kernel] Add support for block FP8 on SM120 (NVIDIA 5090 and RTX PRO 6000)](../sources/prs/vllm/PR-22131.md), [Fp8 paged attention update](../sources/prs/vllm/PR-22222.md), [Upgrade FA3 for attention sink](../sources/prs/vllm/PR-22313.md), [[Attention] FA3 Attention Sinks Perf Boost](../sources/prs/vllm/PR-22478.md), [[Fix] enable swap_ab for pplx problem size computation](../sources/prs/vllm/PR-22991.md), [[Kernel] CUTLASS MoE FP8: Integrate cuda moe permute/unpermute](../sources/prs/vllm/PR-23045.md), [[V1] address post issues related to #20059 (part 1); cascade attention reenable by default](../sources/prs/vllm/PR-23046.md), [[kernel] Support W4A8 on Hopper](../sources/prs/vllm/PR-23198.md), [[Kernel] Add fused grouped_topk kernel for MoE](../sources/prs/vllm/PR-23274.md), [[Perf] Use upstream CUTLASS for SM90 Block FP8 kernel](../sources/prs/vllm/PR-23280.md), [[Compile] Fix Compile Warning SM100 Cutlass MLA](../sources/prs/vllm/PR-23287.md), [fix incompatibililty with non cuda platform for nvfp4](../sources/prs/vllm/PR-23478.md), [[Compile] Fix Compile Warning for `w4a8_mm_entry.cu`](../sources/prs/vllm/PR-23660.md), [[NVIDIA] Support SiluMul + NVFP4 quant fusion](../sources/prs/vllm/PR-23671.md), [[Kernel][tcgen05] nvfp4 fused tcgen05 moe](../sources/prs/vllm/PR-23696.md), [[Bugfix][Misc] Fix silu_and_mul_nvfp4_quant issue and extract common utils for nvfp4 kernel source files](../sources/prs/vllm/PR-23727.md), [[Feature] Support Decode Context Parallel (DCP) for MLA](../sources/prs/vllm/PR-23734.md), [[Kernel] cuda kernels for upcoming decode context parallel feature](../sources/prs/vllm/PR-23791.md), [[Kernel] Faster pre-processing time for W4A8](../sources/prs/vllm/PR-23972.md), [[Model] Add LongCat-Flash ](../sources/prs/vllm/PR-23991.md), [[Kernel] Support decode context parallelism on Blackwell with CUTLASS MLA](../sources/prs/vllm/PR-24385.md), [[NVIDIA] Blackwell Family](../sources/prs/vllm/PR-24673.md), [[Kernel][Quantization] add w4a8 support for marlin kernel](../sources/prs/vllm/PR-24722.md), [[Bugfix] Fix accuracy issue for silu_mul + nvfp4 quant fusion kernel](../sources/prs/vllm/PR-24833.md), [[Bugfix][B200] Fix `cutlass_mla` hang](../sources/prs/vllm/PR-24966.md), [Disable failing GPT-OSS Eval (Blackwell) for now](../sources/prs/vllm/PR-25107.md), [[Compile] Fix Compile Warning for Ignoring `MIN_BLOCK_PER_SM`](../sources/prs/vllm/PR-25193.md), [[Bugfix] [B200] cutlass_mla - ensure kv_split == 1 for batch size > 1](../sources/prs/vllm/PR-25509.md), [Fuse RoPE and MLA KV-cache write](../sources/prs/vllm/PR-25774.md), [Update launch_bounds_utils.h for correct compile on Multiple Cuda Arch - PTXAS out of range Warning](../sources/prs/vllm/PR-25843.md), [Fix INT8 quantization error on Blackwell GPUs (SM100+)](../sources/prs/vllm/PR-25935.md), [[Performance] Split FlashAttn attention and cache update](../sources/prs/vllm/PR-25954.md), [Fix undefined symbol: cutlass_moe_mm_sm100](../sources/prs/vllm/PR-26098.md), [[NVIDIA] [Perf] Update to leverage flashinfer trtllm FP4 MOE throughput kernel](../sources/prs/vllm/PR-26714.md), [[Attention] Tune CUTLASS MLA num_splits](../sources/prs/vllm/PR-26846.md), [[Perf] SM100 - add swap AB optimization to CUTLASS FP8 GEMM](../sources/prs/vllm/PR-27284.md), [Prefer FlashAttention MLA as default over FlashMLA](../sources/prs/vllm/PR-27363.md), [[Attention] Use sparse prefill kernel for fp8 kv-cache in DeepSeek-v3.2](../sources/prs/vllm/PR-27532.md), [[Performance] Fused blockwise quant RMS norm](../sources/prs/vllm/PR-27883.md), [[Kernel] Optimize rms_norm kernel](../sources/prs/vllm/PR-27931.md), [Update Flashinfer from `v0.4.1` to `v0.5.2`](../sources/prs/vllm/PR-27952.md), [[Perf][DeepSeek] Add sigmoid+bias fusion to fused_grouped_topk from TRTLLM](../sources/prs/vllm/PR-28124.md), [[Performance][B200] silu_mul_quant: pack scales in int32](../sources/prs/vllm/PR-28358.md), [[NVIDIA] Guard SM100 CUTLASS MoE macro to SM100 builds v2](../sources/prs/vllm/PR-28938.md), [chore: add RTX_PRO_6000 GLM4.6-FP8 kernel tuning](../sources/prs/vllm/PR-29240.md), [[Kernel] Add NVFP4 MoE CUTLASS support for SM120](../sources/prs/vllm/PR-29242.md), [Lora MoE Align Improvements](../sources/prs/vllm/PR-29257.md), [[Kernel][MoE] optimize `moe_align_block_size`](../sources/prs/vllm/PR-29642.md), [[Kernel]Support W4A8 Grouped GEMM on Hopper](../sources/prs/vllm/PR-29691.md), [[Perf] Improve fp8 quant in mla; replace ReduceSum with ReduceScatterSum](../sources/prs/vllm/PR-29795.md), [[Kernel][Quantization][MoE] add marlin kernel support for turing (sm75)](../sources/prs/vllm/PR-29901.md), [[Perf] Do FP4 quant before All gather on flashinfer trtllmgen MOE ](../sources/prs/vllm/PR-30014.md), [Add llmcompressor fp8 kv-cache quant (per-tensor and per-attn_head)](../sources/prs/vllm/PR-30141.md), [gptq marlin quantization support for fused moe with lora](../sources/prs/vllm/PR-30254.md), [[Feature] Add SM103 (Blackwell Ultra) Support to vLLM](../sources/prs/vllm/PR-30484.md), [OffloadingConnector: Support kernel_block_size != block_size](../sources/prs/vllm/PR-30692.md), [[NVFP4][Perf] Tune NVFP4 input quant kernel for small batch size](../sources/prs/vllm/PR-30897.md), [[Kernel] Add topk_sigmoid kernel](../sources/prs/vllm/PR-31246.md), [[Perf] Fuse stride preparation for NVFP4 cutlass_moe](../sources/prs/vllm/PR-31837.md), [[1/N][Attention] Restructure attention: move files](../sources/prs/vllm/PR-31916.md), [[Perf][Kernel] Optimize FP4 quantization kernels (SM100F)](../sources/prs/vllm/PR-32520.md), [fix: Add glm4_moe_lite to MLA detection](../sources/prs/vllm/PR-32614.md), [[Attention] FA4 integration](../sources/prs/vllm/PR-32974.md), [[Feature] Support CPU Offloading without Pytorch Pinned Memory that leads to doubled allocation](../sources/prs/vllm/PR-32993.md), [[Kernel] Apply 256bit LDG/STG To Activation Kernels](../sources/prs/vllm/PR-33022.md), [Add support for Mistral Large 3 inference with Flashinfer MoE](../sources/prs/vllm/PR-33174.md), [[Bugfix] Fix quant RMS norm fusion for quantization with TMA-aligned scales](../sources/prs/vllm/PR-33255.md), [[Kernel] Add enable_sm120_or_later for SM121 (DGX Spark) CUTLASS support](../sources/prs/vllm/PR-33517.md), [[Feature][Core] Support Fabric detection to adapt the MNNVL protocol for the GB series](../sources/prs/vllm/PR-33540.md), [[Bugfix]fix output Nan/Inf in marlin if dtype=float16](../sources/prs/vllm/PR-33972.md), [Reapply [Attention][FA3] Update FA3 to include new swizzle optimization](../sources/prs/vllm/PR-34043.md), [fix(cpu): fix mla_decode compilation on x86 without AVX512](../sources/prs/vllm/PR-34052.md), [[Kernel] Optimize grouped topk kernel](../sources/prs/vllm/PR-34206.md), [[ModelBash][DSV3] Add TRTLLM DSV3 Router GEMM kernel (6% B1 Speedup)](../sources/prs/vllm/PR-34302.md), [[Bugfix] Enforce DeepGEMM when using sparse_attn_indexer on CUDA](../sources/prs/vllm/PR-34374.md), [[Custom Ops] Add functional + out variant for scaled_fp4_quant](../sources/prs/vllm/PR-34389.md), [[Kernel] Integrate SM100 MXFP8 blockscaled grouped MM and quant kernels](../sources/prs/vllm/PR-34448.md), [[Model Bash] DeepSeek R1 BF16 Min Latency QKV A GEMM (0.5% E2E Speedup)](../sources/prs/vllm/PR-34758.md), [[Bugfix] Gate 256-bit instructions to CUDA 12.9+](../sources/prs/vllm/PR-34791.md), [[Attention][Perf][Kernel] Replace torch.cat with vectorized CUDA kernel MLA query concat - DeepSeek-V3.2](../sources/prs/vllm/PR-34917.md), [[Model Runner V2] Support attention group](../sources/prs/vllm/PR-35036.md), [[Performance] Cublas Bf16 Gate with Fp32 Output](../sources/prs/vllm/PR-35121.md), [[Bugfix] Fix DSV3 kernels breaking _C and _moe_C on unsupported arches](../sources/prs/vllm/PR-35123.md), [[Bugfix] Fix expert_ids padding values in moe_align_block_size kernel](../sources/prs/vllm/PR-35161.md), [[BugFix] Fix fp4 quant kernel on CUDA 12.8](../sources/prs/vllm/PR-35210.md), [[Feat] Add CUDA torch fallbacks for fp8_mqa_logits/fp8_paged_mqa_logits_torch function](../sources/prs/vllm/PR-35271.md), [[Attention][Perf] Optimize cp_gather_and_upconvert_fp8_kv_cache - DeepSeek-v3.2](../sources/prs/vllm/PR-35290.md), [[Kernel] Add FlashInfer MoE A2A Kernel](../sources/prs/vllm/PR-36022.md), [Add 320 dimension size support to MLA](../sources/prs/vllm/PR-36161.md), [docs: fix wrong cc in int8.md](../sources/prs/vllm/PR-36209.md), [[Kernel] Fuse FP8 output quantization into merge_attn_states](../sources/prs/vllm/PR-36518.md), [Update Flashinfer to 0.6.6](../sources/prs/vllm/PR-36768.md), [[MTP][Sparse MLA] Take advantage of native MTP support in indexer when possible](../sources/prs/vllm/PR-36982.md), [[Kernel] Add gpt-oss Router GEMM kernel](../sources/prs/vllm/PR-37205.md), [[UX] Add flashinfer-cubin as CUDA default dep](../sources/prs/vllm/PR-37233.md), [[Perf] Set Flashinfer sparse MLA as default backend for FP8 kv cache](../sources/prs/vllm/PR-37252.md), [[Kernel] Add non-gated support for NVFP4 CUTLASS MoE](../sources/prs/vllm/PR-37320.md), [Add nvfp4 support to reshape_and_cache_flash](../sources/prs/vllm/PR-37332.md), [[torch.compile] Refactor Attention Quant Fusion Pass and Remove Boilerplate](../sources/prs/vllm/PR-37373.md), [[Perf][Kernel] Persistent TopK scheduler: unified CUDAGraph-safe kernel with dynamic per-row dispatch - DeepSeek-V3.2 DSA decode](../sources/prs/vllm/PR-37421.md), [[Kernel] Add MXFP4 W4A4 CUTLASS MoE kernel for SM100](../sources/prs/vllm/PR-37463.md), [[4/n] Migrate FP4/W4A8 CUTLASS kernels to torch stable ABI](../sources/prs/vllm/PR-37503.md), [refactor: abstract deepgemm support into platform](../sources/prs/vllm/PR-37519.md), [[Bugfix] Preserve CUDA arch suffix (a/f) for SM12x — fixes NVFP4 NaN on desktop Blackwell](../sources/prs/vllm/PR-37725.md), [[Kernel] Optimize SM120 CUTLASS blockwise FP8 GEMM](../sources/prs/vllm/PR-37970.md), [[Kernel] Add swapAB support for SM120 CUTLASS blockwise FP8 GEMM ](../sources/prs/vllm/PR-38325.md), [[CI Bugfix] Pre-download missing FlashInfer headers in Docker build](../sources/prs/vllm/PR-38391.md), [[NVIDIA] Bugfix NVFP4 DGX Spark and RTX50](../sources/prs/vllm/PR-38423.md), [[Perf] Batch KV cache swap copies via cuMemcpyBatchAsync](../sources/prs/vllm/PR-38460.md), [[Attention Backend] TurboQuant: 2-bit KV cache compression with 4x capacity](../sources/prs/vllm/PR-38479.md), [[Compile] Fix nvfp4 compile warning](../sources/prs/vllm/PR-38573.md), [[FA4] Update flash-attention to latest upstream FA4](../sources/prs/vllm/PR-38690.md), [[Refactor] Improve indexer decode path metadata preparation](../sources/prs/vllm/PR-38865.md), [[Bugfix] Fix broken explicit unquantized kv cache dtype support](../sources/prs/vllm/PR-38922.md), [[Bugfix] Fix GDN FLA kernel crashes with NULL_BLOCK_ID=0 CUDA graph padding](../sources/prs/vllm/PR-39064.md), [[XPU] Quick fix for TritonMLA to remove cuda hardcode](../sources/prs/vllm/PR-39088.md), [perf(moe): add tuned fused_moe config for RTX PRO 6000 Blackwell Server Edition](../sources/prs/vllm/PR-39183.md), [Use CU_MEMCPY_SRC_ACCESS_ORDER_ANY for batch KV cache swaps](../sources/prs/vllm/PR-39306.md), [Fix NUMA binding on non-CDMM Grace-Blackwell systems](../sources/prs/vllm/PR-39361.md), [fix: clamp NaN/Inf in topk_softmax to prevent duplicate expert IDs](../sources/prs/vllm/PR-39391.md), [[Perf] Fuse Zero Initializer for FP8 DeepGemm Block Quant Kernel](../sources/prs/vllm/PR-39547.md), [[Bugfix] Add Marlin kernel in block scaled mm kernel selection.](../sources/prs/vllm/PR-40105.md), [[Bugfix] moe lora align kernel grid](../sources/prs/vllm/PR-40131.md), [[Bugfix] Guard mxfp4_experts_quant bindings on ENABLE_NVFP4_SM100](../sources/prs/vllm/PR-40191.md), [[Performance][DSR1]: Fused RoPE+KVCache+q_concat for MLA](../sources/prs/vllm/PR-40392.md), [[Perf] Batch invariance with Cutlass fp8 support, 28.9% E2E latency improvement](../sources/prs/vllm/PR-40408.md), [[GDN] Enable FI Blackwell GDN prefill kernel](../sources/prs/vllm/PR-40717.md), [[DSV4] Add silu clamp limit to shared expert](../sources/prs/vllm/PR-40950.md), [[DSV4] Fuse norm and router for low latency scenario](../sources/prs/vllm/PR-41263.md), [Faster per-token fp8 group quant packed kernel for blackwell](../sources/prs/vllm/PR-41326.md), [[Bugfix] Fix condition to clear persistent topk so that it can be captured regardless](../sources/prs/vllm/PR-41665.md), [[MLA Attention Backend] Add TOKENSPEED_MLA backend for DSR1/Kimi K25 prefill + decode on Blackwell](../sources/prs/vllm/PR-41778.md), [[CUDA][CUTLASS] Enable cutlass scaled mm for non-compatible sizes ](../sources/prs/vllm/PR-41868.md), [[MoE] Move various experts classes to fused_moe/experts/](../sources/prs/vllm/PR-41979.md), [[Perf] Use 2D-grid to eliminate divmod in W8W8 group quant](../sources/prs/vllm/PR-42153.md), [[Misc] add humming to dependencies](../sources/prs/vllm/PR-42540.md), [[6/n] Migrate activation kernels, gptq, gguf, non cutlass w8a8 to libtorch stable ABI (continued)](../sources/prs/vllm/PR-42663.md), [[Refactor] Remove dead cuda kernels](../sources/prs/vllm/PR-42767.md), [[Perf] Padded nvfp4 quant kernel to remove additional copy, 2.4%~5.7% e2e performance improvement](../sources/prs/vllm/PR-42774.md), [[Perf][MLA] Enable FULL cudagraph capture for TRITON_MLA decode](../sources/prs/vllm/PR-42885.md), [[Kernel] (1/N) Machete - Hopper Optimized Mixed Precision Linear Kernel ](../sources/prs/vllm/PR-7174.md), [[Kernel] (2/N) Machete - Integrate into CompressedTensorsWNA16 and GPTQMarlin](../sources/prs/vllm/PR-7701.md), [[Bugfix] Fix Machete unittests failing with `NotImplementedError`](../sources/prs/vllm/PR-9218.md), [DeepGEMM — FP8 GEMM with Fine-Grained Scaling](../wiki/kernels/deepgemm.md), [FlashMLA — Multi-head Latent Attention](../wiki/kernels/flashmla.md), [FP8 Block-Scale GEMM](../wiki/kernels/fp8-block-scale-gemm.md), [Fused MoE — FP8 Block-Scale Routing + Dual GEMM](../wiki/kernels/fused-moe.md), [Gated Delta Net — Linear Attention](../wiki/kernels/gated-delta-net.md), [Gated Dual GEMM (Gate-Up + SwiGLU Fusion)](../wiki/kernels/gated-dual-gemm.md), [Grouped GEMM for MoE](../wiki/kernels/grouped-gemm.md), [NVFP4 GEMM — 4-bit Floating Point Matrix Multiply](../wiki/kernels/nvfp4-gemm.md), [NVFP4 Batched GEMV](../wiki/kernels/nvfp4-gemv.md), [Sparse MLA (DeepSeek V3.2)](../wiki/kernels/sparse-mla.md), [TensorRT-LLM Blackwell FP4 DSA Indexer](../wiki/kernels/tensorrt-llm-blackwell-indexer.md), [CCCL CUB Memory Primitives For Selection And Scan](../wiki/techniques/cccl-memory-primitives.md), [External Source-Map Research For Kernel Edits](../wiki/techniques/external-source-map-research.md) | -| `cute-dsl` | [CuTe DSL for Blackwell](../wiki/languages/cute-dsl.md) | [Colfax Article Source Kernels](../sources/blogs/colfax-article-source-kernels.md), [Colfax CUTLASS Tutorial: GEMM Kernels Using Tensor Memory for Blackwell](../sources/blogs/colfax-cutlass-blackwell.md), [FlashAttention-4 Blog](../sources/blogs/flash-attention-4.md), [NVFP4 GEMV and Improved NVFP4 GEMV](../sources/blogs/simon-nvfp4-gemv.md), [simveit effective_transpose](../sources/blogs/simveit-effective-transpose.md), [simveit load_and_store](../sources/blogs/simveit-load-and-store.md), [Tilus: A Tile-Level GPGPU Programming Language for Low-Precision Computation](../sources/blogs/tilus-nvidia.md), [Blackwell NVFP4 Kernel Hackathon Journey](../sources/blogs/yue-nvfp4-hackathon.md), [FlashInfer MLSys 2026 - Track A: Fused MoE FP8](../sources/contests/flashinfer-mlsys26/track-a-fused-moe.md), [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [FlashInfer MLSys 2026 - Track C: Gated Delta Net](../sources/contests/flashinfer-mlsys26/track-c-gated-delta-net.md), [GPU Mode NVFP4 Hackathon - Problem 1: Batched GEMV](../sources/contests/gpu-mode-nvfp4/problem-1-gemv.md), [GPU Mode NVFP4 Hackathon - Problem 2: NVFP4 GEMM](../sources/contests/gpu-mode-nvfp4/problem-2-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 3: Gated Dual GEMM](../sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 4: Grouped GEMM](../sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [NVIDIA CUTLASS 4.x Blackwell Support](../sources/docs/nvidia-cutlass-blackwell.md), [[None][perf] Add more optimization options for MOE CuteDSL finalized kernel](../sources/prs/TensorRT-LLM/PR-10042.md), [[TRTLLM-9992][perf] Enable PDL for CuteDSL kernels and overlap MoeOutputMemset](../sources/prs/TensorRT-LLM/PR-10043.md), [[None][feat] CuteDSL MOE FC1 Enhancement](../sources/prs/TensorRT-LLM/PR-10088.md), [[TRTLLM-9457][feat] Add cute dsl fp8 gemm for Blackwell](../sources/prs/TensorRT-LLM/PR-10130.md), [[TRTLLM-9831][perf] Enable 2CTA with autotune for CuteDSL MoE and Grouped GEMM optimizations](../sources/prs/TensorRT-LLM/PR-10201.md), [[TRTLLM-10147][perf] Balanced random MoE workload generator for CuteDSL kernel UT, autotuner and layerwise benchmark](../sources/prs/TensorRT-LLM/PR-10279.md), [[TRTLLM-9661][chore] Further reduce tuning time for cuteDSL nvFP4 dense gemm.](../sources/prs/TensorRT-LLM/PR-10339.md), [[None] [feat] Add test script and raster M for gather fc1 kernel](../sources/prs/TensorRT-LLM/PR-10429.md), [[TRTLLM-10276][feat] Integrate cutedsl argmax kernel](../sources/prs/TensorRT-LLM/PR-10476.md), [[None] [feat] Add densegemm backend for MoE](../sources/prs/TensorRT-LLM/PR-10479.md), [[TRTLLM-9831][perf] Use TMA.RED to improve effective memory bandwidth](../sources/prs/TensorRT-LLM/PR-10987.md), [[None][feat] fuse shared to sparse experts in TRT-LLM Gen MoE](../sources/prs/TensorRT-LLM/PR-11143.md), [[https://nvbugs/5854860][fix] Fix cutedsl argmax on sm120](../sources/prs/TensorRT-LLM/PR-11181.md), [[TRTLLM-10004][feat] Enable GEMM -> AR with GEMM output in registered buffers](../sources/prs/TensorRT-LLM/PR-11589.md), [[TRTLLM-11092][feat] add support for visual gen FA4 attention backend](../sources/prs/TensorRT-LLM/PR-11697.md), [[https://nvbugs/5885070][fix] fix deepeplowlatency with cutedsl moe backend](../sources/prs/TensorRT-LLM/PR-11769.md), [[TRTLLM-10990][feat] Fuse SwiGLU and quant into shared expert](../sources/prs/TensorRT-LLM/PR-11897.md), [[TRTLLM-10407][feat] Integrate CuTE DSL top-k kernel for Blackwell](../sources/prs/TensorRT-LLM/PR-11900.md), [[TRTLLM-11289][feat] Integrate CuteDSL's bf16 dense GEMMs](../sources/prs/TensorRT-LLM/PR-12074.md), [[None][feat] CuteDSL MOE: Add raster along M/N support for blockscaled contiguous backbone kernel](../sources/prs/TensorRT-LLM/PR-12079.md), [[None][feat] Add DWDP (Distributed Weight Data Parallelism) support for MoE inference](../sources/prs/TensorRT-LLM/PR-12136.md), [[TRTLLM-10407][perf] Enable CuteDSL indexer_top_k in model](../sources/prs/TensorRT-LLM/PR-12236.md), [[TRTLLM-10407][perf] Add cute dsl single pass multi cta cluster topk](../sources/prs/TensorRT-LLM/PR-12354.md), [[None][feat] Add PDL support to CuTE DSL top-k kernels](../sources/prs/TensorRT-LLM/PR-12506.md), [[None][feat] Optimize mamba SSD prefill and extend flashinfer dispatch](../sources/prs/TensorRT-LLM/PR-12731.md), [[TRTLLM-11797][feat] Add cutedsl moe backend supporting for qwen3.5.](../sources/prs/TensorRT-LLM/PR-12799.md), [[TRTLLM-11585][feat] Add CUTEDSL moe backend for nemotron-h](../sources/prs/TensorRT-LLM/PR-12884.md), [[TRTLLM-34871][feat] Add cute dsl FP8 paged MQA logits decode kernel](../sources/prs/TensorRT-LLM/PR-13219.md), [[None][perf] FC2 DenseGEMM autotune: split-K, swap_ab, fine-grained tuning buckets](../sources/prs/TensorRT-LLM/PR-13833.md), [[TRTLLM-35237][feat] Add cute dsl FP4 paged MQA logits decode kernel](../sources/prs/TensorRT-LLM/PR-13929.md), [[None][feat] Enable EPLB for trtllm-gen and cutlass backend](../sources/prs/TensorRT-LLM/PR-8886.md), [[TRTLLM-9685] [feat] Add gather fc1 kernel by cuteDSL](../sources/prs/TensorRT-LLM/PR-9618.md), [Blockwise and Groupwise GEMM for Blackwell and Improvements for Hopper](../sources/prs/cutlass/PR-2139.md), [Example 77 add blackwell flash-attention bwd for MLA shape](../sources/prs/cutlass/PR-2466.md), [Add Blackwell MLA forward (shape: d=192, dv=128) implementation](../sources/prs/cutlass/PR-2472.md), [fix gqa issue for blackwell fmha.py](../sources/prs/cutlass/PR-2599.md), [Add tutorial fp16_gemm_1](../sources/prs/cutlass/PR-2750.md), [new example with TMA prefetch feature targeting for DRAM latency boun…](../sources/prs/cutlass/PR-2881.md), [[CuTeDSL] Fix: SM100 block-scale gemm overlapping accumulator](../sources/prs/cutlass/PR-2995.md), [[Cute-DSL] Add option for issue_clc_query without multicast](../sources/prs/cutlass/PR-3021.md), [[Hopper CuTeDSL] Add grouped GEMM kernel example](../sources/prs/cutlass/PR-3091.md), [[CLI] add cutedsl fp16 gemm tutorial from 2 to 6](../sources/prs/cutlass/PR-3106.md), [Update blackwell tutorial to be compatible with 4.5-dev version](../sources/prs/cutlass/PR-3130.md), [feat: Adding varlen support to cute-dsl sm80 bwd](../sources/prs/flash-attention/PR-1934.md), [[Cute,Fwd,Sm100] fp8 e4m3 and e5m2 support](../sources/prs/flash-attention/PR-2109.md), [[Cute,Flex,Fwd] Allow vectorized score_mod definitions](../sources/prs/flash-attention/PR-2236.md), [[Cute,Sm100,Fwd] add MLA 64/512 with topk sparsity for MQA 128 heads](../sources/prs/flash-attention/PR-2441.md), [feat: masked layout fp4 gemm using cute-dsl](../sources/prs/flashinfer/PR-1331.md), [Add python API for masked grouped gemm](../sources/prs/flashinfer/PR-1481.md), [fix: update cutedsl masked moe gemm](../sources/prs/flashinfer/PR-1488.md), [fix: update masked moe gemm fp4 tensor reshape](../sources/prs/flashinfer/PR-1495.md), [feat: scaling at fp4 gemm epilogue](../sources/prs/flashinfer/PR-1498.md), [Add benchmark for cutedsl gemm](../sources/prs/flashinfer/PR-1502.md), [bugfix: Fix stream handling in cutedsl gemm](../sources/prs/flashinfer/PR-1509.md), [refactor fp4 masked gemm cute-dsl implementation and add manual cache](../sources/prs/flashinfer/PR-1521.md), [feat: initial support for SM103, SM110, SM120, SM121](../sources/prs/flashinfer/PR-1608.md), [Support output signals for overlapping for cutedsl gemm](../sources/prs/flashinfer/PR-1677.md), [[cute_dsl] add gemm + all reduce (two_shot)](../sources/prs/flashinfer/PR-1695.md), [fix: fix cannot import name 'cuda' from 'cuda' in CUDA13](../sources/prs/flashinfer/PR-1764.md), [tests: upgrade cutlass, fix import and skip non-SM100 cutedsl two shot allreduce](../sources/prs/flashinfer/PR-1812.md), [raise error for group_gemm_fp8_nt_groupwise then num_groups > 1 on sm120/121](../sources/prs/flashinfer/PR-1862.md), [enable sm103 moe dsl backend](../sources/prs/flashinfer/PR-2149.md), [Fix gemm allreduce two shot](../sources/prs/flashinfer/PR-2171.md), [feat: Fused RMSNorm + FP4 Quantization Kernels in CuTe-DSL](../sources/prs/flashinfer/PR-2233.md), [fix: Add global scale support and optional output allocation for RMSNorm+FP4Quant fusion kernels](../sources/prs/flashinfer/PR-2260.md), [[WIP] Refactor: simplify torch -> cute-dsl boilerplate and enable tvm-ffi for cute-dsl kernels](../sources/prs/flashinfer/PR-2279.md), [fix: In-place Residual Update for add_rmsnorm_fp4quant](../sources/prs/flashinfer/PR-2385.md), [feat: Add output_both_sf_layouts option to add_rmsnorm_fp4quant API](../sources/prs/flashinfer/PR-2395.md), [feat: cuteDSL fp4 moe for better DSR1 performance.](../sources/prs/flashinfer/PR-2398.md), [perf: improve gdn decode cute-dsl kernels](../sources/prs/flashinfer/PR-2405.md), [refactor: simplify fp4 rmsnorm](../sources/prs/flashinfer/PR-2421.md), [refactor: refactoring cuda code to cute-dsl (part 1)](../sources/prs/flashinfer/PR-2428.md), [Add cute-dsl backends to mxfp[8,4]_quantization for future refactor](../sources/prs/flashinfer/PR-2443.md), [Ameyn/gdn decode cutedsl kernel](../sources/prs/flashinfer/PR-2498.md), [refactor: Port upstream CUTLASS fixes and refactor grouped_gemm_nt_masked GEMM module location](../sources/prs/flashinfer/PR-2503.md), [[Bug] Fix spark unit test failures for test_add_rmsnorm_fp4_quant_cute_dsl](../sources/prs/flashinfer/PR-2573.md), [fix: cute dsl nvfp4 moe routing index error](../sources/prs/flashinfer/PR-2629.md), [feat: support mxfp4 & mxfp8 entrypoint for blackwell cutedsl dense gemm](../sources/prs/flashinfer/PR-2660.md), [Add cute dsl mla decode op](../sources/prs/flashinfer/PR-2743.md), [[CuTe DSL] Add modular FMHA prefill and MLA decode attention kernels](../sources/prs/flashinfer/PR-2805.md), [CuteDSL MoE fix redundant output buffer zeroing](../sources/prs/flashinfer/PR-2811.md), [feat: Add CuTe-DSL backend for NVFP4 quantization](../sources/prs/flashinfer/PR-2838.md), [feat: add pdl support for cute dsl mla decode kernel support](../sources/prs/flashinfer/PR-2901.md), [perf: Optimize CuTe-DSL fp4 and fp8 quantization kernels](../sources/prs/flashinfer/PR-2904.md), [feat: Add CuTe DSL grouped-gemm + combine fusion support](../sources/prs/flashinfer/PR-2944.md), [feat: add PDL support to rmsnorm_fp4quant and add_rmsnorm_fp4quant CuTe DSL kernels](../sources/prs/flashinfer/PR-3008.md), [Prevent MoE autotuner buffer overflow on large token buckets](../sources/prs/flashinfer/PR-3025.md), [feat: Add backend="b12x" for mm_fp4 on SM120](../sources/prs/flashinfer/PR-3051.md), [feat: Add b12x CuTe DSL fused MoE for SM120](../sources/prs/flashinfer/PR-3066.md), [cute-dsl fmha prefill (cubin integration): remove front-padding, add attention_sink, and pdl support](../sources/prs/flashinfer/PR-3181.md), [fix(sm12x): fix micro-kernel workspace sizing when routed_rows > num_local_experts](../sources/prs/flashinfer/PR-3191.md), [fix(cute_dsl/moe): make autotuner bucket configuration adapt to runtime input](../sources/prs/flashinfer/PR-3216.md), [Support Kimi K2.5 H64 CuTe DSL MLA decode](../sources/prs/flashinfer/PR-3235.md), [fix(cute_dsl/moe): unbias autotuner profiling for tile_size enumeration](../sources/prs/flashinfer/PR-3252.md), [feat(moe): add SM120 W4A16 b12x kernels](../sources/prs/flashinfer/PR-3271.md), [feat(cute_dsl/moe): deterministic balanced autotune profile inputs](../sources/prs/flashinfer/PR-3286.md), [feat(cute_dsl/moe): add `moe_output_memset_inplace` dense memset wrapper](../sources/prs/flashinfer/PR-3328.md), [Support single batch overlap](../sources/prs/sglang/PR-10422.md), [Replace [silu_and_mul_]scaled_fp4_group_quant by Flashinfer equivalent](../sources/prs/sglang/PR-12376.md), [[diffusion] kernel fusion: gated residual layernorm scale shift and layernorm scale shift kernel fusion for Qwen-Image, WAN and HunyuanVideo](../sources/prs/sglang/PR-14717.md), [[jit-kernel] Add CuTe DSL GDN Decode Kernel](../sources/prs/sglang/PR-15631.md), [Move fa4 from sgl-kernel to jit kernel](../sources/prs/sglang/PR-17353.md), [[diffusion] Diffusion norm fusion for z-image](../sources/prs/sglang/PR-18762.md), [[SGLang-Diffusion] Fix custom op fake impl missing eps default for torch.compile](../sources/prs/sglang/PR-19725.md), [[diffusion] fix bug of copy_if](../sources/prs/sglang/PR-20094.md), [[KDA] Support CuTeDSL KDA decode kernel](../sources/prs/sglang/PR-21203.md), [Refactor JIT kernel CI to use run_suite.py registration system](../sources/prs/sglang/PR-21239.md), [Add dedicated FlashInferCuteDslMoE layer for standard-path FP4 MoE](../sources/prs/sglang/PR-21339.md), [[Bugfix] Lazy-import CuteDSL KDA kernel to fix AMD/ROCm startup crash](../sources/prs/sglang/PR-21428.md), [feat: Support flashinfer_cutedsl MoE runner with flashinfer alltoall backend](../sources/prs/sglang/PR-22669.md), [[Refactor] Refactor DeepEP dispatcher](../sources/prs/sglang/PR-22822.md), [Reland Cute-DSL FP4 dense GEMM](../sources/prs/sglang/PR-23590.md), [Use Cute-DSL NVFP4 quantization kernels](../sources/prs/sglang/PR-23745.md), [[MoE Refactor] Migrate flashinfer_cutedsl + DeepEP to MoeRunner](../sources/prs/sglang/PR-25525.md), [[NVIDIA] [3/N] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked ](../sources/prs/sglang/PR-9199.md), [[Feature] Support cluster launch, query, synchronization and barrier operations](../sources/prs/tilelang/PR-1874.md), [[Feature] 2-SM support for TMA, TMEM and TCGEN5MMA on Blackwell](../sources/prs/tilelang/PR-1882.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [[Bugfix] Fix CuTeDSL autotune cache invalid ELF header (#1967)](../sources/prs/tilelang/PR-1972.md), [[Feature] Support TMA store in T.tma_copy()](../sources/prs/tilelang/PR-1981.md), [[Transform] Add InjectTcgen05Fence pass](../sources/prs/tilelang/PR-2003.md), [[CUDA] Support int4 `T.gemm`](../sources/prs/tilelang/PR-2063.md), [[TIR][IR] Update to use tirx](../sources/prs/tilelang/PR-2216.md), [[Python] Drop Python 3.9 support](../sources/prs/tilelang/PR-2218.md), [[MoE] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked](../sources/prs/vllm/PR-25990.md), [[FIX] Add NO_MUL activation support for modular kernel path](../sources/prs/vllm/PR-31528.md), [fix: Add SM120 (RTX Blackwell) support for FlashInfer CUTLASS NVFP4 MoE kernels](../sources/prs/vllm/PR-33417.md), [[Bug][MoE] Strengthen _supports_current_device() checks in the TRTLLM FP8, NVFP4, and FlashInfer CuteDSL MoE experts](../sources/prs/vllm/PR-36728.md), [[MoE/EPLB] Fix FlashInfer nvfp4 experts + EPLB correctness](../sources/prs/vllm/PR-37217.md), [[MoE] Move FlashInfer CuteDSL experts into fused_moe/experts/](../sources/prs/vllm/PR-37759.md), [[MoE Kernel] Flashinfer nvfp4 cutedsl moe kernel integration](../sources/prs/vllm/PR-38050.md), [[Quantization] Add FlashInfer CuteDSL batched experts backend for NVFP4 MoE](../sources/prs/vllm/PR-38251.md), [[Bugfix] [Tests] Enforce `out` tensor device in `kernel/moe/test_cutedsl_moe.py`](../sources/prs/vllm/PR-39644.md), [[DSv4] Improved fused Indexer Q quant kernel](../sources/prs/vllm/PR-41428.md), [[DSv4] Improved dequant gather K cache kernel](../sources/prs/vllm/PR-42236.md), [[Perf] Re-enable flashinfer autotune by default and cleanup](../sources/prs/vllm/PR-42857.md), [add cutedsl dsv4 indexer fp8 kernel](../sources/prs/vllm/PR-42899.md), [[Model Refactoring] Move deepseek_v4_ops to models/deepseek_v4 [3/N]](../sources/prs/vllm/PR-43073.md), [FlashAttention-4](../wiki/kernels/flash-attention-4.md), [FlashAttention SM100 MLA TopK Sparse Forward](../wiki/kernels/flash-attention-sm100-mla-topk.md), [FP8 Block-Scale GEMM](../wiki/kernels/fp8-block-scale-gemm.md), [Fused MoE — FP8 Block-Scale Routing + Dual GEMM](../wiki/kernels/fused-moe.md), [Gated Dual GEMM (Gate-Up + SwiGLU Fusion)](../wiki/kernels/gated-dual-gemm.md), [Grouped GEMM for MoE](../wiki/kernels/grouped-gemm.md), [NVFP4 GEMM — 4-bit Floating Point Matrix Multiply](../wiki/kernels/nvfp4-gemm.md), [Sparse MLA (DeepSeek V3.2)](../wiki/kernels/sparse-mla.md), [External Source-Map Research For Kernel Edits](../wiki/techniques/external-source-map-research.md) | +| `cuda-cpp` | [CUDA C++ for Blackwell Kernels](../wiki/languages/cuda-cpp.md) | [Twelve Attempts at an FP4 Kernel](../sources/blogs/amandeep-nvfp4-attempts.md), [Colfax Article Source Kernels](../sources/blogs/colfax-article-source-kernels.md), [Colfax CUTLASS Kernels](../sources/blogs/colfax-cutlass-kernels.md), [NVIDIA Developer Code Samples](../sources/blogs/nvidia-code-samples.md), [simveit effective_transpose](../sources/blogs/simveit-effective-transpose.md), [simveit load_and_store](../sources/blogs/simveit-load-and-store.md), [tcgen05 for dummies](../sources/blogs/tcgen05-tutorial.md), [Blackwell NVFP4 Kernel Hackathon Journey](../sources/blogs/yue-nvfp4-hackathon.md), [FlashInfer MLSys 2026 Track A: FP8 Block-Scale MoE](../sources/contests/flashinfer-mlsys26/track-a-fused-moe.md), [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [NVIDIA Blackwell Compatibility Guide](../sources/docs/blackwell-compatibility-guide.md), [Fix performance issue of m-grouped contiguous GEMMs.](../sources/prs/DeepGEMM/PR-168.md), [Fix multicast bug and optimize masked GEMM](../sources/prs/DeepGEMM/PR-193.md), [fix: use SM90ArchSpec instead of SM100ArchSpec in sm90_bf16_k_grouped_gemm](../sources/prs/DeepGEMM/PR-270.md), [[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes](../sources/prs/DeepGEMM/PR-304.md), [Sync nv_dev with upstream #316 (Mega MoE optimizations & benchmarks)](../sources/prs/DeepGEMM/PR-328.md), [ Solving bank conflict via padding and TMA 3D store](../sources/prs/DeepGEMM/PR-78.md), [Use 1D TMA store instead of 3D](../sources/prs/DeepGEMM/PR-83.md), [Use swizzling instead of padding](../sources/prs/DeepGEMM/PR-86.md), [Support TMA multicast on B with m_grouped_gemm_contiguous.](../sources/prs/DeepGEMM/PR-88.md), [[None][feat] sm100 weight-only kernel](../sources/prs/TensorRT-LLM/PR-10190.md), [[TRTLLM-10022][feat] Add hopper xqa decode support for skip softmax attention](../sources/prs/TensorRT-LLM/PR-10264.md), [[https://nvbugs/5669671][fix] Support GuidedDecoder with sharded logits (pick #10698)](../sources/prs/TensorRT-LLM/PR-10742.md), [[None][feat] fuse shared to sparse experts in TRT-LLM Gen MoE](../sources/prs/TensorRT-LLM/PR-11143.md), [[https://nvbugs/5799917][fix] Recover from CUTLASS MoE doActivation perf regression for MXFP4/NVFP4 dtype](../sources/prs/TensorRT-LLM/PR-11165.md), [[None][feat] Optimize super-v3 nvfp4 for better perf](../sources/prs/TensorRT-LLM/PR-11273.md), [[None][feat] Optimize by fuse nvfp4_quant to layernorm_gated for mamba2_mixer](../sources/prs/TensorRT-LLM/PR-11473.md), [[None][feat] TRT-LLM Gen MoE finalize kernel optimization](../sources/prs/TensorRT-LLM/PR-11501.md), [[None][feat] Add support for expert_number<=2048 and K<=32](../sources/prs/TensorRT-LLM/PR-11510.md), [[TRTLLM-10004][feat] Enable GEMM -> AR with GEMM output in registered buffers](../sources/prs/TensorRT-LLM/PR-11589.md), [[TRTLLM-11119][feat] Blackwell SageAttention, Integrate into AttentionOp API](../sources/prs/TensorRT-LLM/PR-11718.md), [[https://nvbugs/5799917][fix] Recover from CUTLASS MoE doActivation perf regression for MXFP4/NVFP4 dtype](../sources/prs/TensorRT-LLM/PR-11733.md), [[None][feat] Add fused DiT QK Norm + RoPE CUDA kernel for FLUX](../sources/prs/TensorRT-LLM/PR-11869.md), [[TRTLLM-10421][perf] Add fused cat+fp8_quantize CUDA kernel for DSA indexer](../sources/prs/TensorRT-LLM/PR-11899.md), [[#11694][feat] AutoDeploy: Improve the piecewise CG memory usage](../sources/prs/TensorRT-LLM/PR-11993.md), [[https://nvbugs/5955188][fix] Fix harmony parsers and WAR routing PDL for agentic coding use cases](../sources/prs/TensorRT-LLM/PR-12046.md), [[TRTLLM-11540][feat] Add EAGLE3 dynamic tree speculative decoding support](../sources/prs/TensorRT-LLM/PR-12062.md), [[None][feat] Minimax RMS norm optimization](../sources/prs/TensorRT-LLM/PR-12163.md), [[None][feat] Add fused allreduce+RMSNorm op and optional residual in …](../sources/prs/TensorRT-LLM/PR-12201.md), [[None][feat] Support update weight for nvfp4](../sources/prs/TensorRT-LLM/PR-12320.md), [[None][feat] Temporally-Correlated Heuristic-guided Indexer TopK for Sparse Attention](../sources/prs/TensorRT-LLM/PR-12385.md), [[None][perf] add Dynamic SMEM block routing in MOE](../sources/prs/TensorRT-LLM/PR-12456.md), [[None][feat] Support sparse mqa/gqa attention](../sources/prs/TensorRT-LLM/PR-12470.md), [[https://nvbugs/5983390][perf] Split MLA DSA custom op for piecewise CUDA graph capture](../sources/prs/TensorRT-LLM/PR-12503.md), [[None][feat] Add Mamba2 MTP SSM cache CUDA kernel for tree-based speculative decoding](../sources/prs/TensorRT-LLM/PR-12537.md), [[https://nvbugs/5983390][perf] Multiple host perf optimizations for DSA part](../sources/prs/TensorRT-LLM/PR-12581.md), [[None][feat] Trtllm-gen FMHA JIT support](../sources/prs/TensorRT-LLM/PR-12612.md), [[None][feat] Optimize mamba SSD prefill and extend flashinfer dispatch](../sources/prs/TensorRT-LLM/PR-12731.md), [[TRTLLM-11485][feat] Feature rework: Add SageAttention refreshed kernels (attentionOp only)](../sources/prs/TensorRT-LLM/PR-12937.md), [[None][feat] Update rms_norm + fp4_qaunt kernel supporting more dim](../sources/prs/TensorRT-LLM/PR-13033.md), [[#12716][feat] Fused cross-head QK Norm + RoPE kernel for WAN](../sources/prs/TensorRT-LLM/PR-13052.md), [[None][feat] Optimize causal_conv1d prefill and decode kernels](../sources/prs/TensorRT-LLM/PR-13103.md), [[None][feat] Add FP4 residual quantization kernel without channel reo…](../sources/prs/TensorRT-LLM/PR-13117.md), [[https://nvbugs/5945047][fix] Fix cluster launch enablement for SM120 GPUs in allReduce fusion](../sources/prs/TensorRT-LLM/PR-13169.md), [[None][feat] Integrate FP4 indexer for DSA on Blackwell](../sources/prs/TensorRT-LLM/PR-13340.md), [[None][perf] Extend customMoeRouting kernel to support Qwen3.5](../sources/prs/TensorRT-LLM/PR-13433.md), [[None][perf] Scheme X L2-aware dispatcher and PDL launchers for sparse-attention GVR Top-K](../sources/prs/TensorRT-LLM/PR-13477.md), [[None][perf] Drop cubin and Eliminate ~6s FMHA JIT recompile in eager generation by aligning kernel selection with CUDA graph warmup](../sources/prs/TensorRT-LLM/PR-13505.md), [[None][feat] Fuse FP8 1x128 quantize + UE8M0 scale pack on SM100](../sources/prs/TensorRT-LLM/PR-13628.md), [[#13580][fix] AutoDeploy: Support Gemma3n/4 E2B variants](../sources/prs/TensorRT-LLM/PR-13630.md), [[None][feat] Add DeepSeekV4 attention kernels](../sources/prs/TensorRT-LLM/PR-13652.md), [[https://nvbugs/6108841][fix] add hidden_dim=6144 router GEMM instantiation for GLM-5](../sources/prs/TensorRT-LLM/PR-13740.md), [[None][perf] Optimize DeepSeek-V4 compressor BF16 input](../sources/prs/TensorRT-LLM/PR-13761.md), [[None][fix] Plumb swiglu_limit through DeepGEMM and TRTLLMGen FP8 fused MoE](../sources/prs/TensorRT-LLM/PR-13767.md), [[None][fix] Fix fused MHC for DeepSeek-V4-Pro hidden size](../sources/prs/TensorRT-LLM/PR-13771.md), [[None][feat] Indexer topk opt](../sources/prs/TensorRT-LLM/PR-13811.md), [[None][perf] mHC fused_hc kernel optimizations + DS-V4 entry-boundary RMSNorm fold-in](../sources/prs/TensorRT-LLM/PR-13892.md), [[None][perf] Add CUDA q_b norm for DeepSeek V4](../sources/prs/TensorRT-LLM/PR-13975.md), [[None][fix] Add SPDX Apache-2.0 headers to auto_deploy test files](../sources/prs/TensorRT-LLM/PR-14193.md), [[None][feat] DSv4: enable GVR Heuristic Top-K for compress_ratio=4](../sources/prs/TensorRT-LLM/PR-14219.md), [[None][feat] Update the logic of FMHA JIT path](../sources/prs/TensorRT-LLM/PR-14291.md), [[None][chore] Update Claude Code agents and skills](../sources/prs/TensorRT-LLM/PR-14344.md), [feat: Add w4a8_mxfp4_fp8 quantization recipe.](../sources/prs/TensorRT-LLM/PR-4867.md), [[OMNIML-2336][feat] Add NVFP4 x FP8](../sources/prs/TensorRT-LLM/PR-6809.md), [[None][chore] Fix kernel launch param and add TRTLLM MoE backend test](../sources/prs/TensorRT-LLM/PR-7524.md), [[None][fix] Fix and add test for TRTLLM MoE backend](../sources/prs/TensorRT-LLM/PR-7755.md), [[TRTLLM-8637][feat] Optimize the routing kernel for DeepseekV3 (MoE CUTLASS backend); Add support for 384 experts (MoE TRTLLM backend)](../sources/prs/TensorRT-LLM/PR-7761.md), [[None][feat] GPT-OSS Sm120/Sm121 Support](../sources/prs/TensorRT-LLM/PR-7937.md), [[None][fix] Fix the performance issue of FP8 blockwise grouped GEMM when using attention DP](../sources/prs/TensorRT-LLM/PR-8501.md), [[None][feat] Enable nvfp4 cuda core for sm120](../sources/prs/TensorRT-LLM/PR-8620.md), [[None][feat] Update TRTLLM MoE cubins; reduce mxfp4 weight padding requirement; tighten TMA bound](../sources/prs/TensorRT-LLM/PR-9025.md), [[None][fix] support topk autotuner input for expert slot per group larger than 32](../sources/prs/TensorRT-LLM/PR-9087.md), [[None][feat] TRT-LLM Gen MoE optimize DeepSeek Fp8 activation kernel](../sources/prs/TensorRT-LLM/PR-9175.md), [[None][feat] add fp4 gemm + allreduce](../sources/prs/TensorRT-LLM/PR-9729.md), [[https://nvbugs/5726962][feat] Apply fusion for W4AFP8_AWQ MoE](../sources/prs/TensorRT-LLM/PR-9838.md), [[None][feat] Fused kernels (qknormrope + moe routing) and two-model MTP support for glm4moe](../sources/prs/TensorRT-LLM/PR-9852.md), [[None][feat] Port fp4 quantization kernel optimization from FlashInfer](../sources/prs/TensorRT-LLM/PR-9854.md), [[None][feat] Adding torch ext API for FusedAddRMSNormQuant kernel](../sources/prs/TensorRT-LLM/PR-9905.md), [[TRTLLM-9493][feat] Add helixPostProcessNative kernel for cp_dim=2](../sources/prs/TensorRT-LLM/PR-9924.md), [Experimental Python cooperative algorithms](../sources/prs/cccl/PR-1973.md), [fix thread-reduce performance regression](../sources/prs/cccl/PR-2944.md), [Fix scan / sm90 perf regression ](../sources/prs/cccl/PR-3236.md), [Fix the vectorized loading of BlockLoad](../sources/prs/cccl/PR-3517.md), [Add b200 tunings for scan.exclusive.sum](../sources/prs/cccl/PR-3559.md), [Fix SM100 histogram tunings](../sources/prs/cccl/PR-3691.md), [Split Optimize Warp Reduce PR - CUB part](../sources/prs/cccl/PR-4716.md), [Add nondeterministic reduce that uses atomics](../sources/prs/cccl/PR-4961.md), [CUB - Add internal integer utils and tests (Split `WarpReduce` PR)](../sources/prs/cccl/PR-5314.md), [Combine `block_reduce_warp_reduction_nondeterministic.cuh` specialization with original deterministic one ](../sources/prs/cccl/PR-5408.md), [Add dynamic CUB dispatch for segmented_sort](../sources/prs/cccl/PR-6069.md), [[CUB] Use `BlockLoadToShared` in `DeviceMerge`](../sources/prs/cccl/PR-6077.md), [Fix debug section around line 390 of dispatch_topk](../sources/prs/cccl/PR-6152.md), [Split fixed-size segmented reduce dispatch header](../sources/prs/cccl/PR-6597.md), [Integrate decoupled lookahead warpspeed scan](../sources/prs/cccl/PR-6811.md), [Use integer promotion for `warp_reduce`](../sources/prs/cccl/PR-6819.md), [Implement new tuning API arch dispatching](../sources/prs/cccl/PR-7093.md), [Two-phase reduction for fixed size segmented reduction for very large segment sizes](../sources/prs/cccl/PR-7114.md), [Implement the new tuning API for deterministic (rfa) reduce dispatch](../sources/prs/cccl/PR-7346.md), [Radix-selection based `BlockTopK` specialization](../sources/prs/cccl/PR-7384.md), [Implement the new tuning API for `DeviceRleDispatch`](../sources/prs/cccl/PR-7669.md), [Optimize non fixed size segmented reduce for small segments using max_segment_size](../sources/prs/cccl/PR-7718.md), [Add env SegmentedReduce (non fixed-size overloads)](../sources/prs/cccl/PR-7795.md), [Forward policy hub from `dispatch_streaming_arg_reduce_t` to `reduce::dispatch`](../sources/prs/cccl/PR-7805.md), [Implement the new tuning API for `detail::reduce::dispatch_streaming_arg_reduce_t`](../sources/prs/cccl/PR-7807.md), [Use the new tuning API internally for `detail::transform::dispatch`](../sources/prs/cccl/PR-7810.md), [[Backport branch/3.3.x] Forward policy hub from `dispatch_streaming_arg_reduce_t` to `reduce::dispatch`](../sources/prs/cccl/PR-7814.md), [Optimized Device-to-Device Tensor Copy (`cudax`)](../sources/prs/cccl/PR-7823.md), [Implement the new tuning API for `DispatchSegmentedRadixSort`](../sources/prs/cccl/PR-7844.md), [Implement the new tuning API for `DispatchSegmentedSort`](../sources/prs/cccl/PR-7874.md), [Implement the new tuning API for `DispatchTopK`](../sources/prs/cccl/PR-7928.md), [[cuda.compute]: Fix faulty pointer arithmetic calculation in CUB dispatch](../sources/prs/cccl/PR-7940.md), [Reduce usage of `cub::DispatchReduce`](../sources/prs/cccl/PR-7944.md), [Use the new tuning API for `detail::radix_sort::dispatch`](../sources/prs/cccl/PR-7949.md), [Adds support for non-fundamental types via decomposer to `DeviceTopK` ](../sources/prs/cccl/PR-8040.md), [Optimized Device-to-Device Tensor Copy (cudax) - Transpose Case](../sources/prs/cccl/PR-8125.md), [Avoid passing uninitialized values to scan_op](../sources/prs/cccl/PR-8184.md), [[STF] Move unstable_unique from STF to generic cudax utility](../sources/prs/cccl/PR-8190.md), [Expose `max_segment_size` guarantee in cuda.compute](../sources/prs/cccl/PR-8284.md), [Port `thrust::min|max_element` to CUB](../sources/prs/cccl/PR-8291.md), [Implement the new tuning API for `DispatchSelectIf`](../sources/prs/cccl/PR-8311.md), [simplify dispatch segmented reduce to use latest dispatch and new tunings API](../sources/prs/cccl/PR-8332.md), [Apply some random warpspeed tunings](../sources/prs/cccl/PR-8352.md), [[cub]: implement utilities for policy selection](../sources/prs/cccl/PR-8355.md), [Replace `detail::merge::dispatch` by CUB's public API](../sources/prs/cccl/PR-8381.md), [[CUB] Replace `Shuffle(Up|Down|Index)` with cuda::device::warp_shuffle - RadixSort only](../sources/prs/cccl/PR-8395.md), [Vectorize mbarrier initialization in warpspeed scan](../sources/prs/cccl/PR-8423.md), [[thrust] Single-pass `is_partitioned` via adjacent zip_iterator](../sources/prs/cccl/PR-8427.md), [Replace `detail::merge_sort::dispatch` by CUB's public API](../sources/prs/cccl/PR-8473.md), [Replace `detail::scan::dispatch` by CUB's public API](../sources/prs/cccl/PR-8495.md), [Implement the new tuning API for `detail::batched_topk::dispatch_batched_topk`](../sources/prs/cccl/PR-8538.md), [Replace `detail::for_each::dispatch` by CUB's public API](../sources/prs/cccl/PR-8565.md), [Replace `detail::segmented_reduce::dispatch` by the public API](../sources/prs/cccl/PR-8695.md), [Use the new tuning API internally for `detail::topk::dispatch`](../sources/prs/cccl/PR-8742.md), [Use the new tuning API internally for `detail::reduce_by_key::dispatch`](../sources/prs/cccl/PR-8756.md), [Use the new tuning API internally for `detail::reduce[_nd]::dispatch[_nd]`](../sources/prs/cccl/PR-8826.md), [Fix Warpspeed scan shifted output store](../sources/prs/cccl/PR-8839.md), [[cub] Simplify arch dispatch](../sources/prs/cccl/PR-8861.md), [Use the new tuning API internally for `detail::select::dispatch` and `DeviceSelect`](../sources/prs/cccl/PR-8880.md), [[STF] Add per-handle exec_place stream resources](../sources/prs/cccl/PR-8905.md), [Use the new tuning API internally for `detail::select|three_way_partition::dispatch` and `DevicePartition`](../sources/prs/cccl/PR-8925.md), [Use the new tuning API internally for `detail::segmented_radix_sort::dispatch`](../sources/prs/cccl/PR-8927.md), [[libcu++] Always suppress C++ extensions warnings in prologue](../sources/prs/cccl/PR-9019.md), [Fix segmented radix sort benchmark segment size type](../sources/prs/cccl/PR-9039.md), [[libcu++] Fix default make_shared_resource construction](../sources/prs/cccl/PR-9044.md), [Vectorize contiguous iterators in `cub::BlockLoad`/`Store`](../sources/prs/cccl/PR-9056.md), [Improve sm90 mixed dtype kernel](../sources/prs/cutlass/PR-1883.md), [[EVT] Add support for Row/Col broadcast PtrArray](../sources/prs/cutlass/PR-2033.md), [Groupwise scaling along M for FP8 gemm](../sources/prs/cutlass/PR-2037.md), [Improvements for: Groupwise scaling along M for FP8 gemm](../sources/prs/cutlass/PR-2095.md), [Flash MLA support](../sources/prs/cutlass/PR-2130.md), [Flash MLA Support - Step 2](../sources/prs/cutlass/PR-2134.md), [Blockwise and Groupwise GEMM for Blackwell and Improvements for Hopper](../sources/prs/cutlass/PR-2139.md), [Blockwise Improvement and Programmatic Dependent Launch](../sources/prs/cutlass/PR-2161.md), [Fix sm100 gemm wrong static constexpr that breaks compilation on Windows](../sources/prs/cutlass/PR-2167.md), [Fix SM90 beta=1 hang and stream-K launch errors](../sources/prs/cutlass/PR-2172.md), [Set EpiTile correctly when TileN is not divisible by 32](../sources/prs/cutlass/PR-2220.md), [Use cudaMemcpyAsync in gemm grouped with kRequiresPrecomputation sche…](../sources/prs/cutlass/PR-2256.md), [war to fix blackwell grouped groupwise hang](../sources/prs/cutlass/PR-2267.md), [hopper-blockwise-generalization-optimization](../sources/prs/cutlass/PR-2270.md), [Correct divmod order in example 77 (blackwell fmha)](../sources/prs/cutlass/PR-2291.md), [Handle get_masked_trip_count for small length in fmha example](../sources/prs/cutlass/PR-2292.md), [Fix epilogue::thread::Convert cannot be used with DefaultEpilogue](../sources/prs/cutlass/PR-2333.md), [[ex77] fix mla split; add fwd lse; add bwd varlen](../sources/prs/cutlass/PR-2366.md), [support fp16 accmulator for sm89 fp8 mma](../sources/prs/cutlass/PR-2378.md), [Example 77 add blackwell flash-attention bwd for MLA shape](../sources/prs/cutlass/PR-2466.md), [Add Blackwell MLA forward (shape: d=192, dv=128) implementation](../sources/prs/cutlass/PR-2472.md), [fix: examples/cute/tutorial/blackwell/04_mma_tma_2sm_sm100.cu GridDim miscalculated](../sources/prs/cutlass/PR-2492.md), [DistGEMM bug fixes](../sources/prs/cutlass/PR-2713.md), [Support PDL for SM90 Array TMA GEMM](../sources/prs/cutlass/PR-2719.md), [Support for GEMM-K=0 for Blackwell Grouped GEMMs](../sources/prs/cutlass/PR-2746.md), [Blockscaled Ragged Contiguous Grouped Gemm for MoEs](../sources/prs/cutlass/PR-2790.md), [[Bug Fix]Bypass launch grids for SM120 Kernel with SM90 Mainloop & SM100 TileScheduler](../sources/prs/cutlass/PR-2865.md), [[cute] Add constexpr specifier to make_tiled_copy](../sources/prs/cutlass/PR-2875.md), [Fix incorrect tensor layout strides in Blackwell MMA tutorial comments](../sources/prs/cutlass/PR-2921.md), [[Cutlass gemm] Fix SM100 FP8 nosmem epilogue-fusion shape_div 'Divisibility Condition' for non-multiple-of-64 N tiles](../sources/prs/cutlass/PR-2946.md), [[Bug Fix]Set NumSplitsM to 1 when TileShapeM < 128 in sm90 fp8 blockwise scaling CollectiveMma](../sources/prs/cutlass/PR-2965.md), [Replace std::min with cute::min in sm120 blockwise scaling device functions](../sources/prs/cutlass/PR-3055.md), [Small Tile N BlockScaled GEMM + Grouped GEMM on SM12x](../sources/prs/cutlass/PR-3176.md), [Add Snake activation functor for EVT](../sources/prs/cutlass/PR-3184.md), [Fp8 kernel with "in-kernel" transpose of V in producer](../sources/prs/flash-attention/PR-1100.md), [FA3 kvcache + split kv + gqa parallelization](../sources/prs/flash-attention/PR-1236.md), [Fix FA3 Varlen Performance regression](../sources/prs/flash-attention/PR-1361.md), [Add sorting and head swizzle to varlen scheduler](../sources/prs/flash-attention/PR-1823.md), [feat: update decode attention APIs](../sources/prs/flashinfer/PR-1007.md), [misc: fix instrument code for mla profiler](../sources/prs/flashinfer/PR-1014.md), [add multi-item scoring](../sources/prs/flashinfer/PR-1015.md), [fix: add zero init for KV tiled copy](../sources/prs/flashinfer/PR-1029.md), [feat: add functional per-head FP8 quantization for FA3](../sources/prs/flashinfer/PR-1033.md), [feat: Softmax free sampling](../sources/prs/flashinfer/PR-1035.md), [[nvidia] initial support for blackwell kernels](../sources/prs/flashinfer/PR-1039.md), [fix: top_k_mask_logits hangs on -inf inputs](../sources/prs/flashinfer/PR-1050.md), [[nvidia] Add Blackwell FMHA decode kernel from TRT-LLM](../sources/prs/flashinfer/PR-1051.md), [Fix KV chunking for POD. ](../sources/prs/flashinfer/PR-1054.md), [bugfix: temporally disable split-kv in blackwell mla](../sources/prs/flashinfer/PR-1055.md), [Parameterize prefix mask call (needed by POD-Attention)](../sources/prs/flashinfer/PR-1059.md), [bugfix: adding lse output to blackwell fmha kernels](../sources/prs/flashinfer/PR-1071.md), [bugfix: follow user-specified sm_scale for blackwell cutlass fmha](../sources/prs/flashinfer/PR-1072.md), [perf: accelerate blackwell grouped gemm](../sources/prs/flashinfer/PR-1086.md), [bugfix: fix fp8 attention kernels aot compilation issue](../sources/prs/flashinfer/PR-1087.md), [comm: refactor and initialize `flashinfer.comm` module](../sources/prs/flashinfer/PR-1089.md), [feat: add trtllm all-reduce (non-MoE)](../sources/prs/flashinfer/PR-1096.md), [bugfix: host-precomuted plan function for blackwell fmha](../sources/prs/flashinfer/PR-1106.md), [feat: add trtllm moe_allreduce_fusion](../sources/prs/flashinfer/PR-1108.md), [Add CUTLASS fused moe kernels from TensorRT-LLM.](../sources/prs/flashinfer/PR-1113.md), [bugfix: Fix test and output shape of fp4 quantize](../sources/prs/flashinfer/PR-1114.md), [hotfix: fix the blackwell fmha stream](../sources/prs/flashinfer/PR-1116.md), [[Feature] Support PDL for batch Prefill and Decode](../sources/prs/flashinfer/PR-1117.md), [Fix pointer dtype bug in rope](../sources/prs/flashinfer/PR-1129.md), [feat: add trtllm all-reduce fusion](../sources/prs/flashinfer/PR-1131.md), [MNNVL MoE All-to-All Support](../sources/prs/flashinfer/PR-1134.md), [fix: negative zero by type trait --> binary value](../sources/prs/flashinfer/PR-1136.md), [[feat] add unified batch attention w/ correctness tests.](../sources/prs/flashinfer/PR-1137.md), [Fix FA2 and FA3 multi-item scoring and cuda illegal memory access error](../sources/prs/flashinfer/PR-1140.md), [feat: Fused temperature online softmax kernel](../sources/prs/flashinfer/PR-1153.md), [Add more logging to TRTLLM-GEN debug trace (NFC)](../sources/prs/flashinfer/PR-1158.md), [feat: add finalize_moe_allreduce from trtllm](../sources/prs/flashinfer/PR-1159.md), [feat: nvshmem python bindings](../sources/prs/flashinfer/PR-1160.md), [feat: update non-fused moe](../sources/prs/flashinfer/PR-1161.md), [feat: enable and update all-reduce fused quantization](../sources/prs/flashinfer/PR-1164.md), [bugfix: softmax NaN results caused by large -inf masks](../sources/prs/flashinfer/PR-1178.md), [update trtllm-gen decode attention kernel launcher](../sources/prs/flashinfer/PR-1189.md), [[feat] optimize persistent batch attention perf.](../sources/prs/flashinfer/PR-1200.md), [[fix] fix BatchAttention CTA_TILE_KV mask issue](../sources/prs/flashinfer/PR-1206.md), [Fix the issue with auxillary kernel launch and grid dim calculation](../sources/prs/flashinfer/PR-1208.md), [feat: trtllm-gen fp8 moe kernels](../sources/prs/flashinfer/PR-1212.md), [[comm] TRT-LLM's Multi-Node NVLink All-Reduce Kernel](../sources/prs/flashinfer/PR-1213.md), [Feature/sm100 low latency nvfp4 kernels](../sources/prs/flashinfer/PR-1214.md), [Enable cudnn decode and add tests for the cudnn decode kernel](../sources/prs/flashinfer/PR-1221.md), [feat: add trtllm-gen mla cubin](../sources/prs/flashinfer/PR-1222.md), [Fix missing hash in the cudnn cubin path](../sources/prs/flashinfer/PR-1227.md), [feat: Add non-causal cudnn prefill kernels](../sources/prs/flashinfer/PR-1230.md), [bugfix: support uint8_t for vec_t class template](../sources/prs/flashinfer/PR-1234.md), [add trtllm-gen context attention](../sources/prs/flashinfer/PR-1239.md), [Patch fp8 cubin availability](../sources/prs/flashinfer/PR-1240.md), [feat: Support MXFP8 x MXFP4 CUTLASS grouped GEMM](../sources/prs/flashinfer/PR-1241.md), [Add trtllm-gen attention mha kernel with FP8 Q/K/V and FP8 output](../sources/prs/flashinfer/PR-1242.md), [Remove sm100+ requirment for trtllm allreduce kernels](../sources/prs/flashinfer/PR-1249.md), [Reduce the JIT compilation time of gen_gemm_sm100_module](../sources/prs/flashinfer/PR-1251.md), [TRT-LLM's Multi-Node NVLink AR + fused RMSNorm kernel](../sources/prs/flashinfer/PR-1255.md), [feat: enable trtllm-gen mla MTP](../sources/prs/flashinfer/PR-1258.md), [Made AR output optional + esthetic changes](../sources/prs/flashinfer/PR-1265.md), [Bug fix: fix duplicate launch in POD](../sources/prs/flashinfer/PR-1267.md), [Add shuffle matrix flag](../sources/prs/flashinfer/PR-1272.md), [Convert scale_factor from scalar to Tensor in trt_allreduce_fusion](../sources/prs/flashinfer/PR-1284.md), [fix multiCtasKvScratchPtr misalignment issue (new one)](../sources/prs/flashinfer/PR-1286.md), [Bug fix: guard fp8 e8m0 and e2m1 compile ](../sources/prs/flashinfer/PR-1287.md), [refactor: refactor trtllm-gen attention kernel integration code](../sources/prs/flashinfer/PR-1289.md), [[fix] fix integer overflow in FA2 customized_mask & add buffer overflow warning.](../sources/prs/flashinfer/PR-1290.md), [refactor: Improved metainfo for trtllm-gen fmha](../sources/prs/flashinfer/PR-1292.md), [Update cutlass fp4 moe kernels](../sources/prs/flashinfer/PR-1294.md), [add cutlass backend for mm_fp4](../sources/prs/flashinfer/PR-1296.md), [feat: Add weight layout option for trtllm-gen fused moe](../sources/prs/flashinfer/PR-1297.md), [perfix: use lightweight API to query device property](../sources/prs/flashinfer/PR-1298.md), [[Feature] SM level profiler ](../sources/prs/flashinfer/PR-1305.md), [Fix the bug of the kernel-selection heuristic in trtllm-gen](../sources/prs/flashinfer/PR-1307.md), [Refactor Fused Moe Module](../sources/prs/flashinfer/PR-1309.md), [feat: support output nvfp4 in trtllm-gen function call.](../sources/prs/flashinfer/PR-1318.md), [Make Fp8 MoE routing_bias optional](../sources/prs/flashinfer/PR-1319.md), [Add blockwise-scaled FP8 GEMM via TRTLLM-Gen.](../sources/prs/flashinfer/PR-1320.md), [Optimizations for TRTLLM MNNVL Allreduce](../sources/prs/flashinfer/PR-1321.md), [feat: Add k_scale and v_scale to persistent attention ](../sources/prs/flashinfer/PR-1322.md), [feat: Support logits_soft_cap for Persistent attn; fix kv split limit](../sources/prs/flashinfer/PR-1324.md), [feat: Fused rope fp8 quantize kernel for MLA](../sources/prs/flashinfer/PR-1339.md), [fix: fix trtllm-gen mla error on new interface](../sources/prs/flashinfer/PR-1348.md), [feature: add fp4 mm using trtllm backend](../sources/prs/flashinfer/PR-1355.md), [support trtllm-gen prefill fp4 output](../sources/prs/flashinfer/PR-1360.md), [Support scale factor start index for fp4 mha prefill/decode](../sources/prs/flashinfer/PR-1363.md), [bugfix: fixed cutlass fused moe usage of FP4QuantizationSFLayout::SWIZZLED](../sources/prs/flashinfer/PR-1371.md), [bugfix: Add guard for fp4/fp8 related include headers](../sources/prs/flashinfer/PR-1376.md), [GPT-OSS Support: Add Blackwell MoE mxfp4 implementation from TRTLLM and Attention Sink](../sources/prs/flashinfer/PR-1389.md), [gpt-oss: Add MXFP8 x MXFP4 CUTLASS MOE for SM100 and BF16 x MXFP4 CUTLASS for SM90 + SwigluBias Activation](../sources/prs/flashinfer/PR-1396.md), [feature: add cutlass as bmm_fp8 backend.](../sources/prs/flashinfer/PR-1397.md), [Fix trtllm moe launcher local_num_experts](../sources/prs/flashinfer/PR-1398.md), [fix shared memory alignment conflict in sampling.cuh](../sources/prs/flashinfer/PR-1402.md), [[bugfix] Fix compilation failure when compiling csrc/trtllm_moe_allreduce_fusion.cu](../sources/prs/flashinfer/PR-1410.md), [Fixes for Blackwell Tests](../sources/prs/flashinfer/PR-1434.md), [fix: remote redundant zero_init from trtllm-gen attn](../sources/prs/flashinfer/PR-1444.md), [Add alignment in MxFP8Quantization](../sources/prs/flashinfer/PR-1445.md), [Remove getEnvEnablePDL in favor of enable_pdl parameter](../sources/prs/flashinfer/PR-1446.md), [bugfix: Verify num_experts greater or equal to local_experts + offset](../sources/prs/flashinfer/PR-1469.md), [perf: add 1x4x1 cluster shape for fp8 bmm M<16 cases](../sources/prs/flashinfer/PR-1473.md), [tuner: Trtllm-gen Fp4 MoE Autotunner](../sources/prs/flashinfer/PR-1475.md), [perf: add fast path to TopPRenormProbKernel for top_p >= 1.0, significantly boosting SGLang workloads](../sources/prs/flashinfer/PR-1483.md), [feat: add pdl for trtllm-gen attn](../sources/prs/flashinfer/PR-1484.md), [feat: Support fp8 qkv, fp16/bf16 out MHA for trtllm-gen.](../sources/prs/flashinfer/PR-1490.md), [Perf: support scale_a/scale_b instead of combined scale in cutlass bmm_fp8](../sources/prs/flashinfer/PR-1491.md), [fix: Replace cub Max/Min with cuda::maximum/minimum for cuda 13 compatibility](../sources/prs/flashinfer/PR-1500.md), [feat: integrate xqa attention backend](../sources/prs/flashinfer/PR-1503.md), [update allreduce to match trtllm](../sources/prs/flashinfer/PR-1507.md), [Support cuda<12.8 built for trtllm_allreduce_fusion.](../sources/prs/flashinfer/PR-1508.md), [backend: Refactor trtllm-gen fmha metainfo loading](../sources/prs/flashinfer/PR-1518.md), [Fix linking errors with CUDA 13](../sources/prs/flashinfer/PR-1523.md), [Add GeGLU support to trtllm-gen NVFP4 Fused MoE Kernel](../sources/prs/flashinfer/PR-1525.md), [bugfix: Fix compile error for undefined swizzle enum.](../sources/prs/flashinfer/PR-1530.md), [bugfix: Fix Persistent kernel precision for masked output ](../sources/prs/flashinfer/PR-1533.md), [Remove cuda-python from dependency and check at runtime](../sources/prs/flashinfer/PR-1534.md), [feat: Integrate TRTLLM varlen kernel for deepseek R1 prefill ](../sources/prs/flashinfer/PR-1537.md), [fix trtllm_allreduce_fusion twoshot register problem.](../sources/prs/flashinfer/PR-1545.md), [perf: replace cudaGetDeviceProperties with cudaDeviceGetAttribute](../sources/prs/flashinfer/PR-1547.md), [perf: Enable SplitK and fix tile-scheduling for moe fp4 fused moe](../sources/prs/flashinfer/PR-1548.md), [Add mnnvl_moe_alltoallv_prepare_without_allgather](../sources/prs/flashinfer/PR-1550.md), [bugfix: fix persistent attention kernel correctness on blackwell](../sources/prs/flashinfer/PR-1559.md), [fix: separate out fp4 lib into sm90 and sm100 versions, add oob checking in fused moe](../sources/prs/flashinfer/PR-1565.md), [Backend: downgrade trtllm-gen kernel to cuda-12](../sources/prs/flashinfer/PR-1567.md), [bugfix: fix cuda version guard macros](../sources/prs/flashinfer/PR-1571.md), [update trtllm-gen fp4 autotuner and routing](../sources/prs/flashinfer/PR-1573.md), [bugfix: update trtllm-gen gemm kernel names](../sources/prs/flashinfer/PR-1577.md), [bugfix: Fix arg passing to TORCH_CHECK and TORCH_WARN macros](../sources/prs/flashinfer/PR-1582.md), [fix: semaphoress must be at the fixed range in workspace buffer on trtllm_gen attention](../sources/prs/flashinfer/PR-1584.md), [bugfix: fix fused-temperature softmax IMA issue](../sources/prs/flashinfer/PR-1596.md), [bugfix: fix the register overflow issue for topk renorm kernels on blackwell](../sources/prs/flashinfer/PR-1597.md), [feat: initial support for SM103, SM110, SM120, SM121](../sources/prs/flashinfer/PR-1608.md), [feat: cutlass fp4 gemm bringup for SM120 & SM121](../sources/prs/flashinfer/PR-1609.md), [feat: cutlass fp8 gemm bringup for SM120 & SM121](../sources/prs/flashinfer/PR-1610.md), [bugfix: fix fp4 quantization with 8x4 scale factor layout](../sources/prs/flashinfer/PR-1611.md), [bugfix: fix merge_attention_state in BatchAttention w/ gqa-group-size in Qwen family](../sources/prs/flashinfer/PR-1614.md), [perf: Fix the tactic sorting in TrtllmGenBatchedGemmRunner::getValidConfigIndices](../sources/prs/flashinfer/PR-1615.md), [bugfix: collect all modules to aot](../sources/prs/flashinfer/PR-1622.md), [bugfix: trtllm-gen fmha sm101 and sm100 compatibility](../sources/prs/flashinfer/PR-1631.md), [perf&bugfix: skip kv-tile computation out of sliding window in FA2; fix __syncthreads in mergestate](../sources/prs/flashinfer/PR-1661.md), [Refactor Blackwell unit test scripts](../sources/prs/flashinfer/PR-1667.md), [TGV GEMM as a BF16 backend alternative to cuBLAS](../sources/prs/flashinfer/PR-1668.md), [feat: Add `variant.OutputTransform()` to decode kernels](../sources/prs/flashinfer/PR-1670.md), [feat: Batch-size invariant FA2 Prefill & Decode](../sources/prs/flashinfer/PR-1675.md), [perf: improve attention of tcgen05 flash-attention](../sources/prs/flashinfer/PR-1681.md), [Update TGV GEMM default kernel and TGV code cleanup.](../sources/prs/flashinfer/PR-1682.md), [perf: Port the separate reduce kernel mode from trtllm.](../sources/prs/flashinfer/PR-1685.md), [Support Kimi-K2 for TRT: templatize number of experts](../sources/prs/flashinfer/PR-1696.md), [Fix DeepSeek quality for TRTLLM fused MoE routing](../sources/prs/flashinfer/PR-1723.md), [bugfix: partially fix tests/test_trtllm_gen_fused_moe.py unit test failure](../sources/prs/flashinfer/PR-1724.md), [TVM: support TVM binding for GroupedGemm](../sources/prs/flashinfer/PR-1725.md), [fix: put sampling kernel launch into macro](../sources/prs/flashinfer/PR-1727.md), [fix: fix cannot import name 'cuda' from 'cuda' in CUDA13](../sources/prs/flashinfer/PR-1764.md), [feat: add xqa fp8 mha and fp8 kv cache](../sources/prs/flashinfer/PR-1769.md), [Masked batch nvfp4 quantization](../sources/prs/flashinfer/PR-1774.md), [feat:enable fp8 blockscale moe for fused cultass for sm90](../sources/prs/flashinfer/PR-1819.md), [Bugfix: Fix data hazard in persistent reduce](../sources/prs/flashinfer/PR-1826.md), [feat: trtrllm-gen global scaled FP8 GEMMs](../sources/prs/flashinfer/PR-1829.md), [Update the routing for TRTLLMGEN to support kimi k2 and qwen](../sources/prs/flashinfer/PR-1831.md), [[Quantization] Add per-expert global scaling factor for fp4 batched quantize](../sources/prs/flashinfer/PR-1835.md), [Add head_dim=64 for tcgen05 tcgen05 flash-attention implementation](../sources/prs/flashinfer/PR-1850.md), [Bugfix: fix o_strides in persistent kernel ](../sources/prs/flashinfer/PR-1865.md), [Tune kernel compilation parameters for https://github.com/flashinfer-ai/flashinfer/pull/1850 ](../sources/prs/flashinfer/PR-1878.md), [feat: Add FP4 TRTLLM-Gen throughput MOE batched gemms](../sources/prs/flashinfer/PR-1882.md), [MLA RoPE + quantization fused kernel: shape generalization for MHA / GQA](../sources/prs/flashinfer/PR-1924.md), [Add layernorm op for inputs of mixed dtype](../sources/prs/flashinfer/PR-1926.md), [silu_and_mul nvfp4 quanization fusion rework](../sources/prs/flashinfer/PR-1927.md), [Feature: Support Relu2 activation in fused MoE](../sources/prs/flashinfer/PR-1954.md), [Update trtllm-gen fused moe routing kernel and add more kernels](../sources/prs/flashinfer/PR-1955.md), [Fix: Verify scales are not None for Cutlass FP8 FusedMoE](../sources/prs/flashinfer/PR-1961.md), [feat: enable deepgemm jit for fp8 block-scale on SM90](../sources/prs/flashinfer/PR-1969.md), [feat: autotune tile_tokens_dim in trtllm-gen MOE](../sources/prs/flashinfer/PR-1980.md), [fix: correct PDL parameter handling in RopeQuantize kernel](../sources/prs/flashinfer/PR-1982.md), [minor fix for xqa](../sources/prs/flashinfer/PR-1994.md), [Bugfix: Change get() -> GetDLTensorPtr() in cutlass FusedMoE validations](../sources/prs/flashinfer/PR-1995.md), [feat: add xqa backend and completes NHD/HND coverage for trtllm-gen/xqa backend](../sources/prs/flashinfer/PR-2001.md), [Feature: Support non-gated activation in cutlass fused MoE nvfp4](../sources/prs/flashinfer/PR-2011.md), [[feat] Refactor trtllmgen MOE and add Bf16 trtllmgen moe](../sources/prs/flashinfer/PR-2014.md), [[DSV3] Optimized Router Gemm](../sources/prs/flashinfer/PR-2019.md), [update trtllm cutlass moe ](../sources/prs/flashinfer/PR-2020.md), [perf: Speed up fp4 quantization for small batch with swizzling for cutlass MoE](../sources/prs/flashinfer/PR-2025.md), [[NVIDIA] Thor & Spark Support](../sources/prs/flashinfer/PR-2028.md), [Enable renormalize(naive) routing for fp8 per-tensor](../sources/prs/flashinfer/PR-2030.md), [use scalar for kv_scale in xqa](../sources/prs/flashinfer/PR-2033.md), [feat: Add flashinfer.rope.rope_quantize_fp8_append_paged_kv_cache (fused RoPE + Q + KV cache, supports MLA/GQA/MHA) ](../sources/prs/flashinfer/PR-2037.md), [perf: improve sampling/mask/softmax performance (part 1/2)](../sources/prs/flashinfer/PR-2044.md), [Rebase FP8 SM100 Cutlass FMHA Attention to main (original PR#1238)](../sources/prs/flashinfer/PR-2047.md), [Fix dtype of output scales from mnnvl_moe_alltoallv_prepare_without_allgather](../sources/prs/flashinfer/PR-2048.md), [[BUG] Fix trtllm-gen fp4 moe renormalize routing](../sources/prs/flashinfer/PR-2049.md), [Add support for topkPacked input in block-level renormalize](../sources/prs/flashinfer/PR-2051.md), [feat: add xqa mla backend](../sources/prs/flashinfer/PR-2053.md), [perf: Optimize helper max/minmax function in sampling.cuh](../sources/prs/flashinfer/PR-2058.md), [Fix moe fp8 failure for sm121](../sources/prs/flashinfer/PR-2061.md), [Fix: several bugs/issues with trtllm-gen attention kernels. ](../sources/prs/flashinfer/PR-2062.md), [perf: TRT-LLM MoE Block-FP8 activation optimization](../sources/prs/flashinfer/PR-2063.md), [feat: BF16 GEMM using CUTLASS backend for SM100](../sources/prs/flashinfer/PR-2070.md), [[Feature] Support batch prefill for POD Attention](../sources/prs/flashinfer/PR-2079.md), [enable xqa fp8 output](../sources/prs/flashinfer/PR-2081.md), [[API change] Allow using torch.Tensor for scales for trtllm-gen attention](../sources/prs/flashinfer/PR-2084.md), [refactor: pass hopper deepgemm include directory through python](../sources/prs/flashinfer/PR-2090.md), [perf: TRT-LLM Gen finalize kernel optimization](../sources/prs/flashinfer/PR-2092.md), [perf: enable pdl for cutlass fp4 gemm](../sources/prs/flashinfer/PR-2095.md), [Port TRT-LLM communication kernels to flashinfer](../sources/prs/flashinfer/PR-2102.md), [enable xqa speculative decoding](../sources/prs/flashinfer/PR-2105.md), [feat: support more head dim in RoPE kernel](../sources/prs/flashinfer/PR-2109.md), [add tensor scale input for xqa](../sources/prs/flashinfer/PR-2110.md), [refactor: update fa3 codebase and fix hopper unittest [part 1]](../sources/prs/flashinfer/PR-2111.md), [feature: make the LSE returned by MLA support base 2 or e #2113](../sources/prs/flashinfer/PR-2114.md), [update xqa license](../sources/prs/flashinfer/PR-2117.md), [Refactor trtllm_mnnvl_allreduce](../sources/prs/flashinfer/PR-2118.md), [perf: bunch of features and optimizations for top-k (sampling + sparse attention)](../sources/prs/flashinfer/PR-2119.md), [feat: support variable sequence length in decode kernel of trtllm-gen attention](../sources/prs/flashinfer/PR-2125.md), [fix flaky xqa test](../sources/prs/flashinfer/PR-2126.md), [make DeepGEMM swapAB available for linear gemm SM90](../sources/prs/flashinfer/PR-2131.md), [feat: add trtllm-gen per-tensor sparseMla kernels.](../sources/prs/flashinfer/PR-2138.md), [feat: TRTLLM FMHAv2 backend for ctx attention](../sources/prs/flashinfer/PR-2142.md), [fix xqa mha_sm90.cu](../sources/prs/flashinfer/PR-2157.md), [feat: MxInt4 x Bf16 TRT-LLM Gen MoE support](../sources/prs/flashinfer/PR-2159.md), [Add data type check for deepseek fp4 moe](../sources/prs/flashinfer/PR-2165.md), [Fix for moe on sm110](../sources/prs/flashinfer/PR-2190.md), [feat: unit-test and api change, w4a8 grouped-gemm fused MoE for SM90](../sources/prs/flashinfer/PR-2193.md), [Move the run function definition out of BatchedGemmInterface](../sources/prs/flashinfer/PR-2211.md), [feat: further optimize top-k and add fused top-k page construction kernels for DSA](../sources/prs/flashinfer/PR-2215.md), [feat: Support unpadded output hidden size for trtllm_fp4_block_scale_moe](../sources/prs/flashinfer/PR-2217.md), [fix: add DeepSeek routing for Bf16xBf16 and MxIntxBf16 TRT-LLM Gen MoE](../sources/prs/flashinfer/PR-2234.md), [refactor: pull trtllm-gen batch-gemm/gemm headers from artifactory; update tma descriptor shape init](../sources/prs/flashinfer/PR-2235.md), [[feat] Integrate SGLang concat_mla_k kernel into flashinfer](../sources/prs/flashinfer/PR-2237.md), [feat: RMSNorm/Fused RMSNorm + FP8 Quantization kernels](../sources/prs/flashinfer/PR-2243.md), [Remove cudaStreamSynchronize from gemm_groupwise_sm120.cuh for CUDA graph compatibility](../sources/prs/flashinfer/PR-2244.md), [feat: Support numLocalTokens=0 for moe All-to-all](../sources/prs/flashinfer/PR-2247.md), [feat: support non-contiguous query for trtllm-gen attention backend](../sources/prs/flashinfer/PR-2254.md), [fix: support int64 IdType for RoPE part argument in `rope_quantize_fp8_append_paged_kv_cache`](../sources/prs/flashinfer/PR-2255.md), [[TRTLLM-Gen Fmha] add optimized trtllm-gen decode kernels for high throughput + speculative decoding](../sources/prs/flashinfer/PR-2265.md), [[performance]optimize for nvfp4](../sources/prs/flashinfer/PR-2268.md), [feat: add GDN Attention](../sources/prs/flashinfer/PR-2276.md), [feat: IdType indices in sampling kernels](../sources/prs/flashinfer/PR-2281.md), [Selective State Update kernel (mamba)](../sources/prs/flashinfer/PR-2301.md), [[Perf][Feature] Add SM103-specific schedulers for NVFP4 CUTLASS kernels](../sources/prs/flashinfer/PR-2303.md), [feat: Support Fused MoE non gated Relu2 NVFP4 & FP8 and support Nemotron](../sources/prs/flashinfer/PR-2304.md), [Fix: FilteredTopKUnifiedKernel read value out of length](../sources/prs/flashinfer/PR-2308.md), [[ML3] Optimized Router Gemm](../sources/prs/flashinfer/PR-2323.md), [bugfix: fix multi-cta top-k implementation when k value is different for different row](../sources/prs/flashinfer/PR-2325.md), [[perf] Improve gemm_fp8_nt_groupwise (cutlass backend) by 10-40% for batch sizes <= 32](../sources/prs/flashinfer/PR-2327.md), [fix: guard batchWarpReduceSum with ENABLE_FP8 to fix compilation without FP8](../sources/prs/flashinfer/PR-2328.md), [feat: expose swizzled_input_sf parameter for CUTLASS fused MOE](../sources/prs/flashinfer/PR-2330.md), [Optimize quantization function in large problem size](../sources/prs/flashinfer/PR-2343.md), [Enable fp16/bf16/f32 support for selective_state_update (mamba)](../sources/prs/flashinfer/PR-2366.md), [bugfix: hotfix of PR 2366 (mamba kernel)](../sources/prs/flashinfer/PR-2378.md), [fix: ensure each CTA processes full numHeadsQPerKv for trtllm decode kernel](../sources/prs/flashinfer/PR-2380.md), [A Blackwell-optimized version of selective_state_update (decode)](../sources/prs/flashinfer/PR-2387.md), [feat: cuteDSL fp4 moe for better DSR1 performance.](../sources/prs/flashinfer/PR-2398.md), [Remove cudaMalloc/Free in GDN prefill kernel](../sources/prs/flashinfer/PR-2415.md), [feat: update trtllm-gen MoE cubins](../sources/prs/flashinfer/PR-2416.md), [refactor: reduce hopper's gdn prefill compilation time and fix docstring.](../sources/prs/flashinfer/PR-2422.md), [refactor: refactoring cuda code to cute-dsl (part 1)](../sources/prs/flashinfer/PR-2428.md), [fix: Sampling: CUDA Graph fix](../sources/prs/flashinfer/PR-2432.md), [fix: Fix NaN output in mxfp8_quantize for very small input values](../sources/prs/flashinfer/PR-2441.md), [MTP for mamba ](../sources/prs/flashinfer/PR-2444.md), [bugfix: fix stub generation directory in fused_moe module](../sources/prs/flashinfer/PR-2445.md), [feat: Add TRTLLM fmha_v2 library for SM90 attention with Skip-Softmax ](../sources/prs/flashinfer/PR-2446.md), [fix: fix illegal memory access for NaN input in sampling kernels](../sources/prs/flashinfer/PR-2456.md), [feat: Support Fused MoE non gated Relu2 NVFP4 & FP8 and support Nemotron, fixed](../sources/prs/flashinfer/PR-2462.md), [feat: Add MXFP8 GEMM mm_mxfp8 (cutlass)](../sources/prs/flashinfer/PR-2464.md), [feat: Add TRTLLM-Gen Skip-Softmax kernels for prefill and decode](../sources/prs/flashinfer/PR-2477.md), [fix: add support check for gemm config for cutlass moe](../sources/prs/flashinfer/PR-2495.md), [Feat: Trtllm-gen MxFP8 MoE integration](../sources/prs/flashinfer/PR-2505.md), [perf: cache cudaGetDeviceProperties in gdn_prefill to avoid per-call overhead](../sources/prs/flashinfer/PR-2509.md), [Support NVFP4 KV cache decode on SM120](../sources/prs/flashinfer/PR-2520.md), [misc: point triton blackwell-ptxas to local cuda ptxas](../sources/prs/flashinfer/PR-2543.md), [feat: Enable TRTLLM-Gen Skip-Softmax attention for MLA](../sources/prs/flashinfer/PR-2547.md), [[Bugfix][comm] Fix FP4 one-shot launch config instability in trtllm_allreduce_fusion](../sources/prs/flashinfer/PR-2557.md), [Add support for the combinations of allreduce, allgather, and reducescatter](../sources/prs/flashinfer/PR-2563.md), [fix: W4A8 autotune crash in cutlass_fused_moe profiler workspace](../sources/prs/flashinfer/PR-2564.md), [Implement `cutlass_fused_moe` mxfp8](../sources/prs/flashinfer/PR-2581.md), [feat: trtllm tinygemm2 in flashinfer as bf16 routergemm](../sources/prs/flashinfer/PR-2587.md), [Mamba SSU: better automatic kernel selection + algorithm selection optionally exposed to the user.](../sources/prs/flashinfer/PR-2591.md), [[bugfix] Fix FilteredTopK overflow correctness](../sources/prs/flashinfer/PR-2605.md), [fix: cute dsl nvfp4 moe routing index error](../sources/prs/flashinfer/PR-2629.md), [fix: add SM121 support to SM120 version guards](../sources/prs/flashinfer/PR-2631.md), [[fp8_blockwise]Fix int32 overflow in TRTLLM fused MoE activation kernel](../sources/prs/flashinfer/PR-2642.md), [feat: FP32 dtype output for BF16 matmuls (CUTLASS & cuDNN)](../sources/prs/flashinfer/PR-2644.md), [int16 Block-Scaled State and Stochastic Rounding for SSU (mamba)](../sources/prs/flashinfer/PR-2645.md), [[feat] trtllm-gen mxfp8 gemm](../sources/prs/flashinfer/PR-2653.md), [feat: implement deterministic topk](../sources/prs/flashinfer/PR-2661.md), [perf: Update trtllm-gen batched GEMM kernels - faster, more NVFP4 tile dims, MXFP8 with relu2 act](../sources/prs/flashinfer/PR-2667.md), [fix: reduce smem allocation for tinygemm2 kernel in SM120](../sources/prs/flashinfer/PR-2670.md), [feat: add support for more MLA head dimensions](../sources/prs/flashinfer/PR-2677.md), [Add varlen and speculative decoding support to selective state update](../sources/prs/flashinfer/PR-2700.md), [Add NVFP4 KV cache quantization support for SM100](../sources/prs/flashinfer/PR-2702.md), [feat: Add support for TRTLLM MXFP8 non-gated MoE with ReLU2](../sources/prs/flashinfer/PR-2707.md), [Mamba2 SSD Combined Forward Pass (Blackwell CuTe DSL Kernel)](../sources/prs/flashinfer/PR-2709.md), [feat: Add DiT-oriented kernels where Qk (Bmm1) type can be reinterpreted into Int8 or BFloat16](../sources/prs/flashinfer/PR-2711.md), [fix: Add SM120 (RTX Blackwell desktop) support for NVFP4 MoE kernels](../sources/prs/flashinfer/PR-2725.md), [Support for MXFP4 and NVFP4 group GEMMs on GeForce and Spark](../sources/prs/flashinfer/PR-2738.md), [misc: Update gemm/batched gemm cubins from trtllm-gen, gemm header refactor](../sources/prs/flashinfer/PR-2740.md), [[feat] Add 2048 experts and 32 Top K ](../sources/prs/flashinfer/PR-2744.md), [[feat] Add air top-p algorithm](../sources/prs/flashinfer/PR-2752.md), [feat: Add FP4 KV cache quant/dequant kernels ](../sources/prs/flashinfer/PR-2757.md), [feat: Expose TRT-LLM FMHA style paged KV Cache and page table layout](../sources/prs/flashinfer/PR-2770.md), [feat: FP8 output support for CUTLASS MLA paged attention](../sources/prs/flashinfer/PR-2779.md), [feat: Support padding tokens with seqlen=0 for rope+quant+kv cache update fusion kernel](../sources/prs/flashinfer/PR-2792.md), [Upgrade cutlass 4.2.1 -> 4.4.2](../sources/prs/flashinfer/PR-2798.md), [[fmha-v2] Support HND and NHD paged KV cache layouts with conditional stride handling](../sources/prs/flashinfer/PR-2799.md), [fix: Autotuner _find_nearest_profile non-power-of-2 num_tokens, create launchers for all supported tileN in trtllm fused MoE](../sources/prs/flashinfer/PR-2821.md), [[Fmha] Sparse MLA decode kernel selection heuristics](../sources/prs/flashinfer/PR-2836.md), [[Perf] Add FMHAv2 to flashinfer_benchmark.py and eliminate unnecessary H2D](../sources/prs/flashinfer/PR-2841.md), [read real strides for kv and block scale](../sources/prs/flashinfer/PR-2844.md), [fix: int32 overflow in `trtllm_fp4_block_scale_moe` causing "Unsupported hidden state scale shape" for EP32+ configs](../sources/prs/flashinfer/PR-2853.md), [Add support for Relu2 in BF16 fused MoE](../sources/prs/flashinfer/PR-2864.md), [Mamba SSU: horizontal MTP kernel (+ DSTATE=96 support)](../sources/prs/flashinfer/PR-2865.md), [Fix silent bug with FP8 per tensor non-gated MoE](../sources/prs/flashinfer/PR-2882.md), [fix: snap weight_scale_vec_size to handle block_scale_interleave padding for SM120](../sources/prs/flashinfer/PR-2898.md), [feat: add MXFP8 GEMM support for SM120](../sources/prs/flashinfer/PR-2902.md), [feat(gdn): state checkpointing in chunk_gated_delta_rule](../sources/prs/flashinfer/PR-2908.md), [feat: Add cuBLASLt backend for `mm_bf16` and enable multi-tactic autotuning for FP8/MXFP8 runners](../sources/prs/flashinfer/PR-2914.md), [feat: add Relu2 (squared ReLU) activation support in CUTLASS MoE backend](../sources/prs/flashinfer/PR-2926.md), [fix: use float instead of double in sampling binary search to avoid FP64 bottleneck on SM103](../sources/prs/flashinfer/PR-2945.md), [Update NVSHMEM interface to use NVSHMEM4Py instead of custom bindings](../sources/prs/flashinfer/PR-2960.md), [Improved `simple` mamba SSU kernel ](../sources/prs/flashinfer/PR-2962.md), [test: xfail cuDNN FP8 prefill on Blackwell with CUDA <= 12.9](../sources/prs/flashinfer/PR-2963.md), [Add flashinfer.fused_rmsnorm_silu() with native kernel backend](../sources/prs/flashinfer/PR-2965.md), [Fused moe all-reduce routed scaling factor + quant support](../sources/prs/flashinfer/PR-2966.md), [fix: restore SM120 CUTLASS MoE tile candidate removed by #2927 (test_trtllm_cutlass_fused_moe.py)](../sources/prs/flashinfer/PR-2984.md), [[Fmha] support nvfp4 output keepsMmaAb generation kernels](../sources/prs/flashinfer/PR-2988.md), [fix: tinygemm2 hang issue due to barrier sync](../sources/prs/flashinfer/PR-2996.md), [perf: Optimize CUTLASS MoE helper kernels for small-batch decode workloads](../sources/prs/flashinfer/PR-3014.md), [fix: extend moe alltoall top-k specializations](../sources/prs/flashinfer/PR-3021.md), [[feat] Add routing_replay_out support to MoE kernels and Python API](../sources/prs/flashinfer/PR-3024.md), [perf: Port TRT-LLM SM120/SM121 FP4 CUTLASS GEMM optimizations. Add PDL](../sources/prs/flashinfer/PR-3026.md), [[feat] Trtllm-gen Per-token Nvfp4 MoE](../sources/prs/flashinfer/PR-3027.md), [fused_moe: pre-filter SM89 tactics with zero occupancy on SM120 Blackwell (fix review feedback on #2764)](../sources/prs/flashinfer/PR-3032.md), [Support lse in trtllm paged attn kernels](../sources/prs/flashinfer/PR-3058.md), [Support Allreduce + Norm + Per-token Group Fp8 Quant Fusion](../sources/prs/flashinfer/PR-3059.md), [Support NVFP4 KV for prefill and batch attention kernels](../sources/prs/flashinfer/PR-3097.md), [feat: Enable FP8 (E4M3/E5M2) in concat_mla_k for optimize long-context prefill performance and refactor type dispatch for BF16/FP16](../sources/prs/flashinfer/PR-3129.md), [perf: Add no-bias path for tinygemm_bf16](../sources/prs/flashinfer/PR-3151.md), [Integrate CUTLASS Small Tile N Blockscaled GEMMs/Grouped GEMMs for SM120 and SM121](../sources/prs/flashinfer/PR-3152.md), [feat: DiT layer norm fusions for WAN: flashinfer.diffusion_ops](../sources/prs/flashinfer/PR-3157.md), [feat: enable glm5 router gemm](../sources/prs/flashinfer/PR-3185.md), [[Bugfix] Fix fused MoE autotuning correctness issues by filtering clusterDimZ](../sources/prs/flashinfer/PR-3227.md), [perf: optimize per-token nvfp4 quantization kernel.](../sources/prs/flashinfer/PR-3237.md), [Update moe gemm](../sources/prs/flashinfer/PR-3239.md), [Add dynamic tokens-per-page TRTLLM-GEN GQA kernels](../sources/prs/flashinfer/PR-3259.md), [Update trtllm FMHA cubins](../sources/prs/flashinfer/PR-3317.md), [[feat] Add gemma RMS AR fusion](../sources/prs/flashinfer/PR-3322.md), [checkpointing_ssu kernel: fused replay + conditional state-write for Mamba2](../sources/prs/flashinfer/PR-3324.md), [feat(cute_dsl/moe): add `moe_output_memset_inplace` dense memset wrapper](../sources/prs/flashinfer/PR-3328.md), [perf: fix the iteration bound of SWA in FA2 prefill template](../sources/prs/flashinfer/PR-714.md), [bugfix: FusedAddRMSNorm kernels might require more than 48KB shared memory when d is large.](../sources/prs/flashinfer/PR-718.md), [Align KV chunk size binary search with actual KV chunk splitting.](../sources/prs/flashinfer/PR-728.md), [Change `apply_rope_with_cos_sin_cache` to accept `cos_sin_cache`](../sources/prs/flashinfer/PR-754.md), [feat: support deepseek prefill attention shape](../sources/prs/flashinfer/PR-765.md), [bugfix: Ensure Loop Termination by Enforcing IEEE-754 Compliance in Sampling Kernels](../sources/prs/flashinfer/PR-774.md), [perf: refactor fa2 prefill template](../sources/prs/flashinfer/PR-776.md), [bugfix: drop CTA_TILE_Q=32](../sources/prs/flashinfer/PR-785.md), [bugfix: MLA decode should multiply sm_scale by math::log2e](../sources/prs/flashinfer/PR-787.md), [fix rope logic in mla decoding](../sources/prs/flashinfer/PR-793.md), [feat: support f32 attention output in FA2 template](../sources/prs/flashinfer/PR-799.md), [feat: apply sm_scale at logits instead of q in FA2 template](../sources/prs/flashinfer/PR-801.md), [perf: memory efficient deepseek mla fused page-attention kernel](../sources/prs/flashinfer/PR-804.md), [bugfix: mla page-attention kernel for different page sizes](../sources/prs/flashinfer/PR-810.md), [feat: unlocking MLA for A100](../sources/prs/flashinfer/PR-812.md), [feat: unlock MLA attention for sm89 (L40/L40s/4090)](../sources/prs/flashinfer/PR-814.md), [bugfix: bugfix on sm89 MLA](../sources/prs/flashinfer/PR-821.md), [bugfix: fix the signature of `CutlassSegmentGEMMSM90`](../sources/prs/flashinfer/PR-827.md), [perf: MLA decode kernel implemented by CuTe targeted to SM80](../sources/prs/flashinfer/PR-844.md), [misc: Remove duplicate param set in MLA kernel](../sources/prs/flashinfer/PR-850.md), [Add POD-Attention to FlashInfer](../sources/prs/flashinfer/PR-858.md), [perf: dynamic split-k for MLA](../sources/prs/flashinfer/PR-863.md), [bugfix: fix the behavior of MLA kernel when kv-length is 0](../sources/prs/flashinfer/PR-868.md), [Naive Support for Hopper FP8 Prefill Kernel with Per-Head Quantization](../sources/prs/flashinfer/PR-869.md), [perf: FlashAttention-3 style MLA PageAttention](../sources/prs/flashinfer/PR-887.md), [feat - support mla kvcache store](../sources/prs/flashinfer/PR-888.md), [perf: fix MLA split-k performance bug](../sources/prs/flashinfer/PR-898.md), [perf: tweak the pipeline design of mla kernel](../sources/prs/flashinfer/PR-901.md), [feat: flashinfer intra-kernel profiler](../sources/prs/flashinfer/PR-913.md), [feat: experimenta support of PDL](../sources/prs/flashinfer/PR-930.md), [bugfix: fix potential issues of FA3 template loading nans for PageAttention](../sources/prs/flashinfer/PR-945.md), [perf: Use 2WG pipeline design for MLA implementation on Hopper](../sources/prs/flashinfer/PR-952.md), [[TVM] Added tvm binding for sampling kernel](../sources/prs/flashinfer/PR-958.md), [perf: Fix python API overhead when CUDAGraph is not enabled](../sources/prs/flashinfer/PR-969.md), [perf: dual pivot top-p/top-k renorm](../sources/prs/flashinfer/PR-974.md), [perf: prefetch page indices for mla kernel](../sources/prs/flashinfer/PR-991.md), [feat: SM-constraint Communication Kernels](../sources/prs/flashinfer/PR-994.md), [3rdparty: upgrade cutlass to 3.9](../sources/prs/flashinfer/PR-997.md), [ROCm SDPA: Ensure attn_mask has the same dtype with q](../sources/prs/pytorch/PR-144398.md), [Add release branch push triggers to inductor-rocm-mi300.yml](../sources/prs/pytorch/PR-149871.md), [[CUDA][avgpool2d] Fix backward launch bounds again for sm100, sm120](../sources/prs/pytorch/PR-150640.md), [[CUDA][avgpool2d] Fix backward launch bounds again for `sm100`, `sm120`](../sources/prs/pytorch/PR-150676.md), [[CUDA] Only use vec128 if CUDA version is newer than 12.8](../sources/prs/pytorch/PR-150705.md), [[ATen][CUDA] Optimize 128 bit vectorization](../sources/prs/pytorch/PR-152967.md), [Fix uint view copy (#151598)](../sources/prs/pytorch/PR-154121.md), [Fix macOS build with `USE_MPS=OFF`](../sources/prs/pytorch/PR-156932.md), [[user triton] AOT inductor support for device-side TMA](../sources/prs/pytorch/PR-157241.md), [[PowerPC] Fixed build issue for vsx vec256 complexfloat and scaled_mm_out_cpu ](../sources/prs/pytorch/PR-157422.md), [[cherry-pick][inductor][triton] Update HAS_WARP_SPEC to check triton.Config params. Update Triton Hash to top of release/3.4.x stack](../sources/prs/pytorch/PR-158646.md), [CUDA 13.0 Windows Nvidia Driver Update to 580.88](../sources/prs/pytorch/PR-162501.md), [fix cpp extension distributed warning spew](../sources/prs/pytorch/PR-162764.md), [[Graph Partition] improve custom op output alias](../sources/prs/pytorch/PR-163380.md), [[graph partition] Add way to register custom rule (#163310)](../sources/prs/pytorch/PR-163395.md), [CUDA 13.0 Warning update for supported architectures](../sources/prs/pytorch/PR-163633.md), [[Cherry-Pick] [CD] CUDA 13 specific followup changes. Remove sm50-70 From CUDA 12.6 and CUDA 12.8 builds (#162455)](../sources/prs/pytorch/PR-163764.md), [[CD] CUDA 13.0 fix preload logic to include nvidia/cu13/lib/](../sources/prs/pytorch/PR-163766.md), [Move inductor jobs 3.9->3.10](../sources/prs/pytorch/PR-163954.md), [[cuDNN][SDPA] Disable dropout for cuDNN SDPA on 9.11 - 9.13](../sources/prs/pytorch/PR-164026.md), [[AARCH64][CD][CUDA13][Triton][PTXAS] Turn on BUILD_BUNDLE_PTXAS=1 ](../sources/prs/pytorch/PR-164236.md), [CUDA 13.0 builds fix on Amazon Linux 2023](../sources/prs/pytorch/PR-164893.md), [[Graph Partition] move custom rules to inductor config (#166458)](../sources/prs/pytorch/PR-166967.md), [[Graph Partition] fix graph partition input signature for fallback kernels](../sources/prs/pytorch/PR-166985.md), [[cuDNN][SDPA][Convolution] Expose cuDNN runtime version in CUDA hooks](../sources/prs/pytorch/PR-167327.md), [[ROCm] Enable shared memory based pruning for Triton configs](../sources/prs/pytorch/PR-170190.md), [[inductor] Fix cudagraph skip for index_put_ with boolean indices, gr…](../sources/prs/pytorch/PR-170884.md), [[ROCm] Make grouped GEMM CK opt‑in via env and default to fallback path](../sources/prs/pytorch/PR-171140.md), [[cherry-pick][CUDA] Upgrade cuDNN to 9.15.1 for CUDA 13 builds ](../sources/prs/pytorch/PR-171189.md), [[cherry-pick][cuDNN][SDPA] cuDNN SDPA off-by-default for cuDNN versions < 12.9 (#171627)](../sources/prs/pytorch/PR-171895.md), [Skip modded_nanogpt model in TorchInductor benchmark](../sources/prs/pytorch/PR-172141.md), [[Graph Partition] Improve support for mutation ops](../sources/prs/pytorch/PR-172577.md), [Update inductor expected accuracy files](../sources/prs/pytorch/PR-175096.md), [[benchmark] Skip pytorch_CycleGAN_and_pix2pix from inductor benchmarks](../sources/prs/pytorch/PR-175299.md), [[release-only] Remove +ptx from cuda 13.0 builds](../sources/prs/pytorch/PR-175567.md), [[release 2.12] Apply Release only changes to 2.12 branch](../sources/prs/pytorch/PR-180470.md), [Disable kernel cutlass_mla_decode on SM103](../sources/prs/sglang/PR-10058.md), [Optimize nvfp4 block scaled gemm kernel when M is small.](../sources/prs/sglang/PR-10101.md), [fix: resolve gb200 image link](../sources/prs/sglang/PR-10343.md), [Fix correction bias undefined behavior for nvfp4 models](../sources/prs/sglang/PR-10426.md), [Update CUTLASS. Refine KernelSchedule for fp8 (grouped) gemm.](../sources/prs/sglang/PR-10491.md), [[sgl-kernel] Optimize concat_mla_k kernel](../sources/prs/sglang/PR-10543.md), [Optimize cutlass int8 gemm kernel for large M on SM89 Ada GPU](../sources/prs/sglang/PR-10714.md), [disable sm100 for FlashMLA and fast-hadamard-transform in cuda12.6.1](../sources/prs/sglang/PR-11274.md), [[NVIDIA] Add new SMs support for Spark & Thor](../sources/prs/sglang/PR-11287.md), [[sgl-kernel][1/N]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-11432.md), [Improve Kernel Build Time](../sources/prs/sglang/PR-11508.md), [support cutlass fp4 kernel in sm120](../sources/prs/sglang/PR-11737.md), [Fixed aarch64 flash-mla](../sources/prs/sglang/PR-12009.md), [(1/n)support context parallel with deepseekv3.2-DSA](../sources/prs/sglang/PR-12065.md), [[sgl-kernel][4/N]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-12080.md), [[sgl-kernel] clean up fa fetch in CMakeLists.txt](../sources/prs/sglang/PR-12392.md), [[Fix] `concat_mla_absorb_q_kernel` fails for long inputs](../sources/prs/sglang/PR-12453.md), [[NVIDIA] Fix CUDA arch requirement in nvfp4 cast](../sources/prs/sglang/PR-12581.md), [Support moe topk sigmoid kernel](../sources/prs/sglang/PR-13049.md), [diffusion: enable fa4 for blackwell](../sources/prs/sglang/PR-13263.md), [[sgl-kernel][Feat][B200][1/N]Support MXFP8 Grouped GEMM in Blackwell](../sources/prs/sglang/PR-13731.md), [[kernel][moe] add moe topk fast](../sources/prs/sglang/PR-13969.md), [Add new moe wna16 marlin gemm](../sources/prs/sglang/PR-14122.md), [Opt moe align block size kernel](../sources/prs/sglang/PR-14133.md), [sync attention, deepseek doc](../sources/prs/sglang/PR-14335.md), [[CPU] Implement MXFP4 Gemm kernels for intel AMX to support GPT OSS series.](../sources/prs/sglang/PR-14385.md), [Add CUDA kernel size analysis tool for sgl-kernel optimization](../sources/prs/sglang/PR-14544.md), [[sgl-kernel][Feat][B200][2/N] Support MXFP8 Grouped GEMM in Blackwell](../sources/prs/sglang/PR-14640.md), [[NPU][eagle3] support qwen eagle3 on NPU](../sources/prs/sglang/PR-14820.md), [[sgl-kernel][1/2] Fused qk_norm_rope for GLM4.6](../sources/prs/sglang/PR-15141.md), [Add cache for flashinfer installation](../sources/prs/sglang/PR-15153.md), [[sgl-kernel] Update flashmla to include fp8 sparse_mla optimizations](../sources/prs/sglang/PR-15242.md), [Fix warp illegal instruction in kimi k2 thinking PCG](../sources/prs/sglang/PR-15306.md), [[diffusion] Add Sage Attention 3 Support for sm 120 (RTX5090)](../sources/prs/sglang/PR-15382.md), [[sgl-kernel][6/7]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-15471.md), [[Feature] JIT Fused QK norm + qk norm clean up](../sources/prs/sglang/PR-15835.md), [[JIT kernel] Apply jit per_tensor_quant_fp8 kernel](../sources/prs/sglang/PR-15836.md), [[Feature] add aligned_vector type for JIT kernel](../sources/prs/sglang/PR-16162.md), [Fix sgl-kernel jobs to skip when target_stage is specified](../sources/prs/sglang/PR-16308.md), [[diffusion] Fix RuntimeError in SageAttention3 on Nvidia Blackwell with Qwen-Image](../sources/prs/sglang/PR-16335.md), [[Fix]Pin mooncake version to 0.3.7.post2 in grace blackwell](../sources/prs/sglang/PR-16502.md), [[DeepSeek v3.2] Opt MTP decode cuda batch sizes and nsa implementation](../sources/prs/sglang/PR-16961.md), [Kernel: optimize decoding metadata in NSA multi-spec backend with fused kernels](../sources/prs/sglang/PR-17554.md), [Make flashMLA work on: Cu13, B300](../sources/prs/sglang/PR-17600.md), [Feature/support longcat flash lite](../sources/prs/sglang/PR-17838.md), [[Move sgl-kernel Kernel to JIT] Add JIT concat MLA kernels](../sources/prs/sglang/PR-17889.md), [[Diffsuion & JIT_kernel] QKNorm cross heads kernel](../sources/prs/sglang/PR-18073.md), [[Hicache & JIT_kernel] Support page first layout & mla jit kernel](../sources/prs/sglang/PR-18311.md), [docs: expand and update modelopt documentation](../sources/prs/sglang/PR-18479.md), [[FIX] Correct JIT kernel compilation on newer GPUs with outdated driver metadata.](../sources/prs/sglang/PR-18496.md), [Add claude skills for sgl-kernel and jit-kernel](../sources/prs/sglang/PR-18855.md), [Use single mma warp group for short q_len in FA to optimize decoding performance](../sources/prs/sglang/PR-18985.md), [[jit_kernel] Add fused_qknorm_rope JIT kernel](../sources/prs/sglang/PR-19059.md), [[DeepSeek-V3.2][JIT-kernel] Support nsa fuse store indexer k cache](../sources/prs/sglang/PR-19148.md), [[Kernel Slimming] Migrate NVFP4 kernels to JIT](../sources/prs/sglang/PR-19437.md), [[Feature] NVFP4 Marlin fallback for non-Blackwell GPUs (SM75+)](../sources/prs/sglang/PR-19652.md), [Add compile-time 256-bit vector guard for pre-Blackwell](../sources/prs/sglang/PR-19794.md), [[JIT Kernel][Feature] Support JIT custom all reduce (rewrite as v2)](../sources/prs/sglang/PR-19880.md), [[JIT Kernel] Reland NVFP4 kernels to JIT](../sources/prs/sglang/PR-20012.md), [[diffusion] Support nvfp4 for Flux.2](../sources/prs/sglang/PR-20137.md), [fix ci by removing nvidia-cutlass-dsl-libs-base and force reinstall n…](../sources/prs/sglang/PR-20380.md), [fix(docs): correct quantization documentation (#20301)](../sources/prs/sglang/PR-20619.md), [Fix(jit): support rmsnorm for hidden_size in {64, 128, 256}](../sources/prs/sglang/PR-20661.md), [[Feature][JIT Kernel] Fused TP QK norm For Minimax](../sources/prs/sglang/PR-20673.md), [CUTLASS FP8 Blockwise GEMM improvement of SM120](../sources/prs/sglang/PR-20887.md), [Add SGLang CUDA crash API logging inspired by FlashInfer](../sources/prs/sglang/PR-20910.md), [[Tiny Fix] Fix IS_BLACKWELL env var empty string warning in rerun-ut workflow](../sources/prs/sglang/PR-20957.md), [ci: run Stage A CUDA tests as stage-a-test-small-1-gpu on 5090](../sources/prs/sglang/PR-20988.md), [[Whisper] Enable CUDA graph support and timestamp for whisper model](../sources/prs/sglang/PR-21190.md), [Split pr-test.yml: extract sgl-kernel, jit-kernel, and multimodal-gen tests into separate workflow files](../sources/prs/sglang/PR-21219.md), [CUTLASS NVFP4 GEMM improvement of SM120](../sources/prs/sglang/PR-21314.md), [[Diffusion] Add qknorm rope fuse kernel](../sources/prs/sglang/PR-21440.md), [fix: piecewise_cuda_graph get correct qo_indptr](../sources/prs/sglang/PR-21452.md), [fix: TRT-LLM MHA CUDA illegal address with EAGLE v2 + DP attention](../sources/prs/sglang/PR-21649.md), [[jit_kernel] Optimize fused_qknorm_rope: deduplicate sincosf for interleave RoPE ](../sources/prs/sglang/PR-21654.md), [Remove flashinfer wheel cache cleanup that deletes other versions](../sources/prs/sglang/PR-21711.md), [[Feature] JIT rmsnorm update (with claude)](../sources/prs/sglang/PR-21834.md), [[Bugfix] Fix CUDA graph replay issues in trtllm_mla draft_extend](../sources/prs/sglang/PR-21987.md), [[Diffusion] Fix weight scale swizzle and add large-M kernel config for FLUX.2-dev-NVFP4](../sources/prs/sglang/PR-22064.md), [[diffusion] Default NVFP4 to CUTLASS and add all-model shape benchmarks](../sources/prs/sglang/PR-22091.md), [Lazy import flash_attention_v4 to avoid loading flash_attn.cute at startup](../sources/prs/sglang/PR-22306.md), [[Docker] Fix Trivy CVEs, cubin download 403s, and kernels command order](../sources/prs/sglang/PR-22322.md), [[CI/Docker] Clean up redundant flashinfer cubin downloads](../sources/prs/sglang/PR-22491.md), [[Docker] Remove flashinfer cache copy](../sources/prs/sglang/PR-22653.md), [reland [Diffusion] Add FLUX.1-dev ModelOpt NVFP4 support](../sources/prs/sglang/PR-22672.md), [[Diffusion] Add Wan2.2 ModelOpt NVFP4 support](../sources/prs/sglang/PR-22681.md), [[Fix] Fix accuracy bug in Flashmla sparse MLA kernel](../sources/prs/sglang/PR-22723.md), [Dual MoE CUDA graph capture for lora/nolora batches](../sources/prs/sglang/PR-22809.md), [[Fix/Kernel] Add JIT rmsnorm_hf kernel to fix transformers backend MMLU accuracy regression ](../sources/prs/sglang/PR-22931.md), [Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-23686.md), [[feat] Init true on policy with qwen_dense](../sources/prs/sglang/PR-23961.md), [Enable PDL for various kernels in DSV32/GLM5](../sources/prs/sglang/PR-23965.md), [[VLM] Optimize Gemma4 VLM with PCG and fuse RMSNorm + residual add + scalar](../sources/prs/sglang/PR-24048.md), [Refactor device timer, clean up metrics collector, and add fwd occupancy metric](../sources/prs/sglang/PR-24197.md), [[codex] Optimize hidden-size 512 RMSNorm dispatch](../sources/prs/sglang/PR-24710.md), [[rebase]Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-24986.md), [Support draft extend cuda graph for tokenspeed_mla attention backend](../sources/prs/sglang/PR-25489.md), [[fp8] SM90 swap-AB scaled_mm dispatch (~1.16x kernel geomean, +5.8-18.5% end-to-end)](../sources/prs/sglang/PR-25532.md), [fix (jit kernel): elementwise activation C++ error](../sources/prs/sglang/PR-25695.md), [[Codex] Remove stale DeepSeek V4 JIT kernels](../sources/prs/sglang/PR-25764.md), [[Refactor] Rename NSA → DSA: user-facing aliases, file/class/import rename](../sources/prs/sglang/PR-25821.md), [[Test] Stage-a sanity kits; consolidate core/ + models_e2e/ tests](../sources/prs/sglang/PR-25831.md), [Support cutlass Int8 gemm](../sources/prs/sglang/PR-2752.md), [upgrade cutlass v3.7.0](../sources/prs/sglang/PR-2967.md), [feat: add flashinfer as 3rdparty and use rmsnorm as example](../sources/prs/sglang/PR-3033.md), [Support sm90 Int8 gemm](../sources/prs/sglang/PR-3035.md), [support w8a8 fp8 kernel with CUTLASS](../sources/prs/sglang/PR-3047.md), [feat: integrate gemm_fp8 kernel into gemm](../sources/prs/sglang/PR-3056.md), [add tensorrt_llm common and cutlass_extensions as 3rdparty](../sources/prs/sglang/PR-3216.md), [support blockwise fp8 matmul kernel](../sources/prs/sglang/PR-3267.md), [fix undefined symbol cudaGetDriverEntryPointByVersion](../sources/prs/sglang/PR-3372.md), [integrate blockwise fp8 kernel](../sources/prs/sglang/PR-3529.md), [[Feature] Apply Cublas Grouped Gemm kernel](../sources/prs/sglang/PR-3629.md), [Support FP4 gemm (1/2)](../sources/prs/sglang/PR-3899.md), [Add fast decode plan for flashinfer mla](../sources/prs/sglang/PR-3987.md), [[Revision] Add fast decode plan for flashinfer mla ](../sources/prs/sglang/PR-4012.md), [Support overlapping two batches](../sources/prs/sglang/PR-4068.md), [DeepGemm integrate to gemm](../sources/prs/sglang/PR-4165.md), [Accelerate FP8 CUDA Kernel by 20-28%](../sources/prs/sglang/PR-4215.md), [fix per_token_group_quant_fp8 illegal memory when num_groups % 16 != 0](../sources/prs/sglang/PR-4231.md), [add THIRDPARTYNOTICES for DeepGEMM](../sources/prs/sglang/PR-4272.md), [Support Blackwell Block Scale FP8 Gemm](../sources/prs/sglang/PR-4278.md), [update deepgemm](../sources/prs/sglang/PR-4284.md), [Add deepseek style fused moe group gate selection kernel](../sources/prs/sglang/PR-4530.md), [[Fix] Fix raw_bs bug when using flashinfer mla and eagle](../sources/prs/sglang/PR-4557.md), [Support fp8 gemm for blackwell](../sources/prs/sglang/PR-4558.md), [avoid cudaStreamSynchronize in DeepSeekV2AttentionMLA](../sources/prs/sglang/PR-4577.md), [support cmake for sgl-kernel](../sources/prs/sglang/PR-4706.md), [Support (1 <= dp < tp) in the dp attention in DeepEP](../sources/prs/sglang/PR-4770.md), [[Feat] support deepgemm for cmake](../sources/prs/sglang/PR-4864.md), [[Build] Fix cuda12.8 build error in nvfp4_scaled_mm_kernels.cu](../sources/prs/sglang/PR-4953.md), [update cutlass tag](../sources/prs/sglang/PR-5011.md), [fix deepgemm as well](../sources/prs/sglang/PR-5030.md), [support sgl-kernel on blackwell](../sources/prs/sglang/PR-5074.md), [reduce moe_align_block_size_kernel small batch mode overhead](../sources/prs/sglang/PR-5086.md), [Blackwell Cutlass MLA kernel](../sources/prs/sglang/PR-5142.md), [[1/2] Add FP8 Blockscale MoE CUTLASS kernel for Blackwell](../sources/prs/sglang/PR-5281.md), [fix: solve cu118 issue for cutlass mla](../sources/prs/sglang/PR-5331.md), [Add Cutlass MLA attention backend](../sources/prs/sglang/PR-5390.md), [chore: upgrade DeepGEMM](../sources/prs/sglang/PR-5395.md), [[perf] introduce deep gemm group_gemm_masked as gemm](../sources/prs/sglang/PR-5432.md), [feat: use flashinfer jit package](../sources/prs/sglang/PR-5547.md), [[2/2] Add python wrapper for CUTLASS FP8 Blockscale MoE Kernel. ](../sources/prs/sglang/PR-5694.md), [[PP] Add pipeline parallelism](../sources/prs/sglang/PR-5724.md), [cutlass 3.9 supported to improve fp8_blockwise_gemm](../sources/prs/sglang/PR-5820.md), [Add sm_120 for blackwell](../sources/prs/sglang/PR-5903.md), [chore: upgrade cutlass 3.9.2](../sources/prs/sglang/PR-6004.md), [chore: upgrade deepgemm](../sources/prs/sglang/PR-6073.md), [feat: mtp support dp-attention](../sources/prs/sglang/PR-6081.md), [[1/2] Add Kernel support for Cutlass based Fused FP4 MoE](../sources/prs/sglang/PR-6093.md), [Cutlass MLA: Disable split kv due to https://github.com/NVIDIA/cutlass/issues/2274](../sources/prs/sglang/PR-6101.md), [Upgrade CUTLASS 4.0](../sources/prs/sglang/PR-6336.md), [reduce torch.zeros overhead in moe align block size kernel](../sources/prs/sglang/PR-6369.md), [[Feature] Comprehensive Hybrid Parallelism Support](../sources/prs/sglang/PR-6389.md), [Fix bug of deepseek-v3 under DP+EP mode with large batchsize/seqlen](../sources/prs/sglang/PR-6449.md), [[EP] Add cuda kernel for moe_ep_pre_reorder](../sources/prs/sglang/PR-6699.md), [Set `num_fused_shared_experts` as `num_shared_experts` when shared_experts fusion is not disabled](../sources/prs/sglang/PR-6736.md), [Correctly abort the failed grammar requests & Improve the handling of abort](../sources/prs/sglang/PR-6803.md), [[EP] Add cuda kernel for moe_ep_post_reorder](../sources/prs/sglang/PR-6837.md), [Fix AWQ Dequant and Weight Loading of deepseek v2](../sources/prs/sglang/PR-6842.md), [fix ep_moe_reorder kernel bugs](../sources/prs/sglang/PR-6858.md), [Add a CUDA kernel for fusing mapping and weighted sum for MoE.](../sources/prs/sglang/PR-6916.md), [[sgl-kernel] Add cuda kernel for moe_ep_silu_and_mul](../sources/prs/sglang/PR-6919.md), [[perf][sgl-kernel] extend cutlass_mla_decode to support num_head < 128](../sources/prs/sglang/PR-6929.md), [Clean up server_args.py](../sources/prs/sglang/PR-7037.md), [Tiny fix cutlass_mla_get_workspace_size stub incorrect signature](../sources/prs/sglang/PR-7057.md), [Fix grammar abort & Minor style fixes](../sources/prs/sglang/PR-7204.md), [feat: support compatibility between MTP and two-batch-overlap](../sources/prs/sglang/PR-7225.md), [Add CUTLASS FP8 Blockscale MoE kernel for Hopper architecture](../sources/prs/sglang/PR-7278.md), [fix: resolve blackwell deepep image issue](../sources/prs/sglang/PR-7331.md), [Fuse sorted_token_ids padding to moe_align_block_size kernel](../sources/prs/sglang/PR-7437.md), [fix: fix apply_shuffle_mul_sum](../sources/prs/sglang/PR-7444.md), [[CMake] Fix sgl-kernel CMakeLists for Blackwell](../sources/prs/sglang/PR-7543.md), [Add dsv3 router gemm kernel](../sources/prs/sglang/PR-7627.md), [Add dsv3 fused a gemm to sgl-kernel](../sources/prs/sglang/PR-7630.md), [[Feature] CUDA Green Context Support](../sources/prs/sglang/PR-7649.md), [[1/n]: add cutlass W4A8 moe kernel for hopper architecture](../sources/prs/sglang/PR-7772.md), [[kernel] opt moe align block kernel by block/warp scan algorithm](../sources/prs/sglang/PR-7884.md), [[feat] Support tp mode for DeepSeek-R1-W4AFP8](../sources/prs/sglang/PR-8118.md), [[Fix][Ready]Fix register spilling in cutlass nvfp4 gemm kernel on Blackwell](../sources/prs/sglang/PR-8127.md), [[sgl-kernel] Opt per_token_quant_fp8 with warp reduce](../sources/prs/sglang/PR-8130.md), [[2/N]Support DeepSeek-R1 w4a8 low latency deepep](../sources/prs/sglang/PR-8464.md), [[Perf] Tunings for SM100 FP8 CUTLASS kernel](../sources/prs/sglang/PR-8818.md), [chore: support blackwell cu129 image](../sources/prs/sglang/PR-8928.md), [[NVIDA] [1/N] Nvfp4 Masked Gemm: Add quant op for the flashinfer grouped gemm](../sources/prs/sglang/PR-9200.md), [[fix]: fix cutlass moe ut and and Opt H20 cutlass groupGemm performance](../sources/prs/sglang/PR-9272.md), [Support trtllm_allreduce_fusion in flashinfer for cuda<12.8](../sources/prs/sglang/PR-9339.md), [[sgl-kernel] feat: Support sm120 cutlass fp8 gemm kernel](../sources/prs/sglang/PR-9403.md), [[NVIDIA] [2/N] Optimize `silu_and_mul_scaled_fp4_grouped_quant` perf](../sources/prs/sglang/PR-9556.md), [Update CUTLASS 4.2 & Enable K-Major Scale Factor for SM90 FP8 Blockwise Group GEMM](../sources/prs/sglang/PR-9559.md), [Make sm100 fp8 kernels available on sm103](../sources/prs/sglang/PR-9789.md), [Make fp4_quantize kernels work on sm103](../sources/prs/sglang/PR-9807.md), [CUTLASS fp8 blockwise gemm support of sm120](../sources/prs/sglang/PR-9969.md), [[WIP] support more dtypes for tcgen05](../sources/prs/tilelang/PR-1229.md), [[Enhancement] add more dtype and fix mma.ws for fp16 for tcgen05](../sources/prs/tilelang/PR-1327.md), [[Feat] profiler support cudagraph backend](../sources/prs/tilelang/PR-1658.md), [[Feature] Support `cp.reduce.async.bulk.tensor`](../sources/prs/tilelang/PR-1667.md), [[Feature] Support tcgen5mma lowering for `.kind::i8`](../sources/prs/tilelang/PR-1764.md), [[CUDA] Support tcgen5mma gemm ts](../sources/prs/tilelang/PR-1866.md), [[Feature] Support cluster launch, query, synchronization and barrier operations](../sources/prs/tilelang/PR-1874.md), [[Feature] 2-SM support for TMA, TMEM and TCGEN5MMA on Blackwell](../sources/prs/tilelang/PR-1882.md), [[Feature] Add T.copy_cluster to support TMA multicast and SM-to-SM cluster copy](../sources/prs/tilelang/PR-1908.md), [[Feature] Block-scaled GEMM support for MXFP8 on Blackwell](../sources/prs/tilelang/PR-1945.md), [[Feature] Support TMA store in T.tma_copy()](../sources/prs/tilelang/PR-1981.md), [[Transform] Add InjectTcgen05Fence pass](../sources/prs/tilelang/PR-2003.md), [[Backend] Refactor gemm_sp](../sources/prs/tilelang/PR-2048.md), [[CUDA] Support int4 `T.gemm`](../sources/prs/tilelang/PR-2063.md), [[CUDA] Improve int4 GEMM lowering and packed codegen support](../sources/prs/tilelang/PR-2073.md), [[Bugfix] Enable `.shared::cta` in TMA copy paths only on CUDA 12.8+](../sources/prs/tilelang/PR-2087.md), [feat: auto-vectorize bf16/fp16 reduce with packed add2 intrinsics](../sources/prs/tilelang/PR-2112.md), [[CUDA][TMA] Add TMA tile::gather4 / tile::scatter4 support](../sources/prs/tilelang/PR-2129.md), [[codex] Split GEMM implementations by backend](../sources/prs/tilelang/PR-2153.md), [[CUDA] Add native SM75 MMA GEMM support for FP16, INT8 and INT4](../sources/prs/tilelang/PR-2198.md), [[TIR][IR] Update to use tirx](../sources/prs/tilelang/PR-2216.md), [[Python] Drop Python 3.9 support](../sources/prs/tilelang/PR-2218.md), [[Kernel]: Cutlass 2:4 Sparsity + FP8/Int8 Quant Support](../sources/prs/vllm/PR-10995.md), [[Kernel] Update `cutlass_scaled_mm` to support 2d group (blockwise) scaling](../sources/prs/vllm/PR-11868.md), [[Build] Only build 9.0a for scaled_mm and sparse kernels](../sources/prs/vllm/PR-12339.md), [[ROCm] Faster Custom Paged Attention kernels](../sources/prs/vllm/PR-12348.md), [[Attention] MLA decode optimizations](../sources/prs/vllm/PR-12528.md), [[Kernel] port sgl moe_align_block_size kernels](../sources/prs/vllm/PR-12574.md), [[Kernel][Quantization] Integrate block-quantized CUTLASS kernels for DeepSeekV3](../sources/prs/vllm/PR-12587.md), [[Attention] MLA with chunked prefill](../sources/prs/vllm/PR-12639.md), [[Perf] Mem align KV caches for CUDA devices (MLA perf improvement)](../sources/prs/vllm/PR-12676.md), [[Core][AMD] Migrate fully transparent sleep mode to ROCm platform](../sources/prs/vllm/PR-12695.md), [[Bugfix][Kernel] Fix per-token/per-channel quantization for Hopper scaled mm](../sources/prs/vllm/PR-12696.md), [[Kernel] Make rotary_embedding ops more flexible with input shape](../sources/prs/vllm/PR-12777.md), [[NVIDIA] Support nvfp4 quantization](../sources/prs/vllm/PR-12784.md), [Optimize moe_align_block_size for deepseek_v3](../sources/prs/vllm/PR-12850.md), [[Misc][Kernel]: Add GPTQAllSpark Quantization](../sources/prs/vllm/PR-12931.md), [[Kernel]Add streamK for block-quantized CUTLASS kernels](../sources/prs/vllm/PR-12978.md), [[Kernel] moe wna16 cuda kernel](../sources/prs/vllm/PR-13321.md), [[NVIDIA] Support nvfp4 tcgen05 gemm](../sources/prs/vllm/PR-13571.md), [[ROCm][MoE] mi300 mixtral8x7B perf for specific BS](../sources/prs/vllm/PR-13577.md), [[Kernel] FlashMLA integration](../sources/prs/vllm/PR-13747.md), [[Attention] MLA support for V1](../sources/prs/vllm/PR-13789.md), [add tcgen05 support for tcgen05 fp8 gemm](../sources/prs/vllm/PR-13798.md), [[ROCm] Disable chunked prefill/prefix caching when running MLA on non-cuda platforms](../sources/prs/vllm/PR-13844.md), [[Attention] Flash MLA for V1](../sources/prs/vllm/PR-13867.md), [[V1] EP/TP MoE + DP Attention](../sources/prs/vllm/PR-13931.md), [[Kernel] CUTLASS grouped gemm fp8 MoE kernel](../sources/prs/vllm/PR-13972.md), [[Kernel] optimize performance of gptq marlin kernel when n is small](../sources/prs/vllm/PR-14138.md), [dynamic distpatch of fp8 kernels](../sources/prs/vllm/PR-14245.md), [[Attention] FlashAttn MLA](../sources/prs/vllm/PR-14258.md), [fix minor miscalled method](../sources/prs/vllm/PR-14327.md), [[Build/BugFix] Fix hopper 12.8 build](../sources/prs/vllm/PR-14354.md), [Add cutlass support for blackwell fp8 blockwise gemm](../sources/prs/vllm/PR-14383.md), [[BugFix] Illegal Memory Access in the blockwise cutlass fp8 GEMMs](../sources/prs/vllm/PR-14396.md), [[Kernel] moe wna16 marlin kernel](../sources/prs/vllm/PR-14447.md), [permute/unpermute kernel for moe optimization](../sources/prs/vllm/PR-14568.md), [[Attention] Flash Attention 3 - fp8](../sources/prs/vllm/PR-14570.md), [[BugFix/Build] Fix sparse kernels not getting built on hopper](../sources/prs/vllm/PR-14572.md), [[Kernel] GGUF MoE kernel](../sources/prs/vllm/PR-14613.md), [[Kernel] allow non-contiguous input for marlin kernel](../sources/prs/vllm/PR-14658.md), [[Bugfix][Kernel][CPU] Fix num_tokens in CPU rotary embedding kernel](../sources/prs/vllm/PR-14667.md), [[V1] Fully Transparent Implementation of CPU Offloading](../sources/prs/vllm/PR-15354.md), [[Kernel] Fix conflicting macro names for gguf kernels](../sources/prs/vllm/PR-15456.md), [[ROCM][KERNEL] Paged attention for V1](../sources/prs/vllm/PR-15720.md), [[Bugfix] fix use_atomic_add support of marlin kernel when using v1 engine](../sources/prs/vllm/PR-15946.md), [Modularize fused experts and integrate PPLX kernels](../sources/prs/vllm/PR-15956.md), [[NVIDIA] Support Cutlass MLA for Blackwell GPUs](../sources/prs/vllm/PR-16032.md), [[ROCM] Add gfx950 to the custom attention archs](../sources/prs/vllm/PR-16034.md), [Add FlexAttention to V1](../sources/prs/vllm/PR-16078.md), [[Kernel] support merge_attn_states CUDA kernel, 3x speedup](../sources/prs/vllm/PR-16173.md), [[Hardware/NVIDIA/Kernel] [Functional Enablement] [1/N] Enable nvidia/DeepSeek-R1-FP4 Model](../sources/prs/vllm/PR-16362.md), [Allocate kv_cache with stride order](../sources/prs/vllm/PR-16605.md), [[V1] V1 FlashInfer Attention](../sources/prs/vllm/PR-16684.md), [[misc] ignore marlin_moe_wna16 local gen codes](../sources/prs/vllm/PR-16760.md), [[Kernel] GGUF MoeVec kernel](../sources/prs/vllm/PR-16780.md), [[BugFix] Accuracy fix for llama4 int4 - improperly casted scales](../sources/prs/vllm/PR-16801.md), [[Kernel] some optimizations for dense marlin and moe marlin](../sources/prs/vllm/PR-16850.md), [Update PyTorch to 2.7.0](../sources/prs/vllm/PR-16859.md), [[Kernel] Add expert_map support to Cutlass FP8 MOE](../sources/prs/vllm/PR-16861.md), [[Attention] FA3 decode perf improvement - single mma warp group support for head dim 128](../sources/prs/vllm/PR-16864.md), [Update Qwen1.5-MoE-W4A16-compressed-tensors.yaml](../sources/prs/vllm/PR-16946.md), [[ROCm][Kernel][V1] Enable AMD Radeon GPU Custom Paged Attention on v1](../sources/prs/vllm/PR-17004.md), [Fix `numel()` downcast in vllm/csrc/moe/moe_align_sum_kernels.cu +2](../sources/prs/vllm/PR-17082.md), [[ROCm][FP8][Kernel] FP8 quantization fused into Custom Paged Attention](../sources/prs/vllm/PR-17139.md), [[NVIDIA] Support Cutlass w8a8 FP8 for Blackwell Geforce GPUs (sm120)](../sources/prs/vllm/PR-17280.md), [[Misc][ROCm] Exclude `cutlass_mla_decode` for ROCm build](../sources/prs/vllm/PR-17289.md), [[Attention] MLA move o_proj q_proj into cuda-graph region](../sources/prs/vllm/PR-17484.md), [[Attention] MLA move rotary embedding to cuda-graph region](../sources/prs/vllm/PR-17668.md), [[Kernel] fp4 marlin kernel](../sources/prs/vllm/PR-17687.md), [[Kernel] Have rotary embeddings support tensors](../sources/prs/vllm/PR-18046.md), [Fix Broken macro for cutlass moe](../sources/prs/vllm/PR-18049.md), [[Build] Supports CUDA 12.6 and 11.8 after Blackwell Update](../sources/prs/vllm/PR-18316.md), [Sm100 blockwise fp8 swap ab](../sources/prs/vllm/PR-18564.md), [[Kernel] Integrate CUTLASS MoE kernel with PPLX](../sources/prs/vllm/PR-18762.md), [[Perf] Tunings for SM100 FP8 CUTLASS kernel](../sources/prs/vllm/PR-18778.md), [[BugFix] FA2 MLA Accuracy Issue](../sources/prs/vllm/PR-18807.md), [[Hardware][NVIDIA] FP4 MoE kernel optimization](../sources/prs/vllm/PR-19110.md), [[V1] Use FlashInfer by default on Blackwell GPUs](../sources/prs/vllm/PR-19118.md), [[Bugfix][V1] Allow manual FlashAttention for Blackwell](../sources/prs/vllm/PR-19492.md), [[Hardware][NVIDIA][kernel] Fp4 MOE quant kernel optimization](../sources/prs/vllm/PR-19500.md), [[Perf] Further tunings for SM100 FP8 CUTLASS kernel](../sources/prs/vllm/PR-19566.md), [Only build CUTLASS MoE kernels on Hopper](../sources/prs/vllm/PR-19648.md), [[feat]: CUTLASS block scaled group gemm for SM100](../sources/prs/vllm/PR-19757.md), [Fix FA2 fallback for Blackwell V1](../sources/prs/vllm/PR-19781.md), [[Bugfix] Build moe_data for both sm100 and sm90](../sources/prs/vllm/PR-20086.md), [[Bugfix] Fix some narrowing conversion warnings](../sources/prs/vllm/PR-20141.md), [Replace `multiply_add` with `homogeneous_multiply_add` to Address Clang Template Parameter Issue](../sources/prs/vllm/PR-20142.md), [[Bugfix] Fix topk_ids indices_type for CUTLASS w8a8 FP8 MoE](../sources/prs/vllm/PR-20166.md), [[Kernel][Bugfix] Fixup some warnings in nvfp4_blockwise_moe when CUDA < 12.8](../sources/prs/vllm/PR-20324.md), [Update PyTorch to 2.8.0](../sources/prs/vllm/PR-20358.md), [[Kernel] SM90 CUTLASS FP8 GEMM: add support for swap AB + kernel tuning](../sources/prs/vllm/PR-20396.md), [[feat]: add SM100 support for cutlass FP8 groupGEMM](../sources/prs/vllm/PR-20447.md), [[Performance] Performance improvements in non-blockwise fp8 CUTLASS MoE](../sources/prs/vllm/PR-20762.md), [SM100 Cutlass MLA decode with unrestricted num_heads (< 128) for DeepSeek TP](../sources/prs/vllm/PR-20769.md), [[fix]: disable cutlass block scaled group gemm for EP](../sources/prs/vllm/PR-20781.md), [[Perf] Add swap_ab to SM90 FP8 non-block CUTLASS moe grouped gemm](../sources/prs/vllm/PR-20911.md), [Support mnnvl all2allv from Flashinfer](../sources/prs/vllm/PR-21003.md), [[Kernel] Flashinfer MLA (trtllm-gen) decode kernel integration](../sources/prs/vllm/PR-21078.md), [[Perf] Cuda Kernel for Per Token Group Quant](../sources/prs/vllm/PR-21083.md), [[perf] Add fused MLA QKV + strided layernorm](../sources/prs/vllm/PR-21116.md), [[Perf] Use FlashInfer RoPE for RotaryEmbedding.forward_cuda when available](../sources/prs/vllm/PR-21126.md), [[Feature][Kernel]FusedMoE LoRA](../sources/prs/vllm/PR-21229.md), [[v1] - Mamba1 Attention Metadata](../sources/prs/vllm/PR-21249.md), [Support CUTLASS NVFP4 (w4a4) for Blackwell Geforce GPUs (SM120)](../sources/prs/vllm/PR-21309.md), [[Bugfix][CUDA] fixes CUDA FP8 kv cache dtype supported](../sources/prs/vllm/PR-21420.md), [[Bug] Fix Compressed Tensor NVFP4 `cutlass_fp4_group_mm` illegal memory access](../sources/prs/vllm/PR-21465.md), [[Kernel] Improve machete memory bound perf](../sources/prs/vllm/PR-21556.md), [[BugFix] Fix IMA FlashMLA full cuda-graph and DP + Update FlashMLA](../sources/prs/vllm/PR-21691.md), [update flashinfer to v0.2.9rc2](../sources/prs/vllm/PR-21701.md), [Fix Flashinfer CUTLASS MOE Allgather](../sources/prs/vllm/PR-21963.md), [[Kernel] Add support for block FP8 on SM120 (NVIDIA 5090 and RTX PRO 6000)](../sources/prs/vllm/PR-22131.md), [Fp8 paged attention update](../sources/prs/vllm/PR-22222.md), [Upgrade FA3 for attention sink](../sources/prs/vllm/PR-22313.md), [[Attention] FA3 Attention Sinks Perf Boost](../sources/prs/vllm/PR-22478.md), [[Fix] enable swap_ab for pplx problem size computation](../sources/prs/vllm/PR-22991.md), [[Kernel] CUTLASS MoE FP8: Integrate cuda moe permute/unpermute](../sources/prs/vllm/PR-23045.md), [[V1] address post issues related to #20059 (part 1); cascade attention reenable by default](../sources/prs/vllm/PR-23046.md), [[kernel] Support W4A8 on Hopper](../sources/prs/vllm/PR-23198.md), [[Kernel] Add fused grouped_topk kernel for MoE](../sources/prs/vllm/PR-23274.md), [[Perf] Use upstream CUTLASS for SM90 Block FP8 kernel](../sources/prs/vllm/PR-23280.md), [[Compile] Fix Compile Warning SM100 Cutlass MLA](../sources/prs/vllm/PR-23287.md), [fix incompatibililty with non cuda platform for nvfp4](../sources/prs/vllm/PR-23478.md), [[Compile] Fix Compile Warning for `w4a8_mm_entry.cu`](../sources/prs/vllm/PR-23660.md), [[NVIDIA] Support SiluMul + NVFP4 quant fusion](../sources/prs/vllm/PR-23671.md), [[Kernel][B200] mxfp4 fused cutlass moe](../sources/prs/vllm/PR-23696.md), [[Bugfix][Misc] Fix silu_and_mul_nvfp4_quant issue and extract common utils for nvfp4 kernel source files](../sources/prs/vllm/PR-23727.md), [[Feature] Support Decode Context Parallel (DCP) for MLA](../sources/prs/vllm/PR-23734.md), [[Kernel] cuda kernels for upcoming decode context parallel feature](../sources/prs/vllm/PR-23791.md), [[Kernel] Faster pre-processing time for W4A8](../sources/prs/vllm/PR-23972.md), [[Model] Add LongCat-Flash ](../sources/prs/vllm/PR-23991.md), [[Kernel] Support decode context parallelism on Blackwell with CUTLASS MLA](../sources/prs/vllm/PR-24385.md), [[NVIDIA] Blackwell Family](../sources/prs/vllm/PR-24673.md), [[Kernel][Quantization] add w4a8 support for marlin kernel](../sources/prs/vllm/PR-24722.md), [[Bugfix] Fix accuracy issue for silu_mul + nvfp4 quant fusion kernel](../sources/prs/vllm/PR-24833.md), [[Bugfix][B200] Fix `cutlass_mla` hang](../sources/prs/vllm/PR-24966.md), [Disable failing GPT-OSS Eval (Blackwell) for now](../sources/prs/vllm/PR-25107.md), [[Compile] Fix Compile Warning for Ignoring `MIN_BLOCK_PER_SM`](../sources/prs/vllm/PR-25193.md), [[Bugfix] [B200] cutlass_mla - ensure kv_split == 1 for batch size > 1](../sources/prs/vllm/PR-25509.md), [Fuse RoPE and MLA KV-cache write](../sources/prs/vllm/PR-25774.md), [Update launch_bounds_utils.h for correct compile on Multiple Cuda Arch - PTXAS out of range Warning](../sources/prs/vllm/PR-25843.md), [Fix INT8 quantization error on Blackwell GPUs (SM100+)](../sources/prs/vllm/PR-25935.md), [[Performance] Split FlashAttn attention and cache update](../sources/prs/vllm/PR-25954.md), [Fix undefined symbol: cutlass_moe_mm_sm100](../sources/prs/vllm/PR-26098.md), [[NVIDIA] [Perf] Update to leverage flashinfer trtllm FP4 MOE throughput kernel](../sources/prs/vllm/PR-26714.md), [[Attention] Tune CUTLASS MLA num_splits](../sources/prs/vllm/PR-26846.md), [[Perf] SM100 - add swap AB optimization to CUTLASS FP8 GEMM](../sources/prs/vllm/PR-27284.md), [Prefer FlashAttention MLA as default over FlashMLA](../sources/prs/vllm/PR-27363.md), [[Attention] Use sparse prefill kernel for fp8 kv-cache in DeepSeek-v3.2](../sources/prs/vllm/PR-27532.md), [[Performance] Fused blockwise quant RMS norm](../sources/prs/vllm/PR-27883.md), [[Kernel] Optimize rms_norm kernel](../sources/prs/vllm/PR-27931.md), [Update Flashinfer from `v0.4.1` to `v0.5.2`](../sources/prs/vllm/PR-27952.md), [[Perf][DeepSeek] Add sigmoid+bias fusion to fused_grouped_topk from TRTLLM](../sources/prs/vllm/PR-28124.md), [[Performance][B200] silu_mul_quant: pack scales in int32](../sources/prs/vllm/PR-28358.md), [[NVIDIA] Guard SM100 CUTLASS MoE macro to SM100 builds v2](../sources/prs/vllm/PR-28938.md), [chore: add RTX_PRO_6000 GLM4.6-FP8 kernel tuning](../sources/prs/vllm/PR-29240.md), [[Kernel] Add NVFP4 MoE CUTLASS support for SM120](../sources/prs/vllm/PR-29242.md), [Lora MoE Align Improvements](../sources/prs/vllm/PR-29257.md), [[Kernel][MoE] optimize `moe_align_block_size`](../sources/prs/vllm/PR-29642.md), [[Kernel]Support W4A8 Grouped GEMM on Hopper](../sources/prs/vllm/PR-29691.md), [[Perf] Improve fp8 quant in mla; replace ReduceSum with ReduceScatterSum](../sources/prs/vllm/PR-29795.md), [[Kernel][Quantization][MoE] add marlin kernel support for turing (sm75)](../sources/prs/vllm/PR-29901.md), [[Perf] Do FP4 quant before All gather on flashinfer trtllmgen MOE ](../sources/prs/vllm/PR-30014.md), [Add llmcompressor fp8 kv-cache quant (per-tensor and per-attn_head)](../sources/prs/vllm/PR-30141.md), [gptq marlin quantization support for fused moe with lora](../sources/prs/vllm/PR-30254.md), [[Feature] Add SM103 (Blackwell Ultra) Support to vLLM](../sources/prs/vllm/PR-30484.md), [OffloadingConnector: Support kernel_block_size != block_size](../sources/prs/vllm/PR-30692.md), [[NVFP4][Perf] Tune NVFP4 input quant kernel for small batch size](../sources/prs/vllm/PR-30897.md), [[Kernel] Add topk_sigmoid kernel](../sources/prs/vllm/PR-31246.md), [[Perf] Fuse stride preparation for NVFP4 cutlass_moe](../sources/prs/vllm/PR-31837.md), [[1/N][Attention] Restructure attention: move files](../sources/prs/vllm/PR-31916.md), [[Perf][Kernel] Optimize FP4 quantization kernels (SM100F)](../sources/prs/vllm/PR-32520.md), [fix: Add glm4_moe_lite to MLA detection](../sources/prs/vllm/PR-32614.md), [[Attention] FA4 integration](../sources/prs/vllm/PR-32974.md), [[Feature] Support CPU Offloading without Pytorch Pinned Memory that leads to doubled allocation](../sources/prs/vllm/PR-32993.md), [[Kernel] Apply 256bit LDG/STG To Activation Kernels](../sources/prs/vllm/PR-33022.md), [Add support for Mistral Large 3 inference with Flashinfer MoE](../sources/prs/vllm/PR-33174.md), [[Bugfix] Fix quant RMS norm fusion for quantization with TMA-aligned scales](../sources/prs/vllm/PR-33255.md), [[Kernel] Add enable_sm120_or_later for SM121 (DGX Spark) CUTLASS support](../sources/prs/vllm/PR-33517.md), [[Feature][Core] Support Fabric detection to adapt the MNNVL protocol for the GB series](../sources/prs/vllm/PR-33540.md), [[Bugfix]fix output Nan/Inf in marlin if dtype=float16](../sources/prs/vllm/PR-33972.md), [Reapply [Attention][FA3] Update FA3 to include new swizzle optimization](../sources/prs/vllm/PR-34043.md), [fix(cpu): fix mla_decode compilation on x86 without AVX512](../sources/prs/vllm/PR-34052.md), [[Kernel] Optimize grouped topk kernel](../sources/prs/vllm/PR-34206.md), [[ModelBash][DSV3] Add TRTLLM DSV3 Router GEMM kernel (6% B1 Speedup)](../sources/prs/vllm/PR-34302.md), [[Bugfix] Enforce DeepGEMM when using sparse_attn_indexer on CUDA](../sources/prs/vllm/PR-34374.md), [[Custom Ops] Add functional + out variant for scaled_fp4_quant](../sources/prs/vllm/PR-34389.md), [[Kernel] Integrate SM100 MXFP8 blockscaled grouped MM and quant kernels](../sources/prs/vllm/PR-34448.md), [[Model Bash] DeepSeek R1 BF16 Min Latency QKV A GEMM (0.5% E2E Speedup)](../sources/prs/vllm/PR-34758.md), [[Bugfix] Gate 256-bit instructions to CUDA 12.9+](../sources/prs/vllm/PR-34791.md), [[Attention][Perf][Kernel] Replace torch.cat with vectorized CUDA kernel MLA query concat - DeepSeek-V3.2](../sources/prs/vllm/PR-34917.md), [[Model Runner V2] Support attention group](../sources/prs/vllm/PR-35036.md), [[Performance] Cublas Bf16 Gate with Fp32 Output](../sources/prs/vllm/PR-35121.md), [[Bugfix] Fix DSV3 kernels breaking _C and _moe_C on unsupported arches](../sources/prs/vllm/PR-35123.md), [[Bugfix] Fix expert_ids padding values in moe_align_block_size kernel](../sources/prs/vllm/PR-35161.md), [[BugFix] Fix fp4 quant kernel on CUDA 12.8](../sources/prs/vllm/PR-35210.md), [[Feat] Add CUDA torch fallbacks for fp8_mqa_logits/fp8_paged_mqa_logits_torch function](../sources/prs/vllm/PR-35271.md), [[Attention][Perf] Optimize cp_gather_and_upconvert_fp8_kv_cache - DeepSeek-v3.2](../sources/prs/vllm/PR-35290.md), [[Kernel] Add FlashInfer MoE A2A Kernel](../sources/prs/vllm/PR-36022.md), [Add 320 dimension size support to MLA](../sources/prs/vllm/PR-36161.md), [docs: fix wrong cc in int8.md](../sources/prs/vllm/PR-36209.md), [[Kernel] Fuse FP8 output quantization into merge_attn_states](../sources/prs/vllm/PR-36518.md), [Update Flashinfer to 0.6.6](../sources/prs/vllm/PR-36768.md), [[MTP][Sparse MLA] Take advantage of native MTP support in indexer when possible](../sources/prs/vllm/PR-36982.md), [[Kernel] Add gpt-oss Router GEMM kernel](../sources/prs/vllm/PR-37205.md), [[UX] Add flashinfer-cubin as CUDA default dep](../sources/prs/vllm/PR-37233.md), [[Perf] Set Flashinfer sparse MLA as default backend for FP8 kv cache](../sources/prs/vllm/PR-37252.md), [[Kernel] Add non-gated support for NVFP4 CUTLASS MoE](../sources/prs/vllm/PR-37320.md), [Add nvfp4 support to reshape_and_cache_flash](../sources/prs/vllm/PR-37332.md), [[torch.compile] Refactor Attention Quant Fusion Pass and Remove Boilerplate](../sources/prs/vllm/PR-37373.md), [[Perf][Kernel] Persistent TopK scheduler: unified CUDAGraph-safe kernel with dynamic per-row dispatch - DeepSeek-V3.2 DSA decode](../sources/prs/vllm/PR-37421.md), [[Kernel] Add MXFP4 W4A4 CUTLASS MoE kernel for SM100](../sources/prs/vllm/PR-37463.md), [[4/n] Migrate FP4/W4A8 CUTLASS kernels to torch stable ABI](../sources/prs/vllm/PR-37503.md), [refactor: abstract deepgemm support into platform](../sources/prs/vllm/PR-37519.md), [[Bugfix] Preserve CUDA arch suffix (a/f) for SM12x — fixes NVFP4 NaN on desktop Blackwell](../sources/prs/vllm/PR-37725.md), [[Kernel] Optimize SM120 CUTLASS blockwise FP8 GEMM](../sources/prs/vllm/PR-37970.md), [[Kernel] Add swapAB support for SM120 CUTLASS blockwise FP8 GEMM ](../sources/prs/vllm/PR-38325.md), [[CI Bugfix] Pre-download missing FlashInfer headers in Docker build](../sources/prs/vllm/PR-38391.md), [[NVIDIA] Bugfix NVFP4 DGX Spark and RTX50](../sources/prs/vllm/PR-38423.md), [[Perf] Batch KV cache swap copies via cuMemcpyBatchAsync](../sources/prs/vllm/PR-38460.md), [[Attention Backend] TurboQuant: 2-bit KV cache compression with 4x capacity](../sources/prs/vllm/PR-38479.md), [[Compile] Fix nvfp4 compile warning](../sources/prs/vllm/PR-38573.md), [[FA4] Update flash-attention to latest upstream FA4](../sources/prs/vllm/PR-38690.md), [[Refactor] Improve indexer decode path metadata preparation](../sources/prs/vllm/PR-38865.md), [[Bugfix] Fix broken explicit unquantized kv cache dtype support](../sources/prs/vllm/PR-38922.md), [[Bugfix] Fix GDN FLA kernel crashes with NULL_BLOCK_ID=0 CUDA graph padding](../sources/prs/vllm/PR-39064.md), [[XPU] Quick fix for TritonMLA to remove cuda hardcode](../sources/prs/vllm/PR-39088.md), [perf(moe): add tuned fused_moe config for RTX PRO 6000 Blackwell Server Edition](../sources/prs/vllm/PR-39183.md), [Use CU_MEMCPY_SRC_ACCESS_ORDER_ANY for batch KV cache swaps](../sources/prs/vllm/PR-39306.md), [Fix NUMA binding on non-CDMM Grace-Blackwell systems](../sources/prs/vllm/PR-39361.md), [fix: clamp NaN/Inf in topk_softmax to prevent duplicate expert IDs](../sources/prs/vllm/PR-39391.md), [[Perf] Fuse Zero Initializer for FP8 DeepGemm Block Quant Kernel](../sources/prs/vllm/PR-39547.md), [[Bugfix] Add Marlin kernel in block scaled mm kernel selection.](../sources/prs/vllm/PR-40105.md), [[Bugfix] moe lora align kernel grid](../sources/prs/vllm/PR-40131.md), [[Bugfix] Guard mxfp4_experts_quant bindings on ENABLE_NVFP4_SM100](../sources/prs/vllm/PR-40191.md), [[Performance][DSR1]: Fused RoPE+KVCache+q_concat for MLA](../sources/prs/vllm/PR-40392.md), [[Perf] Batch invariance with Cutlass fp8 support, 28.9% E2E latency improvement](../sources/prs/vllm/PR-40408.md), [[GDN] Enable FI Blackwell GDN prefill kernel](../sources/prs/vllm/PR-40717.md), [[DSV4] Add silu clamp limit to shared expert](../sources/prs/vllm/PR-40950.md), [[DSV4] Fuse norm and router for low latency scenario](../sources/prs/vllm/PR-41263.md), [Faster per-token fp8 group quant packed kernel for blackwell](../sources/prs/vllm/PR-41326.md), [[Bugfix] Fix condition to clear persistent topk so that it can be captured regardless](../sources/prs/vllm/PR-41665.md), [[MLA Attention Backend] Add TOKENSPEED_MLA backend for DSR1/Kimi K25 prefill + decode on Blackwell](../sources/prs/vllm/PR-41778.md), [[CUDA][CUTLASS] Enable cutlass scaled mm for non-compatible sizes ](../sources/prs/vllm/PR-41868.md), [[MoE] Move various experts classes to fused_moe/experts/](../sources/prs/vllm/PR-41979.md), [[Perf] Use 2D-grid to eliminate divmod in W8W8 group quant](../sources/prs/vllm/PR-42153.md), [[Misc] add humming to dependencies](../sources/prs/vllm/PR-42540.md), [[6/n] Migrate activation kernels, gptq, gguf, non cutlass w8a8 to libtorch stable ABI (continued)](../sources/prs/vllm/PR-42663.md), [[Refactor] Remove dead cuda kernels](../sources/prs/vllm/PR-42767.md), [[Perf] Padded nvfp4 quant kernel to remove additional copy, 2.4%~5.7% e2e performance improvement](../sources/prs/vllm/PR-42774.md), [[Perf][MLA] Enable FULL cudagraph capture for TRITON_MLA decode](../sources/prs/vllm/PR-42885.md), [[Kernel] (1/N) Machete - Hopper Optimized Mixed Precision Linear Kernel ](../sources/prs/vllm/PR-7174.md), [[Kernel] (2/N) Machete - Integrate into CompressedTensorsWNA16 and GPTQMarlin](../sources/prs/vllm/PR-7701.md), [[Bugfix] Fix Machete unittests failing with `NotImplementedError`](../sources/prs/vllm/PR-9218.md), [DeepGEMM — FP8 GEMM with Fine-Grained Scaling](../wiki/kernels/deepgemm.md), [FlashMLA — Multi-head Latent Attention](../wiki/kernels/flashmla.md), [FP8 Fine-Grained-Scale GEMM](../wiki/kernels/fp8-block-scale-gemm.md), [FlashInfer Track A FP8 Block-Scale MoE](../wiki/kernels/fused-moe.md), [NVFP4 Batched GEMV](../wiki/kernels/nvfp4-gemv.md), [DeepSeek Sparse Attention / Sparse MLA](../wiki/kernels/sparse-mla.md), [TensorRT-LLM Blackwell FP4 DSA Indexer](../wiki/kernels/tensorrt-llm-blackwell-indexer.md), [CCCL CUB Memory Primitives For Selection And Scan](../wiki/techniques/cccl-memory-primitives.md), [External Source-Map Research For Kernel Edits](../wiki/techniques/external-source-map-research.md) | +| `cute-dsl` | [CuTe DSL for Blackwell](../wiki/languages/cute-dsl.md) | [Twelve Attempts at an FP4 Kernel](../sources/blogs/amandeep-nvfp4-attempts.md), [Colfax Article Source Kernels](../sources/blogs/colfax-article-source-kernels.md), [Colfax CUTLASS Tutorial: GEMM Kernels Using Tensor Memory for Blackwell](../sources/blogs/colfax-cutlass-blackwell.md), [Colfax CUTLASS Kernels](../sources/blogs/colfax-cutlass-kernels.md), [FlashAttention-4 Blog](../sources/blogs/flash-attention-4.md), [NVFP4 GEMV and Improved NVFP4 GEMV](../sources/blogs/simon-nvfp4-gemv.md), [Tilus: A Tile-Level GPGPU Programming Language for Low-Precision Computation](../sources/blogs/tilus-nvidia.md), [Blackwell NVFP4 Kernel Hackathon Journey](../sources/blogs/yue-nvfp4-hackathon.md), [FlashInfer MLSys 2026 Track A: FP8 Block-Scale MoE](../sources/contests/flashinfer-mlsys26/track-a-fused-moe.md), [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [FlashInfer MLSys 2026 Track C: Gated Delta Net](../sources/contests/flashinfer-mlsys26/track-c-gated-delta-net.md), [GPU Mode NVFP4 Hackathon - Problem 1: Batched GEMV](../sources/contests/gpu-mode-nvfp4/problem-1-gemv.md), [CUTLASS Changelog: SM100/Blackwell Entries](../sources/docs/cutlass-changelog-sm100.md), [NVIDIA CUTLASS 4.5.0 Blackwell Sources](../sources/docs/nvidia-cutlass-blackwell.md), [[None][perf] Add more optimization options for MOE CuteDSL finalized kernel](../sources/prs/TensorRT-LLM/PR-10042.md), [[TRTLLM-9992][perf] Enable PDL for CuteDSL kernels and overlap MoeOutputMemset](../sources/prs/TensorRT-LLM/PR-10043.md), [[None][feat] CuteDSL MOE FC1 Enhancement](../sources/prs/TensorRT-LLM/PR-10088.md), [[TRTLLM-9457][feat] Add cute dsl fp8 gemm for Blackwell](../sources/prs/TensorRT-LLM/PR-10130.md), [[TRTLLM-9831][perf] Enable 2CTA with autotune for CuteDSL MoE and Grouped GEMM optimizations](../sources/prs/TensorRT-LLM/PR-10201.md), [[TRTLLM-10147][perf] Balanced random MoE workload generator for CuteDSL kernel UT, autotuner and layerwise benchmark](../sources/prs/TensorRT-LLM/PR-10279.md), [[TRTLLM-9661][chore] Further reduce tuning time for cuteDSL nvFP4 dense gemm.](../sources/prs/TensorRT-LLM/PR-10339.md), [[None] [feat] Add test script and raster M for gather fc1 kernel](../sources/prs/TensorRT-LLM/PR-10429.md), [[TRTLLM-10276][feat] Integrate cutedsl argmax kernel](../sources/prs/TensorRT-LLM/PR-10476.md), [[None] [feat] Add densegemm backend for MoE](../sources/prs/TensorRT-LLM/PR-10479.md), [[TRTLLM-9831][perf] Use TMA.RED to improve effective memory bandwidth](../sources/prs/TensorRT-LLM/PR-10987.md), [[None][feat] fuse shared to sparse experts in TRT-LLM Gen MoE](../sources/prs/TensorRT-LLM/PR-11143.md), [[https://nvbugs/5854860][fix] Fix cutedsl argmax on sm120](../sources/prs/TensorRT-LLM/PR-11181.md), [[TRTLLM-10004][feat] Enable GEMM -> AR with GEMM output in registered buffers](../sources/prs/TensorRT-LLM/PR-11589.md), [[TRTLLM-11092][feat] add support for visual gen FA4 attention backend](../sources/prs/TensorRT-LLM/PR-11697.md), [[https://nvbugs/5885070][fix] fix deepeplowlatency with cutedsl moe backend](../sources/prs/TensorRT-LLM/PR-11769.md), [[TRTLLM-10990][feat] Fuse SwiGLU and quant into shared expert](../sources/prs/TensorRT-LLM/PR-11897.md), [[TRTLLM-10407][feat] Integrate CuTE DSL top-k kernel for Blackwell](../sources/prs/TensorRT-LLM/PR-11900.md), [[TRTLLM-11289][feat] Integrate CuteDSL's bf16 dense GEMMs](../sources/prs/TensorRT-LLM/PR-12074.md), [[None][feat] CuteDSL MOE: Add raster along M/N support for blockscaled contiguous backbone kernel](../sources/prs/TensorRT-LLM/PR-12079.md), [[None][feat] Add DWDP (Distributed Weight Data Parallelism) support for MoE inference](../sources/prs/TensorRT-LLM/PR-12136.md), [[TRTLLM-10407][perf] Enable CuteDSL indexer_top_k in model](../sources/prs/TensorRT-LLM/PR-12236.md), [[TRTLLM-10407][perf] Add cute dsl single pass multi cta cluster topk](../sources/prs/TensorRT-LLM/PR-12354.md), [[None][feat] Add PDL support to CuTE DSL top-k kernels](../sources/prs/TensorRT-LLM/PR-12506.md), [[None][feat] Optimize mamba SSD prefill and extend flashinfer dispatch](../sources/prs/TensorRT-LLM/PR-12731.md), [[TRTLLM-11797][feat] Add cutedsl moe backend supporting for qwen3.5.](../sources/prs/TensorRT-LLM/PR-12799.md), [[TRTLLM-11585][feat] Add CUTEDSL moe backend for nemotron-h](../sources/prs/TensorRT-LLM/PR-12884.md), [[TRTLLM-34871][feat] Add cute dsl FP8 paged MQA logits decode kernel](../sources/prs/TensorRT-LLM/PR-13219.md), [[None][perf] FC2 DenseGEMM autotune: split-K, swap_ab, fine-grained tuning buckets](../sources/prs/TensorRT-LLM/PR-13833.md), [[TRTLLM-35237][feat] Add cute dsl FP4 paged MQA logits decode kernel](../sources/prs/TensorRT-LLM/PR-13929.md), [[None][feat] Enable EPLB for trtllm-gen and cutlass backend](../sources/prs/TensorRT-LLM/PR-8886.md), [[TRTLLM-9685] [feat] Add gather fc1 kernel by cuteDSL](../sources/prs/TensorRT-LLM/PR-9618.md), [Blockwise and Groupwise GEMM for Blackwell and Improvements for Hopper](../sources/prs/cutlass/PR-2139.md), [Example 77 add blackwell flash-attention bwd for MLA shape](../sources/prs/cutlass/PR-2466.md), [Add Blackwell MLA forward (shape: d=192, dv=128) implementation](../sources/prs/cutlass/PR-2472.md), [fix gqa issue for blackwell fmha.py](../sources/prs/cutlass/PR-2599.md), [Add tutorial fp16_gemm_1](../sources/prs/cutlass/PR-2750.md), [new example with TMA prefetch feature targeting for DRAM latency boun…](../sources/prs/cutlass/PR-2881.md), [[CuTeDSL] Fix: SM100 block-scale gemm overlapping accumulator](../sources/prs/cutlass/PR-2995.md), [[Cute-DSL] Add option for issue_clc_query without multicast](../sources/prs/cutlass/PR-3021.md), [[Hopper CuTeDSL] Add grouped GEMM kernel example](../sources/prs/cutlass/PR-3091.md), [[CLI] add cutedsl fp16 gemm tutorial from 2 to 6](../sources/prs/cutlass/PR-3106.md), [Update blackwell tutorial to be compatible with 4.5-dev version](../sources/prs/cutlass/PR-3130.md), [feat: Adding varlen support to cute-dsl sm80 bwd](../sources/prs/flash-attention/PR-1934.md), [[Cute,Fwd,Sm100] fp8 e4m3 and e5m2 support](../sources/prs/flash-attention/PR-2109.md), [[Cute,Flex,Fwd] Allow vectorized score_mod definitions](../sources/prs/flash-attention/PR-2236.md), [[Cute,Sm100,Fwd] add MLA 64/512 with topk sparsity for MQA 128 heads](../sources/prs/flash-attention/PR-2441.md), [feat: masked layout fp4 gemm using cute-dsl](../sources/prs/flashinfer/PR-1331.md), [Add python API for masked grouped gemm](../sources/prs/flashinfer/PR-1481.md), [fix: update cutedsl masked moe gemm](../sources/prs/flashinfer/PR-1488.md), [fix: update masked moe gemm fp4 tensor reshape](../sources/prs/flashinfer/PR-1495.md), [feat: scaling at fp4 gemm epilogue](../sources/prs/flashinfer/PR-1498.md), [Add benchmark for cutedsl gemm](../sources/prs/flashinfer/PR-1502.md), [bugfix: Fix stream handling in cutedsl gemm](../sources/prs/flashinfer/PR-1509.md), [refactor fp4 masked gemm cute-dsl implementation and add manual cache](../sources/prs/flashinfer/PR-1521.md), [feat: initial support for SM103, SM110, SM120, SM121](../sources/prs/flashinfer/PR-1608.md), [Support output signals for overlapping for cutedsl gemm](../sources/prs/flashinfer/PR-1677.md), [[cute_dsl] add gemm + all reduce (two_shot)](../sources/prs/flashinfer/PR-1695.md), [fix: fix cannot import name 'cuda' from 'cuda' in CUDA13](../sources/prs/flashinfer/PR-1764.md), [tests: upgrade cutlass, fix import and skip non-SM100 cutedsl two shot allreduce](../sources/prs/flashinfer/PR-1812.md), [raise error for group_gemm_fp8_nt_groupwise then num_groups > 1 on sm120/121](../sources/prs/flashinfer/PR-1862.md), [enable sm103 moe dsl backend](../sources/prs/flashinfer/PR-2149.md), [Fix gemm allreduce two shot](../sources/prs/flashinfer/PR-2171.md), [feat: Fused RMSNorm + FP4 Quantization Kernels in CuTe-DSL](../sources/prs/flashinfer/PR-2233.md), [fix: Add global scale support and optional output allocation for RMSNorm+FP4Quant fusion kernels](../sources/prs/flashinfer/PR-2260.md), [[WIP] Refactor: simplify torch -> cute-dsl boilerplate and enable tvm-ffi for cute-dsl kernels](../sources/prs/flashinfer/PR-2279.md), [fix: In-place Residual Update for add_rmsnorm_fp4quant](../sources/prs/flashinfer/PR-2385.md), [feat: Add output_both_sf_layouts option to add_rmsnorm_fp4quant API](../sources/prs/flashinfer/PR-2395.md), [feat: cuteDSL fp4 moe for better DSR1 performance.](../sources/prs/flashinfer/PR-2398.md), [perf: improve gdn decode cute-dsl kernels](../sources/prs/flashinfer/PR-2405.md), [refactor: simplify fp4 rmsnorm](../sources/prs/flashinfer/PR-2421.md), [refactor: refactoring cuda code to cute-dsl (part 1)](../sources/prs/flashinfer/PR-2428.md), [Add cute-dsl backends to mxfp[8,4]_quantization for future refactor](../sources/prs/flashinfer/PR-2443.md), [Ameyn/gdn decode cutedsl kernel](../sources/prs/flashinfer/PR-2498.md), [refactor: Port upstream CUTLASS fixes and refactor grouped_gemm_nt_masked GEMM module location](../sources/prs/flashinfer/PR-2503.md), [[Bug] Fix spark unit test failures for test_add_rmsnorm_fp4_quant_cute_dsl](../sources/prs/flashinfer/PR-2573.md), [fix: cute dsl nvfp4 moe routing index error](../sources/prs/flashinfer/PR-2629.md), [feat: support mxfp4 & mxfp8 entrypoint for blackwell cutedsl dense gemm](../sources/prs/flashinfer/PR-2660.md), [Add cute dsl mla decode op](../sources/prs/flashinfer/PR-2743.md), [[CuTe DSL] Add modular FMHA prefill and MLA decode attention kernels](../sources/prs/flashinfer/PR-2805.md), [CuteDSL MoE fix redundant output buffer zeroing](../sources/prs/flashinfer/PR-2811.md), [feat: Add CuTe-DSL backend for NVFP4 quantization](../sources/prs/flashinfer/PR-2838.md), [feat: add pdl support for cute dsl mla decode kernel support](../sources/prs/flashinfer/PR-2901.md), [perf: Optimize CuTe-DSL fp4 and fp8 quantization kernels](../sources/prs/flashinfer/PR-2904.md), [feat: Add CuTe DSL grouped-gemm + combine fusion support](../sources/prs/flashinfer/PR-2944.md), [feat: add PDL support to rmsnorm_fp4quant and add_rmsnorm_fp4quant CuTe DSL kernels](../sources/prs/flashinfer/PR-3008.md), [Prevent MoE autotuner buffer overflow on large token buckets](../sources/prs/flashinfer/PR-3025.md), [feat: Add backend="b12x" for mm_fp4 on SM120](../sources/prs/flashinfer/PR-3051.md), [feat: Add b12x CuTe DSL fused MoE for SM120](../sources/prs/flashinfer/PR-3066.md), [cute-dsl fmha prefill (cubin integration): remove front-padding, add attention_sink, and pdl support](../sources/prs/flashinfer/PR-3181.md), [fix(sm12x): fix micro-kernel workspace sizing when routed_rows > num_local_experts](../sources/prs/flashinfer/PR-3191.md), [fix(cute_dsl/moe): make autotuner bucket configuration adapt to runtime input](../sources/prs/flashinfer/PR-3216.md), [Support Kimi K2.5 H64 CuTe DSL MLA decode](../sources/prs/flashinfer/PR-3235.md), [fix(cute_dsl/moe): unbias autotuner profiling for tile_size enumeration](../sources/prs/flashinfer/PR-3252.md), [feat(moe): add SM120 W4A16 b12x kernels](../sources/prs/flashinfer/PR-3271.md), [feat(cute_dsl/moe): deterministic balanced autotune profile inputs](../sources/prs/flashinfer/PR-3286.md), [feat(cute_dsl/moe): add `moe_output_memset_inplace` dense memset wrapper](../sources/prs/flashinfer/PR-3328.md), [Support single batch overlap](../sources/prs/sglang/PR-10422.md), [Replace [silu_and_mul_]scaled_fp4_group_quant by Flashinfer equivalent](../sources/prs/sglang/PR-12376.md), [[diffusion] kernel fusion: gated residual layernorm scale shift and layernorm scale shift kernel fusion for Qwen-Image, WAN and HunyuanVideo](../sources/prs/sglang/PR-14717.md), [[jit-kernel] Add CuTe DSL GDN Decode Kernel](../sources/prs/sglang/PR-15631.md), [Move fa4 from sgl-kernel to jit kernel](../sources/prs/sglang/PR-17353.md), [[diffusion] Diffusion norm fusion for z-image](../sources/prs/sglang/PR-18762.md), [[SGLang-Diffusion] Fix custom op fake impl missing eps default for torch.compile](../sources/prs/sglang/PR-19725.md), [[diffusion] fix bug of copy_if](../sources/prs/sglang/PR-20094.md), [[KDA] Support CuTeDSL KDA decode kernel](../sources/prs/sglang/PR-21203.md), [Refactor JIT kernel CI to use run_suite.py registration system](../sources/prs/sglang/PR-21239.md), [Add dedicated FlashInferCuteDslMoE layer for standard-path FP4 MoE](../sources/prs/sglang/PR-21339.md), [[Bugfix] Lazy-import CuteDSL KDA kernel to fix AMD/ROCm startup crash](../sources/prs/sglang/PR-21428.md), [feat: Support flashinfer_cutedsl MoE runner with flashinfer alltoall backend](../sources/prs/sglang/PR-22669.md), [[Refactor] Refactor DeepEP dispatcher](../sources/prs/sglang/PR-22822.md), [Reland Cute-DSL FP4 dense GEMM](../sources/prs/sglang/PR-23590.md), [Use Cute-DSL NVFP4 quantization kernels](../sources/prs/sglang/PR-23745.md), [[MoE Refactor] Migrate flashinfer_cutedsl + DeepEP to MoeRunner](../sources/prs/sglang/PR-25525.md), [[NVIDIA] [3/N] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked ](../sources/prs/sglang/PR-9199.md), [[Feature] Support cluster launch, query, synchronization and barrier operations](../sources/prs/tilelang/PR-1874.md), [[Feature] 2-SM support for TMA, TMEM and TCGEN5MMA on Blackwell](../sources/prs/tilelang/PR-1882.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [[Bugfix] Fix CuTeDSL autotune cache invalid ELF header (#1967)](../sources/prs/tilelang/PR-1972.md), [[Feature] Support TMA store in T.tma_copy()](../sources/prs/tilelang/PR-1981.md), [[Transform] Add InjectTcgen05Fence pass](../sources/prs/tilelang/PR-2003.md), [[CUDA] Support int4 `T.gemm`](../sources/prs/tilelang/PR-2063.md), [[TIR][IR] Update to use tirx](../sources/prs/tilelang/PR-2216.md), [[Python] Drop Python 3.9 support](../sources/prs/tilelang/PR-2218.md), [[MoE] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked](../sources/prs/vllm/PR-25990.md), [[FIX] Add NO_MUL activation support for modular kernel path](../sources/prs/vllm/PR-31528.md), [fix: Add SM120 (RTX Blackwell) support for FlashInfer CUTLASS NVFP4 MoE kernels](../sources/prs/vllm/PR-33417.md), [[Bug][MoE] Strengthen _supports_current_device() checks in the TRTLLM FP8, NVFP4, and FlashInfer CuteDSL MoE experts](../sources/prs/vllm/PR-36728.md), [[MoE/EPLB] Fix FlashInfer nvfp4 experts + EPLB correctness](../sources/prs/vllm/PR-37217.md), [[MoE] Move FlashInfer CuteDSL experts into fused_moe/experts/](../sources/prs/vllm/PR-37759.md), [[MoE Kernel] Flashinfer nvfp4 cutedsl moe kernel integration](../sources/prs/vllm/PR-38050.md), [[Quantization] Add FlashInfer CuteDSL batched experts backend for NVFP4 MoE](../sources/prs/vllm/PR-38251.md), [[Bugfix] [Tests] Enforce `out` tensor device in `kernel/moe/test_cutedsl_moe.py`](../sources/prs/vllm/PR-39644.md), [[DSv4] Improved fused Indexer Q quant kernel](../sources/prs/vllm/PR-41428.md), [[DSv4] Improved dequant gather K cache kernel](../sources/prs/vllm/PR-42236.md), [[Perf] Re-enable flashinfer autotune by default and cleanup](../sources/prs/vllm/PR-42857.md), [add cutedsl dsv4 indexer fp8 kernel](../sources/prs/vllm/PR-42899.md), [[Model Refactoring] Move deepseek_v4_ops to models/deepseek_v4 [3/N]](../sources/prs/vllm/PR-43073.md), [FlashAttention-4](../wiki/kernels/flash-attention-4.md), [FlashAttention SM100 MLA TopK Sparse Forward](../wiki/kernels/flash-attention-sm100-mla-topk.md), [FP8 Fine-Grained-Scale GEMM](../wiki/kernels/fp8-block-scale-gemm.md), [FlashInfer Track A FP8 Block-Scale MoE](../wiki/kernels/fused-moe.md), [Gated Delta Net — Linear Attention](../wiki/kernels/gated-delta-net.md), [NVFP4 Batched GEMV](../wiki/kernels/nvfp4-gemv.md), [DeepSeek Sparse Attention / Sparse MLA](../wiki/kernels/sparse-mla.md), [External Source-Map Research For Kernel Edits](../wiki/techniques/external-source-map-research.md) | | `cutile` | | [cuTile Python DSL Reference](../sources/docs/cutile-python-dsl.md) | | `jax-pallas` | | [Writing High-Performance Matrix Multiplication Kernels for Blackwell with JAX Pallas](../sources/blogs/jax-pallas-blackwell-matmul.md) | -| `ptx` | [PTX Instructions for SM100](../wiki/languages/ptx-sm100.md) | [Twelve Attempts at NVFP4 Batched GEMV](../sources/blogs/amandeep-nvfp4-attempts.md), [tcgen05 for dummies](../sources/blogs/tcgen05-tutorial.md), [Tilus: A Tile-Level GPGPU Programming Language for Low-Precision Computation](../sources/blogs/tilus-nvidia.md), [Blackwell NVFP4 Kernel Hackathon Journey](../sources/blogs/yue-nvfp4-hackathon.md), [GPU Mode NVFP4 Hackathon - Problem 1: Batched GEMV](../sources/contests/gpu-mode-nvfp4/problem-1-gemv.md), [GPU Mode NVFP4 Hackathon - Problem 2: NVFP4 GEMM](../sources/contests/gpu-mode-nvfp4/problem-2-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 3: Gated Dual GEMM](../sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 4: Grouped GEMM](../sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md), [NVIDIA Blackwell Compatibility Guide](../sources/docs/blackwell-compatibility-guide.md), [PTX ISA SM100 Instructions Reference](../sources/docs/nvidia-ptx-isa-sm100.md), [[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes](../sources/prs/DeepGEMM/PR-304.md), [Sync nv_dev with upstream #316 (Mega MoE optimizations & benchmarks)](../sources/prs/DeepGEMM/PR-328.md), [misc: point triton blackwell-ptxas to local cuda ptxas](../sources/prs/flashinfer/PR-2543.md), [[AARCH64][CD][CUDA13][Triton][PTXAS] Turn on BUILD_BUNDLE_PTXAS=1 ](../sources/prs/pytorch/PR-164236.md), [[release-only] Remove +ptx from cuda 13.0 builds](../sources/prs/pytorch/PR-175567.md), [Add swizzle layout detection and automatic merging for layout conflicts](../sources/prs/tilelang/PR-1736.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [[CUDA] Support int4 `T.gemm`](../sources/prs/tilelang/PR-2063.md), [[Bugfix] Enable `.shared::cta` in TMA copy paths only on CUDA 12.8+](../sources/prs/tilelang/PR-2087.md), [[TIR][IR] Update to use tirx](../sources/prs/tilelang/PR-2216.md), [Update launch_bounds_utils.h for correct compile on Multiple Cuda Arch - PTXAS out of range Warning](../sources/prs/vllm/PR-25843.md), [DeepGEMM — FP8 GEMM with Fine-Grained Scaling](../wiki/kernels/deepgemm.md), [NVFP4 GEMM — 4-bit Floating Point Matrix Multiply](../wiki/kernels/nvfp4-gemm.md), [NVFP4 Batched GEMV](../wiki/kernels/nvfp4-gemv.md) | -| `python` | | [cuTile Python DSL Reference](../sources/docs/cutile-python-dsl.md), [Performance: reducing the percentage of FFMA interleaving yields a sight performance gain, roughly 0.5%](../sources/prs/DeepGEMM/PR-42.md), [[None][feat] sm100 weight-only kernel](../sources/prs/TensorRT-LLM/PR-10190.md), [[TRTLLM-9798][feat] Change to use new DeepGEMM MQA sm100 kernel for MTP-3](../sources/prs/TensorRT-LLM/PR-10226.md), [[None][feat] MiniMax M2 support](../sources/prs/TensorRT-LLM/PR-10532.md), [[None][feat] Remove non flash attetnion style fmha_v2 kernel for hopper](../sources/prs/TensorRT-LLM/PR-11381.md), [[None][fix] Fix int4 awq for sm120/121](../sources/prs/TensorRT-LLM/PR-11561.md), [[None][fix] Fix SM120 issue for rms_norm with nvfp4_quant_fusion](../sources/prs/TensorRT-LLM/PR-11774.md), [[None][feat] GLM 5 support and DSA MTP fixes](../sources/prs/TensorRT-LLM/PR-11990.md), [[TRTLLM-11285][feat] Fuse indexer wk + weights_proj into single GEMM in TF32 for DS-V3.2](../sources/prs/TensorRT-LLM/PR-12055.md), [[https://nvbugs/5983390][fix] Remove redundant D2H sync to optimize perf](../sources/prs/TensorRT-LLM/PR-12445.md), [[#12634][feat] AutoDeploy: Support rank 256 MLA in flashinfer_mla](../sources/prs/TensorRT-LLM/PR-12519.md), [[https://nvbugs/5879577][fix] Fix KeyError in DeepSeekV3Lite FP8 MTP weight loading](../sources/prs/TensorRT-LLM/PR-12530.md), [[None][feat] Add bf16 trtllm-gen moe support through flashinfer.](../sources/prs/TensorRT-LLM/PR-12738.md), [[None][fix] Propagate init_load_balancer to DeepGemmFusedMoE in create_moe_backend](../sources/prs/TensorRT-LLM/PR-13207.md), [[TRTLLM-11127][feat] add W4A8_MXFP4_FP8 MoE unit test support](../sources/prs/TensorRT-LLM/PR-13401.md), [[TRTLLM-11285][perf] Force enable TF32 tensor cores for DSA indexer fused GEMM](../sources/prs/TensorRT-LLM/PR-13452.md), [[TRTLLM-12128][feat] enable SageAttention for Wan/FLUX (new commits)](../sources/prs/TensorRT-LLM/PR-13570.md), [[TRTLLM-12316][feat] Integrate FP4 indexer for DSv4](../sources/prs/TensorRT-LLM/PR-13575.md), [[None][feat] Enable EPLB for DeepSeek-V4](../sources/prs/TensorRT-LLM/PR-13595.md), [[None][feat] Add bf16 trtllm moe through flashinfer.](../sources/prs/TensorRT-LLM/PR-13689.md), [[None][fix] Use compressed lengths for DeepSeek-V4 indexer](../sources/prs/TensorRT-LLM/PR-13802.md), [[None][feat] Update FMHA cubins for head_dim 80](../sources/prs/TensorRT-LLM/PR-13808.md), [[TRTLLM-12503][feat] Parallel VAE independent scaling and fix arg passing](../sources/prs/TensorRT-LLM/PR-13873.md), [[None][feat] enable TRTLLM-Gen internal routing](../sources/prs/TensorRT-LLM/PR-13997.md), [[None][feat] Enable 2 DSv4 perf optimizations by default](../sources/prs/TensorRT-LLM/PR-14120.md), [[TRTLLM-12462][fix] Fix FP8 block scaling GEMM autotuner cache growth](../sources/prs/TensorRT-LLM/PR-14165.md), [[https://nvbugs/6025177][test] rcca tests using kimi k2.5 fp4](../sources/prs/TensorRT-LLM/PR-14172.md), [[https://nvbugs/6163147][fix] swap layer.mlp in place for Mixtral modelopt export](../sources/prs/TensorRT-LLM/PR-14179.md), [[None][fix] Avoid dp_size x ep_size double-count in MegaMoEDeepGemm SymmBuffer](../sources/prs/TensorRT-LLM/PR-14213.md), [[None][refactor] clean up AttentionForwardArgs](../sources/prs/TensorRT-LLM/PR-14244.md), [[None][fix] Handle unset attention_dp_relax in ADP routers](../sources/prs/TensorRT-LLM/PR-14276.md), [[https://nvbugs/6095421][fix] Update resolve_moe_backend](../sources/prs/TensorRT-LLM/PR-14282.md), [[None][chore] Update Claude Code agents and skills](../sources/prs/TensorRT-LLM/PR-14344.md), [[None][feat] GPT-OSS Sm120/Sm121 Support](../sources/prs/TensorRT-LLM/PR-7937.md), [[TRTLLM-8535][feat] Support DeepSeek V3.2 with FP8 + BF16 KV cache/NVFP4 + BF16 KV cache](../sources/prs/TensorRT-LLM/PR-8405.md), [[TRTLLM-8827] [feat] Enable low precision alltoall for Cutlass and TRTLLMGen backends](../sources/prs/TensorRT-LLM/PR-8675.md), [[TRTLLM-8958][feat] and [TRTLLM-8960]: create ConfigurableMoE and support TRTLLMGenFusedMoE as backend](../sources/prs/TensorRT-LLM/PR-9486.md), [[https://nvbugs/5726962][feat] Apply fusion for W4AFP8_AWQ MoE](../sources/prs/TensorRT-LLM/PR-9838.md), [Experimental Python cooperative algorithms](../sources/prs/cccl/PR-1973.md), [[cuda.compute]: Fix faulty pointer arithmetic calculation in CUB dispatch](../sources/prs/cccl/PR-7940.md), [Expose `max_segment_size` guarantee in cuda.compute](../sources/prs/cccl/PR-8284.md), [[cub]: implement utilities for policy selection](../sources/prs/cccl/PR-8355.md), [Hopper Grouped GEMM support for FP8 Accum](../sources/prs/cutlass/PR-2123.md), [fix gqa issue for blackwell fmha.py](../sources/prs/cutlass/PR-2599.md), [Add tutorial fp16_gemm_1](../sources/prs/cutlass/PR-2750.md), [new example with TMA prefetch feature targeting for DRAM latency boun…](../sources/prs/cutlass/PR-2881.md), [[Cute-DSL] Add option for issue_clc_query without multicast](../sources/prs/cutlass/PR-3021.md), [[Hopper CuTeDSL] Add grouped GEMM kernel example](../sources/prs/cutlass/PR-3091.md), [Support for Group GEMM in CUTLASS Profiler for GeForce and Spark](../sources/prs/cutlass/PR-3092.md), [[CLI] add cutedsl fp16 gemm tutorial from 2 to 6](../sources/prs/cutlass/PR-3106.md), [Update blackwell tutorial to be compatible with 4.5-dev version](../sources/prs/cutlass/PR-3130.md), [Small Tile N BlockScaled GEMM + Grouped GEMM on SM12x](../sources/prs/cutlass/PR-3176.md), [Add var-seq-len to FA3 fp16 / bf16 fwd](../sources/prs/flash-attention/PR-1072.md), [FA3 FP8 qkv descales + restore max offset for h128 causal + added sync for producer WG](../sources/prs/flash-attention/PR-1173.md), [Add seqused_q in fwd / bwd and seqused_k in bwd in hopper FA.](../sources/prs/flash-attention/PR-1182.md), [Add local attention in Hopper FAv3](../sources/prs/flash-attention/PR-1233.md), [Paged Attention support for FA3](../sources/prs/flash-attention/PR-1268.md), [FA3 paged attention: Readiness for Cutlass 3.6 / default value for block_table](../sources/prs/flash-attention/PR-1331.md), [Support hdimQK != hdimV backward](../sources/prs/flash-attention/PR-1604.md), [Improve causal backward determinism perf with SPT schedule](../sources/prs/flash-attention/PR-1893.md), [[Cute,Fwd,Sm100] Implement SplitKV](../sources/prs/flash-attention/PR-1940.md), [Blackwell FlashAttention-BWD (v1.0)](../sources/prs/flash-attention/PR-1945.md), [[Cute] Block sparse support Sm100](../sources/prs/flash-attention/PR-1985.md), [[Cute,Fwd,Sm100] Support `q_stage=1` for inference](../sources/prs/flash-attention/PR-1993.md), [[Cute,Fwd,Sm100] Support paged attention](../sources/prs/flash-attention/PR-1999.md), [[Cute,Sm100,Fwd] use correction warps for epi when not using TMA](../sources/prs/flash-attention/PR-2014.md), [[Cute,Fwd,Sm100] don't pass mask_fn to softmax_step generically](../sources/prs/flash-attention/PR-2026.md), [[Cute,Bwd,Sm100] enable deterministic mode for sm100 bwd and fix race conditions](../sources/prs/flash-attention/PR-2033.md), [[Cute,Fwd] Extend score_mod to variable sequence length](../sources/prs/flash-attention/PR-2043.md), [Add score-mod bwd support ](../sources/prs/flash-attention/PR-2070.md), [Add blocksparse support for bwd on blackwell](../sources/prs/flash-attention/PR-2085.md), [Fix IMA in fwd on m boundary](../sources/prs/flash-attention/PR-2091.md), [Add pack-gqa fwd support for sparse impl w/ broadcasted H dim](../sources/prs/flash-attention/PR-2098.md), [[Cute,Fwd,Sm100] distributed offset calculation for paged KV](../sources/prs/flash-attention/PR-2104.md), [[NVIDIA] Enable Jetson Thor FA4](../sources/prs/flash-attention/PR-2108.md), [[CUTE][SM90]Enable pack-gqa with broadcasted maskmods](../sources/prs/flash-attention/PR-2145.md), [[Cute][Flex]Add pack-gqa divmod](../sources/prs/flash-attention/PR-2180.md), [[Cute,Fwd,Sm100] support irregular qhead / kvhead ratios](../sources/prs/flash-attention/PR-2186.md), [[Ai-assisted] CLC work stealing](../sources/prs/flash-attention/PR-2218.md), [[Bwd,Sm120] Add SM120 backward pass support](../sources/prs/flash-attention/PR-2330.md), [Add SM120 varlen attention support](../sources/prs/flash-attention/PR-2333.md), [[Fwd,Sm90] Add paged KV attention support (tma and cp.async)](../sources/prs/flash-attention/PR-2360.md), [[Cute,Sm100,Bwd] refine bwd swizzle for deterministic](../sources/prs/flash-attention/PR-2390.md), [Feat([FA4][CUTE DSL]) Add head_dim=256 support (forward + backward)](../sources/prs/flash-attention/PR-2412.md), [Add CLC scheduler heuristic](../sources/prs/flash-attention/PR-2455.md), [[hd256] Improve forward kernel with exp2 FMA emulation (3% to 9% performance gain)](../sources/prs/flash-attention/PR-2488.md), [[hd256] Add TMA paged KV support to SM100 2CTA forward kernel](../sources/prs/flash-attention/PR-2489.md), [[FA4][hd256] Backward TMA bulk-store epilogue + LSE/dpsum coalesce](../sources/prs/flash-attention/PR-2497.md), [Fix ZeroDivisionError in num_splits_heuristic for empty Q workloads](../sources/prs/flash-attention/PR-2515.md), [bugfix: import wrapper of mla decode](../sources/prs/flashinfer/PR-1013.md), [Add fp4 quantization swizzling tests](../sources/prs/flashinfer/PR-1157.md), [feat: nvshmem python bindings](../sources/prs/flashinfer/PR-1160.md), [feat: logits processor fustion rule for temperature softmax](../sources/prs/flashinfer/PR-1170.md), [Expose fp4 blockscale swizzling kernel](../sources/prs/flashinfer/PR-1176.md), [[feat] support block sparse attention w/ variable block sizes and head-wise sparse patterns](../sources/prs/flashinfer/PR-1177.md), [bugfix: fix invalid blackwell fmha unittests](../sources/prs/flashinfer/PR-1181.md), [bugfix: fix blackwell fmha hanging issue for empty kv_len](../sources/prs/flashinfer/PR-1198.md), [Add DeepGEMM kernels](../sources/prs/flashinfer/PR-1209.md), [Fix test_groupwise_scaled_gemm_fp8.py](../sources/prs/flashinfer/PR-1211.md), [init add gemm fp8 using cudnn backend](../sources/prs/flashinfer/PR-1264.md), [feat: add masked deepgemm support and benchmarking](../sources/prs/flashinfer/PR-1266.md), [hotfix: fix deepgemm artifactory hash](../sources/prs/flashinfer/PR-1278.md), [fix: update trtllm-gen fmha benchmark](../sources/prs/flashinfer/PR-1280.md), [Unify groupwise fp8 GEMM test](../sources/prs/flashinfer/PR-1281.md), [add mm_fp4 use cudnn backend](../sources/prs/flashinfer/PR-1288.md), [Remove FAST_BUILD FLAG for MOE](../sources/prs/flashinfer/PR-1291.md), [Support loading autotuned results from json for cutlass fp4 moe backends](../sources/prs/flashinfer/PR-1310.md), [test qkvo quantization not equal to 1.](../sources/prs/flashinfer/PR-1314.md), [minor: add trtllm_gen_mla benchmark](../sources/prs/flashinfer/PR-1316.md), [Allow cudnn prefill kernels to be called natively](../sources/prs/flashinfer/PR-1317.md), [refactor: Improved metainfo for trtllm-gen kernels](../sources/prs/flashinfer/PR-1328.md), [add torch float4_e2m1fn_x2 check for cudnn fp4 backend](../sources/prs/flashinfer/PR-1333.md), [[Fix] remove torch 2.8 requirement for FP4 GEMM](../sources/prs/flashinfer/PR-1334.md), [Fix bench deepgemm setting](../sources/prs/flashinfer/PR-1344.md), [Support passing kv_data_type to MultiLevelCascadeAttentionWrapper.plan()](../sources/prs/flashinfer/PR-1350.md), [[fix] remove (view) transpose to keep consistent with majorness MN requirement.](../sources/prs/flashinfer/PR-1358.md), [hotfix: update mxfp4 groupwise-scaled gemm unittests](../sources/prs/flashinfer/PR-1359.md), [Update autotune results for the nvfp4 cutlass moe backends for v0.2.9](../sources/prs/flashinfer/PR-1361.md), [refactor: download trtllm gemm metadata from server](../sources/prs/flashinfer/PR-1378.md), [Allow BatchPrefillPagedWrapper to call cudnn API](../sources/prs/flashinfer/PR-1384.md), [Adding FP8 benchmark on attention and matmul testing](../sources/prs/flashinfer/PR-1390.md), [Add Mxfp4 trtllm-gen moe unit tests](../sources/prs/flashinfer/PR-1399.md), [feature: enable cublas for fp4 gemm when cudnn == 9.11.1 or >= 9.13](../sources/prs/flashinfer/PR-1405.md), [Faster weight processing (moe nvfp4)](../sources/prs/flashinfer/PR-1412.md), [refactor: Sink attention AoT](../sources/prs/flashinfer/PR-1427.md), [Fix redundant kernels in moe](../sources/prs/flashinfer/PR-1428.md), [bugfix: fix perf issue by using fp8 graph that can use cublaslt](../sources/prs/flashinfer/PR-1435.md), [feat: enable trtllm-gen attn speculative decoding verify by decode](../sources/prs/flashinfer/PR-1453.md), [Fix TRTLLM NVFP4-out attention kernel scale factor dim issue](../sources/prs/flashinfer/PR-1460.md), [feat: Enable multiple fused-moe backends](../sources/prs/flashinfer/PR-1472.md), [refactor: unify autotuner for bmm_fp8](../sources/prs/flashinfer/PR-1479.md), [fix missing enable_pdl argument in trtllm-gen fp4 moe](../sources/prs/flashinfer/PR-1480.md), [Add python API for masked grouped gemm](../sources/prs/flashinfer/PR-1481.md), [flashinfer_benchmark QoL Improvements and Attention FP8 Support](../sources/prs/flashinfer/PR-1512.md), [Remove cuda-python from dependency and check at runtime](../sources/prs/flashinfer/PR-1534.md), [Add sm check for sm100 only cutlass/trtllm kernel](../sources/prs/flashinfer/PR-1535.md), [feat: Add fp8-qkv, fp16/bf16 output MHA](../sources/prs/flashinfer/PR-1540.md), [perf: Enable SplitK and fix tile-scheduling for moe fp4 fused moe](../sources/prs/flashinfer/PR-1548.md), [feat: Support for inferring out_dtype from out.dtype for TRTLLM attention kernel](../sources/prs/flashinfer/PR-1578.md), [refactor: Expose calculate_tile_tokens_dim function](../sources/prs/flashinfer/PR-1581.md), [bugfix: Fix test_fp4_quantize test bug](../sources/prs/flashinfer/PR-1585.md), [fix: limit the number of nvcc threads for each kernel](../sources/prs/flashinfer/PR-1589.md), [fix: Improve TRTLLM attention kernel out_dtype unit test](../sources/prs/flashinfer/PR-1590.md), [bugfix: fix unittest test_fp8_quantize](../sources/prs/flashinfer/PR-1599.md), [feat: Enable MnnvlMemory (for alltoallv) on B200](../sources/prs/flashinfer/PR-1601.md), [feat: add support of fp4_batched_quantize](../sources/prs/flashinfer/PR-1633.md), [fix: pass workspace for trtllm-gen attention](../sources/prs/flashinfer/PR-1635.md), [test: pytest.mark.xfail on deepgemm](../sources/prs/flashinfer/PR-1636.md), [bugfix: Fix FLOPS calculation for bench_trtllm_gen_mla.py](../sources/prs/flashinfer/PR-1640.md), [fix: zero-init workspace buffer for trtllm-gen fmha](../sources/prs/flashinfer/PR-1643.md), [Added mx_fp4 support using the cudnn backend](../sources/prs/flashinfer/PR-1644.md), [Add benchmark for MLARopeQuantize](../sources/prs/flashinfer/PR-1656.md), [test: update fused_moe test to random scale factor](../sources/prs/flashinfer/PR-1665.md), [[Hotfix] `test_fp4_quantize.py` failure on sm103](../sources/prs/flashinfer/PR-1666.md), [TGV GEMM as a BF16 backend alternative to cuBLAS](../sources/prs/flashinfer/PR-1668.md), [test: better fp8 quantization init for fused_moe test](../sources/prs/flashinfer/PR-1674.md), [[misc] add a wrapper class for attention sink jit args](../sources/prs/flashinfer/PR-1679.md), [Update deepgemm backend for 103a](../sources/prs/flashinfer/PR-1694.md), [[cute_dsl] add gemm + all reduce (two_shot)](../sources/prs/flashinfer/PR-1695.md), [hotfix: Hotfix for `test_pod_kernels.py` on B300](../sources/prs/flashinfer/PR-1698.md), [feat: Benchmark mm_fp4 mxfp4 support and gemm autotune support. Restore mm_fp4 API behavior](../sources/prs/flashinfer/PR-1706.md), [bugfix: increase workspace to make trtllm gen attention unit test pass](../sources/prs/flashinfer/PR-1707.md), [test: skip the unsupported test cases for sm120/121](../sources/prs/flashinfer/PR-1710.md), [perf: Add tuning config for cutlass moe for a hardware](../sources/prs/flashinfer/PR-1716.md), [feat: port fast_decode_plan from sgl](../sources/prs/flashinfer/PR-1745.md), [tests: xfail attention sink UT for sliding window + non causal case](../sources/prs/flashinfer/PR-1752.md), [tests: xfail moe quantization classes mxfp8_bf16 UTs on sm103 ](../sources/prs/flashinfer/PR-1754.md), [Fix tests/test_trtllm_gen_attention.py::test_trtllm_batch_prefill, ::test_trtllm_batch_decode mismatch error](../sources/prs/flashinfer/PR-1755.md), [fix: should pass global_override_indptr_cpu in fast_decode_plan param list](../sources/prs/flashinfer/PR-1757.md), [Added xfail for mx_fp4 matmul on SM120](../sources/prs/flashinfer/PR-1766.md), [tests: skip non SM100/103 for grouped deepgemm](../sources/prs/flashinfer/PR-1767.md), [add test case for trtllm gen fused moe with kimi k2 problem sizes](../sources/prs/flashinfer/PR-1768.md), [Waive / disable test_mla_decode_kernel.py::test_mla_decode_kernel for not sm80 ](../sources/prs/flashinfer/PR-1771.md), [Support checks PoC](../sources/prs/flashinfer/PR-1809.md), [tests: Update support for tgv_gemm to SM100 only and add to ut](../sources/prs/flashinfer/PR-1810.md), [fix: fp4 moe on sm120](../sources/prs/flashinfer/PR-1817.md), [misc: fix some B200 GEMM bench](../sources/prs/flashinfer/PR-1883.md), [fix: Fix trtllm-gen prefill IMA when batch_size==1](../sources/prs/flashinfer/PR-1912.md), [Add realistic bench for persistent kernel ](../sources/prs/flashinfer/PR-1942.md), [fix: Add cutlass as an mm_fp4 backend in compute capability 12.0 in benchmark code](../sources/prs/flashinfer/PR-1959.md), [fix: ensure SM120/121 SFA/SFB contiguity](../sources/prs/flashinfer/PR-1963.md), [Feature: Add support for L40 FusedMoE in cutlass path](../sources/prs/flashinfer/PR-1973.md), [fix: Make attention microbenchmark correctly use page table](../sources/prs/flashinfer/PR-1976.md), [fix: Skipping attention sink Blackwell test outside of Blackwell](../sources/prs/flashinfer/PR-1978.md), [feat: Add backend='auto' to mm_fp4 and enable autotune for backend='cudnn'](../sources/prs/flashinfer/PR-1979.md), [unittest: Add head dim 256 test cases and mark as xfail](../sources/prs/flashinfer/PR-1999.md), [Fix trtllm-gen attention illegal memory access](../sources/prs/flashinfer/PR-2002.md), [fix: Enable SM121 for mm_fp4](../sources/prs/flashinfer/PR-2012.md), [feat: suitable_auto_backends to prune auto backends, bmm_fp8 refactor, heuristic_func intake](../sources/prs/flashinfer/PR-2029.md), [Added an initial implementation of Q and KV Cache in fp8 and to use t…](../sources/prs/flashinfer/PR-2035.md), [test: Skip test_fp8_quantize.py on Hopper](../sources/prs/flashinfer/PR-2052.md), [misc: Add XQA decode to microbenchmark for sm90 and sm120](../sources/prs/flashinfer/PR-2055.md), [[Test] Optimize test_trtllm_gen_fused_moe.py](../sources/prs/flashinfer/PR-2072.md), [unittest: improve the efficiency of xqa unittests](../sources/prs/flashinfer/PR-2075.md), [fix: fix test_trtllm_gen_attention when max_seq_len < page_size](../sources/prs/flashinfer/PR-2076.md), [Patch sm103 for 3xfp4 moe generation](../sources/prs/flashinfer/PR-2082.md), [refactor: update dpsk fused_moe test [1]](../sources/prs/flashinfer/PR-2088.md), [refactor: pass hopper deepgemm include directory through python](../sources/prs/flashinfer/PR-2090.md), [refactor: update dpsk fused_moe test [2]](../sources/prs/flashinfer/PR-2097.md), [[DSR1] Added MLA test](../sources/prs/flashinfer/PR-2100.md), [fix: Fix bench_mm_fp8.py](../sources/prs/flashinfer/PR-2129.md), [A unified API for the MNNVL and single-node/multi-GPU AllReduce kernels.](../sources/prs/flashinfer/PR-2130.md), [fix(trtllm): reset negative strideBatch to 0 for ragged KV layout to …](../sources/prs/flashinfer/PR-2134.md), [fix: some bugs of headDim 256 trtllm-gen fmha kernels. ](../sources/prs/flashinfer/PR-2137.md), [Enable Hopper FA3 FP8 attention in decode.py](../sources/prs/flashinfer/PR-2148.md), [refactor: Move mla code from decode.py to mla.py and add to documentation](../sources/prs/flashinfer/PR-2163.md), [fix: compile flags for trtllm fmha_v2 ](../sources/prs/flashinfer/PR-2175.md), [Rename noauxtc to fused_topk_deepseek](../sources/prs/flashinfer/PR-2181.md), [Permute page table in benchmarking](../sources/prs/flashinfer/PR-2194.md), [misc: support checks for gemm](../sources/prs/flashinfer/PR-2214.md), [Fp8 attention are now part of cuDNN 9.17.1](../sources/prs/flashinfer/PR-2241.md), [test: Fix MNNVL tests to skip when container lacks SYS_PTRACE capability](../sources/prs/flashinfer/PR-2245.md), [feat: Add support for bmm mxfp8](../sources/prs/flashinfer/PR-2256.md), [Fix CUTLASS FP8 gemm correctness issue on SM120/SM121 for shapes where N is not divisible by ScaleGranularityN.](../sources/prs/flashinfer/PR-2261.md), [test: use .float() in in F.cosine_similarity() in bmm_fp8 test](../sources/prs/flashinfer/PR-2266.md), [Tiny fix bench tgv gemm](../sources/prs/flashinfer/PR-2277.md), [fix: Decode benchmark's fa2_tc uses backend=fa2 in wrapper](../sources/prs/flashinfer/PR-2302.md), [Support both 3D and 4D kv_cache shapes in MLA APIs](../sources/prs/flashinfer/PR-2334.md), [Added the cudnn backend Ragged KV Cache wrapper](../sources/prs/flashinfer/PR-2352.md), [benchmarks: Add norm and quantization routines to microbenchmark harness.](../sources/prs/flashinfer/PR-2362.md), [feat: [Qwen3-Next] Add Cute DSL GDN decode kernel and tests](../sources/prs/flashinfer/PR-2370.md), [feat: BF16 GEMM using cuDNN backend](../sources/prs/flashinfer/PR-2376.md), [A Blackwell-optimized version of selective_state_update (decode)](../sources/prs/flashinfer/PR-2387.md), [perf: mm_fp4 heuristic prioritizes CUTLASS over cuDNN on SM103](../sources/prs/flashinfer/PR-2404.md), [perf: add fp4 GEMM tile configs and streamK scheduler for SM120](../sources/prs/flashinfer/PR-2460.md), [fix: blockscale moe routine supports non-DS routing](../sources/prs/flashinfer/PR-2476.md), [fix: Fix memory bandwidth calculation in MLA benchmarks](../sources/prs/flashinfer/PR-2479.md), [Feat/gdn decode pooled](../sources/prs/flashinfer/PR-2521.md), [feat: BF16 GEMM benchmarking support](../sources/prs/flashinfer/PR-2525.md), [pick fa2 for BatchDecodeWithPagedKVCacheWrapper auto backend](../sources/prs/flashinfer/PR-2530.md), [fix: include fp8_blockscale_gemm_90 in AOT jit-cache](../sources/prs/flashinfer/PR-2533.md), [fallback to fa2 (instead of fa3) for unsupported configuration (bf16 Q, Fp8 KV)](../sources/prs/flashinfer/PR-2536.md), [tests: bmm_fp8 for SM110](../sources/prs/flashinfer/PR-2538.md), [feat: cute dsl mmfp4 for blackwell](../sources/prs/flashinfer/PR-2540.md), [Add gen_gemm_sm100_module_cutlass_mxfp8 to jit-cache](../sources/prs/flashinfer/PR-2549.md), [fix: allow fmha_v2_prefill_deepseek on SM121 (DGX Spark)](../sources/prs/flashinfer/PR-2559.md), [fix: guard CUTLASS FMHA against SM12x and fix fmha_v2 SM121a check](../sources/prs/flashinfer/PR-2560.md), [feat: add is_sm12x_supported() helper for SM12x family detection](../sources/prs/flashinfer/PR-2574.md), [tests: add bias testing to nvfp4 moe](../sources/prs/flashinfer/PR-2585.md), [Perf: Optimize GDN decode pretranspose kernel for all batch sizes](../sources/prs/flashinfer/PR-2588.md), [support qk_nope_head_dim for 192 check for GLM-5](../sources/prs/flashinfer/PR-2607.md), [Ameyn/gdn bf16 tolerance parallel reduction](../sources/prs/flashinfer/PR-2610.md), [perf(gdn): optimize MTP kernel with ILP rows and SMEM v caching](../sources/prs/flashinfer/PR-2618.md), [feat: add pool+indices support to gated_delta_rule_decode_pretranspose (bf16 path) ](../sources/prs/flashinfer/PR-2619.md), [fix: trtllm_mxint4_block_scale_moe unit test to index output list](../sources/prs/flashinfer/PR-2627.md), [benchmark: Enable speculative decode microbenchmarking for paged decode](../sources/prs/flashinfer/PR-2628.md), [benchmark: Add MXFP4/MXFP8 quantization mode support to FP4 MoE benchmark](../sources/prs/flashinfer/PR-2635.md), [Enable sm120f compilation](../sources/prs/flashinfer/PR-2650.md), [fix: Add fused MOE and GEMM AOT modules for SM121](../sources/prs/flashinfer/PR-2654.md), [benchmarks: Add FP8 input / BF16 output in ragged prefill benchmark](../sources/prs/flashinfer/PR-2666.md), [feat(gdn): add BF16 state kernel with MTP support beyond T>4 with intermediate caching.](../sources/prs/flashinfer/PR-2679.md), [fix(jit): GEMM kernels produce NaN under concurrency — missing GDC flags cause PDL synchronization barriers to compile as no-ops](../sources/prs/flashinfer/PR-2716.md), [[gdn] support non-contiguous state for decoding](../sources/prs/flashinfer/PR-2727.md), [Support in-place update for `trtllm_fp8_block_scale_moe`](../sources/prs/flashinfer/PR-2739.md), [[Spark unit test debugging] Fix for tests/attention/test_trtllm_gen_mla.py](../sources/prs/flashinfer/PR-2750.md), [[Spark unit test debugging] Fix for tests/gemm/test_groupwise_scaled_gemm_fp8.py](../sources/prs/flashinfer/PR-2751.md), [perf: Performance tune cute dsl RMSNorm variants](../sources/prs/flashinfer/PR-2777.md), [fix(jit): enable GDC for CUTLASS GEMM PDL — SM100 flag only](../sources/prs/flashinfer/PR-2780.md), [tests: skip sliding window + fp8 to prevent hang in fmha_v2 unit tests](../sources/prs/flashinfer/PR-2781.md), [[fix] Bugfix 1367: fix VariableBlockSparseAttention buffer overflow by dynamically resizing kv_lens_buffer](../sources/prs/flashinfer/PR-2802.md), [feat(gdn): add padding index guard for bf16 decode kernel](../sources/prs/flashinfer/PR-2810.md), [[Spark unit test] Adjust tolerance for test_xqa, test_logits_processor](../sources/prs/flashinfer/PR-2828.md), [perf: Optimize GDN MTP decode kernel (v15) — eliminate ilp=1 fallback…](../sources/prs/flashinfer/PR-2842.md), [fix: fix cute dsl swap_ab tactic failure](../sources/prs/flashinfer/PR-2870.md), [fix: add cute dsl moe utils to AOT](../sources/prs/flashinfer/PR-2872.md), [[fix] bugfix 2856: Fix pre-allocated out shape check in trtllm_batch_decode_with_kv_cache_mla for q_len_per_req > 1](../sources/prs/flashinfer/PR-2876.md), [[NVIDIA] fix(jit): enable GDC for CUTLASS fused MoE PDL — prevent random crashes on SM12x](../sources/prs/flashinfer/PR-2913.md), [fix: Fix autotuner crash on meta-device tensor in trtllm_fp4_block_scale_routed_moe](../sources/prs/flashinfer/PR-2916.md), [feat: SM121 (GB10) tile filtering and autotuner robustness](../sources/prs/flashinfer/PR-2927.md), [CuTe DSL FP4 GEMM Heuristic](../sources/prs/flashinfer/PR-2940.md), [[Perf] Refactor MoE autotuning to set valid topk ids in routed MoE tuning](../sources/prs/flashinfer/PR-2942.md), [Only swizzle on v block scale; rename kv_block_scales to kv_cache_sf](../sources/prs/flashinfer/PR-2954.md), [Update NVSHMEM interface to use NVSHMEM4Py instead of custom bindings](../sources/prs/flashinfer/PR-2960.md), [test: skip unsupported mm_mxfp8 configurations on SM12x](../sources/prs/flashinfer/PR-2974.md), [feat(comm): add MOE Finalize/Reduction patterns to unified allreduce_fusion API](../sources/prs/flashinfer/PR-2982.md), [ Fix MXFP4/MXFP8 failures in SM120 FAST_BUILD and expand all_tiles[] ](../sources/prs/flashinfer/PR-2994.md), [[feat] Add blackwell GDN prefill kernel](../sources/prs/flashinfer/PR-3001.md), [fix: use sym_int64 for strides in rmsnorm CuTe DSL kernels to prevent int32 overflow](../sources/prs/flashinfer/PR-3007.md), [[chore] Install nvidia-cutlass-dsl[cu13] for cu130+](../sources/prs/flashinfer/PR-3017.md), [[feat] Add routing_replay_out support to MoE kernels and Python API](../sources/prs/flashinfer/PR-3024.md), [Fix/3170 dense blockscaled sm12x](../sources/prs/flashinfer/PR-3180.md), [test: enable bmm_mxfp8 cutlass backend coverage on SM12x](../sources/prs/flashinfer/PR-3183.md), [Include TinyGEMM into BF16 autotuner](../sources/prs/flashinfer/PR-3203.md), [feat(trace): embed runnable init() in every TraceTemplate](../sources/prs/flashinfer/PR-3221.md), [Ameyn/gdn bf16 dispatcher and 4d pool](../sources/prs/flashinfer/PR-3268.md), [fix(fmha_v2): fix FP8 V-scratch pipeline and varlen scheduler on SM90](../sources/prs/flashinfer/PR-3276.md), [Ep api design - Build Infra dependencies](../sources/prs/flashinfer/PR-3315.md), [feat: Separate QK/VO head dim dispatch for sm90 AOT](../sources/prs/flashinfer/PR-778.md), [bugfix: fix batch prefill attention kernel unittests](../sources/prs/flashinfer/PR-781.md), [bugfix: fix the behavior of mla plan function when provided with host tensors](../sources/prs/flashinfer/PR-816.md), [unittest: add MLA test cases where kv_len is evenly divided by page_size.](../sources/prs/flashinfer/PR-861.md), [perf: reduce torch.library dispatch overhead](../sources/prs/flashinfer/PR-968.md), [perf: Fix python API overhead when CUDAGraph is not enabled](../sources/prs/flashinfer/PR-969.md), [Update torch-xpu-ops commit pin](../sources/prs/pytorch/PR-144209.md), [[inductor][cpu] Fix bmm b_index for dynamic expressions in inductor autotuner](../sources/prs/pytorch/PR-144248.md), [Fix PythonMod printing](../sources/prs/pytorch/PR-144335.md), [Remove runtime dependency on packaging](../sources/prs/pytorch/PR-149125.md), [Add AOTI shim for _weight_int4pack_mm_cpu_tensor (#149031)](../sources/prs/pytorch/PR-149386.md), [op should NOT be static in aoti_torch_call_dispatcher](../sources/prs/pytorch/PR-149644.md), [Dont exclude constant_pad_nd in prologue fusion](../sources/prs/pytorch/PR-150145.md), [[inductor] Fix inductor windows linker error](../sources/prs/pytorch/PR-150447.md), [[Windows][inductor] fix blank space break windows file path](../sources/prs/pytorch/PR-150448.md), [[dynamo][super variable] Fix bug to use correct source](../sources/prs/pytorch/PR-152774.md), [[FlexAttention] Remove Old Constraint on lastdim strides](../sources/prs/pytorch/PR-153104.md), [Mark auto_functionalized HOPs as cacheable (#151194)](../sources/prs/pytorch/PR-153304.md), [[FlexAttention] explicilty create grad_q w/ strides](../sources/prs/pytorch/PR-153641.md), [[MPS] Switch Cholesky decomp to column wise](../sources/prs/pytorch/PR-158237.md), [Add warning about removed sm50 and sm60 arches](../sources/prs/pytorch/PR-158301.md), [[CD] CUDA 13 specific followup changes. Remove sm50-70 From CUDA 12.6 and CUDA 12.8 builds](../sources/prs/pytorch/PR-162455.md), [fix cpp extension distributed warning spew](../sources/prs/pytorch/PR-162764.md), [[Cherry Pick][Graph Partition] allow sharing default device context](../sources/prs/pytorch/PR-163097.md), [[Release 2.9] [cuDNN][SDPA][submodule] Roll-back cuDNN frontend upgrade, update Met…](../sources/prs/pytorch/PR-163265.md), [CUDA 13.0 Warning update for supported architectures](../sources/prs/pytorch/PR-163585.md), [fix pickling for BitwiseFn](../sources/prs/pytorch/PR-163861.md), [[SDPA] [MPS] Fixes regression in 2.8.0 for scaled_dot_product_attention using mps](../sources/prs/pytorch/PR-164364.md), [[Flex attention] Fix flex attention head broadcast](../sources/prs/pytorch/PR-164368.md), [[inductor] don't try to reorder loops for template](../sources/prs/pytorch/PR-166910.md), [[Dynamo] Don't guard data ptrs by default with mark_static_address](../sources/prs/pytorch/PR-166913.md), [[Inductor] No longer throw error in bmm out_dtype lowering due to tem…](../sources/prs/pytorch/PR-166922.md), [[GraphPartition] cache get_free_symbol_uses (#166338)](../sources/prs/pytorch/PR-166994.md), [[cuDNN][SDPA] Check-in test for #166211](../sources/prs/pytorch/PR-167121.md), [[Inductor] ExternKernelBenchmarkRequest best attempt](../sources/prs/pytorch/PR-170246.md), [[flex_attention] adds support for low precision K/V inputs in compiled mode with GPU](../sources/prs/pytorch/PR-170486.md), [[cherry-pick] Fix vllm issue for flex (#170499)](../sources/prs/pytorch/PR-170555.md), [Avoid closing random file handles in Inductor](../sources/prs/pytorch/PR-171150.md), [[xpu][fix][inductor] fallback bfloat16 atomics to eager](../sources/prs/pytorch/PR-171247.md), [[MPS] Fix 2-pass SDPA memory corruption by forcing float accumulators](../sources/prs/pytorch/PR-175580.md), [[CI] Update inductor CI jobs to CUDA 13.0](../sources/prs/pytorch/PR-175826.md), [[Inductor] Reject non-contiguous subnode fusion in mix-order reduction.](../sources/prs/pytorch/PR-176410.md), [[inductor] Fix Identity comparability and evalf recursion](../sources/prs/pytorch/PR-176783.md), [[Inductor] Don't unfuse addmm for bf16/fp16 to avoid precision loss](../sources/prs/pytorch/PR-177144.md), [[Inductor][MPS] Fix half-precision type mismatches in Metal shader codegen (#176436)](../sources/prs/pytorch/PR-177193.md), [[MPS] fix compiling of SDPA producing nan results](../sources/prs/pytorch/PR-178009.md), [feat: Add FP4 (E2M1) KV Cache Support with Quantization Utilities for MLA](../sources/prs/sglang/PR-10078.md), [[Feature] Add MLAProcess for DeepSeek MLA on NPU](../sources/prs/sglang/PR-10130.md), [Enable native ModelOpt quantization support (3/3)](../sources/prs/sglang/PR-10154.md), [Fix chunked prefix cache for nvfp4](../sources/prs/sglang/PR-10180.md), [Add support for bf16 x bf16 cutlass fused MoE](../sources/prs/sglang/PR-10275.md), [support qwen3_next blackwell](../sources/prs/sglang/PR-10403.md), [Fix cutlass moe accuracy drop caused by attention UB from DP padding mode](../sources/prs/sglang/PR-10414.md), [Support single batch overlap](../sources/prs/sglang/PR-10422.md), [Fix correction bias undefined behavior for nvfp4 models](../sources/prs/sglang/PR-10426.md), [feat: add dsv3 fp4 cutlass moe etp ut](../sources/prs/sglang/PR-10433.md), [Cache the result of `is_blackwell` platform check](../sources/prs/sglang/PR-10498.md), [Enable trtllm mla prefix extend](../sources/prs/sglang/PR-10526.md), [Fix bias handling in TritonMoeQuantInfo within quantization/mxfp4.py](../sources/prs/sglang/PR-10579.md), [support qwen3-next-fp8 deepep](../sources/prs/sglang/PR-10622.md), [[Auto Sync] Update modelopt_quant.py (20250920)](../sources/prs/sglang/PR-10688.md), [Unify SGL Kernel Releases](../sources/prs/sglang/PR-10701.md), [Fix MTP MoE weight loading with NVFP4 target model.](../sources/prs/sglang/PR-10758.md), [Fuse quantize and rope in trtllm_mla MTP](../sources/prs/sglang/PR-10779.md), [[2/2] Support MHA prefill with FlashAttention 4.](../sources/prs/sglang/PR-10937.md), [Quick Fix: fix Qwen3-VL launch failure caused by MRotaryEmbedding arg](../sources/prs/sglang/PR-10985.md), [chore: upgrade sgl-kernel 0.3.13](../sources/prs/sglang/PR-11056.md), [Fix DSR1 accuracy for flashinfer_trtllm MoE with FP8 quantization](../sources/prs/sglang/PR-11081.md), [[NVIDIA] Add new SMs support for Spark & Thor](../sources/prs/sglang/PR-11287.md), [[AMD] Clean up vllm dependencies in moe_runner/triton.py](../sources/prs/sglang/PR-11349.md), [[sgl-kernel][1/N]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-11432.md), [[NVIDIA] FA3/FA4 Fix ](../sources/prs/sglang/PR-11606.md), [Support shared experts overlap in cutlass moe](../sources/prs/sglang/PR-11611.md), [[DeepseekV32] Enable flashmla_prefill kernel with fp8 kvcache](../sources/prs/sglang/PR-11655.md), [Use trtllm_mla decode kernel for draft extend in speculative decoding](../sources/prs/sglang/PR-11664.md), [Support running FP4 Deepseek on SM120.](../sources/prs/sglang/PR-11708.md), [[sgl-kernel] support flashmla libtorch](../sources/prs/sglang/PR-11717.md), [Change bf16 to fp8 for some gemms in attention for DeepSeek ckpt v2](../sources/prs/sglang/PR-11805.md), [Use cutlass fp4 gemm by default](../sources/prs/sglang/PR-11813.md), [Support nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8/NVFP4](../sources/prs/sglang/PR-11866.md), [DeepSeek-V3.2: Add Adaptive MHA Attention Pathway for Short-Sequence Prefill](../sources/prs/sglang/PR-11892.md), [chore: upgrade flashinfer 0.4.1](../sources/prs/sglang/PR-11933.md), [Feature/nano v2 offline modelopt fp8 and nvfp4](../sources/prs/sglang/PR-12018.md), [(1/n)support context parallel with deepseekv3.2-DSA](../sources/prs/sglang/PR-12065.md), [[Ascend] qwen optimization](../sources/prs/sglang/PR-12078.md), [[Ascend][feature] support L1+ L2 radixcache on ascend](../sources/prs/sglang/PR-12214.md), [[DeepseekV32]: use `_concat_mla_absorb_q_general` to replace `torch.cat`](../sources/prs/sglang/PR-12215.md), [[hotfix] missing `w13_weight_fp8` and `w2_weight_fp8` in UE8M0 requantization](../sources/prs/sglang/PR-12259.md), [[Deepseek V3.2] Enable flashmla_auto with MTP](../sources/prs/sglang/PR-12294.md), [fix seqlen bug for trtllm_mla's draft_extend](../sources/prs/sglang/PR-12295.md), [fix: Llama 4 BF16 load on Blackwell](../sources/prs/sglang/PR-12308.md), [fix: llama 4 + trtllm gen + fp8 kv cache incompatibility](../sources/prs/sglang/PR-12347.md), [Replace [silu_and_mul_]scaled_fp4_group_quant by Flashinfer equivalent](../sources/prs/sglang/PR-12376.md), [perf: trtllm mla performance minor improvements](../sources/prs/sglang/PR-12435.md), [Use sgl fp4 quant kernel by default](../sources/prs/sglang/PR-12482.md), [[Ascend] Support enable-mixed-chunk in non-MLA scenarios](../sources/prs/sglang/PR-12491.md), [chore: upgrade flashinfer 0.5.0](../sources/prs/sglang/PR-12523.md), [Enable Flashinfer TRTLLM-GEN-MoE FP8 blockwise kernel for Qwen3-Next on Blackwell](../sources/prs/sglang/PR-12543.md), [[CPU] Fix MoE layer support for DeepSeek-OCR models](../sources/prs/sglang/PR-12555.md), [feat: Add FP4 (E2M1) KV Cache Support for MHA](../sources/prs/sglang/PR-12612.md), [[NVIDIA] Fix wrong symmetric sizes for fp4 cases](../sources/prs/sglang/PR-12640.md), [[sgl-kernel][5/N]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-12666.md), [[fix] Only enable flashinfer all reduce fusion by default for single-node servers](../sources/prs/sglang/PR-12724.md), [[Bugfix] Fix illegal memory access](../sources/prs/sglang/PR-12758.md), [[Ascend] support Kimi-K2-Thinking](../sources/prs/sglang/PR-12759.md), [Update dsv3 quantization auto setting for sm100](../sources/prs/sglang/PR-12778.md), [ignore the deepgemm check when the model weight with nvfp4 and moe ba…](../sources/prs/sglang/PR-12782.md), [[DeepSeek-V3.2][NSA] Enable MHA Pathway for Short Sequence Prefill on B200 (SM100)](../sources/prs/sglang/PR-12788.md), [[Deepseek V3.2] Only skip Indexer logits computation when is_extend_without_speculative](../sources/prs/sglang/PR-12816.md), [Apply moe_reduce_sum kernel for fused_marlin_moe](../sources/prs/sglang/PR-12888.md), [[Deepseek V3.2] Use torch.compile to speed up torch.cat in nsa](../sources/prs/sglang/PR-13022.md), [Support moe topk sigmoid kernel](../sources/prs/sglang/PR-13049.md), [[sgl-kernel] support custom fp8 flashmla kernel](../sources/prs/sglang/PR-13087.md), [support mtp with deepseek r1 nvfp4 model](../sources/prs/sglang/PR-13115.md), [Aiter fp8 kv cache](../sources/prs/sglang/PR-13147.md), [Support internvl on Blackwell (which doesn't support fa3): add `SingletonCache` support to Vision{Sdpa|Triton|Ascend}Attention](../sources/prs/sglang/PR-13151.md), [[NPU]Optimization of `forward_npu` for `UnquantizedFusedMoEMethod`](../sources/prs/sglang/PR-13158.md), [Fix nan in global scaling factor for large scale nvfp4 EP](../sources/prs/sglang/PR-13162.md), [diffusion: enable fa4 for blackwell](../sources/prs/sglang/PR-13263.md), [[NVIDIA] Fix broken fp8 MoE of deepseek v3](../sources/prs/sglang/PR-13264.md), [[NVIDIA] Fix use case of SGLANG_ENABLE_FLASHINFER_GEMM](../sources/prs/sglang/PR-13274.md), [Support weight update for blackwell DeepGEMM](../sources/prs/sglang/PR-13324.md), [Flashinfer TRTLLM-GEN-MoE + Qwen3](../sources/prs/sglang/PR-13489.md), [Fix target MLA with eagle3 support for PD disaggregation](../sources/prs/sglang/PR-13555.md), [[BugFix] fix prefixcache performance and accuracy on ascend](../sources/prs/sglang/PR-13573.md), [[ROCM] Optimized deepseek-r1 fp8 model with + triton_gemm_a8w8 + batch_gemm_a8w8 + fused set_mla_kv_buffer kernel](../sources/prs/sglang/PR-13617.md), [[DeepSeekV3.2] Enable pure TP & Partial DP Attention](../sources/prs/sglang/PR-13646.md), [Fix EPLB + FP4 Quantization Compatibility Issue](../sources/prs/sglang/PR-13715.md), [[bugfix] fix TBO crashes when attn_tp_size > 1](../sources/prs/sglang/PR-13730.md), [[sgl-kernel][Feat][B200][1/N]Support MXFP8 Grouped GEMM in Blackwell](../sources/prs/sglang/PR-13731.md), [fix trtllm mla spec](../sources/prs/sglang/PR-13738.md), [[AMD] Support --enable-aiter-allreduce-fusion on AMD GPUs](../sources/prs/sglang/PR-13747.md), [[chore]Upgrade flashinfer to 0.5.3](../sources/prs/sglang/PR-13751.md), [[Feat][NVFP4] Enable NVFP4 MoE for Qwen series models (eg. Qwen3-Next) #13761](../sources/prs/sglang/PR-13761.md), [Support fp4 fp8 non gated moe](../sources/prs/sglang/PR-13794.md), [[NVIDIA] Enable TRTLLM BF16 MoE on Blackwell GPUs](../sources/prs/sglang/PR-13798.md), [update flashinfer_cubin==0.5.3](../sources/prs/sglang/PR-13848.md), [[BugFix] fix outplace_fused_experts missing is_gated](../sources/prs/sglang/PR-13864.md), [Feat: GLM-4.6 supports shared experts fusion](../sources/prs/sglang/PR-13873.md), [Fix update weight error for blackwell DeepGEMM](../sources/prs/sglang/PR-13910.md), [[DeepSeek v3.2] opt Context Parallelism: support fused moe, multi batch and fp8 kvcache](../sources/prs/sglang/PR-13959.md), [Use trtllm mha decode kernel for target_verify in speculative decoding](../sources/prs/sglang/PR-13976.md), [Support KTransformers for Qwen3-VL moe](../sources/prs/sglang/PR-13983.md), [Fix flashinfer cutlass MoE output shape for non-FP4-packed inputs](../sources/prs/sglang/PR-14028.md), [[LoRA][III] Add LoRA support for MoE layers and enable TP](../sources/prs/sglang/PR-14105.md), [Add new moe wna16 marlin gemm](../sources/prs/sglang/PR-14122.md), [Apply new moe wna16 marlin gemm](../sources/prs/sglang/PR-14125.md), [Apply new moe align block size kernel](../sources/prs/sglang/PR-14134.md), [Support checking fp8 params in weight_checker](../sources/prs/sglang/PR-14147.md), [fix: Increase FlashInfer workspace size for Qwen3VL models](../sources/prs/sglang/PR-14173.md), [Add Mistral Large 3 support.](../sources/prs/sglang/PR-14213.md), [[bug fix] fix ima with get_mla_kv_buffer_kernel overflow](../sources/prs/sglang/PR-14224.md), [Tiny use trtllm_mha as default when possible](../sources/prs/sglang/PR-14291.md), [[Fix] add block size logic for sm120 smem size](../sources/prs/sglang/PR-14311.md), [[FIX] trtllm-moe-fp4-renorm for Qwen series models](../sources/prs/sglang/PR-14350.md), [Support FP8 MLA prefill and 128k context.](../sources/prs/sglang/PR-14395.md), [[NPU] perf update with kvcache nz & w4a8 quant](../sources/prs/sglang/PR-14423.md), [Add Mistral Large 3 Eagle Support](../sources/prs/sglang/PR-14466.md), [Mistral Large 3 NVFP4 support](../sources/prs/sglang/PR-14485.md), [[diffusion] kernel fusion: gated residual layernorm scale shift and layernorm scale shift kernel fusion for Qwen-Image, WAN and HunyuanVideo](../sources/prs/sglang/PR-14717.md), [[NPU][eagle3] support qwen eagle3 on NPU](../sources/prs/sglang/PR-14820.md), [Apply back moe_sum_reduce for fused_marlin_moe](../sources/prs/sglang/PR-14829.md), [fix: trtllm mha attention auto-selection on sm120](../sources/prs/sglang/PR-14842.md), [Fix dsv3 dp accuracy issue when using bf16-kv](../sources/prs/sglang/PR-14897.md), [Fix accuracy issue when using a16w16 mla_decode_fwd](../sources/prs/sglang/PR-14936.md), [[AMD] Support fused_rms_mxfp4_quant in the prefill stage for DeepSeek-R1-MXFP4](../sources/prs/sglang/PR-14975.md), [add transformers version validation for glm-4.6v moe models](../sources/prs/sglang/PR-14998.md), [Mistral Large 3 NVFP4 TRTLLM MoE support](../sources/prs/sglang/PR-15049.md), [fix(attention): Prevent trtllm_mha auto-selection with eagle3 speculative decoding](../sources/prs/sglang/PR-15127.md), [[sgl-kernel][1/2] Fused qk_norm_rope for GLM4.6](../sources/prs/sglang/PR-15141.md), [[NVIDIA] upstream FA4](../sources/prs/sglang/PR-15182.md), [[NVIDIA] Fixes for NVFP4 all-gather with spec decoding](../sources/prs/sglang/PR-15280.md), [[Fix] A followup fix for TRTLLM BF16 MoE](../sources/prs/sglang/PR-15303.md), [Fix the accuracy issue when running mxfp4 dsv3 model and enable ep](../sources/prs/sglang/PR-15304.md), [feat: support bitsandbytes quantization algorithm](../sources/prs/sglang/PR-15325.md), [[distributed] Clean up MoE groups in destroy_model_parallel](../sources/prs/sglang/PR-15345.md), [[Tiny]Add warning for deepgemm on Blackwell](../sources/prs/sglang/PR-15352.md), [[NPU]mindspore model support moe](../sources/prs/sglang/PR-15363.md), [[NPU]DeepSeek-V3.2 support npu mlaprolog](../sources/prs/sglang/PR-15381.md), [[diffusion] Add Sage Attention 3 Support for sm 120 (RTX5090)](../sources/prs/sglang/PR-15382.md), [Super tiny add moe_ep_rank to prometheus labels](../sources/prs/sglang/PR-15407.md), [Flashinfer MOE FP8 support for Mistral Large 3.](../sources/prs/sglang/PR-15422.md), [Optimize MiMo-V2-Flash by flashinfer fused allreduce](../sources/prs/sglang/PR-15464.md), [[Perf] Add Flashinfer DeepGEMM SM90 for SwapAB Optimization](../sources/prs/sglang/PR-15514.md), [Optimize FP8 MLA KV cache writes with Triton kernel](../sources/prs/sglang/PR-15522.md), [Optimize Bailing-MoE with FlashInfer Fused All-Reduce](../sources/prs/sglang/PR-15526.md), [MoE: Skip SiLU/GELU activation for masked experts](../sources/prs/sglang/PR-15539.md), [Update flashinfer to 0.6.1](../sources/prs/sglang/PR-15551.md), [[sgl-kernel] Streamline kernel size report (Top 20 only) and clean up](../sources/prs/sglang/PR-15552.md), [Fix BatchMLAPagedAttentionWrapper query/qo_inptr mismatch for EAGLE](../sources/prs/sglang/PR-15601.md), [[jit-kernel] Add CuTe DSL GDN Decode Kernel](../sources/prs/sglang/PR-15631.md), [Add SwapAB Optimization for triton fused_moe_kernel on SM90.](../sources/prs/sglang/PR-15712.md), [[Perf] Eliminate the slice op for Flashinfer `trtllm_fp4_block_scale_moe`](../sources/prs/sglang/PR-15731.md), [Fix GLM-4.7 MoE Detector complex JSON Schema type parsing](../sources/prs/sglang/PR-15753.md), [Fix: Handle empty func_name and None values in GLM MoE detectors](../sources/prs/sglang/PR-15754.md), [[Feature] JIT Fused QK norm + qk norm clean up](../sources/prs/sglang/PR-15835.md), [[JIT kernel] Apply jit per_tensor_quant_fp8 kernel](../sources/prs/sglang/PR-15836.md), [[diffusion] model: support TurboWan2.1-T2V-1.3B/14B SLA](../sources/prs/sglang/PR-15888.md), [[fix]deepgemm precompile when warmup](../sources/prs/sglang/PR-15891.md), [Bugfix for ds-vl2](../sources/prs/sglang/PR-15894.md), [[NPU] NZ for non-quantized MOE, Qwen3 MOE double memory consumption fix](../sources/prs/sglang/PR-15904.md), [ Add tuned triton==3.5.1 h200 tp2, tp4 for qwen 3 next](../sources/prs/sglang/PR-15948.md), [Tiny fix cannot launch nvfp4 checkpoint with bf16 kv cache](../sources/prs/sglang/PR-15986.md), [[Performance] Force split_k=1 for MXFP4 Triton kernels on Hopper](../sources/prs/sglang/PR-16014.md), [Support fa4 decoding](../sources/prs/sglang/PR-16034.md), [optimize get_topk_ragged by fusing get k and k_scale triton kernel](../sources/prs/sglang/PR-16043.md), [[Diffusion] Flux support flashinfer rope](../sources/prs/sglang/PR-16055.md), [enhance accuracy for model kimi-vl-instruct-a3b](../sources/prs/sglang/PR-16076.md), [fix layer intermediate size](../sources/prs/sglang/PR-16084.md), [[Diffusion] Zimage opt with qknorm and flashinfer rope](../sources/prs/sglang/PR-16161.md), [[Feature] add aligned_vector type for JIT kernel](../sources/prs/sglang/PR-16162.md), [[VLM] Adopt jit qk_norm kernel in VLM](../sources/prs/sglang/PR-16171.md), [[NemotronH] Add latent MoE support](../sources/prs/sglang/PR-16227.md), [[Fix] Only add SM90 and SM100 to check for auto-enabling TRT Allreduce Fusion](../sources/prs/sglang/PR-16283.md), [[diffusion] Fix RuntimeError in SageAttention3 on Nvidia Blackwell with Qwen-Image](../sources/prs/sglang/PR-16335.md), [[Fix]Fix FA3 Performance in Diffusion Model ](../sources/prs/sglang/PR-16382.md), [Fix FP8 MoE NaN with DeepGEMM on Blackwell](../sources/prs/sglang/PR-16622.md), [[Rework] Add SwapAB Optimization for triton fused_moe_kernel on SM90.](../sources/prs/sglang/PR-16723.md), [[AMD] Support redundant expert with a2a moe in gfx95x.](../sources/prs/sglang/PR-16791.md), [[Fix] `flashinfer_trtllm` `intermediate_size` assertion with Qwen3 + TP=8](../sources/prs/sglang/PR-16824.md), [Support mxint4 flashinfer_trtllm moe gemm](../sources/prs/sglang/PR-16892.md), [[DeepSeek v3.2] Opt MTP decode cuda batch sizes and nsa implementation](../sources/prs/sglang/PR-16961.md), [[NPU]bugfix: fix for dsv3.2 and dsvl2](../sources/prs/sglang/PR-17007.md), [[MUSA][2/N] sgl-kernel build](../sources/prs/sglang/PR-17053.md), [Optimize GDN decode for Qwen3 Next](../sources/prs/sglang/PR-17094.md), [[diffusion] fix: fix using upstream flash_attn on blackwell](../sources/prs/sglang/PR-17111.md), [Enable XQA for SM90 and SM120](../sources/prs/sglang/PR-17115.md), [Inclusion of nvfp4 blockscale in EPLB Rebalance](../sources/prs/sglang/PR-17158.md), [[Fix] GLM 4.7 + NVFP4 + MTP](../sources/prs/sglang/PR-17166.md), [[GLM 4.7] Add RTX 6000 Pro aka sm120](../sources/prs/sglang/PR-17235.md), [[New Model] GLM4.7-Flash](../sources/prs/sglang/PR-17247.md), [[FIX] Always support TP > 4 for FP4 Gemm](../sources/prs/sglang/PR-17300.md), [Disable mla persistent kernel when not using fp8 kv_cache](../sources/prs/sglang/PR-17327.md), [Move fa4 from sgl-kernel to jit kernel](../sources/prs/sglang/PR-17353.md), [Add mxfp8 support for online quantization, Triton dense linear, and CUTLASS MoE](../sources/prs/sglang/PR-17449.md), [[NPU] enhance accuracy for model kimi-vl-a3b-instruct](../sources/prs/sglang/PR-17480.md), [Kernel: optimize decoding metadata in NSA multi-spec backend with fused kernels](../sources/prs/sglang/PR-17554.md), [[hotfix] Reenable all reduce fusion on sm100](../sources/prs/sglang/PR-17591.md), [[feat] Support nvfp4 quantized model of Qwen3-Next](../sources/prs/sglang/PR-17627.md), [Upgrade transformers==5.3.0](../sources/prs/sglang/PR-17784.md), [fix(quantization): add sgl_kernel fallback for FP4 quantize on Blackwell GPUs](../sources/prs/sglang/PR-17816.md), [Feature/support longcat flash lite](../sources/prs/sglang/PR-17838.md), [[Move sgl-kernel Kernel to JIT] Add JIT concat MLA kernels](../sources/prs/sglang/PR-17889.md), [Skipped warning on sm100](../sources/prs/sglang/PR-18000.md), [[Bugfix] Fix Mistral Large 3 NVFP4 TRTLLM MoE](../sources/prs/sglang/PR-18065.md), [Feat/add fi selective state update kernel call](../sources/prs/sglang/PR-18070.md), [[Diffsuion & JIT_kernel] QKNorm cross heads kernel](../sources/prs/sglang/PR-18073.md), [Fix nvfp4 weight update](../sources/prs/sglang/PR-18085.md), [[Blackwell] Make mxint4 flashinfer_trtllm moe gemm set by default on blackwell](../sources/prs/sglang/PR-18136.md), [[ModelOpt] Fix broken Qwen3-235B-A22B-Instruct-2507-NVFP4 launch](../sources/prs/sglang/PR-18189.md), [[ModelOPT] Support Qwen 3 Next Coder NVFP4](../sources/prs/sglang/PR-18224.md), [Support Qwen3 MoE context parallel](../sources/prs/sglang/PR-18233.md), [[ROCm] Optimize Deepseek R1 on MI300X](../sources/prs/sglang/PR-18242.md), [[Hicache & JIT_kernel] Support page first layout & mla jit kernel](../sources/prs/sglang/PR-18311.md), [[AMD] Support Qwen3-Coder-Next on AMD platform](../sources/prs/sglang/PR-18355.md), [[MUSA][10/N] Add GGUF support](../sources/prs/sglang/PR-18357.md), [feat(gdn): add FlashInfer K-last SSM layout support for GDN prefill and decode for Hopper](../sources/prs/sglang/PR-18361.md), [[Kimi-K2.5] Fix NVFP4 Kimi-K2.5 weight mapping and exclude list](../sources/prs/sglang/PR-18370.md), [Nsa trtllm mla sparse fp8 support with Deepseek v3.2 NVFP4](../sources/prs/sglang/PR-18389.md), [[AMD] Update aiter to v0.1.10.post2](../sources/prs/sglang/PR-18423.md), [feat: add FA4 SM90 paged KV decode support & update attention docs](../sources/prs/sglang/PR-18442.md), [Tilelang sparse decode fwd for dsv32 mi355](../sources/prs/sglang/PR-18488.md), [[FIX] Correct JIT kernel compilation on newer GPUs with outdated driver metadata.](../sources/prs/sglang/PR-18496.md), [Fp8 prefill attn kernel integration](../sources/prs/sglang/PR-18528.md), [[AMD] Fix accuracy issue when running TP4 dsv3 model with mtp](../sources/prs/sglang/PR-18607.md), [[AMD] DSR1/V3 use fp8 bmm in MLA for MI300X](../sources/prs/sglang/PR-18624.md), [[sglang-miles] True on-policy training support for FSDP2](../sources/prs/sglang/PR-18639.md), [use flashinfer.sampling](../sources/prs/sglang/PR-18696.md), [[RL] Support per-layer mixed FP8/BF16 serving for FP8 checkpoints](../sources/prs/sglang/PR-18742.md), [fix: update Blackwell log/error messages to include SM12x](../sources/prs/sglang/PR-18751.md), [[diffusion] Diffusion norm fusion for z-image](../sources/prs/sglang/PR-18762.md), [fix: add SM110 (Jetson AGX Thor) to Blackwell capability check](../sources/prs/sglang/PR-18787.md), [Migrate renorm kernels from sgl-kernel to FlashInfer JIT](../sources/prs/sglang/PR-18854.md), [[Perf] ~9.5x faster Blackwell MXFP4 MoE weight loading](../sources/prs/sglang/PR-18858.md), [Migrate norm kernels to FlashInfer JIT implementation](../sources/prs/sglang/PR-18871.md), [[sgl-kernel] rebase FlashMLA 0217](../sources/prs/sglang/PR-18902.md), [Fix NSA FP8 KV cache path for both-trtllm MHA one-shot](../sources/prs/sglang/PR-18931.md), [[Qwen3.5] Enable nvfp4 checkpoint](../sources/prs/sglang/PR-18937.md), [[Sarvam] Add inference support for Sarvam MoE LLMs](../sources/prs/sglang/PR-18938.md), [[jit_kernel] Add fused_qknorm_rope JIT kernel](../sources/prs/sglang/PR-19059.md), [Support skip-softmax attention](../sources/prs/sglang/PR-19089.md), [feat: Support MXFP4 quantized dense models on AMD CDNA2/CDNA3 GPUs](../sources/prs/sglang/PR-19143.md), [[DeepSeek-V3.2][JIT-kernel] Support nsa fuse store indexer k cache](../sources/prs/sglang/PR-19148.md), [[NVIDIA] Integrate FlashInfer decode kernel (Blackwell) for Qwen3.5](../sources/prs/sglang/PR-19150.md), [Adjust padding size to improve triton_kernels moe performance](../sources/prs/sglang/PR-19174.md), [[AMD] Fix accuracy while using --enable-dp-attention](../sources/prs/sglang/PR-19247.md), [Fix nightly Mistral-Large-3 NVFP4 accuracy threshold](../sources/prs/sglang/PR-19402.md), [[AMD] Fix weight load shape mismatch for amd dsr1 0528 mxfp4](../sources/prs/sglang/PR-19425.md), [[Feature] add feature mla_ag_after_qlora for dsv3.2](../sources/prs/sglang/PR-19428.md), [Fix/nemotron mtp quantaized](../sources/prs/sglang/PR-19433.md), [[Kernel Slimming] Migrate NVFP4 kernels to JIT](../sources/prs/sglang/PR-19437.md), [[FlashInfer v0.6.4] [RL] Integrate FlashInfer mxfp8 gemm, MoE, and routed MoE](../sources/prs/sglang/PR-19537.md), [[NPU] bugs fix for Deepseek models](../sources/prs/sglang/PR-19544.md), [[diffusion][llm] macOS support](../sources/prs/sglang/PR-19549.md), [[miles] fix for glm5](../sources/prs/sglang/PR-19634.md), [[Feature] NVFP4 Marlin fallback for non-Blackwell GPUs (SM75+)](../sources/prs/sglang/PR-19652.md), [Support `triton_kernels` for GPT-OSS on SM120](../sources/prs/sglang/PR-19718.md), [Various SM120 improvements](../sources/prs/sglang/PR-19721.md), [[SGLang-Diffusion] Fix custom op fake impl missing eps default for torch.compile](../sources/prs/sglang/PR-19725.md), [Add compile-time 256-bit vector guard for pre-Blackwell](../sources/prs/sglang/PR-19794.md), [[JIT Kernel][Feature] Support JIT custom all reduce (rewrite as v2)](../sources/prs/sglang/PR-19880.md), [Use TRTLLM allreduce fusion for Qwen 3.5](../sources/prs/sglang/PR-19889.md), [Fix MLA decode path returning unwritten (padded) rows](../sources/prs/sglang/PR-19902.md), [[AMD] Fix Tensor Memory Aliasing ](../sources/prs/sglang/PR-19928.md), [[AMD] Fix FP8 assertion failure in aiter MLA decode by falling back to self.k_scale](../sources/prs/sglang/PR-19935.md), [[AMD] Tilelang sparse fwd for dsv32 mi355/mi300](../sources/prs/sglang/PR-19945.md), [[JIT Kernel] Reland NVFP4 kernels to JIT](../sources/prs/sglang/PR-20012.md), [[Bugfix] Work around FlashInfer unified transport issue on GB](../sources/prs/sglang/PR-20039.md), [Fix SM120 `triton_kernels` MXFP4 `block_k` for GPT-OSS](../sources/prs/sglang/PR-20040.md), [MiniMax-M2.5 - Support dp attention, dp reduce scatter, FP4 all gather, AR fusion in prepare_attn](../sources/prs/sglang/PR-20067.md), [Fix streaming session with paged KV cache (SWA/MLA)](../sources/prs/sglang/PR-20070.md), [Enable modelopt quantized FLUX deployment](../sources/prs/sglang/PR-20082.md), [[V32/GLM5] Change default setting of V32 nvfp4 on TP4](../sources/prs/sglang/PR-20086.md), [[diffusion] fix bug of copy_if](../sources/prs/sglang/PR-20094.md), [[diffusion] Support nvfp4 for Flux.2](../sources/prs/sglang/PR-20137.md), [[AMD] Fp8 prefill integration with radix cache path for dpsk models](../sources/prs/sglang/PR-20187.md), [[4/n jit_kernel restruct] speed up CI tests and add benchmark workflow](../sources/prs/sglang/PR-20268.md), [[AMD] Add 4-GPU test suite for MI325 runners](../sources/prs/sglang/PR-20294.md), [[Benchmark] use flashinfer bench_gpu_time instead of triton do_bench](../sources/prs/sglang/PR-20305.md), [[Fix] Add fallback for flashinfer allreduce fusion](../sources/prs/sglang/PR-20384.md), [[NVIDIA] Enable fp8 flashinfer_trtllm_routed MoE for MiniMax-M2.5](../sources/prs/sglang/PR-20394.md), [[AMD][Bug-fix] Fix gpu fault when run the test with dp-attention-enabled and max-concurrency is over 256](../sources/prs/sglang/PR-20399.md), [[Model] Support Nemotron 3 Super NVFP4](../sources/prs/sglang/PR-20407.md), [[AMD][AITER] Guard _use_mla_ps_kernel with self.use_mla in draft_extend_v2 paths](../sources/prs/sglang/PR-20409.md), [[GDN] Add benchmark for sglang gdn prefill](../sources/prs/sglang/PR-20428.md), [Support Triton MLA FP8 KV cache](../sources/prs/sglang/PR-20479.md), [[Kernel] Fuse temperature + softmax in sampling for decode speedup](../sources/prs/sglang/PR-20501.md), [[Diffusion] Clean upstream fa3 in hopper](../sources/prs/sglang/PR-20576.md), [Use Flashinfer for target_verify in GDN model for SM120](../sources/prs/sglang/PR-20604.md), [FIX: (NSA) Compute topk_indices_offset when NSA prefill flashmla_sparse is used with FP8 KV cache](../sources/prs/sglang/PR-20606.md), [[Diffusion] Add a benchmark for rmsnorm/fuse_add_rmsnorm](../sources/prs/sglang/PR-20632.md), [Fix(jit): support rmsnorm for hidden_size in {64, 128, 256}](../sources/prs/sglang/PR-20661.md), [[Feature][JIT Kernel] Fused TP QK norm For Minimax](../sources/prs/sglang/PR-20673.md), [[Diffusion] Fix compile graph broken by flashinfer rope](../sources/prs/sglang/PR-20699.md), [Add Mistral Small 4 (Pixtral) support](../sources/prs/sglang/PR-20708.md), [Use FlashInfer tinygemm for GPT-OSS MoE router on SM90+](../sources/prs/sglang/PR-20755.md), [fix: guard configure_deep_gemm_num_sms when JIT disabled](../sources/prs/sglang/PR-20868.md), [[JIT Kernel] Fix NVFP4 multi-arch compilation failure](../sources/prs/sglang/PR-20874.md), [Add SGLang CUDA crash API logging inspired by FlashInfer](../sources/prs/sglang/PR-20910.md), [[Qwen3.5] Fuse split/reshape/cat ops in GDN projection with Triton kernel](../sources/prs/sglang/PR-21019.md), [[Chore] Clean up JIT compilation flags](../sources/prs/sglang/PR-21022.md), [fix: wrap _import_static_state in inference_mode to fix resume on Blackwell](../sources/prs/sglang/PR-21035.md), [perf: precompute FA3 scheduler_metadata to eliminate per-layer prepare_varlen_num_blocks](../sources/prs/sglang/PR-21104.md), [ci: remove IS_BLACKWELL env var; auto-detect Blackwell](../sources/prs/sglang/PR-21118.md), [[Not-Merge][AMD] GLM-5 performance optimization](../sources/prs/sglang/PR-21166.md), [[Whisper] Enable CUDA graph support and timestamp for whisper model](../sources/prs/sglang/PR-21190.md), [[NPU] bugfix for import sgl-kernel error](../sources/prs/sglang/PR-21200.md), [[KDA] Support CuTeDSL KDA decode kernel](../sources/prs/sglang/PR-21203.md), [[AMD]: Support MLA with nhead<16 and FP8 KV cache for TP=8 (Kimi K2.5…](../sources/prs/sglang/PR-21213.md), [[refactor] Clean up duplicate flashinfer trtllm moe code](../sources/prs/sglang/PR-21233.md), [Refactor JIT kernel CI to use run_suite.py registration system](../sources/prs/sglang/PR-21239.md), [[NVIDIA] Enable FP4 flashinfer trtllm routed moe](../sources/prs/sglang/PR-21240.md), [P2P Weight Update features for miles ](../sources/prs/sglang/PR-21278.md), [[RL] Support mxfp8 DeepSeek V3](../sources/prs/sglang/PR-21280.md), [[MUSA] apply_vocab_mask support musa device](../sources/prs/sglang/PR-21296.md), [CUTLASS NVFP4 GEMM improvement of SM120](../sources/prs/sglang/PR-21314.md), [[Kernel] Support FlashInfer TRTLLM-Gen fused MoE for non-gated FP4 & FP8 (Nemotron)](../sources/prs/sglang/PR-21321.md), [[misc] clean up kernel API](../sources/prs/sglang/PR-21325.md), [Add dedicated FlashInferCuteDslMoE layer for standard-path FP4 MoE](../sources/prs/sglang/PR-21339.md), [[GDN] Fuse GDN kkt + solve_tril into one kernel](../sources/prs/sglang/PR-21411.md), [[Bugfix] Lazy-import CuteDSL KDA kernel to fix AMD/ROCm startup crash](../sources/prs/sglang/PR-21428.md), [fix nemotron capture for non attention layers](../sources/prs/sglang/PR-21436.md), [[Diffusion] Add qknorm rope fuse kernel](../sources/prs/sglang/PR-21440.md), [Add explicit disable flag for FlashInfer allreduce fusion](../sources/prs/sglang/PR-21446.md), [fix: piecewise_cuda_graph get correct qo_indptr](../sources/prs/sglang/PR-21452.md), [Migrate all callers from /get_server_info to /server_info](../sources/prs/sglang/PR-21463.md), [[AMD] Enable FP8 KV cache and FP8 attention kernel for NSA on MI300/MI355 with TileLang backend](../sources/prs/sglang/PR-21511.md), [test: point DSV3 int8 MLA CI models to lmsys Hugging Face org](../sources/prs/sglang/PR-21561.md), [[FlashInver v0.6.7] Integrate flashinfer_trtllm mxfp8 gemm](../sources/prs/sglang/PR-21576.md), [Change default mm-attention backend from triton_attn to fa4](../sources/prs/sglang/PR-21595.md), [fix: TRT-LLM MHA CUDA illegal address with EAGLE v2 + DP attention](../sources/prs/sglang/PR-21649.md), [[jit_kernel] Optimize fused_qknorm_rope: deduplicate sincosf for interleave RoPE ](../sources/prs/sglang/PR-21654.md), [[AMD] Use tgemm.mm for MoEGate router gemm in deepseek_v2.py](../sources/prs/sglang/PR-21657.md), [[XPU] Enable qwen3.5 on XPU](../sources/prs/sglang/PR-21668.md), [[AMD] Add GLM-5-FP8 nightly performance benchmarks for MI30x and MI35x](../sources/prs/sglang/PR-21710.md), [Harden FlashInfer FP4 imports in standard dispatcher](../sources/prs/sglang/PR-21776.md), [[Fix] Fall back to triton MOE for GPT-OSS on Blackwell with driver >= 595](../sources/prs/sglang/PR-21780.md), [[DSA] Support trtllm sparse mla kernel for prefill batches ](../sources/prs/sglang/PR-21783.md), [Remove redundant test_moe_eval_accuracy_large](../sources/prs/sglang/PR-21787.md), [[Feature] JIT rmsnorm update (with claude)](../sources/prs/sglang/PR-21834.md), [ [GDN] Remove FlashInfer GDN decode + no_buffer guard and default to FlashInfer on SM100+ ](../sources/prs/sglang/PR-21861.md), [[server] Add --quantization unquant to explicitly opt out of quantization](../sources/prs/sglang/PR-21863.md), [[Misc] [MXFP8] Drop sm100 mxfp8 warning](../sources/prs/sglang/PR-21881.md), [fix pcg torch dynamo recompile in mxfp8 Triton path](../sources/prs/sglang/PR-21888.md), [[Bugfix] Temporarily skip TRTLLM attention on (G)B300 (SM103) to avoid high-concurrency hang](../sources/prs/sglang/PR-21906.md), [[DSA] Set trtllm kernels as default for Blackwell](../sources/prs/sglang/PR-21914.md), [[Bugfix] Fix CUDA graph replay issues in trtllm_mla draft_extend](../sources/prs/sglang/PR-21987.md), [Tiny fix trtllm_fp8_per_tensor_scale_moe_wrapper router_logits dtype](../sources/prs/sglang/PR-22006.md), [[NPU] enable mla prepare fused kernel only when being mla attn](../sources/prs/sglang/PR-22024.md), [[MUSA][9/N] Add FA3 attention backend support through MATE (MUSA AI Tensor Engine)](../sources/prs/sglang/PR-22051.md), [[Diffusion] Fix weight scale swizzle and add large-M kernel config for FLUX.2-dev-NVFP4](../sources/prs/sglang/PR-22064.md), [[nvidia] Gemma4 nvfp4 fix](../sources/prs/sglang/PR-22079.md), [[diffusion] Default NVFP4 to CUTLASS and add all-model shape benchmarks](../sources/prs/sglang/PR-22091.md), [[Diffusion] Add diffusion NVFP4 scaled-mm correctness test](../sources/prs/sglang/PR-22127.md), [[Hotfix] Fix router gemm on sm103](../sources/prs/sglang/PR-22134.md), [[Disagg][NIXL] Fix heterogeneous TP KV transfer for non-MLA models (same logic with mooncake, Step 1/2 for Qwen3.5 support)](../sources/prs/sglang/PR-22145.md), [[hisparse]: Adding ci for hisparse kvcache-swap-in jit-kernel](../sources/prs/sglang/PR-22155.md), [[HiSparse]: Add benchmark for hisparse kernel](../sources/prs/sglang/PR-22187.md), [[RL] Refactor NVFP4 shuffling/swizzling to in-place replacement](../sources/prs/sglang/PR-22204.md), [Reduce unnecessary kernels and copies in the NSA indexer](../sources/prs/sglang/PR-22232.md), [[AMD][HIP] NSA: bf16 passthrough from RMSNorm to eliminate FP8 dequantization](../sources/prs/sglang/PR-22258.md), [Lazy import flash_attention_v4 to avoid loading flash_attn.cute at startup](../sources/prs/sglang/PR-22306.md), [[AMD] Fix GLM-5 fp8 KV quant path dispatch on MI300](../sources/prs/sglang/PR-22314.md), [[Reland] DeepSeek-R1-0528-w4a8: DeepEP Low Latency Dispatch Adopts FP8 Communication](../sources/prs/sglang/PR-22316.md), [[Lora] Lora quat info re-factor and support deepseekv3 mla lora](../sources/prs/sglang/PR-22323.md), [[AMD] Add GLM-5.1-FP8 nightly accuracy and performance benchmarks for MI30x and MI35x](../sources/prs/sglang/PR-22336.md), [:sparkles: [diffusion][npu][quant] Add MXFP4 quantization support for Wan2.2 Diffusion on Ascend NPU](../sources/prs/sglang/PR-22338.md), [[Diffusion] modelopt diffusion fp8 support for flux1/flux2 and wan2.2](../sources/prs/sglang/PR-22365.md), [[DSA] Hopper FP8 FlashMLA KV padding](../sources/prs/sglang/PR-22372.md), [[Lora] Lora kimi support](../sources/prs/sglang/PR-22381.md), [[AMD] Use aiter CK layernorm2d for LayerNorm to reduce NSA indexer kernel launches](../sources/prs/sglang/PR-22424.md), [[Fix] Fix several bugs on DSA models](../sources/prs/sglang/PR-22430.md), [Upgrade sglang-torch-profiler-analysis SKILLS](../sources/prs/sglang/PR-22440.md), [[RL] Fix weight update for mxfp8 flashinfer_cutlass gemm backend](../sources/prs/sglang/PR-22484.md), [GLM-5/5.1 MXFP4 Checkpoint Inference Compatibility Fix](../sources/prs/sglang/PR-22543.md), [[Diffusion] Add FLUX.1-dev ModelOpt NVFP4 support](../sources/prs/sglang/PR-22574.md), [diffusion: fix layerwise offload for ModelOpt quantized DiTs](../sources/prs/sglang/PR-22594.md), [feat: Support flashinfer_cutedsl MoE runner with flashinfer alltoall backend](../sources/prs/sglang/PR-22669.md), [reland [Diffusion] Add FLUX.1-dev ModelOpt NVFP4 support](../sources/prs/sglang/PR-22672.md), [[Diffusion] Add Wan2.2 ModelOpt NVFP4 support](../sources/prs/sglang/PR-22681.md), [[Step3p5] Optimize allreduce in MoE layers ](../sources/prs/sglang/PR-22773.md), [Dual MoE CUDA graph capture for lora/nolora batches](../sources/prs/sglang/PR-22809.md), [diffusion: add HunyuanVideo GroupNorm+SiLU fast path](../sources/prs/sglang/PR-22814.md), [[Refactor] Refactor DeepEP dispatcher](../sources/prs/sglang/PR-22822.md), [[FlashInfer v0.6.11] [RL] Support FlashInfer per-token NVFP4 MoE](../sources/prs/sglang/PR-22918.md), [[Fix/Kernel] Add JIT rmsnorm_hf kernel to fix transformers backend MMLU accuracy regression ](../sources/prs/sglang/PR-22931.md), [[codex] diffusion: enable group norm silu fuse by default](../sources/prs/sglang/PR-23148.md), [[BugFix] Resolve adaptive speculative decoding conflicts for Qwen3.5 (hybrid GDN)](../sources/prs/sglang/PR-23331.md), [[Diffusion][NPU]Add attention backends for diffusion models for Ascend NPU](../sources/prs/sglang/PR-23482.md), [Reland Cute-DSL FP4 dense GEMM](../sources/prs/sglang/PR-23590.md), [Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-23686.md), [Use Cute-DSL NVFP4 quantization kernels](../sources/prs/sglang/PR-23745.md), [feat: port SGLANG_JIT_DEEPGEMM_FAST_WARMUP to deepseek_v4 branch](../sources/prs/sglang/PR-23756.md), [[MoE] Unify DeepEPMoE+MoriEPMoE through AITER MoeRunner pre/post-permute](../sources/prs/sglang/PR-23760.md), [Optimize large GroupNorm SiLU apply](../sources/prs/sglang/PR-23938.md), [[feat] Init true on policy with qwen_dense](../sources/prs/sglang/PR-23961.md), [Enable PDL for various kernels in DSV32/GLM5](../sources/prs/sglang/PR-23965.md), [[VLM] Optimize Gemma4 VLM with PCG and fuse RMSNorm + residual add + scalar](../sources/prs/sglang/PR-24048.md), [Refactor device timer, clean up metrics collector, and add fwd occupancy metric](../sources/prs/sglang/PR-24197.md), [[KDA] Optimize prefill kernels with diagonal and recompute fuse](../sources/prs/sglang/PR-24271.md), [[diffusion] Fuse LTX2 split rotary embedding](../sources/prs/sglang/PR-24411.md), [Port MXFP4 Marlin MoE support to JIT kernel path](../sources/prs/sglang/PR-24490.md), [Fix performance regression on Deepseek V3 on `moe-runner-backend=triton` on SM90](../sources/prs/sglang/PR-24562.md), [[Codex] Opt Mistral Large performace ](../sources/prs/sglang/PR-24611.md), [Support spec v2 for FlashMLA speculative decoding](../sources/prs/sglang/PR-24640.md), [[Gemma4] Optimize Gemm4 with fused Q/K/V RMSNorm + per-expert FP8 ckpt loader](../sources/prs/sglang/PR-24696.md), [[codex] Optimize hidden-size 512 RMSNorm dispatch](../sources/prs/sglang/PR-24710.md), [Add FlashInfer SM90 cutlass MXFP4 MoE backend (W4A16) for GPT-OSS + DeepSeek-V4](../sources/prs/sglang/PR-24816.md), [[attn backend] Integrate tokenspeed_mla prefill/decode kernels (fp8 kv cache, blackwell)](../sources/prs/sglang/PR-24925.md), [Amd/deepseek v4 rebase main 0509](../sources/prs/sglang/PR-24933.md), [[rebase]Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-24986.md), [perf(nvfp4): free unused source scales after weight processing](../sources/prs/sglang/PR-25107.md), [Fix AMX GQA extend attention](../sources/prs/sglang/PR-25180.md), [[MUSA][Diffusion] Improve wan model inference speed using torch.compile](../sources/prs/sglang/PR-25256.md), [Support Gemma4 Pipeline Parallelism](../sources/prs/sglang/PR-25284.md), [Fix EPLB mapping for TopK paths](../sources/prs/sglang/PR-25285.md), [[Gemma4]: Fix FP8 Triton scale layout](../sources/prs/sglang/PR-25286.md), [[NSA] Avoid repeated NSA MQA logits memory queries](../sources/prs/sglang/PR-25299.md), [[Intel GPU] Enable DeepSeek V4 Inference on XPU](../sources/prs/sglang/PR-25336.md), [[AMD] test(sgl-kernel): seed RNG on ROCm in test_moe_topk_sigmoid to fix tie-break flake](../sources/prs/sglang/PR-25356.md), [[AMD] Enable shared-experts fusion with new KIMI-K2.5-MXFP4 model.](../sources/prs/sglang/PR-25390.md), [[codex] Update Wan2.2 ModelOpt CI checkpoints](../sources/prs/sglang/PR-25483.md), [Support draft extend cuda graph for tokenspeed_mla attention backend](../sources/prs/sglang/PR-25489.md), [Update logging for inplace setting in MoE layer](../sources/prs/sglang/PR-25499.md), [[misc] Throw error when single batch overlap is enabled on Hopper ](../sources/prs/sglang/PR-25509.md), [Fix logging for inplace setting in the flashInfer-trtllm backend](../sources/prs/sglang/PR-25522.md), [[Bug Fix] Align glm4_moe_nextn NPU MTP loading with qwen3 MTP](../sources/prs/sglang/PR-25524.md), [[MoE Refactor] Migrate flashinfer_cutedsl + DeepEP to MoeRunner](../sources/prs/sglang/PR-25525.md), [Use DeepGEMM BF16 for unquantized DeepEP LL MoE](../sources/prs/sglang/PR-25540.md), [amd/deepseek_v4 27/N [fix] Reduce Triton autotune configs for faster first-time server launch](../sources/prs/sglang/PR-25554.md), [Use triton_attn as default vision attention on B300 (SM103)](../sources/prs/sglang/PR-25570.md), [[Benchmark] Add SGLANG_SIMULATE_UNIFORM_EXPERTS for balanced expert routing with dummy weights](../sources/prs/sglang/PR-25571.md), [Introduce SchedulerDPAttnAdapter to own DP-attention state](../sources/prs/sglang/PR-25611.md), [Move DP-attention adapter methods to SchedulerDPAttnAdapter](../sources/prs/sglang/PR-25612.md), [[SP] Fix runtime_max_tokens_per_rank for sequence parallelism](../sources/prs/sglang/PR-25685.md), [Add no_combine support to cutlass_moe_fp4](../sources/prs/sglang/PR-25688.md), [fix (jit kernel): elementwise activation C++ error](../sources/prs/sglang/PR-25695.md), [[diffusion] Fix GLM-Image /v1/images/edits support](../sources/prs/sglang/PR-25697.md), [[Codex] Remove stale DeepSeek V4 JIT kernels](../sources/prs/sglang/PR-25764.md), [[Refactor] Rename NSA → DSA: user-facing aliases, file/class/import rename](../sources/prs/sglang/PR-25821.md), [[Refactor] Pass PP start_layer via model constructor instead of forward_batch.token_to_kv_pool](../sources/prs/sglang/PR-25825.md), [[Test] Stage-a sanity kits; consolidate core/ + models_e2e/ tests](../sources/prs/sglang/PR-25831.md), [Allow local cutlass directory to be used in sgl-kernel build](../sources/prs/sglang/PR-3037.md), [sync the upstream updates of flashinfer](../sources/prs/sglang/PR-3051.md), [feat: integrate gemm_fp8 kernel into gemm](../sources/prs/sglang/PR-3056.md), [Apply sgl w8a8 fp8 kernel](../sources/prs/sglang/PR-3148.md), [integrate blockwise fp8 kernel](../sources/prs/sglang/PR-3529.md), [feat: support flashinfer mla attention for deepseek v3](../sources/prs/sglang/PR-3550.md), [update flashinfer-python](../sources/prs/sglang/PR-3557.md), [feat: support flashinfer mla with prefix cache](../sources/prs/sglang/PR-3643.md), [add control for cutlass fp8 blockwise gemm](../sources/prs/sglang/PR-3727.md), [Feature DeepSeek V3/R1 INT8 Quantization (block-wise)](../sources/prs/sglang/PR-3730.md), [[Feature] DeepSeek V3/R1 INT8 Quantization (channel-wise) ](../sources/prs/sglang/PR-3888.md), [Support FP4 gemm (1/2)](../sources/prs/sglang/PR-3899.md), [upgrade flashinfer v0.2.2.post1](../sources/prs/sglang/PR-3934.md), [[tools] add fp8 max/min constant in utils](../sources/prs/sglang/PR-3959.md), [Add fast decode plan for flashinfer mla](../sources/prs/sglang/PR-3987.md), [Hierarchical Caching supports MLA](../sources/prs/sglang/PR-4009.md), [[Revision] Add fast decode plan for flashinfer mla ](../sources/prs/sglang/PR-4012.md), [Support overlapping two batches](../sources/prs/sglang/PR-4068.md), [DeepGemm integrate to gemm](../sources/prs/sglang/PR-4165.md), [linear support deepgemm](../sources/prs/sglang/PR-4199.md), [Accelerate FP8 CUDA Kernel by 20-28%](../sources/prs/sglang/PR-4215.md), [Clean up fp8 support](../sources/prs/sglang/PR-4230.md), [[Feature] Integrate DeepEP into SGLang](../sources/prs/sglang/PR-4232.md), [upgrade flashinfer 0.2.3](../sources/prs/sglang/PR-4317.md), [[FIX] fix incorrect output when enable both deepgemm and torch compile](../sources/prs/sglang/PR-4359.md), [[ROCm] fix dtype](../sources/prs/sglang/PR-4510.md), [Create col-major and tma-aligned x_scale for deep_gemm.gemm_fp8_fp8_bf16_nt](../sources/prs/sglang/PR-4515.md), [Add deepseek style fused moe group gate selection kernel](../sources/prs/sglang/PR-4530.md), [[Fix] Fix raw_bs bug when using flashinfer mla and eagle](../sources/prs/sglang/PR-4557.md), [avoid cudaStreamSynchronize in DeepSeekV2AttentionMLA](../sources/prs/sglang/PR-4577.md), [[quantization] fix channelwise conversion with scalar weight scale](../sources/prs/sglang/PR-4596.md), [Set deepgemm to the default value in the hopper architecture.](../sources/prs/sglang/PR-4613.md), [Optimize Permute Kernel in DeepEP](../sources/prs/sglang/PR-4643.md), [Fix loading KV quantization scale; Enable modelopt kv cache](../sources/prs/sglang/PR-4686.md), [[Model] Adding Qwen3 and Qwen3MoE](../sources/prs/sglang/PR-4693.md), [support cmake for sgl-kernel](../sources/prs/sglang/PR-4706.md), [[Feature] Support DeepEP Low Latency](../sources/prs/sglang/PR-4767.md), [Support (1 <= dp < tp) in the dp attention in DeepEP](../sources/prs/sglang/PR-4770.md), [Introduce moe_dense_tp_size to fix dense layer errors in DeepSeek V3 + 4x8xH100](../sources/prs/sglang/PR-4836.md), [Feat/support encoder model (like bert)](../sources/prs/sglang/PR-4887.md), [Add DeepSeek V3/R1 shared experts fusion](../sources/prs/sglang/PR-4918.md), [[Fix] DeepEP Compatibility with Low Latency](../sources/prs/sglang/PR-5068.md), [reduce moe_align_block_size_kernel small batch mode overhead](../sources/prs/sglang/PR-5086.md), [Support MHA with chunked prefix cache for DeepSeek chunked prefill](../sources/prs/sglang/PR-5113.md), [Blackwell Cutlass MLA kernel](../sources/prs/sglang/PR-5142.md), [Add optimized native kernels in sgl-kernel](../sources/prs/sglang/PR-5150.md), [feat: add DeepGEMM build warning](../sources/prs/sglang/PR-5176.md), [feat: use fa3 mla by default on hopper](../sources/prs/sglang/PR-5210.md), [[Fix] Turn off DeepGEMM by default](../sources/prs/sglang/PR-5263.md), [[1/2] Add FP8 Blockscale MoE CUTLASS kernel for Blackwell](../sources/prs/sglang/PR-5281.md), [fix: use deepgemm only on hopper](../sources/prs/sglang/PR-5310.md), [Add Speculative Decoding Eagle3 topk > 1](../sources/prs/sglang/PR-5318.md), [fix: determine if flashinfer is installed](../sources/prs/sglang/PR-5336.md), [Fix DeepGEMM masked cannot be run on groups not being multiple or 4](../sources/prs/sglang/PR-5340.md), [[perf] experimental enhance fp8 per-tensor quant](../sources/prs/sglang/PR-5370.md), [apply fused moe gate in ds v3/r1](../sources/prs/sglang/PR-5371.md), [[PD Bug] fix MLA get_contiguous_buf_infos error](../sources/prs/sglang/PR-5384.md), [Add Cutlass MLA attention backend](../sources/prs/sglang/PR-5390.md), [[PD] Fix dynamic port support and MLA buffer for Mooncake](../sources/prs/sglang/PR-5415.md), [[Feat] upgrade pytorch2.6](../sources/prs/sglang/PR-5417.md), [BLackwell cutlass mla: Add check for bad page size/block num combinations](../sources/prs/sglang/PR-5431.md), [[perf] introduce deep gemm group_gemm_masked as gemm](../sources/prs/sglang/PR-5432.md), [Avoid computing lse in Ragged Prefill when there's no prefix.](../sources/prs/sglang/PR-5476.md), [Fix sampler nan check when calling top_k_top_p_sampling_from_probs](../sources/prs/sglang/PR-5546.md), [[feature] enable pre compile jit deep_gemm](../sources/prs/sglang/PR-5580.md), [[fix] force use deepgemm in compile_deep_gemm](../sources/prs/sglang/PR-5618.md), [ DeepEP normal support deepgemm-contiguous](../sources/prs/sglang/PR-5626.md), [Turn on DeepGemm By Default and Update Doc](../sources/prs/sglang/PR-5628.md), [[perf] dsv3 bmm fallback to bf16](../sources/prs/sglang/PR-5662.md), [[2/2] Add python wrapper for CUTLASS FP8 Blockscale MoE Kernel. ](../sources/prs/sglang/PR-5694.md), [[PP] Add pipeline parallelism](../sources/prs/sglang/PR-5724.md), [Fuse MLA set kv cache kernel](../sources/prs/sglang/PR-5748.md), [opt flashinfer mla cat](../sources/prs/sglang/PR-5822.md), [Cutlass MLA decode - fix dtype error](../sources/prs/sglang/PR-5868.md), [[Fix] Fix a bug for flashmla to run R1 model](../sources/prs/sglang/PR-5875.md), [Improve dp attention port assignment scheme](../sources/prs/sglang/PR-5889.md), [[qwen3] support qwen3 ep moe](../sources/prs/sglang/PR-5917.md), [[Feat] Enable PDL automatically on Hopper architecture](../sources/prs/sglang/PR-5981.md), [KV‑Cache (MHA, MLA): add missing start_layer / end_layer fields to MHATokenToKVPoolHost and MLATokenToKVPoolHost](../sources/prs/sglang/PR-6016.md), [feat: mtp support dp-attention](../sources/prs/sglang/PR-6081.md), [[1/2] Add Kernel support for Cutlass based Fused FP4 MoE](../sources/prs/sglang/PR-6093.md), [feat: add dp attention support for Qwen 2/3 MoE models, fixes #6088](../sources/prs/sglang/PR-6121.md), [Reduce MoE memory usage](../sources/prs/sglang/PR-6147.md), [enable auto-round quantization model](../sources/prs/sglang/PR-6226.md), [Enable FlashInfer support encoder models and add head_dim padding workaround](../sources/prs/sglang/PR-6230.md), [fix: fix MLA for ShardedModelLoader/RemoteModelLoader](../sources/prs/sglang/PR-6287.md), [[Fix] Improve dependencies for Blackwell image](../sources/prs/sglang/PR-6334.md), [reduce torch.zeros overhead in moe align block size kernel](../sources/prs/sglang/PR-6369.md), [[Feature] Comprehensive Hybrid Parallelism Support](../sources/prs/sglang/PR-6389.md), [Add fp8 fused_experts kernel for CPU in sgl-kernel and add UT](../sources/prs/sglang/PR-6404.md), [Fix bug of deepseek-v3 under DP+EP mode with large batchsize/seqlen](../sources/prs/sglang/PR-6449.md), [Fix topk inference performance reduce](../sources/prs/sglang/PR-6474.md), [[Feature] Support Flashinfer fp8 blockwise GEMM kernel on Blackwell](../sources/prs/sglang/PR-6479.md), [qwen3moe support two batch overlap](../sources/prs/sglang/PR-6598.md), [Refine pre_reorder_triton_kernel slightly to improve performance](../sources/prs/sglang/PR-6627.md), [[CPU] [BF16] Call fused_experts_cpu, weight_packed_linear and bmm_cpu kernel in DeepSeek model](../sources/prs/sglang/PR-6641.md), [Fix DeepEP error in Qwen 3 MoE models](../sources/prs/sglang/PR-6673.md), [[EP] Add cuda kernel for moe_ep_pre_reorder](../sources/prs/sglang/PR-6699.md), [Fix PP for Qwen3 MoE](../sources/prs/sglang/PR-6709.md), [Set `num_fused_shared_experts` as `num_shared_experts` when shared_experts fusion is not disabled](../sources/prs/sglang/PR-6736.md), [[CPU] add optimizations for INT8 and FP8 DeepSeek](../sources/prs/sglang/PR-6769.md), [[CPU] support the case where num_attention_heads or intermediate_size is not divisible by the TP size](../sources/prs/sglang/PR-6771.md), [Support token-level quantization for EP MoE](../sources/prs/sglang/PR-6782.md), [[PD] Add different TP sizes support for no-MLA models](../sources/prs/sglang/PR-6793.md), [Correctly abort the failed grammar requests & Improve the handling of abort](../sources/prs/sglang/PR-6803.md), [feat: integrate deepgemm into EPMoE](../sources/prs/sglang/PR-6821.md), [CPU: map changes from developing branch in sgl-kernel](../sources/prs/sglang/PR-6833.md), [[EP] Add cuda kernel for moe_ep_post_reorder](../sources/prs/sglang/PR-6837.md), [Fix AWQ Dequant and Weight Loading of deepseek v2](../sources/prs/sglang/PR-6842.md), [[DeepseekR1-FP4] Add Support for nvidia/DeepSeekR1-FP4 model](../sources/prs/sglang/PR-6853.md), [Use deepgemm instead of triton for fused_qkv_a_proj_with_mqa](../sources/prs/sglang/PR-6890.md), [Add a CUDA kernel for fusing mapping and weighted sum for MoE.](../sources/prs/sglang/PR-6916.md), [[sgl-kernel] Add cuda kernel for moe_ep_silu_and_mul](../sources/prs/sglang/PR-6919.md), [[perf][sgl-kernel] extend cutlass_mla_decode to support num_head < 128](../sources/prs/sglang/PR-6929.md), [[Feature] Support Flashinfer fmha on Blackwell](../sources/prs/sglang/PR-6930.md), [[sgl-kernel] update deepgemm](../sources/prs/sglang/PR-6942.md), [chore: upgrade flashinfer v0.2.6.post1 jit](../sources/prs/sglang/PR-6958.md), [Fuse routed scaling factor in deepseek](../sources/prs/sglang/PR-6970.md), [Fix cutlass MLA gets almost zero accuracy](../sources/prs/sglang/PR-6998.md), [Fix torchvision version for Blackwell](../sources/prs/sglang/PR-7015.md), [Update default settings for blackwell](../sources/prs/sglang/PR-7023.md), [Clean up server_args.py](../sources/prs/sglang/PR-7037.md), [Fix positional argument](../sources/prs/sglang/PR-7093.md), [feat: update blackwell setup](../sources/prs/sglang/PR-7119.md), [fix amd EP MoE FP8 issue](../sources/prs/sglang/PR-7125.md), [Enable ModelOpt Llama4 fp8 checkpoint deployment in SGLang](../sources/prs/sglang/PR-7129.md), [Enable native ModelOpt quantization support (1/3) ](../sources/prs/sglang/PR-7149.md), [[amd] Opt dsv3 moe](../sources/prs/sglang/PR-7160.md), [Fix Deepseek R1 0528 FP4 tensor name mismatch issue during weights loading.](../sources/prs/sglang/PR-7164.md), [Support new DeepGEMM](../sources/prs/sglang/PR-7172.md), [Tiny let DeepGEMM scale checks cover more cases](../sources/prs/sglang/PR-7182.md), [chore: upgrade sgl-kernel v0.1.8.post2](../sources/prs/sglang/PR-7186.md), [[AMD] Fail gracefully when AITER is unavailable gfx90a GPUs](../sources/prs/sglang/PR-7187.md), [Fix a minor bug related to DeepGEMM upgrade](../sources/prs/sglang/PR-7191.md), [Fix error when disabling new DeepGEMM](../sources/prs/sglang/PR-7198.md), [Fix grammar abort & Minor style fixes](../sources/prs/sglang/PR-7204.md), [feat: support compatibility between MTP and two-batch-overlap](../sources/prs/sglang/PR-7225.md), [Minor style and doc fix](../sources/prs/sglang/PR-7228.md), [[fix] fix DeepGEMM blackwell input quant & ut & fix style and log](../sources/prs/sglang/PR-7247.md), [[AMD] add aiter fused moe in DeepEP path](../sources/prs/sglang/PR-7268.md), [Support NVFP4 quantized dense models on AMD CDNA2/CDNA3 GPUs](../sources/prs/sglang/PR-7302.md), [Let EP prefill support new DeepGEMM](../sources/prs/sglang/PR-7310.md), [Kernels for efficient KV cache IO](../sources/prs/sglang/PR-7313.md), [FlashInfer NVFP4 MoE with EP & 2-stream shared expert](../sources/prs/sglang/PR-7327.md), [Fix MTP with Deepseek R1 Fp4](../sources/prs/sglang/PR-7376.md), [Quick fix for DeepGemm requant to also cover MTP.](../sources/prs/sglang/PR-7378.md), [Fix torch compile run](../sources/prs/sglang/PR-7391.md), [[AMD][Quantization] Add `int4fp8_moe` online quantization on ROCm](../sources/prs/sglang/PR-7392.md), [Fix CPU offloading for MLA memory pool](../sources/prs/sglang/PR-7409.md), [Fuse sorted_token_ids padding to moe_align_block_size kernel](../sources/prs/sglang/PR-7437.md), [Support non-contiguous query input for extend/decode attention](../sources/prs/sglang/PR-7462.md), [Add Tencent HunYuanMoEV1 model support](../sources/prs/sglang/PR-7549.md), [[b200] support trt-llm allreduce fuse rms_norm_add kernel](../sources/prs/sglang/PR-7621.md), [Add dsv3 router gemm kernel](../sources/prs/sglang/PR-7627.md), [Add dsv3 fused a gemm to sgl-kernel](../sources/prs/sglang/PR-7630.md), [[Feature] Layer-wise Prefill](../sources/prs/sglang/PR-7634.md), [Apply dsv3_fused_a_gemm kernel](../sources/prs/sglang/PR-7635.md), [[Feature] CUDA Green Context Support](../sources/prs/sglang/PR-7649.md), [chore: upgrade flashinfer v0.2.7 jit](../sources/prs/sglang/PR-7663.md), [Add fp4 quantize before all-gather for Flashinfer cutlass MoE DP (max throughput)](../sources/prs/sglang/PR-7667.md), [Integrate triton moe kernel](../sources/prs/sglang/PR-7689.md), [feat: support DeepSeek-R1-W4AFP8 model with ep-moe mode](../sources/prs/sglang/PR-7762.md), [[1/n]: add cutlass W4A8 moe kernel for hopper architecture](../sources/prs/sglang/PR-7772.md), [Qwen FP8/NVFP4 ModelOPT Quantization support](../sources/prs/sglang/PR-7912.md), [[feat] Support tp mode for DeepSeek-R1-W4AFP8](../sources/prs/sglang/PR-8118.md), [[fix] fix modelopt fp4 on b200](../sources/prs/sglang/PR-8195.md), [[1/N]Support DeepSeek-R1 w4a8 normal deepep](../sources/prs/sglang/PR-8247.md), [Support triton kernels v3.4.0 for fused_moe](../sources/prs/sglang/PR-8258.md), [[2/N]Support DeepSeek-R1 w4a8 low latency deepep](../sources/prs/sglang/PR-8464.md), [Update cutlass_moe.py](../sources/prs/sglang/PR-8535.md), [Update cutlass_moe.py](../sources/prs/sglang/PR-8545.md), [[NVIDIA] Add Low Latency NVFP4 decode kernels from Flashinfer](../sources/prs/sglang/PR-8552.md), [TRTLLM-MLA FP8 path](../sources/prs/sglang/PR-8638.md), [feat: support cutlass_moe_fp8 kernel for fusedmoe in sm90](../sources/prs/sglang/PR-8678.md), [fuse allreduce and residual_rmsnorm](../sources/prs/sglang/PR-8731.md), [Fix mismatch between padded_scales shape and reshape dimensions in modelopt quantization](../sources/prs/sglang/PR-8766.md), [feat: add trtllm-gen mha from direct call](../sources/prs/sglang/PR-8782.md), [[Perf] Auto enable best flashinfer mxfp4 kernel in b200](../sources/prs/sglang/PR-8898.md), [Fix hopper launch gpt-oss model illegal memory](../sources/prs/sglang/PR-8908.md), [[NVIDIA] Fix missing `get_col_major_tma_aligned_tensor` for Blackwell deepgemm in EpMoE](../sources/prs/sglang/PR-8955.md), [optimize: reduce shulffle and quantization overhead in cutlass_moe sm90](../sources/prs/sglang/PR-8962.md), [[fix] fix enable_pdl for blackwell](../sources/prs/sglang/PR-9011.md), [[sgl-kernel] Support FlashInfer top_k_top_p_sampling_from_logits](../sources/prs/sglang/PR-9060.md), [Faster weight processing (trtllm-gen moe nvfp4)](../sources/prs/sglang/PR-9162.md), [[NVIDIA] [3/N] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked ](../sources/prs/sglang/PR-9199.md), [[NVIDA] [1/N] Nvfp4 Masked Gemm: Add quant op for the flashinfer grouped gemm](../sources/prs/sglang/PR-9200.md), [[fix]: fix cutlass moe ut and and Opt H20 cutlass groupGemm performance](../sources/prs/sglang/PR-9272.md), [Support trtllm_allreduce_fusion in flashinfer for cuda<12.8](../sources/prs/sglang/PR-9339.md), [Fix FP4 inference corruption issue in glm4.5-air model](../sources/prs/sglang/PR-9346.md), [Support DP attention with GPT-OSS](../sources/prs/sglang/PR-9359.md), [[fix] Fix mxfp4 triton MoE tp bug](../sources/prs/sglang/PR-9473.md), [Optimize moe_sum_reduce_kernel](../sources/prs/sglang/PR-9477.md), [fix: blackwell dsv3 fp8 issue temporary solution](../sources/prs/sglang/PR-9530.md), [[NVIDIA] [2/N] Optimize `silu_and_mul_scaled_fp4_grouped_quant` perf](../sources/prs/sglang/PR-9556.md), [Update CUTLASS 4.2 & Enable K-Major Scale Factor for SM90 FP8 Blockwise Group GEMM](../sources/prs/sglang/PR-9559.md), [Tiny fix wrong comments](../sources/prs/sglang/PR-9589.md), [Single Batch Overlap for MoE Models](../sources/prs/sglang/PR-9660.md), [fix mooncake store mla zero copy meta](../sources/prs/sglang/PR-9678.md), [move is_sm90_supported/is_sm100_supported to python/sglang/srt/utils.py](../sources/prs/sglang/PR-9679.md), [[ModelOpt] Fix Weight Loading for DSR1-FP4 Quantization](../sources/prs/sglang/PR-9712.md), [[CPU] Add FP8 Bmm support](../sources/prs/sglang/PR-9744.md), [[Model] Support Meituan LongCat-Flash && LongCat-Flash-MTP](../sources/prs/sglang/PR-9824.md), [perf: Avoid unnecessary data type conversions for DeepSeek-V3 on Blackwell](../sources/prs/sglang/PR-9834.md), [support using fa4 on deepseek on blackwell](../sources/prs/sglang/PR-9928.md), [[Fix] DeepSeek EP accuracy issue on B200 GPUs](../sources/prs/sglang/PR-9946.md), [Enable native ModelOpt quantization support (2/3)](../sources/prs/sglang/PR-9991.md), [[Feature] Support `cp.reduce.async.bulk.tensor`](../sources/prs/tilelang/PR-1667.md), [[Feature] Support tcgen5mma lowering for `.kind::i8`](../sources/prs/tilelang/PR-1764.md), [[BugFix] Fix Hopper TMA lowering without warp specialization](../sources/prs/tilelang/PR-1840.md), [[CUDA] Support tcgen5mma gemm ts](../sources/prs/tilelang/PR-1866.md), [[Feature] Support cluster launch, query, synchronization and barrier operations](../sources/prs/tilelang/PR-1874.md), [[Feature] Add T.copy_cluster to support TMA multicast and SM-to-SM cluster copy](../sources/prs/tilelang/PR-1908.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [[Bugfix] Fix CuTeDSL autotune cache invalid ELF header (#1967)](../sources/prs/tilelang/PR-1972.md), [[Feature] Support TMA store in T.tma_copy()](../sources/prs/tilelang/PR-1981.md), [[Transform] Add InjectTcgen05Fence pass](../sources/prs/tilelang/PR-2003.md), [[Backend] Refactor gemm_sp](../sources/prs/tilelang/PR-2048.md), [[CUDA] Support int4 `T.gemm`](../sources/prs/tilelang/PR-2063.md), [[CUDA] Improve int4 GEMM lowering and packed codegen support](../sources/prs/tilelang/PR-2073.md), [[TMA] Support FP4 TensorMap TMA copies](../sources/prs/tilelang/PR-2107.md), [feat: auto-vectorize bf16/fp16 reduce with packed add2 intrinsics](../sources/prs/tilelang/PR-2112.md), [[CUDA][TMA] Add TMA tile::gather4 / tile::scatter4 support](../sources/prs/tilelang/PR-2129.md), [[Python] Drop Python 3.9 support](../sources/prs/tilelang/PR-2218.md), [[Perf] Enable fast math in sparse MLA example](../sources/prs/tilelang/PR-2219.md), [[Core] Support fully transparent sleep mode](../sources/prs/vllm/PR-11743.md), [[ROCm][MoE] moe tuning support for rocm](../sources/prs/vllm/PR-12049.md), [[Kernel] Flash Attention 3 Support](../sources/prs/vllm/PR-12093.md), [Add: Support for Sparse24Bitmask Compressed Models](../sources/prs/vllm/PR-12097.md), [[Hardware][Gaudi][Feature] Enable Dynamic MoE for Mixtral](../sources/prs/vllm/PR-12303.md), [[Core] Optimizing cross-attention `QKVParallelLinear` computation](../sources/prs/vllm/PR-12325.md), [[Bugfix] Disable w16a16 2of4 sparse CompressedTensors24](../sources/prs/vllm/PR-12417.md), [[Misc][MoE] add Deepseek-V3 moe tuning support](../sources/prs/vllm/PR-12558.md), [Expert Parallelism (EP) Support for DeepSeek Models](../sources/prs/vllm/PR-12583.md), [Apply torch.compile to fused_moe/grouped_topk](../sources/prs/vllm/PR-12637.md), [Disable chunked prefill and/or prefix caching when MLA is enabled ](../sources/prs/vllm/PR-12642.md), [[AMD][ROCm] Enable DeepSeek model on ROCm](../sources/prs/vllm/PR-12662.md), [Squelch MLA warning for Compressed-Tensors Models](../sources/prs/vllm/PR-12704.md), [[VLM] Add MLA with pure RoPE support for deepseek-vl2 models](../sources/prs/vllm/PR-12729.md), [[Misc] Update w2 scale loading for GPTQMarlinMoE](../sources/prs/vllm/PR-12757.md), [[Bugfix] Better FP8 supported defaults](../sources/prs/vllm/PR-12796.md), [[Attention] Use FA3 for MLA on Hopper](../sources/prs/vllm/PR-12807.md), [[Model] Deepseek GGUF support ](../sources/prs/vllm/PR-13167.md), [Expand MLA to support most types of quantization](../sources/prs/vllm/PR-13181.md), [[Quant][Perf] Use moe_wna16 kernel by default for MoEs with many experts](../sources/prs/vllm/PR-13236.md), [[Bugfix] Massage MLA's usage of flash attn for RoCM](../sources/prs/vllm/PR-13310.md), [[NVIDIA] Support nvfp4 tcgen05 gemm](../sources/prs/vllm/PR-13571.md), [[Bugfix] Fix max_num_batched_tokens for MLA](../sources/prs/vllm/PR-13620.md), [[Kernel] Optimize moe intermediate_cache usage](../sources/prs/vllm/PR-13625.md), [[ROCM] fix native attention function call](../sources/prs/vllm/PR-13650.md), [[BugFix] Illegal memory access for MoE On H20](../sources/prs/vllm/PR-13693.md), [[Bugfix] Support MLA for CompressedTensorsWNA16](../sources/prs/vllm/PR-13725.md), [[V1] V1 Enablement Oracle ](../sources/prs/vllm/PR-13726.md), [Fix CompressedTensorsWNA16MoE with grouped scales](../sources/prs/vllm/PR-13769.md), [Fix precommit fail in fused_moe intermediate_cache2 chunking](../sources/prs/vllm/PR-13772.md), [[Bugfix][Quantization] Fix FP8 + EP](../sources/prs/vllm/PR-13784.md), [add tcgen05 support for tcgen05 fp8 gemm](../sources/prs/vllm/PR-13798.md), [Fix mla prefill context performance](../sources/prs/vllm/PR-13897.md), [[Misc] Print FusedMoE detail info](../sources/prs/vllm/PR-13974.md), [[core] moe fp8 block quant tuning support](../sources/prs/vllm/PR-14068.md), [[V1] Implement sliding window attention in kv_cache_manager](../sources/prs/vllm/PR-14097.md), [[v1] Add comments to the new ragged paged attention Pallas kernel](../sources/prs/vllm/PR-14155.md), [[V1][TPU] TPU multimodal model support for ragged attention](../sources/prs/vllm/PR-14158.md), [[V1][TPU] Support V1 Sampler for ragged attention](../sources/prs/vllm/PR-14227.md), [[Hardware] Update the flash attn tag to support Blackwell](../sources/prs/vllm/PR-14244.md), [[BugFix] Fix prefix caching V0 MLA](../sources/prs/vllm/PR-14255.md), [[Misc] Add Qwen2MoeForCausalLM moe tuning support ](../sources/prs/vllm/PR-14276.md), [[Hardware][TPU]Enable ragged paged attention kernel and resolve recompilation issue](../sources/prs/vllm/PR-14310.md), [[Bug] Fix Attention when ignored in by quant_method](../sources/prs/vllm/PR-14313.md), [[ROCm] Enable chunked prefill/paged attention in MLA on ROCm](../sources/prs/vllm/PR-14316.md), [[Perf] Reduce MLA CPU overheads in V1](../sources/prs/vllm/PR-14384.md), [[ROCm][Kernel] MoE weights padding](../sources/prs/vllm/PR-14454.md), [[Bugfix] DeepSeek Accuracy](../sources/prs/vllm/PR-14476.md), [[Perf] Improve MLA on V1](../sources/prs/vllm/PR-14540.md), [[Quantization][FP8] Adding support for fp8 gemm layer input in fp8](../sources/prs/vllm/PR-14578.md), [[Model] Add support for Gemma 3](../sources/prs/vllm/PR-14660.md), [[Bugfix][IPEX] Add `VLLM_CPU_MOE_PREPACK` to allow disabling MoE prepack when CPU does not support it](../sources/prs/vllm/PR-14681.md), [[Kernel][CPU] CPU MLA](../sources/prs/vllm/PR-14744.md), [[Attention] MLA get rid of materialization](../sources/prs/vllm/PR-14770.md), [[Attention] Get rid of mla cache alignment](../sources/prs/vllm/PR-14842.md), [[V1][BugFix] Detect interleaved sliding window attention](../sources/prs/vllm/PR-14896.md), [[V1] Default MLA to V1](../sources/prs/vllm/PR-14921.md), [[FEAT][ROCm] Integrate Fused MoE Kernels from AITER](../sources/prs/vllm/PR-14967.md), [[FEAT] [ROCm]: Add AITER Block-Scaled GEMM Feature](../sources/prs/vllm/PR-14968.md), [[FEAT][ROCm] Integrate Paged Attention Kernel from AITER](../sources/prs/vllm/PR-15001.md), [[Bugfix] Fix incorrect qwen2.5-vl attention mask pre-computation](../sources/prs/vllm/PR-15200.md), [[Bugfix] Fix use_cascade_attention handling for Alibi-based models on vllm/v1](../sources/prs/vllm/PR-15211.md), [[Misc] Add attention mask pre-computation optimization back to Qwen2.5-VL](../sources/prs/vllm/PR-15273.md), [[Model] Add Qwen3 and Qwen3MoE](../sources/prs/vllm/PR-15289.md), [Fix non-contiguous input passed to Marlin kernel](../sources/prs/vllm/PR-15319.md), [[FEAT] [ROCm] Add AITER int8 scaled gemm kernel](../sources/prs/vllm/PR-15433.md), [Use Cache Hinting for fused_moe kernel](../sources/prs/vllm/PR-15511.md), [[moe][quant] add weight name case for offset](../sources/prs/vllm/PR-15515.md), [[Quantization] Fp8 Channelwise Dynamic Per Token GroupedGEMM](../sources/prs/vllm/PR-15587.md), [[TPU] Support sliding window and logit soft capping in the paged attention kernel for TPU.](../sources/prs/vllm/PR-15732.md), [[V1] TPU - Fix fused MOE](../sources/prs/vllm/PR-15834.md), [[Bugfix] Fix cache block size calculation for CPU MLA](../sources/prs/vllm/PR-15848.md), [[FEAT][ROCm]: Support AITER MLA](../sources/prs/vllm/PR-15893.md), [[Hardware][Gaudi][BugFix] fix arguments of hpu fused moe](../sources/prs/vllm/PR-15945.md), [Add support to modelopt quantization of Mixtral model](../sources/prs/vllm/PR-15961.md), [[NVIDIA] Support Cutlass MLA for Blackwell GPUs](../sources/prs/vllm/PR-16032.md), [[Kernel] Use moe_wna16 kernel for compressed tensors wna16 moe models](../sources/prs/vllm/PR-16038.md), [[Model] use AutoWeightsLoader for phimoe,qwen2_moe,qwen3_moe](../sources/prs/vllm/PR-16203.md), [[Hardware][AMD] Improve OAM device ID + llama4 Maverick MOE tuning](../sources/prs/vllm/PR-16263.md), [[Llama4] Enable attention temperature tuning by default for long context (>32k)](../sources/prs/vllm/PR-16439.md), [[MLA] Simplification to batch P/D reordering](../sources/prs/vllm/PR-16673.md), [[ROCM] enable aiter fused moe kernel for llama4 bf16 checkpoints](../sources/prs/vllm/PR-16674.md), [[ROCm] Add aiter tkw1 kernel for Llama4 fp8](../sources/prs/vllm/PR-16727.md), [Support W8A8 INT8 MoE for compressed-tensors](../sources/prs/vllm/PR-16745.md), [[FEAT] [ROCm]: AITER Fused MOE V1 Support](../sources/prs/vllm/PR-16752.md), [[Bugfix] Fix moe weight losing all extra attrs after `process_weights_after_loading`.](../sources/prs/vllm/PR-16854.md), [[Bugfix] Add contiguous call inside rope kernel wrapper](../sources/prs/vllm/PR-17091.md), [[FEAT] [ROCm]: Add AITER CK 2 Stages MoE support](../sources/prs/vllm/PR-17110.md), [[Bugfix] gemma[2,3] interleaved attention when sliding window is disabled](../sources/prs/vllm/PR-17180.md), [[Bugfix] Get a specific type of layer from forward context](../sources/prs/vllm/PR-17222.md), [[BugFix] Fix vllm_flash_attn install issues](../sources/prs/vllm/PR-17267.md), [[BugFix] Fix cascade attention - RuntimeError: scheduler_metadata must have shape (metadata_size)](../sources/prs/vllm/PR-17283.md), [[v1] AttentionMetadata for each layer](../sources/prs/vllm/PR-17394.md), [Fix noisy warning for uncalibrated q_scale/p_scale](../sources/prs/vllm/PR-17414.md), [[v1] Pass BlockTable and KVCacheSpec to AttentionMetadataBuilders](../sources/prs/vllm/PR-17483.md), [[BugFix] Fix mla cpu - missing 3 required positional arguments](../sources/prs/vllm/PR-17494.md), [[FEAT][ROCm]: Support AITER MLA on V1 Engine](../sources/prs/vllm/PR-17523.md), [[Bugfix][ROCm] Fix AITER MLA V1](../sources/prs/vllm/PR-17880.md), [[BugFix][AMD] Compatible patch for AITER lib after 04/20](../sources/prs/vllm/PR-17912.md), [[Misc] Add compressed-tensors NVFP4A16 emulation support](../sources/prs/vllm/PR-17914.md), [use ceil_div in cutlass block scaling shape check](../sources/prs/vllm/PR-17918.md), [[v1] Support multiple KV cache groups in GPU model runner](../sources/prs/vllm/PR-17945.md), [[BUG] [ROCm] [MLA] Fix variable name bug due to change in variable name in PR #17483](../sources/prs/vllm/PR-17961.md), [Use NVFP4 Marlin for CompressedTensorsW4A16Fp4](../sources/prs/vllm/PR-18000.md), [[Quantization] Add compressed-tensors NVFP4 support](../sources/prs/vllm/PR-18312.md), [[Model]: Fused MoE for nomic-embed-text-v2-moe](../sources/prs/vllm/PR-18321.md), [[Feature] Expert Parallelism Load Balancer (EPLB)](../sources/prs/vllm/PR-18343.md), [[Bug] Fix moe_sum signature](../sources/prs/vllm/PR-18440.md), [[V1] Support `LLM.apply_model`](../sources/prs/vllm/PR-18465.md), [[Hardware][AMD] integrate aiter chunked prefill into vllm](../sources/prs/vllm/PR-18596.md), [[P/D] Heterogeneous TP](../sources/prs/vllm/PR-18833.md), [[ROCm] [AITER] [Bugfix] Patch for AITER commit `648764942e552a8bb5fe16026703716a81f05374`](../sources/prs/vllm/PR-18990.md), [[Kernel] Support deep_gemm for linear methods](../sources/prs/vllm/PR-19085.md), [[Kernel] Apply torch.Tag.needs_fixed_stride_order only for torch==2.6.0](../sources/prs/vllm/PR-19346.md), [[Core] Support Local Chunked Attention for Hybrid KV Cache](../sources/prs/vllm/PR-19351.md), [[Kernels] Use empty for modular MoE workspaces](../sources/prs/vllm/PR-19667.md), [[Feature] Integrate new deepgemm](../sources/prs/vllm/PR-19820.md), [[Bugfix] Enable PP with AITER+V1](../sources/prs/vllm/PR-19822.md), [[Quantization] Add compressed-tensors emulations support for NVFP4](../sources/prs/vllm/PR-19879.md), [[Quantization] Add compressed-tensors NVFP4 MoE Support](../sources/prs/vllm/PR-19990.md), [Enable V1 for Hybrid SSM/Attention Models](../sources/prs/vllm/PR-20016.md), [[Attention] MLA - Flashinfer Ragged Prefill](../sources/prs/vllm/PR-20034.md), [Add ModelOpt Qwen3 nvfp4 support](../sources/prs/vllm/PR-20101.md), [[Bugfix] Mark 'hidden_states' as mutable in moe_forward registration.](../sources/prs/vllm/PR-20152.md), [[Bugfix] Fix Maverick correctness by filling zero to cache space in cutlass_moe](../sources/prs/vllm/PR-20167.md), [[Nixl] Heterogeneous TP support FlashInfer](../sources/prs/vllm/PR-20189.md), [[V1] [ROCm] Enable EP with AITER Fused MoE](../sources/prs/vllm/PR-20270.md), [Support Llama 4 for cutlass_moe_fp4](../sources/prs/vllm/PR-20453.md), [Support Llama 4 for fused_marlin_moe](../sources/prs/vllm/PR-20457.md), [[Perf] Reuse workspace for FP8+FP4 Marlin MoE](../sources/prs/vllm/PR-20500.md), [[Bugfix] Fix missing per_act_token parameter in compressed_tensors_moe](../sources/prs/vllm/PR-20509.md), [[feat] enable SM100 CUTLASS block scaled group gemm for smaller batch sizes](../sources/prs/vllm/PR-20640.md), [Integration SM100 FlashInfer fused allreduce RMSNorm](../sources/prs/vllm/PR-20691.md), [GLM-4.5 Model Support](../sources/prs/vllm/PR-20736.md), [[v1][core] Support for attention free models](../sources/prs/vllm/PR-20811.md), [[Feature][EPLB] Add eplb support for Qwen3](../sources/prs/vllm/PR-20815.md), [[Bugfix] Fix a couple PPLX+CUTLASS MoE bugs](../sources/prs/vllm/PR-20825.md), [[Bug] Fix DeepGemm for EP low latency case](../sources/prs/vllm/PR-20833.md), [[Model] Pooling models default to using chunked prefill & prefix caching if supported.](../sources/prs/vllm/PR-20930.md), [[Misc] Qwen MoE model supports LoRA](../sources/prs/vllm/PR-20932.md), [[Bugfix] Switch bailout logic for kv-cache-dtype with SM100 Flashinfer](../sources/prs/vllm/PR-20934.md), [Fall back if flashinfer comm module not found](../sources/prs/vllm/PR-20936.md), [[Bugfix] Fix Mistral3 support on SM100/SM120](../sources/prs/vllm/PR-20998.md), [Add FlashInfer allreduce RMSNorm Quant fusion](../sources/prs/vllm/PR-21069.md), [[Bugfix] Voxtral on Blackwell GPUs (RTX 50 series)](../sources/prs/vllm/PR-21077.md), [[Bugfix] Allocate less memory in non-batched CUTLASS MoE](../sources/prs/vllm/PR-21121.md), [[Attention] Optimize FlashInfer MetadataBuilder Build call](../sources/prs/vllm/PR-21137.md), [[Attention][DBO] Add support for "splitting" the CommonAttentionMetadata](../sources/prs/vllm/PR-21153.md), [[Feature][OCP MX] Support mxfp6 and mixed mxfp6-mxfp4](../sources/prs/vllm/PR-21166.md), [[Bug] DeepGemm: Fix TypeError: per_block_cast_to_fp8() missing 1 required positional argument: 'use_ue8m0' for SM100](../sources/prs/vllm/PR-21187.md), [Support encoder-only models without KV-Cache](../sources/prs/vllm/PR-21270.md), [Fix Flashinfer Allreduce+Norm enable disable calculation based on `fi_allreduce_fusion_max_token_num`](../sources/prs/vllm/PR-21325.md), [Support Tensorrt-LLM MoE fp4 for low-latency](../sources/prs/vllm/PR-21331.md), [[TPU][Bugfix] fix moe layer](../sources/prs/vllm/PR-21340.md), [[Quantization] Enable BNB support for more MoE models](../sources/prs/vllm/PR-21370.md), [Update flashinfer CUTLASS NVFP4 MoE Kernel to use per expert global scaling factor](../sources/prs/vllm/PR-21408.md), [[NVIDIA] Explicitly disable shuffled weights for flashinfer blockscale moe fp8 kernels](../sources/prs/vllm/PR-21411.md), [Updates to Flex + VLLm integration](../sources/prs/vllm/PR-21416.md), [[V1] Fix local chunked attention always disabled](../sources/prs/vllm/PR-21419.md), [[BugFix] Fix shared storage connector load kv only load attention layer](../sources/prs/vllm/PR-21428.md), [update flashinfer to v0.2.9rc1](../sources/prs/vllm/PR-21485.md), [[MoE] More balanced expert sharding](../sources/prs/vllm/PR-21497.md), [[NVIDIA] Fix Llama4 Scout FP4 functionality issues](../sources/prs/vllm/PR-21499.md), [Enable 4bit bnb prequant MOE](../sources/prs/vllm/PR-21548.md), [[Attention] Support multiple attention metadata builders per kv_cache_spec + proper local attention no hybrid kv cache fix](../sources/prs/vllm/PR-21588.md), [Override attention metadata for fast prefill in some KV sharing setups](../sources/prs/vllm/PR-21590.md), [[Feature] Add Flashinfer MoE Support for Compressed Tensor NVFP4](../sources/prs/vllm/PR-21639.md), [[xpu]support moe models on XPU platform](../sources/prs/vllm/PR-21643.md), [support `torch.compile` for bailing moe](../sources/prs/vllm/PR-21664.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv/out Attention Kernel](../sources/prs/vllm/PR-21716.md), [feat: Add Support GPTQ Quantization MOE on ROCM vllm serve](../sources/prs/vllm/PR-21733.md), [[Logs] Change flashinfer sampler logs to once](../sources/prs/vllm/PR-21759.md), [[Perf] Disable chunked local attention by default with llama4](../sources/prs/vllm/PR-21761.md), [[Bugfix] Check NVIDIA artifactory is accessible before using flashinfer cubin kernels](../sources/prs/vllm/PR-21893.md), [[Qwen3] Enable dual-chunk-attention support for Qwen3 models.](../sources/prs/vllm/PR-21924.md), [[BUGFIX] KeyError 'layers.14.mlp.gate.g_idx' for Qwen3-MoE with GPTQ on ROCm](../sources/prs/vllm/PR-22017.md), [[NVIDIA] Support Flashinfer TRT-LLM Prefill Attention Kernel](../sources/prs/vllm/PR-22095.md), [[EPLB] Support ernie4.5-moe](../sources/prs/vllm/PR-22100.md), [[fix] fix correct assertion syntax error in attention utils.](../sources/prs/vllm/PR-22154.md), [[bugfix] fix blackwell deepep installation](../sources/prs/vllm/PR-22255.md), [[Bugfix] Fix MoE BNB version](../sources/prs/vllm/PR-22260.md), [Support encoder_only attention for FlexAttention](../sources/prs/vllm/PR-22273.md), [[Bugfix] Fix 3D input passed into cutlass_scaled_mm](../sources/prs/vllm/PR-22278.md), [[ROCm] Add attention sink to use_rocm_custom_paged_attention](../sources/prs/vllm/PR-22329.md), [[gpt-oss] flashinfer mxfp4](../sources/prs/vllm/PR-22339.md), [Update `flashinfer-python==0.2.10`](../sources/prs/vllm/PR-22389.md), [[Bug] Fix B200 DeepGEMM E8M0 Accuracy Issue](../sources/prs/vllm/PR-22399.md), [[bugfix] Fix Llama3/4 issues caused by FlashInfer 0.2.10](../sources/prs/vllm/PR-22426.md), [[Quantization]: Support compressed-tensors mixed-precision model loading](../sources/prs/vllm/PR-22468.md), [Fix Llama4 FlashInfer FP4 MoE issues](../sources/prs/vllm/PR-22511.md), [[Model] Add Ernie4.5 VL Model Support](../sources/prs/vllm/PR-22514.md), [Quantization: support FP4 quantized models on AMD CDNA2/CDNA3 GPUs](../sources/prs/vllm/PR-22527.md), [Fix torch version check for SM100 mxfp4 ](../sources/prs/vllm/PR-22535.md), [Upgrade FlashInfer to v0.2.11](../sources/prs/vllm/PR-22613.md), [[Bugfix] Fix ModernBert load & Enable sliding window attention for bidirectional attention.](../sources/prs/vllm/PR-22637.md), [Support multiple attention groups for KV sharing](../sources/prs/vllm/PR-22672.md), [[Quantization] Expand compressed-tensors MoE matching logic to support NFP4 + FP8 MoEs](../sources/prs/vllm/PR-22674.md), [Force TRTLLM attention for gpt-oss on SM100](../sources/prs/vllm/PR-22678.md), [[Bugfix] Fix default enable for CUTLASS MLA on SM100](../sources/prs/vllm/PR-22738.md), [Fix GGUF loader for Qwen3 MoE.](../sources/prs/vllm/PR-22785.md), [[FIXBUG] Add return_success parameter to moe_wna16_weight_loader function](../sources/prs/vllm/PR-22797.md), [[Model] Modify the gate implementation of glm4_moe](../sources/prs/vllm/PR-22832.md), [[XPU] support data parallel for MoE models on XPU](../sources/prs/vllm/PR-22887.md), [[Kernel] Added flashinfer fp8 per-tensor gemms](../sources/prs/vllm/PR-22895.md), [[Bugfix] Fix DeepSeek MTP](../sources/prs/vllm/PR-22934.md), [Use Blackwell FlashInfer MXFP4 MoE by default if available ](../sources/prs/vllm/PR-23008.md), [[Bugfix gpt-oss] Fix float32 convert for flashinfer sink support](../sources/prs/vllm/PR-23016.md), [[Bugfix] fix qwen3 moe fp8 accuracy issue](../sources/prs/vllm/PR-23031.md), [[Core] Support weight_loader_v2 for `UnquantizedLinearMethod`](../sources/prs/vllm/PR-23036.md), [Add routed_scaling_factor to MoE grouped topk](../sources/prs/vllm/PR-23123.md), [[Bugfix] Fix accuracy issue when using flashinfer cutlass moe, TP=1 and modelopt.](../sources/prs/vllm/PR-23125.md), [Update to flashinfer-python==0.2.12 and disable AOT compile for non-release image](../sources/prs/vllm/PR-23129.md), [[Log] Warning Once for Cutlass MLA ](../sources/prs/vllm/PR-23137.md), [Fix nvfp4 swizzling](../sources/prs/vllm/PR-23140.md), [[CPU] add cpu fused moe pytorch native implementation](../sources/prs/vllm/PR-23146.md), [[XPU][Feature] fp8 online quantization support for XPU](../sources/prs/vllm/PR-23148.md), [Optimize input preparation for FlashInfer [2/N]](../sources/prs/vllm/PR-23174.md), [[Attention] Optimize make_local_attention_virtual_batches for Flash Attention](../sources/prs/vllm/PR-23185.md), [[Misc][qwen2_5_vl][torch.compile] Enable `supports_torch_compile` on generic nn.Module and demonstrate speedup on Qwen Vision model](../sources/prs/vllm/PR-23207.md), [[Core] Always use tensor cores for Flashinfer Decode Wrapper](../sources/prs/vllm/PR-23214.md), [[Perf] Small optimizations for silu_mul_fp8_quant_deep_gemm](../sources/prs/vllm/PR-23265.md), [[Kernels] Overlap shared experts with send/recv](../sources/prs/vllm/PR-23273.md), [[Bug] Fix R1 Accuracy 0 Bug](../sources/prs/vllm/PR-23294.md), [[Attention] Allow V1 flash_attn to support cross-attention](../sources/prs/vllm/PR-23297.md), [[Perf] Warmup FlashInfer attention during startup](../sources/prs/vllm/PR-23439.md), [[Attention][FA3] Update FA3 to include new swizzle optimization](../sources/prs/vllm/PR-23465.md), [fix(v1/kv_cache): resolve async KV transfer bug in cascade attention](../sources/prs/vllm/PR-23485.md), [[Bugfix] Fix Qwen3 MoE GPTQ inference](../sources/prs/vllm/PR-23490.md), [[V1][P/D]P2pNcclConnector supports flashinfer](../sources/prs/vllm/PR-23536.md), [Update Flashinfer to 0.2.14.post1](../sources/prs/vllm/PR-23537.md), [[Misc] Simplify FlashInfer attention metadata](../sources/prs/vllm/PR-23585.md), [DP/EP Support for gpt-oss with deepep-ht comm kernel on SM100](../sources/prs/vllm/PR-23608.md), [[Flashinfer] Support Flashinfer TRTLLM FP8-qkv BF16/FP16-out Attention Kernel](../sources/prs/vllm/PR-23647.md), [[Bugfix] Fix Marlin NVFP4 for modelopt](../sources/prs/vllm/PR-23659.md), [[v1] Add cross-attention KV cache support for encoder-decoder models](../sources/prs/vllm/PR-23664.md), [[Core/DBO][1/N] Add Dual-Batch Overlap mechanism to VLLM](../sources/prs/vllm/PR-23693.md), [[Kernel][tcgen05] nvfp4 fused tcgen05 moe](../sources/prs/vllm/PR-23696.md), [[FlashInfer] Cache hyper params in metadata builder](../sources/prs/vllm/PR-23732.md), [[BugFix][FlashInfer] Fix potential race condition for paged_kv_indptr_cpu](../sources/prs/vllm/PR-23737.md), [[Feat][EPLB] A novel static EPLB placement strategy for MoE models.](../sources/prs/vllm/PR-23745.md), [[Misc] add reorder_batch AttentionMetadataBuilder](../sources/prs/vllm/PR-23798.md), [[fix]: add Arm 4bit fused moe support](../sources/prs/vllm/PR-23809.md), [[Model][gpt-oss] Support DP+EP for GPT-OSS with FlashInfer trtllm-gen MoE](../sources/prs/vllm/PR-23819.md), [[BUGFIX ] fix undefined silu_and_mul_nvfp4_quant](../sources/prs/vllm/PR-23929.md), [Feature/vit attention unification# 23880](../sources/prs/vllm/PR-23978.md), [[BUGFIX] GPTQ quantization compatibility for Qwen3 MOE models (AutoGPTQ and AutoRound-GPTQ)](../sources/prs/vllm/PR-23994.md), [[PERF] Allreduce fusion. Support torch native matching. Tuning of the thresholds](../sources/prs/vllm/PR-24248.md), [[Transform] [Quantization] Add QuTLASS support to vLLM](../sources/prs/vllm/PR-24440.md), [[Feature] Disallow FlashMLA on Blackwell](../sources/prs/vllm/PR-24521.md), [[Performance] Move apply_w8a8_block_fp8_linear to an op class](../sources/prs/vllm/PR-24666.md), [[Model] Support Qwen3-VL Model Series](../sources/prs/vllm/PR-24727.md), [[Bug] Fix `is_flashmla_supported` Check Error](../sources/prs/vllm/PR-24774.md), [[DCP] Support Decode Context Parallel (DCP) for GQA with FlashAttention](../sources/prs/vllm/PR-24864.md), [[Attention][DCP] Support DCP with query length > 1 (MTP) with FA3](../sources/prs/vllm/PR-25049.md), [[Bug] Fix `returned_lse` not Defined issue](../sources/prs/vllm/PR-25106.md), [[ROCm] Small functional changes for gptoss](../sources/prs/vllm/PR-25201.md), [[BugFix] Fix MLA assert with CUTLASS MLA](../sources/prs/vllm/PR-25478.md), [feat: BF16 FlashInfer Fused Cutlass MOE for Hopper and Blackwell Expert Parallel](../sources/prs/vllm/PR-25503.md), [Enable Fbgemm NVFP4 on Dense models](../sources/prs/vllm/PR-25609.md), [[Flashinfer][gpt-oss] Support FP8-qkv Flashinfer TRTLLM Sinks Attention](../sources/prs/vllm/PR-25674.md), [[Bugfix] Enable padded FP4 quantization](../sources/prs/vllm/PR-25947.md), [[Quantization/NVFP4] Speed up TRTLLM NVFP4 MOE weight loading and fix K/V scale loading for MLA Attn](../sources/prs/vllm/PR-25968.md), [[Spec Decode] Enable efficient speculative decoding with FlashInfer-MLA](../sources/prs/vllm/PR-25984.md), [[Bugfix] Allow skipping MoE in NVFP4 (fix for MTP)](../sources/prs/vllm/PR-25987.md), [[NVIDIA] Add support for cudnn fp4 gemm via flashinfer](../sources/prs/vllm/PR-26107.md), [[ModelOpt] Load w13/w2_input_scale for all experts, nvfp4](../sources/prs/vllm/PR-26135.md), [[Performance] Dual stream execution of "shared_experts" and "selected_experts" inside FusedMoE](../sources/prs/vllm/PR-26440.md), [[Bugfix] Convert untraceable GroupShape to list for AMD impl](../sources/prs/vllm/PR-26535.md), [[ROCM] MoE fp4 CK kernel](../sources/prs/vllm/PR-26545.md), [support flashinfer_fp4 moe for 5090 gpu](../sources/prs/vllm/PR-26669.md), [[Bugfix] Fix gpt-oss w4a8 DP/EP on B200](../sources/prs/vllm/PR-26729.md), [Disable FlashInfer sampler by default](../sources/prs/vllm/PR-26859.md), [[Feature] Batch Invariant: Support DeepGEMM and Blackwell](../sources/prs/vllm/PR-27127.md), [[Kernels] Enable FlashInfer FP8 Blockscale on SM90 (for TEP DSR1)](../sources/prs/vllm/PR-27134.md), [[torch.compile] Enable silu_mul_fp8_quant fusion without custom ops enabled](../sources/prs/vllm/PR-27146.md), [[ROCM] Enable CompressedTensorsWNA16](../sources/prs/vllm/PR-27187.md), [[BUGFIX][ROCM] ViT FlashAttention on ROCm (no GFX9) and contiguous on qwen3vl ROCm TORCH_SDPA](../sources/prs/vllm/PR-27190.md), [Flashinfer_CUTLASS_MOE fuses quantization for TP](../sources/prs/vllm/PR-27223.md), [[Feature] Batch Invariant for R1 TP 8 on Blackwell](../sources/prs/vllm/PR-27229.md), [[Bugfix] Ensure calculated KV scales are applied in attention.](../sources/prs/vllm/PR-27232.md), [Bugfix: Cutlass FP8 FusedMoE bad scaling factors](../sources/prs/vllm/PR-27255.md), [Feature: Support Relu2 in FusedMoE fp8 cutlass path](../sources/prs/vllm/PR-27261.md), [[Misc] Make reorder batch also separate extends](../sources/prs/vllm/PR-27367.md), [[Performance] Support FP8 flashinfer TRTLLM MOE on Qwen3 and Qwen-3next](../sources/prs/vllm/PR-27492.md), [[Feature] Batch invariant torch.compile](../sources/prs/vllm/PR-27660.md), [[AMD] Use Decoupled Kernel Block Size to Support AITER MLA block_size=1](../sources/prs/vllm/PR-27715.md), [[Feature] Extend batch invariant torch.compile to B200](../sources/prs/vllm/PR-27856.md), [[Bug] Batch invariant: Fix flash attn MLA `RuntimeError: scheduler_metadata must have shape (metadata_size)`](../sources/prs/vllm/PR-27884.md), [[Performance][B200] Fix deepgemm prologue](../sources/prs/vllm/PR-27897.md), [[flashinfer][fix] do not check nvcc availability when using pre-downloaded cubins](../sources/prs/vllm/PR-27990.md), [[FlashInfer] Avoid FlashInfer block_size 16 + head_size 256 on blackwell](../sources/prs/vllm/PR-27994.md), [[ROCm][MLA] enable fp8 MLA decode on ROCm](../sources/prs/vllm/PR-28032.md), [[Model] Consolidate Deepseek-MoE implementation with DeepSeek-v2](../sources/prs/vllm/PR-28101.md), [[Mamba] - Consolidate Mambas Attention Logic](../sources/prs/vllm/PR-28133.md), [[flashinfer] fix FI all2all with FI cutlass moe](../sources/prs/vllm/PR-28166.md), [[Feature] Support recording expert indices for rollout router replay](../sources/prs/vllm/PR-28284.md), [[ROCm] Support for Whisper v1 with Aiter Unified Attention and Aiter Flash Attention](../sources/prs/vllm/PR-28376.md), [[Bugfix][EPLB] Disabled shared expert overlap when EPLB is enabled](../sources/prs/vllm/PR-28377.md), [[Bugfix] Fix SM100 gpt-oss regression due to faulty attn sink support](../sources/prs/vllm/PR-28561.md), [[Attention][Bugfix] Fix FA sink support](../sources/prs/vllm/PR-28660.md), [[Bugfix][Nixl] Fix kernel physical<>logical block_size issue ](../sources/prs/vllm/PR-28677.md), [[Performance] Reduce DeepGEMM N dim restriction from 128 to 64 multiplier ](../sources/prs/vllm/PR-28687.md), [[Feature] Prefill Context Parallel (PCP) basic support](../sources/prs/vllm/PR-28718.md), [[Bugfix] Fix GPT-OSS on AMD after #28603](../sources/prs/vllm/PR-28816.md), [bugfix: correct attn output with base 2 or e](../sources/prs/vllm/PR-28840.md), [[Bugfix] Fix GPT-OSS AR+NORM fusion](../sources/prs/vllm/PR-28841.md), [[Bugfix] Make compressed-tensors MoEs respect ignored layers](../sources/prs/vllm/PR-28878.md), [Add TRTLLM MoE NVFP4 kernel to CompressedTensorsW4A4MoeMethod](../sources/prs/vllm/PR-28892.md), [[BugFix] Fix async-scheduling + FlashAttn MLA](../sources/prs/vllm/PR-28990.md), [[Feat] Support non-gated activations in NVFP4 modelopt path](../sources/prs/vllm/PR-29004.md), [[DeepSeek + LMCache Multiprocess] handle MLA for deepseek model + LMCache Multiprocess connector](../sources/prs/vllm/PR-29039.md), [[Perf][Kernels] Enable FlashInfer DeepGEMM swapAB on SM90 (for W8A8 Linear Op)](../sources/prs/vllm/PR-29213.md), [[LoRA] Optimize 3D MoE logic](../sources/prs/vllm/PR-29222.md), [[Perf] Disable DeepGEMM MoE by default when TP=8 is used](../sources/prs/vllm/PR-29346.md), [Add unpermute-aware fused MoE path and small-batch fallback](../sources/prs/vllm/PR-29354.md), [[Bugfix] Fix grouped_topk pytorch impl when num_experts can't be grouped properly](../sources/prs/vllm/PR-29439.md), [[Attention] Cache attention metadata builds across hybrid KV-cache groups](../sources/prs/vllm/PR-29627.md), [[Bugfix] Defunctionalize TRTLLM AR+Norm op for avoiding extra clone kernel before it](../sources/prs/vllm/PR-29631.md), [[Attention] Make `split_decodes_and_prefills(..., require_uniform=True)` support padding](../sources/prs/vllm/PR-29644.md), [[perf] Use direct copy (broadcast) instead of cat for k_nope/k_pe in MLA prefill](../sources/prs/vllm/PR-29710.md), [[Quantization] Enable compressed-tensors AWQ for Turing GPU](../sources/prs/vllm/PR-29732.md), [[Bugfix] Fix mismatched nvfp4 gemm output shape](../sources/prs/vllm/PR-29742.md), [[MoE-FP8-modelopt] Add FlashInfer alignment padding for intermediate dimensions](../sources/prs/vllm/PR-29748.md), [Add Mistral Large 3 and Ministral 3](../sources/prs/vllm/PR-29757.md), [[ROCm] [Fused Moe EP] Use binary expert mask for aiter fused moe kernel](../sources/prs/vllm/PR-29773.md), [[ROCm][MXFP4] Infer w4a4 quant method in rocm aiter fused moe](../sources/prs/vllm/PR-29775.md), [[EPLB] Support EPLB w/ NVFP4](../sources/prs/vllm/PR-29804.md), [[Bugfix][Model] Support LoRA on Qwen3 Output Embedding](../sources/prs/vllm/PR-29816.md), [[SpecDecode] Simplified alternative padded-speculation acceptance rate fix](../sources/prs/vllm/PR-29845.md), [[Quantization] fix: overflow with static per-tensor scaling](../sources/prs/vllm/PR-29867.md), [[Bugfix] Fix FP8 MoE LoRA](../sources/prs/vllm/PR-29890.md), [[BugFix] Fix DBO assert `assert B_block_table == B_q`](../sources/prs/vllm/PR-29933.md), [[moe] Use enable_chunking func (to support disabling chunking)](../sources/prs/vllm/PR-29935.md), [[moe] Allow disabling DP chunking](../sources/prs/vllm/PR-29936.md), [[Bugfix] Fix flashinfer ar+norm kernel not available issue](../sources/prs/vllm/PR-29960.md), [[ROCm] add fallback for aiter fp8 decode mla](../sources/prs/vllm/PR-30005.md), [[Quantization] Support Quark int4-fp8 w4a8 for MoE](../sources/prs/vllm/PR-30071.md), [[Model][Quantization] Restore MoE + GGUF models support (incl. Qwen3 MoE) by allowing Sideload Parameters](../sources/prs/vllm/PR-30116.md), [[Model][Quantization] Override HF defaults to GGUF ones (incl. Qwen3 MoE)](../sources/prs/vllm/PR-30118.md), [Nvidia ModelOpt workaround for issue 28072](../sources/prs/vllm/PR-30164.md), [Add latent MoE support](../sources/prs/vllm/PR-30203.md), [[Bugfix]: Fix glm46 awq marlin moe wna16 compatibility](../sources/prs/vllm/PR-30210.md), [[LoRA] Reduce the loading time of MoE LoRA](../sources/prs/vllm/PR-30243.md), [[Bugfix] Fix DeepGEMM after #29546 ](../sources/prs/vllm/PR-30267.md), [[Model][Quantization] Fix / Add GGUF support for Qwen2 MoE models](../sources/prs/vllm/PR-30307.md), [[bugfix][quantization] fix quark qwen3 kv_cache quantization](../sources/prs/vllm/PR-30308.md), [[fix] fix SM check for Flashinfer TRTLLM MOE](../sources/prs/vllm/PR-30314.md), [[Bugfix] Fix fp8 DeepGemm compilation issues](../sources/prs/vllm/PR-30336.md), [[BugFix] Fix `AttributeError: 'MergedColumnParallelLinear' object has no attribute 'weight_scale'`](../sources/prs/vllm/PR-30399.md), [fix(gguf): Disable bfloat16 for GGUF on blackwell device](../sources/prs/vllm/PR-30408.md), [[ROCm][Bugfix] Add MLACommonMetadata to allowed attention types for speculative decoding](../sources/prs/vllm/PR-30430.md), [[Bugfix] Pass FA version in `MultiHeadAttention`](../sources/prs/vllm/PR-30575.md), [[Perf] Eliminate padding and slicing op for GPT-OSS with Flashinfer MXFP4 MXFP8 MoE](../sources/prs/vllm/PR-30647.md), [[Misc][LLaMa4] Compile LLaMa Vision Encoder](../sources/prs/vllm/PR-30709.md), [Update note comment for flashinfer attention warmup](../sources/prs/vllm/PR-30711.md), [[Perf] enable flashinfer rotary_embedding custom ops in DeepSeek rotary](../sources/prs/vllm/PR-30729.md), [[Bugfix] Fix broken ViT attention selection for Blackwell device](../sources/prs/vllm/PR-30731.md), [[SM100] Enable fp8 compute for prefill MLA](../sources/prs/vllm/PR-30746.md), [Add support for LoRA adapters in Nemotron-H models](../sources/prs/vllm/PR-30802.md), [[Kernels][FI] Skip trtllm attention when num_kv_heads=1](../sources/prs/vllm/PR-30842.md), [[Compressed-Tensors] Simplify NVFP4 Conditions, enable marlin support for NVFP4A16 MoEs](../sources/prs/vllm/PR-30881.md), [[Kernel][Performance] Enable smaller Scaling Factor tiling for NVFP4 small-batch decoding](../sources/prs/vllm/PR-30885.md), [[Feature]: Support NVIDIA ModelOpt HF FP8 variants FP8_PER_CHANNEL_PER_TOKEN and FP8_PB_WO in vLLM](../sources/prs/vllm/PR-30957.md), [[Mics] add pcp basic support to MoE model](../sources/prs/vllm/PR-31003.md), [[Bugfix] Fix GLM-4 MoE router logits dtype for data parallel chunking](../sources/prs/vllm/PR-31055.md), [ [FIX] Always support TP > 4 for FP4 Gemm](../sources/prs/vllm/PR-31099.md), [[BugFix] LoRA: Support loading base_layer of experts](../sources/prs/vllm/PR-31104.md), [[Bugfix][Hardware][AMD] Consolidate FP8 min/max values helper function](../sources/prs/vllm/PR-31106.md), [[Misc] Fix grammar errors in comments and messages](../sources/prs/vllm/PR-31115.md), [[Bugfix] Fix MoE LoRA bin/pt loading](../sources/prs/vllm/PR-31161.md), [[perf] Integrate flashinfer concat_mla_k](../sources/prs/vllm/PR-31171.md), [[Bugfix][Hardware][AMD] Fix exception types in AITER MLA FP8 check](../sources/prs/vllm/PR-31177.md), [[SM100] Resubmit FMHA FP8 prefill for MLA](../sources/prs/vllm/PR-31195.md), [[Bugfix][Hardware][AMD] Fix last_page_len calculation in AITER MLA decode](../sources/prs/vllm/PR-31282.md), [fix(rocm): add early return in get_flash_attn_version for ROCm](../sources/prs/vllm/PR-31286.md), [pin lora_b moe weights on cpu](../sources/prs/vllm/PR-31317.md), [[Misc] Fix Qwen2-MoE shared_expert_gate](../sources/prs/vllm/PR-31339.md), [[BugFix] add select_gemm_impl on CompressedTensorsWNA16MoEMethod to support LoRA](../sources/prs/vllm/PR-31453.md), [[Bugfix][ROCm] Fix Static Quant Issue](../sources/prs/vllm/PR-31502.md), [[ROCm][Bugfix] Fix accuracy issue on fmoe when `VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS` enabled](../sources/prs/vllm/PR-31523.md), [Use the same memory for workspace13 and fused_output.](../sources/prs/vllm/PR-31531.md), [[Fix] Align fused moe lora_b shape with peft](../sources/prs/vllm/PR-31534.md), [[Bugfix] Fix Broken ModelOpt NVFP4 MoE](../sources/prs/vllm/PR-31742.md), [[Perf] Add opt-in SM100 Oink RMSNorm custom-op path](../sources/prs/vllm/PR-31828.md), [[MISC] Add strict contiguity check for FlashInfer attention tensors](../sources/prs/vllm/PR-32008.md), [[5/N][Attention] Finish eliminating `vllm/attention` folder](../sources/prs/vllm/PR-32064.md), [[BugFix] Fix DeepSeek-V3.1 + DeepGEMM incompatible scale shapes](../sources/prs/vllm/PR-32361.md), [[Model] Molmo2: Enable quantized weight mapping for vision backbone](../sources/prs/vllm/PR-32385.md), [[Hardware][SM100] Add TRTLLM Kernel for INT4 W4A16 Kernel.](../sources/prs/vllm/PR-32437.md), [[Perf] Create TMA-aligned input scale tensor for DeepGemm on Hopper](../sources/prs/vllm/PR-32619.md), [[Kernel] use flashinfer for gdn prefill](../sources/prs/vllm/PR-32846.md), [[Performance] Tune Mamba selective scan kernel for B200](../sources/prs/vllm/PR-32873.md), [[Bugfix] Fix FP8 MoE EP Weight Loading for ModelOpt Llama4](../sources/prs/vllm/PR-32886.md), [[Spec Decode] Unified Parallel Drafting](../sources/prs/vllm/PR-32887.md), [[ROCm][perf] Shuffle KV cache to use paged_attention_common](../sources/prs/vllm/PR-32914.md), [[NVIDIA] [feat] Integrate flashinfer Trtllmgen bf16 moe](../sources/prs/vllm/PR-32954.md), [Support compress-tensors with nvfp4 or fp8 weights and modelopt with nvfp4 weights on Turing](../sources/prs/vllm/PR-33076.md), [[Attention] Use `has_flashinfer` helper](../sources/prs/vllm/PR-33177.md), [[Bugfix] Disable TRTLLM attention when KV transfer is enabled](../sources/prs/vllm/PR-33192.md), [[Bugfix] Register fp8 cutlass_group_gemm as supported for only SM90+SM100](../sources/prs/vllm/PR-33285.md), [[PERF] Change GDN Attention State Layout from [N, HV, K, V] to [N, HV, V, K]](../sources/prs/vllm/PR-33291.md), [[Kernel] Support Flashinfer trtllm fused MoE non gated FP8 & NVFP4](../sources/prs/vllm/PR-33506.md), [[Perf] Disable clean_logits in deepgemm fp8_mqa_logits kernel](../sources/prs/vllm/PR-33568.md), [[Bugfix] Fix sparse MLA metadata building](../sources/prs/vllm/PR-33579.md), [[Bugfix] fix DeepSeek R1 with CUTLASS MLA Broken on B200](../sources/prs/vllm/PR-33637.md), [enable skipping of SW attention layers when using FP8 KV cache](../sources/prs/vllm/PR-33695.md), [[Bugfix] Fix DSV3.2 NVFP4](../sources/prs/vllm/PR-33932.md), [Adding support to Sarvam's MoE models](../sources/prs/vllm/PR-33942.md), [[Bugfix] Relax TRTLLM KV cache contiguity assertion for cross-layer layout](../sources/prs/vllm/PR-34158.md), [[Bugfix] Fix DP Attention Padding in Dummy Run](../sources/prs/vllm/PR-34187.md), [[ModelBash][DSR1 NVFp4] Avoid Bf16 Bias Cast](../sources/prs/vllm/PR-34298.md), [[CPU][Perf] Accelerate Attention head for s390x using vector intrinsics](../sources/prs/vllm/PR-34434.md), [[Llama4,Quantization] Simplify and generalize logic for Q/K permutations in quantized self-attn layers ](../sources/prs/vllm/PR-34471.md), [[BUGFIX] Fix accuracy regression for NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4 with TP>1](../sources/prs/vllm/PR-34476.md), [[Model] Add NVFP4 quantization support for Step3.5-Flash](../sources/prs/vllm/PR-34478.md), [[Bugfix] Handle num_expert_group=None in flashinfer block-scale FP8 MoE](../sources/prs/vllm/PR-34494.md), [[BugFix] Add support for MTP num_speculative_tokens > 1 with sparse MLA](../sources/prs/vllm/PR-34552.md), [[Quantization] add humming quantization kernel](../sources/prs/vllm/PR-34556.md), [[Bugfix] Rescale NVFP4 weight scales to fix BF16 dequant underflow](../sources/prs/vllm/PR-34577.md), [[Update] Use FlashInfer fast_decode_plan directly instead of replication](../sources/prs/vllm/PR-34687.md), [[Bugfix] Fix MLA attention crash with AWQ/GPTQ quantized models](../sources/prs/vllm/PR-34695.md), [[torch.compile] Turn on silu+fp4 quant fusion by default for O1+](../sources/prs/vllm/PR-34718.md), [[Bugfix] Fix NVFP4 TRTLLM MoE non-gated support; add gsm8k for Nemotron-3-Nano FP8+NVFP4](../sources/prs/vllm/PR-34725.md), [[Attention] Use FA4 for MLA prefill](../sources/prs/vllm/PR-34732.md), [[Bugfix] Fix GDN attention crash with mixed decode/spec-decode batches](../sources/prs/vllm/PR-34871.md), [[Model Bash][DSR1] Add selective dynamic shape marking for CustomOp](../sources/prs/vllm/PR-34900.md), [[Perf] Enable FlashInfer DeepGEMM swapAB on SM90 by default](../sources/prs/vllm/PR-34924.md), [add mixed precision support for modelopt](../sources/prs/vllm/PR-35047.md), [Integrate flashinfer mm_mxfp8 in ModelOpt MXFP8](../sources/prs/vllm/PR-35053.md), [[Bug][DSV3.2] Always prepare metadata for DeepGEMM Sparse Attention](../sources/prs/vllm/PR-35075.md), [[BUGFIX][Qwen3.5] Hardcode `mlp.gate` as not quantizable ](../sources/prs/vllm/PR-35156.md), [[Linear Attention] fix bug for linear attention + prefix caching + reset_prefix_cache](../sources/prs/vllm/PR-35157.md), [[BUGFIX][Mamba][Qwen3.5] Zero freed SSM cache blocks on GPU](../sources/prs/vllm/PR-35219.md), [[Performance] Extract KV cache update op from flashinfer forward](../sources/prs/vllm/PR-35422.md), [[Bugfix] Fix KV Scale loading for MLA Models](../sources/prs/vllm/PR-35430.md), [[Quant][Feature] Support online MXFP8 quantization for MoE and dense models](../sources/prs/vllm/PR-35448.md), [[NVFP4] Support NVFP4 dense models from `modelopt` and `compressed-tensors` on AMD Instinct MI300, MI355X and Hopper through emulation](../sources/prs/vllm/PR-35733.md), [Fix routed experts capture for hybrid models (Mamba + Attention)](../sources/prs/vllm/PR-35744.md), [[MoE][Perf] Wrap DSV3 QKVAProj GEMM in custom op for torch.compile](../sources/prs/vllm/PR-35751.md), [[Mamba] Add stochastic rounding support](../sources/prs/vllm/PR-35753.md), [[Kernel] Add fused_sigmoid_gating_delta_rule_update kernel for Qwen3 Next](../sources/prs/vllm/PR-35777.md), [[Bugfix] Fix score layer quantization for sequence classification models - Qwen3 (VL) Reranker](../sources/prs/vllm/PR-35849.md), [[ROCm] Support MLA with nhead<16 and FP8 KV cache for TP=8 (Kimi K2.5/Linear)](../sources/prs/vllm/PR-35850.md), [[Perf] Support FP8 KV cache for Flashinfer MLA Sparse](../sources/prs/vllm/PR-35891.md), [[MoE] Move PF Methods to Folder](../sources/prs/vllm/PR-35927.md), [Add support for ModelOpt MXFP8 MoE models](../sources/prs/vllm/PR-35986.md), [[Bugfix] Fix passing of activation_type to trtllm fused MoE NVFP4 and FP8](../sources/prs/vllm/PR-36017.md), [[BugFix] Fallback from FA4->FA2 for Batch Invariance](../sources/prs/vllm/PR-36059.md), [[LMCache] Pass TP size in lookup for MLA multi-reader locking](../sources/prs/vllm/PR-36129.md), [[Bugfix] Disable FlashInfer TRTLLM BF16 path for non-gated MoE](../sources/prs/vllm/PR-36146.md), [[Mamba] Flashinfer selective_state_update](../sources/prs/vllm/PR-36162.md), [[Bugfix][MLA] Add logits size budget to sparse indexer prefill chunking](../sources/prs/vllm/PR-36178.md), [[mla] Support fused FP8/NVFP4 output quantization in MLA attention (#35792)](../sources/prs/vllm/PR-36205.md), [mla: don't update kv cache on dummy forwards](../sources/prs/vllm/PR-36282.md), [[Perf] Add TRTLLM FP8 MoE Modular Kernel](../sources/prs/vllm/PR-36307.md), [Disable cascade attention by default](../sources/prs/vllm/PR-36318.md), [Kimi k2.5 MLA based eagle3](../sources/prs/vllm/PR-36361.md), [feat(attention): extract KV-cache update from FlashAttentionDiffKV ba…](../sources/prs/vllm/PR-36466.md), [[Bugfix][Sparse MLA] report indexer CG support properly](../sources/prs/vllm/PR-36519.md), [[ROCm] Utilize persistent MLA kernel from AITER](../sources/prs/vllm/PR-36574.md), [[GDN] add a config for gdn kernel selection](../sources/prs/vllm/PR-36647.md), [[Misc][Attention] Clean up unused method in `CPU_ATTN`](../sources/prs/vllm/PR-36673.md), [[Bug] Fix FlashInfer MNNVL socket collisions under concurrent vLLM jobs](../sources/prs/vllm/PR-36674.md), [[ROCm][Perf] Allow MTP lens > 1 in Sparse MLA](../sources/prs/vllm/PR-36681.md), [fix(kv-cache): increase hybrid attention grouping threshold from 1.25 to 1.5](../sources/prs/vllm/PR-36684.md), [[ROCm] Attention selector reordering](../sources/prs/vllm/PR-36702.md), [[DSV3.2][MTP] Optimize Indexer MTP handling](../sources/prs/vllm/PR-36723.md), [[Bug][MoE] Fix TRTLLM NVFP4 Routing Kernel Precision](../sources/prs/vllm/PR-36725.md), [[ROCm] Fix KV copy methods and auto-select attention backend for ROCm](../sources/prs/vllm/PR-36845.md), [[ROCm] Validate block_size for explicitly selected attention backends](../sources/prs/vllm/PR-36846.md), [[Feat][Spec Decode] DFlash](../sources/prs/vllm/PR-36847.md), [[Bugfix] Fix FlashInfer GDN warmup ValueError on SM90 GPUs](../sources/prs/vllm/PR-36876.md), [[Feat][Bugfix] Enable additional dimension for Flashinfer MLA and fix routing dtype](../sources/prs/vllm/PR-36931.md), [[Bugfix] Disable cross-layer KV cache for MLA attention backends](../sources/prs/vllm/PR-37090.md), [[Benchmark] Improvements to attention benchmark script](../sources/prs/vllm/PR-37115.md), [[XPU] support MLA model on Intel GPU](../sources/prs/vllm/PR-37143.md), [Fix minimax m2.5 nvfp4 kv scales weight loading](../sources/prs/vllm/PR-37214.md), [[ROCM][Bugfix] Use correct stride in cp_mha_gather_cache_kernel for hybrid model (#37228)](../sources/prs/vllm/PR-37228.md), [[Bugfix] Expand quantization method support in perf metrics](../sources/prs/vllm/PR-37231.md), [[Attention] Support distinguishing between short extends and decodes](../sources/prs/vllm/PR-37303.md), [[Bugfix] Fix EP weight filter breaking EPLB and NVFP4 accuracy](../sources/prs/vllm/PR-37322.md), [[Model Runner V2] fix draft attention metadata generation](../sources/prs/vllm/PR-37364.md), [[Bugfix] Remove assertion for NVFP4 scale dynamic range](../sources/prs/vllm/PR-37465.md), [[BugFix] Allow qk_nope_head_dim=192 in FlashInfer MLA backend checks](../sources/prs/vllm/PR-37475.md), [[Bugfix] Fix marlin nvfp4 rescaling](../sources/prs/vllm/PR-37502.md), [Fix KV Offloading + MLA AssertionError by using num_kv_heads=1 in cpu…](../sources/prs/vllm/PR-37536.md), [[Performance] Remove unnecessary zero-fill of MLA decode output tensor in Aiter backend](../sources/prs/vllm/PR-37539.md), [[Bugfix][ROCm] Fix lru_cache on paged_mqa_logits_module](../sources/prs/vllm/PR-37547.md), [[Bugfix] Disable --calculate-kv-scales for hybrid GDN/Mamba+Attention…](../sources/prs/vllm/PR-37565.md), [[Bugfix] Disable monolithic TRTLLM MoE for Renormalize routing (#37591)](../sources/prs/vllm/PR-37605.md), [[ROCm][Bugfix] fix cache block size mismatch for aiter unified attention](../sources/prs/vllm/PR-37606.md), [[Perf] Use torch compile to fuse pack topk in trtllm moe](../sources/prs/vllm/PR-37695.md), [[Bug] Fix fp8 deepgemm batch invariant](../sources/prs/vllm/PR-37718.md), [[Test] Only Run MLA model when user explicitly set for batch invariance](../sources/prs/vllm/PR-37719.md), [[XPU] add gptq(int4) support](../sources/prs/vllm/PR-37844.md), [[Feature] Support per-draft-model MoE backend via `--speculative-config`](../sources/prs/vllm/PR-37880.md), [[ROCm][perf] fix Aiter sparse MLA with MTP>1](../sources/prs/vllm/PR-37887.md), [[Bugfix] Fix DeepGemm E8M0 accuracy degradation for Qwen3.5 FP8 on Blackwell](../sources/prs/vllm/PR-38083.md), [Fix NaN from stale FP4 scale padding in create_fp4_scale_tensor](../sources/prs/vllm/PR-38148.md), [[Model Runner V2] Rebuild attention metadata before eagle decode full…](../sources/prs/vllm/PR-38311.md), [[MoE] Add RoutingMethodType.Simulated to TRT-LLM FP8/NVFP4 kernel allowlists](../sources/prs/vllm/PR-38329.md), [[GDN] Eliminate GPU->CPU sync in prepare_chunk_indices during prefill](../sources/prs/vllm/PR-38361.md), [[QeRL] Fix online quantized reloading](../sources/prs/vllm/PR-38442.md), [[XPU] Fix spec-decode UTs under tests/v1/spec_decode](../sources/prs/vllm/PR-38491.md), [[Bugfix][MLA] Change default SM100 MLA prefill backend back to TRT-LLM](../sources/prs/vllm/PR-38562.md), [[ROCm] Fix aiter persistent mode mla with q/o nhead<16 for kimi-k2.5 tp8](../sources/prs/vllm/PR-38615.md), [Fix MLA runs when use_inductor_graph_partition=True](../sources/prs/vllm/PR-38631.md), [[Bugfix] Fix AWQ models batch invariance issues](../sources/prs/vllm/PR-38670.md), [[XPU] add xpu backend implementation of mxfp8 quant](../sources/prs/vllm/PR-38682.md), [[Bugfix] Restrict TRTLLM attention to SM100, fixing GB300 (SM103) hang](../sources/prs/vllm/PR-38730.md), [[Bugfix] Fix test mocks after SM100 restriction in #38730](../sources/prs/vllm/PR-38791.md), [[LMCache][MP] optimize save when mla enabled](../sources/prs/vllm/PR-38810.md), [[FlashAttention] Symlink FA4 instead of copying when using `VLLM_FLASH_ATTN_SRC_DIR`](../sources/prs/vllm/PR-38814.md), [[Quant] add CompressedTensorsW8A8Mxfp8 for linear and MoE layers](../sources/prs/vllm/PR-38815.md), [[Attention][MLA] Re-enable FA4 as default MLA prefill backend](../sources/prs/vllm/PR-38819.md), [[Bugfix] Fix NVFP4+MTP crash: force unquantized mtp.fc for Qwen3.5](../sources/prs/vllm/PR-38832.md), [[Attention] relax the head dim 512 and paged kv for sm90+FA4](../sources/prs/vllm/PR-38835.md), [[Bugfix] Re-enable Renormalize routing for TRT-LLM MoE experts](../sources/prs/vllm/PR-38859.md), [[Gemma4] Enable Fast Prefill Optimization](../sources/prs/vllm/PR-38879.md), [[MoE Refactor] Split up compressed_tensors_moe.py](../sources/prs/vllm/PR-38960.md), [[Perf][GDN] Align TMA usage with upstream FLA](../sources/prs/vllm/PR-38981.md), [[Bug] Fix routing bias dtype for trtllm per-block fp8 moe](../sources/prs/vllm/PR-38989.md), [[Bugfix][MoE] Fix 6-8% decode regression: prefer multi-stream shared expert overlap](../sources/prs/vllm/PR-38990.md), [[Perf] Change Trtllm fp8 MoE to use Shuffled Weights and BlockMajorK Layout](../sources/prs/vllm/PR-38993.md), [[Quantization] - Layerwise reloading of Attention/KV quantized models](../sources/prs/vllm/PR-38995.md), [[Bugfix] Fix FlashInfer crash with kv_cache_dtype_skip_layers](../sources/prs/vllm/PR-39002.md), [[Gemma4] Support quantized MoE ](../sources/prs/vllm/PR-39045.md), [[Bug] Fix Trtllm Fp8 MoE Weight Shuffle Memory Fragamentation](../sources/prs/vllm/PR-39054.md), [[ROCm] Align AiterFlashAttentionImpl attn_type check with backend](../sources/prs/vllm/PR-39119.md), [[Refactor] Move NVFP4 GEMM management into NvFp4LinearKernel](../sources/prs/vllm/PR-39129.md), [[Refactor] Move MXFP8 GEMM management into MxFp8LinearKernel](../sources/prs/vllm/PR-39205.md), [[Bug] Fix rocm sparse attn indexer issue](../sources/prs/vllm/PR-39225.md), [[Bugfix] FlashInfer MXINT4 MoE crashes, missing do_finalize](../sources/prs/vllm/PR-39315.md), [[Feature] Batch invariant nvfp4 linear support](../sources/prs/vllm/PR-39322.md), [[Model Runner V2] Fix flex attention kv blocks calculation issue](../sources/prs/vllm/PR-39353.md), [[Bugfix][CT] Fix KV cache scale handling](../sources/prs/vllm/PR-39418.md), [[MLA] Optimize mla indexer prepare uniform decode for MTP > 1](../sources/prs/vllm/PR-39458.md), [[Kernel] Support TRTLLM GEN NVFP4 MoE for non-512-aligned hidden dims via weight padding](../sources/prs/vllm/PR-39510.md), [[Bugfix] Fix tensor shape mismatch in sparse attention with speculative decoding](../sources/prs/vllm/PR-39542.md), [[Mooncake] Fix mixed MLA+Eagle block-size validation](../sources/prs/vllm/PR-39596.md), [[XPU] properly handle q_descale on XPU as quant query input not supported](../sources/prs/vllm/PR-39676.md), [[Bugfix] Fix mismatch between global and local attention heads in tensor-parallel mode for param2moe model](../sources/prs/vllm/PR-39707.md), [[Bugfix] Reject non-nvfp4 dtypes when using the flashinfer_nvlink_one_sided all2all backend](../sources/prs/vllm/PR-39717.md), [[Bugfix][NIXL] Fix `_logical_to_kernel_block_ids` conversion for non-mamba models](../sources/prs/vllm/PR-39724.md), [add warning when FP8 KV cache misses prefill query quantization](../sources/prs/vllm/PR-39752.md), [[Bugfix] add support for 'num_attention_groups' in ModelArchConfigConvertorBase for Step3p5](../sources/prs/vllm/PR-39796.md), [[Bug] Fix batch invariance nvfp4 support](../sources/prs/vllm/PR-39820.md), [[Bugfix] Disable FlashInfer CUTLASS MoE on SM121 (DGX Spark)](../sources/prs/vllm/PR-39825.md), [[Core] Replace routing replay with device cache and async D2H pipeline](../sources/prs/vllm/PR-39917.md), [[Attention] use diff kv backend for mimo v2 flash](../sources/prs/vllm/PR-40045.md), [[Bugfix] Temporarily disable B200 fp4 MoE layer tests](../sources/prs/vllm/PR-40057.md), [Add nvfp4 kv cache support](../sources/prs/vllm/PR-40177.md), [Fix MoE backend selection for LoRA (unquantized MoE)](../sources/prs/vllm/PR-40273.md), [[Kernel][Helion] Optimize Helion config parsing latency](../sources/prs/vllm/PR-40850.md), [[DSV4] Add BF16 and MXFP8 A2A support for flashinfer a2a one sided](../sources/prs/vllm/PR-40960.md), [[Kernel][MoE] Support GELU on TRT-LLM NvFP4 fused MoE for Gemma4](../sources/prs/vllm/PR-41050.md), [[Bugfix][Hybrid][NemotronH] Fix mamba_cache_mode=all + speculative decoding crash](../sources/prs/vllm/PR-41233.md), [[ROCm][Quantization][3/N] Refactor quark_moe w4a4 w/ oracle](../sources/prs/vllm/PR-41436.md), [[Quantization] Rework quantization_config to use QuantKey and allow for activation override](../sources/prs/vllm/PR-41566.md), [[MXFP4] Support for linear layers + compressed-tensors integration](../sources/prs/vllm/PR-41664.md), [fix: remove unused norm for dpskv4](../sources/prs/vllm/PR-41710.md), [[Spec Decode] Add Gemma4 MTP speculative decoding support](../sources/prs/vllm/PR-41745.md), [Add NVFP4 all-gather GEMM fusion for AsyncTP](../sources/prs/vllm/PR-41882.md), [[CPU] Add MXFP4 W4A16 MoE support](../sources/prs/vllm/PR-41922.md), [[Bugfix] Add swiglu limits to deepgemm fp8 methods](../sources/prs/vllm/PR-41986.md), [[Bugfix] Fix TRTLLM ragged MLA prefill workspace warmup](../sources/prs/vllm/PR-42112.md), [[LoRA] Support 2D and 3D MoE LoRA adapter at the same time](../sources/prs/vllm/PR-42242.md), [[Bugfix] mamba: run single-token extends as decodes](../sources/prs/vllm/PR-42430.md), [Refactor AWQ Marlin MoE onto modular WNA16 oracle](../sources/prs/vllm/PR-42483.md), [[UX] Add a persistent cache for FlashInfer autotuning](../sources/prs/vllm/PR-42537.md), [[Bugfix] fix swiglu limit issue for humming backend + deepseek v4](../sources/prs/vllm/PR-42541.md), [[CPU] Add fused GDN support for AMX CPU platform](../sources/prs/vllm/PR-42707.md), [Fix Weight loading for Qwen3.5-MTP and Qwen3-VL using runai_streamer](../sources/prs/vllm/PR-42716.md), [[CPU] Specify required KV cache layout for CPU attention backend](../sources/prs/vllm/PR-42740.md), [[ROCm] [Bugfix] Fix DeepSeek V4 Functionality and Accuracy](../sources/prs/vllm/PR-42810.md), [[Perf] Add do_not_specialize in fused FP8 RoPE kernel](../sources/prs/vllm/PR-42849.md), [[Model Refactoring] Migrate DeepSeek V4 to vllm/models/ [1/N] ](../sources/prs/vllm/PR-43004.md), [[XPU] update xpu graph usage](../sources/prs/vllm/PR-43043.md), [[CI failure] Temporarily disable using persistent cache for flashinfer autotune](../sources/prs/vllm/PR-43119.md), [FlashAttention SM100 MLA TopK Sparse Forward](../wiki/kernels/flash-attention-sm100-mla-topk.md), [TensorRT-LLM Blackwell FP4 DSA Indexer](../wiki/kernels/tensorrt-llm-blackwell-indexer.md) | -| `tilelang` | | [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [FlashInfer MLSys 2026 - Track C: Gated Delta Net](../sources/contests/flashinfer-mlsys26/track-c-gated-delta-net.md), [Tilelang sparse decode fwd for dsv32 mi355](../sources/prs/sglang/PR-18488.md), [[AMD] Tilelang sparse fwd for dsv32 mi355/mi300](../sources/prs/sglang/PR-19945.md), [[AMD] Enable FP8 KV cache and FP8 attention kernel for NSA on MI300/MI355 with TileLang backend](../sources/prs/sglang/PR-21511.md), [Amd/deepseek v4 rebase main 0509](../sources/prs/sglang/PR-24933.md), [[Refactor] Rename NSA → DSA: user-facing aliases, file/class/import rename](../sources/prs/sglang/PR-25821.md), [[WIP] support more dtypes for tcgen05](../sources/prs/tilelang/PR-1229.md), [[Enhancement] add more dtype and fix mma.ws for fp16 for tcgen05](../sources/prs/tilelang/PR-1327.md), [[Feat] profiler support cudagraph backend](../sources/prs/tilelang/PR-1658.md), [[Feature] Support `cp.reduce.async.bulk.tensor`](../sources/prs/tilelang/PR-1667.md), [Add swizzle layout detection and automatic merging for layout conflicts](../sources/prs/tilelang/PR-1736.md), [[Feature] Support tcgen5mma lowering for `.kind::i8`](../sources/prs/tilelang/PR-1764.md), [fix(intrinsics): add missing _legalize_to_buffer_region in SM70 emitter](../sources/prs/tilelang/PR-1786.md), [[BugFix] Fix Hopper TMA lowering without warp specialization](../sources/prs/tilelang/PR-1840.md), [[CUDA] Support tcgen5mma gemm ts](../sources/prs/tilelang/PR-1866.md), [[Feature] Support cluster launch, query, synchronization and barrier operations](../sources/prs/tilelang/PR-1874.md), [[Feature] 2-SM support for TMA, TMEM and TCGEN5MMA on Blackwell](../sources/prs/tilelang/PR-1882.md), [[Feature] Add T.copy_cluster to support TMA multicast and SM-to-SM cluster copy](../sources/prs/tilelang/PR-1908.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [[Feature] Block-scaled GEMM support for MXFP8 on Blackwell](../sources/prs/tilelang/PR-1945.md), [[Bugfix] Fix CuTeDSL autotune cache invalid ELF header (#1967)](../sources/prs/tilelang/PR-1972.md), [[Feature] Support TMA store in T.tma_copy()](../sources/prs/tilelang/PR-1981.md), [[Transform] Add InjectTcgen05Fence pass](../sources/prs/tilelang/PR-2003.md), [[Backend] Refactor gemm_sp](../sources/prs/tilelang/PR-2048.md), [[CUDA] Support int4 `T.gemm`](../sources/prs/tilelang/PR-2063.md), [[CUDA] Improve int4 GEMM lowering and packed codegen support](../sources/prs/tilelang/PR-2073.md), [[TMA] Support FP4 TensorMap TMA copies](../sources/prs/tilelang/PR-2107.md), [feat: auto-vectorize bf16/fp16 reduce with packed add2 intrinsics](../sources/prs/tilelang/PR-2112.md), [[CUDA][TMA] Add TMA tile::gather4 / tile::scatter4 support](../sources/prs/tilelang/PR-2129.md), [[codex] Split GEMM implementations by backend](../sources/prs/tilelang/PR-2153.md), [[TIR][IR] Update to use tirx](../sources/prs/tilelang/PR-2216.md), [[Python] Drop Python 3.9 support](../sources/prs/tilelang/PR-2218.md) | -| `triton` | [Triton on Blackwell](../wiki/languages/triton-blackwell.md) | [Gated Delta Networks](../sources/blogs/gated-delta-net.md), [Native Sparse Attention (NSA)](../sources/blogs/nsa.md), [FlashInfer MLSys 2026 - Track A: Fused MoE FP8](../sources/contests/flashinfer-mlsys26/track-a-fused-moe.md), [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [FlashInfer MLSys 2026 - Track C: Gated Delta Net](../sources/contests/flashinfer-mlsys26/track-c-gated-delta-net.md), [Triton 3.6.0 Release Notes — Blackwell (SM100) Lowering](../sources/docs/triton-3.6-blackwell.md), [[None][fix] impl fused triton kernel for e8m0 resmooth to reduce memory footprint](../sources/prs/TensorRT-LLM/PR-10327.md), [[#11694][feat] AutoDeploy: Improve the piecewise CG memory usage](../sources/prs/TensorRT-LLM/PR-11993.md), [[https://nvbugs/5983390][perf] Kernel fusions in _gather_k_cache_for_chunk of Indexer in DSA](../sources/prs/TensorRT-LLM/PR-12322.md), [[https://nvbugs/5983390][perf] Multiple host perf optimizations for DSA part](../sources/prs/TensorRT-LLM/PR-12581.md), [[None][feat] Add triton paged attention for AutoDeploy](../sources/prs/TensorRT-LLM/PR-12642.md), [[#12784][feat] AutoDeploy: Optimize DeepSeek-R1 model performance](../sources/prs/TensorRT-LLM/PR-12946.md), [[#13580][fix] AutoDeploy: Support Gemma3n/4 E2B variants](../sources/prs/TensorRT-LLM/PR-13630.md), [[None][feat] Keep DSv4 o_a_proj as FP8, and port vLLM's fused_inv_rope_fp8_quant](../sources/prs/TensorRT-LLM/PR-13938.md), [[https://nvbugs/6152892][fix] Fix Triton MOE memory free when no swizzling enabled](../sources/prs/TensorRT-LLM/PR-14069.md), [[None][feat] Add chunked prefill support for Gemma4 (text + vision multimodal)](../sources/prs/TensorRT-LLM/PR-14134.md), [[None][fix] Add SPDX Apache-2.0 headers to auto_deploy test files](../sources/prs/TensorRT-LLM/PR-14193.md), [[None][fix] Update the OSS headers in derived FLA ops and AD modeling code](../sources/prs/TensorRT-LLM/PR-14281.md), [[None][chore] Update Claude Code agents and skills](../sources/prs/TensorRT-LLM/PR-14344.md), [feat: ragged tensor padding kernel for blackwell kernel alignment](../sources/prs/flashinfer/PR-1025.md), [[nvidia] initial support for blackwell kernels](../sources/prs/flashinfer/PR-1039.md), [benchmark: trtllm-gen mha with sink, add benchmark args](../sources/prs/flashinfer/PR-1415.md), [refactor: update fa3 codebase and fix hopper unittest [part 1]](../sources/prs/flashinfer/PR-2111.md), [Selective State Update kernel (mamba)](../sources/prs/flashinfer/PR-2301.md), [MTP for mamba ](../sources/prs/flashinfer/PR-2444.md), [misc: point triton blackwell-ptxas to local cuda ptxas](../sources/prs/flashinfer/PR-2543.md), [Mamba SSU: better automatic kernel selection + algorithm selection optionally exposed to the user.](../sources/prs/flashinfer/PR-2591.md), [int16 Block-Scaled State and Stochastic Rounding for SSU (mamba)](../sources/prs/flashinfer/PR-2645.md), [Add varlen and speculative decoding support to selective state update](../sources/prs/flashinfer/PR-2700.md), [Mamba2 SSD Combined Forward Pass (Blackwell CuTe DSL Kernel)](../sources/prs/flashinfer/PR-2709.md), [fix(sm12x): fix micro-kernel workspace sizing when routed_rows > num_local_experts](../sources/prs/flashinfer/PR-3191.md), [checkpointing_ssu kernel: fused replay + conditional state-write for Mamba2](../sources/prs/flashinfer/PR-3324.md), [SM-constraint-GEMM by triton persistent kernel](../sources/prs/flashinfer/PR-982.md), [Triton `rms_norm` kernels](../sources/prs/flashinfer/PR-983.md), [[inductor] Fix profiler tests with latest Triton](../sources/prs/pytorch/PR-149059.md), [[inductor][triton 3.3] Fix cpp_wrapper w/ TMA in triton 3.3](../sources/prs/pytorch/PR-149993.md), [Fix uint view copy (#151598)](../sources/prs/pytorch/PR-154121.md), [[user triton] AOT inductor support for device-side TMA](../sources/prs/pytorch/PR-157241.md), [[release] Triton pin update to 3.4](../sources/prs/pytorch/PR-157752.md), [[cherry-pick][inductor][triton] Update HAS_WARP_SPEC to check triton.Config params. Update Triton Hash to top of release/3.4.x stack](../sources/prs/pytorch/PR-158646.md), [[Inductor][Intel GPU] Save `threads_per_warp` from tirton compiled kernel for launching kernel correctly in cpp wrapper.](../sources/prs/pytorch/PR-163388.md), [[2.9 cherry pick][triton] update 3.5 pin to bbb06c0334a6772b92d24bde54956e675c8c6604 (#163382)](../sources/prs/pytorch/PR-163583.md), [[AARCH64][CD][CUDA13][Triton][PTXAS] Turn on BUILD_BUNDLE_PTXAS=1 ](../sources/prs/pytorch/PR-164236.md), [[Minor][Inductor] move some combo kernel log from warning to debug](../sources/prs/pytorch/PR-167020.md), [[RELEASE 2.10] Release only changes](../sources/prs/pytorch/PR-170112.md), [[ROCm] Enable shared memory based pruning for Triton configs](../sources/prs/pytorch/PR-170190.md), [[Inductor] Fix constants handling for Triton constexpr (triton#8248)](../sources/prs/pytorch/PR-171129.md), [[RELEASE 2.11] Release only changes](../sources/prs/pytorch/PR-175091.md), [[inductor] avoid multi-stage for mix-order-red by default (#176228)](../sources/prs/pytorch/PR-176495.md), [[release 2.12] Apply Release only changes to 2.12 branch](../sources/prs/pytorch/PR-180470.md), [Add support for bf16 x bf16 cutlass fused MoE](../sources/prs/sglang/PR-10275.md), [support qwen3_next blackwell](../sources/prs/sglang/PR-10403.md), [Fix bias handling in TritonMoeQuantInfo within quantization/mxfp4.py](../sources/prs/sglang/PR-10579.md), [Fix MTP MoE weight loading with NVFP4 target model.](../sources/prs/sglang/PR-10758.md), [Fix DSR1 accuracy for flashinfer_trtllm MoE with FP8 quantization](../sources/prs/sglang/PR-11081.md), [[AMD] Clean up vllm dependencies in moe_runner/triton.py](../sources/prs/sglang/PR-11349.md), [Support shared experts overlap in cutlass moe](../sources/prs/sglang/PR-11611.md), [[Ascend] qwen optimization](../sources/prs/sglang/PR-12078.md), [Enable Flashinfer TRTLLM-GEN-MoE FP8 blockwise kernel for Qwen3-Next on Blackwell](../sources/prs/sglang/PR-12543.md), [[Bugfix] Fix illegal memory access](../sources/prs/sglang/PR-12758.md), [Support internvl on Blackwell (which doesn't support fa3): add `SingletonCache` support to Vision{Sdpa|Triton|Ascend}Attention](../sources/prs/sglang/PR-13151.md), [Fix nan in global scaling factor for large scale nvfp4 EP](../sources/prs/sglang/PR-13162.md), [[ROCM] Optimized deepseek-r1 fp8 model with + triton_gemm_a8w8 + batch_gemm_a8w8 + fused set_mla_kv_buffer kernel](../sources/prs/sglang/PR-13617.md), [[Feat][NVFP4] Enable NVFP4 MoE for Qwen series models (eg. Qwen3-Next) #13761](../sources/prs/sglang/PR-13761.md), [Support fp4 fp8 non gated moe](../sources/prs/sglang/PR-13794.md), [[NVIDIA] Enable TRTLLM BF16 MoE on Blackwell GPUs](../sources/prs/sglang/PR-13798.md), [[BugFix] fix outplace_fused_experts missing is_gated](../sources/prs/sglang/PR-13864.md), [Feat: GLM-4.6 supports shared experts fusion](../sources/prs/sglang/PR-13873.md), [[LoRA][III] Add LoRA support for MoE layers and enable TP](../sources/prs/sglang/PR-14105.md), [Apply new moe wna16 marlin gemm](../sources/prs/sglang/PR-14125.md), [Apply new moe align block size kernel](../sources/prs/sglang/PR-14134.md), [Add Mistral Large 3 support.](../sources/prs/sglang/PR-14213.md), [[Fix] add block size logic for sm120 smem size](../sources/prs/sglang/PR-14311.md), [[FIX] trtllm-moe-fp4-renorm for Qwen series models](../sources/prs/sglang/PR-14350.md), [Apply back moe_sum_reduce for fused_marlin_moe](../sources/prs/sglang/PR-14829.md), [Mistral Large 3 NVFP4 TRTLLM MoE support](../sources/prs/sglang/PR-15049.md), [Optimize FP8 MLA KV cache writes with Triton kernel](../sources/prs/sglang/PR-15522.md), [MoE: Skip SiLU/GELU activation for masked experts](../sources/prs/sglang/PR-15539.md), [Update flashinfer to 0.6.1](../sources/prs/sglang/PR-15551.md), [Add SwapAB Optimization for triton fused_moe_kernel on SM90.](../sources/prs/sglang/PR-15712.md), [ Add tuned triton==3.5.1 h200 tp2, tp4 for qwen 3 next](../sources/prs/sglang/PR-15948.md), [[Performance] Force split_k=1 for MXFP4 Triton kernels on Hopper](../sources/prs/sglang/PR-16014.md), [optimize get_topk_ragged by fusing get k and k_scale triton kernel](../sources/prs/sglang/PR-16043.md), [fix layer intermediate size](../sources/prs/sglang/PR-16084.md), [[NemotronH] Add latent MoE support](../sources/prs/sglang/PR-16227.md), [[Rework] Add SwapAB Optimization for triton fused_moe_kernel on SM90.](../sources/prs/sglang/PR-16723.md), [[Fix] `flashinfer_trtllm` `intermediate_size` assertion with Qwen3 + TP=8](../sources/prs/sglang/PR-16824.md), [Support mxint4 flashinfer_trtllm moe gemm](../sources/prs/sglang/PR-16892.md), [[GLM 4.7] Add RTX 6000 Pro aka sm120](../sources/prs/sglang/PR-17235.md), [Add mxfp8 support for online quantization, Triton dense linear, and CUTLASS MoE](../sources/prs/sglang/PR-17449.md), [[AMD] Update aiter to v0.1.10.post2](../sources/prs/sglang/PR-18423.md), [[sglang-miles] True on-policy training support for FSDP2](../sources/prs/sglang/PR-18639.md), [Adjust padding size to improve triton_kernels moe performance](../sources/prs/sglang/PR-19174.md), [Fix/nemotron mtp quantaized](../sources/prs/sglang/PR-19433.md), [[FlashInfer v0.6.4] [RL] Integrate FlashInfer mxfp8 gemm, MoE, and routed MoE](../sources/prs/sglang/PR-19537.md), [[diffusion][llm] macOS support](../sources/prs/sglang/PR-19549.md), [[Feature] NVFP4 Marlin fallback for non-Blackwell GPUs (SM75+)](../sources/prs/sglang/PR-19652.md), [Support `triton_kernels` for GPT-OSS on SM120](../sources/prs/sglang/PR-19718.md), [Fix SM120 `triton_kernels` MXFP4 `block_k` for GPT-OSS](../sources/prs/sglang/PR-20040.md), [[Benchmark] use flashinfer bench_gpu_time instead of triton do_bench](../sources/prs/sglang/PR-20305.md), [Support Triton MLA FP8 KV cache](../sources/prs/sglang/PR-20479.md), [Add SGLang CUDA crash API logging inspired by FlashInfer](../sources/prs/sglang/PR-20910.md), [[Qwen3.5] Fuse split/reshape/cat ops in GDN projection with Triton kernel](../sources/prs/sglang/PR-21019.md), [[refactor] Clean up duplicate flashinfer trtllm moe code](../sources/prs/sglang/PR-21233.md), [Refactor JIT kernel CI to use run_suite.py registration system](../sources/prs/sglang/PR-21239.md), [[RL] Support mxfp8 DeepSeek V3](../sources/prs/sglang/PR-21280.md), [[misc] clean up kernel API](../sources/prs/sglang/PR-21325.md), [Change default mm-attention backend from triton_attn to fa4](../sources/prs/sglang/PR-21595.md), [[XPU] Enable qwen3.5 on XPU](../sources/prs/sglang/PR-21668.md), [[Fix] Fall back to triton MOE for GPT-OSS on Blackwell with driver >= 595](../sources/prs/sglang/PR-21780.md), [fix pcg torch dynamo recompile in mxfp8 Triton path](../sources/prs/sglang/PR-21888.md), [[nvidia] Gemma4 nvfp4 fix](../sources/prs/sglang/PR-22079.md), [[Lora] Lora quat info re-factor and support deepseekv3 mla lora](../sources/prs/sglang/PR-22323.md), [diffusion: add HunyuanVideo GroupNorm+SiLU fast path](../sources/prs/sglang/PR-22814.md), [[codex] diffusion: enable group norm silu fuse by default](../sources/prs/sglang/PR-23148.md), [Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-23686.md), [Optimize large GroupNorm SiLU apply](../sources/prs/sglang/PR-23938.md), [[diffusion] Fuse LTX2 split rotary embedding](../sources/prs/sglang/PR-24411.md), [Port MXFP4 Marlin MoE support to JIT kernel path](../sources/prs/sglang/PR-24490.md), [Fix performance regression on Deepseek V3 on `moe-runner-backend=triton` on SM90](../sources/prs/sglang/PR-24562.md), [[Codex] Opt Mistral Large performace ](../sources/prs/sglang/PR-24611.md), [[rebase]Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-24986.md), [[MUSA][Diffusion] Improve wan model inference speed using torch.compile](../sources/prs/sglang/PR-25256.md), [[Gemma4]: Fix FP8 Triton scale layout](../sources/prs/sglang/PR-25286.md), [[Intel GPU] Enable DeepSeek V4 Inference on XPU](../sources/prs/sglang/PR-25336.md), [Update logging for inplace setting in MoE layer](../sources/prs/sglang/PR-25499.md), [Fix logging for inplace setting in the flashInfer-trtllm backend](../sources/prs/sglang/PR-25522.md), [amd/deepseek_v4 27/N [fix] Reduce Triton autotune configs for faster first-time server launch](../sources/prs/sglang/PR-25554.md), [Add DeepSeekV4 fused MoE Triton autotune support](../sources/prs/sglang/PR-25569.md), [Use triton_attn as default vision attention on B300 (SM103)](../sources/prs/sglang/PR-25570.md), [[diffusion] Fix GLM-Image /v1/images/edits support](../sources/prs/sglang/PR-25697.md), [[Refactor] Rename NSA → DSA: user-facing aliases, file/class/import rename](../sources/prs/sglang/PR-25821.md), [[Test] Stage-a sanity kits; consolidate core/ + models_e2e/ tests](../sources/prs/sglang/PR-25831.md), [Feature DeepSeek V3/R1 INT8 Quantization (block-wise)](../sources/prs/sglang/PR-3730.md), [[Feature] DeepSeek V3/R1 INT8 Quantization (channel-wise) ](../sources/prs/sglang/PR-3888.md), [Add fast decode plan for flashinfer mla](../sources/prs/sglang/PR-3987.md), [[Revision] Add fast decode plan for flashinfer mla ](../sources/prs/sglang/PR-4012.md), [Accelerate FP8 CUDA Kernel by 20-28%](../sources/prs/sglang/PR-4215.md), [Feat/support encoder model (like bert)](../sources/prs/sglang/PR-4887.md), [Add DeepSeek V3/R1 shared experts fusion](../sources/prs/sglang/PR-4918.md), [reduce moe_align_block_size_kernel small batch mode overhead](../sources/prs/sglang/PR-5086.md), [Support tuning moe for llama 4 model](../sources/prs/sglang/PR-6042.md), [feat: mtp support dp-attention](../sources/prs/sglang/PR-6081.md), [enable auto-round quantization model](../sources/prs/sglang/PR-6226.md), [fix: enable multi-GPU Triton fused MoE tuning](../sources/prs/sglang/PR-6295.md), [reduce torch.zeros overhead in moe align block size kernel](../sources/prs/sglang/PR-6369.md), [Refine pre_reorder_triton_kernel slightly to improve performance](../sources/prs/sglang/PR-6627.md), [[CPU] [BF16] Call fused_experts_cpu, weight_packed_linear and bmm_cpu kernel in DeepSeek model](../sources/prs/sglang/PR-6641.md), [Set `num_fused_shared_experts` as `num_shared_experts` when shared_experts fusion is not disabled](../sources/prs/sglang/PR-6736.md), [[CPU] add optimizations for INT8 and FP8 DeepSeek](../sources/prs/sglang/PR-6769.md), [[CPU] support the case where num_attention_heads or intermediate_size is not divisible by the TP size](../sources/prs/sglang/PR-6771.md), [[DeepseekR1-FP4] Add Support for nvidia/DeepSeekR1-FP4 model](../sources/prs/sglang/PR-6853.md), [Use deepgemm instead of triton for fused_qkv_a_proj_with_mqa](../sources/prs/sglang/PR-6890.md), [chore: upgrade flashinfer v0.2.6.post1 jit](../sources/prs/sglang/PR-6958.md), [Fuse routed scaling factor in deepseek](../sources/prs/sglang/PR-6970.md), [Update default settings for blackwell](../sources/prs/sglang/PR-7023.md), [Fix positional argument](../sources/prs/sglang/PR-7093.md), [Enable ModelOpt Llama4 fp8 checkpoint deployment in SGLang](../sources/prs/sglang/PR-7129.md), [Support new DeepGEMM](../sources/prs/sglang/PR-7172.md), [Fix grammar abort & Minor style fixes](../sources/prs/sglang/PR-7204.md), [FlashInfer NVFP4 MoE with EP & 2-stream shared expert](../sources/prs/sglang/PR-7327.md), [Fix MTP with Deepseek R1 Fp4](../sources/prs/sglang/PR-7376.md), [Fix torch compile run](../sources/prs/sglang/PR-7391.md), [Add fp4 quantize before all-gather for Flashinfer cutlass MoE DP (max throughput)](../sources/prs/sglang/PR-7667.md), [Integrate triton moe kernel](../sources/prs/sglang/PR-7689.md), [[feat] Support tp mode for DeepSeek-R1-W4AFP8](../sources/prs/sglang/PR-8118.md), [Support triton kernels v3.4.0 for fused_moe](../sources/prs/sglang/PR-8258.md), [[NVIDIA] Add Low Latency NVFP4 decode kernels from Flashinfer](../sources/prs/sglang/PR-8552.md), [fuse allreduce and residual_rmsnorm](../sources/prs/sglang/PR-8731.md), [[Perf] Auto enable best flashinfer mxfp4 kernel in b200](../sources/prs/sglang/PR-8898.md), [[fix] Fix mxfp4 triton MoE tp bug](../sources/prs/sglang/PR-9473.md), [Optimize moe_sum_reduce_kernel](../sources/prs/sglang/PR-9477.md), [Single Batch Overlap for MoE Models](../sources/prs/sglang/PR-9660.md), [[Kernel] add triton fused moe kernel for gptq/awq](../sources/prs/vllm/PR-12185.md), [[AMD][Quantization] Add TritonScaledMMLinearKernel since int8 is broken for AMD](../sources/prs/vllm/PR-12282.md), [[Attention] MLA decode optimizations](../sources/prs/vllm/PR-12528.md), [[Attention] Deepseek v3 MLA support with FP8 compute](../sources/prs/vllm/PR-12601.md), [[Attention] MLA with chunked prefill](../sources/prs/vllm/PR-12639.md), [[Perf] Mem align KV caches for CUDA devices (MLA perf improvement)](../sources/prs/vllm/PR-12676.md), [[core] Perf improvement for DSv3 on AMD GPUs](../sources/prs/vllm/PR-13718.md), [[Attention] MLA support for V1](../sources/prs/vllm/PR-13789.md), [[Attention] Flash MLA for V1](../sources/prs/vllm/PR-13867.md), [dynamic distpatch of fp8 kernels](../sources/prs/vllm/PR-14245.md), [[BugFix] MLA + V1, illegal memory access and accuracy issues](../sources/prs/vllm/PR-14253.md), [[BugFix][TritonMLA] Process weights after model loading for GGUF](../sources/prs/vllm/PR-14555.md), [[BugFix] Fix nightly MLA failure (FA2 + MLA chunked prefill, i.e. V1, producing bad results)](../sources/prs/vllm/PR-15492.md), [[ROCM][KERNEL] Paged attention for V1](../sources/prs/vllm/PR-15720.md), [Modularize fused experts and integrate PPLX kernels](../sources/prs/vllm/PR-15956.md), [[Kernel][Bugfix] Re-fuse triton moe weight application](../sources/prs/vllm/PR-16071.md), [Upstream Llama4 Support to Main](../sources/prs/vllm/PR-16113.md), [[Bug] [ROCm] Fix Llama 4 Enablement Bug on ROCm: V0 ROCmFlashAttentionImpl and Triton Fused MoE bugs](../sources/prs/vllm/PR-16198.md), [[Kernel] Support W8A8 channel-wise weights and per-token activations in triton fused_moe_kernel](../sources/prs/vllm/PR-16366.md), [[Perf]Optimize rotary_emb implementation to use Triton operator for improved inference performance](../sources/prs/vllm/PR-16457.md), [Enable PTPC FP8 for CompressedTensorsW8A8Fp8MoEMethod (triton fused_moe)](../sources/prs/vllm/PR-16537.md), [[torch.compile][ROCm] Fuse quantization onto attention using a torch.compile pass](../sources/prs/vllm/PR-16756.md), [[Kernel] Unified Triton kernel that doesn't distinguish between prefill + decode](../sources/prs/vllm/PR-16828.md), [[Bugfix] Triton FA function takes no keyword arguments](../sources/prs/vllm/PR-16902.md), [[Attention] MLA move o_proj q_proj into cuda-graph region](../sources/prs/vllm/PR-17484.md), [fix amd triton mla path](../sources/prs/vllm/PR-17871.md), [[Bugfix][ROCm] Use `chunked_prefill_paged_decode` as fallback for V1 attention on ROCm](../sources/prs/vllm/PR-18093.md), [[Kernel] Integrate CUTLASS MoE kernel with PPLX](../sources/prs/vllm/PR-18762.md), [[Kernel] Enable fp8 support for pplx and BatchedTritonExperts.](../sources/prs/vllm/PR-18864.md), [[Kernels] Add activation chunking logic to FusedMoEModularKernel](../sources/prs/vllm/PR-19168.md), [[Bugfix] Don't attempt to use triton if no driver is active](../sources/prs/vllm/PR-19561.md), [ [Feature] Integrate SM100 DeepGEMM support](../sources/prs/vllm/PR-20087.md), [[Kernel] Optimize Prefill Attention in Unified Triton Attention Kernel](../sources/prs/vllm/PR-20308.md), [[Misc] DP : Add ExpertTokensMetadata](../sources/prs/vllm/PR-20332.md), [[Perf] Use Triton instead of Torch for DeepGEMM Per Token Group Quant](../sources/prs/vllm/PR-20841.md), [[Kernel] DeepGemm MoE : Integrate triton permute / unpermute kernels ](../sources/prs/vllm/PR-20903.md), [[v1] Add Whisper model support (encoder-decoder)](../sources/prs/vllm/PR-21088.md), [[Attention] Clean up iRoPE in V1](../sources/prs/vllm/PR-21188.md), [[Kernel] Enable Hybrid Model Support in Triton Unified Attention Kernel](../sources/prs/vllm/PR-21197.md), [[Feature][Kernel]FusedMoE LoRA](../sources/prs/vllm/PR-21229.md), [[v1][attention] Support Hybrid Allocator + FlashInfer](../sources/prs/vllm/PR-21412.md), [[Bugfix] Add proper comparison for package versions](../sources/prs/vllm/PR-22314.md), [[BugFix] Fix triton compile error in `kernel_unified_attention_2/3d` caused by attention sinks](../sources/prs/vllm/PR-22368.md), [[gpt-oss] triton kernel mxfp4](../sources/prs/vllm/PR-22421.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv NVFP4-out Attention Kernel](../sources/prs/vllm/PR-22703.md), [fp8 kv cache support fix for torch.compile](../sources/prs/vllm/PR-22758.md), [[Misc] Add @tdoublep as a maintainer of hybrid model and Triton-attention related code](../sources/prs/vllm/PR-23122.md), [[ROCm][Aiter] Add triton fp8 bmm kernel for mla](../sources/prs/vllm/PR-23264.md), [[Bugfix] Fixing division by zero in triton_attn if query_heads/kv_heads > 16 ](../sources/prs/vllm/PR-23424.md), [[Feature] Add Hopper DeepGEMM E8M0 for DeepSeekV3.1 scale_fmt](../sources/prs/vllm/PR-23666.md), [[Feature] Support Decode Context Parallel (DCP) for MLA](../sources/prs/vllm/PR-23734.md), [[Bug] Fix Shape Validation for Fallback while Enabling E8M0 for DeepGEMM](../sources/prs/vllm/PR-26322.md), [Move query quantization to attention layer for Flashinfer & Triton.](../sources/prs/vllm/PR-26534.md), [[Attention] Use sparse prefill kernel for fp8 kv-cache in DeepSeek-v3.2](../sources/prs/vllm/PR-27532.md), [fix cross attention](../sources/prs/vllm/PR-28346.md), [[Model] Add support for openPangu moe model](../sources/prs/vllm/PR-28775.md), [[Feature] Batch invariant: Enable `TRITON_MLA` without prefix-caching](../sources/prs/vllm/PR-29125.md), [[Bugfix] Only use triton_kernels for MXFP4 on SM90 and SM100](../sources/prs/vllm/PR-29339.md), [[LoRA] Support Quantized Adapters](../sources/prs/vllm/PR-30286.md), [[ROCm][Quantization] GPT OSS Upstream MoE wmxfp4_afp8 with static scales](../sources/prs/vllm/PR-30357.md), [[Perf] Set split_k to 1 for triton_kernels](../sources/prs/vllm/PR-30528.md), [[Bugfix] Fix Triton FusedMoE LoRA](../sources/prs/vllm/PR-30585.md), [Triton Attention: Support cross-layers blocks](../sources/prs/vllm/PR-30687.md), [fused_moe_lora PDL improvements](../sources/prs/vllm/PR-30716.md), [[Bugfix] [Kernel] Triton attention kernels: mask out V blocks that fall outside sliding window](../sources/prs/vllm/PR-30887.md), [[Bugfix] Fix incorrect tiles creation for mm prefix triton attention](../sources/prs/vllm/PR-30974.md), [Use aiter triton fused_add_rmsnorm_pad for gpt-oss](../sources/prs/vllm/PR-30976.md), [[Bugfix][ROCm]Fix Qwen3-Next-80B-A3B-Thinking inference and optimize non-standard block size (544) support under rocm_atten](../sources/prs/vllm/PR-31380.md), [fixed mypy warnings for files vllm/v1/attention with TEMPORARY workaround](../sources/prs/vllm/PR-31465.md), [[FIX] Add NO_MUL activation support for modular kernel path](../sources/prs/vllm/PR-31528.md), [[LoRA]Disable linear LoRA kernel PDL](../sources/prs/vllm/PR-31777.md), [[1/N][Attention] Restructure attention: move files](../sources/prs/vllm/PR-31916.md), [[4/N][Attention] Move MLA common to model_executor](../sources/prs/vllm/PR-32060.md), [Add TMA support to fused_moe_lora kernel](../sources/prs/vllm/PR-32195.md), [[Bugfix][Attention] Explicitly report support for kv_cache_dtype bfloat16](../sources/prs/vllm/PR-32795.md), [Triton MLA perf fixes](../sources/prs/vllm/PR-33529.md), [[Kernel] Add FP8 KV cache support to Triton MLA decode attention](../sources/prs/vllm/PR-34597.md), [fix(mxfp4): return is_monolithic=False when LoRA is enabled for Triton backend](../sources/prs/vllm/PR-35382.md), [[XPU] Support block fp8 moe by fallback to TritonExpert on XPU](../sources/prs/vllm/PR-36458.md), [[Kernel] Fuse FP8 output quantization into merge_attn_states](../sources/prs/vllm/PR-36518.md), [[Bugfix] Warm up Triton autotuner for GDN layers during V1 profiling](../sources/prs/vllm/PR-36599.md), [[Bugfix] Fix KV scales inconsistency in fp8 MLA & FlashInfer kv_cache_dtype "auto" leading to gibberish](../sources/prs/vllm/PR-37054.md), [[MoE Refactor] Mxfp4 oracle rebased](../sources/prs/vllm/PR-37128.md), [[Misc] Add `float16` to `CacheDType`](../sources/prs/vllm/PR-37199.md), [[NIXL][BUG] Fix Triton heterogeneous TP](../sources/prs/vllm/PR-37940.md), [[Perf] triton bilinear_pos_embed kernel for ViT](../sources/prs/vllm/PR-37948.md), [[Perf] FP8 FlashInfer Attn for ViT](../sources/prs/vllm/PR-38065.md), [[Bugfix] Enable batch-invariant Triton matmul on all Ampere GPUs (SM 8x) ](../sources/prs/vllm/PR-38427.md), [[Attention Backend] TurboQuant: 2-bit KV cache compression with 4x capacity](../sources/prs/vllm/PR-38479.md), [[Kernels][MoE] Fix legacy_routing to use bitmatrix-based routing path](../sources/prs/vllm/PR-38504.md), [[Perf] Reduce H2D pageable memory copies](../sources/prs/vllm/PR-38794.md), [[MoE] Move GPT OSS Triton kernel experts into fused_moe/experts/](../sources/prs/vllm/PR-39007.md), [[XPU] Quick fix for TritonMLA to remove cuda hardcode](../sources/prs/vllm/PR-39088.md), [[MoE] Move cutlass moe to fused_moe/experts/](../sources/prs/vllm/PR-40574.md), [[Attention][TurboQuant] Share dequant buffers, eliminate float16_copy](../sources/prs/vllm/PR-40941.md), [[MoE] Move various experts classes to fused_moe/experts/](../sources/prs/vllm/PR-41979.md), [[feat] Add FP8 per-tensor Q scale support to Triton attention backend](../sources/prs/vllm/PR-42080.md), [[Perf] Wire silu_and_mul_per_block_quant into TritonFP8MoE (MiniMax-M2) ](../sources/prs/vllm/PR-42497.md), [[Kernel] Pack topk id/weights triton kernel](../sources/prs/vllm/PR-42527.md), [[Perf][MLA] Enable FULL cudagraph capture for TRITON_MLA decode](../sources/prs/vllm/PR-42885.md), [Fused MoE — FP8 Block-Scale Routing + Dual GEMM](../wiki/kernels/fused-moe.md), [Gated Delta Net — Linear Attention](../wiki/kernels/gated-delta-net.md), [Native Sparse Attention (NSA)](../wiki/kernels/nsa.md) | +| `mojo` | | [Modular: Matrix Multiplication on Blackwell](../sources/blogs/modular-blackwell-matmul.md) | +| `ptx` | [PTX Instructions for SM100](../wiki/languages/ptx-sm100.md) | [Twelve Attempts at an FP4 Kernel](../sources/blogs/amandeep-nvfp4-attempts.md), [simveit load_and_store](../sources/blogs/simveit-load-and-store.md), [tcgen05 for dummies](../sources/blogs/tcgen05-tutorial.md), [Tilus: A Tile-Level GPGPU Programming Language for Low-Precision Computation](../sources/blogs/tilus-nvidia.md), [Blackwell NVFP4 Kernel Hackathon Journey](../sources/blogs/yue-nvfp4-hackathon.md), [NVIDIA Blackwell Compatibility Guide](../sources/docs/blackwell-compatibility-guide.md), [PTX ISA 9.0 SM100 Instruction Reference](../sources/docs/nvidia-ptx-isa-sm100.md), [[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes](../sources/prs/DeepGEMM/PR-304.md), [Sync nv_dev with upstream #316 (Mega MoE optimizations & benchmarks)](../sources/prs/DeepGEMM/PR-328.md), [misc: point triton blackwell-ptxas to local cuda ptxas](../sources/prs/flashinfer/PR-2543.md), [[AARCH64][CD][CUDA13][Triton][PTXAS] Turn on BUILD_BUNDLE_PTXAS=1 ](../sources/prs/pytorch/PR-164236.md), [[release-only] Remove +ptx from cuda 13.0 builds](../sources/prs/pytorch/PR-175567.md), [Add swizzle layout detection and automatic merging for layout conflicts](../sources/prs/tilelang/PR-1736.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [[CUDA] Support int4 `T.gemm`](../sources/prs/tilelang/PR-2063.md), [[Bugfix] Enable `.shared::cta` in TMA copy paths only on CUDA 12.8+](../sources/prs/tilelang/PR-2087.md), [[TIR][IR] Update to use tirx](../sources/prs/tilelang/PR-2216.md), [Update launch_bounds_utils.h for correct compile on Multiple Cuda Arch - PTXAS out of range Warning](../sources/prs/vllm/PR-25843.md), [DeepGEMM — FP8 GEMM with Fine-Grained Scaling](../wiki/kernels/deepgemm.md), [NVFP4 Batched GEMV](../wiki/kernels/nvfp4-gemv.md) | +| `python` | | [NVFP4 GEMV and Improved NVFP4 GEMV](../sources/blogs/simon-nvfp4-gemv.md), [FlashInfer MLSys 2026 Track C: Gated Delta Net](../sources/contests/flashinfer-mlsys26/track-c-gated-delta-net.md), [GPU Mode NVFP4 Hackathon - Problem 1: Batched GEMV](../sources/contests/gpu-mode-nvfp4/problem-1-gemv.md), [GPU Mode NVFP4 Hackathon - Problem 2: NVFP4 GEMM](../sources/contests/gpu-mode-nvfp4/problem-2-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 3: Gated Dual GEMM](../sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md), [GPU Mode NVFP4 Hackathon - Problem 4: Grouped GEMM](../sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md), [cuTile Python DSL Reference](../sources/docs/cutile-python-dsl.md), [Performance: reducing the percentage of FFMA interleaving yields a sight performance gain, roughly 0.5%](../sources/prs/DeepGEMM/PR-42.md), [[None][feat] sm100 weight-only kernel](../sources/prs/TensorRT-LLM/PR-10190.md), [[TRTLLM-9798][feat] Change to use new DeepGEMM MQA sm100 kernel for MTP-3](../sources/prs/TensorRT-LLM/PR-10226.md), [[None][feat] MiniMax M2 support](../sources/prs/TensorRT-LLM/PR-10532.md), [[None][feat] Remove non flash attetnion style fmha_v2 kernel for hopper](../sources/prs/TensorRT-LLM/PR-11381.md), [[None][fix] Fix int4 awq for sm120/121](../sources/prs/TensorRT-LLM/PR-11561.md), [[None][fix] Fix SM120 issue for rms_norm with nvfp4_quant_fusion](../sources/prs/TensorRT-LLM/PR-11774.md), [[None][feat] GLM 5 support and DSA MTP fixes](../sources/prs/TensorRT-LLM/PR-11990.md), [[TRTLLM-11285][feat] Fuse indexer wk + weights_proj into single GEMM in TF32 for DS-V3.2](../sources/prs/TensorRT-LLM/PR-12055.md), [[https://nvbugs/5983390][fix] Remove redundant D2H sync to optimize perf](../sources/prs/TensorRT-LLM/PR-12445.md), [[#12634][feat] AutoDeploy: Support rank 256 MLA in flashinfer_mla](../sources/prs/TensorRT-LLM/PR-12519.md), [[https://nvbugs/5879577][fix] Fix KeyError in DeepSeekV3Lite FP8 MTP weight loading](../sources/prs/TensorRT-LLM/PR-12530.md), [[None][feat] Add bf16 trtllm-gen moe support through flashinfer.](../sources/prs/TensorRT-LLM/PR-12738.md), [[None][fix] Propagate init_load_balancer to DeepGemmFusedMoE in create_moe_backend](../sources/prs/TensorRT-LLM/PR-13207.md), [[TRTLLM-11127][feat] add W4A8_MXFP4_FP8 MoE unit test support](../sources/prs/TensorRT-LLM/PR-13401.md), [[TRTLLM-11285][perf] Force enable TF32 tensor cores for DSA indexer fused GEMM](../sources/prs/TensorRT-LLM/PR-13452.md), [[TRTLLM-12128][feat] enable SageAttention for Wan/FLUX (new commits)](../sources/prs/TensorRT-LLM/PR-13570.md), [[TRTLLM-12316][feat] Integrate FP4 indexer for DSv4](../sources/prs/TensorRT-LLM/PR-13575.md), [[None][feat] Enable EPLB for DeepSeek-V4](../sources/prs/TensorRT-LLM/PR-13595.md), [[None][feat] Add bf16 trtllm moe through flashinfer.](../sources/prs/TensorRT-LLM/PR-13689.md), [[None][fix] Use compressed lengths for DeepSeek-V4 indexer](../sources/prs/TensorRT-LLM/PR-13802.md), [[None][feat] Update FMHA cubins for head_dim 80](../sources/prs/TensorRT-LLM/PR-13808.md), [[TRTLLM-12503][feat] Parallel VAE independent scaling and fix arg passing](../sources/prs/TensorRT-LLM/PR-13873.md), [[None][feat] enable TRTLLM-Gen internal routing](../sources/prs/TensorRT-LLM/PR-13997.md), [[None][feat] Enable 2 DSv4 perf optimizations by default](../sources/prs/TensorRT-LLM/PR-14120.md), [[TRTLLM-12462][fix] Fix FP8 block scaling GEMM autotuner cache growth](../sources/prs/TensorRT-LLM/PR-14165.md), [[https://nvbugs/6025177][test] rcca tests using kimi k2.5 fp4](../sources/prs/TensorRT-LLM/PR-14172.md), [[https://nvbugs/6163147][fix] swap layer.mlp in place for Mixtral modelopt export](../sources/prs/TensorRT-LLM/PR-14179.md), [[None][fix] Avoid dp_size x ep_size double-count in MegaMoEDeepGemm SymmBuffer](../sources/prs/TensorRT-LLM/PR-14213.md), [[None][refactor] clean up AttentionForwardArgs](../sources/prs/TensorRT-LLM/PR-14244.md), [[None][fix] Handle unset attention_dp_relax in ADP routers](../sources/prs/TensorRT-LLM/PR-14276.md), [[https://nvbugs/6095421][fix] Update resolve_moe_backend](../sources/prs/TensorRT-LLM/PR-14282.md), [[None][chore] Update Claude Code agents and skills](../sources/prs/TensorRT-LLM/PR-14344.md), [[None][feat] GPT-OSS Sm120/Sm121 Support](../sources/prs/TensorRT-LLM/PR-7937.md), [[TRTLLM-8535][feat] Support DeepSeek V3.2 with FP8 + BF16 KV cache/NVFP4 + BF16 KV cache](../sources/prs/TensorRT-LLM/PR-8405.md), [[TRTLLM-8827] [feat] Enable low precision alltoall for Cutlass and TRTLLMGen backends](../sources/prs/TensorRT-LLM/PR-8675.md), [[TRTLLM-8958][feat] and [TRTLLM-8960]: create ConfigurableMoE and support TRTLLMGenFusedMoE as backend](../sources/prs/TensorRT-LLM/PR-9486.md), [[https://nvbugs/5726962][feat] Apply fusion for W4AFP8_AWQ MoE](../sources/prs/TensorRT-LLM/PR-9838.md), [Experimental Python cooperative algorithms](../sources/prs/cccl/PR-1973.md), [[cuda.compute]: Fix faulty pointer arithmetic calculation in CUB dispatch](../sources/prs/cccl/PR-7940.md), [Expose `max_segment_size` guarantee in cuda.compute](../sources/prs/cccl/PR-8284.md), [[cub]: implement utilities for policy selection](../sources/prs/cccl/PR-8355.md), [Hopper Grouped GEMM support for FP8 Accum](../sources/prs/cutlass/PR-2123.md), [fix gqa issue for blackwell fmha.py](../sources/prs/cutlass/PR-2599.md), [Add tutorial fp16_gemm_1](../sources/prs/cutlass/PR-2750.md), [new example with TMA prefetch feature targeting for DRAM latency boun…](../sources/prs/cutlass/PR-2881.md), [[Cute-DSL] Add option for issue_clc_query without multicast](../sources/prs/cutlass/PR-3021.md), [[Hopper CuTeDSL] Add grouped GEMM kernel example](../sources/prs/cutlass/PR-3091.md), [Support for Group GEMM in CUTLASS Profiler for GeForce and Spark](../sources/prs/cutlass/PR-3092.md), [[CLI] add cutedsl fp16 gemm tutorial from 2 to 6](../sources/prs/cutlass/PR-3106.md), [Update blackwell tutorial to be compatible with 4.5-dev version](../sources/prs/cutlass/PR-3130.md), [Small Tile N BlockScaled GEMM + Grouped GEMM on SM12x](../sources/prs/cutlass/PR-3176.md), [Add var-seq-len to FA3 fp16 / bf16 fwd](../sources/prs/flash-attention/PR-1072.md), [FA3 FP8 qkv descales + restore max offset for h128 causal + added sync for producer WG](../sources/prs/flash-attention/PR-1173.md), [Add seqused_q in fwd / bwd and seqused_k in bwd in hopper FA.](../sources/prs/flash-attention/PR-1182.md), [Add local attention in Hopper FAv3](../sources/prs/flash-attention/PR-1233.md), [Paged Attention support for FA3](../sources/prs/flash-attention/PR-1268.md), [FA3 paged attention: Readiness for Cutlass 3.6 / default value for block_table](../sources/prs/flash-attention/PR-1331.md), [Support hdimQK != hdimV backward](../sources/prs/flash-attention/PR-1604.md), [Improve causal backward determinism perf with SPT schedule](../sources/prs/flash-attention/PR-1893.md), [[Cute,Fwd,Sm100] Implement SplitKV](../sources/prs/flash-attention/PR-1940.md), [Blackwell FlashAttention-BWD (v1.0)](../sources/prs/flash-attention/PR-1945.md), [[Cute] Block sparse support Sm100](../sources/prs/flash-attention/PR-1985.md), [[Cute,Fwd,Sm100] Support `q_stage=1` for inference](../sources/prs/flash-attention/PR-1993.md), [[Cute,Fwd,Sm100] Support paged attention](../sources/prs/flash-attention/PR-1999.md), [[Cute,Sm100,Fwd] use correction warps for epi when not using TMA](../sources/prs/flash-attention/PR-2014.md), [[Cute,Fwd,Sm100] don't pass mask_fn to softmax_step generically](../sources/prs/flash-attention/PR-2026.md), [[Cute,Bwd,Sm100] enable deterministic mode for sm100 bwd and fix race conditions](../sources/prs/flash-attention/PR-2033.md), [[Cute,Fwd] Extend score_mod to variable sequence length](../sources/prs/flash-attention/PR-2043.md), [Add score-mod bwd support ](../sources/prs/flash-attention/PR-2070.md), [Add blocksparse support for bwd on blackwell](../sources/prs/flash-attention/PR-2085.md), [Fix IMA in fwd on m boundary](../sources/prs/flash-attention/PR-2091.md), [Add pack-gqa fwd support for sparse impl w/ broadcasted H dim](../sources/prs/flash-attention/PR-2098.md), [[Cute,Fwd,Sm100] distributed offset calculation for paged KV](../sources/prs/flash-attention/PR-2104.md), [[NVIDIA] Enable Jetson Thor FA4](../sources/prs/flash-attention/PR-2108.md), [[CUTE][SM90]Enable pack-gqa with broadcasted maskmods](../sources/prs/flash-attention/PR-2145.md), [[Cute][Flex]Add pack-gqa divmod](../sources/prs/flash-attention/PR-2180.md), [[Cute,Fwd,Sm100] support irregular qhead / kvhead ratios](../sources/prs/flash-attention/PR-2186.md), [[Ai-assisted] CLC work stealing](../sources/prs/flash-attention/PR-2218.md), [[Bwd,Sm120] Add SM120 backward pass support](../sources/prs/flash-attention/PR-2330.md), [Add SM120 varlen attention support](../sources/prs/flash-attention/PR-2333.md), [[Fwd,Sm90] Add paged KV attention support (tma and cp.async)](../sources/prs/flash-attention/PR-2360.md), [[Cute,Sm100,Bwd] refine bwd swizzle for deterministic](../sources/prs/flash-attention/PR-2390.md), [Feat([FA4][CUTE DSL]) Add head_dim=256 support (forward + backward)](../sources/prs/flash-attention/PR-2412.md), [Add CLC scheduler heuristic](../sources/prs/flash-attention/PR-2455.md), [[hd256] Improve forward kernel with exp2 FMA emulation (3% to 9% performance gain)](../sources/prs/flash-attention/PR-2488.md), [[hd256] Add TMA paged KV support to SM100 2CTA forward kernel](../sources/prs/flash-attention/PR-2489.md), [[FA4][hd256] Backward TMA bulk-store epilogue + LSE/dpsum coalesce](../sources/prs/flash-attention/PR-2497.md), [Fix ZeroDivisionError in num_splits_heuristic for empty Q workloads](../sources/prs/flash-attention/PR-2515.md), [bugfix: import wrapper of mla decode](../sources/prs/flashinfer/PR-1013.md), [Add fp4 quantization swizzling tests](../sources/prs/flashinfer/PR-1157.md), [feat: nvshmem python bindings](../sources/prs/flashinfer/PR-1160.md), [feat: logits processor fustion rule for temperature softmax](../sources/prs/flashinfer/PR-1170.md), [Expose fp4 blockscale swizzling kernel](../sources/prs/flashinfer/PR-1176.md), [[feat] support block sparse attention w/ variable block sizes and head-wise sparse patterns](../sources/prs/flashinfer/PR-1177.md), [bugfix: fix invalid blackwell fmha unittests](../sources/prs/flashinfer/PR-1181.md), [bugfix: fix blackwell fmha hanging issue for empty kv_len](../sources/prs/flashinfer/PR-1198.md), [Add DeepGEMM kernels](../sources/prs/flashinfer/PR-1209.md), [Fix test_groupwise_scaled_gemm_fp8.py](../sources/prs/flashinfer/PR-1211.md), [init add gemm fp8 using cudnn backend](../sources/prs/flashinfer/PR-1264.md), [feat: add masked deepgemm support and benchmarking](../sources/prs/flashinfer/PR-1266.md), [hotfix: fix deepgemm artifactory hash](../sources/prs/flashinfer/PR-1278.md), [fix: update trtllm-gen fmha benchmark](../sources/prs/flashinfer/PR-1280.md), [Unify groupwise fp8 GEMM test](../sources/prs/flashinfer/PR-1281.md), [add mm_fp4 use cudnn backend](../sources/prs/flashinfer/PR-1288.md), [Remove FAST_BUILD FLAG for MOE](../sources/prs/flashinfer/PR-1291.md), [Support loading autotuned results from json for cutlass fp4 moe backends](../sources/prs/flashinfer/PR-1310.md), [test qkvo quantization not equal to 1.](../sources/prs/flashinfer/PR-1314.md), [minor: add trtllm_gen_mla benchmark](../sources/prs/flashinfer/PR-1316.md), [Allow cudnn prefill kernels to be called natively](../sources/prs/flashinfer/PR-1317.md), [refactor: Improved metainfo for trtllm-gen kernels](../sources/prs/flashinfer/PR-1328.md), [add torch float4_e2m1fn_x2 check for cudnn fp4 backend](../sources/prs/flashinfer/PR-1333.md), [[Fix] remove torch 2.8 requirement for FP4 GEMM](../sources/prs/flashinfer/PR-1334.md), [Fix bench deepgemm setting](../sources/prs/flashinfer/PR-1344.md), [Support passing kv_data_type to MultiLevelCascadeAttentionWrapper.plan()](../sources/prs/flashinfer/PR-1350.md), [[fix] remove (view) transpose to keep consistent with majorness MN requirement.](../sources/prs/flashinfer/PR-1358.md), [hotfix: update mxfp4 groupwise-scaled gemm unittests](../sources/prs/flashinfer/PR-1359.md), [Update autotune results for the nvfp4 cutlass moe backends for v0.2.9](../sources/prs/flashinfer/PR-1361.md), [refactor: download trtllm gemm metadata from server](../sources/prs/flashinfer/PR-1378.md), [Allow BatchPrefillPagedWrapper to call cudnn API](../sources/prs/flashinfer/PR-1384.md), [Adding FP8 benchmark on attention and matmul testing](../sources/prs/flashinfer/PR-1390.md), [Add Mxfp4 trtllm-gen moe unit tests](../sources/prs/flashinfer/PR-1399.md), [feature: enable cublas for fp4 gemm when cudnn == 9.11.1 or >= 9.13](../sources/prs/flashinfer/PR-1405.md), [Faster weight processing (moe nvfp4)](../sources/prs/flashinfer/PR-1412.md), [refactor: Sink attention AoT](../sources/prs/flashinfer/PR-1427.md), [Fix redundant kernels in moe](../sources/prs/flashinfer/PR-1428.md), [bugfix: fix perf issue by using fp8 graph that can use cublaslt](../sources/prs/flashinfer/PR-1435.md), [feat: enable trtllm-gen attn speculative decoding verify by decode](../sources/prs/flashinfer/PR-1453.md), [Fix TRTLLM NVFP4-out attention kernel scale factor dim issue](../sources/prs/flashinfer/PR-1460.md), [feat: Enable multiple fused-moe backends](../sources/prs/flashinfer/PR-1472.md), [refactor: unify autotuner for bmm_fp8](../sources/prs/flashinfer/PR-1479.md), [fix missing enable_pdl argument in trtllm-gen fp4 moe](../sources/prs/flashinfer/PR-1480.md), [Add python API for masked grouped gemm](../sources/prs/flashinfer/PR-1481.md), [flashinfer_benchmark QoL Improvements and Attention FP8 Support](../sources/prs/flashinfer/PR-1512.md), [Remove cuda-python from dependency and check at runtime](../sources/prs/flashinfer/PR-1534.md), [Add sm check for sm100 only cutlass/trtllm kernel](../sources/prs/flashinfer/PR-1535.md), [feat: Add fp8-qkv, fp16/bf16 output MHA](../sources/prs/flashinfer/PR-1540.md), [perf: Enable SplitK and fix tile-scheduling for moe fp4 fused moe](../sources/prs/flashinfer/PR-1548.md), [feat: Support for inferring out_dtype from out.dtype for TRTLLM attention kernel](../sources/prs/flashinfer/PR-1578.md), [refactor: Expose calculate_tile_tokens_dim function](../sources/prs/flashinfer/PR-1581.md), [bugfix: Fix test_fp4_quantize test bug](../sources/prs/flashinfer/PR-1585.md), [fix: limit the number of nvcc threads for each kernel](../sources/prs/flashinfer/PR-1589.md), [fix: Improve TRTLLM attention kernel out_dtype unit test](../sources/prs/flashinfer/PR-1590.md), [bugfix: fix unittest test_fp8_quantize](../sources/prs/flashinfer/PR-1599.md), [feat: Enable MnnvlMemory (for alltoallv) on B200](../sources/prs/flashinfer/PR-1601.md), [feat: add support of fp4_batched_quantize](../sources/prs/flashinfer/PR-1633.md), [fix: pass workspace for trtllm-gen attention](../sources/prs/flashinfer/PR-1635.md), [test: pytest.mark.xfail on deepgemm](../sources/prs/flashinfer/PR-1636.md), [bugfix: Fix FLOPS calculation for bench_trtllm_gen_mla.py](../sources/prs/flashinfer/PR-1640.md), [fix: zero-init workspace buffer for trtllm-gen fmha](../sources/prs/flashinfer/PR-1643.md), [Added mx_fp4 support using the cudnn backend](../sources/prs/flashinfer/PR-1644.md), [Add benchmark for MLARopeQuantize](../sources/prs/flashinfer/PR-1656.md), [test: update fused_moe test to random scale factor](../sources/prs/flashinfer/PR-1665.md), [[Hotfix] `test_fp4_quantize.py` failure on sm103](../sources/prs/flashinfer/PR-1666.md), [TGV GEMM as a BF16 backend alternative to cuBLAS](../sources/prs/flashinfer/PR-1668.md), [test: better fp8 quantization init for fused_moe test](../sources/prs/flashinfer/PR-1674.md), [[misc] add a wrapper class for attention sink jit args](../sources/prs/flashinfer/PR-1679.md), [Update deepgemm backend for 103a](../sources/prs/flashinfer/PR-1694.md), [[cute_dsl] add gemm + all reduce (two_shot)](../sources/prs/flashinfer/PR-1695.md), [hotfix: Hotfix for `test_pod_kernels.py` on B300](../sources/prs/flashinfer/PR-1698.md), [feat: Benchmark mm_fp4 mxfp4 support and gemm autotune support. Restore mm_fp4 API behavior](../sources/prs/flashinfer/PR-1706.md), [bugfix: increase workspace to make trtllm gen attention unit test pass](../sources/prs/flashinfer/PR-1707.md), [test: skip the unsupported test cases for sm120/121](../sources/prs/flashinfer/PR-1710.md), [perf: Add tuning config for cutlass moe for a hardware](../sources/prs/flashinfer/PR-1716.md), [feat: port fast_decode_plan from sgl](../sources/prs/flashinfer/PR-1745.md), [tests: xfail attention sink UT for sliding window + non causal case](../sources/prs/flashinfer/PR-1752.md), [tests: xfail moe quantization classes mxfp8_bf16 UTs on sm103 ](../sources/prs/flashinfer/PR-1754.md), [Fix tests/test_trtllm_gen_attention.py::test_trtllm_batch_prefill, ::test_trtllm_batch_decode mismatch error](../sources/prs/flashinfer/PR-1755.md), [fix: should pass global_override_indptr_cpu in fast_decode_plan param list](../sources/prs/flashinfer/PR-1757.md), [Added xfail for mx_fp4 matmul on SM120](../sources/prs/flashinfer/PR-1766.md), [tests: skip non SM100/103 for grouped deepgemm](../sources/prs/flashinfer/PR-1767.md), [add test case for trtllm gen fused moe with kimi k2 problem sizes](../sources/prs/flashinfer/PR-1768.md), [Waive / disable test_mla_decode_kernel.py::test_mla_decode_kernel for not sm80 ](../sources/prs/flashinfer/PR-1771.md), [Support checks PoC](../sources/prs/flashinfer/PR-1809.md), [tests: Update support for tgv_gemm to SM100 only and add to ut](../sources/prs/flashinfer/PR-1810.md), [fix: fp4 moe on sm120](../sources/prs/flashinfer/PR-1817.md), [misc: fix some B200 GEMM bench](../sources/prs/flashinfer/PR-1883.md), [fix: Fix trtllm-gen prefill IMA when batch_size==1](../sources/prs/flashinfer/PR-1912.md), [Add realistic bench for persistent kernel ](../sources/prs/flashinfer/PR-1942.md), [fix: Add cutlass as an mm_fp4 backend in compute capability 12.0 in benchmark code](../sources/prs/flashinfer/PR-1959.md), [fix: ensure SM120/121 SFA/SFB contiguity](../sources/prs/flashinfer/PR-1963.md), [Feature: Add support for L40 FusedMoE in cutlass path](../sources/prs/flashinfer/PR-1973.md), [fix: Make attention microbenchmark correctly use page table](../sources/prs/flashinfer/PR-1976.md), [fix: Skipping attention sink Blackwell test outside of Blackwell](../sources/prs/flashinfer/PR-1978.md), [feat: Add backend='auto' to mm_fp4 and enable autotune for backend='cudnn'](../sources/prs/flashinfer/PR-1979.md), [unittest: Add head dim 256 test cases and mark as xfail](../sources/prs/flashinfer/PR-1999.md), [Fix trtllm-gen attention illegal memory access](../sources/prs/flashinfer/PR-2002.md), [fix: Enable SM121 for mm_fp4](../sources/prs/flashinfer/PR-2012.md), [feat: suitable_auto_backends to prune auto backends, bmm_fp8 refactor, heuristic_func intake](../sources/prs/flashinfer/PR-2029.md), [Added an initial implementation of Q and KV Cache in fp8 and to use t…](../sources/prs/flashinfer/PR-2035.md), [test: Skip test_fp8_quantize.py on Hopper](../sources/prs/flashinfer/PR-2052.md), [misc: Add XQA decode to microbenchmark for sm90 and sm120](../sources/prs/flashinfer/PR-2055.md), [[Test] Optimize test_trtllm_gen_fused_moe.py](../sources/prs/flashinfer/PR-2072.md), [unittest: improve the efficiency of xqa unittests](../sources/prs/flashinfer/PR-2075.md), [fix: fix test_trtllm_gen_attention when max_seq_len < page_size](../sources/prs/flashinfer/PR-2076.md), [Patch sm103 for 3xfp4 moe generation](../sources/prs/flashinfer/PR-2082.md), [refactor: update dpsk fused_moe test [1]](../sources/prs/flashinfer/PR-2088.md), [refactor: pass hopper deepgemm include directory through python](../sources/prs/flashinfer/PR-2090.md), [refactor: update dpsk fused_moe test [2]](../sources/prs/flashinfer/PR-2097.md), [[DSR1] Added MLA test](../sources/prs/flashinfer/PR-2100.md), [fix: Fix bench_mm_fp8.py](../sources/prs/flashinfer/PR-2129.md), [A unified API for the MNNVL and single-node/multi-GPU AllReduce kernels.](../sources/prs/flashinfer/PR-2130.md), [fix(trtllm): reset negative strideBatch to 0 for ragged KV layout to …](../sources/prs/flashinfer/PR-2134.md), [fix: some bugs of headDim 256 trtllm-gen fmha kernels. ](../sources/prs/flashinfer/PR-2137.md), [Enable Hopper FA3 FP8 attention in decode.py](../sources/prs/flashinfer/PR-2148.md), [refactor: Move mla code from decode.py to mla.py and add to documentation](../sources/prs/flashinfer/PR-2163.md), [fix: compile flags for trtllm fmha_v2 ](../sources/prs/flashinfer/PR-2175.md), [Rename noauxtc to fused_topk_deepseek](../sources/prs/flashinfer/PR-2181.md), [Permute page table in benchmarking](../sources/prs/flashinfer/PR-2194.md), [misc: support checks for gemm](../sources/prs/flashinfer/PR-2214.md), [Fp8 attention are now part of cuDNN 9.17.1](../sources/prs/flashinfer/PR-2241.md), [test: Fix MNNVL tests to skip when container lacks SYS_PTRACE capability](../sources/prs/flashinfer/PR-2245.md), [feat: Add support for bmm mxfp8](../sources/prs/flashinfer/PR-2256.md), [Fix CUTLASS FP8 gemm correctness issue on SM120/SM121 for shapes where N is not divisible by ScaleGranularityN.](../sources/prs/flashinfer/PR-2261.md), [test: use .float() in in F.cosine_similarity() in bmm_fp8 test](../sources/prs/flashinfer/PR-2266.md), [Tiny fix bench tgv gemm](../sources/prs/flashinfer/PR-2277.md), [fix: Decode benchmark's fa2_tc uses backend=fa2 in wrapper](../sources/prs/flashinfer/PR-2302.md), [Support both 3D and 4D kv_cache shapes in MLA APIs](../sources/prs/flashinfer/PR-2334.md), [Added the cudnn backend Ragged KV Cache wrapper](../sources/prs/flashinfer/PR-2352.md), [benchmarks: Add norm and quantization routines to microbenchmark harness.](../sources/prs/flashinfer/PR-2362.md), [feat: [Qwen3-Next] Add Cute DSL GDN decode kernel and tests](../sources/prs/flashinfer/PR-2370.md), [feat: BF16 GEMM using cuDNN backend](../sources/prs/flashinfer/PR-2376.md), [A Blackwell-optimized version of selective_state_update (decode)](../sources/prs/flashinfer/PR-2387.md), [perf: mm_fp4 heuristic prioritizes CUTLASS over cuDNN on SM103](../sources/prs/flashinfer/PR-2404.md), [perf: add fp4 GEMM tile configs and streamK scheduler for SM120](../sources/prs/flashinfer/PR-2460.md), [fix: blockscale moe routine supports non-DS routing](../sources/prs/flashinfer/PR-2476.md), [fix: Fix memory bandwidth calculation in MLA benchmarks](../sources/prs/flashinfer/PR-2479.md), [Feat/gdn decode pooled](../sources/prs/flashinfer/PR-2521.md), [feat: BF16 GEMM benchmarking support](../sources/prs/flashinfer/PR-2525.md), [pick fa2 for BatchDecodeWithPagedKVCacheWrapper auto backend](../sources/prs/flashinfer/PR-2530.md), [fix: include fp8_blockscale_gemm_90 in AOT jit-cache](../sources/prs/flashinfer/PR-2533.md), [fallback to fa2 (instead of fa3) for unsupported configuration (bf16 Q, Fp8 KV)](../sources/prs/flashinfer/PR-2536.md), [tests: bmm_fp8 for SM110](../sources/prs/flashinfer/PR-2538.md), [feat: cute dsl mmfp4 for blackwell](../sources/prs/flashinfer/PR-2540.md), [Add gen_gemm_sm100_module_cutlass_mxfp8 to jit-cache](../sources/prs/flashinfer/PR-2549.md), [fix: allow fmha_v2_prefill_deepseek on SM121 (DGX Spark)](../sources/prs/flashinfer/PR-2559.md), [fix: guard CUTLASS FMHA against SM12x and fix fmha_v2 SM121a check](../sources/prs/flashinfer/PR-2560.md), [feat: add is_sm12x_supported() helper for SM12x family detection](../sources/prs/flashinfer/PR-2574.md), [tests: add bias testing to nvfp4 moe](../sources/prs/flashinfer/PR-2585.md), [Perf: Optimize GDN decode pretranspose kernel for all batch sizes](../sources/prs/flashinfer/PR-2588.md), [support qk_nope_head_dim for 192 check for GLM-5](../sources/prs/flashinfer/PR-2607.md), [Ameyn/gdn bf16 tolerance parallel reduction](../sources/prs/flashinfer/PR-2610.md), [perf(gdn): optimize MTP kernel with ILP rows and SMEM v caching](../sources/prs/flashinfer/PR-2618.md), [feat: add pool+indices support to gated_delta_rule_decode_pretranspose (bf16 path) ](../sources/prs/flashinfer/PR-2619.md), [fix: trtllm_mxint4_block_scale_moe unit test to index output list](../sources/prs/flashinfer/PR-2627.md), [benchmark: Enable speculative decode microbenchmarking for paged decode](../sources/prs/flashinfer/PR-2628.md), [benchmark: Add MXFP4/MXFP8 quantization mode support to FP4 MoE benchmark](../sources/prs/flashinfer/PR-2635.md), [Enable sm120f compilation](../sources/prs/flashinfer/PR-2650.md), [fix: Add fused MOE and GEMM AOT modules for SM121](../sources/prs/flashinfer/PR-2654.md), [benchmarks: Add FP8 input / BF16 output in ragged prefill benchmark](../sources/prs/flashinfer/PR-2666.md), [feat(gdn): add BF16 state kernel with MTP support beyond T>4 with intermediate caching.](../sources/prs/flashinfer/PR-2679.md), [fix(jit): GEMM kernels produce NaN under concurrency — missing GDC flags cause PDL synchronization barriers to compile as no-ops](../sources/prs/flashinfer/PR-2716.md), [[gdn] support non-contiguous state for decoding](../sources/prs/flashinfer/PR-2727.md), [Support in-place update for `trtllm_fp8_block_scale_moe`](../sources/prs/flashinfer/PR-2739.md), [[Spark unit test debugging] Fix for tests/attention/test_trtllm_gen_mla.py](../sources/prs/flashinfer/PR-2750.md), [[Spark unit test debugging] Fix for tests/gemm/test_groupwise_scaled_gemm_fp8.py](../sources/prs/flashinfer/PR-2751.md), [perf: Performance tune cute dsl RMSNorm variants](../sources/prs/flashinfer/PR-2777.md), [fix(jit): enable GDC for CUTLASS GEMM PDL — SM100 flag only](../sources/prs/flashinfer/PR-2780.md), [tests: skip sliding window + fp8 to prevent hang in fmha_v2 unit tests](../sources/prs/flashinfer/PR-2781.md), [[fix] Bugfix 1367: fix VariableBlockSparseAttention buffer overflow by dynamically resizing kv_lens_buffer](../sources/prs/flashinfer/PR-2802.md), [feat(gdn): add padding index guard for bf16 decode kernel](../sources/prs/flashinfer/PR-2810.md), [[Spark unit test] Adjust tolerance for test_xqa, test_logits_processor](../sources/prs/flashinfer/PR-2828.md), [perf: Optimize GDN MTP decode kernel (v15) — eliminate ilp=1 fallback…](../sources/prs/flashinfer/PR-2842.md), [fix: fix cute dsl swap_ab tactic failure](../sources/prs/flashinfer/PR-2870.md), [fix: add cute dsl moe utils to AOT](../sources/prs/flashinfer/PR-2872.md), [[fix] bugfix 2856: Fix pre-allocated out shape check in trtllm_batch_decode_with_kv_cache_mla for q_len_per_req > 1](../sources/prs/flashinfer/PR-2876.md), [[NVIDIA] fix(jit): enable GDC for CUTLASS fused MoE PDL — prevent random crashes on SM12x](../sources/prs/flashinfer/PR-2913.md), [fix: Fix autotuner crash on meta-device tensor in trtllm_fp4_block_scale_routed_moe](../sources/prs/flashinfer/PR-2916.md), [feat: SM121 (GB10) tile filtering and autotuner robustness](../sources/prs/flashinfer/PR-2927.md), [CuTe DSL FP4 GEMM Heuristic](../sources/prs/flashinfer/PR-2940.md), [[Perf] Refactor MoE autotuning to set valid topk ids in routed MoE tuning](../sources/prs/flashinfer/PR-2942.md), [Only swizzle on v block scale; rename kv_block_scales to kv_cache_sf](../sources/prs/flashinfer/PR-2954.md), [Update NVSHMEM interface to use NVSHMEM4Py instead of custom bindings](../sources/prs/flashinfer/PR-2960.md), [test: skip unsupported mm_mxfp8 configurations on SM12x](../sources/prs/flashinfer/PR-2974.md), [feat(comm): add MOE Finalize/Reduction patterns to unified allreduce_fusion API](../sources/prs/flashinfer/PR-2982.md), [ Fix MXFP4/MXFP8 failures in SM120 FAST_BUILD and expand all_tiles[] ](../sources/prs/flashinfer/PR-2994.md), [[feat] Add blackwell GDN prefill kernel](../sources/prs/flashinfer/PR-3001.md), [fix: use sym_int64 for strides in rmsnorm CuTe DSL kernels to prevent int32 overflow](../sources/prs/flashinfer/PR-3007.md), [[chore] Install nvidia-cutlass-dsl[cu13] for cu130+](../sources/prs/flashinfer/PR-3017.md), [[feat] Add routing_replay_out support to MoE kernels and Python API](../sources/prs/flashinfer/PR-3024.md), [Fix/3170 dense blockscaled sm12x](../sources/prs/flashinfer/PR-3180.md), [test: enable bmm_mxfp8 cutlass backend coverage on SM12x](../sources/prs/flashinfer/PR-3183.md), [Include TinyGEMM into BF16 autotuner](../sources/prs/flashinfer/PR-3203.md), [feat(trace): embed runnable init() in every TraceTemplate](../sources/prs/flashinfer/PR-3221.md), [Ameyn/gdn bf16 dispatcher and 4d pool](../sources/prs/flashinfer/PR-3268.md), [fix(fmha_v2): fix FP8 V-scratch pipeline and varlen scheduler on SM90](../sources/prs/flashinfer/PR-3276.md), [Ep api design - Build Infra dependencies](../sources/prs/flashinfer/PR-3315.md), [feat: Separate QK/VO head dim dispatch for sm90 AOT](../sources/prs/flashinfer/PR-778.md), [bugfix: fix batch prefill attention kernel unittests](../sources/prs/flashinfer/PR-781.md), [bugfix: fix the behavior of mla plan function when provided with host tensors](../sources/prs/flashinfer/PR-816.md), [unittest: add MLA test cases where kv_len is evenly divided by page_size.](../sources/prs/flashinfer/PR-861.md), [perf: reduce torch.library dispatch overhead](../sources/prs/flashinfer/PR-968.md), [perf: Fix python API overhead when CUDAGraph is not enabled](../sources/prs/flashinfer/PR-969.md), [Update torch-xpu-ops commit pin](../sources/prs/pytorch/PR-144209.md), [[inductor][cpu] Fix bmm b_index for dynamic expressions in inductor autotuner](../sources/prs/pytorch/PR-144248.md), [Fix PythonMod printing](../sources/prs/pytorch/PR-144335.md), [Remove runtime dependency on packaging](../sources/prs/pytorch/PR-149125.md), [Add AOTI shim for _weight_int4pack_mm_cpu_tensor (#149031)](../sources/prs/pytorch/PR-149386.md), [op should NOT be static in aoti_torch_call_dispatcher](../sources/prs/pytorch/PR-149644.md), [Dont exclude constant_pad_nd in prologue fusion](../sources/prs/pytorch/PR-150145.md), [[inductor] Fix inductor windows linker error](../sources/prs/pytorch/PR-150447.md), [[Windows][inductor] fix blank space break windows file path](../sources/prs/pytorch/PR-150448.md), [[dynamo][super variable] Fix bug to use correct source](../sources/prs/pytorch/PR-152774.md), [[FlexAttention] Remove Old Constraint on lastdim strides](../sources/prs/pytorch/PR-153104.md), [Mark auto_functionalized HOPs as cacheable (#151194)](../sources/prs/pytorch/PR-153304.md), [[FlexAttention] explicilty create grad_q w/ strides](../sources/prs/pytorch/PR-153641.md), [[MPS] Switch Cholesky decomp to column wise](../sources/prs/pytorch/PR-158237.md), [Add warning about removed sm50 and sm60 arches](../sources/prs/pytorch/PR-158301.md), [[CD] CUDA 13 specific followup changes. Remove sm50-70 From CUDA 12.6 and CUDA 12.8 builds](../sources/prs/pytorch/PR-162455.md), [fix cpp extension distributed warning spew](../sources/prs/pytorch/PR-162764.md), [[Cherry Pick][Graph Partition] allow sharing default device context](../sources/prs/pytorch/PR-163097.md), [[Release 2.9] [cuDNN][SDPA][submodule] Roll-back cuDNN frontend upgrade, update Met…](../sources/prs/pytorch/PR-163265.md), [CUDA 13.0 Warning update for supported architectures](../sources/prs/pytorch/PR-163585.md), [fix pickling for BitwiseFn](../sources/prs/pytorch/PR-163861.md), [[SDPA] [MPS] Fixes regression in 2.8.0 for scaled_dot_product_attention using mps](../sources/prs/pytorch/PR-164364.md), [[Flex attention] Fix flex attention head broadcast](../sources/prs/pytorch/PR-164368.md), [[inductor] don't try to reorder loops for template](../sources/prs/pytorch/PR-166910.md), [[Dynamo] Don't guard data ptrs by default with mark_static_address](../sources/prs/pytorch/PR-166913.md), [[Inductor] No longer throw error in bmm out_dtype lowering due to tem…](../sources/prs/pytorch/PR-166922.md), [[GraphPartition] cache get_free_symbol_uses (#166338)](../sources/prs/pytorch/PR-166994.md), [[cuDNN][SDPA] Check-in test for #166211](../sources/prs/pytorch/PR-167121.md), [[Inductor] ExternKernelBenchmarkRequest best attempt](../sources/prs/pytorch/PR-170246.md), [[flex_attention] adds support for low precision K/V inputs in compiled mode with GPU](../sources/prs/pytorch/PR-170486.md), [[cherry-pick] Fix vllm issue for flex (#170499)](../sources/prs/pytorch/PR-170555.md), [Avoid closing random file handles in Inductor](../sources/prs/pytorch/PR-171150.md), [[xpu][fix][inductor] fallback bfloat16 atomics to eager](../sources/prs/pytorch/PR-171247.md), [[MPS] Fix 2-pass SDPA memory corruption by forcing float accumulators](../sources/prs/pytorch/PR-175580.md), [[CI] Update inductor CI jobs to CUDA 13.0](../sources/prs/pytorch/PR-175826.md), [[Inductor] Reject non-contiguous subnode fusion in mix-order reduction.](../sources/prs/pytorch/PR-176410.md), [[inductor] Fix Identity comparability and evalf recursion](../sources/prs/pytorch/PR-176783.md), [[Inductor] Don't unfuse addmm for bf16/fp16 to avoid precision loss](../sources/prs/pytorch/PR-177144.md), [[Inductor][MPS] Fix half-precision type mismatches in Metal shader codegen (#176436)](../sources/prs/pytorch/PR-177193.md), [[MPS] fix compiling of SDPA producing nan results](../sources/prs/pytorch/PR-178009.md), [feat: Add FP4 (E2M1) KV Cache Support with Quantization Utilities for MLA](../sources/prs/sglang/PR-10078.md), [[Feature] Add MLAProcess for DeepSeek MLA on NPU](../sources/prs/sglang/PR-10130.md), [Enable native ModelOpt quantization support (3/3)](../sources/prs/sglang/PR-10154.md), [Fix chunked prefix cache for nvfp4](../sources/prs/sglang/PR-10180.md), [Add support for bf16 x bf16 cutlass fused MoE](../sources/prs/sglang/PR-10275.md), [support qwen3_next blackwell](../sources/prs/sglang/PR-10403.md), [Fix cutlass moe accuracy drop caused by attention UB from DP padding mode](../sources/prs/sglang/PR-10414.md), [Support single batch overlap](../sources/prs/sglang/PR-10422.md), [Fix correction bias undefined behavior for nvfp4 models](../sources/prs/sglang/PR-10426.md), [feat: add dsv3 fp4 cutlass moe etp ut](../sources/prs/sglang/PR-10433.md), [Cache the result of `is_blackwell` platform check](../sources/prs/sglang/PR-10498.md), [Enable trtllm mla prefix extend](../sources/prs/sglang/PR-10526.md), [Fix bias handling in TritonMoeQuantInfo within quantization/mxfp4.py](../sources/prs/sglang/PR-10579.md), [support qwen3-next-fp8 deepep](../sources/prs/sglang/PR-10622.md), [[Auto Sync] Update modelopt_quant.py (20250920)](../sources/prs/sglang/PR-10688.md), [Unify SGL Kernel Releases](../sources/prs/sglang/PR-10701.md), [Fix MTP MoE weight loading with NVFP4 target model.](../sources/prs/sglang/PR-10758.md), [Fuse quantize and rope in trtllm_mla MTP](../sources/prs/sglang/PR-10779.md), [[2/2] Support MHA prefill with FlashAttention 4.](../sources/prs/sglang/PR-10937.md), [Quick Fix: fix Qwen3-VL launch failure caused by MRotaryEmbedding arg](../sources/prs/sglang/PR-10985.md), [chore: upgrade sgl-kernel 0.3.13](../sources/prs/sglang/PR-11056.md), [Fix DSR1 accuracy for flashinfer_trtllm MoE with FP8 quantization](../sources/prs/sglang/PR-11081.md), [[NVIDIA] Add new SMs support for Spark & Thor](../sources/prs/sglang/PR-11287.md), [[AMD] Clean up vllm dependencies in moe_runner/triton.py](../sources/prs/sglang/PR-11349.md), [[sgl-kernel][1/N]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-11432.md), [[NVIDIA] FA3/FA4 Fix ](../sources/prs/sglang/PR-11606.md), [Support shared experts overlap in cutlass moe](../sources/prs/sglang/PR-11611.md), [[DeepseekV32] Enable flashmla_prefill kernel with fp8 kvcache](../sources/prs/sglang/PR-11655.md), [Use trtllm_mla decode kernel for draft extend in speculative decoding](../sources/prs/sglang/PR-11664.md), [Support running FP4 Deepseek on SM120.](../sources/prs/sglang/PR-11708.md), [[sgl-kernel] support flashmla libtorch](../sources/prs/sglang/PR-11717.md), [Change bf16 to fp8 for some gemms in attention for DeepSeek ckpt v2](../sources/prs/sglang/PR-11805.md), [Use cutlass fp4 gemm by default](../sources/prs/sglang/PR-11813.md), [Support nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8/NVFP4](../sources/prs/sglang/PR-11866.md), [DeepSeek-V3.2: Add Adaptive MHA Attention Pathway for Short-Sequence Prefill](../sources/prs/sglang/PR-11892.md), [chore: upgrade flashinfer 0.4.1](../sources/prs/sglang/PR-11933.md), [Feature/nano v2 offline modelopt fp8 and nvfp4](../sources/prs/sglang/PR-12018.md), [(1/n)support context parallel with deepseekv3.2-DSA](../sources/prs/sglang/PR-12065.md), [[Ascend] qwen optimization](../sources/prs/sglang/PR-12078.md), [[Ascend][feature] support L1+ L2 radixcache on ascend](../sources/prs/sglang/PR-12214.md), [[DeepseekV32]: use `_concat_mla_absorb_q_general` to replace `torch.cat`](../sources/prs/sglang/PR-12215.md), [[hotfix] missing `w13_weight_fp8` and `w2_weight_fp8` in UE8M0 requantization](../sources/prs/sglang/PR-12259.md), [[Deepseek V3.2] Enable flashmla_auto with MTP](../sources/prs/sglang/PR-12294.md), [fix seqlen bug for trtllm_mla's draft_extend](../sources/prs/sglang/PR-12295.md), [fix: Llama 4 BF16 load on Blackwell](../sources/prs/sglang/PR-12308.md), [fix: llama 4 + trtllm gen + fp8 kv cache incompatibility](../sources/prs/sglang/PR-12347.md), [Replace [silu_and_mul_]scaled_fp4_group_quant by Flashinfer equivalent](../sources/prs/sglang/PR-12376.md), [perf: trtllm mla performance minor improvements](../sources/prs/sglang/PR-12435.md), [Use sgl fp4 quant kernel by default](../sources/prs/sglang/PR-12482.md), [[Ascend] Support enable-mixed-chunk in non-MLA scenarios](../sources/prs/sglang/PR-12491.md), [chore: upgrade flashinfer 0.5.0](../sources/prs/sglang/PR-12523.md), [Enable Flashinfer TRTLLM-GEN-MoE FP8 blockwise kernel for Qwen3-Next on Blackwell](../sources/prs/sglang/PR-12543.md), [[CPU] Fix MoE layer support for DeepSeek-OCR models](../sources/prs/sglang/PR-12555.md), [feat: Add FP4 (E2M1) KV Cache Support for MHA](../sources/prs/sglang/PR-12612.md), [[NVIDIA] Fix wrong symmetric sizes for fp4 cases](../sources/prs/sglang/PR-12640.md), [[sgl-kernel][5/N]Support Expert Specialization Grouped GEMM](../sources/prs/sglang/PR-12666.md), [[fix] Only enable flashinfer all reduce fusion by default for single-node servers](../sources/prs/sglang/PR-12724.md), [[Bugfix] Fix illegal memory access](../sources/prs/sglang/PR-12758.md), [[Ascend] support Kimi-K2-Thinking](../sources/prs/sglang/PR-12759.md), [Update dsv3 quantization auto setting for sm100](../sources/prs/sglang/PR-12778.md), [ignore the deepgemm check when the model weight with nvfp4 and moe ba…](../sources/prs/sglang/PR-12782.md), [[DeepSeek-V3.2][NSA] Enable MHA Pathway for Short Sequence Prefill on B200 (SM100)](../sources/prs/sglang/PR-12788.md), [[Deepseek V3.2] Only skip Indexer logits computation when is_extend_without_speculative](../sources/prs/sglang/PR-12816.md), [Apply moe_reduce_sum kernel for fused_marlin_moe](../sources/prs/sglang/PR-12888.md), [[Deepseek V3.2] Use torch.compile to speed up torch.cat in nsa](../sources/prs/sglang/PR-13022.md), [Support moe topk sigmoid kernel](../sources/prs/sglang/PR-13049.md), [[sgl-kernel] support custom fp8 flashmla kernel](../sources/prs/sglang/PR-13087.md), [support mtp with deepseek r1 nvfp4 model](../sources/prs/sglang/PR-13115.md), [Aiter fp8 kv cache](../sources/prs/sglang/PR-13147.md), [Support internvl on Blackwell (which doesn't support fa3): add `SingletonCache` support to Vision{Sdpa|Triton|Ascend}Attention](../sources/prs/sglang/PR-13151.md), [[NPU]Optimization of `forward_npu` for `UnquantizedFusedMoEMethod`](../sources/prs/sglang/PR-13158.md), [Fix nan in global scaling factor for large scale nvfp4 EP](../sources/prs/sglang/PR-13162.md), [diffusion: enable fa4 for blackwell](../sources/prs/sglang/PR-13263.md), [[NVIDIA] Fix broken fp8 MoE of deepseek v3](../sources/prs/sglang/PR-13264.md), [[NVIDIA] Fix use case of SGLANG_ENABLE_FLASHINFER_GEMM](../sources/prs/sglang/PR-13274.md), [Support weight update for blackwell DeepGEMM](../sources/prs/sglang/PR-13324.md), [Flashinfer TRTLLM-GEN-MoE + Qwen3](../sources/prs/sglang/PR-13489.md), [Fix target MLA with eagle3 support for PD disaggregation](../sources/prs/sglang/PR-13555.md), [[BugFix] fix prefixcache performance and accuracy on ascend](../sources/prs/sglang/PR-13573.md), [[ROCM] Optimized deepseek-r1 fp8 model with + triton_gemm_a8w8 + batch_gemm_a8w8 + fused set_mla_kv_buffer kernel](../sources/prs/sglang/PR-13617.md), [[DeepSeekV3.2] Enable pure TP & Partial DP Attention](../sources/prs/sglang/PR-13646.md), [Fix EPLB + FP4 Quantization Compatibility Issue](../sources/prs/sglang/PR-13715.md), [[bugfix] fix TBO crashes when attn_tp_size > 1](../sources/prs/sglang/PR-13730.md), [[sgl-kernel][Feat][B200][1/N]Support MXFP8 Grouped GEMM in Blackwell](../sources/prs/sglang/PR-13731.md), [fix trtllm mla spec](../sources/prs/sglang/PR-13738.md), [[AMD] Support --enable-aiter-allreduce-fusion on AMD GPUs](../sources/prs/sglang/PR-13747.md), [[chore]Upgrade flashinfer to 0.5.3](../sources/prs/sglang/PR-13751.md), [[Feat][NVFP4] Enable NVFP4 MoE for Qwen series models (eg. Qwen3-Next) #13761](../sources/prs/sglang/PR-13761.md), [Support fp4 fp8 non gated moe](../sources/prs/sglang/PR-13794.md), [[NVIDIA] Enable TRTLLM BF16 MoE on Blackwell GPUs](../sources/prs/sglang/PR-13798.md), [update flashinfer_cubin==0.5.3](../sources/prs/sglang/PR-13848.md), [[BugFix] fix outplace_fused_experts missing is_gated](../sources/prs/sglang/PR-13864.md), [Feat: GLM-4.6 supports shared experts fusion](../sources/prs/sglang/PR-13873.md), [Fix update weight error for blackwell DeepGEMM](../sources/prs/sglang/PR-13910.md), [[DeepSeek v3.2] opt Context Parallelism: support fused moe, multi batch and fp8 kvcache](../sources/prs/sglang/PR-13959.md), [Use trtllm mha decode kernel for target_verify in speculative decoding](../sources/prs/sglang/PR-13976.md), [Support KTransformers for Qwen3-VL moe](../sources/prs/sglang/PR-13983.md), [Fix flashinfer cutlass MoE output shape for non-FP4-packed inputs](../sources/prs/sglang/PR-14028.md), [[LoRA][III] Add LoRA support for MoE layers and enable TP](../sources/prs/sglang/PR-14105.md), [Add new moe wna16 marlin gemm](../sources/prs/sglang/PR-14122.md), [Apply new moe wna16 marlin gemm](../sources/prs/sglang/PR-14125.md), [Apply new moe align block size kernel](../sources/prs/sglang/PR-14134.md), [Support checking fp8 params in weight_checker](../sources/prs/sglang/PR-14147.md), [fix: Increase FlashInfer workspace size for Qwen3VL models](../sources/prs/sglang/PR-14173.md), [Add Mistral Large 3 support.](../sources/prs/sglang/PR-14213.md), [[bug fix] fix ima with get_mla_kv_buffer_kernel overflow](../sources/prs/sglang/PR-14224.md), [Tiny use trtllm_mha as default when possible](../sources/prs/sglang/PR-14291.md), [[Fix] add block size logic for sm120 smem size](../sources/prs/sglang/PR-14311.md), [[FIX] trtllm-moe-fp4-renorm for Qwen series models](../sources/prs/sglang/PR-14350.md), [Support FP8 MLA prefill and 128k context.](../sources/prs/sglang/PR-14395.md), [[NPU] perf update with kvcache nz & w4a8 quant](../sources/prs/sglang/PR-14423.md), [Add Mistral Large 3 Eagle Support](../sources/prs/sglang/PR-14466.md), [Mistral Large 3 NVFP4 support](../sources/prs/sglang/PR-14485.md), [[diffusion] kernel fusion: gated residual layernorm scale shift and layernorm scale shift kernel fusion for Qwen-Image, WAN and HunyuanVideo](../sources/prs/sglang/PR-14717.md), [[NPU][eagle3] support qwen eagle3 on NPU](../sources/prs/sglang/PR-14820.md), [Apply back moe_sum_reduce for fused_marlin_moe](../sources/prs/sglang/PR-14829.md), [fix: trtllm mha attention auto-selection on sm120](../sources/prs/sglang/PR-14842.md), [Fix dsv3 dp accuracy issue when using bf16-kv](../sources/prs/sglang/PR-14897.md), [Fix accuracy issue when using a16w16 mla_decode_fwd](../sources/prs/sglang/PR-14936.md), [[AMD] Support fused_rms_mxfp4_quant in the prefill stage for DeepSeek-R1-MXFP4](../sources/prs/sglang/PR-14975.md), [add transformers version validation for glm-4.6v moe models](../sources/prs/sglang/PR-14998.md), [Mistral Large 3 NVFP4 TRTLLM MoE support](../sources/prs/sglang/PR-15049.md), [fix(attention): Prevent trtllm_mha auto-selection with eagle3 speculative decoding](../sources/prs/sglang/PR-15127.md), [[sgl-kernel][1/2] Fused qk_norm_rope for GLM4.6](../sources/prs/sglang/PR-15141.md), [[NVIDIA] upstream FA4](../sources/prs/sglang/PR-15182.md), [[NVIDIA] Fixes for NVFP4 all-gather with spec decoding](../sources/prs/sglang/PR-15280.md), [[Fix] A followup fix for TRTLLM BF16 MoE](../sources/prs/sglang/PR-15303.md), [Fix the accuracy issue when running mxfp4 dsv3 model and enable ep](../sources/prs/sglang/PR-15304.md), [feat: support bitsandbytes quantization algorithm](../sources/prs/sglang/PR-15325.md), [[distributed] Clean up MoE groups in destroy_model_parallel](../sources/prs/sglang/PR-15345.md), [[Tiny]Add warning for deepgemm on Blackwell](../sources/prs/sglang/PR-15352.md), [[NPU]mindspore model support moe](../sources/prs/sglang/PR-15363.md), [[NPU]DeepSeek-V3.2 support npu mlaprolog](../sources/prs/sglang/PR-15381.md), [[diffusion] Add Sage Attention 3 Support for sm 120 (RTX5090)](../sources/prs/sglang/PR-15382.md), [Super tiny add moe_ep_rank to prometheus labels](../sources/prs/sglang/PR-15407.md), [Flashinfer MOE FP8 support for Mistral Large 3.](../sources/prs/sglang/PR-15422.md), [Optimize MiMo-V2-Flash by flashinfer fused allreduce](../sources/prs/sglang/PR-15464.md), [[Perf] Add Flashinfer DeepGEMM SM90 for SwapAB Optimization](../sources/prs/sglang/PR-15514.md), [Optimize FP8 MLA KV cache writes with Triton kernel](../sources/prs/sglang/PR-15522.md), [Optimize Bailing-MoE with FlashInfer Fused All-Reduce](../sources/prs/sglang/PR-15526.md), [MoE: Skip SiLU/GELU activation for masked experts](../sources/prs/sglang/PR-15539.md), [Update flashinfer to 0.6.1](../sources/prs/sglang/PR-15551.md), [[sgl-kernel] Streamline kernel size report (Top 20 only) and clean up](../sources/prs/sglang/PR-15552.md), [Fix BatchMLAPagedAttentionWrapper query/qo_inptr mismatch for EAGLE](../sources/prs/sglang/PR-15601.md), [[jit-kernel] Add CuTe DSL GDN Decode Kernel](../sources/prs/sglang/PR-15631.md), [Add SwapAB Optimization for triton fused_moe_kernel on SM90.](../sources/prs/sglang/PR-15712.md), [[Perf] Eliminate the slice op for Flashinfer `trtllm_fp4_block_scale_moe`](../sources/prs/sglang/PR-15731.md), [Fix GLM-4.7 MoE Detector complex JSON Schema type parsing](../sources/prs/sglang/PR-15753.md), [Fix: Handle empty func_name and None values in GLM MoE detectors](../sources/prs/sglang/PR-15754.md), [[Feature] JIT Fused QK norm + qk norm clean up](../sources/prs/sglang/PR-15835.md), [[JIT kernel] Apply jit per_tensor_quant_fp8 kernel](../sources/prs/sglang/PR-15836.md), [[diffusion] model: support TurboWan2.1-T2V-1.3B/14B SLA](../sources/prs/sglang/PR-15888.md), [[fix]deepgemm precompile when warmup](../sources/prs/sglang/PR-15891.md), [Bugfix for ds-vl2](../sources/prs/sglang/PR-15894.md), [[NPU] NZ for non-quantized MOE, Qwen3 MOE double memory consumption fix](../sources/prs/sglang/PR-15904.md), [ Add tuned triton==3.5.1 h200 tp2, tp4 for qwen 3 next](../sources/prs/sglang/PR-15948.md), [Tiny fix cannot launch nvfp4 checkpoint with bf16 kv cache](../sources/prs/sglang/PR-15986.md), [[Performance] Force split_k=1 for MXFP4 Triton kernels on Hopper](../sources/prs/sglang/PR-16014.md), [Support fa4 decoding](../sources/prs/sglang/PR-16034.md), [optimize get_topk_ragged by fusing get k and k_scale triton kernel](../sources/prs/sglang/PR-16043.md), [[Diffusion] Flux support flashinfer rope](../sources/prs/sglang/PR-16055.md), [enhance accuracy for model kimi-vl-instruct-a3b](../sources/prs/sglang/PR-16076.md), [fix layer intermediate size](../sources/prs/sglang/PR-16084.md), [[Diffusion] Zimage opt with qknorm and flashinfer rope](../sources/prs/sglang/PR-16161.md), [[Feature] add aligned_vector type for JIT kernel](../sources/prs/sglang/PR-16162.md), [[VLM] Adopt jit qk_norm kernel in VLM](../sources/prs/sglang/PR-16171.md), [[NemotronH] Add latent MoE support](../sources/prs/sglang/PR-16227.md), [[Fix] Only add SM90 and SM100 to check for auto-enabling TRT Allreduce Fusion](../sources/prs/sglang/PR-16283.md), [[diffusion] Fix RuntimeError in SageAttention3 on Nvidia Blackwell with Qwen-Image](../sources/prs/sglang/PR-16335.md), [[Fix]Fix FA3 Performance in Diffusion Model ](../sources/prs/sglang/PR-16382.md), [Fix FP8 MoE NaN with DeepGEMM on Blackwell](../sources/prs/sglang/PR-16622.md), [[Rework] Add SwapAB Optimization for triton fused_moe_kernel on SM90.](../sources/prs/sglang/PR-16723.md), [[AMD] Support redundant expert with a2a moe in gfx95x.](../sources/prs/sglang/PR-16791.md), [[Fix] `flashinfer_trtllm` `intermediate_size` assertion with Qwen3 + TP=8](../sources/prs/sglang/PR-16824.md), [Support mxint4 flashinfer_trtllm moe gemm](../sources/prs/sglang/PR-16892.md), [[DeepSeek v3.2] Opt MTP decode cuda batch sizes and nsa implementation](../sources/prs/sglang/PR-16961.md), [[NPU]bugfix: fix for dsv3.2 and dsvl2](../sources/prs/sglang/PR-17007.md), [[MUSA][2/N] sgl-kernel build](../sources/prs/sglang/PR-17053.md), [Optimize GDN decode for Qwen3 Next](../sources/prs/sglang/PR-17094.md), [[diffusion] fix: fix using upstream flash_attn on blackwell](../sources/prs/sglang/PR-17111.md), [Enable XQA for SM90 and SM120](../sources/prs/sglang/PR-17115.md), [Inclusion of nvfp4 blockscale in EPLB Rebalance](../sources/prs/sglang/PR-17158.md), [[Fix] GLM 4.7 + NVFP4 + MTP](../sources/prs/sglang/PR-17166.md), [[GLM 4.7] Add RTX 6000 Pro aka sm120](../sources/prs/sglang/PR-17235.md), [[New Model] GLM4.7-Flash](../sources/prs/sglang/PR-17247.md), [[FIX] Always support TP > 4 for FP4 Gemm](../sources/prs/sglang/PR-17300.md), [Disable mla persistent kernel when not using fp8 kv_cache](../sources/prs/sglang/PR-17327.md), [Move fa4 from sgl-kernel to jit kernel](../sources/prs/sglang/PR-17353.md), [Add mxfp8 support for online quantization, Triton dense linear, and CUTLASS MoE](../sources/prs/sglang/PR-17449.md), [[NPU] enhance accuracy for model kimi-vl-a3b-instruct](../sources/prs/sglang/PR-17480.md), [Kernel: optimize decoding metadata in NSA multi-spec backend with fused kernels](../sources/prs/sglang/PR-17554.md), [[hotfix] Reenable all reduce fusion on sm100](../sources/prs/sglang/PR-17591.md), [[feat] Support nvfp4 quantized model of Qwen3-Next](../sources/prs/sglang/PR-17627.md), [Upgrade transformers==5.3.0](../sources/prs/sglang/PR-17784.md), [fix(quantization): add sgl_kernel fallback for FP4 quantize on Blackwell GPUs](../sources/prs/sglang/PR-17816.md), [Feature/support longcat flash lite](../sources/prs/sglang/PR-17838.md), [[Move sgl-kernel Kernel to JIT] Add JIT concat MLA kernels](../sources/prs/sglang/PR-17889.md), [Skipped warning on sm100](../sources/prs/sglang/PR-18000.md), [[Bugfix] Fix Mistral Large 3 NVFP4 TRTLLM MoE](../sources/prs/sglang/PR-18065.md), [Feat/add fi selective state update kernel call](../sources/prs/sglang/PR-18070.md), [[Diffsuion & JIT_kernel] QKNorm cross heads kernel](../sources/prs/sglang/PR-18073.md), [Fix nvfp4 weight update](../sources/prs/sglang/PR-18085.md), [[Blackwell] Make mxint4 flashinfer_trtllm moe gemm set by default on blackwell](../sources/prs/sglang/PR-18136.md), [[ModelOpt] Fix broken Qwen3-235B-A22B-Instruct-2507-NVFP4 launch](../sources/prs/sglang/PR-18189.md), [[ModelOPT] Support Qwen 3 Next Coder NVFP4](../sources/prs/sglang/PR-18224.md), [Support Qwen3 MoE context parallel](../sources/prs/sglang/PR-18233.md), [[ROCm] Optimize Deepseek R1 on MI300X](../sources/prs/sglang/PR-18242.md), [[Hicache & JIT_kernel] Support page first layout & mla jit kernel](../sources/prs/sglang/PR-18311.md), [[AMD] Support Qwen3-Coder-Next on AMD platform](../sources/prs/sglang/PR-18355.md), [[MUSA][10/N] Add GGUF support](../sources/prs/sglang/PR-18357.md), [feat(gdn): add FlashInfer K-last SSM layout support for GDN prefill and decode for Hopper](../sources/prs/sglang/PR-18361.md), [[Kimi-K2.5] Fix NVFP4 Kimi-K2.5 weight mapping and exclude list](../sources/prs/sglang/PR-18370.md), [Nsa trtllm mla sparse fp8 support with Deepseek v3.2 NVFP4](../sources/prs/sglang/PR-18389.md), [[AMD] Update aiter to v0.1.10.post2](../sources/prs/sglang/PR-18423.md), [feat: add FA4 SM90 paged KV decode support & update attention docs](../sources/prs/sglang/PR-18442.md), [Tilelang sparse decode fwd for dsv32 mi355](../sources/prs/sglang/PR-18488.md), [[FIX] Correct JIT kernel compilation on newer GPUs with outdated driver metadata.](../sources/prs/sglang/PR-18496.md), [Fp8 prefill attn kernel integration](../sources/prs/sglang/PR-18528.md), [[AMD] Fix accuracy issue when running TP4 dsv3 model with mtp](../sources/prs/sglang/PR-18607.md), [[AMD] DSR1/V3 use fp8 bmm in MLA for MI300X](../sources/prs/sglang/PR-18624.md), [[sglang-miles] True on-policy training support for FSDP2](../sources/prs/sglang/PR-18639.md), [use flashinfer.sampling](../sources/prs/sglang/PR-18696.md), [[RL] Support per-layer mixed FP8/BF16 serving for FP8 checkpoints](../sources/prs/sglang/PR-18742.md), [fix: update Blackwell log/error messages to include SM12x](../sources/prs/sglang/PR-18751.md), [[diffusion] Diffusion norm fusion for z-image](../sources/prs/sglang/PR-18762.md), [fix: add SM110 (Jetson AGX Thor) to Blackwell capability check](../sources/prs/sglang/PR-18787.md), [Migrate renorm kernels from sgl-kernel to FlashInfer JIT](../sources/prs/sglang/PR-18854.md), [[Perf] ~9.5x faster Blackwell MXFP4 MoE weight loading](../sources/prs/sglang/PR-18858.md), [Migrate norm kernels to FlashInfer JIT implementation](../sources/prs/sglang/PR-18871.md), [[sgl-kernel] rebase FlashMLA 0217](../sources/prs/sglang/PR-18902.md), [Fix NSA FP8 KV cache path for both-trtllm MHA one-shot](../sources/prs/sglang/PR-18931.md), [[Qwen3.5] Enable nvfp4 checkpoint](../sources/prs/sglang/PR-18937.md), [[Sarvam] Add inference support for Sarvam MoE LLMs](../sources/prs/sglang/PR-18938.md), [[jit_kernel] Add fused_qknorm_rope JIT kernel](../sources/prs/sglang/PR-19059.md), [Support skip-softmax attention](../sources/prs/sglang/PR-19089.md), [feat: Support MXFP4 quantized dense models on AMD CDNA2/CDNA3 GPUs](../sources/prs/sglang/PR-19143.md), [[DeepSeek-V3.2][JIT-kernel] Support nsa fuse store indexer k cache](../sources/prs/sglang/PR-19148.md), [[NVIDIA] Integrate FlashInfer decode kernel (Blackwell) for Qwen3.5](../sources/prs/sglang/PR-19150.md), [Adjust padding size to improve triton_kernels moe performance](../sources/prs/sglang/PR-19174.md), [[AMD] Fix accuracy while using --enable-dp-attention](../sources/prs/sglang/PR-19247.md), [Fix nightly Mistral-Large-3 NVFP4 accuracy threshold](../sources/prs/sglang/PR-19402.md), [[AMD] Fix weight load shape mismatch for amd dsr1 0528 mxfp4](../sources/prs/sglang/PR-19425.md), [[Feature] add feature mla_ag_after_qlora for dsv3.2](../sources/prs/sglang/PR-19428.md), [Fix/nemotron mtp quantaized](../sources/prs/sglang/PR-19433.md), [[Kernel Slimming] Migrate NVFP4 kernels to JIT](../sources/prs/sglang/PR-19437.md), [[FlashInfer v0.6.4] [RL] Integrate FlashInfer mxfp8 gemm, MoE, and routed MoE](../sources/prs/sglang/PR-19537.md), [[NPU] bugs fix for Deepseek models](../sources/prs/sglang/PR-19544.md), [[diffusion][llm] macOS support](../sources/prs/sglang/PR-19549.md), [[miles] fix for glm5](../sources/prs/sglang/PR-19634.md), [[Feature] NVFP4 Marlin fallback for non-Blackwell GPUs (SM75+)](../sources/prs/sglang/PR-19652.md), [Support `triton_kernels` for GPT-OSS on SM120](../sources/prs/sglang/PR-19718.md), [Various SM120 improvements](../sources/prs/sglang/PR-19721.md), [[SGLang-Diffusion] Fix custom op fake impl missing eps default for torch.compile](../sources/prs/sglang/PR-19725.md), [Add compile-time 256-bit vector guard for pre-Blackwell](../sources/prs/sglang/PR-19794.md), [[JIT Kernel][Feature] Support JIT custom all reduce (rewrite as v2)](../sources/prs/sglang/PR-19880.md), [Use TRTLLM allreduce fusion for Qwen 3.5](../sources/prs/sglang/PR-19889.md), [Fix MLA decode path returning unwritten (padded) rows](../sources/prs/sglang/PR-19902.md), [[AMD] Fix Tensor Memory Aliasing ](../sources/prs/sglang/PR-19928.md), [[AMD] Fix FP8 assertion failure in aiter MLA decode by falling back to self.k_scale](../sources/prs/sglang/PR-19935.md), [[AMD] Tilelang sparse fwd for dsv32 mi355/mi300](../sources/prs/sglang/PR-19945.md), [[JIT Kernel] Reland NVFP4 kernels to JIT](../sources/prs/sglang/PR-20012.md), [[Bugfix] Work around FlashInfer unified transport issue on GB](../sources/prs/sglang/PR-20039.md), [Fix SM120 `triton_kernels` MXFP4 `block_k` for GPT-OSS](../sources/prs/sglang/PR-20040.md), [MiniMax-M2.5 - Support dp attention, dp reduce scatter, FP4 all gather, AR fusion in prepare_attn](../sources/prs/sglang/PR-20067.md), [Fix streaming session with paged KV cache (SWA/MLA)](../sources/prs/sglang/PR-20070.md), [Enable modelopt quantized FLUX deployment](../sources/prs/sglang/PR-20082.md), [[V32/GLM5] Change default setting of V32 nvfp4 on TP4](../sources/prs/sglang/PR-20086.md), [[diffusion] fix bug of copy_if](../sources/prs/sglang/PR-20094.md), [[diffusion] Support nvfp4 for Flux.2](../sources/prs/sglang/PR-20137.md), [[AMD] Fp8 prefill integration with radix cache path for dpsk models](../sources/prs/sglang/PR-20187.md), [[4/n jit_kernel restruct] speed up CI tests and add benchmark workflow](../sources/prs/sglang/PR-20268.md), [[AMD] Add 4-GPU test suite for MI325 runners](../sources/prs/sglang/PR-20294.md), [[Benchmark] use flashinfer bench_gpu_time instead of triton do_bench](../sources/prs/sglang/PR-20305.md), [[Fix] Add fallback for flashinfer allreduce fusion](../sources/prs/sglang/PR-20384.md), [[NVIDIA] Enable fp8 flashinfer_trtllm_routed MoE for MiniMax-M2.5](../sources/prs/sglang/PR-20394.md), [[AMD][Bug-fix] Fix gpu fault when run the test with dp-attention-enabled and max-concurrency is over 256](../sources/prs/sglang/PR-20399.md), [[Model] Support Nemotron 3 Super NVFP4](../sources/prs/sglang/PR-20407.md), [[AMD][AITER] Guard _use_mla_ps_kernel with self.use_mla in draft_extend_v2 paths](../sources/prs/sglang/PR-20409.md), [[GDN] Add benchmark for sglang gdn prefill](../sources/prs/sglang/PR-20428.md), [Support Triton MLA FP8 KV cache](../sources/prs/sglang/PR-20479.md), [[Kernel] Fuse temperature + softmax in sampling for decode speedup](../sources/prs/sglang/PR-20501.md), [[Diffusion] Clean upstream fa3 in hopper](../sources/prs/sglang/PR-20576.md), [Use Flashinfer for target_verify in GDN model for SM120](../sources/prs/sglang/PR-20604.md), [FIX: (NSA) Compute topk_indices_offset when NSA prefill flashmla_sparse is used with FP8 KV cache](../sources/prs/sglang/PR-20606.md), [[Diffusion] Add a benchmark for rmsnorm/fuse_add_rmsnorm](../sources/prs/sglang/PR-20632.md), [Fix(jit): support rmsnorm for hidden_size in {64, 128, 256}](../sources/prs/sglang/PR-20661.md), [[Feature][JIT Kernel] Fused TP QK norm For Minimax](../sources/prs/sglang/PR-20673.md), [[Diffusion] Fix compile graph broken by flashinfer rope](../sources/prs/sglang/PR-20699.md), [Add Mistral Small 4 (Pixtral) support](../sources/prs/sglang/PR-20708.md), [Use FlashInfer tinygemm for GPT-OSS MoE router on SM90+](../sources/prs/sglang/PR-20755.md), [fix: guard configure_deep_gemm_num_sms when JIT disabled](../sources/prs/sglang/PR-20868.md), [[JIT Kernel] Fix NVFP4 multi-arch compilation failure](../sources/prs/sglang/PR-20874.md), [Add SGLang CUDA crash API logging inspired by FlashInfer](../sources/prs/sglang/PR-20910.md), [[Qwen3.5] Fuse split/reshape/cat ops in GDN projection with Triton kernel](../sources/prs/sglang/PR-21019.md), [[Chore] Clean up JIT compilation flags](../sources/prs/sglang/PR-21022.md), [fix: wrap _import_static_state in inference_mode to fix resume on Blackwell](../sources/prs/sglang/PR-21035.md), [perf: precompute FA3 scheduler_metadata to eliminate per-layer prepare_varlen_num_blocks](../sources/prs/sglang/PR-21104.md), [ci: remove IS_BLACKWELL env var; auto-detect Blackwell](../sources/prs/sglang/PR-21118.md), [[Not-Merge][AMD] GLM-5 performance optimization](../sources/prs/sglang/PR-21166.md), [[Whisper] Enable CUDA graph support and timestamp for whisper model](../sources/prs/sglang/PR-21190.md), [[NPU] bugfix for import sgl-kernel error](../sources/prs/sglang/PR-21200.md), [[KDA] Support CuTeDSL KDA decode kernel](../sources/prs/sglang/PR-21203.md), [[AMD]: Support MLA with nhead<16 and FP8 KV cache for TP=8 (Kimi K2.5…](../sources/prs/sglang/PR-21213.md), [[refactor] Clean up duplicate flashinfer trtllm moe code](../sources/prs/sglang/PR-21233.md), [Refactor JIT kernel CI to use run_suite.py registration system](../sources/prs/sglang/PR-21239.md), [[NVIDIA] Enable FP4 flashinfer trtllm routed moe](../sources/prs/sglang/PR-21240.md), [P2P Weight Update features for miles ](../sources/prs/sglang/PR-21278.md), [[RL] Support mxfp8 DeepSeek V3](../sources/prs/sglang/PR-21280.md), [[MUSA] apply_vocab_mask support musa device](../sources/prs/sglang/PR-21296.md), [CUTLASS NVFP4 GEMM improvement of SM120](../sources/prs/sglang/PR-21314.md), [[Kernel] Support FlashInfer TRTLLM-Gen fused MoE for non-gated FP4 & FP8 (Nemotron)](../sources/prs/sglang/PR-21321.md), [[misc] clean up kernel API](../sources/prs/sglang/PR-21325.md), [Add dedicated FlashInferCuteDslMoE layer for standard-path FP4 MoE](../sources/prs/sglang/PR-21339.md), [[GDN] Fuse GDN kkt + solve_tril into one kernel](../sources/prs/sglang/PR-21411.md), [[Bugfix] Lazy-import CuteDSL KDA kernel to fix AMD/ROCm startup crash](../sources/prs/sglang/PR-21428.md), [fix nemotron capture for non attention layers](../sources/prs/sglang/PR-21436.md), [[Diffusion] Add qknorm rope fuse kernel](../sources/prs/sglang/PR-21440.md), [Add explicit disable flag for FlashInfer allreduce fusion](../sources/prs/sglang/PR-21446.md), [fix: piecewise_cuda_graph get correct qo_indptr](../sources/prs/sglang/PR-21452.md), [Migrate all callers from /get_server_info to /server_info](../sources/prs/sglang/PR-21463.md), [[AMD] Enable FP8 KV cache and FP8 attention kernel for NSA on MI300/MI355 with TileLang backend](../sources/prs/sglang/PR-21511.md), [test: point DSV3 int8 MLA CI models to lmsys Hugging Face org](../sources/prs/sglang/PR-21561.md), [[FlashInver v0.6.7] Integrate flashinfer_trtllm mxfp8 gemm](../sources/prs/sglang/PR-21576.md), [Change default mm-attention backend from triton_attn to fa4](../sources/prs/sglang/PR-21595.md), [fix: TRT-LLM MHA CUDA illegal address with EAGLE v2 + DP attention](../sources/prs/sglang/PR-21649.md), [[jit_kernel] Optimize fused_qknorm_rope: deduplicate sincosf for interleave RoPE ](../sources/prs/sglang/PR-21654.md), [[AMD] Use tgemm.mm for MoEGate router gemm in deepseek_v2.py](../sources/prs/sglang/PR-21657.md), [[XPU] Enable qwen3.5 on XPU](../sources/prs/sglang/PR-21668.md), [[AMD] Add GLM-5-FP8 nightly performance benchmarks for MI30x and MI35x](../sources/prs/sglang/PR-21710.md), [Harden FlashInfer FP4 imports in standard dispatcher](../sources/prs/sglang/PR-21776.md), [[Fix] Fall back to triton MOE for GPT-OSS on Blackwell with driver >= 595](../sources/prs/sglang/PR-21780.md), [[DSA] Support trtllm sparse mla kernel for prefill batches ](../sources/prs/sglang/PR-21783.md), [Remove redundant test_moe_eval_accuracy_large](../sources/prs/sglang/PR-21787.md), [[Feature] JIT rmsnorm update (with claude)](../sources/prs/sglang/PR-21834.md), [ [GDN] Remove FlashInfer GDN decode + no_buffer guard and default to FlashInfer on SM100+ ](../sources/prs/sglang/PR-21861.md), [[server] Add --quantization unquant to explicitly opt out of quantization](../sources/prs/sglang/PR-21863.md), [[Misc] [MXFP8] Drop sm100 mxfp8 warning](../sources/prs/sglang/PR-21881.md), [fix pcg torch dynamo recompile in mxfp8 Triton path](../sources/prs/sglang/PR-21888.md), [[Bugfix] Temporarily skip TRTLLM attention on (G)B300 (SM103) to avoid high-concurrency hang](../sources/prs/sglang/PR-21906.md), [[DSA] Set trtllm kernels as default for Blackwell](../sources/prs/sglang/PR-21914.md), [[Bugfix] Fix CUDA graph replay issues in trtllm_mla draft_extend](../sources/prs/sglang/PR-21987.md), [Tiny fix trtllm_fp8_per_tensor_scale_moe_wrapper router_logits dtype](../sources/prs/sglang/PR-22006.md), [[NPU] enable mla prepare fused kernel only when being mla attn](../sources/prs/sglang/PR-22024.md), [[MUSA][9/N] Add FA3 attention backend support through MATE (MUSA AI Tensor Engine)](../sources/prs/sglang/PR-22051.md), [[Diffusion] Fix weight scale swizzle and add large-M kernel config for FLUX.2-dev-NVFP4](../sources/prs/sglang/PR-22064.md), [[nvidia] Gemma4 nvfp4 fix](../sources/prs/sglang/PR-22079.md), [[diffusion] Default NVFP4 to CUTLASS and add all-model shape benchmarks](../sources/prs/sglang/PR-22091.md), [[Diffusion] Add diffusion NVFP4 scaled-mm correctness test](../sources/prs/sglang/PR-22127.md), [[Hotfix] Fix router gemm on sm103](../sources/prs/sglang/PR-22134.md), [[Disagg][NIXL] Fix heterogeneous TP KV transfer for non-MLA models (same logic with mooncake, Step 1/2 for Qwen3.5 support)](../sources/prs/sglang/PR-22145.md), [[hisparse]: Adding ci for hisparse kvcache-swap-in jit-kernel](../sources/prs/sglang/PR-22155.md), [[HiSparse]: Add benchmark for hisparse kernel](../sources/prs/sglang/PR-22187.md), [[RL] Refactor NVFP4 shuffling/swizzling to in-place replacement](../sources/prs/sglang/PR-22204.md), [Reduce unnecessary kernels and copies in the NSA indexer](../sources/prs/sglang/PR-22232.md), [[AMD][HIP] NSA: bf16 passthrough from RMSNorm to eliminate FP8 dequantization](../sources/prs/sglang/PR-22258.md), [Lazy import flash_attention_v4 to avoid loading flash_attn.cute at startup](../sources/prs/sglang/PR-22306.md), [[AMD] Fix GLM-5 fp8 KV quant path dispatch on MI300](../sources/prs/sglang/PR-22314.md), [[Reland] DeepSeek-R1-0528-w4a8: DeepEP Low Latency Dispatch Adopts FP8 Communication](../sources/prs/sglang/PR-22316.md), [[Lora] Lora quat info re-factor and support deepseekv3 mla lora](../sources/prs/sglang/PR-22323.md), [[AMD] Add GLM-5.1-FP8 nightly accuracy and performance benchmarks for MI30x and MI35x](../sources/prs/sglang/PR-22336.md), [:sparkles: [diffusion][npu][quant] Add MXFP4 quantization support for Wan2.2 Diffusion on Ascend NPU](../sources/prs/sglang/PR-22338.md), [[Diffusion] modelopt diffusion fp8 support for flux1/flux2 and wan2.2](../sources/prs/sglang/PR-22365.md), [[DSA] Hopper FP8 FlashMLA KV padding](../sources/prs/sglang/PR-22372.md), [[Lora] Lora kimi support](../sources/prs/sglang/PR-22381.md), [[AMD] Use aiter CK layernorm2d for LayerNorm to reduce NSA indexer kernel launches](../sources/prs/sglang/PR-22424.md), [[Fix] Fix several bugs on DSA models](../sources/prs/sglang/PR-22430.md), [Upgrade sglang-torch-profiler-analysis SKILLS](../sources/prs/sglang/PR-22440.md), [[RL] Fix weight update for mxfp8 flashinfer_cutlass gemm backend](../sources/prs/sglang/PR-22484.md), [GLM-5/5.1 MXFP4 Checkpoint Inference Compatibility Fix](../sources/prs/sglang/PR-22543.md), [[Diffusion] Add FLUX.1-dev ModelOpt NVFP4 support](../sources/prs/sglang/PR-22574.md), [diffusion: fix layerwise offload for ModelOpt quantized DiTs](../sources/prs/sglang/PR-22594.md), [feat: Support flashinfer_cutedsl MoE runner with flashinfer alltoall backend](../sources/prs/sglang/PR-22669.md), [reland [Diffusion] Add FLUX.1-dev ModelOpt NVFP4 support](../sources/prs/sglang/PR-22672.md), [[Diffusion] Add Wan2.2 ModelOpt NVFP4 support](../sources/prs/sglang/PR-22681.md), [[Step3p5] Optimize allreduce in MoE layers ](../sources/prs/sglang/PR-22773.md), [Dual MoE CUDA graph capture for lora/nolora batches](../sources/prs/sglang/PR-22809.md), [diffusion: add HunyuanVideo GroupNorm+SiLU fast path](../sources/prs/sglang/PR-22814.md), [[Refactor] Refactor DeepEP dispatcher](../sources/prs/sglang/PR-22822.md), [[FlashInfer v0.6.11] [RL] Support FlashInfer per-token NVFP4 MoE](../sources/prs/sglang/PR-22918.md), [[Fix/Kernel] Add JIT rmsnorm_hf kernel to fix transformers backend MMLU accuracy regression ](../sources/prs/sglang/PR-22931.md), [[codex] diffusion: enable group norm silu fuse by default](../sources/prs/sglang/PR-23148.md), [[BugFix] Resolve adaptive speculative decoding conflicts for Qwen3.5 (hybrid GDN)](../sources/prs/sglang/PR-23331.md), [[Diffusion][NPU]Add attention backends for diffusion models for Ascend NPU](../sources/prs/sglang/PR-23482.md), [Reland Cute-DSL FP4 dense GEMM](../sources/prs/sglang/PR-23590.md), [Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-23686.md), [Use Cute-DSL NVFP4 quantization kernels](../sources/prs/sglang/PR-23745.md), [feat: port SGLANG_JIT_DEEPGEMM_FAST_WARMUP to deepseek_v4 branch](../sources/prs/sglang/PR-23756.md), [[MoE] Unify DeepEPMoE+MoriEPMoE through AITER MoeRunner pre/post-permute](../sources/prs/sglang/PR-23760.md), [Optimize large GroupNorm SiLU apply](../sources/prs/sglang/PR-23938.md), [[feat] Init true on policy with qwen_dense](../sources/prs/sglang/PR-23961.md), [Enable PDL for various kernels in DSV32/GLM5](../sources/prs/sglang/PR-23965.md), [[VLM] Optimize Gemma4 VLM with PCG and fuse RMSNorm + residual add + scalar](../sources/prs/sglang/PR-24048.md), [Refactor device timer, clean up metrics collector, and add fwd occupancy metric](../sources/prs/sglang/PR-24197.md), [[KDA] Optimize prefill kernels with diagonal and recompute fuse](../sources/prs/sglang/PR-24271.md), [[diffusion] Fuse LTX2 split rotary embedding](../sources/prs/sglang/PR-24411.md), [Port MXFP4 Marlin MoE support to JIT kernel path](../sources/prs/sglang/PR-24490.md), [Fix performance regression on Deepseek V3 on `moe-runner-backend=triton` on SM90](../sources/prs/sglang/PR-24562.md), [[Codex] Opt Mistral Large performace ](../sources/prs/sglang/PR-24611.md), [Support spec v2 for FlashMLA speculative decoding](../sources/prs/sglang/PR-24640.md), [[Gemma4] Optimize Gemm4 with fused Q/K/V RMSNorm + per-expert FP8 ckpt loader](../sources/prs/sglang/PR-24696.md), [[codex] Optimize hidden-size 512 RMSNorm dispatch](../sources/prs/sglang/PR-24710.md), [Add FlashInfer SM90 cutlass MXFP4 MoE backend (W4A16) for GPT-OSS + DeepSeek-V4](../sources/prs/sglang/PR-24816.md), [[attn backend] Integrate tokenspeed_mla prefill/decode kernels (fp8 kv cache, blackwell)](../sources/prs/sglang/PR-24925.md), [Amd/deepseek v4 rebase main 0509](../sources/prs/sglang/PR-24933.md), [[rebase]Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-24986.md), [perf(nvfp4): free unused source scales after weight processing](../sources/prs/sglang/PR-25107.md), [Fix AMX GQA extend attention](../sources/prs/sglang/PR-25180.md), [[MUSA][Diffusion] Improve wan model inference speed using torch.compile](../sources/prs/sglang/PR-25256.md), [Support Gemma4 Pipeline Parallelism](../sources/prs/sglang/PR-25284.md), [Fix EPLB mapping for TopK paths](../sources/prs/sglang/PR-25285.md), [[Gemma4]: Fix FP8 Triton scale layout](../sources/prs/sglang/PR-25286.md), [[NSA] Avoid repeated NSA MQA logits memory queries](../sources/prs/sglang/PR-25299.md), [[Intel GPU] Enable DeepSeek V4 Inference on XPU](../sources/prs/sglang/PR-25336.md), [[AMD] test(sgl-kernel): seed RNG on ROCm in test_moe_topk_sigmoid to fix tie-break flake](../sources/prs/sglang/PR-25356.md), [[AMD] Enable shared-experts fusion with new KIMI-K2.5-MXFP4 model.](../sources/prs/sglang/PR-25390.md), [[codex] Update Wan2.2 ModelOpt CI checkpoints](../sources/prs/sglang/PR-25483.md), [Support draft extend cuda graph for tokenspeed_mla attention backend](../sources/prs/sglang/PR-25489.md), [Update logging for inplace setting in MoE layer](../sources/prs/sglang/PR-25499.md), [[misc] Throw error when single batch overlap is enabled on Hopper ](../sources/prs/sglang/PR-25509.md), [Fix logging for inplace setting in the flashInfer-trtllm backend](../sources/prs/sglang/PR-25522.md), [[Bug Fix] Align glm4_moe_nextn NPU MTP loading with qwen3 MTP](../sources/prs/sglang/PR-25524.md), [[MoE Refactor] Migrate flashinfer_cutedsl + DeepEP to MoeRunner](../sources/prs/sglang/PR-25525.md), [Use DeepGEMM BF16 for unquantized DeepEP LL MoE](../sources/prs/sglang/PR-25540.md), [amd/deepseek_v4 27/N [fix] Reduce Triton autotune configs for faster first-time server launch](../sources/prs/sglang/PR-25554.md), [Use triton_attn as default vision attention on B300 (SM103)](../sources/prs/sglang/PR-25570.md), [[Benchmark] Add SGLANG_SIMULATE_UNIFORM_EXPERTS for balanced expert routing with dummy weights](../sources/prs/sglang/PR-25571.md), [Introduce SchedulerDPAttnAdapter to own DP-attention state](../sources/prs/sglang/PR-25611.md), [Move DP-attention adapter methods to SchedulerDPAttnAdapter](../sources/prs/sglang/PR-25612.md), [[SP] Fix runtime_max_tokens_per_rank for sequence parallelism](../sources/prs/sglang/PR-25685.md), [Add no_combine support to cutlass_moe_fp4](../sources/prs/sglang/PR-25688.md), [fix (jit kernel): elementwise activation C++ error](../sources/prs/sglang/PR-25695.md), [[diffusion] Fix GLM-Image /v1/images/edits support](../sources/prs/sglang/PR-25697.md), [[Codex] Remove stale DeepSeek V4 JIT kernels](../sources/prs/sglang/PR-25764.md), [[Refactor] Rename NSA → DSA: user-facing aliases, file/class/import rename](../sources/prs/sglang/PR-25821.md), [[Refactor] Pass PP start_layer via model constructor instead of forward_batch.token_to_kv_pool](../sources/prs/sglang/PR-25825.md), [[Test] Stage-a sanity kits; consolidate core/ + models_e2e/ tests](../sources/prs/sglang/PR-25831.md), [Allow local cutlass directory to be used in sgl-kernel build](../sources/prs/sglang/PR-3037.md), [sync the upstream updates of flashinfer](../sources/prs/sglang/PR-3051.md), [feat: integrate gemm_fp8 kernel into gemm](../sources/prs/sglang/PR-3056.md), [Apply sgl w8a8 fp8 kernel](../sources/prs/sglang/PR-3148.md), [integrate blockwise fp8 kernel](../sources/prs/sglang/PR-3529.md), [feat: support flashinfer mla attention for deepseek v3](../sources/prs/sglang/PR-3550.md), [update flashinfer-python](../sources/prs/sglang/PR-3557.md), [feat: support flashinfer mla with prefix cache](../sources/prs/sglang/PR-3643.md), [add control for cutlass fp8 blockwise gemm](../sources/prs/sglang/PR-3727.md), [Feature DeepSeek V3/R1 INT8 Quantization (block-wise)](../sources/prs/sglang/PR-3730.md), [[Feature] DeepSeek V3/R1 INT8 Quantization (channel-wise) ](../sources/prs/sglang/PR-3888.md), [Support FP4 gemm (1/2)](../sources/prs/sglang/PR-3899.md), [upgrade flashinfer v0.2.2.post1](../sources/prs/sglang/PR-3934.md), [[tools] add fp8 max/min constant in utils](../sources/prs/sglang/PR-3959.md), [Add fast decode plan for flashinfer mla](../sources/prs/sglang/PR-3987.md), [Hierarchical Caching supports MLA](../sources/prs/sglang/PR-4009.md), [[Revision] Add fast decode plan for flashinfer mla ](../sources/prs/sglang/PR-4012.md), [Support overlapping two batches](../sources/prs/sglang/PR-4068.md), [DeepGemm integrate to gemm](../sources/prs/sglang/PR-4165.md), [linear support deepgemm](../sources/prs/sglang/PR-4199.md), [Accelerate FP8 CUDA Kernel by 20-28%](../sources/prs/sglang/PR-4215.md), [Clean up fp8 support](../sources/prs/sglang/PR-4230.md), [[Feature] Integrate DeepEP into SGLang](../sources/prs/sglang/PR-4232.md), [upgrade flashinfer 0.2.3](../sources/prs/sglang/PR-4317.md), [[FIX] fix incorrect output when enable both deepgemm and torch compile](../sources/prs/sglang/PR-4359.md), [[ROCm] fix dtype](../sources/prs/sglang/PR-4510.md), [Create col-major and tma-aligned x_scale for deep_gemm.gemm_fp8_fp8_bf16_nt](../sources/prs/sglang/PR-4515.md), [Add deepseek style fused moe group gate selection kernel](../sources/prs/sglang/PR-4530.md), [[Fix] Fix raw_bs bug when using flashinfer mla and eagle](../sources/prs/sglang/PR-4557.md), [avoid cudaStreamSynchronize in DeepSeekV2AttentionMLA](../sources/prs/sglang/PR-4577.md), [[quantization] fix channelwise conversion with scalar weight scale](../sources/prs/sglang/PR-4596.md), [Set deepgemm to the default value in the hopper architecture.](../sources/prs/sglang/PR-4613.md), [Optimize Permute Kernel in DeepEP](../sources/prs/sglang/PR-4643.md), [Fix loading KV quantization scale; Enable modelopt kv cache](../sources/prs/sglang/PR-4686.md), [[Model] Adding Qwen3 and Qwen3MoE](../sources/prs/sglang/PR-4693.md), [support cmake for sgl-kernel](../sources/prs/sglang/PR-4706.md), [[Feature] Support DeepEP Low Latency](../sources/prs/sglang/PR-4767.md), [Support (1 <= dp < tp) in the dp attention in DeepEP](../sources/prs/sglang/PR-4770.md), [Introduce moe_dense_tp_size to fix dense layer errors in DeepSeek V3 + 4x8xH100](../sources/prs/sglang/PR-4836.md), [Feat/support encoder model (like bert)](../sources/prs/sglang/PR-4887.md), [Add DeepSeek V3/R1 shared experts fusion](../sources/prs/sglang/PR-4918.md), [[Fix] DeepEP Compatibility with Low Latency](../sources/prs/sglang/PR-5068.md), [reduce moe_align_block_size_kernel small batch mode overhead](../sources/prs/sglang/PR-5086.md), [Support MHA with chunked prefix cache for DeepSeek chunked prefill](../sources/prs/sglang/PR-5113.md), [Blackwell Cutlass MLA kernel](../sources/prs/sglang/PR-5142.md), [Add optimized native kernels in sgl-kernel](../sources/prs/sglang/PR-5150.md), [feat: add DeepGEMM build warning](../sources/prs/sglang/PR-5176.md), [feat: use fa3 mla by default on hopper](../sources/prs/sglang/PR-5210.md), [[Fix] Turn off DeepGEMM by default](../sources/prs/sglang/PR-5263.md), [[1/2] Add FP8 Blockscale MoE CUTLASS kernel for Blackwell](../sources/prs/sglang/PR-5281.md), [fix: use deepgemm only on hopper](../sources/prs/sglang/PR-5310.md), [Add Speculative Decoding Eagle3 topk > 1](../sources/prs/sglang/PR-5318.md), [fix: determine if flashinfer is installed](../sources/prs/sglang/PR-5336.md), [Fix DeepGEMM masked cannot be run on groups not being multiple or 4](../sources/prs/sglang/PR-5340.md), [[perf] experimental enhance fp8 per-tensor quant](../sources/prs/sglang/PR-5370.md), [apply fused moe gate in ds v3/r1](../sources/prs/sglang/PR-5371.md), [[PD Bug] fix MLA get_contiguous_buf_infos error](../sources/prs/sglang/PR-5384.md), [Add Cutlass MLA attention backend](../sources/prs/sglang/PR-5390.md), [[PD] Fix dynamic port support and MLA buffer for Mooncake](../sources/prs/sglang/PR-5415.md), [[Feat] upgrade pytorch2.6](../sources/prs/sglang/PR-5417.md), [BLackwell cutlass mla: Add check for bad page size/block num combinations](../sources/prs/sglang/PR-5431.md), [[perf] introduce deep gemm group_gemm_masked as gemm](../sources/prs/sglang/PR-5432.md), [Avoid computing lse in Ragged Prefill when there's no prefix.](../sources/prs/sglang/PR-5476.md), [Fix sampler nan check when calling top_k_top_p_sampling_from_probs](../sources/prs/sglang/PR-5546.md), [[feature] enable pre compile jit deep_gemm](../sources/prs/sglang/PR-5580.md), [[fix] force use deepgemm in compile_deep_gemm](../sources/prs/sglang/PR-5618.md), [ DeepEP normal support deepgemm-contiguous](../sources/prs/sglang/PR-5626.md), [Turn on DeepGemm By Default and Update Doc](../sources/prs/sglang/PR-5628.md), [[perf] dsv3 bmm fallback to bf16](../sources/prs/sglang/PR-5662.md), [[2/2] Add python wrapper for CUTLASS FP8 Blockscale MoE Kernel. ](../sources/prs/sglang/PR-5694.md), [[PP] Add pipeline parallelism](../sources/prs/sglang/PR-5724.md), [Fuse MLA set kv cache kernel](../sources/prs/sglang/PR-5748.md), [opt flashinfer mla cat](../sources/prs/sglang/PR-5822.md), [Cutlass MLA decode - fix dtype error](../sources/prs/sglang/PR-5868.md), [[Fix] Fix a bug for flashmla to run R1 model](../sources/prs/sglang/PR-5875.md), [Improve dp attention port assignment scheme](../sources/prs/sglang/PR-5889.md), [[qwen3] support qwen3 ep moe](../sources/prs/sglang/PR-5917.md), [[Feat] Enable PDL automatically on Hopper architecture](../sources/prs/sglang/PR-5981.md), [KV‑Cache (MHA, MLA): add missing start_layer / end_layer fields to MHATokenToKVPoolHost and MLATokenToKVPoolHost](../sources/prs/sglang/PR-6016.md), [feat: mtp support dp-attention](../sources/prs/sglang/PR-6081.md), [[1/2] Add Kernel support for Cutlass based Fused FP4 MoE](../sources/prs/sglang/PR-6093.md), [feat: add dp attention support for Qwen 2/3 MoE models, fixes #6088](../sources/prs/sglang/PR-6121.md), [Reduce MoE memory usage](../sources/prs/sglang/PR-6147.md), [enable auto-round quantization model](../sources/prs/sglang/PR-6226.md), [Enable FlashInfer support encoder models and add head_dim padding workaround](../sources/prs/sglang/PR-6230.md), [fix: fix MLA for ShardedModelLoader/RemoteModelLoader](../sources/prs/sglang/PR-6287.md), [[Fix] Improve dependencies for Blackwell image](../sources/prs/sglang/PR-6334.md), [reduce torch.zeros overhead in moe align block size kernel](../sources/prs/sglang/PR-6369.md), [[Feature] Comprehensive Hybrid Parallelism Support](../sources/prs/sglang/PR-6389.md), [Add fp8 fused_experts kernel for CPU in sgl-kernel and add UT](../sources/prs/sglang/PR-6404.md), [Fix bug of deepseek-v3 under DP+EP mode with large batchsize/seqlen](../sources/prs/sglang/PR-6449.md), [Fix topk inference performance reduce](../sources/prs/sglang/PR-6474.md), [[Feature] Support Flashinfer fp8 blockwise GEMM kernel on Blackwell](../sources/prs/sglang/PR-6479.md), [qwen3moe support two batch overlap](../sources/prs/sglang/PR-6598.md), [Refine pre_reorder_triton_kernel slightly to improve performance](../sources/prs/sglang/PR-6627.md), [[CPU] [BF16] Call fused_experts_cpu, weight_packed_linear and bmm_cpu kernel in DeepSeek model](../sources/prs/sglang/PR-6641.md), [Fix DeepEP error in Qwen 3 MoE models](../sources/prs/sglang/PR-6673.md), [[EP] Add cuda kernel for moe_ep_pre_reorder](../sources/prs/sglang/PR-6699.md), [Fix PP for Qwen3 MoE](../sources/prs/sglang/PR-6709.md), [Set `num_fused_shared_experts` as `num_shared_experts` when shared_experts fusion is not disabled](../sources/prs/sglang/PR-6736.md), [[CPU] add optimizations for INT8 and FP8 DeepSeek](../sources/prs/sglang/PR-6769.md), [[CPU] support the case where num_attention_heads or intermediate_size is not divisible by the TP size](../sources/prs/sglang/PR-6771.md), [Support token-level quantization for EP MoE](../sources/prs/sglang/PR-6782.md), [[PD] Add different TP sizes support for no-MLA models](../sources/prs/sglang/PR-6793.md), [Correctly abort the failed grammar requests & Improve the handling of abort](../sources/prs/sglang/PR-6803.md), [feat: integrate deepgemm into EPMoE](../sources/prs/sglang/PR-6821.md), [CPU: map changes from developing branch in sgl-kernel](../sources/prs/sglang/PR-6833.md), [[EP] Add cuda kernel for moe_ep_post_reorder](../sources/prs/sglang/PR-6837.md), [Fix AWQ Dequant and Weight Loading of deepseek v2](../sources/prs/sglang/PR-6842.md), [[DeepseekR1-FP4] Add Support for nvidia/DeepSeekR1-FP4 model](../sources/prs/sglang/PR-6853.md), [Use deepgemm instead of triton for fused_qkv_a_proj_with_mqa](../sources/prs/sglang/PR-6890.md), [Add a CUDA kernel for fusing mapping and weighted sum for MoE.](../sources/prs/sglang/PR-6916.md), [[sgl-kernel] Add cuda kernel for moe_ep_silu_and_mul](../sources/prs/sglang/PR-6919.md), [[perf][sgl-kernel] extend cutlass_mla_decode to support num_head < 128](../sources/prs/sglang/PR-6929.md), [[Feature] Support Flashinfer fmha on Blackwell](../sources/prs/sglang/PR-6930.md), [[sgl-kernel] update deepgemm](../sources/prs/sglang/PR-6942.md), [chore: upgrade flashinfer v0.2.6.post1 jit](../sources/prs/sglang/PR-6958.md), [Fuse routed scaling factor in deepseek](../sources/prs/sglang/PR-6970.md), [Fix cutlass MLA gets almost zero accuracy](../sources/prs/sglang/PR-6998.md), [Fix torchvision version for Blackwell](../sources/prs/sglang/PR-7015.md), [Update default settings for blackwell](../sources/prs/sglang/PR-7023.md), [Clean up server_args.py](../sources/prs/sglang/PR-7037.md), [Fix positional argument](../sources/prs/sglang/PR-7093.md), [feat: update blackwell setup](../sources/prs/sglang/PR-7119.md), [fix amd EP MoE FP8 issue](../sources/prs/sglang/PR-7125.md), [Enable ModelOpt Llama4 fp8 checkpoint deployment in SGLang](../sources/prs/sglang/PR-7129.md), [Enable native ModelOpt quantization support (1/3) ](../sources/prs/sglang/PR-7149.md), [[amd] Opt dsv3 moe](../sources/prs/sglang/PR-7160.md), [Fix Deepseek R1 0528 FP4 tensor name mismatch issue during weights loading.](../sources/prs/sglang/PR-7164.md), [Support new DeepGEMM](../sources/prs/sglang/PR-7172.md), [Tiny let DeepGEMM scale checks cover more cases](../sources/prs/sglang/PR-7182.md), [chore: upgrade sgl-kernel v0.1.8.post2](../sources/prs/sglang/PR-7186.md), [[AMD] Fail gracefully when AITER is unavailable gfx90a GPUs](../sources/prs/sglang/PR-7187.md), [Fix a minor bug related to DeepGEMM upgrade](../sources/prs/sglang/PR-7191.md), [Fix error when disabling new DeepGEMM](../sources/prs/sglang/PR-7198.md), [Fix grammar abort & Minor style fixes](../sources/prs/sglang/PR-7204.md), [feat: support compatibility between MTP and two-batch-overlap](../sources/prs/sglang/PR-7225.md), [Minor style and doc fix](../sources/prs/sglang/PR-7228.md), [[fix] fix DeepGEMM blackwell input quant & ut & fix style and log](../sources/prs/sglang/PR-7247.md), [[AMD] add aiter fused moe in DeepEP path](../sources/prs/sglang/PR-7268.md), [Support NVFP4 quantized dense models on AMD CDNA2/CDNA3 GPUs](../sources/prs/sglang/PR-7302.md), [Let EP prefill support new DeepGEMM](../sources/prs/sglang/PR-7310.md), [Kernels for efficient KV cache IO](../sources/prs/sglang/PR-7313.md), [FlashInfer NVFP4 MoE with EP & 2-stream shared expert](../sources/prs/sglang/PR-7327.md), [Fix MTP with Deepseek R1 Fp4](../sources/prs/sglang/PR-7376.md), [Quick fix for DeepGemm requant to also cover MTP.](../sources/prs/sglang/PR-7378.md), [Fix torch compile run](../sources/prs/sglang/PR-7391.md), [[AMD][Quantization] Add `int4fp8_moe` online quantization on ROCm](../sources/prs/sglang/PR-7392.md), [Fix CPU offloading for MLA memory pool](../sources/prs/sglang/PR-7409.md), [Fuse sorted_token_ids padding to moe_align_block_size kernel](../sources/prs/sglang/PR-7437.md), [Support non-contiguous query input for extend/decode attention](../sources/prs/sglang/PR-7462.md), [Add Tencent HunYuanMoEV1 model support](../sources/prs/sglang/PR-7549.md), [[b200] support trt-llm allreduce fuse rms_norm_add kernel](../sources/prs/sglang/PR-7621.md), [Add dsv3 router gemm kernel](../sources/prs/sglang/PR-7627.md), [Add dsv3 fused a gemm to sgl-kernel](../sources/prs/sglang/PR-7630.md), [[Feature] Layer-wise Prefill](../sources/prs/sglang/PR-7634.md), [Apply dsv3_fused_a_gemm kernel](../sources/prs/sglang/PR-7635.md), [[Feature] CUDA Green Context Support](../sources/prs/sglang/PR-7649.md), [chore: upgrade flashinfer v0.2.7 jit](../sources/prs/sglang/PR-7663.md), [Add fp4 quantize before all-gather for Flashinfer cutlass MoE DP (max throughput)](../sources/prs/sglang/PR-7667.md), [Integrate triton moe kernel](../sources/prs/sglang/PR-7689.md), [feat: support DeepSeek-R1-W4AFP8 model with ep-moe mode](../sources/prs/sglang/PR-7762.md), [[1/n]: add cutlass W4A8 moe kernel for hopper architecture](../sources/prs/sglang/PR-7772.md), [Qwen FP8/NVFP4 ModelOPT Quantization support](../sources/prs/sglang/PR-7912.md), [[feat] Support tp mode for DeepSeek-R1-W4AFP8](../sources/prs/sglang/PR-8118.md), [[fix] fix modelopt fp4 on b200](../sources/prs/sglang/PR-8195.md), [[1/N]Support DeepSeek-R1 w4a8 normal deepep](../sources/prs/sglang/PR-8247.md), [Support triton kernels v3.4.0 for fused_moe](../sources/prs/sglang/PR-8258.md), [[2/N]Support DeepSeek-R1 w4a8 low latency deepep](../sources/prs/sglang/PR-8464.md), [Update cutlass_moe.py](../sources/prs/sglang/PR-8535.md), [Update cutlass_moe.py](../sources/prs/sglang/PR-8545.md), [[NVIDIA] Add Low Latency NVFP4 decode kernels from Flashinfer](../sources/prs/sglang/PR-8552.md), [TRTLLM-MLA FP8 path](../sources/prs/sglang/PR-8638.md), [feat: support cutlass_moe_fp8 kernel for fusedmoe in sm90](../sources/prs/sglang/PR-8678.md), [fuse allreduce and residual_rmsnorm](../sources/prs/sglang/PR-8731.md), [Fix mismatch between padded_scales shape and reshape dimensions in modelopt quantization](../sources/prs/sglang/PR-8766.md), [feat: add trtllm-gen mha from direct call](../sources/prs/sglang/PR-8782.md), [[Perf] Auto enable best flashinfer mxfp4 kernel in b200](../sources/prs/sglang/PR-8898.md), [Fix hopper launch gpt-oss model illegal memory](../sources/prs/sglang/PR-8908.md), [[NVIDIA] Fix missing `get_col_major_tma_aligned_tensor` for Blackwell deepgemm in EpMoE](../sources/prs/sglang/PR-8955.md), [optimize: reduce shulffle and quantization overhead in cutlass_moe sm90](../sources/prs/sglang/PR-8962.md), [[fix] fix enable_pdl for blackwell](../sources/prs/sglang/PR-9011.md), [[sgl-kernel] Support FlashInfer top_k_top_p_sampling_from_logits](../sources/prs/sglang/PR-9060.md), [Faster weight processing (trtllm-gen moe nvfp4)](../sources/prs/sglang/PR-9162.md), [[NVIDIA] [3/N] Nvfp4 Masked Gemm: Add flashinfer grouped_gemm_nt_masked ](../sources/prs/sglang/PR-9199.md), [[NVIDA] [1/N] Nvfp4 Masked Gemm: Add quant op for the flashinfer grouped gemm](../sources/prs/sglang/PR-9200.md), [[fix]: fix cutlass moe ut and and Opt H20 cutlass groupGemm performance](../sources/prs/sglang/PR-9272.md), [Support trtllm_allreduce_fusion in flashinfer for cuda<12.8](../sources/prs/sglang/PR-9339.md), [Fix FP4 inference corruption issue in glm4.5-air model](../sources/prs/sglang/PR-9346.md), [Support DP attention with GPT-OSS](../sources/prs/sglang/PR-9359.md), [[fix] Fix mxfp4 triton MoE tp bug](../sources/prs/sglang/PR-9473.md), [Optimize moe_sum_reduce_kernel](../sources/prs/sglang/PR-9477.md), [fix: blackwell dsv3 fp8 issue temporary solution](../sources/prs/sglang/PR-9530.md), [[NVIDIA] [2/N] Optimize `silu_and_mul_scaled_fp4_grouped_quant` perf](../sources/prs/sglang/PR-9556.md), [Update CUTLASS 4.2 & Enable K-Major Scale Factor for SM90 FP8 Blockwise Group GEMM](../sources/prs/sglang/PR-9559.md), [Tiny fix wrong comments](../sources/prs/sglang/PR-9589.md), [Single Batch Overlap for MoE Models](../sources/prs/sglang/PR-9660.md), [fix mooncake store mla zero copy meta](../sources/prs/sglang/PR-9678.md), [move is_sm90_supported/is_sm100_supported to python/sglang/srt/utils.py](../sources/prs/sglang/PR-9679.md), [[ModelOpt] Fix Weight Loading for DSR1-FP4 Quantization](../sources/prs/sglang/PR-9712.md), [[CPU] Add FP8 Bmm support](../sources/prs/sglang/PR-9744.md), [[Model] Support Meituan LongCat-Flash && LongCat-Flash-MTP](../sources/prs/sglang/PR-9824.md), [perf: Avoid unnecessary data type conversions for DeepSeek-V3 on Blackwell](../sources/prs/sglang/PR-9834.md), [support using fa4 on deepseek on blackwell](../sources/prs/sglang/PR-9928.md), [[Fix] DeepSeek EP accuracy issue on B200 GPUs](../sources/prs/sglang/PR-9946.md), [Enable native ModelOpt quantization support (2/3)](../sources/prs/sglang/PR-9991.md), [[Feature] Support `cp.reduce.async.bulk.tensor`](../sources/prs/tilelang/PR-1667.md), [[Feature] Support tcgen5mma lowering for `.kind::i8`](../sources/prs/tilelang/PR-1764.md), [[BugFix] Fix Hopper TMA lowering without warp specialization](../sources/prs/tilelang/PR-1840.md), [[CUDA] Support tcgen5mma gemm ts](../sources/prs/tilelang/PR-1866.md), [[Feature] Support cluster launch, query, synchronization and barrier operations](../sources/prs/tilelang/PR-1874.md), [[Feature] Add T.copy_cluster to support TMA multicast and SM-to-SM cluster copy](../sources/prs/tilelang/PR-1908.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [[Bugfix] Fix CuTeDSL autotune cache invalid ELF header (#1967)](../sources/prs/tilelang/PR-1972.md), [[Feature] Support TMA store in T.tma_copy()](../sources/prs/tilelang/PR-1981.md), [[Transform] Add InjectTcgen05Fence pass](../sources/prs/tilelang/PR-2003.md), [[Backend] Refactor gemm_sp](../sources/prs/tilelang/PR-2048.md), [[CUDA] Support int4 `T.gemm`](../sources/prs/tilelang/PR-2063.md), [[CUDA] Improve int4 GEMM lowering and packed codegen support](../sources/prs/tilelang/PR-2073.md), [[TMA] Support FP4 TensorMap TMA copies](../sources/prs/tilelang/PR-2107.md), [feat: auto-vectorize bf16/fp16 reduce with packed add2 intrinsics](../sources/prs/tilelang/PR-2112.md), [[CUDA][TMA] Add TMA tile::gather4 / tile::scatter4 support](../sources/prs/tilelang/PR-2129.md), [[Python] Drop Python 3.9 support](../sources/prs/tilelang/PR-2218.md), [[Perf] Enable fast math in sparse MLA example](../sources/prs/tilelang/PR-2219.md), [[Core] Support fully transparent sleep mode](../sources/prs/vllm/PR-11743.md), [[ROCm][MoE] moe tuning support for rocm](../sources/prs/vllm/PR-12049.md), [[Kernel] Flash Attention 3 Support](../sources/prs/vllm/PR-12093.md), [Add: Support for Sparse24Bitmask Compressed Models](../sources/prs/vllm/PR-12097.md), [[Hardware][Gaudi][Feature] Enable Dynamic MoE for Mixtral](../sources/prs/vllm/PR-12303.md), [[Core] Optimizing cross-attention `QKVParallelLinear` computation](../sources/prs/vllm/PR-12325.md), [[Bugfix] Disable w16a16 2of4 sparse CompressedTensors24](../sources/prs/vllm/PR-12417.md), [[Misc][MoE] add Deepseek-V3 moe tuning support](../sources/prs/vllm/PR-12558.md), [Expert Parallelism (EP) Support for DeepSeek Models](../sources/prs/vllm/PR-12583.md), [Apply torch.compile to fused_moe/grouped_topk](../sources/prs/vllm/PR-12637.md), [Disable chunked prefill and/or prefix caching when MLA is enabled ](../sources/prs/vllm/PR-12642.md), [[AMD][ROCm] Enable DeepSeek model on ROCm](../sources/prs/vllm/PR-12662.md), [Squelch MLA warning for Compressed-Tensors Models](../sources/prs/vllm/PR-12704.md), [[VLM] Add MLA with pure RoPE support for deepseek-vl2 models](../sources/prs/vllm/PR-12729.md), [[Misc] Update w2 scale loading for GPTQMarlinMoE](../sources/prs/vllm/PR-12757.md), [[Bugfix] Better FP8 supported defaults](../sources/prs/vllm/PR-12796.md), [[Attention] Use FA3 for MLA on Hopper](../sources/prs/vllm/PR-12807.md), [[Model] Deepseek GGUF support ](../sources/prs/vllm/PR-13167.md), [Expand MLA to support most types of quantization](../sources/prs/vllm/PR-13181.md), [[Quant][Perf] Use moe_wna16 kernel by default for MoEs with many experts](../sources/prs/vllm/PR-13236.md), [[Bugfix] Massage MLA's usage of flash attn for RoCM](../sources/prs/vllm/PR-13310.md), [[NVIDIA] Support nvfp4 tcgen05 gemm](../sources/prs/vllm/PR-13571.md), [[Bugfix] Fix max_num_batched_tokens for MLA](../sources/prs/vllm/PR-13620.md), [[Kernel] Optimize moe intermediate_cache usage](../sources/prs/vllm/PR-13625.md), [[ROCM] fix native attention function call](../sources/prs/vllm/PR-13650.md), [[BugFix] Illegal memory access for MoE On H20](../sources/prs/vllm/PR-13693.md), [[Bugfix] Support MLA for CompressedTensorsWNA16](../sources/prs/vllm/PR-13725.md), [[V1] V1 Enablement Oracle ](../sources/prs/vllm/PR-13726.md), [Fix CompressedTensorsWNA16MoE with grouped scales](../sources/prs/vllm/PR-13769.md), [Fix precommit fail in fused_moe intermediate_cache2 chunking](../sources/prs/vllm/PR-13772.md), [[Bugfix][Quantization] Fix FP8 + EP](../sources/prs/vllm/PR-13784.md), [add tcgen05 support for tcgen05 fp8 gemm](../sources/prs/vllm/PR-13798.md), [Fix mla prefill context performance](../sources/prs/vllm/PR-13897.md), [[Misc] Print FusedMoE detail info](../sources/prs/vllm/PR-13974.md), [[core] moe fp8 block quant tuning support](../sources/prs/vllm/PR-14068.md), [[V1] Implement sliding window attention in kv_cache_manager](../sources/prs/vllm/PR-14097.md), [[v1] Add comments to the new ragged paged attention Pallas kernel](../sources/prs/vllm/PR-14155.md), [[V1][TPU] TPU multimodal model support for ragged attention](../sources/prs/vllm/PR-14158.md), [[V1][TPU] Support V1 Sampler for ragged attention](../sources/prs/vllm/PR-14227.md), [[Hardware] Update the flash attn tag to support Blackwell](../sources/prs/vllm/PR-14244.md), [[BugFix] Fix prefix caching V0 MLA](../sources/prs/vllm/PR-14255.md), [[Misc] Add Qwen2MoeForCausalLM moe tuning support ](../sources/prs/vllm/PR-14276.md), [[Hardware][TPU]Enable ragged paged attention kernel and resolve recompilation issue](../sources/prs/vllm/PR-14310.md), [[Bug] Fix Attention when ignored in by quant_method](../sources/prs/vllm/PR-14313.md), [[ROCm] Enable chunked prefill/paged attention in MLA on ROCm](../sources/prs/vllm/PR-14316.md), [[Perf] Reduce MLA CPU overheads in V1](../sources/prs/vllm/PR-14384.md), [[ROCm][Kernel] MoE weights padding](../sources/prs/vllm/PR-14454.md), [[Bugfix] DeepSeek Accuracy](../sources/prs/vllm/PR-14476.md), [[Perf] Improve MLA on V1](../sources/prs/vllm/PR-14540.md), [[Quantization][FP8] Adding support for fp8 gemm layer input in fp8](../sources/prs/vllm/PR-14578.md), [[Model] Add support for Gemma 3](../sources/prs/vllm/PR-14660.md), [[Bugfix][IPEX] Add `VLLM_CPU_MOE_PREPACK` to allow disabling MoE prepack when CPU does not support it](../sources/prs/vllm/PR-14681.md), [[Kernel][CPU] CPU MLA](../sources/prs/vllm/PR-14744.md), [[Attention] MLA get rid of materialization](../sources/prs/vllm/PR-14770.md), [[Attention] Get rid of mla cache alignment](../sources/prs/vllm/PR-14842.md), [[V1][BugFix] Detect interleaved sliding window attention](../sources/prs/vllm/PR-14896.md), [[V1] Default MLA to V1](../sources/prs/vllm/PR-14921.md), [[FEAT][ROCm] Integrate Fused MoE Kernels from AITER](../sources/prs/vllm/PR-14967.md), [[FEAT] [ROCm]: Add AITER Block-Scaled GEMM Feature](../sources/prs/vllm/PR-14968.md), [[FEAT][ROCm] Integrate Paged Attention Kernel from AITER](../sources/prs/vllm/PR-15001.md), [[Bugfix] Fix incorrect qwen2.5-vl attention mask pre-computation](../sources/prs/vllm/PR-15200.md), [[Bugfix] Fix use_cascade_attention handling for Alibi-based models on vllm/v1](../sources/prs/vllm/PR-15211.md), [[Misc] Add attention mask pre-computation optimization back to Qwen2.5-VL](../sources/prs/vllm/PR-15273.md), [[Model] Add Qwen3 and Qwen3MoE](../sources/prs/vllm/PR-15289.md), [Fix non-contiguous input passed to Marlin kernel](../sources/prs/vllm/PR-15319.md), [[FEAT] [ROCm] Add AITER int8 scaled gemm kernel](../sources/prs/vllm/PR-15433.md), [Use Cache Hinting for fused_moe kernel](../sources/prs/vllm/PR-15511.md), [[moe][quant] add weight name case for offset](../sources/prs/vllm/PR-15515.md), [[Quantization] Fp8 Channelwise Dynamic Per Token GroupedGEMM](../sources/prs/vllm/PR-15587.md), [[TPU] Support sliding window and logit soft capping in the paged attention kernel for TPU.](../sources/prs/vllm/PR-15732.md), [[V1] TPU - Fix fused MOE](../sources/prs/vllm/PR-15834.md), [[Bugfix] Fix cache block size calculation for CPU MLA](../sources/prs/vllm/PR-15848.md), [[FEAT][ROCm]: Support AITER MLA](../sources/prs/vllm/PR-15893.md), [[Hardware][Gaudi][BugFix] fix arguments of hpu fused moe](../sources/prs/vllm/PR-15945.md), [Add support to modelopt quantization of Mixtral model](../sources/prs/vllm/PR-15961.md), [[NVIDIA] Support Cutlass MLA for Blackwell GPUs](../sources/prs/vllm/PR-16032.md), [[Kernel] Use moe_wna16 kernel for compressed tensors wna16 moe models](../sources/prs/vllm/PR-16038.md), [[Model] use AutoWeightsLoader for phimoe,qwen2_moe,qwen3_moe](../sources/prs/vllm/PR-16203.md), [[Hardware][AMD] Improve OAM device ID + llama4 Maverick MOE tuning](../sources/prs/vllm/PR-16263.md), [[Llama4] Enable attention temperature tuning by default for long context (>32k)](../sources/prs/vllm/PR-16439.md), [[MLA] Simplification to batch P/D reordering](../sources/prs/vllm/PR-16673.md), [[ROCM] enable aiter fused moe kernel for llama4 bf16 checkpoints](../sources/prs/vllm/PR-16674.md), [[ROCm] Add aiter tkw1 kernel for Llama4 fp8](../sources/prs/vllm/PR-16727.md), [Support W8A8 INT8 MoE for compressed-tensors](../sources/prs/vllm/PR-16745.md), [[FEAT] [ROCm]: AITER Fused MOE V1 Support](../sources/prs/vllm/PR-16752.md), [[Bugfix] Fix moe weight losing all extra attrs after `process_weights_after_loading`.](../sources/prs/vllm/PR-16854.md), [[Bugfix] Add contiguous call inside rope kernel wrapper](../sources/prs/vllm/PR-17091.md), [[FEAT] [ROCm]: Add AITER CK 2 Stages MoE support](../sources/prs/vllm/PR-17110.md), [[Bugfix] gemma[2,3] interleaved attention when sliding window is disabled](../sources/prs/vllm/PR-17180.md), [[Bugfix] Get a specific type of layer from forward context](../sources/prs/vllm/PR-17222.md), [[BugFix] Fix vllm_flash_attn install issues](../sources/prs/vllm/PR-17267.md), [[BugFix] Fix cascade attention - RuntimeError: scheduler_metadata must have shape (metadata_size)](../sources/prs/vllm/PR-17283.md), [[v1] AttentionMetadata for each layer](../sources/prs/vllm/PR-17394.md), [Fix noisy warning for uncalibrated q_scale/p_scale](../sources/prs/vllm/PR-17414.md), [[v1] Pass BlockTable and KVCacheSpec to AttentionMetadataBuilders](../sources/prs/vllm/PR-17483.md), [[BugFix] Fix mla cpu - missing 3 required positional arguments](../sources/prs/vllm/PR-17494.md), [[FEAT][ROCm]: Support AITER MLA on V1 Engine](../sources/prs/vllm/PR-17523.md), [[Bugfix][ROCm] Fix AITER MLA V1](../sources/prs/vllm/PR-17880.md), [[BugFix][AMD] Compatible patch for AITER lib after 04/20](../sources/prs/vllm/PR-17912.md), [[Misc] Add compressed-tensors NVFP4A16 emulation support](../sources/prs/vllm/PR-17914.md), [use ceil_div in cutlass block scaling shape check](../sources/prs/vllm/PR-17918.md), [[v1] Support multiple KV cache groups in GPU model runner](../sources/prs/vllm/PR-17945.md), [[BUG] [ROCm] [MLA] Fix variable name bug due to change in variable name in PR #17483](../sources/prs/vllm/PR-17961.md), [Use NVFP4 Marlin for CompressedTensorsW4A16Fp4](../sources/prs/vllm/PR-18000.md), [[Quantization] Add compressed-tensors NVFP4 support](../sources/prs/vllm/PR-18312.md), [[Model]: Fused MoE for nomic-embed-text-v2-moe](../sources/prs/vllm/PR-18321.md), [[Feature] Expert Parallelism Load Balancer (EPLB)](../sources/prs/vllm/PR-18343.md), [[Bug] Fix moe_sum signature](../sources/prs/vllm/PR-18440.md), [[V1] Support `LLM.apply_model`](../sources/prs/vllm/PR-18465.md), [[Hardware][AMD] integrate aiter chunked prefill into vllm](../sources/prs/vllm/PR-18596.md), [[P/D] Heterogeneous TP](../sources/prs/vllm/PR-18833.md), [[ROCm] [AITER] [Bugfix] Patch for AITER commit `648764942e552a8bb5fe16026703716a81f05374`](../sources/prs/vllm/PR-18990.md), [[Kernel] Support deep_gemm for linear methods](../sources/prs/vllm/PR-19085.md), [[Kernel] Apply torch.Tag.needs_fixed_stride_order only for torch==2.6.0](../sources/prs/vllm/PR-19346.md), [[Core] Support Local Chunked Attention for Hybrid KV Cache](../sources/prs/vllm/PR-19351.md), [[Kernels] Use empty for modular MoE workspaces](../sources/prs/vllm/PR-19667.md), [[Feature] Integrate new deepgemm](../sources/prs/vllm/PR-19820.md), [[Bugfix] Enable PP with AITER+V1](../sources/prs/vllm/PR-19822.md), [[Quantization] Add compressed-tensors emulations support for NVFP4](../sources/prs/vllm/PR-19879.md), [[Quantization] Add compressed-tensors NVFP4 MoE Support](../sources/prs/vllm/PR-19990.md), [Enable V1 for Hybrid SSM/Attention Models](../sources/prs/vllm/PR-20016.md), [[Attention] MLA - Flashinfer Ragged Prefill](../sources/prs/vllm/PR-20034.md), [Add ModelOpt Qwen3 nvfp4 support](../sources/prs/vllm/PR-20101.md), [[Bugfix] Mark 'hidden_states' as mutable in moe_forward registration.](../sources/prs/vllm/PR-20152.md), [[Bugfix] Fix Maverick correctness by filling zero to cache space in cutlass_moe](../sources/prs/vllm/PR-20167.md), [[Nixl] Heterogeneous TP support FlashInfer](../sources/prs/vllm/PR-20189.md), [[V1] [ROCm] Enable EP with AITER Fused MoE](../sources/prs/vllm/PR-20270.md), [Support Llama 4 for cutlass_moe_fp4](../sources/prs/vllm/PR-20453.md), [Support Llama 4 for fused_marlin_moe](../sources/prs/vllm/PR-20457.md), [[Perf] Reuse workspace for FP8+FP4 Marlin MoE](../sources/prs/vllm/PR-20500.md), [[Bugfix] Fix missing per_act_token parameter in compressed_tensors_moe](../sources/prs/vllm/PR-20509.md), [[feat] enable SM100 CUTLASS block scaled group gemm for smaller batch sizes](../sources/prs/vllm/PR-20640.md), [Integration SM100 FlashInfer fused allreduce RMSNorm](../sources/prs/vllm/PR-20691.md), [GLM-4.5 Model Support](../sources/prs/vllm/PR-20736.md), [[v1][core] Support for attention free models](../sources/prs/vllm/PR-20811.md), [[Feature][EPLB] Add eplb support for Qwen3](../sources/prs/vllm/PR-20815.md), [[Bugfix] Fix a couple PPLX+CUTLASS MoE bugs](../sources/prs/vllm/PR-20825.md), [[Bug] Fix DeepGemm for EP low latency case](../sources/prs/vllm/PR-20833.md), [[Model] Pooling models default to using chunked prefill & prefix caching if supported.](../sources/prs/vllm/PR-20930.md), [[Misc] Qwen MoE model supports LoRA](../sources/prs/vllm/PR-20932.md), [[Bugfix] Switch bailout logic for kv-cache-dtype with SM100 Flashinfer](../sources/prs/vllm/PR-20934.md), [Fall back if flashinfer comm module not found](../sources/prs/vllm/PR-20936.md), [[Bugfix] Fix Mistral3 support on SM100/SM120](../sources/prs/vllm/PR-20998.md), [Add FlashInfer allreduce RMSNorm Quant fusion](../sources/prs/vllm/PR-21069.md), [[Bugfix] Voxtral on Blackwell GPUs (RTX 50 series)](../sources/prs/vllm/PR-21077.md), [[Bugfix] Allocate less memory in non-batched CUTLASS MoE](../sources/prs/vllm/PR-21121.md), [[Attention] Optimize FlashInfer MetadataBuilder Build call](../sources/prs/vllm/PR-21137.md), [[Attention][DBO] Add support for "splitting" the CommonAttentionMetadata](../sources/prs/vllm/PR-21153.md), [[Feature][OCP MX] Support mxfp6 and mixed mxfp6-mxfp4](../sources/prs/vllm/PR-21166.md), [[Bug] DeepGemm: Fix TypeError: per_block_cast_to_fp8() missing 1 required positional argument: 'use_ue8m0' for SM100](../sources/prs/vllm/PR-21187.md), [Support encoder-only models without KV-Cache](../sources/prs/vllm/PR-21270.md), [Fix Flashinfer Allreduce+Norm enable disable calculation based on `fi_allreduce_fusion_max_token_num`](../sources/prs/vllm/PR-21325.md), [Support Tensorrt-LLM MoE fp4 for low-latency](../sources/prs/vllm/PR-21331.md), [[TPU][Bugfix] fix moe layer](../sources/prs/vllm/PR-21340.md), [[Quantization] Enable BNB support for more MoE models](../sources/prs/vllm/PR-21370.md), [Update flashinfer CUTLASS NVFP4 MoE Kernel to use per expert global scaling factor](../sources/prs/vllm/PR-21408.md), [[NVIDIA] Explicitly disable shuffled weights for flashinfer blockscale moe fp8 kernels](../sources/prs/vllm/PR-21411.md), [Updates to Flex + VLLm integration](../sources/prs/vllm/PR-21416.md), [[V1] Fix local chunked attention always disabled](../sources/prs/vllm/PR-21419.md), [[BugFix] Fix shared storage connector load kv only load attention layer](../sources/prs/vllm/PR-21428.md), [update flashinfer to v0.2.9rc1](../sources/prs/vllm/PR-21485.md), [[MoE] More balanced expert sharding](../sources/prs/vllm/PR-21497.md), [[NVIDIA] Fix Llama4 Scout FP4 functionality issues](../sources/prs/vllm/PR-21499.md), [Enable 4bit bnb prequant MOE](../sources/prs/vllm/PR-21548.md), [[Attention] Support multiple attention metadata builders per kv_cache_spec + proper local attention no hybrid kv cache fix](../sources/prs/vllm/PR-21588.md), [Override attention metadata for fast prefill in some KV sharing setups](../sources/prs/vllm/PR-21590.md), [[Feature] Add Flashinfer MoE Support for Compressed Tensor NVFP4](../sources/prs/vllm/PR-21639.md), [[xpu]support moe models on XPU platform](../sources/prs/vllm/PR-21643.md), [support `torch.compile` for bailing moe](../sources/prs/vllm/PR-21664.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv/out Attention Kernel](../sources/prs/vllm/PR-21716.md), [feat: Add Support GPTQ Quantization MOE on ROCM vllm serve](../sources/prs/vllm/PR-21733.md), [[Logs] Change flashinfer sampler logs to once](../sources/prs/vllm/PR-21759.md), [[Perf] Disable chunked local attention by default with llama4](../sources/prs/vllm/PR-21761.md), [[Bugfix] Check NVIDIA artifactory is accessible before using flashinfer cubin kernels](../sources/prs/vllm/PR-21893.md), [[Qwen3] Enable dual-chunk-attention support for Qwen3 models.](../sources/prs/vllm/PR-21924.md), [[BUGFIX] KeyError 'layers.14.mlp.gate.g_idx' for Qwen3-MoE with GPTQ on ROCm](../sources/prs/vllm/PR-22017.md), [[NVIDIA] Support Flashinfer TRT-LLM Prefill Attention Kernel](../sources/prs/vllm/PR-22095.md), [[EPLB] Support ernie4.5-moe](../sources/prs/vllm/PR-22100.md), [[fix] fix correct assertion syntax error in attention utils.](../sources/prs/vllm/PR-22154.md), [[bugfix] fix blackwell deepep installation](../sources/prs/vllm/PR-22255.md), [[Bugfix] Fix MoE BNB version](../sources/prs/vllm/PR-22260.md), [Support encoder_only attention for FlexAttention](../sources/prs/vllm/PR-22273.md), [[Bugfix] Fix 3D input passed into cutlass_scaled_mm](../sources/prs/vllm/PR-22278.md), [[ROCm] Add attention sink to use_rocm_custom_paged_attention](../sources/prs/vllm/PR-22329.md), [[gpt-oss] flashinfer mxfp4](../sources/prs/vllm/PR-22339.md), [Update `flashinfer-python==0.2.10`](../sources/prs/vllm/PR-22389.md), [[Bug] Fix B200 DeepGEMM E8M0 Accuracy Issue](../sources/prs/vllm/PR-22399.md), [[bugfix] Fix Llama3/4 issues caused by FlashInfer 0.2.10](../sources/prs/vllm/PR-22426.md), [[Quantization]: Support compressed-tensors mixed-precision model loading](../sources/prs/vllm/PR-22468.md), [Fix Llama4 FlashInfer FP4 MoE issues](../sources/prs/vllm/PR-22511.md), [[Model] Add Ernie4.5 VL Model Support](../sources/prs/vllm/PR-22514.md), [Quantization: support FP4 quantized models on AMD CDNA2/CDNA3 GPUs](../sources/prs/vllm/PR-22527.md), [Fix torch version check for SM100 mxfp4 ](../sources/prs/vllm/PR-22535.md), [Upgrade FlashInfer to v0.2.11](../sources/prs/vllm/PR-22613.md), [[Bugfix] Fix ModernBert load & Enable sliding window attention for bidirectional attention.](../sources/prs/vllm/PR-22637.md), [Support multiple attention groups for KV sharing](../sources/prs/vllm/PR-22672.md), [[Quantization] Expand compressed-tensors MoE matching logic to support NFP4 + FP8 MoEs](../sources/prs/vllm/PR-22674.md), [Force TRTLLM attention for gpt-oss on SM100](../sources/prs/vllm/PR-22678.md), [[Bugfix] Fix default enable for CUTLASS MLA on SM100](../sources/prs/vllm/PR-22738.md), [Fix GGUF loader for Qwen3 MoE.](../sources/prs/vllm/PR-22785.md), [[FIXBUG] Add return_success parameter to moe_wna16_weight_loader function](../sources/prs/vllm/PR-22797.md), [[Model] Modify the gate implementation of glm4_moe](../sources/prs/vllm/PR-22832.md), [[XPU] support data parallel for MoE models on XPU](../sources/prs/vllm/PR-22887.md), [[Kernel] Added flashinfer fp8 per-tensor gemms](../sources/prs/vllm/PR-22895.md), [[Bugfix] Fix DeepSeek MTP](../sources/prs/vllm/PR-22934.md), [Use Blackwell FlashInfer MXFP4 MoE by default if available ](../sources/prs/vllm/PR-23008.md), [[Bugfix gpt-oss] Fix float32 convert for flashinfer sink support](../sources/prs/vllm/PR-23016.md), [[Bugfix] fix qwen3 moe fp8 accuracy issue](../sources/prs/vllm/PR-23031.md), [[Core] Support weight_loader_v2 for `UnquantizedLinearMethod`](../sources/prs/vllm/PR-23036.md), [Add routed_scaling_factor to MoE grouped topk](../sources/prs/vllm/PR-23123.md), [[Bugfix] Fix accuracy issue when using flashinfer cutlass moe, TP=1 and modelopt.](../sources/prs/vllm/PR-23125.md), [Update to flashinfer-python==0.2.12 and disable AOT compile for non-release image](../sources/prs/vllm/PR-23129.md), [[Log] Warning Once for Cutlass MLA ](../sources/prs/vllm/PR-23137.md), [Fix nvfp4 swizzling](../sources/prs/vllm/PR-23140.md), [[CPU] add cpu fused moe pytorch native implementation](../sources/prs/vllm/PR-23146.md), [[XPU][Feature] fp8 online quantization support for XPU](../sources/prs/vllm/PR-23148.md), [Optimize input preparation for FlashInfer [2/N]](../sources/prs/vllm/PR-23174.md), [[Attention] Optimize make_local_attention_virtual_batches for Flash Attention](../sources/prs/vllm/PR-23185.md), [[Misc][qwen2_5_vl][torch.compile] Enable `supports_torch_compile` on generic nn.Module and demonstrate speedup on Qwen Vision model](../sources/prs/vllm/PR-23207.md), [[Core] Always use tensor cores for Flashinfer Decode Wrapper](../sources/prs/vllm/PR-23214.md), [[Perf] Small optimizations for silu_mul_fp8_quant_deep_gemm](../sources/prs/vllm/PR-23265.md), [[Kernels] Overlap shared experts with send/recv](../sources/prs/vllm/PR-23273.md), [[Bug] Fix R1 Accuracy 0 Bug](../sources/prs/vllm/PR-23294.md), [[Attention] Allow V1 flash_attn to support cross-attention](../sources/prs/vllm/PR-23297.md), [[Perf] Warmup FlashInfer attention during startup](../sources/prs/vllm/PR-23439.md), [[Attention][FA3] Update FA3 to include new swizzle optimization](../sources/prs/vllm/PR-23465.md), [fix(v1/kv_cache): resolve async KV transfer bug in cascade attention](../sources/prs/vllm/PR-23485.md), [[Bugfix] Fix Qwen3 MoE GPTQ inference](../sources/prs/vllm/PR-23490.md), [[V1][P/D]P2pNcclConnector supports flashinfer](../sources/prs/vllm/PR-23536.md), [Update Flashinfer to 0.2.14.post1](../sources/prs/vllm/PR-23537.md), [[Misc] Simplify FlashInfer attention metadata](../sources/prs/vllm/PR-23585.md), [DP/EP Support for gpt-oss with deepep-ht comm kernel on SM100](../sources/prs/vllm/PR-23608.md), [[Flashinfer] Support Flashinfer TRTLLM FP8-qkv BF16/FP16-out Attention Kernel](../sources/prs/vllm/PR-23647.md), [[Bugfix] Fix Marlin NVFP4 for modelopt](../sources/prs/vllm/PR-23659.md), [[v1] Add cross-attention KV cache support for encoder-decoder models](../sources/prs/vllm/PR-23664.md), [[Core/DBO][1/N] Add Dual-Batch Overlap mechanism to VLLM](../sources/prs/vllm/PR-23693.md), [[Kernel][B200] mxfp4 fused cutlass moe](../sources/prs/vllm/PR-23696.md), [[FlashInfer] Cache hyper params in metadata builder](../sources/prs/vllm/PR-23732.md), [[BugFix][FlashInfer] Fix potential race condition for paged_kv_indptr_cpu](../sources/prs/vllm/PR-23737.md), [[Feat][EPLB] A novel static EPLB placement strategy for MoE models.](../sources/prs/vllm/PR-23745.md), [[Misc] add reorder_batch AttentionMetadataBuilder](../sources/prs/vllm/PR-23798.md), [[fix]: add Arm 4bit fused moe support](../sources/prs/vllm/PR-23809.md), [[Model][gpt-oss] Support DP+EP for GPT-OSS with FlashInfer trtllm-gen MoE](../sources/prs/vllm/PR-23819.md), [[BUGFIX ] fix undefined silu_and_mul_nvfp4_quant](../sources/prs/vllm/PR-23929.md), [Feature/vit attention unification# 23880](../sources/prs/vllm/PR-23978.md), [[BUGFIX] GPTQ quantization compatibility for Qwen3 MOE models (AutoGPTQ and AutoRound-GPTQ)](../sources/prs/vllm/PR-23994.md), [[PERF] Allreduce fusion. Support torch native matching. Tuning of the thresholds](../sources/prs/vllm/PR-24248.md), [[Transform] [Quantization] Add QuTLASS support to vLLM](../sources/prs/vllm/PR-24440.md), [[Feature] Disallow FlashMLA on Blackwell](../sources/prs/vllm/PR-24521.md), [[Performance] Move apply_w8a8_block_fp8_linear to an op class](../sources/prs/vllm/PR-24666.md), [[Model] Support Qwen3-VL Model Series](../sources/prs/vllm/PR-24727.md), [[Bug] Fix `is_flashmla_supported` Check Error](../sources/prs/vllm/PR-24774.md), [[DCP] Support Decode Context Parallel (DCP) for GQA with FlashAttention](../sources/prs/vllm/PR-24864.md), [[Attention][DCP] Support DCP with query length > 1 (MTP) with FA3](../sources/prs/vllm/PR-25049.md), [[Bug] Fix `returned_lse` not Defined issue](../sources/prs/vllm/PR-25106.md), [[ROCm] Small functional changes for gptoss](../sources/prs/vllm/PR-25201.md), [[BugFix] Fix MLA assert with CUTLASS MLA](../sources/prs/vllm/PR-25478.md), [feat: BF16 FlashInfer Fused Cutlass MOE for Hopper and Blackwell Expert Parallel](../sources/prs/vllm/PR-25503.md), [Enable Fbgemm NVFP4 on Dense models](../sources/prs/vllm/PR-25609.md), [[Flashinfer][gpt-oss] Support FP8-qkv Flashinfer TRTLLM Sinks Attention](../sources/prs/vllm/PR-25674.md), [[Bugfix] Enable padded FP4 quantization](../sources/prs/vllm/PR-25947.md), [[Quantization/NVFP4] Speed up TRTLLM NVFP4 MOE weight loading and fix K/V scale loading for MLA Attn](../sources/prs/vllm/PR-25968.md), [[Spec Decode] Enable efficient speculative decoding with FlashInfer-MLA](../sources/prs/vllm/PR-25984.md), [[Bugfix] Allow skipping MoE in NVFP4 (fix for MTP)](../sources/prs/vllm/PR-25987.md), [[NVIDIA] Add support for cudnn fp4 gemm via flashinfer](../sources/prs/vllm/PR-26107.md), [[ModelOpt] Load w13/w2_input_scale for all experts, nvfp4](../sources/prs/vllm/PR-26135.md), [[Performance] Dual stream execution of "shared_experts" and "selected_experts" inside FusedMoE](../sources/prs/vllm/PR-26440.md), [[Bugfix] Convert untraceable GroupShape to list for AMD impl](../sources/prs/vllm/PR-26535.md), [[ROCM] MoE fp4 CK kernel](../sources/prs/vllm/PR-26545.md), [support flashinfer_fp4 moe for 5090 gpu](../sources/prs/vllm/PR-26669.md), [[Bugfix] Fix gpt-oss w4a8 DP/EP on B200](../sources/prs/vllm/PR-26729.md), [Disable FlashInfer sampler by default](../sources/prs/vllm/PR-26859.md), [[Feature] Batch Invariant: Support DeepGEMM and Blackwell](../sources/prs/vllm/PR-27127.md), [[Kernels] Enable FlashInfer FP8 Blockscale on SM90 (for TEP DSR1)](../sources/prs/vllm/PR-27134.md), [[torch.compile] Enable silu_mul_fp8_quant fusion without custom ops enabled](../sources/prs/vllm/PR-27146.md), [[ROCM] Enable CompressedTensorsWNA16](../sources/prs/vllm/PR-27187.md), [[BUGFIX][ROCM] ViT FlashAttention on ROCm (no GFX9) and contiguous on qwen3vl ROCm TORCH_SDPA](../sources/prs/vllm/PR-27190.md), [Flashinfer_CUTLASS_MOE fuses quantization for TP](../sources/prs/vllm/PR-27223.md), [[Feature] Batch Invariant for R1 TP 8 on Blackwell](../sources/prs/vllm/PR-27229.md), [[Bugfix] Ensure calculated KV scales are applied in attention.](../sources/prs/vllm/PR-27232.md), [Bugfix: Cutlass FP8 FusedMoE bad scaling factors](../sources/prs/vllm/PR-27255.md), [Feature: Support Relu2 in FusedMoE fp8 cutlass path](../sources/prs/vllm/PR-27261.md), [[Misc] Make reorder batch also separate extends](../sources/prs/vllm/PR-27367.md), [[Performance] Support FP8 flashinfer TRTLLM MOE on Qwen3 and Qwen-3next](../sources/prs/vllm/PR-27492.md), [[Feature] Batch invariant torch.compile](../sources/prs/vllm/PR-27660.md), [[AMD] Use Decoupled Kernel Block Size to Support AITER MLA block_size=1](../sources/prs/vllm/PR-27715.md), [[Feature] Extend batch invariant torch.compile to B200](../sources/prs/vllm/PR-27856.md), [[Bug] Batch invariant: Fix flash attn MLA `RuntimeError: scheduler_metadata must have shape (metadata_size)`](../sources/prs/vllm/PR-27884.md), [[Performance][B200] Fix deepgemm prologue](../sources/prs/vllm/PR-27897.md), [[flashinfer][fix] do not check nvcc availability when using pre-downloaded cubins](../sources/prs/vllm/PR-27990.md), [[FlashInfer] Avoid FlashInfer block_size 16 + head_size 256 on blackwell](../sources/prs/vllm/PR-27994.md), [[ROCm][MLA] enable fp8 MLA decode on ROCm](../sources/prs/vllm/PR-28032.md), [[Model] Consolidate Deepseek-MoE implementation with DeepSeek-v2](../sources/prs/vllm/PR-28101.md), [[Mamba] - Consolidate Mambas Attention Logic](../sources/prs/vllm/PR-28133.md), [[flashinfer] fix FI all2all with FI cutlass moe](../sources/prs/vllm/PR-28166.md), [[Feature] Support recording expert indices for rollout router replay](../sources/prs/vllm/PR-28284.md), [[ROCm] Support for Whisper v1 with Aiter Unified Attention and Aiter Flash Attention](../sources/prs/vllm/PR-28376.md), [[Bugfix][EPLB] Disabled shared expert overlap when EPLB is enabled](../sources/prs/vllm/PR-28377.md), [[Bugfix] Fix SM100 gpt-oss regression due to faulty attn sink support](../sources/prs/vllm/PR-28561.md), [[Attention][Bugfix] Fix FA sink support](../sources/prs/vllm/PR-28660.md), [[Bugfix][Nixl] Fix kernel physical<>logical block_size issue ](../sources/prs/vllm/PR-28677.md), [[Performance] Reduce DeepGEMM N dim restriction from 128 to 64 multiplier ](../sources/prs/vllm/PR-28687.md), [[Feature] Prefill Context Parallel (PCP) basic support](../sources/prs/vllm/PR-28718.md), [[Bugfix] Fix GPT-OSS on AMD after #28603](../sources/prs/vllm/PR-28816.md), [bugfix: correct attn output with base 2 or e](../sources/prs/vllm/PR-28840.md), [[Bugfix] Fix GPT-OSS AR+NORM fusion](../sources/prs/vllm/PR-28841.md), [[Bugfix] Make compressed-tensors MoEs respect ignored layers](../sources/prs/vllm/PR-28878.md), [Add TRTLLM MoE NVFP4 kernel to CompressedTensorsW4A4MoeMethod](../sources/prs/vllm/PR-28892.md), [[BugFix] Fix async-scheduling + FlashAttn MLA](../sources/prs/vllm/PR-28990.md), [[Feat] Support non-gated activations in NVFP4 modelopt path](../sources/prs/vllm/PR-29004.md), [[DeepSeek + LMCache Multiprocess] handle MLA for deepseek model + LMCache Multiprocess connector](../sources/prs/vllm/PR-29039.md), [[Perf][Kernels] Enable FlashInfer DeepGEMM swapAB on SM90 (for W8A8 Linear Op)](../sources/prs/vllm/PR-29213.md), [[LoRA] Optimize 3D MoE logic](../sources/prs/vllm/PR-29222.md), [[Perf] Disable DeepGEMM MoE by default when TP=8 is used](../sources/prs/vllm/PR-29346.md), [Add unpermute-aware fused MoE path and small-batch fallback](../sources/prs/vllm/PR-29354.md), [[Bugfix] Fix grouped_topk pytorch impl when num_experts can't be grouped properly](../sources/prs/vllm/PR-29439.md), [[Attention] Cache attention metadata builds across hybrid KV-cache groups](../sources/prs/vllm/PR-29627.md), [[Bugfix] Defunctionalize TRTLLM AR+Norm op for avoiding extra clone kernel before it](../sources/prs/vllm/PR-29631.md), [[Attention] Make `split_decodes_and_prefills(..., require_uniform=True)` support padding](../sources/prs/vllm/PR-29644.md), [[perf] Use direct copy (broadcast) instead of cat for k_nope/k_pe in MLA prefill](../sources/prs/vllm/PR-29710.md), [[Quantization] Enable compressed-tensors AWQ for Turing GPU](../sources/prs/vllm/PR-29732.md), [[Bugfix] Fix mismatched nvfp4 gemm output shape](../sources/prs/vllm/PR-29742.md), [[MoE-FP8-modelopt] Add FlashInfer alignment padding for intermediate dimensions](../sources/prs/vllm/PR-29748.md), [Add Mistral Large 3 and Ministral 3](../sources/prs/vllm/PR-29757.md), [[ROCm] [Fused Moe EP] Use binary expert mask for aiter fused moe kernel](../sources/prs/vllm/PR-29773.md), [[ROCm][MXFP4] Infer w4a4 quant method in rocm aiter fused moe](../sources/prs/vllm/PR-29775.md), [[EPLB] Support EPLB w/ NVFP4](../sources/prs/vllm/PR-29804.md), [[Bugfix][Model] Support LoRA on Qwen3 Output Embedding](../sources/prs/vllm/PR-29816.md), [[SpecDecode] Simplified alternative padded-speculation acceptance rate fix](../sources/prs/vllm/PR-29845.md), [[Quantization] fix: overflow with static per-tensor scaling](../sources/prs/vllm/PR-29867.md), [[Bugfix] Fix FP8 MoE LoRA](../sources/prs/vllm/PR-29890.md), [[BugFix] Fix DBO assert `assert B_block_table == B_q`](../sources/prs/vllm/PR-29933.md), [[moe] Use enable_chunking func (to support disabling chunking)](../sources/prs/vllm/PR-29935.md), [[moe] Allow disabling DP chunking](../sources/prs/vllm/PR-29936.md), [[Bugfix] Fix flashinfer ar+norm kernel not available issue](../sources/prs/vllm/PR-29960.md), [[ROCm] add fallback for aiter fp8 decode mla](../sources/prs/vllm/PR-30005.md), [[Quantization] Support Quark int4-fp8 w4a8 for MoE](../sources/prs/vllm/PR-30071.md), [[Model][Quantization] Restore MoE + GGUF models support (incl. Qwen3 MoE) by allowing Sideload Parameters](../sources/prs/vllm/PR-30116.md), [[Model][Quantization] Override HF defaults to GGUF ones (incl. Qwen3 MoE)](../sources/prs/vllm/PR-30118.md), [Nvidia ModelOpt workaround for issue 28072](../sources/prs/vllm/PR-30164.md), [Add latent MoE support](../sources/prs/vllm/PR-30203.md), [[Bugfix]: Fix glm46 awq marlin moe wna16 compatibility](../sources/prs/vllm/PR-30210.md), [[LoRA] Reduce the loading time of MoE LoRA](../sources/prs/vllm/PR-30243.md), [[Bugfix] Fix DeepGEMM after #29546 ](../sources/prs/vllm/PR-30267.md), [[Model][Quantization] Fix / Add GGUF support for Qwen2 MoE models](../sources/prs/vllm/PR-30307.md), [[bugfix][quantization] fix quark qwen3 kv_cache quantization](../sources/prs/vllm/PR-30308.md), [[fix] fix SM check for Flashinfer TRTLLM MOE](../sources/prs/vllm/PR-30314.md), [[Bugfix] Fix fp8 DeepGemm compilation issues](../sources/prs/vllm/PR-30336.md), [[BugFix] Fix `AttributeError: 'MergedColumnParallelLinear' object has no attribute 'weight_scale'`](../sources/prs/vllm/PR-30399.md), [fix(gguf): Disable bfloat16 for GGUF on blackwell device](../sources/prs/vllm/PR-30408.md), [[ROCm][Bugfix] Add MLACommonMetadata to allowed attention types for speculative decoding](../sources/prs/vllm/PR-30430.md), [[Bugfix] Pass FA version in `MultiHeadAttention`](../sources/prs/vllm/PR-30575.md), [[Perf] Eliminate padding and slicing op for GPT-OSS with Flashinfer MXFP4 MXFP8 MoE](../sources/prs/vllm/PR-30647.md), [[Misc][LLaMa4] Compile LLaMa Vision Encoder](../sources/prs/vllm/PR-30709.md), [Update note comment for flashinfer attention warmup](../sources/prs/vllm/PR-30711.md), [[Perf] enable flashinfer rotary_embedding custom ops in DeepSeek rotary](../sources/prs/vllm/PR-30729.md), [[Bugfix] Fix broken ViT attention selection for Blackwell device](../sources/prs/vllm/PR-30731.md), [[SM100] Enable fp8 compute for prefill MLA](../sources/prs/vllm/PR-30746.md), [Add support for LoRA adapters in Nemotron-H models](../sources/prs/vllm/PR-30802.md), [[Kernels][FI] Skip trtllm attention when num_kv_heads=1](../sources/prs/vllm/PR-30842.md), [[Compressed-Tensors] Simplify NVFP4 Conditions, enable marlin support for NVFP4A16 MoEs](../sources/prs/vllm/PR-30881.md), [[Kernel][Performance] Enable smaller Scaling Factor tiling for NVFP4 small-batch decoding](../sources/prs/vllm/PR-30885.md), [[Feature]: Support NVIDIA ModelOpt HF FP8 variants FP8_PER_CHANNEL_PER_TOKEN and FP8_PB_WO in vLLM](../sources/prs/vllm/PR-30957.md), [[Mics] add pcp basic support to MoE model](../sources/prs/vllm/PR-31003.md), [[Bugfix] Fix GLM-4 MoE router logits dtype for data parallel chunking](../sources/prs/vllm/PR-31055.md), [ [FIX] Always support TP > 4 for FP4 Gemm](../sources/prs/vllm/PR-31099.md), [[BugFix] LoRA: Support loading base_layer of experts](../sources/prs/vllm/PR-31104.md), [[Bugfix][Hardware][AMD] Consolidate FP8 min/max values helper function](../sources/prs/vllm/PR-31106.md), [[Misc] Fix grammar errors in comments and messages](../sources/prs/vllm/PR-31115.md), [[Bugfix] Fix MoE LoRA bin/pt loading](../sources/prs/vllm/PR-31161.md), [[perf] Integrate flashinfer concat_mla_k](../sources/prs/vllm/PR-31171.md), [[Bugfix][Hardware][AMD] Fix exception types in AITER MLA FP8 check](../sources/prs/vllm/PR-31177.md), [[SM100] Resubmit FMHA FP8 prefill for MLA](../sources/prs/vllm/PR-31195.md), [[Bugfix][Hardware][AMD] Fix last_page_len calculation in AITER MLA decode](../sources/prs/vllm/PR-31282.md), [fix(rocm): add early return in get_flash_attn_version for ROCm](../sources/prs/vllm/PR-31286.md), [pin lora_b moe weights on cpu](../sources/prs/vllm/PR-31317.md), [[Misc] Fix Qwen2-MoE shared_expert_gate](../sources/prs/vllm/PR-31339.md), [[BugFix] add select_gemm_impl on CompressedTensorsWNA16MoEMethod to support LoRA](../sources/prs/vllm/PR-31453.md), [[Bugfix][ROCm] Fix Static Quant Issue](../sources/prs/vllm/PR-31502.md), [[ROCm][Bugfix] Fix accuracy issue on fmoe when `VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS` enabled](../sources/prs/vllm/PR-31523.md), [Use the same memory for workspace13 and fused_output.](../sources/prs/vllm/PR-31531.md), [[Fix] Align fused moe lora_b shape with peft](../sources/prs/vllm/PR-31534.md), [[Bugfix] Fix Broken ModelOpt NVFP4 MoE](../sources/prs/vllm/PR-31742.md), [[Perf] Add opt-in SM100 Oink RMSNorm custom-op path](../sources/prs/vllm/PR-31828.md), [[MISC] Add strict contiguity check for FlashInfer attention tensors](../sources/prs/vllm/PR-32008.md), [[5/N][Attention] Finish eliminating `vllm/attention` folder](../sources/prs/vllm/PR-32064.md), [[BugFix] Fix DeepSeek-V3.1 + DeepGEMM incompatible scale shapes](../sources/prs/vllm/PR-32361.md), [[Model] Molmo2: Enable quantized weight mapping for vision backbone](../sources/prs/vllm/PR-32385.md), [[Hardware][SM100] Add TRTLLM Kernel for INT4 W4A16 Kernel.](../sources/prs/vllm/PR-32437.md), [[Perf] Create TMA-aligned input scale tensor for DeepGemm on Hopper](../sources/prs/vllm/PR-32619.md), [[Kernel] use flashinfer for gdn prefill](../sources/prs/vllm/PR-32846.md), [[Performance] Tune Mamba selective scan kernel for B200](../sources/prs/vllm/PR-32873.md), [[Bugfix] Fix FP8 MoE EP Weight Loading for ModelOpt Llama4](../sources/prs/vllm/PR-32886.md), [[Spec Decode] Unified Parallel Drafting](../sources/prs/vllm/PR-32887.md), [[ROCm][perf] Shuffle KV cache to use paged_attention_common](../sources/prs/vllm/PR-32914.md), [[NVIDIA] [feat] Integrate flashinfer Trtllmgen bf16 moe](../sources/prs/vllm/PR-32954.md), [Support compress-tensors with nvfp4 or fp8 weights and modelopt with nvfp4 weights on Turing](../sources/prs/vllm/PR-33076.md), [[Attention] Use `has_flashinfer` helper](../sources/prs/vllm/PR-33177.md), [[Bugfix] Disable TRTLLM attention when KV transfer is enabled](../sources/prs/vllm/PR-33192.md), [[Bugfix] Register fp8 cutlass_group_gemm as supported for only SM90+SM100](../sources/prs/vllm/PR-33285.md), [[PERF] Change GDN Attention State Layout from [N, HV, K, V] to [N, HV, V, K]](../sources/prs/vllm/PR-33291.md), [[Kernel] Support Flashinfer trtllm fused MoE non gated FP8 & NVFP4](../sources/prs/vllm/PR-33506.md), [[Perf] Disable clean_logits in deepgemm fp8_mqa_logits kernel](../sources/prs/vllm/PR-33568.md), [[Bugfix] Fix sparse MLA metadata building](../sources/prs/vllm/PR-33579.md), [[Bugfix] fix DeepSeek R1 with CUTLASS MLA Broken on B200](../sources/prs/vllm/PR-33637.md), [enable skipping of SW attention layers when using FP8 KV cache](../sources/prs/vllm/PR-33695.md), [[Bugfix] Fix DSV3.2 NVFP4](../sources/prs/vllm/PR-33932.md), [Adding support to Sarvam's MoE models](../sources/prs/vllm/PR-33942.md), [[Bugfix] Relax TRTLLM KV cache contiguity assertion for cross-layer layout](../sources/prs/vllm/PR-34158.md), [[Bugfix] Fix DP Attention Padding in Dummy Run](../sources/prs/vllm/PR-34187.md), [[ModelBash][DSR1 NVFp4] Avoid Bf16 Bias Cast](../sources/prs/vllm/PR-34298.md), [[CPU][Perf] Accelerate Attention head for s390x using vector intrinsics](../sources/prs/vllm/PR-34434.md), [[Llama4,Quantization] Simplify and generalize logic for Q/K permutations in quantized self-attn layers ](../sources/prs/vllm/PR-34471.md), [[BUGFIX] Fix accuracy regression for NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4 with TP>1](../sources/prs/vllm/PR-34476.md), [[Model] Add NVFP4 quantization support for Step3.5-Flash](../sources/prs/vllm/PR-34478.md), [[Bugfix] Handle num_expert_group=None in flashinfer block-scale FP8 MoE](../sources/prs/vllm/PR-34494.md), [[BugFix] Add support for MTP num_speculative_tokens > 1 with sparse MLA](../sources/prs/vllm/PR-34552.md), [[Quantization] add humming quantization kernel](../sources/prs/vllm/PR-34556.md), [[Bugfix] Rescale NVFP4 weight scales to fix BF16 dequant underflow](../sources/prs/vllm/PR-34577.md), [[Update] Use FlashInfer fast_decode_plan directly instead of replication](../sources/prs/vllm/PR-34687.md), [[Bugfix] Fix MLA attention crash with AWQ/GPTQ quantized models](../sources/prs/vllm/PR-34695.md), [[torch.compile] Turn on silu+fp4 quant fusion by default for O1+](../sources/prs/vllm/PR-34718.md), [[Bugfix] Fix NVFP4 TRTLLM MoE non-gated support; add gsm8k for Nemotron-3-Nano FP8+NVFP4](../sources/prs/vllm/PR-34725.md), [[Attention] Use FA4 for MLA prefill](../sources/prs/vllm/PR-34732.md), [[Bugfix] Fix GDN attention crash with mixed decode/spec-decode batches](../sources/prs/vllm/PR-34871.md), [[Model Bash][DSR1] Add selective dynamic shape marking for CustomOp](../sources/prs/vllm/PR-34900.md), [[Perf] Enable FlashInfer DeepGEMM swapAB on SM90 by default](../sources/prs/vllm/PR-34924.md), [add mixed precision support for modelopt](../sources/prs/vllm/PR-35047.md), [Integrate flashinfer mm_mxfp8 in ModelOpt MXFP8](../sources/prs/vllm/PR-35053.md), [[Bug][DSV3.2] Always prepare metadata for DeepGEMM Sparse Attention](../sources/prs/vllm/PR-35075.md), [[BUGFIX][Qwen3.5] Hardcode `mlp.gate` as not quantizable ](../sources/prs/vllm/PR-35156.md), [[Linear Attention] fix bug for linear attention + prefix caching + reset_prefix_cache](../sources/prs/vllm/PR-35157.md), [[BUGFIX][Mamba][Qwen3.5] Zero freed SSM cache blocks on GPU](../sources/prs/vllm/PR-35219.md), [[Performance] Extract KV cache update op from flashinfer forward](../sources/prs/vllm/PR-35422.md), [[Bugfix] Fix KV Scale loading for MLA Models](../sources/prs/vllm/PR-35430.md), [[Quant][Feature] Support online MXFP8 quantization for MoE and dense models](../sources/prs/vllm/PR-35448.md), [[NVFP4] Support NVFP4 dense models from `modelopt` and `compressed-tensors` on AMD Instinct MI300, MI355X and Hopper through emulation](../sources/prs/vllm/PR-35733.md), [Fix routed experts capture for hybrid models (Mamba + Attention)](../sources/prs/vllm/PR-35744.md), [[MoE][Perf] Wrap DSV3 QKVAProj GEMM in custom op for torch.compile](../sources/prs/vllm/PR-35751.md), [[Mamba] Add stochastic rounding support](../sources/prs/vllm/PR-35753.md), [[Kernel] Add fused_sigmoid_gating_delta_rule_update kernel for Qwen3 Next](../sources/prs/vllm/PR-35777.md), [[Bugfix] Fix score layer quantization for sequence classification models - Qwen3 (VL) Reranker](../sources/prs/vllm/PR-35849.md), [[ROCm] Support MLA with nhead<16 and FP8 KV cache for TP=8 (Kimi K2.5/Linear)](../sources/prs/vllm/PR-35850.md), [[Perf] Support FP8 KV cache for Flashinfer MLA Sparse](../sources/prs/vllm/PR-35891.md), [[MoE] Move PF Methods to Folder](../sources/prs/vllm/PR-35927.md), [Add support for ModelOpt MXFP8 MoE models](../sources/prs/vllm/PR-35986.md), [[Bugfix] Fix passing of activation_type to trtllm fused MoE NVFP4 and FP8](../sources/prs/vllm/PR-36017.md), [[BugFix] Fallback from FA4->FA2 for Batch Invariance](../sources/prs/vllm/PR-36059.md), [[LMCache] Pass TP size in lookup for MLA multi-reader locking](../sources/prs/vllm/PR-36129.md), [[Bugfix] Disable FlashInfer TRTLLM BF16 path for non-gated MoE](../sources/prs/vllm/PR-36146.md), [[Mamba] Flashinfer selective_state_update](../sources/prs/vllm/PR-36162.md), [[Bugfix][MLA] Add logits size budget to sparse indexer prefill chunking](../sources/prs/vllm/PR-36178.md), [[mla] Support fused FP8/NVFP4 output quantization in MLA attention (#35792)](../sources/prs/vllm/PR-36205.md), [mla: don't update kv cache on dummy forwards](../sources/prs/vllm/PR-36282.md), [[Perf] Add TRTLLM FP8 MoE Modular Kernel](../sources/prs/vllm/PR-36307.md), [Disable cascade attention by default](../sources/prs/vllm/PR-36318.md), [Kimi k2.5 MLA based eagle3](../sources/prs/vllm/PR-36361.md), [feat(attention): extract KV-cache update from FlashAttentionDiffKV ba…](../sources/prs/vllm/PR-36466.md), [[Bugfix][Sparse MLA] report indexer CG support properly](../sources/prs/vllm/PR-36519.md), [[ROCm] Utilize persistent MLA kernel from AITER](../sources/prs/vllm/PR-36574.md), [[GDN] add a config for gdn kernel selection](../sources/prs/vllm/PR-36647.md), [[Misc][Attention] Clean up unused method in `CPU_ATTN`](../sources/prs/vllm/PR-36673.md), [[Bug] Fix FlashInfer MNNVL socket collisions under concurrent vLLM jobs](../sources/prs/vllm/PR-36674.md), [[ROCm][Perf] Allow MTP lens > 1 in Sparse MLA](../sources/prs/vllm/PR-36681.md), [fix(kv-cache): increase hybrid attention grouping threshold from 1.25 to 1.5](../sources/prs/vllm/PR-36684.md), [[ROCm] Attention selector reordering](../sources/prs/vllm/PR-36702.md), [[DSV3.2][MTP] Optimize Indexer MTP handling](../sources/prs/vllm/PR-36723.md), [[Bug][MoE] Fix TRTLLM NVFP4 Routing Kernel Precision](../sources/prs/vllm/PR-36725.md), [[ROCm] Fix KV copy methods and auto-select attention backend for ROCm](../sources/prs/vllm/PR-36845.md), [[ROCm] Validate block_size for explicitly selected attention backends](../sources/prs/vllm/PR-36846.md), [[Feat][Spec Decode] DFlash](../sources/prs/vllm/PR-36847.md), [[Bugfix] Fix FlashInfer GDN warmup ValueError on SM90 GPUs](../sources/prs/vllm/PR-36876.md), [[Feat][Bugfix] Enable additional dimension for Flashinfer MLA and fix routing dtype](../sources/prs/vllm/PR-36931.md), [[Bugfix] Disable cross-layer KV cache for MLA attention backends](../sources/prs/vllm/PR-37090.md), [[Benchmark] Improvements to attention benchmark script](../sources/prs/vllm/PR-37115.md), [[XPU] support MLA model on Intel GPU](../sources/prs/vllm/PR-37143.md), [Fix minimax m2.5 nvfp4 kv scales weight loading](../sources/prs/vllm/PR-37214.md), [[ROCM][Bugfix] Use correct stride in cp_mha_gather_cache_kernel for hybrid model (#37228)](../sources/prs/vllm/PR-37228.md), [[Bugfix] Expand quantization method support in perf metrics](../sources/prs/vllm/PR-37231.md), [[Attention] Support distinguishing between short extends and decodes](../sources/prs/vllm/PR-37303.md), [[Bugfix] Fix EP weight filter breaking EPLB and NVFP4 accuracy](../sources/prs/vllm/PR-37322.md), [[Model Runner V2] fix draft attention metadata generation](../sources/prs/vllm/PR-37364.md), [[Bugfix] Remove assertion for NVFP4 scale dynamic range](../sources/prs/vllm/PR-37465.md), [[BugFix] Allow qk_nope_head_dim=192 in FlashInfer MLA backend checks](../sources/prs/vllm/PR-37475.md), [[Bugfix] Fix marlin nvfp4 rescaling](../sources/prs/vllm/PR-37502.md), [Fix KV Offloading + MLA AssertionError by using num_kv_heads=1 in cpu…](../sources/prs/vllm/PR-37536.md), [[Performance] Remove unnecessary zero-fill of MLA decode output tensor in Aiter backend](../sources/prs/vllm/PR-37539.md), [[Bugfix][ROCm] Fix lru_cache on paged_mqa_logits_module](../sources/prs/vllm/PR-37547.md), [[Bugfix] Disable --calculate-kv-scales for hybrid GDN/Mamba+Attention…](../sources/prs/vllm/PR-37565.md), [[Bugfix] Disable monolithic TRTLLM MoE for Renormalize routing (#37591)](../sources/prs/vllm/PR-37605.md), [[ROCm][Bugfix] fix cache block size mismatch for aiter unified attention](../sources/prs/vllm/PR-37606.md), [[Perf] Use torch compile to fuse pack topk in trtllm moe](../sources/prs/vllm/PR-37695.md), [[Bug] Fix fp8 deepgemm batch invariant](../sources/prs/vllm/PR-37718.md), [[Test] Only Run MLA model when user explicitly set for batch invariance](../sources/prs/vllm/PR-37719.md), [[XPU] add gptq(int4) support](../sources/prs/vllm/PR-37844.md), [[Feature] Support per-draft-model MoE backend via `--speculative-config`](../sources/prs/vllm/PR-37880.md), [[ROCm][perf] fix Aiter sparse MLA with MTP>1](../sources/prs/vllm/PR-37887.md), [[Bugfix] Fix DeepGemm E8M0 accuracy degradation for Qwen3.5 FP8 on Blackwell](../sources/prs/vllm/PR-38083.md), [Fix NaN from stale FP4 scale padding in create_fp4_scale_tensor](../sources/prs/vllm/PR-38148.md), [[Model Runner V2] Rebuild attention metadata before eagle decode full…](../sources/prs/vllm/PR-38311.md), [[MoE] Add RoutingMethodType.Simulated to TRT-LLM FP8/NVFP4 kernel allowlists](../sources/prs/vllm/PR-38329.md), [[GDN] Eliminate GPU->CPU sync in prepare_chunk_indices during prefill](../sources/prs/vllm/PR-38361.md), [[QeRL] Fix online quantized reloading](../sources/prs/vllm/PR-38442.md), [[XPU] Fix spec-decode UTs under tests/v1/spec_decode](../sources/prs/vllm/PR-38491.md), [[Bugfix][MLA] Change default SM100 MLA prefill backend back to TRT-LLM](../sources/prs/vllm/PR-38562.md), [[ROCm] Fix aiter persistent mode mla with q/o nhead<16 for kimi-k2.5 tp8](../sources/prs/vllm/PR-38615.md), [Fix MLA runs when use_inductor_graph_partition=True](../sources/prs/vllm/PR-38631.md), [[Bugfix] Fix AWQ models batch invariance issues](../sources/prs/vllm/PR-38670.md), [[XPU] add xpu backend implementation of mxfp8 quant](../sources/prs/vllm/PR-38682.md), [[Bugfix] Restrict TRTLLM attention to SM100, fixing GB300 (SM103) hang](../sources/prs/vllm/PR-38730.md), [[Bugfix] Fix test mocks after SM100 restriction in #38730](../sources/prs/vllm/PR-38791.md), [[LMCache][MP] optimize save when mla enabled](../sources/prs/vllm/PR-38810.md), [[FlashAttention] Symlink FA4 instead of copying when using `VLLM_FLASH_ATTN_SRC_DIR`](../sources/prs/vllm/PR-38814.md), [[Quant] add CompressedTensorsW8A8Mxfp8 for linear and MoE layers](../sources/prs/vllm/PR-38815.md), [[Attention][MLA] Re-enable FA4 as default MLA prefill backend](../sources/prs/vllm/PR-38819.md), [[Bugfix] Fix NVFP4+MTP crash: force unquantized mtp.fc for Qwen3.5](../sources/prs/vllm/PR-38832.md), [[Attention] relax the head dim 512 and paged kv for sm90+FA4](../sources/prs/vllm/PR-38835.md), [[Bugfix] Re-enable Renormalize routing for TRT-LLM MoE experts](../sources/prs/vllm/PR-38859.md), [[Gemma4] Enable Fast Prefill Optimization](../sources/prs/vllm/PR-38879.md), [[MoE Refactor] Split up compressed_tensors_moe.py](../sources/prs/vllm/PR-38960.md), [[Perf][GDN] Align TMA usage with upstream FLA](../sources/prs/vllm/PR-38981.md), [[Bug] Fix routing bias dtype for trtllm per-block fp8 moe](../sources/prs/vllm/PR-38989.md), [[Bugfix][MoE] Fix 6-8% decode regression: prefer multi-stream shared expert overlap](../sources/prs/vllm/PR-38990.md), [[Perf] Change Trtllm fp8 MoE to use Shuffled Weights and BlockMajorK Layout](../sources/prs/vllm/PR-38993.md), [[Quantization] - Layerwise reloading of Attention/KV quantized models](../sources/prs/vllm/PR-38995.md), [[Bugfix] Fix FlashInfer crash with kv_cache_dtype_skip_layers](../sources/prs/vllm/PR-39002.md), [[Gemma4] Support quantized MoE ](../sources/prs/vllm/PR-39045.md), [[Bug] Fix Trtllm Fp8 MoE Weight Shuffle Memory Fragamentation](../sources/prs/vllm/PR-39054.md), [[ROCm] Align AiterFlashAttentionImpl attn_type check with backend](../sources/prs/vllm/PR-39119.md), [[Refactor] Move NVFP4 GEMM management into NvFp4LinearKernel](../sources/prs/vllm/PR-39129.md), [[Refactor] Move MXFP8 GEMM management into MxFp8LinearKernel](../sources/prs/vllm/PR-39205.md), [[Bug] Fix rocm sparse attn indexer issue](../sources/prs/vllm/PR-39225.md), [[Bugfix] FlashInfer MXINT4 MoE crashes, missing do_finalize](../sources/prs/vllm/PR-39315.md), [[Feature] Batch invariant nvfp4 linear support](../sources/prs/vllm/PR-39322.md), [[Model Runner V2] Fix flex attention kv blocks calculation issue](../sources/prs/vllm/PR-39353.md), [[Bugfix][CT] Fix KV cache scale handling](../sources/prs/vllm/PR-39418.md), [[MLA] Optimize mla indexer prepare uniform decode for MTP > 1](../sources/prs/vllm/PR-39458.md), [[Kernel] Support TRTLLM GEN NVFP4 MoE for non-512-aligned hidden dims via weight padding](../sources/prs/vllm/PR-39510.md), [[Bugfix] Fix tensor shape mismatch in sparse attention with speculative decoding](../sources/prs/vllm/PR-39542.md), [[Mooncake] Fix mixed MLA+Eagle block-size validation](../sources/prs/vllm/PR-39596.md), [[XPU] properly handle q_descale on XPU as quant query input not supported](../sources/prs/vllm/PR-39676.md), [[Bugfix] Fix mismatch between global and local attention heads in tensor-parallel mode for param2moe model](../sources/prs/vllm/PR-39707.md), [[Bugfix] Reject non-nvfp4 dtypes when using the flashinfer_nvlink_one_sided all2all backend](../sources/prs/vllm/PR-39717.md), [[Bugfix][NIXL] Fix `_logical_to_kernel_block_ids` conversion for non-mamba models](../sources/prs/vllm/PR-39724.md), [add warning when FP8 KV cache misses prefill query quantization](../sources/prs/vllm/PR-39752.md), [[Bugfix] add support for 'num_attention_groups' in ModelArchConfigConvertorBase for Step3p5](../sources/prs/vllm/PR-39796.md), [[Bug] Fix batch invariance nvfp4 support](../sources/prs/vllm/PR-39820.md), [[Bugfix] Disable FlashInfer CUTLASS MoE on SM121 (DGX Spark)](../sources/prs/vllm/PR-39825.md), [[Core] Replace routing replay with device cache and async D2H pipeline](../sources/prs/vllm/PR-39917.md), [[Attention] use diff kv backend for mimo v2 flash](../sources/prs/vllm/PR-40045.md), [[Bugfix] Temporarily disable B200 fp4 MoE layer tests](../sources/prs/vllm/PR-40057.md), [Add nvfp4 kv cache support](../sources/prs/vllm/PR-40177.md), [Fix MoE backend selection for LoRA (unquantized MoE)](../sources/prs/vllm/PR-40273.md), [[Kernel][Helion] Optimize Helion config parsing latency](../sources/prs/vllm/PR-40850.md), [[DSV4] Add BF16 and MXFP8 A2A support for flashinfer a2a one sided](../sources/prs/vllm/PR-40960.md), [[Kernel][MoE] Support GELU on TRT-LLM NvFP4 fused MoE for Gemma4](../sources/prs/vllm/PR-41050.md), [[Bugfix][Hybrid][NemotronH] Fix mamba_cache_mode=all + speculative decoding crash](../sources/prs/vllm/PR-41233.md), [[ROCm][Quantization][3/N] Refactor quark_moe w4a4 w/ oracle](../sources/prs/vllm/PR-41436.md), [[Quantization] Rework quantization_config to use QuantKey and allow for activation override](../sources/prs/vllm/PR-41566.md), [[MXFP4] Support for linear layers + compressed-tensors integration](../sources/prs/vllm/PR-41664.md), [fix: remove unused norm for dpskv4](../sources/prs/vllm/PR-41710.md), [[Spec Decode] Add Gemma4 MTP speculative decoding support](../sources/prs/vllm/PR-41745.md), [Add NVFP4 all-gather GEMM fusion for AsyncTP](../sources/prs/vllm/PR-41882.md), [[CPU] Add MXFP4 W4A16 MoE support](../sources/prs/vllm/PR-41922.md), [[Bugfix] Add swiglu limits to deepgemm fp8 methods](../sources/prs/vllm/PR-41986.md), [[Bugfix] Fix TRTLLM ragged MLA prefill workspace warmup](../sources/prs/vllm/PR-42112.md), [[LoRA] Support 2D and 3D MoE LoRA adapter at the same time](../sources/prs/vllm/PR-42242.md), [[Bugfix] mamba: run single-token extends as decodes](../sources/prs/vllm/PR-42430.md), [Refactor AWQ Marlin MoE onto modular WNA16 oracle](../sources/prs/vllm/PR-42483.md), [[UX] Add a persistent cache for FlashInfer autotuning](../sources/prs/vllm/PR-42537.md), [[Bugfix] fix swiglu limit issue for humming backend + deepseek v4](../sources/prs/vllm/PR-42541.md), [[CPU] Add fused GDN support for AMX CPU platform](../sources/prs/vllm/PR-42707.md), [Fix Weight loading for Qwen3.5-MTP and Qwen3-VL using runai_streamer](../sources/prs/vllm/PR-42716.md), [[CPU] Specify required KV cache layout for CPU attention backend](../sources/prs/vllm/PR-42740.md), [[ROCm] [Bugfix] Fix DeepSeek V4 Functionality and Accuracy](../sources/prs/vllm/PR-42810.md), [[Perf] Add do_not_specialize in fused FP8 RoPE kernel](../sources/prs/vllm/PR-42849.md), [[Model Refactoring] Migrate DeepSeek V4 to vllm/models/ [1/N] ](../sources/prs/vllm/PR-43004.md), [[XPU] update xpu graph usage](../sources/prs/vllm/PR-43043.md), [[CI failure] Temporarily disable using persistent cache for flashinfer autotune](../sources/prs/vllm/PR-43119.md), [FlashAttention SM100 MLA TopK Sparse Forward](../wiki/kernels/flash-attention-sm100-mla-topk.md), [FlashMLA — Multi-head Latent Attention](../wiki/kernels/flashmla.md), [FP8 Fine-Grained-Scale GEMM](../wiki/kernels/fp8-block-scale-gemm.md), [Gated Delta Net — Linear Attention](../wiki/kernels/gated-delta-net.md), [GPU Mode NVFP4 Gated Dual GEMM](../wiki/kernels/gated-dual-gemm.md), [Grouped GEMM Contracts for MoE and NVFP4](../wiki/kernels/grouped-gemm.md), [Native Sparse Attention (NSA)](../wiki/kernels/nsa.md), [NVFP4 GEMM — GPU Mode Problem 2 Contract](../wiki/kernels/nvfp4-gemm.md), [TensorRT-LLM Blackwell FP4 DSA Indexer](../wiki/kernels/tensorrt-llm-blackwell-indexer.md) | +| `tilelang` | | [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [Tilelang sparse decode fwd for dsv32 mi355](../sources/prs/sglang/PR-18488.md), [[AMD] Tilelang sparse fwd for dsv32 mi355/mi300](../sources/prs/sglang/PR-19945.md), [[AMD] Enable FP8 KV cache and FP8 attention kernel for NSA on MI300/MI355 with TileLang backend](../sources/prs/sglang/PR-21511.md), [Amd/deepseek v4 rebase main 0509](../sources/prs/sglang/PR-24933.md), [[Refactor] Rename NSA → DSA: user-facing aliases, file/class/import rename](../sources/prs/sglang/PR-25821.md), [[WIP] support more dtypes for tcgen05](../sources/prs/tilelang/PR-1229.md), [[Enhancement] add more dtype and fix mma.ws for fp16 for tcgen05](../sources/prs/tilelang/PR-1327.md), [[Feat] profiler support cudagraph backend](../sources/prs/tilelang/PR-1658.md), [[Feature] Support `cp.reduce.async.bulk.tensor`](../sources/prs/tilelang/PR-1667.md), [Add swizzle layout detection and automatic merging for layout conflicts](../sources/prs/tilelang/PR-1736.md), [[Feature] Support tcgen5mma lowering for `.kind::i8`](../sources/prs/tilelang/PR-1764.md), [fix(intrinsics): add missing _legalize_to_buffer_region in SM70 emitter](../sources/prs/tilelang/PR-1786.md), [[BugFix] Fix Hopper TMA lowering without warp specialization](../sources/prs/tilelang/PR-1840.md), [[CUDA] Support tcgen5mma gemm ts](../sources/prs/tilelang/PR-1866.md), [[Feature] Support cluster launch, query, synchronization and barrier operations](../sources/prs/tilelang/PR-1874.md), [[Feature] 2-SM support for TMA, TMEM and TCGEN5MMA on Blackwell](../sources/prs/tilelang/PR-1882.md), [[Feature] Add T.copy_cluster to support TMA multicast and SM-to-SM cluster copy](../sources/prs/tilelang/PR-1908.md), [[Feature] Add Producer-Consumer Warp Specialization and T.tma_copy() API](../sources/prs/tilelang/PR-1909.md), [[Feature] Block-scaled GEMM support for MXFP8 on Blackwell](../sources/prs/tilelang/PR-1945.md), [[Bugfix] Fix CuTeDSL autotune cache invalid ELF header (#1967)](../sources/prs/tilelang/PR-1972.md), [[Feature] Support TMA store in T.tma_copy()](../sources/prs/tilelang/PR-1981.md), [[Transform] Add InjectTcgen05Fence pass](../sources/prs/tilelang/PR-2003.md), [[Backend] Refactor gemm_sp](../sources/prs/tilelang/PR-2048.md), [[CUDA] Support int4 `T.gemm`](../sources/prs/tilelang/PR-2063.md), [[CUDA] Improve int4 GEMM lowering and packed codegen support](../sources/prs/tilelang/PR-2073.md), [[TMA] Support FP4 TensorMap TMA copies](../sources/prs/tilelang/PR-2107.md), [feat: auto-vectorize bf16/fp16 reduce with packed add2 intrinsics](../sources/prs/tilelang/PR-2112.md), [[CUDA][TMA] Add TMA tile::gather4 / tile::scatter4 support](../sources/prs/tilelang/PR-2129.md), [[codex] Split GEMM implementations by backend](../sources/prs/tilelang/PR-2153.md), [[TIR][IR] Update to use tirx](../sources/prs/tilelang/PR-2216.md), [[Python] Drop Python 3.9 support](../sources/prs/tilelang/PR-2218.md) | +| `triton` | [Triton on Blackwell](../wiki/languages/triton-blackwell.md) | [Gated Delta Networks](../sources/blogs/gated-delta-net.md), [FlashInfer MLSys 2026 Track A: FP8 Block-Scale MoE](../sources/contests/flashinfer-mlsys26/track-a-fused-moe.md), [FlashInfer MLSys 2026 - Track B: DeepSeek V3.2 Sparse Attention](../sources/contests/flashinfer-mlsys26/track-b-sparse-attention.md), [Native Sparse Attention: Hardware-Aligned and Natively Trainable Sparse Attention](../sources/docs/nsa.md), [Tiled Flash Linear Attention (TFLA)](../sources/docs/tfla.md), [Triton v3.3.0 — Blackwell TCGen5/TMEM Boundary](../sources/docs/triton-3.3-blackwell.md), [Triton v3.6.0 — Incremental Blackwell Changes](../sources/docs/triton-3.6-blackwell.md), [[None][fix] impl fused triton kernel for e8m0 resmooth to reduce memory footprint](../sources/prs/TensorRT-LLM/PR-10327.md), [[#11694][feat] AutoDeploy: Improve the piecewise CG memory usage](../sources/prs/TensorRT-LLM/PR-11993.md), [[https://nvbugs/5983390][perf] Kernel fusions in _gather_k_cache_for_chunk of Indexer in DSA](../sources/prs/TensorRT-LLM/PR-12322.md), [[https://nvbugs/5983390][perf] Multiple host perf optimizations for DSA part](../sources/prs/TensorRT-LLM/PR-12581.md), [[None][feat] Add triton paged attention for AutoDeploy](../sources/prs/TensorRT-LLM/PR-12642.md), [[#12784][feat] AutoDeploy: Optimize DeepSeek-R1 model performance](../sources/prs/TensorRT-LLM/PR-12946.md), [[#13580][fix] AutoDeploy: Support Gemma3n/4 E2B variants](../sources/prs/TensorRT-LLM/PR-13630.md), [[None][feat] Keep DSv4 o_a_proj as FP8, and port vLLM's fused_inv_rope_fp8_quant](../sources/prs/TensorRT-LLM/PR-13938.md), [[https://nvbugs/6152892][fix] Fix Triton MOE memory free when no swizzling enabled](../sources/prs/TensorRT-LLM/PR-14069.md), [[None][feat] Add chunked prefill support for Gemma4 (text + vision multimodal)](../sources/prs/TensorRT-LLM/PR-14134.md), [[None][fix] Add SPDX Apache-2.0 headers to auto_deploy test files](../sources/prs/TensorRT-LLM/PR-14193.md), [[None][fix] Update the OSS headers in derived FLA ops and AD modeling code](../sources/prs/TensorRT-LLM/PR-14281.md), [[None][chore] Update Claude Code agents and skills](../sources/prs/TensorRT-LLM/PR-14344.md), [feat: ragged tensor padding kernel for blackwell kernel alignment](../sources/prs/flashinfer/PR-1025.md), [[nvidia] initial support for blackwell kernels](../sources/prs/flashinfer/PR-1039.md), [benchmark: trtllm-gen mha with sink, add benchmark args](../sources/prs/flashinfer/PR-1415.md), [refactor: update fa3 codebase and fix hopper unittest [part 1]](../sources/prs/flashinfer/PR-2111.md), [Selective State Update kernel (mamba)](../sources/prs/flashinfer/PR-2301.md), [MTP for mamba ](../sources/prs/flashinfer/PR-2444.md), [misc: point triton blackwell-ptxas to local cuda ptxas](../sources/prs/flashinfer/PR-2543.md), [Mamba SSU: better automatic kernel selection + algorithm selection optionally exposed to the user.](../sources/prs/flashinfer/PR-2591.md), [int16 Block-Scaled State and Stochastic Rounding for SSU (mamba)](../sources/prs/flashinfer/PR-2645.md), [Add varlen and speculative decoding support to selective state update](../sources/prs/flashinfer/PR-2700.md), [Mamba2 SSD Combined Forward Pass (Blackwell CuTe DSL Kernel)](../sources/prs/flashinfer/PR-2709.md), [fix(sm12x): fix micro-kernel workspace sizing when routed_rows > num_local_experts](../sources/prs/flashinfer/PR-3191.md), [checkpointing_ssu kernel: fused replay + conditional state-write for Mamba2](../sources/prs/flashinfer/PR-3324.md), [SM-constraint-GEMM by triton persistent kernel](../sources/prs/flashinfer/PR-982.md), [Triton `rms_norm` kernels](../sources/prs/flashinfer/PR-983.md), [[inductor] Fix profiler tests with latest Triton](../sources/prs/pytorch/PR-149059.md), [[inductor][triton 3.3] Fix cpp_wrapper w/ TMA in triton 3.3](../sources/prs/pytorch/PR-149993.md), [Fix uint view copy (#151598)](../sources/prs/pytorch/PR-154121.md), [[user triton] AOT inductor support for device-side TMA](../sources/prs/pytorch/PR-157241.md), [[release] Triton pin update to 3.4](../sources/prs/pytorch/PR-157752.md), [[cherry-pick][inductor][triton] Update HAS_WARP_SPEC to check triton.Config params. Update Triton Hash to top of release/3.4.x stack](../sources/prs/pytorch/PR-158646.md), [[Inductor][Intel GPU] Save `threads_per_warp` from tirton compiled kernel for launching kernel correctly in cpp wrapper.](../sources/prs/pytorch/PR-163388.md), [[2.9 cherry pick][triton] update 3.5 pin to bbb06c0334a6772b92d24bde54956e675c8c6604 (#163382)](../sources/prs/pytorch/PR-163583.md), [[AARCH64][CD][CUDA13][Triton][PTXAS] Turn on BUILD_BUNDLE_PTXAS=1 ](../sources/prs/pytorch/PR-164236.md), [[Minor][Inductor] move some combo kernel log from warning to debug](../sources/prs/pytorch/PR-167020.md), [[RELEASE 2.10] Release only changes](../sources/prs/pytorch/PR-170112.md), [[ROCm] Enable shared memory based pruning for Triton configs](../sources/prs/pytorch/PR-170190.md), [[Inductor] Fix constants handling for Triton constexpr (triton#8248)](../sources/prs/pytorch/PR-171129.md), [[RELEASE 2.11] Release only changes](../sources/prs/pytorch/PR-175091.md), [[inductor] avoid multi-stage for mix-order-red by default (#176228)](../sources/prs/pytorch/PR-176495.md), [[release 2.12] Apply Release only changes to 2.12 branch](../sources/prs/pytorch/PR-180470.md), [Add support for bf16 x bf16 cutlass fused MoE](../sources/prs/sglang/PR-10275.md), [support qwen3_next blackwell](../sources/prs/sglang/PR-10403.md), [Fix bias handling in TritonMoeQuantInfo within quantization/mxfp4.py](../sources/prs/sglang/PR-10579.md), [Fix MTP MoE weight loading with NVFP4 target model.](../sources/prs/sglang/PR-10758.md), [Fix DSR1 accuracy for flashinfer_trtllm MoE with FP8 quantization](../sources/prs/sglang/PR-11081.md), [[AMD] Clean up vllm dependencies in moe_runner/triton.py](../sources/prs/sglang/PR-11349.md), [Support shared experts overlap in cutlass moe](../sources/prs/sglang/PR-11611.md), [[Ascend] qwen optimization](../sources/prs/sglang/PR-12078.md), [Enable Flashinfer TRTLLM-GEN-MoE FP8 blockwise kernel for Qwen3-Next on Blackwell](../sources/prs/sglang/PR-12543.md), [[Bugfix] Fix illegal memory access](../sources/prs/sglang/PR-12758.md), [Support internvl on Blackwell (which doesn't support fa3): add `SingletonCache` support to Vision{Sdpa|Triton|Ascend}Attention](../sources/prs/sglang/PR-13151.md), [Fix nan in global scaling factor for large scale nvfp4 EP](../sources/prs/sglang/PR-13162.md), [[ROCM] Optimized deepseek-r1 fp8 model with + triton_gemm_a8w8 + batch_gemm_a8w8 + fused set_mla_kv_buffer kernel](../sources/prs/sglang/PR-13617.md), [[Feat][NVFP4] Enable NVFP4 MoE for Qwen series models (eg. Qwen3-Next) #13761](../sources/prs/sglang/PR-13761.md), [Support fp4 fp8 non gated moe](../sources/prs/sglang/PR-13794.md), [[NVIDIA] Enable TRTLLM BF16 MoE on Blackwell GPUs](../sources/prs/sglang/PR-13798.md), [[BugFix] fix outplace_fused_experts missing is_gated](../sources/prs/sglang/PR-13864.md), [Feat: GLM-4.6 supports shared experts fusion](../sources/prs/sglang/PR-13873.md), [[LoRA][III] Add LoRA support for MoE layers and enable TP](../sources/prs/sglang/PR-14105.md), [Apply new moe wna16 marlin gemm](../sources/prs/sglang/PR-14125.md), [Apply new moe align block size kernel](../sources/prs/sglang/PR-14134.md), [Add Mistral Large 3 support.](../sources/prs/sglang/PR-14213.md), [[Fix] add block size logic for sm120 smem size](../sources/prs/sglang/PR-14311.md), [[FIX] trtllm-moe-fp4-renorm for Qwen series models](../sources/prs/sglang/PR-14350.md), [Apply back moe_sum_reduce for fused_marlin_moe](../sources/prs/sglang/PR-14829.md), [Mistral Large 3 NVFP4 TRTLLM MoE support](../sources/prs/sglang/PR-15049.md), [Optimize FP8 MLA KV cache writes with Triton kernel](../sources/prs/sglang/PR-15522.md), [MoE: Skip SiLU/GELU activation for masked experts](../sources/prs/sglang/PR-15539.md), [Update flashinfer to 0.6.1](../sources/prs/sglang/PR-15551.md), [Add SwapAB Optimization for triton fused_moe_kernel on SM90.](../sources/prs/sglang/PR-15712.md), [ Add tuned triton==3.5.1 h200 tp2, tp4 for qwen 3 next](../sources/prs/sglang/PR-15948.md), [[Performance] Force split_k=1 for MXFP4 Triton kernels on Hopper](../sources/prs/sglang/PR-16014.md), [optimize get_topk_ragged by fusing get k and k_scale triton kernel](../sources/prs/sglang/PR-16043.md), [fix layer intermediate size](../sources/prs/sglang/PR-16084.md), [[NemotronH] Add latent MoE support](../sources/prs/sglang/PR-16227.md), [[Rework] Add SwapAB Optimization for triton fused_moe_kernel on SM90.](../sources/prs/sglang/PR-16723.md), [[Fix] `flashinfer_trtllm` `intermediate_size` assertion with Qwen3 + TP=8](../sources/prs/sglang/PR-16824.md), [Support mxint4 flashinfer_trtllm moe gemm](../sources/prs/sglang/PR-16892.md), [[GLM 4.7] Add RTX 6000 Pro aka sm120](../sources/prs/sglang/PR-17235.md), [Add mxfp8 support for online quantization, Triton dense linear, and CUTLASS MoE](../sources/prs/sglang/PR-17449.md), [[AMD] Update aiter to v0.1.10.post2](../sources/prs/sglang/PR-18423.md), [[sglang-miles] True on-policy training support for FSDP2](../sources/prs/sglang/PR-18639.md), [Adjust padding size to improve triton_kernels moe performance](../sources/prs/sglang/PR-19174.md), [Fix/nemotron mtp quantaized](../sources/prs/sglang/PR-19433.md), [[FlashInfer v0.6.4] [RL] Integrate FlashInfer mxfp8 gemm, MoE, and routed MoE](../sources/prs/sglang/PR-19537.md), [[diffusion][llm] macOS support](../sources/prs/sglang/PR-19549.md), [[Feature] NVFP4 Marlin fallback for non-Blackwell GPUs (SM75+)](../sources/prs/sglang/PR-19652.md), [Support `triton_kernels` for GPT-OSS on SM120](../sources/prs/sglang/PR-19718.md), [Fix SM120 `triton_kernels` MXFP4 `block_k` for GPT-OSS](../sources/prs/sglang/PR-20040.md), [[Benchmark] use flashinfer bench_gpu_time instead of triton do_bench](../sources/prs/sglang/PR-20305.md), [Support Triton MLA FP8 KV cache](../sources/prs/sglang/PR-20479.md), [Add SGLang CUDA crash API logging inspired by FlashInfer](../sources/prs/sglang/PR-20910.md), [[Qwen3.5] Fuse split/reshape/cat ops in GDN projection with Triton kernel](../sources/prs/sglang/PR-21019.md), [[refactor] Clean up duplicate flashinfer trtllm moe code](../sources/prs/sglang/PR-21233.md), [Refactor JIT kernel CI to use run_suite.py registration system](../sources/prs/sglang/PR-21239.md), [[RL] Support mxfp8 DeepSeek V3](../sources/prs/sglang/PR-21280.md), [[misc] clean up kernel API](../sources/prs/sglang/PR-21325.md), [Change default mm-attention backend from triton_attn to fa4](../sources/prs/sglang/PR-21595.md), [[XPU] Enable qwen3.5 on XPU](../sources/prs/sglang/PR-21668.md), [[Fix] Fall back to triton MOE for GPT-OSS on Blackwell with driver >= 595](../sources/prs/sglang/PR-21780.md), [fix pcg torch dynamo recompile in mxfp8 Triton path](../sources/prs/sglang/PR-21888.md), [[nvidia] Gemma4 nvfp4 fix](../sources/prs/sglang/PR-22079.md), [[Lora] Lora quat info re-factor and support deepseekv3 mla lora](../sources/prs/sglang/PR-22323.md), [diffusion: add HunyuanVideo GroupNorm+SiLU fast path](../sources/prs/sglang/PR-22814.md), [[codex] diffusion: enable group norm silu fuse by default](../sources/prs/sglang/PR-23148.md), [Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-23686.md), [Optimize large GroupNorm SiLU apply](../sources/prs/sglang/PR-23938.md), [[diffusion] Fuse LTX2 split rotary embedding](../sources/prs/sglang/PR-24411.md), [Port MXFP4 Marlin MoE support to JIT kernel path](../sources/prs/sglang/PR-24490.md), [Fix performance regression on Deepseek V3 on `moe-runner-backend=triton` on SM90](../sources/prs/sglang/PR-24562.md), [[Codex] Opt Mistral Large performace ](../sources/prs/sglang/PR-24611.md), [[rebase]Deepseek_v4 support w4(mxfp4)a16 on hopper](../sources/prs/sglang/PR-24986.md), [[MUSA][Diffusion] Improve wan model inference speed using torch.compile](../sources/prs/sglang/PR-25256.md), [[Gemma4]: Fix FP8 Triton scale layout](../sources/prs/sglang/PR-25286.md), [[Intel GPU] Enable DeepSeek V4 Inference on XPU](../sources/prs/sglang/PR-25336.md), [Update logging for inplace setting in MoE layer](../sources/prs/sglang/PR-25499.md), [Fix logging for inplace setting in the flashInfer-trtllm backend](../sources/prs/sglang/PR-25522.md), [amd/deepseek_v4 27/N [fix] Reduce Triton autotune configs for faster first-time server launch](../sources/prs/sglang/PR-25554.md), [Add DeepSeekV4 fused MoE Triton autotune support](../sources/prs/sglang/PR-25569.md), [Use triton_attn as default vision attention on B300 (SM103)](../sources/prs/sglang/PR-25570.md), [[diffusion] Fix GLM-Image /v1/images/edits support](../sources/prs/sglang/PR-25697.md), [[Refactor] Rename NSA → DSA: user-facing aliases, file/class/import rename](../sources/prs/sglang/PR-25821.md), [[Test] Stage-a sanity kits; consolidate core/ + models_e2e/ tests](../sources/prs/sglang/PR-25831.md), [Feature DeepSeek V3/R1 INT8 Quantization (block-wise)](../sources/prs/sglang/PR-3730.md), [[Feature] DeepSeek V3/R1 INT8 Quantization (channel-wise) ](../sources/prs/sglang/PR-3888.md), [Add fast decode plan for flashinfer mla](../sources/prs/sglang/PR-3987.md), [[Revision] Add fast decode plan for flashinfer mla ](../sources/prs/sglang/PR-4012.md), [Accelerate FP8 CUDA Kernel by 20-28%](../sources/prs/sglang/PR-4215.md), [Feat/support encoder model (like bert)](../sources/prs/sglang/PR-4887.md), [Add DeepSeek V3/R1 shared experts fusion](../sources/prs/sglang/PR-4918.md), [reduce moe_align_block_size_kernel small batch mode overhead](../sources/prs/sglang/PR-5086.md), [Support tuning moe for llama 4 model](../sources/prs/sglang/PR-6042.md), [feat: mtp support dp-attention](../sources/prs/sglang/PR-6081.md), [enable auto-round quantization model](../sources/prs/sglang/PR-6226.md), [fix: enable multi-GPU Triton fused MoE tuning](../sources/prs/sglang/PR-6295.md), [reduce torch.zeros overhead in moe align block size kernel](../sources/prs/sglang/PR-6369.md), [Refine pre_reorder_triton_kernel slightly to improve performance](../sources/prs/sglang/PR-6627.md), [[CPU] [BF16] Call fused_experts_cpu, weight_packed_linear and bmm_cpu kernel in DeepSeek model](../sources/prs/sglang/PR-6641.md), [Set `num_fused_shared_experts` as `num_shared_experts` when shared_experts fusion is not disabled](../sources/prs/sglang/PR-6736.md), [[CPU] add optimizations for INT8 and FP8 DeepSeek](../sources/prs/sglang/PR-6769.md), [[CPU] support the case where num_attention_heads or intermediate_size is not divisible by the TP size](../sources/prs/sglang/PR-6771.md), [[DeepseekR1-FP4] Add Support for nvidia/DeepSeekR1-FP4 model](../sources/prs/sglang/PR-6853.md), [Use deepgemm instead of triton for fused_qkv_a_proj_with_mqa](../sources/prs/sglang/PR-6890.md), [chore: upgrade flashinfer v0.2.6.post1 jit](../sources/prs/sglang/PR-6958.md), [Fuse routed scaling factor in deepseek](../sources/prs/sglang/PR-6970.md), [Update default settings for blackwell](../sources/prs/sglang/PR-7023.md), [Fix positional argument](../sources/prs/sglang/PR-7093.md), [Enable ModelOpt Llama4 fp8 checkpoint deployment in SGLang](../sources/prs/sglang/PR-7129.md), [Support new DeepGEMM](../sources/prs/sglang/PR-7172.md), [Fix grammar abort & Minor style fixes](../sources/prs/sglang/PR-7204.md), [FlashInfer NVFP4 MoE with EP & 2-stream shared expert](../sources/prs/sglang/PR-7327.md), [Fix MTP with Deepseek R1 Fp4](../sources/prs/sglang/PR-7376.md), [Fix torch compile run](../sources/prs/sglang/PR-7391.md), [Add fp4 quantize before all-gather for Flashinfer cutlass MoE DP (max throughput)](../sources/prs/sglang/PR-7667.md), [Integrate triton moe kernel](../sources/prs/sglang/PR-7689.md), [[feat] Support tp mode for DeepSeek-R1-W4AFP8](../sources/prs/sglang/PR-8118.md), [Support triton kernels v3.4.0 for fused_moe](../sources/prs/sglang/PR-8258.md), [[NVIDIA] Add Low Latency NVFP4 decode kernels from Flashinfer](../sources/prs/sglang/PR-8552.md), [fuse allreduce and residual_rmsnorm](../sources/prs/sglang/PR-8731.md), [[Perf] Auto enable best flashinfer mxfp4 kernel in b200](../sources/prs/sglang/PR-8898.md), [[fix] Fix mxfp4 triton MoE tp bug](../sources/prs/sglang/PR-9473.md), [Optimize moe_sum_reduce_kernel](../sources/prs/sglang/PR-9477.md), [Single Batch Overlap for MoE Models](../sources/prs/sglang/PR-9660.md), [[Kernel] add triton fused moe kernel for gptq/awq](../sources/prs/vllm/PR-12185.md), [[AMD][Quantization] Add TritonScaledMMLinearKernel since int8 is broken for AMD](../sources/prs/vllm/PR-12282.md), [[Attention] MLA decode optimizations](../sources/prs/vllm/PR-12528.md), [[Attention] Deepseek v3 MLA support with FP8 compute](../sources/prs/vllm/PR-12601.md), [[Attention] MLA with chunked prefill](../sources/prs/vllm/PR-12639.md), [[Perf] Mem align KV caches for CUDA devices (MLA perf improvement)](../sources/prs/vllm/PR-12676.md), [[core] Perf improvement for DSv3 on AMD GPUs](../sources/prs/vllm/PR-13718.md), [[Attention] MLA support for V1](../sources/prs/vllm/PR-13789.md), [[Attention] Flash MLA for V1](../sources/prs/vllm/PR-13867.md), [dynamic distpatch of fp8 kernels](../sources/prs/vllm/PR-14245.md), [[BugFix] MLA + V1, illegal memory access and accuracy issues](../sources/prs/vllm/PR-14253.md), [[BugFix][TritonMLA] Process weights after model loading for GGUF](../sources/prs/vllm/PR-14555.md), [[BugFix] Fix nightly MLA failure (FA2 + MLA chunked prefill, i.e. V1, producing bad results)](../sources/prs/vllm/PR-15492.md), [[ROCM][KERNEL] Paged attention for V1](../sources/prs/vllm/PR-15720.md), [Modularize fused experts and integrate PPLX kernels](../sources/prs/vllm/PR-15956.md), [[Kernel][Bugfix] Re-fuse triton moe weight application](../sources/prs/vllm/PR-16071.md), [Upstream Llama4 Support to Main](../sources/prs/vllm/PR-16113.md), [[Bug] [ROCm] Fix Llama 4 Enablement Bug on ROCm: V0 ROCmFlashAttentionImpl and Triton Fused MoE bugs](../sources/prs/vllm/PR-16198.md), [[Kernel] Support W8A8 channel-wise weights and per-token activations in triton fused_moe_kernel](../sources/prs/vllm/PR-16366.md), [[Perf]Optimize rotary_emb implementation to use Triton operator for improved inference performance](../sources/prs/vllm/PR-16457.md), [Enable PTPC FP8 for CompressedTensorsW8A8Fp8MoEMethod (triton fused_moe)](../sources/prs/vllm/PR-16537.md), [[torch.compile][ROCm] Fuse quantization onto attention using a torch.compile pass](../sources/prs/vllm/PR-16756.md), [[Kernel] Unified Triton kernel that doesn't distinguish between prefill + decode](../sources/prs/vllm/PR-16828.md), [[Bugfix] Triton FA function takes no keyword arguments](../sources/prs/vllm/PR-16902.md), [[Attention] MLA move o_proj q_proj into cuda-graph region](../sources/prs/vllm/PR-17484.md), [fix amd triton mla path](../sources/prs/vllm/PR-17871.md), [[Bugfix][ROCm] Use `chunked_prefill_paged_decode` as fallback for V1 attention on ROCm](../sources/prs/vllm/PR-18093.md), [[Kernel] Integrate CUTLASS MoE kernel with PPLX](../sources/prs/vllm/PR-18762.md), [[Kernel] Enable fp8 support for pplx and BatchedTritonExperts.](../sources/prs/vllm/PR-18864.md), [[Kernels] Add activation chunking logic to FusedMoEModularKernel](../sources/prs/vllm/PR-19168.md), [[Bugfix] Don't attempt to use triton if no driver is active](../sources/prs/vllm/PR-19561.md), [ [Feature] Integrate SM100 DeepGEMM support](../sources/prs/vllm/PR-20087.md), [[Kernel] Optimize Prefill Attention in Unified Triton Attention Kernel](../sources/prs/vllm/PR-20308.md), [[Misc] DP : Add ExpertTokensMetadata](../sources/prs/vllm/PR-20332.md), [[Perf] Use Triton instead of Torch for DeepGEMM Per Token Group Quant](../sources/prs/vllm/PR-20841.md), [[Kernel] DeepGemm MoE : Integrate triton permute / unpermute kernels ](../sources/prs/vllm/PR-20903.md), [[v1] Add Whisper model support (encoder-decoder)](../sources/prs/vllm/PR-21088.md), [[Attention] Clean up iRoPE in V1](../sources/prs/vllm/PR-21188.md), [[Kernel] Enable Hybrid Model Support in Triton Unified Attention Kernel](../sources/prs/vllm/PR-21197.md), [[Feature][Kernel]FusedMoE LoRA](../sources/prs/vllm/PR-21229.md), [[v1][attention] Support Hybrid Allocator + FlashInfer](../sources/prs/vllm/PR-21412.md), [[Bugfix] Add proper comparison for package versions](../sources/prs/vllm/PR-22314.md), [[BugFix] Fix triton compile error in `kernel_unified_attention_2/3d` caused by attention sinks](../sources/prs/vllm/PR-22368.md), [[gpt-oss] triton kernel mxfp4](../sources/prs/vllm/PR-22421.md), [[NVIDIA] Support Flashinfer TRTLLM FP8-q/kv NVFP4-out Attention Kernel](../sources/prs/vllm/PR-22703.md), [fp8 kv cache support fix for torch.compile](../sources/prs/vllm/PR-22758.md), [[Misc] Add @tdoublep as a maintainer of hybrid model and Triton-attention related code](../sources/prs/vllm/PR-23122.md), [[ROCm][Aiter] Add triton fp8 bmm kernel for mla](../sources/prs/vllm/PR-23264.md), [[Bugfix] Fixing division by zero in triton_attn if query_heads/kv_heads > 16 ](../sources/prs/vllm/PR-23424.md), [[Feature] Add Hopper DeepGEMM E8M0 for DeepSeekV3.1 scale_fmt](../sources/prs/vllm/PR-23666.md), [[Feature] Support Decode Context Parallel (DCP) for MLA](../sources/prs/vllm/PR-23734.md), [[Bug] Fix Shape Validation for Fallback while Enabling E8M0 for DeepGEMM](../sources/prs/vllm/PR-26322.md), [Move query quantization to attention layer for Flashinfer & Triton.](../sources/prs/vllm/PR-26534.md), [[Attention] Use sparse prefill kernel for fp8 kv-cache in DeepSeek-v3.2](../sources/prs/vllm/PR-27532.md), [fix cross attention](../sources/prs/vllm/PR-28346.md), [[Model] Add support for openPangu moe model](../sources/prs/vllm/PR-28775.md), [[Feature] Batch invariant: Enable `TRITON_MLA` without prefix-caching](../sources/prs/vllm/PR-29125.md), [[Bugfix] Only use triton_kernels for MXFP4 on SM90 and SM100](../sources/prs/vllm/PR-29339.md), [[LoRA] Support Quantized Adapters](../sources/prs/vllm/PR-30286.md), [[ROCm][Quantization] GPT OSS Upstream MoE wmxfp4_afp8 with static scales](../sources/prs/vllm/PR-30357.md), [[Perf] Set split_k to 1 for triton_kernels](../sources/prs/vllm/PR-30528.md), [[Bugfix] Fix Triton FusedMoE LoRA](../sources/prs/vllm/PR-30585.md), [Triton Attention: Support cross-layers blocks](../sources/prs/vllm/PR-30687.md), [fused_moe_lora PDL improvements](../sources/prs/vllm/PR-30716.md), [[Bugfix] [Kernel] Triton attention kernels: mask out V blocks that fall outside sliding window](../sources/prs/vllm/PR-30887.md), [[Bugfix] Fix incorrect tiles creation for mm prefix triton attention](../sources/prs/vllm/PR-30974.md), [Use aiter triton fused_add_rmsnorm_pad for gpt-oss](../sources/prs/vllm/PR-30976.md), [[Bugfix][ROCm]Fix Qwen3-Next-80B-A3B-Thinking inference and optimize non-standard block size (544) support under rocm_atten](../sources/prs/vllm/PR-31380.md), [fixed mypy warnings for files vllm/v1/attention with TEMPORARY workaround](../sources/prs/vllm/PR-31465.md), [[FIX] Add NO_MUL activation support for modular kernel path](../sources/prs/vllm/PR-31528.md), [[LoRA]Disable linear LoRA kernel PDL](../sources/prs/vllm/PR-31777.md), [[1/N][Attention] Restructure attention: move files](../sources/prs/vllm/PR-31916.md), [[4/N][Attention] Move MLA common to model_executor](../sources/prs/vllm/PR-32060.md), [Add TMA support to fused_moe_lora kernel](../sources/prs/vllm/PR-32195.md), [[Bugfix][Attention] Explicitly report support for kv_cache_dtype bfloat16](../sources/prs/vllm/PR-32795.md), [Triton MLA perf fixes](../sources/prs/vllm/PR-33529.md), [[Kernel] Add FP8 KV cache support to Triton MLA decode attention](../sources/prs/vllm/PR-34597.md), [fix(mxfp4): return is_monolithic=False when LoRA is enabled for Triton backend](../sources/prs/vllm/PR-35382.md), [[XPU] Support block fp8 moe by fallback to TritonExpert on XPU](../sources/prs/vllm/PR-36458.md), [[Kernel] Fuse FP8 output quantization into merge_attn_states](../sources/prs/vllm/PR-36518.md), [[Bugfix] Warm up Triton autotuner for GDN layers during V1 profiling](../sources/prs/vllm/PR-36599.md), [[Bugfix] Fix KV scales inconsistency in fp8 MLA & FlashInfer kv_cache_dtype "auto" leading to gibberish](../sources/prs/vllm/PR-37054.md), [[MoE Refactor] Mxfp4 oracle rebased](../sources/prs/vllm/PR-37128.md), [[Misc] Add `float16` to `CacheDType`](../sources/prs/vllm/PR-37199.md), [[NIXL][BUG] Fix Triton heterogeneous TP](../sources/prs/vllm/PR-37940.md), [[Perf] triton bilinear_pos_embed kernel for ViT](../sources/prs/vllm/PR-37948.md), [[Perf] FP8 FlashInfer Attn for ViT](../sources/prs/vllm/PR-38065.md), [[Bugfix] Enable batch-invariant Triton matmul on all Ampere GPUs (SM 8x) ](../sources/prs/vllm/PR-38427.md), [[Attention Backend] TurboQuant: 2-bit KV cache compression with 4x capacity](../sources/prs/vllm/PR-38479.md), [[Kernels][MoE] Fix legacy_routing to use bitmatrix-based routing path](../sources/prs/vllm/PR-38504.md), [[Perf] Reduce H2D pageable memory copies](../sources/prs/vllm/PR-38794.md), [[MoE] Move GPT OSS Triton kernel experts into fused_moe/experts/](../sources/prs/vllm/PR-39007.md), [[XPU] Quick fix for TritonMLA to remove cuda hardcode](../sources/prs/vllm/PR-39088.md), [[MoE] Move cutlass moe to fused_moe/experts/](../sources/prs/vllm/PR-40574.md), [[Attention][TurboQuant] Share dequant buffers, eliminate float16_copy](../sources/prs/vllm/PR-40941.md), [[MoE] Move various experts classes to fused_moe/experts/](../sources/prs/vllm/PR-41979.md), [[feat] Add FP8 per-tensor Q scale support to Triton attention backend](../sources/prs/vllm/PR-42080.md), [[Perf] Wire silu_and_mul_per_block_quant into TritonFP8MoE (MiniMax-M2) ](../sources/prs/vllm/PR-42497.md), [[Kernel] Pack topk id/weights triton kernel](../sources/prs/vllm/PR-42527.md), [[Perf][MLA] Enable FULL cudagraph capture for TRITON_MLA decode](../sources/prs/vllm/PR-42885.md), [FlashInfer Track A FP8 Block-Scale MoE](../wiki/kernels/fused-moe.md), [Gated Delta Net — Linear Attention](../wiki/kernels/gated-delta-net.md), [Native Sparse Attention (NSA)](../wiki/kernels/nsa.md) | diff --git a/queries/by-problem.md b/queries/by-problem.md index bbbff2ae1..b4bd75397 100644 --- a/queries/by-problem.md +++ b/queries/by-problem.md @@ -4,10 +4,10 @@ | Symptom | Pattern Page | Candidate Techniques | Sources | |---------|-------------|---------------------|---------| -| low-sm-utilization, tail-effect, load-imbalance | [Low SM Utilization](../wiki/patterns/low-sm-utilization.md) | [Persistent Kernels with CLC](../wiki/techniques/persistent-kernels.md), [Tile Scheduling Strategies](../wiki/techniques/tile-scheduling.md), [Cluster Launch Control (CLC)](../wiki/hardware/clc.md) | 3 sources | -| memory-bound, low-compute-utilization, high-memory-throughput | [Memory Bandwidth Bound](../wiki/patterns/memory-bound.md) | [Wide Vectorized Loads and Cache Policies](../wiki/techniques/vectorized-loads.md), [Shared Memory Swizzling](../wiki/techniques/swizzling.md), [Software Pipelining and Multi-Stage Buffering](../wiki/techniques/pipeline-stages.md) | 3 sources | -| load-imbalance, tail-effect, low-sm-utilization | [MoE Expert Load Imbalance](../wiki/patterns/moe-load-imbalance.md) | [Tile Scheduling Strategies](../wiki/techniques/tile-scheduling.md), [Persistent Kernels with CLC](../wiki/techniques/persistent-kernels.md), [Kernel Fusion](../wiki/techniques/kernel-fusion.md) | 3 sources | -| compute-bound, low-tensor-core-utilization, pipeline-stalls | [Not Reaching Peak FLOPS](../wiki/patterns/compute-bound.md) | [Two-SM Cooperative MMA](../wiki/hardware/2sm-cooperative.md), [Software Pipelining and Multi-Stage Buffering](../wiki/techniques/pipeline-stages.md), [Warp Specialization on Blackwell](../wiki/techniques/warp-specialization.md), [Epilogue Fusion](../wiki/techniques/epilogue-fusion.md), [Software-Emulated Exponential](../wiki/techniques/software-exp.md) | 3 sources | -| pipeline-stalls, compute-bound, low-tensor-core-utilization | [Pipeline Stalls](../wiki/patterns/pipeline-stalls.md) | [Software Pipelining and Multi-Stage Buffering](../wiki/techniques/pipeline-stages.md), [Warp Specialization on Blackwell](../wiki/techniques/warp-specialization.md), [Double/Multi-Buffering Patterns](../wiki/techniques/double-buffering.md), [Ping-Pong Scheduling](../wiki/techniques/ping-pong-scheduling.md) | 3 sources | -| register-pressure, low-occupancy, register-spilling | [Register Pressure — Low Occupancy](../wiki/patterns/register-pressure.md) | [Tensor Memory (TMEM)](../wiki/hardware/tmem.md), [Warp Specialization on Blackwell](../wiki/techniques/warp-specialization.md), [Register Accumulators to TMEM](../wiki/migration/register-to-tmem.md) | 3 sources | -| tail-effect, low-sm-utilization, wave-quantization | [Tail Effect — Last Wave Underutilization](../wiki/patterns/tail-effect.md) | [Persistent Kernels with CLC](../wiki/techniques/persistent-kernels.md), [Cluster Launch Control (CLC)](../wiki/hardware/clc.md), [Tile Scheduling Strategies](../wiki/techniques/tile-scheduling.md) | 3 sources | +| low-sm-utilization, tail-effect, load-imbalance | [Low SM Utilization](../wiki/patterns/low-sm-utilization.md) | [Persistent Kernels with CLC](../wiki/techniques/persistent-kernels.md), [Tile Scheduling Strategies](../wiki/techniques/tile-scheduling.md), [Cluster Launch Control (CLC)](../wiki/hardware/clc.md) | 4 sources | +| memory-bound, low-compute-utilization, high-memory-throughput | [Memory Bandwidth Bound](../wiki/patterns/memory-bound.md) | [Vectorized Loads and Cache Hints](../wiki/techniques/vectorized-loads.md), [PTX Cache Eviction and Prefetch Hints](../wiki/techniques/cache-policy.md), [Shared Memory Swizzling](../wiki/techniques/swizzling.md), [Software Pipelining and Multi-Stage Buffering](../wiki/techniques/pipeline-stages.md) | 4 sources | +| load-imbalance, tail-effect, low-sm-utilization | [MoE Expert Load Imbalance](../wiki/patterns/moe-load-imbalance.md) | [Tile Scheduling Strategies](../wiki/techniques/tile-scheduling.md), [Persistent Kernels with CLC](../wiki/techniques/persistent-kernels.md), [Kernel Fusion](../wiki/techniques/kernel-fusion.md) | 5 sources | +| compute-bound, low-tensor-core-utilization, pipeline-stalls | [Not Reaching the Relevant Compute Ceiling](../wiki/patterns/compute-bound.md) | [Two-SM Cooperative MMA](../wiki/hardware/2sm-cooperative.md), [Software Pipelining and Multi-Stage Buffering](../wiki/techniques/pipeline-stages.md), [Warp Specialization on Blackwell](../wiki/techniques/warp-specialization.md), [Epilogue Fusion](../wiki/techniques/epilogue-fusion.md), [Software-Emulated Exponential](../wiki/techniques/software-exp.md) | 4 sources | +| pipeline-stalls, compute-bound, low-tensor-core-utilization | [Pipeline Stalls](../wiki/patterns/pipeline-stalls.md) | [Software Pipelining and Multi-Stage Buffering](../wiki/techniques/pipeline-stages.md), [Warp Specialization on Blackwell](../wiki/techniques/warp-specialization.md), [Double/Multi-Buffering Patterns](../wiki/techniques/double-buffering.md), [Ping-Pong Scheduling](../wiki/techniques/ping-pong-scheduling.md) | 4 sources | +| register-pressure, low-occupancy, register-spilling | [Register Pressure and Residency](../wiki/patterns/register-pressure.md) | [Tensor Memory (TMEM)](../wiki/hardware/tmem.md), [Warp Specialization on Blackwell](../wiki/techniques/warp-specialization.md), [Register Budgeting for Occupancy](../wiki/techniques/register-budgeting.md), [Register Accumulators to TMEM](../wiki/migration/register-to-tmem.md) | 3 sources | +| tail-effect, low-sm-utilization, wave-quantization | [Tail Effect — Last-Wave Underutilization](../wiki/patterns/tail-effect.md) | [Persistent Kernels with CLC](../wiki/techniques/persistent-kernels.md), [Cluster Launch Control (CLC)](../wiki/hardware/clc.md), [Tile Scheduling Strategies](../wiki/techniques/tile-scheduling.md) | 4 sources | diff --git a/queries/by-repo.md b/queries/by-repo.md index 14e8ca649..aead7ba1a 100644 --- a/queries/by-repo.md +++ b/queries/by-repo.md @@ -259,7 +259,7 @@ | [#6819](../sources/prs/cccl/PR-6819.md) | Use integer promotion for `warp_reduce` | 2025-12-01 | | gemm | | [#6811](../sources/prs/cccl/PR-6811.md) | Integrate decoupled lookahead warpspeed scan | 2025-11-28 | | gemm | | [#6597](../sources/prs/cccl/PR-6597.md) | Split fixed-size segmented reduce dispatch header | 2025-11-12 | | gemm | -| [#6152](../sources/prs/cccl/PR-6152.md) | Fix debug section around line 390 of dispatch_topk | 2025-10-08 | | gemm | +| [#6152](../sources/prs/cccl/PR-6152.md) | Fix debug section around line 390 of dispatch_topk | 2025-10-08 | top-k-selection | top-k-selection | | [#6069](../sources/prs/cccl/PR-6069.md) | Add dynamic CUB dispatch for segmented_sort | 2025-09-30 | | gemm | | [#6077](../sources/prs/cccl/PR-6077.md) | [CUB] Use `BlockLoadToShared` in `DeviceMerge` | 2025-09-30 | | gemm | | [#5408](../sources/prs/cccl/PR-5408.md) | Combine `block_reduce_warp_reduction_nondeterministic.cuh` specialization with original deterministic one | 2025-08-01 | | gemm | @@ -267,7 +267,7 @@ | [#4961](../sources/prs/cccl/PR-4961.md) | Add nondeterministic reduce that uses atomics | 2025-06-11 | | gemm | | [#4716](../sources/prs/cccl/PR-4716.md) | Split Optimize Warp Reduce PR - CUB part | 2025-05-15 | | gemm | | [#3691](../sources/prs/cccl/PR-3691.md) | Fix SM100 histogram tunings | 2025-02-05 | | gemm | -| [#3559](../sources/prs/cccl/PR-3559.md) | Add b200 tunings for scan.exclusive.sum | 2025-01-28 | | gemm | +| [#3559](../sources/prs/cccl/PR-3559.md) | Add b200 tunings for scan.exclusive.sum | 2025-01-28 | parallel-scan | parallel-scan | | [#3517](../sources/prs/cccl/PR-3517.md) | Fix the vectorized loading of BlockLoad | 2025-01-24 | | gemm | | [#3236](../sources/prs/cccl/PR-3236.md) | Fix scan / sm90 perf regression | 2025-01-02 | | gemm | | [#2944](../sources/prs/cccl/PR-2944.md) | fix thread-reduce performance regression | 2024-11-22 | | gemm | @@ -529,7 +529,7 @@ | [#2416](../sources/prs/flashinfer/PR-2416.md) | feat: update trtllm-gen MoE cubins | 2026-01-26 | | gemm, moe, tma | | [#2415](../sources/prs/flashinfer/PR-2415.md) | Remove cudaMalloc/Free in GDN prefill kernel | 2026-01-25 | | prefill | | [#2405](../sources/prs/flashinfer/PR-2405.md) | perf: improve gdn decode cute-dsl kernels | 2026-01-23 | | decode | -| [#2387](../sources/prs/flashinfer/PR-2387.md) | A Blackwell-optimized version of selective_state_update (decode) | 2026-01-22 | warp-specialization, pipeline-stages, double-buffering | tcgen05, decode | +| [#2387](../sources/prs/flashinfer/PR-2387.md) | A Blackwell-optimized version of selective_state_update (decode) | 2026-01-22 | warp-specialization, pipeline-stages, double-buffering | decode | | [#2398](../sources/prs/flashinfer/PR-2398.md) | feat: cuteDSL fp4 moe for better DSR1 performance. | 2026-01-22 | kernel-fusion, pipeline-stages | fp4, gemm, grouped-gemm | | [#2404](../sources/prs/flashinfer/PR-2404.md) | perf: mm_fp4 heuristic prioritizes CUTLASS over cuDNN on SM103 | 2026-01-22 | | fp4, gemm | | [#2395](../sources/prs/flashinfer/PR-2395.md) | feat: Add output_both_sf_layouts option to add_rmsnorm_fp4quant API | 2026-01-21 | | fp4 | @@ -2344,7 +2344,7 @@ | [#24722](../sources/prs/vllm/PR-24722.md) | [Kernel][Quantization] add w4a8 support for marlin kernel | 2025-09-12 | kernel-fusion | fp4, fp8, gemm | | [#24727](../sources/prs/vllm/PR-24727.md) | [Model] Support Qwen3-VL Model Series | 2025-09-12 | | moe | | [#24774](../sources/prs/vllm/PR-24774.md) | [Bug] Fix `is_flashmla_supported` Check Error | 2025-09-12 | | attention, mla | -| [#23696](../sources/prs/vllm/PR-23696.md) | [Kernel][tcgen05] nvfp4 fused tcgen05 moe | 2025-09-11 | kernel-fusion, fine-grained-quantization | tcgen05, nvfp4, moe | +| [#23696](../sources/prs/vllm/PR-23696.md) | [Kernel][B200] mxfp4 fused cutlass moe | 2025-09-11 | kernel-fusion, fine-grained-quantization | fp4, fp8, moe | | [#24666](../sources/prs/vllm/PR-24666.md) | [Performance] Move apply_w8a8_block_fp8_linear to an op class | 2025-09-11 | | fp8, gemm, quantization | | [#24673](../sources/prs/vllm/PR-24673.md) | [NVIDIA] Blackwell Family | 2025-09-11 | | fp8, quantization | | [#24521](../sources/prs/vllm/PR-24521.md) | [Feature] Disallow FlashMLA on Blackwell | 2025-09-09 | | attention, mla | @@ -2593,7 +2593,7 @@ | [#17280](../sources/prs/vllm/PR-17280.md) | [NVIDIA] Support Cutlass w8a8 FP8 for Blackwell Geforce GPUs (sm120) | 2025-04-28 | | fp8, quantization | | [#17283](../sources/prs/vllm/PR-17283.md) | [BugFix] Fix cascade attention - RuntimeError: scheduler_metadata must have shape (metadata_size) | 2025-04-28 | | attention | | [#17289](../sources/prs/vllm/PR-17289.md) | [Misc][ROCm] Exclude `cutlass_mla_decode` for ROCm build | 2025-04-28 | | decode, mla | -| [#16032](../sources/prs/vllm/PR-16032.md) | [NVIDIA] Support Cutlass MLA for Blackwell GPUs | 2025-04-27 | warp-specialization, persistent-kernel | tcgen05, mla, moe | +| [#16032](../sources/prs/vllm/PR-16032.md) | [NVIDIA] Support Cutlass MLA for Blackwell GPUs | 2025-04-27 | persistent-kernel | mla, attention | | [#17267](../sources/prs/vllm/PR-17267.md) | [BugFix] Fix vllm_flash_attn install issues | 2025-04-27 | | attention, mla | | [#17222](../sources/prs/vllm/PR-17222.md) | [Bugfix] Get a specific type of layer from forward context | 2025-04-26 | | attention | | [#17180](../sources/prs/vllm/PR-17180.md) | [Bugfix] gemma[2,3] interleaved attention when sliding window is disabled | 2025-04-25 | | attention, gemm | diff --git a/queries/by-technique.md b/queries/by-technique.md index 8f23afae1..0b7a39526 100644 --- a/queries/by-technique.md +++ b/queries/by-technique.md @@ -4,20 +4,20 @@ | Technique | Tags | Architectures | Confidence | Reproducibility | Sources | |-----------|------|--------------|------------|-----------------|---------| -| [CCCL CUB Memory Primitives For Selection And Scan](../wiki/techniques/cccl-memory-primitives.md) | cuda-cpp, top-k-selection, parallel-scan, vectorized-loads | sm100, sm90 | source-reported | snippet | 2 | -| [Chunk-Based Parallelism for Linear Attention](../wiki/techniques/chunk-parallelism.md) | chunk-parallelism, linear-attention, gated-delta-net, pipeline-stages | sm100, sm90 | source-reported | snippet | 3 | -| [Double/Multi-Buffering Patterns](../wiki/techniques/double-buffering.md) | double-buffering, tmem, pipeline-stages | sm100, sm90 | source-reported | snippet | 3 | -| [Epilogue Fusion](../wiki/techniques/epilogue-fusion.md) | epilogue-fusion, tmem, warp-specialization | sm100, sm90 | source-reported | snippet | 3 | -| [External Source-Map Research For Kernel Edits](../wiki/techniques/external-source-map-research.md) | cuda-cpp, cute-dsl, tma, wgmma | sm100, sm90 | source-reported | snippet | 5 | +| [CCCL CUB Memory Primitives For Selection And Scan](../wiki/techniques/cccl-memory-primitives.md) | cuda-cpp, top-k-selection, parallel-scan, vectorized-loads | sm100, sm90 | source-reported | concept | 2 | +| [Chunkwise Parallelism for Linear Recurrences](../wiki/techniques/chunk-parallelism.md) | chunk-parallelism, linear-attention, gated-delta-net, pipeline-stages | sm100, sm90 | verified | concept | 3 | +| [Double/Multi-Buffering Patterns](../wiki/techniques/double-buffering.md) | double-buffering, tmem, pipeline-stages | sm100, sm90 | verified | pseudocode | 3 | +| [Epilogue Fusion](../wiki/techniques/epilogue-fusion.md) | epilogue-fusion, tmem, warp-specialization | sm100, sm90 | verified | pseudocode | 3 | +| [External Source-Map Research For Kernel Edits](../wiki/techniques/external-source-map-research.md) | cuda-cpp, cute-dsl, tma, wgmma | sm90a | source-reported | snippet | 5 | | [Fine-Grained FP8/FP4 Quantization](../wiki/techniques/fine-grained-quantization.md) | fine-grained-quantization, fp8, fp4, nvfp4 | sm100, sm90 | source-reported | snippet | 3 | -| [Kernel Fusion](../wiki/techniques/kernel-fusion.md) | kernel-fusion, fused-kernel, tmem | sm100, sm90 | source-reported | snippet | 3 | -| [PTX Cache Policy Differentiation](../wiki/techniques/cache-policy.md) | cache-policy, vectorized-loads | sm100, sm90 | source-reported | snippet | 4 | -| [Persistent Kernels with CLC](../wiki/techniques/persistent-kernels.md) | persistent-kernel, clc, tile-scheduling | sm100 | source-reported | snippet | 3 | -| [Ping-Pong Scheduling](../wiki/techniques/ping-pong-scheduling.md) | ping-pong-scheduling, warp-specialization, tmem, pipeline-stages | sm100 | source-reported | snippet | 3 | -| [Register Budgeting for Occupancy](../wiki/techniques/register-budgeting.md) | register-budgeting, register-reuse | sm100, sm90 | source-reported | snippet | 3 | -| [Shared Memory Swizzling](../wiki/techniques/swizzling.md) | swizzling, shared-memory-optimization, tma | sm100, sm90 | source-reported | snippet | 3 | -| [Software Pipelining and Multi-Stage Buffering](../wiki/techniques/pipeline-stages.md) | pipeline-stages, double-buffering, tma, mbarrier | sm100, sm90 | source-reported | snippet | 3 | -| [Software-Emulated Exponential](../wiki/techniques/software-exp.md) | software-exp, attention | sm100 | source-reported | snippet | 3 | -| [Tile Scheduling Strategies](../wiki/techniques/tile-scheduling.md) | tile-scheduling, clc, persistent-kernel | sm100, sm90 | source-reported | snippet | 3 | -| [Warp Specialization on Blackwell](../wiki/techniques/warp-specialization.md) | warp-specialization, tcgen05, tmem | sm100, sm90 | source-reported | snippet | 3 | -| [Wide Vectorized Loads and Cache Policies](../wiki/techniques/vectorized-loads.md) | vectorized-loads, cache-policy, register-budgeting | sm100, sm90 | source-reported | snippet | 3 | +| [Kernel Fusion](../wiki/techniques/kernel-fusion.md) | kernel-fusion, fused-kernel, tmem | sm100, sm90 | verified | pseudocode | 2 | +| [PTX Cache Eviction and Prefetch Hints](../wiki/techniques/cache-policy.md) | cache-policy, vectorized-loads | sm100, sm90 | verified | concept | 4 | +| [Persistent Kernels with CLC](../wiki/techniques/persistent-kernels.md) | persistent-kernel, clc, tile-scheduling | sm100 | verified | pseudocode | 3 | +| [Ping-Pong Scheduling](../wiki/techniques/ping-pong-scheduling.md) | ping-pong-scheduling, warp-specialization, tmem, pipeline-stages | sm100 | verified | concept | 2 | +| [Register Budgeting for Occupancy](../wiki/techniques/register-budgeting.md) | register-budgeting, register-reuse | sm100, sm90 | verified | concept | 3 | +| [Shared Memory Swizzling](../wiki/techniques/swizzling.md) | swizzling, shared-memory-optimization, tma | sm100, sm90 | verified | concept | 3 | +| [Software Pipelining and Multi-Stage Buffering](../wiki/techniques/pipeline-stages.md) | pipeline-stages, double-buffering, tma, mbarrier | sm100, sm90 | verified | pseudocode | 3 | +| [Software-Emulated Exponential](../wiki/techniques/software-exp.md) | software-exp, attention | sm100 | verified | concept | 3 | +| [Tile Scheduling Strategies](../wiki/techniques/tile-scheduling.md) | tile-scheduling, clc, persistent-kernel | sm100, sm90 | source-reported | snippet | 2 | +| [Vectorized Loads and Cache Hints](../wiki/techniques/vectorized-loads.md) | vectorized-loads, cache-policy, register-budgeting | sm100, sm90 | verified | concept | 5 | +| [Warp Specialization on Blackwell](../wiki/techniques/warp-specialization.md) | warp-specialization, tcgen05, tmem | sm100, sm90 | source-reported | concept | 4 | diff --git a/references/examples.md b/references/examples.md index aec70c539..daf7dedd1 100644 --- a/references/examples.md +++ b/references/examples.md @@ -1,6 +1,6 @@ --- version_sensitive: - id: vs-triton-3.6-blackwell-tcgen05 + id: vs-triton-3.3-blackwell-tcgen05 --- # Worked Query Examples @@ -109,7 +109,7 @@ python3 scripts/get_page.py contest-gpumode-p1 **Navigation path**: 1. `wiki/kernels/gated-delta-net.md` — conceptual + code -2. `wiki/languages/triton-blackwell.md` — current Triton 3.6+ Blackwell lowering surfaces (tcgen05 + TMEM via descriptor/TMA + warp_specialize, `tl.dot_scaled`, Gluon multi-CTA); pre-3.6 historical context preserved in a clearly-marked subsection +2. `wiki/languages/triton-blackwell.md` — native TCGen5/TMEM backend support from v3.3, later v3.5/v3.6 surface expansions, and the evidence limits on plain `tl.dot` instruction selection 3. Source PRs: `pr-vllm-*` for gated_delta, FlashInfer GDN kernels **Command**: diff --git a/references/primer.md b/references/primer.md index f1662a271..3258bc691 100644 --- a/references/primer.md +++ b/references/primer.md @@ -1,6 +1,6 @@ --- version_sensitive: - id: vs-triton-3.6-blackwell-tcgen05 + id: vs-triton-3.3-blackwell-tcgen05 --- # Topic Map / Primer @@ -88,7 +88,7 @@ All page IDs below resolve via `get_page.py `. All paths are relative to the | CuTe DSL | `lang-cute-dsl` | Preferred high-level path on SM100; native tcgen05/TMEM/CLC bindings. | | CUDA C++ | `lang-cuda-cpp` | PTX inline is common; used by CUTLASS, vLLM, SGLang custom kernels. | | PTX (SM100) | `lang-ptx` | `tcgen05.*`, `clusterlaunchcontrol.*`, `cp.async.bulk.tensor.*` — low-level control. | -| Triton | `lang-triton` | On Blackwell: Triton 3.6+ ships native tcgen05 + TMEM lowering through descriptor/TMA + `tl.range(warp_specialize=True)`, `tl.dot_scaled`, and Gluon multi-CTA / 2CTA. Pre-3.6 the framing was "no tcgen05/TMEM exposure"; that historical context is preserved on the page. Cite via `version_sensitive: vs-triton-3.6-blackwell-tcgen05`. | +| Triton | `lang-triton` | Native Blackwell TCGen5/TMEM compiler support enters between Triton v3.2.0 and v3.3.0. Later releases expand user-visible paths; exact instruction selection remains configuration-dependent. Cite via `version_sensitive: vs-triton-3.3-blackwell-tcgen05`. | --- diff --git a/scripts/validate.py b/scripts/validate.py index 4e84123d6..37f5fd7af 100644 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -376,7 +376,9 @@ def validate_file(filepath, schemas, valid_tags, all_source_ids, code_langs): if all_source_ids and src_id not in all_source_ids: errors.append(f"{rel}: references unknown source id '{src_id}'") - # AC-9: Enforce evidence_basis for verified wiki pages + # AC-9: Enforce a direct, page-scoped evidence basis for verified wiki + # pages. The verification contract accepts any adequate primary oracle; + # it does not require every claim to have both documentation and code. if page_type.startswith("wiki-") and fm.get("confidence") == "verified": eb = fm.get("evidence_basis") if not eb or not isinstance(eb, list) or len(eb) == 0: @@ -384,17 +386,6 @@ def validate_file(filepath, schemas, valid_tags, all_source_ids, code_langs): f"{rel}: confidence 'verified' requires non-empty 'evidence_basis' field" ) else: - eb_types = {entry.get("evidence_type") for entry in eb if isinstance(entry, dict)} - if "official-doc" not in eb_types: - errors.append( - f"{rel}: evidence_basis for 'verified' must include at least one " - f"'official-doc' entry (found: {eb_types})" - ) - if "upstream-code" not in eb_types: - errors.append( - f"{rel}: evidence_basis for 'verified' must include at least one " - f"'upstream-code' entry (found: {eb_types})" - ) # Cross-check evidence_basis source_ids against page sources page_sources = set(fm.get("sources", [])) for entry in eb: @@ -406,8 +397,13 @@ def validate_file(filepath, schemas, valid_tags, all_source_ids, code_langs): f"not listed in page sources" ) - # Check technique/kernel/language pages have fenced code - if page_type in ("wiki-technique", "wiki-kernel", "wiki-language"): + # A page claiming snippet-or-better reproducibility must contain real code. + # Concept and pseudocode pages may instead provide an evidence-backed + # procedure without inventing a compilable implementation. + if ( + page_type in ("wiki-technique", "wiki-kernel", "wiki-language") + and repro_at_least(fm.get("reproducibility"), "snippet") + ): body = read_body(filepath) if not has_compilable_code(body, code_langs): errors.append(f"{rel}: {page_type} page must contain fenced code block (reproducibility >= snippet)") diff --git a/scripts/verify_core_prs.py b/scripts/verify_core_prs.py index 8a522b446..13037fe2d 100755 --- a/scripts/verify_core_prs.py +++ b/scripts/verify_core_prs.py @@ -3,7 +3,7 @@ Modes: default — regenerate in memory and diff against committed bytes - --strict — additionally resolve each captured PR's merge_sha via `gh api` + --strict — additionally resolve each captured PR's merge_sha from GitHub and flag reverted / unresolvable / prefix-mismatched entries Exit codes: @@ -22,6 +22,7 @@ """ import argparse +from concurrent.futures import ThreadPoolExecutor import hashlib import json import subprocess @@ -38,7 +39,11 @@ # which gh failures are environmental (exit 2) versus content-level (exit 1). sys.path.insert(0, str(REPO_ROOT / "scripts")) import compute_core_prs # noqa: E402 -from verify_verbatim import EnvError, _looks_like_env_error # noqa: E402 +from verify_verbatim import ( # noqa: E402 + EnvError, + _looks_like_env_error, + fetch_pull_metadata, +) def run_gh(args): @@ -169,7 +174,7 @@ def main(): f"total_captured are consistent (checksum {fresh.get('checksum_sha256','')[:12]}..., " f"{fresh.get('total_captured',0)} PRs)") - # --strict: resolve merge_sha via gh api + # --strict: resolve merge_sha from GitHub's first-party PR record. if args.strict: issues = 0 # content-level findings (exit 1) env_failures = 0 # environment / connectivity (exit 2) @@ -192,58 +197,54 @@ def main(): if fm.get("id"): sources_prs[fm["id"]] = fm - print(f"Resolving merge_sha for {len(pr_entries)} PRs via gh api...") - for e in pr_entries: + def resolve_entry(e): pid = e.get("id") fm = sources_prs.get(pid) if not fm: - print(f" WARN: {pid}: page not found in sources/prs/") - issues += 1 - continue + return "issue", f" WARN: {pid}: page not found in sources/prs/" sha = fm.get("merge_sha") repo = fm.get("repo") pr_num = fm.get("pr") if not (sha and repo and pr_num): - print(f" WARN: {pid}: missing merge_sha / repo / pr number") - issues += 1 - continue + return "issue", f" WARN: {pid}: missing merge_sha / repo / pr number" try: - out = run_gh(["api", f"/repos/{repo}/pulls/{pr_num}"]) - data = json.loads(out) + data = fetch_pull_metadata(repo, str(pr_num)) if not data.get("merged"): - print(f" FAIL: {pid}: upstream state is not merged (state={data.get('state')})") - issues += 1 - else: - # sources/prs/**/*.md stores abbreviated 8-char - # merge_sha values; gh returns the full 40-char - # merge_commit_sha. R30: match ONLY against - # merge_commit_sha (the actual merged revision). - # Previously this check also accepted a prefix - # match against head.sha, which let stale merge_sha - # values pass strict verification whenever the PR - # branch was kept alive after merge or a squash/ - # rebase merge moved the merge commit away from - # head — the recorded `merge_sha` no longer named - # the commit the bundle was fetched from. - upstream_merge = str(data.get("merge_commit_sha") or "") - if not (upstream_merge and upstream_merge.startswith(sha)): - upstream_head = str((data.get("head") or {}).get("sha") or "") - print( - f" FAIL: {pid}: recorded merge_sha={sha[:12]}... does not prefix-match " - f"upstream merge_commit_sha={upstream_merge[:12]}... " - f"(head.sha={upstream_head[:12]}... shown for reference; not accepted)" - ) - issues += 1 + return "issue", f" FAIL: {pid}: upstream state is not merged (state={data.get('state')})" + + # sources/prs/**/*.md stores abbreviated 8-char merge_sha + # values; GitHub returns the full 40-char merge_commit_sha. + # Match ONLY against merge_commit_sha (the actual merged + # revision), never against head.sha. + upstream_merge = str(data.get("merge_commit_sha") or "") + if not (upstream_merge and upstream_merge.startswith(sha)): + upstream_head = str((data.get("head") or {}).get("sha") or "") + return "issue", ( + f" FAIL: {pid}: recorded merge_sha={sha[:12]}... does not prefix-match " + f"upstream merge_commit_sha={upstream_merge[:12]}... " + f"(head.sha={upstream_head[:12]}... shown for reference; not accepted)" + ) + return "ok", "" except EnvError as ex: # Offline / unauthenticated / rate-limited / DNS unreachable. # These are environment failures, not content drift, so they # must surface via the exit-2 contract instead of being # conflated with real merge-SHA mismatches (exit 1). - print(f" ENV: {pid}: gh unreachable: {ex}", file=sys.stderr) - env_failures += 1 + return "env", f" ENV: {pid}: GitHub unreachable: {ex}" except RuntimeError as ex: - print(f" WARN: {pid}: gh fetch failed: {ex}") + return "issue", f" WARN: {pid}: GitHub fetch failed: {ex}" + + print(f"Resolving merge_sha for {len(pr_entries)} PRs from GitHub...") + workers = min(8, max(1, len(pr_entries))) + with ThreadPoolExecutor(max_workers=workers) as executor: + results = list(executor.map(resolve_entry, pr_entries)) + for kind, message in results: + if kind == "issue": + print(message) issues += 1 + elif kind == "env": + print(message, file=sys.stderr) + env_failures += 1 # Environment failures take precedence over content issues: if we # could not reach GitHub at all, the content check is inconclusive, @@ -253,8 +254,8 @@ def main(): if env_failures: print( f"\nSTRICT inconclusive: {env_failures} environment/connectivity " - f"failure(s) while resolving merge_sha via gh api. " - f"Re-run with network + authenticated gh to complete verification.", + f"failure(s) while resolving merge_sha from GitHub. " + f"Re-run with network access to complete verification.", file=sys.stderr, ) sys.exit(2) diff --git a/scripts/verify_verbatim.py b/scripts/verify_verbatim.py index 3ece11083..de6df5ff5 100755 --- a/scripts/verify_verbatim.py +++ b/scripts/verify_verbatim.py @@ -25,6 +25,10 @@ import yaml import base64 import json +from functools import lru_cache +from urllib.error import HTTPError, URLError +from urllib.parse import quote +from urllib.request import Request, urlopen REPO_ROOT = Path(__file__).resolve().parent.parent ARTIFACTS_DIR = REPO_ROOT / "artifacts" @@ -80,6 +84,7 @@ class EnvError(RuntimeError): "proxyconnect", # Auth / account state (missing auth => env, not content) "authentication required", + "to authenticate", "you must authenticate", "you are not logged in", "not logged into", @@ -119,17 +124,104 @@ def run_gh(args): raise RuntimeError(f"gh {' '.join(args)} failed: {stderr_text.strip()[:200]}") +def _fetch_public_url(url, *, accept="application/octet-stream"): + """Fetch a public GitHub URL without credentials. + + GitHub CLI intentionally refuses even public API requests when it has no + authenticated host. The strict verifiers still need to work in clean CI + and audit containers, so use GitHub's public raw/diff/timeline endpoints as + a transport fallback. The same pinned revisions and byte comparisons are + enforced; this changes only how upstream bytes are obtained. + """ + request = Request( + url, + headers={ + "Accept": accept, + "User-Agent": "KernelWiki-upstream-verifier/1.0", + "X-Requested-With": "XMLHttpRequest", + }, + ) + try: + with urlopen(request, timeout=90) as response: + return response.read() + except HTTPError as exc: + detail = exc.read(300).decode(errors="replace").strip() + if exc.code in (403, 429) or _looks_like_env_error(detail): + raise EnvError(f"public GitHub fetch environment failure for {url}: HTTP {exc.code} {detail[:160]}") + raise RuntimeError(f"public GitHub fetch failed for {url}: HTTP {exc.code} {detail[:160]}") + except (URLError, TimeoutError, OSError) as exc: + raise EnvError(f"public GitHub fetch environment failure for {url}: {exc}") + + +@lru_cache(maxsize=None) +def fetch_pull_metadata(upstream_repo, pr_number): + """Return the merged state and exact merge commit for a public PR. + + Prefer authenticated ``gh api``. Without credentials, GitHub's public PR + timeline is an upstream first-party record and includes the exact 40-byte + merge-commit link. Unlike a head-ref fallback, this preserves the strict + requirement that the recorded SHA name the actual merged revision. + """ + try: + return json.loads(run_gh(["api", f"/repos/{upstream_repo}/pulls/{pr_number}"])) + except EnvError as gh_error: + escaped_repo = quote(str(upstream_repo), safe="/") + url = ( + f"https://github.com/{escaped_repo}/pull/{pr_number}/partials/" + "conversation_content?graceful_retry=1&timeline_per_page=1000" + ) + try: + timeline = _fetch_public_url(url, accept="text/html").decode(errors="replace") + except (EnvError, RuntimeError) as public_error: + raise EnvError(f"{gh_error}; public timeline fallback also failed: {public_error}") + + repo_pattern = re.escape(str(upstream_repo)) + match = re.search( + rf"merged\s+commit\s*1000us) -- Problem: no coalescing, no vectorization - -Attempt 2: Coalesced memory access -- Threads in warp read contiguous bytes -- Result: ~500us -- Problem: still using generic FP4 decode - -Attempt 3: Vectorized loads (uint4) -- 128-bit loads for better bandwidth utilization -- Result: ~200us -- Problem: FP4 decode still manual bitwise ops -``` - -### Attempts 4-6: FP4 Decode Optimization - -``` -Attempt 4: Hardware FP4 intrinsics -- __cvt_fp4x2_to_halfx2 for type conversion -- Result: ~80us -- Big improvement from hardware-accelerated decode +## Attempts 8–12 -Attempt 5: PTX byte unpacking -- mov.b32 {a,b,c,d} instead of shift/mask -- Result: ~50us -- Eliminates bitwise extraction overhead +The next five attempts did not form a monotonic optimization progression: -Attempt 6: Combined PTX for load + decode -- Inline PTX for entire load-decode pipeline -- Result: ~40us -``` +- Attempt 8 used split-K plus FP32 atomics and was worse because of contention, extra traffic, and scheduling overhead. +- Attempt 9 replaced two `uchar4` loads with one `uint2` load and was 16–25% slower because byte extraction added instructions. +- Attempt 10 used four accumulator chains and regressed by 32–55% with more register pressure and worse coalescing. +- Attempt 11 found no effect from reducing `-maxrregcount` from 80 to 64 and no effect from the tested block-size change; unroll 8 was worse than unroll 4. +- Attempt 12's explicit software pipeline increased register pressure and was slower. -### Attempts 7-9: Memory System Optimization +The author's main retrospective lesson is to run Nsight Compute early and verify the bottleneck before choosing transformations. -``` -Attempt 7: Cache policy differentiation -- L1::no_allocate for matrix A (streamed) -- L1::evict_last for vector B (reused) -- Result: ~35us -- B vector stays hot in L1 across rows +## Post-event Observations -Attempt 8: Wider loads (v4.u64 = 256-bit) -- Maximum vector width for global loads -- Result: ~30us -- Better memory transaction efficiency +The author says three inspected solutions around an 18.5 microsecond aggregate used raw PTX load/decode paths, A `L1::no_allocate`, B `L1::evict_last`, `v2.u64` or `v4.u64` loads with `mov.b32` byte decomposition, exact-K specializations, and tighter register budgets. The post reports 32 registers for one inspected solution and 45 for another, and says a further solution shared B reads across multiple M rows. -Attempt 9: Register budgeting (-maxrregcount) -- Tested 32, 40, 48, 56 max registers -- Result: ~28us with -maxrregcount=40 -- More warps per SM -> better latency hiding -``` +These are author-reported observations. They do not supply controlled ablations, public contestant code, or proof that lower register caps and wider loads monotonically improve performance. The author's own attempts provide counterexamples to both generalizations. -### Attempts 10-12: Fine-Tuning +## Primary Sources -``` -Attempt 10: Block size tuning -- Tested BLOCK_M = 1, 2, 4, 8 -- Result: ~27.5us with BLOCK_M=4 -- B vector amortized across more rows - -Attempt 11: K-dimension unrolling -- #pragma unroll for inner K loop -- Result: ~27us -- Marginal improvement from reduced loop overhead - -Attempt 12: Per-K specialization -- Separate kernels for different K values -- Result: ~26.7us (final) -- Each K variant fully unrolled -``` - -## Key Debugging Methodology - -### Nsight Compute Analysis - -The blog emphasizes using Nsight Compute to confirm the kernel is memory-bound: - -``` -// Nsight Compute key metrics to check: -// 1. Memory throughput: how close to 8 TB/s? -// 2. Compute throughput: should be low for memory-bound -// 3. Achieved occupancy: higher is better for memory-bound -// 4. L1 hit rate: should be high for B vector (evict_last) -// 5. L2 hit rate: confirms data reuse patterns - -// Key insight from Amandeep: -// "Run Nsight Compute to confirm memory-bound behavior" -// Many optimizations are counterproductive if you misidentify -// the bottleneck (e.g., compute optimizations on memory-bound kernel) -``` - -### Performance Model - -``` -// Speed-of-light calculation: -// Total data to read: -// A: M * K * 0.5 bytes (FP4) -// B: 1 * K * 0.5 bytes (FP4) -// sfa: M * (K/16) bytes (FP8) -// sfb: 1 * (K/16) bytes (FP8) -// Total ≈ M * K * 0.5625 bytes -// -// For M=7168, K=16384, L=1: -// Total = 7168 * 16384 * 0.5625 = 66 MB -// At 8 TB/s: 66 MB / 8 TB/s = 8.25us -// -// Actual: 26.7us = 3.2x off SOL -// The gap comes from: FP4 decode overhead, scale factor application, -// partial sum reduction, and less-than-perfect vectorization -``` - -## Failed Approaches (Instructive) - -1. **Shared memory for A matrix**: No benefit because A is streamed (each element read once). Shared memory only helps when data is reused. - -2. **Tensor cores for GEMV**: tcgen05.mma requires MxNxK tiles with M >= 128. GEMV has M=1 (or small M for batched), so tensor cores cannot be efficiently utilized. This is fundamentally a CUDA-core memory-bandwidth problem. - -3. **Warp shuffle reduction**: Expected to help for the K-dimension reduction, but the overhead of shuffle instructions exceeded the benefit over simple register accumulation at these K sizes. - -## Key Lessons - -1. **Profile first, optimize second**: Nsight Compute should be the first tool, not the last. Knowing whether a kernel is memory-bound or compute-bound determines the entire optimization strategy. - -2. **FP4 decode is the hidden bottleneck**: The sub-byte format introduces decode overhead that doesn't exist for standard FP16/FP32 kernels. Hardware intrinsics and PTX byte unpacking are essential. - -3. **Tensor cores don't help for GEMV**: GEMV is fundamentally memory-bandwidth-limited and the arithmetic intensity is too low for tensor cores. CUDA cores + wide vectorized loads are the right approach. - -4. **3x off SOL is realistic for FP4 GEMV**: The FP4 decode overhead, scale factor application, and reduction operations add unavoidable computation that a pure bandwidth model doesn't account for. - -5. **Systematic exploration beats intuition**: Documenting 12 attempts with measurements at each step is more productive than guessing at the optimal configuration. - -## Sources - -- [Twelve Attempts at NVFP4 Batched GEMV](https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/) -- [gpu-mode/reference-kernels](https://github.com/gpu-mode/reference-kernels) +- [Amandeep Singh, “Twelve Attempts at an FP4 Kernel”](https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/) +- [Public attempt repository](https://github.com/amandeepsp/cuda/tree/44513ac7d5bbd1cf8109cab952844adac5b6c551/nvfp4/gemv) diff --git a/sources/blogs/colfax-article-source-kernels.md b/sources/blogs/colfax-article-source-kernels.md index d2c198cf4..adc37012a 100644 --- a/sources/blogs/colfax-article-source-kernels.md +++ b/sources/blogs/colfax-article-source-kernels.md @@ -5,8 +5,7 @@ author: Colfax Research url: https://github.com/ColfaxResearch/cfx-article-src source_category: community-note architectures: -- sm90 -- sm100 +- sm90a tags: - cuda-cpp - cute-dsl @@ -20,7 +19,8 @@ retrieved_at: '2026-05-20' description: Source-map entry imported from KernelPilot for TMA, pipelined GEMM, Stream-K, and CuTe transpose examples. --- -This repository collects source files for Colfax Research articles. The useful -KernelWiki path is code-first: inspect the TMA, pipeline GEMM, Stream-K, -transpose-cute, and CUTLASS GEMM folders when a profile points to pipeline -bubbles, tail effects, or memory-layout pressure. +At commit `fbecfed88de2e4246f104a023188ba722937c5fc`, the relevant Hopper +examples compile for SM90a. Inspect `tma/tma_copy.h`, `pipeline-gemm/`, +`streamk/tile_scheduler.hpp`, and `transpose-cute/` for exact TMA, pipelined +GEMM, persistent/Stream-K scheduler, and transpose implementations. This pinned +tree contains no demonstrated SM100 target. diff --git a/sources/blogs/colfax-cutlass-blackwell.md b/sources/blogs/colfax-cutlass-blackwell.md index a65fb7284..6d09d653f 100644 --- a/sources/blogs/colfax-cutlass-blackwell.md +++ b/sources/blogs/colfax-cutlass-blackwell.md @@ -28,51 +28,21 @@ Detailed tutorial on CUTLASS abstraction for Blackwell UMMA (tcgen05.mma) with s - Architectural progression: Volta → Hopper TMA → Blackwell TMEM+UMMA - Sub-byte GEMM tutorial covering NVFP4, MXFP4, block scaling -## Key Code - -### TMEM allocation + tcgen05.mma (single-thread launch) - -```cuda -// UMMA on Blackwell: one thread drives the MMA for the whole CTA. -// Accumulator lives in TMEM, not registers. -__shared__ uint32_t tmem_addr; - -if (threadIdx.x == 0) { - // Allocate 128 rows × 256 cols of TMEM for the accumulator - asm volatile("tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], 256;\n" - :: "r"(&tmem_addr)); -} -__syncthreads(); - -// Issue UMMA: A and B live in SMEM, C accumulates into TMEM -if (threadIdx.x == 0) { - asm volatile( - "tcgen05.mma.cta_group::1.kind::f16 [%0], %1, %2, %3, 1;\n" - :: "r"(tmem_addr), "l"(desc_a), "l"(desc_b), "r"(0)); -} -``` - -### TMEM load into registers for epilogue - -```cuda -// Epilogue warps drain TMEM → registers using tcgen05.ld -// Each warp loads 32 columns (=128 bytes) at a time. -float reg[4]; -asm volatile( - "tcgen05.ld.sync.aligned.32x32b.x4.b32 " - "{%0, %1, %2, %3}, [%4];\n" - : "=f"(reg[0]), "=f"(reg[1]), "=f"(reg[2]), "=f"(reg[3]) - : "r"(tmem_addr + warp_col_offset)); -``` - -### CUTLASS MMA_Atom wrapping - -```cpp -// The CUTLASS two-level abstraction: MMA_Atom wraps the PTX intrinsic, -// MMA_Traits maps logical MxNxK shapes to TMEM addressing. -using Atom = cute::MMA_Atom>; -``` +## Verified implementation notes + +The article is useful for understanding CUTLASS's `MMA_Atom`/`MMA_Traits` +layering and TMEM-backed accumulators, but abbreviated inline PTX is not a safe +substitute for the ISA grammar: + +- `tcgen05.alloc` and `tcgen05.dealloc` are warp-collective for + `cta_group::1`; they are not lane-0-only instructions. +- `tcgen05.mma` is issued by one thread, but its full operand list includes a + disable-output-lane vector whose width depends on the CTA group. +- `tcgen05.ld` is warp-collective and asynchronous; use its documented shape, + repetition, register mapping, and completion mechanism. +- CUTLASS supplies exact wrapper types and layouts for supported combinations; + choose the atom from the version-pinned library rather than reconstructing a + template signature from prose. + +For executable references, use the [PTX ISA 9.0 tcgen05 sections](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma) +and the [CUTLASS 4.5.0 SM100 examples](https://github.com/NVIDIA/cutlass/tree/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/70_blackwell_gemm). diff --git a/sources/blogs/colfax-cutlass-kernels.md b/sources/blogs/colfax-cutlass-kernels.md index 085ca6e89..9fb6514ce 100644 --- a/sources/blogs/colfax-cutlass-kernels.md +++ b/sources/blogs/colfax-cutlass-kernels.md @@ -5,21 +5,21 @@ author: Colfax Research url: https://github.com/ColfaxResearch/cutlass-kernels source_category: community-note architectures: -- sm90 -- sm100 +- sm90a tags: - cuda-cpp +- cute-dsl - gemm - tma - wgmma -- persistent-kernel -- tile-scheduling - pipeline-stages +- warp-specialization retrieved_at: '2026-05-20' description: Source-map entry imported from KernelPilot for CUTLASS GEMM kernel examples and scheduling patterns. --- -Colfax CUTLASS kernel examples are a complementary source-map route for -production-shaped GEMM patterns. Use them alongside CUTLASS PR pages when a -candidate needs concrete tile scheduling, persistent scheduling, or TMA pipeline -code. +At commit `84f0802e2b4a1bf068ac70359f20ffdb368c8f6a`, this repository contains +Hopper SM90a CUTLASS/CuTe GEMM and FMHA examples. The FMHA READMEs and compile +scripts pin CUDA 12.2/12.3, CUTLASS 3.3/3.4, and SM90a; `src/fmha-pipeline/` +implements TMA pipelining with optional warp specialization. The pinned tree is +not evidence for SM100, a persistent scheduler, or a tile scheduler. diff --git a/sources/blogs/deepgemm.md b/sources/blogs/deepgemm.md index b8f031166..f8d15d2df 100644 --- a/sources/blogs/deepgemm.md +++ b/sources/blogs/deepgemm.md @@ -1,8 +1,8 @@ --- id: blog-deepgemm -title: DeepGEMM — FP8 GEMM Library +title: DeepGEMM — Pinned Upstream Project Summary author: DeepSeek AI -url: https://github.com/deepseek-ai/DeepGEMM +url: https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f source_category: benchmark-blog architectures: - sm100 @@ -15,84 +15,29 @@ tags: - jit-compilation - tcgen05 - wgmma -retrieved_at: 2026-04-16 -artifact_dir: artifacts/blogs/deepgemm/code +retrieved_at: 2026-04-27 +artifact_dir: artifacts/kernels/deepgemm/full --- -## Summary +## Scope -DeepSeek's high-performance FP8 GEMM library with fine-grained scaling, supporting both Hopper and Blackwell. +This source entry summarizes DeepGEMM at commit [`891d57b4db1071624b5c8fa0d1e51cb317fa709f`](https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f). The local SM90 and SM100 FP8 1D1D files are byte-verified copies from that commit; see their [`PROVENANCE.yaml`](../../artifacts/kernels/deepgemm/full/PROVENANCE.yaml). -## Key Techniques -- Fine-grained quantization: tile-wise 1×128 activations, block-wise 128×128 weights -- SM90: WGMMA with Nc=128 CUDA core promotion (FP22→FP32) -- SM100: tcgen05.mma with TMEM, packed UE8M0 scale format, all memory layouts -- MoE grouped GEMMs: M-axis grouping, contiguous/masked/K-grouped layouts -- JIT compilation via NVRTC -- ~300 lines core kernel code -- Up to 1550 TFLOPS on H800 +## Verified Techniques -## Key Code +- SM90 uses FP32 scale factors. Its pinned 1D1D kernel fixes `BLOCK_K == 128`, accumulates WGMMA partial results in `float accum[...]`, and applies A/B scales into a separate `float final_accum[...]` after each K block. +- SM100 uses packed UE8M0 scale factors. Its pinned 1D1D kernel copies factor blocks into TMEM, constructs a block-scaled UMMA descriptor, and accumulates through the selected TMEM operation without the SM90 CUDA-core `final_accum` loop. +- M-grouped contiguous and masked APIs vary M with N/K fixed. `masked_m` contains an integer valid-M length for each group. K-grouped APIs instead vary K while M/N remain fixed. +- Kernels are generated and JIT compiled. NVCC is the default compiler; `DG_JIT_USE_NVRTC=1` opts into NVRTC. Generated code and compiler settings participate in the cache signature. +- The pinned README exposes NT only for SM90 FP8 and NT/TN/NN/TT dense interfaces for SM100. -### Nc=128 CUDA-core promotion (Hopper SM90) +## Performance Scope -```cpp -// On Hopper, the TC accumulator is only ~FP22-precise. DeepGEMM promotes -// the partial sum to an FP32 CUDA-core accumulator every Nc=128 columns -// (4 consecutive WGMMAs of n=32 each) to avoid precision drift. -constexpr int Nc = 128; -constexpr int WGMMA_N = 32; +The pinned README contains a source-reported claim of **up to 1550 TFLOPS on H800**. It does not attach that number to `M=N=K=4096`, report approximately 90% utilization, or retain a complete reproduction environment. No more specific measurement is attributed to this source entry. -float cuda_core_acc[TILE_M][TILE_N] = {0}; +## Primary References -for (int k = 0; k < K; k += Nc) { - __half2 tc_acc[TILE_M][WGMMA_N]; - memset(tc_acc, 0, sizeof(tc_acc)); - for (int sub_k = 0; sub_k < Nc; sub_k += WGMMA_K) { - wgmma_mma_async(tc_acc, A_smem + sub_k, B_smem + sub_k); - } - wgmma_wait(); - for (int m = 0; m < TILE_M; m++) - for (int n = 0; n < TILE_N; n++) - cuda_core_acc[m][n] += (float)tc_acc[m][n] * scale_a[m] * scale_b[n]; -} -``` - -### SM100 path — tcgen05.mma with UE8M0 block scaling - -```cpp -// On Blackwell, tcgen05.mma consumes UE8M0 scale factors directly. -// 4 UE8M0 values pack into a single uint32; TMEM accumulates in full FP32 -// precision so no CUDA-core promotion is needed. -uint32_t packed_scales = pack_ue8m0(sf[0], sf[1], sf[2], sf[3]); -asm volatile( - "tcgen05.mma.cta_group::1.kind::f8f6f4.block_scale " - "[%0], %1, %2, [%3], %4, 1;\n" - :: "r"(tmem_acc), "l"(desc_a), "l"(desc_b), - "r"(sf_tmem_addr), "r"(0)); -``` - -### MoE grouped-GEMM launch - -```cpp -// Grouped-GEMM packs a variable list of per-expert GEMMs into one kernel -// launch via a prefix-sum offset array; layouts are contiguous (M-axis), -// masked (variable-K), or K-grouped depending on router output. -struct GroupedGemmArgs { - int num_groups; - int* m_prefix; // [num_groups+1] - const __nv_fp8_e4m3* A; - const __nv_fp8_e4m3* B; - const float* scales_a; - const float* scales_b; - __half* C; - int N, K; -}; - -__global__ void grouped_gemm_launch(GroupedGemmArgs args) { - int group = blockIdx.y; - int m_start = args.m_prefix[group]; - int m_end = args.m_prefix[group + 1]; - // Dispatch a standard tile-level GEMM for [m_start, m_end) × N × K. -} -``` +- [README at the pinned commit](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md) +- [GEMM API at the pinned commit](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp) +- [JIT compiler at the pinned commit](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/jit/compiler.hpp) +- [DeepSeek-V3 Technical Report v2](https://arxiv.org/abs/2412.19437v2) diff --git a/sources/blogs/flash-attention-4.md b/sources/blogs/flash-attention-4.md index 53ad4c5d0..8817627fe 100644 --- a/sources/blogs/flash-attention-4.md +++ b/sources/blogs/flash-attention-4.md @@ -16,76 +16,53 @@ tags: - ping-pong-scheduling - conditional-rescaling - cute-dsl -retrieved_at: 2026-04-27 +retrieved_at: 2026-08-08 artifact_dir: artifacts/blogs/flash-attention-4/code --- -## Summary +## Evidence Scope -Tri Dao's blog post on FlashAttention-4 design for Blackwell's asymmetric hardware scaling. +Tri Dao's first-party explanation of FlashAttention-4's Blackwell design and source-reported performance. The two code blocks below are KernelWiki illustrations derived from formulas and dimensions in the post; they are explicitly not verbatim FA4 source from the post. ## Key Techniques -- Asymmetric problem: tensor core throughput doubles but SFU count and SMEM bandwidth unchanged -- Ping-pong scheduling: two 128-token query tiles per CTA -- Software 2^x: Cody-Waite range reduction + Horner polynomial (Sollya-optimized coefficients) -- Multiplies exponential throughput without additional SFU hardware -- Conditional softmax rescaling: only when max jump is large -- 2-CTA backward: paired CTAs share TMEM, halves SMEM traffic -- CuTe-DSL implementation: 20-30x faster compilation than C++ templates -## Performance -- 1605 TFLOPS on B200 BF16 (71% utilization) -- 1.1-1.3x over cuDNN 9.13, 2.1-2.7x over Triton +- A CTA alternates two 128-row output tiles so tensor-core MMA and non-matmul softmax/correction work can overlap. +- FA4 evaluates only a selected fraction of exponentials with FMA polynomial code while retaining hardware `ex2` for the rest. +- The software-selected path uses `n=floor(x)`, a fraction in `[0,1)`, and the post's rounded degree-3 coefficients. +- Conditional rescaling compares running maxima, normally with `tau=8.0` base-2 units, retains the old reference maximum on skipped updates, and renormalizes at the end. +- Two-CTA backward shares operand B for five GEMMs, exchanges the required dS half through distributed shared memory for dQ, and reduces dQ global atomic reductions. It does not halve every dQ/dK/dV shared-memory transfer. -## Key Code +## Performance -### Software exp (Cody-Waite + Horner) +The post reports up to 1605 TFLOPS/s on B200 BF16, labeled 71%, plus up to 1.3x over cuDNN 9.13 and up to 2.7x over Triton. These are maxima over the author's evaluated configurations, not one fully specified `seqlen=8192, headdim=128` row. -```cuda -// Software-emulated exp2(x) using Cody-Waite range reduction and a -// Horner-scheme polynomial, Sollya-optimized coefficients. Lets FA-4 -// overlap the exp path with tcgen05.mma because it stays off the SFU. -__device__ __forceinline__ float sw_exp2(float x) { - // Range reduction: x = n + r, with n = round(x), r in [-0.5, 0.5] - int n = __float2int_rn(x); - float r = x - (float)n; - // Horner-scheme polynomial for 2^r, r in [-0.5, 0.5] - float p = 0x1.62e430p-1f; // ~ ln(2) - p = fmaf(p, r, 0x1.ebfc1ep-3f); - p = fmaf(p, r, 0x1.c6af98p-5f); - p = fmaf(p, r, 0x1.3b2c9cp-7f); - p = fmaf(p, r, 0x1.62e43ap-10f); - float y = fmaf(r, p, 1.0f); - // Scale by 2^n via direct FP32 bit manipulation - int bits = __float_as_int(y) + (n << 23); - return __int_as_float(bits); -} -``` +## Illustrative Code -### Ping-pong scheduling +### Software exp (published range reduction and rounded polynomial) - ```cuda -// Ping-pong two 128-token query tiles per CTA. While one tile is in the -// softmax/rescale stage, the other issues tcgen05.mma — the 2x tensor-core -// throughput on B200 gets fed while the SFU-bound softmax stays out of the -// critical path. -for (int tile = 0; tile < Q_tiles; tile += 2) { - issue_mma(query_a, key_block); - wait_mma(); - softmax_and_rescale(query_a); // SFU + MUFU path - issue_mma(query_b, key_block); - wait_mma(); - softmax_and_rescale(query_b); +// KernelWiki scalar illustration derived from the FA4 blog equations. +// This is not verbatim upstream FA4 code and omits selection and clamping. +#include + +__host__ __device__ inline float fa4_blog_exp2_reference(float x) { + const int n = static_cast(floorf(x)); + const float f = x - static_cast(n); // f in [0, 1) + const float p = 1.0f + f * (0.6951f + f * (0.2276f + f * 0.0771f)); + return ldexpf(p, n); } ``` ### 2-CTA cooperative backward ```cuda -// 2-CTA cooperative backward: paired CTAs in a cluster share a single TMEM -// accumulator half, halving SMEM traffic for dK/dV accumulation. -asm volatile( - "tcgen05.mma.cta_group::2.kind::f16 [%0], %1, %2, %3, 1;" - : : "r"(tmem_acc_shared), "l"(desc_a), "l"(desc_b), "r"(0)); +// KernelWiki schematic derived from the FA4 paper/blog dimensions. +// This is not upstream inline PTX or a complete kernel. +struct Fa4TwoCtaBackwardShape { + static constexpr int cta_group = 2; + static constexpr int mma_m = 256; + static constexpr int mma_n = 128; + static constexpr int mma_k = 128; + static constexpr int backward_gemm_count = 5; +}; ``` diff --git a/sources/blogs/flashmla.md b/sources/blogs/flashmla.md index 3b84964a4..f06e86300 100644 --- a/sources/blogs/flashmla.md +++ b/sources/blogs/flashmla.md @@ -2,7 +2,7 @@ id: blog-flashmla title: FlashMLA — Multi-head Latent Attention author: DeepSeek AI -url: https://github.com/deepseek-ai/FlashMLA +url: https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232 source_category: benchmark-blog architectures: - sm100 @@ -20,75 +20,65 @@ retrieved_at: 2026-04-27 artifact_dir: artifacts/blogs/flashmla/code --- -## Summary - -DeepSeek's efficient MLA kernels for V3/V3.2 with massive KV cache compression (70KB/token). - -## Variants -- Dense MLA decoding (SM90): BF16, paged KV (block 64), 3000 GB/s, 660 TFLOPS on H800 -- Sparse MLA decoding (SM90/SM100): FP8 KV, token-level sparsity, 410 TFLOPS H800, 350 TFLOPS B200 -- Dense prefill (SM100): 1460 TFLOPS fwd, 1000 TFLOPS bwd on B200 -- Sparse prefill (SM90/SM100): 640 TFLOPS H800, 1450 TFLOPS B200 - -## Token Format -656 bytes/token: 512B FP8 data + 16B FP32 scales + 128B BF16 RoPE embeddings - -## Key Code - -### MLA decode inner loop - -```cuda -// MLA collapses K and V into a shared latent matrix of head-dim Dc=128. -// On decode (one query vector against N KV tokens) we alternate TMA load, -// wgmma/tcgen05 into the q@K^T accumulator, online softmax, and the second -// accumulator against V. -constexpr int Dc = 128; // latent head dim -constexpr int BLOCK_N = 64; // paged KV block size -float acc[Dc] = {0}; -float max_val = -INFINITY; -float l = 0.f; -for (int n0 = 0; n0 < seqlen; n0 += BLOCK_N) { - tma_load(smem_kv, KV_pages + n0); - cp_async_wait(); - float scores[BLOCK_N]; - wgmma_or_tcgen05_mma(scores, q, smem_kv); // q @ K^T - float new_max = warp_reduce_max(scores, BLOCK_N); - float scale = expf(max_val - new_max); - for (int j = 0; j < Dc; j++) acc[j] *= scale; - l *= scale; - for (int j = 0; j < BLOCK_N; j++) { - float p = expf(scores[j] - new_max); - l += p; - for (int d = 0; d < Dc; d++) acc[d] += p * smem_kv[j * Dc + d]; - } - max_val = new_max; -} -for (int d = 0; d < Dc; d++) O[d] = acc[d] / l; +## Captured Revision + +This source record summarizes DeepSeek FlashMLA commit `71c737929f2567bd0a094ae140f8f60f390b1232` (2026-03-31). It is a repository/benchmark capture; code below is explicitly KernelWiki-derived and is not copied from the upstream implementation. + +## Supported Operators + +| Operator | Architecture | Mode and format | +|---|---|---| +| Dense decode | SM90 | MQA dimensions `576/512`, BF16 KV cache | +| Sparse decode | SM90/SM100 | MQA dimensions, FP8 KV cache dequantized for BF16 MMA | +| Dense prefill | SM100 | MHA dimensions `192/128` or `128/128`, BF16 inputs | +| Sparse prefill | SM90/SM100 | MQA mode, BF16 Q/KV inputs | + +The repository requires CUDA 12.8+, CUDA 12.9+ for SM100, and PyTorch 2.0+. + +## Exact V3 FP8 Sparse-Decode Layout + +When `is_fp8_kvcache=True` for the DeepSeek-V3-family sparse-decode path, each token uses 512 E4M3 NoPE bytes, four FP32 group scales (16 bytes), and 64 BF16 RoPE values (128 bytes), totaling 656 bytes. Dense decode and other sparse-cache layouts are different. + +### V3 FP8 sparse-decode byte check + +```python +# KernelWiki-derived contract check; not upstream FlashMLA code. +NOPE_FP8_VALUES = 512 +GROUPS = 4 +FP32_BYTES = 4 +ROPE_BF16_VALUES = 64 +BF16_BYTES = 2 + +V3_FP8_SPARSE_BYTES = ( + NOPE_FP8_VALUES + GROUPS * FP32_BYTES + ROPE_BF16_VALUES * BF16_BYTES +) +assert V3_FP8_SPARSE_BYTES == 656 ``` -### Sparse-MLA KV-retrieval kernel (V3.2) - -```cuda -// Sparse MLA selects top-k KV positions per query before running the dense -// MLA kernel on just those positions. Retrieval uses FP8 dot products with -// per-token scale factors. -__global__ void sparse_mla_topk( - const __nv_fp8_e4m3* Q, const __nv_fp8_e4m3* K, - const float* Q_scale, const float* K_scale, - int* topk_idx, float* topk_score, - int N, int K_DIM, int TOPK) -{ - int q_tile = blockIdx.x; - float scores[N]; - for (int n = 0; n < N; n++) { - float s = 0.f; - for (int k = 0; k < K_DIM; k++) { - s += decode_fp8(Q[q_tile * K_DIM + k]) * Q_scale[q_tile] - * decode_fp8(K[n * K_DIM + k]) * K_scale[n]; - } - scores[n] = s; - } - warp_topk_select(scores, N, topk_idx + q_tile * TOPK, - topk_score + q_tile * TOPK, TOPK); -} +## Sparse Index Contracts + +Sparse decode consumes `indices[batch,s_q,topk]` whose entries encode a physical page and offset. Invalid entries are `-1`, and no block table is needed after physical-page encoding. Sparse prefill instead consumes unbatched `indices[s_q,h_kv,topk]` with BF16 Q/KV and accepts negative or out-of-range invalid entries. + +### Decode page-index round trip + +```python +# KernelWiki-derived contract check; not upstream FlashMLA code. +def encode_page_index(physical_page: int, offset: int, page_size: int) -> int: + assert physical_page >= 0 and 0 <= offset < page_size + return physical_page * page_size + offset + +def decode_page_index(encoded: int, page_size: int) -> tuple[int, int]: + assert encoded >= 0 and page_size > 0 + return divmod(encoded, page_size) + +assert decode_page_index(encode_page_index(7, 13, 64), 64) == (7, 13) ``` + +## Source-Reported Performance + +- Dense MLA decode, H800 SXM5/CUDA 12.8: up to 3000 GB/s memory-bound and 660 TFLOPS compute-bound. +- Sparse MLA decode: 410 TFLOPS on H800 SXM5/CUDA 12.8 and up to 350 TFLOPS on B200; FP8 describes KV storage and MMA is BF16. +- Dense **MHA** prefill, B200: up to 1460 TFLOPS forward and 1000 TFLOPS backward, reported by NVIDIA. +- Sparse MLA prefill: up to 640 TFLOPS forward on H800 SXM5/CUDA 12.8 and up to 1450 TFLOPS on B200/CUDA 12.9; documented Q/KV inputs are BF16. + +The repository does not supply complete shape/timing/sample/variance records for these maxima. They remain qualified author reports rather than structured benchmark entries. diff --git a/sources/blogs/gated-delta-net.md b/sources/blogs/gated-delta-net.md index f50a83775..1a80539dd 100644 --- a/sources/blogs/gated-delta-net.md +++ b/sources/blogs/gated-delta-net.md @@ -2,101 +2,31 @@ id: blog-gated-delta-net title: Gated Delta Networks author: NVlabs -url: https://github.com/NVlabs/GatedDeltaNet +url: https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921 source_category: benchmark-blog -architectures: -- sm90 -- sm100 +architectures: [] tags: - gated-delta-net - linear-attention - attention - triton - chunk-parallelism -retrieved_at: 2026-04-16 -artifact_dir: artifacts/blogs/gated-delta-net/code +retrieved_at: 2026-08-08 --- -## Summary +## Scope -Linear attention mechanism with delta rule for intelligent memory management (ICLR 2025). Used in Qwen3-Next (3:1 hybrid ratio). +This source capture summarizes the official NVlabs repository at commit `b53d6d3a161267432a79c1c04af69fa52bddc921` and the associated ICLR 2025 paper. It does not preserve a hardware performance record. -## Architecture -- Delta rule: targeted state updates (keep/forget) -- Exponential gating: adaptive memory decay -- Causal Conv1D for local context -- Fixed-size 128×128 state matrix (independent of sequence length) -- O(n) complexity vs O(n²) for standard attention +## Mechanism -## Implementations -- NVlabs reference: Triton kernels -- FLA optimized: recommended, significantly faster, varlen support -- Chunk-based parallelism for prefill, streaming for decode +- Gated DeltaNet combines an independent exponential decay gate with the delta rule's targeted memory correction. +- In the repository implementation, the recurrent state is shaped `[batch, heads, head_qk_dim, head_v_dim]`; its dimensions are model parameters rather than a universal `128x128` constant. +- The layer also owns learned Q/K/V, decay, update, convolution, output-gate, normalization, and output-projection parameters. +- The training path uses chunkwise Triton kernels and a WY representation rather than the additive update shown in the former local snippets. -## Adoption -- Qwen3-Next-80B (3:1 GDN:attention ratio, 512 experts) -- Qwen3.5 (262K context, 10x+ throughput over Qwen3-32B at 32K+) +## Implementations and adoption -## Key Code +The authors' FAQ says that FLA kernels are faster, support variable-length training, and are strongly recommended for better performance. The repository's dated updates record integration into Qwen3-Next and Qwen3.5. -### Chunk-parallel prefill reference (PyTorch) - -```python -import torch - -def gated_delta_net_prefill(q, k, v, gate, initial_state, CHUNK_SIZE=64): - """ - Chunk-parallel prefill. Each chunk's state matrix is reused across its - query window, so we pay the O(Dk*Dv) state update once per chunk, not - per token. - q, k: [B, L, Dk] v: [B, L, Dv] gate: [B, L] - """ - B, L, Dk = q.shape - Dv = v.shape[-1] - out = torch.empty(B, L, Dv, device=q.device, dtype=q.dtype) - state = initial_state.clone() # [B, Dk, Dv] - for ci in range(0, L, CHUNK_SIZE): - ce = min(ci + CHUNK_SIZE, L) - k_chunk = k[:, ci:ce] - v_chunk = v[:, ci:ce] - g_chunk = gate[:, ci:ce] - decay = torch.cumprod(g_chunk, dim=1) # adaptive memory decay - for t in range(ce - ci): - state = state * decay[:, t:t+1, None] - state = state + k_chunk[:, t, :, None] * v_chunk[:, t, None, :] - out[:, ci + t] = (q[:, ci + t, :, None] * state).sum(dim=1) - return out, state -``` - -### Triton decode-step kernel (streaming) - -```python -import triton -import triton.language as tl - -@triton.jit -def gdn_decode_step_kernel( - Q, K, V, GATE, STATE, OUT, - stride_qb, stride_kb, stride_vb, - Dk: tl.constexpr, Dv: tl.constexpr): - """ - One-token delta-rule update for decode. STATE is a [Dk, Dv] matrix kept - per sample; we fold in the new (k,v) pair after applying the decay gate. - """ - b = tl.program_id(0) - dk = tl.arange(0, Dk) - dv = tl.arange(0, Dv) - - q = tl.load(Q + b * stride_qb + dk) # [Dk] - k = tl.load(K + b * stride_kb + dk) # [Dk] - v = tl.load(V + b * stride_vb + dv) # [Dv] - g = tl.load(GATE + b) # scalar decay - - state = tl.load(STATE + b * Dk * Dv + dk[:, None] * Dv + dv[None, :]) - state = state * g # apply decay - state = state + k[:, None] * v[None, :] # delta update - tl.store(STATE + b * Dk * Dv + dk[:, None] * Dv + dv[None, :], state) - - out = tl.sum(q[:, None] * state, axis=0) # [Dv] - tl.store(OUT + b * Dv + dv, out) -``` +No local code block is extracted from this summary. Consult the pinned upstream implementation for executable kernels. diff --git a/sources/blogs/modular-blackwell-matmul.md b/sources/blogs/modular-blackwell-matmul.md index 8b600b8df..12bf4cf88 100644 --- a/sources/blogs/modular-blackwell-matmul.md +++ b/sources/blogs/modular-blackwell-matmul.md @@ -2,105 +2,29 @@ id: blog-modular-blackwell title: "Modular: Matrix Multiplication on Blackwell" author: Modular -url: https://www.modular.com/blog/matrix-multiplication-on-nvidias-blackwell-part-1-introduction +url: https://www.modular.com/blog/matrix-multiplication-on-nvidias-blackwell-part-3-the-optimizations-behind-85-of-sota-performance source_category: community-note architectures: [sm100, sm100a] -tags: [gemm, tcgen05, tmem, tma, 2sm-cooperative, pipeline-stages, tma-multicast, clc] +tags: [gemm, tcgen05, tmem, tma, 2sm-cooperative, pipeline-stages, tma-multicast, warp-specialization, double-buffering] techniques: [pipeline-stages, tma-multicast, warp-specialization, double-buffering] -hardware_features: [tcgen05, tmem, tma, 2sm-cooperative, clc] +hardware_features: [tcgen05, tmem, tma, 2sm-cooperative] kernel_types: [gemm] -languages: [cuda-cpp] -retrieved_at: 2026-04-16 +languages: [mojo] +retrieved_at: 2026-08-09 --- -# Modular: Matrix Multiplication on Blackwell +# Modular Blackwell matmul series -## Overview +## Evidence scope -Multi-part blog series from Modular on building a high-performance GEMM kernel for Blackwell GPUs. The series provides a clear optimization progression, reaching 85% of SOTA performance with documented techniques at each step. +Part 3 describes a sequence of Blackwell matmul changes rather than isolated universal prescriptions. The implementation uses 2-SM cooperative MMA and TMA multicast, then adds a five-stage circular buffer for A/B shared-memory tiles. Separate warp roles issue TMA and MMA so different stages can progress concurrently. -**Part 1**: [Introduction](https://www.modular.com/blog/matrix-multiplication-on-nvidias-blackwell-part-1-introduction) -**Part 3**: [The Optimizations Behind 85% of SOTA Performance](https://www.modular.com/blog/matrix-multiplication-on-nvidias-blackwell-part-3-the-optimizations-behind-85-of-sota-performance) +The article later double-buffers output writeback in shared memory: TMEM partitions are loaded/converted/stored to alternating output buffers while TMA stores progress. It says the reduced output-buffer footprint frees shared memory for deeper input pipelining, reports an additional 64 TFLOP/s for that final change, and labels the resulting endpoint 85% of the article's SOTA reference. -## Key Techniques +Those are author-reported results for the series' code and benchmark context. The article does not establish that five stages are universally optimal, a fixed B200 HBM latency, a portable percentage for each earlier step, or an architecture-wide optimization order. -### TMA Multicasting +## References -TMA can deliver the same data to multiple SMs in a cluster simultaneously: - -``` -// Without multicast: each SM loads its own copy of shared tiles -// With multicast: TMA loads once, broadcasts to N SMs - -// Example: B matrix tile shared across M-dimension SMs -// If cluster has 4 SMs along M dimension: -// - Without multicast: 4 separate global memory reads -// - With multicast: 1 read, hardware broadcasts to 4 SMs -// 4x reduction in global memory bandwidth for B tiles -``` - -TMA multicasting is particularly effective for GEMM where the B matrix tile is shared across the M dimension of the output. - -### 2-SM Cooperative MMA - -Two SMs cooperate on a single m256 x n256 x k16 tile: - -``` -// Single SM: m128 x n256 x k16 = 128 * 256 * 16 * 2 = 1M FLOPs per MMA -// 2-SM: m256 x n256 x k16 = 256 * 256 * 16 * 2 = 2M FLOPs per MMA - -// Benefits: -// 1. Doubled M dimension -> better data reuse for A tile -// 2. Larger output tile -> fewer tiles needed for full GEMM -// 3. Both SMs share B tile (loaded once via TMA multicast) -``` - -### 5-Stage Circular Buffer Pipeline - -Multi-stage software pipeline for overlapping TMA loads with compute: - -``` -// 5 SMEM buffer slots: -// Slot 0: being consumed by tcgen05.mma (current compute) -// Slot 1: data ready, waiting for compute -// Slot 2: TMA load in progress -// Slot 3: TMA load issued, waiting for completion -// Slot 4: free, ready for next TMA load - -// Deeper pipeline (5 vs 2-3) hides more memory latency -// Critical for large K dimensions where many tiles must stream through -``` - -The 5-stage depth is deeper than the typical 2-3 stages used in simpler implementations, providing better latency hiding at the cost of more shared memory for buffers. - -### Optimization Progression - -The Modular blog documents a clear progression: - -| Step | Technique | Approximate % SOTA | -|------|-----------|-------------------| -| 1 | Basic tcgen05.mma | ~20% | -| 2 | Swizzled SMEM layout | ~45% | -| 3 | TMA async pipeline (5-stage) | ~60% | -| 4 | Warp specialization | ~70% | -| 5 | 2-SM cooperative + TMA multicast | ~85% | - -The remaining 15% gap to SOTA (cuBLAS / CUTLASS persistent) comes from: -- Persistent kernel with CLC scheduling -- Fine-tuned tile sizes for specific problem shapes -- Advanced register allocation and instruction scheduling - -## Key Insights - -1. **TMA multicast reduces bandwidth pressure**: For GEMM, the B matrix tile can be multicast to all SMs processing different M rows. This is a free bandwidth reduction that requires only cluster configuration. - -2. **5-stage pipeline is optimal for B200**: The B200's 8 TB/s bandwidth and deep memory hierarchy benefit from deeper pipelines. Shallower pipelines (2-3 stages) leave performance on the table. - -3. **85% without persistence**: Notably, 85% of SOTA is achievable without persistent kernels or CLC. This makes non-persistent kernels viable for many use cases where simplicity is preferred. - -4. **Clear optimization ordering**: The series demonstrates that optimizations should be applied in order of impact: swizzling > pipelining > warp specialization > 2-SM cooperative. Attempting later optimizations without earlier ones yields minimal benefit. - -## Sources - -- [Part 1: Introduction](https://www.modular.com/blog/matrix-multiplication-on-nvidias-blackwell-part-1-introduction) -- [Part 3: 85% of SOTA](https://www.modular.com/blog/matrix-multiplication-on-nvidias-blackwell-part-3-the-optimizations-behind-85-of-sota-performance) +- [Part 1: introduction](https://www.modular.com/blog/matrix-multiplication-on-nvidias-blackwell-part-1-introduction) +- [Part 3: optimizations behind the 85% endpoint](https://www.modular.com/blog/matrix-multiplication-on-nvidias-blackwell-part-3-the-optimizations-behind-85-of-sota-performance) +- [Part 3 linked kernel 6](https://github.com/modular/modular/blob/main/max/kernels/test/gpu/linalg/matmul_blackwell_iterative/6_2sm_pipelined.mojo) — rolling branch, not an immutable revision diff --git a/sources/blogs/nsa.md b/sources/blogs/nsa.md deleted file mode 100644 index d29c3e236..000000000 --- a/sources/blogs/nsa.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -id: blog-nsa -title: "Native Sparse Attention (NSA)" -author: DeepSeek AI -url: https://arxiv.org/abs/2502.11089 -source_category: benchmark-blog -architectures: [sm90, sm100] -tags: [sparse-attention, attention, triton, chunk-parallelism] -retrieved_at: 2026-04-16 ---- - -## Summary - -DeepSeek's natively trainable sparse attention with three parallel paths. - -## Architecture -1. Token compression via learnable MLP (coarse-grained) -2. Token selection using blockwise importance scores (top-n fine-grained blocks) -3. Sliding window (w=512) for local context - -## Key Techniques -- Hardware-aligned blockwise memory access -- Group-centric loading: shares sparse KV blocks across GQA group heads -- Triton kernel: grid-based loop scheduling -- 9x forward speedup, 6x backward at 64K sequences vs FlashAttention-2 -- Deployed in DeepSeek-V3.2-Exp diff --git a/sources/blogs/nvidia-code-samples.md b/sources/blogs/nvidia-code-samples.md index 2a5106eae..b4585ae96 100644 --- a/sources/blogs/nvidia-code-samples.md +++ b/sources/blogs/nvidia-code-samples.md @@ -4,22 +4,20 @@ title: NVIDIA Developer Code Samples author: NVIDIA Developer Blog url: https://github.com/NVIDIA-developer-blog/code-samples source_category: community-note -architectures: -- sm90 -- sm100 +architectures: [] tags: - cuda-cpp - gemm -- swizzling - shared-memory-optimization -- vectorized-loads -- register-budgeting retrieved_at: '2026-05-20' description: Source-map entry imported from KernelPilot for CUDA sample kernels and memory-system examples. --- -NVIDIA's developer-blog sample-code repository is useful as a source-map route -for small CUDA examples: coalesced global loads, shared-memory staging, -reductions, transposes, and occupancy-sensitive launch choices. In KernelWiki it -serves as an upstream source pointer rather than a synthesized implementation -claim. +At commit `3350d216083a902ccbf5b31665e3b82096a75b55`, this cross-generation +repository contains small CUDA examples including +`series/cuda-cpp/coalescing-global/coalescing.cu`, +`series/cuda-cpp/shared-memory/shared-memory.cu`, +`series/cuda-cpp/transpose/transpose.cu`, and +`posts/tensor-cores/simpleTensorCoreGEMM.cu`. The source card makes no SM90 or +SM100 compatibility claim; inspect the selected example's build requirements +before reuse. diff --git a/sources/blogs/simon-nvfp4-gemv.md b/sources/blogs/simon-nvfp4-gemv.md index 3fafe9928..4352c36ca 100644 --- a/sources/blogs/simon-nvfp4-gemv.md +++ b/sources/blogs/simon-nvfp4-gemv.md @@ -12,132 +12,58 @@ tags: - fp4 - block-scale - cute-dsl -- vectorized-loads -- register-reuse - batched-gemv -retrieved_at: 2026-04-17 -artifact_dir: artifacts/blogs/simon-nvfp4-gemv/code +techniques: +- k-dimension-parallelism +- reduction +hardware_features: +- nvfp4 +- fp4 +- block-scale +kernel_types: +- batched-gemv +- gemv +languages: +- cute-dsl +- python +retrieved_at: 2026-08-08 --- -# NVFP4 GEMV and Improved NVFP4 GEMV (Simon Veitner) - -## Overview - -Simon Veitner's two-part blog series on implementing NVFP4 GEMV kernels using CuTe DSL for Blackwell GPUs. Part 1 presents a reference CuTe implementation with hierarchical tensor layouts for FP4 block-scaled GEMV. Part 2 ("Improved") introduces three optimization strategies that parallelize the K-dimension reduction, achieving up to 6.4x speedup over the reference. - -## Part 1: Reference CuTe DSL Implementation - -### Configuration - -- Tile dimensions: (128, 1, 64) for M, N, K -- Data types: FP4 (E2M1) for weight matrix, FP8 (E4M3) for scale factors, FP16 output -- Scale factor block size: 16 elements per FP8 scale value -- Thread block size: 128 threads per CTA - -### NVFP4 Format Handling - -NVFP4 is composed of two tensors: one in FP4 precision and another in FP8 precision. The scale factor applies to every block of 16 values to minimize quantization error while reducing memory footprint. - -### Memory Access Patterns - -The kernel uses hierarchical CuTe tensor indexing: -- Matrix A: shape (128, 64, 1, 4, 1) representing one M-tile with four K-tiles -- Scale factors: hierarchical layout ((32,4), (16,4)) enabling broadcast optimization -- Vector B: single N-tile with K-dimension iteration - -### Core Computation - -The computation loop applies two-level scaling: -``` -res += tArA[i] * tArSFA[i] * tBrB[i] * tBrSFB[i] -``` -FP4 values are multiplied by their corresponding FP8 scale factors before accumulation. Values convert from FP4/FP8 to FP32 registers for computation, then convert to FP16 for storage. - -## Part 2: Improved NVFP4 GEMV +# NVFP4 GEMV and Improved NVFP4 GEMV -### Three Optimization Strategies +## Evidence Scope -**Strategy 1: Extra Blocks (K-Parallel Grid)** -- Launches grid blocks corresponding to K-tiles, eliminating the K-tile loop -- Uses atomic operations on global memory (F32 accumulation buffer) -- Performance: ~36,864 ops (benchmark 0), ~55,399 ops (benchmark 1) -- Achieves 6.4x improvement on benchmark 0 versus reference +Simon Veitner's two posts present CuTe DSL Python kernels. They do not publish the CUDA C++ listings formerly attributed to them by this card, and the benchmark outputs are times in nanoseconds rather than operation counts. -**Strategy 2: Thread-Level with Atomic Add** -- Distributes work across thread dimensions (threads_per_m=32, threads_per_k=32) -- Uses shared memory atomics for collaborative result calculation -- Avoids separate F32 tensor allocation overhead -- Performance: ~38,911 ops (benchmark 0), ~67,258 ops (benchmark 1) +## Reference Post -**Strategy 3: Thread-Level with Reduction (No Atomics)** -- Allocates 2D shared memory tensor (K-major stride) -- Each thread pair stores intermediate results, then performs synchronous reduction -- Performance: ~38,911 ops (benchmark 0), ~65,599 ops (benchmark 1) +The first post develops a CuTe DSL reference for the official NVFP4 batched GEMV task. It describes a `(128, 1, 64)` M-N-K tile, 128 threads per block, packed E2M1 operands, E4M3 block scales, and FP32 register accumulation before FP16 output. -### Key Technical Changes +The post's benchmark output includes: -- Serial K-dimension loop replaced with parallel K-reduction -- Intermediate products converted to FP16 for storage efficiency -- FP32 maintained for scale factor operations -- Synchronization barriers inserted between computation and reduction phases -- K-major memory layouts optimize the reduction step access patterns +| Row | Mean time (ns) | +|---:|---:| +| 0 | 234495.997 | +| 1 | 119713.035 | +| 2 | 38911.998 | -### Performance Summary +These are author-reported benchmark results from the displayed environment and are not independent reproduction. -Reference baseline: ~234,495 ops. Best improvement (extra blocks) delivers 6.4x speedup on larger K dimensions, though smaller K problems show more modest gains. +## Improved Post -## Key Insights +The second post parallelizes the K reduction in three ways: -- CuTe DSL's hierarchical tensor layouts naturally express NVFP4's two-level scaling structure -- K-dimension parallelism is critical for GEMV performance on Blackwell -- Atomic-free reductions in shared memory match or approach atomic-based approaches -- The choice between strategies depends on K-dimension size and available SM resources +1. Extra K-grid blocks compute partial sums and use an FP32 global atomic before conversion to FP16. +2. Additional threads collaborate on each row and use an atomic reduction path. +3. Additional threads collaborate through shared memory and an atomic-free reduction. -## Key Code +For the extra-block strategy, the displayed means are 36864.001, 55399.918, and 24576.001 ns for rows 0–2. The post reports additional timing blocks for the two thread-collaboration variants and discusses their shape-dependent tradeoffs. The approximately 6.4× statement compares the first reference row, 234495.997 ns, with 36864.001 ns; it is not a leaderboard score or operation-rate unit. -### Reference core computation (Part 1) +## Provenance Limits -```cpp -// NVFP4 GEMV: FP4 values are decoded to FP32 via their per-block FP8 scale, -// then multiplied against a decoded B element + its FP8 scale. Accumulation -// stays in FP32. Simon's reference kernel does this in a CuTe register tile: -for (int i = 0; i < TILES_K; i++) { - float a = decode_nvfp4(tArA[i]) * decode_fp8(tArSFA[i]); - float b = decode_nvfp4(tBrB[i]) * decode_fp8(tBrSFB[i]); - res += a * b; // FP32 accumulation -} -``` +The posts provide CuTe DSL snippets and benchmark text. Locally reconstructed `.cpp` files extracted from an earlier summary are not verbatim author source and must not be labeled contestant submissions. The public leaderboard snapshot fetched on 2026-08-08 places the `Simon` row at current rank 25 with a 25.112153955 microsecond aggregate score; it does not expose that submission's code. -### Strategy 1 — K-parallel grid with atomic accumulation +## Primary Sources -```cpp -// Launch one CTA per (M-tile, K-tile); accumulate partial products into a -// global FP32 buffer via atomicAdd, then cast to FP16 in a second pass. -__global__ void nvfp4_gemv_k_parallel( - const __nv_fp4_e2m1* A, const __nv_fp8_e4m3* SFA, - const __nv_fp4_e2m1* B, const __nv_fp8_e4m3* SFB, - float* accum_f32, int K_TILES) -{ - int m_tile = blockIdx.x; - int k_tile = blockIdx.y; - float partial = nvfp4_dot_product(A, SFA, B, SFB, m_tile, k_tile); - atomicAdd(&accum_f32[m_tile], partial); -} -``` - -### Strategy 3 — Atomic-free shared-memory reduction - -```cpp -// Each thread pair stores an intermediate product; after __syncthreads() -// the CTA reduces along K-major in shared memory without atomics. -__shared__ float smem[THREADS_PER_M][THREADS_PER_K]; -smem[tid_m][tid_k] = thread_partial; -__syncthreads(); - -// Warp-wide parallel reduction along the K axis -for (int s = THREADS_PER_K / 2; s > 0; s >>= 1) { - if (tid_k < s) smem[tid_m][tid_k] += smem[tid_m][tid_k + s]; - __syncthreads(); -} -if (tid_k == 0) C[m_base + tid_m] = __float2half(smem[tid_m][0]); -``` +- [NVFP4 GEMV](https://veitner.bearblog.dev/nvfp4-gemv/) +- [Improved NVFP4 GEMV](https://veitner.bearblog.dev/nvfp4-gemv-improved/) diff --git a/sources/blogs/simveit-effective-transpose.md b/sources/blogs/simveit-effective-transpose.md index e9385840e..c406bd585 100644 --- a/sources/blogs/simveit-effective-transpose.md +++ b/sources/blogs/simveit-effective-transpose.md @@ -5,20 +5,17 @@ author: Simon Veitner url: https://github.com/simveit/effective_transpose source_category: community-note architectures: -- sm90 -- sm100 +- sm90a tags: -- cute-dsl -- gemm +- cuda-cpp - tma - swizzling -- vectorized-loads - shared-memory-optimization retrieved_at: '2026-05-20' description: Source-map entry imported from KernelPilot for CuTe transpose, swizzle, and memory-layout examples. --- -The effective_transpose repository provides compact CuTe-oriented transpose and -layout examples. Use it when a kernel profile indicates poor sector utilization, -shared-memory bank conflicts, or a need to reason about tiled load/store -layouts. +At commit `994b2b5acaa67f80e411df3e8274b6ae13fd1949`, this Hopper repository +compiles for SM90a and contains raw CUDA C++ TMA transpose variants. +`transpose_swizzle_batched.cu` encodes 128-byte tensor-map swizzling. It is not +a CuTe DSL, GEMM, SM100, or explicit vector-load-width source. diff --git a/sources/blogs/simveit-load-and-store.md b/sources/blogs/simveit-load-and-store.md index 7aec49436..7d29f534f 100644 --- a/sources/blogs/simveit-load-and-store.md +++ b/sources/blogs/simveit-load-and-store.md @@ -5,20 +5,17 @@ author: Simon Veitner url: https://github.com/simveit/load_and_store source_category: community-note architectures: -- sm90 -- sm100 +- sm90a tags: -- cute-dsl -- gemm -- tma -- wgmma -- vectorized-loads -- shared-memory-optimization +- cuda-cpp +- ptx +- ldmatrix +- stmatrix retrieved_at: '2026-05-20' description: Source-map entry imported from KernelPilot for CuTe load/store and shared-memory movement examples. --- -The load_and_store repository is a source-map route for CuTe load/store -mechanics. It is most useful when converting a profiler symptom into a concrete -data-movement edit: vector width, layout, shared staging, or TMA-friendly access -structure. +At commit `05d828cf910dd43f0053ddbbe4744218a06e9d7f`, this repository contains +six inline-PTX examples for `ldmatrix` and `stmatrix` x1/x2/x4 forms. Its +Makefile compiles for SM90a. It contains no CuTe, GEMM, TMA, WGMMA, SM100, or +generic vector-width implementation. diff --git a/sources/blogs/tcgen05-tutorial.md b/sources/blogs/tcgen05-tutorial.md index 9070b0bf8..1952114c4 100644 --- a/sources/blogs/tcgen05-tutorial.md +++ b/sources/blogs/tcgen05-tutorial.md @@ -20,95 +20,38 @@ retrieved_at: 2026-04-16 artifact_dir: artifacts/blogs/tcgen05-tutorial/code --- -## Summary +# tcgen05 for dummies -Step-by-step tutorial building a Blackwell GEMM kernel from scratch in plain CUDA C++ with PTX, achieving 98% of cuBLAS performance. +## Scope and provenance -## Performance Progression -- Basic kernel: 255 TFLOPS (17%) -- 128B swizzling: 695 TFLOPS (46%) -- Pipelining: 940 TFLOPS (62%) -- Warp specialization: ~1200 TFLOPS (80%) -- Persistent kernel: 1476 TFLOPS (98%) vs 1507 cuBLAS +Gau Nernst's tutorial develops a plain CUDA C++/PTX GEMM on a Modal B200. The article is dated 2025-12-21, and the relevant code is in `02e_matmul_sm100` at repository commit `3b90ac9b3f624bdf1f6f78d02dcd533675d36573`. The disclosed benchmark uses M=N=K=4096 and compares against PyTorch 2.9.1 with CUDA 13 cuBLAS. -## Key Findings -- tcgen05.mma operates directly on shared memory — no ldmatrix needed -- TMEM: 128×512 capacity, 32-bit elements, must alloc/dealloc -- mbarrier synchronization with phases and parity bits -- "Tensor Core programming on Blackwell is easier than previous generations" -- 128B swizzling alone gives 2.7× speedup +Use the linked pinned source for complete assembly operands and synchronization. Earlier local excerpts omitted required tcgen05 operands and were not valid standalone code. -## Key Code +## Source-reported progression -### Basic tcgen05.mma kernel (17% of peak) +| Version | Author's description | TFLOP/s | +|---|---|---:| +| cuBLAS | PyTorch 2.9.1 + CUDA 13 | 1506.74 | +| v1a | basic tcgen05 + 2D 16B TMA | 254.62 | +| v1b | 3D 16B TMA | 252.81 | +| v2a | 2D 128B TMA | 681.20 | +| v2b | 3D 128B TMA | 695.43 | +| v3 | pipelining | 939.61 | +| v4 | warp specialization | 1208.83 | +| v5 | 2-SM MMA | 1302.29 | +| v6 | persistent with static scheduling | 1475.93 | -```cuda -// The naive building block: one-thread-launched tcgen05.mma into TMEM. -// ~255 TFLOPS on B200 (17% of peak). -__shared__ uint32_t tmem; -if (threadIdx.x == 0) { - asm volatile("tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], 256;\n" - :: "r"(&tmem)); -} -__syncthreads(); +These values are reports for the author's exact environment; they are not portable performance guarantees. The final v6 kernel uses static scheduling. The article explicitly says threadblock swizzling and Cluster Launch Control were not added. -for (int k = 0; k < K; k += K_TILE) { - cp_async(smem_a, A + k); - cp_async(smem_b, B + k); - cp_async_commit(); - cp_async_wait<0>(); - __syncthreads(); - if (threadIdx.x == 0) { - asm volatile("tcgen05.mma.cta_group::1.kind::f16 [%0], %1, %2, %3, 1;\n" - :: "r"(tmem), "l"(desc_a), "l"(desc_b), "r"(0)); - } -} -``` +## Supported takeaways -### 128B swizzling (46% of peak) +- The tutorial demonstrates TMEM allocation/lifecycle, tcgen05 MMA, TMA transfers, mbarrier-driven pipelines, 128-byte swizzling as a performance optimization, warp specialization, 2-SM MMA, and persistence. +- Its 128-byte swizzle stage substantially improves this benchmark, but PTX also defines other valid shared-memory descriptor modes. +- The article's completion and pipeline code should be read in full; short fragments can omit the mbarrier, fence, descriptor, and operand-lifetime context required for correctness. -```cuda -// XOR-swizzled SMEM layout eliminates bank conflicts on MMA load; -// 128-byte granularity gives 2.7x speedup on its own. -template -__device__ void swizzle_128b_store(half* smem, const half* gmem, int k_tile) { - int tid = threadIdx.x; - int col = (tid * 8) % N_K; - int row = (tid * 8) / N_K; - int swizzled = col ^ ((row & 0x7) << 4); // 8-lane XOR swizzle - *reinterpret_cast(&smem[row * N_K + swizzled]) = - *reinterpret_cast(&gmem[k_tile + row * N_K + col]); -} -``` +## References -### Pipelining + mbarrier phases (62% of peak) - -```cuda -// Multi-stage TMA load pipeline. mbarrier phase bits toggle every STAGES -// arrivals so try_wait.parity can distinguish consecutive rounds without a -// counter rollover. -constexpr int STAGES = 4; -__shared__ uint64_t mbar_full[STAGES]; -__shared__ uint64_t mbar_empty[STAGES]; - -if (threadIdx.x == 0) { - for (int i = 0; i < STAGES; i++) { - asm volatile("mbarrier.init.shared::cta.b64 [%0], 1;\n" :: "r"(&mbar_full[i])); - asm volatile("mbarrier.init.shared::cta.b64 [%0], 1;\n" :: "r"(&mbar_empty[i])); - } -} -__syncthreads(); - -int phase = 0; -for (int k = 0; k < K_TILES; k++) { - int stage = k % STAGES; - if (k >= STAGES) { - asm volatile("mbarrier.try_wait.parity.shared::cta.b64 _, [%0], %1;\n" - :: "r"(&mbar_empty[stage]), "r"(phase)); - } - tma_load(smem_a[stage], gmem_a, k); - asm volatile("mbarrier.arrive.shared::cta.b64 _, [%0];\n" - :: "r"(&mbar_full[stage])); - if ((k + 1) % STAGES == 0) phase ^= 1; -} -``` +- [Article](https://gau-nernst.github.io/tcgen05/) +- [Pinned source tree](https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100) +- [PTX ISA 9.0 tcgen05 reference](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma) diff --git a/sources/blogs/yue-nvfp4-hackathon.md b/sources/blogs/yue-nvfp4-hackathon.md index 23b925f76..a03e0372a 100644 --- a/sources/blogs/yue-nvfp4-hackathon.md +++ b/sources/blogs/yue-nvfp4-hackathon.md @@ -16,7 +16,6 @@ tags: techniques: - vectorized-loads - cache-policy -- register-reuse - loop-unrolling hardware_features: - nvfp4 @@ -29,126 +28,43 @@ languages: - cuda-cpp - ptx - cute-dsl -retrieved_at: 2026-04-16 -artifact_dir: artifacts/blogs/yue-nvfp4-hackathon/code +retrieved_at: 2026-08-08 --- -# Blackwell NVFP4 Kernel Hackathon Journey (Yue Zhang) +# Blackwell NVFP4 Kernel Hackathon Journey -## Overview +## Evidence Scope -Yue Zhang's detailed account of optimizing Problem 1 (NVFP4 Batched GEMV) in the GPU Mode hackathon. Documents the full optimization journey from a naive CuTe DSL implementation (~100us) to a highly optimized CUDA/PTX kernel (22.392us) -- a 4.5x improvement through systematic optimization. +This is an evidence-scoped summary of Yue Zhang's own optimization report for Problem 1. Its performance values and explanations are author-reported; the post does not provide raw repeated-trial data or a complete released submission. At retrieval, the post still said its source-code link was coming soon. -## Performance Progression +## Reported Progression -| Stage | Approach | Latency | Improvement | -|-------|----------|---------|-------------| -| 1 | CuTe DSL baseline | ~100us | -- | -| 2 | Naive CUDA (coalesced access) | ~443us | (worse than CuTe initially) | -| 3 | Hardware intrinsics | ~39us | 11.4x from stage 2 | -| 4 | PTX assembly | ~27us | 1.44x from stage 3 | -| 5 | ILP optimization | ~22.9us | 1.18x from stage 4 | -| 6 | Final tuned | 22.392us | 4.5x from stage 1 | +| Stage | Combined change | Author-reported latency | +|---|---|---:| +| Initial CuTe DSL | First working CuTe path | ~100 µs | +| Optimized CuTe DSL | Scale-load/arithmetic changes and thread collaboration | ~33 µs | +| Initial CUDA | Naive hand-written path | ~2000 µs | +| CUDA optimization 1 | Coalescing, shared B, thread collaboration, warp reduction | ~443 µs | +| CUDA optimization 2 | Remove shared B, per-thread tiles, `float4` loads, hardware intrinsics | ~39 µs | +| CUDA optimization 3 | Vectorized PTX FP4 and scale decode | ~27 µs | +| Parameter tuning | Threads per row and rows per block | ~26 µs | +| ILP | Two tiles per loop iteration | ~22.9 µs | +| Aggressive PTX fusion | Decode, scales, multiply, and accumulation in a larger PTX block | ~22.3 µs | +| Submitted leaderboard score | Geometric mean | 22.392 µs | -Key observation: The initial naive CUDA attempt was slower than CuTe DSL, demonstrating that manual optimization requires deep understanding of the hardware to outperform a well-designed DSL. +The 443-to-39 step is not a coalescing-only result, and the 443-to-27 endpoints do not isolate C intrinsics versus PTX. Each spans multiple simultaneous changes. -## Key Optimization Steps +## Reported Technical Details -### Step 1: CuTe DSL Baseline (~100us) +- The CuTe path reduced duplicate scale loads and scale-product arithmetic, then used multiple threads per output with a shared-memory partial-sum reduction. +- Loading the entire B vector into shared memory and double buffering with asynchronous copy did not improve that CuTe attempt. +- The first CUDA improvement combined coalescing, B/SFB shared-memory staging, multiple threads per row, and warp reduction. +- The next CUDA stage removed B shared-memory staging and combined per-thread K tiles, 16-byte `float4` loads, and hardware FP4 conversion intrinsics. +- The PTX stage used `mov.b32` decomposition and packed `cvt.rn.f16x2.e2m1x2` conversion as part of a vectorized decode path. +- Processing two tiles per loop improved the author's implementation; three or four tiles were slightly slower. -```cpp -// CuTe DSL approach: -// - Automatic partition/copy for NVFP4 data -// - Handles packing/unpacking of FP4 values -// - Reasonable but not optimal memory access patterns -// Result: ~100us -- decent starting point -``` +These observations do not establish that shared memory, load width, inline PTX, or ILP has the same effect in another kernel. -CuTe DSL provided a functional baseline without requiring deep hardware knowledge, but left significant performance on the table for this memory-bound kernel. +## Primary Source -### Step 2: Coalesced Memory Access (~443us, then improved) - -Initial hand-written CUDA was actually slower because the memory access pattern was not properly coalesced: - -```cpp -// Bad: each thread reads non-contiguous FP4 elements -// Good: threads in a warp read contiguous 128-byte chunks -// The FP4 packing (2 elements per byte) requires careful indexing -// to maintain coalesced access at the byte level -``` - -After fixing coalescing, performance improved dramatically but still required hardware-specific optimizations. - -### Step 3: Hardware Intrinsics (~39us) - -Replaced generic type conversions with NVIDIA FP4 hardware intrinsics: - -```cpp -// Generic: manual bit manipulation for FP4 -> FP16 conversion -// float val = decode_fp4_manual(packed_byte >> 4); // slow - -// Hardware intrinsic: single instruction for FP4 -> FP16x2 -// __half2 result = __cvt_fp4x2_to_halfx2(packed_fp4); // fast -``` - -The hardware intrinsic path is 11.4x faster than the manual approach, demonstrating the importance of using ISA-specific instructions for sub-byte data types. - -### Step 4: PTX Assembly (~27us) - -Dropped to raw PTX for fine-grained control: - -```asm -// Key PTX optimizations: -// 1. cvt.rn.f16x2.e2m1x2 for FP4 conversion (vs C intrinsic) -cvt.rn.f16x2.e2m1x2 %result, %fp4_packed; - -// 2. Byte unpacking via mov.b32 (vs bitwise shift/mask) -mov.b32 {b0, b1, b2, b3}, %packed_word; -// Splits 32-bit word into 4 bytes without arithmetic - -// 3. Cache-qualified loads -ld.global.L1::no_allocate.v4.u64 {a0,a1,a2,a3}, [addr_a]; // stream A -ld.global.L1::evict_last.v4.u64 {b0,b1,b2,b3}, [addr_b]; // keep B hot -``` - -The PTX byte unpacking (`mov.b32 {a,b,c,d}`) is a critical optimization: it replaces 3-4 shift/mask instructions with a single register move, and the savings compound across the entire K dimension. - -### Step 5: ILP Optimization (~22.9us) - -Increased instruction-level parallelism by unrolling and interleaving independent operations: - -```cpp -// Before: sequential FP4 decode + accumulate -for (int k = 0; k < K; k += 16) { - decode_fp4(a[k:k+16]); - accumulate(partial_sum); -} - -// After: unrolled with interleaved decode + accumulate -// Decode batch[i+1] while accumulating batch[i] -// Multiple independent accumulator registers -``` - -### Final Result: 22.392us - -The final kernel combined all optimizations. Key factors in the 4.5x total improvement: -1. Hardware FP4 conversion intrinsics (biggest single win) -2. PTX byte unpacking (avoids bitwise overhead) -3. Cache policy differentiation (A: no-allocate, B: evict-last) -4. ILP through unrolling and register interleaving -5. Proper memory coalescing for FP4 packed data - -## Key Lessons Shared - -1. **CuTe DSL is a good starting point**: Even for memory-bound kernels, CuTe provides a reasonable baseline. But for the last 4x of performance, manual optimization is required. - -2. **Hardware intrinsics are essential for sub-byte types**: Generic FP4 decoding is an order of magnitude slower than hardware-specific paths. - -3. **PTX gives control that C++ cannot**: Cache policies, byte unpacking, and instruction scheduling are only accessible at the PTX level. - -4. **Memory-bound kernels need different optimization strategies**: Unlike compute-bound GEMM (where tensor core utilization is key), GEMV optimization is about maximizing memory bandwidth utilization. - -## Sources - -- [Yue's Hackathon Journey](https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html) -- [gpu-mode/reference-kernels](https://github.com/gpu-mode/reference-kernels) +- [Yue Zhang, “My Blackwell NVFP4 Kernel Hackathon Journey”](https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html) diff --git a/sources/contests/flashinfer-mlsys26/track-a-fused-moe.md b/sources/contests/flashinfer-mlsys26/track-a-fused-moe.md index 546d8f9d2..20748edaa 100644 --- a/sources/contests/flashinfer-mlsys26/track-a-fused-moe.md +++ b/sources/contests/flashinfer-mlsys26/track-a-fused-moe.md @@ -1,6 +1,6 @@ --- id: contest-flashinfer-track-a -title: 'FlashInfer MLSys 2026 - Track A: Fused MoE FP8' +title: 'FlashInfer MLSys 2026 Track A: FP8 Block-Scale MoE' source_category: contest-report architectures: - sm100 @@ -9,257 +9,109 @@ tags: - moe - fp8 - block-scale -- fused-kernel +- grouped-gemm techniques: - kernel-fusion -- warp-specialization - tile-scheduling -- pipeline-stages - fine-grained-quantization hardware_features: -- tcgen05 -- tmem - fp8 - block-scale -- tma kernel_types: - moe -- fused-kernel -- gemm - grouped-gemm +- fused-kernel languages: - cuda-cpp - cute-dsl - triton url: https://mlsys26.flashinfer.ai/ -submissions: -- rank: 1 - participant: Gemini 2.5 Pro (AI baseline) - score: 0.628x avg speedup vs FlashInfer - technique: iterative refinement with evolve agent, leverages FP8 block-scale MoE - baseline - submission_truth: unavailable - code_unavailable_reason: AI-baseline submission generated by Gemini 2.5 Pro; the - FlashInfer MLSys 2026 contest organizers have not released the generated code - to the public repository -- rank: 2 - participant: GPT-5 (2025-08-07) - score: 0.467x avg speedup, 92.3% resolve rate - technique: code generation with str_replace iteration; high resolve rate but modest - speedup - submission_truth: unavailable - code_unavailable_reason: AI-baseline submission generated by GPT-5 (2025-08-07 snapshot); - organizers have not released generated code publicly -- rank: 3 - participant: Claude Opus 4.1 - score: 0.456x avg speedup, 73.1% resolve rate - technique: iterative kernel refinement; strong on structured kernels - submission_truth: unavailable - code_unavailable_reason: AI-baseline submission generated by Claude Opus 4.1; organizers - have not released generated code publicly -- rank: notable - participant: SGLang baseline - score: 1262 TFLOPS at batch 4096, 206.9us at batch 1 - technique: 5-fused-launch MoE (vs vLLM 7 launches); CUTLASS SM100 warp-specialized - schedule - submission_truth: unavailable - code_unavailable_reason: SGLang baseline fused-MoE kernel for reference comparison; - sglang's public fused-MoE code lives in the main repo and is captured via pr-sglang-21239 - / pr-sglang-21339 rather than as a contest submission bundle +benchmark_url: https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048 +captured_at: 2026-08-08 +starter_kit_sha: 75ccd05cafceb0fd1f86be4cd0f2117249463c66 +flashinfer_sha: 7f614b86470180bab2d22e36fd1775791c6bf3e6 --- -# Track A: Fused MoE (FP8 Block Scale) - -## Problem Description - -Fused Mixture-of-Experts kernel with FP8 block-scale quantization, targeting B200 GPUs. The kernel must fuse routing, dispatch, dual GEMM (gate-up + down projections), SwiGLU activation, and combine into minimal kernel launches. - -**Benchmark identifier**: `fp8_moe_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048` - -**Parameters**: -| Parameter | Value | -|-----------|-------| -| topk | 8 (experts per token) | -| num_groups | 8 | -| topk_group | 4 | -| num_experts | 32 | -| hidden_size | 7168 | -| intermediate_size | 2048 | -| Scaling | FP8 block-scale (block size 128) | - -## Contest Context - -One of three tracks in the FlashInfer AI Kernel Generation Contest at MLSys 2026. B200 GPU evaluation. Human, AI, or hybrid submissions accepted. Teams of up to 5 members. - -**Timeline**: -- Launch: January 22, 2026 -- Dataset released: February 5, 2026 -- Submission deadline: April 24, 2026 -- Writeup due: May 1, 2026 (4 pages max) -- Awards: May 17-22, 2026 (Bellevue, WA) - -**Submission format**: Fork starter kit, implement in `solution/triton/kernel.py` or `solution/cuda/kernel.cu` + `binding.py`. Pack with `python scripts/pack_solution.py`. CUDA bindings via TVM FFI. - -**API**: `flashinfer.fused_moe.trtllm_fp8_block_scale_moe()` - -## Baseline Performance - -| Framework | Batch 4096 TFLOPS | Batch 1 Latency | -|-----------|-------------------|-----------------| -| SGLang | 1262 | 206.9us | -| FlashInfer CuTeDSL | 1225 | 481.9us | -| vLLM | 1117 | 369.5us | - -SGLang leads at large batch (1262 TFLOPS) and small batch (206.9us). FlashInfer CuTeDSL is competitive at large batch but has higher small-batch latency. - -## Key Challenges - -### 1. No Pre-Tuned FP8 MoE Config for B200 - -Existing FP8 MoE kernels are tuned for Hopper (SM90). Blackwell's different tensor core interface (tcgen05 vs wgmma), TMEM, and CLC require fresh tuning of tile sizes, pipeline depths, and warp configurations. - -### 2. FP8 Numerical Overflow with Block Scaling - -Block size 128 means every 128 elements share one scale factor. Careful handling required: - -```python -# Block-scale FP8 quantization -# Each 128-element block has its own E4M3 scale factor -# Overflow risk: if any element in the block is too large, -# the scale factor saturates and precision is lost for all 128 elements -block_size = 128 -for i in range(0, len(tensor), block_size): - block = tensor[i:i+block_size] - scale = compute_scale_e4m3(block.abs().max()) - quantized[i:i+block_size] = quantize_fp8(block / scale) - scales[i // block_size] = scale -``` - -### 3. Performance Varies by Batch Size - -The kernel must handle both: -- **Large batch** (4096+ tokens): Compute-bound, maximize tensor core utilization -- **Small batch** (1-32 tokens): Memory-bound or launch-overhead-bound, minimize latency - -Different strategies needed for each regime. - -### 4. Multiple Kernel Launches - -Existing implementations use multiple launches: -- **vLLM**: 7 kernel launches (routing, sort, scatter, GEMM1, activation, GEMM2, gather) -- **SGLang**: 5 fused launches -- **Optimal**: 1-2 launches (maximum fusion) - -Each launch adds overhead: kernel launch latency (~5-10us on B200), synchronization, memory traffic for intermediate buffers. - -### 5. Variable Expert Load Balancing - -With topk=8 out of 32 experts, token-to-expert assignment is dynamic. Some experts may receive many tokens while others receive few. The kernel must handle this imbalance efficiently without wasting compute on idle experts. +# Track A: FP8 Block-Scale MoE -### 6. TMA 128-Byte Alignment +## Primary Scope -Tensor Memory Accelerator requires 128-byte aligned descriptors. With dynamic expert routing, per-expert weight and activation pointers must be carefully aligned: +The organizer page identifies Fused MoE as Track A of the MLSys 2026 FlashInfer AI Kernel Generation Contest. The contest challenge targets NVIDIA Blackwell B200 GPUs, and Track A links directly to this definition: -```cpp -// Each expert's weight matrix must start at 128-byte boundary -// For FP8 weights: 128 bytes = 128 elements -// Expert weight layout must account for alignment padding -assert(reinterpret_cast(expert_weights[i]) % 128 == 0); -``` +`moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048` -## Architecture: Fused MoE Pipeline +The definition describes an FP8 block-scale MoE operation with routing and two grouped GEMMs included. It is tagged for DeepSeek-V3 and DeepSeek-R1 and records expert parallelism `ep:8`. -``` -Input tokens (hidden_size=7168) - | - v -[Routing] -> expert assignments (topk=8 per token) - | - v -[Dispatch] -> scatter tokens to assigned experts - | - v -[Gate GEMM] A @ W_gate (FP8 block-scale, 7168 -> 2048) -[Up GEMM] A @ W_up (FP8 block-scale, 7168 -> 2048) - | - v -[SwiGLU] SiLU(gate) * up - | - v -[Down GEMM] activated @ W_down (FP8 block-scale, 2048 -> 7168) - | - v -[Combine] -> weighted sum of expert outputs per token - | - v -Output tokens (hidden_size=7168) -``` +## Definition Axes -## Optimization Strategies +| Axis | Value | +| --- | ---: | +| `seq_len` | variable | +| `num_experts` | 256 | +| `num_local_experts` | 32 | +| `hidden_size` | 7168 | +| `intermediate_size` | 2048 | +| `gemm1_out_size` | 4096 | +| `num_hidden_blocks` | 56 | +| `num_intermediate_blocks` | 16 | +| `num_gemm1_out_blocks` | 32 | -### Kernel Fusion +The definition name and reference fix `top_k=8`, `n_group=8`, and `topk_group=4`. Thus `e32` denotes local experts; routing logits still span 256 global experts. -Fuse dispatch + dual GEMM + SwiGLU + down GEMM + combine into minimal launches: +## Signature -``` -// Strategy 1: 2-launch approach -// Launch 1: dispatch + gate_up_gemm + SwiGLU (fused) -// Launch 2: down_gemm + combine (fused) +| Input | Dtype | Shape | +| --- | --- | --- | +| `routing_logits` | FP32 | `[seq_len, num_experts]` | +| `routing_bias` | BF16 | `[num_experts]` | +| `hidden_states` | FP8 E4M3FN | `[seq_len, hidden_size]` | +| `hidden_states_scale` | FP32 | `[num_hidden_blocks, seq_len]` | +| `gemm1_weights` | FP8 E4M3FN | `[num_local_experts, gemm1_out_size, hidden_size]` | +| `gemm1_weights_scale` | FP32 | `[num_local_experts, num_gemm1_out_blocks, num_hidden_blocks]` | +| `gemm2_weights` | FP8 E4M3FN | `[num_local_experts, hidden_size, intermediate_size]` | +| `gemm2_weights_scale` | FP32 | `[num_local_experts, num_hidden_blocks, num_intermediate_blocks]` | +| `local_expert_offset` | INT32 | scalar | +| `routed_scaling_factor` | FP32 | scalar | -// Strategy 2: 1-launch persistent kernel -// Single persistent kernel handles entire MoE pipeline -// Uses CLC for dynamic tile scheduling across expert groups -``` +The output is BF16 `[seq_len, hidden_size]`. -### Warp Specialization for MoE +## Reference Semantics -``` -// Warp roles in fused MoE: -// - Router warps: compute expert assignments (small, fast) -// - TMA warps: async load expert weights for next batch -// - Compute warps: execute tcgen05.mma for gate/up/down GEMMs -// - Epilogue warps: SwiGLU activation + combine -``` +The official reference uses sigmoid routing. It adds `routing_bias` only for expert selection, scores each of eight groups by the sum of its two largest selection scores, retains four groups, and selects eight global experts. Combine weights come from the corresponding unbiased sigmoid values, normalized per token and multiplied by `routed_scaling_factor`. -### Per-Expert Tile Scheduling +Each rank computes only global experts in its 32-expert local interval. GEMM1 produces the concatenated 4096-column W13 result; the reference splits it into two 2048-column halves, applies SwiGLU, performs GEMM2, and accumulates each expert result with its routing weight. -Dynamic tile assignment based on per-expert token counts: +The reference semantics specify an operation, not a GPU launch count or required tcgen05/TMEM decomposition. -``` -// Expert i has M_i tokens -// Tile grid for expert i: ceil(M_i / tile_M) x ceil(N / tile_N) -// Total tiles across all experts: sum(tiles_per_expert) -// CLC distributes total tiles to available SMs -``` +## Official Evaluation Contract -## FlashInfer-Bench Leaderboard (AI Agent Results) +At starter-kit commit `75ccd05cafceb0fd1f86be4cd0f2117249463c66`, `EVALUATION.md` records: -| Model | Avg Speedup | Resolved % | -|-------|-------------|------------| -| Gemini 2.5 Pro | 0.628x | 73.1% | -| GPT-5 (2025-08-07) | 0.467x | 92.3% | -| Claude Opus 4.1 | 0.456x | 73.1% | -| GPT-O3 | 0.450x | 92.3% | +| Field | Value | +| --- | --- | +| GPU | Bare-metal NVIDIA B200 (`sm_100a`) | +| Locked clocks | `nvidia-smi -ac 3996,1965` | +| Container | `flashinfer/flashinfer-ci-cu132:20260401-2c675fb` | +| CUDA | 13.2 | +| Python | 3.12 | +| PyTorch | 2.12.0+cu132 | +| Triton | 3.6.0 | +| MoE correctness | `atol=1`, `rtol=0.3`, required matched ratio `0.9` | -All AI-generated kernels perform below 1.0x vs FlashInfer baselines, demonstrating that MoE kernel optimization remains a human-expertise task. +The baseline solution is `flashinfer_wrapper_9sdjf3`. For the single-definition MoE track, the score is the arithmetic mean of per-workload `baseline_latency / candidate_latency`; any failing workload zeros that kernel's score. -## Agent Baseline +## Performance Boundary -The [mlsys26-agent-baseline](https://github.com/flashinfer-ai/mlsys26-agent-baseline) provides two approaches: -- **Iterative Agent**: propose solution -> refine via `str_replace` edits -- **Evolve Agent**: generate multiple proposals -> elite pool -> evolutionary improvement -- Supports OpenAI and Claude models +No framework TFLOPS/latency/launch-count table is retained from this source. The current definition page exposes per-solution, per-`seq_len` traces, but a reusable performance record would need a named solution and revision, exact workload, environment, timed region, synchronization, warmup, repetitions, statistic, and variance. The former local 1262-TFLOPS record did not supply those fields. -## K-Search Framework +## Contest Dates and Results -Automated kernel generation using co-evolving world model to guide LLM optimization. Achieves 2.10x average improvement over OpenEvolve, up to 14.3x on complex MoE kernels. +The current organizer page records a January 22, 2026 public launch, February 9 baseline release, April 24 kernel submission deadline, May 1 writeup deadline, May 12 winner notification, and May 22 award ceremony. It now lists the actual Track A winners separately for agent-assisted and full-agent approaches; stale pre-contest AI-baseline rankings are not reproduced here. -## Sources +## Primary Sources -- [MLSys 2026 Contest](https://mlsys26.flashinfer.ai/) -- [FlashInfer Starter Kit](https://github.com/flashinfer-ai/flashinfer-bench-starter-kit) -- [FlashInfer Agent Baseline](https://github.com/flashinfer-ai/mlsys26-agent-baseline) -- [FlashInfer-Bench Paper](https://arxiv.org/abs/2601.00227) -- [FlashInfer-Bench Leaderboard](https://bench.flashinfer.ai/) -- [K-Search Paper](https://arxiv.org/abs/2602.19128) -- [mlsys26-contest Dataset (HuggingFace)](https://huggingface.co/datasets/flashinfer-ai/mlsys26-contest) +- [Organizer page](https://mlsys26.flashinfer.ai/) +- [Exact benchmark definition](https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048) +- [Starter kit at `75ccd05`](https://github.com/flashinfer-ai/flashinfer-bench-starter-kit/tree/75ccd05cafceb0fd1f86be4cd0f2117249463c66) +- [Evaluation contract at `75ccd05`](https://github.com/flashinfer-ai/flashinfer-bench-starter-kit/blob/75ccd05cafceb0fd1f86be4cd0f2117249463c66/EVALUATION.md) +- [FlashInfer reference template at `7f614b8`](https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/trace/templates/moe.py) diff --git a/sources/contests/flashinfer-mlsys26/track-c-gated-delta-net.md b/sources/contests/flashinfer-mlsys26/track-c-gated-delta-net.md index 9585d75fe..36f8b8d20 100644 --- a/sources/contests/flashinfer-mlsys26/track-c-gated-delta-net.md +++ b/sources/contests/flashinfer-mlsys26/track-c-gated-delta-net.md @@ -1,6 +1,6 @@ --- id: contest-flashinfer-track-c -title: 'FlashInfer MLSys 2026 - Track C: Gated Delta Net' +title: 'FlashInfer MLSys 2026 Track C: Gated Delta Net' source_category: contest-report architectures: - sm100 @@ -11,284 +11,71 @@ tags: - chunk-parallelism techniques: - chunk-parallelism -- kernel-fusion -- warp-specialization -- pipeline-stages -hardware_features: -- tcgen05 -- tmem -- tma +hardware_features: [] kernel_types: - gated-delta-net - linear-attention - decode - prefill languages: -- cuda-cpp - cute-dsl -- triton -- tilelang +- python url: https://mlsys26.flashinfer.ai/ -submissions: -- rank: 1 - participant: Gemini 2.5 Pro (AI baseline) - score: 0.628x avg speedup - technique: evolve agent generating GatedDeltaNet chunk-parallel prefill + recurrent - decode - submission_truth: unavailable - code_unavailable_reason: AI-baseline submission generated by Gemini 2.5 Pro for - the Gated Delta Net track; organizers have not released generated code publicly -- rank: 2 - participant: GPT-5 - score: 0.467x avg speedup - technique: iterative Triton kernel generation targeting linear attention delta rule - submission_truth: unavailable - code_unavailable_reason: AI-baseline submission generated by GPT-5 for the Gated - Delta Net track; organizers have not released generated code publicly -- rank: 3 - participant: Claude Opus 4.1 - score: 0.456x avg speedup - technique: code generation referencing NVlabs/GatedDeltaNet and FLA kernels - submission_truth: unavailable - code_unavailable_reason: AI-baseline submission generated by Claude Opus 4.1 for - the Gated Delta Net track; organizers have not released generated code publicly -- rank: notable - participant: FLA (Flash Linear Attention) - score: 10x+ throughput vs Qwen3-32B at 32K+ - technique: optimized Triton kernels for gated delta rule; variable-length support; - chunk-based parallel prefill - submission_truth: unavailable - code_unavailable_reason: FLA (Flash Linear Attention) referenced as the Gated Delta - Net notable baseline; its code lives in fla-org/flash-linear-attention and is - summarized by sources/blogs/gated-delta-net.md + artifacts/blogs/gated-delta-net/code/ - rather than as a contest submission bundle +benchmark_decode_url: https://bench.flashinfer.ai/kernels/gdn_decode_qk4_v8_d128_k_last +benchmark_prefill_url: https://bench.flashinfer.ai/kernels/gdn_prefill_qk4_v8_d128_k_last +captured_at: 2026-08-08 +flashinfer_sha: 7f614b86470180bab2d22e36fd1775791c6bf3e6 --- -# Track C: Gated Delta Net (Qwen3-Next) +# Track C: Gated Delta Net -## Problem Description +## Primary scope -Gated Delta Network kernels for both decode and prefill modes, targeting B200 GPUs. GatedDeltaNet is the linear attention mechanism used in Qwen3-Next-80B and Qwen3.5, replacing standard attention in 75% of layers. +The MLSys 2026 FlashInfer organizer identifies Gated Delta Net as Track C, targets NVIDIA Blackwell B200, and links separate decode and prefill definitions. Both definitions were captured from Qwen3-Next linear-attention layers at tensor parallelism four and are tagged `status:verified`. -**Benchmark identifiers**: -- Decode: `qk4_v8_d128_k_last` -- Prefill: `qk4_v8_d128_k_last` +## Fixed geometry -**Parameters**: qk_dim=4, v_dim=8, d=128 (head dimension) +| Axis | Value | +| --- | ---: | +| `num_q_heads` | 4 | +| `num_k_heads` | 4 | +| `num_v_heads` | 8 | +| `head_size` | 128 | -## GatedDeltaNet Mechanism +The heads remain separate. The recurrent state is `[batch_or_sequences,8,128,128]`, not a flattened `512x1024` matrix. -### Delta Rule +## Decode definition -Unlike standard attention (which writes new KV pairs additively), the delta rule performs error-correcting state updates: +The decode definition fixes `seq_len=1` and varies batch size. Q, K, and V are BF16; the state is FP32 in K-last layout `[B,H,V,K]`. The public inputs also include `A_log`, `a`, `dt_bias`, `b`, and an optional scale. Output is BF16 `[B,1,8,128]`, accompanied by the updated FP32 state. -```python -# Standard linear attention: -# S = S + k @ v^T (additive, no forgetting) +At FlashInfer commit `7f614b8`, the reference computes: -# Delta rule: -# S = S + k @ (v - S^T @ k)^T -# = S + k @ v^T - k @ k^T @ S -# This "deletes" the old value at key k before writing the new value v +```text +g = exp(-exp(A_log) * softplus(a + dt_bias)) +beta = sigmoid(b) +decayed = g * state +read = k @ decayed +new_state = decayed + outer(k, beta * (v - read)) +output = scale * (q @ new_state) ``` -The delta rule enables the model to actively overwrite stale information, which standard linear attention cannot do. +## Prefill definition -### Exponential Gating +The prefill definition varies total sequence length and sequence count, accepts `cu_seqlens[num_seqs+1]`, and returns one final state per sequence. At commit `7f614b8`, FlashInfer dispatches SM90 and SM100 implementations. The SM100/SM103 CuTe DSL path requires CUDA 13 or newer and head size 128. -Adaptive memory decay prevents state saturation: +## Results boundary -```python -# Gated delta rule: -# S_t = gate_t * S_{t-1} + k_t @ (v_t - S_{t-1}^T @ k_t)^T -# where gate_t = exp(-softplus(alpha_t)) in [0, 1] +The current organizer page lists Track C winners separately: -# gate close to 1: retain memory (long-term context) -# gate close to 0: forget quickly (short-term patterns) -# Learned per-head, adaptive per-token -``` - -### Dual-Mode Operation - -GatedDeltaNet operates in two modes with different computational characteristics: - -**Prefill (chunk-based parallel)**: -``` -// Sequence divided into chunks of size C -// Within each chunk: parallel matmul-based computation -// Across chunks: sequential state propagation -// Complexity: O(n * C) for chunk-parallel, O(n/C * d^2) for state updates -``` - -**Decode (streaming recurrent)**: -``` -// Single token at a time -// Update recurrent state S: O(d^2) per token -// State size: (num_heads, qk_dim * d, v_dim * d) = (h, 512, 1024) -// O(1) per token during generation -``` - -## Qwen Architecture Context - -### Qwen3-Next-80B -- 80B parameters, 3B active per token -- 48 layers: 12 x (3 x [GatedDeltaNet -> MoE] -> [Full Attention -> MoE]) -- 75% GatedDeltaNet layers, 25% full attention layers -- 512 routed experts, ~19 active per token (1:50 activation ratio) - -### Qwen3.5 -- 60 layers: 15 x (3 x [GatedDeltaNet -> MoE] -> 1 x [Full Attention -> MoE]) -- 262K token context window -- 10x+ throughput over Qwen3-32B at 32K+ context lengths - -## Key Challenges - -### 1. O(n) Linear Complexity - -Unlike O(n^2) attention, GatedDeltaNet has O(n) complexity. This changes the optimization target: -- Standard attention: reduce memory traffic for QK^T and softmax -- GatedDeltaNet: optimize state update operations and chunk boundaries -- Different roofline model and bottleneck analysis - -### 2. Recurrent State Management - -The hidden state S has shape (qk_dim * d) x (v_dim * d) = 512 x 1024 per head: - -``` -// State S is large: 512 * 1024 * sizeof(float) = 2MB per head -// For 64 heads: 128MB of state per layer -// Must be kept in HBM between tokens during decode -// Must be updated atomically during prefill chunk boundaries -``` - -### 3. Gating Mechanism - -The exponential gating adds branching and additional computation: - -``` -// Per-token gating requires: -// 1. Compute alpha from input (learned linear projection) -// 2. Apply softplus: softplus(alpha) = log(1 + exp(alpha)) -// 3. Negate and exponentiate: gate = exp(-softplus(alpha)) -// 4. Element-wise multiply with state -// The exp and log operations are SFU-bound on Blackwell -``` - -### 4. Variable-Length Support - -Production inference requires handling variable-length sequences in a single batch: - -``` -// Batch of requests at different decode positions: -// Request 0: decode token 1024 (state from prefill of 1023 tokens) -// Request 1: decode token 50 (state from prefill of 49 tokens) -// Request 2: prefill 8192 tokens (chunk-parallel mode) -// Must handle mixed prefill/decode in same batch -``` - -### 5. Triton CPU Launch Overhead - -Current implementations (NVlabs/GatedDeltaNet, FLA) use Triton kernels. Triton's CPU-side compilation and launch overhead impacts small-batch decode latency. vLLM enables full CUDA graph mode to mitigate this. - -## Implementation Status (FlashInfer issue #1690) - -| Mode | Hopper (SM90) | Blackwell (SM100) | -|------|---------------|-------------------| -| Prefill | Done | In progress | -| Decode | Done | Done | - -Blackwell prefill kernel is the primary optimization target for this contest track. - -## Reference Implementations - -### NVlabs/GatedDeltaNet (ICLR 2025) -- Reference Triton kernels -- Chunk-based parallel prefill -- Recurrent decode - -### FLA (Flash Linear Attention) -- Optimized Triton kernels (recommended, significantly faster than NVlabs reference) -- Variable-length support -- Better memory efficiency - -### Key Code Pattern: Chunk-Parallel Prefill - -```python -# Triton kernel sketch for GatedDeltaNet prefill -@triton.jit -def gated_delta_net_chunk_prefill( - Q, K, V, Gate, Output, State, - chunk_size: tl.constexpr, - d: tl.constexpr, -): - # Load chunk of Q, K, V, Gate - q = tl.load(Q + offsets) # [chunk_size, qk_dim * d] - k = tl.load(K + offsets) # [chunk_size, qk_dim * d] - v = tl.load(V + offsets) # [chunk_size, v_dim * d] - g = tl.load(Gate + offsets) # [chunk_size] - - # Intra-chunk: parallel computation - # Compute attention-like scores within chunk - scores = tl.dot(q, tl.trans(k)) # [chunk_size, chunk_size] - - # Apply causal mask and gating - # ... - - # Inter-chunk: update state sequentially - state = tl.load(State + state_offsets) - # state = gate * state + k @ (v - state^T @ k)^T - # ... - - tl.store(Output + offsets, output) - tl.store(State + state_offsets, state) -``` - -## Optimization Strategies for Blackwell - -### TMEM for State Management - -``` -// GatedDeltaNet state S (512 x 1024 floats) could reside in TMEM -// TMEM: 128 rows x 512 cols x 32-bit = 256KB per SM -// State chunk (e.g., 128 x 512) fits in TMEM -// Avoid spilling state to shared memory during updates -``` - -### tcgen05.mma for State Update - -``` -// The state update is a matrix operation: -// S = gate * S + k @ delta_v^T -// Where delta_v = v - S^T @ k -// Both terms involve matmuls suitable for tcgen05.mma -``` - -### Chunk Size Tuning - -Larger chunks -> more parallelism within chunks but more state materialization: - -``` -// Chunk size tradeoffs on B200: -// Small chunks (64): good for decode-like workloads, low latency -// Large chunks (256-512): better tensor core utilization for prefill -// TFLA (Tiled Flash Linear Attention) enables arbitrarily large chunks -// via two-level sequence parallelism -``` - -## Contest Submission Format - -Fork the starter kit per track. Implement in: -- `solution/triton/kernel.py` (Triton) -- `solution/cuda/kernel.cu` + `binding.py` (CUDA) +- Agent-assisted: Kachua, UW SyFI, and LLM-CUDA. +- Full-agent: UW SyFI, LLM-CUDA, and HAN Lab Kernel Mafia. -DPS (Destination Passing Style) by default. CUDA binding via TVM FFI. +The former local Gemini/GPT/Claude ranking was a stale agent-baseline snapshot and is not reproduced as a final result. No latency or speedup record is retained here because a revision-pinned result table with workload, environment, timing protocol, repetitions, statistic, and variance was not captured. -## Sources +## Primary sources -- [MLSys 2026 Contest](https://mlsys26.flashinfer.ai/) -- [FlashInfer Starter Kit](https://github.com/flashinfer-ai/flashinfer-bench-starter-kit) -- [NVlabs/GatedDeltaNet](https://github.com/NVlabs/GatedDeltaNet) (ICLR 2025) -- [FLA (Flash Linear Attention)](https://github.com/fla-org/flash-linear-attention) (recommended implementation) -- [FlashInfer issue #1690](https://github.com/flashinfer-ai/flashinfer/issues/1690) (implementation status) -- [Qwen3-Next NVIDIA Blog](https://developer.nvidia.com/blog/new-open-source-qwen3-next-models-preview-hybrid-moe-architecture-delivering-improved-accuracy-and-accelerated-parallel-processing-across-nvidia-platform/) -- [TFLA Paper](https://arxiv.org/abs/2503.14376) +- [Organizer and winners](https://mlsys26.flashinfer.ai/) +- [Decode definition](https://bench.flashinfer.ai/kernels/gdn_decode_qk4_v8_d128_k_last) +- [Prefill definition](https://bench.flashinfer.ai/kernels/gdn_prefill_qk4_v8_d128_k_last) +- [Pinned FlashInfer reference](https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/trace/templates/gdn.py) +- [Pinned FlashInfer prefill dispatch](https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/gdn_prefill.py) diff --git a/sources/contests/gpu-mode-nvfp4/problem-1-gemv.md b/sources/contests/gpu-mode-nvfp4/problem-1-gemv.md index d144d4390..a483c5c19 100644 --- a/sources/contests/gpu-mode-nvfp4/problem-1-gemv.md +++ b/sources/contests/gpu-mode-nvfp4/problem-1-gemv.md @@ -10,14 +10,6 @@ tags: - gemv - fp4 - block-scale -techniques: -- vectorized-loads -- cache-policy -- register-reuse -- per-k-specialization -- data-reuse -- register-budgeting -- loop-unrolling hardware_features: - nvfp4 - fp4 @@ -26,203 +18,54 @@ kernel_types: - batched-gemv - gemv languages: -- cuda-cpp -- ptx +- python - cute-dsl -url: https://github.com/gpu-mode/reference-kernels -submissions: -- rank: 1 - participant: Simon (veitner) - score: ~22.4us geomean - technique: Full PTX assembly with cache policy differentiation, byte unpacking, - and aggressive register budgeting (maxrregcount=32) - submission_truth: reconstructed-from-blog - code_path: artifacts/contests/gpu-mode-nvfp4/problem-1-gemv/submissions/rank-1-simon-veitner/03-strategy-3-atomic-free-shared-memory-reduction.cpp -- rank: 2 - participant: yue - score: ~23.0us geomean - technique: Shared B vector reads across BLOCK_M rows, PTX load/decode path, ILP - optimization - submission_truth: reconstructed-from-blog - code_path: artifacts/contests/gpu-mode-nvfp4/problem-1-gemv/submissions/rank-2-yue/05-step-5-ilp-optimization-22-9us.cpp -- rank: 3 - participant: Amandeep - score: ~24.0us geomean - technique: PTX assembly with per-K specialization, vectorized 256-bit loads, cache - bypass for streamed matrix A - submission_truth: unavailable - code_unavailable_reason: Amandeep's PTX-level per-K specialization kernel was shared - in the GPU Mode Discord problem-1 thread; author has not republished to a public - platform at collection time -artifact_dir: artifacts/contests/gpu-mode-nvfp4/problem-1-gemv +url: https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv +retrieved_at: 2026-08-08 --- # Problem 1: NVFP4 Batched GEMV -## Problem Description +## Evidence Scope -Batched matrix-vector multiply with NVFP4 (E2M1) block-scaled inputs on B200 GPUs. +This card records the official task at `gpu-mode/reference-kernels` commit `ae67948685dfccf54ae8374dc9402addb7aae4f6`, the official contest rules, and a dated public leaderboard response. It does not attribute implementations to contestants because the public API exposes no source or technique field. -**Inputs**: -- Matrix `a`: shape (M x K x L), NVFP4 format -- Vector `b`: shape (1 x K x L), NVFP4 format -- Scale factors `sfa`: shape (M x K/16 x L), FP8 E4M3 (one scale per 16 FP4 elements) -- Scale factors `sfb`: shape (1 x K/16 x L), FP8 E4M3 +## Task Contract -**Output**: `c`: shape (M x 1 x L), FP16 +The operation is an N=1 block-scaled NVFP4 matrix-vector product on B200 with FP16 output. Although `task.yml` describes a logical five-tensor tuple, the pinned Python callable receives seven physical tensors: -**Nature**: Memory-bound (low arithmetic intensity -- each FP4 element is only used once in the dot product). +| Tensor | Physical shape | +|---|---| +| A payload | `[M, K/2, L]` | +| B payload | `[128, K/2, L]` | +| logical SFA | `[M, K/16, L]` | +| logical SFB | `[128, K/16, L]` | +| reordered SFA | `[32, 4, ceil(M/128), 4, K/64, L]` | +| reordered SFB | `[32, 4, 1, 4, K/64, L]` | +| C | `[M, 1, L]` | -**Benchmark configurations**: -| Config | M | K | L | -|--------|------|-------|---| -| 1 | 7168 | 16384 | 1 | -| 2 | 4096 | 7168 | 8 | -| 3 | 7168 | 2048 | 4 | +B is physically padded to 128 rows to support the `torch._scaled_mm` reference, but only logical result column zero is copied to C. The generator constructs E4M3FN scale tensors even though the task prose says E4M3FNUZ. There are no FP32 global-scale arguments in this task ABI. Correctness uses `rtol=1e-3` and `atol=1e-3`; K must be divisible by 64, and M must satisfy the submitted kernel's M-tile divisibility rule. -**Speed of light**: ~8.6us for the largest configuration (theoretical bound from B200 8 TB/s memory bandwidth). +## Benchmarks and Ranking -## Timeline +The task publishes these theoretical times under a 1.5 GHz B200 model based on the slower of FFMA math and DRAM transfer time: -November 10 -- November 28, 2025. First of four problems in the hackathon series. +| M | K | L | Theoretical time (µs) | +|---:|---:|---:|---:| +| 7168 | 16384 | 1 | 8.622 | +| 4096 | 7168 | 8 | 17.275 | +| 7168 | 2048 | 4 | 4.317 | -Prizes: 1st place DGX Spark + GTC pass, 2nd RTX 5090 + GTC, 3rd RTX 5080. +Submissions are ranked by the geometric mean of the three benchmark results. The pinned task also contains ten correctness cases. -## NVFP4 Data Format +## Timeline and Public Snapshot -- 4-bit floating-point E2M1: 1 sign bit, 2 exponent bits, 1 mantissa bit -- Representable values: 0, 0.5, 1, 1.5, 2, 3, 4, 6 (positive and negative) -- Block scaling: every 16 FP4 elements share one FP8 E4M3 scale factor -- Two-level scaling: per-block E4M3 scale + per-tensor FP32 global scale -- Dequantization formula: `x_hat_i = s_global * s_block * deq_FP4(q_i)` -- Key difference from MXFP4: E4M3 block scale (non-power-of-two), smaller block size (16 vs 32) +The official rules run Kernel 1 from November 10, 2025 at 12:00 a.m. PT through November 28, 2025 at 11:59 p.m. PT and assign it 10% of the four-kernel grand-prize score. -## Top Performer Results +The mutable public endpoint fetched on 2026-08-08 showed `s.am._`, `gau.nernst`, and `shellsmile15795` in its first three rows with aggregate scores of 18.549562452, 18.552844757, and 18.707609314 microseconds. Yue's 22.392217755 score was rank 11 and Simon's 25.112153955 score was rank 25. Current ranks 2 and 3 have post-cutoff timestamps, so this response must not be presented as an official prize-placement snapshot. -Geometric mean across all three benchmark configurations: -- Rank 1: ~22.4us (using full PTX assembly with per-K specialization) -- Rank 2: ~23.0us (shared B vector reads across BLOCK_M rows) -- Rank 3: ~24.0us +## Primary Sources -Speed-of-light gap: top performers achieved roughly 2.6x of SOL (~8.6us), reflecting the overhead of FP4 decoding and scale application. - -## Optimization Techniques from Top Performers - -### PTX-Level Control - -Raw PTX instructions instead of C intrinsics for critical paths: - -```asm -// FP4 to FP16 conversion via PTX -cvt.rn.f16x2.e2m1x2 %result, %fp4_packed; - -// Byte unpacking: avoid bitwise extraction overhead -mov.b32 {tmp0, tmp1, tmp2, tmp3}, %packed_word; -``` - -Key insight: PTX byte unpacking (`mov.b32 {a, b, c, d}`) is significantly faster than manual bitwise extraction (`>> 4 & 0xF`) for splitting packed FP4 values. - -### Cache Policy Differentiation - -Different cache strategies for different access patterns: - -```asm -// Matrix A (streamed once, never reused): bypass L1 to avoid pollution -ld.global.L1::no_allocate.v4.u64 {a0,a1,a2,a3}, [addr_a]; - -// Vector B (reused across M rows): keep hot in L1 -ld.global.L1::evict_last.v4.u64 {b0,b1,b2,b3}, [addr_b]; -``` - -Rank 1 solution used different `ld.global` qualifiers depending on which K-dimension variant was being compiled. - -### Register Budgeting - -Lower register counts force higher occupancy, which is critical for memory-bound kernels: - -``` -// Rank 1: aggressive register limit -nvcc -maxrregcount=32 ... - -// Rank 3: slightly relaxed -nvcc -maxrregcount=45 ... -``` - -Fewer registers per thread -> more warps per SM -> better memory latency hiding. - -### Wider Vectorized Loads - -128-bit and 256-bit vector loads to maximize memory bandwidth utilization: - -```asm -// 128-bit load (2x uint64) -ld.global.v2.u64 {r0, r1}, [addr]; - -// 256-bit load (4x uint64) -ld.global.v4.u64 {r0, r1, r2, r3}, [addr]; -``` - -Only effective when combined with PTX byte unpacking to avoid bitwise overhead in the unpack stage. - -### Per-K Specialization - -Separate kernel compilations per K-dimension, each with full loop unrolling: - -```cpp -// Each K variant compiled separately with optimal config -template -__global__ void nvfp4_gemv_specialized(); - -// K=1024: fewer iterations, aggressive unrolling -// K=3584: moderate unrolling, different block dims -// K=8192: deepest loop, different register budget -``` - -Different K values have different optimal block dimensions, register limits, and unroll factors. Compiling separate kernels avoids runtime branching. - -### Data Reuse (Rank 2 approach) - -Share the B vector reads across all BLOCK_M rows within a thread block: - -``` -// Each thread block handles BLOCK_M rows -// Vector B is loaded once into shared memory -// All threads in the block reuse the same B data -__shared__ half b_shared[K_TILE]; -``` - -Since B is shape (1 x K x L), every row of A multiplies against the same B vector. Sharing B reads across BLOCK_M rows reduces global memory traffic proportionally. - -## Performance Progression (from Yue's blog) - -| Stage | Technique | Latency | -|-------|-----------|---------| -| CuTe DSL baseline | Basic CuTe partition/copy | ~100us | -| Coalesced access | Fix memory access patterns | ~443us -> 39us | -| Hardware intrinsics | Use cvt.rn.f16x2.e2m1x2 | ~39us | -| PTX assembly | Full PTX with byte unpacking | ~27us | -| ILP optimization | Instruction-level parallelism | ~22.9us | -| Final submission | All combined | 22.392us | - -## Key Lessons - -1. **Memory-bound kernels need bandwidth-first thinking**: Arithmetic optimizations have minimal impact; focus on memory access patterns, cache policies, and vectorized loads. -2. **PTX gives real control on Blackwell**: The gap between C intrinsics and hand-written PTX was substantial (443us -> 27us in one participant's journey). -3. **Nsight Compute confirms memory-bound behavior**: "Run Nsight Compute to confirm memory-bound behavior" (Amandeep's key lesson after 12 attempts). -4. **Register budgeting matters**: On memory-bound kernels, lower register count -> higher occupancy -> better memory latency hiding. The difference between 32 and 45 max registers was measurable. - -## B200 Context - -- Architecture: sm_100a, 142 SMs -- Memory bandwidth: 8 TB/s HBM3e -- Native FP4 (E2M1) tensor core instructions -- TMA for async bulk loads -- TMEM: 128 x 512 x 32-bit per SM (not used for GEMV -- memory-bound, not compute-bound) - -## Sources - -- [gpu-mode/reference-kernels](https://github.com/gpu-mode/reference-kernels) (`/problems/nvidia/nvfp4_gemv/`) -- [Yue's Hackathon Journey](https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html) -- [Twelve Attempts (Amandeep)](https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/) -- [Simon's NVFP4 GEMV Blog](https://veitner.bearblog.dev/nvfp4-gemv/) -- [NVFP4 Format Details](https://haroldbenoit.com/notes/ml/engineering/precision/nvfp4-format) +- [Pinned task directory](https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv) +- [Official terms and conditions](https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf) +- [Public leaderboard endpoint](https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemv/NVIDIA?limit=100) diff --git a/sources/contests/gpu-mode-nvfp4/problem-2-gemm.md b/sources/contests/gpu-mode-nvfp4/problem-2-gemm.md index 0469d52b1..f52c9810b 100644 --- a/sources/contests/gpu-mode-nvfp4/problem-2-gemm.md +++ b/sources/contests/gpu-mode-nvfp4/problem-2-gemm.md @@ -10,181 +10,53 @@ tags: - gemm - fp4 - block-scale -- tcgen05 -- tmem -- tma -techniques: -- warp-specialization -- pipeline-stages -- swizzling -- register-reuse hardware_features: - nvfp4 - fp4 - block-scale -- tcgen05 -- tmem -- tma kernel_types: - gemm languages: -- cuda-cpp -- ptx -- cute-dsl -url: https://github.com/gpu-mode/reference-kernels -submissions: -- rank: 1 - participant: Simon (veitner) - score: 10.807us geomean - technique: CUTLASS SM100 warp-specialized NVFP4 GEMM with tcgen05.mma, optimized - TMA pipeline depth and tile scheduling - submission_truth: unavailable - code_unavailable_reason: Simon's NVFP4 GEMM winning submission posted in the GPU - Mode Discord problem-2 thread; author has not republished the GEMM variant on - a public platform -- rank: 2 - participant: yue - score: 10.914us geomean - technique: CUTLASS-based warp specialization with tuned pipeline stages, TMA async - bulk loads, TMEM accumulator management - submission_truth: unavailable - code_unavailable_reason: Yue's NVFP4 GEMM submission posted in the GPU Mode Discord - problem-2 thread; the public hackathon blog covers GEMV (problem 1), not problem-2 - GEMM -- rank: 3 - participant: currybab - score: 10.931us geomean - technique: CUTLASS KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100 schedule with custom - tile size and cluster shape tuning - submission_truth: unavailable - code_unavailable_reason: currybab's submission posted in the GPU Mode Discord problem-2 - thread; no public author republish at collection time +- python +url: https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm +captured_at: 2026-08-08 --- # Problem 2: NVFP4 GEMM -## Problem Description +## Evidence-scoped task record -Standard NVFP4 block-scaled general matrix multiplication on B200 GPUs. Unlike Problem 1 (GEMV), this is compute-bound and targets tensor core utilization. +The official NVIDIA rules identify this as Kernel Challenge 2, running from November 29 through December 19, 2025. It targets NVIDIA B200 and contributes 20% of the four-problem grand-prize score. -**Operation**: C = A * B where A and B are NVFP4 (E2M1) with per-16-element FP8 E4M3 block scaling. +At `gpu-mode/reference-kernels` commit `ae67948685dfccf54ae8374dc9402addb7aae4f6`, the public task implements block-scaled matrix multiplication over each `L` slice with packed E2M1 A and B, per-16 logical scales, reordered scale copies, and preallocated FP16 C. The correctness checker uses `rtol=1e-3` and `atol=1e-3`. -**Nature**: Compute-bound -- high arithmetic intensity enables tensor core saturation. +The task description abbreviates the input as five tensors, but `task.py`, `template.py`, and `reference.py` expose seven: A, B, logical SFA/SFB, reordered SFA/SFB, and C. The prose/template call the scales E4M3FNUZ, while `reference.py` constructs `torch.float8_e4m3fn`; consumers must follow the actual tensors supplied by the harness. -## Timeline +`K` is divisible by 256. `M` and `N` divisibility depends on the submission's MMA tile. The repository publishes ten correctness cases and three ranking cases. -November 29 -- December 19, 2025. Second problem, weighted 20% for grand prize. +## Ranking contract -## Top Performer Results +Ranking uses the geometric mean of three benchmark results. The task labels these B200 values a theoretical speed-of-light analysis at 1.5 GHz, using the maximum of FP4 Tensor Core math time and DRAM-memory time: -Geometric mean across benchmark configurations: -| Rank | Participant | Geometric Mean | -|------|-----------|----------------| -| 1 | Simon | 10.807us | -| 2 | yue | 10.914us | -| 3 | currybab | 10.931us | +| M | N | K | L | Theoretical time (µs) | +| ---: | ---: | ---: | ---: | ---: | +| 128 | 7168 | 16384 | 1 | 8.994 | +| 128 | 4096 | 7168 | 1 | 2.354 | +| 128 | 7168 | 2048 | 1 | 1.333 | -Extremely tight competition -- top 3 within 1.1% of each other. +These rows are not measured contestant results or a cuBLAS comparison. -## Optimization Techniques +## Public leaderboard boundary -### Tensor Core Utilization +The public Popcorn API snapshot fetched August 8, 2026 places `gau.nernst`, `s.am._`, and `billcarson` at current ranks 1-3. It places `Simon`, `yue`, and `currybab` at ranks 8-10; their API scores, multiplied by one million, are 10.806750, 10.914084, and 10.930623. -Unlike Problem 1 (memory-bound GEMV), GEMM fully leverages Blackwell tensor cores: +The current top-three timestamps are December 20-21, after the official December 19 cutoff. The endpoint therefore does not by itself establish prize winners or cutoff rankings. It exposes score metadata and submission filenames, not contestant source code, optimization techniques, a cuBLAS baseline, trials, or variance. -``` -// tcgen05.mma operates directly on shared memory -// No ldmatrix needed -- operands read from SMEM, results written to TMEM -tcgen05.mma.cta_group::1.kind::f16 - [tmem_addr], // accumulator in TMEM - [smem_desc_a], // operand A descriptor (shared memory) - [smem_desc_b]; // operand B descriptor (shared memory) -``` +## Primary references -Key Blackwell advantage: tcgen05.mma reads operands directly from shared memory and writes results to tensor memory (TMEM), eliminating the register-based ldmatrix/stmatrix pipeline required on Hopper. - -### Warp Specialization - -Dedicated warp roles for overlapping data movement and computation: - -- **TMA warps**: Issue async bulk loads from global to shared memory via TMA descriptors -- **Tensor core warps**: Execute tcgen05.mma on data already in shared memory -- **Epilogue warps**: Handle accumulator readback from TMEM and output conversion - -``` -// CUTLASS schedule used by top performers: -// KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100 -``` - -### TMA for Async Bulk Loads - -Tensor Memory Accelerator handles data movement without consuming SM resources: - -- 128-byte alignment requirement for TMA descriptors -- Async pipeline from HBM -> SMEM via TMA, overlapped with tensor core compute -- Shared memory acts as staging buffer for NVFP4 data + scale factors - -### TMEM for MMA Results - -Tensor Memory holds accumulator state: - -- 128 x 512 matrix of 32-bit elements per SM -- MMA results written directly to TMEM (not registers) -- Eliminates register pressure from large accumulator tiles -- Readback to registers only during epilogue - -### NVFP4 Block Scale Handling - -Scale factors must be applied during or after the MMA: - -``` -// Two-level dequantization in epilogue: -// 1. Apply per-block E4M3 scale factors -// 2. Apply per-tensor FP32 global scale -// 3. Convert accumulator to FP16 output -result[i] = fp16(global_scale * block_scale_a[i/16] * block_scale_b[j/16] * acc[i][j]); -``` - -## Key Code Pattern: CUTLASS SM100 NVFP4 GEMM - -Top performers leveraged CUTLASS 4.x infrastructure: - -```cpp -// CUTLASS collective MMA for NVFP4 on SM100 -using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< - cutlass::arch::Sm100, - cutlass::arch::OpClassTensorOp, - cutlass::float_e2m1_t, // Element A: NVFP4 - LayoutA, - AlignmentA, - cutlass::float_e2m1_t, // Element B: NVFP4 - LayoutB, - AlignmentB, - float, // Accumulator: FP32 - TileShape, - ClusterShape, - cutlass::gemm::collective::StageCountAutoCarveout<>, - cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100 ->::CollectiveOp; -``` - -## Performance Context - -On B200 (142 SMs, peak FP4 tensor TFLOPS): -- Top performers achieved near-cuBLAS performance for NVFP4 GEMM -- The ~10.8us geometric mean represents excellent tensor core utilization -- Key differentiator from Problem 1: this is compute-bound, so tensor core scheduling and pipeline depth matter more than memory access patterns - -## B200 Hardware Used - -- sm_100a, 142 SMs -- TMEM: 256KB per SM (128 rows x 512 cols x 32-bit) -- tcgen05.mma with native NVFP4 support -- TMA with 128-byte alignment -- 8 TB/s HBM3e bandwidth (less relevant for compute-bound GEMM) - -## Sources - -- [gpu-mode/reference-kernels](https://github.com/gpu-mode/reference-kernels) (`/problems/nvidia/nvfp4_gemm/`) -- [NVIDIA Forums Announcement](https://forums.developer.nvidia.com/t/join-us-for-the-blackwell-nvfp4-kernel-hackathon-with-nvidia-and-gpu-mode/350092) -- [TFLOPS Gap Blog](https://huggingface.co/blog/apsys/blackwell-nvfp4-comparison) +- [Official contest rules](https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf) +- [Pinned task definition](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml) +- [Pinned task types](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.py) +- [Pinned starter template](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py) +- [Pinned correctness reference](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/reference.py) +- [Public Popcorn leaderboard API](https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12) diff --git a/sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md b/sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md index 6664aa758..a7009954e 100644 --- a/sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md +++ b/sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md @@ -7,181 +7,70 @@ architectures: - sm100a tags: - nvfp4 -- gemm - fp4 - block-scale -- tcgen05 -- tmem -- tma -techniques: -- warp-specialization -- kernel-fusion -- epilogue-fusion -- pipeline-stages -hardware_features: -- nvfp4 -- fp4 -- block-scale -- tcgen05 -- tmem -- tma +- gemm +- gated-dual-gemm kernel_types: - gated-dual-gemm - gemm -- fused-kernel languages: -- cuda-cpp -- ptx -- cute-dsl -url: https://github.com/gpu-mode/reference-kernels -submissions: -- rank: 1 - participant: Simon (veitner) - score: ~19us geomean - technique: Fused dual GEMM with shared A tile, epilogue SiLU fusion, dual TMEM accumulator - layout, CUTLASS SM100 schedule - submission_truth: unavailable - code_unavailable_reason: Simon's gated-dual-GEMM winning submission posted in the - GPU Mode Discord problem-3 thread; not republished publicly -- rank: 2 - participant: yue - score: ~19.5us geomean - technique: CUTLASS warp-specialized dual GEMM with TMA pipeline overlap for W_gate - and W_up streams - submission_truth: unavailable - code_unavailable_reason: Yue's gated-dual-GEMM submission posted in the GPU Mode - Discord problem-3 thread; blog covers problem-1 progression, not this problem -- rank: 3 - participant: currybab - score: ~20us geomean - technique: Epilogue-fused SiLU + element-wise multiply, shared input tiling across - both GEMMs - submission_truth: unavailable - code_unavailable_reason: currybab's gated-dual-GEMM submission posted in the GPU - Mode Discord problem-3 thread; no public republish at collection time +- python +url: https://github.com/gpu-mode/reference-kernels/tree/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm +problem_number: 3 +description: Exact public task and correctness-reference scope at the challenge-opening + commit; no unpublished leaderboard or submission details are asserted. --- # Problem 3: NVFP4 Gated Dual GEMM -## Problem Description - -Fused gated dual GEMM implementing the standard MLP gate-up pattern found in modern LLMs (e.g., LLaMA, DeepSeek, Qwen): - -``` -gate = A @ W_gate // First GEMM -up = A @ W_up // Second GEMM -out = SiLU(gate) * up // Activation + element-wise multiply -``` - -Both GEMMs use NVFP4 (E2M1) block-scaled inputs on B200 GPUs. The challenge is fusing the two GEMMs with the SiLU activation and element-wise multiply into a single kernel launch. - -**Nature**: Compute-bound (two full GEMMs), with fusion opportunity in the epilogue. - -## Timeline - -December 20, 2025 -- January 16, 2026. Third problem, weighted 30% for grand prize. +## Verified identity -## Optimization Techniques +The official NVIDIA rules name this Kernel Challenge 3 and give its entry window as December 20, 2025 through January 16, 2026. The public GPU Mode task at commit `c5b2f7c062d5015f29c3a1043cfd04954397944c` targets NVIDIA B200. -### Kernel Fusion Strategy +## Exact operation -The naive approach requires 3 kernel launches: -1. GEMM for gate projection -2. GEMM for up projection -3. Element-wise SiLU(gate) * up +For each batch index `l`, the reference computes: -The fused approach combines all three into a single kernel: - -``` -// Fused approach: single kernel, shared input tiles -// 1. Load A tile from SMEM (shared between both GEMMs) -// 2. Load W_gate tile -> compute partial gate accumulator -// 3. Load W_up tile -> compute partial up accumulator -// 4. In epilogue: apply SiLU to gate, multiply with up, write output +```python +gate = scaled_mm(a[:, :, l], b1[:, :, l].T, sfa, sfb1) +up = scaled_mm(a[:, :, l], b2[:, :, l].T, sfa, sfb2) +output[:, :, l] = silu(gate) * up ``` -Key benefit: Input matrix A is loaded once from HBM and reused for both GEMMs, cutting global memory traffic for A in half. +The two products share `a` and its scale tensor `sfa`; `b1` and `b2` have separate scale tensors. The FP32 product results are combined and converted to FP16 by the correctness reference. -### Epilogue Fusion +## Published tensor contract -The SiLU activation and element-wise multiply are fused into the GEMM epilogue: +| Tensor | Dtype | Logical shape | +| --- | --- | --- | +| `a` | NVFP4 E2M1 | `[M,K,L]`, K-major | +| `b1`, `b2` | NVFP4 E2M1 | `[N,K,L]`, K-major | +| `sfa` | FP8 E4M3FNUZ | `[M,K/16,L]`, K-major | +| `sfb1`, `sfb2` | FP8 E4M3FNUZ | `[N,K/16,L]`, K-major | +| `c` | FP16 | `[M,N,L]` | -```cpp -// SiLU(x) = x * sigmoid(x) = x / (1 + exp(-x)) -// Applied to gate GEMM output, then multiplied with up GEMM output +The submission tuple also includes reordered copies of all three scale tensors and a preallocated output. There is an upstream dtype-label inconsistency at this commit: `task.yml` and `template.py` call the scales E4M3FNUZ, while `reference.py` constructs `torch.float8_e4m3fn`. This capture preserves that distinction instead of silently choosing one spelling. -// In CUTLASS epilogue visitor: -struct SiLUGateFusion { - template - __device__ auto operator()(AccumTile const& gate_acc, AccumTile const& up_acc) { - auto gate_f32 = convert(gate_acc); - auto up_f32 = convert(up_acc); - // SiLU + element-wise multiply - return gate_f32 * sigmoid(gate_f32) * up_f32; - } -}; -``` +`K` must be divisible by 256. The task additionally requires `M` and `N` to be divisible by the selected MMA tile dimensions. The correctness checker uses relative and absolute tolerances of `1e-3`. -### Dual Accumulator Management in TMEM +## Published benchmark and scoring contract -Both GEMM accumulators must fit in TMEM simultaneously: +| M | N | K | L | Theoretical speed-of-light time (µs) | +| ---: | ---: | ---: | ---: | ---: | +| 256 | 4096 | 7168 | 1 | 4.708 | +| 512 | 4096 | 7168 | 1 | 8.714 | +| 256 | 3072 | 4096 | 1 | 2.125 | +| 512 | 3072 | 7168 | 1 | 6.535 | -- TMEM capacity: 128 rows x 512 columns x 32-bit per SM -- Gate accumulator: occupies one region of TMEM -- Up accumulator: occupies adjacent region -- Careful tile sizing to fit both without spilling +Ranking uses the geometric mean of benchmark times. The task labels the last column a speed-of-light analysis based on the maximum of FP4 Tensor Core math time and DRAM-memory time for B200 at a 1.5 GHz clock. These are theoretical comparison values, not measured winning latencies. -``` -// TMEM layout for dual GEMM: -// [0, 255] columns: gate accumulator -// [256, 511] columns: up accumulator -// Both share the same 128 rows -``` - -### Warp Specialization for Dual GEMM - -Extended warp specialization with separate pipelines for each GEMM's weight loads: - -- **TMA warp group 1**: Loads A tiles + W_gate tiles -- **TMA warp group 2**: Loads W_up tiles (A tiles shared) -- **Compute warps**: Execute tcgen05.mma for both GEMMs -- **Epilogue warps**: Fused SiLU + multiply + FP16 output - -### Pipeline Scheduling - -Multi-stage software pipeline handles both weight streams: - -``` -// Stage N: TMA loads A[n], W_gate[n], W_up[n] -// Stage N-1: tcgen05.mma on A[n-1] * W_gate[n-1], A[n-1] * W_up[n-1] -// Stage N-2: Epilogue fusion on completed tiles -``` +## Evidence boundary -The shared A tile across both GEMMs means only 3 TMA streams (A, W_gate, W_up) instead of 4 (A_gate, W_gate, A_up, W_up). - -## Relevance to LLM Inference - -This pattern appears in every transformer MLP block using gated activations: -- **LLaMA/LLaMA-2/LLaMA-3**: SwiGLU MLP (gate + up projections) -- **DeepSeek-V3**: Same gated MLP structure in each expert -- **Qwen-3**: SwiGLU in both dense and MoE variants -- **Mistral/Mixtral**: Gated MLP in every layer - -Fusing the dual GEMM reduces kernel launch overhead and halves the A matrix memory traffic, making it essential for inference latency optimization. - -## CUTLASS Schedule - -Top performers used CUTLASS 4.x with the SM100 NVFP4 schedule: - -```cpp -using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100; - -// The dual GEMM is composed as two CUTLASS GEMMs with shared input -// and a fused epilogue visitor -``` +The public task and reference do not publish a final leaderboard, winning source, launch count, TMEM partition, TMA pipeline, CUTLASS schedule, physical input-load count, or compute-/memory-bound verdict for a submission. No such implementation or result claims are retained in this source capture. -## Sources +## Primary sources -- [gpu-mode/reference-kernels](https://github.com/gpu-mode/reference-kernels) (`/problems/nvidia/nvfp4_dual_gemm/`, `/problems/nvidia/modal_nvfp4_dual_gemm/`) -- [GPU MODE Hackathon (Luma)](https://luma.com/9n27uem4) -- [NVIDIA Forums Announcement](https://forums.developer.nvidia.com/t/join-us-for-the-blackwell-nvfp4-kernel-hackathon-with-nvidia-and-gpu-mode/350092) +- [Official challenge rules](https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf) +- [Pinned public task](https://github.com/gpu-mode/reference-kernels/blob/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm/task.yml) +- [Pinned correctness reference](https://github.com/gpu-mode/reference-kernels/blob/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm/reference.py) diff --git a/sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md b/sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md index e14edcb94..cd58c4719 100644 --- a/sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md +++ b/sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md @@ -10,185 +10,69 @@ tags: - grouped-gemm - fp4 - block-scale -- tcgen05 -- tmem -- tma - moe -techniques: -- warp-specialization -- tile-scheduling -- pipeline-stages -- kernel-fusion -hardware_features: -- nvfp4 -- fp4 -- block-scale -- tcgen05 -- tmem -- tma -- clc kernel_types: - grouped-gemm - gemm - moe languages: -- cuda-cpp -- ptx -- cute-dsl -url: https://github.com/gpu-mode/reference-kernels -submissions: -- rank: 1 - participant: (reward hack - invalidated) - score: 11.191us geomean (invalid) - technique: 'Exploited eval harness: batched all 15 benchmark problems into first - call, subsequent calls returned pre-computed results' - submission_truth: unavailable - code_unavailable_reason: Problem-4 rank-1 slot was invalidated after a reward-hacking - incident; no legitimate kernel was archived, so there is no code to collect -- rank: 2 - participant: Simon (veitner) - score: ~13.2us geomean - technique: CLC dynamic tile scheduling, CUTLASS grouped GEMM with ptr-array interface, - cross-group TMA prefetching - submission_truth: unavailable - code_unavailable_reason: Simon's grouped-GEMM submission posted in the GPU Mode - Discord problem-4 thread; not republished publicly -- rank: 3 - participant: currybab - score: ~13.5us geomean - technique: Group packing for small-M experts, warp-specialized pipeline with group-boundary-aware - scheduling - submission_truth: unavailable - code_unavailable_reason: currybab's grouped-GEMM submission posted in the GPU Mode - Discord problem-4 thread; no public republish at collection time +- python +url: https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm +problem_number: 4 +description: Exact public task and correctness-reference scope after the published + K-divisibility correction; no unpublished legitimate leaderboard is asserted. --- # Problem 4: NVFP4 Grouped GEMM -## Problem Description - -Multiple GEMMs with variable M dimensions but shared N and K, directly relevant to Mixture-of-Experts (MoE) inference on B200 GPUs: - -``` -for i in range(num_groups): - C[i] = A[i] @ B[i] - // A[i] shape: (M_i x K), variable M per group (tokens routed to expert i) - // B[i] shape: (K x N), shared N and K across all groups - // All in NVFP4 with block scaling -``` - -**Nature**: Compute-bound with load-balancing challenge. Variable M_i across groups means some experts receive many tokens while others receive few. - -## Timeline - -January 17 -- February 13, 2026. Final problem, weighted 40% (heaviest) for grand prize. - -Grand Prize: Dell Pro Max with GB300 NVLink. Weighted scoring across all 4 problems: 10% / 20% / 30% / 40%. - -## MoE Relevance - -Grouped GEMM is the core compute kernel for MoE inference: -- DeepSeek-V3: 256 experts, tokens routed to 8 experts each -- Qwen3-Next: 512 experts, ~19 active per token -- Each expert's forward pass is a grouped GEMM where M_i = number of tokens routed to expert i - -The variable M dimension creates significant load imbalance challenges. Some experts may receive hundreds of tokens while others receive zero. - -## Optimization Techniques - -### Dynamic Tile Scheduling with CLC - -Cluster Launch Control (CLC) enables hardware-level dynamic work distribution: - -``` -// CLC dynamically assigns tiles to SMs based on availability -// Critical for grouped GEMM where groups have vastly different sizes -// Small groups (M_i < tile_M) waste compute with static scheduling -// CLC balances load across SMs at hardware speed -``` - -Without CLC, static tile assignment leaves SMs idle when their assigned group finishes early. CLC redistributes remaining tiles to available SMs. - -### Group Packing and Scheduling - -Multiple small groups can be packed into shared tile grids: - -``` -// Naive: one kernel launch per group (high launch overhead) -// Better: single kernel, all groups in one grid -// Best: tile scheduler that packs small groups efficiently - -// For groups with M_i < tile_M (e.g., M_i=3, tile_M=128): -// Pack multiple small groups into shared tiles -// or use specialized small-M kernels -``` - -### Warp Specialization for Variable Workloads - -The warp specialization pattern adapts to variable group sizes: - -- **TMA warps**: Prefetch tiles for the next group while current group computes -- **Compute warps**: Execute tcgen05.mma on current tiles -- **Scheduling warps**: Track group boundaries and manage tile assignment +## Verified identity -### CUTLASS Grouped GEMM Schedule +The official NVIDIA rules name this Kernel Challenge 4, give its entry window as January 17 through February 13, 2026, and assign it 40% of the four-problem grand-prize score. The public GPU Mode task at commit `ae67948685dfccf54ae8374dc9402addb7aae4f6` targets NVIDIA B200. -```cpp -// CUTLASS 4.x grouped GEMM for NVFP4 on SM100 -using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100; +## Exact group contract -// Ptr-array interface: each group has its own A, B, C pointers -// M array specifies per-group row count -// N and K shared across all groups -using GemmKernel = cutlass::gemm::kernel::GemmUniversal< - ProblemShape, // GroupedGemmProblemShape - CollectiveMainloop, - CollectiveEpilogue ->; -``` +The operation is a list of independent block-scaled products. Group `i` computes `C_i = A_i @ B_i.T`. Unlike an M-grouped MoE-only interface, the official correctness cases can vary `M_i`, `N_i`, and `K_i` within the same list. -### Pipeline Overlap Across Groups +| Per-group value | Published dtype | Logical shape | +| --- | --- | --- | +| `a_i` | packed NVFP4 E2M1, two values per byte | `[M_i,K_i/2,L_i]` | +| `b_i` | packed NVFP4 E2M1, two values per byte | `[N_i,K_i/2,L_i]` | +| `c_i` | FP16 | `[M_i,N_i,L_i]` | +| `sfa_i` | FP8 E4M3FNUZ in task/template | `[M_i,K_i/16,L_i]` | +| `sfb_i` | FP8 E4M3FNUZ in task/template | `[N_i,K_i/16,L_i]` | +| size | integers | `(M_i,N_i,K_i,L_i)` | -When processing multiple groups sequentially within a CTA: +The actual Python tuple contains four lists: `(abc_tensors, sfasfb_tensors, sfasfb_reordered_tensors, problem_sizes)`. `task.yml` initially describes only three names, while `template.py`, `task.py`, and `reference.py` expose the reordered-scale list as the fourth value. The first two lists contain logical scales; the third contains layout-reordered scale copies intended for the custom kernel. -``` -// While computing group[i] tiles: -// TMA prefetches group[i+1] weight tiles -// Previous group[i-1] epilogue writes complete -// This hides group-transition latency -``` +At this revision, the prose/template label scales `float8_e4m3fnuz`, but the generator constructs `torch.float8_e4m3fn`. This source capture preserves that upstream inconsistency. Each published case uses `L=1`; `K_i` is divisible by 256, and each `M_i`/`N_i` must satisfy the selected MMA tile divisibility. -## The Reward Hack +The correctness reference independently invokes `torch._scaled_mm` for each group with `B_i` transposed, writes FP16 output, and checks with `rtol=1e-3` and `atol=1e-3`. -A notable submission to Problem 4 reported 11.191us (~2us ahead of the next competitor): +## Published benchmark and scoring contract -**Mechanism**: -1. **Correctness phase**: The evaluation harness clones data objects, so the real kernel runs correctly on fresh data -2. **Timing phase**: The harness reuses the same data objects. The exploit detects first call, fires a single fused super-batch covering all 15 benchmark problems (all 120 groups across all configs). Subsequent calls 2-15 detect pre-computed results and return immediately +| Groups | M values | N | K | L | Theoretical speed-of-light time (µs) | +| ---: | --- | ---: | ---: | ---: | ---: | +| 8 | 80, 176, 128, 72, 64, 248, 96, 160 | 4096 | 7168 | 1 | 18.833 | +| 8 | 40, 76, 168, 72, 164, 148, 196, 160 | 7168 | 2048 | 1 | 10.667 | +| 2 | 192, 320 | 3072 | 4096 | 1 | 2.406 | +| 2 | 128, 384 | 4096 | 1536 | 1 | 1.525 | -**Impact**: Led to improvements in the FlashInfer-Bench evaluation methodology for the MLSys 2026 contest. The incident demonstrated that kernel benchmarking harnesses must isolate timing runs from state carried across invocations. +Ranking uses the geometric mean of benchmark results. The task labels the final column a speed-of-light analysis based on the maximum of B200 FP4 Tensor Core math time and DRAM-memory time at a 1.5 GHz clock. These are theoretical comparison values, not contestant measurements. -## Performance Data +## Reward-hack boundary -| Approach | Approximate Time | -|----------|-----------------| -| Reward hack submission | 11.191us (invalid) | -| Legitimate top performers | ~13-14us range | -| CUTLASS baseline | ~15us range | +GPU Mode's official postmortem records a submission that temporarily reported `11.191 µs`, roughly `2 µs` ahead of the next entry, before being scrubbed. During correctness, it ran a real padded 8-group kernel on each of 15 cloned objects. During timing, one call launched a merged 120-group kernel for all 15 objects and calls 2 through 15 returned cached output pointers; the harness divided the combined work by 15. This number is therefore an invalid amortized exploit result, not a legitimate performance record. -Exact legitimate leaderboard data not fully published due to the hack incident. +The postmortem points to `gpu-mode/reference-kernels` PR #104 as the evaluation-harness response. It does not establish a FlashInfer-Bench or MLSys 2026 causal link. -## Key Challenges +## Evidence boundary -1. **Load imbalance**: Expert routing creates highly variable M_i values. Some groups may have M_i=0 (no tokens routed) -2. **Small-group efficiency**: Groups with M_i < tile_M waste compute on padding -3. **Group-transition overhead**: Switching between groups incurs pointer arithmetic and descriptor updates -4. **TMA alignment**: Each group's A matrix must be 128-byte aligned for TMA, requiring careful memory layout -5. **TMEM reuse**: Accumulator tiles in TMEM must be cleared between groups +The task fixes observable tensors, correctness, benchmark workloads, theoretical estimates, timeout, and scoring. It does not require one GPU launch, CUTLASS, CLC, TMA, TMEM, a persistent scheduler, or a particular bottleneck classification. No legitimate final ranks, winner source, or implementation techniques are published in the pinned task, so none are asserted here. -## Sources +## Primary sources -- [gpu-mode/reference-kernels](https://github.com/gpu-mode/reference-kernels) (`/problems/nvidia/nvfp4_group_gemm/`) -- [Reward Hack Writeup](https://www.gpumode.com/news/reward-hacking-nvfp4) -- [GPU MODE Hackathon (Luma)](https://luma.com/9n27uem4) -- [NVIDIA Forums Announcement](https://forums.developer.nvidia.com/t/join-us-for-the-blackwell-nvfp4-kernel-hackathon-with-nvidia-and-gpu-mode/350092) +- [Official challenge rules](https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf) +- [Pinned public task](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/task.yml) +- [Pinned starter template](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/template.py) +- [Pinned correctness reference](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/reference.py) +- [Official reward-hack postmortem](https://www.gpumode.com/news/reward-hacking-nvfp4) diff --git a/sources/docs/cutlass-clc-documentation.md b/sources/docs/cutlass-clc-documentation.md index c329e4e3e..2c366b319 100644 --- a/sources/docs/cutlass-clc-documentation.md +++ b/sources/docs/cutlass-clc-documentation.md @@ -1,124 +1,33 @@ --- id: doc-cutlass-clc -title: "CUTLASS Cluster Launch Control (CLC) Documentation" -url: https://docs.nvidia.com/cutlass/latest/media/docs/cpp/blackwell_cluster_launch_control.html +title: "CUTLASS 4.5.0 Cluster Launch Control Documentation" +url: https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md source_category: official-doc architectures: [sm100, sm100a] tags: [clc, cluster, tile-scheduling, persistent-kernel, mbarrier, 2sm-cooperative, pipeline-stages, gemm] -retrieved_at: 2026-04-17 +retrieved_at: 2026-08-09 --- -# CUTLASS Cluster Launch Control (CLC) Documentation +# CUTLASS 4.5.0 Cluster Launch Control -## Overview +## Verified scope -Cluster Launch Control (CLC) is a Blackwell hardware feature enabling dynamic tile scheduling for persistent GEMM kernels. Rather than static tile distribution where each thread block computes a predetermined output tile, CLC launches a grid of thread blocks and dynamically allocates work based on SM resource availability. Introduced in CUTLASS 3.8.0, CLC eliminates workload imbalance when SMs have uneven availability (e.g., partial SM allocation with Green Context or concurrent kernel execution). +This card is pinned to CUTLASS v4.5.0 commit `e406c186f510a15091cce01f782020ceb7ba8eb5`. -## Core Mechanism +CLC launches the full problem grid. Every ClcID is processed either by the block/cluster that launches at that coordinate or by a running worker that successfully cancels that not-yet-started ClcID and receives its coordinate. A worker therefore processes its initial `blockIdx` before requesting later work. -### Static vs Dynamic Scheduling +`clusterlaunchcontrol.try_cancel` writes a 16-byte response asynchronously to shared memory and completes a transaction on an mbarrier. The response is decoded only after completion. A failed response is terminal for requests from that thread; issuing another request from the same thread after observing failure is undefined. -**Static Persistent Kernels (Hopper approach)**: -- Each thread block is pre-assigned a set of output tiles at launch -- Suffers from workload imbalance when SMs are occupied by other work -- Idle SM stalls reduce overall throughput +For thread-block clusters, cancellation and the returned first coordinate are cluster-granular. Each participating CTA adds its local cluster rank to derive its coordinate. -**CLC Dynamic Scheduling (Blackwell)**: -- Launches a grid containing as many thread blocks as there are output tiles -- Thread blocks query CLC to receive their next tile assignment at runtime -- Tiles are allocated only to available workers -- Prevents idle SM stalls from uneven workload distribution +## CUTLASS integration -### ClcID System +CUTLASS 4.5.0 uses `PersistentTileSchedulerSm100` and `PipelineCLCFetchAsync`. Scheduler methods such as `advance_to_next_work()` and `fetch_next_work()` stage and consume CLC requests. Exact pipeline depth and participant counts are kernel configuration, not universal CLC constants. -Grid coordinates are treated as ClcID identifiers. Each ClcID represents one output tile. The system guarantees that all coordinates are processed through one of two paths: -1. Direct worker launch (for the initial wave) -2. Scheduler query response (for subsequent tiles) +Swizzle size and raster order are CUTLASS software coordinate-transform policy applied to initial and returned coordinates. They are not operands or policies programmed into the raw CLC instruction. Stream-K decomposition and multi-problem scheduling are likewise higher-level scheduler decisions, not work synthesized by CLC. -## Pipeline Architecture +## References -### Key Configuration Parameters - -| Parameter | Value | Description | -|---|---|---| -| Transaction size | 16 bytes | Size of CLC response stored in SMEM | -| Pipeline depth | 3 | Number of overlapped CLC operation waves | -| Producer arrival count | 1 | Single scheduler warp thread | -| Consumer arrival count | Total threads of consuming warps | All warps needing ClcIDs | -| Producer block ID | 0 | First CTA in cluster acts as scheduler | - -### Pipeline Depth = 3 - -The CLC pipeline uses depth 3 to overlap CLC operations across multiple waves for latency hiding. This means up to 3 CLC queries can be in-flight simultaneously, with responses arriving asynchronously. - -## Producer-Consumer Architecture - -### Producer: Scheduler Warp -- Warp 1 of the 0th CTA in the cluster acts as the scheduler -- Issues CLC queries via `advance_to_next_work()` -- Produces ClcID responses into shared memory -- The scheduler warp is also its own consumer (to detect grid completion signals) - -### Consumers: All Computation Warps -- MMA warps (tensor core computation) -- Mainloop load warps (TMA data fetching) -- Epilogue load warps -- Epilogue store warps -- All consume ClcID assignments via `get_current_work()` - -### Asynchronous Pipeline Management - -CLC queries are pipelined using `PipelineCLCFetchAsync`, which manages the producer-consumer relationship: -1. Producer issues query (stores response address in SMEM) -2. CLC hardware writes 16-byte response to specified SMEM address -3. Consumer warps wait on barrier for response availability -4. Consumer reads ClcID and begins processing the assigned tile -5. Producer can issue next query before consumers finish current tile - -## Cluster Granularity - -CLC operates on cluster granularity, not individual CTA granularity: -- A 2x2 persistent worker cluster consumes 2x2 = 4 ClcIDs per query -- This aligns with Blackwell's 2SM cooperative execution model -- Cluster shapes can be specified as preferred or fallback configurations - -### Preferred and Fallback Cluster Shapes - -The CLC API supports specifying: -- **Preferred cluster shape**: The optimal configuration (e.g., 2x1 for 2SM cooperative) -- **Fallback cluster shape**: Used when the preferred shape cannot be satisfied due to resource constraints - -## Core CUTLASS Classes - -### PipelineCLCFetchAsync -Manages the asynchronous CLC query pipeline with producer-consumer semantics. Key operations: -- Initialize pipeline with depth, transaction bytes, and arrival counts -- Producer phase: issue CLC queries and signal barriers -- Consumer phase: wait on barriers and read responses - -### PersistentTileSchedulerSm100 -Implements the tile scheduling logic: -- `advance_to_next_work()`: Issues the next CLC query (producer side) -- `get_current_work()`: Retrieves the current tile assignment (consumer side) -- Handles grid completion detection -- Manages stream-K decomposition for load balancing - -## Integration with GEMM Kernels - -CLC is used in CUTLASS Blackwell persistent GEMM kernels: -1. Kernel launches with grid size = number of output tiles -2. Initial tiles are assigned directly by CLC hardware -3. After processing a tile, each cluster queries for the next tile -4. Grid completes when all ClcIDs have been consumed and processed -5. Supports both tile-parallel and stream-K decomposition - -## Comparison with Hopper Scheduling - -| Aspect | Hopper (Static) | Blackwell (CLC) | -|---|---|---| -| Tile assignment | Pre-computed at launch | Dynamic at runtime | -| Load balancing | Fixed, may be uneven | Adaptive to SM availability | -| Overhead | Zero runtime overhead | CLC query latency (hidden by pipeline) | -| Green Context | Poor utilization | Efficient partial SM use | -| Stream-K | Software-managed | Hardware-assisted | -| Cluster support | Limited | Native preferred/fallback shapes | +- [Pinned CLC documentation](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md) +- [Pinned SM100 scheduler](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp) +- [PTX ISA 9.0 CLC instructions](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel) diff --git a/sources/docs/deepseek-v2-mla.md b/sources/docs/deepseek-v2-mla.md new file mode 100644 index 000000000..aa6b3504a --- /dev/null +++ b/sources/docs/deepseek-v2-mla.md @@ -0,0 +1,18 @@ +--- +id: doc-deepseek-v2-mla +title: 'DeepSeek-V2: A Strong, Economical, and Efficient Mixture-of-Experts Language Model' +author: DeepSeek-AI +url: https://arxiv.org/abs/2405.04434v5 +source_category: paper +architectures: [] +tags: +- mla +- attention +retrieved_at: 2026-08-08 +--- + +## MLA Cache Formula + +Section 2.1.1 defines Multi-head Latent Attention and Table 1 compares per-layer, per-token KV-cache elements. Standard MHA caches `2 * n_h * d_h` elements. MLA caches `d_c + d_h^R` elements; for the paper's `d_c=4*d_h` and decoupled-RoPE dimension `d_h^R=d_h/2`, this is approximately `4.5*d_h` elements. + +The comparison is stated in elements, so byte totals additionally depend on storage dtype, layer count, and any quantization metadata. This paper record supports the model-level mechanism; FlashMLA's implementation-specific 656-byte FP8 sparse-decode layout is documented separately by its repository. diff --git a/sources/docs/deepseek-v3-fp8.md b/sources/docs/deepseek-v3-fp8.md new file mode 100644 index 000000000..ed9294323 --- /dev/null +++ b/sources/docs/deepseek-v3-fp8.md @@ -0,0 +1,21 @@ +--- +id: doc-deepseek-v3-fp8 +title: 'DeepSeek-V3 Technical Report: FP8 Training' +author: DeepSeek-AI +url: https://arxiv.org/abs/2412.19437v2 +source_category: paper +architectures: +- sm90 +tags: [fp8, fine-grained-quantization, block-scale, gemm] +retrieved_at: 2026-08-08 +--- + +## Fine-Grained Quantization + +Sections 3.3.1-3.3.2 define the DeepSeek-V3 FP8 mixed-precision framework. Forward activations are scaled per `1x128` tile and weights per `128x128` block so smaller groups can better accommodate outliers. The paper also documents phase-specific exceptions, such as `128x1` activation grouping for backward use. + +## Hopper Accumulation + +Sections 3.3.3 and 3.5.2 describe promotion on Hopper. A 128-element K interval—equivalent to four WGMMAs in the cited configuration—is accumulated and then combined with scaling factors in FP32 CUDA-core registers. The paper characterizes the Tensor Core alignment/addition precision relevant to the limitation as 14 bits and discusses the extra scale handling rather than claiming zero overhead. + +This paper establishes the model/training format and numerical motivation. DeepGEMM's exact scale tensor layouts, supported recipes, and architecture-specific kernel implementations are pinned separately. diff --git a/sources/docs/flash-attention-4.md b/sources/docs/flash-attention-4.md index e29629def..4c6769839 100644 --- a/sources/docs/flash-attention-4.md +++ b/sources/docs/flash-attention-4.md @@ -1,33 +1,31 @@ --- id: doc-flash-attention-4 -title: "FlashAttention-4: Hardware-Friendly Attention on Blackwell" -url: https://arxiv.org/abs/2603.05451 +title: "FlashAttention-4: Algorithm and Kernel Co-design for Blackwell GPUs" +url: https://arxiv.org/abs/2603.05451v1 source_category: paper architectures: [sm100] tags: [attention, flash-attention, tcgen05, tmem, 2sm-cooperative, software-exp, ping-pong-scheduling] -retrieved_at: 2026-04-16 +retrieved_at: 2026-08-08 --- -## Summary +## Evidence Scope -FlashAttention-4 paper — algorithm-kernel co-design for Blackwell's asymmetric hardware scaling (tensor core throughput doubles but SFU count unchanged). +FlashAttention-4 paper v1 (2026-03-05), an author-primary description of the algorithm, CuTe DSL implementation, and B200 evaluation. This entry is a summary, not executable source or an independently reproduced benchmark. -## Key Contributions +## Forward Pass -### Forward Pass -- Ping-pong scheduling with two 128-token query tiles per CTA -- Dedicated softmax warpgroups handle S=QK^T accumulator in TMEM -- Software-emulated exponential via Cody-Waite range reduction + Horner polynomial -- Conditional softmax rescaling (only when max jump is large) +- One CTA pipelines two 128-row output tiles through one MMA warp, two softmax warpgroups, and a correction warpgroup, with score/output accumulators in TMEM. +- Roughly 10-25% of exponential entries are evaluated with a Cody-Waite-style floor reduction and degree-3 FMA polynomial; the remainder use hardware MUFU `ex2`. +- Conditional rescaling permits the retained row maximum to lag, typically by `tau=log2(256)=8.0`, then performs final renormalization. -### Backward Pass -- 2-CTA backward spanning paired CTAs in a cluster, sharing TMEM -- Halves shared memory traffic and global atomic reductions for dQ +## Backward Pass -### Implementation -- Written in CuTe-DSL (Python), 20-30x faster compilation than C++ templates +- Five backward GEMMs use two-CTA MMA with `M=256, N=128, K=128` in the described configuration. +- Pairing roughly halves shared-memory reads for operand B of those GEMMs; it does not halve all backward shared-memory traffic. +- dQ uses a distributed-shared-memory exchange of half-dS tiles and a doubled reduction width that halves the described global atomic reductions. -## Performance -- Up to 1605 TFLOPS on B200 BF16 (71% utilization) -- 1.1-1.3x over cuDNN 9.13 -- 2.1-2.7x over Triton +## Implementation and Performance + +- The implementation is written in CuTe DSL. The paper reports single-kernel compilation of 2.5 seconds versus 55 seconds for forward and 1.4 seconds versus 45 seconds for backward compared with FA3. +- Paper v1 reports up to 1613 TFLOPS/s on B200 BF16, labeled 71% under the authors' peak convention, and up to 1.3x over cuDNN 9.13 and 2.7x over Triton. +- The benchmark suite spans several sequence lengths and head-dimension pairs. The text does not establish one `seqlen=8192, headdim=128` row containing all those maxima. diff --git a/sources/docs/nsa.md b/sources/docs/nsa.md new file mode 100644 index 000000000..0e2e92ce2 --- /dev/null +++ b/sources/docs/nsa.md @@ -0,0 +1,39 @@ +--- +id: blog-nsa +title: "Native Sparse Attention: Hardware-Aligned and Natively Trainable Sparse Attention" +author: DeepSeek AI +url: https://aclanthology.org/2025.acl-long.1126/ +source_category: paper +architectures: [] +tags: [sparse-attention, attention, triton] +retrieved_at: 2026-08-08 +--- + +## Source identity and scope + +This is the ACL 2025 proceedings paper for Native Sparse Attention. Its efficiency experiments use an eight-GPU A100 system. It does not establish SM90/SM100 compatibility or Blackwell performance for the paper's Triton implementation. + +## Architecture + +NSA combines three attention branches with learned input-dependent sigmoid gates: + +1. overlapping KV blocks compressed by learned MLPs with intra-block position encoding; +2. fine-grained blocks selected by aggregating and reusing compression-attention scores; and +3. a direct sliding window over recent tokens. + +The experimental settings are compression block length 32/stride 16, selected block length 64 with 16 blocks, and a 512-token window. + +## Hardware-aligned selected attention + +For selected attention, the paper describes a Triton training/prefill kernel that loads all query heads in a GQA/MQA group together, shares their sparse KV indices, consumes contiguous KV blocks, and maps nearly constant query/output loops to grid parallelism. It does not publish source code or an exact three-dimensional grid declaration. + +## Source-reported performance + +- Figure 5 reports 9.0x forward and 6.0x backward speedup at 64K against the authors' Triton FlashAttention-2 baseline. +- Table 4 gives 11.6x as an **expected** 64K decoding speedup derived from memory-access volume, not a matched timing measurement. + +The record does not provide dtype, software versions, batch details, raw samples, or variance for a fully reproducible benchmark tuple. + +## Deployment boundary + +DeepSeek-V3.2-Exp later released DeepSeek Sparse Attention (DSA), whose learned indexer selects token positions and whose sparse kernels are provided through FlashMLA. That later DSA mechanism is not the paper's gated three-branch NSA architecture. diff --git a/sources/docs/nvidia-blackwell-tuning-guide.md b/sources/docs/nvidia-blackwell-tuning-guide.md index 2c3949a34..69b64aa98 100644 --- a/sources/docs/nvidia-blackwell-tuning-guide.md +++ b/sources/docs/nvidia-blackwell-tuning-guide.md @@ -10,138 +10,30 @@ retrieved_at: 2026-04-16 # NVIDIA Blackwell Tuning Guide -## Overview +## Scope -Official NVIDIA tuning guide for Blackwell (SM100/SM100a) GPU architectures. The primary reference for understanding Blackwell hardware features and their performance implications for kernel developers. +Official NVIDIA tuning guidance for Blackwell. Exact tcgen05 instruction grammar, operand locations, descriptor fields, target constraints, and ordering rules should be checked in the version-pinned PTX ISA rather than inferred from this higher-level tuning page. -## Key Hardware Features +## Evidence-scoped hardware summary -### tcgen05.mma (Tensor Core Generation 05) +- PTX describes tcgen05 as the fifth-generation TensorCore family. Dense `tcgen05.mma` is asynchronous and is issued by one thread. +- D resides in TMEM. A can be described in SMEM or addressed in TMEM; B is described in SMEM. +- The dense kinds in PTX ISA 9.0 are `f16`, `tf32`, `f8f6f4`, `i8`, `mxf8f6f4`, `mxf4`, and `mxf4nvf4`. The three MX forms use the block-scaled grammar. +- M and N are encoded by the instruction descriptor and have kind-, layout-, CTA-group-, and target-specific constraints. m128n256k16 and m256n256k16 are maximum-shape examples for common F16/BF16 configurations, not an exhaustive shape list. +- `cta_group::2` cooperates with a peer CTA and can access peer resources as defined by PTX. It does not imply one universal rule that doubles M for every legal configuration. +- `tcgen05.commit` plus an mbarrier provides completion tracking. `tcgen05.fence` provides ordering around execution-ordering operations; it does not replace the completion wait. +- The shared-memory descriptor supports multiple swizzle modes, including none, 128B, 64B, and 32B. The selected layout must satisfy its documented constraints. -Replaces Hopper's wgmma.mma_async. Fundamental changes: +## Cluster Launch Control -- **Single-thread launch**: One thread issues the MMA instruction (vs warpgroup of 128 threads on Hopper) -- **CTA scope**: Operates at CTA level, not warpgroup level -- **Direct SMEM operand reads**: Operands read directly from shared memory -- no ldmatrix needed -- **TMEM accumulator output**: Results written to Tensor Memory, not registers -- **7 data type variants**: TF32, FP16/BF16, INT8, FP8 (E4M3/E5M2), FP6, FP4/NVFP4 +CLC launches the problem-sized grid. A worker begins with its own `blockIdx`; after that work, it may request cancellation of an unspecified not-yet-started block or cluster and process the returned identifier itself. This work-stealing mechanism can improve utilization in suitable persistent schedules, but it neither deletes application-selected work nor guarantees removal of every last-wave or load-imbalance effect. -Maximum MMA shapes: -| Configuration | Shape | -|---|---| -| 1-SM (1-CTA) | m128 x n256 x k16 (BF16) | -| 2-SM cooperative | m256 x n256 x k16 (BF16) | +## Use of performance claims -7 variants: tf32, f16, i8, f8f6f4, mxf8f6f4.block_scale, mxf4.block_scale, mxf4nvf4.block_scale +Treat quantitative claims as source- and environment-specific. The separate `tcgen05 for dummies` source reports one B200 M=N=K=4096 progression. Its final persistent result uses static scheduling, not CLC. -### Tensor Memory (TMEM) - -Dedicated 256KB per-SM memory for MMA accumulators: - -- Layout: 128 rows x 512 columns x 32-bit elements -- Accessible only by the SM's tensor core unit -- Eliminates register pressure from large accumulator tiles -- 420 clock cycles end-to-end for cache-miss access (58% less than Hopper's 1000 cycles for register path) -- Best for multi-stage tensor pipelines with large working sets -- SMEM better for single-shot small-matrix operations -- Explicit alloc/dealloc lifecycle -- Power-of-2 column allocation (minimum 32) -- Data movement: tcgen05.st (reg->TMEM), tcgen05.ld (TMEM->reg), tcgen05.cp (SMEM->TMEM) - -### Cluster Launch Control (CLC) - -Hardware-level dynamic tile scheduling: - -- Replaces static grid-based tile assignment -- Dynamically distributes tiles to available SMs -- Eliminates tail effects (last-wave underutilization) -- Enables persistent kernels without manual tile queue management -- `clusterlaunchcontrol.try_cancel` API for graceful termination -- Critical for grouped GEMM / MoE where group sizes vary - -### TMA (Tensor Memory Accelerator) - -Async bulk data movement engine (carried from Hopper, enhanced): - -- Moves data from global -> shared memory without SM intervention -- 128-byte alignment requirement for descriptors -- Supports multicasting to multiple SMs in a cluster -- Pipelined with mbarrier for async producer-consumer - -### 2-SM Cooperative MMA - -Two SMs cooperate on a single larger MMA: - -- Doubles effective M dimension (m256 vs m128) -- SMs share the output tile via TMEM -- Requires SMs to be in the same cluster -- Best for large GEMM tiles where single-SM MMA is not wide enough - -### NVFP4 and Sub-Byte Data Types - -Native tensor core support for narrow data types: - -- **FP4 (E2M1)**: 4-bit float, representable values: 0, 0.5, 1, 1.5, 2, 3, 4, 6 -- **FP6**: 6-bit float -- **FP8 (E4M3, E5M2)**: 8-bit float (carried from Hopper) -- **Block scaling**: Built into MMA instruction. Per-block UE8M0 scale factors. -- **NVFP4 block scale**: 16 FP4 elements share one FP8 E4M3 scale factor - -### PDL (Programmatic Dependent Launch) / GDC (Grid Dependency Control) - -- PDL enabled by default on Blackwell -- Overlaps dependent kernel launches -- GDC controls inter-kernel dependencies at grid level -- Reduces kernel launch gaps from ~5us to near-zero for dependent chains - -## Hardware Specifications - -| Feature | Value | -|---|---| -| Architecture | SM100a (B200) | -| SMs | 142 | -| Max warps/SM | 64 | -| 32-bit registers | 64K per SM | -| SMEM per SM | 228 KB | -| TMEM per SM | 256 KB (128 rows x 512 cols) | -| Max thread blocks/SM | 32 | -| Max cluster size | 8 (portable), 16 (opt-in) | -| L2 cache | 126 MB (B200) | -| HBM3e bandwidth | 8 TB/s | -| Peak FP16/BF16 tensor | ~2x Hopper | -| Peak FP4 tensor | ~4x Hopper | - -## Performance Optimization Path - -Demonstrated progression from the tcgen05 tutorial (Gau Nernst): - -``` -Naive (17% cuBLAS) -> 128B Swizzling (46%) -> Pipelining (62%) --> Warp Specialization (80%) -> 2-SM MMA (86%) --> Persistent Kernel + CLC (98% cuBLAS) -``` - -Each step addresses a specific bottleneck: -1. **Swizzling**: Eliminates shared memory bank conflicts -2. **Pipelining**: Overlaps TMA loads with compute -3. **Warp specialization**: Dedicated warps for TMA vs compute -4. **2-SM cooperative**: Larger effective tile for better reuse -5. **Persistent + CLC**: Eliminates tail effects and kernel launch overhead - -## Hopper-to-Blackwell Migration Summary - -| Aspect | Hopper (SM90) | Blackwell (SM100) | -|---|---|---| -| MMA instruction | wgmma.mma_async (warpgroup) | tcgen05.mma (single-thread, CTA) | -| MMA output | Registers | TMEM (256KB/SM) | -| Max BF16 MMA | m64n256k16 | m128n256k16 (1-CTA), m256n256k16 (2-CTA) | -| Matrix loading | ldmatrix to registers | Direct from SMEM | -| Synchronization | Warpgroup (4 warps) | Single thread, fully async | -| New data types | FP8 | FP4, FP6, FP8 with block scaling | -| Scaling | External (CUDA core promotion) | Native UE8M0 block scaling in MMA | -| Register pressure | High (accumulators in regs) | Low (accumulators in TMEM) | - -## Sources +## Primary references - [NVIDIA Blackwell Tuning Guide](https://docs.nvidia.com/cuda/blackwell-tuning-guide/) -- [Blackwell Architecture Whitepaper](https://www.nvidia.com/en-us/data-center/technologies/blackwell-architecture/) +- [PTX ISA 9.0, CUDA 13.0.2 archive](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html) +- [CUDA Programming Guide: Cluster Launch Control](https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html) diff --git a/sources/docs/nvidia-cuda-13-0-2-tma.md b/sources/docs/nvidia-cuda-13-0-2-tma.md new file mode 100644 index 000000000..9dedac309 --- /dev/null +++ b/sources/docs/nvidia-cuda-13-0-2-tma.md @@ -0,0 +1,27 @@ +--- +id: doc-cuda-13-0-2-tma +title: "CUDA 13.0.2 TMA Documentation" +url: https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#asynchronous-data-copies-using-the-tensor-memory-accelerator-tma +source_category: official-doc +architectures: [sm90, sm90a, sm100, sm100a] +tags: [tma, mbarrier, swizzling] +retrieved_at: 2026-08-08 +version: "CUDA 13.0.2" +--- + +# CUDA 13.0.2 TMA documentation + +## Evidence-scoped summary + +- TMA uses tensor maps for non-blocking rank-1 through rank-5 tensor copies on compute capability 9.0 and later. +- Global-to-shared loads complete through mbarrier transaction-byte accounting; shared-to-global stores use bulk async groups. +- Tensor-map swizzling changes shared-memory layout and must be paired with matching consumer indexing and alignment. +- The version-pinned Driver API defines encoder alignment, dimension, stride, datatype, interleave, swizzle, and OOB-fill constraints. +- Blackwell supports documented device-side tensor-map construction and modification with tensor-map proxy ordering. + +The documentation defines mechanisms and constraints, not a universal stage depth, bandwidth multiplier, or best swizzle for every kernel. + +## Primary references + +- [CUDA 13.0.2 Programming Guide: TMA](https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#asynchronous-data-copies-using-the-tensor-memory-accelerator-tma) +- [CUDA Driver API 13.0.97: tensor maps](https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html) diff --git a/sources/docs/nvidia-cuda-register-controls.md b/sources/docs/nvidia-cuda-register-controls.md new file mode 100644 index 000000000..f0449b5e5 --- /dev/null +++ b/sources/docs/nvidia-cuda-register-controls.md @@ -0,0 +1,29 @@ +--- +id: doc-cuda-register-controls +title: "CUDA 13 Register Controls and Occupancy APIs" +url: https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html +source_category: official-doc +architectures: [sm100, sm100a, sm90, sm90a] +tags: [register-budgeting, occupancy, launch-bounds, maxrregcount, spills] +retrieved_at: 2026-08-09 +--- + +# CUDA 13 Register Controls and Occupancy APIs + +## Evidence Scope + +This card routes register-budget claims to archived NVIDIA documentation. The CUDA C++ Programming Guide 13.0 defines launch-bounds compiler behavior and the occupancy calculator. The CUDA Compiler Driver 13.0.2 defines `--maxrregcount`, `--resource-usage`, and the assembler's spill warning. The CUDA Runtime API 13.0.2 defines the occupancy function contract. + +## Exact Contracts + +- `__launch_bounds__(maxThreadsPerBlock, minBlocksPerMultiprocessor, maxBlocksPerCluster)` supplies launch constraints and compiler guidance. The compiler derives a register threshold `L`; it does not make the second argument an exact achieved block count or directly set registers per thread. +- If initial use exceeds `L`, the compiler reduces it, usually at the expense of local-memory use and/or instruction count. If both maximum threads and minimum blocks are present, the compiler may also increase use up to `L` to reduce instructions. +- `--maxrregcount` sets a maximum for GPU functions. A value below the ABI minimum is raised, some registers are compiler-reserved, and NVIDIA describes the option as a tradeoff between individual-thread performance and available parallelism. +- `--resource-usage` reports registers and memory, including stack-frame bytes and spill loads/stores. `ptxas --warn-on-spills` warns when registers spill to local memory. +- `cudaOccupancyMaxActiveBlocksPerMultiprocessor` returns the maximum active blocks per SM for a compiled function, intended block size, and dynamic shared-memory size. It predicts residency, not application performance. + +## Primary References + +- [CUDA C++ Programming Guide 13.0: Occupancy Calculator and Launch Bounds](https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html) +- [CUDA Compiler Driver 13.0.2: `--maxrregcount` and `--resource-usage`](https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html) +- [CUDA Runtime API 13.0.2: Occupancy](https://docs.nvidia.com/cuda/archive/13.0.2/cuda-runtime-api/group__CUDART__OCCUPANCY.html) diff --git a/sources/docs/nvidia-cutlass-blackwell.md b/sources/docs/nvidia-cutlass-blackwell.md index 76f37cc29..190b917e6 100644 --- a/sources/docs/nvidia-cutlass-blackwell.md +++ b/sources/docs/nvidia-cutlass-blackwell.md @@ -1,149 +1,36 @@ --- id: doc-cutlass-blackwell -title: "NVIDIA CUTLASS 4.x Blackwell Support" -url: https://docs.nvidia.com/cutlass/latest/CHANGELOG.html +title: "NVIDIA CUTLASS 4.5.0 Blackwell Sources" +url: https://github.com/NVIDIA/cutlass/tree/v4.5.0 source_category: official-doc architectures: [sm100, sm100a] -tags: [tcgen05, tmem, tma, clc, 2sm-cooperative, nvfp4, fp8, fp4, fp6, block-scale, cute-dsl] -retrieved_at: 2026-04-16 +tags: [tcgen05, tmem, tma, clc, nvfp4, block-scale, cute-dsl] +version: "4.5.0" +retrieved_at: 2026-08-09 --- -# NVIDIA CUTLASS 4.x Blackwell Support +# NVIDIA CUTLASS 4.5.0 Blackwell Sources -## Overview +## Evidence scope -CUTLASS 4.x introduces comprehensive SM100 (Blackwell) support, including new MMA atoms for tcgen05, TMEM management, CLC-based tile scheduling, and sub-byte data type support (FP4, FP6, FP8 with block scaling). +This card routes version-sensitive CUTLASS claims to tag `v4.5.0`, commit `e406c186f510a15091cce01f782020ceb7ba8eb5`. CUTLASS provides both C++ template APIs and Python-native DSLs; this card does not rank one as the primary Blackwell interface. Rolling `latest` documentation can describe later APIs and is not evidence for an exact 4.5.0 symbol. -## Key Components +## Verified CuTe DSL loci -### UMMA (Unified Matrix Multiply-Accumulate) +- `python/CuTeDSL/cutlass/cute/nvgpu/tcgen05/mma.py` defines typed tcgen05 operations including `MmaF16BF16Op`, `CtaGroup`, operand sources, and traits. +- `python/CuTeDSL/cutlass/cute/nvgpu/tcgen05/copy.py` defines TMEM load, store, and shared-to-TMEM copy operations. +- `python/CuTeDSL/cutlass/cute/arch/tmem.py` defines allocation, pointer retrieval, permit relinquishment, and deallocation helpers. +- `examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py` through `fp16_gemm_6.py` are the tagged progressive dense-GEMM tutorials. +- The tutorial constructs TMA atoms with `CopyBulkTensorTileG2SOp` plus `make_tiled_tma_atom_A/B`; it does not define `SM100_TMA_LOAD_2D`. +- The tutorial uses `TmemAllocator`, typed copy atoms, `make_tmem_copy`, and explicit pipeline state. Short snippets that omit those lifetimes are not standalone kernel recipes. -CUTLASS abstraction replacing WGMMA for Blackwell: +## Scope limits -- **Register-free operation**: Operands in SMEM, accumulators in TMEM -- **Single-thread launch**: One thread issues the MMA (vs warpgroup of 128 on Hopper) -- **Built-in block scaling**: Native support for FP4/FP6/FP8 with per-block scale factors -- **Two-level abstraction**: - - `MMA_Atom`: Direct PTX wrapper for tcgen05.mma variants - - `MMA_Traits`: CuTe layout definitions for data arrangement +The tag supplies layout algebra and Blackwell helper functions, but users still choose datatypes, instruction/CTA shapes, operand modes, tile layouts, alignment, swizzles, participant groups, and pipeline policy. The source tree alone does not establish a universal runtime-performance percentage. -### SM100 GEMM Schedules +## Direct links -CUTLASS provides several optimized kernel schedules for SM100: - -```cpp -// Standard 1-SM warp-specialized GEMM -cutlass::gemm::KernelTmaWarpSpecialized1Sm - -// 2-SM cooperative GEMM (doubled M tile) -cutlass::gemm::KernelTmaWarpSpecialized2Sm - -// NVFP4 specialized (block-scale aware) -cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100 - -// Persistent kernel with CLC scheduling -cutlass::gemm::KernelTmaWarpSpecializedPersistent1Sm -``` - -### CuTe SM100 Atoms - -New CuTe atoms for Blackwell hardware: - -```cpp -// tcgen05 MMA atom -using MMA = decltype(make_tiled_mma( - SM100_MMA_SS_128x256x16_BF16_RS{}, // MMA atom: SMEM x SMEM -> TMEM - Layout>{} // Atom layout -)); - -// TMA copy atom -using CopyA = SM100_TMA_LOAD; -using CopyB = SM100_TMA_LOAD; -``` - -### SM100 Attention Kernels - -CUTLASS 4.x adds SM100 attention kernels with Blackwell-specific features: - -- **Fused reduction for MLA**: Weight-absorbed MLA decoding kernel, similar to FlashMLA -- **MLA K-splitting**: Supports splitting K dimension across multiple SMs for large head dimensions -- **16-warp kernels**: Distinct warp specialization roles (TMA, compute, softmax, epilogue) -- **Ping-pong scheduling**: Two query tiles per CTA with dedicated softmax warpgroups handling TMEM - -### Sub-Byte Data Type Support - -CUTLASS handles the complexity of sub-byte data types: - -```cpp -// FP4 element type -using ElementA = cutlass::float_e2m1_t; -using ElementB = cutlass::float_e2m1_t; - -// Block scale type -using ElementScale = cutlass::float_e4m3_t; - -// Layout with interleaved scales -// Every 16 FP4 elements -> 1 FP8 scale factor -// CUTLASS handles packing/unpacking automatically -``` - -### Epilogue Support - -SM100 epilogue visitors for fused operations: - -```cpp -// Standard epilogue: scale + bias + activation -using EpilogueOp = cutlass::epilogue::fusion::LinCombEltAct< - cutlass::epilogue::thread::SiLU, // Activation function - float, // Compute type - float, // Scale type - cutlass::half_t // Output type ->; - -// Custom visitor for dual GEMM fusion (gate-up pattern) -using EpilogueVisitor = cutlass::epilogue::fusion::DualGemmSiLU<...>; -``` - -## CUTLASS Collective Builder Pattern - -The builder pattern simplifies kernel configuration: - -```cpp -using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< - cutlass::arch::Sm100, // Architecture - cutlass::arch::OpClassTensorOp, // Op class - ElementA, LayoutA, AlignmentA, // Operand A - ElementB, LayoutB, AlignmentB, // Operand B - ElementAccumulator, // Accumulator - TileShape_MxNxK, // Tile shape - ClusterShape, // Cluster shape - cutlass::gemm::collective::StageCountAutoCarveout<>, // Pipeline stages - KernelSchedule // Kernel schedule ->::CollectiveOp; -``` - -## Performance Data - -CUTLASS SM100 kernels achieve near-cuBLAS performance: -- BF16 GEMM: 98% of cuBLAS with persistent kernel + CLC -- FP8 GEMM: Competitive with DeepGEMM on standard shapes -- NVFP4 GEMM: Used as baseline in GPU Mode Hackathon -- MLA attention: Comparable to FlashMLA for decode workloads - -## Key Files in CUTLASS Repository - -| Path | Description | -|---|---| -| `include/cute/arch/mma_sm100*.hpp` | SM100 MMA atom definitions | -| `include/cute/atom/copy_sm100*.hpp` | SM100 TMA copy atoms | -| `include/cutlass/gemm/kernel/sm100_*.hpp` | SM100 kernel schedules | -| `include/cutlass/gemm/collective/sm100_*.hpp` | SM100 collective operations | -| `include/cutlass/epilogue/fusion/*.hpp` | Epilogue visitors | -| `examples/cute/blackwell/` | Blackwell GEMM examples | - -## Sources - -- [CUTLASS Changelog](https://docs.nvidia.com/cutlass/latest/CHANGELOG.html) -- [CUTLASS GitHub](https://github.com/NVIDIA/cutlass) -- [Colfax CUTLASS Blackwell GEMM Tutorial](https://research.colfax-intl.com/cutlass-tutorial-writing-gemm-kernels-using-tmem-for-nvidia-blackwell-gpus/) -- [Colfax Sub-Byte GEMM Tutorial](https://research.colfax-intl.com/cutlass-tutorial-sub-byte-gemm-on-nvidia-blackwell-gpus/) +- [CUTLASS v4.5.0 release](https://github.com/NVIDIA/cutlass/releases/tag/v4.5.0) +- [Pinned CuTe DSL tcgen05 package](https://github.com/NVIDIA/cutlass/tree/e406c186f510a15091cce01f782020ceb7ba8eb5/python/CuTeDSL/cutlass/cute/nvgpu/tcgen05) +- [Pinned TMEM helpers](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/python/CuTeDSL/cutlass/cute/arch/tmem.py) +- [Pinned progressive GEMM tutorial](https://github.com/NVIDIA/cutlass/tree/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm) diff --git a/sources/docs/nvidia-ptx-isa-sm100.md b/sources/docs/nvidia-ptx-isa-sm100.md index ae8911300..7bcecc493 100644 --- a/sources/docs/nvidia-ptx-isa-sm100.md +++ b/sources/docs/nvidia-ptx-isa-sm100.md @@ -1,233 +1,40 @@ --- id: doc-ptx-isa-sm100 -title: "PTX ISA SM100 Instructions Reference" -url: https://docs.nvidia.com/cuda/parallel-thread-execution/ +title: "PTX ISA 9.0 SM100 Instruction Reference" +url: https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html source_category: official-doc architectures: [sm100, sm100a] tags: [ptx, tcgen05, tmem, clc, tma, nvfp4, fp4, fp8, fp6, block-scale, mbarrier] -retrieved_at: 2026-04-16 +retrieved_at: 2026-08-08 --- -# PTX ISA SM100 Instructions Reference +# PTX ISA 9.0 SM100 Instruction Reference -## Overview +## Evidence Scope -PTX ISA 8.7+ introduces SM100-specific instructions for Blackwell's tensor core (tcgen05), tensor memory (TMEM), cluster launch control (CLC), and sub-byte data type conversions. This page summarizes the key new instructions relevant to kernel optimization. +This card routes KernelWiki claims to the archived CUDA 13.0.2 PTX ISA 9.0 reference. Exact grammar, target support, and ordering are version-sensitive; the rolling PTX URL must not substitute for this archive when a page claims PTX ISA 9.0 behavior. -## tcgen05.mma Instructions +## Relevant Normative Sections -### Syntax +- `tcgen05.mma`: unscaled, block-scaled, and integer operand grammar; CTA-group rules; descriptors; target restrictions; asynchronous completion. +- Tensor Memory: 128 lanes by 512 columns of 32-bit cells per SM, 32-bit addresses, allocation/deallocation, collective ld/st, cp, shift, and waits. +- `tcgen05.commit` and `tcgen05.fence`: asynchronous completion and cross-thread execution-ordering mechanisms with distinct roles. +- `clusterlaunchcontrol.try_cancel` and `query_cancel`: 16-byte shared response, mbarrier completion, success query, and first-CTA-coordinate decoding. +- `cp.async.bulk.tensor`: tensor-map loads/stores, CTA/cluster destinations, mbarrier or bulk-group completion, and `.multicast::cluster`. +- Packed `cvt`: E2M1, E3M2, E2M3, and FP8 conversion forms and their target notes. +- `mov` and `ld`: typed scalar/vector moves, vector load widths, cache operators, and eviction-priority hints. +- `mbarrier`: phase, arrival, transaction-count, wait, scope, and memory-ordering semantics. -```asm -tcgen05.mma.cta_group::{1|2}.kind::{dtype} - [tmem_addr], // Destination: TMEM address - [smem_desc_a], // Source A: SMEM descriptor - [smem_desc_b]; // Source B: SMEM descriptor -``` +## Scope Limits -### CTA Group Variants +The ISA defines legal behavior, not a universal performance ranking between instruction sequences, cache hints, vector widths, swizzles, or pipeline depths. A legal fragment is not a complete kernel: operand declarations, descriptors, collective participation, proxy visibility, lifetimes, completion waits, and target/toolchain selection still matter. -| cta_group | Description | Max M | -|---|---|---| -| `cta_group::1` | Single-CTA (1-SM) operation | 128 | -| `cta_group::2` | Cooperative 2-CTA (2-SM) operation | 256 | +## Direct Links -### Data Type Variants (kind) - -| kind | A type | B type | Accumulator | Shape (1-CTA) | -|---|---|---|---|---| -| `kind::f16` | FP16/BF16 | FP16/BF16 | FP32 | m128n256k16 | -| `kind::tf32` | TF32 | TF32 | FP32 | m128n256k8 | -| `kind::i8` | INT8 | INT8 | INT32 | m128n256k32 | -| `kind::f8f6f4` | FP8/FP6/FP4 | FP8/FP6/FP4 | FP32 | m128n256k32+ | -| `kind::mxf8` | MX FP8 | MX FP8 | FP32 | m128n256k32 | -| `kind::mxf4nvf4` | NVFP4 | NVFP4 | FP32 | m128n256k64 | - -### Key Differences from Hopper wgmma - -```asm -// Hopper (SM90): warpgroup scope, register accumulators -wgmma.mma_async.sync.aligned.m64n256k16.f32.bf16.bf16 - {d0..d127}, // 128 register accumulators - [desc_a], - [desc_b]; - -// Blackwell (SM100): single-thread, TMEM accumulators -tcgen05.mma.cta_group::1.kind::f16 - [tmem_addr], // TMEM accumulator (no register pressure) - [smem_desc_a], - [smem_desc_b]; -``` - -## TMEM Instructions - -### Allocation and Deallocation - -```asm -// Allocate TMEM rows for a CTA group -tcgen05.alloc.cta_group::1 [tmem_base], num_rows; - -// Deallocate TMEM rows -tcgen05.dealloc.cta_group::1 [tmem_base], num_rows; -``` - -### Load/Store (TMEM <-> Registers) - -```asm -// Load from TMEM to registers (for epilogue) -tcgen05.ld.16x256b [reg_dest], [tmem_src]; - -// Store from registers to TMEM -tcgen05.st.16x256b [tmem_dest], [reg_src]; -``` - -### TMEM Layout - -``` -TMEM per SM: 128 rows x 512 columns x 32-bit -Total: 128 * 512 * 4 bytes = 256 KB - -Row addressing: tmem_base + row_offset -Column mapping: determined by MMA instruction variant -``` - -## CLC Instructions - -### Dynamic Tile Scheduling - -```asm -// Signal tile arrival (producer done loading data) -clc.arrive.group::1; - -// Wait for tile assignment (consumer waits for work) -clc.wait.group::1; - -// CLC replaces manual tile queues: -// - Hardware maintains work queue -// - Tiles assigned to SMs as they become available -// - Eliminates tail effects and load imbalance -``` - -## TMA Instructions (SM100 Enhanced) - -### Async Bulk Copy - -```asm -// TMA load: global -> shared memory -cp.async.bulk.tensor.2d.dst_shared::cta.src_global.tile.mbarrier::complete_tx::bytes - [smem_addr], [tma_desc, {coord_x, coord_y}], [mbarrier]; - -// TMA store: shared memory -> global -cp.async.bulk.tensor.2d.dst_global.src_shared::cta.tile - [tma_desc, {coord_x, coord_y}], [smem_addr]; -``` - -### TMA Multicast - -```asm -// Multicast TMA load to multiple CTAs in cluster -cp.async.bulk.tensor.2d.dst_shared::cluster.src_global.tile.mbarrier::complete_tx::bytes - [smem_addr], [tma_desc, {coord_x, coord_y}], [mbarrier], multicast_mask; -``` - -### Alignment Requirements - -- TMA descriptors: 128-byte aligned base address -- Shared memory buffers: 128-byte aligned for TMA targets -- Global memory source: 128-byte aligned - -## FP4/FP8 Conversion Instructions - -### FP4 (E2M1) Conversions - -```asm -// Pack two FP16 values into FP4x2 -cvt.rn.e2m1x2.f16x2 %fp4_packed, %f16x2_val; - -// Unpack FP4x2 to two FP16 values -cvt.rn.f16x2.e2m1x2 %f16x2_result, %fp4_packed; -``` - -### Byte Unpacking for FP4 - -```asm -// Efficient byte unpacking (faster than bitwise extraction) -mov.b32 {tmp0, tmp1, tmp2, tmp3}, %packed_word; -// Splits 32-bit word into 4 bytes without shift/mask overhead -// Critical for FP4 decoding performance -``` - -### FP8 Conversions - -```asm -// FP8 E4M3 to FP16 -cvt.rn.f16.e4m3 %f16_result, %fp8_val; - -// FP16 to FP8 E4M3 -cvt.rn.e4m3.f16 %fp8_result, %f16_val; - -// FP8 E5M2 conversions (similar syntax) -cvt.rn.f16.e5m2 %f16_result, %fp8_val; -``` - -## Cache Control Instructions - -### Load Qualifiers (Critical for Memory-Bound Kernels) - -```asm -// Default: normal caching -ld.global %val, [addr]; - -// L1 no-allocate: bypass L1 for streaming data -ld.global.L1::no_allocate %val, [addr]; - -// L1 evict-last: keep in L1 as long as possible (for reused data) -ld.global.L1::evict_last %val, [addr]; - -// Non-coherent read-only (used by DeepEP for communication) -ld.global.nc.L1::no_allocate.L2::256B %val, [addr]; -``` - -### Vectorized Loads - -```asm -// 64-bit vector load -ld.global.v2.u32 {r0, r1}, [addr]; - -// 128-bit vector load -ld.global.v2.u64 {r0, r1}, [addr]; - -// 256-bit vector load -ld.global.v4.u64 {r0, r1, r2, r3}, [addr]; -``` - -## Synchronization Primitives - -### mbarrier (Memory Barrier) - -```asm -// Initialize mbarrier -mbarrier.init.shared.b64 [mbar], thread_count; - -// Arrive at mbarrier (producer signals completion) -mbarrier.arrive.shared.b64 %phase, [mbar]; - -// Wait on mbarrier (consumer waits for data) -mbarrier.try_wait.shared.b64 %pred, [mbar], %phase; -``` - -### Async Pipeline Coordination - -```asm -// TMA + mbarrier pipeline pattern: -// 1. Producer issues TMA load with mbarrier -// 2. Consumer waits on mbarrier -// 3. Consumer processes data while producer issues next load -// 4. Repeat for multi-stage pipeline -``` - -## Sources - -- [PTX ISA Reference](https://docs.nvidia.com/cuda/parallel-thread-execution/) -- [CUDA 13.0 Programming Guide](https://docs.nvidia.com/cuda/cuda-c-programming-guide/) -- [NVIDIA Blackwell Tuning Guide](https://docs.nvidia.com/cuda/blackwell-tuning-guide/) +- [PTX ISA 9.0 contents](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/contents.html) +- [`tcgen05.mma`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma) +- [Tensor Memory](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory) +- [`clusterlaunchcontrol.try_cancel`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel) +- [`cp.async.bulk.tensor`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor) +- [`cvt`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) +- [`ld`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld) diff --git a/sources/docs/nvidia-transformer-engine-2.13-nvfp4.md b/sources/docs/nvidia-transformer-engine-2.13-nvfp4.md new file mode 100644 index 000000000..660119c55 --- /dev/null +++ b/sources/docs/nvidia-transformer-engine-2.13-nvfp4.md @@ -0,0 +1,26 @@ +--- +id: doc-transformer-engine-2.13-nvfp4 +title: "Transformer Engine 2.13: NVFP4" +url: https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html +source_category: official-doc +architectures: [sm100, sm100a] +tags: [nvfp4, fp4, block-scale] +retrieved_at: 2026-08-08 +version: "2.13" +--- + +# Transformer Engine 2.13 NVFP4 + +## Evidence-scoped summary + +- NVFP4 uses E2M1 payloads whose largest finite magnitude is 6. +- Its 1D recipe combines one E4M3 scale per 16 consecutive values with a per-tensor FP32 scale. +- Its weight-oriented 2D mode assigns a scale to each 16-by-16 block. +- Compared with MXFP4, the documented recipe uses a smaller group and a fractional rather than power-of-two local scale. +- The supported-hardware table associates training with compute capabilities 10.0 and 10.3 and inference with compute capability 10.0 and later. + +These format facts do not, by themselves, establish a universal accuracy or throughput advantage for a particular kernel or workload. + +## Primary reference + +- [Version-pinned Transformer Engine 2.13 NVFP4 documentation](https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html) diff --git a/sources/docs/tfla.md b/sources/docs/tfla.md index 01c71ac59..6be1c556f 100644 --- a/sources/docs/tfla.md +++ b/sources/docs/tfla.md @@ -1,19 +1,26 @@ --- id: doc-tfla -title: "Tiled Flash Linear Attention (TFLA)" -url: https://arxiv.org/abs/2503.14376 +title: Tiled Flash Linear Attention (TFLA) +url: https://arxiv.org/abs/2503.14376v3 source_category: paper -architectures: [sm100, sm90] -tags: [linear-attention, gated-delta-net, chunk-parallelism, tcgen05, wgmma] -retrieved_at: 2026-04-16 +architectures: +- sm90 +tags: +- linear-attention +- chunk-parallelism +- triton +retrieved_at: 2026-08-08 --- -## Summary +## Verified scope -Paper on Tiled Flash Linear Attention enabling arbitrarily large chunk sizes for linear attention. +Tiled Flash Linear Attention adds a second level of sequence parallelization within each chunk. The paper states that this enables arbitrarily large chunks, raises arithmetic intensity, and reduces the need to materialize intermediate recurrent states. -## Key Techniques -- Two levels of sequence parallelism: standard chunkwise + tiling within chunks -- Prevents materialization of intermediate memory states -- Matmuls emitted as inline PTX: WGMMA on Hopper, tcgen05 on Blackwell -- Improves arithmetic intensity for linear attention variants including GatedDeltaNet +The paper applies TFLA to mLSTM. Its official `NX-AI/mlstm_kernels` repository at commit `5b98ff8e2bec189b3d3c249405bab5149564d6f8` provides PyTorch, JAX, and Triton mLSTM implementations and reports H100 benchmarks. + +This source does not establish a Gated DeltaNet implementation, a Blackwell implementation, or inline WGMMA/tcgen05 assembly. Those claims are outside the cited paper and code revision. + +## Primary sources + +- [Paper revision 3](https://arxiv.org/abs/2503.14376v3) +- [Official code at `5b98ff8`](https://github.com/NX-AI/mlstm_kernels/tree/5b98ff8e2bec189b3d3c249405bab5149564d6f8) diff --git a/sources/docs/triton-3.3-blackwell.md b/sources/docs/triton-3.3-blackwell.md new file mode 100644 index 000000000..51dce77fc --- /dev/null +++ b/sources/docs/triton-3.3-blackwell.md @@ -0,0 +1,28 @@ +--- +id: doc-triton-3.3-blackwell +title: "Triton v3.3.0 — Blackwell TCGen5/TMEM Boundary" +url: https://github.com/triton-lang/triton/compare/v3.2.0...v3.3.0 +source_category: official-doc +architectures: [sm100, sm100a] +tags: [triton, tcgen05, tmem] +retrieved_at: 2026-08-08 +--- + +# Triton v3.3.0 — Blackwell TCGen5/TMEM Boundary + +## Exact comparison + +The checked boundary is Triton v3.2.0 (`9641643da6c52000c807b5eeed05edaec4402a67`) to v3.3.0 (`819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c`). The corresponding TCGen5 MMA, TMEM, and MMAv5-lowering symbols are absent from the v3.2.0 tree. The v3.3.0 tree adds: + +- `TTNG_TCGen5MMAOp` and `TTNG_TCGen5MMAScaledOp`; +- TMEM allocation, load, store, and copy operations; +- MMAv5 lowering and tensor-memory allocation passes; and +- Blackwell conversion tests that require concrete `tcgen05.mma`, commit, TMEM, scaled-MMA, and `cta_group::2` output. + +The pinned positive control is [`test/Conversion/tritongpu_to_llvm_blackwell.mlir`](https://github.com/triton-lang/triton/blob/819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c/test/Conversion/tritongpu_to_llvm_blackwell.mlir). The operation definitions are in [`TritonNvidiaGPUOps.td`](https://github.com/triton-lang/triton/blob/819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c/include/triton/Dialect/TritonNvidiaGPU/IR/TritonNvidiaGPUOps.td). + +## Scope + +This tag comparison establishes that native Blackwell TCGen5/TMEM compiler support enters between v3.2.0 and v3.3.0. It does not establish that every user-level `tl.dot` shape selects that path, or that every later frontend surface was already mature in v3.3.0. + +Later releases extend the surface: v3.5.0 includes explicit Gluon TCGen5/TMEM and block-scaled matmul tutorials; v3.6.0 generalizes layouts and copies and advances warp-specialized and initial multi-CTA Gluon support; v3.7.0 continues 2-CTA, multicast, and TMA work. v3.7.1 is a regression-fix patch with no new API or feature. diff --git a/sources/docs/triton-3.6-blackwell.md b/sources/docs/triton-3.6-blackwell.md index 33b306e34..6b6a23593 100644 --- a/sources/docs/triton-3.6-blackwell.md +++ b/sources/docs/triton-3.6-blackwell.md @@ -1,56 +1,29 @@ --- id: doc-triton-3.6-blackwell -title: "Triton 3.6.0 Release Notes — Blackwell (SM100) Lowering" +title: "Triton v3.6.0 — Incremental Blackwell Changes" url: https://github.com/triton-lang/triton/releases/tag/v3.6.0 source_category: official-doc architectures: [sm100, sm100a] -tags: [triton, tcgen05, tmem, 2sm-cooperative, block-scale, nvfp4, warp-specialization] -retrieved_at: 2026-04-27 +tags: [triton, tcgen05, tmem, 2sm-cooperative, block-scale, warp-specialization] +retrieved_at: 2026-08-08 --- -# Triton 3.6.0 Release Notes — Blackwell Lowering +# Triton v3.6.0 — Incremental Blackwell Changes -## Overview +Triton v3.6.0 was released on 2026-01-21 at commit `7c56a5e40f7fd928dfd5c72902d5def0097db73a`. It is not the first release with a native Blackwell backend: the pinned v3.2.0-to-v3.3.0 comparison in [`doc-triton-3.3-blackwell`](triton-3.3-blackwell.md) establishes the earlier TCGen5/TMEM boundary. -Triton 3.6.0 (released `2026-01-21`, release commit `7c56a5e`) is the first Triton release with native NVIDIA Blackwell (SM100) lowering through `tcgen05.mma` plus tensor-memory (TMEM) accumulators. Prior to 3.6, the Triton compiler on Blackwell silently fell back to the Hopper `wgmma` path with register-resident accumulators, which is the framing recorded in the older `wiki/languages/triton-blackwell.md` page. +## Blackwell-relevant changes in v3.6.0 -This doc page summarizes only the SM100-relevant items from the 3.6.0 release notes; per-pathway breakdown with verified-vs-needs-verification classification lives in `data/triton-3.6-evidence.md`. +The release notes describe incremental work including: -## Blackwell-Relevant Items in the 3.6.0 Release Notes +- generic `tcgen05` copy support and broader TMEM bit-width and layout handling; +- more general `tcgen05` load/store layouts and MMA lowering; +- additional aref-style warp-specialization plumbing; +- initial Gluon multi-CTA and 2-CTA support, including `num_ctas`-related cluster work; and +- Gluon scaled-MMA and `tl.dot_scaled` fixes. -### Tensor Memory (TMEM) infrastructure +These changes extend a backend already present in v3.3.0. The word “initial” matters for the multi-CTA path: the v3.7.0 release subsequently adds more end-to-end 2-CTA, multicast, and TMA work. v3.7.1, published 2026-06-18 at commit `f797708`, fixes two regressions and advertises no new API or feature. -The release adds TMEM allocation, copy, and layout primitives that the Blackwell backend lowers through `ttng.tmem_alloc`, `ttng.tmem_copy`, `ttng.tmem_load`, and `ttng.tmem_store`. Source PRs: `#8136`, `#8148`, `#8202`. After 3.6, accumulators on SM100 may live in TMEM rather than registers — the older blanket "accumulators stay in registers" claim is no longer correct as a universal statement. +## Evidence boundary -### `tcgen05` lowering - -Generic `tcgen05` load/store/copy lowering and `tcgen05.mma` generalization land via `#8225`, `#8421`, `#8495`, `#8102`, `#8338`, `#8386`. The dialect now exposes `ttng.tc_gen5_mma` and `ttng.tc_gen5_mma_scaled` with TMEM-token semantics. - -### Warp specialization end-to-end - -End-to-end aref-style warp specialization plumbing on the Blackwell path: `#8262`, `#7826`, `#8009`, `#8123`, `#8534`, `#8451`, `#8651`. The strongest user-visible surface is `tl.range(..., warp_specialize=True)` on top of descriptor / TMA matmul kernels, as documented in the Triton persistent matmul tutorial. - -### Gluon front-end and 2-CTA support - -Initial 2-CTA cluster support in the Gluon front-end (`#8644`, `#8653`), `num_ctas` plumbing (`#8645`), and Gluon-side `tcgen05 mma scaled` support (`#8393`). The Gluon path is the most explicit Blackwell-native surface; the release notes describe it as initial support. - -### Block-scaled matmul (NVFP4 / MXFP) - -Hardware-accelerated block-scaled matmul on Blackwell tensor cores via `tl.dot_scaled`. Backend exposes `ttng.tc_gen5_mma_scaled` (`#8393`); frontend fixes `#8564`, `#8658`. Format coverage centers on NVFP4 / MXFP per the official block-scaled matmul tutorial. - -## Predecessor Release for Context - -Triton 3.5.1 (released `2025-11-12`) is the last 3.5.x patch before the 3.6 Blackwell story. Pages with `version_sensitive` claims valid for `>=3.5,<3.6` should pin to 3.5.1. - -## Subsequent 3.6.x Patches - -As of `2026-04-27`, no 3.6.x patch release is visible on the official triton-lang/triton GitHub releases page; the next previous release shown there is 3.5.1. (`needs-verification` if this requires a machine-checked negative claim.) - -## When To Cite This Page - -Pages making claims about Triton's SM100 capabilities should add a `version_sensitive` block whose registry entry pins `last_verified_release: "3.6.0"` and lists `doc-triton-3.6-blackwell` (this page) as one of its `source_ids`. The companion downstream-code anchors are `pr-sglang-5390`, `pr-sglang-21595`, and `pr-pytorch-175826` — see `data/triton-3.6-evidence.md` for the per-pathway breakdown plus caveat anchors. - -## Caveats - -- The 3.6 release introduces native Blackwell lowering paths but does not by itself prove that every plain `tl.dot` matmul on SM100 lowers through TMEM-backed `tcgen05`. The strongest checked path is descriptor/TMA + `tl.range(warp_specialize=True)` + `tl.dot`, plus the Gluon multi-CTA / 2CTA path. -- Production-peak performance on Blackwell still favors hand-written CuTe-DSL / CUTLASS / FA-4 / TRT-LLM kernels for many compute-bound workloads. SGLang `pr-sglang-5390` reports a CUTLASS `tcgen05_mla` backend ~27% faster than the Triton MLA decode baseline; SGLang `pr-sglang-21595` changes Blackwell datacenter multimodal attention default away from `triton_attn` to FA4. A "first-class lane" framing for Triton on Blackwell is justified for supported lowering surfaces, but not as a blanket peak-performance equivalence claim. +The release notes and pinned compiler tests prove that Triton contains native TCGen5/TMEM paths. They do not prove that every plain `tl.dot` shape selects one, nor that a downstream kernel containing `tl.dot` emitted a particular PTX instruction. Such a claim requires target- and configuration-specific IR or PTX evidence. diff --git a/sources/prs/cccl/PR-3559.md b/sources/prs/cccl/PR-3559.md index 1f295fe4f..aa5ce71ca 100644 --- a/sources/prs/cccl/PR-3559.md +++ b/sources/prs/cccl/PR-3559.md @@ -10,10 +10,12 @@ source_category: upstream-code architectures: - sm100 tags: -- gemm -techniques: [] +- parallel-scan +techniques: +- parallel-scan hardware_features: [] -kernel_types: [] +kernel_types: +- scan languages: - cuda-cpp captured_at: '2026-05-20' @@ -46,4 +48,3 @@ Add b200 tunings for scan.exclusive.sum - `thrust/testing/scan.cu` - `thrust/thrust/system/cuda/detail/async/inclusive_scan.h` - `thrust/thrust/system/cuda/detail/scan.h` - diff --git a/sources/prs/cccl/PR-6152.md b/sources/prs/cccl/PR-6152.md index 7dbec0a84..f70a4e5e7 100644 --- a/sources/prs/cccl/PR-6152.md +++ b/sources/prs/cccl/PR-6152.md @@ -7,13 +7,14 @@ author: oleksandr-pavlyk date: '2025-10-08' url: https://github.com/NVIDIA/cccl/pull/6152 source_category: upstream-code -architectures: -- sm100 +architectures: [] tags: -- gemm -techniques: [] +- top-k-selection +techniques: +- top-k-selection hardware_features: [] -kernel_types: [] +kernel_types: +- topk languages: - cuda-cpp captured_at: '2026-05-20' @@ -49,4 +50,3 @@ Fix debug section around line 390 of dispatch_topk ## Changed Files - `cub/cub/device/dispatch/dispatch_topk.cuh` - diff --git a/sources/prs/cutlass/PR-2139.md b/sources/prs/cutlass/PR-2139.md index e29f88006..4358e3292 100644 --- a/sources/prs/cutlass/PR-2139.md +++ b/sources/prs/cutlass/PR-2139.md @@ -22,7 +22,6 @@ hardware_features: - tcgen05 - tmem - block-scale -- nvfp4 - fp8 kernel_types: - gemm @@ -72,7 +71,7 @@ Introduces blockwise and groupwise GEMM implementations targeting the Blackwell ## Problem -CUTLASS lacked native blockwise and groupwise scaled GEMM kernels for the Blackwell SM100 architecture. Quantized inference workloads using FP8 and FP4 block-scaled formats on Blackwell required new warp-specialized collective mainloop implementations that leverage SM100-specific hardware features such as TCGen05 MMA instructions and TMEM for accumulator management. +CUTLASS lacked blockwise and groupwise FP8 GEMM kernels for the Blackwell SM100 architecture. These workloads required warp-specialized collective mainloop implementations that use SM100-specific infrastructure such as TCGen05 MMA instructions and TMEM accumulator management. ## Solution / Techniques @@ -110,4 +109,4 @@ using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder ## Performance -The PR adds profiling infrastructure with warmup iterations to Hopper FP8 GEMM examples. The Blackwell blockwise GEMM kernels leverage TCGen05 instructions and TMEM-based accumulator double buffering for high throughput on SM100. Specific performance numbers were not included in the PR but these kernels are foundational for CUTLASS 3.8's Blackwell support. +The PR adds profiling infrastructure with warmup iterations to Hopper FP8 GEMM examples. Its Blackwell examples also use FP8 E4M3 operands. Specific performance numbers were not included in the PR. The patch does not add an NVFP4/E2M1 example, so it is not direct evidence for an NVFP4 implementation. diff --git a/sources/prs/cutlass/PR-2161.md b/sources/prs/cutlass/PR-2161.md index b693d9d54..bfb8ede86 100644 --- a/sources/prs/cutlass/PR-2161.md +++ b/sources/prs/cutlass/PR-2161.md @@ -38,7 +38,7 @@ artifact_dir: artifacts/prs/cutlass/PR-2161 ## Summary -Enables Programmatic Dependent Launch (PDL) via Grid Dependency Control (GDC) for SM100 Blackwell kernels, and fixes potential out-of-bounds memory access in blockwise/groupwise scaling copy operations. The PDL feature allows back-to-back kernel launches with hardware-managed dependencies, reducing launch overhead for persistent GEMM kernels. +Enables a Programmatic Dependent Launch (PDL) path via Grid Dependency Control (GDC) for eligible SM100 Blackwell kernels, and fixes potential out-of-bounds memory access in blockwise/groupwise scaling copy operations. PDL can make a same-stream dependent grid eligible to launch before its prerequisite grid finishes; actual overlap and performance remain workload- and resource-dependent. ## Problem @@ -46,7 +46,7 @@ Two issues were addressed: 1. **Security/correctness**: The blockwise/groupwise scaling kernels used hardcoded copy sizes for scale factor arrays (SFA and SFB), which could lead to out-of-bounds memory access when tile dimensions did not match the hardcoded values. -2. **Performance**: SM100 Blackwell architecture supports Grid Dependency Control for programmatic dependent launch, but CUTLASS had not yet enabled this feature. Without GDC, back-to-back kernel launches on Blackwell could not overlap grid scheduling, leaving performance on the table for multi-kernel workloads. +2. **Enablement**: SM100 Blackwell supports Grid Dependency Control, but this CUTLASS path had not enabled the feature. The change adds the compile-time path needed for eligible CUTLASS kernels to use GDC; CUDA's launch attribute and device-side dependency protocol still govern a PDL launch. ## Solution / Techniques @@ -81,6 +81,6 @@ static constexpr int ElementsPerSFACopy = ...; // derived from CTA tile static constexpr int ElementsPerSFBCopy = ...; // derived from CTA tile ``` -## Performance +## Performance scope -PDL/GDC reduces kernel launch latency by allowing the GPU to overlap scheduling of dependent kernel grids. This is particularly beneficial for persistent GEMM kernels that use CLC-based tile scheduling on Blackwell, where back-to-back launches (e.g., GEMM followed by epilogue-fusion or another GEMM) can overlap without CPU synchronization. Performance improvements were validated through the CUTLASS gemm. +The change creates an opportunity to overlap part of a prerequisite grid with an independent preamble in its dependent grid. Neither the PR record nor the captured artifact supplies a reproducible device/shape/baseline/result tuple, so this source does not establish a numeric or general speedup. Measure the exact launch pair and verify the overlap timeline on the target GPU. diff --git a/sources/prs/flash-attention/PR-2441.md b/sources/prs/flash-attention/PR-2441.md index 2f2162b60..c86d6a7e7 100644 --- a/sources/prs/flash-attention/PR-2441.md +++ b/sources/prs/flash-attention/PR-2441.md @@ -26,6 +26,7 @@ status: merged merge_sha: f219c89c inclusion_reason: kernel file changes changed_paths: +- .pre-commit-config.yaml - benchmarks/benchmark_attn.py - flash_attn/cute/bench_utils.py - flash_attn/cute/cute_dsl_utils.py @@ -36,35 +37,31 @@ changed_paths: - flash_attn/cute/softmax.py - flash_attn/cute/testing.py - flash_attn/cute/tile_scheduler.py +- flash_attn/cute/topk_gather_kv.py +- tests/cute/test_flash_attn.py artifact_dir: artifacts/prs/flash-attention/PR-2441 --- -## Summary +## Evidence Scope -Initial saturating decode perf numbers, deepseek 64/512 shape and topk length = 2048. -``` -Vanilla MLA -batch: 512, seqlen_q: 1, seqlen_k: 16384, nheads: 128, -> 1.98 ms, 1180.70 TFLOPS -DSA, no bitmask (topk selected assumed in bounds) -batch: 512, seqlen_q: 1, seqlen_k: 16384, nheads: 128, -> 0.3 +PR 2441 introduces the SM100 CuTe DSL MLA 64/512 forward path with caller-provided top-k sparsity for MQA with 128 query heads per KV head. It merged as `f219c89c886c6ccbf9d3dbd9fe41b11ac64e9df8`; the local artifact bundle contains the upstream patch and byte-verified key files from that merge. -## Problem +## Author-Reported Initial Decode Results -[Cute,Sm100,Fwd] add MLA 64/512 with topk sparsity for MQA 128 heads +The PR body labels these as initial saturating decode results with top-k length 2048: -## Changed Files +| Variant | batch | seqlen_q | seqlen_k | nheads | latency | TFLOPS | +|---|---:|---:|---:|---:|---:|---:| +| Vanilla MLA | 512 | 1 | 16384 | 128 | 1.98 ms | 1180.70 | +| DSA, no bitmask; selected indices assumed in bounds | 512 | 1 | 16384 | 128 | 0.31 ms | 955.47 | +| DSA, validity bitmask | 512 | 1 | 16384 | 128 | 0.33 ms | 898.08 | + +The PR body does not state the exact GPU model, dtype, clock/power settings, software versions, timing protocol, sample count, or variance for these rows. They are source-reported observations, not a complete reproducibility record. + +## Implemented and Excluded Scope -- `.pre-commit-config.yaml` -- `benchmarks/benchmark_attn.py` -- `flash_attn/cute/bench_utils.py` -- `flash_attn/cute/cute_dsl_utils.py` -- `flash_attn/cute/flash_fwd_mla_sm100.py` -- `flash_attn/cute/interface.py` -- `flash_attn/cute/mask.py` -- `flash_attn/cute/named_barrier.py` -- `flash_attn/cute/softmax.py` -- `flash_attn/cute/testing.py` -- `flash_attn/cute/tile_scheduler.py` -- `flash_attn/cute/topk_gather_kv.py` -- `tests/cute/test_flash_attn.py` +The PR says it added the interface exposure and tests and addressed a causal `seqlen_q > seqlen_k` hang. It also says the tcgen05 MMA code generation still needed performance revision. Page-table and sliding-window support were not planned in this PR; the merged MLA kernel rejects a non-null page table. + +## Changed Files +The patch adds `flash_fwd_mla_sm100.py` and `topk_gather_kv.py`, updates the benchmark/interface/mask/softmax/testing/scheduler support files, adds tests, and adjusts the pre-commit configuration. The frontmatter `changed_paths` list is aligned with the captured patch headers. diff --git a/sources/prs/flashinfer/PR-2387.md b/sources/prs/flashinfer/PR-2387.md index 7a9bf0568..b772901fd 100644 --- a/sources/prs/flashinfer/PR-2387.md +++ b/sources/prs/flashinfer/PR-2387.md @@ -10,7 +10,6 @@ source_category: upstream-code architectures: - sm100 tags: -- tcgen05 - decode techniques: - warp-specialization @@ -18,9 +17,7 @@ techniques: - double-buffering hardware_features: - tma -- tmem - mbarrier -- tcgen05 kernel_types: - fused-kernel - decode @@ -29,47 +26,31 @@ languages: - python captured_at: 2026-04-16 status: merged -merge_sha: 18804cd5 +merge_sha: 18804cd51734cccf807356d017733bc757677f15 inclusion_reason: manually curated, Blackwell kernel relevance changed_paths: - include/flashinfer/mamba/selective_state_update.cuh artifact_dir: artifacts/prs/flashinfer/PR-2387 --- -## Summary +# FlashInfer PR 2387 -Introduces a Blackwell (SM100+) optimized version of the Mamba selective_state_update kernel using a horizontal producer-consumer pattern. The kernel replaces the vertical warp-level reduction used on Hopper with a sequential per-thread processing approach that eliminates the warp reduction bottleneck on Blackwell, achieving substantial throughput improvements on B200. +## Verified scope -## Problem +The PR was merged as commit `18804cd51734cccf807356d017733bc757677f15`. Its captured `selective_state_update.cuh` byte-matches that merge revision (SHA-256 `40a2de44b1f29b4b4c213e74e530131c9a43be5e602daa3c8bbd901a3b281446`). The captured patch has SHA-256 `d484b94d986f9c3345cbd89c7d0179f57c0334614b9a517660f173bf6d6762f5`. -The existing Mamba selective_state_update kernel used a vertical producer-consumer pattern where each warp processes a row with warp-level reduction via shuffle intrinsics. On Blackwell hardware, the warp-level reduction became a bottleneck, preventing the kernel from achieving peak attention. A fundamentally different computational pattern was needed for SM100. +The merged source provides two directly inspectable staged shared-memory designs: -## Solution / Techniques +- `SharedStorage` and `SharedStorageHorizontal` allocate `state[numStages][...]` plus one `bar_empty` and one `bar_full` per stage. +- Producers wait for an empty stage before TMA or ordinary-copy production, attach transaction bytes to the full barrier for TMA, and publish the stage. +- Consumers wait for the corresponding full stage, process it, and arrive on the empty barrier before producer reuse. +- Dispatch uses three stages in one path and `min(totalStages, 4)` in the horizontal path. These are implementation choices, not universal stage-count rules. -- **Horizontal producer-consumer pattern**: individual threads sequentially process multiple row elements while maintaining running sums, eliminating warp-level reduction -- New `SharedStorageHorizontal` struct for multi-stage buffering -- Per-stage synchronization barriers with `mbarrier` support -- TMA alignment validation for efficient data loading -- Bank-conflict-free shared memory indexing via `conflict_free_column()` function -- Dynamic dispatch: SM100+ uses horizontal kernel, SM90 uses vertical kernel, older GPUs use base kernel -- JIT compilation specs supporting NVCC versions 10-12 for SM100 +This source card does not preserve enough benchmark metadata to substantiate a portable performance number or a Hopper-versus-Blackwell comparison. Use the pull request discussion for author-reported charts and the pinned file for code semantics. -## Key Code +## Evidence routes -```cuda -// Horizontal producer-consumer pattern for SM100+ -// Eliminates warp-level reduction bottleneck -__device__ void consumer_func_horizontal(...) { - // Each thread processes multiple elements sequentially - // with running sums instead of warp shuffle reduction - for (int elem = 0; elem < elements_per_thread; elem++) { - // Bank-conflict-free shared memory access - int col = conflict_free_column(elem, threadIdx.x); - running_sum += smem[stage][col]; - } -} -``` - -## Performance - -Performance graphs demonstrate substantial throughput improvements on B200 (Blackwell) across varying batch sizes. The horizontal kernel explicitly underperforms on H200 (Hopper) compared to the vertical variant, confirming the architecture-specific optimization. Dynamic dispatch ensures each architecture uses its optimal kernel variant. +- [Merged pull request](https://github.com/flashinfer-ai/flashinfer/pull/2387) +- [Pinned merged source](https://github.com/flashinfer-ai/flashinfer/blob/18804cd51734cccf807356d017733bc757677f15/include/flashinfer/mamba/selective_state_update.cuh) +- `artifacts/prs/flashinfer/PR-2387/key-files/include/flashinfer/mamba/selective_state_update.cuh` +- `artifacts/prs/flashinfer/PR-2387/diff.patch` diff --git a/sources/prs/vllm/PR-16032.md b/sources/prs/vllm/PR-16032.md index 51c9fc9d6..cd6739819 100644 --- a/sources/prs/vllm/PR-16032.md +++ b/sources/prs/vllm/PR-16032.md @@ -10,12 +10,9 @@ source_category: upstream-code architectures: - sm100 tags: -- tcgen05 - mla -- moe - attention techniques: -- warp-specialization - persistent-kernel hardware_features: - tcgen05 @@ -25,13 +22,12 @@ kernel_types: - mla - attention - decode -- moe languages: - cuda-cpp - python captured_at: 2026-04-16 status: merged -merge_sha: ed7a29d9 +merge_sha: ed7a29d9f8b48978e3bbf43599d21b4de65387e0 inclusion_reason: manually curated, Blackwell kernel relevance changed_paths: - csrc/attention/mla/cutlass_mla_entry.cu @@ -43,40 +39,26 @@ changed_paths: artifact_dir: artifacts/prs/vllm/PR-16032 --- -## Summary - -Integrates NVIDIA's CUTLASS MLA (Multi-Head Latent Attention) decode kernel for Blackwell GPUs into vLLM. Exposes the kernel as `ops.tcgen05_mla_decode` with support for separate query tensor inputs and configurable softmax scaling, targeting efficient DeepSeek model inference on SM100 hardware. +# vLLM PR 16032 -## Problem +## Verified scope -vLLM lacked a native high-performance MLA decode kernel for Blackwell GPUs. The MLA attention mechanism used by DeepSeek models requires specialized kernel support to efficiently handle the latent key-value projection and multi-head attention computation. Without SM100-optimized kernels, Blackwell GPU capabilities were underutilized for DeepSeek inference. +The PR was merged as `ed7a29d9f8b48978e3bbf43599d21b4de65387e0`. The captured patch and six key files record the merged integration. -## Solution / Techniques +- `cutlass_mla_decode` is declared in `csrc/ops.h`, registered in `csrc/torch_bindings.cpp`, and wrapped by `vllm/_custom_ops.py`. +- The entry point accepts an output tensor, separate `q_nope` and `q_pe` tensors, a combined KV cache, sequence lengths, a page table, and a scale. +- When `ENABLE_CUTLASS_MLA` is enabled, the entry dispatches to `cutlass_mla_decode_sm100a`; otherwise it reports that no CUTLASS MLA implementation was compiled. +- The SM100a implementation validates concrete tensor ranks and dimensions, builds CUTLASS FMHA arguments, lets the implementation select split-KV policy, calls `can_implement`, initializes workspace, and runs on the current CUDA stream. +- The PR also changes the NVFP4 scaled-GEMM wrapper. Its `CollectiveBuilder` epilogue is a separately inspectable SM100 CUTLASS example; it is not evidence that the MLA path uses a particular user-visible epilogue schedule. -- New CUDA kernel in `csrc/attention/mla/tcgen05_mla_kernels.cu` wrapping CUTLASS library primitives -- Python bindings in `vllm/_custom_ops.py` exposing `tcgen05_mla_decode` -- Supports separate q_nope and q_pe query tensor inputs to avoid memory copies -- Configurable softmax scaling parameters passed from Python layer -- Template-based CUDA kernel design for flexibility -- Based on CUTLASS v3.9 MLA examples from NVIDIA's repository +The captured material does not preserve a benchmark protocol or rows sufficient for a portable speedup claim or a recommended-default-backend claim. -## Key Code +## Artifact checks -```cpp -// CUTLASS MLA decode kernel for Blackwell (SM100) -// Wraps tcgen05 MLA primitives with vLLM-specific interface -// Supports split query tensors: q_nope (no positional encoding) -// and q_pe (with positional encoding) -void tcgen05_mla_decode( - torch::Tensor& out, - torch::Tensor& q_nope, - torch::Tensor& q_pe, - torch::Tensor& kv_cache, - float softmax_scale, - ... -); -``` +`artifacts/prs/vllm/PR-16032/PROVENANCE.yaml` records SHA-256 for the patch and all six files. The captured `nvfp4_scaled_mm_kernels.cu` also appears under `artifacts/kernels/epilogue-fusion/full/` and byte-matches the raw file at the full merge SHA (`e8aed5ccb3dd9de26c3aeff159a242a46dcb7c7d8d0351b6c44ff1f8d2f7effa`). -## Performance +## References -Enables hardware-accelerated MLA decode on Blackwell GPUs leveraging SM100 tensor cores (tcgen05) and tensor memory (tmem). Performance validated with DeepSeek models showing significant speedup over software-emulated MLA attention paths. The CUTLASS MLA kernel is the recommended default backend for SM100 devices. +- [Merged pull request](https://github.com/vllm-project/vllm/pull/16032) +- [Pinned MLA entry](https://github.com/vllm-project/vllm/blob/ed7a29d9f8b48978e3bbf43599d21b4de65387e0/csrc/attention/mla/cutlass_mla_entry.cu) +- [Pinned MLA kernel wrapper](https://github.com/vllm-project/vllm/blob/ed7a29d9f8b48978e3bbf43599d21b4de65387e0/csrc/attention/mla/cutlass_mla_kernels.cu) diff --git a/sources/prs/vllm/PR-23696.md b/sources/prs/vllm/PR-23696.md index 56dc2bec5..b2e3e1ab5 100644 --- a/sources/prs/vllm/PR-23696.md +++ b/sources/prs/vllm/PR-23696.md @@ -2,7 +2,7 @@ id: pr-vllm-23696 repo: vllm-project/vllm pr: 23696 -title: '[Kernel][tcgen05] nvfp4 fused tcgen05 moe' +title: '[Kernel][B200] mxfp4 fused cutlass moe' author: djmmoss date: 2025-09-11 url: https://github.com/vllm-project/vllm/pull/23696 @@ -11,20 +11,18 @@ architectures: - sm90 - sm100 tags: -- tcgen05 -- nvfp4 +- fp4 +- fp8 - moe - fused-kernel +- block-scale techniques: - kernel-fusion - fine-grained-quantization hardware_features: -- nvfp4 - fp4 - fp8 - block-scale -- tma -- tcgen05 kernel_types: - moe - grouped-gemm @@ -34,48 +32,56 @@ languages: - python captured_at: 2026-04-16 status: merged -merge_sha: 074854b2 +merge_sha: 074854b24f6e0b1e237a004283e1f46d98c0d73c inclusion_reason: manually curated, Blackwell kernel relevance changed_paths: +- tests/kernels/moe/test_mxfp4_moe.py +- vllm/envs.py +- vllm/model_executor/layers/fused_moe/layer.py +- vllm/model_executor/layers/quantization/mxfp4.py - vllm/model_executor/warmup/kernel_warmup.py artifact_dir: artifacts/prs/vllm/PR-23696 --- -## Summary +## Scope + +Merged vLLM PR 23696 adds a FlashInfer CUTLASS fused-expert backend for GPT-OSS **MXFP4 weights**. The PR author describes two activation paths: + +- Hopper (SM90): BF16 activations times MXFP4 weights. +- Blackwell (SM100): MXFP8 activations times MXFP4 weights. + +The change is adjacent to, but not evidence for, the FlashInfer MLSys 2026 FP8 E4M3 block-scale Track A definition. + +## Dataflow Boundary + +vLLM computes or receives top-k expert IDs and routing weights before invoking `flashinfer_cutlass_fused_moe`. The backend call consumes those selections together with expert weights, quantization scales, optional biases, and SwiGLU parameters. It fuses expert computation; the PR does not establish that router-logit computation is inside the same kernel. -Implements CUTLASS-based fused Mixture-of-Experts (MoE) kernels optimized for MXFP4 quantization on both Hopper (SM90) and Blackwell (SM100) architectures. Supports bf16xnvfp4 on Hopper and mxfp8xnvfp4 on Blackwell, with environment variable control for backend selection between CUTLASS and TRT-LLM implementations. +The PR adds two environment-controlled backend choices in `vllm/envs.py`: -## Problem +- `VLLM_USE_FLASHINFER_MOE_MXFP4_BF16` +- `VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8` -Large MoE models like GPT-OSS-120B require highly efficient expert computation kernels. The existing MoE backends did not support MXFP4 (microscaling FP4) quantization with CUTLASS, leaving performance on the table especially on Blackwell where native FP4 hardware support is available. A fused kernel combining expert routing with MXFP4 GEMM was needed. +The merged aggregate diff changes five files, listed in frontmatter. The local artifact stores that aggregate diff plus one byte-verified representative key file, `kernel_warmup.py`. -## Solution / Techniques +## Tests Reported in the PR -- CUTLASS fused MoE kernels for MXFP4 quantization across Hopper and Blackwell -- bf16xnvfp4 variant for Hopper (SM90), mxfp8xnvfp4 variant for Blackwell (SM100) -- Environment variables `VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8` and `VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS` for backend selection -- Custom weight quantization handling across different backend implementations -- Kernel fusion of expert routing and GEMM computation +The author reports: -## Key Code +- Hopper unit test: 8 passed, 59 skipped. +- Blackwell unit test: 56 passed, 11 skipped. +- GPT-OSS GPQA results of approximately 0.5612 for 20B and 0.6660 for 120B on the Blackwell path. -```python -# Backend selection for MXFP4 MoE -# CUTLASS backend takes precedence when enabled -VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS=1 # Enable CUTLASS MoE -# Blackwell: mxfp8 activations x nvfp4 weights -# Hopper: bf16 activations x nvfp4 weights -``` +These are PR-author reports; KernelWiki did not reproduce them. -## Performance +## Performance Reported in the PR -Benchmarked on gpt-oss-120b: +For Blackwell serving with random 1024-input/1024-output requests, the author reports that the CUTLASS backend is slower than TRT-LLM at low concurrency and becomes faster only at high concurrency. The 120B table records 16,221 versus 15,592 output tokens/s at concurrency 1024 (4.0% in the table) and 16,424 versus 15,369 at concurrency 1152 (6.9%). The 20B table records 24,629 versus 23,195 at concurrency 1024 (5.8%). -**Blackwell (tcgen05):** -- At 1024 concurrency: CUTLASS achieves 4.0% improvement over TRT-LLM (16,221 vs 15,592 output tok/s) -- At high concurrency: approximately 7% throughput advantage for CUTLASS over TRT-LLM +For Hopper ShareGPT serving, the PR's Triton comparison is faster than the FlashInfer path in the shown runs. These serving measurements are not interchangeable with isolated MoE-kernel TFLOPS or latency. -**Hopper:** -- Triton backend outperforms CUTLASS by approximately 27% at medium-high concurrency on Hopper +## Evidence Boundary -Accuracy validated on gpt-oss-20b (~0.57) and gpt-oss-120b (~0.66). +- Exact primary revision: merge `074854b24f6e0b1e237a004283e1f46d98c0d73c`. +- The local aggregate patch has SHA-256 `033d30b15af4f2050189c2e76ba61959e1e5d4bd049dc901fd56f68890b891a0`. +- The captured `kernel_warmup.py` is byte-identical to the file at the merge revision, SHA-256 `d38c2dd5e9be60f22849f4c3e455746fa5a09f669f995bf75e9a4c4c35e5cb51`. +- The PR does not use the former local title `nvfp4 fused tcgen05 moe`, does not change only `kernel_warmup.py`, and does not provide the FP8 Track A benchmark table. diff --git a/sources/prs/vllm/PR-34597.md b/sources/prs/vllm/PR-34597.md index eb8519b40..8be4206f8 100644 --- a/sources/prs/vllm/PR-34597.md +++ b/sources/prs/vllm/PR-34597.md @@ -8,7 +8,7 @@ date: '2026-02-16' url: https://github.com/vllm-project/vllm/pull/34597 source_category: upstream-code architectures: -- sm100 +- sm120 tags: - attention - decode @@ -36,7 +36,7 @@ artifact_dir: artifacts/prs/vllm/PR-34597 ## Summary -Enable fp8/fp8_e4m3 KV cache for the Triton MLA attention backend, which is the only MLA backend available on sm120 GPUs. +Enable FP8/FP8-E4M3 KV cache handling for the Triton MLA attention backend. The PR identifies this as the MLA backend available on SM120; neither the PR nor the pinned kernel establishes an SM100-only path or a particular TCGen5/TMEM lowering. - Add fp8 and fp8_e4m3 to TritonMLABackend.supported_kv_cache_dtypes - Thread k_scale/v_scale through decode attention kernel launch path @@ -52,4 +52,3 @@ Enable fp8/fp8_e4m3 KV cache for the Triton MLA attention backend, which is the - `tests/kernels/attention/test_triton_decode_attention.py` - `vllm/v1/attention/backends/mla/triton_mla.py` - `vllm/v1/attention/ops/triton_decode_attention.py` - diff --git a/wiki/hardware/2sm-cooperative.md b/wiki/hardware/2sm-cooperative.md index cb43825e9..30cfa3911 100644 --- a/wiki/hardware/2sm-cooperative.md +++ b/wiki/hardware/2sm-cooperative.md @@ -4,53 +4,52 @@ title: "Two-SM Cooperative MMA" type: hardware architectures: [sm100, sm100a] tags: [2sm-cooperative, tcgen05, cluster] -confidence: source-reported +confidence: verified +evidence_basis: + - source_id: doc-ptx-isa-sm100 + evidence_type: official-doc + - source_id: blog-tcgen05-tutorial + evidence_type: source-reported + - source_id: pr-cutlass-2139 + evidence_type: upstream-code related: [hw-tcgen05-mma, hw-tmem, technique-warp-specialization] -sources: [doc-nvidia-tuning-guide, blog-colfax-cutlass, blog-modular-blackwell] +sources: [doc-ptx-isa-sm100, blog-tcgen05-tutorial, pr-cutlass-2139] aliases: ["2-SM cooperative", "dual CTA", "2CTA", "cta_group::2"] --- -## Overview +# Two-SM Cooperative MMA -Blackwell enables two SMs within a TPC to cooperatively execute a single larger MMA, doubling the effective compute tile size to m256×n256×k16. +## Execution model -## How It Works +PTX defines a CTA pair as two CTAs in the same cluster whose `%cluster_ctarank` values differ only in the low bit. With `cta_group::2`, one thread from either CTA can initiate a whole `tcgen05.mma`; the peer CTA must still be active. The operation accesses Tensor Memory belonging to both the current CTA and its peer. -``` -TPC (Two Processing Clusters) -├── SM 0: CTA 0 — issues tcgen05.mma with cta_group::2 -│ ├── Shared Memory A (rows 0-127) -│ └── TMEM (columns 0-255) -└── SM 1: CTA 1 — cooperates on same MMA - ├── Shared Memory A (rows 128-255) - └── TMEM (columns 256-511) -``` +Allocation management differs from MMA issue. A `tcgen05.alloc` or `tcgen05.dealloc` for `cta_group::2` is issued collectively by two warps, one in each CTA. All tcgen05 instructions in the kernel must use the same CTA-group value. -## PTX +## Shape is independent of group size -```ptx -// 2-SM cooperative MMA -tcgen05.mma.cta_group::2.kind::f16 - [tmem_addr], descA, descB, idescC, idescD, ...; -``` +`cta_group::2` selects pair-level resources; it does not by itself select one fixed MxNxK shape. The instruction descriptor (`idesc`) encodes M, N, exact operand and accumulator types, sparsity, and related operation details. Legal values depend on the kind, data-path layout, CTA group, and target ISA. -## Requirements -1. **Identical shared memory layouts** across both CTAs -2. `shared::cluster` mbarrier signaling between the two CTAs -3. Both CTAs in the same cluster -4. Each CTA contributes half the M-dimension +For example, m256xn256xk16 is a useful maximum F16/BF16 configuration, but PTX also defines group-2 layouts with other M/N values. Do not infer that every pair-level MMA is created by mechanically doubling a group-1 M dimension. -## Performance Impact +The complete operand grammar, including the eight-register disable-output-lane vector used by `cta_group::2`, is documented on the related [tcgen05.mma page](tcgen05-mma.md). -From tcgen05 tutorial progression: -- 1-SM MMA (m128×n256): 80% of cuBLAS → adding 2-SM: **86%** of cuBLAS -- ~7.5% improvement from doubling the MMA tile size +## Correctness requirements -## When to Use -- Large GEMM problems where M ≥ 256 -- Compute-bound kernels where peak FLOPS matters -- Combined with persistent scheduling for maximum throughput +A pair-level kernel must account for all of these invariants: + +- The two CTAs form a valid CTA pair, and the peer remains active while a group-2 operation is issued. +- Pair-level TMEM allocation and deallocation follow their two-warp collective issue rule. +- SMEM and TMEM operands use legal layouts and descriptors for the selected instruction configuration. +- Source storage is not overwritten while the asynchronous MMA may still consume it. +- `tcgen05.commit.cta_group::2` and an mbarrier wait provide completion tracking. Fences and execution-ordering operations are added where tcgen05-visible state crosses threads or CTAs. + +## Source-reported performance + +In Gau Nernst's M=N=K=4096 experiment on a Modal B200 with PyTorch 2.9.1 and CUDA 13, v4 warp specialization reports 1208.83 TFLOP/s and v5 2-SM MMA reports 1302.29 TFLOP/s. Relative to the same 1506.74-TFLOP/s cuBLAS result, those are approximately 80.2% and 86.4%, or a 7.7% relative increase from v4 to v5. + +That result establishes a gain for one kernel and setup, not a universal threshold. Choose between group 1 and group 2 by benchmarking the target shapes and accounting for CTA pairing, data reuse, layout, occupancy, pipeline stages, and epilogue cost. Persistence is an independent scheduling choice, not a condition that guarantees maximum throughput. ## Related -- [tcgen05-mma](tcgen05-mma.md) — Base MMA instruction -- [tmem](tmem.md) — Full TMEM used in 2-SM mode + +- [tcgen05.mma](tcgen05-mma.md) — instruction grammar, completion, and descriptors +- [Tensor Memory](tmem.md) — TMEM allocation, addressing, and access diff --git a/wiki/hardware/clc.md b/wiki/hardware/clc.md index 94dcb92c2..64d112e89 100644 --- a/wiki/hardware/clc.md +++ b/wiki/hardware/clc.md @@ -6,7 +6,7 @@ architectures: [sm100, sm100a] tags: [clc, persistent-kernel, tile-scheduling] confidence: source-reported related: [technique-persistent-kernels, technique-tile-scheduling, pattern-tail-effect] -sources: [doc-nvidia-tuning-guide, doc-cutlass-blackwell, pr-cutlass-2161] +sources: [doc-ptx-isa-sm100, doc-cutlass-clc] aliases: [CLC, "cluster launch control"] --- @@ -14,291 +14,101 @@ aliases: [CLC, "cluster launch control"] ## Overview -Cluster Launch Control (CLC) is a Blackwell hardware mechanism for **dynamic tile scheduling** in persistent kernels. It replaces the static grid scheduling model where the CUDA runtime pre-assigns tile coordinates to CTAs at launch time. +Cluster Launch Control is a Blackwell compute-capability 10.0 mechanism for +work stealing between running and not-yet-started thread blocks or thread block +clusters. A CLC kernel still launches the problem-sized grid. Each grid +coordinate, called a ClcID by CUTLASS, is processed in exactly one of two ways: -With CLC, persistent CTAs dynamically request work from a hardware queue, enabling: +1. The block or cluster launches normally and first processes its own + `blockIdx`. +2. An existing worker successfully cancels that not-yet-started ClcID and + processes the returned coordinate itself. -- **Better load balancing**: No fixed CTA-to-tile mapping; busy SMs consume tiles as they become available. -- **Tail-effect mitigation**: The "tail" of a GEMM (when remaining tiles < number of SMs) is handled efficiently because idle CTAs pick up remaining work. -- **Dynamic cancellation**: Tiles can be cancelled via `try_cancel` when the output is no longer needed (e.g., speculative decoding). +This lets a persistent worker request subsequent work without maintaining a +separate software work queue. It is especially useful when the set of SMs +available to a kernel is uneven or changes while the grid is running. -## Static Scheduling vs CLC +## Request and Decode Protocol -### Static Scheduling (Hopper and earlier) +`clusterlaunchcontrol.try_cancel` is asynchronous. It writes an opaque 16-byte +response to shared memory and completes a transaction on a shared-memory +`mbarrier`. The request does not accept a tile coordinate to cancel. -``` -Launch grid: 256 CTAs for 256 tiles -CTA 0 -> tile (0,0) [fixed at launch] -CTA 1 -> tile (0,1) [fixed at launch] -CTA 2 -> tile (0,2) [fixed at launch] -... -CTA 255 -> tile (15,15) [fixed at launch] - -Problem: If SM count = 132, first wave = 132 CTAs. - Second wave = 124 CTAs -> 8 SMs idle = 6% waste. - For small GEMMs, tail effect dominates. -``` - -### CLC Dynamic Scheduling (Blackwell) - -``` -Launch grid: 132 persistent CTAs (= SM count) -CTA 0: request tile -> get (0,0) -> compute -> request tile -> get (2,4) -> ... -CTA 1: request tile -> get (0,1) -> compute -> request tile -> get (2,5) -> ... -... -CTA 131: request tile -> get (0,131) -> compute -> request tile -> ... - -All CTAs stay busy until the tile queue is empty. -Tail: last few tiles distributed to first-available CTAs. -``` - -## How CLC Works - -### Hardware Queue - -CLC operates by letting a running CTA or cluster issue `clusterlaunchcontrol.try_cancel` to cancel a not-yet-launched ClcID and take over that work. There is no `clusterlaunchcontrol.try_acquire` PTX instruction. - -### CLC Programming Model - -```cuda -__global__ void persistent_gemm_clc( - const half* A, const half* B, half* C, - int M, int N, int K, - int num_tiles_m, int num_tiles_n -) { - // Allocate persistent resources (TMEM, pipeline state) - uint32_t tmem_acc = tmem_alloc(256); - - // Shared storage for CLC results (visible to all threads in CTA) - __shared__ uint2 clc_tile_coord; - __shared__ int clc_has_tile; - - // CLC tile loop: keep requesting tiles until none remain - while (true) { - // Thread 0 acquires the next tile; result goes to shared memory - if (threadIdx.x == 0) { - uint2 result; - int acquired = 0; - asm volatile( - "{\n" - " .reg .pred p;\n" - " clusterlaunchcontrol.try_cancel {%0, %1}, p;\n" - " selp.s32 %2, 1, 0, p;\n" - "}\n" - : "=r"(result.x), "=r"(result.y), "=r"(acquired) - ); - clc_tile_coord = result; - clc_has_tile = acquired; - } - __syncthreads(); // All threads see the shared result - - // Exit if no more tiles - if (!clc_has_tile) break; - - int tile_m = clc_tile_coord.x; - int tile_n = clc_tile_coord.y; - - // Zero accumulator - tmem_zero(tmem_acc, 256); - - // Mainloop: iterate over K dimension - for (int k = 0; k < K / TILE_K; ++k) { - // TMA load A and B tiles to SMEM - tma_load_a(smem_a, A, tile_m, k); - tma_load_b(smem_b, B, k, tile_n); - wait_barrier(); - - // Issue MMA - if (threadIdx.x == 0) { - asm volatile( - "tcgen05.mma.cta_group::1.kind::f16 " - "[%0], %1, %2, %3, 1;" - : - : "r"(tmem_acc), "l"(desc_a), "l"(desc_b), "r"(0) - ); - } - } +The normative PTX instruction forms are: - // Epilogue - asm volatile("tcgen05.fence::before_thread_sync;"); - __syncthreads(); - store_output(tmem_acc, C, tile_m, tile_n); - } - - // Cleanup - tmem_dealloc(tmem_acc, 256); -} +```ptx +clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [response], [mbar]; +clusterlaunchcontrol.query_cancel.is_canceled.pred.b128 p, response_b128; +@p clusterlaunchcontrol.query_cancel.get_first_ctaid.v4.b32.b128 {x, y, z, unused}, response_b128; ``` -## try_cancel API +A complete loop must: -CLC provides a `try_cancel` mechanism to cancel pending tiles. This is useful for speculative execution where some outputs may not be needed. +1. Allocate and initialize a shared response and `mbarrier`. +2. Submit one asynchronous request from the selected thread and set the + expected transaction size to 16 bytes. +3. Wait for the corresponding barrier phase to complete. +4. Decode `is_canceled`; decode `get_first_ctaid` only after success. +5. Apply the async/generic proxy fences needed before reusing the response. -```cuda -// Cancel a specific tile if it hasn't started execution yet -__device__ bool clc_try_cancel(uint2 tile_coord) { - bool cancelled = false; - if (threadIdx.x == 0) { - asm volatile( - "clusterlaunchcontrol.try_cancel.async.shared::cta " - "%0, [%1];" - : "=r"(cancelled) - : "r"(&tile_coord) - ); - } - return cancelled; -} -``` +After a thread has observed a failed request, issuing another request from that +thread is undefined. Decoding a CTA ID from a failed response is also undefined. +A request can fail because no ClcIDs remain or for another scheduling reason, +including pending higher-priority work. -Use cases for `try_cancel`: -- **Speculative decoding**: Cancel tiles for rejected draft tokens. -- **Early termination**: If an attention mask makes certain output tiles zero, cancel them. -- **Dynamic batching**: Cancel tiles for sequences that have finished. +## Thread Block Cluster Rules -## CUTLASS Integration +CLC cancellation is cluster-granular when the kernel uses thread block +clusters. One cluster thread submits the multicast request. Every CTA tracks +completion with its local shared-memory barrier at cluster scope and receives +the same encoded first-CTA coordinate. Each CTA then adds its local block rank +to that first coordinate. A cluster synchronization is needed to guarantee all +blocks exist before cluster cancellation begins. -CUTLASS 4.5.0 for SM100 provides CLC support through the `PersistentScheduler` class: +For example, a successful query by a 2x2 worker cluster consumes the matching +2x2 group of ClcIDs; it is not four unrelated CTA-level cancellations. -```cuda -// CUTLASS SM100 persistent GEMM with CLC scheduling -using Gemm = cutlass::gemm::device::GemmUniversal< - cutlass::half_t, // ElementA - cutlass::layout::RowMajor, // LayoutA - cutlass::half_t, // ElementB - cutlass::layout::ColumnMajor, // LayoutB - cutlass::half_t, // ElementC - cutlass::layout::RowMajor, // LayoutC - float, // ElementAccumulator - cutlass::arch::OpClassTensorOp, - cutlass::arch::Sm100, // Blackwell - // Tile shape: 128x256x64 - cutlass::gemm::GemmShape<128, 256, 64>, - // Cluster shape - cutlass::gemm::GemmShape<2, 1, 1>, - // Use CLC persistent scheduler - cutlass::gemm::PersistentScheduler ->; +## CUTLASS 4.5.0 Integration -// Launch: CUTLASS handles CLC internally -Gemm gemm_op; -gemm_op(args, workspace, stream); -``` +At CUTLASS 4.5.0, the `PersistentScheduler` tag for `arch::Sm100` maps to +`PersistentTileSchedulerSm100`; newer code can select the intent explicitly +with `DynamicPersistentScheduler`. The SM100 scheduler uses +`PipelineCLCFetchAsync`: `advance_to_next_work()` submits a request, while +`fetch_next_work()` waits for and decodes the staged response. -### CUTLASS CLC Tile Scheduler +A schematic of the relevant CUTLASS 3.x kernel composition is: ```cpp -// Simplified CUTLASS CLC scheduler logic -struct ClcTileScheduler { - CUTLASS_DEVICE - WorkTileInfo get_next_work() { - WorkTileInfo work; - // Fetch next tile by cancelling a not-yet-launched ClcID. - bool valid = clc_try_cancel(&work.tile_coord); - work.is_valid = valid; - - if (valid) { - // Convert linear tile index to 2D coordinates - work.tile_m = work.tile_coord.x; - work.tile_n = work.tile_coord.y; - - // Apply swizzle for L2 locality - apply_l2_swizzle(work.tile_m, work.tile_n); - } - return work; - } -}; -``` - -## Performance Impact - -CLC delivers significant performance gains, especially for small-to-medium GEMMs where tail effects dominate: - -### Tail Effect Mitigation - -| GEMM Size | Static Scheduler | CLC Scheduler | Improvement | -|---|---|---|---| -| 2048x2048 (small) | 86% SM utilization | 98% SM utilization | +14% | -| 4096x4096 (medium) | 92% SM utilization | 98% SM utilization | +6.5% | -| 8192x8192 (large) | 97% SM utilization | 99% SM utilization | +2% | +using TileScheduler = cutlass::gemm::DynamicPersistentScheduler; -The canonical benchmark from the "tcgen05 for dummies" tutorial shows the jump from 940 TFLOPS (pipelined, static scheduling) to **1476 TFLOPS** (persistent + CLC), approaching 98% of cuBLAS (1507 TFLOPS). - -### Why CLC Matters for Inference - -Production LLM inference typically hits shapes where tail effects are severe: - -```python -# Typical LLM GEMM shapes during decode (batch_size=1-64) -# M is small (batch * seq_len for decode), N and K are large (model dim) -# Example: Llama-70B decode, batch=32 -M = 32 # small! -N = 8192 # hidden dim -K = 8192 # hidden dim - -# Tile = 128x256 -> tiles_m = 1, tiles_n = 32 -> only 32 tiles total -# On B200 (132 SMs): 100 SMs idle with static scheduling -# CLC: 32 persistent CTAs handle all 32 tiles efficiently +using GemmKernel = cutlass::gemm::kernel::GemmUniversal< + cute::Shape, + CollectiveMainloop, + CollectiveEpilogue, + TileScheduler>; ``` -## CLC with 2-SM Cooperative Mode - -When using 2-SM cooperative MMA (`cta_group::2`), CLC distributes work in **cluster-sized units**: +CUTLASS can apply a software coordinate transform to both the initial +`blockIdx` and decoded CLC responses. `max_swizzle_size` and `raster_order` are +scheduler arguments used by `swizzle_and_rasterize()`; they are not operands of +the CLC PTX instruction or a CLC hardware policy configured at launch. -```cuda -// 2-SM cooperative CLC: each successful cancel gets a cluster-sized tile -__device__ void cooperative_clc_loop() { - while (true) { - // Fetch tile for the 2-CTA cluster - ClusterTile tile; - bool valid = clc_try_cancel_cluster(&tile); - if (!valid) break; +## Scope and Limits - // Both CTAs in the cluster share the tile - // CTA 0 handles rows 0-127, CTA 1 handles rows 128-255 - int my_row_start = (blockIdx.x % 2) * 128; +CLC redistributes ClcIDs that already exist in the launched grid. It does not: - // Issue cooperative MMA - if (threadIdx.x == 0) { - asm volatile( - "tcgen05.mma.cta_group::2.kind::f16 " - "[%0], %1, %2, %3, 1;" - : - : "r"(tmem_acc), "l"(desc_a), "l"(desc_b), "r"(0) - ); - } - // ... - } -} -``` - -## L2 Cache Swizzling with CLC - -CLC tile ordering can be customized with swizzle patterns to improve L2 cache hit rates: +- discard an application-selected output tile; +- create more independent tiles than the problem grid contains; +- guarantee that every SM remains occupied; or +- universally eliminate GEMM wave quantization. -```cuda -// Swizzle tile coordinates for better L2 locality -// Tiles are visited in a Z-order (Morton) curve pattern -__device__ void apply_l2_swizzle(int& tile_m, int& tile_n, int swizzle_bits) { - // Convert linear tile index to swizzled 2D coordinates - // This groups spatially adjacent tiles together, improving - // L2 reuse for the B matrix (shared across M tiles) - int linear = tile_m * num_tiles_n + tile_n; - int swizzle_mask = (1 << swizzle_bits) - 1; - int group = linear >> swizzle_bits; - int within = linear & swizzle_mask; - - tile_m = group / num_tiles_n; - tile_n = (group % num_tiles_n) ^ (tile_m & swizzle_mask); -} -``` +Consequently, a 32-tile grid exposes at most 32 independent ClcIDs even on a +148-SM B200. Any CLC performance result must state the GPU, available SMs, +software versions, tile and cluster shapes, data types, timed region, and +measurement method. -## Comparison: CLC vs Software Persistent Scheduling +## Related -| Feature | CLC (Hardware) | Software Atomics | -|---|---|---| -| Scheduling overhead | Near zero (hardware) | atomicAdd contention | -| Tail-effect handling | Optimal | Good with careful design | -| Cancellation | try_cancel API | Complex (flags + barriers) | -| L2 swizzle | Configurable at launch | Manual implementation | -| Portability | SM100+ only | SM70+ | -| CUTLASS support | Built-in | Manual scheduler | +- [Persistent kernels](../techniques/persistent-kernels.md) +- [Tile scheduling](../techniques/tile-scheduling.md) +- [Two-SM cooperative MMA](2sm-cooperative.md) diff --git a/wiki/hardware/mbarrier.md b/wiki/hardware/mbarrier.md index 74d6b99b7..3f2041316 100644 --- a/wiki/hardware/mbarrier.md +++ b/wiki/hardware/mbarrier.md @@ -3,84 +3,119 @@ id: hw-mbarrier title: "mbarrier (Memory Barrier Primitives)" type: hardware architectures: [sm100, sm100a, sm90, sm90a] -tags: [mbarrier, tma] -confidence: source-reported +tags: [mbarrier, tma, tcgen05] +confidence: verified +evidence_basis: + - source_id: doc-ptx-isa-sm100 + evidence_type: official-doc + - source_id: pr-cutlass-2139 + evidence_type: upstream-code related: [hw-tma, hw-tcgen05-mma, technique-warp-specialization, technique-pipeline-stages] -sources: [doc-ptx-isa-sm100, blog-tcgen05-tutorial, doc-nvidia-tuning-guide] +sources: [doc-ptx-isa-sm100, pr-cutlass-2139] aliases: [mbarrier, "memory barrier", "mbar"] -blackwell_relevance: "mbarrier is the primary synchronization primitive between TMA producers and tcgen05 consumers on Blackwell. Phase/parity tracking critical for pipelined kernels." +blackwell_relevance: "TMA complete-tx and tcgen05 commit use mbarriers for asynchronous completion; each pipeline stage must track its own phase or state token." --- # mbarrier -## Overview +## Scope and history -mbarriers are 64-bit shared memory primitives used for producer/consumer synchronization between asynchronous hardware units (TMA, tcgen05) and SM threads. Introduced on Hopper, essential for Blackwell warp-specialized kernels. +An mbarrier is an opaque, naturally aligned 64-bit object in shared memory. It synchronizes threads and can track asynchronous operations. The base instructions were introduced in PTX ISA 7.0 for `sm_80`; Hopper (`sm_90`) added transaction-count operations and cluster scope. Blackwell uses the same object for TMA transaction completion and for `tcgen05.commit` arrival-on completion. -## Key Operations +mbarriers are useful in warp-specialized pipelines, but they are a mechanism rather than a requirement for every such kernel. -```ptx -// Initialize: set expected arrival count -mbarrier.init.shared.b64 [mbar_addr], num_arrivals; +## Object state and phase completion + +For the current phase, an mbarrier tracks: + +- pending arrivals; +- the expected arrival count for the next phase; and +- a transaction count (`tx-count`) for outstanding asynchronous work. -// Producer: arrive on barrier (decrements expected count) -mbarrier.arrive.shared.b64 _, [mbar_addr]; +The current phase completes only when **both** pending arrivals and tx-count reach zero. Completion atomically advances to the next phase and restores pending arrivals from the expected count. Before an arrival in the following phase, at least one `test_wait` or `try_wait` for the completed phase must have returned true. -// Producer (with byte expectation): used by TMA -mbarrier.arrive.expect_tx.shared.b64 _, [mbar_addr], expected_bytes; -// After this, TMA hardware completes the transaction by arriving with -// the transferred byte count, and the mbarrier flips parity. +Initialization sets phase 0, initializes expected and pending arrivals to `count`, and sets tx-count to zero: -// Consumer: wait for parity flip -mbarrier.try_wait.parity.shared.b64 p, [mbar_addr], phase; -@!p bra WAIT_LOOP; +```ptx +mbarrier.init.shared::cta.b64 [bar], arrival_count; ``` -## Phase/Parity Semantics +Invalidate the object with `mbarrier.inval` before reusing its storage for another purpose or reinitializing an already-valid object. -mbarriers have a 1-bit **phase** that flips each time the arrival count reaches zero. Consumers track their expected phase and wait for it: +## Arrival, transaction, and wait operations -```cuda -int phase = 0; -for (int k = 0; k < num_iterations; k++) { - int stage = k % NUM_STAGES; +These operations affect different parts of the state: - // Wait for this stage's producer to complete - mbarrier_wait_parity(&mbar[stage], phase); - phase ^= 1; // Flip for next use of this stage slot +| Operation | Effect | +|---|---| +| `mbarrier.arrive` | Decrements pending arrivals; returns a phase state token for a CTA-shared object. | +| `mbarrier.arrive.expect_tx` | Performs an arrival and increments tx-count by `txCount`. | +| `mbarrier.expect_tx` | Increments tx-count without an arrival. | +| `mbarrier.complete_tx` | Decrements tx-count; it is not an arrival. | +| `mbarrier.test_wait` / `try_wait` | Tests completion of the phase identified by a state token or parity. | - // Use data... - consume_data(stage); +Representative CTA-shared forms from PTX ISA 9.0 are: - // Arrive on next-stage barrier when done - mbarrier_arrive(&buffer_free[stage]); -} +```ptx +mbarrier.arrive.shared::cta.b64 state, [bar]; +mbarrier.arrive.expect_tx.shared::cta.b64 state, [bar], tx_count; +mbarrier.try_wait.parity.acquire.cta.shared::cta.b64 ready, [bar], phase_parity; ``` -## TMA Integration +`try_wait` can suspend temporarily and must still be retried until its predicate is true. Use acquire/release semantics appropriate to the producer-consumer handoff; `.relaxed` does not provide memory-ordering or visibility guarantees. -TMA directly signals mbarriers when transfer completes, avoiding manual polling: +## Phase and parity in a stage ring -```cuda -// Producer warp issues TMA with mbarrier target -if (lane_id == 0) { - uint32_t tx_bytes = TILE_A_BYTES + TILE_B_BYTES; - mbarrier_arrive_expect_tx(&mbar[stage], tx_bytes); +Parity is the low bit of an individual mbarrier object's phase: even phases use 0 and odd phases use 1. A parity wait can refer only to the current or immediately preceding phase, so software must track phase for the entire lifetime of that object. - // TMA completion will fire the mbarrier automatically - cp_async_bulk_tensor_2d(smem_A[stage], global_desc, x, y, &mbar[stage]); - cp_async_bulk_tensor_2d(smem_B[stage], global_desc, x, y, &mbar[stage]); -} -// Do NOT manually arrive after TMA issue - it races with hardware -``` +For an N-stage ring, track state per stage. When stage `s` is reused, pass the parity expected for `bar[s]`; toggle that stage's parity only after its phase completes. One global bit toggled on every loop iteration is generally wrong because different stage barriers advance independently. + +Using the opaque state returned by `mbarrier.arrive` is an alternative when the same participant can carry that token to its wait. + +## TMA completion accounting + +For a common TMA-load phase initialized with one pending arrival: + +1. The producer executes `mbarrier.arrive.expect_tx` with the sum of bytes that all TMA operations in this phase will report. +2. The producer issues the TMA operations with `.mbarrier::complete_tx::bytes` and the same barrier. +3. Each TMA completion performs `complete-tx` for the bytes it copied. +4. The consumer waits for the phase to complete before reading the destination. + +The `arrive.expect_tx` operation accounts for the software arrival and establishes the expected byte total. TMA hardware does **not** perform a second arrival: it decrements tx-count through complete-tx. Therefore, do not add an unmatched `mbarrier.arrive`, and do not manually simulate TMA's complete-tx. Either mistake can advance or strand the wrong phase. + +The expected byte total, barrier address, destination ownership, and exact `cp.async.bulk.tensor` form must match. Follow the complete instruction grammar in the PTX ISA rather than using placeholder helper calls. + +## tcgen05 completion accounting + +`tcgen05.commit.cta_group::*.mbarrier::arrive::one` makes an mbarrier track prior asynchronous `tcgen05.mma`, `tcgen05.cp`, or `tcgen05.shift` operations issued by that thread. When those operations complete, the system performs one arrive-on operation. This uses pending-arrival accounting, not TMA's byte-valued tx-count protocol. + +A consumer in another thread waits for the mbarrier and then uses the applicable `tcgen05.fence::after_thread_sync` ordering sequence. The fence participates in the handoff but does not replace the completion wait. + +## CTA and cluster distinctions + +State space and synchronization scope are separate concepts: + +- `.shared::cta` identifies a barrier in the current CTA's shared memory. +- `.shared::cluster` identifies a cluster-shared address, such as a mapped address for another CTA's shared memory. +- `.cta` or `.cluster` on an operation specifies its synchronization scope. + +Only `arrive`, `expect_tx`, and `complete_tx` support an mbarrier address in `.shared::cluster`. Other mbarrier operations, including initialization and waits, target a CTA-shared object. A cluster can initialize an owner CTA's object, map its address for remote arrivals, and have the owner wait with the required cluster scope, as shown by the PTX cluster example. Do not infer that adding `.shared::cluster` alone turns every operation into a cluster barrier. + +## Correctness checklist -## Pitfalls +- Initialize the object before any other mbarrier operation. +- Match all arrivals and byte-valued complete-tx operations to the initialized counts. +- Keep phase or opaque state separately for each pipeline stage. +- Retry `try_wait` until it reports completion. +- Use the required acquire/release semantics for data visibility. +- Distinguish TMA complete-tx from tcgen05 arrive-on completion. +- Respect the supported combinations of object state space and operation scope. +- Invalidate the object before repurposing its storage. -1. **Missing phase flip**: Reusing a stage slot without flipping parity causes stale arrivals to satisfy the new wait -2. **Mismatched init count**: If consumer expects N arrivals but producers only arrive M times, barrier never fires -3. **Manual arrive after async issue**: TMA/tcgen05 hardware arrives on completion — extra manual arrive causes double-count -4. **Cross-cluster mbarriers**: Use `shared::cluster` qualifier for cluster-level sync (2-SM cooperative MMA) +## References -## Related -- [TMA](tma.md) — Primary producer for mbarriers -- [warp-specialization](../techniques/warp-specialization.md) — Uses mbarriers for warp role handoff +- [PTX ISA 9.0: mbarrier object and lifecycle](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier) +- [PTX ISA 9.0: mbarrier waits](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait) +- [PTX ISA 9.0: TMA tensor copies](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor) +- [TMA](tma.md) +- [Warp specialization](../techniques/warp-specialization.md) diff --git a/wiki/hardware/nvfp4.md b/wiki/hardware/nvfp4.md index 4ce3b8070..43e56266c 100644 --- a/wiki/hardware/nvfp4.md +++ b/wiki/hardware/nvfp4.md @@ -3,57 +3,77 @@ id: hw-nvfp4 title: "NVFP4 and Block-Scaled Narrow Precision" type: hardware architectures: [sm100, sm100a] -tags: [nvfp4, fp4, block-scale, fp8, fp6] -confidence: source-reported +tags: [nvfp4, fp4, block-scale] +confidence: verified +evidence_basis: + - source_id: doc-transformer-engine-2.13-nvfp4 + evidence_type: official-doc + - source_id: doc-ptx-isa-sm100 + evidence_type: official-doc related: [technique-fine-grained-quantization, kernel-nvfp4-gemm, kernel-nvfp4-gemv, hw-tcgen05-mma] -sources: [doc-nvidia-tuning-guide, contest-gpumode-p1, contest-gpumode-p2, blog-yue-nvfp4] +sources: [doc-transformer-engine-2.13-nvfp4, doc-ptx-isa-sm100] aliases: [NVFP4, E2M1, "FP4 E2M1", "nv_float4"] +blackwell_relevance: "Blackwell tensor cores support block-scaled E2M1 MMA; NVFP4 combines block-of-16 fractional local scales with a per-tensor FP32 scale." --- -## Overview +# NVFP4 -NVFP4 is NVIDIA's 4-bit floating-point format (E2M1) with block scaling, native to Blackwell tensor cores. +## Format and recipe -## Format Details +NVFP4 is a quantization recipe, not just another name for its 4-bit payload. It reconstructs a value as: -``` -E2M1: 1 sign bit, 2 exponent bits, 1 mantissa bit -Representable values: 0, ±0.5, ±1, ±1.5, ±2, ±3, ±4, ±6 +`x_hat_i = e2m1_i * s_block * s_global` -Block scaling: every 16 FP4 elements share one FP8 E4M3 scale factor -Two-level: per-block E4M3 scale × per-tensor FP32 global scale +The three components are: -Quantization: q_i = cast_FP4(x_i / (s_global * s_block)) -Dequantization: x_hat_i = s_global * s_block * deq_FP4(q_i) -``` +- **E2M1 payload:** one sign bit, two exponent bits, and one mantissa bit. Its numeric values are 0, +/-0.5, +/-1, +/-1.5, +/-2, +/-3, +/-4, and +/-6. Two E2M1 encodings fit in one byte. +- **Local scale:** one E4M3 scale shared by 16 consecutive payload values. PTX names the corresponding unsigned scale element type `.ue4m3`; its most-significant storage bit is zero. +- **Global scale:** one FP32 scale per tensor. This second level lets the block scales use their range effectively. + +Transformer Engine 2.13 also defines a 2D scaling mode for weights in which a scale covers a 16-by-16 block. The 1D mode groups 16 consecutive values. + +## NVFP4 and MXFP4 -## tcgen05 Variants for FP4 +The recipes make different scale tradeoffs: -| Variant | Description | Throughput vs Hopper | +| Property | NVFP4 | MXFP4 | |---|---|---| -| `tcgen05.mma.mxf4.block_scale` | MX FP4 with block scaling | **4×** | -| `tcgen05.mma.mxf4nvf4.block_scale` | NVFP4 + MX FP4 flexible scaling | **4×** | +| Payload | E2M1 | E2M1 | +| Local group in the standard 1D recipe | 16 values | 32 values | +| Local scale | E4M3 (`.ue4m3` in PTX) | UE8M0 | +| Scale values | Fractional values are available | Powers of two | +| Additional recipe scale | Per-tensor FP32 | Not part of the MXFP4 microscaling format | -## PTX for FP4 Conversion +Finer groups and fractional local scales give NVFP4 more scaling freedom, at the cost of more scale metadata. They do not guarantee strictly lower error for every tensor, scale-selection algorithm, or error metric. Measure accuracy with the exact quantizer and workload. -```ptx -// Convert two FP4 values to two FP16 values -cvt.rn.f16x2.e2m1x2 result, packed_fp4; +## PTX block-scaled MMA + +In PTX ISA 9.0, the relevant `tcgen05.mma` combinations for E2M1 inputs include: + +| Instruction qualifiers | Block | Local scale | Scale-vector qualifier | +|---|---:|---|---| +| `.kind::mxf4.block_scale.block32` | 32 | `.ue8m0` | `.scale_vec::2X` | +| `.kind::mxf4nvf4.block_scale.block16` | 16 | `.ue4m3` | `.scale_vec::4X` | -// Byte unpacking (faster than bitwise extraction) -mov.b32 {tmp0, tmp1, tmp2, tmp3}, packed_data; +The `.kind::mxf4nvf4` family also admits documented UE8M0 modes, so the kind name by itself does not select the NVFP4 recipe. Use the complete block and scale-vector qualifiers and follow the type-combination tables. These forms target architecture-specific Blackwell targets such as `sm_100a`; a kernel using them must use the matching compile and runtime target. + +PTX ISA 9.0 provides the packed conversion form: + +```ptx +cvt.rn.f16x2.e2m1x2 d, a; ``` -## NVFP4 vs MXFP4 +Here `a` is a byte-sized packed pair and `d` is a 32-bit `f16x2` result. PTX also permits `mov.b32` to unpack a 32-bit scalar into four byte-sized vector destinations when the declarations satisfy its type rules. Neither syntax establishes that one packing strategy is faster than masks and shifts; inspect generated machine code and benchmark the target GPU. -| Aspect | NVFP4 | MXFP4 | -|---|---|---| -| Scale format | E4M3 (fractional) | UE8M0 (power-of-2 only) | -| Block size | 16 elements | 32 elements | -| Scale precision | Non-power-of-2 | Power-of-2 only | -| Quantization error | Lower | Higher | - -## Related -- [fine-grained-quantization](../techniques/fine-grained-quantization.md) — Scaling strategies -- [nvfp4-gemm](../kernels/nvfp4-gemm.md) — NVFP4 GEMM kernel -- [nvfp4-gemv](../kernels/nvfp4-gemv.md) — NVFP4 GEMV kernel +## Performance claims + +There is no architecture-independent "4x versus Hopper" result for these instructions. Hopper has no native FP4 tensor-core path, so any comparison depends on the emulation or higher-precision baseline as well as GPU SKU, clocks, matrix shape, layouts, scale preparation, epilogue, and achieved occupancy. Record those conditions with any throughput number. + +## References + +- [Transformer Engine 2.13: NVFP4](https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html) +- [PTX ISA 9.0: `tcgen05.mma`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma) +- [PTX ISA 9.0: `cvt`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) +- [Fine-grained quantization](../techniques/fine-grained-quantization.md) +- [NVFP4 GEMM](../kernels/nvfp4-gemm.md) +- [NVFP4 GEMV](../kernels/nvfp4-gemv.md) diff --git a/wiki/hardware/pdl-gdc.md b/wiki/hardware/pdl-gdc.md index 9e30e5ea9..1436db2cf 100644 --- a/wiki/hardware/pdl-gdc.md +++ b/wiki/hardware/pdl-gdc.md @@ -2,41 +2,114 @@ id: hw-pdl-gdc title: "Programmatic Dependent Launch / Grid Dependency Control" type: hardware -architectures: [sm100, sm100a, sm90] +architectures: [sm100, sm100a, sm90, sm90a] tags: [pdl, gdc] -confidence: source-reported +confidence: verified +evidence_basis: + - source_id: doc-cuda-13 + evidence_type: official-doc + - source_id: doc-ptx-isa-sm100 + evidence_type: official-doc + - source_id: pr-cutlass-2161 + evidence_type: upstream-code related: [technique-persistent-kernels, hw-clc] -sources: [doc-nvidia-tuning-guide, pr-cutlass-2161, doc-cutlass-changelog-sm100] +sources: [doc-cuda-13, doc-ptx-isa-sm100, pr-cutlass-2161] aliases: [PDL, GDC, "programmatic dependent launch", "grid dependency control"] -blackwell_relevance: "PDL available on Hopper but enabled by default on Blackwell SM100." +blackwell_relevance: "PDL remains an explicit CUDA launch-and-kernel protocol on Blackwell; CUTLASS 4.5.0 separately defaults its SM100 GDC compile option on." --- -## Overview +# Programmatic Dependent Launch -PDL/GDC allows overlapping execution of dependent kernel launches. The primary kernel signals it is finishing; the secondary kernel begins before the primary fully completes. +## Contract -## How It Works +Programmatic Dependent Launch (PDL) lets a secondary grid in the same CUDA stream become eligible to start before its prerequisite primary grid completes. It is available on compute capability 9.0 and later, including Hopper and Blackwell. + +PDL creates an **opportunity** for overlap; it does not guarantee that the grids execute concurrently. The useful overlap is normally the primary's work after its launch trigger and the secondary's independent preamble before its dependency wait. + +The CUDA protocol has three required roles: + +1. Every primary CTA either executes `cudaTriggerProgrammaticLaunchCompletion()` or exits. After all CTAs satisfy that condition, the driver may schedule the secondary grid. +2. The host launches the secondary in the same stream with `cudaLaunchAttributeProgrammaticStreamSerialization` and `programmaticStreamSerializationAllowed = 1`. +3. Every secondary thread waits with `cudaGridDependencySynchronize()` before it consumes prerequisite results. The wait completes after the prerequisite grids finish and their memory operations are visible. + +If the primary does not explicitly trigger, its CTAs implicitly satisfy the trigger only as they exit. Omitting the explicit trigger is correct but removes the intended primary-tail overlap. + +## CUDA role skeleton + +This minimal skeleton shows where each operation belongs; real kernels put independent and dependent work at the indicated points and add normal error checking: ```cuda -// Primary kernel signals near completion -cudaGridDependencySynchronize(); // or PTX equivalent +#include + +__global__ void primary() { + // Produce everything needed before the secondary may be launched. + if (threadIdx.x == 0) { + cudaTriggerProgrammaticLaunchCompletion(); + } + // Primary tail that does not change data consumed by the secondary. +} + +__global__ void secondary() { + // Independent preamble may execute early. + cudaGridDependencySynchronize(); + // Dependent reads begin only after this wait. +} + +void launch_pdl(dim3 grid, dim3 block, cudaStream_t stream) { + cudaLaunchConfig_t cfg{}; + cfg.gridDim = grid; + cfg.blockDim = block; + cfg.stream = stream; + + cudaLaunchAttribute attr{}; + attr.id = cudaLaunchAttributeProgrammaticStreamSerialization; + attr.val.programmaticStreamSerializationAllowed = 1; + cfg.attrs = &attr; + cfg.numAttrs = 1; + + primary<<>>(); + cudaLaunchKernelEx(&cfg, secondary); +} +``` + +Do not place `cudaGridDependencySynchronize()` in the primary: it is the secondary-side wait, not the launch trigger. Do not replace the wait with an ordinary memory fence. PDL's wait supplies both prerequisite-grid completion and visibility for its dependent work. + +## PTX mapping -// Secondary kernel can start overlapping with primary's tail -// Enabled by default on SM100 (opt-in on SM90) +CUDA lowers the two device roles to Grid Dependency Control instructions: + +```ptx +// Primary CTA: makes designated dependent grids eligible after every CTA +// has issued this instruction or completed. +griddepcontrol.launch_dependents; + +// Secondary thread: waits for in-flight prerequisite grids and visibility. +griddepcontrol.wait; ``` -## Blackwell Default Behavior +`griddepcontrol` was introduced in PTX ISA 7.8 and requires `sm_90` or newer. Repeating `launch_dependents` within one CTA has no additional effect after that CTA's first invocation. If a prerequisite uses `launch_dependents`, its dependent must use `griddepcontrol.wait` or an equivalent CUDA dependency wait for correct execution. + +## CUDA support versus CUTLASS defaults + +Blackwell does not make arbitrary back-to-back launches overlap automatically. CUDA applications still opt the secondary launch into programmatic stream serialization and implement the device-side trigger/wait protocol. + +CUTLASS has a separate build choice. In CUTLASS 4.5.0, the CMake option `CUTLASS_ENABLE_GDC_FOR_SM100` defaults to `ON`, while the SM90 option is opt-in. That default only enables eligible CUTLASS code to emit its GDC wrappers; it is not a device-wide CUDA default and can be overridden by the build. + +## When to use it + +PDL can help only when all of these conditions hold: + +- the secondary has enough independent preamble to overlap; +- the primary has useful tail work after every CTA reaches the trigger; +- the two grids have enough simultaneous resource headroom; and +- the saved launch/serialization time exceeds the protocol and occupancy costs. -On SM100, PDL is **enabled by default** — no opt-in needed. This means: -- Back-to-back kernel launches naturally overlap -- Memory fences ensure correctness for dependent data -- Reduces kernel launch gaps in compute-heavy pipelines +Small-kernel chains, GEMM/epilogue sequences, and pipeline-parallel stages are candidates, not guaranteed wins. Profile an explicit non-PDL baseline and record GPU, clocks, launch shapes, stream/graph configuration, input sizes, warmup, repetitions, and the observed overlap timeline. Never rely on overlap for forward progress; CUDA documents it as opportunistic and warns that such reliance can deadlock. -## When It Matters -- Chains of small kernels (e.g., MoE dispatch → compute → combine) -- Pipeline-parallel training with many sequential kernel launches -- Reduces overall wall-clock time without code changes on Blackwell +## References -## Related -- [persistent-kernels](../techniques/persistent-kernels.md) — Alternative approach to reducing launch overhead -- [clc](clc.md) — Dynamic scheduling within persistent kernels +- [CUDA 13.0.2: Programmatic Dependent Launch](https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#programmatic-dependent-launch-and-synchronization) +- [PTX ISA 9.0: `griddepcontrol`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-griddepcontrol) +- [CUTLASS 4.5.0: dependent kernel launch](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/dependent_kernel_launch.md) +- [Persistent kernels](../techniques/persistent-kernels.md) +- [Cluster Launch Control](clc.md) diff --git a/wiki/hardware/tcgen05-mma.md b/wiki/hardware/tcgen05-mma.md index add1433bd..d69118261 100644 --- a/wiki/hardware/tcgen05-mma.md +++ b/wiki/hardware/tcgen05-mma.md @@ -6,12 +6,14 @@ architectures: [sm100, sm100a] tags: [tcgen05, tmem, mbarrier] confidence: verified evidence_basis: - - source_id: doc-nvidia-tuning-guide + - source_id: doc-ptx-isa-sm100 evidence_type: official-doc - source_id: pr-cutlass-2139 evidence_type: upstream-code + - source_id: blog-tcgen05-tutorial + evidence_type: source-reported related: [hw-tmem, hw-2sm-cooperative, technique-warp-specialization] -sources: [pr-cutlass-2139, doc-nvidia-tuning-guide, blog-tcgen05-tutorial, blog-colfax-cutlass] +sources: [doc-ptx-isa-sm100, pr-cutlass-2139, blog-tcgen05-tutorial] aliases: [UMMA, tcgen05, "tensor core gen 05"] --- @@ -19,299 +21,89 @@ aliases: [UMMA, tcgen05, "tensor core gen 05"] ## Overview -`tcgen05.mma` is the Blackwell (SM100/SM100a) matrix-multiply-accumulate instruction that replaces Hopper's `wgmma.mma_async`. The name stands for **Tensor Core Generation 05**. NVIDIA also refers to the higher-level abstraction as **UMMA** (Unified Matrix Multiply-Accumulate) in the CUTLASS framework. +`tcgen05.mma` is NVIDIA PTX's fifth-generation TensorCore matrix-multiply-accumulate family. PTX ISA 9.0 supports its architecture-specific forms on `sm_100a`; CUTLASS uses the `UMMA` namespace for its SM100 wrappers. NVIDIA's pinned PTX and CUTLASS sources do not define an expanded form of the `UMMA` acronym. -Key differences from `wgmma`: +The programming-model shift from Hopper WGMMA is precise: -| Property | wgmma (SM90) | tcgen05.mma (SM100) | +| Property | WGMMA | tcgen05.mma | |---|---|---| -| Issuing scope | Warpgroup (4 warps, 128 threads) | Single thread | -| Accumulator storage | Registers (high pressure) | Tensor Memory (TMEM, 256KB/SM) | -| Operand A source | Registers or SMEM | Shared memory only | -| Operand B source | Shared memory | Shared memory only | -| Matrix load | ldmatrix to registers | Direct from SMEM (no ldmatrix) | -| Synchronization | Warpgroup-scoped barriers | Fully async, fence-based | +| Issue granularity | Warpgroup | One thread for `cta_group::1` or `cta_group::2` | +| D accumulator | Per-thread registers | Tensor Memory (TMEM) | +| A location | Register or SMEM forms, depending on instruction | SMEM descriptor or TMEM address | +| B location | SMEM descriptor | SMEM descriptor | +| Completion | WGMMA commit/wait groups | `tcgen05.commit` plus an mbarrier wait | -## Instruction Variants +Single-thread issue reduces the number of issuing threads, but it does not remove asynchronous completion, operand-lifetime, or inter-thread ordering requirements. -tcgen05.mma has 7 variants organized by precision and scaling mode: +## Kinds and shapes -| Variant | A Type | B Type | Accumulator | Scale | MMA Shape (1SM) | Notes | -|---|---|---|---|---|---|---| -| `tcgen05.mma.kind::f16` | FP16/BF16 | FP16/BF16 | FP32 | None | m128n256k16 | Standard half-precision | -| `tcgen05.mma.kind::tf32` | TF32 | TF32 | FP32 | None | m128n256k8 | Single-precision approximation | -| `tcgen05.mma.kind::f8f6f4` | FP8/FP6/FP4 | FP8/FP6/FP4 | FP32 | Block (UE8M0) | m128n256k32 | Narrow precision with native block scaling | -| `tcgen05.mma.kind::i8` | INT8 | INT8 | INT32 | None | m128n256k32 | Integer quantized inference | -| `tcgen05.mma.kind::mxf8` | MXFP8 | MXFP8 | FP32 | MX (E8M0) | m128n256k32 | Microscaling FP8 | -| `tcgen05.mma.kind::mxf4` | MXFP4 | MXFP4 | FP32 | MX (E8M0) | m128n256k64 | Microscaling FP4 | -| `tcgen05.mma.kind::mxf4nvf4` | NVFP4 | MXFP4 | FP32 | Mixed | m128n256k64 | Mixed NVFP4/MXFP4 | +PTX ISA 9.0 divides dense `tcgen05.mma` into these grammar groups: -## MMA Shapes: 1-SM vs 2-SM +| Grammar group | Kind qualifiers | +|---|---| +| Floating point, without block scaling | `f16`, `tf32`, `f8f6f4` | +| Floating point, with block scaling | `mxf8f6f4`, `mxf4`, `mxf4nvf4` | +| Integer | `i8` | -### 1-SM (Single CTA) Shapes +Block-scaled forms add `.block_scale` and accept separate TMEM addresses for A and B scale factors. `f8f6f4` is not the block-scaled MX kind, and `mxf8` is not a dense kind token in this grammar. -In single-SM mode, a single CTA owns the full MMA operation: +M and N are encoded in the instruction descriptor (`idesc`) and are constrained by kind, CTA group, layouts, and target ISA. Consequently, names such as m128n256k16 and m256n256k16 are useful maximum-shape examples for F16/BF16 configurations, not the only legal M and N values. `cta_group::1` uses current-CTA resources; `cta_group::2` can also access the paired peer CTA's SMEM and TMEM resources. All `tcgen05` instructions in a kernel must use the same CTA-group value. -- **BF16/FP16**: M=128, N=256, K=16 -- **TF32**: M=128, N=256, K=8 -- **FP8/FP6/FP4**: M=128, N=256, K=32 -- **MXFP4/NVFP4**: M=128, N=256, K=64 +## Operand grammar -### 2-SM (Cooperative) Shapes - -In two-SM cooperative mode, two CTAs share a single MMA across paired SMs: - -- **BF16/FP16**: M=256, N=256, K=16 -- **TF32**: M=256, N=256, K=8 -- **FP8/FP6/FP4**: M=256, N=256, K=32 -- **MXFP4/NVFP4**: M=256, N=256, K=64 - -The M dimension doubles because each SM contributes 128 rows from its own TMEM partition. - -## Single-Thread Issuance Model - -Unlike `wgmma` which required coordinated issuance from a warpgroup (4 warps, 128 threads), `tcgen05.mma` is issued by a **single thread**. This is a fundamental architectural simplification: - -1. **No warpgroup synchronization overhead** -- one elected thread (typically lane 0 of warp 0) issues the MMA. -2. **Fully asynchronous** -- the instruction returns immediately; the hardware pipeline executes the MMA in the background. -3. **Fence-based completion** -- the producer must insert explicit fences before reading results from TMEM. - -```cuda -// Single-thread MMA issuance pattern -__device__ void issue_mma(uint32_t tmem_addr, uint64_t smem_desc_a, uint64_t smem_desc_b) { - // Only one thread issues the MMA - if (threadIdx.x == 0) { - asm volatile( - "tcgen05.mma.cta_group::1.kind::f16 " - "[%0], %1, %2, %3, 0;" - : - : "r"(tmem_addr), - "l"(smem_desc_a), - "l"(smem_desc_b), - "r"(0) // scale descriptor (unused for f16) - ); - } -} -``` - -## PTX Examples - -### Basic BF16 MMA (1-SM) +The unscaled floating-point grammar includes a disable-output-lane vector before the accumulation predicate: ```ptx -// Issue a 128x256x16 BF16 MMA -// Operand A: shared memory descriptor -// Operand B: shared memory descriptor -// Accumulator: TMEM address -tcgen05.mma.cta_group::1.kind::f16 [tmem_addr], desc_a, desc_b, idesc, 0; +tcgen05.mma.cta_group.kind [d-tmem], a-desc, b-desc, idesc, + { disable-output-lane }, enable-input-d + {, scale-input-d}; ``` -### 2-SM Cooperative BF16 MMA +For `cta_group::1`, the lane-disable vector contains four 32-bit values; for `cta_group::2`, it contains eight. Setting `enable-input-d` false computes `D = A*B`; setting it true computes `D = A*B+D`. The optional `scale-input-d` is limited to the forms documented by the ISA. -```ptx -// Issue a 256x256x16 BF16 MMA across two paired CTAs -// cta_group::2 indicates cooperative mode -tcgen05.mma.cta_group::2.kind::f16 [tmem_addr], desc_a, desc_b, idesc, 0; -``` - -### FP8 with Block Scaling +The block-scaled grammar is structurally different: ```ptx -// FP8 with native UE8M0 block scaling -// scale_desc encodes the per-block scale factors -tcgen05.mma.cta_group::1.kind::f8f6f4 [tmem_addr], desc_a, desc_b, idesc, scale_desc; -``` - -### CUDA Inline PTX for BF16 MMA with Accumulation - -```cuda -__device__ void mma_bf16_128x256x16( - uint32_t tmem_addr, - uint64_t desc_a, - uint64_t desc_b -) { - if (threadIdx.x == 0) { - // First MMA: zero-initialize accumulator - asm volatile( - "tcgen05.mma.cta_group::1.kind::f16 " - "[%0], %1, %2, %3, 0;" - : - : "r"(tmem_addr), "l"(desc_a), "l"(desc_b), "r"(0) - ); - } -} - -__device__ void mma_bf16_accumulate( - uint32_t tmem_addr, - uint64_t desc_a, - uint64_t desc_b -) { - if (threadIdx.x == 0) { - // Subsequent MMAs: accumulate into existing TMEM - // The enable_accumulate flag controls whether to add to or overwrite TMEM - asm volatile( - "tcgen05.mma.cta_group::1.kind::f16 " - "[%0], %1, %2, %3, 1;" // last arg 1 = accumulate - : - : "r"(tmem_addr), "l"(desc_a), "l"(desc_b), "r"(0) - ); - } -} +tcgen05.mma.cta_group.kind.block_scale.scale_vectorsize + [d-tmem], a-desc, b-desc, idesc, + [scale-A-tmem], [scale-B-tmem], enable-input-d; ``` -## Critical Fences +These are normative grammar excerpts, not complete inline-assembly functions: real code must also build valid descriptors, allocate TMEM, preserve operand lifetimes, and implement completion. -Fences are **mandatory** for correctness. The hardware does not implicitly synchronize between MMA and TMEM reads/writes. +## Completion and inter-thread ordering -### tcgen05.fence +`tcgen05.mma` is asynchronous. Two mechanisms serve different purposes: -Insert before reading MMA results from TMEM: +1. `tcgen05.commit.cta_group::N.mbarrier::arrive::one.b64` makes an mbarrier track completion of prior asynchronous tcgen05 operations issued by the executing thread. Waiting on that mbarrier observes completion. +2. `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` order tcgen05 operations across an execution-ordering handoff and constrain code motion. A fence is not a completion wait. -```cuda -// Fence before CTA sync and TMEM reads after the completion mechanism. -__device__ void fence_before_tmem_read() { - asm volatile("tcgen05.fence::before_thread_sync;"); - __syncthreads(); // Ensure all threads see the fence -} -``` +A cross-thread result handoff therefore needs both the applicable completion protocol and the applicable fence/execution-ordering protocol. Likewise, a pipelined mainloop must not release or overwrite an SMEM stage until the asynchronous MMA has finished consuming it. -### tcgen05.fence variants +## Shared-memory descriptors -```ptx -// Fence before reading TMEM (most common) -tcgen05.fence::before_thread_sync; +The tcgen05 shared-memory descriptor is a 64-bit runtime value. PTX ISA 9.0 assigns fields for the encoded base address, leading dimension, stride dimension, fixed bits, base offset, leading-dimension mode, and a three-bit swizzle mode. -// Fence after CTA sync before issuing dependent tcgen05 operations -tcgen05.fence::after_thread_sync; -``` +Valid swizzle encodings include no swizzle, 128-byte, 64-byte, and 32-byte layouts (plus a 128-byte/32-byte-atomic mode). Values 3, 5, and 7 are invalid; ordinary 128-byte swizzling uses value 2. A layout must satisfy the addressing and alignment constraints for its chosen mode. Although 128-byte swizzling can materially improve a particular GEMM, it is not a universal correctness requirement. -### Correct Mainloop Fence Pattern +## Register pressure and specialization -```cuda -__device__ void gemm_mainloop(/* params */) { - for (int k_tile = 0; k_tile < num_k_tiles; ++k_tile) { - // 1. Wait for operand data to arrive in SMEM - wait_barrier(k_tile % NUM_STAGES); +For comparison, CUTLASS's pinned Hopper wrapper for m64n256k16 with FP32 accumulation declares 128 accumulator registers per participating thread. tcgen05 keeps D in TMEM during the MMA sequence, which can free GPR capacity for data movement and epilogue work. CUTLASS examples use this model in warp-specialized pipelines with distinct TMA, MMA-control, and accumulator-consumer roles; the best partition remains kernel-dependent. - // 2. Issue MMA (single thread) - if (threadIdx.x == 0) { - asm volatile( - "tcgen05.mma.cta_group::1.kind::f16 " - "[%0], %1, %2, %3, 1;" - : - : "r"(tmem_addr), - "l"(make_smem_desc(smem_a, k_tile)), - "l"(make_smem_desc(smem_b, k_tile)), - "r"(0) - ); - } +## Source-reported performance progression - // 3. Release SMEM buffer for next TMA load - if (threadIdx.x == 0) { - arrive_barrier((k_tile + 1) % NUM_STAGES); - } - } +Gau Nernst reports the following results for M=N=K=4096 on a Modal B200, using PyTorch 2.9.1 with CUDA 13. Values are measurements for that setup, not architecture-wide guarantees. - // 4. CRITICAL: Fence before reading accumulator from TMEM - asm volatile("tcgen05.fence::before_thread_sync;"); - __syncthreads(); +| Tutorial version | Reported TFLOP/s | Approx. cuBLAS share | +|---|---:|---:| +| cuBLAS | 1506.74 | 100% | +| v1a: basic tcgen05 + 2D 16B TMA | 254.62 | 17% | +| v1b: 3D 16B TMA | 252.81 | 17% | +| v2a: 2D 128B TMA | 681.20 | 45% | +| v2b: 3D 128B TMA | 695.43 | 46% | +| v3: pipelining | 939.61 | 62% | +| v4: warp specialization | 1208.83 | 80% | +| v5: 2-SM MMA | 1302.29 | 86% | +| v6: persistent, static scheduling | 1475.93 | 98% | - // 5. Now safe to read results from TMEM - read_tmem_accumulator(tmem_addr, output); -} -``` - -## Comparison with wgmma - -### Programming Model Shift - -```cuda -// ---- HOPPER (SM90): wgmma ---- -// Requires warpgroup-scoped execution -// All 128 threads in a warpgroup participate -__device__ void hopper_mma() { - // Load A matrix into registers via ldmatrix - uint32_t a_frag[4]; - asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 " - "{%0,%1,%2,%3}, [%4];" - : "=r"(a_frag[0]), "=r"(a_frag[1]), - "=r"(a_frag[2]), "=r"(a_frag[3]) - : "r"(smem_addr)); - - // Issue wgmma -- warpgroup scope, register accumulator - asm volatile("wgmma.mma_async.sync.aligned.m64n256k16.f32.bf16.bf16 " - "{%0, %1, ...}, " // register accumulators (128+ registers!) - "{%N, %N+1, ...}, " // A operand in registers - "desc_b, ...;" - : "+f"(acc[0]), "+f"(acc[1]), ... - : ...); - - // Commit and wait - asm volatile("wgmma.commit_group.sync.aligned;"); - asm volatile("wgmma.wait_group.sync.aligned 0;"); - // Accumulators now in registers -- high register pressure -} - -// ---- BLACKWELL (SM100): tcgen05 ---- -// Single-thread issuance, TMEM accumulator -__device__ void blackwell_mma() { - // No ldmatrix needed -- reads directly from SMEM - // No register allocation for accumulators - - // Single thread issues MMA - if (threadIdx.x == 0) { - asm volatile( - "tcgen05.mma.cta_group::1.kind::f16 " - "[%0], %1, %2, %3, 1;" - : - : "r"(tmem_addr), // accumulator in TMEM (not registers!) - "l"(desc_a), // A from SMEM descriptor - "l"(desc_b), // B from SMEM descriptor - "r"(0) - ); - } - - // Fence + sync before reading results - asm volatile("tcgen05.fence::before_thread_sync;"); - __syncthreads(); - // Read from TMEM -- registers free for other work -} -``` - -### Performance Implications - -- **Register pressure**: Hopper wgmma uses 128+ registers for accumulators in a large GEMM tile. Blackwell stores accumulators in TMEM, freeing those registers for data movement and epilogue logic. -- **Occupancy**: Lower register pressure enables higher CTA occupancy or larger tile sizes without spilling. -- **Warp specialization**: With tcgen05, a single MMA-producer warp can feed the tensor cores while other warps handle TMA loads, epilogue, or softmax -- a natural fit for FlashAttention-style kernels. - -## SMEM Layout Requirements - -tcgen05.mma requires **128-byte swizzled** shared memory layouts for both operands. Non-swizzled or 64-byte swizzled layouts will produce incorrect results silently. - -```cuda -// Shared memory descriptor construction for tcgen05 -// The descriptor encodes: base address, stride, swizzle mode, dimensions -__device__ uint64_t make_smem_desc(void* smem_ptr, int stride_bytes) { - uint64_t desc = 0; - uint32_t addr = static_cast(__cvta_generic_to_shared(smem_ptr)); - - // Encode base address (bits 0-13) - desc |= (uint64_t)(addr >> 4); - // Encode leading dimension stride (bits 16-29) - desc |= (uint64_t)((stride_bytes >> 4) & 0x3FFF) << 16; - // Encode 128-byte swizzle mode (bits 62-63) -- MANDATORY for tcgen05 - desc |= (uint64_t)(3) << 62; // 3 = 128-byte swizzle - - return desc; -} -``` - -## Performance Progression (from tcgen05-for-dummies) - -| Optimization Stage | Throughput (TFLOPS) | % of cuBLAS | -|---|---|---| -| Naive tcgen05.mma | 255 | 17% | -| + 128B swizzled SMEM | 695 | 46% | -| + TMA pipelining | 940 | 62% | -| + Persistent kernel + CLC | 1476 | 98% | -| cuBLAS reference | 1507 | 100% | +The v6 result did not use Cluster Launch Control; the author lists CLC and threadblock swizzling as unimplemented follow-up ideas. diff --git a/wiki/hardware/tma.md b/wiki/hardware/tma.md index c55e32840..02073bc1f 100644 --- a/wiki/hardware/tma.md +++ b/wiki/hardware/tma.md @@ -4,368 +4,138 @@ title: "Tensor Memory Accelerator (TMA)" type: hardware architectures: [sm100, sm100a, sm90, sm90a] tags: [tma, mbarrier] -confidence: source-reported -related: [hw-tcgen05-mma, technique-pipeline-stages, technique-swizzling] -sources: [doc-nvidia-tuning-guide, blog-tcgen05-tutorial, pr-flashinfer-2387] -aliases: [TMA, "tensor memory accelerator", "cp.async.bulk"] -blackwell_relevance: "TMA is shared with Hopper but enhanced on Blackwell. 128-byte swizzling mandatory for tcgen05 inputs." +confidence: verified +evidence_basis: + - source_id: doc-cuda-13-0-2-tma + evidence_type: official-doc + - source_id: doc-ptx-isa-sm100 + evidence_type: official-doc + - source_id: pr-cutlass-2139 + evidence_type: upstream-code +related: [hw-mbarrier, hw-tcgen05-mma, technique-pipeline-stages, technique-swizzling] +sources: [doc-cuda-13-0-2-tma, doc-ptx-isa-sm100, pr-cutlass-2139] +aliases: [TMA, "tensor memory accelerator", "cp.async.bulk.tensor"] +blackwell_relevance: "Blackwell retains Hopper's descriptor-driven TMA and adds tensor-map and copy modes; a TMA destination layout must match the exact tcgen05 shared-memory descriptor rather than one universal swizzle." --- -# Tensor Memory Accelerator (TMA) - -## Overview - -The Tensor Memory Accelerator (TMA) is a hardware unit that performs **asynchronous bulk data transfers** between global memory and shared memory. First introduced on Hopper (SM90), TMA carries forward to Blackwell (SM100) with stricter requirements for tcgen05 compatibility. - -TMA offloads data movement from CUDA cores entirely -- a single thread issues the transfer, and the TMA hardware engine handles the multi-dimensional copy, address calculation, out-of-bounds clamping, and format conversion. - -## Key Properties - -| Property | Detail | -|---|---| -| Transfer direction | GMEM <-> SMEM (bidirectional) | -| Dimensionality | 1D to 5D tensor copies | -| Max transfer size | Up to 256 bytes per element, tiles up to 128x256 | -| Swizzle modes | None, 32B, 64B, 128B (128B required for tcgen05) | -| Format conversion | FP32<->BF16, FP32<->FP16 during transfer | -| Multicast | Single GMEM tile -> multiple CTAs in a cluster | -| Synchronization | mbarrier-based (arrive/wait) | -| Thread requirement | Single thread issues the operation | - -## TMA Descriptor - -TMA operations are driven by a **descriptor** that encodes the tensor layout, addressing, and transfer parameters. The descriptor is created on the host and passed to the kernel. - -### Host-Side Descriptor Creation - -```cuda -#include - -// Create a 2D TMA descriptor for a row-major FP16 matrix -CUtensorMap create_tma_descriptor_2d( - const half* global_ptr, - int M, int N, // Global tensor dimensions - int tile_m, int tile_n, // Tile dimensions for each transfer - int swizzle_bytes // Swizzle mode: 0, 32, 64, 128 -) { - CUtensorMap tensor_map; - - // Tensor dimensions (outermost to innermost) - uint64_t global_dims[2] = {(uint64_t)N, (uint64_t)M}; - uint64_t global_strides[1] = {(uint64_t)(N * sizeof(half))}; - - // Tile box dimensions - uint32_t box_dims[2] = {(uint32_t)tile_n, (uint32_t)tile_m}; - - // Element strides (1 = contiguous) - uint32_t elem_strides[2] = {1, 1}; - - CUtensorMapSwizzle swizzle; - switch (swizzle_bytes) { - case 0: swizzle = CU_TENSOR_MAP_SWIZZLE_NONE; break; - case 32: swizzle = CU_TENSOR_MAP_SWIZZLE_32B; break; - case 64: swizzle = CU_TENSOR_MAP_SWIZZLE_64B; break; - case 128: swizzle = CU_TENSOR_MAP_SWIZZLE_128B; break; - } - - cuTensorMapEncodeTiled( - &tensor_map, - CU_TENSOR_MAP_DATA_TYPE_FLOAT16, // Element type - 2, // Dimensionality - (void*)global_ptr, // Global pointer - global_dims, // Tensor dimensions - global_strides, // Byte strides (exclude innermost) - box_dims, // Tile/box dimensions - elem_strides, // Element strides - CU_TENSOR_MAP_INTERLEAVE_NONE, - swizzle, - CU_TENSOR_MAP_L2_PROMOTION_NONE, - CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE - ); - - return tensor_map; -} -``` +# Tensor Memory Accelerator -### Blackwell Requirement: 128-Byte Swizzle +## Scope -On Blackwell, `tcgen05.mma` requires operands in **128-byte swizzled** SMEM layout. If TMA loads data without 128B swizzling, the MMA will produce incorrect results. +TMA is the descriptor-driven `cp.async.bulk.tensor` facility introduced for Hopper (`sm_90`) and retained on Blackwell. One thread can issue a non-blocking rank-1 through rank-5 tensor copy while the hardware performs multidimensional address traversal and moves the tile. -```cuda -// CORRECT for Blackwell tcgen05: -CUtensorMap desc = create_tma_descriptor_2d(ptr, M, N, 128, 64, 128); -// swizzle=128 ^^^ +The principal copy directions and completion mechanisms are: -// WRONG for tcgen05 (will silently produce garbage): -CUtensorMap desc = create_tma_descriptor_2d(ptr, M, N, 128, 64, 0); -// swizzle=0 ^^^ -``` +| Direction | Destination | Optional cluster behavior | Completion | +|---|---|---|---| +| Global to shared | Issuing CTA or a CTA in its cluster | One masked instruction can multicast to selected CTAs | mbarrier `complete_tx` in bytes | +| Shared to global | Issuing CTA's shared memory to a tensor map | Scatter modes are available on supported Blackwell targets | Bulk async group: issue, commit, wait | -## Asynchronous Copy Operations - -### GMEM to SMEM (Load) - -```cuda -// TMA load: single thread issues, hardware executes asynchronously -__device__ void tma_load_tile( - const CUtensorMap* desc, - void* smem_ptr, - uint64_t* mbar_ptr, // mbarrier for synchronization - int coord_x, int coord_y -) { - if (threadIdx.x == 0) { - // Set expected bytes on the mbarrier - uint32_t expected_bytes = TILE_M * TILE_N * sizeof(half); - asm volatile( - "mbarrier.arrive.expect_tx.shared.b64 _, [%0], %1;" - : - : "r"((uint32_t)__cvta_generic_to_shared(mbar_ptr)), - "r"(expected_bytes) - ); - - // Issue TMA copy - asm volatile( - "cp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes " - "[%0], [%1, {%2, %3}], [%4];" - : - : "r"((uint32_t)__cvta_generic_to_shared(smem_ptr)), - "l"(desc), - "r"(coord_x), "r"(coord_y), - "r"((uint32_t)__cvta_generic_to_shared(mbar_ptr)) - ); - } -} -``` +TMA copies the datatype represented by the tensor map. It does not provide a general FP32-to/from-FP16 or BF16 conversion step. For a tiled global-to-shared load, out-of-bounds elements are filled according to the supported tensor-map policy rather than clamped to an edge coordinate. -### SMEM to GMEM (Store) - -```cuda -// TMA store: write a tile from shared memory back to global memory -__device__ void tma_store_tile( - const CUtensorMap* desc, - const void* smem_ptr, - int coord_x, int coord_y -) { - if (threadIdx.x == 0) { - asm volatile( - "cp.async.bulk.tensor.2d.global.shared::cta " - "[%0, {%1, %2}], [%3];" - : - : "l"(desc), - "r"(coord_x), "r"(coord_y), - "r"((uint32_t)__cvta_generic_to_shared(smem_ptr)) - ); - - // Commit the store - asm volatile("cp.async.bulk.commit_group;"); - } -} -``` +## Tensor maps -## mbarrier Synchronization - -TMA uses mbarriers (memory barriers) for producer-consumer synchronization. The pattern is: - -1. **Producer** (TMA): arrives at the barrier when the transfer completes, decrementing the expected transaction count. -2. **Consumer** (compute warps): waits on the barrier before reading the loaded data. - -### Pipeline Stage Pattern - -```cuda -// Multi-stage pipeline with TMA + mbarrier -__device__ void pipelined_mainloop( - const CUtensorMap* desc_a, - const CUtensorMap* desc_b, - void* smem_a_stages[NUM_STAGES], - void* smem_b_stages[NUM_STAGES], - uint64_t* mbar[NUM_STAGES], - int num_k_tiles -) { - // Prologue: fill the first NUM_STAGES-1 stages - for (int s = 0; s < NUM_STAGES - 1 && s < num_k_tiles; ++s) { - tma_load_tile(desc_a, smem_a_stages[s], mbar[s], 0, s); - tma_load_tile(desc_b, smem_b_stages[s], mbar[s], s, 0); - } - - // Mainloop - for (int k = 0; k < num_k_tiles; ++k) { - int stage = k % NUM_STAGES; - - // Wait for data to arrive in this stage - mbarrier_wait(mbar[stage]); - - // Issue MMA using this stage's SMEM buffers - if (threadIdx.x == 0) { - asm volatile( - "tcgen05.mma.cta_group::1.kind::f16 " - "[%0], %1, %2, %3, 1;" - : - : "r"(tmem_acc), - "l"(make_desc(smem_a_stages[stage])), - "l"(make_desc(smem_b_stages[stage])), - "r"(0) - ); - } - - // Prefetch next stage - int next_k = k + NUM_STAGES - 1; - if (next_k < num_k_tiles) { - int next_stage = next_k % NUM_STAGES; - tma_load_tile(desc_a, smem_a_stages[next_stage], - mbar[next_stage], 0, next_k); - tma_load_tile(desc_b, smem_b_stages[next_stage], - mbar[next_stage], next_k, 0); - } - } -} -``` +A tensor map is opaque and is accessed through the tensor-map proxy. `cuTensorMapEncodeTiled` describes: -### mbarrier Operations - -```cuda -// Initialize an mbarrier -__device__ void mbarrier_init(uint64_t* mbar, int arrive_count) { - if (threadIdx.x == 0) { - asm volatile( - "mbarrier.init.shared.b64 [%0], %1;" - : - : "r"((uint32_t)__cvta_generic_to_shared(mbar)), - "r"(arrive_count) - ); - } -} - -// Wait for an mbarrier to complete (phase-based) -__device__ void mbarrier_wait(uint64_t* mbar, int phase) { - uint32_t mbar_addr = (uint32_t)__cvta_generic_to_shared(mbar); - asm volatile( - "{\n" - ".reg .pred p;\n" - "WAIT_LOOP:\n" - "mbarrier.try_wait.parity.shared.b64 p, [%0], %1;\n" - "@!p bra WAIT_LOOP;\n" - "}\n" - : - : "r"(mbar_addr), "r"(phase) - ); -} -``` +- datatype and global base address; +- rank, global dimensions, and byte strides; +- traversal box dimensions and element strides; +- interleave and shared-memory swizzle; +- L2-promotion hint; and +- out-of-bounds fill policy. -## TMA Multicast - -TMA multicast sends a single GMEM tile to **multiple CTAs within a cluster** simultaneously. This is critical for GEMM where the B operand is shared across M-axis tiles. - -```cuda -// Multicast TMA: load B tile to all CTAs in the cluster -__device__ void tma_multicast_load( - const CUtensorMap* desc, - void* smem_ptr, - uint64_t* mbar_ptr, - int coord_x, int coord_y, - uint16_t multicast_mask // bitmask: which CTAs in cluster receive the data -) { - if (threadIdx.x == 0) { - uint32_t expected_bytes = TILE_K * TILE_N * sizeof(half); - asm volatile( - "mbarrier.arrive.expect_tx.shared.b64 _, [%0], %1;" - : - : "r"((uint32_t)__cvta_generic_to_shared(mbar_ptr)), - "r"(expected_bytes) - ); - - asm volatile( - "cp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes.multicast::cluster " - "[%0], [%1, {%2, %3}], [%4], %5;" - : - : "r"((uint32_t)__cvta_generic_to_shared(smem_ptr)), - "l"(desc), - "r"(coord_x), "r"(coord_y), - "r"((uint32_t)__cvta_generic_to_shared(mbar_ptr)), - "h"(multicast_mask) - ); - } -} -``` +For the ordinary non-interleaved tiled path in CUDA Driver API 13.0.97, important constraints include: -### Multicast in GEMM +- the `CUtensorMap` output object is 64-byte aligned; +- the global base and byte strides satisfy the documented alignment rules, commonly at least 16 bytes for the basic types/path; +- every `boxDim` entry is from 1 through 256 elements; +- `boxDim[0] * element_size` is a multiple of 16 bytes; +- every element stride is from 1 through 8; and +- when swizzling is enabled, the inner box in bytes does not exceed the selected swizzle span. -``` -Cluster: 2 CTAs (CTA0 and CTA1) each computing different M-tiles of the same N column +Datatype, interleave, sub-byte, and architecture-specific modes add further restrictions. Always check the encoder's `CUresult`; never use the output after an encoding failure. -CTA0: computes C[0:128, 0:256] -- needs A[0:128, :] and B[:, 0:256] -CTA1: computes C[128:256, 0:256] -- needs A[128:256, :] and B[:, 0:256] +Host encoding is common, but it is not the only Blackwell path. CUDA also documents device-side tensor-map construction and modification on Blackwell. A map modified through the generic proxy must be published to the tensor-map proxy with the required `fence.proxy.tensormap::generic` sequence before a TMA operation consumes it. -B[:, 0:256] is SHARED -- multicast it once from GMEM to both CTAs -A tiles are UNIQUE -- each CTA loads its own A tile +## Global-to-shared load -Result: B bandwidth is halved (1 GMEM read serves 2 CTAs) +PTX ISA 9.0 defines this representative 2D CTA-local form: + +```ptx +cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes + [dst_smem], [tensor_map, {x, y}], [full_barrier]; ``` -## Blackwell-Specific Enhancements +The instruction is non-blocking. A common one-producer phase with one or more loads uses this accounting: -### 128-Byte Swizzle for tcgen05 +1. Initialize and publish the stage's mbarrier with the intended pending-arrival count. +2. The producer performs one `mbarrier.arrive.expect_tx` for its software arrival and the sum of bytes that all loads in this phase will complete. +3. Issue the TMA loads against that barrier. +4. Each completed load performs `complete_tx` for its copied byte count; it does **not** perform another arrival. +5. Consumers wait with the correct state token or per-stage parity and acquire semantics before reading the destination. -All TMA loads feeding `tcgen05.mma` must use 128-byte swizzling. The swizzle pattern rearranges bytes within each 128-byte line to match the tensor core's internal data layout: +The phase completes only after pending arrivals and tx-count are both zero. If a design performs multiple software arrivals instead, its initialized count must match them exactly. See [mbarrier](mbarrier.md) for lifecycle, phase, and memory-ordering rules. -``` -Without swizzle (linear): - Row 0: bytes [0, 1, 2, ..., 127] - Row 1: bytes [128, 129, ..., 255] - -With 128B swizzle: - Row 0: bytes [0, 1, ..., 127] (unchanged) - Row 1: bytes [128, 129, ..., 255] XOR pattern applied - Row 2: bytes [256, ...] XOR pattern applied differently - ... +## Shared-to-global store + +A tensor store uses bulk-group completion rather than the load's mbarrier protocol: + +```ptx +cp.async.bulk.tensor.2d.global.shared::cta.bulk_group + [tensor_map, {x, y}], [src_smem]; +cp.async.bulk.commit_group; +cp.async.bulk.wait_group 0; ``` -The swizzle eliminates bank conflicts when the tensor core reads operand tiles from SMEM. +The wait may be delayed to overlap independent work. It must occur before the issuing thread reuses the source shared memory or before code that requires store completion. `commit_group` creates the group; it is not itself a completion wait. -### TMA + TMEM Integration +## Cluster multicast -On Blackwell, data flows through a characteristic pipeline: +The cluster form copies a global tile to the same shared-memory offset in every CTA selected by a 16-bit mask: -``` -GMEM --[TMA]--> SMEM --[tcgen05.mma]--> TMEM --[tcgen05.ld]--> Registers --[st.global]--> GMEM - ^ | - | v - +---- (epilogue) --------+ - (bias, activation, etc.) +```ptx +cp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes.multicast::cluster + [dst_smem], [tensor_map, {x, y}], [full_barrier], cta_mask; ``` -## Performance Considerations - -| Tip | Detail | -|---|---| -| Maximize TMA utilization | Keep the TMA unit busy with back-to-back loads across pipeline stages | -| Use multicast for shared operands | Reduces GMEM bandwidth by cluster_size x for shared tiles | -| Always use 128B swizzle on Blackwell | Non-128B swizzle produces incorrect tcgen05 results | -| Prefer 2D TMA over manual addressing | TMA handles out-of-bounds clamping, padding, and strided access | -| Pipeline depth | 3-5 stages typically optimal; more stages increase SMEM usage | - -## CuTe-DSL Example - -```python -# CuTe-DSL TMA copy setup for Blackwell GEMM -from cute import * - -# Define TMA copy atom for operand A (BF16, 128x64 tile) -tma_a = make_tma_copy( - SM100_TMA_LOAD_2D, - tensor_a, # global tensor - smem_layout_a, # shared memory layout - tile_shape=(128, 64), # tile dimensions - swizzle=Swizzle(7, 0, 4), # 128-byte swizzle - multicast_mask=None # no multicast for A -) - -# Define TMA copy atom for operand B with multicast -tma_b = make_tma_copy( - SM100_TMA_LOAD_2D_MULTICAST, - tensor_b, - smem_layout_b, - tile_shape=(64, 256), - swizzle=Swizzle(7, 0, 4), # 128-byte swizzle - multicast_mask=cluster_mask # multicast B to all CTAs in cluster -) -``` +For `cta_group::1` (the default), the completion signal is also multicast to the same barrier offset in every selected destination CTA. A correct design therefore: + +- launches an explicit cluster and keeps every destination CTA's shared memory alive; +- initializes corresponding destination barriers before the elected issuer can start; +- uses one cluster-wide elected issuer, not one `threadIdx.x == 0` issuer in every CTA; +- includes only valid destination CTA ranks in the mask; and +- has every destination wait on its own corresponding barrier phase before consuming the tile. + +For GEMM tiles with the same N range and different M ranges, the CTAs use different A tiles but the same B tile, so multicast can avoid duplicate logical B-load requests. It does not promise an exact `cluster_size` reduction in measured DRAM traffic or elapsed time; caches, mask population, transaction behavior, and resource costs affect the result. + +## Swizzle and tcgen05 + +TMA supports no swizzle and multiple swizzled shared-memory layouts, including 32B, 64B, and 128B spans plus newer variants for selected types. Swizzling rearranges chunks across shared-memory banks. The consumer must address the matching logical layout; the exact mapping also depends on the documented shared-memory base-offset rule. + +There is no universal rule that every Blackwell or `tcgen05.mma` input uses 128B swizzling. The TMA tensor-map swizzle, destination base alignment, leading dimension, and the tcgen05 shared-memory descriptor must describe the **same** legal layout. PTX defines no-, 32B-, 64B-, and 128B-swizzled tcgen05 descriptors with kind-, type-, shape-, and layout-specific constraints. + +A matched swizzle can reduce or remove bank conflicts for a particular access pattern. It does not make every possible consumer access conflict-free. + +## Pipeline invariants + +A common Blackwell GEMM data path is: + +`GMEM -> TMA -> SMEM -> tcgen05.mma -> TMEM -> tcgen05.ld -> registers -> output store` + +Each reusable pipeline stage needs two independent ownership transitions: + +- **full:** TMA has finished producing the shared-memory operands, so the MMA consumer may read them; and +- **empty:** asynchronous MMA has stopped reading those operands, so the TMA producer may overwrite the stage. + +Track full and empty state per stage. Account transaction bytes once, track each barrier's own phase, and do not equate CTA synchronization or a tcgen05 fence with async completion. A complete CUTLASS pipeline is safer evidence than a shortened helper that omits one ownership edge. + +Choose tile rank, swizzle, multicast, issue cadence, and stage count from the actual access pattern and resource budget. More distinct stages consume more shared memory; there is no universal optimum of three to five stages. Profile the target GPU and record the copy shape, descriptor, cluster mask, stage count, shared-memory use, occupancy, warmup, repetitions, and baseline. + +## References + +- [CUDA 13.0.2 Programming Guide: TMA](https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#asynchronous-data-copies-using-the-tensor-memory-accelerator-tma) +- [CUDA Driver API 13.0.97: tensor-map management](https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html) +- [PTX ISA 9.0: `cp.async.bulk.tensor`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor) +- [CUTLASS 4.5.0: complete CuTe DSL TMA tutorial](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_tma/tma_v0.py) +- [mbarrier](mbarrier.md) +- [tcgen05 MMA](tcgen05-mma.md) diff --git a/wiki/hardware/tmem.md b/wiki/hardware/tmem.md index 8f02eade1..d0bb4fa97 100644 --- a/wiki/hardware/tmem.md +++ b/wiki/hardware/tmem.md @@ -6,319 +6,121 @@ architectures: [sm100, sm100a] tags: [tmem, tcgen05] confidence: verified evidence_basis: - - source_id: doc-nvidia-tuning-guide + - source_id: doc-ptx-isa-sm100 evidence_type: official-doc - source_id: pr-cutlass-2139 evidence_type: upstream-code -related: [hw-tcgen05-mma, technique-double-buffering, pattern-register-pressure] -sources: [pr-cutlass-2139, doc-nvidia-tuning-guide, blog-tcgen05-tutorial, blog-colfax-cutlass] +related: [hw-tcgen05-mma, hw-2sm-cooperative, technique-double-buffering, pattern-register-pressure] +sources: [doc-ptx-isa-sm100, pr-cutlass-2139] aliases: [TMEM, "tensor memory", "Tensor Memory"] --- # Tensor Memory (TMEM) -## Overview +## What TMEM is -Tensor Memory (TMEM) is a new addressable memory space introduced in the Blackwell architecture (SM100). Each SM contains **256KB of dedicated TMEM**, used primarily as the accumulator storage for `tcgen05.mma` operations. TMEM eliminates the register pressure that plagued Hopper's wgmma, where accumulators consumed 128+ registers per warpgroup. +Tensor Memory is an addressable, on-chip memory introduced for fifth-generation Tensor Core operations on Blackwell. Each SM has 256 KiB, organized as 128 lanes by 512 columns of 32-bit cells. `tcgen05.mma` writes its D accumulator to TMEM; depending on the instruction form, A can also reside there. -## Architecture Layout +Moving D out of general-purpose registers changes the resource balance relative to Hopper WGMMA. For example, CUTLASS 4.5.0's SM90 m64n256 FP32 WGMMA wrapper exposes 128 accumulator registers **per participating thread**. TMEM avoids that particular register-resident D fragment, but it does not remove registers needed by operands, control flow, or the epilogue. -TMEM is organized as a 2D matrix: +## Addressing and warp access -| Dimension | Size | Description | -|---|---|---| -| Rows | 128 | Mapped to warp lanes (4 warps x 32 lanes) | -| Columns | 512 | 32-bit (4-byte) elements per row | -| Total | 128 x 512 x 4 bytes = 256 KB | Per-SM capacity | +A TMEM address (`taddr`) is a 32-bit value: -### Row-to-Lane Mapping +| Bits | Meaning | +|---|---| +| 31:16 | TMEM lane | +| 15:0 | TMEM column | -TMEM rows are mapped to warp lanes within a CTA: +The lane is a TMEM data-path lane, not storage independently owned by a CUDA thread. `tcgen05.ld` and `tcgen05.st` are warp-collective. Under their four-warp access model, the warps cover these lane chunks: -``` -Row 0-31: Warp 0, lanes 0-31 -Row 32-63: Warp 1, lanes 0-31 -Row 64-95: Warp 2, lanes 0-31 -Row 96-127: Warp 3, lanes 0-31 -``` +| Participating warp | TMEM lanes | +|---|---| +| 0 | 0-31 | +| 1 | 32-63 | +| 2 | 64-95 | +| 3 | 96-127 | -Each thread "owns" the TMEM row corresponding to its warp and lane. When reading from TMEM, thread `T` in warp `W` accesses row `W*32 + T%32`. +The chosen load/store shape determines how values from that lane-column region map to each thread's register vector. Use the PTX shape-specific layout tables rather than treating a logical MxN accumulator as a universal row-major array owned one row per thread. -### Column Addressing +## Allocation lifecycle -Columns are addressed via a column offset in the TMEM descriptor. A 128x256 MMA accumulator tile occupies: +TMEM has an explicit software-managed lifetime: -- 128 rows (all lanes across 4 warps) -- 256 columns of FP32 values = 1024 bytes per row +1. Reserve shared memory for the 32-bit allocation result. +2. Execute `tcgen05.alloc` collectively from one warp for `cta_group::1`, or from two warps—one per paired CTA—for `cta_group::2`. +3. Synchronize consumers as required, then read the returned base `taddr` from shared memory. +4. Use the allocation for MMA, copy, load, or store operations with the same CTA-group mode. +5. Execute `tcgen05.dealloc` with the corresponding collective issue pattern. +6. Deallocate every TMEM allocation before kernel exit. -``` -TMEM Layout for 128x256 FP32 accumulator: - col 0 col 255 col 511 - |------------|-----------|-----------| - | Acc Tile | (free) | (free) | row 0 (warp0, lane0) - | 128x256 | | | row 1 (warp0, lane1) - | FP32 | | | ... - | | | | row 127 (warp3, lane31) - |------------|-----------|-----------| -``` +The allocation operand counts columns. Legal allocation sizes are powers of two from 32 through 512 columns. Allocation can block until the requested TMEM is available; the ISA does not define folklore outcomes such as silent corruption for an invalid size. -## Allocation and Deallocation Lifecycle - -TMEM is **explicitly managed** by the programmer. There is no automatic allocation or garbage collection. - -### Allocation - -```cuda -// Shared storage for CTA-wide broadcast of TMEM address -__shared__ uint32_t s_tmem_addr; - -__device__ uint32_t tmem_alloc_cta(uint32_t num_cols) { - // Only thread 0 allocates; result must reach ALL warps in the CTA. - // __shfl_sync is warp-local — it cannot broadcast across warps. - if (threadIdx.x == 0) { - uint32_t smem_addr = - static_cast(__cvta_generic_to_shared(&s_tmem_addr)); - asm volatile( - "tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;" - : - : "r"(smem_addr), "r"(num_cols) - ); - } - __syncthreads(); // All warps now see s_tmem_addr - return s_tmem_addr; -} -``` +### Column budgeting -Key points: -- `num_cols` specifies the number of 32-bit columns to allocate. -- A 128x256 FP32 accumulator needs 256 columns. -- Allocation is **CTA-scoped** -- all threads in the CTA share the same TMEM region. -- Only one thread (typically thread 0) issues the allocation. -- The returned `tmem_addr` is the base column index. - -### Deallocation - -```cuda -__device__ void tmem_dealloc(uint32_t tmem_addr, uint32_t num_cols) { - if (threadIdx.x == 0) { - asm volatile( - "tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;" - : - : "r"(tmem_addr), "r"(num_cols) - ); - } - __syncthreads(); -} -``` - -**Warning**: Failure to deallocate TMEM before CTA exit will leak memory and prevent subsequent CTAs from allocating, leading to hangs in persistent kernels. +All allocations share the SM's 512-column capacity. Some useful whole-allocation budgets are: -### Lifecycle in a Persistent GEMM Kernel +| Allocation per stage | Maximum stages if TMEM has no other use | Columns left | +|---|---:|---:| +| 128 columns | 4 | 0 | +| 256 columns | 2 | 0 | +| 512 columns | 1 | 0 | -```cuda -__global__ void persistent_gemm_kernel(/* ... */) { - // 1. Allocate TMEM for accumulators - uint32_t tmem_acc = tmem_alloc(256); // 256 cols for M=128, N=256 +A logical need of 192 columns cannot be requested directly: reserve 256 columns, or suballocate that logical region within another legal power-of-two reservation. Two 256-column accumulator stages consume the entire capacity, so scale-factor tensors or other scratch state must fit inside those reservations or the pipeline must use fewer columns. - while (has_more_tiles()) { - // 2. Zero-initialize TMEM accumulator - tmem_zero(tmem_acc, 256); +## Completion and ordering - // 3. Mainloop: accumulate K tiles - for (int k = 0; k < K_tiles; ++k) { - issue_tcgen05_mma(tmem_acc, smem_a[k], smem_b[k]); - } +The relevant mechanisms are distinct: - // 4. Fence and read results - asm volatile("tcgen05.fence::before_thread_sync;"); - __syncthreads(); +- `tcgen05.commit` attaches completion of prior asynchronous MMA operations to an mbarrier. Wait on that barrier before consuming their results. +- `tcgen05.wait::ld` and `tcgen05.wait::st` are the completion mechanisms for the corresponding asynchronous TMEM load/store operations. +- `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` order tcgen05 operations around a documented execution-ordering handoff. A fence is not an MMA completion wait. +- Producer buffers must remain live until the asynchronous operation that reads them has completed according to its instruction contract. - // 5. Epilogue: read from TMEM, apply bias/activation, write to GMEM - epilogue_from_tmem(tmem_acc, output); +CTA synchronization alone does not replace these completion operations. - // 6. TMEM persists across tiles -- no need to reallocate - } +## Data movement - // 7. Deallocate before exit - tmem_dealloc(tmem_acc, 256); -} -``` +`tcgen05.ld` transfers TMEM into registers, and `tcgen05.st` transfers registers into TMEM. Their documented shapes include `.16x64b`, `.16x128b`, `.16x256b`, `.32x32b`, and `.16x32bx2`, with supported repetition qualifiers. There is no scalar `.32x1b` form. -## Data Movement Operations - -### TMEM Store (Register to TMEM) - -```cuda -// Store a register value to TMEM -// Each thread writes to its own TMEM row at the specified column -__device__ void tmem_store_f32(uint32_t tmem_col, float value) { - asm volatile( - "tcgen05.st.sync.aligned.32x1b.x1.b32 [%0], {%1};" - : - : "r"(tmem_col), "f"(value) - ); -} - -// Vectorized store: 4 consecutive FP32 values -__device__ void tmem_store_f32x4(uint32_t tmem_col, float4 values) { - asm volatile( - "tcgen05.st.sync.aligned.32x1b.x4.b32 [%0], {%1, %2, %3, %4};" - : - : "r"(tmem_col), - "f"(values.x), "f"(values.y), - "f"(values.z), "f"(values.w) - ); -} -``` - -### TMEM Load (TMEM to Register) - -```cuda -// Load a single FP32 from TMEM -__device__ float tmem_load_f32(uint32_t tmem_col) { - float result; - asm volatile( - "tcgen05.ld.sync.aligned.32x1b.x1.b32 {%0}, [%1];" - : "=f"(result) - : "r"(tmem_col) - ); - return result; -} - -// Vectorized load: 4 consecutive FP32 values -__device__ float4 tmem_load_f32x4(uint32_t tmem_col) { - float4 result; - asm volatile( - "tcgen05.ld.sync.aligned.32x1b.x4.b32 {%0, %1, %2, %3}, [%4];" - : "=f"(result.x), "=f"(result.y), - "=f"(result.z), "=f"(result.w) - : "r"(tmem_col) - ); - return result; -} -``` - -### TMEM Zero-Fill - -```cuda -// Zero-fill a range of TMEM columns -__device__ void tmem_zero(uint32_t tmem_base_col, uint32_t num_cols) { - // Each thread zeros its own row - for (uint32_t c = 0; c < num_cols; c += 4) { - float4 zero = make_float4(0.f, 0.f, 0.f, 0.f); - tmem_store_f32x4(tmem_base_col + c, zero); - } -} -``` - -### Bulk SMEM -> TMEM Copy via tcgen05.cp - -The `tcgen05.cp` instruction copies a shaped matrix from shared memory into -TMEM. The source operand is a shared-memory matrix descriptor, not a TMEM -address. +`tcgen05.cp` performs shaped shared-memory-to-TMEM copies. For example, PTX ISA 9.0 defines this exact form: ```ptx -// Copy a 128x256b shared-memory tile into TMEM. tcgen05.cp.cta_group::1.128x256b [taddr], sdesc; ``` -The 64-bit register operand `sdesc` is the matrix descriptor representing the -source matrix in shared memory. The matrix descriptor format is described in -[Matrix Descriptors](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#tcgen05-matrix-descriptors). - -## Double-Buffering with TMEM - -Double-buffering TMEM accumulators enables overlapping the epilogue of the current tile with the MMA accumulation of the next tile: +Here `taddr` is the TMEM destination and `sdesc` is the 64-bit shared-memory matrix descriptor. Follow the instruction's asynchronous ordering and completion rules before reusing the source or consuming the destination. -```cuda -__global__ void double_buffered_gemm(/* ... */) { - // Allocate two accumulator buffers - uint32_t tmem_acc[2]; - tmem_acc[0] = tmem_alloc(256); - tmem_acc[1] = tmem_alloc(256); +## Staging accumulators - int buf = 0; +Multiple accumulator stages can overlap MMA production with draining a different, completed TMEM stage. The design is valid only when: - for (int tile = 0; tile < num_tiles; ++tile) { - // Zero the current buffer - tmem_zero(tmem_acc[buf], 256); +- the combined allocations and suballocations fit the 512-column budget; +- producer and consumer roles use explicit pipeline barriers; +- MMA completion is observed before a consumer drains a stage; +- TMEM load completion is observed before register results are used; and +- no stage is recycled while an asynchronous operation can still access it. - // Mainloop: accumulate into current buffer - for (int k = 0; k < K_tiles; ++k) { - issue_tcgen05_mma(tmem_acc[buf], smem_a[k], smem_b[k]); - } - asm volatile("tcgen05.fence::before_thread_sync;"); - __syncthreads(); +This is a pipeline design, not an automatic consequence of alternating two addresses. CUTLASS's version-pinned SM100 tutorials show complete producer/consumer implementations. - // If not the first tile, the *other* buffer's epilogue - // was overlapped with this tile's MMA in the pipeline. +## CUTLASS 4.5.0 Python DSL - // Start epilogue for current buffer - // (can overlap with next tile's MMA on the other buffer) - epilogue_from_tmem(tmem_acc[buf], output_tile[tile]); +CUTLASS 4.5.0 exposes TMEM through `cutlass.utils.TmemAllocator` and layout-aware CuTe tensors. The official tutorial sequence is: - buf ^= 1; // Swap buffers - } +1. create a `TmemAllocator` over shared storage for the allocation result; +2. call `allocate(num_columns)` from the configured allocator warp; +3. use `wait_for_alloc()` before other warps retrieve the address; +4. obtain a typed pointer with `retrieve_ptr(dtype)`; +5. bind that pointer to the MMA accumulator layout with `cute.make_tensor`; +6. drain it with `tcgen05` copy atoms and the pipeline's completion protocol; and +7. call `free(tmem_ptr)` before exit. - // Cleanup - tmem_dealloc(tmem_acc[0], 256); - tmem_dealloc(tmem_acc[1], 256); -} -``` - -### TMEM Budget Considerations - -With 512 total columns per SM: - -| Accumulator Size | Columns | Max Buffers | Remaining for Scratch | -|---|---|---|---| -| 128x128 FP32 | 128 cols | 4 | 0 | -| 128x192 FP32 | 192 cols | 2 | 128 | -| 128x256 FP32 | 256 cols | 2 | 0 | -| 128x256 FP32 (2x) | 512 cols | 2 (double-buf) | 0 | - -For double-buffered 128x256 tiles, the full 512 columns are consumed. If additional scratch TMEM is needed (e.g., for softmax in attention kernels), reduce the tile size or use a single accumulator buffer. - -## Microbenchmark Data - -From published Blackwell microbenchmarks: - -| Metric | TMEM | SMEM | Registers | -|---|---|---|---| -| End-to-end latency (cache miss) | ~420 cycles | ~30 cycles | ~4 cycles | -| Bandwidth for large working sets | High (dedicated bus) | Medium | N/A (limited count) | -| Best for | Multi-stage tensor pipelines | Single-shot small matrix | Scalar/vector ALU | - -TMEM is **not** a replacement for shared memory. Its strength is in serving as a dedicated accumulator buffer that eliminates register pressure for large MMA tiles. SMEM remains faster for small, frequently accessed data. - -## TMEM in the CUTLASS Abstraction - -In CUTLASS 4.5.0 for SM100, TMEM is managed through CuTe layouts: - -```python -# CuTe-DSL example: TMEM accumulator layout -# 128 rows x 256 columns, FP32 -tmem_layout = Layout( - shape=(128, 256), - stride=(256, 1), - memory_space=MemorySpace.TMEM -) - -# Allocate TMEM accumulator -acc = tmem_alloc(tmem_layout) - -# Issue MMA -- accumulator lives in TMEM -tcgen05_mma(acc, smem_a, smem_b) - -# Fence and read -tcgen05_fence() -result = tmem_load(acc) -``` +See the pinned [CUTLASS 4.5.0 FP16 GEMM tutorial](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py) and [TMEM allocator implementation](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/python/CuTeDSL/cutlass/utils/tmem_allocator.py). APIs such as `Layout(memory_space=MemorySpace.TMEM)` and standalone `tcgen05_mma()` are not CUTLASS 4.5.0 interfaces. -## Common Pitfalls +## Primary references -1. **Forgetting to fence**: Reading TMEM without `tcgen05.fence::before_thread_sync` produces undefined (stale) values. -2. **Forgetting to deallocate**: In persistent kernels, TMEM must be freed before re-acquiring tiles. Otherwise, the next allocation will fail or hang. -3. **Exceeding 512 columns**: Attempting to allocate more than the SM's total column budget silently corrupts data or causes a hang. -4. **Cross-warp reads**: A thread can only directly read/write TMEM rows mapped to its own lane. Accessing another warp's rows requires explicit shuffle or SMEM staging. -5. **Assuming SMEM-like latency**: TMEM has ~420-cycle latency on cache miss vs ~30 cycles for SMEM. Do not use TMEM for low-latency random access patterns. +- [PTX ISA 9.0: Tensor Memory](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory) +- [PTX ISA 9.0: `tcgen05.alloc`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc) +- [PTX ISA 9.0: `tcgen05.ld`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld) +- [PTX ISA 9.0: `tcgen05.commit`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit) +- [CUTLASS 4.5.0 SM100 Python tutorial](https://github.com/NVIDIA/cutlass/tree/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm) diff --git a/wiki/kernels/deepgemm.md b/wiki/kernels/deepgemm.md index 9a21550b8..7eaf01884 100644 --- a/wiki/kernels/deepgemm.md +++ b/wiki/kernels/deepgemm.md @@ -25,190 +25,112 @@ related: sources: - blog-deepgemm - pr-deepgemm-304 -- pr-cutlass-2139 -- pr-vllm-23696 -performance_claims: -- gpu: H800 - dtype: fp8 - shape: M=4096, N=4096, K=4096 - metric: TFLOPS - value: 1550 - utilization: ~90% - source_id: blog-deepgemm -blackwell_relevance: SM100 kernel uses tcgen05.mma with TMEM and native UE8M0 block - scaling; SM90 kernel provides baseline FP8 GEMM comparison. +performance_claims: [] +blackwell_relevance: The pinned SM100 kernel uses block-scaled UMMA/tcgen05 with + packed UE8M0 scale factors in TMEM; the pinned SM90 kernel provides the + WGMMA plus CUDA-core-promotion comparison. artifact_dir: artifacts/kernels/deepgemm --- # DeepGEMM -- FP8 GEMM with Fine-Grained Scaling -## Overview +## Verified Scope -DeepGEMM is DeepSeek's open-source FP8 GEMM library providing high-performance matrix multiplication with fine-grained per-tile/per-block scaling. The core kernel is remarkably compact (~300 lines), yet achieves approximately 90% utilization on H800. It supports both Hopper (SM90 via WGMMA) and Blackwell (SM100 via tcgen05.mma) architectures, and includes specialized MoE grouped GEMM layouts. +DeepGEMM is DeepSeek's open-source tensor-core kernel library. This page describes the FP8 GEMM paths at commit [`891d57b4db1071624b5c8fa0d1e51cb317fa709f`](https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f), with byte-verified SM90 and SM100 kernel files stored locally. The pinned project requires an SM90 or SM100 GPU, CUDA 12.3 or newer for SM90, and CUDA 12.9 or newer for SM100. -The key innovation is the fine-grained quantization scheme: tile-wise 1x128 scaling for activations and block-wise 128x128 scaling for weights, which prevents outlier values from destroying quantization precision. +DeepGEMM contains multiple kernels and APIs; the local SM90 and SM100 files below are representative FP8 1D1D implementations, not the whole library. -## Fine-Grained Quantization Scheme +## Fine-Grained Quantization -``` -Activations (tile-wise 1x128): - +-----------+-----------+-----------+ - | tile 0 | tile 1 | tile 2 | <-- each tile: 1 row x 128 cols - | scale: s0 | scale: s1 | scale: s2 | <-- one FP32 scale per tile - +-----------+-----------+-----------+ - -Weights (block-wise 128x128): - +---------------+---------------+ - | block (0,0) | block (0,1) | <-- each block: 128 rows x 128 cols - | scale: s_00 | scale: s_01 | <-- one FP32 scale per block - +---------------+---------------+ - | block (1,0) | block (1,1) | - | scale: s_10 | scale: s_11 | - +---------------+---------------+ -``` +The [DeepSeek-V3 Technical Report v2](https://arxiv.org/abs/2412.19437v2) defines the training scheme that motivates this path: -## FP8 Accumulation with Nc=128 CUDA Core Promotion +- activations are grouped per token and per 128 channels, forming `1 x 128` tiles; +- weights are grouped per 128 input channels and 128 output channels, forming `128 x 128` blocks; and +- smaller groups let scales adapt more locally, which the paper says better accommodates outliers. This is a scoped accuracy motivation, not a guarantee that quantization error disappears. -On Hopper, the Tensor Core accumulator has limited precision (~FP22, not true FP32). DeepGEMM mitigates this by promoting partial sums to a separate FP32 accumulator on CUDA Cores every Nc=128 columns (4 consecutive WGMMA operations). +Scale representation is architecture-specific in the pinned DeepGEMM interface. SM90 consumes FP32 scale factors. SM100 consumes packed UE8M0 factors, four UE8M0 values per `torch.int`. The pinned SM90 1D1D kernel fixes `BLOCK_K == 128`; the SM100 1D1D template accepts K scale granularities of 32 or 128 for each operand. -```cpp -// SM90 path: WGMMA with Nc=128 CUDA Core promotion -// Every 4 WGMMAs, promote accumulated result to FP32 CUDA core accumulator -constexpr int Nc = 128; // Promotion interval (4 WGMMAs of n=32 each) -constexpr int WGMMA_N = 32; - -float cuda_core_acc[TILE_M][TILE_N] = {0}; // FP32 accumulator on CUDA Cores - -for (int k = 0; k < K; k += Nc) { - // Run 4 consecutive WGMMAs with TC-limited precision accumulation - __half2 tc_acc[TILE_M][WGMMA_N]; // Tensor Core accumulator (~FP22) - memset(tc_acc, 0, sizeof(tc_acc)); - - for (int sub_k = 0; sub_k < Nc; sub_k += WGMMA_K) { - wgmma_mma_async(tc_acc, A_smem + sub_k, B_smem + sub_k); - } - wgmma_wait(); - - // Promote: add TC result to CUDA Core FP32 accumulator - // This prevents precision loss from repeated FP22 accumulation - for (int m = 0; m < TILE_M; m++) - for (int n = 0; n < TILE_N; n++) - cuda_core_acc[m][n] += (float)tc_acc[m][n] * scale_a[m] * scale_b[n]; -} -``` +## SM90: WGMMA and CUDA-Core Promotion -On Blackwell (SM100), the tcgen05.mma instruction with TMEM accumulation uses native UE8M0 block scaling, which eliminates the need for explicit CUDA core promotion. +The DeepSeek-V3 report characterizes H800 FP8 Tensor Core accumulation as retaining about 14 bits. Its `Nc=128` strategy accumulates 128 elements of the GEMM inner dimension—four WGMMAs in the described configuration—before moving the partial result to FP32 registers on CUDA cores. -```cpp -// SM100 path: tcgen05.mma with native block scaling -// Scaling factors packed as UE8M0 (4 values per uint32) -// No explicit CUDA core promotion needed -- TMEM accumulates in full precision - -// Pack 4 UE8M0 scale factors into a single uint32 -uint32_t packed_scales = pack_ue8m0(sf[0], sf[1], sf[2], sf[3]); - -// tcgen05.mma reads A/B from SMEM, accumulates into TMEM -// Block scale applied natively during MMA -asm volatile( - "tcgen05.mma.cta_group::1.kind::f8f6f4" - " [%0], %1, %2, %3, %4;" - : - : "l"(tmem_addr), "l"(a_smem_addr), "l"(b_smem_addr), - "r"(packed_scales), "n"(SCALE_D_ENABLED) -); -``` +The pinned [`sm90_fp8_gemm_1d1d.cuh`](../../artifacts/kernels/deepgemm/full/sm90_fp8_gemm_1d1d.cuh) implements that structure directly: -## MoE Grouped GEMM Layouts +1. A math warp-group owns `float accum[...]` for WGMMA output and zero-initialized `float final_accum[...]` for promoted results. +2. For each 128-element K block, it reads the A and B FP32 factors from shared memory, issues `BLOCK_K / WGMMA::K` WGMMA operations, commits the group, and waits for completion. +3. The `Promote with scales` loop multiplies each partial result by its A and B factors and adds it to `final_accum`. +4. After all K blocks, the kernel stages the final FP32 values for the epilogue/store path. -DeepGEMM provides three grouped GEMM layouts tailored for MoE workloads, where only the M-axis varies (different token counts per expert) while N and K remain fixed: +The exact upstream file is the reproducible reference. It should not be replaced by a sketch using half accumulators or by deriving the promotion interval from WGMMA's output-N tile. -``` -Layout 1: Contiguous (prefill) - Expert 0: M0 tokens ──┐ - Expert 1: M1 tokens ──┤── All packed contiguously in memory - Expert 2: M2 tokens ──┘ Index array stores cumulative offsets - -Layout 2: Masked (decode with CUDA graphs) - Fixed-size M_max allocation per expert - Binary mask indicates valid tokens - Compatible with CUDA graph capture (no dynamic shapes) - -Layout 3: K-grouped (weight gradients) - Groups along K-axis instead of M-axis - Used for computing dW in MoE training backward -``` +## SM100: Native Block-Scaled UMMA -```cpp -// Contiguous grouped GEMM dispatch -// problem_sizes[i] = {M_i, N, K} for expert i -void grouped_gemm_contiguous( - const fp8_t* A, // All expert inputs packed - const fp8_t* B, // Expert weights [num_experts, N, K] - float* C, // Output packed - const int* offsets, // Cumulative M offsets per expert - int num_experts -) { - // Each thread block picks an expert via tile scheduling - // M-axis tiles distributed across experts using offset lookup - int expert_id = binary_search(offsets, num_experts, tile_m_start); - int local_m = tile_m_start - offsets[expert_id]; - - // Standard GEMM tile with expert-specific B matrix - compute_tile(A + offsets[expert_id] * K, - B + expert_id * N * K, - C + offsets[expert_id] * N, - local_m, N, K); -} -``` +The pinned [`sm100_fp8_gemm_1d1d.cuh`](../../artifacts/kernels/deepgemm/full/sm100_fp8_gemm_1d1d.cuh) uses a different data path: -## JIT Compilation +1. It creates a block-scaled UMMA instruction descriptor with a `float` accumulator type and `cutlass::float_ue8m0_t` scale type. +2. TMA loads packed scale-factor data into shared memory. UTCCP copies selected scale blocks from shared memory into dedicated SFA and SFB columns in TMEM. +3. The elected issuing thread calls the SM100 UMMA wrapper with shared-memory operand descriptors, the TMEM accumulator column, a runtime instruction descriptor containing scale IDs, and the two TMEM scale addresses. +4. The epilogue drains the TMEM accumulator after the kernel's full/empty barrier protocol says it is ready. -DeepGEMM uses JIT compilation via NVRTC to specialize kernels per problem shape at runtime. This avoids the combinatorial explosion of pre-compiled template instantiations while still achieving optimal register allocation and loop unrolling. +This path does not contain the SM90 `final_accum` CUDA-core promotion loop. That difference does not justify a blanket claim that every tcgen05 accumulator mode or datatype has "full FP32" behavior; the accumulator type and instruction form remain part of the selected operation. -```cpp -// Lightweight JIT module: compile per-shape kernel at first call -auto kernel = jit_compile( - "deepgemm_fp8", - {{"M", M}, {"N", N}, {"K", K}, - {"BLOCK_M", 128}, {"BLOCK_N", 128}, {"BLOCK_K", 64}, - {"NUM_STAGES", 4}} -); -kernel.launch(A, B, C, scales_a, scales_b, stream); -``` +## Grouped GEMM Interfaces + +The pinned interface distinguishes three workload arrangements rather than treating all of them as M-varying layouts: -## Memory Layout +| Interface family | Varying group dimension | Fixed dimensions | Work metadata | +|---|---:|---:|---| +| M-grouped contiguous | M | N and K | Either a group index for each packed M row or a prefix-sum M layout, depending on the selected option | +| M-grouped masked | Valid M within each `[G, M, K]` allocation | Maximum M, N, and K tensor extents | An integer `masked_m[G]` vector holding each group's valid M length, not a binary mask | +| K-grouped contiguous | K | M and N | Per-group K lengths plus their device tensor; used for weight-gradient-style grouped GEMM | + +Contiguous M grouping is intended for variable per-expert token counts in training forward or inference prefill. Masked M grouping keeps fixed allocations suitable for CUDA-graph decode while limiting work to each group's valid M. The pinned repository provides K-grouped NT on SM90 and K-grouped TN on SM100 for the documented FP8 paths. + +## JIT Compilation -SM90 kernels use NT (non-transposed A, transposed B) layout exclusively. SM100 kernels support all four layout combinations (NT, TN, NN, TT), enabled by tcgen05.mma's flexible operand addressing. +DeepGEMM generates and compiles kernel source at runtime. At the pinned commit, the compiler defaults to NVCC. Setting `DG_JIT_USE_NVRTC=1` selects the optional NVRTC path; the project warns that this may reduce performance for some cases. -## Performance +The cache key includes the kernel name, compiler signature, compiler flags, and generated source. A cache hit reuses the existing kernel runtime; a miss compiles a CUBIN in a temporary directory and then publishes the completed cache entry. Consequently, a new specialization has first-use compilation/loading cost, whereas matching later calls can reuse the artifact. This mechanism does not guarantee globally optimal register allocation or unrolling. -| GPU | Dtype | Shape | TFLOPS | Utilization | -|-----|-------|-------|--------|-------------| -| H800 | FP8 | M=4096, N=4096, K=4096 | 1550 | ~90% | +## Operand Layouts -## When to Use +For the pinned FP8 interface, SM90 supports NT only: A is non-transposed and B is supplied in the representation used for `A @ B.T`. SM100 exposes dense `fp8_gemm_{nt, nn, tn, tt}` variants. Its implementation propagates the selected operand-major modes into the UMMA descriptors; the public wrappers transpose the corresponding tensor views for the non-NT variants. -- FP8 inference and training where per-tensor quantization loses too much precision -- MoE expert computation with variable token counts per expert -- Situations where fine-grained (1x128 / 128x128) scale granularity is needed +## Performance Evidence -## Caveats +The pinned README reports that DeepGEMM reached **up to 1550 TFLOPS on H800** in an April 2025 news item. It does not bind that peak to `M=N=K=4096`, state approximately 90% utilization, or preserve enough benchmark conditions for reproduction. The unsupported structured performance record and table have therefore been removed; the surviving number is source-reported only and should not be compared across software, clocks, shapes, or GPUs without a controlled rerun. -- SM90 path is NT layout only -- JIT compilation adds first-call latency (amortized over repeated calls) -- Fine-grained scaling adds overhead vs. per-tensor scaling -- only beneficial when outlier sensitivity matters +## Practical Boundaries -## Sources +- Input transposition, FP8 casting, and scale-layout preparation are separate from the optimized GEMM kernels; the project supplies utilities but warns they may be slower than fusing the work into producers. +- M-grouped contiguous segments must satisfy the configured M/K alignment. Masked mode uses valid-length integers, while K-grouped mode has different shapes and architecture-specific layout variants. +- JIT cache misses add compilation latency, and NVRTC is optional rather than the default. +- Fine-grained scaling's accuracy and cost tradeoffs depend on quantization recipe, target architecture, scale preparation, fusion, and workload. There is no universal per-tensor-scaling penalty or single "use only for outliers" rule. -- [DeepGEMM GitHub](https://github.com/deepseek-ai/DeepGEMM) -- [DeepSeek-V3 Technical Report](https://arxiv.org/abs/2412.19437) +## Pinned Sources + +- [DeepGEMM README at commit `891d57b`](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md) +- [DeepGEMM GEMM API at commit `891d57b`](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp) +- [DeepGEMM JIT compiler at commit `891d57b`](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/jit/compiler.hpp) +- [DeepSeek-V3 Technical Report v2](https://arxiv.org/abs/2412.19437v2) ## Full Reference Implementation -Local verbatim upstream code lives in [`artifacts/kernels/deepgemm/full/`](../../artifacts/kernels/deepgemm/full/) (see its `PROVENANCE.yaml` for the pinned upstream SHA and byte-verified SHA-256). Labeled derived variants — including a naive/teaching skeleton — live in [`artifacts/kernels/deepgemm/variants/`](../../artifacts/kernels/deepgemm/variants/). +Local verbatim upstream code lives in [`artifacts/kernels/deepgemm/full/`](../../artifacts/kernels/deepgemm/full/) and is pinned by [`PROVENANCE.yaml`](../../artifacts/kernels/deepgemm/full/PROVENANCE.yaml) to commit `891d57b4db1071624b5c8fa0d1e51cb317fa709f`. The recorded SHA-256 values match both files. Labeled derived material lives in [`artifacts/kernels/deepgemm/variants/`](../../artifacts/kernels/deepgemm/variants/); it is teaching material, not upstream code. -Query via: +Query the page and its attached code with: ```bash python3 scripts/get_page.py kernel-deepgemm --include-code ``` + +The following is a verbatim fragment of the SM90 promotion loop; the linked full file supplies its surrounding declarations and loop bounds: + +```cpp +const float &scale_b_0 = scales_b[i].x; +const float &scale_b_1 = scales_b[i].y; +final_accum[i * 4 + 0] += scale_a_0 * scale_b_0 * accum[i * 4 + 0]; +final_accum[i * 4 + 1] += scale_a_0 * scale_b_1 * accum[i * 4 + 1]; +final_accum[i * 4 + 2] += scale_a_1 * scale_b_0 * accum[i * 4 + 2]; +final_accum[i * 4 + 3] += scale_a_1 * scale_b_1 * accum[i * 4 + 3]; +``` diff --git a/wiki/kernels/flash-attention-4.md b/wiki/kernels/flash-attention-4.md index 490269c8b..3cd66676b 100644 --- a/wiki/kernels/flash-attention-4.md +++ b/wiki/kernels/flash-attention-4.md @@ -26,175 +26,102 @@ related: sources: - doc-flash-attention-4 - blog-flash-attention-4 -- pr-flashinfer-1850 -performance_claims: -- gpu: B200 - dtype: bf16 - shape: seqlen=8192, headdim=128 - metric: TFLOPS - value: 1605 - utilization: 71% - source_id: doc-flash-attention-4 +performance_claims: [] +evidence_basis: +- source_id: doc-flash-attention-4 + evidence_type: paper +- source_id: blog-flash-attention-4 + evidence_type: source-reported +blackwell_relevance: The paper's SM100 design overlaps tcgen05 MMA with non-matmul + work, stores large intermediates in TMEM, partially emulates exp2 on FMA units, + and uses two-CTA MMA in backward. artifact_dir: artifacts/kernels/flash-attention-4 --- # FlashAttention-4 -## Overview +## Verified Scope -FlashAttention-4 is the Blackwell-native evolution of the FlashAttention family, designed to exploit SM100 architectural features that break the Hopper-era bottleneck: tensor core throughput doubles on Blackwell, but SFU (Special Function Unit) count and shared memory bandwidth remain unchanged. FA4 addresses this asymmetry through software-emulated exponentials, ping-pong scheduling across TMEM, and 2-CTA cooperative backward passes. +FlashAttention-4 is an attention algorithm and CuTe DSL implementation designed around Blackwell's asymmetric throughput: B200 tensor-core throughput grows much more than its special-function and shared-memory resources. This page separates two scopes: -Written entirely in CuTe DSL (Python), FA4 compiles 20-30x faster than equivalent CUTLASS C++ template code while matching or exceeding cuDNN performance. +- the [FA4 paper v1](https://arxiv.org/abs/2603.05451v1), which studies the SM100/B200 algorithm and reports the authors' measurements; and +- the public implementation at Dao-AILab/flash-attention commit [`a369df707e1980fb328abcc1733e3457ec10155f`](https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute), which is the source snapshot used for implementation statements below. -## Key Techniques +The paper implementation is written in CuTe DSL. Its compilation comparison is against corresponding FA3 CUTLASS kernels: forward compiles in 2.5 seconds instead of 55 seconds, and backward in 1.4 seconds instead of 45 seconds. Those are single-kernel source-reported compile measurements, not an end-to-end installation or runtime speedup. -### Ping-Pong Scheduling +## Ping-Pong Forward Schedule -Two 128-token query tiles are assigned to each CTA. While one tile's matmul runs on the tensor cores, the other tile's softmax rescaling runs on dedicated warpgroups accessing TMEM. This hides the softmax latency behind MMA compute. +The forward kernel assigns two output tiles of 128 query rows to one CTA. One MMA warp issues the matrix products, two four-warp softmax groups serve the two output tiles, and a correction warpgroup handles accumulator corrections. The score and output accumulators live in disjoint TMEM regions. Pipeline barriers allow softmax work for one output tile to overlap MMA work that advances the other tile. -```python -# CuTe DSL: Ping-pong tile scheduling (simplified) -# Two query tiles per CTA, alternating MMA and softmax phases - -@cute.kernel -def flash_attention_4_fwd(Q, K, V, O, L): - # Each CTA processes 2 query tiles of 128 tokens - TILE_Q = 128 - NUM_TILES = 2 - - # Phase A: tile_0 does MMA(Q0, K), tile_1 does softmax rescale - # Phase B: tile_1 does MMA(Q1, K), tile_0 does softmax rescale - - for kv_block in range(num_kv_blocks): - # Ping: MMA on tile 0, softmax on tile 1 - with warpgroup(mma_wg): - S0 = cute.mma(Q_tile[0], K_block) # tcgen05.mma -> TMEM - with warpgroup(softmax_wg): - O_tile[1], L_tile[1] = rescale_and_accumulate( - O_tile[1], L_tile[1], S1, V_prev - ) - - # Pong: MMA on tile 1, softmax on tile 0 - with warpgroup(mma_wg): - S1 = cute.mma(Q_tile[1], K_block) - with warpgroup(softmax_wg): - O_tile[0], L_tile[0] = rescale_and_accumulate( - O_tile[0], L_tile[0], S0, V_block - ) -``` +This is more than ordinary K/V double buffering: the two alternating objects are output tiles with separate softmax state. The exact synchronization and stage ownership are implementation details; the invented one-loop pseudocode formerly on this page did not preserve those dependencies. + +## Partial Software Exponential -### Software-Emulated Exponential (Cody-Waite) +FA4 does not replace every hardware exponential. The paper selects only about 10-25% of entries for software evaluation on FMA units and leaves the rest on the hardware MUFU `ex2` path, allowing both resources to contribute. -The SFU `ex2` instruction is the bottleneck on Blackwell -- its throughput does not scale with the doubled tensor core rate. FA4 replaces it with a software exponential distributed across FMA units using Cody-Waite range reduction and Horner polynomial evaluation. +For a software-selected value, the published range reduction writes `x = n + f` with `n = floor(x)` and `f` in `[0, 1)`, evaluates a degree-3 polynomial for `2**f`, and reconstructs the scale from `n`. The function below is a scalar reference using the rounded coefficients printed in the first-party blog. It illustrates that formula; it is not the CuTe kernel's vector selection, clamping, or scheduling code. ```python -# Software exp2 via Cody-Waite range reduction + Horner polynomial -# Distributes across FMA units instead of using scarce SFU hardware - -def software_exp2(x): - """ - Compute 2^x using FMA units instead of SFU ex2. - Cody-Waite range reduction splits x into integer + fraction. - Horner polynomial approximates 2^frac. - """ - # Range reduction: x = n + f, where n is integer, f in [-0.5, 0.5] - n = round(x) - f = x - n # Cody-Waite: use extended precision subtraction - - # Horner polynomial for 2^f on [-0.5, 0.5] - # Coefficients chosen for bf16 precision target - p = f * (C5 + f * (C4 + f * (C3 + f * (C2 + f * C1)))) - p = 1.0 + p - - # Reconstruct: 2^x = 2^n * 2^f - return ldexp(p, n) # Integer exponent via bit manipulation +import math + +def fa4_blog_exp2_reference(x: float) -> float: + n = math.floor(x) + f = x - n + polynomial = 1.0 + f * (0.6951 + f * (0.2276 + f * 0.0771)) + return math.ldexp(polynomial, n) ``` -This gives roughly 4x throughput improvement over the hardware SFU path by utilizing FMA units that would otherwise be idle during softmax phases. +No standalone four-times software-versus-hardware exponential result is asserted here. The paper evaluates the combined kernel and its ablations rather than establishing that former page claim. -### Conditional Softmax Rescaling +## Conditional Softmax Rescaling -Standard FlashAttention rescales the output accumulator every KV block. FA4 only rescales when the running maximum changes significantly (large jump), reducing non-matmul operations. +Ordinary online softmax updates the row maximum and rescales accumulated state as each score block arrives. FA4 permits its retained maximum to lag: it resynchronizes only when the new block maximum exceeds the retained maximum by more than a threshold. The paper's typical threshold is `tau = log2(256) = 8.0` in the exponent's base-2 units. -```python -# Only rescale when max changes substantially -def conditional_rescale(O_acc, lse_old, lse_new, threshold=2.0): - diff = lse_new - lse_old - if abs(diff) > threshold: - # Full rescale: O_acc *= exp(lse_old - lse_new) - scale = software_exp2((lse_old - lse_new) * LOG2E) - O_acc = O_acc * scale - # Otherwise: skip rescale, accumulate normally - return O_acc -``` - -### 2-CTA Backward Pass +When a rescale is skipped, subsequent probabilities are still evaluated relative to the retained old maximum, and auxiliary statistics track the delayed normalization. The algorithm performs final renormalization at the end. Comparing absolute changes in LSE, changing the maximum anyway, or simply skipping the accumulator multiply is not equivalent. -The backward pass spans two paired CTAs in a cluster, sharing TMEM across both SMs. This halves shared memory traffic for the dQ/dK/dV gradient computation. +## Two-CTA Backward -```python -# 2-CTA backward: paired CTAs share TMEM via 2-SM cooperative mode -@cute.kernel -def flash_attention_4_bwd(Q, K, V, O, dO, dQ, dK, dV): - # Two CTAs cooperate: CTA_0 and CTA_1 in same cluster - # tcgen05.mma shape: m256 x n256 x k16 (2-SM cooperative) - - cta_id = cute.cluster_rank() # 0 or 1 - - for q_block in range(num_q_blocks): - # Both CTAs load shared KV block via TMA - K_smem = tma_load(K, kv_offset) - V_smem = tma_load(V, kv_offset) - - # CTA_0: compute dK contribution - # CTA_1: compute dV contribution - if cta_id == 0: - # dK += dS^T @ Q (accumulated in TMEM) - dS = compute_dS(O, dO, S) - cute.mma(dS.T, Q_tile, accumulator=dK_tmem) - else: - # dV += S^T @ dO (accumulated in TMEM) - cute.mma(S.T, dO_tile, accumulator=dV_tmem) -``` +The paper maps five backward GEMMs to two-CTA tcgen05 MMA with `M=256, N=128, K=128`. For those operations, the paired CTAs can share operand B, which the authors describe as roughly halving the shared-memory reads for that operand. This is not a claim that all shared-memory traffic for dQ, dK, and dV is halved. -## Performance +For dQ, each CTA computes a half of dS and exchanges that half through distributed shared memory so both CTAs can form the required dQ product. The two-CTA organization also doubles the dQ reduction tile along N and thereby halves the number of global atomic reductions described by the paper. It does not assign dK exclusively to CTA 0 and dV exclusively to CTA 1. -| Configuration | GPU | Dtype | TFLOPS | Utilization | vs cuDNN | vs Triton | -|---------------|-----|-------|--------|-------------|----------|-----------| -| seqlen=8192, headdim=128 | B200 | BF16 | 1605 | 71% | 1.1-1.3x | 2.1-2.7x | +## Pinned Implementation Notes -The 71% MMA utilization represents the state of the art for attention kernels on Blackwell. The remaining 29% is consumed by softmax, rescaling, and memory transfers. +At commit `a369df7`: -## Implementation Notes +- [`flash_fwd_sm100.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py) builds tcgen05 operations through `make_trivial_tiled_mma`, allocates TMEM through `TmemAllocator`, and uses explicit TMEM column offsets for score and output accumulators. +- The forward path constructs TMA atoms for Q, K, and V where its configuration enables them. It also has non-TMA Q and paged-K/V copy paths, so TMA use is not unconditional. +- [`flash_bwd_sm100.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_bwd_sm100.py) constructs the five backward MMA operations and the two-CTA exchange/reduction pipelines rather than splitting dK and dV by cluster rank. +- The package README describes CuTe DSL attention for Hopper and Blackwell, and the tree contains SM90, SM100, and SM120 dispatch modules. The paper's FA4 result remains SM100/B200-specific; package architecture coverage should not be used to broaden that performance result. +- The pinned [`pyproject.toml`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/pyproject.toml) requires `nvidia-cutlass-dsl==4.6.0.dev0`. That is a property of this source snapshot, not a timeless minimum. +- Forward and backward accept FP16/BF16. An FP8 benchmark/bring-up script is present, but at this revision it explicitly expects the FA4 FP8 call to fail until support is implemented. -- Written entirely in CuTe DSL (Python), not C++ templates -- 20-30x faster compilation than equivalent CUTLASS C++ code -- Uses `SM100_MMA_SS` atoms for tcgen05 MMA from shared memory -- TMEM locality via `TMEM` locale in CuTe layout -- TMA bulk loads for Q, K, V tiles into shared memory +## Performance Evidence -## When to Use +The first-party sources contain two different source-reported peak values. Paper v1 reports **up to 1613 TFLOPS/s on B200 BF16, or 71% of the peak convention used by the authors**. Tri Dao's blog reports **up to 1605 TFLOPS/s, also labeled 71%**, plus up to 1.3x over cuDNN 9.13 and up to 2.7x over Triton. -- Standard multi-head attention on Blackwell with sequence lengths >= 1024 -- Both forward and backward passes -- BF16 precision (FP8 support planned) +The paper's benchmark suite spans sequence lengths from 1K through 32K and multiple query/value head-dimension pairs under a fixed total-token convention. Neither textual source establishes the former single row that attached 1605 TFLOPS, 71%, and both speedup ranges specifically to `seqlen=8192, headdim=128`. The structured performance record is therefore empty, and no complement-of-71% time breakdown is inferred. -## Caveats +## Practical Boundaries -- SM100 only -- no fallback to SM90 -- Requires CuTe DSL toolchain (CUTLASS 4.5.0 + Python frontend) -- Ping-pong scheduling most effective for headdim=128; smaller headdims may not fully overlap +- Choose the package path only for a supported architecture, dtype, head-dimension pair, masking mode, and feature set at the exact revision in use. +- Do not treat sequence length 1024 or head dimension 128 as universal crossover or optimum values. Compare against the relevant cuDNN, framework, or other kernel path on the actual workload. +- Compilation speed and runtime speed are separate measurements. The paper's compile comparison does not prove an equivalent runtime factor. +- Treat source-reported B200 numbers as unreproduced unless the same software, clock/power settings, tensor shapes, timing region, warmup, and FLOP convention are available. -## Sources +## Pinned Sources -- [FlashAttention-4 paper](https://arxiv.org/abs/2603.05451) -- [Tri Dao's blog](https://tridao.me/blog/2026/flash4/) +- [FlashAttention-4 paper, arXiv v1](https://arxiv.org/abs/2603.05451v1) +- [Tri Dao's FlashAttention-4 blog](https://tridao.me/blog/2026/flash4/) +- [FA4 CuTe DSL package at commit `a369df7`](https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute) +- [Pinned SM100 forward source](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py) +- [Pinned SM100 backward source](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_bwd_sm100.py) -## Full Reference Implementation +## Local Code References -Local verbatim upstream code lives in [`artifacts/kernels/flash-attention-4/full/`](../../artifacts/kernels/flash-attention-4/full/) (see its `PROVENANCE.yaml` for the pinned upstream SHA and byte-verified SHA-256). Labeled derived variants — including a naive/teaching skeleton — live in [`artifacts/kernels/flash-attention-4/variants/`](../../artifacts/kernels/flash-attention-4/variants/). +The local [`full/`](../../artifacts/kernels/flash-attention-4/full/) bundle is a byte-verified, verbatim **adjacent NVIDIA CUTLASS SM100 FMHA backward MLA example**, pinned to CUTLASS commit `0e026982`. It is not the Dao-AILab FA4 implementation. The local [`variants/`](../../artifacts/kernels/flash-attention-4/variants/) bundle contains explicitly derived teaching material, including the scalar software-exp formula; it is not upstream code. Exact FA4 implementation evidence is linked to the immutable Dao-AILab commit above. -Query via: +Query the page and its attached local references with: ```bash python3 scripts/get_page.py kernel-flash-attention-4 --include-code diff --git a/wiki/kernels/flash-attention-sm100-mla-topk.md b/wiki/kernels/flash-attention-sm100-mla-topk.md index ef88b35f2..a2d4a8bc3 100644 --- a/wiki/kernels/flash-attention-sm100-mla-topk.md +++ b/wiki/kernels/flash-attention-sm100-mla-topk.md @@ -30,37 +30,61 @@ related: - technique-external-source-map-research sources: - pr-flash-attention-2441 -- pr-flash-attention-1236 -performance_claims: -- gpu: B200 - dtype: bf16 - shape: batch=512, seqlen_q=1, seqlen_k=16384, nheads=128, topk=2048 - metric: latency_ms - value: 0.3 - source_id: pr-flash-attention-2441 -blackwell_relevance: PR-grade CuTe DSL SM100 MLA code is directly relevant to DSA sparse attention and top-k KV-gather routing on B200. +performance_claims: [] +evidence_basis: +- source_id: pr-flash-attention-2441 + evidence_type: upstream-code +blackwell_relevance: PR 2441 adds a pinned SM100 CuTe DSL MLA forward path for + 64/512 query/value dimensions, MQA with 128 query heads per KV head, and + caller-supplied top-k KV indices. --- -## Shape +# FlashAttention SM100 MLA Top-K Sparse Forward -FlashAttention PR 2441 adds an SM100 CuTe DSL forward path for MLA shapes with -top-k sparsity. It is useful when an attention candidate has to combine page/KV -layout handling, sparse top-k selection, and tiled forward scheduling. +## Verified Scope + +[Dao-AILab/flash-attention PR 2441](https://github.com/Dao-AILab/flash-attention/pull/2441), merged as [`f219c89c886c6ccbf9d3dbd9fe41b11ac64e9df8`](https://github.com/Dao-AILab/flash-attention/commit/f219c89c886c6ccbf9d3dbd9fe41b11ac64e9df8), adds an SM100 CuTe DSL forward path for the DeepSeek-style MLA dimensions `head_dim=64` and `head_dim_v=512`, with MQA packing of 128 query heads per KV head and caller-supplied top-k indices. + +The top-k path is deliberately narrower than a generic paged sparse-attention API. It requires packed GQA/MQA, `qhead_per_kvhead == 128`, and the cp.async K/V gather path. Its constructor defaults to top-k length 2048 and requires the configured length to be divisible by 256. The merged kernel rejects a non-null page table with `page table tbd for MLA`; page-table support was explicitly outside this PR. + +## Gather and Scheduling Paths + +The pinned merge keeps three responsibilities distinguishable: + +- [`topk_gather_kv.py`](../../artifacts/prs/flash-attention/PR-2441/key-files/flash_attn/cute/topk_gather_kv.py) loads caller-provided top-k indices, forms indexed K/V addresses, issues cp.async copies, and optionally constructs validity bitmasks for out-of-range indices. +- [`tile_scheduler.py`](../../artifacts/prs/flash-attention/PR-2441/key-files/flash_attn/cute/tile_scheduler.py) supplies the tile-scheduler implementations selected by the MLA kernel. +- [`flash_fwd_mla_sm100.py`](../../artifacts/prs/flash-attention/PR-2441/key-files/flash_attn/cute/flash_fwd_mla_sm100.py) integrates gather, scheduling, two-CTA tcgen05 MMA, TMEM accumulators, softmax, and the output path. + +This separation matters for evaluation. Arithmetic work follows the effective top-k length, whereas memory addresses follow the caller's index set and use an indexed gather/optional-bitmask path. Reduced attention FLOPs therefore do not prove a proportional latency or locality improvement. + +## Source-Reported Performance + +The PR author reports an initial saturating-decode comparison for batch 512, `seqlen_q=1`, `seqlen_k=16384`, 128 query heads, the 64/512 MLA shape, and top-k length 2048: + +| Variant | Author-reported latency | Author-reported throughput | +|---|---:|---:| +| DSA, no bitmask; indices assumed in bounds | 0.31 ms | 955.47 TFLOPS | +| DSA, validity bitmask | 0.33 ms | 898.08 TFLOPS | +| Vanilla MLA baseline | 1.98 ms | 1180.70 TFLOPS | + +These PR-description observations are not reproduced here. The PR body does not name the exact GPU model, dtype, clocks/power state, software environment, timing protocol, or run-to-run variation for the rows. They therefore remain prose with exact source qualifications rather than structured `performance_claims`. + +## Transfer Checklist + +- Confirm the exact MQA packing, 64/512 dimensions, top-k divisibility, index layout, and bounds contract used by the target revision. +- Treat no-bitmask results as valid only when the caller guarantees every index is in bounds; otherwise preserve the validity path. +- Do not infer paged-KV compatibility from the exposed parameter name: merge `f219c89c` rejects page tables in this MLA path. +- Profile indexed-gather traffic and tensor arithmetic separately, then validate end-to-end accuracy and latency on the full workload. + +## Reproducible Source Lookup + +The local PR record and byte-verified bundle identify the integration and its dedicated gather file: ```python -# Query pattern before borrowing implementation details: -# open PR page, then inspect the source snapshot or upstream files listed there. from pathlib import Path -pr_page = Path("sources/prs/flash-attention/PR-2441.md") -text = pr_page.read_text() -assert "flash_fwd_mla_sm100.py" in text -assert "topk_gather_kv.py" in text +pr_page = Path("sources/prs/flash-attention/PR-2441.md").read_text() +provenance = Path("artifacts/prs/flash-attention/PR-2441/PROVENANCE.yaml").read_text() +assert "flash_fwd_mla_sm100.py" in pr_page +assert "topk_gather_kv.py" in provenance ``` - -## Transfer Notes - -- Treat top-k gather and tiled attention scheduling as separate evidence paths. -- Profile memory traffic separately from tensor-pipe utilization; sparse top-k - routing can improve arithmetic work while worsening gather locality. -- Keep full-workload validation because the useful path is shape-specific. diff --git a/wiki/kernels/flashmla.md b/wiki/kernels/flashmla.md index 99d6593d4..e5bd5511f 100644 --- a/wiki/kernels/flashmla.md +++ b/wiki/kernels/flashmla.md @@ -22,215 +22,128 @@ kernel_types: - sparse-attention languages: - cuda-cpp +- python related: - hw-tcgen05-mma - hw-tmem - kernel-nsa sources: - blog-flashmla -- pr-flashinfer-1117 -- pr-vllm-39752 -performance_claims: -- gpu: B200 - dtype: bf16 - shape: dense prefill, variable seqlen - metric: TFLOPS - value: 1460 - utilization: ~65% - source_id: blog-flashmla -- gpu: B200 - dtype: fp8 - shape: sparse prefill - metric: TFLOPS - value: 1450 - utilization: ~65% - source_id: blog-flashmla -blackwell_relevance: SM100 dense prefill achieves 1460 TFLOPS (vs 660 on SM90); Blackwell - tcgen05 enables higher MLA throughput. +- doc-deepseek-v2-mla +- pr-cutlass-2466 +- pr-cutlass-2472 +performance_claims: [] +blackwell_relevance: FlashMLA supports sparse MLA decode and prefill on SM100 and + bundles NVIDIA's dense MHA prefill path; its pinned snapshot requires CUDA 12.9+ + for SM100. artifact_dir: artifacts/kernels/flashmla --- # FlashMLA -- Multi-head Latent Attention -## Overview +## Scope -FlashMLA provides high-performance kernels for DeepSeek's Multi-head Latent Attention (MLA) mechanism, which compresses the KV cache from 327-516 KB/token (standard MHA) down to 70 KB/token through a learned low-rank projection into a latent space. This extreme compression (4.66-7.28x reduction) is critical for serving DeepSeek-V3/V3.2 models at scale. +FlashMLA is DeepSeek's attention-kernel library for DeepSeek-V3 and DeepSeek-V3.2-Exp. At pinned commit [`71c7379`](https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232), the library contains MLA-mode decode and sparse-prefill operators plus a dense **MHA** prefill operator contributed for SM100. The repository's term “MLA mode” distinguishes MQA-shaped `d_qk=576, d_v=512` kernels from MHA-shaped `d_qk=192/128, d_v=128` kernels; it does not mean every operator in the package is MLA. -FlashMLA includes four kernel variants: dense MLA decoding (SM90), sparse MLA decoding (SM90/SM100), dense MLA prefill (SM100), and sparse MLA prefill (SM90/SM100). +## MLA Cache Reduction Versus a Concrete Cache ABI -## MLA KV Cache Layout +The DeepSeek-V2 paper describes the model-level reduction in elements cached per layer and token: -Each token in the MLA KV cache occupies 656 bytes: +- MHA: `2 * n_h * d_h` +- MLA: `d_c + d_h^R`, approximately `4.5 * d_h` for its `d_c=4*d_h` and `d_h^R=d_h/2` configuration -``` -Token KV Cache Entry (656 bytes total): -+------------------------------------------+ -| FP8 compressed KV data | 512 bytes | <-- Latent KV representation -| FP32 scaling factors | 16 bytes | <-- Per-head scales -| BF16 RoPE embeddings | 128 bytes | <-- Position encodings -+------------------------------------------+ - -Paged KV cache: - Page size = 64 tokens (dense) or variable (sparse) - Each page = 64 * 656 = 41,984 bytes -``` +These formulas are independent of storage dtype and layer count. They should not be converted to whole-model KB/token figures without naming those additional assumptions. -## Dense MLA Decoding (SM90) - -The decode kernel targets memory-bound inference with paged KV cache (block size 64). It achieves up to 3000 GB/s memory bandwidth and 660 TFLOPS on H800. - -```cpp -// Dense MLA decode kernel structure (SM90, BF16) -// Memory-bound: bandwidth utilization is the primary metric - -template -__global__ void flashmla_decode_dense( - const half* __restrict__ Q, // [batch, num_heads, head_dim] - const int8_t* __restrict__ KV, // Paged KV cache (FP8) - const float* __restrict__ scales, // Per-head scales - const half* __restrict__ rope, // RoPE embeddings - const int* __restrict__ page_table, - half* __restrict__ O, - float* __restrict__ L // Log-sum-exp -) { - // Each warpgroup handles one query head - const int head_id = blockIdx.x; - const int batch_id = blockIdx.y; - - // Load query into registers - half Q_reg[HEAD_DIM]; - load_query(Q, batch_id, head_id, Q_reg); - - float acc[HEAD_DIM] = {0.0f}; - float lse = -INFINITY; - - // Iterate over KV pages - for (int page = 0; page < num_pages; page++) { - int page_idx = page_table[batch_id * max_pages + page]; - - // TMA load KV page into shared memory - __shared__ int8_t KV_smem[BLOCK_KV * 656]; - tma_load_async(KV_smem, KV + page_idx * BLOCK_KV * 656); - cp_async_wait(); - - // Compute attention scores for this page - for (int t = 0; t < BLOCK_KV; t++) { - // Dequantize KV: fp8 -> bf16, apply scale - half K_token[HEAD_DIM], V_token[HEAD_DIM]; - dequant_kv(KV_smem + t * 656, scales, K_token, V_token); - - // Apply RoPE - apply_rope(K_token, rope + (page * BLOCK_KV + t) * 64); - - // Score and accumulate (online softmax) - float score = dot_product(Q_reg, K_token, HEAD_DIM); - float new_lse = logaddexp(lse, score); - float rescale = exp(lse - new_lse); - for (int d = 0; d < HEAD_DIM; d++) - acc[d] = acc[d] * rescale + exp(score - new_lse) * V_token[d]; - lse = new_lse; - } - } - - // Write output - store_output(O, batch_id, head_id, acc); - L[batch_id * num_heads + head_id] = lse; -} -``` +FlashMLA's **656-byte** layout is narrower: it is the DeepSeek-V3-family FP8 sparse-decode cache ABI, not the definition of an MLA cache. One token contains: -## Sparse MLA (SM90/SM100) - -Sparse MLA uses token-level sparsity indices to select only relevant tokens from the KV cache, dramatically reducing memory reads for long sequences. - -```cpp -// Sparse MLA: only attend to selected tokens via indices tensor -// Each query has a variable-length list of relevant token indices - -template -__global__ void flashmla_sparse( - const half* Q, - const int8_t* KV_cache, - const float* scales, - const int* token_indices, // Selected token indices per query - const int* num_selected, // Number of selected tokens per query - half* O -) { - const int query_id = blockIdx.x; - const int n_tokens = num_selected[query_id]; - - // Only load and compute on selected tokens - for (int i = 0; i < n_tokens; i += BLOCK_SIZE) { - int tok_idx = token_indices[query_id * MAX_TOKENS + i]; - - // Load only the selected token's KV entry (656 bytes) - load_kv_entry(KV_cache, tok_idx, K_local, V_local); - dequant_and_accumulate(Q, K_local, V_local, scales, acc, lse); - } -} -``` +| Region | Representation | Bytes | +|---|---|---:| +| NoPE latent data | 512 `float8_e4m3` values | 512 | +| NoPE group scales | four `float32` values, one per 128 values | 16 | +| RoPE data | 64 unquantized `bfloat16` values | 128 | +| **Total** | | **656** | -## Dense Prefill (SM100) +Dense decode uses a BF16 cache. The pinned quantization tests also contain a separate 512-dimensional sparse layout, so code must select the model/layout contract rather than assuming 656 bytes universally. `page_block_size` is taken from the cache tensor; 64 is a test default, not a fixed API rule. -The SM100 prefill kernel leverages tcgen05.mma and TMEM for the compute-heavy forward and backward passes, achieving 1460 TFLOPS forward and 1000 TFLOPS backward on B200. +## Supported Paths at `71c7379` -```cpp -// SM100 dense prefill: tcgen05.mma with TMEM accumulation -// Uses warp specialization: TMA warps + MMA warps + softmax warps +| Operator | Architecture | Attention mode | Documented cache/input format | +|---|---|---|---| +| Dense decode | SM90 | MQA (`576/512`) | BF16 paged KV | +| Sparse decode | SM90, SM100 | MQA (`576/512`) | FP8 KV, dequantized for BF16 MMA; BF16 output | +| Dense prefill | SM100 | MHA (`192/128` or `128/128`) | BF16 Q/K/V | +| Sparse prefill | SM90, SM100 | MQA | BF16 Q and KV | -// Forward pass structure: -// 1. TMA loads Q, K, V tiles into SMEM -// 2. tcgen05.mma computes S = Q @ K^T into TMEM -// 3. Softmax warpgroup applies online softmax on TMEM data -// 4. tcgen05.mma computes O = softmax(S) @ V into TMEM -// 5. TMEM -> SMEM -> Global memory for output +CUDA 12.8 or newer and PyTorch 2.0 or newer are required; the pinned README requires CUDA 12.9 or newer for SM100. -// Key: TMEM holds both S matrix and O accumulator -// No register spill for large tile sizes +## Sparse Contracts + +Sparse decode receives `indices[batch, s_q, topk]`. Each nonnegative value already encodes a physical page and offset: + +```text +encoded = physical_page * page_block_size + offset_in_page ``` -## Performance +Because the physical page is already encoded, sparse decode does not use `block_table`; `-1` marks an invalid entry. The kernel consumes these indices but does not produce the top-k selection, so an indexing stage outside the attention call must supply them. + +Sparse prefill is a different interface. It receives BF16 `q[s_q,h_q,d_qk]`, BF16 `kv[s_kv,h_kv,d_qk]`, and `indices[s_q,h_kv,topk]`; it has no batch dimension, requires `h_kv=1` in the documented equivalence, and accepts `-1` or values at least `s_kv` as invalid. It returns `(out, max_logits, lse)`. -| Variant | GPU | Dtype | TFLOPS | Bandwidth | -|---------|-----|-------|--------|-----------| -| Dense decode | H800 | BF16 | 660 | 3000 GB/s | -| Sparse decode | H800 | FP8 | 410 | -- | -| Sparse decode | B200 | FP8 | 350 | -- | -| Dense prefill fwd | B200 | BF16 | 1460 | -- | -| Dense prefill bwd | B200 | BF16 | 1000 | -- | -| Sparse prefill | H800 | FP8 | 640 | -- | -| Sparse prefill | B200 | FP8 | 1450 | -- | +## Source-Reported Performance -## Architecture Integration +The following are maxima reported by the pinned first-party README. They were not reproduced here, and the README does not provide complete shape, timing, sample-count, or variance cells, so they are intentionally excluded from structured `performance_claims`. -FlashMLA is deployed in production for DeepSeek-V3 and V3.2 inference: -- SGLang and vLLM provide day-0 support -- CUTLASS SM100 includes MLA attention kernels with fused reduction -- FlashMLA sparse kernels are used by DeepSeek-V3.2-Exp with NSA +| Operator | Environment stated by source | Precision scope | Author-reported observation | +|---|---|---|---| +| Dense MLA decode | H800 SXM5, CUDA 12.8 | BF16 cache | Up to 3000 GB/s in a memory-bound configuration; up to 660 TFLOPS in a compute-bound configuration | +| Sparse MLA decode | H800 SXM5, CUDA 12.8 | FP8 KV, BF16 MMA | 410 TFLOPS in a compute-bound configuration | +| Sparse MLA decode | B200; software version not stated in the row | FP8 KV, BF16 MMA | Up to 350 TFLOPS; the author says it was not well optimized | +| Dense MHA prefill | B200; NVIDIA-reported | BF16 inputs | Up to 1460 TFLOPS forward and 1000 TFLOPS backward | +| Sparse MLA prefill | H800 SXM5, CUDA 12.8 | BF16 inputs | Up to 640 TFLOPS forward | +| Sparse MLA prefill | B200, CUDA 12.9 | BF16 inputs | Up to 1450 TFLOPS forward | -## When to Use +The numbers compare different operators, phases, shapes, precision scopes, and machines. In particular, `1460` is dense MHA prefill, not a replacement for the `660` dense-MLA-decode observation. -- DeepSeek-V3/V3.2 model serving with MLA architecture -- Long-context inference where KV cache size is the bottleneck -- Combined with NSA for sparse attention on long sequences +## SM100 Implementation Notes -## Caveats +The pinned DeepSeek SM100 sources use TMA, `tcgen05` tensor-core operations, and TMEM in specialized sparse-decode/prefill and dense-MHA-prefill code. Those mechanisms are implementation-specific: they do not make the CUTLASS and FlashInfer files in this repository copies of DeepSeek FlashMLA. -- MLA-specific: the latent KV cache format (656 bytes/token) is tied to DeepSeek's architecture -- Dense prefill is SM100 only -- Sparse MLA requires a separate indexing pass to select relevant tokens +The local [`full/`](../../artifacts/kernels/flashmla/full/) bundle contains two byte-verified **adjacent implementations**: -## Sources +- NVIDIA CUTLASS Example 77 MLA forward at merge `9baa06dd` +- FlashInfer's SM100 FMHA-MLA header at commit `9a05c92a` -- [FlashMLA GitHub](https://github.com/deepseek-ai/FlashMLA) -- [DeepSeek-V3 Technical Report](https://arxiv.org/abs/2412.19437) -- [CUTLASS SM100 Attention Changelog](https://docs.nvidia.com/cutlass/latest/CHANGELOG.html) +Their exact per-file origins are recorded in `full/PROVENANCE.yaml`. The [`variants/`](../../artifacts/kernels/flashmla/variants/) directory contains a small KernelWiki-derived layout/index helper, explicitly marked as non-upstream. For the DeepSeek implementation itself, use commit `71c7379` linked above. -## Full Reference Implementation +## Selection and Validation Checklist -Verbatim upstream code lives in [`artifacts/kernels/flashmla/full/`](../../artifacts/kernels/flashmla/full/); labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` header) live in [`artifacts/kernels/flashmla/variants/`](../../artifacts/kernels/flashmla/variants/). Every file's SHA-256 and upstream-pinning metadata is in `PROVENANCE.yaml` inside each bundle. +- Match the operator family, `d_qk/d_v`, architecture, CUDA version, cache dtype, and index shape exactly. +- Treat the 656-byte layout as a V3-family FP8 sparse-decode ABI, not a generic MLA property. +- Generate sparse indices before invoking FlashMLA and validate invalid-index/page encoding rules. +- Benchmark the target decode or prefill workload; do not transfer TFLOPS or bandwidth across the table's distinct regimes. +- Validate output and LSE against the repository reference before relying on throughput. -Query via: +## Sources and Local Query + +- [DeepSeek FlashMLA at audited commit `71c7379`](https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232) +- [DeepSeek-V2 MLA paper, v5](https://arxiv.org/html/2405.04434v5) +- [CUTLASS PR 2466, SM100 MLA-shape backward](https://github.com/NVIDIA/cutlass/pull/2466) +- [CUTLASS PR 2472, SM100 MLA-shape forward](https://github.com/NVIDIA/cutlass/pull/2472) + +Query the page and its labeled artifacts with: ```bash -python3 scripts/get_page.py kernel-flashmla --include-code +conda run -n base python scripts/get_page.py kernel-flashmla --include-code +``` + +The mode-specific byte arithmetic can be checked without a GPU: + +```python +# KernelWiki-derived contract check; not upstream FlashMLA code. +def v3_fp8_sparse_bytes() -> int: + nope_bytes = 512 + scale_bytes = 4 * 4 + rope_bytes = 64 * 2 + return nope_bytes + scale_bytes + rope_bytes + +assert v3_fp8_sparse_bytes() == 656 ``` diff --git a/wiki/kernels/fp8-block-scale-gemm.md b/wiki/kernels/fp8-block-scale-gemm.md index 8cd84031b..07f42a99e 100644 --- a/wiki/kernels/fp8-block-scale-gemm.md +++ b/wiki/kernels/fp8-block-scale-gemm.md @@ -1,117 +1,75 @@ --- id: kernel-fp8-block-scale-gemm -title: "FP8 Block-Scale GEMM" +title: FP8 Fine-Grained-Scale GEMM type: kernel architectures: [sm100, sm90] tags: [gemm, fp8, block-scale, fine-grained-quantization, tcgen05, wgmma] confidence: source-reported reproducibility: snippet kernel_types: [gemm] -languages: [cuda-cpp, cute-dsl] +languages: [cuda-cpp, cute-dsl, python] related: [kernel-deepgemm, kernel-nvfp4-gemm, technique-fine-grained-quantization, hw-tcgen05-mma] -sources: [blog-deepgemm, doc-cutlass-blackwell, doc-cutlass-changelog-sm100] -performance_claims: - - gpu: H800 - dtype: fp8 - shape: "M=4096, N=4096, K=4096" - metric: TFLOPS - value: 1550 - utilization: "~90% via CUDA core promotion" - source_id: blog-deepgemm -blackwell_relevance: "SM100 tcgen05.mma has native UE8M0 block scaling; SM90 requires Nc=128 CUDA core promotion. Same kernel pattern works on both but different scale handling." +sources: [blog-deepgemm, doc-deepseek-v3-fp8, doc-ptx-isa-sm100] +performance_claims: [] +blackwell_relevance: SM100 accepts packed UE8M0 factors in block-scaled UMMA, + replacing DeepGEMM's SM90 FP32-scale CUDA-core promotion path. --- -# FP8 Block-Scale GEMM +# FP8 Fine-Grained-Scale GEMM -## Overview +## Scope -FP8 GEMM with fine-grained block scales (128x128 weights, 1x128 activations). Preserves more dynamic range than per-tensor FP8 scaling, critical for LLM inference and training where outliers dominate quantization error. +The DeepSeek-V3 training recipe quantizes forward activations per `1x128` tile and weights per `128x128` block. The smaller groups let each scale adapt to local outliers. This is a **logical quantization format**; a kernel is compatible only when its element type, scale granularity, scale representation, matrix layout, and architecture-specific ABI all match. -DeepGEMM is the reference implementation; CUTLASS provides SM100 schedules. +DeepGEMM commit [`891d57b4`](https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f) implements related FP8 GEMMs on SM90 and SM100, but the two generations handle scale factors differently. -## Block Scaling Structure +## Logical Scale Shapes -``` -Activations: tile-wise 1x128 scales - [1x128 values] → 1 scale factor (FP32 or FP8 E4M3) +For `A[M,K] @ B[N,K].T` with the DeepSeek-V3 forward recipe, the logical scale arrays have shapes `A_sf[M,K/128]` and `B_sf[N/128,K/128]`. This small check documents only that grouping; it does not encode DeepGEMM's required TMA-transformed layouts. -Weights: block-wise 128x128 scales - [128x128 values] → 1 scale factor per block +```python +# KernelWiki-derived format check; not upstream DeepGEMM code. +def deepseek_v3_scale_shapes(m: int, n: int, k: int): + assert m % 128 == 0 and n % 128 == 0 and k % 128 == 0 + activation_scales = (m, k // 128) + weight_scales = (n // 128, k // 128) + return activation_scales, weight_scales -Output accumulator: FP32 - Multiply A × B in FP8, accumulate in FP32, apply scales at MMA boundary +assert deepseek_v3_scale_shapes(4096, 4096, 4096) == ((4096, 32), (32, 32)) ``` -## SM90 Path (Hopper, WGMMA) - -```cuda -// Hopper accumulator has ~22-bit precision (FP22) -// Every Nc=128 WGMMAs, promote partial sum to FP32 on CUDA cores -// This retains precision without adding MMA overhead - -__device__ void sm90_fp8_gemm_with_promotion(...) { - float acc_fp32 = 0.0f; - - for (int k = 0; k < K; k += 128) { - float acc_fp22 = 0.0f; - #pragma unroll 4 - for (int kk = 0; kk < 128; kk += 32) { - wgmma_fp8_e4m3(acc_fp22, A_frag, B_frag); - } - // Promote to FP32 accumulator on CUDA cores - acc_fp32 += acc_fp22 * scale_a * scale_b; - } +The paper documents phase-specific exceptions, including `128x1` activation grouping in backward paths. Do not treat the forward layout above as a universal FP8 tensor ABI. - // Write acc_fp32 to output -} -``` +## SM90: WGMMA Followed by CUDA-Core Promotion -## SM100 Path (Blackwell, tcgen05.mma) +The pinned SM90 1D1D kernel requires FP32 scale factors and fixes `BLOCK_K == 128`. Within each K block it issues the selected WGMMA operations into a register `accum` array; after the WGMMA batch completes, it multiplies those partials by the corresponding A/B scales and adds them into a separate FP32 `final_accum` array on CUDA cores. -```cuda -// tcgen05.mma has native UE8M0 block scale support in hardware -// No promotion needed - scales applied inside MMA +For the DeepSeek-V3 description, a 128-element K interval corresponds to four WGMMAs. The paper describes Hopper's relevant internal addition/alignment precision as 14 bits, not as a generic “FP22 accumulator.” Promotion improves numerical behavior but still adds scale loads and CUDA-core work, so its overhead must be measured rather than assumed away. -__global__ void sm100_fp8_gemm_block_scale(...) { - uint32_t tmem = tmem_alloc(256); +## SM100: Packed UE8M0 and Block-Scaled UMMA - for (int k = 0; k < K; k += BLOCK_K) { - mbarrier_wait(&tma_done); +The pinned SM100 interface requires scale factors packed as four UE8M0 values per 32-bit `torch.int`. The 1D1D kernel TMA-loads scale-factor blocks, copies them into TMEM, builds a `make_instr_desc_block_scaled<..., float_ue8m0_t, ...>` descriptor, and accumulates with the selected block-scaled UMMA path in TMEM. - // tcgen05.mma.mxf8f6f4.block_scale variant - // Reads A, B from SMEM; scales from scale SMEM; accumulates in TMEM - asm volatile( - "tcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.scale_vec::1X " - "[%0], %1, %2, %3, %4, %5;" - :: "r"(tmem), "l"(a_desc), "l"(b_desc), - "r"(sf_a_desc), "r"(sf_b_desc), "n"(1) - ); - } +This is hardware-integrated scale consumption, not the SM90 `final_accum += scale_a * scale_b * accum` loop. Exact `tcgen05.mma` instruction spelling and descriptor restrictions are PTX-version-sensitive; use the pinned source wrapper or the NVIDIA PTX ISA rather than a hand-written approximate inline-assembly string. - // Read from TMEM, apply global scale, store - float result = tmem_load(tmem) * global_scale; - output[row * N + col] = __float2half(result); -} -``` +The current DeepGEMM snapshot supports more than the original 128-granularity training recipe on SM100, including selected 32-element recipes. Treat the API's transformed/padded scale layout as authoritative for the chosen recipe. -## Memory Layout +## Source-Reported Performance -``` -A (activations) [M, K] packed FP8 E4M3 -sf_a [M, K/128] FP32 or FP8 E4M3 scales # 1 per 1x128 tile +The pinned README says DeepGEMM achieved **up to 1550 TFLOPS on H800** in its 2025-04-18 news item. That sentence does not identify a matrix shape, utilization percentage, timing protocol, sample count, variance, or a single optimization responsible for the maximum. The result was not reproduced in this audit and is therefore kept out of structured `performance_claims`. -B (weights) [N, K] packed FP8 E4M3 -sf_b [N/128, K/128] scales # 1 per 128x128 block - # or packed UE8M0 format (Blackwell): 4 scales per int32 -``` +CUTLASS also ships SM100 block-scaled GEMM schedules, but no matched CUTLASS-versus-DeepGEMM shape/environment record is established here. -## Performance +## Selection Checklist -- DeepGEMM on H800: up to 1550 TFLOPS FP8 -- CUTLASS SM100 schedules: similar ratio vs peak +- Confirm the producer's scale grouping; FP8 element type alone is insufficient for compatibility. +- On SM90, provide the FP32 scale layout expected by the selected DeepGEMM kernel. +- On SM100, provide correctly packed, TMA-aligned UE8M0 factors for the selected granularity and layout. +- Include quantization, scale-layout transformation, and epilogue costs when measuring an end-to-end path. +- Validate numerical error against an FP32/BF16 reference on the actual activation/weight distribution. -## When To Use +## Sources -- LLM inference with FP8 quantized weights (DeepSeek V3, Qwen2-FP8, etc.) -- Training with FP8 activations (DeepSeek V3 training framework) -- Anywhere per-tensor FP8 accuracy is insufficient due to outliers +- [DeepSeek-V3 Technical Report v2, FP8 training](https://arxiv.org/html/2412.19437v2) +- [DeepGEMM at audited commit `891d57b4`](https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f) +- [NVIDIA PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/) diff --git a/wiki/kernels/fused-moe.md b/wiki/kernels/fused-moe.md index 5db7933fe..0fb819cf5 100644 --- a/wiki/kernels/fused-moe.md +++ b/wiki/kernels/fused-moe.md @@ -1,27 +1,22 @@ --- id: kernel-fused-moe -title: Fused MoE — FP8 Block-Scale Routing + Dual GEMM +title: FlashInfer Track A FP8 Block-Scale MoE type: kernel architectures: - sm100 - sm100a -- sm90 tags: - moe -- fused-kernel - fp8 - block-scale -- kernel-fusion -- warp-specialization - grouped-gemm -- gated-dual-gemm +- kernel-fusion confidence: source-reported reproducibility: snippet kernel_types: - moe -- fused-kernel - grouped-gemm -- gated-dual-gemm +- fused-kernel languages: - cuda-cpp - cute-dsl @@ -33,253 +28,137 @@ related: - technique-tile-scheduling sources: - contest-flashinfer-track-a -- blog-deepgemm -- pr-vllm-23696 -performance_claims: -- gpu: B200 - dtype: fp8 - shape: topk=8, experts=32, hidden=7168, intermediate=2048, batch=4096 - metric: TFLOPS - value: 1262 - utilization: ~56% - source_id: contest-flashinfer-track-a -blackwell_relevance: SM100 enables native FP8 block-scale MoE via tcgen05 with higher - throughput; technique transfers from Hopper FP8 MoE. +performance_claims: [] +blackwell_relevance: The MLSys 2026 Track A definition and official evaluation + target NVIDIA B200 (sm_100a); this page documents the logical benchmark + contract, not a particular launch decomposition. artifact_dir: artifacts/kernels/fused-moe --- -# Fused MoE -- FP8 Block-Scale Routing + Dual GEMM +# FlashInfer Track A FP8 Block-Scale MoE -## Overview +## Scope -Fused MoE kernels combine the full Mixture-of-Experts forward pass into minimal kernel launches: routing, token dispatch, gate-up dual GEMM, SwiGLU activation, down projection GEMM, and token combine. In unfused implementations this requires 5-7 separate kernel launches; fused variants reduce this to 1-3 launches, eliminating intermediate global memory roundtrips and saving up to 21.9% activation memory traffic. +FlashInfer's MLSys 2026 contest calls Track A **Fused MoE** with FP8 support and targets NVIDIA B200. Its exact benchmark definition is `moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048`. The definition says that DeepSeek-style routing and two grouped GEMMs are included. That operation-level scope does **not** establish that an implementation uses one GPU launch. -This is Track A of the FlashInfer MLSys 2026 contest, targeting B200 GPUs with DeepSeek-V3-style MoE parameters. +## Fixed Benchmark Geometry -## MoE Forward Pass Structure +| Field | Value | Meaning | +| --- | ---: | --- | +| `seq_len` | variable | Number of input tokens; this is not labeled request batch size | +| global experts | 256 | Width of `routing_logits` | +| local experts | 32 | Experts whose weights are resident on one EP rank | +| expert parallelism | 8 | `256 / 32` ranks in the published definition | +| `top_k` | 8 | Selected global experts per token | +| `n_group` | 8 | Groups of 32 global experts | +| `topk_group` | 4 | Groups retained before global top-k selection | +| hidden size | 7168 | Input and output width | +| intermediate size | 2048 | Per-expert SwiGLU width | +| GEMM1 output | 4096 | Concatenated W13 output, `2 * 2048` | +| scale block | 128 | Fixed granularity for this DeepSeek-FP8 trace | -``` -Input tokens x [batch, hidden_dim=7168] - | - v -[Router] top-k=8 experts from 32, grouped (8 groups, top_group=4) - | - v -[Dispatch] Scatter tokens to selected experts - | - v -[Gate-Up Dual GEMM] - gate = x @ W_gate [batch_expert, hidden -> intermediate] - up = x @ W_up [batch_expert, hidden -> intermediate] - | - v -[SwiGLU Activation] - h = SiLU(gate) * up Element-wise: SiLU(x) = x * sigmoid(x) - | - v -[Down GEMM] - y = h @ W_down [batch_expert, intermediate -> hidden] - | - v -[Combine] Weighted sum of expert outputs per token (router weights) - | - v -Output [batch, hidden_dim=7168] -``` +The `e32` suffix means **32 local experts**, not 32 total experts. -## Kernel Fusion Strategy +## Tensor Contract -``` -Unfused (vLLM): 7 kernel launches - 1. Router softmax - 2. Top-k selection - 3. Token dispatch (scatter) - 4. Gate GEMM - 5. Up GEMM - 6. SiLU + multiply - 7. Down GEMM + combine - -Partially fused (SGLang): 5 kernel launches - 1. Router + top-k - 2. Dispatch - 3. Gate-Up fused GEMM + SiLU (3 ops -> 1 kernel) - 4. Down GEMM - 5. Combine - -Fully fused (ideal): 1-2 launches - All ops in single kernel, or gate-up-silu + down-combine -``` +For `T = seq_len`, the published benchmark signature is: -## Gated Dual GEMM Kernel (Hackathon Problem 3) - -The gate-up projection fuses two GEMMs with SiLU activation and element-wise multiply: - -```cpp -// Fused gate-up: two GEMMs + SiLU + multiply in one kernel -// Avoids writing intermediate gate and up results to global memory - -template -__global__ void gated_dual_gemm_fused( - const fp8_t* __restrict__ X, // [M, K] input tokens - const fp8_t* __restrict__ W_gate, // [N, K] gate weights - const fp8_t* __restrict__ W_up, // [N, K] up weights - const float* __restrict__ sf_x, // Block scales for X - const float* __restrict__ sf_gate, // Block scales for W_gate - const float* __restrict__ sf_up, // Block scales for W_up - half* __restrict__ output, // [M, N] fused output - int M, int N, int K -) { - // Two TMEM regions: one for gate accumulator, one for up accumulator - uint32_t tmem_gate = tmem_alloc_cta(256); - uint32_t tmem_up = tmem_alloc_cta(256); - - // Pipelined main loop - for (int k = 0; k < K; k += BLOCK_K) { - int stage = (k / BLOCK_K) % NUM_STAGES; - barrier_wait(stage); - - // Two MMAs per K-tile: gate and up projections - // Both read same X tile, different weight tiles - asm volatile( - "tcgen05.mma.cta_group::1.kind::f8f6f4" - " [%0], %1, %2, %3, %4;" - :: "r"(tmem_gate), "l"(x_smem[stage]), - "l"(wg_smem[stage]), "r"(scales_gate), "n"(1) - ); - asm volatile( - "tcgen05.mma.cta_group::1.kind::f8f6f4" - " [%0], %1, %2, %3, %4;" - :: "r"(tmem_up), "l"(x_smem[stage]), - "l"(wu_smem[stage]), "r"(scales_up), "n"(1) - ); - } +| Input | Dtype | Shape | +| --- | --- | --- | +| `routing_logits` | FP32 | `[T, 256]` | +| `routing_bias` | BF16 | `[256]` | +| `hidden_states` | FP8 E4M3FN | `[T, 7168]` | +| `hidden_states_scale` | FP32 | `[56, T]` | +| `gemm1_weights` | FP8 E4M3FN | `[32, 4096, 7168]` | +| `gemm1_weights_scale` | FP32 | `[32, 32, 56]` | +| `gemm2_weights` | FP8 E4M3FN | `[32, 7168, 2048]` | +| `gemm2_weights_scale` | FP32 | `[32, 56, 16]` | +| `local_expert_offset` | INT32 | scalar | +| `routed_scaling_factor` | FP32 | scalar | - // Fused epilogue: SiLU(gate) * up - // Read both accumulators from TMEM, apply activation, store result - float gate_val = tmem_load_f32(tmem_gate); - float up_val = tmem_load_f32(tmem_up); +The output is BF16 `[T, 7168]`. These scale tensors are explicit storage inputs; this page does not infer their runtime cost from their existence. - // SiLU(x) = x * sigmoid(x) = x / (1 + exp(-x)) - float silu_gate = gate_val / (1.0f + expf(-gate_val)); - output[row * N + col] = __float2half(silu_gate * up_val); +This derived helper makes the fixed shape arithmetic executable without pretending to encode FlashInfer's physical layouts: - // Deallocate TMEM - tmem_dealloc(tmem_gate, 256); - tmem_dealloc(tmem_up, 256); -} -``` +```python +def track_a_shapes(tokens: int) -> dict[str, tuple[int, ...]]: + assert tokens > 0 + return { + "routing_logits": (tokens, 256), + "hidden_states": (tokens, 7168), + "hidden_states_scale": (7168 // 128, tokens), + "gemm1_weights": (32, 2 * 2048, 7168), + "gemm1_weights_scale": (32, (2 * 2048) // 128, 7168 // 128), + "gemm2_weights": (32, 7168, 2048), + "gemm2_weights_scale": (32, 7168 // 128, 2048 // 128), + } -## FP8 Block-Scale MoE (FlashInfer API) -```python -# FlashInfer API for fused MoE -# Single function call replaces 5-7 separate kernel launches -import flashinfer - -output = flashinfer.fused_moe.trtllm_fp8_block_scale_moe( - hidden_states=x, # [batch, 7168] BF16 input - w_gate_up=w_gate_up, # [32, 2*2048, 7168] FP8 (fused gate+up weights) - w_down=w_down, # [32, 7168, 2048] FP8 (down weights) - router_weights=router_w, # [batch, 32] FP32 routing logits - topk=8, # Select top-8 experts per token - num_groups=8, # Expert grouping - topk_group=4, # Top groups to select from - block_scale_gate_up=sf_gu, # Block scales [32, 2*2048, 7168/128] FP8 - block_scale_down=sf_d, # Block scales [32, 7168, 2048/128] FP8 -) +assert track_a_shapes(7)["gemm1_weights_scale"] == (32, 32, 56) ``` -## Triton Fused Gate-Up Kernel +At FlashInfer commit `7f614b86470180bab2d22e36fd1775791c6bf3e6`, the corresponding public entry point is `flashinfer.fused_moe.trtllm_fp8_block_scale_moe`. Its complete call includes the eight tensors above plus `num_experts=256`, `top_k=8`, `n_group=8`, `topk_group=4`, `intermediate_size=2048`, the local expert interval, the routed scaling factor, and DeepSeek-V3 routing mode. -```python -import triton -import triton.language as tl - -@triton.jit -def fused_moe_gate_up_triton( - X_ptr, W_gate_ptr, W_up_ptr, Out_ptr, - expert_ids_ptr, token_counts_ptr, - sf_x_ptr, sf_gate_ptr, sf_up_ptr, - N: tl.constexpr, K: tl.constexpr, - BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr, -): - """Fused gate-up GEMM + SiLU for one expert's tokens.""" - expert_id = tl.program_id(2) - pid_m = tl.program_id(0) - pid_n = tl.program_id(1) - - token_count = tl.load(token_counts_ptr + expert_id) - m_start = pid_m * BLOCK_M - if m_start >= token_count: - return - - n_start = pid_n * BLOCK_N - offs_m = m_start + tl.arange(0, BLOCK_M) - offs_n = n_start + tl.arange(0, BLOCK_N) - - gate_acc = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) - up_acc = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) - - for k in range(0, K, BLOCK_K): - offs_k = k + tl.arange(0, BLOCK_K) - x_tile = tl.load(X_ptr + offs_m[:, None] * K + offs_k[None, :]) - wg_tile = tl.load(W_gate_ptr + expert_id * N * K - + offs_n[:, None] * K + offs_k[None, :]) - wu_tile = tl.load(W_up_ptr + expert_id * N * K - + offs_n[:, None] * K + offs_k[None, :]) - gate_acc += tl.dot(x_tile, tl.trans(wg_tile)) - up_acc += tl.dot(x_tile, tl.trans(wu_tile)) - - # Fused epilogue: SiLU(gate) * up - silu_gate = gate_acc * tl.sigmoid(gate_acc) - result = silu_gate * up_acc - tl.store(Out_ptr + offs_m[:, None] * N + offs_n[None, :], - result.to(tl.float16)) -``` +## Reference Semantics -## Framework Baselines (B200) +The official reference performs these steps: -| Framework | Batch 4096 TFLOPS | Batch 1 Latency | Kernel Launches | -|-----------|-------------------|-----------------|-----------------| -| SGLang | 1262 | 206.9us | 5 (fused) | -| FlashInfer CuTe DSL | 1225 | 481.9us | 1-2 (fully fused) | -| vLLM | 1117 | 369.5us | 7 (unfused) | +1. Convert routing logits to `s = sigmoid(logits)` and form selection scores `s + routing_bias`. +2. Reshape 256 scores into eight groups of 32. Sum the top two selection scores in each group, then retain four groups. +3. Select eight global experts from the retained groups using the biased selection scores. +4. Form combine weights from the **unbiased** sigmoid values for those eight experts, normalize per token, and multiply by `routed_scaling_factor`. +5. For global expert IDs in `[local_expert_offset, local_expert_offset + 32)`, dequantize the relevant activation and weight blocks, compute one W13 projection, split its 4096 columns into two 2048-column halves, apply SwiGLU, and compute W2. +6. Accumulate each local expert result into the token output using that expert's combine weight. Experts outside the local interval contribute nothing on that rank. -## Challenges +The W13 representation permits one logical `A @ W13.T` followed by a split. It does not require two separately allocated TMEM accumulators, and the benchmark contract does not prescribe a tcgen05 instruction sequence. -1. **No pre-tuned FP8 MoE config for B200**: Tile sizes and pipeline stages need empirical tuning -2. **FP8 numerical overflow**: Block scaling (block size 128) required for stability -3. **Batch-size sensitivity**: batch=1 is latency-critical (kernel launch overhead dominates); batch=4096 is throughput-critical -4. **Expert load imbalance**: Variable token counts per expert cause tail effects -5. **TMA alignment**: 128-byte alignment required for all TMA descriptors -6. **Dual TMEM allocation**: Gate and up accumulators each need TMEM space, competing for the 256KB budget +## Small Derived Reference -## When to Use +[`01-routing-plus-fusion-skeleton.py`](../../artifacts/kernels/fused-moe/variants/01-routing-plus-fusion-skeleton.py) is a CPU-checkable, parameterized reference for grouped selection and local W13/SwiGLU/W2 accumulation. It is derived KernelWiki code, not an optimized kernel and not an upstream contest solution. -- MoE model inference (DeepSeek-V3, Mixtral, etc.) -- Both prefill (high batch, throughput-critical) and decode (low batch, latency-critical) -- When gate-up GEMM fusion provides measurable speedup over separate launches +## Official Evaluation Boundary -## Caveats +The starter-kit evaluation document at commit `75ccd05cafceb0fd1f86be4cd0f2117249463c66` records: -- Full fusion (routing through combine) is extremely complex to implement correctly -- Expert load imbalance is the primary practical bottleneck -- CUDA graph compatibility requires masked layout (fixed allocation per expert) -- Small expert token counts cause thin-GEMM inefficiency on tensor cores -- FP8 block scaling adds memory overhead for scale factor storage +- bare-metal NVIDIA B200 (`sm_100a`) with clocks locked to `3996,1965`; +- container `flashinfer/flashinfer-ci-cu132:20260401-2c675fb`; +- CUDA 13.2, Python 3.12, PyTorch 2.12.0+cu132, and Triton 3.6.0; +- correctness gates `atol=1`, `rtol=0.3`, and matched ratio `0.9` for the MoE command; and +- an arithmetic mean of per-workload `FlashInfer baseline latency / candidate latency` as the single-definition MoE score. -## Sources +The current primary sources do not support the former framework TFLOPS/latency table, its launch counts, or the structured 1262-TFLOPS record. No performance result is retained here. The official trace axis is `seq_len`; relabeling its endpoints as prefill/decode or batch size requires a separate serving experiment. + +## Implementation Risks That Must Be Measured -- [FlashInfer MLSys 2026 Contest](https://mlsys26.flashinfer.ai/) -- [GPU Mode NVFP4 Hackathon Problem 3](https://github.com/gpu-mode/reference-kernels) -- [DeepGEMM MoE](https://github.com/deepseek-ai/DeepGEMM) -- [SGLang Fused MoE](https://github.com/sgl-project/sglang) +- Routing produces different token counts per expert, hence different grouped-GEMM M dimensions. A scheduler can mitigate that imbalance; no single bottleneck is universal across token counts and tactics. +- Small M or partially filled GEMM tiles can reduce utilization. This is a workload-dependent risk, not a retained performance result. +- At the pinned FlashInfer revision, runtime autotuning enumerates valid tactics and selects GEMM1 and GEMM2 tactics over token buckets. Record the exact revision and tuning state in any measurement. +- The FP8 element type alone is insufficient for compatibility. Routing method, global/local expert mapping, scale dtype and granularity, tensor layout, activation, and output semantics must all match. +- CUDA-graph layout requirements are backend-specific. DeepGEMM's masked grouped layout is one documented decode case when the CPU does not know expert token counts; it is not a universal graph requirement. +- TMA alignment depends on the tensor-map configuration. CUDA 13.2.1 requires a 64-byte-aligned tensor-map object and ordinarily a 16-byte-aligned global base, with additional requirements for selected interleave, dtype, and swizzle modes; it does not require every expert pointer to be 128-byte aligned. -## Full Reference Implementation +## Adjacent Local Artifacts + +The files under [`full/`](../../artifacts/kernels/fused-moe/full/) are mixed **adjacent references**, not a full implementation of this benchmark: + +- `vllm-PR-23696-dual-gemm.patch` is the aggregate merged diff for vLLM's MXFP4-weight fused expert-compute integration (BF16 activations on Hopper and MXFP8 activations on Blackwell). +- `flashinfer_cutedsl.py` is byte-identical to an SGLang FP4 CuteDSL runner at commit `c554dc5c64b661f2c53225b03a76359eaddc39e4`. +- `moe-grouped-gemm-launch.cpp` is local illustrative pseudocode for varying per-expert M segments. + +None supplies the exact Track A FP8 implementation or substantiates performance/launch claims. Their hashes and exact scopes are recorded in the bundle provenance. + +## Sources -Verbatim upstream code lives in [`artifacts/kernels/fused-moe/full/`](../../artifacts/kernels/fused-moe/full/); labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` header) live in [`artifacts/kernels/fused-moe/variants/`](../../artifacts/kernels/fused-moe/variants/). Every file's SHA-256 and upstream-pinning metadata is in `PROVENANCE.yaml` inside each bundle. +- [MLSys 2026 FlashInfer contest](https://mlsys26.flashinfer.ai/) +- [Exact FP8 MoE benchmark definition](https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048) +- [Starter-kit evaluation contract at `75ccd05`](https://github.com/flashinfer-ai/flashinfer-bench-starter-kit/blob/75ccd05cafceb0fd1f86be4cd0f2117249463c66/EVALUATION.md) +- [FlashInfer trace reference at `7f614b8`](https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/trace/templates/moe.py) +- [FlashInfer API implementation at `7f614b8`](https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/fused_moe/core.py) +- [CUDA 13.2.1 tensor-map requirements](https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html) +- [CUTLASS efficient-GEMM small-dimension discussion](https://github.com/NVIDIA/cutlass/blob/main/media/docs/cpp/efficient_gemm.md) +- [DeepGEMM grouped-layout documentation at `891d57b4`](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md) -Query via: +Query the page and its explicitly labeled artifacts with: ```bash python3 scripts/get_page.py kernel-fused-moe --include-code diff --git a/wiki/kernels/gated-delta-net.md b/wiki/kernels/gated-delta-net.md index 39930b930..d3b0941d7 100644 --- a/wiki/kernels/gated-delta-net.md +++ b/wiki/kernels/gated-delta-net.md @@ -18,254 +18,111 @@ kernel_types: - prefill - attention languages: +- cute-dsl - triton -- cuda-cpp +- python related: - technique-pipeline-stages sources: - blog-gated-delta-net +- contest-flashinfer-track-c +- blog-qwen3-next-architecture - doc-tfla - pr-vllm-37303 -performance_claims: -- gpu: H100 - dtype: bf16 - shape: seqlen=8192, qk_dim=4, v_dim=8, d=128 - metric: speedup - value: 10 - utilization: vs Qwen3-32B at 32K+ context, O(n) linear complexity - source_id: blog-gated-delta-net -blackwell_relevance: Blackwell prefill kernel in progress; decode done for both SM90/SM100. - TFLA uses tcgen05 PTX on Blackwell. +performance_claims: [] +blackwell_relevance: FlashInfer commit 7f614b8 implements an SM100/SM103 CuTe + DSL prefill path that requires CUDA 13 or newer and head size 128. artifact_dir: artifacts/kernels/gated-delta-net --- -# Gated Delta Net -- Linear Attention +# Gated Delta Net — Linear Attention -## Overview +## Verified mechanism -Gated Delta Networks (GatedDeltaNet) replace standard O(n^2) attention with an O(n) linear attention mechanism that uses a delta rule for error-correcting memory updates and exponential gating for adaptive decay. Published at ICLR 2025 by NVlabs, GatedDeltaNet is deployed in production in Qwen3-Next-80B (3:1 hybrid ratio with full attention) and Qwen3.5 (262K context window). - -The key advantage is O(1) per-token cost during decoding: the recurrent state is a fixed-size matrix that gets updated with each new token, eliminating the KV cache growth problem entirely. - -## Architecture in Qwen3-Next - -``` -Qwen3-Next-80B: 48 layers -Pattern: 12 x (3 x [GatedDeltaNet -> MoE] -> [Full Attention -> MoE]) - -Layer distribution: - - 36 GatedDeltaNet layers (75%): O(n) linear attention - - 12 Full Attention layers (25%): Standard GQA for global retrieval - - All layers followed by MoE: 512 experts, ~19 active per token - -Total: 80B parameters, only 3B active per token -``` - -## Delta Rule Mechanism - -Unlike standard linear attention (which uses simple additive updates to the recurrent state), the delta rule performs targeted error-correcting updates: +Gated DeltaNet is a recurrent linear-attention architecture published at ICLR 2025. For one head with state `S` shaped `[K,V]`, the FlashInfer reference at commit `7f614b8` implements the update in this equivalent form: ```python -# Standard linear attention (additive): -# S_t = S_{t-1} + v_t @ k_t^T -# -# Delta rule (error-correcting): -# S_t = S_{t-1} + (v_t - S_{t-1} @ k_t) @ k_t^T -# ^^^^^^^^^^^^^^^^^^^^^^^^ -# Error correction term - -def delta_rule_step(S, k, v, beta, alpha): - """ - Single step of gated delta rule update. - - S: recurrent state [d_k, d_v] (the "memory matrix") - k: key vector [d_k] - v: value vector [d_v] - beta: exponential gate (learned, controls decay) - alpha: delta gate (learned, controls update strength) - """ - # Retrieve what the current state "thinks" about this key - v_retrieved = S @ k # [d_v] - - # Error: difference between true value and retrieved value - delta = v - v_retrieved # [d_v] - - # Gated update: decay old state, add error-corrected new info - S_new = beta * S + alpha * (delta[:, None] @ k[None, :]) - # ^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - # decay error-correcting update - - return S_new +def gdn_step(S, q, k, v, A_log, a, dt_bias, b, scale): + g = exp(-exp(A_log) * softplus(a + dt_bias)) + beta = sigmoid(b) + decayed = g * S + read = k @ decayed + S_new = decayed + outer(k, beta * (v - read)) + output = scale * (q @ S_new) + return output, S_new ``` -## Chunk-Based Parallel Prefill - -During prefill, sequences are divided into chunks that can be processed in parallel. Within each chunk, the inter-token dependencies are resolved via a causal linear recurrence; across chunks, the recurrent state is propagated sequentially. - -```python -import triton -import triton.language as tl - -@triton.jit -def gated_delta_net_chunk_fwd( - Q_ptr, K_ptr, V_ptr, Beta_ptr, O_ptr, State_ptr, - SEQ_LEN: tl.constexpr, - CHUNK_SIZE: tl.constexpr, # e.g., 64 or 128 - D_QK: tl.constexpr, # qk_dim * d = 4 * 128 = 512 - D_V: tl.constexpr, # v_dim * d = 8 * 128 = 1024 -): - """ - Chunk-parallel forward pass for GatedDeltaNet. - Two levels of parallelism: - 1. Across chunks (parallel after inter-chunk state propagation) - 2. Within chunks (parallel via matrix operations) - """ - chunk_id = tl.program_id(0) - batch_id = tl.program_id(1) - head_id = tl.program_id(2) +The subtraction is the delta-rule correction: the update moves the value retrieved at `k` toward `v`, while `g` independently decays the previous state. The recurrent state has no prior-token axis, so a single GDN decode step has work and storage fixed with respect to context length. Its per-head state update is still proportional to `K*V`; “constant” here means constant in the number of earlier tokens. - chunk_start = chunk_id * CHUNK_SIZE +## Qwen3-Next architecture - # Load recurrent state from previous chunk - # S shape: [D_QK, D_V] - S = tl.load(State_ptr + (batch_id * NUM_HEADS + head_id) * D_QK * D_V - + tl.arange(0, D_QK)[:, None] * D_V - + tl.arange(0, D_V)[None, :]) +The immutable `Qwen3-Next-80B-A3B-Instruct` configuration at revision `9c7f2fbe` records: - # Intra-chunk computation - for t in range(CHUNK_SIZE): - pos = chunk_start + t +| Field | Value | +| --- | --- | +| Layers | 48 | +| Hybrid layout | `12 * (3 * Gated DeltaNet -> MoE, 1 * Gated Attention -> MoE)` | +| GDN heads | 16 QK heads, 32 value heads, head dimension 128 | +| MoE | 512 experts; 10 routed experts plus one shared expert | +| Parameters | 80B total, 3B activated | +| Native context | 262,144 tokens | - # Load q, k, v, beta for this token - q = tl.load(Q_ptr + pos * D_QK + tl.arange(0, D_QK)) - k = tl.load(K_ptr + pos * D_QK + tl.arange(0, D_QK)) - v = tl.load(V_ptr + pos * D_V + tl.arange(0, D_V)) - beta = tl.load(Beta_ptr + pos) +Thus 36 layers use GDN and 12 use full Gated Attention. The fixed GDN state does not eliminate cache growth for the whole hybrid model: the full-attention layers retain their own context-dependent cache. - # Output: query the state - o = tl.sum(S * q[:, None], axis=0) # [D_V] +Qwen's architecture article attributes attention-sink and massive-activation mitigation to the output gate in its full Gated Attention path and says the gate *helps* address those effects. That is separate from the GDN recurrence and is not a requirement imposed by the GDN kernel contract. - # Delta rule update - v_retrieved = tl.sum(S * k[:, None], axis=0) - delta = v - v_retrieved +## FlashInfer-Bench Track C contracts - # Gated state update - S = beta * S + delta[:, None] * k[None, :] # [D_QK, D_V] +The MLSys 2026 organizer identifies Gated Delta Net as Track C and links separate verified definitions captured from Qwen3-Next with tensor parallelism four. - # Store output - tl.store(O_ptr + pos * D_V + tl.arange(0, D_V), o) - - # Store final state for next chunk - tl.store(State_ptr + (batch_id * NUM_HEADS + head_id) * D_QK * D_V - + tl.arange(0, D_QK)[:, None] * D_V - + tl.arange(0, D_V)[None, :], S) -``` - -## Streaming Decode Kernel - -During autoregressive decoding, each new token only requires one state update -- O(1) per token regardless of context length. - -```python -@triton.jit -def gated_delta_net_decode( - Q_ptr, K_ptr, V_ptr, Beta_ptr, O_ptr, State_ptr, - D_QK: tl.constexpr, - D_V: tl.constexpr, -): - """ - Single-token decode: O(1) per token. - The recurrent state replaces the KV cache entirely. - State size: D_QK * D_V (e.g., 512 * 1024 = 512K floats per head) - """ - batch_id = tl.program_id(0) - head_id = tl.program_id(1) - - # Load persistent recurrent state - state_offset = (batch_id * NUM_HEADS + head_id) * D_QK * D_V - S = tl.load(State_ptr + state_offset - + tl.arange(0, D_QK)[:, None] * D_V - + tl.arange(0, D_V)[None, :]) - - # Load new token - q = tl.load(Q_ptr + tl.arange(0, D_QK)) - k = tl.load(K_ptr + tl.arange(0, D_QK)) - v = tl.load(V_ptr + tl.arange(0, D_V)) - beta = tl.load(Beta_ptr) - - # Query state for output - o = tl.sum(S * q[:, None], axis=0) - - # Update state with delta rule - v_retrieved = tl.sum(S * k[:, None], axis=0) - delta = v - v_retrieved - S = beta * S + delta[:, None] * k[None, :] - - # Store - tl.store(O_ptr + tl.arange(0, D_V), o) - tl.store(State_ptr + state_offset - + tl.arange(0, D_QK)[:, None] * D_V - + tl.arange(0, D_V)[None, :], S) -``` - -## TFLA: Tiled Flash Linear Attention - -TFLA adds a second level of tiling within chunks, enabling arbitrarily large chunk sizes. It emits matmuls as inline PTX assembly for both Hopper (WGMMA) and Blackwell (tcgen05). - -```cpp -// TFLA: Inline PTX for Blackwell tcgen05 matmul within chunk tiles -// Two levels of parallelism: standard chunkwise + tiling within chunks - -// SM100 path: tcgen05.mma for intra-chunk matrix operations -asm volatile( - "tcgen05.mma.cta_group::1.kind::f16f16f32" - " [%0], %1, %2;" - : - : "l"(tmem_addr), "l"(a_smem_addr), "l"(b_smem_addr) -); -``` +| Axis | Decode | Prefill | +| --- | --- | --- | +| Q heads / K heads / V heads | `4 / 4 / 8` | `4 / 4 / 8` | +| Head size | `128` | `128` | +| Token axis | `seq_len=1`, variable `batch_size` | variable `total_seq_len` and `num_seqs` | +| Q/K/V dtype | BF16 | BF16 | +| State | FP32 `[B,8,128,128]` | FP32 `[N,8,128,128]` | +| Variable-length metadata | none | `cu_seqlens[N+1]` | -## Implementations +Decode exposes `A_log`, `a`, `dt_bias`, and `b` for the decay and update gates, plus an optional scale. It returns BF16 output `[B,1,8,128]` and the updated state. Prefill returns BF16 output `[total_seq_len,8,128]` and one final state per sequence. -Two main implementations exist: +For this exact geometry, one value-head state contains `128*128 = 16,384` FP32 values, or 64 KiB. All eight value heads contain 131,072 FP32 values, or 512 KiB, per sequence and layer. The heads are independent states; they must not be flattened into one `512x1024` matrix. -| Implementation | Source | Notes | -|----------------|--------|-------| -| NVlabs/GatedDeltaNet | Reference | Triton kernels, research quality | -| FLA (Flash Linear Attention) | Recommended | Optimized, significantly faster, variable-length support | +## Implementations and architecture scope -## FlashInfer MLSys 2026 Contest +- NVlabs commit `b53d6d3` is the authors' PyTorch/Triton research implementation. Its README recommends FLA for faster kernels and variable-length functionality. +- FlashInfer commit `7f614b8` supplies the exact decode/prefill reference contracts used above. Its prefill wrapper dispatches SM90 and SM100 implementations; the SM100/SM103 path uses CuTe DSL, requires CUDA 13 or newer, and fixes head size 128. +- vLLM merge `e1d85e5c` gives its recurrent-attention backend uniform-batch CUDA-graph support for decode. CUDA graphs reduce CPU launch setup cost, but their benefit remains workload-dependent. -GatedDeltaNet is Track C of the FlashInfer MLSys 2026 contest: -- Parameters: qk_dim=4, v_dim=8, d=128 -- Benchmarks: decode `qk4_v8_d128_k_last`, prefill `qk4_v8_d128_k_last` -- Status: decode done for both Hopper and Blackwell; prefill done on Hopper, in progress for Blackwell +TFLA (`arXiv:2503.14376v3`) is related linear-recurrence work, but its published application and official code are for mLSTM. They use a second level of sequence parallelization to permit arbitrarily large chunks. They do not establish a GDN implementation or an inline WGMMA/tcgen05 path, so no TFLA assembly is presented here. -## When to Use +## Performance boundary and use -- Long-context inference (32K+) where O(n) scaling provides major throughput gains -- Hybrid architectures combining linear attention (75% of layers) with full attention (25%) -- Streaming decode where O(1) per-token cost eliminates KV cache growth +GDN sequence mixing scales linearly with sequence length, but asymptotic complexity is not a measured speedup. Qwen reports a 10x whole-model inference-throughput comparison against Qwen3-32B for contexts over 32K, while also warning that efficiency depends strongly on implementation. That result does not isolate this kernel, name a GPU, or describe the Track C qk4/v8/d128 workloads, so it is not stored as a kernel performance record. -## Caveats +Use an exact backend measurement for the intended batch, sequence distribution, dtype, state layout, software revision, and GPU. Also account for the context-growing full-attention cache in a hybrid model and for the GDN layer's learned projections, gates, convolution, and state; it is not a weight-compatible runtime substitute for a trained softmax-attention layer. -- Triton-based kernels have CPU launch overhead impacting small decode batches; use CUDA graph mode via vLLM -- Recurrent state size (D_QK * D_V per head) can be large -- 512K floats for typical configs -- Quality depends on the learned gating; not a drop-in replacement for standard attention without retraining -- Attention output gating (in Qwen3.5) is required to eliminate Attention Sink and Massive Activation problems +## Local artifacts -## Sources +The `full/` bundle contains one byte-pinned SGLang file from merge `5bdc07d974f6cf236fa765a685453ea5e587a838`. It fuses projection-output split/reshape/concatenation for Qwen3-Next/Qwen3.5; it is adjacent preprocessing, not a GDN recurrence, prefill, or decode implementation. -- [GatedDeltaNet (ICLR 2025)](https://github.com/NVlabs/GatedDeltaNet) -- [TFLA paper](https://arxiv.org/abs/2503.14376) -- [Qwen3-Next NVIDIA blog](https://developer.nvidia.com/blog/new-open-source-qwen3-next-models-preview-hybrid-moe-architecture-delivering-improved-accuracy-and-accelerated-parallel-processing-across-nvidia-platform/) -- [FlashInfer MLSys 2026 Contest](https://mlsys26.flashinfer.ai/) +The `variants/` bundle contains a standard-library, one-head recurrence check derived from the FlashInfer reference. It checks the compact update against the independently expanded remove/write form and rejects an additive-only negative control. It is not a tuned GPU kernel. -## Full Reference Implementation +## Primary sources -Verbatim upstream code lives in [`artifacts/kernels/gated-delta-net/full/`](../../artifacts/kernels/gated-delta-net/full/); labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` header) live in [`artifacts/kernels/gated-delta-net/variants/`](../../artifacts/kernels/gated-delta-net/variants/). Every file's SHA-256 and upstream-pinning metadata is in `PROVENANCE.yaml` inside each bundle. +- [Gated DeltaNet paper](https://arxiv.org/abs/2412.06464) +- [NVlabs implementation at `b53d6d3`](https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921) +- [Qwen3-Next model card and configuration at `9c7f2fbe`](https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/tree/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7) +- [MLSys 2026 organizer](https://mlsys26.flashinfer.ai/) +- [Exact decode definition](https://bench.flashinfer.ai/kernels/gdn_decode_qk4_v8_d128_k_last) +- [Exact prefill definition](https://bench.flashinfer.ai/kernels/gdn_prefill_qk4_v8_d128_k_last) +- [FlashInfer GDN trace at `7f614b8`](https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/trace/templates/gdn.py) +- [FlashInfer prefill dispatch at `7f614b8`](https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/gdn_prefill.py) +- [CUDA Graphs programming guide](https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cuda-graphs.html) +- [TFLA v3](https://arxiv.org/abs/2503.14376v3) Query via: ```bash -python3 scripts/get_page.py kernel-gated-delta-net --include-code +conda run -n base python scripts/get_page.py kernel-gated-delta-net --include-code ``` diff --git a/wiki/kernels/gated-dual-gemm.md b/wiki/kernels/gated-dual-gemm.md index 8ec0d42d3..9c815c952 100644 --- a/wiki/kernels/gated-dual-gemm.md +++ b/wiki/kernels/gated-dual-gemm.md @@ -1,17 +1,16 @@ --- id: kernel-gated-dual-gemm -title: Gated Dual GEMM (Gate-Up + SwiGLU Fusion) +title: GPU Mode NVFP4 Gated Dual GEMM type: kernel architectures: - sm100 -- sm90 +- sm100a tags: - gated-dual-gemm - gemm - fused-kernel -- kernel-fusion - nvfp4 -- tmem +- block-scale confidence: source-reported reproducibility: snippet kernel_types: @@ -19,8 +18,7 @@ kernel_types: - gemm - fused-kernel languages: -- cuda-cpp -- cute-dsl +- python related: - kernel-nvfp4-gemm - kernel-fused-moe @@ -28,100 +26,77 @@ related: - technique-epilogue-fusion sources: - contest-gpumode-p3 -- blog-deepgemm -- blog-tflops-gap-fp4-moe -- pr-vllm-23696 -performance_claims: -- gpu: B200 - dtype: nvfp4 - shape: M=1024 N=2*2048 K=7168 (gate-up MLP) - metric: latency_us - value: 18.5 - utilization: compute-bound - source_id: contest-gpumode-p3 -blackwell_relevance: TMEM holds two accumulators simultaneously (gate, up), enabling - single-kernel fusion that Hopper register file could not handle efficiently. +performance_claims: [] +blackwell_relevance: The official challenge targets NVIDIA B200 and its NVFP4 + block-scaled tensor contract; no winning instruction schedule is asserted. artifact_dir: artifacts/kernels/gated-dual-gemm --- -# Gated Dual GEMM +# GPU Mode NVFP4 Gated Dual GEMM -## Overview +## Verified scope -Gated dual GEMM fuses two matrix multiplications with activation and elementwise operations — the canonical MLP gate-up pattern used by LLaMA, Qwen, DeepSeek, and most modern LLMs. Fusion eliminates two global memory roundtrips compared to separate gate/up GEMM + SwiGLU. +The official NVIDIA rules identify NVFP4 Gated Dual GEMM as Kernel Challenge 3 of the Blackwell NVFP4 Hackathon, open from December 20, 2025 through January 16, 2026. The public GPU Mode problem at challenge-opening commit `c5b2f7c` targets NVIDIA B200. -This was Problem 3 of the GPU Mode NVFP4 Hackathon. +For each batch index, the required result is two block-scaled matrix products sharing the same left operand, with SiLU applied only to the first branch: -## Fused Operation - -``` -Given x, W_gate, W_up (weights shared same K dimension) -Standard (unfused): - gate = x @ W_gate (GEMM 1: reads x, W_gate, writes gate) - up = x @ W_up (GEMM 2: reads x, W_up, writes up) - silu = gate * sigmoid(gate) (elementwise: reads gate, writes silu) - out = silu * up (elementwise: reads silu, up, writes out) - -Fused: - out = SiLU(x @ W_gate) * (x @ W_up) (single kernel, no intermediate GMEM) +```python +def gated_dual_result(a, b1, b2, scale_a, scale_b1, scale_b2, scaled_mm): + gate = scaled_mm(a, b1.T, scale_a, scale_b1) + up = scaled_mm(a, b2.T, scale_a, scale_b2) + return silu(gate) * up ``` -## Kernel Structure (Blackwell) - -```cuda -template -__global__ void gated_dual_gemm_nvfp4( - const nvfp4_t* __restrict__ X, // [M, K] input - const nvfp4_t* __restrict__ W_gate, // [N, K] gate weights - const nvfp4_t* __restrict__ W_up, // [N, K] up weights - const fp8_t* sf_x, const fp8_t* sf_gate, const fp8_t* sf_up, - half* __restrict__ output, // [M, N] output - int M, int N, int K -) { - // Two TMEM regions: one accumulator per GEMM - uint32_t tmem_gate = tmem_alloc(256); - uint32_t tmem_up = tmem_alloc(256); - - for (int k = 0; k < K; k += BLOCK_K) { - int stage = (k / BLOCK_K) % NUM_STAGES; - mbarrier_wait(&tma_done[stage]); - - // Two MMAs per K-tile: gate and up - // Both read same X tile, different weight tiles - tcgen05_mma(x_smem[stage], wg_smem[stage], tmem_gate); - tcgen05_mma(x_smem[stage], wu_smem[stage], tmem_up); - } - - // Fused epilogue: SiLU(gate) * up - float g = tmem_load(tmem_gate); - float u = tmem_load(tmem_up); - float s = g / (1.0f + expf(-g)); // SiLU - output[row * N + col] = __float2half(s * u); - - tmem_dealloc(tmem_gate, 256); - tmem_dealloc(tmem_up, 256); -} -``` +This fixes result semantics and branch order. It does not require the submission entry point to use one GPU launch or forbid intermediate storage. + +## Published tensor contract + +| Tensor | Published dtype | Logical shape | +| --- | --- | --- | +| `a` | NVFP4 E2M1 | `[M,K,L]`, K-major | +| `b1`, `b2` | NVFP4 E2M1 | `[N,K,L]`, K-major | +| `sfa` | FP8 E4M3FNUZ | `[M,K/16,L]`, K-major | +| `sfb1`, `sfb2` | FP8 E4M3FNUZ | `[N,K/16,L]`, K-major | +| `c` | FP16 | `[M,N,L]` | + +The submission tuple also supplies layout-reordered copies of `sfa`, `sfb1`, and `sfb2`, plus preallocated `c`. `K` is divisible by 256; `M` and `N` must be divisible by the selected MMA tile dimensions. The correctness checker uses `rtol=1e-3` and `atol=1e-3`. + +The pinned upstream files contain one dtype-label inconsistency: `task.yml` and `template.py` call the scales E4M3FNUZ, while `reference.py` constructs `torch.float8_e4m3fn`. A backend must follow the actual submission objects rather than treating those spellings as interchangeable. + +## Published workloads and theoretical bounds + +| M | N | K | L | Task speed-of-light time (µs) | +| ---: | ---: | ---: | ---: | ---: | +| 256 | 4096 | 7168 | 1 | 4.708 | +| 512 | 4096 | 7168 | 1 | 8.714 | +| 256 | 3072 | 4096 | 1 | 2.125 | +| 512 | 3072 | 7168 | 1 | 6.535 | + +Ranking uses the geometric mean of the four benchmark times. The task explicitly labels the final column a speed-of-light analysis based on the maximum of B200 FP4 Tensor Core math time and DRAM-memory time at a 1.5 GHz clock. These values are theoretical comparison bounds, not measured contestant results. The former `M=1024, N=4096, K=7168, 18.5 µs` record is not one of the published workloads and has been removed. + +## Implementation boundary + +The shared `a` and `sfa` inputs create an opportunity to reuse data across the two products, and a genuinely fused epilogue can avoid writing both full-precision products to global memory. Those are optimization possibilities, not task guarantees. + +The public task/reference does not establish a final leaderboard, winning source, single-launch decomposition, physical load count, TMEM accumulator partition, TMA pipeline, CUTLASS schedule, or compute-/memory-bound classification. If an implementation uses `tcgen05`, TMEM, or TMA, its exact descriptors, scale layouts, synchronization, allocation, and architecture requirements must be verified from that implementation and the version-matched ISA. -## Key Optimizations +Compatibility is correspondingly narrower than “any dual-output operation”: a model/backend must match the two same-shaped NVFP4 products, scale storage and layouts, first-branch SiLU, FP16 output, batching, and divisibility rules. Grouped per-expert MoE execution is a different contract unless an implementation explicitly supplies that grouping layer. -1. **X reuse**: Same X tile feeds both MMAs — loaded once, used twice -2. **TMEM dual accumulator**: Blackwell's 512-column TMEM fits two 256-col accumulators side-by-side -3. **Fused epilogue**: SiLU and multiply happen after TMEM load, no intermediate SMEM -4. **Shared SFA**: X's block scales apply to both gate and up computations +## Local artifacts -## When To Use +The `full/` bundle contains the byte-pinned official `task.yml` from commit `c5b2f7c`; it is the problem specification, not an optimized submission. The unrelated third-party schedule extract and duplicate vLLM MXFP4 MoE patch formerly stored here were removed; the latter remains preserved in its own PR artifact collection. -- MLP layers in modern LLMs (LLaMA, Qwen, DeepSeek, Mistral) -- Any dual-output operation sharing one input -- MoE expert computations (expand to per-expert fused kernels) +The `variants/` bundle contains a standard-library semantic reference for two small dense products followed by `SiLU(gate) * up`. Its self-test compares compact and expanded forms and rejects the wrong alternative that gates the second branch. It does not model NVFP4 packing, block scales, GPU execution, or performance. -## Full Reference Implementation +## Primary sources -The reference bundle lives in [`artifacts/kernels/gated-dual-gemm/full/`](../../artifacts/kernels/gated-dual-gemm/full/) and combines the upstream vLLM PR-23696 diff (`vllm-PR-23696-gated-dual-gemm.patch`, `mode: upstream-patch`) with an extracted CUTLASS-schedule snippet from the `tflops-gap-fp4-moe` blog (`blackwell-cutlass-schedules-and-tma.cu`, `mode: extracted`). Labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` header) live in [`artifacts/kernels/gated-dual-gemm/variants/`](../../artifacts/kernels/gated-dual-gemm/variants/). Every file's SHA-256 and upstream-pinning metadata is in `PROVENANCE.yaml` inside each bundle. +- [Official challenge rules](https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf) +- [Pinned public task](https://github.com/gpu-mode/reference-kernels/blob/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm/task.yml) +- [Pinned starter template](https://github.com/gpu-mode/reference-kernels/blob/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm/template.py) +- [Pinned correctness reference](https://github.com/gpu-mode/reference-kernels/blob/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm/reference.py) Query via: ```bash -python3 scripts/get_page.py kernel-gated-dual-gemm --include-code +conda run -n base python scripts/get_page.py kernel-gated-dual-gemm --include-code ``` diff --git a/wiki/kernels/grouped-gemm.md b/wiki/kernels/grouped-gemm.md index 553fc11b2..4d18cfa0f 100644 --- a/wiki/kernels/grouped-gemm.md +++ b/wiki/kernels/grouped-gemm.md @@ -1,208 +1,145 @@ --- id: kernel-grouped-gemm -title: "Grouped GEMM for MoE" +title: Grouped GEMM Contracts for MoE and NVFP4 type: kernel -architectures: [sm100, sm100a, sm90] -tags: [grouped-gemm, moe, gemm, fp8, nvfp4, tcgen05, persistent-kernel, tile-scheduling] +architectures: +- sm100 +- sm100a +- sm90 +tags: +- grouped-gemm +- moe +- gemm +- fp8 +- nvfp4 +- tile-scheduling confidence: source-reported reproducibility: snippet -kernel_types: [grouped-gemm, gemm, moe] -languages: [cuda-cpp, cute-dsl] -related: [kernel-fused-moe, kernel-deepgemm, hw-tcgen05-mma, hw-clc, technique-persistent-kernels, technique-tile-scheduling] -sources: [contest-gpumode-p4, blog-deepgemm, doc-cutlass-blackwell] -performance_claims: - - gpu: B200 - dtype: nvfp4 - shape: "variable M, shared N=K, 15 groups" - metric: latency_us - value: 11.2 - utilization: "compute-bound" - source_id: contest-gpumode-p4 -blackwell_relevance: "SM100 CLC enables dynamic tile scheduling critical for variable-M grouped GEMM in MoE workloads." +kernel_types: +- grouped-gemm +- gemm +- moe +languages: +- python +related: +- kernel-fused-moe +- kernel-deepgemm +- technique-tile-scheduling +- hw-clc +sources: +- contest-gpumode-p4 +- blog-deepgemm +- blog-gpu-mode-reward-hack +performance_claims: [] +blackwell_relevance: The official challenge targets B200 with independent + per-group NVFP4 shapes; no particular CUTLASS, CLC, TMA, or launch topology + is required by the observable contract. --- -# Grouped GEMM for MoE +# Grouped GEMM Contracts for MoE and NVFP4 -## Overview +## Verified scope -Grouped GEMM computes multiple matrix multiplications with variable M dimensions but shared N and K, directly targeting MoE (Mixture of Experts) inference where each expert processes a different number of tokens. This is the most practically important kernel pattern for MoE serving: during inference, the router sends different token counts to each expert, and grouped GEMM batches all expert computations into a single kernel launch. +The official NVIDIA rules identify NVFP4 Grouped GEMM as Kernel Challenge 4 of the Blackwell NVFP4 Hackathon, open from January 17 through February 13, 2026. It carries 40% of the four-problem grand-prize score. The corrected public task at commit `ae67948` targets NVIDIA B200. -Grouped GEMM was Problem 4 (heaviest weight: 40%) of the GPU Mode NVFP4 Hackathon, and is also the core of DeepGEMM's MoE support. +“Grouped GEMM” does not imply one universal shape contract. Two relevant interfaces differ materially: -## Problem Structure +- The GPU Mode challenge accepts a list of independent problems; `M_i`, `N_i`, and `K_i` can all differ by group. +- DeepGEMM's M-grouped MoE APIs vary M while holding N and K fixed. It separately provides a K-grouped interface for MoE weight backward. -``` -Standard GEMM: C = A @ B (single problem) -Grouped GEMM: C_i = A_i @ B_i for i in [0, num_groups) - -MoE specialization: - - N and K are FIXED (same expert architecture) - - Only M varies (different token counts per expert) - - B_i may be different weight matrices (per-expert weights) - -Example (DeepSeek-V3, 256 experts, top-8 routing): - Group 0: M=47 tokens -> Expert 0 weights [N, K] - Group 1: M=23 tokens -> Expert 1 weights [N, K] - ... - Group 255: M=31 tokens -> Expert 255 weights [N, K] -``` +Neither interface definition proves that a conforming implementation uses exactly one GPU launch. -## DeepGEMM Grouped Layouts - -DeepGEMM provides three layouts optimized for different MoE phases: - -```cpp -// Layout 1: Contiguous (prefill) -// All expert inputs packed sequentially with cumulative offset array -// Memory: [Expert0 (M0 rows)] [Expert1 (M1 rows)] [Expert2 (M2 rows)]... -// Index: offsets[0]=0 offsets[1]=M0 offsets[2]=M0+M1 -struct ContiguousLayout { - const fp8_t* A; // Packed input [sum(M_i), K] - const fp8_t* B; // Expert weights [num_experts, N, K] - float* C; // Packed output [sum(M_i), N] - const int* offsets; // Cumulative M offsets [num_experts + 1] -}; - -// Layout 2: Masked (decode with CUDA graphs) -// Fixed allocation per expert, binary mask for valid tokens -// Compatible with CUDA graph capture (static shapes) -struct MaskedLayout { - const fp8_t* A; // [num_experts, M_max, K] - const fp8_t* B; // [num_experts, N, K] - float* C; // [num_experts, M_max, N] - const bool* mask; // [num_experts, M_max] validity flags -}; - -// Layout 3: K-grouped (weight gradients in training backward) -// Groups along K-axis instead of M-axis -struct KGroupedLayout { - const fp8_t* A; // [M, sum(K_i)] - const fp8_t* B; // Per-group B matrices with different K - float* C; // [M, N] - const int* k_offsets; // Cumulative K offsets -}; -``` +## GPU Mode Problem 4 contract -## CUTLASS Grouped GEMM on SM100 - -```cpp -// CUTLASS schedule for grouped GEMM on Blackwell -using Schedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100; - -// PtrArray mode: array of pointers to per-group A, B, C matrices -// TMA handles variable-offset loads via per-group descriptors -// CLC distributes tiles across groups dynamically - -using GemmKernel = cutlass::gemm::kernel::GemmGrouped< - cutlass::gemm::GemmShape<128, 256, 128>, // Tile shape - cutlass::arch::Sm100, - cutlass::float_e2m1_t, // NVFP4 operand type - cutlass::float_e2m1_t, - float, - cutlass::layout::RowMajor, - cutlass::layout::ColumnMajor ->; - -// Launch: single kernel handles all groups -GemmKernel::Arguments args{ - num_groups, - problem_sizes, // [num_groups] array of {M_i, N, K} - ptr_A, ptr_B, ptr_C, - scale_factors_A, scale_factors_B -}; -GemmKernel kernel; -kernel.run(args, stream); -``` +For each group `i`, the correctness reference computes `C_i = A_i @ B_i.T` with block scales and FP16 output. -## Tile Scheduling for Variable M - -### Static Scheduling (Precomputed) - -```cpp -// Precompute tile-to-expert mapping on host before launch -struct TileInfo { - int expert_id; - int tile_m_start; // Local M offset within this expert - int tile_n_start; -}; - -std::vector build_tile_schedule( - const int* M_per_expert, int num_experts, int N -) { - std::vector schedule; - for (int e = 0; e < num_experts; e++) { - for (int m = 0; m < M_per_expert[e]; m += BLOCK_M) - for (int n = 0; n < N; n += BLOCK_N) - schedule.push_back({e, m, n}); - } - return schedule; // Copy to device; blockIdx.x indexes into this -} -``` +| Per-group value | Published dtype | Logical shape | +| --- | --- | --- | +| `a_i` | packed NVFP4 E2M1, two values per byte | `[M_i,K_i/2,L_i]` | +| `b_i` | packed NVFP4 E2M1, two values per byte | `[N_i,K_i/2,L_i]` | +| `c_i` | FP16 | `[M_i,N_i,L_i]` | +| `sfa_i` | FP8 E4M3FNUZ in task/template | `[M_i,K_i/16,L_i]` | +| `sfb_i` | FP8 E4M3FNUZ in task/template | `[N_i,K_i/16,L_i]` | +| problem size | integers | `(M_i,N_i,K_i,L_i)` | -### Dynamic Scheduling (CLC / Persistent Kernel) - -```cpp -// Persistent kernel with atomic tile counter -// Each thread block loops, grabbing tiles until all are processed -__device__ int g_tile_counter = 0; - -__global__ void grouped_gemm_persistent( - const fp8_t** A_ptrs, const fp8_t** B_ptrs, float** C_ptrs, - const int* M_per_expert, int N, int K, - const TileInfo* tile_map, int total_tiles -) { - while (true) { - int tile_id = atomicAdd(&g_tile_counter, 1); - if (tile_id >= total_tiles) return; - - TileInfo info = tile_map[tile_id]; - int M_e = M_per_expert[info.expert_id]; - - // Effective BLOCK_M may be smaller for the last tile of an expert - int eff_m = min(BLOCK_M, M_e - info.tile_m_start); - - // TMA load + tcgen05.mma for this tile - tma_load(A_ptrs[info.expert_id] + info.tile_m_start * K, ...); - tma_load(B_ptrs[info.expert_id] + info.tile_n_start, ...); - tcgen05_mma(...); - - // Store result - store_tile(C_ptrs[info.expert_id] + info.tile_m_start * N - + info.tile_n_start, eff_m, BLOCK_N); - } -} -``` +The submission object actually contains four lists: logical A/B/C tensors, logical scales, reordered scale copies, and problem sizes. The `task.yml` prose lists only three tuple members, but `task.py`, `template.py`, and `reference.py` expose the reordered scales as the fourth. Each published case has `L=1`; `K_i` is divisible by 256, and `M_i`/`N_i` must satisfy the selected MMA tile divisibility. -## The Reward Hack +The pinned upstream files also disagree on the scale dtype suffix: task/template text says E4M3FNUZ, while `reference.py` constructs `torch.float8_e4m3fn`. A submission must follow the actual tensors it receives rather than silently treating those names as interchangeable. The correctness checker uses `rtol=1e-3` and `atol=1e-3`. -The 1st-place submission to Problem 4 exploited the evaluation harness: +For small non-empty ordinary-number matrices, this CPU-only reference isolates the group semantics. Each `b` is stored as `[N,K]`, so its rows are the columns of the mathematical right operand: +```python +def grouped_reference(groups): + outputs = [] + for a, b in groups: + k = len(a[0]) + if any(len(row) != k for row in a) or any(len(row) != k for row in b): + raise ValueError("ragged or incompatible K") + outputs.append([ + [sum(x * y for x, y in zip(a_row, b_row)) for b_row in b] + for a_row in a + ]) + return outputs ``` -Correctness phase: harness clones data -> real kernel runs correctly -Timing phase: harness reuses same objects -> - Call 1: fires 120-group super-batch (all 15 benchmark problems fused) - Calls 2-15: detect pre-computed results, skip computation -``` -This reported 11.191us (~2us ahead of second place). It led to improvements in the FlashInfer-Bench evaluation methodology for the MLSys 2026 contest. +It models neither NVFP4 packing nor block scales; those remain part of the challenge ABI above. + +## DeepGEMM's different grouped layouts + +The following shapes describe the pinned FP8/FP4 grouped APIs; scale tensors and layout conversions are additional required inputs. + +| Mode | Main tensors | Group descriptor | Documented role | +| --- | --- | --- | --- | +| M-grouped contiguous | A `[M,K]`, B `[G,N,K]`, D `[M,N]` | `grouped_layout` is either one expert ID per packed row (with `-1` padding) or one prefix-sum end per group | Training forward or inference prefill; N/K fixed and expert segments M-block aligned | +| M-grouped masked | A `[G,Mmax,K]`, B `[G,N,K]`, D `[G,Mmax,N]` | `masked_m` is one int32 valid-row count per group | CUDA-graph decode; fixed allocation while computing valid portions | +| K-grouped contiguous | packed A `[sum(K_i),M]`, B `[sum(K_i),N]`, D `[G,M,N]` | host and device K-size lists; optional C has `[G,M,N]` | MoE weight backward; M/N fixed | + +These are concrete library contracts, not generic C++ structs. Compatibility also depends on the documented architecture, dtype/scale format, operand-major mode, alignment, output dtype, and recipe constraints. -## When to Use +## Published workloads and theoretical bounds -- MoE inference: dispatch tokens to experts, compute per-expert projections -- Any workload with multiple GEMMs sharing N and K but varying M -- Prefill (contiguous layout) and decode (masked layout for CUDA graph compatibility) +| Groups | M values | N | K | L | Task speed-of-light time (µs) | +| ---: | --- | ---: | ---: | ---: | ---: | +| 8 | 80, 176, 128, 72, 64, 248, 96, 160 | 4096 | 7168 | 1 | 18.833 | +| 8 | 40, 76, 168, 72, 164, 148, 196, 160 | 7168 | 2048 | 1 | 10.667 | +| 2 | 192, 320 | 3072 | 4096 | 1 | 2.406 | +| 2 | 128, 384 | 4096 | 1536 | 1 | 1.525 | -## Caveats +Ranking uses the geometric mean. The task labels these microsecond values a speed-of-light analysis derived from the maximum of B200 FP4 Tensor Core math time and DRAM-memory time at a 1.5 GHz clock. They are theoretical comparison values, not measured contestant results. -- Expert load imbalance is the primary practical bottleneck (see [tail-effect](../patterns/tail-effect.md)) -- Small M per expert causes thin-GEMM inefficiency on tensor cores -- Masked layout wastes compute on padding when M distribution is skewed -- CLC dynamic scheduling adds hardware overhead vs static precomputed schedules -- TMA alignment (128 bytes) constrains minimum tile dimensions +## The scrubbed reward hack -## Sources +GPU Mode's official postmortem records a submission that temporarily reached the number-one leaderboard position with a reported `11.191 µs`, roughly `2 µs` ahead of the next entry, and was scrubbed minutes after the competition. -- [GPU Mode NVFP4 Hackathon](https://github.com/gpu-mode/reference-kernels) -- [DeepGEMM Grouped GEMM](https://github.com/deepseek-ai/DeepGEMM) -- [Reward Hack Writeup](https://www.gpumode.com/news/reward-hacking-nvfp4) -- [CUTLASS SM100 documentation](https://docs.nvidia.com/cutlass/latest/CHANGELOG.html) +During correctness, it ran a real padded 8-group kernel on each of 15 cloned data objects. During timing, the first call launched one merged 120-group kernel covering all 15 objects; calls 2 through 15 returned cached output pointers. The harness then divided the combined timing by 15. The reported number is evidence about the exploit, not a valid per-call performance record. + +The official post points to `gpu-mode/reference-kernels` PR #104 as the harness response. It does not attribute a FlashInfer-Bench or MLSys 2026 methodology change to this incident. + +## Implementation and performance boundary + +The public challenge constrains outputs, tolerances, workloads, and scoring. It does not require CUTLASS, CLC, TMA, TMEM, a persistent kernel, a static schedule, or one launch. The former CUTLASS and CUDA sketches were removed because they were not executable instances of the named APIs. + +CUTLASS documents that small M or N can leave threads outside the useful problem bounds, and that a small M/N grid with large K can launch too few threadblocks to use every multiprocessor. This is a possible shape effect, not proof that every grouped workload has the same bottleneck. GPU Mode's postmortem, for example, measured substantial fixed setup cost in its studied implementation. + +CLC itself uses an asynchronous cancellation request, shared response, mbarrier completion, and response decoding after a worker's initial block. That is different from a global `atomicAdd` tile queue. Whether CLC improves end-to-end time relative to a static or software-persistent scheduler requires a workload- and implementation-specific measurement. + +TMA does not have one universal 128-byte alignment rule from which minimum tile sizes follow. CUDA 13.2.1's tiled tensor-map API generally documents 16-byte global-address and stride alignment, a 64-byte descriptor, and feature-specific 32-byte constraints. An implementation must check the exact descriptor mode and version it uses. + +## Primary sources + +- [Official challenge rules](https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf) +- [Pinned public task](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/task.yml) +- [Pinned starter template](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/template.py) +- [Pinned correctness reference](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/reference.py) +- [Pinned DeepGEMM grouped overview](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md#grouped-gemms-contiguous-layout) +- [Pinned DeepGEMM grouped APIs](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp) +- [Official reward-hack postmortem](https://www.gpumode.com/news/reward-hacking-nvfp4) +- [CUDA 13.2.1 tensor-map constraints](https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html) +- [CUDA 13.2 CLC programming guide](https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html) +- [Pinned CUTLASS efficient-GEMM guide](https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/media/docs/cpp/efficient_gemm.md) + +Query via: + +```bash +conda run -n base python scripts/get_page.py kernel-grouped-gemm +``` diff --git a/wiki/kernels/nsa.md b/wiki/kernels/nsa.md index c42bcb286..723052f79 100644 --- a/wiki/kernels/nsa.md +++ b/wiki/kernels/nsa.md @@ -2,196 +2,116 @@ id: kernel-nsa title: "Native Sparse Attention (NSA)" type: kernel -architectures: [sm90, sm100] +architectures: [] tags: [sparse-attention, attention, triton] confidence: source-reported reproducibility: snippet kernel_types: [sparse-attention, attention] -languages: [triton] +languages: [python, triton] related: [kernel-flashmla, technique-pipeline-stages] -sources: [blog-nsa, blog-flashmla, blog-vllm-deepseek-v3-sparse] -performance_claims: - - gpu: H100 - dtype: bf16 - shape: "seqlen=65536" - metric: speedup - value: 9.0 - utilization: "vs FlashAttention-2 forward" - source_id: blog-nsa -blackwell_relevance: "Sparse attention patterns transfer directly; Blackwell's larger L2 (126MB) and higher bandwidth benefit sparse block fetches." +sources: [blog-nsa, blog-flashmla] +performance_claims: [] +blackwell_relevance: "The ACL paper benchmarks an A100 Triton implementation. It provides no SM90/SM100 compatibility result or Blackwell performance comparison; porting and benchmarking are separate work." --- # Native Sparse Attention (NSA) -## Overview +## Verified scope -Native Sparse Attention (NSA) is a hardware-aligned sparse attention mechanism published at ACL 2025, designed to reduce attention compute for long sequences (64K+) without sacrificing quality. Unlike post-hoc sparsity approaches, NSA is natively trainable end-to-end. It decomposes attention into three parallel paths -- token compression, token selection, and sliding window -- then fuses their outputs. +Native Sparse Attention is the three-branch, natively trainable sparse-attention design published in the ACL 2025 proceedings. The paper targets long-context training and inference and reports quality comparable to or better than full attention on its evaluated tasks. Those are paper-scoped results, not a guarantee for an arbitrary model or post-training conversion. -NSA achieves 9x forward speedup and 6x backward speedup at 64K sequences versus FlashAttention-2, and 11.6x decoding speedup at 64K context. It is deployed in DeepSeek-V3.2-Exp combined with FlashMLA sparse kernels. +The paper's efficiency experiments use an eight-GPU A100 system. The paper does not establish Hopper or Blackwell compatibility for its Triton implementation. -## Three-Path Architecture +## Exact three-branch mechanism -``` -Input Query Q - | - +---> [Compression Path] Learned MLP creates coarse-grained KV - | | representations (token compression) - | v - | S_compressed = Q @ K_compressed^T - | - +---> [Selection Path] Blockwise importance scores select - | | top-n fine-grained token blocks - | v - | S_selected = Q @ K_selected^T (sparse) - | - +---> [Sliding Window] Local context window (w=512) - | - v - S_local = Q @ K_local^T (banded) - -Output = Combine(softmax(S_compressed) @ V_compressed, - softmax(S_selected) @ V_selected, - softmax(S_local) @ V_local) -``` +For token representation \(h_t\), NSA computes three attention outputs and combines them with learned, input-dependent gates: -## Triton Kernel Implementation +\[ +o_t = g_t^{cmp} o_t^{cmp} + g_t^{slc} o_t^{slc} + g_t^{win} o_t^{win}, +\qquad +g_t^c = \operatorname{sigmoid}(\operatorname{MLP}_c(h_t)). +\] -The core NSA kernel is implemented in Triton with group-centric data loading and grid-based scheduling. +| Branch | Paper-defined role | +|---|---| +| Compression | Learned MLPs with intra-block position encoding compress overlapping KV blocks into coarse-grained representations. | +| Selection | Compression-attention scores are reused and aggregated into fine-grained block-importance scores; top-n blocks are selected and attended at full token resolution. | +| Sliding window | Recent tokens are attended directly to preserve local context. | -```python -import triton -import triton.language as tl - -@triton.jit -def nsa_sparse_attention_fwd( - Q_ptr, K_ptr, V_ptr, O_ptr, - block_indices_ptr, # Selected block indices per query - num_selected: tl.constexpr, - BLOCK_SIZE: tl.constexpr, - HEAD_DIM: tl.constexpr, - NUM_HEADS: tl.constexpr, - GROUP_SIZE: tl.constexpr, # GQA group size -): - """ - Sparse attention forward: only compute attention over selected KV blocks. - Group-centric loading: shares sparse KV blocks across all query heads - in a GQA group, minimizing redundant KV transfers. - """ - pid = tl.program_id(0) - head_id = tl.program_id(1) - batch_id = tl.program_id(2) - - # GQA: determine which KV head group this query head belongs to - kv_head_id = head_id // GROUP_SIZE - - # Load query tile - q_offset = batch_id * NUM_HEADS * HEAD_DIM + head_id * HEAD_DIM - q = tl.load(Q_ptr + q_offset + tl.arange(0, HEAD_DIM)) - - # Accumulator for online softmax - acc = tl.zeros([HEAD_DIM], dtype=tl.float32) - lse = float("-inf") - - # Iterate over selected blocks (sparse) - for block_idx in range(num_selected): - # Load block index for this query - sel_offset = (batch_id * NUM_HEADS + kv_head_id) * num_selected + block_idx - kv_block_start = tl.load(block_indices_ptr + sel_offset) * BLOCK_SIZE - - # Group-centric: load KV block once, shared across GROUP_SIZE query heads - k_offsets = kv_block_start + tl.arange(0, BLOCK_SIZE) - k_block = tl.load(K_ptr + (batch_id * kv_head_id * SEQ_LEN + k_offsets[:, None]) * HEAD_DIM - + tl.arange(0, HEAD_DIM)[None, :]) - v_block = tl.load(V_ptr + (batch_id * kv_head_id * SEQ_LEN + k_offsets[:, None]) * HEAD_DIM - + tl.arange(0, HEAD_DIM)[None, :]) - - # Compute attention scores for this block - scores = tl.sum(q[None, :] * k_block, axis=1) # [BLOCK_SIZE] - - # Online softmax update - block_max = tl.max(scores) - new_lse = tl.where(lse > block_max, - lse + tl.log(1.0 + tl.exp(block_max - lse)), - block_max + tl.log(1.0 + tl.exp(lse - block_max))) - - # Rescale accumulator and add new contribution - old_scale = tl.exp(lse - new_lse) - new_scale = tl.exp(scores - new_lse) - acc = acc * old_scale + tl.sum(new_scale[:, None] * v_block, axis=0) - lse = new_lse - - # Store output - o_offset = batch_id * NUM_HEADS * HEAD_DIM + head_id * HEAD_DIM - tl.store(O_ptr + o_offset + tl.arange(0, HEAD_DIM), acc) -``` +The main experimental configuration uses compression block length 32 and stride 16, selected block length 64 with 16 selected blocks, and a 512-token sliding window. These are experiment settings, not universal NSA constants. + +### Executable branch-combination reference -## Sliding Window Component +This CPU function checks only the learned gated-sum semantics above. It is KernelWiki-derived, uses ordinary numbers, and makes no claim about the paper's GPU layout or performance. ```python -@triton.jit -def nsa_sliding_window_fwd( - Q_ptr, K_ptr, V_ptr, O_ptr, - seq_pos, - WINDOW_SIZE: tl.constexpr, # 512 - HEAD_DIM: tl.constexpr, -): - """Local sliding window attention for recent context.""" - pid = tl.program_id(0) # query position - - # Window bounds - window_start = tl.maximum(0, seq_pos - WINDOW_SIZE) - window_end = seq_pos - - # Standard dense attention within window - # This is O(w*d) per query, where w=512 - q = tl.load(Q_ptr + pid * HEAD_DIM + tl.arange(0, HEAD_DIM)) - acc = tl.zeros([HEAD_DIM], dtype=tl.float32) - lse = float("-inf") - - for pos in range(window_start, window_end, BLOCK_SIZE): - k = tl.load(K_ptr + pos * HEAD_DIM + tl.arange(0, HEAD_DIM)) - v = tl.load(V_ptr + pos * HEAD_DIM + tl.arange(0, HEAD_DIM)) - score = tl.sum(q * k) - # Online softmax accumulation - new_lse = tl.where(lse > score, - lse + tl.log(1 + tl.exp(score - lse)), - score + tl.log(1 + tl.exp(lse - score))) - acc = acc * tl.exp(lse - new_lse) + v * tl.exp(score - new_lse) - lse = new_lse - - tl.store(O_ptr + pid * HEAD_DIM + tl.arange(0, HEAD_DIM), acc) +def gated_branch_sum(branch_outputs, gates): + """Combine equal-width branch vectors with one [0, 1] gate per branch.""" + if len(branch_outputs) != len(gates) or not branch_outputs: + raise ValueError("one gate is required for each non-empty branch set") + width = len(branch_outputs[0]) + if any(len(branch) != width for branch in branch_outputs): + raise ValueError("branch widths must match") + if any(not 0.0 <= gate <= 1.0 for gate in gates): + raise ValueError("sigmoid gates must lie in [0, 1]") + return [ + sum(gate * branch[i] for branch, gate in zip(branch_outputs, gates)) + for i in range(width) + ] + + +assert gated_branch_sum([[1.0, 2.0], [10.0, 20.0], [-2.0, 4.0]], + [0.5, 0.25, 1.0]) == [1.0, 10.0] ``` -## Hardware-Aligned Design +Removing the gates is a useful negative control: the raw branch sum for the same vectors is `[9.0, 26.0]`, not `[1.0, 10.0]`. + +## Paper-described Triton design -NSA's sparsity pattern is explicitly designed for GPU memory access efficiency: +The authors say compression and sliding-window attention can use existing FlashAttention-2-style kernels, while selected attention needs a specialized Triton kernel for training and prefill. Its source-described structure is: -1. **Blockwise memory access**: Selected tokens are organized in contiguous blocks (not scattered individual tokens), exploiting spatial locality for contiguous GPU memory loads -2. **Group-centric loading**: In GQA configurations, all query heads in a group share the same sparse KV blocks, minimizing redundant KV transfers to shared memory -3. **Grid-based scheduling**: Triton grid dimensions map directly to (query_block, head, batch), avoiding dynamic scheduling overhead +1. Load all query heads in one GQA/MQA group into SRAM so they share selected sparse KV indices. +2. Load selected KV data as contiguous blocks rather than scattered individual tokens. +3. Put the nearly constant query and output loops in Triton grid parallelism; keep the selected-block loop inside a program because its count is approximately constant. -## Performance +The paper does not publish the former page's kernel listing, specify a grid exactly as `(query_block, head, batch)`, or claim that such a grid eliminates dynamic scheduling overhead. The removed listings were non-executable and mathematically incorrect: the selected-attention sketch omitted within-block softmax normalization, and the sliding-window sketch stepped by a block size while loading only one token. -| Sequence Length | Forward Speedup | Backward Speedup | Decoding Speedup | -|----------------|-----------------|-------------------|-------------------| -| 64K | 9.0x | 6.0x | 11.6x | +## Source-reported efficiency -All speedups measured against FlashAttention-2 on H100 with BF16 precision. +| Quantity | Source scope at sequence/context length 65,536 | +|---|---| +| Training/prefill forward | 9.0x versus the authors' Triton FlashAttention-2 baseline; Figure 5 timing result | +| Training/prefill backward | 6.0x versus the same baseline; Figure 5 timing result | +| Decoding | 11.6x **expected** speedup from Table 4's memory-access volumes: 65,536 full-attention tokens versus 5,632 NSA-equivalent tokens | -## When to Use +The setup states eight A100 GPUs, GQA group count 4, 16 query heads per group, key dimension 192, and value dimension 128. The paper does not provide the benchmark dtype, software versions, batch details, raw samples, or variance. Consequently these values are retained only as explicitly labeled author reports, and `performance_claims` remains empty rather than encoding a falsely reproducible tuple. -- Long-context inference (32K+ tokens) where full attention is prohibitively expensive -- Models with GQA (grouped query attention) where KV sharing amplifies sparse access efficiency -- End-to-end trainable settings requiring differentiable sparsity +## NSA is not the later DSA deployment -## Caveats +DeepSeek-V3.2-Exp later introduced **DeepSeek Sparse Attention (DSA)**. Its pinned first-party inference code uses a learned indexer to select up to 2,048 token positions and masks attention to those positions. Its README points to DeepGEMM for indexer-logit kernels and FlashMLA for sparse-attention kernels. FlashMLA likewise says its sparse kernels power DSA. -- The compression path requires a learned MLP, adding parameters and training cost -- Token selection adds a two-pass overhead (score all blocks, then select top-n) -- Triton implementation has CPU launch overhead impacting small-batch decode; CUDA graph mode recommended -- Quality depends on training the sparsity selection jointly with the model +That deployed DSA path is related sparse-attention work, but it is not evidence that the ACL paper's gated compression/selection/window NSA architecture was deployed in V3.2-Exp. -## Sources +## Applicability and boundaries -- [NSA paper (ACL 2025)](https://arxiv.org/abs/2502.11089) -- [PyTorch reference implementation](https://github.com/lucidrains/native-sparse-attention-pytorch) +- NSA requires a model trained for its compression, selection, window, and gate mechanism; do not treat it as a drop-in sparse mask for an arbitrary checkpoint. +- GQA/MQA lets query heads in a group share selected blocks, matching the paper's group-centric kernel design. +- The paper evaluates long contexts through 64K but defines no universal 32K threshold. Compare quality, sequence distribution, memory use, and measured latency or throughput on the target workload. +- CUDA Graphs can reduce repeated-launch CPU overhead, but they are not an NSA-specific default. Profile first: NVIDIA's guidance says the largest gains occur for CPU-bound workflows and that GPU-bound workloads may see little benefit or regress. +- The compression MLP adds learned parameters and computation. The paper does not isolate a standalone cost for that component. +- The paper notes that a Triton implementation can retain abstraction overhead relative to native CUDA; it does not publish an NSA-specific CPU-launch profile. + +## Primary sources + +- [NSA in the ACL 2025 Anthology](https://aclanthology.org/2025.acl-long.1126/) +- [ACL proceedings PDF](https://aclanthology.org/2025.acl-long.1126.pdf) +- [DeepSeek-V3.2-Exp at pinned commit 87e509a](https://github.com/deepseek-ai/DeepSeek-V3.2-Exp/tree/87e509a2e5a100d221c97df52c6e8be7835f0057) +- [FlashMLA at pinned commit 71c7379](https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232) +- [NVIDIA CUDA Graph performance troubleshooting](https://docs.nvidia.com/dl-cuda-graph/troubleshooting/performance-issues.html) +- [Third-party lucidrains PyTorch implementation](https://github.com/lucidrains/native-sparse-attention-pytorch) + +## Query + +```bash +conda run -n base python scripts/get_page.py kernel-nsa +``` diff --git a/wiki/kernels/nvfp4-gemm.md b/wiki/kernels/nvfp4-gemm.md index 68cb8a49a..55533e04d 100644 --- a/wiki/kernels/nvfp4-gemm.md +++ b/wiki/kernels/nvfp4-gemm.md @@ -1,6 +1,6 @@ --- id: kernel-nvfp4-gemm -title: NVFP4 GEMM — 4-bit Floating Point Matrix Multiply +title: NVFP4 GEMM — GPU Mode Problem 2 Contract type: kernel architectures: - sm100 @@ -12,237 +12,154 @@ tags: - block-scale - tcgen05 - tmem -- warp-specialization -confidence: source-reported +- tma +confidence: verified +evidence_basis: +- source_id: doc-transformer-engine-2.13-nvfp4 + evidence_type: official-doc +- source_id: doc-ptx-isa-sm100 + evidence_type: official-doc reproducibility: snippet kernel_types: - gemm languages: -- cuda-cpp -- cute-dsl -- ptx +- python related: - hw-nvfp4 - hw-tcgen05-mma - hw-tmem +- hw-tma - kernel-nvfp4-gemv -- technique-warp-specialization sources: - contest-gpumode-p2 -- doc-cutlass-blackwell -- pr-cutlass-2139 -performance_claims: -- gpu: B200 - dtype: nvfp4 - shape: standard GEMM configs - metric: latency_us - value: 10.807 - utilization: near cuBLAS - source_id: contest-gpumode-p2 -artifact_dir: artifacts/kernels/nvfp4-gemm +- doc-transformer-engine-2.13-nvfp4 +- doc-ptx-isa-sm100 +- doc-cuda-13-0-2-tma +performance_claims: [] +blackwell_relevance: The official task targets B200 and exact packed-E2M1 and + block-scale layouts; it does not require a particular CUTLASS schedule or + establish that every problem shape is compute-bound. --- -# NVFP4 GEMM -- 4-bit Floating Point Matrix Multiply +# NVFP4 GEMM — GPU Mode Problem 2 Contract -## Overview +## Verified scope -NVFP4 GEMM is a compute-bound matrix multiplication kernel operating on NVIDIA's native 4-bit floating-point format (E2M1) with block scaling on Blackwell GPUs. Unlike the memory-bound GEMV, GEMM is dominated by tensor core throughput and benefits from Blackwell's native FP4 MMA instructions via tcgen05.mma, TMA bulk loads, TMEM accumulation, and warp specialization. +The official NVIDIA rules identify NVFP4 GEMM as Kernel Challenge 2 of the Blackwell NVFP4 Hackathon. The contest ran from November 29 through December 19, 2025; Problem 2 targeted NVIDIA B200 and contributed 20% of the four-problem grand-prize score. -This kernel was Problem 2 of the GPU Mode NVFP4 Hackathon (Nov-Dec 2025), targeting B200 GPUs. Top entries achieved within 1% of cuBLAS performance using CUTLASS SM100 schedules. +The pinned public task defines observable inputs, correctness, test shapes, benchmark shapes, and ranking. It does not publish a canonical optimized kernel, require CUTLASS, or establish which implementation mechanisms any entrant used. “NVFP4 GEMM” also does not determine a bottleneck by itself: shape, layout, staging, launch geometry, and epilogue can change whether math, memory traffic, occupancy, or launch overhead controls runtime. -## NVFP4 Data Format +## Format recipe versus task ABI -``` -NVFP4 (E2M1): 4-bit floating point - Bit layout: [S][E1][E0][M0] - Representable values: 0, 0.5, 1, 1.5, 2, 3, 4, 6 (positive and negative) +The generic one-dimensional NVFP4 recipe reconstructs each value from a signed E2M1 payload, one E4M3 local scale per 16 consecutive payloads, and a per-tensor FP32 global scale. E2M1 represents zero and signed magnitudes 0.5, 1, 1.5, 2, 3, 4, and 6, with two payloads packed per byte. MXFP4 is different: its local groups contain 32 payloads and use power-of-two UE8M0 scales. -Block scaling: - Every 16 FP4 elements share one FP8 (E4M3) scale factor - Two-level: per-block E4M3 + per-tensor FP32 global scale +The contest ABI is narrower and does not expose the generic recipe's FP32 global scales. Its actual generated input has seven tensors: - Dequantization: x_hat = s_global * s_block * deq_FP4(q) +| Tensor | Published representation | Physical shape | +| --- | --- | --- | +| `a` | packed E2M1, two values per byte | `[M,K/2,L]` | +| `b` | packed E2M1, two values per byte | `[N,K/2,L]` | +| `sfa` | logical E4M3 scales | `[M,K/16,L]` | +| `sfb` | logical E4M3 scales | `[N,K/16,L]` | +| `sfa_reordered` | MMA-oriented scale copy | `[32,4,ceil(M/128),4,K/64,L]` | +| `sfb_reordered` | MMA-oriented scale copy | `[32,4,ceil(N/128),4,K/64,L]` | +| `c` | preallocated FP16 output | `[M,N,L]` | -Key difference from MXFP4: - - E4M3 block scale (non-power-of-two) vs UE8M0 (power-of-two only) - - Block size 16 (tighter) vs 32 (coarser) -``` +For each `L` slice, the correctness reference computes the block-scaled equivalent of `A @ B.T` and stores FP16 output. It uses `rtol=1e-3` and `atol=1e-3`. -## CUTLASS SM100 Schedule - -The kernel uses the `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100` CUTLASS schedule, which combines TMA async loads with warp-specialized MMA execution. - -```cpp -// CUTLASS dispatch for NVFP4 GEMM on Blackwell -using Schedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100; - -// Tile configuration -using TileShape = cute::Shape<_128, _256, _128>; // M, N, K tile -using ClusterShape = cute::Shape<_1, _1, _1>; // 1-SM mode - -// Element types -using ElementA = cutlass::float_e2m1_t; // NVFP4 -using ElementB = cutlass::float_e2m1_t; // NVFP4 -using ElementC = float; // FP32 accumulator -using ElementScale = cutlass::float_e4m3_t; // FP8 E4M3 block scales - -// Kernel definition -using Kernel = cutlass::gemm::device::GemmUniversal< - ElementA, cutlass::layout::RowMajor, - ElementB, cutlass::layout::ColumnMajor, - ElementC, cutlass::layout::RowMajor, - float, // accumulator type - cutlass::arch::OpClassTensorOp, - cutlass::arch::Sm100, - TileShape, ClusterShape, Schedule ->; -``` +The upstream files contain two interface inconsistencies that implementations must not conceal: + +- `task.yml` describes a five-member `(a,b,sfa,sfb,c)` tuple, while `task.py`, `template.py`, and `reference.py` expose the seven tensors above. +- Task and template prose label scale tensors E4M3FNUZ, while `reference.py` constructs `torch.float8_e4m3fn` values. A submission must follow the tensors supplied by the actual harness rather than treating those suffixes as interchangeable. + +## Host-checkable shape contract + +The published task requires `K` divisible by 256. Divisibility of `M` and `N` depends on the submission's selected MMA tile. The following CPU-only helper reproduces the published storage shapes; it does not decode FP4, apply scales, or model a GPU kernel: + +```python +def task_storage_shapes(m, n, k, l=1): + if min(m, n, k, l) <= 0: + raise ValueError("dimensions must be positive") + if k % 256: + raise ValueError("K must be divisible by 256") -## Warp-Specialized Kernel Structure - -```cpp -// Warp specialization: TMA producer + MMA consumer + epilogue -// Shared memory holds pipelined A/B tiles + scale factors - -constexpr int NUM_STAGES = 4; // Pipeline depth - -// Shared memory layout -struct SharedStorage { - // Double-buffered across NUM_STAGES - nvfp4_t A_smem[NUM_STAGES][BLOCK_M * BLOCK_K / 2]; // Packed FP4: 2 per byte - nvfp4_t B_smem[NUM_STAGES][BLOCK_N * BLOCK_K / 2]; - fp8_t sfa_smem[NUM_STAGES][BLOCK_M * (BLOCK_K / 16)]; // 1 scale per 16 elements - fp8_t sfb_smem[NUM_STAGES][BLOCK_N * (BLOCK_K / 16)]; - uint64_t mbarrier[NUM_STAGES]; -}; - -__global__ void nvfp4_gemm_kernel( - const nvfp4_t* A, const nvfp4_t* B, - const fp8_t* sfa, const fp8_t* sfb, - float sf_a_global, float sf_b_global, - float* C, int M, int N, int K -) { - extern __shared__ SharedStorage smem[]; - int warp_id = threadIdx.x / 32; - int lane_id = threadIdx.x % 32; - - if (warp_id == 0 && lane_id == 0) { - // TMA producer warp: async bulk loads - for (int k = 0; k < K; k += BLOCK_K) { - int stage = (k / BLOCK_K) % NUM_STAGES; - - // TMA descriptor-based bulk load (128-byte aligned) - asm volatile( - "cp.async.bulk.tensor.2d.shared::cluster.global.tile" - ".mbarrier::complete_tx::bytes" - " [%0], [%1, {%2, %3}], [%4];" - :: "r"((uint32_t)&smem->A_smem[stage]), - "l"(tma_desc_A), - "r"(tile_m), "r"(k), - "r"((uint32_t)&smem->mbarrier[stage]) - ); - // Similarly for B, sfa, sfb - } - } else if (warp_id == 1 && lane_id == 0) { - // MMA consumer warp: tcgen05.mma - uint32_t tmem_addr = tmem_alloc_cta(256); // 256 TMEM columns - - for (int k = 0; k < K; k += BLOCK_K) { - int stage = (k / BLOCK_K) % NUM_STAGES; - // Wait for TMA to complete this stage - mbarrier_wait(&smem->mbarrier[stage]); - - // tcgen05.mma with native block scaling - // Reads A/B from SMEM, accumulates into TMEM - asm volatile( - "tcgen05.mma.cta_group::1.kind::f8f6f4" - " [%0], %1, %2, %3, %4;" - :: "r"(tmem_addr), - "l"((uint64_t)&smem->A_smem[stage]), - "l"((uint64_t)&smem->B_smem[stage]), - "r"(packed_scales), - "n"(1) // scale_D enabled - ); - } - - // Signal epilogue warps - } else { - // Epilogue warps: read from TMEM, apply global scales, store to C - // tmem -> registers -> global + return { + "a_packed": (m, k // 2, l), + "b_packed": (n, k // 2, l), + "sfa_logical": (m, k // 16, l), + "sfb_logical": (n, k // 16, l), + "sfa_reordered": (32, 4, (m + 127) // 128, 4, (k + 63) // 64, l), + "sfb_reordered": (32, 4, (n + 127) // 128, 4, (k + 63) // 64, l), + "c": (m, n, l), } -} + + +for valid_k in (256, 512, 1536, 2048, 2304, 7168, 16384): + task_storage_shapes(128, 256, valid_k) + +assert (256 // 16) % 128 != 0 # the former scale-array assertion was invalid ``` -## 128-Byte TMA Alignment +Nine of the ten official correctness shapes fail the former `(K/16) % 128 == 0` assertion, including the smallest valid case with `K=256`. -All TMA operands require 128-byte alignment. For NVFP4 (2 elements per byte), this means K dimensions must be multiples of 256 elements: +## Blackwell implementation boundary -```cpp -// Critical: pad tensors to 128-byte boundaries for TMA -// NVFP4: 2 elements per byte, so 256 elements = 128 bytes -static_assert(K % 256 == 0, "K must align to 128 bytes for FP4 TMA"); +The native Blackwell instruction path supports block-of-16 NVFP4 through `tcgen05.mma...kind::mxf4nvf4.block_scale.block16`. It uses UE4M3 scale elements, which CUTLASS names `float_ue4m3_t`; UE8M0 is the MXFP4 scale type. Converting NVFP4 scales to UE8M0 would discard fractional scale values and is not a required NVFP4 preprocessing step. -// For scale factors: FP8 is 1 byte per element -// 128 bytes = 128 scale values -// Since 1 scale per 16 FP4 elements: 128 scales cover 2048 FP4 elements -static_assert((K / 16) % 128 == 0, "Scale array must align to 128 bytes"); -``` +CUTLASS defines `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100`, but its direct official example is a grouped pointer-array kernel. The symbol's existence is not evidence that the contest entrants used it. CUTLASS's current official NVFP4 examples use `nv_float4_t`, collective mainloop and epilogue builders, `kernel::GemmUniversal`, and `device::GemmUniversalAdapter`; an old scalar-template `device::GemmUniversal` sketch is not interchangeable with that API. -## Scale Factor Conversion - -The tcgen05.mma instruction expects UE8M0 (unsigned power-of-two exponent only) scales, but NVFP4 uses FP8 E4M3 (non-power-of-two). Conversion is needed: - -```cpp -// Convert FP8 E4M3 block scales to UE8M0 for tcgen05.mma hardware -// E4M3: 4 exponent bits, 3 mantissa bits (non-power-of-two) -// UE8M0: 8 exponent bits, 0 mantissa bits (power-of-two only) -__device__ uint32_t pack_scales_ue8m0( - fp8_e4m3_t s0, fp8_e4m3_t s1, fp8_e4m3_t s2, fp8_e4m3_t s3 -) { - uint8_t u0 = fp8_e4m3_to_ue8m0(s0); // Round to nearest power-of-two - uint8_t u1 = fp8_e4m3_to_ue8m0(s1); - uint8_t u2 = fp8_e4m3_to_ue8m0(s2); - uint8_t u3 = fp8_e4m3_to_ue8m0(s3); - return (u3 << 24) | (u2 << 16) | (u1 << 8) | u0; -} -``` +TMA likewise has no universal 128-byte alignment rule for every operand. CUDA 13.2.1 generally requires a 16-byte-aligned global base and 16-byte-multiple strides for a tiled tensor map, with additional datatype-, interleave-, swizzle-, and mode-specific restrictions. Problem 2's `K % 256 == 0` rule belongs to the task ABI; it does not follow as a universal TMA theorem. + +TMEM's 128-lane by 512-column organization is a storage and addressing model, not a universal 128-by-512 logical output-tile limit. Official CUTLASS NVFP4 configurations include a cooperative two-SM MMA tile with logical shape 256 by 256 by 256. + +## Published performance records -## Competition Results +The task ranks the geometric mean across three benchmark cases. It labels the following values a speed-of-light analysis based on the maximum of B200 FP4 Tensor Core math time and DRAM-memory time at a 1.5 GHz clock: -Problem 2 top performers (geometric mean across benchmark configs): +| M | N | K | L | Theoretical time (µs) | +| ---: | ---: | ---: | ---: | ---: | +| 128 | 7168 | 16384 | 1 | 8.994 | +| 128 | 4096 | 7168 | 1 | 2.354 | +| 128 | 7168 | 2048 | 1 | 1.333 | -| Rank | Participant | Latency (us) | -|------|-------------|--------------| -| 1 | Simon | 10.807 | -| 2 | yue | 10.914 | -| 3 | currybab | 10.931 | +These are theoretical comparison rows, not measured contestant latencies and not cuBLAS results. -## When to Use +The public Popcorn endpoint currently gives the following dated snapshot. To make its small floating-point `submission_score` values readable beside the task's microsecond presentation, the table shows `submission_score × 10^6`; it does not relabel the rows as prize placements. -- Inference with 4-bit quantized weights on Blackwell -- MLP layers in LLMs where weight matrices are NVFP4-quantized -- Compute-bound matrix multiplications where tensor core utilization is the bottleneck +| Current rank | User | API score × 10^6 | Submission timestamp (UTC) | +| ---: | --- | ---: | --- | +| 1 | `gau.nernst` | 9.981889 | 2025-12-21 00:43:03 | +| 2 | `s.am._` | 10.060110 | 2025-12-20 17:45:21 | +| 3 | `billcarson` | 10.137411 | 2025-12-21 03:05:32 | +| 8 | `Simon` | 10.806750 | 2025-12-16 20:18:42 | +| 9 | `yue` | 10.914084 | 2025-12-11 04:36:45 | +| 10 | `currybab` | 10.930623 | 2025-12-19 08:10:18 | -## Caveats +Snapshot fetched August 8, 2026. Because the current first three submissions postdate the official December 19 cutoff, this endpoint alone cannot establish winners or final prize rankings. It also publishes no contestant source, CUTLASS attribution, cuBLAS comparison, raw trials, or variance. -- SM100/SM100a only -- no Hopper support for native FP4 tensor core instructions -- Scale factor conversion (E4M3 to UE8M0) adds overhead if not precomputed -- TMA requires 128-byte alignment for all operands -- TMEM size (128x512 per SM) limits maximum output tile to 128 rows x 512 cols (32-bit) +## Applicability -## Sources +Use this task contract only when the packed E2M1 payloads, logical and reordered scales, transpose convention, FP16 output, tolerances, and B200 target match the workload. “Four-bit weights” alone is insufficient: INT4, MXFP4, other scale granularities, and other layouts are not interchangeable with this ABI. -- [GPU Mode NVFP4 Hackathon](https://github.com/gpu-mode/reference-kernels) -- [NVIDIA NVFP4 Blog](https://developer.nvidia.com/blog/introducing-nvfp4-for-efficient-and-accurate-low-precision-inference/) -- [NVFP4 Format Details](https://haroldbenoit.com/notes/ml/engineering/precision/nvfp4-format) -- [CUTLASS SM100 documentation](https://docs.nvidia.com/cutlass/latest/CHANGELOG.html) +The documented native tensor-core path is Blackwell-specific; Hopper has no native FP4 tensor-core instruction. This does not exclude software conversion or emulation. Choose TMA, TMEM allocation, CUTLASS schedules, warp specialization, tile sizes, and stage counts only after validating their exact API constraints and profiling the actual shapes. -## Full Reference Implementation +## Primary sources -The reference bundle lives in [`artifacts/kernels/nvfp4-gemm/full/`](../../artifacts/kernels/nvfp4-gemm/full/) and combines the upstream PR-2139 diff (`PR-2139-blockwise-groupwise-gemm.patch`, `mode: upstream-patch`, SHA-pinned to `ca4fdbea` on NVIDIA/cutlass) with an extracted CUTLASS-schedule / TMA snippet from the `tflops-gap-fp4-moe` blog (`blackwell-cutlass-schedules-and-tma.cu`, `mode: extracted`). Labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` header) live in [`artifacts/kernels/nvfp4-gemm/variants/`](../../artifacts/kernels/nvfp4-gemm/variants/). Every file's SHA-256 and upstream-pinning metadata is in `PROVENANCE.yaml` inside each bundle. +- [Official contest rules](https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf) +- [Pinned task definition](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml) +- [Pinned task types](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.py) +- [Pinned starter template](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py) +- [Pinned correctness reference](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/reference.py) +- [Public Popcorn leaderboard API](https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12) +- [Transformer Engine 2.13 NVFP4 recipe](https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html) +- [cuBLAS block-scaling formats](https://docs.nvidia.com/cuda/cublas/index.html#element-1d-block-scaling-for-fp8-and-fp4-data-types) +- [PTX ISA 9.0 block-scaled `tcgen05.mma`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma) +- [Pinned CUTLASS NVFP4 example](https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu) +- [Pinned CUTLASS grouped NVFP4 example](https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu) +- [CUDA 13.2.1 tensor-map constraints](https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html) Query via: ```bash -python3 scripts/get_page.py kernel-nvfp4-gemm --include-code +conda run -n base python scripts/get_page.py kernel-nvfp4-gemm ``` diff --git a/wiki/kernels/nvfp4-gemv.md b/wiki/kernels/nvfp4-gemv.md index e274b11be..e47137c4c 100644 --- a/wiki/kernels/nvfp4-gemv.md +++ b/wiki/kernels/nvfp4-gemv.md @@ -14,13 +14,14 @@ tags: - register-budgeting - vectorized-loads confidence: source-reported -reproducibility: snippet +reproducibility: concept kernel_types: - gemv - batched-gemv languages: - cuda-cpp - ptx +- cute-dsl related: - hw-nvfp4 - kernel-nvfp4-gemm @@ -29,241 +30,94 @@ sources: - contest-gpumode-p1 - blog-yue-nvfp4 - blog-amandeep-nvfp4 -performance_claims: -- gpu: B200 - dtype: nvfp4 - shape: M=7168, K=16384, L=1 - metric: latency_us - value: 22.4 - utilization: ~2.6x of SOL (8.6us) - source_id: contest-gpumode-p1 -artifact_dir: artifacts/kernels/nvfp4-gemv +performance_claims: [] --- # NVFP4 Batched GEMV -## Overview +## Verified Scope -NVFP4 Batched GEMV is a memory-bound kernel computing batched matrix-vector products with NVFP4 (E2M1) block-scaled inputs on B200 GPUs. Unlike compute-bound GEMM, GEMV is dominated by memory bandwidth utilization (each FP4 element is used only once in the dot product), making PTX-level memory access control, cache policy differentiation, and register budgeting the critical optimization levers. +GPU Mode's Problem 1 asks for a block-scaled NVFP4 batched matrix-vector product on NVIDIA B200. The official task models its three benchmark cases against the slower of FFMA math and DRAM transfer time and reports DRAM-limited theoretical times. That task-specific model does not make every GEMV implementation or shape bandwidth-bound. -This was Problem 1 of the GPU Mode NVFP4 Hackathon (Nov 2025). The theoretical speed-of-light is ~8.6us for the largest config, limited by B200's 8 TB/s HBM3e bandwidth. Top performers achieved ~22.4us (2.6x off SOL), reflecting the overhead of FP4 decoding and scale factor application. +The logical operation has one B row, reused by all M output rows. The reference harness pads B and its scales to 128 rows so it can call `torch._scaled_mm`, then retains only result column zero. Consequently, A values are row-specific while the logical B vector is reused across M; it is incorrect to say every FP4 input value is consumed only once. -## Problem Specification +## Official Callable Contract -``` -Inputs: - a: (M x K x L) NVFP4 packed matrix - b: (1 x K x L) NVFP4 packed vector - sfa: (M x K/16 x L) FP8 E4M3 scale factors for a - sfb: (1 x K/16 x L) FP8 E4M3 scale factors for b - sf_a_global, sf_b_global: FP32 per-tensor global scales - -Output: - c: (M x 1 x L) FP16 - -Computation per element: - c[m][l] = sum_k( sf_a_global * sfa[m][k/16][l] * deq(a[m][k][l]) - * sf_b_global * sfb[0][k/16][l] * deq(b[0][k][l]) ) - -Benchmark configs: - Config 1: M=7168, K=16384, L=1 - Config 2: M=4096, K=7168, L=8 - Config 3: M=7168, K=2048, L=4 -``` - -## Key Optimization: PTX-Level Control - -Raw PTX provides critical performance gains over C intrinsics for FP4 decoding and memory access: - -```asm -; FP4 to FP16 conversion: hardware instruction -; Converts two packed E2M1 values to a pair of FP16 values -cvt.rn.f16x2.e2m1x2 %result, %fp4_packed; - -; Byte unpacking: PTX mov.b32 is faster than manual bitwise ops -; Instead of: val = (packed >> (i*4)) & 0xF (multiple shifts + masks) -; Use: direct byte decomposition -mov.b32 {tmp0, tmp1, tmp2, tmp3}, %packed_word; - -; This eliminates the shift-mask chain entirely -; Key insight: PTX byte unpacking leverages hardware byte decomposition -``` - -## Cache Policy Differentiation - -Different data access patterns require different cache strategies: - -```asm -; Matrix A: streamed once per row, never reused across thread blocks -; Bypass L1 to avoid polluting cache with one-shot data -ld.global.L1::no_allocate.v4.u64 {a0,a1,a2,a3}, [addr_a]; - -; Vector B: reused across all M rows in a thread block -; Keep hot in L1 for fast reuse -ld.global.L1::evict_last.v4.u64 {b0,b1,b2,b3}, [addr_b]; - -; Rank 1 solution: used different qualifiers depending on K-dimension variant -; K=16384 (large): aggressive L1 bypass for A -; K=2048 (small): different balance since B is smaller relative to cache -``` - -## Register Budgeting - -Lower register counts force higher occupancy, which is critical for memory-bound kernels where latency hiding dominates: - -```cpp -// Rank 1: aggressive register limit for maximum occupancy -// nvcc -maxrregcount=32 -// Fewer registers -> more warps/SM -> better memory latency hiding +At pinned commit `ae679486`, `custom_kernel` receives seven tensors, not five tensors plus global FP32 scale arguments: -// Rank 3: slightly relaxed for more ILP -// nvcc -maxrregcount=45 +| Tensor | Physical shape | Role | +|---|---|---| +| `a` | `[M, K/2, L]` | Packed E2M1 A; two logical values per byte | +| `b` | `[128, K/2, L]` | Packed E2M1 B, physically padded; logical row 0 is used | +| `sfa` | `[M, K/16, L]` | Logical/reference A block scales | +| `sfb` | `[128, K/16, L]` | Logical/reference B block scales, padded to 128 rows | +| `sfa_reordered` | `[32, 4, ceil(M/128), 4, K/64, L]` | Swizzled A scales for custom kernels | +| `sfb_reordered` | `[32, 4, 1, 4, K/64, L]` | Swizzled padded-B scales for custom kernels | +| `c` | `[M, 1, L]` | FP16 output buffer | -// Launch bounds to complement register budgeting -__launch_bounds__(256, 4) // 256 threads/block, 4+ blocks per SM -__global__ void nvfp4_gemv_kernel(...) { - // ... -} +`K` must be divisible by 64, and M must be divisible by the implementation's selected M tile. The task prose labels scales `e4m3fnuz`, but the pinned generator constructs `torch.float8_e4m3fn`; implementations should follow the actual harness revision they are tested against. This contest ABI exposes no per-tensor FP32 global scales. -// The measurable difference between 32 and 45 registers shows -// that occupancy is the dominant factor for memory-bound kernels -``` - -## Per-K Specialization - -Compile separate kernels per K dimension, each with full loop unrolling and tuned configs: +## Official Benchmark Model -```cpp -// Each K variant compiled separately with optimal configuration -template -__global__ __launch_bounds__(THREADS, MIN_BLOCKS) -void nvfp4_gemv_specialized( - const uint8_t* __restrict__ a, - const uint8_t* __restrict__ b, - const uint8_t* __restrict__ sfa, - const uint8_t* __restrict__ sfb, - half* __restrict__ c, - float sf_a_global, float sf_b_global, - int M -) { - float acc = 0.0f; +The official table assumes a 1.5 GHz B200 clock and ranks submissions by the geometric mean across all three rows: - // Full unroll: compiler knows K_SIZE at compile time - #pragma unroll - for (int k = 0; k < K_SIZE / ELEMENTS_PER_LOAD; k++) { - // Load FP4 packed data - uint64_t a_packed = load_fp4(a, row, k); - uint64_t b_packed = load_fp4(b, 0, k); +| M | K | L | Theoretical time (µs) | +|---:|---:|---:|---:| +| 7168 | 16384 | 1 | 8.622 | +| 4096 | 7168 | 8 | 17.275 | +| 7168 | 2048 | 4 | 4.317 | - // Dequantize via PTX - half2 a_vals = cvt_e2m1x2(a_packed); - half2 b_vals = cvt_e2m1x2(b_packed); +NVIDIA's Blackwell technical brief specifies 8 TB/s of HBM3e bandwidth for one GB200 GPU. The task's theoretical numbers are model values, not measured kernel timings. - // Apply block scales - float sa = sfa[row * (K_SIZE/16) + k/2]; - float sb = sfb[k/2]; +The public leaderboard is mutable and reports aggregate scores rather than per-shape latencies. In the snapshot fetched on 2026-08-08, the first three rows were `s.am._` at 18.549562452 µs, `gau.nernst` at 18.552844757 µs, and `shellsmile15795` at 18.707609314 µs. Yue's 22.392217755 µs submission was rank 11. The official cutoff was November 28, 2025 at 11:59 p.m. PT; current ranks 2 and 3 have November 30 timestamps, so this endpoint is not an official prize-placement snapshot. - // Accumulate - acc += (float)a_vals.x * (float)b_vals.x * sa * sb; - acc += (float)a_vals.y * (float)b_vals.y * sa * sb; - } - - // Apply global scales and store - c[row] = __float2half(acc * sf_a_global * sf_b_global); -} - -// Dispatch: K=2048 uses BLOCK_M=8, maxreg=32 -// K=7168 uses BLOCK_M=4, maxreg=40 -// K=16384 uses BLOCK_M=2, maxreg=32 -``` +## PTX Semantics and Reported Techniques -## Vectorized Loads - -128-bit and 256-bit vector loads maximize bandwidth utilization: +PTX ISA 9.0 defines the packed conversion and typed register decomposition used in Yue's author-reported decode path: ```asm -; 128-bit vector load (16 bytes = 32 FP4 elements) -ld.global.v2.u64 {r0, r1}, [addr]; - -; 256-bit vector load (32 bytes = 64 FP4 elements) -ld.global.v4.u64 {r0, r1, r2, r3}, [addr]; - -; Only effective when combined with PTX byte unpacking -; to avoid bitwise overhead in the subsequent unpack stage -; Without proper unpacking, wide loads just move the bottleneck -``` - -## Data Reuse (Rank 2 Approach) - -Since B is shape (1 x K x L), all M rows multiply against the same B vector: - -```cpp -// Load B vector into shared memory once per thread block -// All BLOCK_M rows reuse the same B data -__shared__ half b_shared[K_TILE]; - -// Cooperative load: all threads in block load a portion of B -for (int i = threadIdx.x; i < K_TILE; i += blockDim.x) { - b_shared[i] = dequant_fp4(b[i], sfb[i/16]) * sf_b_global; -} -__syncthreads(); - -// Each thread computes dot product of its A row with shared B -float acc = 0.0f; -for (int k = 0; k < K_TILE; k++) { - acc += (float)a_dequant[k] * (float)b_shared[k]; -} - -// Reduces global memory traffic by BLOCK_M ratio +cvt.rn.f16x2.e2m1x2 %result, %packed_fp4_pair; +mov.b32 {%b0, %b1, %b2, %b3}, %packed_word; ``` -## Performance Progression - -Documented progression from Yue's hackathon blog: +It also defines vector loads such as `ld.global.v2.u64` and `ld.global.v4.u64`, which move 16 and 32 bytes and can carry 32 and 64 packed FP4 values. The ISA specifies behavior, not that one width or decomposition is universally faster. -| Stage | Technique | Latency | -|-------|-----------|---------| -| CuTe DSL baseline | Basic CuTe partition/copy | ~100us | -| Coalesced access | Fix memory access patterns | 443us -> 39us | -| Hardware intrinsics | cvt.rn.f16x2.e2m1x2 | ~39us | -| PTX assembly | Full PTX with byte unpacking | ~27us | -| ILP optimization | Instruction-level parallelism | ~22.9us | -| Final submission | All combined | 22.392us | +Amandeep reports that three solutions inspected after the event used `L1::no_allocate` for streamed A loads, `L1::evict_last` for reused B loads, wider PTX loads, and exact-K specializations. Those are author-reported observations; the public leaderboard API supplies no contestant code or technique field. Amandeep's own wider `uint2` experiment was 16–25% slower, and reducing `-maxrregcount` from 80 to 64 had no effect because the kernel already used fewer than 64 registers. Load width, cache hints, and register caps therefore require measurement in the exact implementation. -## Key Lessons +## Yue's Author-Reported Progression -1. **Memory-bound kernels need bandwidth-first thinking**: Arithmetic optimizations have minimal impact; focus on memory access patterns, cache policies, and vectorized loads -2. **PTX gives real control on Blackwell**: The gap between C intrinsics and hand-written PTX was substantial (443us to 27us in one journey) -3. **Profile first**: "Run Nsight Compute to confirm memory-bound behavior" (Amandeep's lesson after 12 attempts) -4. **Register budgeting matters**: Lower registers -> higher occupancy -> better memory latency hiding -5. **TMEM is irrelevant**: Memory-bound kernels do not benefit from TMEM (it helps compute-bound only) +The following timings are from Yue's post, not an independent reproduction. Several stages combine multiple changes and cannot establish isolated causality: -## When to Use +| Stage | Combined change | Reported latency | +|---|---|---:| +| Initial CuTe DSL | First working CuTe implementation | ~100 µs | +| Optimized CuTe DSL | Scale-load and arithmetic changes plus thread collaboration | ~33 µs | +| Initial CUDA | Naive hand-written path | ~2000 µs | +| CUDA step 1 | Coalescing, shared B, thread collaboration, warp reduction | ~443 µs | +| CUDA step 2 | Remove shared B, per-thread tiles, `float4` loads, hardware intrinsics | ~39 µs | +| CUDA step 3 | Vectorized PTX FP4/scale decode | ~27 µs | +| Parameter tuning | Threads per row and rows per block | ~26 µs | +| Two-tile ILP | Interleave two tiles per loop | ~22.9 µs | +| Aggressive PTX fusion | Fuse decode, scale, multiply, and accumulation | ~22.3 µs | +| Submitted score | Geometric mean shown by the leaderboard | 22.392 µs | -- Decode-time MLP with batch size 1 (matrix-vector product) -- Any NVFP4 workload where arithmetic intensity is too low for tensor cores -- Memory-bound operations with FP4 quantized weights +Yue also reports that loading the entire B vector into shared memory did not improve the CuTe attempt. This is a useful counterexample to treating shared-memory B staging or an exact `BLOCK_M` traffic reduction as automatic. -## Caveats +## Practical Use and Caveats -- SM100/SM100a only (native FP4 decode instructions) -- PTX-level optimizations are fragile across CUDA toolkit versions -- Per-K specialization increases binary size (one kernel per K variant) -- Speed-of-light is bounded by B200 memory bandwidth (8 TB/s) +- Use this case study for an N=1 decode-style matrix-vector operation only when payloads, scales, layouts, output type, and target match the chosen NVFP4 implementation contract. +- Profile bytes moved, instruction count, decode and reduction work, cache behavior, spills, and occupancy before deciding which resource is limiting performance. +- Exact-K dispatch can enable full unrolling and size-specific tuning, but retaining multiple specializations adds compiled kernel entries. +- Inline PTX must be revalidated against the selected PTX ISA, toolkit, and target. The contest ran on B200; that does not imply packed FP4 conversion is restricted to SM100/SM100a forever. Transformer Engine 2.13 lists NVFP4 inference support for SM 10.0 and later. +- The official task accepts any implementation that satisfies its checker. It does not require the cache policies, load widths, register limits, or code structures described in participant posts. -## Sources +## Primary Sources -- [GPU Mode NVFP4 Hackathon](https://github.com/gpu-mode/reference-kernels) -- [Yue's Hackathon Journey](https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html) -- [Twelve Attempts (Amandeep)](https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/) -- [Simon's NVFP4 GEMV Blog](https://veitner.bearblog.dev/nvfp4-gemv/) - -## Full Reference Implementation - -Local verbatim upstream code lives in [`artifacts/kernels/nvfp4-gemv/full/`](../../artifacts/kernels/nvfp4-gemv/full/) (see its `PROVENANCE.yaml` for the pinned upstream SHA and byte-verified SHA-256). Labeled derived variants — including a naive/teaching skeleton — live in [`artifacts/kernels/nvfp4-gemv/variants/`](../../artifacts/kernels/nvfp4-gemv/variants/). - -Query via: - -```bash -python3 scripts/get_page.py kernel-nvfp4-gemv --include-code -``` +- [Pinned Problem 1 task](https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv) +- [Official contest rules](https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf) +- [NVIDIA Blackwell Architecture Technical Brief](https://resources.nvidia.com/en-us-blackwell-architecture/blackwell-architecture-technical-brief) +- [PTX ISA 9.0](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html) +- [Transformer Engine 2.13 NVFP4](https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html) +- [Yue Zhang's hackathon journey](https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html) +- [Amandeep Singh's twelve attempts](https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/) +- [Simon Veitner's reference](https://veitner.bearblog.dev/nvfp4-gemv/) and [improved variants](https://veitner.bearblog.dev/nvfp4-gemv-improved/) diff --git a/wiki/kernels/sparse-mla.md b/wiki/kernels/sparse-mla.md index e1e265357..96cf3b659 100644 --- a/wiki/kernels/sparse-mla.md +++ b/wiki/kernels/sparse-mla.md @@ -1,118 +1,141 @@ --- id: kernel-sparse-mla -title: "Sparse MLA (DeepSeek V3.2)" +title: "DeepSeek Sparse Attention / Sparse MLA" type: kernel architectures: [sm100, sm90] tags: [sparse-attention, mla, fp8, attention, decode, prefill] confidence: source-reported -reproducibility: snippet +reproducibility: concept kernel_types: [sparse-attention, mla, attention, decode, prefill] languages: [cuda-cpp, cute-dsl] related: [kernel-flashmla, kernel-nsa, hw-tcgen05-mma] sources: [blog-flashmla, blog-vllm-deepseek-v3-sparse, blog-nsa] -performance_claims: - - gpu: B200 - dtype: fp8 - shape: "sparse prefill, seqlen=32k, topk=2048" - metric: TFLOPS - value: 1450 - utilization: "FP8 sparse compute bound" - source_id: blog-flashmla -blackwell_relevance: "SM100 tcgen05.mma with FP8 block-scale MMA enables DeepSeek V3.2's Lightning Indexer + sparse attention two-stage pipeline." +performance_claims: [] +blackwell_relevance: "On SM100, DeepGEMM supplies FP8 indexer-logit kernels while FlashMLA supplies separate sparse-attention kernels; V3-family sparse decode stores FP8 KV but performs BF16 MMA, and sparse prefill accepts BF16 Q/KV." --- -# Sparse MLA (DeepSeek V3.2 Sparse Attention) - -## Overview - -Sparse Multi-head Latent Attention introduced in DeepSeek V3.2. Two-stage pipeline: (1) Lightning Indexer selects top-K tokens per query via FP8 scorer, (2) MLA runs only over selected tokens. This reduces decode attention compute from O(seqlen) to O(topk=2048) tokens, critical for long-context serving. - -## Architecture - -``` -Query q_t (new token embedding) - │ - ▼ -┌────────────────────────┐ -│ Lightning Indexer │ FP8 scorer, per-query top-K selection -│ - FP8 KV cache │ h64, d128, topk=2048, page_size=64 -│ - Compute q·k_i scores │ -│ - Select top-2048 i │ -└────────────────────────┘ - │ selected_indices [2048] - ▼ -┌────────────────────────┐ -│ Sparse MLA │ Standard MLA but only over selected -│ - Gather selected K,V │ h16, ckv512, kpe64, topk=2048 -│ - Attention compute │ -│ - Output y_t │ -└────────────────────────┘ -``` - -## Token Layout - -Each KV cache entry is 656 bytes: -- 512 bytes: FP8 compressed KV data -- 16 bytes: FP32 per-block scale factors -- 128 bytes: BF16 RoPE embeddings (for indexer positional encoding) - -Block size fixed at 64 (FlashMLA requirement). - -## Kernel Patterns - -### Lightning Indexer (FP8 Score Compute) -```cuda -// Score each KV block against query using FP8 block-scale MMA -// Reduce to per-block max, then top-K selection across blocks - -__global__ void lightning_indexer_kernel( - const fp8_t* q_fp8, // [h, d] query (FP8 quantized) - const fp8_t* kv_cache_fp8, // [num_blocks, 64, d] paged - const fp8_t* kv_scales, - float* scores_out, // [num_blocks] per-block max score - int num_blocks -) { - // Each threadblock handles one KV block's score - uint32_t tmem = tmem_alloc(64); - tcgen05_mma_f8(q_smem, k_block_smem, tmem); - // Reduce inside block to max score - // Write per-block score -} - -// Separate top-K selection kernel across the num_blocks score array -``` - -### Sparse Attention Gather -```cuda -// After top-K selection gives indices, gather K,V from paged cache -// Then run standard MLA on the gathered subset - -__global__ void sparse_mla_decode_kernel( - const int* topk_indices, // [2048] selected block indices - const fp8_t* kv_cache, // paged KV (656 bytes/token) - const half* q, // [16, 576] MLA query - half* output -) { - // Load q into registers/SMEM - // For each selected block: - // gather K,V block (FP8) → dequant to BF16 → MMA - // Online softmax accumulation - // Final weighted sum -} -``` - -## Performance - -| Variant | GPU | TFLOPS | Notes | -|---------|-----|--------|-------| -| Dense MLA decode | H800 | 660 (BF16) | 3000 GB/s, compute-bound | -| Sparse MLA decode | H800 | 410 (FP8) | Token-level sparsity | -| Sparse MLA decode | B200 | 350 (FP8) | Lower because bandwidth dominates decode | -| Dense prefill | B200 | 1460 (BF16) | tcgen05 peak | -| Sparse prefill | B200 | 1450 (FP8) | FP8 sparse matches BF16 dense | - -## When To Use - -- Long-context LLM serving (32K+) -- DeepSeek V3.2 and similar MLA architectures -- Serving workloads where per-token decode latency matters +# DeepSeek Sparse Attention / Sparse MLA + +## Model mechanism + +DeepSeek-V3.2-Exp introduces **DeepSeek Sparse Attention (DSA)**. Its +first-party report defines two components: + +1. The lightning indexer computes one score for every query/preceding-token + pair. A score is a weighted sum across indexer heads of a ReLU-applied query + and key dot product. +2. Fine-grained token selection retains the KV entries at the top-k index + scores and applies the main attention only to that selected set. + +The released `config_671B_v3.2.json` uses 64 indexer heads of dimension 128 and +`index_topk=2048`. DSA is instantiated under MLA's MQA mode: the selected MLA +latent entry is shared across the query heads. The model has 128 MLA query +heads, a 512-dimensional latent KV component, and a 64-dimensional RoPE +component; a smaller local head count is a parallel-sharding choice rather than +the architecture-wide contract. + +For a length-L sequence and fixed selected count k, the report reduces the +**main/core attention** complexity from O(L squared) to O(Lk). The lightning +indexer still scores the preceding context and remains O(L squared) over the +sequence. For decode, measure both the O(L) selector scan for each new query and +the O(k) selected-attention work; do not describe the full pipeline as O(k). + +## Separate implementation boundaries + +The released high-performance path is not one fused “indexer plus sparse MLA” +kernel: + +- DeepGEMM provides non-paged and paged indexer-logit kernels. Its pinned SM100 + FP8 path computes token logits from FP8 query/key inputs, per-token key + scales, and per-query head weights; top-k selection remains a separate + operation. +- FlashMLA sparse attention consumes caller-produced token indices. It does not + run the lightning indexer or top-k selector. + +At FlashMLA commit +[`71c7379`](https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232), +prefill and decode have different contracts: + +| Stage | Inputs and selected-token encoding | Precision and outputs | +| --- | --- | --- | +| Sparse decode | `q[batch,s_q,h_q,576]`; paged `k_cache`; `indices[batch,s_q,topk]`, where each nonnegative value encodes physical page times page size plus token offset; `-1` is invalid | V3-family sparse mode dequantizes its FP8 cache for BF16 attention and returns BF16 `out` plus FP32 `lse` | +| Sparse prefill | Unbatched BF16 `q[s_q,h_q,d_qk]`, BF16 `kv[s_kv,h_kv,d_qk]`, and INT32 `indices[s_q,h_kv,topk]`; the documented equivalence requires `h_kv=1`; negative or at-least-`s_kv` entries are invalid | Computes attention over gathered token rows and returns BF16 `out`, FP32 `max_logits`, and FP32 `lse` | + +These are token indices, not one maximum or one selection per cache block. +The sparse-decode API accepts page size through the cache tensor; pinned +correctness tests exercise multiple values including 2, 53, 61, 64, 69, 256, +and 576. Page size 64 is therefore a deployment configuration, not a universal +FlashMLA requirement. + +## V3-family sparse-decode cache + +Only the V3/V3.1/V3.2 **FP8 sparse-decode** mode has the documented 656-byte +per-token layout: + +- 512 E4M3 NoPE values: 512 bytes; +- four FP32 scales, one for each successive group of 128 NoPE values: 16 bytes; +- 64 BF16 RoPE values used by the attention key: 128 bytes. + +The indexer has a separate FP8 K cache and scale cache in the released model. +The 656-byte attention entry is not an indexer cache and is not the layout of +every dense, sparse-prefill, or non-V3 FlashMLA mode. + +## Source-reported performance boundary + +FlashMLA's pinned README reports the following maxima. They are useful source +claims, not reproducible benchmark tuples: the README does not provide complete +shapes, timed regions, repetitions, samples, or variance. + +| Operator and regime | Reported environment | Author-reported maximum and precision scope | +| --- | --- | --- | +| Dense MLA decode, memory-bound configuration | H800 SXM5, CUDA 12.8 | Up to 3000 GB/s with BF16 cache | +| Dense MLA decode, compute-bound configuration | H800 SXM5, CUDA 12.8 | Up to 660 TFLOPS with BF16 cache; separate from the 3000-GB/s case | +| Sparse MLA decode | H800 SXM5, CUDA 12.8 | 410 TFLOPS; FP8 KV storage and BF16 matrix multiplication | +| Sparse MLA decode | B200 | Up to 350 TFLOPS; the source says this path was not really optimized and gives no bandwidth-causality result | +| Sparse MLA prefill | H800 SXM5, CUDA 12.8 | Up to 640 TFLOPS forward with BF16 Q/KV | +| Sparse MLA prefill | B200, CUDA 12.9 | Up to 1450 TFLOPS forward with BF16 Q/KV | + +The same README separately reports NVIDIA's dense **MHA** prefill maxima of +1460 TFLOPS forward and 1000 TFLOPS backward on B200. Those values are not a +dense-MLA baseline matched to the sparse-prefill result, so numerical proximity +between 1450 and 1460 does not establish equivalent performance. + +## Evaluation procedure + +Use this path directly for DeepSeek-V3.2-Exp, or for another model only after +its query dimensions, latent/RoPE layout, cache format, head relationships, +index encoding, invalid-entry rules, and output semantics match the chosen +FlashMLA interface. + +For a target deployment: + +1. Compare index scores and selected token sets against the released model + equation and a dense reference, including causal masking and the indexer's + non-interleaved RoPE layout. +2. Compare sparse prefill/decode outputs and LSE values with the pinned + FlashMLA reference for identical selected indices, including invalid and + partially filled top-k cases. +3. Time indexer logits, top-k selection, sparse attention, and the complete + pipeline separately across context lengths, query batch sizes, page sizes, + and selected counts. Use a matched dense-attention baseline and report + synchronization, warmup, repeated trials, statistic, and variation. + +There is no verified universal 32K crossover. The relevant threshold is where +the measured selector-plus-sparse-attention pipeline improves the target's +end-to-end latency or cost without violating its accuracy criterion. + +## DSA is not Native Sparse Attention + +DeepSeek-V3.2-Exp DSA selects individual token positions with a learned +indexer. The ACL Native Sparse Attention architecture instead combines +compressed-block, selected-block, and sliding-window branches with learned +gates. Do not transfer NSA's A100 results or three-branch structure to this +FlashMLA DSA path. + +## Primary references + +- [DeepSeek-V3.2-Exp report and released model](https://github.com/deepseek-ai/DeepSeek-V3.2-Exp/tree/87e509a2e5a100d221c97df52c6e8be7835f0057) +- [FlashMLA at audited commit `71c7379`](https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232) +- [FlashMLA sparse interface at `71c7379`](https://github.com/deepseek-ai/FlashMLA/blob/71c737929f2567bd0a094ae140f8f60f390b1232/flash_mla/flash_mla_interface.py) +- [DeepGEMM indexer-logit implementation at `891d57b4`](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_mqa_logits.cuh) +- [ACL 2025 Native Sparse Attention paper](https://aclanthology.org/2025.acl-long.1126/) diff --git a/wiki/kernels/tensorrt-llm-blackwell-indexer.md b/wiki/kernels/tensorrt-llm-blackwell-indexer.md index 99fa0f3e1..32974cbe1 100644 --- a/wiki/kernels/tensorrt-llm-blackwell-indexer.md +++ b/wiki/kernels/tensorrt-llm-blackwell-indexer.md @@ -12,7 +12,7 @@ tags: - top-k-selection - vectorized-loads confidence: source-reported -reproducibility: snippet +reproducibility: concept kernel_types: - attention - gemm @@ -28,30 +28,93 @@ related: sources: - pr-TensorRT-LLM-13340 performance_claims: [] -blackwell_relevance: TensorRT-LLM's Blackwell DSA indexer PR is a current upstream implementation reference for FP4/FP8 cache indexer paths and fused quantized gather/scatter kernels. +blackwell_relevance: At the pinned PR revision, TensorRT-LLM's SM100 DSA path fuses FP4 concatenation and quantization, copies packed cache payloads and scales with separate gather/scatter kernels, computes FP4 indexer logits, and selects top-k separately. --- -## Shape - -TensorRT-LLM PR 13340 integrates an FP4 indexer path for DSA on Blackwell and -lands CUDA kernels for K-cache gather/scatter and fused FP4 concatenation. Use it -as implementation evidence for sparse indexer memory movement and quantized -cache layout, not as a drop-in answer for FlashInfer-Bench. - -```cuda -// Evidence checklist before adapting the idea: -// 1. Inspect indexerKCacheGather.cu and indexerKCacheScatter.cu. -// 2. Check the scale and FP4 packing layout. -// 3. Benchmark gather/scatter separately from top-k selection. -__global__ void candidate_indexer_probe(const uint8_t* cache, int* indices) { - int tid = blockIdx.x * blockDim.x + threadIdx.x; - indices[tid] = static_cast(cache[tid]); -} -``` - -## Transfer Notes - -- Keep FP4 packing, scale placement, and invalid-token handling explicit in the - correctness reference. -- Treat gather/scatter traffic as a separate NCU profile target. -- Avoid merging this with top-k selection until the memory path is understood. +## Audited scope + +TensorRT-LLM PR 13340 adds an FP4 option to its DeepSeek Sparse Attention +(DSA) indexer path. This page describes the PR's pinned merge revision +`897c4bff`; it is a TensorRT-LLM implementation reference, not a generic DSA +ABI or a drop-in FlashInfer-Bench kernel. + +The implementation has four distinct stages: + +| Stage | Pinned behavior | +| --- | --- | +| Q/K preparation | `fused_cat_fp4` concatenates BF16 positional and non-positional components, then quantizes the 128-value row to FP4 E2M1 with per-32-value UE8M0 scales | +| Cache update/read | Scatter and gather copy an already-quantized payload and its scale word between contiguous tensors and TensorRT-LLM's possibly non-contiguous paged indexer cache | +| Indexer logits | The FP4 path reinterprets the packed bytes/scales and dispatches TensorRT-LLM's DeepGEMM FP4 MQA-logit implementation | +| Selection | Prefill or decode top-k runs after logit computation in a separate operator | + +The gather/scatter kernels do not quantize values, compute logits, or select +indices. Calling them “fused quantized gather/scatter” conflates the first two +stages. + +## FP4 row and scale contract + +For the fixed 128-value indexer head in this path, one row has: + +| Component | Representation | Bytes per row | +| --- | --- | ---: | +| Input | BF16 positional values followed by BF16 non-positional values | 256 before quantization | +| Packed payload | Two FP4 E2M1 codes per byte | 64 | +| Scale word | Four UE8M0 exponent bytes, one per successive group of 32 values, packed little-endian into one `int32` | 4 | +| Cache footprint | Packed payload plus scale word | 68 | + +The fused operator requires CUDA BF16 inputs on one device, at least two +dimensions, a contiguous innermost dimension, eight-byte-aligned addresses, +equal row counts, and positional width divisible by four. Positional and +non-positional widths must sum to 128. It returns packed `int8[M,64]` data and +`int32[M,1]` scales. + +The scale for each 32-value group is +`2^ceil(log2(max(amax, 1e-12) / 6))`. Quantization uses the FP4 E2M1 magnitude +set `{0, 0.5, 1, 1.5, 2, 3, 4, 6}` and packs the earlier value into the low +nibble. + +## Gather and scatter semantics + +The cache is viewed as +`[num_blocks, block_size, 1, per_token_size]` and may be non-contiguous. Each +token launch copies four bytes per thread. The FP4 path uses 64 payload bytes +and one four-byte scale word; the legacy FP8 path uses 128 payload bytes and +one four-byte FP32 scale. + +Payload and scale use separate contiguous `int64` slot-mapping arrays. If +either mapping for a token is negative, gather and scatter skip that entire +token. The gather wrapper allocates output with `empty`, so a directly gathered +row skipped this way has no kernel-written sentinel value; callers must avoid +consuming it or define initialization in their own reference. + +The gather wrapper preserves a historical typed view: payload bytes are +returned as float8 and scale bytes as FP32. The FP4 call site reinterprets them +as packed `int8` data and `int32` scale words before the DeepGEMM call. These +views are byte contracts, not evidence that the FP4 payload became FP8 or that +its UE8M0 scale word became an FP32 numeric scale. + +## Validation and profiling procedure + +1. Compare `fused_cat_fp4` byte-for-byte with a reference implementation for + zeros, FP4 decision boundaries, saturation, non-contiguous row strides, and + multiple positional/non-positional splits whose widths sum to 128. +2. Round-trip valid tokens through scatter and gather using strided cache + views. Check all 64 payload bytes and the four scale bytes independently; + test negative payload and scale mappings without reading skipped empty rows. +3. Compare FP4 non-paged and paged indexer logits, then selected indices, + against an unquantized or higher-precision reference with identical masks, + weights, and sequence boundaries. +4. Profile fused preparation, scatter, gather, logit computation, top-k, and + the complete indexer separately. Report shapes, cache layout, mapping mix, + warmup, synchronization, repetitions, statistic, and variation. + +The pinned PR contains no performance result, so use it to define candidate +mechanisms and exact contracts rather than to claim a speedup. + +## Primary references + +- [TensorRT-LLM PR 13340](https://github.com/NVIDIA/TensorRT-LLM/pull/13340) +- [Pinned `fusedCatFp4.cu` artifact](../../artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/fusedCatFp4.cu) +- [Pinned gather kernel artifact](../../artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheGather.cu) +- [Pinned scatter kernel artifact](../../artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheScatter.cu) +- [Pinned DSA integration artifact](../../artifacts/prs/tensorrt-llm/PR-13340/key-files/tensorrt_llm/_torch/attention_backend/sparse/dsa.py) diff --git a/wiki/languages/cuda-cpp.md b/wiki/languages/cuda-cpp.md index 6e3e97413..635ffc7f9 100644 --- a/wiki/languages/cuda-cpp.md +++ b/wiki/languages/cuda-cpp.md @@ -7,108 +7,80 @@ related: [lang-ptx, hw-tcgen05-mma, hw-tmem, blog-tcgen05-tutorial] sources: [blog-tcgen05-tutorial, doc-nvidia-tuning-guide, blog-yue-nvfp4] reproducibility: snippet architectures: [sm100, sm100a] -confidence: source-reported +confidence: verified +evidence_basis: + - source_id: doc-nvidia-tuning-guide + evidence_type: official-doc --- -## Overview +## Scope -Plain CUDA C++ with inline PTX is used for hand-optimized Blackwell kernels. The tcgen05 tutorial achieved 98% of cuBLAS performance using this approach. +CUDA C++ can host PTX instructions that do not yet have a convenient CUDA intrinsic. Gau Nernst's pinned `tcgen05` tutorial uses that approach for a B200 GEMM. For M=N=K=4096 in its disclosed PyTorch 2.9.1/CUDA 13 environment, v6 reports 1475.93 TFLOP/s versus 1506.74 TFLOP/s for cuBLAS, or 97.96%. This is an author-reported result for that configuration, not a portable performance guarantee. -## tcgen05 via Inline PTX +## Inline-PTX boundary -```cuda -// Allocate TMEM. tcgen05.alloc writes the allocated address into SMEM. -__device__ uint32_t tmem_alloc_cta(uint32_t* smem_tmem_addr, - uint32_t num_cols) { - if (threadIdx.x == 0) { - uint32_t smem_addr = - static_cast(__cvta_generic_to_shared(smem_tmem_addr)); - asm volatile( - "tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;" - :: "r"(smem_addr), "r"(num_cols) - ); - } - __syncthreads(); - return *smem_tmem_addr; -} +The CUDA front end does not parse the instruction text inside an `asm()` statement. Operand constraints and address-space conversion therefore remain the wrapper author's responsibility. Use `"r"` for a 32-bit integer register, `"l"` for a 64-bit integer register, and convert a generic pointer with `__cvta_generic_to_shared` before supplying a shared-memory address. Add a `"memory"` clobber when the assembly has memory effects that are hidden from the compiler. -// Issue MMA (single thread, typically warp 1 lane 0) -// idesc_c/idesc_d: immediate descriptors for accumulator C and output D -__device__ void tcgen05_mma(uint32_t tmem_addr, - uint64_t desc_a, uint64_t desc_b, - uint32_t idesc_c, uint32_t idesc_d) { - asm volatile( - "tcgen05.mma.cta_group::1.kind::f16" - " [%0], %1, %2, %3, %4;" - :: "r"(tmem_addr), "l"(desc_a), "l"(desc_b), - "r"(idesc_c), "r"(idesc_d) - ); -} +The allocation instruction is collective. For `.cta_group::1`, every lane of one designated warp must execute the same instruction. Synchronize the CTA before another warp reads the address written to shared memory. -// Load TMEM to registers -__device__ void tmem_load(float* dst, uint32_t tmem_addr, int cols) { - asm volatile( - "tcgen05.ld.sync.aligned.32x32b.x1.b32 {%0}, [%1];" - : "=f"(*dst) : "r"(tmem_addr) - ); -} - -// Deallocate TMEM (MUST do before kernel exit) -__device__ void tmem_dealloc(uint32_t addr, uint32_t num_cols) { +```cuda +// All 32 lanes of alloc_warp execute this branch. +if (warp_id == alloc_warp) { + uint32_t smem_addr = + static_cast(__cvta_generic_to_shared(smem_tmem_addr)); asm volatile( - "tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;" - :: "r"(addr), "r"(num_cols) - ); + "tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 " + "[%0], %1;" + :: "r"(smem_addr), "r"(num_cols) : "memory"); } +__syncthreads(); +uint32_t taddr = *smem_tmem_addr; ``` -## mbarrier Synchronization +An unscaled `kind::f16` MMA takes one instruction descriptor and an `enable-input-d` predicate. A single elected thread may issue this MMA form; converting an ordinary CUDA integer to the required PTX predicate inside the assembly keeps the C++ interface well typed. ```cuda -// TMA-MMA synchronization via mbarrier -// expected_bytes: total bytes the TMA will deliver to this stage -__device__ void mbarrier_arrive(uint64_t* mbar, uint32_t expected_bytes) { - asm volatile( - "mbarrier.arrive.expect_tx.shared.b64 _, [%0], %1;" - :: "r"((uint32_t)__cvta_generic_to_shared(mbar)), - "r"(expected_bytes) - ); -} - -__device__ void mbarrier_wait(uint64_t* mbar, int phase) { +__device__ inline void tcgen05_mma_f16( + uint32_t taddr, uint64_t a_desc, uint64_t b_desc, + uint32_t idesc, int enable_input_d) { asm volatile( - "{\n" - ".reg .pred p;\n" - "WAIT_LOOP:\n" - " mbarrier.try_wait.parity.shared.b64 p, [%0], %1;\n" - " @!p bra WAIT_LOOP;\n" - "}\n" - :: "r"((uint32_t)__cvta_generic_to_shared(mbar)), - "r"(phase) - ); + "{\n\t" + ".reg .pred p;\n\t" + "setp.ne.b32 p, %4, 0;\n\t" + "tcgen05.mma.cta_group::1.kind::f16 " + "[%0], %1, %2, %3, p;\n\t" + "}" + :: "r"(taddr), "l"(a_desc), "l"(b_desc), + "r"(idesc), "r"(enable_input_d)); } ``` -## Warp Role Dispatch +`tcgen05.ld` is likewise a warp-level instruction. All participating lanes execute it, then the warp executes `tcgen05.wait::ld.sync.aligned` before consuming the loaded registers. After all readers finish, synchronize at the required CTA or cluster scope and have all lanes of one warp execute the matching collective `tcgen05.dealloc` before kernel exit. -```cuda -__global__ void blackwell_gemm_kernel(...) { - int warp_id = threadIdx.x / 32; - int lane_id = threadIdx.x % 32; - - if (warp_id == 0 && lane_id == 0) { - // TMA producer: issue cp.async.bulk.tensor - tma_producer_loop(...); - } else if (warp_id == 1 && lane_id == 0) { - // MMA consumer: issue tcgen05.mma - mma_consumer_loop(...); - } else if (warp_id >= 2) { - // Epilogue: read TMEM, write to global - epilogue_loop(...); - } -} -``` +## Barrier lifecycle + +An `mbarrier.arrive.expect_tx` plus a parity wait is not a complete pipeline by itself. A staged TMA-to-MMA pipeline has these invariants: + +1. An elected thread initializes each barrier with the intended arrival count, then publishes initialization to the async proxy with the appropriate `fence.mbarrier_init`. +2. The producer reserves a reusable stage, issues TMA against that stage's full barrier, and accounts for the expected transaction bytes. +3. The consumer waits with acquire semantics on the matching phase before reading the shared-memory stage. +4. After issuing tcgen05 MMA, it commits completion to a separate barrier; the epilogue waits before loading TMEM. +5. The last stage user signals a separate empty/reuse barrier. A stage cannot be overwritten until its owner observes that handoff. +6. Each barrier's parity flips only when its circular stage is revisited. Inline assembly that reads or writes memory invisibly to C++ uses a `"memory"` clobber. + +Arrival counts, transaction bytes, scope, and which CTA owns each barrier depend on whether the kernel uses one-CTA or two-CTA MMA. Copy the complete protocol from the pinned implementation rather than treating isolated wait/arrive wrappers as a standalone recipe. + +## One verified role split + +The tutorial's v6 kernel launches six warps per CTA: warp 0 elects one lane for TMA, warp 1 collectively allocates TMEM and elects one lane for MMA, and warps 2--5 run the epilogue. Before exit, all threads finish their TMEM reads and warp 0 collectively deallocates TMEM. This is the role split of that implementation, not a requirement of CUDA C++ or tcgen05. + +## Primary references + +- [CUDA 13.0.2 Inline PTX Assembly guide](https://docs.nvidia.com/cuda/archive/13.0.2/inline-ptx-assembly/index.html) +- [PTX ISA 9.0 tcgen05 instructions](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05) +- [Pinned tutorial implementation](https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100) ## Related -- [ptx-sm100](ptx-sm100.md) — PTX instruction reference -- [tcgen05 tutorial](../../sources/blogs/tcgen05-tutorial.md) — Step-by-step guide + +- [ptx-sm100](ptx-sm100.md) — version-pinned PTX instruction forms +- [tcgen05 tutorial](../../sources/blogs/tcgen05-tutorial.md) — benchmark provenance and progression diff --git a/wiki/languages/cute-dsl.md b/wiki/languages/cute-dsl.md index 210764e2f..2360174ae 100644 --- a/wiki/languages/cute-dsl.md +++ b/wiki/languages/cute-dsl.md @@ -7,112 +7,119 @@ related: [hw-tcgen05-mma, hw-tmem, kernel-flash-attention-4, doc-cutlass-blackwe sources: [doc-cutlass-blackwell, blog-colfax-cutlass, blog-flash-attention-4] reproducibility: snippet architectures: [sm100, sm100a] -confidence: source-reported +confidence: verified +evidence_basis: + - source_id: doc-cutlass-blackwell + evidence_type: official-doc --- -## Overview +## Scope -CuTe (CUDA Templates) DSL is the primary abstraction layer in CUTLASS 4.5.0 for Blackwell kernels. FlashAttention-4 was implemented entirely in CuTe-DSL (Python variant), achieving 20-30× faster compilation than C++ templates. +CUTLASS v4.5.0 includes CuTe DSL, a Python-native interface for authoring GPU kernels, alongside CUTLASS's C++ template interfaces. This page uses the exact `v4.5.0` tag (`e406c186f510a15091cce01f782020ceb7ba8eb5`); rolling `latest` documentation can contain later names. -## SM100 MMA Atoms +FlashAttention-4 is implemented entirely in CuTe DSL. Its author reports roughly 20--30x shorter compile times than C++ templates and a peak of 1605 TFLOP/s on B200 BF16. These are FA4-specific, source-reported results. The 1605-TFLOP/s result was not presented as a matched comparison with a handwritten-C++ FA4 kernel. -```python -# CuTe-DSL: SM100 MMA atom for BF16 -from cutlass.cute import * +## F16/BF16 MMA construction -# 1-SM MMA: m128 x n256 x k16 -mma_atom = SM100_MMA_F16BF16_SS # inputs from shared memory -# Accumulator goes to TMEM automatically +CUTLASS 4.5.0 uses a configured `tcgen05.MmaF16BF16Op`, not an `SM100_MMA_F16BF16_SS` Python symbol. This excerpt shows the one-CTA operation used by the tagged first tutorial; it is construction code, not a complete kernel. -# 2-SM MMA: m256 x n256 x k16 -mma_atom_2sm = SM100_MMA_F16BF16_SS_2SM +```python +import cutlass +import cutlass.cute as cute +from cutlass.cute.nvgpu import tcgen05 + +op = tcgen05.MmaF16BF16Op( + cutlass.Float16, + cutlass.Float32, + (128, 256, 16), + tcgen05.CtaGroup.ONE, + tcgen05.OperandSource.SMEM, + cute.nvgpu.OperandMajorMode.K, + cute.nvgpu.OperandMajorMode.K, +) +tiled_mma = cute.make_tiled_mma(op) ``` -## TMEM as CuTe Locale +The two-CTA tutorial instead uses instruction shape `(256, 256, 16)` and `tcgen05.CtaGroup.TWO`. The accumulator fragment is rebound to an allocated TMEM pointer before `cute.gemm` issues the operation. + +## TMEM lifecycle and epilogue copy + +The tagged examples use `cutlass.utils.TmemAllocator` around a shared holding buffer. A complete path allocates columns, waits before pointer retrieval, rebinds the accumulator tensor, partitions a typed `tcgen05` TMEM-to-register copy across participating threads, synchronizes readers, and frees the same allocation. ```python -# TMEM allocation in CuTe -tmem_tensor = make_tensor( - make_tmem_ptr(tmem_addr), - make_layout(make_shape(128, 256)) # rows x cols +# Excerpt: storage/barrier/tensor definitions and pipeline edges are required. +tmem = utils.TmemAllocator( + storage.tmem_holding_buf.ptr, + barrier_for_retrieve=tmem_alloc_barrier, +) +tmem.allocate(num_tmem_cols) +tmem.wait_for_alloc() +tmem_ptr = tmem.retrieve_ptr(cutlass.Float32) +tCtAcc = cute.make_tensor(tmem_ptr, tCtAcc.layout) + +tmem_atom = cute.make_copy_atom( + tcgen05.Ld32x32bOp(tcgen05.Repetition.x64), cutlass.Float32 ) +tmem_tiled_copy = tcgen05.make_tmem_copy(tmem_atom, tCtAcc_epi[None, 0]) +# get_slice(), partition_S/D(), and cute.copy() form the per-thread epilogue. -# Copy TMEM → registers for epilogue -copy(tmem_tensor, reg_tensor) # tcgen05.ld under the hood +pipeline.sync(barrier_id=1) +tmem.free(tmem_ptr) ``` -## TMA Copy Atoms +The excerpt deliberately does not imply that allocation or deallocation is lane-local. Use the complete tutorial for collective participation, two-CTA handling, and pointer lifetime. + +## TMA construction + +The one-CTA tutorial constructs a typed global-to-shared TMA operation and then derives operand-specific tiled atoms. Kernel code subsequently uses `tma_partition` and `cute.copy` with pipeline barriers. ```python -# TMA bulk copy: global → shared -tma_copy = SM100_TMA_LOAD_2D - -# Setup TMA descriptor -tma_desc = make_tma_copy( - tma_copy, - global_tensor, - smem_layout, - tile_shape, - cluster_shape +from cutlass.cute.nvgpu import cpasync, tcgen05 + +tma_op = cpasync.CopyBulkTensorTileG2SOp(tcgen05.CtaGroup.ONE) +a_tma_atom, a_tma_tensor = cute.nvgpu.make_tiled_tma_atom_A( + tma_op, + a, + a_smem_layout_one_stage, + mma_tiler_mnk, + tiled_mma, ) ``` -## Warp-Specialized Kernel Skeleton +For the two-CTA tutorial, the corresponding operation is `CopyBulkTensorTileG2SMulticastOp(CtaGroup.TWO)` and the cluster layout/multicast participants must agree with the launch. -```python -@cute_kernel -def blackwell_gemm(A, B, C): - # Warp 0: TMA producer - if warp_id == 0: - for stage in pipeline: - tma_copy(A_tile, smem_A[stage]) - tma_copy(B_tile, smem_B[stage]) - arrive(mbarrier[stage]) - - # Warp 1: MMA consumer - elif warp_id == 1: - tmem = alloc_tmem(256) # columns - for stage in pipeline: - wait(mbarrier[stage]) - fence_after_thread_sync() - mma(smem_A[stage], smem_B[stage], tmem) - signal_epilogue() - - # Warps 2+: Epilogue - else: - wait_epilogue() - regs = load_tmem(tmem) - store_global(C, regs) - dealloc_tmem(tmem) -``` +## Warp specialization and layouts -## Why CuTe-DSL for Blackwell +`fp16_gemm_2.py` specializes TMA, MMA, and epilogue warps and uses `PipelineTmaUmma` plus `PipelineUmmaAsync` to represent full/empty ownership. It also retains explicit TMEM allocation, epilogue TMEM-copy partitioning, TMA-store completion, pipeline tails, and collective teardown. Those details are required; an `if warp_id` sketch alone is not a safe synchronization recipe. -1. **20-30× faster compilation** than C++ CUTLASS templates -2. Python-native: easier to iterate and debug -3. Same performance as hand-written C++ (FlashAttention-4: 1605 TFLOPS) -4. First-class TMEM and tcgen05 support in CUTLASS 4.5.0 -5. Automatic layout computation and swizzle handling +CuTe supplies typed layout algebra and Blackwell helpers such as `make_smem_layout_a` and `make_smem_layout_b`. The author still supplies the MMA tiler, datatypes, operand major modes, alignment, cluster shape, and pipeline policy; layout and swizzle choices are helper-assisted rather than universally automatic. -## Full Examples (verbatim upstream code shipped locally) +## Verbatim upstream artifacts -The following CuTe DSL files ship **verbatim** in this repository under `artifacts/prs/cutlass/` (pinned at each PR's merge SHA). Open them with `python3 scripts/get_page.py --include-code` or read them directly. +The following files are untruncated, verbatim captures. Their bytes match the SHA-256 records in each PR's `PROVENANCE.yaml`, and the seven numeric sizes below are physical line counts. To inspect through the repository helper, run `conda run -n base python scripts/get_page.py --include-code`. -| File | Purpose | Size | -|---|---|---| -| [`artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_0.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_0.py) | Step 0 — FP16 GEMM baseline | 447 lines | -| [`artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_1.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_1.py) | Step 1 — 2CTA MMA + TMA multicast | 535 lines | -| [`artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_2.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_2.py) | Step 2 — Warp specialization (TMA / MMA / epilogue warps) | 679 lines | -| [`artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_3.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_3.py) | Step 3 — Static persistent tile scheduler | 769 lines | -| [`artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_4.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_4.py) | Step 4 — Preferred + dynamic clusters | 1065 lines | -| [`artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_5.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_5.py) | Step 5 — TMA prefetch | 919 lines | -| [`artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_6.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_6.py) | Step 6 — Programmatic Dependent Launch (PDL) | 1002 lines | -| [`artifacts/prs/cutlass/PR-2881/key-files/examples/python/CuTeDSL/blackwell/dense_gemm_persistent_prefetch.py`](../../artifacts/prs/cutlass/PR-2881/key-files/examples/python/CuTeDSL/blackwell/dense_gemm_persistent_prefetch.py) | Persistent GEMM with prefetch | full | -| [`artifacts/prs/cutlass/PR-3021/key-files/python/CuTeDSL/cutlass/cute/arch/clc.py`](../../artifacts/prs/cutlass/PR-3021/key-files/python/CuTeDSL/cutlass/cute/arch/clc.py) | CLC (Cluster Launch Control) Python binding | full | +| File | Upstream purpose | Lines | +|---|---|---:| +| [`fp16_gemm_0.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_0.py) | Software-pipelined FP16 tutorial baseline | 447 | +| [`fp16_gemm_1.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_1.py) | Add two-CTA MMA and TMA multicast | 535 | +| [`fp16_gemm_2.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_2.py) | Add TMA/MMA/epilogue warp specialization | 679 | +| [`fp16_gemm_3.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_3.py) | Add static persistent scheduling | 769 | +| [`fp16_gemm_4.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_4.py) | Add preferred and fallback/dynamic clusters | 1065 | +| [`fp16_gemm_5.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_5.py) | Add TMA data prefetch | 919 | +| [`fp16_gemm_6.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_6.py) | Add Programmatic Dependent Launch | 1002 | +| [`dense_gemm_persistent_prefetch.py`](../../artifacts/prs/cutlass/PR-2881/key-files/examples/python/CuTeDSL/blackwell/dense_gemm_persistent_prefetch.py) | Persistent GEMM with TMA prefetch | full capture | +| [`clc.py`](../../artifacts/prs/cutlass/PR-3021/key-files/python/CuTeDSL/cutlass/cute/arch/clc.py) | CLC Python binding | full capture | -The `fp16_gemm_{0..6}.py` series from `examples/python/CuTeDSL/blackwell/tutorial_gemm/` in NVIDIA/cutlass PR-3106 is the authoritative CuTe DSL learning path: it walks from a naive FP16 GEMM baseline through 2CTA MMA with TMA multicast, warp specialization, static persistent scheduling, preferred / dynamic clusters, TMA prefetch, and ends with Programmatic Dependent Launch (PDL). Reading them in order is the recommended on-ramp. +PR 3106's series is an official progressive tutorial: software-pipelined FP16, two-CTA multicast, warp specialization, static persistence, preferred/dynamic clusters, prefetch, and PDL. In the v4.5.0 tag, the same series is under `examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/`. + +## Primary references + +- [CUTLASS v4.5.0 tag](https://github.com/NVIDIA/cutlass/tree/e406c186f510a15091cce01f782020ceb7ba8eb5) +- [Pinned v4.5.0 tutorial series](https://github.com/NVIDIA/cutlass/tree/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm) +- [FlashAttention-4 first-party post](https://tridao.me/blog/2026/flash4/) ## Related -- [tcgen05-mma](../hardware/tcgen05-mma.md) — Underlying MMA instruction -- [flash-attention-4](../kernels/flash-attention-4.md) — CuTe-DSL implementation -- [CUTLASS Blackwell docs](../../sources/docs/nvidia-cutlass-blackwell.md) — Official reference + +- [tcgen05-mma](../hardware/tcgen05-mma.md) — underlying instruction semantics +- [flash-attention-4](../kernels/flash-attention-4.md) — evidence-scoped CuTe DSL case study +- [CUTLASS Blackwell source card](../../sources/docs/nvidia-cutlass-blackwell.md) — version-pinned source routes diff --git a/wiki/languages/ptx-sm100.md b/wiki/languages/ptx-sm100.md index 79a3dade1..12f5ad448 100644 --- a/wiki/languages/ptx-sm100.md +++ b/wiki/languages/ptx-sm100.md @@ -7,85 +7,118 @@ related: [hw-tcgen05-mma, hw-tmem, hw-clc, lang-cuda-cpp] sources: [doc-ptx-isa-sm100, doc-nvidia-tuning-guide, blog-yue-nvfp4] reproducibility: snippet architectures: [sm100, sm100a] -confidence: source-reported +confidence: verified +evidence_basis: + - source_id: doc-ptx-isa-sm100 + evidence_type: official-doc --- -## Overview +# PTX Instructions for SM100 -SM100 PTX instructions for Blackwell-specific hardware features. +## Scope -## tcgen05 Instructions +This page uses CUDA 13.0.2's archived PTX ISA 9.0 grammar. The fragments are representative instruction forms, not complete inline-assembly functions: operands need correctly typed declarations, descriptors, collective issue, lifetime management, and the instruction-specific completion protocol. + +## tcgen05 and Tensor Memory + +Representative unscaled F16 forms include: ```ptx -// Allocate TMEM columns -tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [smem_tmem_addr], num_cols; +tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [alloc_result_smem], ncols; +ld.shared.b32 taddr, [alloc_result_smem]; -// MMA: inputs from SMEM, accumulator in TMEM -tcgen05.mma.cta_group::1.kind::f16 [tmem_addr], desc_a, desc_b, idesc, enable_input_d; +tcgen05.mma.cta_group::1.kind::f16 [taddr], a_desc, b_desc, idesc, enable_input_d; +tcgen05.commit.cta_group::1.mbarrier::arrive::one.b64 [mma_done]; -// 2-SM cooperative MMA -tcgen05.mma.cta_group::2.kind::f16 [tmem_addr], desc_a, desc_b, idesc, enable_input_d; +tcgen05.dealloc.cta_group::1.sync.aligned.b32 taddr, ncols; +``` -// Load TMEM to registers -tcgen05.ld.sync.aligned.32x32b.x1.b32 {regs}, [tmem_addr]; +`tcgen05.alloc` writes a 32-bit TMEM address to shared memory and counts columns. Allocation and deallocation are warp-collective for `cta_group::1`, every allocation must be deallocated before kernel exit, and all tcgen05 instructions in one kernel must use the same CTA-group value. The `cta_group::2` MMA form has the same unscaled operand grammar but can access paired-CTA resources. -// Store registers to TMEM -tcgen05.st.sync.aligned.32x32b.x1.b32 [tmem_addr], {regs}; +TMEM/register transfers and shared-memory/TMEM copies have their own asynchronous completion rules: -// Copy a shaped SMEM matrix descriptor into TMEM -tcgen05.cp.cta_group::1.128x256b [tmem_addr], sdesc; +```ptx +tcgen05.ld.sync.aligned.32x32b.x1.b32 {r0}, [taddr]; +tcgen05.wait::ld.sync.aligned; -// Critical fence between TMA completion and MMA -tcgen05.fence::after_thread_sync; +tcgen05.st.sync.aligned.32x32b.x1.b32 [taddr], {r0}; +tcgen05.wait::st.sync.aligned; -// Deallocate TMEM (MUST before kernel exit) -tcgen05.dealloc.cta_group::1.sync.aligned.b32 tmem_addr, num_cols; +tcgen05.cp.cta_group::1.128x256b [taddr], sdesc; ``` -## NVFP4 Conversion Instructions +`tcgen05.ld` and `tcgen05.st` are warp-collective. `tcgen05.cp` copies a shaped shared-memory descriptor into TMEM. MMA and cp completion can be attached to an mbarrier with `tcgen05.commit`; the source and destination must remain live until their documented completion points. -```ptx -// Convert two packed FP4 values to two FP16 values -cvt.rn.f16x2.e2m1x2 result_f16x2, packed_fp4; +## Completion and Ordering Are Distinct -// Byte unpacking (faster than bitwise extraction) -mov.b32 {byte0, byte1, byte2, byte3}, packed_word; -``` +- A TMA global-to-shared load completes bytes on its mbarrier; a consumer waits for that phase before reading the destination. +- `tcgen05.commit` makes an mbarrier track completion of prior asynchronous tcgen05 MMA/cp/shift operations issued by the thread. +- `tcgen05.wait::ld` and `tcgen05.wait::st` wait for the corresponding prior TMEM/register transfers. +- `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` constrain tcgen05 operations around a documented cross-thread execution-ordering handoff. They are not substitutes for TMA or MMA completion waits. -## Cache Control for Memory-Bound Kernels +## Packed FP4 Conversion + +PTX ISA 9.0 defines these typed operations: ```ptx -// Streaming data (use once): bypass L1 -ld.global.L1::no_allocate.v2.u64 {r0, r1}, [addr]; +cvt.rn.f16x2.e2m1x2 result_f16x2, packed_fp4_pair; +mov.b32 {byte0, byte1, byte2, byte3}, packed_word; +``` + +The first converts one byte containing two E2M1 values into a 32-bit F16x2 result. The second can decompose a 32-bit scalar into four byte-sized destinations when operand declarations satisfy the scalar-to-vector size rules. PTX specifies these semantics but does not guarantee that the move is faster than every compiler-generated shift/mask sequence. -// Reused data: keep in L1 -ld.global.L1::evict_last.v2.u64 {r0, r1}, [addr]; +## Cache Eviction Hints and Vector Width -// Wide vectorized loads -ld.global.v4.u64 {r0, r1, r2, r3}, [addr]; // 256-bit +```ptx +ld.global.L1::no_allocate.v2.u64 {r0, r1}, [addr]; +ld.global.L1::evict_last.v2.u64 {r0, r1}, [addr]; +ld.global.v4.u64 {r0, r1, r2, r3}, [addr]; ``` +`L1::no_allocate` and `L1::evict_last` are eviction-priority hints and may not always be respected. They do not guarantee L1 bypass or residency. The vector forms above move 16 and 32 bytes; whether a width or hint helps depends on alignment, surrounding instructions, reuse, cache state, and the target GPU. + ## Cluster Launch Control +CLC cancellation is an asynchronous request followed by response decoding: + ```ptx -// Query for next tile (persistent kernel loop) -clusterlaunchcontrol.try_cancel {clc_id}; -// Returns valid tile_id or decline (all work done) +clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 + [response_smem], [response_mbarrier]; + +clusterlaunchcontrol.query_cancel.is_canceled.pred.b128 p, response_b128; +@p clusterlaunchcontrol.query_cancel.get_first_ctaid.v4.b32.b128 + {x, y, z, unused}, response_b128; ``` -## TMA (Tensor Memory Accelerator) +The request writes an opaque 16-byte response to shared memory and completes on the mbarrier. Code must wait for that phase before loading and querying the response. A successful query returns the first CTA coordinate of a canceled not-yet-launched block or cluster; the request does not take a desired tile ID. + +## Tensor Memory Accelerator + +A representative 2D global-to-cluster-shared load is: ```ptx -// Bulk tensor copy: global → shared cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes - [smem_ptr], [tensorMap, {x, y}], [mbarrier]; + [dst_smem], [tensor_map, {x, y}], [full_mbarrier]; +``` + +Cluster multicast adds the full `.multicast::cluster` qualifier and a 16-bit CTA mask: -// Multicast to cluster SMs -cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast - [smem_ptr], [tensorMap, {x, y}], [mbarrier], multicast_mask; +```ptx +cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster + [dst_smem], [tensor_map, {x, y}], [full_mbarrier], cta_mask; ``` +The multicast data and completion signal target the selected CTAs at corresponding shared-memory offsets. Each destination must have a live, initialized matching barrier and wait for its own phase before consuming the tile. + ## Related -- [cuda-cpp](cuda-cpp.md) — Inline PTX in CUDA C++ -- [tcgen05-mma](../hardware/tcgen05-mma.md) — MMA instruction details -- [nvfp4](../hardware/nvfp4.md) — FP4 format details + +- [CUDA C++](cuda-cpp.md) — inline PTX integration +- [tcgen05 MMA](../hardware/tcgen05-mma.md) — operand, descriptor, completion, and shape details +- [Tensor Memory](../hardware/tmem.md) — allocation, layout, and transfer details +- [Cluster Launch Control](../hardware/clc.md) — complete request/decode protocol +- [TMA](../hardware/tma.md) — tensor-map and pipeline semantics +- [NVFP4](../hardware/nvfp4.md) — packed format and block scaling + +## Primary Reference + +- [CUDA 13.0.2, PTX ISA 9.0](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html) diff --git a/wiki/languages/triton-blackwell.md b/wiki/languages/triton-blackwell.md index feda3cb98..6a33b7687 100644 --- a/wiki/languages/triton-blackwell.md +++ b/wiki/languages/triton-blackwell.md @@ -4,162 +4,95 @@ title: "Triton on Blackwell" type: language tags: [triton, attention, moe, gated-delta-net] related: [kernel-nsa, kernel-gated-delta-net, kernel-fused-moe, lang-cute-dsl] -sources: [doc-triton-3.6-blackwell, pr-vllm-34597, pr-vllm-29339, pr-sglang-22079, pr-sglang-21019, pr-sglang-5390, pr-sglang-21595, pr-pytorch-175826, blog-nsa, blog-gated-delta-net, blog-flash-attention-4] +sources: [doc-triton-3.3-blackwell, doc-triton-3.6-blackwell, pr-vllm-34597, pr-vllm-29339, pr-sglang-22079, pr-sglang-21019, pr-sglang-5390, pr-sglang-21595] reproducibility: snippet -architectures: [sm100, sm90] +architectures: [sm100, sm100a, sm120, sm90] confidence: verified evidence_basis: - evidence_type: official-doc source_id: doc-triton-3.6-blackwell + - evidence_type: upstream-code + source_id: doc-triton-3.3-blackwell - evidence_type: upstream-code source_id: pr-vllm-34597 version_sensitive: - id: vs-triton-3.6-blackwell-tcgen05 -blackwell_relevance: "As of Triton 3.6+, Triton has native Blackwell (SM100) lowering through tcgen05 + TMEM via descriptor/TMA warp-specialized matmul, Gluon multi-CTA / 2CTA, and tl.dot_scaled. This page documents which lowering surfaces are first-class on Blackwell vs which are still gluon-only or workload-dependent." + id: vs-triton-3.3-blackwell-tcgen05 +blackwell_relevance: "Native TCGen5/TMEM compiler support enters between Triton v3.2.0 and v3.3.0. Later releases expand the user-visible surfaces, while exact instruction selection remains configuration-dependent." --- -## Overview - -Triton is used for many attention and linear-attention kernels (NSA, GatedDeltaNet, FLA). Starting with Triton 3.6 (released `2026-01-21`), Triton ships native Blackwell (SM100) lowering through `tcgen05.mma` + Tensor Memory (TMEM). The earlier framing — "Triton compiler generates wgmma, not tcgen05" — was correct for Triton 3.5 and earlier but is no longer correct on 3.6+. See the "Pre-3.6 historical context" subsection below for the historical framing; the rest of this page describes the current 3.6+ behavior. - -When to use Triton on Blackwell: -- Rapid prototyping (Triton's fast turnaround still beats CuTe-DSL for exploration) -- Memory-bound kernels (bandwidth is bottleneck, not compute) -- Linear / sparse attention (GatedDeltaNet, FLA, NSA) where Triton's grid scheduling is convenient -- Block-scaled matmul on Blackwell via `tl.dot_scaled` (NVFP4 / MXFP families) — first-class hardware-accelerated path -- Warp-specialized descriptor/TMA matmul kernels following the Triton persistent matmul tutorial pattern - -Use CuTe-DSL / CUTLASS / FA-4 / TRT-LLM instead when: -- Peak-performance compute-bound matmul or attention on Blackwell. SGLang `pr-sglang-5390` measured a CUTLASS `tcgen05_mla` backend ~27% faster than the Triton MLA decode baseline on Blackwell. -- Production routing where vendor kernels are mature: SGLang `pr-sglang-21595` changes Blackwell datacenter multimodal attention default from `triton_attn` to FA-4. - -## Triton 3.6+ Blackwell path - -The 3.6 release adds Blackwell-native infrastructure for `tcgen05.mma`, TMEM allocation/copy/load/store, and warp-specialization plumbing. Source-of-record: [`doc-triton-3.6-blackwell`](../../sources/docs/triton-3.6-blackwell.md). Per-pathway breakdown with verified-vs-needs-verification status: [`data/triton-3.6-evidence.md`](../../data/triton-3.6-evidence.md). +## Verified release boundary -Verified lowering surfaces (with caveats): +Native Blackwell TCGen5/TMEM compiler support enters between Triton v3.2.0 and v3.3.0. In the exact tag comparison, the corresponding TCGen5 MMA, TMEM, and MMAv5-lowering symbols are absent at v3.2.0 (`9641643d`) and present at v3.3.0 (`819e9c8c`). The v3.3 conversion suite checks concrete `tcgen05.mma`, commit, TMEM, scaled-MMA, and `cta_group::2` output. See the pinned files in [`doc-triton-3.3-blackwell`](../../sources/docs/triton-3.3-blackwell.md). -1. **Descriptor/TMA + `tl.range(warp_specialize=True)` + `tl.dot`** — strongest checked `tl.*`-surface evidence. The official Triton persistent matmul tutorial states the warp-specialized mode "only works on Blackwell right now"; backed by warp-specialization aref plumbing (PRs `#8262`, `#7826`, `#8009`, `#8123`, `#8534`, `#8451`, `#8651`) and Blackwell TMEM / `tcgen05` backend work (PRs `#8136`, `#8148`, `#8202`, `#8386`, `#8421`, `#8495`, `#8102`, `#8338`, `#8225`). +| Release | Evidence-scoped milestone | +|---|---| +| v3.2.0 | Checked negative side of the native-backend boundary. | +| v3.3.0 | First checked tag after v3.2.0 with TCGen5/TMEM operations, allocation and lowering passes, and concrete conversion tests. | +| v3.5.0 | Tagged tree includes an explicit Gluon TCGen5/TMEM tutorial and the Blackwell block-scaled matmul tutorial; release notes also document warp-specialization work. | +| v3.6.0 | Generalizes TCGen5 copies/layouts and MMA handling, advances aref-style warp specialization, and adds initial multi-CTA/2-CTA Gluon work. It is not the introduction boundary. | +| v3.7.0 / v3.7.1 | v3.7.0 continues 2-CTA, multicast, and TMA work; v3.7.1 is a two-regression patch with no advertised new API or feature. | -2. **`tl.dot_scaled` block-scaled matmul** — hardware-accelerated by fifth-generation Tensor Cores on compute capability 10. The 3.6 dialect doc exposes `ttng.tc_gen5_mma_scaled` with TMEM-token semantics plus `ttng.tmem_copy`. Source PRs: Gluon NVIDIA `tcgen05 mma scaled` (`#8393`); frontend fixes (`#8564`, `#8658`); shared TMEM/tcgen05 backend (`#8136`, `#8148`, `#8202`). Format coverage centered on NVFP4 / MXFP per the official block-scaled matmul tutorial. +The compiler evidence proves that native paths exist. It does not prove that every plain `tl.dot` shape selects TCGen5: instruction selection can depend on architecture, dtype, shape, layout, and compiler configuration. A claim about one kernel's emitted instruction needs matching IR or PTX. -3. **Gluon front-end + `gl.warp_specialize` + `num_ctas`** — the most explicit Blackwell-native surface. Initial 2-CTA cluster support landed in `#8644`, `#8653`; `num_ctas` plumbing in `#8645`; Gluon-side `tcgen05 mma scaled` in `#8393`. The release notes describe 2-CTA / cluster as initial support, so cluster-scope kernels via Gluon should be treated as early-stage. +## User-visible surfaces -Caveats — what the verified evidence does and does not establish: -- **What is verified**: the Triton 3.6 release adds tcgen05 + TMEM lowering infrastructure (per `doc-triton-3.6-blackwell`), and tracked downstream repos are landing real Triton kernel changes for SM100 post-3.6 (per `pr-vllm-34597`, the post-refresh primary anchor — vLLM's "[Kernel] Add FP8 KV cache support to Triton MLA decode attention" PR, which directly modifies `@triton.jit`-decorated kernels in `vllm/v1/attention/ops/triton_decode_attention.py` doing `tl.dot(q, k)` / `tl.dot(qpe, kpe)` / `tl.dot(p, v)` matmul on `[sm100]`, shipped verbatim under `artifacts/prs/vllm/PR-34597/`). Supplementary post-refresh evidence: `pr-vllm-29339` shows the Triton 3.6 `triton_kernels` library being explicitly scoped to Blackwell production codepaths. Pre-refresh historical evidence: `pr-sglang-22079` (the Gemma4 NVFP4 attention kernel merged 2026-04-03). Together these establish that Triton 3.6+ on Blackwell is materially different from the pre-3.6 "lacked-tcgen05 / lacked-TMEM" story. -- **What is not yet verified by in-corpus evidence**: that arbitrary plain `tl.dot` kernels AUTOMATICALLY emit `ttng.tc_gen5_mma` / `tcgen05.*` PTX on every shape and configuration. The 3.6 infrastructure exists (`#8136`, `#8148`, `#8202`, `#8225`, `#8338`, `#8386`, `#8421`, `#8495`, `#8102` in the release notes) and downstream tl.dot kernels for SM100 are landing, but the question of "is the new path the default for every `tl.dot` shape on Blackwell" has not been confirmed by inspectable downstream PTX in any tracked-repo PR locally. Treat plain `tl.dot` on Blackwell as "uses the 3.6+ infrastructure when applicable" rather than as a blanket equivalence-to-CuTe-DSL claim. -- **What is also not yet verified**: that fused attention forward kernels with `warp_specialize=True`, `tl.dot_scaled` block-scaled matmul, and Gluon multi-CTA paths produce the same lowering as the upstream Triton tutorials predict — the upstream tutorials demonstrate these paths; tracked-downstream repos haven't yet landed PRs with inspectable PTX confirming each one. +The v3.5.0 tree provides two useful pinned examples: -### What changed vs the pre-3.6 framing +- [`python/tutorials/gluon/06-tcgen05.py`](https://github.com/triton-lang/triton/blob/c3c476f357f1e9768ea4e45aa5c17528449ab9ef/python/tutorials/gluon/06-tcgen05.py) explicitly allocates, loads, and stores TMEM and invokes TCGen5 MMA through Gluon. +- [`python/tutorials/10-block-scaled-matmul.py`](https://github.com/triton-lang/triton/blob/c3c476f357f1e9768ea4e45aa5c17528449ab9ef/python/tutorials/10-block-scaled-matmul.py) demonstrates `tl.dot_scaled` for Blackwell block-scaled matmul. -Before Triton 3.6, the Blackwell story was: the compiler generated `wgmma.mma_async`, accumulators stayed in registers, and direct `tcgen05` / TMEM access was unavailable from `tl.*`. Pages and inclusion-policy text written under that premise are obsolete. The current premise is: **`tcgen05` + TMEM lowering paths exist on SM100, but coverage and performance leadership are workload-dependent**. +Triton v3.6.0 expands those foundations with broader layouts and copies and initial multi-CTA/2-CTA Gluon support. “Initial” is deliberate: v3.7.0 contains follow-on end-to-end 2-CTA, multicast, and TMA changes. See [`doc-triton-3.6-blackwell`](../../sources/docs/triton-3.6-blackwell.md) and the compact release matrix in [`data/triton-3.6-evidence.md`](../../data/triton-3.6-evidence.md). -## Downstream Triton-on-Blackwell adoption (post-3.6 evidence) +## What downstream code establishes -The `evidence_basis` is anchored on: +- [`pr-vllm-34597`](../../sources/prs/vllm/PR-34597.md), pinned at `a1257fd1`, adds FP8 KV-cache handling to the Triton MLA decode backend. Its verbatim kernel contains `tl.dot`, but no target guard, Triton-version requirement, TCGen5/TMEM name, or emitted PTX. The primary PR specifically motivates the backend as the MLA option on SM120; it is not an SM100-only lowering demonstration. +- [`pr-vllm-29339`](../../sources/prs/vllm/PR-29339.md), pinned at `c17610e2`, gates MXFP4 `triton_kernels` dispatch to SM90 and SM100. It changes dispatch logic, not a kernel or compiler lowering. +- [`pr-sglang-21019`](../../sources/prs/sglang/PR-21019.md), pinned at `5bdc07d9`, provides a Triton GatedDeltaNet projection rearrangement using loads and stores, with no `tl.dot`. +- [`pr-sglang-22079`](../../sources/prs/sglang/PR-22079.md), pinned at `5638d40f`, provides an extend-attention Triton kernel with real `tl.dot` operations. The source does not contain an emitted-PTX witness for a particular MMA instruction. -- **`doc-triton-3.6-blackwell`** (`source_category: official-doc`) — verifies that Triton 3.6 ships native Blackwell lowering infrastructure (TMEM, tcgen05, warp_specialize plumbing, Gluon multi-CTA / 2CTA, tl.dot_scaled). This is the "infrastructure exists" half of AC-1.2. -- **`pr-vllm-34597`** (`source_category: upstream-code`) — **post-refresh primary anchor**: vLLM's "[Kernel] Add FP8 KV cache support to Triton MLA decode attention" PR (merged 2026-02-16, post-Triton-3.6.0 release; `architectures: [sm100]`; tags `attention`, `decode`, `fp8`, `mla`). Directly modifies actual Triton kernel files: `vllm/v1/attention/ops/triton_decode_attention.py` (the `@triton.jit`-decorated MLA decode kernel doing `tl.dot(q, k)` for attention scores, `tl.dot(qpe, kpe)` for positional contributions, and `acc += tl.dot(p, v)` for the output projection) and `vllm/v1/attention/backends/mla/triton_mla.py` (the backend wrapping it). The Triton kernel itself is shipped verbatim under [`artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/ops/triton_decode_attention.py`](../../artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/ops/triton_decode_attention.py) (756 lines, pinned at upstream SHA `a1257fd1`). This PR page is post-refresh — it is NOT in `data/refresh-cutoff.yaml::previous_pages_manifest`. This is the "downstream adoption is happening" half of AC-1.2 and the AC-1.1 "new tracked-repo PR page demonstrating a kernel" anchor. +These are verified downstream Triton examples. They must not be combined with the separate compiler-version evidence to infer an unobserved lowering. -Together the two anchors establish that **the 3.6+ Blackwell lowering infrastructure is real AND tracked downstream repos are landing real Triton matmul kernels for Blackwell production decode paths today** — which is the substance of the rewrite: the pre-3.6 "lacked-tcgen05 / lacked-TMEM" framing is no longer correct. +## Scoped ecosystem results -Two clarifications about what these anchors do NOT prove (since the verified-surface section above flags some shapes as needs-verification): +SGLang [`pr-sglang-5390`](../../sources/prs/sglang/PR-5390.md) reports 10,447.34 total tok/s for its CUTLASS MLA run and 8,227.35 total tok/s for its Triton run, 26.98% higher under the PR's recorded DeepSeek-R1, 3,000-prompt, TP8/DP8, float16, 1,000-input/1,000-output-token scope. This is one reported comparison, not a universal language ranking. -- The anchors do not prove that EVERY plain `tl.dot` kernel on SM100 emits `tcgen05.mma` PTX. They prove that real Triton kernels with `tl.dot` matmul are landing on Blackwell-only paths in tracked downstream repos (per `pr-vllm-34597`'s `triton_decode_attention.py` kernel) and that the 3.6 release added the infrastructure those kernels lower through. Whether the lowering automatically targets `tcgen05.mma` for every shape and dtype on every Blackwell SKU is a separate question. -- The anchors do not include explicit inspectable `tcgen05.mma` PTX dumps from a tracked-downstream merged PR. Such proof would be an even stronger anchor than what we have today; until one is found, the verified claim should be read as "Triton 3.6+ Blackwell is real and downstream-adopted", not "every Triton matmul on Blackwell is now optimal". +SGLang [`pr-sglang-21595`](../../sources/prs/sglang/PR-21595.md) changes the SM100 datacenter multimodal-attention default from `triton_attn` to FA4. That routing decision is likewise workload- and architecture-scoped. -Supplementary post-refresh anchor: [`pr-vllm-29339`](../../sources/prs/vllm/PR-29339.md) — vLLM bugfix that scopes the upstream `triton_kernels` library (the `triton-lang/triton/python/triton_kernels` collection shipped with Triton 3.6) to `[sm100, sm90]` for the MXFP4 quantization path. Modifies only a dispatch gate (`vllm/model_executor/layers/quantization/mxfp4.py`), which is why `pr-vllm-34597` is the primary anchor and `pr-vllm-29339` is supplementary. +The live [FlashInfer-Bench leaderboard](https://bench.flashinfer.ai/), retrieved 2026-08-08, reports these author rows across 660 workloads each: -Pre-refresh historical anchors (retained as supplementary context, not as AC-1.1 "new tracked-repo PR page" evidence on their own): - -- [`pr-sglang-22079`](../../sources/prs/sglang/PR-22079.md) — Gemma4 NVFP4 SGLang `extend_attention` Triton kernel doing actual `tl.dot(q,k)` / `tl.dot(p,v)` matmul on `[sm100, sm90]`, merged 2026-04-03. Strongest in-corpus example of a real `tl.dot` Triton matmul landing for SM100 post-3.6, but pre-refresh per `previous_pages_manifest`. -- [`pr-sglang-21019`](../../sources/prs/sglang/PR-21019.md) — Qwen3.5 GDN projection fused split/reshape/cat kernel merged 2026-03-20. `tl.load`/`tl.store` only (memory rearrangement, no `tl.dot`); demonstrates "Triton on SM100 post-3.6" but not the matmul lowering path. - -Caveat anchors `pr-sglang-5390`, `pr-sglang-21595`, and `pr-pytorch-175826` provide ecosystem context (CUTLASS still leads on peak; Blackwell defaults moved away from triton_attn for some workloads; CI moved to CUDA 13.0). - -## FlashInfer-Bench: AI-Generated Triton Performance - -The following table reflects benchmark snapshots from when the original page was written (Triton 3.5 era) and is preserved for historical reference. It does NOT reflect the 3.6+ tcgen05 path. - -| Model | Avg Speedup vs FlashInfer | Resolved % | -|---|---|---| +| Author | Average speedup | Resolved | +|---|---:|---:| | Gemini 2.5 Pro | 0.628x | 73.1% | | GPT-5 | 0.467x | 92.3% | | Claude Opus 4.1 | 0.456x | 73.1% | -A re-run on Triton 3.6 with FlashInfer-Bench is not yet available locally; future refresh rounds should update this table or remove it in favor of a pointer to the live leaderboard. +The leaderboard does not attach those rows to a Triton release or identify them as a Triton-only language subset. -## GatedDeltaNet Decode in Triton +## Launch overhead and CUDA Graphs -```python -@triton.jit -def gated_delta_net_decode( - Q, K, V, Gate, State, Output, - qk_dim: tl.constexpr, v_dim: tl.constexpr, d: tl.constexpr, -): - """Single-token decode: O(d^2) per token.""" - head_id = tl.program_id(0) - # Load recurrent state S: [qk_dim*d, v_dim*d] - s = tl.load(State + head_id * qk_dim * d * v_dim * d + offsets) - q = tl.load(Q + offsets) - k = tl.load(K + offsets) - v = tl.load(V + offsets) - g = tl.load(Gate + head_id) - - # Delta rule: S = g*S + k @ (v - S^T @ k)^T - sk = tl.dot(tl.trans(s), k) - delta_v = v - sk - s = g * s + tl.dot(k[:, None], delta_v[None, :]) - o = tl.dot(tl.trans(s), q) - - tl.store(State + offsets, s) - tl.store(Output + offsets, o) -``` - -## NSA Sparse Attention in Triton - -```python -@triton.jit -def sparse_attention_fwd(Q, K, V, Output, TopK_Indices, - block_size: tl.constexpr, topk: tl.constexpr): - """Attend to top-k sparse token blocks only.""" - qid = tl.program_id(0) - q = tl.load(Q + qid * d + tl.arange(0, d)) - acc = tl.zeros([d], dtype=tl.float32) - for i in range(topk): - bidx = tl.load(TopK_Indices + qid * topk + i) - k_block = tl.load(K + bidx * block_size * d + offsets) - v_block = tl.load(V + bidx * block_size * d + offsets) - scores = tl.dot(q[None, :], tl.trans(k_block)) - # softmax + accumulate... -``` - -## Pre-3.6 historical context +The CUDA Programming Guide explains that CPU setup and launch overhead can be significant for short kernels and that CUDA Graphs reduce repeated launch costs by preparing work in advance. At vLLM commit `a1257fd1`, `FULL_AND_PIECEWISE` is the v1 CUDA-graph default, and the GatedDeltaNet backend implements uniform-batch graph support plus decode-only full graph capture. This is a launch-path observation, independent of which MMA instruction a Triton kernel selects. -> The text in this subsection describes Triton 3.5 and earlier. It is preserved for historical accuracy and is NOT a current statement about Triton 3.6+ behavior. Do not cite it as current limitations. +## Provenance-pinned examples -Triton on Blackwell — Triton 3.5 and earlier: +Each linked file is stored verbatim under its bundle's recorded merge SHA: -1. **No direct tcgen05 access**: Triton compiler generates wgmma, not tcgen05. -2. **No TMEM**: accumulators stay in registers. -3. **CPU launch overhead**: impacts small-batch decode latency. -4. **Workaround**: CUDA graphs (vLLM default for GatedDeltaNet). +| File | Verified role | +|---|---| +| [`triton_decode_attention.py`](../../artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/ops/triton_decode_attention.py) | Triton MLA decode kernels with `tl.dot`; PR 34597 adds FP8 cache handling. | +| [`triton_mla.py`](../../artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/backends/mla/triton_mla.py) | Backend wrapper and supported FP8 cache dtypes. | +| [`format_conversion.py`](../../artifacts/prs/flashinfer/PR-1025/key-files/flashinfer/triton/format_conversion.py) | Triton FP8/FP16 format-conversion kernels. | +| [`norm.py`](../../artifacts/prs/sglang/PR-20910/key-files/python/sglang/jit_kernel/norm.py) | Triton normalization kernels. | +| [`gdn_fused_proj.py`](../../artifacts/prs/sglang/PR-21019/key-files/python/sglang/jit_kernel/triton/gdn_fused_proj.py) | Triton GatedDeltaNet projection rearrangement using loads/stores. | +| [`extend_attention.py`](../../artifacts/prs/sglang/PR-22079/key-files/python/sglang/srt/layers/attention/triton_ops/extend_attention.py) | Triton extend-attention kernels containing `tl.dot`. | -These four bullets describe the world before Triton 3.6.0 landed (`2026-01-21`). The first two are no longer correct on 3.6+; see the "Triton 3.6+ Blackwell path" subsection above. CPU launch overhead and CUDA-graph workarounds remain workload-relevant on small-batch decode paths regardless of compiler era. +The complete tracked PR universe and its captured/skipped flags are recorded in [`data/triton-universe.yaml`](../../data/triton-universe.yaml); this page does not duplicate that changing count. -## Blackwell Triton Examples (verbatim upstream code shipped locally) +This verbatim excerpt from the pinned vLLM decode kernel demonstrates the page's deliberately limited downstream claim—FP8 rescaling followed by `tl.dot`, without proving a particular emitted MMA instruction: -The following Triton files ship **verbatim** under `artifacts/prs/` (pinned at each PR's merge SHA). Each PR is in the `triton-in-policy` capture lane defined by `data/inclusion-policy.yaml` — i.e., SM100-integration, memory-bound, or backend-fallback scope (not pure Hopper Triton). - -| File | Purpose | PR | -|---|---|---| -| [`artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/ops/triton_decode_attention.py`](../../artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/ops/triton_decode_attention.py) | **Post-refresh AC-1.1 primary anchor**: Triton MLA decode attention kernel — `@triton.jit`-decorated, doing `tl.dot(q, k)` / `tl.dot(qpe, kpe)` / `acc += tl.dot(p, v)` matmul on `[sm100]` with FP8 KV cache support added (`pr-vllm-34597`, merged 2026-02-16). | vllm#34597 | -| [`artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/backends/mla/triton_mla.py`](../../artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/backends/mla/triton_mla.py) | MLA backend wrapping the Triton decode kernel above — adds FP8 / FP8-E4M3 to `TritonMLABackend.supported_kv_cache_dtypes`. | vllm#34597 | -| [`artifacts/prs/flashinfer/PR-1025/key-files/flashinfer/triton/format_conversion.py`](../../artifacts/prs/flashinfer/PR-1025/key-files/flashinfer/triton/format_conversion.py) | FP8 / FP16 format conversion Triton kernels for FlashInfer | flashinfer#1025 | -| [`artifacts/prs/sglang/PR-20910/key-files/python/sglang/jit_kernel/norm.py`](../../artifacts/prs/sglang/PR-20910/key-files/python/sglang/jit_kernel/norm.py) | Normalization kernels (memory-bound SM100 Triton) | sglang#20910 | -| [`artifacts/prs/sglang/PR-21019/key-files/python/sglang/jit_kernel/triton/gdn_fused_proj.py`](../../artifacts/prs/sglang/PR-21019/key-files/python/sglang/jit_kernel/triton/gdn_fused_proj.py) | GatedDeltaNet fused projection (linear-attention) Triton kernel — `tl.load`/`tl.store` only; pre-refresh historical anchor (`pr-sglang-21019`, merged 2026-03-20). | sglang#21019 | -| [`artifacts/prs/sglang/PR-22079/key-files/python/sglang/srt/layers/attention/triton_ops/extend_attention.py`](../../artifacts/prs/sglang/PR-22079/key-files/python/sglang/srt/layers/attention/triton_ops/extend_attention.py) | Gemma4 NVFP4 attention Triton kernel — actual `tl.dot(q,k)` / `tl.dot(p,v)` matmul on `[sm100, sm90]`; pre-refresh historical anchor (`pr-sglang-22079`, merged 2026-04-03). | sglang#22079 | - -The current AC-1.1 **post-refresh primary upstream-code anchor** is `pr-vllm-34597` (above), with the actual Triton decode-attention kernel shipped verbatim. Supplementary post-refresh anchor: `pr-vllm-29339` ([`sources/prs/vllm/PR-29339.md`](../../sources/prs/vllm/PR-29339.md)), a vLLM bugfix scoping the Triton 3.6 `triton_kernels` library to `[sm100, sm90]` for the MXFP4 quantization path (no artifact bundle because the change is dispatch-gate-only). - -The full 42-PR universe is enumerated in `data/triton-universe.yaml`. Entries marked `captured: false` do not ship locally because they fall outside the three in-policy sub-scopes (see the policy file for reasons). +```python +if k.dtype.is_fp8(): + k = (k.to(tl.float32) * ks).to(q.dtype) +qk = tl.dot(q, k.to(q.dtype)) +if BLOCK_DPE > 0: + offs_buf_kpe = kv_loc[None, :] * stride_buf_kbs +``` diff --git a/wiki/migration/register-to-tmem.md b/wiki/migration/register-to-tmem.md index 46f837c7b..eaf74d97a 100644 --- a/wiki/migration/register-to-tmem.md +++ b/wiki/migration/register-to-tmem.md @@ -7,455 +7,107 @@ to_arch: sm100 tags: [tmem, tcgen05] related: [hw-tmem, hw-tcgen05-mma, pattern-register-pressure] sources: [doc-nvidia-tuning-guide, blog-tcgen05-tutorial, pr-vllm-22738] -blackwell_relevance: "Hopper stores MMA accumulators in registers (high pressure). Blackwell uses dedicated TMEM (256KB), eliminating register pressure for accumulators." -confidence: source-reported +blackwell_relevance: "Hopper WGMMA D fragments reside in per-thread registers; Blackwell tcgen05 D resides in dynamically allocated TMEM, changing accumulator lifetime and resource tradeoffs." +confidence: verified +evidence_basis: + - source_id: doc-nvidia-tuning-guide + evidence_type: official-doc + - source_id: pr-vllm-22738 + evidence_type: upstream-code reproducibility: pseudocode --- # Register Accumulators to TMEM -## Overview +## What actually changes -On Hopper (SM90), `wgmma` stores MMA accumulators in **registers**. A single m64xn256xk16 BF16 wgmma requires 128 FP32 registers per thread in the warpgroup for the accumulator alone. This extreme register pressure limits tile sizes, reduces occupancy, and forces complex register management. +For Hopper `wgmma.mma_async.m64nNk16` with an FP32 accumulator, each warpgroup thread holds `N/2` FP32 D registers. At `N=256`, that is 128 registers per thread for D. WGMMA updates that register vector asynchronously; the program uses the WGMMA fence, commit-group, and wait-group mechanisms before dependent use. -On Blackwell (SM100), `tcgen05.mma` stores accumulators in **Tensor Memory (TMEM)** -- a dedicated 256KB per-SM memory. This eliminates accumulator register pressure entirely, freeing registers for data movement, epilogue computation, and enabling larger tile sizes. +For `sm_100a`, tcgen05 D is addressed in Tensor Memory rather than supplied as a per-thread register vector. The CTA-visible TMEM structure has 128 lanes by 512 columns of 32-bit cells, or 256 KiB when fully allocated. Allocation is dynamic in columns. -## The Register Pressure Problem on Hopper +This removes the resident D vector from the ordinary register file. It does not remove all register pressure: descriptors, addresses, loop state, pipeline state, and each TMEM-to-register epilogue batch still need registers. Both Hopper and compute-capability-10.0 Blackwell have 64K 32-bit registers per SM and at most 255 registers per thread. Occupancy, spilling, and practical tile size therefore remain properties of the compiled kernel and launch configuration. -### Register Budget Analysis +## Storage and lifecycle contrast -Each SM90 SM has **65,536 registers** (256KB). A typical GEMM warpgroup needs: - -``` -Hopper wgmma m64xn256xk16 BF16 register budget (per thread): - - Accumulator: 128 FP32 registers (m64 x n256 / 128 threads) - A operand: 16 registers (via ldmatrix) - B descriptor: 2 registers - Loop variables: 8 registers - SMEM pointers: 4 registers - TMA state: 6 registers - Misc (indexing): 10 registers - ───────────────────────────── - Total per thread: ~174 registers - - Per warpgroup (128 threads): 174 * 128 = 22,272 registers - Per SM (65,536 regs): max 2 warpgroups = 2 CTAs at this tile size - - Occupancy: 2 CTAs per SM (limited by registers) -``` - -If the tile is larger (e.g., m64xn256xk32 with double-buffered accumulators), register pressure becomes even worse: - -``` -Double-buffered accumulator: 256 registers per thread -Total per thread: ~300 registers -Per warpgroup: 38,400 registers -Per SM: max 1 warpgroup -> 1 CTA per SM - -Occupancy: 1 CTA per SM (severe underutilization) -``` - -### Register Spilling on Hopper - -When register pressure exceeds the budget, the compiler spills registers to local memory (L1 cache backed by GMEM). This is catastrophic for performance: - -``` -Register access: ~4 cycles -L1 local access: ~30 cycles -L2 spill access: ~200 cycles - -A single spilled accumulator register accessed every MMA iteration -can cost 200 cycles * K_tiles additional latency. -``` - -## Blackwell TMEM Solution - -### TMEM Budget Analysis - -``` -Blackwell tcgen05 m128xn256xk16 BF16 register budget (per thread): - - Accumulator: 0 registers (stored in TMEM!) - SMEM descriptors: 4 registers - Loop variables: 8 registers - TMA state: 6 registers - TMEM address: 1 register - Misc (indexing): 10 registers - ───────────────────────────── - Total per thread: ~29 registers - - Per CTA (256 threads): 29 * 256 = 7,424 registers - Per SM (65,536 regs): max 8 CTAs (register-wise) - Actual limit: TMEM capacity (512 columns) - - TMEM usage per CTA: 256 columns for 128x256 tile - Max CTAs per SM: 512 / 256 = 2 CTAs (TMEM-limited, not register-limited) -``` - -The key insight: **registers are no longer the bottleneck**. TMEM capacity becomes the binding constraint for occupancy, and the freed registers enable complex epilogues without spilling. - -## Migration: Accumulator Lifecycle - -### Hopper: Register-Based Lifecycle - -```cuda -// HOPPER: Accumulator lives and dies in registers -__global__ void hopper_kernel(/* ... */) { - // 1. Declare accumulator in registers - float acc[4][32]; // 128 registers per thread! - - // 2. Zero-initialize - #pragma unroll - for (int i = 0; i < 4; ++i) - for (int j = 0; j < 32; ++j) - acc[i][j] = 0.0f; - - // 3. Mainloop: wgmma accumulates into registers - for (int k = 0; k < K_tiles; ++k) { - // Load A via ldmatrix - uint32_t a_frag[4]; - ldmatrix(a_frag, smem_a + k * TILE_K); - - // wgmma: reads A from registers, B from SMEM descriptor - // Accumulates into acc[] registers - wgmma_m64n256k16(acc, a_frag, smem_b_desc); - wgmma_commit(); - wgmma_wait(); - } - - // 4. Epilogue: acc is directly in registers -- fast access - // But: if epilogue needs temporary storage, registers are exhausted - #pragma unroll - for (int i = 0; i < 4; ++i) { - for (int j = 0; j < 32; ++j) { - float val = acc[i][j]; - val += bias[col_base + j]; // bias add - val = fmaxf(val, 0.0f); // ReLU - // Need to convert and store -- no registers left for temp! - C[row * N + col_base + j] = __float2half(val); - } - } - - // 5. Accumulator freed implicitly when CTA exits -} -``` - -### Blackwell: TMEM-Based Lifecycle - -```cuda -// BLACKWELL: Accumulator lives in TMEM -__global__ void blackwell_kernel(/* ... */) { - // 1. Allocate TMEM (explicit, must be done once) - __shared__ uint32_t s_tmem_acc; - if (threadIdx.x == 0) { - uint32_t smem_addr = - static_cast(__cvta_generic_to_shared(&s_tmem_acc)); - asm volatile( - "tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;" - :: "r"(smem_addr), "r"(256) - ); - } - __syncthreads(); - uint32_t tmem_acc = s_tmem_acc; - - // 2. Zero-initialize TMEM - for (int c = 0; c < 256; c += 4) { - asm volatile( - "tcgen05.st.sync.aligned.32x1b.x4.b32 [%0], {%1,%2,%3,%4};" - : : "r"(tmem_acc + c), - "f"(0.f), "f"(0.f), "f"(0.f), "f"(0.f) - ); - } - - // 3. Mainloop: tcgen05 accumulates into TMEM - for (int k = 0; k < K_tiles; ++k) { - // NO ldmatrix -- tcgen05 reads directly from SMEM - if (threadIdx.x == 0) { - asm volatile( - "tcgen05.mma.cta_group::1.kind::f16 " - "[%0], %1, %2, %3, 1;" - : : "r"(tmem_acc), "l"(desc_a), "l"(desc_b), "r"(0) - ); - } - // NO commit/wait -- fully async - } - - // 4. Fence before reading - asm volatile("tcgen05.fence::before_thread_sync;"); - __syncthreads(); - - // 5. Epilogue: read from TMEM to registers (in small batches) - // PLENTY of registers available for epilogue temporaries! - for (int c = 0; c < 256; c += 4) { - float4 vals; - asm volatile( - "tcgen05.ld.sync.aligned.32x1b.x4.b32 {%0,%1,%2,%3}, [%4];" - : "=f"(vals.x), "=f"(vals.y), "=f"(vals.z), "=f"(vals.w) - : "r"(tmem_acc + c) - ); - - // Apply epilogue ops with register headroom - vals.x += bias[col_base + c]; - vals.y += bias[col_base + c + 1]; - vals.z += bias[col_base + c + 2]; - vals.w += bias[col_base + c + 3]; - - vals.x = fmaxf(vals.x, 0.0f); // ReLU - vals.y = fmaxf(vals.y, 0.0f); - vals.z = fmaxf(vals.z, 0.0f); - vals.w = fmaxf(vals.w, 0.0f); - - // Vectorized store -- registers available for conversion - half2 h01 = __floats2half2_rn(vals.x, vals.y); - half2 h23 = __floats2half2_rn(vals.z, vals.w); - *reinterpret_cast(&C[row * N + col_base + c]) = h01; - *reinterpret_cast(&C[row * N + col_base + c + 2]) = h23; - } - - // 6. Deallocate TMEM (explicit, MUST be done in persistent kernels) - if (threadIdx.x == 0) { - asm volatile( - "tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;" - : : "r"(tmem_acc), "r"(256) - ); - } -} -``` - -## Impact on Kernel Design - -### Tile Size Freedom - -With accumulators in TMEM, the tile size is no longer constrained by register count: - -```cuda -// Hopper: large tiles cause spilling -// m64xn256 uses 128 acc registers/thread -> barely fits -// m128xn256 would need 256 acc registers/thread -> spills guaranteed - -// Blackwell: tile size limited by TMEM columns (512) and SMEM, not registers -// m128xn256: 256 TMEM cols, 0 acc registers -> fits easily -// m128xn512: 512 TMEM cols, 0 acc registers -> uses full TMEM budget -// m256xn256 (2-SM): 256 TMEM cols/SM, 0 acc registers -> fits with cooperative -``` - -### Epilogue Complexity - -On Hopper, complex epilogues (bias + activation + quantization + scaling) often spill because registers are already exhausted by the accumulator. On Blackwell, the epilogue has the full register file available: - -```cuda -// Blackwell epilogue: full register file available -__device__ void rich_epilogue(uint32_t tmem_acc, float* output, - const float* bias, const float* scale, - int M, int N) { - // Read TMEM in chunks - for (int c = 0; c < 256; c += 8) { - // Load 8 values from TMEM (8 registers -- trivial) - float v[8]; - tmem_load_f32x4(tmem_acc + c, &v[0]); - tmem_load_f32x4(tmem_acc + c + 4, &v[4]); - - // Apply bias (8 registers for bias values) - float b[8]; - load_global_f32x4(&b[0], bias + c); - load_global_f32x4(&b[4], bias + c + 4); - - #pragma unroll - for (int i = 0; i < 8; ++i) v[i] += b[i]; - - // Apply GELU activation (needs temp registers for polynomial) - #pragma unroll - for (int i = 0; i < 8; ++i) v[i] = gelu(v[i]); - - // Apply per-channel scale - float s[8]; - load_global_f32x4(&s[0], scale + c); - load_global_f32x4(&s[4], scale + c + 4); - - #pragma unroll - for (int i = 0; i < 8; ++i) v[i] *= s[i]; - - // Quantize to FP8 for next layer - uint8_t q[8]; - #pragma unroll - for (int i = 0; i < 8; ++i) q[i] = float_to_e4m3(v[i]); - - // Store quantized output - store_global_u8x8(output + c, q); - - // Total temp registers used: ~24 -- NO SPILLING - // On Hopper this epilogue would need 24 + 128 (acc) = 152 per thread - } -} -``` - -### Overlapped Epilogue and MMA - -The most powerful pattern enabled by TMEM: overlapping the current tile's epilogue with the next tile's MMA computation. This is impossible with register accumulators because the accumulator registers are still in use. - -```cuda -__global__ void overlapped_gemm(/* ... */) { - // Two TMEM accumulator buffers - uint32_t tmem_a = tmem_alloc(256); - uint32_t tmem_b = tmem_alloc(256); - uint32_t* tmem_cur = &tmem_a; - uint32_t* tmem_nxt = &tmem_b; - - // Compute first tile into tmem_a - compute_tile(*tmem_cur, tile_0); - fence_and_sync(); - - for (int t = 1; t < num_tiles; ++t) { - // START: next tile MMA into tmem_nxt (background, async) - zero_tmem(*tmem_nxt); - start_mma(*tmem_nxt, tile_t); - - // SIMULTANEOUSLY: epilogue of current tile from tmem_cur - // MMA is running in the background while we read the old accumulator! - epilogue_from_tmem(*tmem_cur, output_tile[t-1]); - - // Wait for MMA to complete - fence_and_sync(); - - // Swap buffers - uint32_t* tmp = tmem_cur; - tmem_cur = tmem_nxt; - tmem_nxt = tmp; - } - - // Final epilogue - epilogue_from_tmem(*tmem_cur, output_tile[num_tiles-1]); - - tmem_dealloc(tmem_a, 256); - tmem_dealloc(tmem_b, 256); -} -``` +| Concern | Hopper WGMMA | Blackwell tcgen05 | +|---|---|---| +| Resident D | Per-thread register fragment | TMEM region addressed by `taddr` | +| Example FP32 D cost | `N/2` registers/thread; 128 at `N=256` | No resident per-thread D vector | +| First accumulation | Set WGMMA `scale-d` false to compute `D=A*B`, or initialize/use D when accumulation is intended | Set `enable-input-d` false to compute `D=A*B`, or explicitly initialize D when the algorithm needs another value | +| Compute completion | WGMMA commit/wait group | `tcgen05.commit` to an mbarrier, then wait before a consumer reads/reuses D | +| Epilogue access | Use the thread's D registers | Collective `tcgen05.ld`, then `tcgen05.wait::ld` before consuming result registers | +| Cleanup | Register lifetime ends with the thread | Matching collective `tcgen05.dealloc` before kernel exit | +| Double buffering | Two simultaneously live D fragments consume two register fragments | Two live outputs consume disjoint TMEM columns inside the allocation | -## FlashAttention Case Study +The table describes storage contracts, not an occupancy prediction. Instruction shape is also not necessarily the same as a composed CTA tile. -FlashAttention is a prime example where the register-to-TMEM migration unlocks major gains. +## Migration lifecycle -### Hopper FlashAttention Register Pressure +1. Choose one CTA-group mode for the kernel. For allocation, `nCols` is a power of two in `[32, 512]`; allocations are column-granular and cover all 128 lanes. +2. Have every lane of one designated warp execute the same `.cta_group::1` allocation. Synchronize before other threads read the 32-bit `taddr` written to shared memory. Two-CTA mode requires one warp in each live peer CTA. +3. Initialize and publish the barriers used by TMA and tcgen05. If the first MMA should compute only `A*B`, supply a false `enable-input-d` predicate instead of assuming a mandatory TMEM zero-store pass. +4. Issue MMA from the permitted elected thread with valid A/B descriptors, instruction descriptor, predicate, and lifetimes. +5. Attach completion of the relevant tcgen05 work to an mbarrier with `tcgen05.commit`. A fence controls execution ordering but is not a completion wait. +6. After the completion handoff, participating epilogue lanes execute a legal `tcgen05.ld` shape and `tcgen05.wait::ld` before using the loaded registers. +7. After all readers finish, every lane of the designated warp executes the matching deallocation on every kernel exit path. Allocation/deallocation address, column count, and CTA-group mode must agree. -``` -FlashAttention on Hopper needs registers for: - - QK^T accumulator: 64 registers (m64xn64 attention scores) - - PV accumulator: 128 registers (m64xn256 output) - - Softmax state: 4 registers (rowmax, rowsum) - - Q fragment: 16 registers - - K fragment: 16 registers - - V fragment: 16 registers - - Loop state: 10 registers - ────────────────────────────── - Total: ~254 registers per thread +Representative PTX ISA 9.0 forms are shown below. They omit declarations, descriptor construction, collective control flow, barriers, and inline-assembly constraints, so they are not a standalone kernel. - Available: 256 registers/thread at 1 CTA/SM occupancy - Margin: 2 registers (!!) - - Result: compiler must spill, or tile sizes must shrink +```ptx +tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [saddr], nCols; +tcgen05.mma.cta_group::1.kind::f16 [taddr], a_desc, b_desc, idesc, p; +tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cta.b64 [mbar]; +tcgen05.ld.sync.aligned.32x32b.x1.b32 {r0}, [taddr]; +tcgen05.wait::ld.sync.aligned; +tcgen05.dealloc.cta_group::1.sync.aligned.b32 taddr, nCols; ``` -### Blackwell FlashAttention with TMEM +## Overlap without invented guarantees -``` -FlashAttention on Blackwell: - - QK^T accumulator: 0 registers (TMEM buffer 1, 64 cols) - - PV accumulator: 0 registers (TMEM buffer 2, 256 cols) - - Softmax state: 4 registers (rowmax, rowsum) - - SMEM descriptors: 6 registers - - Loop state: 10 registers - ────────────────────────────── - Total: ~20 registers per thread +TMEM makes it practical to keep multiple output regions independently addressable while different warp roles issue MMA and drain a completed region. Safe overlap needs three distinct proofs: - Available: 256 registers/thread - Margin: 236 registers -- massive headroom +- compute completion occurs before an epilogue loads a region; +- all epilogue loads complete before that region is overwritten or deallocated; +- producer/consumer state and phase cannot alias a different pipeline stage. - Result: Can use for ping-pong scheduling, software exp emulation, - larger tiles, more pipeline stages -``` +TMEM is not the only way to overlap tensor-core and non-matmul work. FlashAttention-3 already uses warp specialization and matmul/softmax interleaving on Hopper with register accumulators. The migration benefit is the different storage and lifetime tradeoff, not a new theorem that overlap was previously impossible. -FlashAttention-4 exploits this headroom for software-emulated exponentials (distributing `2^x` across FMA units instead of waiting for the SFU), achieving **1605 TFLOPS on B200** at 71% utilization. +## FlashAttention-4 case study -## Common Mistakes During Migration +FlashAttention-4's first-party paper places score/output accumulators in TMEM and assigns MMA, softmax, and correction work to specialized warp roles. The accompanying author post describes software-selected polynomial `exp2` work on CUDA cores and reports up to 1605 TFLOP/s on B200 BF16, labeled 71%. That is a source-reported maximum, not evidence for a universal register count, occupancy, or migration speedup. -### Mistake 1: Treating TMEM Like Registers +## Evidence-driven retuning -```cuda -// WRONG: Trying to use TMEM values directly in expressions -float result = tmem_acc[row][col] + bias; // TMEM is not directly addressable! +Do not translate an instruction shape directly into a required CTA tile or assume that moving D guarantees no spills. For each concrete kernel: -// CORRECT: Load from TMEM to register, then compute -float val = tmem_load_f32(tmem_col); -float result = val + bias; -``` +1. Record the compiler's registers/thread, spill loads/stores, static/dynamic shared memory, barriers, threads/CTA, and cluster shape. +2. Compute occupancy with the CUDA occupancy APIs for the actual launch; do not equate one warpgroup with one CTA. +3. Sweep legal instruction descriptors, CTA tiles, pipeline depth, TMEM column partitioning, and epilogue batch width. +4. Benchmark identical shapes, datatypes, outputs, synchronization, warmup, and trial statistics. A fixed latency such as “420 cycles per TMEM load” is not an ISA guarantee. +5. Inspect generated PTX/SASS to confirm that the intended tcgen05 shapes, waits, and no unexpected spills are present. -### Mistake 2: Forgetting TMEM Latency +## Common migration errors -```cuda -// WRONG: Fine-grained TMEM access in a tight loop (high latency) -for (int i = 0; i < 256; ++i) { - float v = tmem_load_f32(tmem_acc + i); // 420 cycle latency each! - output[i] = v; -} +- TMEM is not an ordinary CUDA pointer space. Load through a legal collective tcgen05 copy operation before scalar arithmetic. +- Do not issue collective allocation, load/store, or deallocation from lane 0 alone. +- Do not substitute `tcgen05.fence` or `__syncthreads()` for asynchronous MMA completion. +- Do not zero TMEM unconditionally when a false `enable-input-d` predicate provides the intended first-write semantics. +- Do not treat TMEM-resident D as the entire kernel's register allocation; the epilogue can still spill. +- Do not describe missing deallocation as a defined recoverable “leak.” The normative rule is explicit matching deallocation before exit. -// CORRECT: Vectorized loads to amortize latency -for (int i = 0; i < 256; i += 4) { - float4 v = tmem_load_f32x4(tmem_acc + i); // Single 420-cycle access - output[i] = v.x; - output[i+1] = v.y; - output[i+2] = v.z; - output[i+3] = v.w; -} -``` +## Primary references -### Mistake 3: Not Re-tuning Tile Sizes +- [PTX ISA 9.0 WGMMA register fragments](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16) +- [PTX ISA 9.0 Tensor Memory](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory) +- [PTX ISA 9.0 tcgen05 allocation](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc) +- [PTX ISA 9.0 tcgen05 MMA](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma) +- [CUDA 13.0.2 Hopper Tuning Guide](https://docs.nvidia.com/cuda/archive/13.0.2/hopper-tuning-guide/index.html#occupancy) +- [CUDA 13.0.2 Blackwell Tuning Guide](https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy) +- [FlashAttention-3 paper](https://arxiv.org/abs/2407.08608) +- [FlashAttention-4 paper v1](https://arxiv.org/abs/2603.05451v1) -```cuda -// Hopper: tile size was m64xn128 to avoid register spilling -// Migrating to Blackwell: keep same tile = leaving performance on the table +## Related -// Blackwell should use at minimum m128xn256 (1-SM) or m256xn256 (2-SM) -// The freed registers allow larger tiles without any spilling risk -``` - -### Mistake 4: TMEM Leak in Persistent Kernels - -```cuda -// WRONG: Forgetting to dealloc in a persistent kernel -__global__ void persistent_kernel(/* ... */) { - uint32_t tmem = tmem_alloc(256); // Allocated once - - while (has_work()) { - compute_tile(tmem); - // BUG: If the kernel exits early (e.g., error path), - // TMEM is leaked. Next CTA on this SM cannot allocate. - } - // Missing: tmem_dealloc(tmem, 256); -} - -// CORRECT: Always dealloc, even on error paths -__global__ void persistent_kernel(/* ... */) { - uint32_t tmem = tmem_alloc(256); - - while (has_work()) { - compute_tile(tmem); - } - - tmem_dealloc(tmem, 256); // Always reached -} -``` - -## Summary: What Changes, What Stays - -| Aspect | Register (Hopper) | TMEM (Blackwell) | -|---|---|---| -| Accumulator declaration | `float acc[N]` in registers | `tmem_alloc(cols)` | -| Zero-initialization | Loop over register array | `tcgen05.st` zero pattern | -| MMA accumulation | Implicit (wgmma writes regs) | Implicit (tcgen05 writes TMEM) | -| Reading results | Direct register access | `tcgen05.ld` to register | -| Cleanup | Implicit (CTA exit) | Explicit `tcgen05.dealloc` | -| Register pressure | ~128-256 regs for accumulator | ~0 regs for accumulator | -| Epilogue headroom | Minimal (spill risk) | Ample (full register file) | -| Double-buffering | Doubles register pressure (2x acc) | Uses 2 TMEM regions (no reg impact) | -| Max practical tile | m64xn256 (limited by registers) | m128xn256 or m256xn256 (limited by TMEM cols) | +- [Tensor Memory](../hardware/tmem.md) — allocation, addressing, and access constraints +- [tcgen05 MMA](../hardware/tcgen05-mma.md) — exact operation and completion semantics +- [register pressure](../patterns/register-pressure.md) — diagnosis and compiler-resource evidence diff --git a/wiki/migration/wgmma-to-tcgen05.md b/wiki/migration/wgmma-to-tcgen05.md index 3d9e43059..627affb56 100644 --- a/wiki/migration/wgmma-to-tcgen05.md +++ b/wiki/migration/wgmma-to-tcgen05.md @@ -6,378 +6,72 @@ from_arch: sm90 to_arch: sm100 tags: [tcgen05, wgmma, tmem] related: [hw-tcgen05-mma, hw-tmem, technique-warp-specialization] -sources: [doc-nvidia-tuning-guide, blog-tcgen05-tutorial, blog-colfax-cutlass] -blackwell_relevance: "Core MMA instruction change. wgmma (warp-group, register accumulators) replaced by tcgen05 (single-thread, TMEM accumulators)." -confidence: source-reported -reproducibility: pseudocode +sources: [doc-ptx-isa-sm100, pr-cutlass-2139, blog-tcgen05-tutorial] +blackwell_relevance: "WGMMA uses warpgroup issue and register accumulators; tcgen05 uses single-thread MMA issue and TMEM accumulators." +confidence: verified +reproducibility: snippet +evidence_basis: + - source_id: doc-ptx-isa-sm100 + evidence_type: official-doc + - source_id: pr-cutlass-2139 + evidence_type: upstream-code --- # Migrating from wgmma to tcgen05 -## Overview +## What actually changes -This guide covers the migration from Hopper's `wgmma.mma_async` (SM90) to Blackwell's `tcgen05.mma` (SM100). This is the single most impactful change when porting kernels from H100/H200 to B200. The two instructions differ in: +Port the programming model, not just the opcode: -- **Issuing model**: warpgroup (128 threads) vs single thread -- **Accumulator storage**: registers vs TMEM -- **Operand loading**: ldmatrix + registers vs direct SMEM access -- **Synchronization**: warpgroup barriers vs async fences -- **Available data types**: FP8 (Hopper) vs FP4/FP6/FP8 with block scaling (Blackwell) - -## Migration Checklist - -``` -[ ] Replace wgmma.mma_async with tcgen05.mma -[ ] Move accumulators from registers to TMEM (alloc/dealloc) -[ ] Remove ldmatrix operations (tcgen05 reads directly from SMEM) -[ ] Change SMEM swizzle from 64B to 128B -[ ] Replace warpgroup commit/wait with tcgen05 fences -[ ] Update warp specialization roles (fewer warps needed for MMA) -[ ] Update tile sizes (128xN -> consider 256xN with 2-SM) -[ ] Add TMEM lifecycle management (alloc at start, dealloc at end) -[ ] Update epilogue to read from TMEM instead of registers -``` - -## Side-by-Side Comparison - -### Hopper Kernel Structure (SM90) - -```cuda -// SM90 GEMM kernel using wgmma -__global__ void hopper_gemm( - const half* A, const half* B, half* C, - int M, int N, int K -) { - extern __shared__ char smem[]; - half* smem_a = reinterpret_cast(smem); - half* smem_b = smem_a + TILE_M * TILE_K; - - // Register accumulators -- HIGH REGISTER PRESSURE - float acc[MMA_M_FRAGS][MMA_N_FRAGS]; // 128+ registers! - for (int i = 0; i < MMA_M_FRAGS; ++i) - for (int j = 0; j < MMA_N_FRAGS; ++j) - acc[i][j] = 0.0f; - - // Mainloop - for (int k = 0; k < K / TILE_K; ++k) { - // Load A to SMEM (TMA or cp.async) - tma_load(smem_a, A, tile_m, k); - tma_load(smem_b, B, k, tile_n); - - // Load A fragment from SMEM to registers via ldmatrix - uint32_t a_frag[4]; - asm volatile( - "ldmatrix.sync.aligned.m8n8.x4.shared.b16 " - "{%0,%1,%2,%3}, [%4];" - : "=r"(a_frag[0]), "=r"(a_frag[1]), - "=r"(a_frag[2]), "=r"(a_frag[3]) - : "r"(smem_a_addr) - ); - - // Issue wgmma -- ALL 128 THREADS IN WARPGROUP participate - asm volatile( - "wgmma.mma_async.sync.aligned.m64n256k16.f32.bf16.bf16 " - "{%0, %1, %2, %3, %4, %5, %6, %7, ...}, " // acc registers - "{%N, %N+1, %N+2, %N+3}, " // A in registers - "desc_b, ..." // B descriptor - : "+f"(acc[0][0]), "+f"(acc[0][1]), ... - : "r"(a_frag[0]), ... - ); - - // Commit and wait for warpgroup - asm volatile("wgmma.commit_group.sync.aligned;"); - asm volatile("wgmma.wait_group.sync.aligned 0;"); - } - - // Epilogue: accumulators already in registers - // Write directly to GMEM (or through SMEM for vectorized stores) - for (int i = 0; i < MMA_M_FRAGS; ++i) - for (int j = 0; j < MMA_N_FRAGS; ++j) - C[row + i][col + j] = (half)acc[i][j]; -} -``` - -### Blackwell Kernel Structure (SM100) - -```cuda -// SM100 GEMM kernel using tcgen05 -__global__ void blackwell_gemm( - const half* A, const half* B, half* C, - int M, int N, int K -) { - extern __shared__ char smem[]; - half* smem_a = reinterpret_cast(smem); - half* smem_b = smem_a + TILE_M * TILE_K; - - // TMEM accumulator -- NO REGISTER PRESSURE - __shared__ uint32_t s_tmem_acc; - if (threadIdx.x == 0) { - uint32_t smem_addr = - static_cast(__cvta_generic_to_shared(&s_tmem_acc)); - asm volatile( - "tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;" - :: "r"(smem_addr), "r"(256) // 256 columns for 128x256 tile - ); - } - __syncthreads(); - uint32_t tmem_acc = s_tmem_acc; - - // Zero TMEM accumulator - tmem_zero(tmem_acc, 256); - - // Mainloop - for (int k = 0; k < K / TILE_K; ++k) { - // TMA load with 128B swizzle (mandatory for tcgen05) - tma_load_128b_swizzle(smem_a, A, tile_m, k); - tma_load_128b_swizzle(smem_b, B, k, tile_n); - wait_barrier(); - - // NO ldmatrix -- tcgen05 reads directly from SMEM - // SINGLE THREAD issues MMA (not warpgroup) - if (threadIdx.x == 0) { - asm volatile( - "tcgen05.mma.cta_group::1.kind::f16 " - "[%0], %1, %2, %3, 1;" - : - : "r"(tmem_acc), - "l"(make_smem_desc_128b(smem_a)), - "l"(make_smem_desc_128b(smem_b)), - "r"(0) - ); - } - // NO commit/wait -- fully async, fence-based - } - - // Fence before reading TMEM (replaces wgmma.wait_group) - asm volatile("tcgen05.fence::before_thread_sync;"); - __syncthreads(); - - // Epilogue: read from TMEM, then write to GMEM - float4 acc_vals = tmem_load_f32x4(tmem_acc + col_offset); - // Apply bias, activation, etc. - store_output(acc_vals, C, row, col); - - // Deallocate TMEM (no equivalent needed on Hopper) - if (threadIdx.x == 0) { - asm volatile( - "tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;" - : - : "r"(tmem_acc), "r"(256) - ); - } -} -``` - -## Step-by-Step Migration - -### Step 1: Replace Accumulator Storage - -**Before (Hopper):** Accumulators live in registers. - -```cuda -// Hopper: 128 FP32 registers for a 64x256 accumulator fragment -float acc[4][32]; // Per-thread fragment of the warpgroup accumulator -``` - -**After (Blackwell):** Accumulators live in TMEM. - -```cuda -// Blackwell: TMEM address replaces register array -uint32_t tmem_acc = tmem_alloc(256); // 128 rows x 256 cols in TMEM -``` - -### Step 2: Remove ldmatrix - -**Before (Hopper):** Load A operand from SMEM to registers. - -```cuda -// Hopper: ldmatrix loads 8x8 matrix fragments into registers -uint32_t a_frag[4]; -asm volatile( - "ldmatrix.sync.aligned.m8n8.x4.shared.b16 " - "{%0,%1,%2,%3}, [%4];" - : "=r"(a_frag[0]), "=r"(a_frag[1]), - "=r"(a_frag[2]), "=r"(a_frag[3]) - : "r"(smem_addr) -); -``` - -**After (Blackwell):** No equivalent needed. tcgen05 reads A directly from SMEM via descriptor. - -```cuda -// Blackwell: just construct the SMEM descriptor -uint64_t desc_a = make_smem_desc_128b(smem_a_ptr); -// Pass desc_a directly to tcgen05.mma -- no register staging -``` - -### Step 3: Change SMEM Swizzle from 64B to 128B - -**Before (Hopper):** 64-byte or 128-byte swizzle both work for wgmma. - -```cuda -// Hopper TMA descriptor: 64B swizzle is common -CUtensorMap desc = create_tma_desc(ptr, M, N, tile_m, tile_n, - CU_TENSOR_MAP_SWIZZLE_64B); -``` - -**After (Blackwell):** 128-byte swizzle is mandatory. - -```cuda -// Blackwell TMA descriptor: MUST use 128B swizzle -CUtensorMap desc = create_tma_desc(ptr, M, N, tile_m, tile_n, - CU_TENSOR_MAP_SWIZZLE_128B); -``` - -### Step 4: Replace wgmma Issue with tcgen05 Issue - -**Before (Hopper):** - -```cuda -// Hopper: all 128 threads in warpgroup issue wgmma -asm volatile( - "wgmma.mma_async.sync.aligned.m64n256k16.f32.bf16.bf16 " - "{%0,%1,...}, {%N,...}, desc_b, ...;" - : "+f"(acc[0]), "+f"(acc[1]), ... - : "r"(a_frag[0]), ... -); -asm volatile("wgmma.commit_group.sync.aligned;"); -``` - -**After (Blackwell):** - -```cuda -// Blackwell: single thread issues tcgen05 -if (threadIdx.x == 0) { - asm volatile( - "tcgen05.mma.cta_group::1.kind::f16 " - "[%0], %1, %2, %3, 1;" - : - : "r"(tmem_acc), "l"(desc_a), "l"(desc_b), "r"(0) - ); -} -// No commit needed -- fully async -``` - -### Step 5: Replace wgmma.wait_group with tcgen05 Fence - -**Before (Hopper):** - -```cuda -// Hopper: wait for outstanding wgmma groups -asm volatile("wgmma.wait_group.sync.aligned 0;"); -// Accumulators now ready in registers -``` - -**After (Blackwell):** - -```cuda -// Blackwell: fence before reading TMEM -asm volatile("tcgen05.fence::before_thread_sync;"); -__syncthreads(); -// TMEM accumulators now ready for reading -``` - -### Step 6: Update Epilogue - -**Before (Hopper):** - -```cuda -// Hopper: accumulators in registers, directly usable -float result = acc[frag_m][frag_n]; -// Apply bias -result += bias[col]; -// Apply activation -result = relu(result); -// Store to GMEM -C[row * N + col] = (half)result; -``` - -**After (Blackwell):** - -```cuda -// Blackwell: must read from TMEM first -float result = tmem_load_f32(tmem_acc + col_offset); -// Apply bias -result += bias[col]; -// Apply activation -result = relu(result); -// Store to GMEM -C[row * N + col] = (half)result; -``` - -## Warp Specialization Changes - -### Hopper Warp Specialization (Typical) - -``` -Warp 0-3: Warpgroup 0 -- MMA producer (all 128 threads issue wgmma) -Warp 4-7: Warpgroup 1 -- MMA producer (backup/double-buffer) -Warp 8-11: Data movement (TMA loads, SMEM management) -Warp 12: Tile scheduler - -Total: 13+ warps, 416+ threads -``` +| Concern | SM90 WGMMA | SM100 tcgen05.mma | +|---|---|---| +| MMA issue | Warpgroup collective | One thread for group-1 or group-2 MMA | +| D accumulator | Per-thread registers | TMEM | +| A operand | SMEM-descriptor or register forms, depending on instruction | SMEM-descriptor or TMEM-address forms | +| B operand | SMEM descriptor | SMEM descriptor | +| MMA completion | Commit/wait groups | `tcgen05.commit` plus mbarrier wait | +| Cross-thread ordering | WGMMA-specific rules | tcgen05 fences composed with an execution-ordering operation | +| Narrow floating formats | FP8 WGMMA forms | FP8/FP6/FP4 plus block-scaled MX/NVFP4 kinds | -### Blackwell Warp Specialization (Typical) +The exact old WGMMA form matters. Do not assume every Hopper kernel loads A through `ldmatrix`: descriptor-sourced WGMMA forms already read A from SMEM. Likewise, tcgen05 can source A from SMEM or TMEM. -``` -Warp 0: MMA producer (single thread issues tcgen05) -Warp 1-2: TMA data movement (load A, load B, manage barriers) -Warp 3: Epilogue (read TMEM, apply post-ops, store to GMEM) -Warp 4: Tile scheduler / CLC management +## Dependency-ordered migration checklist -Total: 5 warps, 160 threads (can be fewer) -``` +1. Identify the exact WGMMA operand form, accumulator type, shape, and group-completion points in the SM90 kernel. +2. Choose a legal tcgen05 kind, A source, CTA group, instruction descriptor, and data-path layout for the SM100 target. +3. Replace register-resident D with a TMEM allocation. Group-1 allocation/deallocation is warp-collective; group 2 requires one warp in each CTA of the pair. +4. Keep A/B backing storage unchanged until all asynchronous MMA consumers have completed. +5. Replace WGMMA group completion with `tcgen05.commit` and an mbarrier wait. Add tcgen05 fences only where an execution-ordering handoff must order tcgen05-visible state across threads or CTAs. +6. Transfer completed accumulator values from TMEM to registers with `tcgen05.ld`, observe its completion/access rules, then run the epilogue. +7. Deallocate every dynamic TMEM allocation before kernel exit. Use the same `cta_group` value for all tcgen05 instructions in the kernel. +8. Retune CTA/cluster shapes, pipeline stages, thread roles, descriptors, and epilogue scheduling on the target workload. -With tcgen05, far fewer warps are dedicated to MMA because only one thread is needed to issue the instruction. The freed warps can be repurposed for: -- More aggressive data prefetching -- Overlapped epilogue (read TMEM while next tile's MMA is computing) -- Softmax or other reductions (FlashAttention-style) +## TMEM lifecycle -## Tile Size Migration +`tcgen05.alloc` writes a 32-bit TMEM address to shared memory. Allocation size is expressed in columns, in power-of-two multiples permitted by PTX. It is a synchronous warp instruction: a lane-0-only call is invalid. `tcgen05.dealloc` has the same issue granularity, and PTX requires all allocated TMEM to be deallocated before kernel exit—not only in persistent kernels. -| Hopper | Blackwell (1-SM) | Blackwell (2-SM) | -|---|---|---| -| m64 x n256 x k16 | m128 x n256 x k16 | m256 x n256 x k16 | -| m64 x n128 x k16 | m128 x n128 x k16 | m256 x n128 x k16 | +The MMA issuer and the allocation participants are different concepts. One thread initiates a group-1 MMA, but one full warp collectively allocates or deallocates its TMEM. A group-2 allocation uses one warp from each peer CTA. -Blackwell's base MMA tile is 2x larger in M (128 vs 64). When migrating: -- CTA tile size of 128x256 maps naturally to a single tcgen05 MMA -- CTA tile size of 64x256 on Hopper should be doubled to 128x256 -- For 2-SM mode, consider 256x256 tiles +## Completion, ordering, and epilogue -## Common Migration Pitfalls +`tcgen05.mma` is asynchronous. `tcgen05.commit.cta_group::N.mbarrier::arrive::one.b64` makes an mbarrier track prior MMA work issued by the current thread; waiting on that barrier observes completion. A `tcgen05.fence` is an ordering and code-motion primitive, not a completion wait, so fence plus `__syncthreads()` is not a substitute for commit/mbarrier. -1. **Forgetting 128B swizzle**: The most common silent-failure bug. wgmma works with 64B swizzle; tcgen05 does not. Results will be numerically wrong but the kernel won't crash. +After completion and any necessary thread handoff, epilogue warps use `tcgen05.ld` to bring their accessible TMEM lanes into registers. Ordinary bias, activation, conversion, and global-store code then operates on those register values. Preserve the `tcgen05.ld` warp-access mapping and use `tcgen05.wait::ld` where its asynchronous completion must be observed. -2. **Not deallocating TMEM**: On Hopper, register accumulators are freed implicitly when the CTA exits. On Blackwell, TMEM must be explicitly deallocated in persistent kernels. +## Layout and shape selection -3. **Synchronization model mismatch**: Replacing `wgmma.wait_group` with `__syncthreads()` alone is insufficient. The `tcgen05.fence::before_thread_sync` must precede the syncthreads. +Do not mechanically convert every 64-byte swizzle to 128-byte swizzling. The tcgen05 shared-memory descriptor defines valid no-swizzle, 128B, 64B, and 32B modes, subject to mode-specific alignment and layout constraints. The 128B choice can be much faster for a particular tile, but it is not a universal correctness condition. -4. **Over-allocating threads**: On Hopper, you need 128 threads per warpgroup for MMA. Blindly keeping the same thread count on Blackwell wastes resources since only 1 thread issues tcgen05. +Similarly, there is no universal Hopper-to-Blackwell rule that doubles M. WGMMA and tcgen05 each expose multiple shapes; tcgen05 M/N are encoded in `idesc` and constrained by kind, layout, CTA group, and target ISA. Treat m128xn256xk16 and m256xn256xk16 as useful F16/BF16 maximum examples, not a complete migration table. -5. **Register pressure assumptions**: Code tuned for Hopper's high register pressure (e.g., reduced tile sizes, manual spilling) may be overly conservative on Blackwell. Re-tune tile sizes to take advantage of freed registers. +## Warp specialization -## CUTLASS Migration +Single-thread MMA issue can free instruction-issue capacity for TMA, descriptor preparation, epilogue, reductions, or scheduling. It does not determine the kernel's total thread count: TMEM allocation, TMEM loads, TMA, epilogue, and barriers remain warp- or CTA-level work. Start from a pinned SM100 implementation and measure a role partition rather than copying fixed warp counts from an SM90 kernel. -If using CUTLASS, the migration is largely handled by changing the arch tag and kernel schedule: +Moving D out of the GPR file also changes the register budget. Revisit tile size, pipeline depth, and launch bounds, but do not assume that lower accumulator pressure automatically increases occupancy; SMEM, TMEM, barriers, threads, and registers used by other roles can become limiting resources. -```cpp -// Hopper CUTLASS GEMM -using GemmHopper = cutlass::gemm::device::GemmUniversal< - /* ... */ - cutlass::arch::Sm90, - /* ... */ - cutlass::gemm::collective::KernelScheduleSm90CpAsyncWarpSpecialized ->; +## CUTLASS migration -// Blackwell CUTLASS GEMM -- change arch + schedule -using GemmBlackwell = cutlass::gemm::device::GemmUniversal< - /* ... */ - cutlass::arch::Sm100, // <-- changed - /* ... */ - cutlass::gemm::collective::KernelScheduleSm100CpAsyncWarpSpecialized // <-- changed ->; -``` +CUTLASS does not reduce this port to replacing an architecture tag and one schedule token. In pinned SM100 examples, the collective builders are re-instantiated with SM100 operator classes, tile and cluster shapes, stage policy, mainloop schedule, and epilogue schedule; the resulting collectives feed `cutlass::gemm::kernel::GemmUniversal` and a device adapter. Schedule tags are configuration-specific—for example, the pinned tree defines `KernelTmaWarpSpecialized1SmSm100` and `KernelTmaWarpSpecialized2SmSm100`, not `KernelScheduleSm100CpAsyncWarpSpecialized`. -CUTLASS handles the internal differences (TMEM allocation, descriptor construction, fence insertion, 128B swizzle) automatically through its `MMA_Atom` and `MMA_Traits` abstractions for SM100. +Use a complete pinned example such as CUTLASS `examples/70_blackwell_gemm/70_blackwell_fp16_gemm.cu`, then revalidate alignment, workspace, hardware support, dispatch, and numerical results for the migrated configuration. diff --git a/wiki/patterns/compute-bound.md b/wiki/patterns/compute-bound.md index e8cdba34c..f5d354219 100644 --- a/wiki/patterns/compute-bound.md +++ b/wiki/patterns/compute-bound.md @@ -1,47 +1,67 @@ --- id: pattern-compute-bound -title: "Not Reaching Peak FLOPS" +title: "Not Reaching the Relevant Compute Ceiling" type: pattern tags: [tcgen05, 2sm-cooperative, pipeline-stages, warp-specialization] symptoms: [compute-bound, low-tensor-core-utilization, pipeline-stalls] candidate_techniques: [hw-2sm-cooperative, technique-pipeline-stages, technique-warp-specialization, technique-epilogue-fusion, technique-software-exp] -related: [pattern-low-sm-utilization, pattern-register-pressure] -sources: [doc-nvidia-tuning-guide, blog-tcgen05-tutorial, blog-flash-attention-4] +related: [pattern-pipeline-stalls, pattern-low-sm-utilization, pattern-register-pressure] +sources: [doc-nvidia-tuning-guide, blog-tcgen05-tutorial, doc-flash-attention-4, blog-flash-attention-4] +confidence: verified +evidence_basis: + - source_id: doc-nvidia-tuning-guide + evidence_type: official-doc +reproducibility: concept --- -## Symptom +## Classify the Gap First -Tensor core utilization below 70%. Memory bandwidth is not saturated. Kernel is compute-bound but not reaching peak FLOPS. +“Below peak FLOPS” is not itself a bottleneck diagnosis. Select the peak for the executed datatype, instruction kind, sparsity/scaling mode, clocks, and number of participating SMs. Then use arithmetic intensity and achieved compute/memory ceilings to decide whether the measured kernel lies on the compute side of the relevant roofline. Unsaturated DRAM bandwidth and tensor-core utilization below an arbitrary threshold such as 70% do not prove compute boundedness. -## Likely Causes +Also distinguish whole-kernel throughput from tensor-core active time. A correct kernel can have efficient MMA intervals yet spend material time in TMA readiness, CUDA-core transforms, reductions, synchronization, epilogue work, or the grid tail. Scheduler under-issue plus source/SASS correlation identifies where issue opportunity is lost; a high stall sample count alone does not establish the cause. -1. **Pipeline bubbles**: MMA stalled waiting for data from TMA -2. **Non-matmul overhead**: Softmax, activation functions, reductions consuming cycles -3. **Single-SM MMA tiles too small**: Not fully utilizing available compute -4. **Epilogue blocking mainloop**: TMEM reads blocking next MMA +## Separate the Limiting Edge -## Candidate Techniques - -| Technique | Effect | +| Observation and matched control | Supported inference if runtime also improves | |---|---| -| [2-SM cooperative](../hardware/2sm-cooperative.md) | Double effective MMA tile (m256×n256), 2× compute per cycle | -| [Pipeline stages](../techniques/pipeline-stages.md) | Overlap TMA load with MMA compute | -| [Warp specialization](../techniques/warp-specialization.md) | Dedicated warps for TMA/MMA/epilogue, no stalls | -| [Epilogue fusion](../techniques/epilogue-fusion.md) | Overlap epilogue with next tile's MMA | -| [Software exponential](../techniques/software-exp.md) | Distribute non-matmul ops across FMA units (FA4) | - -## Example: FlashAttention-4 - -``` -// Problem: Blackwell doubles tensor core throughput but SFU count unchanged -// SFU bottleneck: exp() for softmax -// -// Solution: Software 2^x via Cody-Waite + Horner polynomial -// Distributes across FMA units, multiplying exponential throughput -// Result: 1605 TFLOPS (71% utilization) on B200 -``` - -## Caveats -- 2-SM cooperative requires cluster configuration and identical SMEM layouts -- Pipeline depth tuning is workload-dependent (3-5 stages typical) -- Software-emulated transcendentals trade accuracy for throughput +| An extra legal SMEM operand stage reduces TMA-full waits | Operand production was exposed on the tested K loop | +| Separating producer/MMA roles reduces control or dependency gaps | The prior role schedule was on the critical issue path | +| A second TMEM output region reduces mainloop-to-epilogue waits | Accumulator reuse was serialized by the epilogue | +| Moving only selected non-MMA operations changes their pipeline activity and runtime | Those operations contributed to the tested critical path | +| A legal 2-SM variant changes traffic/reuse and aggregate throughput | Cooperative mapping helped that shape and resource configuration | + +For every comparison, hold math, datatype, tile coverage, launch environment, warmup, and timing statistic fixed. Record registers, spills, SMEM, TMEM columns, occupancy, cluster size, achieved bandwidth, scheduler issue activity, and per-pipeline activity. A change in utilization without lower end-to-end time is not a win. + +## Candidate-Specific Checks + +### Pipeline stages + +Enumerate stage counts that compile and fit the complete shared-storage allocation. More stages can overlap TMA production with MMA consumption, but they also consume SMEM, change occupancy, and enlarge prologue/tail costs. There is no architecture-wide “three to five stages” range; the cited tutorial's beneficial v3 kernel uses two stages. + +### Warp specialization and output overlap + +Dedicated producer, MMA, or epilogue warps can remove role switching from a loop and allow independent work to progress, but add live warps, registers, and synchronization. They do not eliminate stalls. If the epilogue owns the only output region, compare one versus multiple disjoint TMEM regions and prove MMA-completion and epilogue-load completion before each handoff. + +### 2-SM cooperative MMA + +`cta_group::2` requires a valid cluster and exact kind-, shape-, layout-, descriptor-, and peer-resource constraints. It does not universally mean `m256n256`, require identical SMEM layouts, or promise twice the compute per cycle: one cooperative operation consumes resources from two CTAs. Compare total and per-SM throughput against a legal one-CTA mapping and measure whether peer operand reuse or a larger aggregate tile reduces traffic enough to repay coordination and occupancy costs. + +### Non-MMA work + +Optimize non-MMA work only after showing that it lies on the critical path. Fusion may remove intermediate traffic; tile interleaving may expose independent work; an approximation may exchange numerical error for throughput. Validate error against the application oracle rather than assuming a software transcendental is interchangeable with the hardware operation. + +## FlashAttention-4 Case Study + +FA4's B200 analysis reports 8192 BF16 MMA operations per clock per SM versus 4096 on Hopper, while exponential throughput is 16 operations per clock per SM on both. Its response is a coordinated design: two-output-tile scheduling, selected software exponentials, conditional rescaling, TMEM partitioning, and pipeline/register choices. + +The software path uses base-2 range reduction with `n=floor(x)` and a cubic FMA polynomial for a selected fraction of exponential evaluations; other values still use hardware `ex2`. The paper evaluates approximation error and end-to-end accuracy. This is not a claim that every exponential uses Cody-Waite reduction or that polynomial evaluation universally multiplies exponential throughput. + +The paper reports up to 1613 TFLOP/s and 71% for complete FA4 forward kernels; the author blog reports up to 1605 TFLOP/s and 71%. Neither number isolates software exponentiation or any other single technique. Use FA4 as evidence that non-MMA work can become material after tensor-core throughput increases, not as a recipe for an unrelated compute-bound kernel. + +## Primary References + +- [Nsight Compute 2025.3 roofline and profiling guidance](https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html) +- [PTX ISA 9.0 tcgen05 MMA forms](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma) +- [Pinned two-stage tutorial pipeline](https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu) +- [FlashAttention-4 paper v1](https://arxiv.org/html/2603.05451v1) +- [FlashAttention-4 author blog](https://tridao.me/blog/2026/flash4/) diff --git a/wiki/patterns/low-sm-utilization.md b/wiki/patterns/low-sm-utilization.md index 99cfe4372..c908111dd 100644 --- a/wiki/patterns/low-sm-utilization.md +++ b/wiki/patterns/low-sm-utilization.md @@ -6,37 +6,60 @@ tags: [persistent-kernel, clc, tile-scheduling] symptoms: [low-sm-utilization, tail-effect, load-imbalance] candidate_techniques: [technique-persistent-kernels, technique-tile-scheduling, hw-clc] related: [pattern-tail-effect, pattern-compute-bound] -sources: [doc-nvidia-tuning-guide, blog-tcgen05-tutorial, pr-cutlass-2161] +sources: [doc-nvidia-tuning-guide, blog-tcgen05-tutorial, doc-cutlass-clc, doc-ptx-isa-sm100] +confidence: verified +evidence_basis: + - source_id: doc-nvidia-tuning-guide + evidence_type: official-doc +reproducibility: concept --- -## Symptom +## Define the Missing Parallelism -SM utilization below 60% despite sufficient occupancy. Nsight Compute shows idle SMs during portions of kernel execution. +Low SM utilization means that fewer SMs perform useful work than the workload could profitably use during a material part of its measured time. It is not defined by a universal 60% threshold. Theoretical occupancy describes how many blocks or warps can reside from resource limits; it does not prove that the grid supplies those workers, that they are simultaneously active, or that their work is balanced. -## Likely Causes +Collect a synchronized kernel time and a time-resolved view where supported. Record the physical and application-constrained SM count, grid and cluster dimensions, blocks resident per SM, logical work count, work duration distribution, and active/idle intervals. Aggregate SM activity alone cannot distinguish a small grid, a partial final wave, variable-duration work, resource-limited residency, or phase-local serial work. -1. **Tail effect**: Last wave of tiles leaves most SMs idle (see [tail-effect](tail-effect.md)) -2. **Load imbalance**: Some tiles take longer than others (variable computation per tile) -3. **Static scheduling**: Fixed tile-to-SM assignment doesn't adapt to runtime conditions -4. **Grid too small**: Fewer threadblocks than SMs +## Distinguish Four Cases -## Candidate Techniques - -| Technique | Applicability | Effect | +| Case | Discriminating evidence | Appropriate next experiment | |---|---|---| -| [CLC](../hardware/clc.md) | SM100 only | Dynamic tile assignment, eliminates load imbalance | -| [Persistent kernels](../techniques/persistent-kernels.md) | SM90+ | Eliminates tail effect, one-time launch overhead | -| [Tile scheduling](../techniques/tile-scheduling.md) | SM90+ | Better L2 locality, reduce load variance | - -## Examples - -``` -// tcgen05 tutorial progression: -// Without persistent/CLC: 86% of cuBLAS (some SMs idle at wave boundaries) -// With persistent + CLC: 98% of cuBLAS (all SMs stay busy) -``` - -## Caveats -- CLC only available on SM100 datacenter GPUs (not SM120 consumer) -- Persistent kernels complicate debugging and profiling -- For non-persistent kernels, ensure grid size >> SM count +| Too little independent work | Logical block/cluster count is below available one-wave worker capacity | Change problem decomposition, accounting for reduction and synchronization cost | +| Wave quantization | Equal-duration data-parallel work has a small nonzero final-wave remainder | Compare tile shapes or a decomposition that changes tile count | +| Variable work duration | Per-worker tile counts/times have a long tail despite enough pending work | Compare static and dynamic acquisition with identical tile/decomposition | +| Residency or phase limitation | Grid is large, but resources or serial phases limit active blocks/warps | Change the limiting resource or phase; more grid blocks alone do not help | + +Static assignment is nonadaptive, but it is not automatically imbalanced. A grid with fewer independent blocks than SMs cannot occupy every SM, while a grid much larger than the SM count can still show a tail or long-running stragglers. Do not prescribe `grid size >> SM count` without a decomposition that creates useful independent work. + +## Scheduler Choices + +Persistence, coordinate order, CLC reassignment, and K decomposition are separate controls: + +- A static persistent worker can process multiple logical tiles by grid stride. It can amortize setup and change wave behavior but cannot guarantee removal of the final tail. +- A row/column raster or swizzle changes coordinate order and may change operand locality. It does not guarantee a better L2 hit rate or lower work variance. +- Cluster Launch Control lets a running worker cancel an unspecified not-yet-started block or cluster from the launched grid and process the returned ID. It redistributes existing work; it cannot create independent tiles, discard required work, or guarantee equal worker times. +- Stream-K or Split-K can create additional partitions when tile-level parallelism is insufficient, at the cost of partial-result reduction, workspace, synchronization, and possible determinism changes. + +For CLC, compare static and dynamic scheduling with identical math, tile/cluster shapes, problem-sized grid, resource limits, and timing method. Record successful and failed requests plus per-worker work counts where instrumentation permits. Include ordinary and intentionally uneven SM availability as separate cases. + +PTX ISA 9.0 says `clusterlaunchcontrol.try_cancel` requires `sm_100` or higher; its cluster-wide multicast qualifier explicitly lists `sm_120a` and the SM120 family. Therefore CLC is not categorically excluded from SM120 by the ISA. The pinned CUTLASS 4.5.0 persistent scheduler discussed in this wiki is specifically an SM100 integration; do not generalize a library route into an architecture exclusion. + +## Source-Reported Tutorial Result + +In Gau Nernst's `M=N=K=4096` B200 experiment, v5 reports 1302.29 TFLOP/s (86.43% of its 1506.74-TFLOP/s cuBLAS value), while v6 reports 1475.93 TFLOP/s (97.96%). The v6 endpoint combines static persistence, a changed output pipeline, and epilogue-specialized warps on top of earlier changes. The author says CLC was not added and observes that overlap remains imperfect. This is not a CLC result, an all-SMs-busy measurement, or an isolated tail-effect ablation. + +## Verification Checklist + +1. Prove exact logical-work coverage and no duplicates for initial and reassigned coordinates. +2. Compare requested work with available worker capacity and calculate the equal-duration final-wave remainder as a diagnostic baseline. +3. Instrument work count and time per worker to separate shortage from variable-duration imbalance. +4. Hold decomposition constant when testing static versus dynamic acquisition; test decomposition separately. +5. Report end-to-end time along with active-SM distribution, scheduler overhead, cache traffic, occupancy, and regressions. + +## Primary References + +- [CUDA Programming Guide: Cluster Launch Control](https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html) +- [PTX ISA 9.0 CLC target and request semantics](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel) +- [CUTLASS 4.5.0 CLC scheduling](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md) +- [Pinned tutorial v6](https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu) +- [Tutorial result progression](https://gau-nernst.github.io/tcgen05/) diff --git a/wiki/patterns/memory-bound.md b/wiki/patterns/memory-bound.md index 4e4362773..a9e86b263 100644 --- a/wiki/patterns/memory-bound.md +++ b/wiki/patterns/memory-bound.md @@ -4,47 +4,64 @@ title: "Memory Bandwidth Bound" type: pattern tags: [vectorized-loads, cache-policy, shared-memory-optimization] symptoms: [memory-bound, low-compute-utilization, high-memory-throughput] -candidate_techniques: [technique-vectorized-loads, technique-swizzling, technique-pipeline-stages] +candidate_techniques: [technique-vectorized-loads, technique-cache-policy, technique-swizzling, technique-pipeline-stages] related: [pattern-compute-bound, kernel-nvfp4-gemv] -sources: [blog-yue-nvfp4, blog-amandeep-nvfp4, doc-nvidia-tuning-guide] +sources: [blog-amandeep-nvfp4, blog-yue-nvfp4, doc-nvidia-tuning-guide, doc-ptx-isa-sm100] +confidence: verified +evidence_basis: + - source_id: doc-nvidia-tuning-guide + evidence_type: official-doc +reproducibility: concept --- -## Symptom +## Establish the Memory Roof -Nsight Compute shows high DRAM throughput but low tensor core utilization. Arithmetic intensity below the roofline knee point. +A kernel is memory-bandwidth bound when its measured arithmetic intensity places it on the memory side of the relevant roofline and its useful throughput is limited by an attained memory ceiling. Compute intensity from the operations actually performed and bytes transferred at the memory level under study. Compare against a measured ceiling for the same device, clocks, memory level, datatype path, and environment; a nominal product bandwidth is only an upper-bound input. -## Likely Causes +High DRAM throughput, low tensor-core activity, or a workload label such as GEMV is not sufficient alone. Record useful/requested bytes, transferred sectors or bytes, achieved bandwidth, cache behavior, scheduler issue/stalls, and end-to-end time. This separates four cases that can otherwise look similar: -1. **Low arithmetic intensity**: Operations like GEMV, small batch decode, or reduction kernels -2. **Poor data reuse**: Each data element used only once -3. **Inefficient memory access**: Uncoalesced loads, L1 cache thrashing +| Evidence | More precise interpretation | +|---|---| +| Useful bytes and transferred bytes are close; attained bandwidth is near the measured roof | Plausible bandwidth-ceiling limit | +| Transferred bytes substantially exceed useful bytes | Coalescing, overfetch, cache, or redundant-traffic problem | +| Bandwidth is below the roof and warps lack ready work | Latency, dependency, insufficient concurrency, or issue problem | +| Bandwidth is high only during one phase | Phase balance or fusion opportunity; whole-kernel boundedness remains unproven | -## Candidate Techniques +Single-use data can lower operations per transferred byte, but “poor reuse” is actionable only if additional legal reuse exists. Uncoalesced access and cache interference may increase traffic or latency; they are not synonyms for saturation of the DRAM bandwidth ceiling. -| Technique | Effect | -|---|---| -| [Vectorized loads](../techniques/vectorized-loads.md) | 128/256-bit loads maximize bandwidth utilization | -| [Cache policies](../techniques/vectorized-loads.md) | L1::no_allocate for streaming, L1::evict_last for reuse | -| [Register budgeting](../techniques/vectorized-loads.md) | -maxrregcount increases occupancy | -| [TMA multicast](../hardware/tma.md) | Share loaded data across SMs in cluster | -| [Swizzling](../techniques/swizzling.md) | Eliminate bank conflicts in shared memory | - -## Examples - -```cuda -// NVFP4 GEMV: memory-bound optimization -// Key insight: profile FIRST to confirm memory-bound behavior -// "The single most important thing could have been running Nsight Compute" -// — Amandeep (12 Attempts at an FP4 Kernel) - -// Optimization priorities for memory-bound kernels: -// 1. Maximize memory bandwidth (wide loads, coalescing) -// 2. Reduce register count (higher occupancy) -// 3. Differentiate cache policies per access pattern -// 4. DON'T optimize compute (it's not the bottleneck) -``` - -## Caveats -- Always profile before optimizing — wrong assumption wastes effort -- B200 has 8 TB/s bandwidth; speed-of-light calculation determines achievable performance -- ILP and compute optimizations have diminishing returns for memory-bound kernels +## Controlled Candidate Tests + +### Load width and coalescing + +Choose only legal, naturally aligned vector forms and handle tails separately. Compare scalar/narrow and wider variants with identical mapping. Record instruction count, requested and transferred bytes, transactions, registers, spills, achieved bandwidth, and time. Wider instructions do not guarantee fewer hardware transactions or higher bandwidth. + +### Cache policy + +PTX L1 eviction priorities and L2 prefetch controls are hints. `no_allocate` does not guarantee bypass, and `evict_last` does not guarantee residence. Characterize address order, reuse distance, working set, and interfering streams, then vary one hint at a time against the default. Retain it only when a representative end-to-end comparison improves. + +### Register budget and concurrency + +Inspect compiled registers, spills, SMEM, threads, and the actual occupancy-limiting resource before applying `-maxrregcount`. A cap may be clamped by ABI requirements, introduce spill traffic, or leave occupancy unchanged. Higher theoretical occupancy is useful only if it raises ready work or attained bandwidth enough to reduce runtime. + +### TMA multicast and shared-memory layout + +TMA multicast can issue one global-to-shared tensor copy to selected CTAs' shared-memory destinations within a cluster. It is relevant when those CTAs consume the same operand and the cluster/lifetime/barrier costs are valid. It does not help a stream with no inter-CTA reuse. + +Swizzling changes a shared-memory address mapping and can reduce conflicts for a specified access pattern. It does not universally eliminate conflicts and is not by itself a DRAM optimization. Validate the legal tensor-map/layout constraints and compare bank-conflict, traffic, and time measurements. + +## NVFP4 GEMV Negative Controls + +The cited Amandeep NVFP4 GEMV report is useful precisely because plausible memory-oriented changes failed. A wider `uint2` load was 16–25% slower on the reported shapes, and reducing `maxrregcount` from 80 to 64 had no effect. These observations do not prove that wide loads or register caps are generally harmful; they refute a fixed optimization priority based only on the “memory-bound GEMV” label. + +Use the same discipline for compute work. If the kernel is genuinely at its attainable memory roof, compute-only instruction reductions may have little effect. But address calculation, decoding, and instruction-level parallelism can affect memory issue and latency hiding before that roof is reached. State a predicted counter change, alter one variable, and keep or reject the hypothesis from matched time and profiler evidence. + +## Reproduction Record + +Report input shapes/distributions, useful work and bytes, cache state, warmup/repetitions/statistic, GPU and clocks, software versions, generated instructions, resource usage, roofline assumptions, achieved memory-level bandwidth, requested/transferred efficiency, scheduler activity, and correctness tolerance. Include negative and regressing variants so a category heuristic is not mistaken for causal evidence. + +## Primary References + +- [Nsight Compute 2025.3 Profiling Guide](https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html) +- [PTX ISA 9.0 cache-operation semantics](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld) +- [PTX ISA 9.0 TMA multicast](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor) +- [Amandeep NVFP4 GEMV attempts](https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/) diff --git a/wiki/patterns/moe-load-imbalance.md b/wiki/patterns/moe-load-imbalance.md index 845d84bc2..9751cee67 100644 --- a/wiki/patterns/moe-load-imbalance.md +++ b/wiki/patterns/moe-load-imbalance.md @@ -6,50 +6,114 @@ tags: [moe, grouped-gemm, tile-scheduling, clc] symptoms: [load-imbalance, tail-effect, low-sm-utilization] candidate_techniques: [technique-tile-scheduling, technique-persistent-kernels, technique-kernel-fusion] related: [kernel-grouped-gemm, kernel-fused-moe, pattern-tail-effect] -sources: [contest-gpumode-p4, contest-flashinfer-track-a, blog-deepgemm] +sources: [contest-gpumode-p4, contest-flashinfer-track-a, blog-deepgemm, doc-cutlass-clc, blog-gpu-mode-reward-hack] --- # MoE Expert Load Imbalance -## Symptom +## Diagnose the level that is imbalanced -MoE grouped GEMM shows uneven per-expert compute time. Some SMs finish their expert quickly and sit idle while others are still processing. Overall latency is dominated by the slowest expert. +Do not infer one expert per physical SM from a grouped-GEMM trace. CUDA assigns +thread blocks to available SMs, while grouped kernels may split an expert into +multiple output tiles or let one resident worker process several logical tiles. +Measure at each relevant level: -## Likely Causes +- routed tokens per expert for the exact batch and prefill/decode phase; +- valid rows and output-tile counts per expert; +- completed tiles and elapsed work per logical worker and, when instrumented, + per SM; +- dispatch, grouped-GEMM, combine, and end-to-end times per expert-parallel + rank. -1. **Skewed token distribution**: Router sends 80% of tokens to 20% of experts (common in trained MoE models) -2. **Static tile assignment**: Precomputed tile→SM mapping cannot rebalance at runtime -3. **Masked layout waste**: Fixed M_max per expert wastes compute on padding rows -4. **Small-M per expert**: When M < BLOCK_M, thin-GEMM underutilizes tensor cores +Small expert segments and partial output tiles can expose too little parallel +work or leave lanes predicated out. The effect depends on the kernel tile, +other GEMM dimensions, resident-worker count, and competing implementation; +there is no universal `M < BLOCK_M` diagnosis or minimum viable size. -## Candidate Techniques +Uniform expected routing, a larger batch, or an auxiliary balancing loss may +change token-count skew, but none proves balanced tile counts, worker +durations, communication, or runtime. Treat imbalance as absent only when the +measured distributions are narrow and a matched balancing variant does not +materially improve the timed operation. -| Technique | Effect | -|---|---| -| [CLC (Cluster Launch Control)](../hardware/clc.md) | Hardware dynamic tile assignment — fastest SMs grab more tiles | -| [Persistent kernels](../techniques/persistent-kernels.md) | Amortize launch overhead; loop over dynamic work queue | -| [Contiguous layout](../kernels/grouped-gemm.md) | Pack variable-M experts sequentially; offsets array indexes expert boundaries | -| [Masked layout](../kernels/grouped-gemm.md) | Good for CUDA graph capture; wastes compute on padding | -| [K-grouped layout](../kernels/grouped-gemm.md) | For weight gradient computation with variable K per expert | -| [EPLB (Expert Parallel Load Balancer)](https://github.com/deepseek-ai/EPLB) | Replicate heavy experts across GPUs; 1.49x prefill speedup, 2.54x decode | +## Grouped-layout choices -## Example: Reward Hack in GPU Mode Problem 4 +At pinned DeepGEMM commit +[`891d57b4`](https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f), +the three grouped interfaces have different axes and metadata: -The 1st-place submission exploited the evaluation harness rather than truly balancing load: -- Correctness phase: real kernel ran on cloned data -- Timing phase: detected reused objects, fired 120-group super-batch in call 1, returned cached results for calls 2-15 +| Layout | Pinned contract | Relevant control | +| --- | --- | --- | +| M-grouped contiguous | Pack `A` and `D` along variable M with N/K fixed; identify groups by per-row expert IDs or per-group prefix-sum ends; segments are M-block aligned | Record valid rows, alignment padding, tile counts, and metadata form | +| M-grouped masked | Allocate `[G, M_max, ...]`, pass one valid-M count per group, and compute valid portions; the fixed allocation is documented for a CUDA-graph decode case | Separate allocated rows from valid rows and inspect edge predication in generated code | +| K-grouped contiguous | Pack variable K with M/N fixed for MoE weight backward | Do not use its K-axis contract to describe forward token imbalance | -This highlighted that even careful tile scheduling can be outrun by algorithmic restructuring — and prompted the MLSys 2026 FlashInfer contest to add runtime isolation + subprocess eval. +A fixed `M_max` allocation therefore does not establish that all padding rows +are computed. Any residual edge-tile or predication cost needs generated-code +and profile evidence for the selected kernel. -## Caveats +## Scheduler comparisons -- CLC only available on SM100 datacenter (not SM120 consumer) -- Dynamic scheduling has small per-tile overhead vs static precomputed -- Small experts may not benefit — minimum viable tile size is a floor -- EPLB works at cluster scale, not single-device +Keep persistence, work acquisition, and decomposition as separate variables: -## When NOT An Issue +1. A static persistent worker can advance through a deterministic logical-work + sequence. CUDA still decides where its block runs; the rule is not a + precomputed tile-to-physical-SM map. +2. In a CLC-backed scheduler, a selected thread requests cancellation of an + unspecified block or cluster that has not launched. A successful response + returns that existing grid coordinate; a request can fail. CLC does not + create tiles or make the fastest SM autonomously steal arbitrary work. +3. The initial `blockIdx` in CUTLASS's pinned CLC scheduler is static and later + requests are asynchronous, pipelined, and cluster-granular. There is no + universal per-tile latency bound: compare request activity and end-to-end + time against the matched static scheduler. +4. Splitting a large expert or K range may expose more independent work, but it + can add partial-result and reduction costs. Hold math, tile shape, cluster + shape, launch resources, and output semantics constant in the comparison. -- Uniform routing (rare in practice) -- Very large batch sizes (statistics average out) -- Training with auxiliary load balancing loss +PTX ISA 9.0 specifies CLC for `sm_100` or higher and explicitly includes +SM120-family targets for the multicast form. CUTLASS 4.5.0's documented +`PersistentTileSchedulerSm100` integration is a narrower library example, not +the complete ISA compatibility boundary. + +## Expert placement is a separate system layer + +[DeepSeek EPLB at commit +`d52c72d`](https://github.com/deepseek-ai/EPLB/tree/d52c72d5b2f2fb4c41afbf8eb21366820239913d) +takes per-logical-expert load statistics, replicates heavily loaded logical +experts, and packs physical experts across configured GPUs and nodes. It is a +host-side expert-parallel placement planner, not a device-kernel tile +scheduler; use it as a complement only when multi-GPU expert placement is in +the measured system scope. + +An LMSYS/SGLang study on a 96-H100 deployment reported `1.49x` prefill and +`2.54x` decode throughput speedups in its large-scale EPLB ablation. The same +study says it used in-distribution data and that production distribution +shifts require further testing. Those figures are scoped results, not generic +EPLB speedups. + +## GPU Mode Problem 4 reward-hack boundary + +GPU Mode's official postmortem records a temporarily first-place submission +that combined a real grouped-GEMM kernel with a timing-harness exploit. The +correctness path ran a padded eight-group computation on each of 15 cloned +objects. During timing, the first call merged them into one 120-group launch, +calls 2 through 15 returned cached output pointers, and the harness divided the +combined timing by 15. + +This is evidence about evaluator state reuse, not a valid load-balancing +performance result. The postmortem identifies `gpu-mode/reference-kernels` +PR 104 as the harness response; neither that record nor the pinned FlashInfer +MLSys 2026 evaluation contract establishes a causal link between this incident +and the later contest design. + +## Primary references + +- [CUDA thread-block scheduling](https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html) +- [PTX ISA 9.0 CLC request semantics](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel) +- [CUTLASS 4.5.0 CLC scheduler](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md) +- [DeepGEMM grouped interfaces at `891d57b4`](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md) +- [DeepSeek EPLB at `d52c72d`](https://github.com/deepseek-ai/EPLB/tree/d52c72d5b2f2fb4c41afbf8eb21366820239913d) +- [LMSYS/SGLang 96-H100 deployment study](https://www.lmsys.org/blog/2025-05-05-large-scale-ep/) +- [GPU Mode reward-hack postmortem](https://www.gpumode.com/news/reward-hacking-nvfp4) +- [Pinned FlashInfer MLSys 2026 evaluator](https://github.com/flashinfer-ai/flashinfer-bench-starter-kit/blob/75ccd05cafceb0fd1f86be4cd0f2117249463c66/EVALUATION.md) diff --git a/wiki/patterns/pipeline-stalls.md b/wiki/patterns/pipeline-stalls.md index 6aa9236ad..d0bd46c28 100644 --- a/wiki/patterns/pipeline-stalls.md +++ b/wiki/patterns/pipeline-stalls.md @@ -6,53 +6,70 @@ tags: [pipeline-stages, warp-specialization, tma, tcgen05, mbarrier] symptoms: [pipeline-stalls, compute-bound, low-tensor-core-utilization] candidate_techniques: [technique-pipeline-stages, technique-warp-specialization, technique-double-buffering, technique-ping-pong-scheduling] related: [pattern-compute-bound, pattern-tail-effect] -sources: [blog-tcgen05-tutorial, blog-flash-attention-4, doc-nvidia-tuning-guide] +sources: [blog-tcgen05-tutorial, doc-flash-attention-4, doc-nvidia-tuning-guide, doc-ptx-isa-sm100] +confidence: verified +evidence_basis: + - source_id: doc-ptx-isa-sm100 + evidence_type: official-doc +reproducibility: concept --- -# Pipeline Stalls +## Symptom, Not Diagnosis -## Symptom +A pipeline stall is lost issue opportunity caused by a producer-consumer dependency on the critical path. Low TMA or tensor-core activity, a warp sampled at a barrier wait, or a phase-local utilization drop is only a lead. Expected dependency waits, memory latency, execution-pipeline contention, load imbalance, prologue/tail work, and synchronization defects can produce similar observations. -Nsight Compute shows TMA or tcgen05 units idle despite nominally compute-bound workload. Tensor core utilization drops during specific phases of the kernel. Warp-level profiling reveals threads blocked on `mbarrier.try_wait` more than expected. +Nsight Compute's Warp State Statistics describes why sampled warps could not issue, while Scheduler Statistics shows whether schedulers had eligible warps and issued instructions. NVIDIA cautions that stalls are not necessarily performance-limiting and should be prioritized when schedulers fail to issue. Metric availability and names vary by tool and chip, so query the installed profiler and use its shipped sections rather than assuming a Blackwell metric name. PM sampling can add a timeline on supported systems, but warp sampling itself has no time resolution. -## Likely Causes +## Correctness Gate: Audit the Wait Edge -1. **Insufficient pipeline depth**: 2 stages cannot hide a 3-cycle latency chain -2. **Incorrect mbarrier phase tracking**: Consumer observes stale arrivals, waits for next -3. **Missing `tcgen05.fence::after_thread_sync`**: MMA reads SMEM before TMA transfer fully visible -4. **Single-tile scheduling**: All warps serialized on one tile's softmax/epilogue -5. **Producer over-arrives**: Manual `mbarrier_arrive` after async TMA — hardware + manual both arrive, next stage gets stale release +Do this before tuning around an mbarrier wait: -## Candidate Techniques +1. Identify the exact barrier object, owner, initialized arrival count, initial phase, and storage stage for the wait. +2. For a TMA global-to-shared load, account for the producer's software arrival and expected transaction bytes. `mbarrier.arrive.expect_tx` performs an arrival and adds transaction bytes; the TMA `.mbarrier::complete_tx::bytes` operation decrements transaction bytes, not a second arrival. +3. Wait for the matching phase of that same object. Phase parity changes when that barrier completes a phase and is reused; “flip after every wait” is not a valid global rule when a thread waits on multiple objects or phases. +4. A successful acquire wait supplies the documented visibility for associated prior `cp.async.bulk` work before the consumer reads SMEM. +5. Before reusing SMEM operands read by asynchronous MMA, attach completion of the relevant tcgen05 work with `tcgen05.commit` and observe its mbarrier. Issue is not completion. +6. When asynchronous tcgen05 operations cross a thread handoff, place `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` around the applicable execution-ordering operation. The after fence is not a generic replacement for the TMA acquire wait. +7. Verify prologue, steady-state wraparound, short loops, and the producer/consumer tails. A correct steady-state loop can still hang or reuse live storage at a boundary. -| Technique | Effect | -|---|---| -| [Pipeline stages](../techniques/pipeline-stages.md) | Increase NUM_STAGES (3-5 typical on Blackwell) | -| [Warp specialization](../techniques/warp-specialization.md) | Dedicated warps for TMA/MMA/epilogue eliminate role-switching stalls | -| [Double-buffering](../techniques/double-buffering.md) | TMEM buffer A while MMA runs on buffer B | -| [Ping-pong scheduling](../techniques/ping-pong-scheduling.md) | Two query tiles alternate softmax/MMA (FA4 pattern) | +An extra software arrival is erroneous only relative to the barrier's initialized and phase-specific accounting. Diagnose pending arrivals and transaction bytes separately instead of assuming that TMA hardware performs another arrival. -## Diagnosis Checklist +## Causal Diagnosis Workflow -``` -1. Profile with Nsight Compute, check tensor core active cycles -2. Inspect mbarrier wait stalls in warp state breakdown -3. Verify phase tracking increments correctly (each wait should flip parity) -4. Check that TMA uses arrive_expect_tx + mbarrier target (not manual arrive) -5. Ensure tcgen05.fence::after_thread_sync between TMA wait and MMA issue -6. Measure pipeline depth: can you add more NUM_STAGES? -``` +1. Establish a synchronized end-to-end timing regression on fixed inputs and confirm outputs against a reference. Record GPU, clocks, toolkit, build flags, kernel name, launch shape, and resource use. +2. Collect Speed-of-Light, Scheduler Statistics, Warp State Statistics, and source/SASS correlation with the sections available in the installed Nsight Compute. Account for replay and sampling effects. +3. Map the dominant not-issued locations to concrete edges: TMA-full wait, MMA-completion wait, output-buffer reuse, CTA barrier, queue starvation, or pipeline prologue/tail. +4. State a prediction. For example: if TMA readiness is the critical edge, a legal extra operand stage should reduce that wait and runtime; if only the tail is exposed, the steady-state wait distribution should remain largely unchanged. +5. Change one variable and repeat identical correctness, warmup, timing, and profiler collection. Reject a cause when its predicted counter and time movement do not occur. -## Example Progression (tcgen05 tutorial) +| Controlled change | Plausible target | Required controls and costs | +|---|---|---| +| Vary legal SMEM stage count | Producer cannot stay far enough ahead of MMA | Same tile/math/roles; record SMEM, occupancy, short-loop behavior, and tail | +| Separate long-lived warp roles | Role handoff or control work is on the issue path | Same stages and tile; record registers, active warps, and per-role idle/backpressure | +| Add a second TMEM output region | Epilogue holds the only accumulator region | Prove disjoint lifetime; record TMEM columns and epilogue/MMA completion waits | +| Interleave two query/output tiles | Softmax/correction and MMA have independent ready work | Prove separate state and dependencies; compare one versus two query stages | -- 1-stage: 62% of cuBLAS (TMA blocks MMA) -- 3-stage pipelined: 70% (hide most TMA latency) -- Warp specialized: 80% (no role switching) -- Add 2-SM MMA: 86% (larger tile, more reuse) -- Persistent + CLC: 98% (eliminate tail effect) +None is a universal cure. More stages consume storage and can reduce occupancy; specialization adds warps and synchronization; TMEM buffering consumes columns; multi-tile schedules increase live state. A memory-bound roofline classification also does not prove pipelining is useless: test it only when the proposed change has a specific issue-gap, latency, or transaction-efficiency prediction. -## Caveats +## Source-Reported Tutorial Progression -- Too many stages consume SMEM; exceeds 228KB budget -- Phase tracking bugs are notoriously hard to debug — add assertions in development -- Profile first — pipeline is a waste of effort on memory-bound kernels +Gau Nernst reports the following for one `M=N=K=4096` BF16 GEMM on a Modal B200 using PyTorch 2.9.1 and CUDA 13. Percentages below are computed from the reported 1506.74-TFLOP/s cuBLAS value. + +| Version | Author's cumulative version label | TFLOP/s | cuBLAS ratio | +|---|---|---:|---:| +| v1a | basic tcgen05 + 2D 16-byte TMA | 254.62 | 16.90% | +| v2b | 3D 128-byte TMA | 695.43 | 46.15% | +| v3 | pipelining | 939.61 | 62.36% | +| v4 | warp specialization | 1208.83 | 80.23% | +| v5 | 2-SM MMA | 1302.29 | 86.43% | +| v6 | persistent kernel with static scheduling | 1475.93 | 97.96% | + +The pinned v3 source instantiates two stages, not three. Each row is a cumulative source version rather than an isolated microbenchmark of the named mechanism. The final version uses static scheduling; the author explicitly says Cluster Launch Control was not added. The result is evidence that these changes helped that implementation and shape, not a Blackwell progression template. + +## Primary References + +- [Nsight Compute 2025.3 Profiling Guide](https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html) +- [PTX ISA 9.0 mbarrier waits and visibility](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait) +- [PTX ISA 9.0 tcgen05 execution ordering](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence) +- [Pinned tutorial v3 source](https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu) +- [Tutorial progression](https://gau-nernst.github.io/tcgen05/) diff --git a/wiki/patterns/register-pressure.md b/wiki/patterns/register-pressure.md index 6a4495402..0f5f82550 100644 --- a/wiki/patterns/register-pressure.md +++ b/wiki/patterns/register-pressure.md @@ -1,47 +1,55 @@ --- id: pattern-register-pressure -title: "Register Pressure — Low Occupancy" +title: "Register Pressure and Residency" type: pattern tags: [tmem, register-reuse, warp-specialization] symptoms: [register-pressure, low-occupancy, register-spilling] -candidate_techniques: [hw-tmem, technique-warp-specialization, migration-register-to-tmem] +candidate_techniques: [hw-tmem, technique-warp-specialization, technique-register-budgeting, migration-register-to-tmem] related: [pattern-compute-bound, hw-tmem] -sources: [doc-nvidia-tuning-guide, blog-tcgen05-tutorial, pr-vllm-16032] +sources: [doc-nvidia-tuning-guide, blog-tcgen05-tutorial, doc-ptx-isa-sm100] +confidence: verified +evidence_basis: + - source_id: doc-nvidia-tuning-guide + evidence_type: official-doc +reproducibility: concept --- -## Symptom +## Diagnose a Register-Limited Kernel -Occupancy below target due to high register usage per thread. Nsight Compute shows register spilling to local memory. +Register pressure is performance-relevant only when compiled register allocation or spills constrain useful scheduling or add material local-memory traffic. “Occupancy below target” is not enough: theoretical occupancy is a residency limit, achieved occupancy is runtime behavior, and maximum occupancy is not a universal optimum. -## Likely Causes +For the exact binary and launch, record registers per thread, allocation granularity, spill stores/loads, local-memory traffic, threads and warps per CTA, SMEM, cluster shape, theoretical limiting resource, achieved active/eligible warps, scheduler stalls, and end-to-end time. Inspect generated SASS/source correlation to find the live ranges or spill sites. Then change one source or compiler factor and require the predicted resource/stall movement plus a runtime improvement. -1. **Accumulator registers**: On Hopper, large MMA tiles consume many registers for accumulators -2. **Epilogue state**: Data transformation in epilogue requires additional registers -3. **Complex control flow**: Many live variables across branches +Common contributors include a resident MMA fragment, an epilogue whose temporaries overlap the mainloop, descriptors/addresses and pipeline state, unrolled loops, and values live across branches. These are hypotheses about compiled liveness, not conclusions from source complexity alone. -## Candidate Techniques +## Candidate Tests -| Technique | Applicability | Effect | -|---|---|---| -| [TMEM](../hardware/tmem.md) | SM100 only | Moves accumulators to dedicated 256KB memory | -| [Warp specialization](../techniques/warp-specialization.md) | SM100+ | Different warps handle different roles, reducing per-warp register needs | -| [Register-to-TMEM migration](../migration/register-to-tmem.md) | SM90→SM100 | Systematic approach to moving accumulators off registers | +### Shorten or separate live ranges -## Blackwell Solution +Move epilogue work after accumulator lifetime when dependencies allow, reduce unnecessary unrolling, or split a long-lived role. Warp specialization exists on Hopper as well as Blackwell; it can shorten one role's live set, but adds role state and synchronization and does not guarantee fewer registers. Compare compiled resources and time for matched role layouts. -``` -// Hopper: 64×256 MMA tile accumulator = 64*256*4 bytes in registers per warp group -// → ~128 registers per thread just for accumulators -// -// Blackwell: TMEM holds accumulators -// → 0 registers for accumulators -// → ~128 registers freed for other work or higher occupancy -// -// TMEM: 256 KB per SM, 128 rows × 512 columns × 32-bit -// Largest 1-SM MMA uses half → double-buffering possible -``` +### Test a register cap -## Caveats -- TMEM only available on SM100 datacenter (not SM120 consumer) -- TMEM requires explicit alloc/dealloc lifecycle -- TMEM→register transfer adds latency (offset by freeing registers) +Compare an uncapped build with selected `-maxrregcount` values. Record whether the requested cap is effective, the occupancy-limiting resource, spill traffic, instructions, and runtime. ABI minima can constrain the cap, another resource can remain limiting, and spills can erase any benefit. + +### Move resident D from registers to TMEM + +For Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, that is 128 32-bit registers (512 bytes) per thread for D. This exact fragment arithmetic does not include other live state or guarantee a particular resident-block count. + +On the SM100 tcgen05 path, resident D is in TMEM, organized as 128 lanes by 512 columns of 32-bit cells. That removes the long-lived per-thread WGMMA D fragment, but not all accumulator-related register work: addresses, descriptors, barriers, loop state, `tcgen05.ld` destinations, conversion, and epilogue temporaries still consume registers. Logical capacity and required columns depend on MMA kind, shape, and packing, so a largest-operation or always-double-bufferable rule cannot be inferred from raw cell count. + +The migration must implement collective allocation/address publication, tcgen05 MMA completion, cross-thread ordering where applicable, collective TMEM loads and `tcgen05.wait::ld`, consumer completion, and collective deallocation before exit. The described TMEM/tcgen05 path targets SM100-class data-center architectures and is not the SM120 consumer MMA path. + +## Evaluate the Tradeoff + +TMEM loads are asynchronous operations with explicit completion waits. Whether their issue/wait and epilogue work are repaid by lower long-lived register use is an empirical whole-kernel question. Compare the Hopper and Blackwell designs only with equivalent math and precision; for same-architecture alternatives, compare TMEM region count/layout while holding tile and launch policy fixed. + +Report compiled registers and spills, TMEM columns, SMEM, barriers, occupancy limit, achieved warps, pipeline/scoreboard stalls, correctness, and time. Include shapes where register occupancy changes and shapes where it does not. A reduction in register count without a useful residency, issue, traffic, or time improvement is not sufficient. + +## Primary References + +- [Nsight Compute 2025.3 register and occupancy analysis](https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html) +- [PTX ISA 9.0 WGMMA accumulator fragments](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma) +- [PTX ISA 9.0 Tensor Memory](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory) +- [PTX ISA 9.0 tcgen05 load/wait](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld) +- [Pinned SM100 tutorial implementation](https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100) diff --git a/wiki/patterns/tail-effect.md b/wiki/patterns/tail-effect.md index fa8055ea0..25df121c5 100644 --- a/wiki/patterns/tail-effect.md +++ b/wiki/patterns/tail-effect.md @@ -1,44 +1,60 @@ --- id: pattern-tail-effect -title: "Tail Effect — Last Wave Underutilization" +title: "Tail Effect — Last-Wave Underutilization" type: pattern tags: [persistent-kernel, clc, tile-scheduling] symptoms: [tail-effect, low-sm-utilization, wave-quantization] candidate_techniques: [technique-persistent-kernels, hw-clc, technique-tile-scheduling] related: [pattern-low-sm-utilization] -sources: [doc-nvidia-tuning-guide, blog-tcgen05-tutorial, pr-cutlass-2161] +sources: [doc-nvidia-tuning-guide, blog-tcgen05-tutorial, doc-cutlass-clc, doc-ptx-isa-sm100] +confidence: verified +evidence_basis: + - source_id: doc-cutlass-clc + evidence_type: official-doc +reproducibility: concept --- -## Symptom +## Exact Simplified Model -Performance drops for problem sizes where total_tiles % num_SMs != 0. The last wave of tiles runs with many SMs idle. +For `T` equal-duration independent tiles and `W` available one-tile workers, with no K decomposition, write: -## Likely Causes +```text +T = qW + r, 0 <= r < W +``` -1. **Wave quantization**: Grid of N tiles on M SMs takes ceil(N/M) waves; last wave may use only N%M SMs -2. **Static assignment**: stride-by-gridDim leaves remainder tiles on few SMs -3. **Non-persistent launch**: each kernel launch has fixed grid, no dynamic rebalancing +There are `q` full waves. If `r>0`, a final partial wave uses `r` workers and has instantaneous worker utilization `r/W`; if `r=0` and `T>0`, the final wave is full. For example, `T=150` and `W=142` gives one full wave plus eight tiles: the partial wave has `8/142 = 5.63%` worker utilization under these assumptions. -## Candidate Techniques +This is an analytical model, not a statement that a B200 has 142 SMs or that physical SM count always equals `W`. Block resources can make multiple blocks resident per SM, cluster shape changes scheduling granularity, application policies may restrict available SMs, and tile durations can differ. Measure the actual worker capacity and timeline. -| Technique | Effect | -|---|---| -| [CLC](../hardware/clc.md) | Hardware dynamic scheduling, SMs grab tiles on-demand | -| [Persistent kernels](../techniques/persistent-kernels.md) | SM-count grid, iterate over tiles, no wave boundary | -| [Tile scheduling](../techniques/tile-scheduling.md) | Raster order, swizzle patterns for better distribution | +Under the equal-duration model, the time fraction attributable to at most one partial wave shrinks as the number of full waves grows. There is no universal “below four times the SM count” cutoff: the remainder, wave duration, other phases, and imbalance determine significance. -## Example +## Confirm a Last-Wave Cause -``` -// B200: 142 SMs -// Problem: 150 tiles -// Without CLC: 2 waves (142 + 8), last wave uses only 8 SMs (5.6%) -// With CLC: single persistent wave, all 142 SMs stay busy -// -// Impact: 86% → 98% of cuBLAS (tcgen05 tutorial data) -``` +1. Record logical tiles/clusters, grid and cluster dimensions, available worker capacity, residency limits, and per-work-item duration. +2. Predict wave count and the final remainder from the simplified model. +3. Use time-resolved activity or instrumented worker records to locate the underfilled interval at the end, rather than inferring it from aggregate utilization. +4. Sweep nearby problem sizes or tile shapes. A wave-quantization hypothesis predicts a sawtooth response aligned with changes in the remainder, after controlling total work. +5. Separate unequal tile durations: a long straggler is load imbalance even if the tile-count remainder is zero. + +CUDA schedules ordinary grid blocks onto available SMs; a static grid-stride loop fixes logical worker indices, not `blockIdx`-to-physical-SM placement. A fixed grid therefore does not mean CUDA lacks dynamic block scheduling. + +## Scheduler and Decomposition Choices + +- A static persistent loop lets a resident CTA process multiple logical work items. Worker count is selected from problem decomposition, cluster/resource limits, and policy—not necessarily one CTA per physical SM. Finite work still has a final subset and may remain imbalanced. +- Cluster Launch Control lets a selected thread request cancellation of an unspecified not-yet-started block or cluster and process the returned grid ID. CLC redistributes existing IDs; it cannot turn eight remaining independent tiles into 142 or 148 concurrent tiles. +- Raster order and swizzle are coordinate transforms. Test them for locality while holding work acquisition fixed; they do not create work or guarantee balanced durations. +- Stream-K or Split-K may create more independent partitions when tile-level work is insufficient. Evaluate reduction traffic, workspace, synchronization, determinism, and numerical effects separately. + +Compare static and CLC-backed persistence with identical math, decomposition, tile/cluster shapes, grid, resources, and timing. Record CLC request results and per-worker work/time distribution. PTX ISA 9.0 requires `sm_100` or higher for CLC and explicitly lists SM120-family support for the cluster-wide multicast qualifier; the pinned CUTLASS 4.5.0 route used here is specifically its SM100 scheduler. + +## Tutorial Scope + +Gau Nernst's Modal B200 has 148 SMs. In the author's `M=N=K=4096` experiment, v5 reports 1302.29 TFLOP/s (86.43% of cuBLAS) and v6 reports 1475.93 TFLOP/s (97.96%). The v6 change is static persistence plus output-pipeline/epilogue-role changes accumulated over earlier versions. The author explicitly did not add CLC and says overlap remains imperfect. Those results cannot be used as a CLC ablation or proof that all SMs stay busy. + +## Primary References -## Caveats -- Only significant for moderate tile counts (< 4× SM count) -- For very large problems, tail effect is amortized across many waves -- CLC only on SM100 datacenter +- [CUDA Programming Guide: block and cluster scheduling](https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html) +- [PTX ISA 9.0 CLC](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel) +- [CUTLASS 4.5.0 CLC scheduling](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md) +- [Pinned tutorial v6 source](https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu) +- [Tutorial result progression](https://gau-nernst.github.io/tcgen05/) diff --git a/wiki/techniques/cache-policy.md b/wiki/techniques/cache-policy.md index 96a882705..17549582d 100644 --- a/wiki/techniques/cache-policy.md +++ b/wiki/techniques/cache-policy.md @@ -1,51 +1,61 @@ --- id: technique-cache-policy -title: "PTX Cache Policy Differentiation" +title: "PTX Cache Eviction and Prefetch Hints" type: technique architectures: [sm100, sm90] tags: [cache-policy, vectorized-loads] -confidence: source-reported -reproducibility: snippet +confidence: verified +evidence_basis: + - source_id: doc-ptx-isa-sm100 + evidence_type: official-doc +reproducibility: concept prerequisites: [] related: [technique-vectorized-loads, kernel-nvfp4-gemv, pattern-memory-bound] -sources: [blog-yue-nvfp4, blog-amandeep-nvfp4, blog-simon-nvfp4-gemv, doc-ptx-isa-sm100] -blackwell_relevance: "Same PTX cache hints on both archs; higher B200 bandwidth (8TB/s) amplifies the impact of correct cache policy selection." +sources: [doc-ptx-isa-sm100, blog-yue-nvfp4, blog-amandeep-nvfp4, contest-gpumode-p1] +blackwell_relevance: "PTX 9.0 exposes L1/L2 eviction priorities and L2 prefetch-size hints on SM100; these remain performance hints rather than cache-residency guarantees." --- -# Cache Policy Differentiation +# PTX Cache Eviction and Prefetch Hints -## Overview +## Contract -PTX cache qualifiers (`L1::no_allocate`, `L1::evict_last`, `L1::evict_first`) let kernels hint to hardware how to handle cache admission for specific loads. Critical for memory-bound kernels where the L1 working set matters more than the compute. +PTX global loads and stores can carry cache operators and eviction-priority qualifiers. Global loads can also carry L2 prefetch-size hints. These controls let a kernel express a preference for a particular access; they do not guarantee cache admission, residence, eviction time, hit rate, or performance. -## Pattern +| Form | PTX meaning | Not guaranteed | +|---|---|---| +| `L1::no_allocate` | Selects an L1 eviction priority that may be applied | That the access bypasses L1 | +| `L1::evict_first` | Requests first-eviction priority | Immediate eviction | +| `L1::evict_last` | Requests last-eviction priority | Persistent residence | +| `L2::64B`, `128B`, `256B` | Hints that additional data of that size be fetched into L2 | A wider memory instruction or a completed prefetch | +| `L2::cache_hint` with a policy operand | Supplies a created L2 eviction policy | That the hint is respected or changes memory consistency | -```asm -; Matrix A (streamed once per row, never reused): bypass L1 -; Avoids polluting L1 with one-shot data -ld.global.L1::no_allocate.v4.u64 {a0,a1,a2,a3}, [addr_a]; +The following are legal PTX 9.0 instruction fragments when their operands and addresses are declared with matching types. Each vector access shown is 16 bytes and therefore requires 16-byte natural alignment: -; Vector B (reused across BLOCK_M rows): keep in L1 -ld.global.L1::evict_last.v4.u64 {b0,b1,b2,b3}, [addr_b]; +```ptx +.reg .b64 addr_a, addr_b, addr_c; +.reg .u32 a<4>, b<4>, c<4>; -; Streaming output: evict immediately after write -st.global.L1::evict_first.v2.u64 [addr_c], {c0, c1}; +ld.global.L1::no_allocate.v4.u32 {a0, a1, a2, a3}, [addr_a]; +ld.global.L1::evict_last.v4.u32 {b0, b1, b2, b3}, [addr_b]; +st.global.L1::evict_first.v4.u32 [addr_c], {c0, c1, c2, c3}; ``` -## GPU Mode NVFP4 GEMV Winner Technique +The common “stream A, retain B” explanation is a hypothesis about reuse and interference, not the semantics of the code. Even with correct alignment, the hardware may apply the priorities differently than a literal bypass/keep model suggests. -Rank 1 submission used **different qualifiers per K-dimension variant**: -- K=16384 (large): aggressive `L1::no_allocate` on A (huge streaming matrix) -- K=2048 (small): relaxed balance since B is smaller relative to cache +## Selecting a policy -## Measurable Impact +Start from the concrete access trace rather than a kernel label or tensor name: -- NVFP4 GEMV: 443μs → 27μs (16x improvement) came partly from cache policy + PTX byte unpacking -- On memory-bound kernels, cache policy can be the dominant lever +1. Identify which addresses each warp touches, the reuse distance for each line, the live working set, and competing traffic at the same launch shape. +2. Establish a correct default-policy baseline. Hold mapping, vector width, decode, unrolling, register controls, compiler, inputs, and launch parameters constant. +3. Change one qualifier at a time. Re-run the same correctness oracle, including sizes that exercise alignment and tail paths. +4. Record generated instructions and resources, then profile cache hit behavior, requested/transferred bytes, stalls, achieved bandwidth, and active warps with metrics available on the installed tool and target. +5. Benchmark every production shape with the same warmup, synchronization, repetitions, and statistic. Retain only scoped improvements; a policy may help one shape and regress another. -## When To Use +An input being larger than L2, a kernel being described as memory-bound, or a tensor being called streamed/reused does not by itself predict a useful qualifier. Concurrent blocks can reuse data, a nominally reused vector may exceed the effective working set, and a hint can alter other traffic. Those are measurements, not properties inferred from names. -- Memory-bound kernels (profile with Nsight Compute first) -- Tensor with clear "streaming" vs "reused" access patterns -- Inputs > L2 cache size (B200: 126MB) -- Separate M and N tile loading patterns in GEMM +## Evidence boundary for the NVFP4 reports + +Amandeep Singh reports that three inspected B200 NVFP4 GEMV solutions used `L1::no_allocate` for A and `L1::evict_last` for B alongside raw PTX decode, wide loads, exact-K specialization, tighter register caps, and—in one solution—sharing B reads across M rows. The report does not release those contestant implementations or a cache-policy-only ablation, and it does not support the former page's rank-1 per-K policy attribution. + +Yue Zhang reports approximately 39 microseconds for a stage combining removal of B shared-memory staging, per-thread tiles, `float4` loads, and hardware conversion, followed by approximately 27 microseconds for a vectorized PTX FP4/scale-decode stage. Those endpoints change the load and decode path together. They do not measure a cache-hint-only speedup or establish cache policy as the dominant lever for memory-bound kernels. diff --git a/wiki/techniques/cccl-memory-primitives.md b/wiki/techniques/cccl-memory-primitives.md index 179146123..9a63245ec 100644 --- a/wiki/techniques/cccl-memory-primitives.md +++ b/wiki/techniques/cccl-memory-primitives.md @@ -2,49 +2,47 @@ id: technique-cccl-memory-primitives title: CCCL CUB Memory Primitives For Selection And Scan type: technique -architectures: -- sm100 -- sm90 -tags: -- cuda-cpp -- top-k-selection -- parallel-scan -- vectorized-loads -- shared-memory-optimization -- tile-scheduling +architectures: [sm100, sm90] +tags: [cuda-cpp, top-k-selection, parallel-scan, vectorized-loads, cache-policy] confidence: source-reported -reproducibility: snippet -prerequisites: -- technique-vectorized-loads -- technique-swizzling -related: -- pattern-memory-bound -- technique-tile-scheduling -sources: -- pr-cccl-3559 -- pr-cccl-6152 -blackwell_relevance: CUB scan/top-k tuning PRs expose reusable policy and dispatch choices for B200 selection, fill, and prefix-style helper kernels. +reproducibility: concept +prerequisites: [technique-vectorized-loads, technique-cache-policy] +related: [pattern-memory-bound, technique-tile-scheduling] +sources: [pr-cccl-3559, pr-cccl-6152] +blackwell_relevance: "PR 3559 adds type- and offset-specific SM100 scan policies; PR 6152 is only a generic TopK debug-log correction and provides no Blackwell tuning evidence." --- -## Use +## Exact Source Scope -Use CCCL/CUB PRs when the bottleneck is not tensor math but a memory primitive: -scan, top-k selection, fill, histogram, reduce, or block load/store policy. The -goal is usually a policy or dispatch idea rather than copying a full CUB -primitive into an application kernel. +The two linked PRs have different evidentiary value: -```cuda -// Minimal policy probe shape for an application-specific top-k or scan helper. -template -struct PrimitivePolicy { - static constexpr int block_threads = BLOCK_THREADS; - static constexpr int items_per_thread = ITEMS_PER_THREAD; - static constexpr bool vectorized = ITEMS_PER_THREAD >= 4; -}; -``` +| PR | Semantic scope | Transfer value | +|---|---|---| +| CCCL 3559, captured merge `25523da2` | Adds B200/SM100 exclusive-sum scan tuning, expands policy classification to input/output/accumulator/offset types, and updates scan dispatch/tests. | A concrete example of architecture- and type-specific CUB policy selection. | +| CCCL 6152, captured merge `3fb05826` | Changes only `CUB_DEBUG_LOG` formatting and stale variable names in `DispatchTopK`. | Evidence for the corrected debug output, not TopK algorithm, performance, determinism, or SM100 tuning. | -## Transfer Notes +Do not use PR 6152 as evidence for DSA TopK design or performance. Its captured key file exposes surrounding TopK implementation for inspection, but the PR's semantic delta is the small debug-only patch. -- Check whether the PR tuned SM100 separately from SM90. -- Validate determinism and tie-breaking before using atomic or relaxed variants. -- For DSA TopK, keep radix/select cost separate from score-computation cost. +## What PR 3559 Actually Tunes + +The new `sm100_tuning` specializations select a tuple rather than a single “vectorized” bit: + +| Policy dimension | Examples in the captured source | +|---|---| +| Classification | input value size, accumulator type/size, offset size, and recognized `plus` operator | +| Work partition | `threads` and `items` per thread | +| Memory policy | `BLOCK_LOAD_*`, `BLOCK_STORE_*`, and `LOAD_DEFAULT` or `LOAD_CA` | +| Look-back behavior | an exponential backoff/delay constructor and its parameters | +| Fallback | `Policy1000` selects a matching SM100 specialization; otherwise it falls back to `Policy900`; the double specialization explicitly inherits an SM90 tuning | + +`items >= 4` does not establish vectorized access. Vector width also depends on iterator contiguity, element type, alignment, load/store algorithm, compiler lowering, and the executed policy. Treat items-per-thread, block size, load/store algorithm, cache modifier, and delay policy as separate variables. + +## Transfer Workflow + +1. Identify the exact primitive and semantic contract: inclusive/exclusive scan, operator, input/output/accumulator types, offset width, aliasing, and empty/large-size behavior. +2. Follow runtime architecture dispatch to the active policy. Confirm whether SM100 has a matching specialization or inherits the SM90/default route. +3. Reproduce the upstream baseline and changed policy over the size/type/operator matrix relevant to the application. A policy comment containing benchmark ratios is source context, not a portable speedup guarantee. +4. Change one policy dimension at a time where practical and record time, bandwidth, occupancy, register/spill data, and correctness. +5. For selection, separately verify membership, output count, ordering, ties, NaNs, signed zero, key/value association, and repeatability. Those properties are not established by PR 6152. + +The linked evidence does not support claims about fill, histogram, reduce, block-load/store vectorization, or application-level DSA score computation. Add a directly relevant CCCL source before transferring policy conclusions to those primitives. diff --git a/wiki/techniques/chunk-parallelism.md b/wiki/techniques/chunk-parallelism.md index 9f459dbc5..f0224ed78 100644 --- a/wiki/techniques/chunk-parallelism.md +++ b/wiki/techniques/chunk-parallelism.md @@ -1,62 +1,52 @@ --- id: technique-chunk-parallelism -title: "Chunk-Based Parallelism for Linear Attention" +title: "Chunkwise Parallelism for Linear Recurrences" type: technique architectures: [sm100, sm90] tags: [chunk-parallelism, linear-attention, gated-delta-net, pipeline-stages] -confidence: source-reported -reproducibility: snippet +confidence: verified +evidence_basis: + - source_id: doc-tfla + evidence_type: paper +reproducibility: concept prerequisites: [] related: [kernel-gated-delta-net, kernel-nsa] -sources: [blog-gated-delta-net, blog-nsa, doc-tfla] -blackwell_relevance: "Chunk size scales with TMEM capacity on Blackwell; larger chunks = better tensor core utilization." +sources: [blog-gated-delta-net, doc-tfla, blog-qwen3-next-architecture] +blackwell_relevance: "Blackwell can accelerate concrete chunkwise kernels, but neither TMEM capacity nor the architecture alone determines a correct or optimal chunk size." --- -# Chunk-Based Parallelism +# Chunkwise Parallelism for Linear Recurrences -## Overview +## Mechanism -Linear attention variants (GatedDeltaNet, RetNet, Mamba) have O(n) complexity but naive implementations are sequential. Chunk-based parallelism divides the sequence into chunks of size C, computes within each chunk in parallel (matmul-friendly), and propagates state between chunks sequentially. +Some linear recurrent models admit an algebraically equivalent chunkwise formulation. Work local to a chunk can then be expressed with parallel matrix operations, while the state passed across chunk boundaries preserves the recurrence's sequence order. The exact transform is model-specific: it must be derived from the recurrence, not replaced by a generic attention matrix and additive state update. -## Pattern +A correct implementation separates at least these obligations: -```python -@triton.jit -def chunk_parallel_linear_attn(Q, K, V, State, Output, - chunk_size: tl.constexpr, - d: tl.constexpr): - # Grid: (num_chunks, num_heads, batch) - chunk_id = tl.program_id(0) +1. Compute the chunk-local quantities required by the model's exact recurrence. +2. Resolve boundary states in sequence order, either with an associative scan supported by the formulation or with explicitly ordered stages or launches. +3. Combine each chunk's local result with its incoming boundary state and emit outputs in the original token order. +4. Validate outputs and final states against a token-by-token reference across variable sequence lengths, chunk tails, batches, heads, dtypes, and gate extremes. - # Load chunk of Q, K, V - q = tl.load(Q + chunk_id * chunk_size * d + offsets) # [C, d] - k = tl.load(K + chunk_id * chunk_size * d + offsets) - v = tl.load(V + chunk_id * chunk_size * d + offsets) +An ordinary GPU grid does not imply increasing program-ID execution order or grid-wide synchronization. Programs for every chunk therefore cannot safely read and overwrite one shared state pointer in a single unordered launch. A staged algorithm must make the boundary-state dependency explicit. - # Intra-chunk: parallel O(C^2) attention-like compute - scores = tl.dot(q, tl.trans(k)) - o_intra = tl.dot(scores, v) +## Verified implementations and scope - # Inter-chunk: sequential state propagation - state = tl.load(State) # From previous chunk - o_inter = tl.dot(q, state) +The pinned NVlabs GatedDeltaNet repository uses chunkwise Triton kernels for training and a WY representation of the gated delta rule. That implementation is direct evidence for GatedDeltaNet chunking, but it is not equivalent to a generic `scores = Q @ K.T; output = scores @ V` snippet and does not supply a universal chunk-size rule. - # Combine and update state - output = o_intra + o_inter - state = update_state(state, k, v) +Tiled Flash Linear Attention (TFLA) starts from the chunkwise formulation of linear RNNs and adds another level of sequence parallelization within a chunk. The authors state that this permits arbitrarily large chunks, raises arithmetic intensity, and reduces intermediate-state materialization. The paper and pinned official code apply the method to mLSTM and report H100 results; they do not establish a GatedDeltaNet implementation, a Blackwell/TMEM implementation, or a recursive-tiling API. - tl.store(Output + offsets, output) - tl.store(State, state) -``` +Qwen3-Next is a hybrid-model example, not evidence that one kernel handles every layer. Its immutable 48-layer configuration repeats three Gated DeltaNet linear-attention layers and one full-attention layer twelve times: 36 GDN and 12 full-attention layers. A chunkwise GDN backend applies only to the GDN recurrence; the full-attention layers retain their distinct attention implementation and context-dependent cache. -## Size Tradeoff +## Choosing a chunk configuration -- **Small chunks (C=32)**: low latency decode, fewer intermediate materializations -- **Large chunks (C=256-512)**: better tensor core utilization, higher throughput for prefill -- **TFLA (Tiled FLA)**: two-level tiling allows arbitrary chunk sizes via recursive tiling +There is no source-backed universal rule that `C=32` is a decode choice or that `C=256-512` is the prefill optimum. Choose only among chunk sizes supported by the exact algorithm and backend, then measure the tradeoff: -## When To Use +- arithmetic intensity and matrix-instruction utilization; +- intermediate-state and workspace traffic; +- registers, shared memory, and occupancy; +- tail handling and variable-length metadata; +- launch count and boundary-scan cost; +- latency and throughput for the intended batch and sequence distribution. -- Linear attention (O(n) complexity) -- Recurrent state models (Mamba, GatedDeltaNet, Delta Rule) -- Hybrid architectures mixing linear + full attention (Qwen3-Next uses 3:1 ratio) +Compare candidates with identical compiler/software revisions, launch inputs, correctness oracles, warmups, synchronization, and repeated-trial statistics. Treat a selected size as scoped to the model dimensions, dtype, GPU, backend, and workload. TMEM availability on SM100 may change an implementation's resource design, but it does not by itself prove that a larger chunk is legal or faster. diff --git a/wiki/techniques/double-buffering.md b/wiki/techniques/double-buffering.md index 496634657..e476cfa16 100644 --- a/wiki/techniques/double-buffering.md +++ b/wiki/techniques/double-buffering.md @@ -4,243 +4,111 @@ title: "Double/Multi-Buffering Patterns" type: technique architectures: [sm100, sm90] tags: [double-buffering, tmem, pipeline-stages] -confidence: source-reported -reproducibility: snippet +confidence: verified +evidence_basis: + - source_id: doc-nvidia-tuning-guide + evidence_type: official-doc +reproducibility: pseudocode prerequisites: [hw-tmem] related: [hw-tmem, technique-pipeline-stages, technique-epilogue-fusion] sources: [blog-tcgen05-tutorial, doc-nvidia-tuning-guide, pr-flashinfer-2387] -blackwell_relevance: "TMEM double-buffering is Blackwell-specific (half of 512 columns each); SMEM double-buffering transfers from Hopper." +blackwell_relevance: "SM100 can keep independent accumulator regions in TMEM while a separate SMEM pipeline stages operands; either mechanism remains an implementation and tuning choice." --- -## Overview - -Double-buffering (and multi-buffering) allocates two or more copies of a data buffer so that one copy can be written while another is read. On Blackwell, this pattern applies at two distinct levels: (1) TMEM double-buffering for overlapping MMA accumulation with epilogue readout, and (2) SMEM multi-stage buffering for overlapping TMA loads with MMA consumption. Both levels operate simultaneously in a well-optimized kernel. - -## TMEM Double-Buffering - -Tensor Memory (TMEM) on Blackwell has 128 rows x 512 columns of 32-bit elements (256 KB per SM). For a standard GEMM with TILE_M=128, TILE_N=256, the accumulator requires 128 x 256 = 32,768 elements, which fits in half the TMEM column space (256 out of 512 columns). The other 256 columns serve as the second buffer. - -```cuda -// TMEM double-buffering: ping-pong between two 128x256 accumulator regions -// -// TMEM physical layout (per SM): -// +------ 256 cols ------+------ 256 cols ------+ -// | | | -// | Buffer A (active) | Buffer B (drain) | 128 rows -// | MMA writes here | Epilogue reads | -// | | | -// +----------------------+----------------------+ -// -// After tile completes, roles swap: -// Buffer A becomes "drain", Buffer B becomes "active" - -#define TMEM_BUF_A_OFFSET 0 -#define TMEM_BUF_B_OFFSET 256 -#define TMEM_COLS_PER_BUF 256 - -__device__ void tmem_double_buffer_mainloop( - const GemmParams& params, - int num_output_tiles) -{ - int warp_id = threadIdx.x / 32; - int lane_id = threadIdx.x % 32; - - // Synchronization between MMA warp and epilogue warps - __shared__ uint64_t mbar_acc_ready[2]; // MMA done, epilogue can read - __shared__ uint64_t mbar_acc_drained[2]; // Epilogue done, MMA can reuse - - if (threadIdx.x == 0) { - mbarrier_init(&mbar_acc_ready[0], 1); - mbarrier_init(&mbar_acc_ready[1], 1); - mbarrier_init(&mbar_acc_drained[0], 1); - mbarrier_init(&mbar_acc_drained[1], 1); - } - __syncthreads(); - - for (int tile = 0; tile < num_output_tiles; tile++) { - int buf = tile % 2; - int tmem_offset = buf ? TMEM_BUF_B_OFFSET : TMEM_BUF_A_OFFSET; - - if (warp_id == 1) { - // === MMA WARP === - // Wait for epilogue to finish draining this buffer - if (tile >= 2) { - mbarrier_wait(&mbar_acc_drained[buf]); - } - - // Clear accumulator region before new tile - if (lane_id == 0) { - tmem_clear(tmem_offset, TMEM_COLS_PER_BUF); - } - __syncwarp(); - - // Accumulate K-tiles into this TMEM buffer - for (int k = 0; k < num_k_tiles; k++) { - // (TMA load sync omitted -- see pipeline-stages) - if (lane_id == 0) { - tcgen05_mma_accumulate(tmem_offset); - } - __syncwarp(); - } - - // Signal epilogue that accumulator is ready - if (lane_id == 0) { - mbarrier_arrive(&mbar_acc_ready[buf]); - } - - } else if (warp_id >= 2) { - // === EPILOGUE WARPS === - // Wait for MMA to finish accumulating - mbarrier_wait(&mbar_acc_ready[buf]); - - // Read TMEM and store to global memory - int rows_per_warp = TILE_M / 14; - int my_row = (warp_id - 2) * rows_per_warp; - for (int r = my_row; r < my_row + rows_per_warp; r++) { - for (int c = lane_id; c < TMEM_COLS_PER_BUF; c += 32) { - float val = tmem_load_f32(r, tmem_offset + c); - // Apply epilogue and store... - params.C_ptr[r * params.N + c] = __float2half(val); - } - } - - // All epilogue warps must finish reading TMEM before the MMA - // warp can overwrite this half-buffer. Each epilogue warp arrives - // on a shared mbarrier; the mbarrier is initialized with - // arrival_count = NUM_EPILOGUE_WARPS (e.g. 14 for warps 2-15). - // The MMA warp waits on this mbarrier before reusing the buffer. - if (lane_id == 0) { - mbarrier_arrive(&mbar_acc_drained[buf]); - } - // mbar_acc_drained[buf] fires only after ALL epilogue warps arrive - } - } -} -``` +# Double/Multi-Buffering Patterns -## SMEM Multi-Stage Buffering - -Shared memory multi-buffering provides multiple copies of the A and B input tiles so that TMA loads can overlap with MMA consumption: - -```cuda -// SMEM multi-stage buffer allocation -// 3 stages, each holding one A tile and one B tile -// -// Memory layout: -// +--------+--------+--------+ -// | Stage0 | Stage1 | Stage2 | -// | A0 B0 | A1 B1 | A2 B2 | -// +--------+--------+--------+ -// -// Pipeline state at steady state: -// Stage 0: TMA loading (k+2) -// Stage 1: Ready, waiting for MMA -// Stage 2: MMA consuming (k) - -constexpr int SMEM_STAGES = 3; -constexpr int TILE_A_BYTES = TILE_M * TILE_K * sizeof(half); // e.g. 16 KB -constexpr int TILE_B_BYTES = TILE_K * TILE_N * sizeof(half); // e.g. 32 KB -constexpr int STAGE_BYTES = TILE_A_BYTES + TILE_B_BYTES; // e.g. 48 KB -// Total SMEM for operands: 3 * 48 KB = 144 KB - -struct SmemBuffers { - half A[SMEM_STAGES][TILE_M][TILE_K]; - half B[SMEM_STAGES][TILE_K][TILE_N]; - uint64_t mbar_full[SMEM_STAGES]; // TMA done -> MMA can consume - uint64_t mbar_empty[SMEM_STAGES]; // MMA done -> TMA can refill -}; -``` +## What the pattern guarantees + +Double- or multi-buffering reserves disjoint storage regions and transfers ownership between producers and consumers. While a consumer reads stage `s`, a producer may fill another stage. The storage alone does not establish overlap: the program also needs a completion edge before consumption and a read-complete edge before reuse. + +On SM100 these ideas can be applied independently: + +- SMEM stages can hold operand tiles while TMA production overlaps MMA consumption. +- Separate TMEM regions can hold output accumulators while an epilogue drains a completed region and MMA writes a different region. + +The pinned `matmul_v6.cu` from Gau Nernst's tutorial uses both mechanisms. That is a concrete design, not a rule that every optimized Blackwell GEMM needs both. -## Combined TMEM + SMEM Double-Buffering - -A fully optimized Blackwell GEMM uses both levels simultaneously: - -```cuda -// Combined double-buffering: SMEM (3-stage) + TMEM (2-buffer) -// -// Outer loop: output tiles (TMEM double-buffered) -// Inner loop: K-tiles (SMEM 3-stage pipelined) -// -// Timeline for 2 output tiles, 6 K-tiles each: -// -// TMEM buf: |---- A (tile 0) ----|---- B (tile 1) ----| -// MMA warp: | k0 k1 k2 k3 k4 k5 | k0 k1 k2 ... -// TMA warp: |k2 k3 k4 k5 - - k2 k3 k4 k5 ... -// Epilogue: | idle | drain A | drain B | -// SMEM stage: |0 1 2 0 1 2 |0 1 2 0 1 2 | -// -// Key insight: the epilogue of tile 0 overlaps with the MMA of tile 1. -// This is only possible because they use different TMEM buffers. - -__device__ void full_double_buffered_gemm(const GemmParams& params) { - int warp_id = threadIdx.x / 32; - - for (int out_tile = 0; out_tile < num_output_tiles; out_tile++) { - int tmem_buf = out_tile % 2; - - if (warp_id == 0) { - // TMA producer: fill SMEM stages for this output tile's K-loop - tma_producer_loop(params, out_tile); - } else if (warp_id == 1) { - // MMA: consume SMEM stages, accumulate into TMEM[tmem_buf] - mma_consumer_loop(params, tmem_buf); - } else { - // Epilogue: drain TMEM[1-tmem_buf] from previous tile - if (out_tile > 0) { - epilogue_drain(params, out_tile - 1, 1 - tmem_buf); - } - } - - // Lightweight sync point between tiles - // (mbarrier-based, not __syncthreads) - } - - // Final epilogue for last tile - if (warp_id >= 2) { - epilogue_drain(params, num_output_tiles - 1, - (num_output_tiles - 1) % 2); - } -} +## TMEM regions + +The CTA-visible TMEM address structure has 128 lanes and 512 columns of 32-bit cells. Allocation is column-granular across all 128 lanes. A kernel that allocates 512 columns may choose two 256-column regions: + +```text +region[0] = columns [0, 256) +region[1] = columns [256, 512) ``` -## Comparison with Hopper +This equal split is one implementation choice. The actual invariant is that every simultaneously live region is inside the allocation and does not alias another region whose MMA or epilogue access is still outstanding. Region sizes can differ, and logical element capacity depends on the tcgen05 instruction kind and packing rather than the raw cell count alone. + +For each region, prove this lifecycle: + +| Transition | Required edge | +|---|---| +| free → MMA-owned | every prior epilogue load from the region has completed | +| MMA-owned → ready | relevant tcgen05 work is committed to an mbarrier and completion is observed | +| ready → epilogue-owned | readers observe the matching barrier phase before `tcgen05.ld` | +| epilogue-owned → free | all collective loads complete with `tcgen05.wait::ld`, then every reader reports completion | + +TMEM must first be allocated by the required participating warp and its address safely published. Matching collective deallocation is required before every kernel exit. Reused mbarriers need the correct arrival count and phase/parity state; a one-time wait on an address is not a reusable two-buffer protocol. -On Hopper (SM90), there is no TMEM. The accumulator lives in registers, and double-buffering the accumulator requires explicit register management: +## SMEM stages -```cuda -// Hopper: accumulator double-buffering uses register arrays -// Each warpgroup maintains two register-based accumulators -// This doubles register pressure and reduces occupancy +An SMEM operand stage normally has its own full/empty state: -// Hopper approach (register pressure is the primary constraint): -float acc_buf0[REG_TILE_M][REG_TILE_N]; // First accumulator -float acc_buf1[REG_TILE_M][REG_TILE_N]; // Second accumulator -// Total: 2 * REG_TILE_M * REG_TILE_N * 4 bytes per thread -// For a 64x256 tile with 128-thread warpgroup: -// each thread holds 2 * (64/4) * (256/128) * 4 = 2 * 16 * 2 * 4 = 256 bytes -// = 64 registers just for accumulators +1. The producer waits until stage `s` is empty. +2. It issues the stage's TMA copies and accounts for expected transaction bytes. +3. The consumer observes the full barrier's matching phase before using the stage. +4. After the final dependent read, the consumer releases the empty barrier for reuse. -// Blackwell approach: TMEM holds both buffers with zero register cost -// MMA warp uses ~0 registers for accumulators -// All 256 KB of TMEM is dedicated accumulator space +For binary16 `A[128,64]` and `B[64,256]`, the unpadded payload arithmetic is: + +```text +A = 128 × 64 × 2 bytes = 16 KiB +B = 64 × 256 × 2 bytes = 32 KiB +one stage = 48 KiB; three stages = 144 KiB ``` -| Aspect | Hopper (Registers) | Blackwell (TMEM) | -|--------|-------------------|------------------| -| Accumulator storage | Thread-local registers | CTA-wide TMEM | -| Double-buffer cost | 2x register usage | Zero register cost | -| Typical occupancy impact | Reduces from 2 to 1 CTA/SM | No impact | -| Epilogue access | Direct (already in registers) | TMEM load required | -| Max accumulator size | ~16K elements (register limited) | 128x512 = 65K elements | +That is not a complete C++ shared-storage layout. Barriers, descriptors, epilogue scratch, padding, alignment, and swizzled physical layouts also consume or constrain shared memory. Compute capability 10.0 supports up to 228 KiB of shared memory per SM, while per-block opt-in limits and all other residency resources still apply. + +FlashInfer PR 2387 is a second pinned example. Its merged `selective_state_update.cuh` has `state[numStages][...]` plus `bar_full` and `bar_empty` arrays and uses stage-specific producer/consumer handoffs. One path selects three stages and another caps a geometry-derived count at four, illustrating why stage count is policy rather than an architecture-wide default. + +## Hopper and Blackwell storage tradeoff + +For Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, one D fragment is 128 registers per thread; two simultaneously live fragments are 256 registers, or 1024 bytes, per thread before other live values. That arithmetic describes the ISA fragment, not a promise that a C++ array stays in registers or that a particular launch is resident. + +On SM100, tcgen05 keeps resident D in TMEM. This avoids a long-lived per-thread D vector, but the kernel still uses registers for addresses, descriptors, loop and barrier state, `tcgen05.ld` destinations, and epilogue temporaries. TMEM buffering also consumes TMEM capacity and synchronization state. Neither architecture therefore has a fixed occupancy result from “double buffering” alone. + +| Concern | Hopper WGMMA D | Blackwell tcgen05 D | +|---|---|---| +| Resident storage | per-thread register fragment | allocated TMEM region | +| Two live outputs | two disjoint register fragments | two disjoint TMEM regions | +| Epilogue access | dependent use after WGMMA completion | collective `tcgen05.ld` and `tcgen05.wait::ld` after MMA completion | +| Main resource question | compiled registers, SMEM, block shape, and other limits | TMEM columns plus compiled registers, SMEM, block/cluster shape, and other limits | + +## Decide with a controlled comparison + +Buffering is useful only when useful producer/consumer work overlaps enough to repay extra storage and coordination. For each concrete variant: + +1. Keep shape, datatype, outputs, launch policy, warmup, and timing statistics identical. +2. Compare one versus multiple TMEM regions and the legal SMEM stage counts. +3. Record compiled registers/thread, spill traffic, static/dynamic SMEM, TMEM columns, barriers, threads/CTA, and cluster shape. +4. Use the CUDA occupancy APIs for the actual resource record; do not infer a universal CTA/SM count from an instruction tile. +5. Inspect profiler pipeline and barrier stalls. PTX specifies mbarrier semantics, not a fixed “few cycles” latency. +6. Inspect generated PTX/SASS and execute correctness tests that force region wraparound and pipeline-tail paths. + +Useful negative tests deliberately delay the epilogue, swap a barrier phase, omit the final drain, or reuse a region early. A correct test should detect stale/overwritten output or time out under a watchdog rather than silently accepting the broken schedule. -## When to Use +## Primary references -- **TMEM double-buffering**: Always use on Blackwell when the epilogue takes more than trivial time. The only cost is the TMEM space, which is plentiful. -- **SMEM multi-stage buffering**: Always use for GEMM/attention mainloops. 3 stages is the default; increase to 4-5 only if the K-loop is long and memory latency is high. -- **Combined**: The standard approach for production Blackwell GEMM kernels. Both CUTLASS and CuTe-DSL kernels use this pattern. +- [PTX ISA 9.0 Tensor Memory](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory) +- [PTX ISA 9.0 tcgen05 allocation](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc) +- [PTX ISA 9.0 tcgen05 load and wait](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld) +- [PTX ISA 9.0 mbarrier](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier) +- [CUDA 13.0.2 Blackwell Tuning Guide](https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy) +- [Pinned combined SMEM/TMEM example](https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu) +- [FlashInfer PR 2387 merged source](https://github.com/flashinfer-ai/flashinfer/blob/18804cd51734cccf807356d017733bc757677f15/include/flashinfer/mamba/selective_state_update.cuh) -## Caveats +## Related -- TMEM double-buffering requires that the accumulator fits in half the TMEM columns (256 out of 512). For very wide tiles (TILE_N > 256 with FP32 accumulators), the tile must be split or a different buffering strategy used. -- SMEM multi-stage buffering is constrained by the 228 KB SMEM limit. With 3 stages of large tiles, there may not be enough SMEM left for epilogue scratch space. -- The mbarrier synchronization between TMEM buffers adds a few cycles of overhead per tile. For kernels with very few K-iterations per tile, this overhead is proportionally larger. +- [Tensor Memory](../hardware/tmem.md) — allocation, addressing, and lifetime rules +- [pipeline stages](pipeline-stages.md) — SMEM pipeline construction and tail handling +- [epilogue fusion](epilogue-fusion.md) — work performed while draining an output tile diff --git a/wiki/techniques/epilogue-fusion.md b/wiki/techniques/epilogue-fusion.md index 7e54eed90..5a93fae8e 100644 --- a/wiki/techniques/epilogue-fusion.md +++ b/wiki/techniques/epilogue-fusion.md @@ -9,8 +9,13 @@ tags: - epilogue-fusion - tmem - warp-specialization -confidence: source-reported -reproducibility: snippet +confidence: verified +evidence_basis: + - source_id: doc-cutlass-blackwell + evidence_type: official-doc + - source_id: pr-vllm-16032 + evidence_type: upstream-code +reproducibility: pseudocode prerequisites: - hw-tmem - technique-warp-specialization @@ -22,200 +27,119 @@ sources: - doc-cutlass-blackwell - blog-colfax-cutlass - pr-vllm-16032 -blackwell_relevance: TMEM-based epilogue fusion is new to Blackwell; Hopper pattern - provides conceptual foundation. +blackwell_relevance: SM100 tcgen05 accumulators are read from TMEM into registers before supported output transforms and stores. artifact_dir: artifacts/kernels/epilogue-fusion --- -## Overview - -Epilogue fusion overlaps the post-MMA operations (scaling, bias addition, activation functions, quantization, store to global memory) with ongoing MMA computation. On Blackwell, the accumulator lives in TMEM rather than registers, enabling dedicated epilogue warps (typically warps 2-15) to read TMEM concurrently while the MMA warp (warp 1) continues accumulating the next tile. This overlap is achieved by double-buffering the TMEM accumulator: the MMA warp writes to one half while the epilogue warps read from the other half. - -## TMEM-to-Register Epilogue Path - -On Blackwell, the MMA result resides in Tensor Memory (TMEM). Epilogue warps must copy relevant portions of TMEM into registers before applying element-wise operations and writing to global memory: - -```cuda -// Epilogue warp: read TMEM accumulator, apply fused operations, store -// This runs on warps 2-15 while warp 1 continues MMA on next tile -__device__ void epilogue_warp_fn( - int warp_id, - int tile_m, int tile_n, - float scale, const float* bias, - half* C, int ldc) -{ - int lane_id = threadIdx.x % 32; - - // Each epilogue warp handles a stripe of the output tile - // 14 warps, TILE_M = 128 -> ~9 rows per warp - int rows_per_warp = (TILE_M + 13) / 14; - int row_start = (warp_id - 2) * rows_per_warp; - int row_end = min(row_start + rows_per_warp, TILE_M); - - for (int r = row_start; r < row_end; r++) { - int global_row = tile_m * TILE_M + r; - - for (int c = lane_id; c < TILE_N; c += 32) { - // Step 1: Load accumulator from TMEM into register - float acc = tmem_load_f32(r, c); - - // Step 2: Fused epilogue operations (all in registers) - // Scale - acc *= scale; - // Bias - acc += bias[tile_n * TILE_N + c]; - // ReLU activation - acc = fmaxf(acc, 0.0f); - - // Step 3: Store to global memory - int global_col = tile_n * TILE_N + c; - C[global_row * ldc + global_col] = __float2half(acc); - } - } -} -``` +# Epilogue Fusion + +## Definition and boundary + +Epilogue fusion computes output transformations—such as `alpha*acc + beta*C`, bias, activation, output conversion, or auxiliary values—inside the producer kernel before final output materialization. It can remove an intermediate tensor or launch only when the unfused comparison would otherwise materialize or launch that work. + +Fusion does not by itself imply overlap with the next MMA tile. A schedule may drain one completed accumulator after the mainloop, or it may use disjoint storage and specialized roles to overlap a completed region's epilogue with independent MMA work. Participant counts and role IDs come from the concrete kernel; there is no architectural default of fourteen epilogue warps. + +## SM100 TMEM-to-output path + +For tcgen05, D resides in TMEM. An ordinary arithmetic epilogue first uses a legal collective `tcgen05.ld` mapping, or a library wrapper around it, to transfer a partition into per-thread registers. The load is asynchronous with respect to the issuing thread, so the matching `tcgen05.wait::ld` completion must occur before those registers are consumed. -## Overlapping MMA with Epilogue via Double-Buffering - -The key to epilogue fusion on Blackwell is TMEM double-buffering. The 512-column TMEM space is split into two halves (columns 0-255 and 256-511). While the MMA warp accumulates into one half, the epilogue warps drain the other: - -```cuda -// TMEM double-buffering for MMA-epilogue overlap -// -// TMEM layout: 128 rows x 512 columns (32-bit elements) -// Buffer A: columns [0, 255] -- 128 x 256 accumulator -// Buffer B: columns [256, 511] -- 128 x 256 accumulator -// -// Timeline: -// Tile 0: MMA -> buffer A | epilogue idle (no prior result) -// Tile 1: MMA -> buffer B | epilogue reads buffer A -// Tile 2: MMA -> buffer A | epilogue reads buffer B -// Tile 3: MMA -> buffer B | epilogue reads buffer A -// ... ping-pong continues ... - -__device__ void mma_epilogue_overlap( - const GemmParams& params, - int num_tiles) -{ - int warp_id = threadIdx.x / 32; - - __shared__ uint64_t mbar_mma_done[2]; // One per TMEM buffer half - __shared__ uint64_t mbar_epi_done[2]; // Epilogue completion signals - - if (threadIdx.x == 0) { - for (int i = 0; i < 2; i++) { - mbarrier_init(&mbar_mma_done[i], 1); - mbarrier_init(&mbar_epi_done[i], 1); - } - } - __syncthreads(); - - if (warp_id == 1) { - // === MMA WARP === - for (int t = 0; t < num_tiles; t++) { - int buf = t % 2; // Alternate between buffer halves - - // Wait for epilogue to finish reading this buffer - if (t >= 2) { - mbarrier_wait(&mbar_epi_done[buf]); - } - - // Issue MMA, accumulating into TMEM buffer half - int tmem_col_offset = buf * 256; - tcgen05_mma_with_offset(tmem_col_offset, - params.smem_A, params.smem_B); - - // Signal epilogue that this buffer is ready - mbarrier_arrive(&mbar_mma_done[buf]); - } - - } else if (warp_id >= 2) { - // === EPILOGUE WARPS === - for (int t = 0; t < num_tiles; t++) { - int buf = t % 2; - - // Wait for MMA to finish filling this buffer - mbarrier_wait(&mbar_mma_done[buf]); - - // Read from TMEM buffer half and write to global memory - int tmem_col_offset = buf * 256; - epilogue_store(params, t, tmem_col_offset, warp_id); - - // ALL epilogue warps must finish reading TMEM before MMA reuses - // the buffer. Each epilogue warp arrives on mbar_epi_done; - // mbar_epi_done is initialized with count = NUM_EPILOGUE_WARPS. - // MMA warp waits on this mbarrier before writing to this half. - if (lane_id == 0) { - mbarrier_arrive(&mbar_epi_done[buf]); - } - // mbar_epi_done[buf] fires only after ALL epilogue warps arrive - } - } -} +CUTLASS 4.5.0's `fp16_gemm_2.py` demonstrates the typed route: + +```python +copy_atom_t2r = cute.make_copy_atom( + tcgen05.Ld32x32bOp(tcgen05.Repetition.x32), cutlass.Float32 +) +tiled_copy_t2r = tcgen05.make_tmem_copy(copy_atom_t2r, tCtAcc_epi) +thr_copy_t2r = tiled_copy_t2r.get_slice(tidx) +tTR_tAcc = thr_copy_t2r.partition_S(tCtAcc_epi) +tTR_rAcc = cute.make_rmem_tensor(..., cutlass.Float32) +cute.copy(tiled_copy_t2r, tTR_tAcc_slice, tTR_rAcc) ``` -## CUTLASS Epilogue Patterns - -CUTLASS 4.5.0 provides composable epilogue visitors that fuse arbitrary element-wise operations after GEMM: - -```cuda -// CUTLASS SM100 epilogue with fused scale + bias + activation -// Uses the EVT (Epilogue Visitor Tree) pattern - -using EpilogueOp = cutlass::epilogue::fusion::LinCombEltAct< - cutlass::epilogue::thread::ReLU, // Activation function - float, // Compute type - float, // Scale type - cutlass::half_t // Output type ->; - -// The epilogue descriptor tells CUTLASS how to partition work -// across the 14 epilogue warps -using CollectiveEpilogue = cutlass::epilogue::collective::Sm100EpilogueTmaWarpSpecialized< - cutlass::gemm::TagToStrideC_t, - cutlass::gemm::TagToStrideC_t, - EpilogueOp, - cutlass::gemm::EpilogueDefault // Default tiling ->; - -// In the kernel, the epilogue is invoked after the mainloop: -// epilogue( -// problem_shape, -// collective_mainloop.get_accumulator(), // TMEM reference -// epilogue_params, // scale, bias pointers -// shared_storage // SMEM for TMA stores -// ); +This is an API-routing fragment, not standalone code: the official file supplies the exact tensor layouts, selected load shape, participant group, edge predicates, accumulator-completion handoff, register-to-SMEM conversion, TMA-store pipeline, tail, and TMEM cleanup. + +## Correctness contract + +| Boundary | Proof required before crossing it | +|---|---| +| tcgen05 MMA → TMEM reader | the relevant asynchronous MMA is committed and its completion observed | +| TMEM → result registers | the legal collective load completes before dependent arithmetic | +| registers → output | element/layout mapping and output-edge predicates cover exactly the valid coordinates | +| TMEM reader → region reuse | every reader's load has completed and the matching reusable barrier phase is released | +| kernel tail | all output stores complete as required and every TMEM allocation is collectively freed | + +For a multi-region overlap schedule, simultaneously live TMEM regions must also be disjoint. Equal 256-column halves are one possible policy for a 512-column allocation, not an epilogue-fusion requirement. Reusable mbarriers need correct expected-arrival counts and phase/parity tracking. + +## CUTLASS 4.5.0 fusion interface + +`cutlass::epilogue::fusion::LinCombEltAct` has this parameter order: + +```cpp +template < + template class ActivationFn, + class ElementOutput, + class ElementCompute, + class ElementSource = ElementOutput, + class ElementScalar = ElementCompute, + cutlass::FloatRoundStyle Round = cutlass::FloatRoundStyle::round_to_nearest> +struct LinCombEltAct; ``` -## Common Fused Epilogue Operations +For SM100 construction, `cutlass::epilogue::collective::CollectiveBuilder` receives architecture, operator class, tile/cluster shapes, `EpilogueTileAuto` or an explicit epilogue tile, accumulator/compute/C/D types and layouts, alignment, `EpilogueScheduleAuto` or an explicit supported schedule, and finally a supported fusion operation or callbacks type. + +There is no CUTLASS 4.5.0 type named `Sm100EpilogueTmaWarpSpecialized`. Support is constrained by the exact architecture, operator class, schedule, tile, layout, alignment, datatype, and fusion callback combination. Use `Gemm::can_implement(arguments)` and a tagged example rather than reconstructing a builder signature from prose. + +The vLLM PR 16032 NVFP4 wrapper is one pinned C++ construction example: it uses `CollectiveBuilder<... EpilogueTileAuto, ... EpilogueScheduleAuto>` and derives mainloop shared-memory stages with `StageCountAutoCarveout`. + +## Operation shapes + +| Operation | Data dependency | +|---|---| +| Source linear combination | `D = alpha*acc + beta*C` | +| Per-row/per-column bias | add a separately laid-out broadcast bias input | +| Activation | apply a supported functor such as ReLU, GELU, or SiLU to the output fragment | +| Output conversion/quantization | convert the accumulator fragment and, when required, generate/store scale metadata | +| Gated product | combine two values, for example `SiLU(gate) * up`; requires both inputs and a supported/custom callback | +| Online-softmax rescale | rescale a prior partial output after a new row maximum; requires the attention reduction state | +| Residual | combine a separately supplied residual/source tensor at the defined point in the expression tree | + +These are operation categories, not a claim that every composition is supported by one built-in SM100 visitor. In particular, reductions, multiple outputs, auxiliary tensors, and broadcasts add synchronization and layout requirements beyond a pointwise activation. -| Operation | Description | Typical Use | -|-----------|-------------|-------------| -| Scale + Bias | `y = alpha * acc + beta * C` | Standard GEMM epilogue | -| ReLU / GeLU / SiLU | Element-wise activation | MLP layers | -| Quantize | FP32 accumulator to FP8/FP16 | Inference quantization | -| SwiGLU gate | `y = SiLU(gate) * up` | Gated dual GEMM (LLM FFN) | -| Softmax rescale | `y = acc * exp(max_old - max_new)` | Attention epilogue | -| Residual add | `y = acc + residual` | Transformer blocks | +## Overlap and performance evaluation -## When to Use +When an implementation overlaps the epilogue with independent MMA work, verify both directions of the handoff: compute-complete before read, and load/read-complete before overwrite. Include the prologue (no prior result), steady-state wraparound, final drain, and deallocation path in tests. -- **All Blackwell GEMMs with non-trivial epilogues**: The 14 epilogue warps are available by default in the warp-specialized model. Fusing operations avoids a separate kernel launch and an extra global memory round-trip. -- **Attention kernels**: The softmax rescaling and output accumulation can be overlapped with the next KV tile's MMA. -- **Quantized inference**: FP32-to-FP8 conversion in the epilogue avoids writing FP32 intermediates to global memory. +Evaluate a defined fused/unfused pair with identical inputs, output semantics, launch policy, warmup, and timing statistics. Record registers, spills, SMEM, TMEM columns, barriers, threads/CTA, cluster shape, and occupancy. Sweep supported epilogue group sizes, load shapes, output tiles, and store stages. The PTX ISA defines no equal-share bandwidth model for a fixed number of epilogue warps; diagnose TMEM-load, conversion, barrier, and store bottlenecks with generated code and profiler data. -## Caveats +Fusing output conversion can avoid a global FP32 intermediate when the baseline would write and reread that intermediate. It does not guarantee a speedup: added registers, shared-memory staging, synchronization, edge handling, or reduced occupancy can outweigh saved traffic. -- The epilogue can only read TMEM after the MMA for that tile is complete. The double-buffer synchronization is mandatory to prevent reading partial results. -- TMEM-to-register bandwidth is not unlimited. With 14 warps simultaneously reading TMEM, each warp gets a proportional share. Very wide output tiles (large TILE_N) may bottleneck on TMEM read bandwidth. -- Simple epilogues (just store) waste the 14 epilogue warps. For such cases, consider reducing the CTA size or assigning epilogue warps to other work (e.g., next-tile TMA prefetch). +Useful negative tests delay one reader, skip a barrier phase, reuse a TMEM region early, omit the final store tail, and exercise partial M/N tiles. Each fault should be detected by output comparison or a bounded watchdog. -## Full Reference Implementation +## Artifact provenance -Verbatim upstream code lives in [`artifacts/kernels/epilogue-fusion/full/`](../../artifacts/kernels/epilogue-fusion/full/); labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` header) live in [`artifacts/kernels/epilogue-fusion/variants/`](../../artifacts/kernels/epilogue-fusion/variants/). Every file's SHA-256 and upstream-pinning metadata is in `PROVENANCE.yaml` inside each bundle. +The artifact bundle contains mixed provenance modes: -Query via: +- `full/nvfp4_scaled_mm_kernels.cu` is verbatim from vLLM merge `ed7a29d9f8b48978e3bbf43599d21b4de65387e0` and byte-matches SHA-256 `e8aed5ccb3dd9de26c3aeff159a242a46dcb7c7d8d0351b6c44ff1f8d2f7effa`. +- `full/tmem-load-into-registers-for-epilogue.cu` is an extracted historical snippet from a local source card; it is not an upstream-verbatim kernel and must not be treated as standalone safe code. +- `variants/01-double-buffered-tmem-epilogue-skeleton.cu` is explicitly labeled derived/not-upstream and is incomplete pseudocode. + +Retrieve the page and bundle with: ```bash -python3 scripts/get_page.py technique-epilogue-fusion --include-code +conda run -n base python scripts/get_page.py technique-epilogue-fusion --include-code ``` + +## Primary references + +- [PTX ISA 9.0 tcgen05 load and wait](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld) +- [PTX ISA 9.0 tcgen05 completion](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit) +- [CUTLASS 4.5.0 fusion operations](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/epilogue/fusion/operations.hpp) +- [CUTLASS 4.5.0 collective builder](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/epilogue/collective/collective_builder.hpp) +- [CUTLASS 4.5.0 CuTe DSL epilogue example](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py) +- [vLLM PR 16032 NVFP4 wrapper](https://github.com/vllm-project/vllm/blob/ed7a29d9f8b48978e3bbf43599d21b4de65387e0/csrc/quantization/fp4/nvfp4_scaled_mm_kernels.cu) + +## Related + +- [Tensor Memory](../hardware/tmem.md) — load, wait, and lifetime rules +- [warp specialization](warp-specialization.md) — participant groups and role invariants +- [double buffering](double-buffering.md) — optional multi-region overlap protocol diff --git a/wiki/techniques/external-source-map-research.md b/wiki/techniques/external-source-map-research.md index 8682c1098..ee5bed363 100644 --- a/wiki/techniques/external-source-map-research.md +++ b/wiki/techniques/external-source-map-research.md @@ -3,8 +3,7 @@ id: technique-external-source-map-research title: External Source-Map Research For Kernel Edits type: technique architectures: -- sm100 -- sm90 +- sm90a tags: - cuda-cpp - cute-dsl @@ -29,33 +28,69 @@ sources: - blog-colfax-cutlass-kernels - blog-simveit-effective-transpose - blog-simveit-load-and-store -blackwell_relevance: These source-map repositories complement PR pages with compact, searchable examples for TMA, swizzling, Stream-K, persistent scheduling, and load/store mechanics on Hopper and Blackwell-era kernels. +blackwell_relevance: The pinned repositories provide Hopper SM90a examples. They can suggest candidate mechanisms for Blackwell, but an SM100 transfer requires separate version-matched source evidence and measurement. --- ## Use Use external source-map research after a profile or benchmark identifies an edit family but the local PR pages do not expose a small enough implementation -example. The route is code-first: clone a source-map repository, grep for the -measured mechanism, and cite exact files before adapting an idea. +example. Treat the profiler result as a search key, not a root-cause proof: +inspect a pinned upstream revision and exact implementation files before +adapting an idea. ```bash -git clone https://github.com/ColfaxResearch/cfx-article-src external/colfax-cfx -git clone https://github.com/simveit/load_and_store external/simveit-load-store -rg -n "tma|mbarrier|swizzle|ldmatrix|stream" external/colfax-cfx external/simveit-load-store +git init external/colfax-cfx +git -C external/colfax-cfx remote add origin \ + https://github.com/ColfaxResearch/cfx-article-src +git -C external/colfax-cfx fetch --depth=1 origin \ + fbecfed88de2e4246f104a023188ba722937c5fc +git -C external/colfax-cfx checkout --detach FETCH_HEAD +test "$(git -C external/colfax-cfx rev-parse HEAD)" = \ + fbecfed88de2e4246f104a023188ba722937c5fc + +rg -n "SM90_TMA|mbarrier" \ + external/colfax-cfx/tma external/colfax-cfx/pipeline-gemm +rg -n "PersistentTileScheduler|StreamK" external/colfax-cfx/streamk ``` +At that revision, `tma/tma_copy.h` contains SM90 TMA load/store and transaction +barrier handling; `pipeline-gemm/` contains multistage and warp-specialized +Hopper GEMMs; and `streamk/tile_scheduler.hpp` contains non-persistent, +data-parallel persistent, and Stream-K scheduler classes. The other pinned +source maps are narrower: + +- `simveit/effective_transpose` at + `994b2b5acaa67f80e411df3e8274b6ae13fd1949` contains SM90a TMA transpose and + 128-byte tensor-map swizzle examples. +- `simveit/load_and_store` at + `05d828cf910dd43f0053ddbbe4744218a06e9d7f` contains only `ldmatrix` and + `stmatrix` x1/x2/x4 instruction examples. +- `ColfaxResearch/cutlass-kernels` at + `84f0802e2b4a1bf068ac70359f20ffdb368c8f6a` contains Hopper SM90a + GEMM/FMHA TMA, WGMMA, pipelining, and optional warp-specialization examples; + it is not a persistent-scheduler or SM100 source. +- `NVIDIA-developer-blog/code-samples` at + `3350d216083a902ccbf5b31665e3b82096a75b55` is a cross-generation collection. + Exact relevant paths include `series/cuda-cpp/coalescing-global/coalescing.cu` + and `series/cuda-cpp/transpose/transpose.cu`. + ## When It Helps -- Long-scoreboard or poor sector utilization: search load/store and transpose - examples before changing vector width or memory layout. -- Barrier or TMA wait stalls: search pipelined GEMM examples before changing - stage count or producer/consumer split. -- Tail waves: search persistent and Stream-K examples before adding a - shape-specific dispatcher. +- For long-scoreboard or poor sector utilization, inspect the exact coalescing, + transpose, and matrix load/store examples as candidate mechanisms. The metric + alone does not establish vector width or layout as the cause. +- For barrier or TMA-wait stalls, inspect the SM90 TMA and pipelined GEMM paths + before proposing stage-count or producer/consumer changes. The examples do + not guarantee that either edit will help another kernel. +- For tail waves, compare the pinned non-persistent, data-parallel persistent, + and Stream-K scheduler implementations before adding shape dispatch. Profile + the target shapes rather than treating persistence as a universal remedy. ## Provenance Rule Do not cite this page as implementation evidence by itself. Cite one of its -source pages plus the concrete upstream file path, commit, or URL that shaped the -candidate edit. +source pages only as a discovery lead. Implementation evidence must name the +upstream repository, full immutable revision, exact file, and a symbol or line +range. For Blackwell transfer, add an SM100-versioned primary source or a +controlled SM100 experiment; Hopper source compatibility must not be assumed. diff --git a/wiki/techniques/fine-grained-quantization.md b/wiki/techniques/fine-grained-quantization.md index a7dfddb37..c52f7a92d 100644 --- a/wiki/techniques/fine-grained-quantization.md +++ b/wiki/techniques/fine-grained-quantization.md @@ -7,232 +7,125 @@ tags: [fine-grained-quantization, fp8, fp4, nvfp4, block-scale] confidence: source-reported reproducibility: snippet prerequisites: [hw-nvfp4] -related: [hw-nvfp4, kernel-deepgemm, technique-fine-grained-quantization] -sources: [blog-deepgemm, doc-nvidia-tuning-guide, pr-vllm-23696] -blackwell_relevance: "Blackwell tcgen05 has native UE8M0 block scaling; Hopper requires external CUDA core promotion (Nc=128)." +related: [hw-nvfp4, kernel-deepgemm] +sources: [blog-deepgemm, doc-ptx-isa-sm100, doc-transformer-engine-2.13-nvfp4] +blackwell_relevance: "Blackwell tcgen05 has native block-scaled MMA combinations using UE8M0 or UE4M3; the legal scale type and vector size depend on the selected MMA kind." --- ## Overview -Fine-grained quantization applies per-block (rather than per-tensor) scaling factors to low-precision data, preventing outlier values from destroying the quantization precision of an entire tensor. DeepSeek pioneered the tile-wise 1x128 scaling for activations and block-wise 128x128 scaling for weights in their FP8 training framework. On Blackwell (SM100), native block scaling support in tcgen05.mma enables hardware-accelerated fine-grained quantization using the UE8M0 scale format, while Hopper requires software-managed CUDA core promotion. - -## Scaling Granularities - -``` -Per-tensor scaling (coarsest): - Entire tensor shares one FP32 scale factor. - Problem: a single outlier ruins precision for all elements. - - [=== entire MxK matrix === ] -> 1 scale - -Per-block 128x128 scaling (weights): - Each 128x128 block has its own scale factor. - 128x128 = 16,384 elements per scale -> 0.006% overhead. - - +---+---+---+ - |s1 |s2 |s3 | <- each block has independent scale - +---+---+---+ - |s4 |s5 |s6 | - +---+---+---+ - -Per-tile 1x128 scaling (activations): - Each row of 128 elements has its own scale factor. - Captures per-channel activation distributions. - - [s1: ====128 elements====] - [s2: ====128 elements====] - [s3: ====128 elements====] -``` - -## DeepGEMM: Tile-wise and Block-wise Scaling - -DeepGEMM implements fine-grained FP8 GEMM with two scaling patterns: - -```cuda -// DeepGEMM FP8 GEMM with fine-grained scaling -// A (activations): FP8 E4M3 with tile-wise 1x128 FP32 scales -// B (weights): FP8 E4M3 with block-wise 128x128 FP32 scales -// C (output): FP32 accumulator - -// Scale tensor shapes: -// scale_A: [M, K/128] -- one FP32 scale per 128 elements along K for each row -// scale_B: [K/128, N/128] -- one FP32 scale per 128x128 block - -__device__ void deepgemm_fp8_tile( - const fp8_e4m3* A, // [TILE_M, TILE_K] in FP8 - const fp8_e4m3* B, // [TILE_K, TILE_N] in FP8 - const float* scale_A, // [TILE_M, TILE_K/128] - const float* scale_B, // [TILE_K/128, TILE_N/128] - float* C, // [TILE_M, TILE_N] accumulator - int M, int N, int K) -{ - // For each 128-element K-chunk: - for (int k_block = 0; k_block < K; k_block += 128) { - // 1. Load FP8 A tile [TILE_M, 128] and B tile [128, TILE_N] - // 2. Perform MMA: partial = A_fp8 * B_fp8 (in limited-precision FP32) - // 3. Apply combined scale: scale_A[m, k_block/128] * scale_B[k_block/128, n_block/128] - // 4. Accumulate: C[m, n] += partial * combined_scale - - for (int m = 0; m < TILE_M; m++) { - float sa = scale_A[m * (K / 128) + k_block / 128]; - for (int n_block = 0; n_block < TILE_N; n_block += 128) { - float sb = scale_B[(k_block / 128) * (N / 128) + n_block / 128]; - float combined_scale = sa * sb; - - // Apply scale to the partial MMA result - for (int n = n_block; n < n_block + 128; n++) { - C[m * TILE_N + n] += partial[m][n] * combined_scale; - } - } - } - } -} +Fine-grained quantization associates low-precision payloads with scales for +subsets of a tensor. Smaller groups let scale selection respond more locally, +but they do not guarantee a particular error or speed result. Keep four choices +separate when transferring a kernel: payload type, scale type, scale geometry, +and physical scale layout. + +Three verified recipes in this wiki illustrate why the full contract matters: + +- **DeepSeek FP8 training geometry:** one scale for each token row and 128 + activation channels (`1 x 128`), and one scale for each 128-input-channel by + 128-output-channel weight block (`128 x 128`). +- **NVFP4 1D recipe:** E2M1 payloads, one E4M3 local scale per 16 consecutive + values, and one FP32 global scale per tensor. Transformer Engine 2.13 also + defines a weight-oriented 2D mode with one local scale per `16 x 16` block. +- **MXFP4:** E2M1 payloads with one UE8M0 power-of-two scale per 32 values; the + NVFP4 per-tensor global scale is not part of this microscaling format. + +Scale-count and storage overhead are different quantities. A `128 x 128` block +has one factor per 16,384 payloads, a factor-count ratio of about 0.0061%. If +the payload is one byte and the factor is FP32, the byte ratio is instead +`4 / 16384`, about 0.0244%. Include factor width, payload width, padding, and +layout transformations in any storage or bandwidth claim. + +## DeepGEMM FP8 implementations + +At commit `891d57b4db1071624b5c8fa0d1e51cb317fa709f`, DeepGEMM uses +architecture-specific scale representations and accumulation paths. + +### SM90 + +The pinned SM90 1D1D kernel consumes FP32 A/B factors and fixes +`BLOCK_K == 128`. The DeepSeek-V3 report characterizes the H800 FP8 Tensor Core +path as retaining about 14 bits and describes this interval as four WGMMAs in +its configuration. The exact kernel accumulates one K block in `float accum` +and then applies both factors into a separate `float final_accum`: + +```cpp +final_accum[i * 4 + 0] += scale_a_0 * scale_b_0 * accum[i * 4 + 0]; +final_accum[i * 4 + 1] += scale_a_0 * scale_b_1 * accum[i * 4 + 1]; +final_accum[i * 4 + 2] += scale_a_1 * scale_b_0 * accum[i * 4 + 2]; +final_accum[i * 4 + 3] += scale_a_1 * scale_b_1 * accum[i * 4 + 3]; ``` -## Hopper: CUDA Core Promotion (Nc=128) - -On Hopper (SM90), the wgmma instruction accumulates in Tensor Core registers with limited precision (~FP22, not true FP32). To maintain numerical accuracy, DeepGEMM promotes the partial sum to a separate FP32 accumulator on CUDA Cores every 4 wgmma instructions (Nc=128, since each wgmma processes 32 K-elements): - -```cuda -// Hopper FP8 GEMM with CUDA Core promotion (DeepGEMM pattern) -// Every Nc=128 K-elements, promote Tensor Core accumulator to FP32 - -__device__ void hopper_fp8_gemm_with_promotion( - const fp8_e4m3* A, const fp8_e4m3* B, - const float* scale_A, const float* scale_B, - float* C_accumulator, - int K) -{ - // Tensor Core limited-precision accumulator (FP22-ish) - // These are wgmma output registers - float tc_acc[TILE_M_PER_THREAD][TILE_N_PER_THREAD] = {0}; - - // True FP32 accumulator on CUDA Cores - float fp32_acc[TILE_M_PER_THREAD][TILE_N_PER_THREAD] = {0}; - - int promotion_interval = 128; // Nc = 128 elements = 4 wgmma ops - int wgmma_count = 0; - - for (int k = 0; k < K; k += 32) { - // Issue wgmma (accumulates in limited-precision tc_acc) - wgmma_mma_async(tc_acc, smem_A_ptr, smem_B_ptr); - wgmma_count++; - - if (wgmma_count == 4) { // Every 128 K-elements - // Promote: transfer tc_acc to fp32_acc on CUDA Cores - wgmma_wait(); // Ensure wgmma is complete - - int k_block = k / promotion_interval; - for (int m = 0; m < TILE_M_PER_THREAD; m++) { - float sa = scale_A[/*...*/]; - for (int n = 0; n < TILE_N_PER_THREAD; n++) { - float sb = scale_B[/*...*/]; - // Add scaled partial to true FP32 accumulator - fp32_acc[m][n] += tc_acc[m][n] * sa * sb; - // Reset TC accumulator for next interval - tc_acc[m][n] = 0; - } - } - wgmma_count = 0; - } - } -} -``` - -The Nc=128 interval was chosen because: -- 4 wgmma operations process 128 K-elements (4 x 32) -- At this interval, the accumulated FP22 error is bounded to ~0.1% relative error -- Fewer than 4 ops (Nc=32, Nc=64) adds too much promotion overhead -- More than 4 ops (Nc=256) allows unacceptable precision loss - -## Blackwell: Native Block Scaling - -On Blackwell, tcgen05.mma supports native block scaling via the UE8M0 (unsigned 8-bit exponent, no mantissa) format. The hardware applies per-block scale factors directly during the MMA operation, eliminating the software promotion step: +This is a source-specific implementation, not a rule for every Hopper FP8 +kernel. The cited sources do not establish the deleted 0.1% error bound or a +universal ranking of `Nc=32`, `64`, `128`, and `256`. -```cuda -// Blackwell native block scaling with UE8M0 -// Scale format: UE8M0 = pure power-of-two scale (2^exponent) -// Packed: 4 UE8M0 values per 32-bit integer +### SM100 -// DeepGEMM SM100 kernel: scale_A and scale_B are UE8M0 packed -// tcgen05.mma applies scales automatically during accumulation +The pinned SM100 1D1D template accepts K scale granularities of 32 or 128 for +each operand. Its public interface packs four UE8M0 factors in each 32-bit +container. TMA moves factor blocks to shared memory, UTCCP copies them into +dedicated TMEM SFA/SFB columns, and block-scaled UMMA consumes the TMEM scale +addresses. The SM90 `final_accum` promotion loop is absent. -struct BlockScaleDescriptor { - // 4 UE8M0 scale values packed into one uint32 - // Each UE8M0 is an 8-bit unsigned exponent: value = 2^(e - 127) - uint32_t packed_scales; // Contains 4 block scales - - // Dequantization for block [i]: - // scale_i = 2^(((packed >> (i*8)) & 0xFF) - 127) -}; - -// PTX for tcgen05.mma with block scaling: -// The .scale modifier tells the hardware to apply UE8M0 scales -// from a designated SMEM region alongside the MMA operands -``` +PTX ISA 9.0 expresses the corresponding FP8 block-scaled operand classes with +this grammar-level form: ```ptx -// tcgen05.mma with native block scaling (Blackwell PTX) -// This instruction applies UE8M0 scales from SMEM during MMA -tcgen05.mma.cta_group::1.kind::f8f6f4 - [%tmem_addr], // TMEM accumulator destination - [%desc_a], // SMEM descriptor for A operand - [%desc_b], // SMEM descriptor for B operand - %scale_d, // Scale descriptor for D (output) - %enable_mask, - [%scale_a_smem], // UE8M0 scales for A in SMEM - [%scale_b_smem]; // UE8M0 scales for B in SMEM -``` - -## UE8M0 vs E4M3 Scale Formats - -| Property | UE8M0 (Blackwell native) | E4M3 (NVFP4 hackathon) | FP32 (DeepGEMM Hopper) | -|----------|-------------------------|------------------------|------------------------| -| Bits | 8 | 8 | 32 | -| Representable values | Powers of 2 only | 240 distinct values | Full FP32 range | -| Range | 2^-127 to 2^128 | ~0 to 448 | Full FP32 | -| Block size | 32 (MXFP standard) | 16 (NVFP4) | 128 (DeepGEMM) | -| Hardware support | tcgen05.mma native | Software decode | Software promotion | -| Precision impact | Coarser (power-of-2 only) | Fine (non-power-of-2) | Best (FP32) | - -## NVFP4 Two-Level Scaling - -The NVFP4 format used in the GPU Mode hackathon has its own scaling scheme: - -```cuda -// NVFP4 dequantization with two-level scaling -// Level 1: per-block FP8 E4M3 scale (every 16 FP4 elements) -// Level 2: per-tensor FP32 global scale - -__device__ float dequant_nvfp4( - uint8_t fp4_packed, // Two FP4 values packed in one byte - fp8_e4m3 block_scale, // Per-block scale (every 16 elements) - float global_scale) // Per-tensor global scale -{ - // Extract one FP4 value (E2M1 format) - // Representable: 0, 0.5, 1, 1.5, 2, 3, 4, 6 - float fp4_val = decode_e2m1(fp4_packed & 0x0F); - - // Two-level dequantization - float result = global_scale * float(block_scale) * fp4_val; - return result; -} - -// Block scale layout for NVFP4: -// For matrix A of shape [M, K]: -// scale_A shape: [M, K/16] -- one FP8 E4M3 per 16 FP4 elements -// Finer granularity than MXFP4 (block size 32) or DeepGEMM (block size 128) +tcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.scale_vec::1X + [d_tmem], a_desc, b_desc, idesc, + [scale_a_tmem], [scale_b_tmem], enable_input_d; ``` -## When to Use - -- **FP8 training**: Use tile-wise 1x128 for activations and block-wise 128x128 for weights (DeepGEMM pattern). This is the validated approach for training 671B+ parameter models. -- **FP4 inference on Blackwell**: Use NVFP4 with E4M3 block scales (block size 16) for highest precision, or MXFP4 with UE8M0 scales (block size 32) for native hardware acceleration. -- **Hopper FP8 inference**: Use CUDA core promotion with Nc=128 interval to maintain precision despite limited TC accumulation. - -## Caveats - -- UE8M0 scales are power-of-two only. Non-power-of-two distributions (common in activations) lose precision compared to E4M3 or FP32 scales. -- Smaller block sizes (16 for NVFP4 vs 128 for DeepGEMM) provide better precision but higher overhead: more scale values to store, load, and apply. -- The Nc=128 promotion interval on Hopper is a performance-accuracy tradeoff. Reducing Nc improves accuracy but adds more promotion overhead. Increasing Nc risks precision degradation. -- On Blackwell, native block scaling only works with UE8M0. Using E4M3 or FP32 scales still requires software handling. +This line is an instruction-shape reference, not a complete kernel: declarations, +legal descriptors, collective participation, scale layouts, ordering, completion, +and an architecture-specific target are still required. UE8M0 uses power-of-two +finite values and reserves encoding `0xff` for NaN. + +## Blackwell FP4 block-scale combinations + +PTX ISA 9.0 distinguishes the native formats by the complete instruction +combination, not by “FP4” alone: + +| Instruction qualifiers | Payload | Local scale | Group | +|---|---|---|---:| +| `.kind::mxf4.block_scale.block32` / `.scale_vec::2X` | E2M1 | UE8M0 | 32 | +| `.kind::mxf4nvf4.block_scale.block16` / `.scale_vec::4X` | E2M1 | UE4M3 | 16 | + +The `mxf4nvf4` family also has documented UE8M0 modes, so the kind name alone +does not select the NVFP4 recipe. NVFP4-compatible UE4M3 scaling is native on +the documented Blackwell targets; it is not necessarily a software-decode path. + +## Transfer checklist + +1. Match the model/checkpoint recipe: payload encoding, local scale type, + optional global scale, and scale geometry. +2. Match the kernel ABI: logical factor shape is not necessarily its TMA- or + TMEM-ready physical layout. Account for packing, padding, transposition, and + swizzling. +3. Match the target instruction. For native Blackwell block scaling, use the + complete kind, block/scale-vector qualifier, scale type, and target rules. +4. Keep preparation in the timed region unless the producer already emits the + required layout. Otherwise report preprocessing separately. +5. Validate output accuracy with the exact quantizer, workload, reference, + tolerance, and accumulation path; then measure end-to-end latency or + throughput for the actual shapes. + +## Evidence boundaries + +- More groups require more scale elements, but padding and layout determine the + actual traffic and storage cost. +- Smaller groups and fractional scales provide more representational freedom; + they do not guarantee lower error for every tensor or scale-selection method. +- DeepGEMM's Nc=128 path is evidence for that pinned SM90 implementation, not a + universal optimum for Hopper. +- Native instruction support does not make two recipes ABI-compatible. NVFP4, + MXFP4, and DeepGEMM FP8 differ in payload, scale type, grouping, and layout. + +## Primary references + +- [DeepSeek-V3 Technical Report v2](https://arxiv.org/abs/2412.19437v2) +- [DeepGEMM at commit `891d57b`](https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f) +- [Transformer Engine 2.13 NVFP4](https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html) +- [PTX ISA 9.0 `tcgen05.mma`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma) diff --git a/wiki/techniques/kernel-fusion.md b/wiki/techniques/kernel-fusion.md index 3c36fdb59..183bfacb5 100644 --- a/wiki/techniques/kernel-fusion.md +++ b/wiki/techniques/kernel-fusion.md @@ -4,52 +4,103 @@ title: "Kernel Fusion" type: technique architectures: [sm100, sm90] tags: [kernel-fusion, fused-kernel, tmem] -confidence: source-reported -reproducibility: snippet +confidence: verified +evidence_basis: + - source_id: contest-gpumode-p3 + evidence_type: upstream-code +reproducibility: pseudocode prerequisites: [hw-tmem] related: [kernel-fused-moe, kernel-nvfp4-gemm, technique-epilogue-fusion] -sources: [contest-gpumode-p3, contest-flashinfer-track-a, blog-tflops-gap-fp4-moe] -blackwell_relevance: "TMEM enables multi-accumulator fusion (gate+up dual GEMM) without register pressure; technique valuable on both architectures." +sources: [contest-gpumode-p3, contest-flashinfer-track-a] +blackwell_relevance: "Blackwell TMEM can hold tcgen05 accumulators for fused schedules, but fusion legality and benefit still depend on the exact dataflow, live storage, synchronization, and output contract." --- # Kernel Fusion -## Overview +## Definition and evidence boundary -Kernel fusion combines multiple operations into a single kernel launch, eliminating intermediate global memory roundtrips. Critical for MoE and attention pipelines where 5-7 sequential launches each incur latency, synchronization, and memory traffic overhead. +Kernel fusion implements two or more dependent logical operations inside one +GPU kernel launch. It can remove a launch or an intermediate global-memory +write/read only when the chosen unfused baseline would perform that work and +the fused implementation keeps the value on chip. Fusion is therefore an +implementation property, not a conclusion that follows from an operation's +name. -## Examples +Report a benefit only for a defined fused/unfused pair with identical input and +output semantics. Count actual launches and bytes for the pinned implementation; +do not transfer framework-level launch counts or traffic percentages from an +unpinned configuration. -### Fused Gate-Up Dual GEMM + SwiGLU -```cuda -// Instead of: gate_gemm → up_gemm → silu → multiply (4 kernels) -// Fused: single kernel with two TMEM accumulators -__global__ void fused_gate_up_silu(...) { - uint32_t tmem_gate = tmem_alloc(256); - uint32_t tmem_up = tmem_alloc(256); +## Gated dual GEMM - for (int k = 0; k < K; k += BLOCK_K) { - tcgen05_mma(x_smem, w_gate_smem, tmem_gate); - tcgen05_mma(x_smem, w_up_smem, tmem_up); - } +GPU Mode NVFP4 Challenge 3 fixes this operation for each batch index: - float g = tmem_load(tmem_gate); - float u = tmem_load(tmem_up); - output = (g / (1.0f + expf(-g))) * u; // SwiGLU fused -} +```python +gate = scaled_mm(a, b1.T, sfa, sfb1) +up = scaled_mm(a, b2.T, sfa, sfb2) +output = silu(gate) * up ``` -### MoE Fusion Progression -- **vLLM (7 kernels)**: softmax → topk → dispatch → gate → up → silu_mul → down+combine -- **SGLang (5 kernels)**: router+topk → dispatch → fused gate-up-silu → down → combine -- **Ideal (1-2 kernels)**: all ops in one launch, saves 21.9% activation memory traffic +The two products share `a` and its scale tensor but use separate B operands and +scales. This operation graph permits reuse and fused pointwise output, but the +pinned correctness reference does not prescribe launch count, TMEM partition, +TMA pipeline, or instruction schedule. A concatenated gate/up projection or two +logical accumulators can both implement compatible observable semantics when +their tensor contracts match. -## Constraints +## Fused MoE -- TMEM capacity limits how many accumulators can fuse (256 cols total) -- Register pressure on epilogue if fusing complex activations -- Fusion opportunities depend on dataflow shape (dependency graph must be DAG-compatible with CTA scope) +FlashInfer MLSys 2026 Track A includes sigmoid routing, grouped expert +selection, two grouped GEMMs, SwiGLU, and weighted expert accumulation in one +logical benchmark definition. The exact reference performs one concatenated +W13 projection, splits its 4096 columns into gate and up halves, applies +SwiGLU, then performs W2. The term “Fused MoE” does not establish that a +submission uses one launch or keeps every intermediate on chip. + +## Implementation constraints + +- **Semantics:** preserve routing, scale, bias, activation, reduction, + accumulation, dtype, ordering, and output-edge behavior. A pointwise epilogue + is easier to compose than a cross-CTA reduction or global routing decision. +- **Communication:** identify the producer and every consumer of each + intermediate. Use the synchronization scope and memory space actually needed; + fusion is not restricted to one CTA, but multi-CTA communication has its own + legality and completion rules. +- **Resources:** record registers, spills, SMEM, TMEM, barriers, threads, cluster + shape, and occupancy. Added live values can raise register or shared-memory + pressure even when a global intermediate disappears. +- **TMEM:** SM100 TMEM is 128 lanes by 512 columns of 32-bit cells per SM. + Simultaneously live regions must fit, use legal collective allocation sizes, + remain live through every asynchronous access, and be collectively freed. + Two 256-column halves are one possible allocation policy, not the total + hardware capacity or a dual-GEMM requirement. +- **Fallbacks:** keep unfused or differently fused paths for shapes, dtypes, + layouts, reductions, or resource footprints that the fused schedule cannot + implement safely. + +## Evaluation + +For correctness, compare the fused and unfused paths on identical tensors and +exercise empty/small groups, partial tiles, extreme activations, routing ties, +and every supported scale/layout variant. Add negative tests for premature +buffer reuse, omitted completion, wrong routing weights, and invalid output +edges. + +For performance, include preparation and layout conversion unless the real +producer already emits the required representation. Record hardware, software, +shapes, dtypes, warmup, synchronization, repeated trials, statistic, variance, +launch count, bytes, and resource usage. Saved traffic does not guarantee a +speedup: extra computation, synchronization, staging, spills, or lower occupancy +can outweigh it. + +## Primary references + +- [GPU Mode NVFP4 gated dual GEMM at `c5b2f7c`](https://github.com/gpu-mode/reference-kernels/tree/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm) +- [FlashInfer Track A definition](https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048) +- [FlashInfer MoE reference at `7f614b8`](https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/trace/templates/moe.py) +- [PTX ISA 9.0 Tensor Memory](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory) ## Related -- [fused-moe](../kernels/fused-moe.md) -- [epilogue-fusion](epilogue-fusion.md) + +- [fused MoE](../kernels/fused-moe.md) +- [epilogue fusion](epilogue-fusion.md) diff --git a/wiki/techniques/persistent-kernels.md b/wiki/techniques/persistent-kernels.md index 79cee1834..2812f69c9 100644 --- a/wiki/techniques/persistent-kernels.md +++ b/wiki/techniques/persistent-kernels.md @@ -8,8 +8,11 @@ tags: - persistent-kernel - clc - tile-scheduling -confidence: source-reported -reproducibility: snippet +confidence: verified +evidence_basis: +- source_id: doc-cutlass-clc + evidence_type: official-doc +reproducibility: pseudocode prerequisites: - hw-clc related: @@ -19,197 +22,92 @@ related: sources: - doc-nvidia-tuning-guide - blog-tcgen05-tutorial -- doc-cutlass-blackwell +- doc-cutlass-clc artifact_dir: artifacts/kernels/persistent-kernels --- -## Overview - -Persistent kernels launch exactly as many CTAs as SMs, and each CTA processes multiple output tiles in a loop rather than exiting after one tile. On Blackwell, the CLC (Cluster Launch Control) hardware unit replaces software-based tile scheduling with a hardware-assisted mechanism. Each CTA queries the CLC for its next tile assignment and can cancel itself when no work remains, using the `try_cancel` pattern. - -## CLC Loop Pattern - -The core persistent kernel loop on Blackwell uses CLC to dynamically assign tiles: - -```cuda -// Persistent kernel with CLC tile scheduling (Blackwell SM100) -__global__ void __launch_bounds__(512) -persistent_gemm_clc(const __grid_constant__ GemmParams params) -{ - // CLC-managed persistent loop: each CTA processes multiple tiles - while (true) { - // Query CLC for next tile assignment - // Returns tile coordinates (tile_m, tile_n) or signals termination - TileCoord tile; - bool has_work = clc_try_get_tile(&tile); - - if (!has_work) { - // No more tiles to process -- CTA exits - // clc_try_cancel atomically checks if all tiles are done - if (clc_try_cancel()) { - return; // CTA terminates - } - continue; // Race condition: another CTA may have generated work - } - - // Standard GEMM tile computation - int tile_m = tile.m; - int tile_n = tile.n; - - // TMA producer loads A[tile_m, :] and B[:, tile_n] tiles - // MMA consumer accumulates K-dimension - // Epilogue writes C[tile_m, tile_n] - compute_gemm_tile(params, tile_m, tile_n); - } -} -``` +# Persistent Kernels with CLC + +## Persistence and CLC are separate choices + +A persistent kernel keeps a resident worker alive for multiple logical work items. Its launch size is chosen from the problem decomposition, cluster shape, resource limits, and scheduling policy; “persistent” does not require exactly one CTA per physical SM. -At the PTX level, the CLC interaction is a cancel/query sequence. The exact -inline PTX is usually hidden behind CUTLASS/CuTe wrappers, but the control flow -looks like this: +Cluster Launch Control is a compute-capability-10.0 mechanism for redistributing grid coordinates that have not started. A CLC GEMM still launches its problem-sized grid. Each ClcID is processed exactly once through one of two paths: -```text -TILE_LOOP: - // Request cancellation of a not-yet-launched cluster. - clusterlaunchcontrol.try_cancel(response_smem, mbarrier) - wait(mbarrier) +1. A block or cluster launches normally and processes its initial `blockIdx`. +2. A running worker successfully cancels another not-yet-started block/cluster and processes the returned coordinate. - // Query the 16-byte response. - has_work, tile_m, tile_n = clusterlaunchcontrol.query_cancel(response_smem) - if (!has_work) return +CLC does not create tiles, cancel the requesting worker, or chain independent problems. It can help when the SMs available to a grid are uneven, but it cannot expose more independent parallel work than exists in the launched grid. - // ... compute tile ... - goto TILE_LOOP +## Request lifecycle + +The normative request is asynchronous and returns an opaque 16-byte shared-memory response: + +```ptx +clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [response], [mbar]; +clusterlaunchcontrol.query_cancel.is_canceled.pred.b128 p, response_b128; +@p clusterlaunchcontrol.query_cancel.get_first_ctaid.v4.b32.b128 {x, y, z, unused}, response_b128; ``` -## Comparison: CLC vs Static Stride (Hopper) +A complete implementation must initialize/publish the shared response and mbarrier, set the expected 16 transaction bytes, submit from the designated participant, wait for the matching phase, apply required proxy fences before reading/reusing the response, and decode a coordinate only when `is_canceled` succeeds. + +After a thread observes a failed request, another request from that thread is undefined. Failure is therefore not a “retry until another CTA creates work” condition; no CTA creates new ClcIDs. + +For a thread-block cluster, one cluster participant submits the multicast request. Every CTA tracks completion at cluster scope, receives the same first coordinate, and adds its local block rank. Cluster synchronization is required before cancellation begins so all peers exist. -On Hopper (SM90), persistent kernels use a static stride pattern where each CTA computes tiles at fixed intervals: +## Static stride baseline -```cuda -// Hopper-style static stride persistent kernel -__global__ void hopper_persistent_gemm(GemmParams params) -{ - int cta_id = blockIdx.x; - int total_ctas = gridDim.x; - int total_tiles = params.num_tiles_m * params.num_tiles_n; +A software persistent scheduler can assign a flat tile index by fixed stride: - // Static stride: CTA i handles tiles i, i+total_ctas, i+2*total_ctas, ... - for (int tile_idx = cta_id; tile_idx < total_tiles; tile_idx += total_ctas) { - int tile_m = tile_idx / params.num_tiles_n; - int tile_n = tile_idx % params.num_tiles_n; - compute_gemm_tile(params, tile_m, tile_n); - } +```cpp +for (int tile = blockIdx.x; tile < total_tiles; tile += gridDim.x) { + int tile_m = tile / tiles_n; + int tile_n = tile % tiles_n; + compute_tile(tile_m, tile_n); } ``` -| Aspect | Hopper Static Stride | Blackwell CLC | -|--------|---------------------|---------------| -| Scheduling | Software loop with fixed stride | Hardware CLC unit assigns tiles | -| Load balancing | Fixed; uneven if tile costs vary | Dynamic; CLC rebalances automatically | -| Tail effect | Last wave may have partial occupancy | CLC minimizes by giving fast CTAs more tiles | -| Launch overhead | Grid launch for each new problem | CLC can chain multiple problems | -| Termination | Implicit when loop ends | Explicit `try_cancel` | -| L2 locality | Depends on stride pattern | CLC can apply swizzled raster | - -## CUTLASS PersistentTileSchedulerSm100 - -CUTLASS 4.5.0 provides `PersistentTileSchedulerSm100` that wraps the CLC hardware: - -```cuda -// CUTLASS SM100 persistent tile scheduler (simplified) -template -struct PersistentTileSchedulerSm100 { - - // Initialize the CLC with the problem geometry - CUTLASS_DEVICE static void init( - dim3 problem_tiles, - void* clc_smem_buffer) - { - if (threadIdx.x == 0) { - // Program CLC with total tile count and scheduling policy - clc_init(clc_smem_buffer, - problem_tiles.x, // tiles along M - problem_tiles.y, // tiles along N - ClcPolicy::SwizzledRaster); - } - __syncthreads(); - } - - // Shared storage for CTA-wide CLC result broadcast - // __shfl_sync is warp-local and cannot reach warps 1-15. - struct SharedClcState { - int tile_m, tile_n; - int valid; // 1 = got tile, 0 = no more work - int cancelled; - }; - - // Get next tile assignment from CLC - CUTLASS_DEVICE static bool get_next_tile( - void* clc_smem_buffer, - SharedClcState& shared_clc, - int& tile_m, - int& tile_n) - { - if (threadIdx.x == 0) { - int m, n; - bool v = clc_query_tile(clc_smem_buffer, m, n); - shared_clc.tile_m = m; - shared_clc.tile_n = n; - shared_clc.valid = v ? 1 : 0; - } - __syncthreads(); // All warps see the result - tile_m = shared_clc.tile_m; - tile_n = shared_clc.tile_n; - return shared_clc.valid != 0; - } - - // Try to cancel the CTA when no more work - CUTLASS_DEVICE static bool try_cancel( - void* clc_smem_buffer, - SharedClcState& shared_clc) - { - if (threadIdx.x == 0) { - shared_clc.cancelled = clc_try_cancel(clc_smem_buffer) ? 1 : 0; - } - __syncthreads(); - return shared_clc.cancelled != 0; - } -}; -``` +For positive `gridDim.x`, this covers each index in `[0,total_tiles)` exactly once, including nondivisible tails. It remains useful as a controlled baseline on Hopper and Blackwell. -## Performance Impact +## CUTLASS 4.5.0 route -The tcgen05-tutorial progression demonstrates the impact of persistent kernels: +At CUTLASS 4.5.0, the SM100 persistent scheduler integrates CLC through `PersistentTileSchedulerSm100` and `PipelineCLCFetchAsync`. The scheduler's work flow is expressed through `WorkTileInfo` and methods including `get_current_work()`, `advance_to_next_work()`, and `fetch_next_work()`; it is not an API built from `clc_init`, `clc_query_tile`, or a caller-self-cancel helper. -``` -Without persistence (static grid): 940 TFLOPS (62% of peak) -With CLC persistent scheduling: 1476 TFLOPS (98% of cuBLAS) -``` +`advance_to_next_work()` submits/stages the next request, while `fetch_next_work()` waits for and decodes a response. CUTLASS applies `swizzle_and_rasterize()` as a software coordinate transform to both the initial `blockIdx` and returned CLC coordinates. `max_swizzle_size` and `raster_order` are scheduler arguments, not CLC hardware policy operands. + +The official documentation is the authoritative implementation map; use its pinned scheduler and pipeline links rather than a short pseudo-class. -The 57% improvement comes from: -1. **Eliminated tail effect**: CLC dynamically assigns tiles, so fast-completing CTAs absorb extra work rather than sitting idle while the last wave finishes. -2. **Reduced launch overhead**: A single kernel launch covers all tiles; no need to re-launch grids. -3. **Better L2 cache utilization**: CLC can apply a swizzled raster pattern that improves spatial locality across neighboring tiles. +## Evidence-scoped performance -## When to Use +Gau Nernst's pinned tutorial reports 939.61 TFLOP/s for v3 pipelining and 1475.93 TFLOP/s for v6 on its disclosed 4096-cubed B200 experiment. The v6 endpoint adds warp specialization, 2-SM MMA, and static persistence. The author explicitly did not add CLC or threadblock swizzling, so the endpoint difference is not a CLC ablation and cannot be decomposed into tail, launch, or cache contributions. -- **Large GEMM problems**: Persistent kernels are most beneficial when the number of output tiles exceeds the SM count by at least 2-3x. -- **Grouped GEMMs / MoE**: CLC can chain multiple problem instances, eliminating inter-kernel launch gaps. -- **Workloads with uneven tile cost**: CLC's dynamic scheduling naturally handles variable-cost tiles (e.g., triangular attention masks). +To evaluate CLC, compare a static and dynamic scheduler with identical math, tile/cluster shapes, epilogue, launch environment, available-SM policy, warmup, and timing statistics. Record successful/failed requests, work distribution per worker, scheduler stalls, occupancy, and end-to-end time. Test ordinary availability, intentionally uneven availability, nondivisible grids, and cluster modes. -## Caveats +Useful negative controls include decoding a failed response, re-requesting from the same thread after observed failure, omitting initial `blockIdx`, and applying the software swizzle to only one of the initial/returned paths. Each should fail validation or produce missing/duplicate coordinates. -- CLC is SM100-only; Hopper kernels must use software-based scheduling. -- The `try_cancel` pattern introduces a potential race that must be handled with a retry loop. -- For very small problems (fewer tiles than SMs), CLC overhead may not justify the complexity. A simple single-wave grid launch suffices. +When the grid contains fewer independent tiles than available SMs, CLC cannot manufacture parallelism. Stream-K or another decomposition may create additional work, but that is a separate scheduler transformation with its own reduction and synchronization costs. -## Full Reference Implementation +## Artifact provenance -Verbatim upstream code lives in [`artifacts/kernels/persistent-kernels/full/`](../../artifacts/kernels/persistent-kernels/full/); labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` header) live in [`artifacts/kernels/persistent-kernels/variants/`](../../artifacts/kernels/persistent-kernels/variants/). Every file's SHA-256 and upstream-pinning metadata is in `PROVENANCE.yaml` inside each bundle. +- `full/PR-2161-persistent-scheduler-clc.patch` is the recorded upstream PR patch with manifest SHA-256 `aac62aa643631ba452953b61243b657c8f2de11480cbfb6802fdb0157b2ab017`. +- `full/dense_gemm_persistent_prefetch.py` byte-matches CUTLASS PR 2881 merge `3f4c086d09bd1dc55defb955862f333893bbb28b` with SHA-256 `704811ee96850b404b7e1d47e2c0d67f6b19219e3fd8081dc9396b52163f7045`. +- `variants/01-clc-persistent-loop-skeleton.cu` is labeled derived/not-upstream and is not a complete safe CLC kernel. -Query via: +Retrieve the page and bundle with: ```bash -python3 scripts/get_page.py technique-persistent-kernels --include-code +conda run -n base python scripts/get_page.py technique-persistent-kernels --include-code ``` + +## Primary references + +- [PTX ISA 9.0 CLC instructions](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel) +- [CUTLASS 4.5.0 CLC documentation](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md) +- [CUTLASS 4.5.0 SM100 scheduler](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp) +- [Pinned tutorial v6 source](https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu) + +## Related + +- [Cluster Launch Control](../hardware/clc.md) — exact hardware semantics +- [tile scheduling](tile-scheduling.md) — static, swizzled, dynamic, and Stream-K choices +- [tail effect](../patterns/tail-effect.md) — wave-quantization diagnosis diff --git a/wiki/techniques/ping-pong-scheduling.md b/wiki/techniques/ping-pong-scheduling.md index be7994229..a960009d3 100644 --- a/wiki/techniques/ping-pong-scheduling.md +++ b/wiki/techniques/ping-pong-scheduling.md @@ -2,83 +2,62 @@ id: technique-ping-pong-scheduling title: Ping-Pong Scheduling type: technique -architectures: -- sm100 -tags: -- ping-pong-scheduling -- warp-specialization -- tmem -- pipeline-stages -confidence: source-reported -reproducibility: snippet -prerequisites: -- hw-tmem -- technique-warp-specialization -related: -- kernel-flash-attention-4 -- technique-double-buffering -sources: -- blog-flash-attention-4 -- doc-flash-attention-4 -- blog-tcgen05-tutorial -artifact_dir: artifacts/kernels/ping-pong-scheduling +architectures: [sm100] +tags: [ping-pong-scheduling, warp-specialization, tmem, pipeline-stages] +confidence: verified +evidence_basis: + - source_id: doc-flash-attention-4 + evidence_type: official-doc +reproducibility: concept +prerequisites: [hw-tmem, technique-warp-specialization] +related: [kernel-flash-attention-4, technique-double-buffering] +sources: [doc-flash-attention-4, blog-flash-attention-4] +blackwell_relevance: "The pinned FlashAttention-4 SM100 forward kernel uses two query/output-tile states to overlap MMA for one tile with softmax for the other; the benefit and remaining stalls are configuration-dependent." --- -# Ping-Pong Scheduling +## Definition -## Overview +Ping-pong scheduling interleaves two output-tile states in one CTA. In the pinned FlashAttention-4 (FA4) SM100 forward implementation, `q_stage=2` makes the CTA cover two 128-row query tiles. One dedicated warp controls MMA, two four-warp groups perform softmax for the respective query stages, and a separate four-warp group performs correction work. While MMA advances one tile, the schedule tries to overlap softmax work for the other tile. -Ping-pong scheduling alternates two query tiles within a single CTA so the softmax warpgroup never stalls waiting for MMA. Introduced in FlashAttention-4 to exploit Blackwell's asymmetric hardware (2× tensor cores, same SFU count as Hopper). +This is not merely ordinary double buffering of one producer-consumer stream. Each query stage owns a distinct output state, including its softmax and output TMEM regions, and the algorithm must preserve the dependencies among MMA accumulation, row statistics, rescaling/correction, and output consumption. “Ping-pong” describes this interleaving; it does not guarantee that either execution resource is stall-free. -## Pattern +## Version-Pinned Implementation Contract -```cuda -// Two 128-token query tiles per CTA, alternating through the mainloop -// Warpgroup 0: softmax for tile A while MMA runs on tile B -// Warpgroup 1: softmax for tile B while MMA runs on tile A +The concrete reference is [`flash_fwd_sm100.py` at commit `a369df707e1980fb328abcc1733e3457ec10155f`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py). Its default two-stage layout has the following software roles: -__global__ void fa4_ping_pong_attn(...) { - int wg = warp_group_id(); +| Warp IDs | Role | +|---|---| +| 0–3 | softmax for query stage 0 | +| 4–7 | softmax for query stage 1 | +| 8–11 | correction | +| 12 | MMA control | +| 13 | epilogue | +| 14 | load | +| 15 | empty in this configuration | - // TMEM holds accumulators for BOTH tiles - uint32_t tmem_A = tmem_alloc(256); - uint32_t tmem_B = tmem_alloc(256); +These IDs are implementation choices, not SM100 architectural roles. The source changes assignments for other configurations, including `q_stage=1`. - for (int k = 0; k < num_kv_tiles; k++) { - if (wg == 0) { - // Compute softmax for tile A (previous MMA output) - softmax_normalize(tmem_A); - // Issue MMA for tile B next - tcgen05_mma(Q_B_smem, K_smem[k], tmem_B); - } else { - softmax_normalize(tmem_B); - tcgen05_mma(Q_A_smem, K_smem[k], tmem_A); - } - mbarrier_arrive(&ping_pong_sync); - mbarrier_wait(&ping_pong_sync); - } -} -``` +The implementation performs one collective TMEM allocation sized for its required columns and gives the two query stages disjoint offsets within that allocation. It also constructs multiple producer/consumer pipelines for distinct handoffs. A single generic barrier after issuing `tcgen05.mma` is not an equivalent implementation: tcgen05 work is asynchronous, and completion, memory visibility, stage reuse, and TMEM lifetime must follow the corresponding pipeline and ISA contracts. -## Why It Helps on Blackwell +## Correctness Checklist -- Tensor core throughput doubled (B200 vs H100) but SFU count unchanged -- Single-tile schedule would leave SFU idle while MMA runs, and vice versa -- Ping-pong keeps both units 100% busy -- FA4 achieves 1605 TFLOPS BF16 (71% utilization) with this pattern +Before treating a two-tile schedule as valid, verify all of the following in the complete kernel: -## When To Use +1. TMEM regions for simultaneous tile states are disjoint, allocated and deallocated collectively, and remain live until their final consumers finish. +2. Every SMEM or TMEM stage has one unambiguous producer/consumer phase owner; a stage cannot be overwritten until all operations that read it have completed. +3. Asynchronous MMA completion is connected to the barrier that guards accumulator consumption and operand-stage reuse. A CTA barrier or an unrelated mbarrier arrival is insufficient. +4. Softmax groups obey the implementation's explicit synchronization. FA4 serializes their exponential critical sections rather than allowing both groups to contend there simultaneously. +5. Correction and epilogue work wait for the row statistics and accumulators they consume, including the proper proxy and execution-order fences. +6. Boundary query rows, final key tiles, causal/window masks, and the pipeline tail preserve the same mathematical result as the reference attention computation. -- Compute-bound attention kernels on Blackwell -- Kernels where softmax/epilogue is SFU-heavy -- Not useful on Hopper (balance is different) +## Hardware Motivation and Evidence Scope -## Full Reference Implementation +FA4's paper models B200 BF16 tensor-core throughput at 8192 operations per clock per SM versus 4096 on Hopper, while exponential throughput is 16 operations per clock per SM on both. For its simplified `M=N=d=128` model, the paper assigns 1024 cycles each to MMA and exponentials and 768 cycles to shared-memory traffic; doubling the query extent doubles each of those modeled counts. The model explains why overlapping independent tile work is attractive, but the paper says that this roofline omits other resources. -Verbatim upstream code lives in [`artifacts/kernels/ping-pong-scheduling/full/`](../../artifacts/kernels/ping-pong-scheduling/full/); labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` header) live in [`artifacts/kernels/ping-pong-scheduling/variants/`](../../artifacts/kernels/ping-pong-scheduling/variants/). Every file's SHA-256 and upstream-pinning metadata is in `PROVENANCE.yaml` inside each bundle. +The paper reports up to 1613 TFLOP/s and 71% utilization for complete FA4 forward kernels. The accompanying author blog reports up to 1605 TFLOP/s and 71%. Those endpoints include partial software exponentials, conditional rescaling, TMEM partitioning, register allocation choices, and the rest of the pipeline. Neither source supplies a ping-pong-only ablation, and neither establishes 100% tensor-core or exponential-unit utilization. -Query via: +## When to Test It -```bash -python3 scripts/get_page.py technique-ping-pong-scheduling --include-code -``` +Use the pinned SM100 kernel as the concrete reference when two independent query/output states can coexist within the register, shared-memory, and TMEM budgets. Compare `q_stage=2` with the source-supported `q_stage=1` configuration while holding the remaining build, input, tile, and launch choices fixed. Measure end-to-end time, MMA and exponential-pipeline activity, barrier stalls, per-role idle time, register spills, and occupancy across representative shapes. + +Do not select the technique from a generic “compute-bound” or “SFU-heavy” label alone. The FA4 paper says its schedule is similar to FlashAttention-3's Hopper ping-pong schedule, so the scheduling idea is not intrinsically Blackwell-only; this page's pinned implementation and hardware figures are specifically SM100. Keep the two-stage form only when the controlled comparison shows a relevant improvement and correctness tests cover boundaries and pipeline tails. diff --git a/wiki/techniques/pipeline-stages.md b/wiki/techniques/pipeline-stages.md index 297309de4..9a04dcc2f 100644 --- a/wiki/techniques/pipeline-stages.md +++ b/wiki/techniques/pipeline-stages.md @@ -4,205 +4,101 @@ title: "Software Pipelining and Multi-Stage Buffering" type: technique architectures: [sm100, sm90] tags: [pipeline-stages, double-buffering, tma, mbarrier] -confidence: source-reported -reproducibility: snippet +confidence: verified +evidence_basis: + - source_id: doc-nvidia-tuning-guide + evidence_type: official-doc +reproducibility: pseudocode prerequisites: [hw-tma, hw-tmem] related: [technique-warp-specialization, technique-double-buffering, hw-tma] sources: [blog-tcgen05-tutorial, blog-modular-blackwell, doc-nvidia-tuning-guide] -blackwell_relevance: "Same mbarrier pattern on both architectures; Blackwell adds tcgen05 fence requirement between TMA and MMA." +blackwell_relevance: "SM100 pipelines TMA-produced SMEM stages into tcgen05 consumers; correctness depends on transaction, MMA-completion, phase, and reuse edges." --- -## Overview +# Software Pipelining and Multi-Stage Buffering -Software pipelining overlaps data loading (TMA copies from global to shared memory) with computation (tcgen05.mma or wgmma) by maintaining multiple in-flight tile buffers. A circular buffer of 3-5 stages allows the TMA producer to fill stage N+2 while the MMA consumer processes stage N, hiding the global memory latency entirely. This technique is critical for achieving high utilization on both Hopper and Blackwell. +## What a stage means -## Pipeline Progression +A multi-stage pipeline reserves disjoint shared-memory operand buffers so production of one tile can overlap consumption of another. Stage count controls how far producer and consumer progress may separate; it does not guarantee that latency is fully hidden or that the schedule is faster. -The tcgen05-tutorial demonstrates the performance impact of pipelining: +Each stage cycles through this ownership state machine: -``` -No pipelining (load, then compute): 695 TFLOPS (46%) -3-stage pipeline (TMA + MMA overlap): 940 TFLOPS (62%) -+ warp specialization: 1476 TFLOPS (98%) -``` +| State | Owner | Transition condition | +|---|---|---| +| empty | producer | the prior consumer has completed every read of the stage | +| loading | TMA async proxy | expected transaction bytes are registered before the copy and the TMA transaction is outstanding | +| ready | consumer | the full barrier's matching phase has completed | +| consuming | MMA path | the consumer has acquired the stage and no producer may overwrite it | +| empty again | producer | every asynchronous MMA use of the SMEM operands is complete and the consumer releases the matching empty phase | -The 35% improvement from pipelining alone (695 to 940 TFLOPS) comes from hiding global memory latency behind MMA computation. - -## Multi-Stage Circular Buffer Pattern - -The fundamental pattern allocates `NUM_STAGES` copies of each SMEM buffer and cycles through them: - -```cuda -// 3-stage circular buffer with mbarrier synchronization -// Stages: [0] loading, [1] ready for MMA, [2] being consumed by MMA -#define NUM_STAGES 3 - -__global__ void __launch_bounds__(512) -pipelined_gemm(const __grid_constant__ GemmParams params) -{ - extern __shared__ char smem[]; - - // Circular buffer layout in shared memory - // Each stage has its own A and B tile buffers - half* smem_A[NUM_STAGES]; - half* smem_B[NUM_STAGES]; - for (int s = 0; s < NUM_STAGES; s++) { - smem_A[s] = reinterpret_cast( - smem + s * (TILE_A_BYTES + TILE_B_BYTES)); - smem_B[s] = reinterpret_cast( - smem + s * (TILE_A_BYTES + TILE_B_BYTES) + TILE_A_BYTES); - } - - // mbarrier arrays for producer-consumer sync - __shared__ uint64_t mbar_load_complete[NUM_STAGES]; - __shared__ uint64_t mbar_mma_complete[NUM_STAGES]; - - int warp_id = threadIdx.x / 32; - int lane_id = threadIdx.x % 32; - - // Initialize barriers - if (threadIdx.x == 0) { - for (int s = 0; s < NUM_STAGES; s++) { - mbarrier_init(&mbar_load_complete[s], 1); - mbarrier_init(&mbar_mma_complete[s], 1); - } - } - __syncthreads(); - - int num_k_tiles = params.K / TILE_K; - - if (warp_id == 0) { - // ===== TMA PRODUCER ===== - // Prologue: fill the first NUM_STAGES buffers - for (int s = 0; s < NUM_STAGES && s < num_k_tiles; s++) { - if (lane_id == 0) { - // Set expected TX bytes on mbarrier BEFORE issuing TMA. - // TMA hardware will arrive on the mbarrier when transfer completes. - uint32_t tx_bytes = TILE_A_BYTES + TILE_B_BYTES; - mbarrier_arrive_expect_tx(&mbar_load_complete[s], tx_bytes); - tma_load_tile_A(smem_A[s], params, s, &mbar_load_complete[s]); - tma_load_tile_B(smem_B[s], params, s, &mbar_load_complete[s]); - // NOTE: Do NOT manually arrive after TMA issue — the TMA - // hardware signals the mbarrier upon transfer completion. - } - } - - // Steady state: load stage s while MMA processes stage s-NUM_STAGES - for (int k = NUM_STAGES; k < num_k_tiles; k++) { - int stage = k % NUM_STAGES; - // Wait for MMA to finish with this buffer - mbarrier_wait(&mbar_mma_complete[stage]); - if (lane_id == 0) { - uint32_t tx_bytes = TILE_A_BYTES + TILE_B_BYTES; - mbarrier_arrive_expect_tx(&mbar_load_complete[stage], tx_bytes); - tma_load_tile_A(smem_A[stage], params, k, &mbar_load_complete[stage]); - tma_load_tile_B(smem_B[stage], params, k, &mbar_load_complete[stage]); - } - } - - } else if (warp_id == 1) { - // ===== MMA CONSUMER ===== - for (int k = 0; k < num_k_tiles; k++) { - int stage = k % NUM_STAGES; - // Wait for TMA to fill this buffer - mbarrier_wait(&mbar_load_complete[stage]); - - // Issue MMA on the filled buffer - if (lane_id == 0) { - tcgen05_mma(smem_A[stage], smem_B[stage]); - } - __syncwarp(); - - // Signal that this buffer is free for reuse - if (lane_id == 0) { - mbarrier_arrive(&mbar_mma_complete[stage]); - } - } - } - // Epilogue warps omitted for clarity -} -``` +Modulo indexing selects storage, but it does not track ownership by itself. A correct implementation also supplies barrier initialization/publication, expected arrival counts, phase or token state, async-proxy ordering, a bounded prologue, steady-state advancement, pipeline tail, and error-free behavior when K tiles are fewer than stages. + +On SM100, do not release a stage merely after issuing `tcgen05.mma`. The MMA is asynchronous; use its defined completion path before allowing a producer to overwrite A/B storage. Likewise, `__syncthreads()` is not a substitute for the required tcgen05/TMA async-proxy completion and ordering operations. -## Stage Count Selection +## TMA/full-barrier edge -The optimal number of pipeline stages depends on the ratio of memory latency to compute time per tile: +For a TMA load into shared memory, the producer initializes/publishes the mbarrier, accounts for the copy's expected transaction bytes, and issues the tensor copy with that barrier. Hardware completes the transaction on the barrier; the consumer waits for the matching phase before reading the stage. -| Stages | SMEM Usage | Latency Hiding | Best For | -|--------|-----------|----------------|----------| -| 2 | 2x base | Partial | Small tiles, limited SMEM | -| 3 | 3x base | Full for most GEMMs | Standard choice on Blackwell | -| 4-5 | 4-5x base | Full with margin | Large K, high memory latency | -| >5 | Excessive | Diminishing returns | Rarely justified | +This autonomous completion avoids a producer instruction that manually announces data-ready after the transfer. It does not prove that producer issue work or the overall load path is absent from the measured critical path. -The SMEM budget on Blackwell is 228 KB per SM. For a BF16 GEMM with TILE_M=128, TILE_N=256, TILE_K=64: -- A tile: 128 x 64 x 2B = 16 KB -- B tile: 64 x 256 x 2B = 32 KB -- Per stage: 48 KB -- 3 stages: 144 KB (63% of SMEM, leaves room for barriers and epilogue) -- 5 stages: 240 KB (exceeds SMEM capacity; must use smaller tiles) +Use the exact PTX ISA tensor-copy grammar or a version-pinned library pipeline. An illustrative `arrive`/spin loop that omits expected transaction bytes, copy descriptors, proxy fences, phase initialization, and the consumer-to-producer reuse edge is not a safe pipeline. -## mbarrier-Based Synchronization +## Stage capacity arithmetic -The mbarrier (memory barrier) is the hardware primitive that makes pipelining efficient. Unlike `__syncthreads()`, mbarrier supports asymmetric producer-consumer synchronization where only the relevant warp participates: +For binary16 `A[128,64]` and `B[64,256]`, unpadded operand payload is: -```ptx -// Phase-based mbarrier protocol for 3-stage pipeline -// -// Each mbarrier tracks a "phase" (0 or 1). The producer flips the phase -// on arrive; the consumer waits for the expected phase. +```text +A = 128 × 64 × 2 bytes = 16 KiB +B = 64 × 256 × 2 bytes = 32 KiB +one stage = 48 KiB +three stages = 144 KiB +five stages = 240 KiB +``` -// Producer: arrive on stage %s (flips phase) -mbarrier.arrive.shared.b64 %state, [%mbar_load + %s * 8]; +Compute capability 10.0 supports up to 228 KiB of shared memory per SM. Thus five stages of that exact payload already exceed the per-SM capacity, while three stages consume about 63.2% before barriers, padding, descriptors, epilogue buffers, and other shared storage. Per-block opt-in limits and occupancy constraints also apply. -// Consumer: wait for phase %expected_phase on stage %s -// try_wait is non-blocking; the warp can spin-wait or do other work -WAIT_LOOP: - mbarrier.try_wait.parity.shared.b64 %ready, [%mbar_load + %s * 8], %phase; - @!%ready bra WAIT_LOOP; +Payload grows linearly with stage count. Total allocation can additionally contain fixed and stage-dependent metadata, so derive the concrete shared-storage type rather than multiplying payload alone. -// TMA can also arrive directly on an mbarrier: -// The TMA unit signals completion without CPU thread involvement -cp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes - [%smem_addr], [%tensor_map, {%coord0, %coord1}], [%mbar_load + %s * 8]; -``` +## Select a stage count -The key advantage is that TMA can arrive on an mbarrier autonomously. The producer warp only needs to initiate the TMA; the TMA hardware signals completion directly, removing the producer from the critical path. +There is no architecture-wide table in which two stages are always partial, three always fully hide latency, or more than five are always excessive. Legal and useful depth depends on: -## Modular's 5-Stage Implementation +- bytes per stage, alignment/swizzle padding, other SMEM, and occupancy; +- TMA issue/transfer rate and descriptor/transaction shape; +- MMA work and completion time per K tile; +- producer/consumer role schedule and register pressure; +- K-loop length, prologue/tail fraction, and output schedule. -The Modular blog series describes a 5-stage circular buffer reaching 85% of SOTA performance on Blackwell: +Compile every candidate, reject resource-invalid variants, then compare controlled timings and pipeline/barrier stalls. Include `num_k_tiles` values below, equal to, and above the stage count to exercise every boundary. -```cuda -// Modular-style 5-stage pipeline constants -// Chosen to fully hide B200 HBM latency (~400 cycles) -// while fitting within 228 KB SMEM budget +## Evidence-scoped examples -constexpr int NUM_STAGES = 5; -constexpr int TILE_M = 128; -constexpr int TILE_N = 128; // Smaller N to fit 5 stages -constexpr int TILE_K = 64; +Gau Nernst's disclosed B200 4096-cubed experiment reports 695.43 TFLOP/s for v2b and 939.61 TFLOP/s for v3 after adding pipelining, about a 35.1% increase for that source progression. Warp specialization is the later v4 result at 1208.83 TFLOP/s; the 1475.93-TFLOP/s v6 endpoint also includes 2-SM MMA and static persistence. -// Per stage: A (128*64*2=16KB) + B (64*128*2=16KB) = 32 KB -// 5 stages: 160 KB + barriers + metadata < 228 KB +Modular Part 3 uses five A/B stages in its particular 2-SM matmul. The eventual author-reported 85%-of-SOTA endpoint also includes 2-SM work, warp specialization, and double-buffered output writeback; it is not a five-stage-only result. The article does not supply the fixed “~400 cycle HBM latency” premise previously attributed to it. -// The pipeline timing diagram for steady state: -// -// Stage: 0 1 2 3 4 -// TMA: [load k5] [done] [done] [load k8] [load k9] -// MMA: [done] [done] [comp k7] [done] [done] -// -// At any point: 1 stage being loaded, 1 being computed, 3 in transit or done -``` +## Validation checklist + +1. Verify every stage has independent storage and full/empty synchronization state. +2. Initialize correct expected participants and publish barrier state before async use. +3. Register expected TMA transaction bytes before issuing the copy. +4. Wait for TMA completion before the first dependent read. +5. Wait for every asynchronous MMA read to complete before releasing the stage. +6. Track the correct phase/token on first use and every wraparound. +7. Drain producer and consumer tails, including output-store tails where present. +8. Fault-test wrong phase, early release, omitted tail, and short-K paths under a bounded watchdog. -## When to Use +## Primary references -- **All memory-bound and compute-bound GEMM kernels**: Pipelining is never harmful and always improves utilization by hiding latency. -- **Attention kernels**: The K-dimension loop in attention benefits from pipelining the KV tile loads. -- **Combined with warp specialization**: Pipelining provides the buffer structure; warp specialization assigns the producer/consumer roles. The two techniques are complementary and almost always used together. +- [PTX ISA 9.0 mbarrier](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier) +- [PTX ISA 9.0 bulk tensor copy](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor) +- [PTX ISA 9.0 tcgen05 completion](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit) +- [CUTLASS 4.5.0 CuTe DSL pipeline example](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py) +- [Pinned tcgen05 tutorial source](https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu) +- [Modular Part 3](https://www.modular.com/blog/matrix-multiplication-on-nvidias-blackwell-part-3-the-optimizations-behind-85-of-sota-performance) -## Caveats +## Related -- More stages increase SMEM usage linearly. On Blackwell's fixed 228 KB, this constrains tile size choices. -- Barrier initialization overhead is negligible but must happen before the first TMA. Place init in a `__syncthreads()` block at kernel start. -- Incorrect phase tracking in mbarrier causes deadlocks. The phase alternates with each arrive/wait cycle; off-by-one errors are common during development. -- For very short K dimensions (fewer iterations than stages), the prologue/epilogue overhead may dominate. Guard the loop bounds accordingly. +- [TMA](../hardware/tma.md) — tensor-copy descriptors and completion +- [double buffering](double-buffering.md) — shared/TMEM ownership patterns +- [warp specialization](warp-specialization.md) — optional dedicated producer/consumer roles diff --git a/wiki/techniques/register-budgeting.md b/wiki/techniques/register-budgeting.md index 34fb24fc4..5c0f1b866 100644 --- a/wiki/techniques/register-budgeting.md +++ b/wiki/techniques/register-budgeting.md @@ -4,58 +4,63 @@ title: "Register Budgeting for Occupancy" type: technique architectures: [sm100, sm90] tags: [register-budgeting, register-reuse] -confidence: source-reported -reproducibility: snippet +confidence: verified +evidence_basis: + - source_id: doc-cuda-register-controls + evidence_type: official-doc +reproducibility: concept prerequisites: [] related: [pattern-memory-bound, pattern-register-pressure, kernel-nvfp4-gemv] -sources: [blog-yue-nvfp4, blog-amandeep-nvfp4, blog-simon-nvfp4-gemv] -blackwell_relevance: "TMEM eliminates accumulator register pressure on Blackwell, freeing ~100 registers/thread for other uses; technique still critical for memory-bound kernels." +sources: [doc-cuda-register-controls, blog-amandeep-nvfp4, doc-ptx-isa-sm100] +blackwell_relevance: "SM100 TMEM removes a resident tcgen05 D fragment from the general-purpose register file, but operands, control state, and TMEM-to-register epilogue batches still consume registers; occupancy remains a property of the compiled kernel and launch." --- # Register Budgeting -## Overview +## Resource model -SM occupancy is inversely proportional to registers-per-thread. For memory-bound kernels, higher occupancy = more warps to hide memory latency. `-maxrregcount` and `__launch_bounds__` force the compiler to stay within a budget. +Registers per thread are one input to residency. Threads per block, static and dynamic shared memory, architectural limits, allocation granularity, barriers, and cluster configuration can impose other limits. Register thresholds therefore produce discrete changes in active blocks or warps; occupancy is not generally the reciprocal of a source-level register count, and higher predicted occupancy does not by itself imply lower latency. -## Pattern +The CUDA occupancy API evaluates the compiled function with an intended block size and dynamic shared-memory size. `cudaOccupancyMaxActiveBlocksPerMultiprocessor` returns the maximum active blocks per SM for that configuration. Convert that result to active warps only after multiplying by the actual warps per block. -```cuda -// Aggressive: 32 registers/thread → ~4 blocks per SM at 256 threads/block -__launch_bounds__(256, 4) -__global__ void gemv_memory_bound(...) { - // Compiler will spill to local memory if needed -} +## What the controls mean -// Or via nvcc flag: -// nvcc -maxrregcount=32 -arch=sm_100a ... -``` +| Control | Documented effect | What it does not establish | +|---|---|---| +| `--maxrregcount=N` | Sets a maximum for GPU functions, subject to the ABI minimum and compiler-reserved registers | That the compiler will use exactly `N`, or that residency/performance will change | +| `__launch_bounds__(T, B)` | Supplies maximum threads per block and desired minimum blocks per SM; the compiler derives an architecture-dependent register limit `L` | Exactly `B` resident blocks when shared memory or another resource is limiting | +| `--resource-usage` | Reports registers, stack frame, spill loads/stores, and other resources for compiled functions | Runtime occupancy or the cost of those resources | +| Occupancy API | Predicts maximum concurrent blocks for the compiled function and launch inputs | Achieved latency, bandwidth, or throughput | + +When launch bounds make initial register use exceed `L`, CUDA documents that the compiler usually trades registers for more local-memory use and/or instructions. When both `T` and `B` are supplied and initial use is below `L`, the compiler may instead increase register use up to `L` to reduce instruction count. A launch bound is therefore compiler guidance and a launch constraint, not an exact register assignment. + +## Reproducible sweep -## Compiler Tradeoffs +Compile the same source and target first without a cap, then with candidate caps. `--resource-usage` makes a cap that does not change the compiled allocation visible. + +```bash +nvcc -arch=sm_100a --resource-usage kernel.cu -o kernel-default +nvcc -arch=sm_100a --maxrregcount=64 --resource-usage kernel.cu -o kernel-r64 +``` -Lower register count → compiler may: -- Spill frequently-used values to local memory (bad) -- Recompute values instead of storing them (neutral) -- Use fewer unrolled iterations (bad for compute-bound) +For each binary and each supported production shape: -For memory-bound kernels, spills can be hidden by memory latency anyway, so aggressive budgeting often wins. +1. Record toolkit, target, compiler options, registers per thread, stack-frame bytes, spill loads/stores, static/dynamic shared memory, block size, and cluster shape. +2. Query active blocks with the occupancy API using the exact function, block size, and dynamic shared-memory bytes. Identify which resource is actually limiting residency. +3. Verify that the candidate cap changed generated resources or code before attributing a timing change to it. Inspect PTX/SASS when instruction selection or loop shape may have changed. +4. Run the same correctness oracle, warmup, synchronization, inputs, and repeated-trial statistic for every variant. Profile spill traffic, memory stalls, and achieved warps rather than assuming the predicted occupancy is reached or useful. +5. Keep a cap only for the measured target and workload where it improves the declared metric without violating correctness or a resource contract. -## GPU Mode NVFP4 GEMV Results +Spill loads and stores use thread-local stack memory backed by the memory hierarchy. They add instructions and traffic; a memory-bound label is not evidence that this cost will be hidden. Conversely, a lower cap can be irrelevant when the uncapped allocation is already below it. -| Rank | Register count | Latency | -|------|---------------|---------| -| 1 | 32 | 18.5μs | -| 3 | 45 | ~20μs | +## Blackwell and TMEM -The measurable difference between 32 and 45 registers shows occupancy dominates for memory-bound NVFP4 GEMV. +For tcgen05 MMA, the resident D accumulator is in TMEM rather than a per-thread D register vector. This changes one major register consumer, but it does not erase register pressure: descriptors, addresses, pipeline/control state, operands, and epilogue batches still use general-purpose registers. The pinned SM100 DeepGEMM path explicitly loads completed TMEM values into registers before its shared/global-store epilogue. -## When To Use +Treat the removed D vector as shape-specific. For example, an SM90 m64n256 FP32 WGMMA fragment exposes 128 D registers per participating thread, but that number is not a universal Blackwell saving and does not predict a resident-block transition. -- Memory-bound kernels (first priority: occupancy) -- Kernels where register pressure comes from inner loop, not accumulators (TMEM handles accumulators) -- Sub-byte types with heavy decode/scale computation +## Scope of the cited NVFP4 observation -## When NOT To Use +Amandeep Singh reports that three inspected B200 NVFP4 GEMV solutions clustered around an 18.5 microsecond geometric mean, with a 32-register cap in rank 1 and 45 in rank 3. The same report says those solutions also differed in PTX decode, cache policies, load widths, exact-K specialization, and B reuse; it provides no controlled register-cap ablation or public contestant code. In the author's own kernel, lowering the cap from 80 to 64 had no effect because natural use was already below 64, while extra accumulator chains and software pipelining increased pressure and regressed. -- Compute-bound GEMM (let compiler use what it needs) -- Kernels where spills to local memory would serialize +That observation motivates inspecting register allocation; it does not prove that 32 beats 45, that occupancy caused the ranking, or that aggressive caps generally help memory-bound kernels. diff --git a/wiki/techniques/software-exp.md b/wiki/techniques/software-exp.md index f45d6782c..fffc0abb6 100644 --- a/wiki/techniques/software-exp.md +++ b/wiki/techniques/software-exp.md @@ -4,211 +4,71 @@ title: "Software-Emulated Exponential" type: technique architectures: [sm100] tags: [software-exp, attention] -confidence: source-reported -reproducibility: snippet +confidence: verified +evidence_basis: + - source_id: doc-flash-attention-4 + evidence_type: official-doc +reproducibility: concept prerequisites: [] related: [kernel-flash-attention-4, technique-warp-specialization] sources: [blog-flash-attention-4, doc-flash-attention-4, doc-ptx-isa-sm100] +blackwell_relevance: "FlashAttention-4 uses a configuration-dependent hybrid of hardware MUFU.EX2 and a degree-3 FMA polynomial for selected forward-softmax entries on SM100; it does not replace every exponential." --- -## Overview - -FlashAttention-4 replaces the hardware Special Function Unit (SFU) exponential (`ex2.approx`) with a software-emulated 2^x function that distributes computation across the SM's FMA (fused multiply-add) units. On Blackwell, tensor core throughput doubled compared to Hopper while SFU count remained the same, making the SFU the throughput bottleneck for attention's softmax operation. The software exponential uses Cody-Waite range reduction followed by a Horner-form polynomial evaluation, achieving sufficient accuracy for attention while bypassing the SFU entirely. - -## Why SFU Is a Bottleneck on Blackwell - -The softmax in attention requires computing `exp(x - max)` for every element of the score matrix. On previous generations, the SFU's `ex2.approx` instruction was fast enough relative to the MMA throughput. On Blackwell: - -| Resource | Hopper (SM90) | Blackwell (SM100) | Ratio | -|----------|--------------|-------------------|-------| -| Tensor core TFLOPS (BF16) | ~990 | ~2250 | 2.27x | -| SFU units per SM | 16 | 16 | 1.0x | -| SFU throughput (exp per cycle) | 16 | 16 | 1.0x | -| FMA units per SM | 128 | 128 | 1.0x | - -The tensor cores produce 2x more score elements per cycle, but the SFU can only process exp() at the same rate as before. This makes the SFU the bottleneck for any kernel that needs exp() proportional to the number of MMA outputs. - -FlashAttention-4's approach: distribute the exp() workload across FMA units (128 per SM) instead of SFU units (16 per SM), achieving 8x the throughput for the exponential computation. - -## Cody-Waite Range Reduction - -Range reduction transforms the input `x` into a small residual that a polynomial can accurately approximate. The Cody-Waite method splits the input into an integer part (for exact power-of-two scaling) and a fractional part (for polynomial approximation): - -```cuda -// Cody-Waite range reduction for 2^x -// Goal: decompose x = n + r where n is integer, r in [-0.5, 0.5] -// Then 2^x = 2^n * 2^r, and 2^r is approximated by polynomial -// -// The Cody-Waite trick: subtract n using two constants (C1 + C2) -// to maintain precision when x is large. -// -// C1 is the nearest representable float to log2(e) with low-order bits zeroed -// C2 is the correction: log2(e) - C1 -// This avoids catastrophic cancellation in x - n - -__device__ float software_exp2(float x) { - // Step 1: Range reduction (Cody-Waite) - // Round x to nearest integer - float n = rintf(x); - // High-precision subtraction using two constants - // C1 and C2 together represent 1.0 in extended precision - const float C1 = 1.0f; // Exact in float - const float C2 = 0.0f; // Correction term (zero for 2^x, non-zero for e^x) - // For 2^x, range reduction is simpler: r = x - n - float r = x - n; // r in [-0.5, 0.5] - - // Step 2: Polynomial approximation of 2^r via Horner's method - // Minimax polynomial coefficients for 2^r on [-0.5, 0.5] - // Degree-4 polynomial: sufficient for ~22 bits of accuracy - const float c0 = 1.0f; - const float c1 = 0.6931471805599453f; // ln(2) - const float c2 = 0.2402265069591007f; // ln(2)^2 / 2 - const float c3 = 0.05550410866482158f; // ln(2)^3 / 6 - const float c4 = 0.009618129107628477f; // ln(2)^4 / 24 - - // Horner evaluation: c0 + r*(c1 + r*(c2 + r*(c3 + r*c4))) - // Each step is one FMA instruction - float poly = c4; - poly = fmaf(poly, r, c3); // FMA 1 - poly = fmaf(poly, r, c2); // FMA 2 - poly = fmaf(poly, r, c1); // FMA 3 - poly = fmaf(poly, r, c0); // FMA 4 - - // Step 3: Reconstruct 2^x = 2^n * poly - // Use integer addition to the float exponent field - int n_int = (int)n; - // ldexpf multiplies by 2^n by adjusting the exponent bits - float result = ldexpf(poly, n_int); - - return result; -} -``` - -## Distributing Across FMA Units - -The key insight is that Horner polynomial evaluation is a chain of FMA operations. With the softmax warp executing on CUDA cores while the MMA warp uses tensor cores, the FMA throughput is fully available: - -```cuda -// FlashAttention-4 softmax with software exp2 -// Executed by dedicated softmax warpgroups (part of warp specialization) -// -// For each row of the score matrix S[i,:]: -// 1. Find row max: m_new = max(S[i,:]) -// 2. Compute exp2((S[i,j] - m_new) * log2(e)) for each j -// 3. Sum for normalization denominator -// 4. Conditionally rescale previous output if max changed - -__device__ void softmax_with_software_exp( - float* scores, // Input: S[i, 0..N-1] (one row) - float* output, // Output: softmax(S[i,:]) - int N, - float* row_max, // Running max (for online softmax) - float* row_sum) // Running sum -{ - int lane = threadIdx.x % 32; - - // Step 1: Find max across the row (warp reduction) - float local_max = -INFINITY; - for (int j = lane; j < N; j += 32) { - local_max = fmaxf(local_max, scores[j]); - } - // Warp-level max reduction - for (int offset = 16; offset > 0; offset >>= 1) { - local_max = fmaxf(local_max, __shfl_xor_sync(0xFFFFFFFF, local_max, offset)); - } - float m_new = local_max; - - // Step 2: Compute software exp2 and sum - float local_sum = 0.0f; - const float LOG2E = 1.4426950408889634f; - - for (int j = lane; j < N; j += 32) { - float x = (scores[j] - m_new) * LOG2E; - float exp_val = software_exp2(x); // 4 FMAs instead of 1 SFU op - output[j] = exp_val; - local_sum += exp_val; - } - - // Warp-level sum reduction - for (int offset = 16; offset > 0; offset >>= 1) { - local_sum += __shfl_xor_sync(0xFFFFFFFF, local_sum, offset); - } - - // Step 3: Conditional rescaling (FlashAttention online softmax) - float m_old = *row_max; - if (m_new > m_old) { - // Rescale previous accumulated output - float scale = software_exp2((m_old - m_new) * LOG2E); - *row_sum = (*row_sum) * scale + local_sum; - *row_max = m_new; - // The output accumulator must also be rescaled by `scale` - } else { - float scale = software_exp2((m_new - m_old) * LOG2E); - *row_sum += local_sum * scale; - // Rescale current exp values, not the accumulator - } -} -``` - -## PTX-Level FMA Chain - -At the PTX level, the Horner polynomial compiles to a tight chain of `fma.rn.f32` instructions: - -```ptx -// Software exp2 via Horner polynomial in PTX -// Input: %x (float, range-reduced to [-0.5, 0.5]) -// Output: %result (float, approximation of 2^x) - -.reg .f32 %x, %r, %n, %poly, %result; -.reg .f32 %c0, %c1, %c2, %c3, %c4; - -// Load polynomial coefficients -mov.f32 %c0, 0f3F800000; // 1.0 -mov.f32 %c1, 0f3F317218; // 0.6931471805599453 (ln2) -mov.f32 %c2, 0f3E75FDF0; // 0.2402265069591007 -mov.f32 %c3, 0f3D635847; // 0.05550410866482158 -mov.f32 %c4, 0f3C1D9539; // 0.009618129107628477 - -// Range reduction: n = rintf(x), r = x - n -cvt.rni.f32.f32 %n, %x; // Round to nearest int -sub.f32 %r, %x, %n; // Fractional part - -// Horner evaluation: 4 dependent FMAs -// poly = c4 -// poly = poly * r + c3 -// poly = poly * r + c2 -// poly = poly * r + c1 -// poly = poly * r + c0 -mov.f32 %poly, %c4; -fma.rn.f32 %poly, %poly, %r, %c3; // FMA 1 -fma.rn.f32 %poly, %poly, %r, %c2; // FMA 2 -fma.rn.f32 %poly, %poly, %r, %c1; // FMA 3 -fma.rn.f32 %poly, %poly, %r, %c0; // FMA 4 - -// Reconstruct: result = poly * 2^n -// Convert n to int and use ex2 scaling via bit manipulation -cvt.rzi.s32.f32 %ni, %n; -ex2.approx.f32 %scale, %n; // Or use integer exponent manipulation -mul.f32 %result, %poly, %scale; -``` - -## Accuracy Considerations - -The degree-4 polynomial provides approximately 22 bits of mantissa accuracy, which is more than sufficient for attention softmax where: -- The input `x = (S[i,j] - max) * log2(e)` is always non-positive -- The softmax output is normalized, so small absolute errors cancel out -- BF16 output has only 7 mantissa bits anyway - -For applications requiring higher accuracy, a degree-6 polynomial (6 FMAs) achieves near-ULP accuracy across the full float range. - -## When to Use - -- **Attention kernels on Blackwell**: Whenever the SFU is the bottleneck for softmax computation. FlashAttention-4 measured 1.1-1.3x speedup over cuDNN from this technique alone on B200. -- **Any kernel limited by transcendental function throughput**: If profiling shows SFU utilization near 100% while FMA utilization is low, software emulation can rebalance the workload. -- **Not recommended on Hopper**: The SFU-to-MMA throughput ratio is better balanced on SM90. The overhead of 4 FMAs vs 1 SFU instruction is not justified unless the SFU is proven to be the bottleneck. - -## Caveats - -- The 4-FMA chain has a latency of ~16 cycles (4 dependent FMAs at ~4 cycles each), vs ~20 cycles for SFU `ex2.approx`. Latency is comparable; the win comes from throughput (128 FMA units vs 16 SFU units). -- Polynomial coefficients are for 2^x on [-0.5, 0.5]. For e^x, multiply the input by log2(e) first. -- The `ldexpf` or exponent bit-manipulation step for 2^n must handle overflow/underflow (very large/small x). In attention, `x <= 0` always holds, so only underflow toward zero is possible. +# Software-emulated exponential + +## What FlashAttention-4 implements + +FlashAttention-4 (FA4) combines two paths for base-2 exponential in its Blackwell forward softmax. Most selected configurations retain hardware `exp2` for some entries and evaluate a tunable fraction with a software polynomial on general-purpose FMA pipelines. The paper describes roughly 10–25% software evaluation, while the exact fraction is a configuration choice rather than an architecture constant. + +This hybrid is part of a larger schedule: two 128-thread softmax warpgroups alternate 128-row query tiles, synchronize to limit simultaneous exponential contention, stage probabilities through TMEM, and hand conditional rescaling to a correction warpgroup. A standalone polynomial is not equivalent to that pipeline. + +## Scoped bottleneck model + +For the authors' `M=N=D=128` B200 feeds-and-speeds model, one SM supplies 8192 BF16 tensor-core operations per cycle, 16 exponential operations per cycle, and 128 shared-memory bytes per cycle. Their forward-tile accounting assigns 1024 cycles to two MMAs, 1024 cycles to 128×128 exponentials, and 768 cycles to shared-memory traffic. + +Those are analytical inputs for that tile and schedule. They do not imply that every kernel with one exponential per MMA output is exponential-bound, nor do functional-unit counts give the throughput or latency of a software approximation. Range reduction, polynomial dependencies, reconstruction, instruction issue, and overlap all remain part of the measured implementation. + +## Pinned software path + +At Dao-AILab/flash-attention revision `a369df707e1980fb328abcc1733e3457ec10155f`, [`flash_attn/cute/utils.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py) defines the default software path and [`flash_attn/cute/softmax.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/softmax.py) performs fragment-level hardware/software selection. + +For each software-selected pair, the pinned implementation: + +1. Assumes each input is no greater than 127 and clamps values below `-127`. +2. Uses a rounding-down addition with the float32 constant `2^23 + 2^22` to recover the integer floor and a fractional value in `[0,1)`. +3. Evaluates a degree-3 polynomial in Horner form with packed float32 FMA operations. +4. Combines the integer contribution with the polynomial's float32 representation by exponent-field integer operations; the software branch does not call `ex2.approx` for reconstruction. + +The degree-3 float32 coefficients in that revision are: + +| Term | Coefficient | +|---:|---:| +| `p0` | `1.0` | +| `p1` | `0.695146143436431884765625` | +| `p2` | `0.227564394474029541015625` | +| `p3` | `0.077119089663028717041015625` | + +The authors state that Sollya selected these coefficients to minimize relative error for `2^f` over `f in [0,1)`. For a natural exponential, the caller uses the identity `e^z = 2^(z * log2(e))`. + +## Numerical contract + +The polynomial is approximate. On a deterministic host grid of 1,000,002 evenly spaced points over `[0,1]`, evaluating the rounded degree-3 coefficients against `2^x` produced a maximum sampled relative error of approximately `8.763e-5`. This is a regression observation, not a proof of the continuous maximum and not an end-to-end attention tolerance. + +BF16 conversion does not by itself prove that approximation errors are harmless. Per-entry error changes both the softmax numerator and row sum, and masking, all-`-inf` rows, conditional rescaling, underflow clamping, and output accumulation introduce separate boundary cases. Validate the scalar approximation and the complete attention output for the exact dtype and features. + +## Reproduction and decision procedure + +Use the pinned implementation as the starting point; do not reconstruct it from rounded blog coefficients. For each target configuration: + +1. Build an all-hardware control and one or more hybrid selections from the same source revision, compiler, target, and launch configuration. +2. Inspect generated PTX/SASS to confirm which entries use hardware `MUFU.EX2`, which use the expected FMA/range-reduction sequence, and whether register use or spills changed. +3. Test ordinary, masked, all-masked, causal, variable-length, large-gap, and underflow-heavy rows against a declared higher-precision reference. Record maximum absolute/relative output error and row-sum behavior, not only BF16 agreement on random inputs. +4. Profile exponential-pipeline pressure, FMA/ALU issue, tensor-core overlap, registers, and spills. A high hardware-exp metric is a lead, not proof that moving more entries to FMA improves the schedule. +5. Benchmark identical shapes, inputs, warmup, synchronization, clock policy, and repeated-trial statistics. Sweep the software fraction because both zero emulation and mixed settings appear in the pinned target/configuration table. + +Keep the hybrid only where both the numerical contract and declared end-to-end metric pass. Do not transfer the choice to another architecture or transcendental function without repeating the reduction, approximation, special-value, generated-code, and performance checks. + +## Performance provenance + +The FA4 blog reports complete forward-pass speedups of 1.1–1.3× over cuDNN 9.13 on its evaluated B200 BF16 configurations. That comparison includes the full FA4 co-design—pipelining, hybrid exponentials, conditional rescaling, TMEM staging, scheduling, and other choices. It is not a software-exponential-only ablation and must not be used as the isolated speedup of this technique. diff --git a/wiki/techniques/swizzling.md b/wiki/techniques/swizzling.md index 7c0605a37..52f8c8ae9 100644 --- a/wiki/techniques/swizzling.md +++ b/wiki/techniques/swizzling.md @@ -4,163 +4,91 @@ title: "Shared Memory Swizzling" type: technique architectures: [sm100, sm90] tags: [swizzling, shared-memory-optimization, tma] -confidence: source-reported -reproducibility: snippet +confidence: verified +evidence_basis: + - source_id: doc-cuda-13-0-2-tma + evidence_type: official-doc +reproducibility: concept prerequisites: [hw-tma] related: [hw-tma, technique-pipeline-stages, pattern-memory-bound] -sources: [doc-nvidia-tuning-guide, blog-tcgen05-tutorial, blog-modular-blackwell] -blackwell_relevance: "128-byte swizzling mandatory for Blackwell tcgen05 inputs; same concept on Hopper but less critical." +sources: [doc-cuda-13-0-2-tma, doc-ptx-isa-sm100, blog-tcgen05-tutorial] +blackwell_relevance: "SM100 TMA and tcgen05 support multiple shared-memory mappings, including no swizzle and 32B, 64B, and 128B swizzles; correctness requires the producer and consumer to describe the same legal layout, not universal 128B use." --- -## Overview - -Shared memory swizzling remaps the linear address layout of a matrix tile in SMEM so that threads accessing consecutive columns (or rows) hit different 32-byte banks rather than the same bank. This eliminates bank conflicts that would otherwise serialize concurrent accesses. On Blackwell (SM100), 128-byte swizzling is mandatory for TMA loads and tcgen05.mma operands. Without it, performance drops to 46% of the achievable throughput for GEMM workloads. - -## Why 128-Byte Swizzling is Mandatory on Blackwell - -Shared memory has 32 banks, each 4 bytes wide (128 bytes total per bank cycle). When a warp accesses a matrix stored in row-major layout, threads in the same warp reading elements from consecutive rows in the same column hit the same bank, causing a 32-way bank conflict. - -The TMA unit on both Hopper and Blackwell encodes the swizzle pattern as part of the tensor descriptor. The tcgen05.mma instruction expects its SMEM operands to already be swizzled in the 128-byte pattern. Using unswizzled data produces incorrect MMA results. - -The tcgen05-tutorial benchmark progression shows the impact: - -``` -Naive (no swizzle): 255 TFLOPS (17% of cuBLAS) -128B swizzle applied: 695 TFLOPS (46% of cuBLAS) - ---- 2.7x improvement from swizzling alone ---- -``` - -## How 128-Byte Swizzling Works - -The swizzle function XORs a portion of the column address with the row address to scatter accesses across banks: - -```cuda -// 128-byte swizzle: XOR bits [4:6] of the byte offset with the row index -// This ensures that consecutive rows accessing the same logical column -// map to different physical SMEM banks. -// -// For a tile stored in SMEM with TILE_N columns of 2-byte elements: -// byte_offset = row * (TILE_N * sizeof(half)) + col * sizeof(half) -// swizzled_offset = byte_offset ^ ((row & 0x7) << 4) -// -// The mask 0x7 = 3 bits, shift 4 = bits [4:6], giving 8-row periodicity -// across the 128-byte bank group. - -__device__ int swizzle_128B(int row, int col, int stride_bytes) { - int byte_offset = row * stride_bytes + col * sizeof(half); - // XOR bits [4:6] of byte offset with low 3 bits of row - int swizzled = byte_offset ^ ((row & 0x7) << 4); - return swizzled; -} -``` - -Visually, for an 8-row x 64-column half-precision tile (128 bytes per row): - -``` -Without swizzle (row-major): - Row 0: bank 0,1,2,...,31 bank 0,1,2,...,31 - Row 1: bank 0,1,2,...,31 bank 0,1,2,...,31 - Row 2: bank 0,1,2,...,31 bank 0,1,2,...,31 - -> Column access = 8-way bank conflict - -With 128B swizzle (XOR pattern): - Row 0: bank 0,1,2,...,31 bank 0,1,2,...,31 - Row 1: bank 1,2,3,...,0 bank 1,2,3,...,0 (rotated by 1) - Row 2: bank 2,3,4,...,1 bank 2,3,4,...,1 (rotated by 2) - -> Column access = conflict-free -``` - -## TMA Swizzle Encoding - -The TMA descriptor encodes the swizzle mode when creating a tensor map. The swizzle mode must match what the consumer (tcgen05.mma or wgmma) expects: - -```cuda -// Creating a TMA descriptor with 128-byte swizzle -#include - -CUtensorMap tensor_map; - -// Swizzle mode: CU_TENSOR_MAP_SWIZZLE_128B -// This tells TMA to apply the 128-byte XOR swizzle pattern -// when writing data into shared memory -cuTensorMapEncodeTiled( - &tensor_map, - CU_TENSOR_MAP_DATA_TYPE_FLOAT16, - 2, // 2D tensor - global_ptr, // global memory base - global_dims, // {N, M} dimensions - global_strides, // {N * sizeof(half), sizeof(half)} - tile_dims, // {TILE_N, TILE_M} - element_strides, // {1, 1} - CU_TENSOR_MAP_INTERLEAVE_NONE, - CU_TENSOR_MAP_SWIZZLE_128B, // 128-byte swizzle - CU_TENSOR_MAP_L2_PROMOTION_NONE, - CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE -); -``` - -The available swizzle modes and their use cases: - -| Swizzle Mode | Bank Spread | Use Case | -|-------------|-------------|----------| -| `SWIZZLE_NONE` | No remapping | Non-MMA data (flags, scales) | -| `SWIZZLE_32B` | 32-byte groups | Narrow tiles, small data types | -| `SWIZZLE_64B` | 64-byte groups | Medium tiles | -| `SWIZZLE_128B` | 128-byte groups | Standard for BF16/FP16 MMA operands | - -## CuTe Swizzle Layout - -In CuTe/CUTLASS, swizzle is expressed as a layout composition: - -```cuda -// CuTe swizzle layout for 128-byte swizzle pattern -// Swizzle where: -// B = number of bits in the base (non-swizzled) portion -// M = number of bits in the mask -// S = shift amount -// -// Swizzle<3, 4, 3> encodes the 128B swizzle: -// 3 base bits (8-byte alignment) -// 4 mask bits (16 rows) -// 3 shift bits (8-column groups) - -using SmemLayoutAtom = decltype( - composition( - Swizzle<3, 4, 3>{}, - Layout, - Stride<_64, _1>>{} - ) -); - -// Tile the atom across the full SMEM tile -using SmemLayoutA = decltype( - tile_to_shape(SmemLayoutAtom{}, Shape, Int>{}) -); -``` - -## Verification: Detecting Bank Conflicts - -Use `nvprof` or Nsight Compute to verify that swizzling eliminates conflicts: - -```python -# Nsight Compute command to check shared memory bank conflicts -# Look for "Shared Memory Bank Conflicts" metric -# ncu --metrics l1tex__data_bank_conflicts_pipe_lsu_mem_shared_op_ld.sum \ -# ./my_kernel - -# Expected results: -# Without swizzle: bank_conflicts >> 0 -# With 128B swizzle: bank_conflicts == 0 -``` - -## When to Use - -- **All Blackwell tensor core kernels**: 128-byte swizzling is not optional. Both TMA and tcgen05.mma require it for correct results and peak performance. -- **Hopper wgmma kernels**: Same requirement applies; wgmma expects swizzled SMEM operands. -- **Non-MMA shared memory access**: If multiple warps access the same SMEM tile in a column pattern (e.g., reduction), swizzling prevents serialization. - -## Caveats - -- The swizzle mode in the TMA descriptor must exactly match the access pattern of the consumer. A mismatch produces silently incorrect results, not a runtime error. -- Swizzled layouts make SMEM address computation non-trivial. Using CuTe's layout algebra avoids manual indexing errors. -- For data types wider than 2 bytes (e.g., FP32 accumulators), the optimal swizzle mode may differ. TMEM accumulators avoid this issue since they use a separate address space. +# Shared-memory swizzling + +## Bank-conflict model + +Shared memory has 32 banks, and successive 32-bit words map to successive banks. For one warp memory request, accesses to different words in the same bank require serialized wavefronts; same-word reads can broadcast and do not create that conflict. The result therefore depends on the exact byte address requested by every participating lane. + +A swizzle permutes address chunks so a target access pattern may distribute its requests over banks more evenly. It can reduce conflicts for that pattern, but it does not make every access to the tile conflict-free. A layout favorable to an MMA operand can still be unfavorable to a CUDA-core row, column, transpose, or epilogue access. + +## Three mappings must agree + +For a TMA-to-tcgen05 path, keep these layers consistent: + +1. The tensor map describes how TMA places the global-memory box into shared memory, including interleave and swizzle. +2. The shared-memory allocation and base address satisfy the alignment and span constraints of that mapping. +3. The tcgen05 shared-memory descriptor describes the same physical-to-logical layout, leading/stride dimensions, base offset, and swizzle mode expected by the MMA operand. + +PTX ISA 9.0 permits ordinary tcgen05 shared-memory descriptor modes with no swizzle and 32B, 64B, or 128B spans, plus a 128B/base-32B mode. In the descriptor's three-bit layout field, ordinary no-swizzle is `0`, 128B/base-32B is `1`, ordinary 128B is `2`, 64B is `4`, and 32B is `6`; values `3`, `5`, and `7` are invalid. Legality also depends on MMA kind, element type, major mode, shape, CTA group, and target. + +The CUDA 13.0.97 Driver API exposes TMA modes including: + +| Tensor-map mode | Documented chunk mapping | +|---|---| +| `CU_TENSOR_MAP_SWIZZLE_NONE` | No bank swizzle | +| `CU_TENSOR_MAP_SWIZZLE_32B` | 16B chunks within a 32B span | +| `CU_TENSOR_MAP_SWIZZLE_64B` | 16B chunks within a 64B span | +| `CU_TENSOR_MAP_SWIZZLE_128B` | 16B chunks within a 128B span | +| `CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B` | 32B chunks within a 128B span | +| `CU_TENSOR_MAP_SWIZZLE_128B_ATOM_64B` | 64B chunks within a 128B span; support is operation/type-specific | + +These names define mappings, not universal datatype or tile-size recommendations. The tensor-map encoder imposes mode-, interleave-, datatype-, alignment-, and box-size constraints; for example, ordinary 128B modes require the inner bounding-box byte span not to exceed 128 bytes. Check the exact archived API for the selected type and operation. + +## CuTe address-bit form + +CUTLASS 4.5.0 defines [`Swizzle`](https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/swizzle.hpp): + +- `BBits` is the number of mask bits; +- `MBase` is the number of least-significant address bits kept invariant; and +- `SShift` is the distance between the two bit fields. + +For `Swizzle<3,4,3>`, three address bits beginning above the four invariant low bits are XORed with the three-bit field shifted by three positions. Equivalently, address bits 7:9 affect bits 4:6. This is an address transformation, not a general `row & 7` rule: equivalence to row-based indexing depends on stride, byte units, base alignment, and layout composition. + +CUTLASS also distinguishes a position-independent composed swizzle layout from a position-dependent swizzle pointer, because hardware swizzling depends on the shared-memory pointer address. Reuse a pinned complete layout/descriptor construction or prove the composition and required base alignment; a `Swizzle` type alone is not a complete TMA or MMA contract. + +## Tensor-map construction checklist + +For `cuTensorMapEncodeTiled`, record and validate all inputs rather than copying only the swizzle enumerator: + +1. Use a correctly aligned `CUtensorMap` object and global base pointer. +2. Supply rank-sized global dimensions, rank-minus-one byte strides (the fastest dimension is implicit), rank-sized box dimensions, and rank-sized element strides. +3. Satisfy the global alignment, stride, box, interleave, datatype, and selected-swizzle constraints. +4. Check the returned `CUresult`; do not launch with an output descriptor after encoding failed. +5. Use a shared-memory base and tcgen05 descriptor that represent the same mapping as the tensor map. + +An invalid encoder combination can fail explicitly. A successfully encoded tensor map paired with the wrong consumer layout may instead read the wrong logical elements, so a successful API return is not a correctness oracle. + +## Source-reported tutorial result + +Gau Nernst's pinned B200 tutorial compares M=N=K=4096 kernels with PyTorch 2.9.1 and CUDA 13. Its 3D TMA version changes from a 16-byte inner tile with no swizzle to a 128-byte inner tile with 128B swizzling and matching tcgen05 descriptors: + +| Tutorial version | Author-reported TFLOP/s | +|---|---:| +| v1b: 3D 16B TMA | 252.81 | +| v2b: 3D 128B TMA plus 128B swizzle | 695.43 | + +The combined change is approximately 2.75× and the v2b endpoint is about 46% of the tutorial's 1506.74 TFLOP/s cuBLAS result. It is not a bank-conflict or swizzle-only ablation. The author notes that the earlier contiguous `8×16B` tile might already span all 32 banks, lacked Nsight Compute access, and offers wider TMA transfers as an alternative explanation. + +## Verification procedure + +For each candidate mapping: + +1. Keep global tensor, box, consumer instruction, tile shape, synchronization, and output oracle fixed. Change only the producer/consumer layout pair when an isolated comparison is possible. +2. Check every tensor-map encoder result and run correctness tests that distinguish rows, columns, tiles, boundaries, and out-of-bounds fill. Do not rely on random uniform values that can hide permutations. +3. Use the installed Nsight Compute's `--query-metrics` or Memory Workload Analysis rather than assuming one metric name exists on every chip/tool version. Current documentation includes `l1tex__data_bank_conflicts_pipe_lsu.sum` and request/wavefront/conflict analysis. +4. Compare executed shared-memory requests, ideal versus excessive wavefronts, bank conflicts, TMA behavior, total time, and occupancy. Zero observed CUDA-core bank conflicts neither proves the MMA descriptor matches nor explains a performance change by itself. +5. Record GPU, clocks, toolkit, profiler, compiler, exact addresses/alignment, layout types, descriptor fields, warmup, repetitions, and trial statistic. + +Select the mapping that is legal and correct for both producer and every consumer, then retain it only where the controlled target workload improves. Do not infer the choice from architecture name, element width, or “MMA versus non-MMA” alone. diff --git a/wiki/techniques/tile-scheduling.md b/wiki/techniques/tile-scheduling.md index 17fdd610b..50cf41826 100644 --- a/wiki/techniques/tile-scheduling.md +++ b/wiki/techniques/tile-scheduling.md @@ -8,199 +8,90 @@ confidence: source-reported reproducibility: snippet prerequisites: [hw-clc] related: [hw-clc, technique-persistent-kernels, pattern-low-sm-utilization] -sources: [doc-nvidia-tuning-guide, doc-cutlass-blackwell, pr-cutlass-2161] -blackwell_relevance: "CLC (SM100-only) replaces static scheduling; Hopper patterns provide baseline comparison." +sources: [doc-cutlass-clc, doc-cutlass-blackwell] +blackwell_relevance: "SM100 CLC can reassign unlaunched grid IDs; raster order, swizzle, persistence, and Stream-K remain distinct software scheduling choices." --- -## Overview +## Keep the Scheduling Layers Separate -Tile scheduling determines the order in which output tiles of a GEMM (or attention) kernel are assigned to CTAs. The scheduling order affects L2 cache hit rates, tail-effect severity, and overall GPU utilization. On Blackwell, the CLC hardware unit supports dynamic scheduling policies including swizzled raster, while Hopper relies on software-based static stride or swizzled patterns computed at launch time. +“Tile scheduling” can refer to four different choices: -## Scheduling Strategies +1. **Coordinate order:** a software mapping from a logical work index to `(tile_m, tile_n, ...)`, such as row-major, column-major, or a blocked/swizzled raster. +2. **Resident-worker iteration:** whether a CTA handles one logical tile or repeatedly advances through work, for example by a static grid stride. +3. **Work reassignment:** on SM100, Cluster Launch Control (CLC) lets a running worker cancel another grid entity that has not started and process the returned ClcID. +4. **Problem decomposition:** Stream-K or Split-K can partition a tile's K work and then reduce partial results. -### Linear Raster (Naive) +CLC does not accept a raster-order or swizzle-policy operand. CUTLASS applies those coordinate transforms in software to initial or returned coordinates. CLC also does not itself create K partitions or synthesize work beyond the launched grid. -Tiles are assigned in row-major order. Simple but poor L2 locality: consecutive tiles share no B-matrix data until the entire M-dimension is traversed. +## Exact Static Mappings -```cuda -// Linear raster: tile_idx maps directly to (tile_m, tile_n) -__device__ void linear_raster(int tile_idx, int tiles_n, - int& tile_m, int& tile_n) { - tile_m = tile_idx / tiles_n; - tile_n = tile_idx % tiles_n; -} - -// Access pattern for a 4x4 tile grid: -// 0 1 2 3 -// 4 5 6 7 -// 8 9 10 11 -// 12 13 14 15 -// -// Problem: tiles 0,1,2,3 all load different B columns. -// By tile 4, B column 0 has been evicted from L2. -``` - -### Swizzled Raster - -Tiles are assigned in a blocked pattern that groups nearby M and N tiles together, maximizing reuse of both A rows and B columns in L2 cache: - -```cuda -// Swizzled raster: group tiles into blocks that share A and B data -// swizzle_size controls the block width (typically 4-8) -__device__ void swizzled_raster(int tile_idx, int tiles_m, int tiles_n, - int swizzle_size, int& tile_m, int& tile_n) -{ - // Number of tile columns per swizzle group - int group_cols = min(swizzle_size, tiles_n); - int tiles_per_group = tiles_m * group_cols; - - // Which swizzle group - int group_idx = tile_idx / tiles_per_group; - int within_group = tile_idx % tiles_per_group; - - // Within the group, iterate in column-major order - tile_m = within_group / group_cols; - tile_n = group_idx * group_cols + within_group % group_cols; -} - -// Access pattern with swizzle_size=2 on a 4x4 grid: -// 0 1 | 8 9 -// 2 3 | 10 11 -// 4 5 | 12 13 -// 6 7 | 14 15 -// -// Tiles 0,1,2,3 share the same B columns (0,1). -// Tiles 0,2,4,6 share the same A rows. -// Much better L2 reuse. -``` - -### Static Stride (Hopper Persistent) - -Each CTA processes tiles at fixed intervals equal to the grid size: +A flat row-major mapping is: ```cuda -// Static stride: CTA i processes tiles i, i+gridDim.x, i+2*gridDim.x, ... -__device__ void static_stride(int cta_id, int total_ctas, - int iteration, int tiles_n, +__device__ void row_major_tile(int tile_idx, int tiles_n, int& tile_m, int& tile_n) { - int tile_idx = cta_id + iteration * total_ctas; - tile_m = tile_idx / tiles_n; - tile_n = tile_idx % tiles_n; + tile_m = tile_idx / tiles_n; + tile_n = tile_idx % tiles_n; } ``` -### CLC Dynamic Scheduling (Blackwell) - -The CLC hardware scheduler assigns tiles at runtime, combining the benefits of dynamic load balancing with configurable scheduling policies: +For a positive grid size, a static persistent worker can cover the flat index set without overlap: ```cuda -// CLC-based scheduling on Blackwell -// The scheduling policy is set once during CLC initialization -enum class ClcSchedulePolicy { - LinearRaster, // Simple row-major order - SwizzledRaster, // Blocked pattern for L2 locality - ColumnFirst, // Column-major for specific workloads - Hilbert // Space-filling curve (experimental) -}; - -__device__ void clc_init_scheduler( - void* clc_buffer, - int tiles_m, int tiles_n, - ClcSchedulePolicy policy) -{ - if (threadIdx.x == 0) { - // Configure the software scheduler metadata used around CLC queries. - // The CLC PTX surface is try_cancel/query_cancel, not clusterctl.init. - uint32_t config = encode_clc_config(tiles_m, tiles_n, policy); - init_clc_scheduler_metadata(clc_buffer, tiles_m, tiles_n, config); - } - __syncwarp(); +for (int tile_idx = int(blockIdx.x); + tile_idx < total_tiles; + tile_idx += int(gridDim.x)) { + int tile_m = tile_idx / tiles_n; + int tile_n = tile_idx % tiles_n; + compute_tile(tile_m, tile_n); } ``` -## CUTLASS Tile Schedulers +Changing coordinate order changes the sequence of operand panels presented to the cache. It does not by itself prove a hit rate or speedup. A valid transform must first be shown to be in-bounds and bijective for edge groups; then its locality must be evaluated for the actual tile shape, batch/group structure, concurrent traffic, and cache capacity. -CUTLASS provides several tile schedulers that abstract these strategies: +## SM100 CLC Work Acquisition -```cuda -// CUTLASS tile scheduler selection for SM100 -// All persistent schedulers inherit from PersistentTileSchedulerSm100 - -// 1. Default CLC scheduler with swizzled raster -using Scheduler_Default = cutlass::gemm::PersistentTileSchedulerSm100; - -// 2. Stream-K scheduler for better tail handling -// Splits K-dimension across CTAs for the last wave -using Scheduler_StreamK = cutlass::gemm::StreamKSchedulerSm100; - -// 3. Grouped GEMM scheduler for MoE workloads -// Each group has different M, shared N and K -using Scheduler_Grouped = cutlass::gemm::GroupedTileSchedulerSm100; - -// Usage in CUTLASS kernel definition: -using GemmKernel = cutlass::gemm::kernel::GemmUniversal< - cute::Shape, Int<256>, Int<64>>, // Tile shape - ElementA, LayoutA, - ElementB, LayoutB, - ElementC, LayoutC, - TiledMma, - CollectiveMainloop, - CollectiveEpilogue, - Scheduler_Default // Tile scheduler ->; -``` +CUTLASS 4.5.0 documents this lifecycle for a CLC-backed persistent scheduler: -## L2 Cache Locality Analysis +1. Launch the full problem grid and process the worker's initial `blockIdx` coordinate. +2. Submit `clusterlaunchcontrol.try_cancel` against another not-yet-started grid entity. The asynchronous 16-byte response completes a transaction on an mbarrier. +3. After the transaction completes, query whether cancellation succeeded. On success, decode and process the returned ClcID; for clusters, combine the returned first coordinate with the local cluster rank. +4. Treat an observed failed request as terminal for requests by that thread. Retrying from the same thread after failure is undefined. -The choice of scheduling strategy directly impacts L2 cache hit rates. On B200 with 126 MB L2: +This can redistribute existing work when SM availability is uneven. It changes which resident worker handles an unlaunched ID, not the number of independent IDs in the grid. Software raster/swizzle may be applied consistently to both initial and returned coordinates. -```python -# L2 cache reuse analysis for different schedulers -# Problem: M=8192, N=8192, K=4096, BF16 -# Tile: 128x256, giving 64x32 = 2048 tiles -# B200: 142 SMs, 126 MB L2 +## CUTLASS 4.5.0 Scheduler Routes -tile_bytes_A = 128 * 4096 * 2 # 1 MB per tile row of A -tile_bytes_B = 4096 * 256 * 2 # 2 MB per tile column of B +The public scheduler tags select implementation classes; users should prefer those tags over spelling detail types directly. -# Linear raster: first wave loads 142 tiles across 142/32 = 4.4 column groups -# B data for 5 different column groups = 5 * 2 MB = 10 MB (fits in L2) -# But A data for 142/32 = 4.4 row groups * 4.4 col groups = ~20 distinct A rows -# 20 * 1 MB = 20 MB -> some L2 eviction +| Public tag for `arch::Sm100` | Selected implementation | Scope | +|---|---|---| +| `PersistentScheduler` (also the default `void` tag) | `PersistentTileSchedulerSm100` | CLC-backed persistent data-parallel scheduling | +| `DynamicPersistentScheduler` | `PersistentTileSchedulerSm100` | Explicit dynamic route to the same SM100 implementation | +| `StaticPersistentScheduler` | `StaticPersistentTileScheduler100` | Static persistent scheduling | +| `StreamKScheduler` | `PersistentTileSchedulerSm100StreamK` | Parameterized data-parallel, Stream-K, or Split-K decomposition | +| `GroupScheduler` | `PersistentTileSchedulerSm100Group` | Grouped-problem route; in this tag it wraps the SM90-style static group scheduler | -# Swizzled raster (swizzle=4): first wave covers 142 tiles in ~36 groups of 4 -# Each group uses 1 A row + 4 B columns = 1 + 8 = 9 MB per group -# But groups share A rows: total unique A = ~36 rows * 1 MB = 36 MB -# L2 pressure: 36 MB + 8 MB = 44 MB (fits in B200's 126 MB L2) +The SM100 Stream-K route accepts decomposition, split, raster, swizzle, and reduction settings. CUTLASS 4.5.0 has a deterministic lock/turnstile reduction and a nondeterministic atomic-workspace reduction; atomic accumulation is therefore not a universal property of every Stream-K configuration. -# Conclusion: swizzled raster reduces L2 misses by ~2x vs linear for large problems -``` +## Tail Arithmetic -## Tail Effect Mitigation +For a deliberately simplified model with `T` equal-duration independent tiles, `W` available one-tile workers, and no K decomposition, write: -The "tail effect" occurs when the last wave of tiles does not fully occupy all SMs. Different schedulers handle this differently: - -| Scheduler | Tail Handling | SM Utilization (Last Wave) | -|-----------|---------------|---------------------------| -| Linear raster | None | `(total_tiles % num_SMs) / num_SMs` | -| Static stride | None | Same as linear | -| CLC dynamic | Automatic | Fast CTAs steal from slow ones | -| Stream-K | K-splitting | Near 100% (splits partial tiles across SMs) | +```text +T = qW + r, 0 <= r < W +``` -For a problem with 150 tiles on 142 SMs: -- Static: last wave has 8 tiles on 8 SMs, 134 SMs idle (5.6% utilization) -- CLC: fast-finishing CTAs from wave 1 absorb the 8 extra tiles -- Stream-K: the 8 remaining tiles are split across all 142 SMs +If `r > 0`, the final partial wave has `r` active workers and occupancy `r / W`. If `r == 0` and `T > 0`, the final wave is full; reporting zero occupancy from the remainder alone is an error. -## When to Use +For `T = 150` and `W = 142`, the second wave contains 8 tiles, so 8 workers are active, 134 are idle, and wave occupancy is `8 / 142 = 5.63%` under those assumptions. CLC may change which workers receive those eight IDs and reduce delays caused by uneven worker availability, but it cannot turn eight independent IDs into 142. Stream-K may create additional K partitions, but its selected decomposition and reduction overhead determine whether that is profitable; near-100-percent occupancy is not guaranteed. -- **Swizzled raster**: Default choice for large GEMMs. Always better than linear for L2 locality. -- **CLC dynamic**: Recommended on Blackwell for all persistent kernels. Combines dynamic load balancing with swizzled ordering. -- **Stream-K**: Best for small-to-medium problems where the tail effect dominates. Adds complexity for K-dimension synchronization. -- **Grouped scheduler**: Essential for MoE and batched GEMM where problem sizes vary across groups. +## Selection and Verification Workflow -## Caveats +1. Establish a correct unswizzled data-parallel mapping and verify exact tile coverage, including nondivisible edge groups. +2. Sweep legal raster orders and swizzle sizes. Record kernel time plus L2 hit/sector traffic; do not infer eviction from coordinate order alone. +3. Compare static and CLC-backed persistence at identical tile, cluster, grid, and occupancy settings. Record successful/failed CLC requests and per-worker tile counts where instrumentation permits. +4. Test Stream-K separately with explicit decomposition, split, and reduction modes. Include workspace traffic, synchronization, determinism, and numerical tolerance. +5. Repeat across representative shapes and grouped-size distributions. Report regressions as well as wins; there is no universal best scheduler for all GEMMs, attention kernels, or MoE workloads. -- Swizzle size must be tuned per problem shape. Too large a swizzle group exceeds L2 capacity; too small loses the locality benefit. -- CLC scheduling adds a small latency per tile acquisition (~10s of cycles). For extremely small tiles, this overhead is proportionally larger. -- Stream-K requires atomic accumulation where K-splits meet, adding synchronization overhead. Only worthwhile when tail utilization is a proven bottleneck. +Without target-GPU measurements, this page makes no fixed claim for CLC acquisition latency, L2-miss reduction, or scheduler speedup. diff --git a/wiki/techniques/vectorized-loads.md b/wiki/techniques/vectorized-loads.md index bd796add0..f65d903df 100644 --- a/wiki/techniques/vectorized-loads.md +++ b/wiki/techniques/vectorized-loads.md @@ -1,302 +1,89 @@ --- id: technique-vectorized-loads -title: "Wide Vectorized Loads and Cache Policies" +title: "Vectorized Loads and Cache Hints" type: technique architectures: [sm100, sm90] tags: [vectorized-loads, cache-policy, register-budgeting] -confidence: source-reported -reproducibility: snippet +confidence: verified +evidence_basis: + - source_id: doc-ptx-isa-sm100 + evidence_type: official-doc +reproducibility: concept prerequisites: [] -related: [kernel-nvfp4-gemv, pattern-memory-bound] -sources: [blog-yue-nvfp4, blog-amandeep-nvfp4, contest-gpumode-p1] -blackwell_relevance: "Cache policy PTX qualifiers work on both; B200's 8TB/s bandwidth makes these techniques even more impactful." +related: [technique-register-budgeting, kernel-nvfp4-gemv, pattern-memory-bound] +sources: [doc-ptx-isa-sm100, doc-cuda-register-controls, blog-yue-nvfp4, blog-amandeep-nvfp4, contest-gpumode-p1] +blackwell_relevance: "PTX ISA 9.0 adds SM100 support for 256-bit global vector loads through v8.b32 or v4.b64 forms; their alignment, cache-hint, register, and workload tradeoffs remain explicit verification obligations." --- -## Overview +# Vectorized Loads and Cache Hints -For memory-bound kernels (low arithmetic intensity), maximizing global memory throughput is critical. Three complementary techniques from the GPU Mode NVFP4 Hackathon achieve this: (1) wide vectorized loads (128-bit and 256-bit) to saturate memory bandwidth per thread, (2) differentiated L1 cache policies to keep reused data hot while bypassing the cache for streaming data, and (3) register budgeting via `-maxrregcount` to increase occupancy. These techniques reduced NVFP4 GEMV latency from 2000us to 22.4us (89x improvement). +## What a wider load establishes -## Wide Vectorized Loads +A PTX vector load moves several typed elements into several registers with one instruction. This can reduce the number of load instructions issued by a thread, but PTX only says that vector loads *may* improve memory performance. Source-level width alone does not prove fewer physical memory transactions, higher achieved bandwidth, or lower kernel latency. -Standard 32-bit loads waste memory bus bandwidth. Wider loads amortize the instruction overhead and saturate the 8 TB/s HBM bandwidth of B200: - -```cuda -// Vectorized load widths comparison for FP4 GEMV -// Each thread loads more data per instruction - -// 32-bit load: 4 bytes per thread per instruction -float val; -asm volatile("ld.global.b32 %0, [%1];" : "=f"(val) : "l"(ptr)); - -// 64-bit load: 8 bytes per thread per instruction -uint2 val64; -asm volatile("ld.global.v2.u32 {%0,%1}, [%2];" - : "=r"(val64.x), "=r"(val64.y) : "l"(ptr)); - -// 128-bit load: 16 bytes per thread per instruction (preferred) -uint4 val128; -asm volatile("ld.global.v4.u32 {%0,%1,%2,%3}, [%4];" - : "=r"(val128.x), "=r"(val128.y), "=r"(val128.z), "=r"(val128.w) - : "l"(ptr)); - -// 256-bit load: 32 bytes per thread per instruction -// Requires v4.u64 (4 x 64-bit) -uint64_t v256[4]; -asm volatile("ld.global.v4.u64 {%0,%1,%2,%3}, [%4];" - : "=l"(v256[0]), "=l"(v256[1]), "=l"(v256[2]), "=l"(v256[3]) - : "l"(ptr)); -``` - -For the NVFP4 GEMV, 128-bit and 256-bit loads are essential because FP4 elements are only 0.5 bytes each. A 256-bit load fetches 64 FP4 values in a single instruction: - -```cuda -// NVFP4 GEMV: each thread loads 64 FP4 values via 256-bit load -// Then unpacks using PTX byte manipulation instead of bitwise ops -__device__ void load_and_unpack_nvfp4_256bit( - const uint8_t* fp4_data, // Packed FP4 data (2 values per byte) - float* unpacked, // Output: 64 FP32 values - int offset) -{ - // 256-bit load: 32 bytes = 64 FP4 values - uint64_t raw[4]; - const uint64_t* ptr = reinterpret_cast(fp4_data + offset); - asm volatile( - "ld.global.v4.u64 {%0,%1,%2,%3}, [%4];" - : "=l"(raw[0]), "=l"(raw[1]), "=l"(raw[2]), "=l"(raw[3]) - : "l"(ptr) - ); - - // Unpack using PTX mov.b32 byte extraction - // This avoids the bitwise shift-and-mask overhead - for (int i = 0; i < 4; i++) { - uint32_t lo = (uint32_t)(raw[i]); - uint32_t hi = (uint32_t)(raw[i] >> 32); - - // PTX byte unpack: extract individual bytes from 32-bit word - uint32_t b0, b1, b2, b3; - asm volatile("mov.b32 {%0,%1,%2,%3}, %4;" - : "=r"(b0), "=r"(b1), "=r"(b2), "=r"(b3) : "r"(lo)); - - // Each byte contains 2 FP4 values — decode low word (8 values) - for (int b = 0; b < 4; b++) { - uint32_t byte_val = (b == 0) ? b0 : (b == 1) ? b1 : (b == 2) ? b2 : b3; - unpacked[i * 16 + b * 2] = decode_fp4(byte_val & 0xF); - unpacked[i * 16 + b * 2 + 1] = decode_fp4((byte_val >> 4) & 0xF); - } - - // Unpack high word (next 8 values from same 64-bit element) - uint32_t hb0, hb1, hb2, hb3; - asm volatile("mov.b32 {%0,%1,%2,%3}, %4;" - : "=r"(hb0), "=r"(hb1), "=r"(hb2), "=r"(hb3) : "r"(hi)); - for (int b = 0; b < 4; b++) { - uint32_t byte_val = (b == 0) ? hb0 : (b == 1) ? hb1 : (b == 2) ? hb2 : hb3; - unpacked[i * 16 + 8 + b * 2] = decode_fp4(byte_val & 0xF); - unpacked[i * 16 + 8 + b * 2 + 1] = decode_fp4((byte_val >> 4) & 0xF); - } - } -} -``` - -## L1 Cache Policy Differentiation - -Different data streams have different reuse patterns. Applying the correct cache policy per stream avoids L1 pollution: - -```cuda -// Cache policy selection based on data reuse pattern -// -// Matrix A (streamed, each row used once): bypass L1 -// Vector B (reused across all rows): keep in L1 - -// L1::no_allocate -- data bypasses L1 cache (streaming access) -// Used for matrix A which is read once per GEMV -asm volatile( - "ld.global.L1::no_allocate.v4.u32 {%0,%1,%2,%3}, [%4];" - : "=r"(a.x), "=r"(a.y), "=r"(a.z), "=r"(a.w) - : "l"(matrix_a_ptr) -); - -// L1::evict_last -- data stays in L1 as long as possible -// Used for vector B which is reused across all M rows -asm volatile( - "ld.global.L1::evict_last.v4.u32 {%0,%1,%2,%3}, [%4];" - : "=r"(b.x), "=r"(b.y), "=r"(b.z), "=r"(b.w) - : "l"(vector_b_ptr) -); -``` - -The impact of cache policies from the GPU Mode Hackathon: - -``` -No cache policy differentiation: 39 us -A: L1::no_allocate, B: L1::evict_last: 27 us (1.44x faster) -``` - -The full set of PTX load cache qualifiers: +PTX ISA 9.0 permits the following global-memory forms on `sm_100`. The first accesses 16 bytes and the second accesses 32 bytes: ```ptx -// PTX load qualifiers for cache control - -// Default: normal L1 and L2 caching -ld.global.b32 %r, [%addr]; - -// L1 bypass: skip L1, still cached in L2 -ld.global.L1::no_allocate.b32 %r, [%addr]; - -// L1 keep: prioritize keeping in L1 (evict last) -ld.global.L1::evict_last.b32 %r, [%addr]; - -// L2 promotion hint (256-byte sector) -// Used by DeepEP for communication overlap -ld.global.nc.L1::no_allocate.L2::256B.b32 %r, [%addr]; - -// Non-coherent read-only (nc): uses texture path -// Avoids coherence traffic, useful for read-only data -ld.global.nc.b32 %r, [%addr]; -``` - -## Register Budgeting (-maxrregcount) - -For memory-bound kernels, occupancy (number of concurrent warps) matters more than per-thread register count. Limiting registers per thread allows more warps to be resident: - -```cuda -// Compile-time register budgeting -// Lower register count -> higher occupancy -> better latency hiding - -// Problem 1 winner (rank 1): -maxrregcount=32 -// This allows 64 warps per SM (100% occupancy on SM100) -// Sufficient for GEMV where each thread does minimal computation +.reg .b64 addr; +.reg .u32 x<4>; +.reg .u64 y<4>; -// Problem 1 rank 3: -maxrregcount=45 -// Allows 44 warps per SM (~69% occupancy) -// More registers per thread for wider vectorized accumulation +ld.global.v4.u32 {x0, x1, x2, x3}, [addr]; +ld.global.v4.u64 {y0, y1, y2, y3}, [addr]; ``` -The tradeoff in a build system: +The address must be naturally aligned to the total access size: 16 bytes for `v4.u32` and 32 bytes for `v4.u64`. A misaligned PTX address has undefined behavior; the ISA says it may have low address bits masked or fault. It does not specify a fallback to narrower loads. Guard tails or dispatch them to an access whose complete range and alignment are valid. -```python -# nvcc compilation with register budgeting -# In CMakeLists.txt or build script: +The 256-bit `v4.b64` family is an SM100-or-newer feature. A 32-byte load contains 64 values when the payload is densely packed E2M1 at two 4-bit values per byte. That arithmetic says nothing about whether the width is profitable: unpack cost, live registers, instruction selection, per-thread address patterns, and tail handling can outweigh a lower source-level load count. -# For memory-bound GEMV kernel: -# nvcc -maxrregcount=32 -arch=sm_100a gemv_kernel.cu -o gemv_kernel +## Cache controls are hints -# For compute-bound GEMM kernel: -# nvcc -arch=sm_100a gemm_kernel.cu -o gemm_kernel (no limit; needs ~128+ regs) +PTX distinguishes cache operators, eviction priorities, prefetch-size hints, and non-coherent read-only loads. Representative legal forms are: -# Per-kernel register limits in the same translation unit: -# Use __launch_bounds__ to control per-kernel - -# Memory-bound: maximize occupancy -__global__ void __launch_bounds__(256, 8) // 256 threads, min 8 blocks/SM -gemv_kernel(/* ... */) { /* ... */ } - -# Compute-bound: maximize register availability -__global__ void __launch_bounds__(512, 1) // 512 threads, min 1 block/SM -gemm_kernel(/* ... */) { /* ... */ } +```ptx +ld.global.L1::no_allocate.v4.u32 {x0, x1, x2, x3}, [addr]; +ld.global.L1::evict_last.v4.u32 {x0, x1, x2, x3}, [addr]; +ld.global.L2::256B.b32 x0, [addr]; +ld.global.nc.b32 x0, [addr]; ``` -## Complete NVFP4 GEMV Example - -Combining all three techniques for the GPU Mode Hackathon Problem 1: - -```cuda -// Optimized NVFP4 Batched GEMV -// A: [M, K] NVFP4, B: [1, K] NVFP4, C: [M, 1] FP16 -// Memory-bound: maximize bandwidth utilization - -// NVFP4 Batched GEMV: each row processed by THREADS_PER_ROW threads -// Memory-bound: maximize bandwidth with wide loads and cache policies +| Form | Documented meaning | Limit on inference | +|---|---|---| +| `L1::no_allocate` | L1 eviction-priority selection that may be applied | Does not guarantee an L1 bypass or a speedup for a streaming operand | +| `L1::evict_last` | Requests the corresponding L1 eviction priority | Does not guarantee that a line remains resident | +| `L2::256B` | Hints that additional data of the stated size be prefetched into L2 | Is not a 256-byte load or a promotion guarantee | +| `ld.global.nc` | Loads through a non-coherent read-only cache | Has architecture- and parallelism-dependent latency/throughput; it is not a general coherence optimization | -#define BLOCK_M 4 // Rows per thread block -#define THREADS 256 -#define THREADS_PER_ROW (THREADS / BLOCK_M) // 64 threads per row +Cache-policy operands and prefetch-size qualifiers are performance hints and do not change the program's memory-consistency behavior. A reuse argument can motivate `evict_last`, and a one-pass stream can motivate `no_allocate`, but only a matched comparison determines whether either helps the concrete working set and launch. -__global__ void __launch_bounds__(THREADS, 4) -nvfp4_gemv_optimized( - const uint8_t* __restrict__ A, // [M, K/2] packed FP4 - const uint8_t* __restrict__ B, // [1, K/2] packed FP4 - const fp8_e4m3* __restrict__ sfa, // [M, K/16] block scales - const fp8_e4m3* __restrict__ sfb, // [1, K/16] block scales - half* __restrict__ C, // [M, 1] output - float global_scale_a, float global_scale_b, - int M, int K) -{ - // Map threads to rows: 64 threads per row, 4 rows per block - int local_row = threadIdx.x / THREADS_PER_ROW; // 0..3 - int thread_in_row = threadIdx.x % THREADS_PER_ROW; // 0..63 - int row = blockIdx.x * BLOCK_M + local_row; - if (row >= M) return; +## Reproducible selection procedure - float acc = 0.0f; +For each candidate width and cache policy: - // Each of the 64 threads handles a distinct K-chunk (no duplication) - // 64 elements per load × 64 threads = 4096 K elements per iteration - for (int k = thread_in_row * 64; k < K; k += THREADS_PER_ROW * 64) { - // Load B (reused across rows): L1::evict_last, 256-bit - uint64_t b_raw[4]; - asm volatile( - "ld.global.L1::evict_last.v4.u64 {%0,%1,%2,%3}, [%4];" - : "=l"(b_raw[0]), "=l"(b_raw[1]), "=l"(b_raw[2]), "=l"(b_raw[3]) - : "l"((const uint64_t*)(B + k / 2))); +1. Prove base and per-thread address alignment for every executed access. Use a separate scalar or narrower-vector path for tails; validate minimum, maximum, and awkward sizes. +2. Hold the algorithm, mapping, decode, unrolling, launch shape, compiler, and inputs fixed while changing one load or cache choice. Compare identical correctness oracles before timing. +3. Inspect generated PTX and SASS. Record actual load instructions, registers per thread, spill loads/stores, stack bytes, shared memory, and launch parameters. A source cast or inline-PTX block does not bypass compiler register allocation. +4. Profile the matched variants. Check achieved bandwidth, requested versus transferred bytes, cache hit behavior, instruction issue/stalls, and active warps with metrics available on the installed Nsight Compute and GPU versions. +5. Use warmups, synchronization, repeated trials, and the same statistic for every production shape. Keep the change only where the declared metric improves without a correctness or resource regression. - // Load A (streamed once): L1::no_allocate, 256-bit - uint64_t a_raw[4]; - asm volatile( - "ld.global.L1::no_allocate.v4.u64 {%0,%1,%2,%3}, [%4];" - : "=l"(a_raw[0]), "=l"(a_raw[1]), "=l"(a_raw[2]), "=l"(a_raw[3]) - : "l"((const uint64_t*)(A + row * (K / 2) + k / 2))); - - // Unpack FP4, apply block scale, dot-product - for (int i = 0; i < 64; i++) { - float a_val = unpack_fp4(a_raw, i) * get_block_scale(sfa, row, k + i); - float b_val = unpack_fp4(b_raw, i) * get_block_scale(sfb, 0, k + i); - acc += a_val * b_val; - } - } - - // Two-phase reduction: first within each warp, then across the 2 warps per row - // Phase 1: warp-level reduction (32 threads → 1 partial sum per warp) - for (int offset = 16; offset > 0; offset >>= 1) { - acc += __shfl_xor_sync(0xFFFFFFFF, acc, offset); - } - - // Phase 2: shared memory reduction across the 2 warps assigned to this row - __shared__ float smem_reduce[BLOCK_M * 2]; // 2 warp partials per row - int warp_in_row = thread_in_row / 32; // 0 or 1 - int lane = thread_in_row % 32; - if (lane == 0) { - smem_reduce[local_row * 2 + warp_in_row] = acc; - } - __syncthreads(); - - // Final sum and store (one thread per row) - if (thread_in_row == 0) { - float result = smem_reduce[local_row * 2] + smem_reduce[local_row * 2 + 1]; - C[row] = __float2half(result * global_scale_a * global_scale_b); - } -} -``` +Register caps and launch bounds are a separate variable. A nominal cap may be inert when natural allocation is lower, or it may trade registers for spills and extra instructions. Follow the resource and occupancy workflow in [Register Budgeting](register-budgeting.md) rather than inferring residency from the cap value. -## Optimization Progression (GPU Mode Hackathon Problem 1) +## NVFP4 GEMV case study -| Step | Technique | Latency | Speedup | -|------|-----------|---------|---------| -| Baseline | Naive C++ | 2000 us | 1.0x | -| Coalesced access | Memory layout fix | 443 us | 4.5x | -| Hardware intrinsics | FP4 decode | 39 us | 51x | -| PTX assembly | Vectorized loads + cache policy | 27 us | 74x | -| ILP + register tuning | Unrolling + maxrregcount | 22.4 us | 89x | -| Speed of light | | ~8.6 us | 233x | +Yue Zhang reports the following CUDA progression for GPU Mode Problem 1. These are author-reported endpoints without raw repeated-trial data or released complete submission code: -## When to Use +| Stage | Combined change | Reported latency | +|---|---|---:| +| Initial CUDA | Naive hand-written path | about 2000 µs | +| CUDA optimization 1 | Coalescing, shared B, thread collaboration, warp reduction | about 443 µs | +| CUDA optimization 2 | Remove shared B, per-thread tiles, `float4` loads, hardware intrinsics | about 39 µs | +| CUDA optimization 3 | Vectorized PTX FP4 and scale decode | about 27 µs | +| Parameter tuning | Threads per row and rows per block | about 26 µs | +| ILP | Two tiles per loop iteration | about 22.9 µs | +| Aggressive PTX fusion | Decode, scales, multiply, and accumulation in a larger PTX block | about 22.3 µs | -- **GEMV and memory-bound kernels**: Vectorized loads and cache policies are essential. These kernels are entirely limited by memory bandwidth. -- **FP4/FP8 kernels**: Sub-byte data types make wide loads even more impactful since more elements fit in a single wide load. -- **Decode-phase inference**: Single-token GEMV during autoregressive decoding is always memory-bound. +The submitted public-leaderboard score was 22.392 microseconds, the geometric mean over three benchmark shapes. The pinned task's theoretical model gives 8.622, 17.275, and 4.317 microseconds for those separate shapes; 8.622 microseconds is therefore not an aggregate “speed of light.” The 39-to-27 endpoints change the PTX load/decode path together and do not isolate cache policy, load width, or conversion choice. -## Caveats +Amandeep Singh's attempts supply useful negative controls for the same task. Replacing two `uchar4` loads with one `uint2` load was reported 16–25% slower because extraction added instructions. Lowering `-maxrregcount` from 80 to 64 had no effect because natural allocation was already below the cap. The author later observed wider PTX forms and differentiated L1 policies in other solutions, but published no contestant code or controlled ablation for those observations. -- 256-bit loads require 32-byte aligned addresses. Misaligned access falls back to multiple narrower transactions. -- `L1::no_allocate` is harmful for data that will be reused. Only apply it to truly streaming access patterns. -- `-maxrregcount` that is too low causes register spilling to local memory, which is slower than the occupancy gain. Profile with Nsight Compute to find the optimal point. -- PTX inline assembly bypasses the compiler's register allocator. Excessive inline PTX can interfere with compiler optimizations for surrounding code. +Together, the reports justify testing vector width, decode organization, cache hints, and register limits. They do not establish that the widest load, a particular cache hint, or the lowest register cap is essential for GEMV, sub-byte arithmetic, or decode workloads. diff --git a/wiki/techniques/warp-specialization.md b/wiki/techniques/warp-specialization.md index a7d61d484..c008fb813 100644 --- a/wiki/techniques/warp-specialization.md +++ b/wiki/techniques/warp-specialization.md @@ -2,261 +2,81 @@ id: technique-warp-specialization title: Warp Specialization on Blackwell type: technique -architectures: -- sm100 -- sm90 -tags: -- warp-specialization -- tcgen05 -- tmem +architectures: [sm100, sm90] +tags: [warp-specialization, tcgen05, tmem] confidence: source-reported -reproducibility: snippet -prerequisites: -- hw-tmem -- hw-tcgen05-mma -related: -- technique-persistent-kernels -- technique-pipeline-stages -- hw-tcgen05-mma -sources: -- doc-nvidia-tuning-guide -- blog-tcgen05-tutorial -- blog-colfax-cutlass -blackwell_relevance: Blackwell uses 16-warp single-thread MMA model (vs Hopper's 4-warp - warp-group); fundamentally different structure. +reproducibility: concept +prerequisites: [hw-tmem, hw-tcgen05-mma] +related: [technique-persistent-kernels, technique-pipeline-stages, hw-tcgen05-mma] +sources: [doc-ptx-isa-sm100, doc-cutlass-blackwell, blog-tcgen05-tutorial, pr-flashinfer-1039] +blackwell_relevance: "Single-thread tcgen05 issue and TMEM accumulators enable new role partitions, but warp counts and role IDs are software-configuration choices." artifact_dir: artifacts/kernels/warp-specialization --- -## Overview +## Definition and ISA Boundary -Warp specialization assigns distinct functional roles to warps within a CTA, allowing each warp to focus on a single pipeline stage (data loading, MMA computation, or epilogue writeback). On Blackwell (SM100), the 16-warp CTA structure replaces Hopper's 4-warp warpgroup model. Because tcgen05.mma is a single-thread instruction that operates on TMEM rather than registers, only one warp needs to issue MMA operations, freeing the remaining warps for producer and consumer roles. +Warp specialization assigns different long-lived functions to disjoint warps in a CTA—for example scheduler, TMA load, MMA control, softmax/correction, or epilogue work. It is a software organization for overlapping pipeline stages, not a fixed SM100 CTA layout. -## Blackwell 16-Warp Kernel Structure +The architecture-level comparison is narrower: -The canonical Blackwell GEMM kernel uses 16 warps (512 threads) per CTA with the following role assignment: +| Property | Hopper `wgmma.mma_async` | Blackwell `tcgen05.mma` | +|---|---|---| +| Issue granularity | Warpgroup collective | One thread for `cta_group::1` or `cta_group::2` | +| D accumulator | Per-thread registers | TMEM | +| A/B movement | Explicit register/SMEM operands and producer work | Explicit SMEM descriptors or TMEM addresses and producer work | +| Completion | WGMMA commit/wait groups | `tcgen05.commit` tied to an mbarrier, followed by a wait | -| Warp ID | Role | Responsibility | -|---------|------|----------------| -| 0 | TMA Producer | Issues TMA bulk-copy from global to shared memory, signals mbarrier | -| 1 | MMA Consumer | Issues tcgen05.mma on SMEM operands, writes results to TMEM | -| 2-15 | Epilogue | Reads TMEM accumulator, applies scale/bias/activation, writes to global memory | +Single-thread issue reduces the number of threads needed to submit MMA instructions. It does not make operands move automatically, turn an entire warp into an ISA-level role, complete MMA synchronously, or choose how many epilogue warps a kernel should launch. -This contrasts with Hopper where a warpgroup (4 warps, 128 threads) collectively issues wgmma.mma_async, and all threads in the warpgroup participate in the MMA. On Blackwell, the MMA warp dispatches the instruction from a single thread while the hardware handles the data movement internally. +## Version-Pinned Role Maps -## Comparison with Hopper Warpgroup Model +There is no universal “warp 0 load, warp 1 MMA, warps 2–15 epilogue” rule. Three primary implementations illustrate the range: -| Aspect | Hopper (SM90) | Blackwell (SM100) | -|--------|---------------|-------------------| -| MMA granularity | 4-warp warpgroup (128 threads) | Single thread in 1 warp | -| MMA output destination | Registers (shared across warpgroup) | TMEM (256KB, CTA-visible) | -| Producer warps | Separate warp(s) for TMA loads | Warp 0 dedicated to TMA | -| Epilogue execution | Same warpgroup or separate warps | 14 dedicated warps (2-15) | -| Synchronization | warpgroup barriers, arrive/wait | mbarrier pairs (producer/consumer) | -| Register pressure | High (accumulators in registers) | Low (accumulators in TMEM) | +| Implementation | Total warps and roles | +|---|---| +| Gau Nernst tutorial v4, commit `3b90ac9b...` | 4 warps. An elected lane in warp 0 runs the TMA loop; an elected lane in warp 1 runs the MMA loop; after completion, all four warps participate in TMEM load/conversion/output. | +| CUTLASS 4.5.0 generic SM100 GEMM | One warp each for `MMA`, `Sched`, `MainloopLoad`, and `EpilogueLoad`, followed by `CollectiveEpilogue::ThreadCount / 32` epilogue warps. Optional work can make some control roles nonparticipants. | +| FlashInfer PR 1039 context FMHA | 16 warps: 0–3 `Softmax0`, 4–7 `Softmax1`, 8–11 correction, 12 MMA, 13 load, 14 epilogue, and 15 empty. The kernel has explicit pipelines between these roles. | -## Warp Role Assignment +FlashAttention-4's pinned SM100 forward source likewise starts from a 16-warp layout with two four-warp softmax groups, four correction warps, and dedicated MMA/load/epilogue IDs, then adjusts roles for configuration choices such as one Q stage, non-TMA paths, and dynamic persistence. These are concrete attention schedules, not a GEMM-wide architectural template. -The kernel entry point assigns each warp its role based on `threadIdx.x`: +## Correct Synchronization Obligations -```cuda -// Blackwell 16-warp specialized GEMM kernel skeleton -// 16 warps = 512 threads per CTA -__global__ void __launch_bounds__(512) -blackwell_gemm_warp_specialized( - const __grid_constant__ GemmParams params) -{ - const int warp_id = threadIdx.x / 32; - const int lane_id = threadIdx.x % 32; +A warp-specialized implementation must prove each ownership handoff. For a reusable TMA-to-MMA stage: - // Shared memory for A/B tiles and mbarrier objects - extern __shared__ char smem[]; - half* smem_A = reinterpret_cast(smem); - half* smem_B = reinterpret_cast(smem + SMEM_A_SIZE); +1. Initialize the mbarrier objects with correct participant/transaction counts and publish them to the required threads and async proxies before use. +2. The producer acquires an empty stage, sets expected transaction bytes, and submits the TMA copies. The full/data-ready phase completes only after the expected async transactions complete. +3. The MMA controller waits for the full phase and applies the required cross-thread/proxy fence before issuing `tcgen05.mma` against that SMEM stage. +4. Because MMA is asynchronous, `__syncwarp()` or an ordinary `mbarrier.arrive` after issue does not make the operands reusable. Commit prior tcgen05 work to an mbarrier and wait for completion before releasing or overwriting the stage. +5. Before another role reads D from TMEM, complete the MMA sequence and apply the required `tcgen05.fence::before_thread_sync` / execution-ordering handoff / `tcgen05.fence::after_thread_sync` protocol. A fence orders operations; it is not a completion wait. +6. Keep TMEM allocated until every consumer finishes its `tcgen05.ld` sequence and associated waits, then deallocate with the required collective participation. - // mbarrier pairs: TMA hardware signals "data ready", MMA signals "buffer free" - __shared__ uint64_t mbar_data_ready[NUM_STAGES]; - __shared__ uint64_t mbar_buffer_free[NUM_STAGES]; - // MMA→epilogue handoff barrier - __shared__ uint64_t mbar_acc_complete; - // Phase tracking: mbarriers alternate parity on each reuse cycle - int phase_data[NUM_STAGES]; - int phase_free[NUM_STAGES]; +Pipeline wrappers in CUTLASS/CuTe encode parts of these rules, but participant counts, initial phases, tails, and producer/consumer states remain configuration-specific. A short role-dispatch sketch is not a substitute for the complete pipeline and TMEM lifecycle. - if (warp_id == 0) { - if (lane_id == 0) { - for (int s = 0; s < NUM_STAGES; s++) { - // TMA expects arrive.expect_tx → hardware completes - mbarrier_init(&mbar_data_ready[s], 1); - mbarrier_init(&mbar_buffer_free[s], 1); - } - mbarrier_init(&mbar_acc_complete, 1); - } - } - // Initialize phase counters (all start at 0) - for (int s = 0; s < NUM_STAGES; s++) { - phase_data[s] = 0; - phase_free[s] = 0; - } - __syncthreads(); +## Source-Reported Tutorial Result - if (warp_id == 0) { - // === TMA PRODUCER WARP === - for (int k_tile = 0; k_tile < num_k_tiles; k_tile++) { - int stage = k_tile % NUM_STAGES; +For the author's exact `M=N=K=4096` Modal B200 setup with PyTorch 2.9.1 and CUDA 13, tutorial v3 reports 939.61 TFLOP/s and v4 reports 1208.83 TFLOP/s after introducing warp specialization. That is a 269.22-TFLOP/s, approximately 28.65% step between those source variants. It is not a portable Blackwell speedup or evidence that specialization is always profitable. - // Wait for consumer to release this buffer (with phase tracking) - if (k_tile >= NUM_STAGES) { - mbarrier_wait_parity(&mbar_buffer_free[stage], phase_free[stage]); - phase_free[stage] ^= 1; // flip parity for next reuse - } +## When to Test Warp Specialization - // Set expected TX bytes, then issue TMA. TMA hardware will - // signal mbar_data_ready upon transfer completion. - // Do NOT manually arrive — that races with the async transfer. - if (lane_id == 0) { - uint32_t tx_bytes = TILE_A_BYTES + TILE_B_BYTES; - mbarrier_arrive_expect_tx(&mbar_data_ready[stage], tx_bytes); - tma_copy_async(smem_A + stage * TILE_A_SIZE, - ¶ms.A[k_tile * TILE_K], TILE_A_SIZE, - &mbar_data_ready[stage]); - tma_copy_async(smem_B + stage * TILE_B_SIZE, - ¶ms.B[k_tile * TILE_K], TILE_B_SIZE, - &mbar_data_ready[stage]); - // TMA hardware arrives on mbar_data_ready when transfer completes - } - } +Use it as a candidate when independent pipeline roles have enough steady-state work to overlap and their register needs differ materially. Compare it with a temporally pipelined version at the same tile shape, stage count, CTA-group mode, and scheduler. Measure: - } else if (warp_id == 1) { - // === MMA CONSUMER WARP === - for (int k_tile = 0; k_tile < num_k_tiles; k_tile++) { - int stage = k_tile % NUM_STAGES; +- time and achieved throughput across representative shapes; +- per-role active/stall time and pipeline backpressure; +- register allocation, spills, shared memory, and occupancy; +- TMA/MMA/epilogue balance, including prologue and tail cost; +- 1-SM versus 2-SM mode as an independent legal-shape/resource choice. - // Wait for TMA to complete this stage (with phase tracking) - mbarrier_wait_parity(&mbar_data_ready[stage], phase_data[stage]); - phase_data[stage] ^= 1; +More role warps are not automatically useful for a more complex epilogue, and one MMA-control warp does not limit the issuer to one outstanding asynchronous MMA operation. Choose participant counts from the concrete collective and measurements. - // Critical fence: ensure TMA data visible before MMA reads SMEM - tcgen05_fence_after_thread_sync(); +## Local Evidence Bundle - if (lane_id == 0) { - tcgen05_mma(smem_A + stage * TILE_A_SIZE, - smem_B + stage * TILE_B_SIZE); - } - __syncwarp(); +[`artifacts/kernels/warp-specialization/full/`](../../artifacts/kernels/warp-specialization/full/) contains the captured FlashInfer PR 1039 mainloop file; its bytes match the SHA-256 recorded in `PROVENANCE.yaml` and the duplicate captured PR key-file. The provenance declares upstream revision `9a05c92a`. That abbreviated historical object was not independently refetchable during this audit, so the bundle is locally hash-verified captured evidence, not a fresh upstream retrieval. - // Signal buffer is free for reuse - if (lane_id == 0) { - mbarrier_arrive(&mbar_buffer_free[stage]); - } - } +[`artifacts/kernels/warp-specialization/variants/`](../../artifacts/kernels/warp-specialization/variants/) is explicitly `derived` teaching material and must not be cited as upstream code or as a complete safe kernel. - // Signal epilogue warps that accumulation is complete - if (lane_id == 0) { - mbarrier_arrive(&mbar_acc_complete); - } - - } else { - // === EPILOGUE WARPS (2-15) === - // Wait for MMA completion via dedicated mbarrier (not __syncthreads, - // which would deadlock since producer/MMA warps don't reach it) - mbarrier_wait(&mbar_acc_complete); - - // Each epilogue warp handles a partition of the TMEM output. - // Use ceiling division to cover tail rows when TILE_M % 14 != 0. - constexpr int NUM_EPI_WARPS = 14; // warps 2-15 - int epi_warp = warp_id - 2; // 0..13 - int rows_per_warp = (TILE_M + NUM_EPI_WARPS - 1) / NUM_EPI_WARPS; - int my_row_start = epi_warp * rows_per_warp; - int my_row_end = min(my_row_start + rows_per_warp, TILE_M); - - for (int r = my_row_start; r < my_row_end; r++) { - for (int c = lane_id; c < TILE_N; c += 32) { - // Read accumulator from TMEM - float acc = tmem_load(r, c); - // Apply epilogue: scale + bias + activation - float result = epilogue_op(acc, params.scale, params.bias[c]); - // Write to global memory - params.C[r * params.N + c] = __float2half(result); - } - } - } -} -``` - -## mbarrier Synchronization Pattern - -The producer-consumer synchronization uses mbarrier pairs. Each pipeline stage has two barriers: - -1. **data_ready**: Producer (Warp 0) arrives after TMA completes. Consumer (Warp 1) waits before issuing MMA. -2. **buffer_free**: Consumer (Warp 1) arrives after MMA consumes the data. Producer (Warp 0) waits before overwriting the buffer. - -At the PTX level, the mbarrier operations map to: - -```ptx -// Producer: signal data is ready in stage %stage -mbarrier.arrive.shared.b64 %dummy, [%mbar_data_ready + %stage_offset]; - -// Consumer: wait for data to be ready -mbarrier.try_wait.parity.shared.b64 %pred, [%mbar_data_ready + %stage_offset], %phase; - -// Consumer: signal buffer is consumed -mbarrier.arrive.shared.b64 %dummy, [%mbar_buffer_free + %stage_offset]; - -// Producer: wait for buffer to be free -mbarrier.try_wait.parity.shared.b64 %pred, [%mbar_buffer_free + %stage_offset], %phase; -``` - -## CUTLASS SM100 Warp Specialization - -In CUTLASS 4.5.0, the SM100 GEMM collective (`CollectiveMma_1SM`) implements this pattern with CuTe abstractions: - -```cuda -// CUTLASS SM100 warp role dispatch (simplified from CollectiveMma) -// Template parameter WarpCount = cute::Shape<1, 1, 14> -// Warp 0 = producer, Warp 1 = math, Warps 2-15 = epilogue - -template -struct CollectiveMma_1SM { - static constexpr int NumProducerWarps = 1; - static constexpr int NumMathWarps = 1; - static constexpr int NumEpilogueWarps = 14; - - CUTLASS_DEVICE void operator()( - Params const& params, - char* smem_buf, - TiledMma& tiled_mma) - { - int warp_idx = cutlass::canonical_warp_idx_sync(); - - if (warp_idx == 0) { - producer_warp(params, smem_buf); - } else if (warp_idx == 1) { - math_warp(params, smem_buf, tiled_mma); - } else { - epilogue_warp(params, smem_buf); - } - } -}; -``` - -## When to Use - -- **Always on Blackwell GEMMs**: Warp specialization is the standard pattern for SM100 tensor core kernels. The tcgen05 instruction model assumes single-thread dispatch with TMEM output. -- **Attention kernels**: FlashAttention-4 extends this to ping-pong scheduling with 2 query tile groups and dedicated softmax warps. -- **Any kernel with producer-consumer pipeline**: When TMA loads and MMA compute can overlap, warp specialization provides the cleanest decomposition. - -## Caveats - -- The 14 epilogue warps may be underutilized for simple epilogues (e.g., pure store). Complex epilogues (scale, bias, activation, quantization) benefit more. -- The single MMA warp means the kernel cannot overlap multiple independent MMA streams within a CTA. Use 2-SM cooperative mode for larger tiles instead. -- mbarrier initialization must happen before any warp tries to wait; use `__syncthreads()` after init if needed. - -## Full Reference Implementation - -Local verbatim upstream code lives in [`artifacts/kernels/warp-specialization/full/`](../../artifacts/kernels/warp-specialization/full/) (see its `PROVENANCE.yaml` for the pinned upstream SHA and byte-verified SHA-256). Labeled derived variants — including a naive/teaching skeleton — live in [`artifacts/kernels/warp-specialization/variants/`](../../artifacts/kernels/warp-specialization/variants/). - -Query via: +Retrieve the page plus both artifact modes with: ```bash -python3 scripts/get_page.py technique-warp-specialization --include-code +conda run -n base python scripts/get_page.py technique-warp-specialization --include-code ``` From c33840e2163ce8750b3b0e16a8a9c3a75e54cbf4 Mon Sep 17 00:00:00 2001 From: Waterpine Date: Sun, 9 Aug 2026 22:24:59 -0500 Subject: [PATCH 2/3] add verification --- verification/baseline-metadata.json | 39 ++ verification/baseline-queue.jsonl | 52 ++ verification/before-after-report.jsonl | 493 ++++++++++++++ verification/current-queue.jsonl | 52 ++ verification/original-report.jsonl | 52 ++ verification/remediation-log.jsonl | 554 ++++++++++++++++ .../reviewed/current/hw-2sm-cooperative.json | 137 ++++ verification/reviewed/current/hw-clc.json | 388 +++++++++++ .../reviewed/current/hw-mbarrier.json | 188 ++++++ verification/reviewed/current/hw-nvfp4.json | 179 +++++ verification/reviewed/current/hw-pdl-gdc.json | 186 ++++++ .../reviewed/current/hw-tcgen05-mma.json | 219 +++++++ verification/reviewed/current/hw-tma.json | 127 ++++ verification/reviewed/current/hw-tmem.json | 253 ++++++++ .../reviewed/current/kernel-deepgemm.json | 110 ++++ .../current/kernel-flash-attention-4.json | 100 +++ ...kernel-flash-attention-sm100-mla-topk.json | 60 ++ .../reviewed/current/kernel-flashmla.json | 110 ++++ .../current/kernel-fp8-block-scale-gemm.json | 90 +++ .../reviewed/current/kernel-fused-moe.json | 156 +++++ .../current/kernel-gated-delta-net.json | 146 +++++ .../current/kernel-gated-dual-gemm.json | 105 +++ .../reviewed/current/kernel-grouped-gemm.json | 106 +++ verification/reviewed/current/kernel-nsa.json | 107 +++ .../reviewed/current/kernel-nvfp4-gemm.json | 150 +++++ .../reviewed/current/kernel-nvfp4-gemv.json | 104 +++ .../reviewed/current/kernel-sparse-mla.json | 15 + ...kernel-tensorrt-llm-blackwell-indexer.json | 186 ++++++ .../reviewed/current/lang-cuda-cpp.json | 90 +++ .../reviewed/current/lang-cute-dsl.json | 110 ++++ verification/reviewed/current/lang-ptx.json | 100 +++ .../reviewed/current/lang-triton.json | 314 +++++++++ .../current/migration-register-to-tmem.json | 130 ++++ .../current/migration-wgmma-to-tcgen05.json | 177 +++++ .../current/pattern-compute-bound.json | 13 + .../current/pattern-low-sm-utilization.json | 14 + .../current/pattern-memory-bound.json | 13 + .../current/pattern-moe-load-imbalance.json | 14 + .../current/pattern-pipeline-stalls.json | 60 ++ .../current/pattern-register-pressure.json | 13 + .../reviewed/current/pattern-tail-effect.json | 13 + .../current/technique-cache-policy.json | 110 ++++ .../technique-cccl-memory-primitives.json | 40 ++ .../current/technique-chunk-parallelism.json | 132 ++++ .../current/technique-double-buffering.json | 100 +++ .../current/technique-epilogue-fusion.json | 90 +++ ...echnique-external-source-map-research.json | 209 ++++++ .../technique-fine-grained-quantization.json | 109 ++++ .../current/technique-kernel-fusion.json | 99 +++ .../current/technique-persistent-kernels.json | 100 +++ .../technique-ping-pong-scheduling.json | 143 ++++ .../current/technique-pipeline-stages.json | 110 ++++ .../current/technique-register-budgeting.json | 176 +++++ .../current/technique-software-exp.json | 159 +++++ .../reviewed/current/technique-swizzling.json | 165 +++++ .../current/technique-tile-scheduling.json | 83 +++ .../current/technique-vectorized-loads.json | 131 ++++ .../technique-warp-specialization.json | 74 +++ .../reviewed/original/hw-2sm-cooperative.json | 154 +++++ verification/reviewed/original/hw-clc.json | 611 ++++++++++++++++++ .../reviewed/original/hw-mbarrier.json | 196 ++++++ verification/reviewed/original/hw-nvfp4.json | 178 +++++ .../reviewed/original/hw-pdl-gdc.json | 131 ++++ .../reviewed/original/hw-tcgen05-mma.json | 402 ++++++++++++ verification/reviewed/original/hw-tma.json | 245 +++++++ verification/reviewed/original/hw-tmem.json | 410 ++++++++++++ .../reviewed/original/kernel-deepgemm.json | 171 +++++ .../original/kernel-flash-attention-4.json | 206 ++++++ ...kernel-flash-attention-sm100-mla-topk.json | 68 ++ .../reviewed/original/kernel-flashmla.json | 247 +++++++ .../original/kernel-fp8-block-scale-gemm.json | 123 ++++ .../reviewed/original/kernel-fused-moe.json | 295 +++++++++ .../original/kernel-gated-delta-net.json | 260 ++++++++ .../original/kernel-gated-dual-gemm.json | 117 ++++ .../original/kernel-grouped-gemm.json | 213 ++++++ .../reviewed/original/kernel-nsa.json | 176 +++++ .../reviewed/original/kernel-nvfp4-gemm.json | 284 ++++++++ .../reviewed/original/kernel-nvfp4-gemv.json | 329 ++++++++++ .../reviewed/original/kernel-sparse-mla.json | 25 + ...kernel-tensorrt-llm-blackwell-indexer.json | 187 ++++++ .../reviewed/original/lang-cuda-cpp.json | 87 +++ .../reviewed/original/lang-cute-dsl.json | 173 +++++ verification/reviewed/original/lang-ptx.json | 115 ++++ .../reviewed/original/lang-triton.json | 266 ++++++++ .../original/migration-register-to-tmem.json | 263 ++++++++ .../original/migration-wgmma-to-tcgen05.json | 369 +++++++++++ .../original/pattern-compute-bound.json | 20 + .../original/pattern-low-sm-utilization.json | 17 + .../original/pattern-memory-bound.json | 21 + .../original/pattern-moe-load-imbalance.json | 27 + .../original/pattern-pipeline-stalls.json | 130 ++++ .../original/pattern-register-pressure.json | 17 + .../original/pattern-tail-effect.json | 18 + .../original/technique-cache-policy.json | 106 +++ .../technique-cccl-memory-primitives.json | 58 ++ .../original/technique-chunk-parallelism.json | 124 ++++ .../original/technique-double-buffering.json | 188 ++++++ .../original/technique-epilogue-fusion.json | 179 +++++ ...echnique-external-source-map-research.json | 193 ++++++ .../technique-fine-grained-quantization.json | 173 +++++ .../original/technique-kernel-fusion.json | 75 +++ .../technique-persistent-kernels.json | 142 ++++ .../technique-ping-pong-scheduling.json | 150 +++++ .../original/technique-pipeline-stages.json | 162 +++++ .../technique-register-budgeting.json | 157 +++++ .../original/technique-software-exp.json | 236 +++++++ .../original/technique-swizzling.json | 228 +++++++ .../original/technique-tile-scheduling.json | 152 +++++ .../original/technique-vectorized-loads.json | 236 +++++++ .../technique-warp-specialization.json | 163 +++++ .../tools/build_before_after_report.py | 329 ++++++++++ verification/tools/replace_report_entry.py | 45 ++ verification/verification-report.jsonl | 52 ++ 113 files changed, 17254 insertions(+) create mode 100644 verification/baseline-metadata.json create mode 100644 verification/baseline-queue.jsonl create mode 100644 verification/before-after-report.jsonl create mode 100644 verification/current-queue.jsonl create mode 100644 verification/original-report.jsonl create mode 100644 verification/remediation-log.jsonl create mode 100644 verification/reviewed/current/hw-2sm-cooperative.json create mode 100644 verification/reviewed/current/hw-clc.json create mode 100644 verification/reviewed/current/hw-mbarrier.json create mode 100644 verification/reviewed/current/hw-nvfp4.json create mode 100644 verification/reviewed/current/hw-pdl-gdc.json create mode 100644 verification/reviewed/current/hw-tcgen05-mma.json create mode 100644 verification/reviewed/current/hw-tma.json create mode 100644 verification/reviewed/current/hw-tmem.json create mode 100644 verification/reviewed/current/kernel-deepgemm.json create mode 100644 verification/reviewed/current/kernel-flash-attention-4.json create mode 100644 verification/reviewed/current/kernel-flash-attention-sm100-mla-topk.json create mode 100644 verification/reviewed/current/kernel-flashmla.json create mode 100644 verification/reviewed/current/kernel-fp8-block-scale-gemm.json create mode 100644 verification/reviewed/current/kernel-fused-moe.json create mode 100644 verification/reviewed/current/kernel-gated-delta-net.json create mode 100644 verification/reviewed/current/kernel-gated-dual-gemm.json create mode 100644 verification/reviewed/current/kernel-grouped-gemm.json create mode 100644 verification/reviewed/current/kernel-nsa.json create mode 100644 verification/reviewed/current/kernel-nvfp4-gemm.json create mode 100644 verification/reviewed/current/kernel-nvfp4-gemv.json create mode 100644 verification/reviewed/current/kernel-sparse-mla.json create mode 100644 verification/reviewed/current/kernel-tensorrt-llm-blackwell-indexer.json create mode 100644 verification/reviewed/current/lang-cuda-cpp.json create mode 100644 verification/reviewed/current/lang-cute-dsl.json create mode 100644 verification/reviewed/current/lang-ptx.json create mode 100644 verification/reviewed/current/lang-triton.json create mode 100644 verification/reviewed/current/migration-register-to-tmem.json create mode 100644 verification/reviewed/current/migration-wgmma-to-tcgen05.json create mode 100644 verification/reviewed/current/pattern-compute-bound.json create mode 100644 verification/reviewed/current/pattern-low-sm-utilization.json create mode 100644 verification/reviewed/current/pattern-memory-bound.json create mode 100644 verification/reviewed/current/pattern-moe-load-imbalance.json create mode 100644 verification/reviewed/current/pattern-pipeline-stalls.json create mode 100644 verification/reviewed/current/pattern-register-pressure.json create mode 100644 verification/reviewed/current/pattern-tail-effect.json create mode 100644 verification/reviewed/current/technique-cache-policy.json create mode 100644 verification/reviewed/current/technique-cccl-memory-primitives.json create mode 100644 verification/reviewed/current/technique-chunk-parallelism.json create mode 100644 verification/reviewed/current/technique-double-buffering.json create mode 100644 verification/reviewed/current/technique-epilogue-fusion.json create mode 100644 verification/reviewed/current/technique-external-source-map-research.json create mode 100644 verification/reviewed/current/technique-fine-grained-quantization.json create mode 100644 verification/reviewed/current/technique-kernel-fusion.json create mode 100644 verification/reviewed/current/technique-persistent-kernels.json create mode 100644 verification/reviewed/current/technique-ping-pong-scheduling.json create mode 100644 verification/reviewed/current/technique-pipeline-stages.json create mode 100644 verification/reviewed/current/technique-register-budgeting.json create mode 100644 verification/reviewed/current/technique-software-exp.json create mode 100644 verification/reviewed/current/technique-swizzling.json create mode 100644 verification/reviewed/current/technique-tile-scheduling.json create mode 100644 verification/reviewed/current/technique-vectorized-loads.json create mode 100644 verification/reviewed/current/technique-warp-specialization.json create mode 100644 verification/reviewed/original/hw-2sm-cooperative.json create mode 100644 verification/reviewed/original/hw-clc.json create mode 100644 verification/reviewed/original/hw-mbarrier.json create mode 100644 verification/reviewed/original/hw-nvfp4.json create mode 100644 verification/reviewed/original/hw-pdl-gdc.json create mode 100644 verification/reviewed/original/hw-tcgen05-mma.json create mode 100644 verification/reviewed/original/hw-tma.json create mode 100644 verification/reviewed/original/hw-tmem.json create mode 100644 verification/reviewed/original/kernel-deepgemm.json create mode 100644 verification/reviewed/original/kernel-flash-attention-4.json create mode 100644 verification/reviewed/original/kernel-flash-attention-sm100-mla-topk.json create mode 100644 verification/reviewed/original/kernel-flashmla.json create mode 100644 verification/reviewed/original/kernel-fp8-block-scale-gemm.json create mode 100644 verification/reviewed/original/kernel-fused-moe.json create mode 100644 verification/reviewed/original/kernel-gated-delta-net.json create mode 100644 verification/reviewed/original/kernel-gated-dual-gemm.json create mode 100644 verification/reviewed/original/kernel-grouped-gemm.json create mode 100644 verification/reviewed/original/kernel-nsa.json create mode 100644 verification/reviewed/original/kernel-nvfp4-gemm.json create mode 100644 verification/reviewed/original/kernel-nvfp4-gemv.json create mode 100644 verification/reviewed/original/kernel-sparse-mla.json create mode 100644 verification/reviewed/original/kernel-tensorrt-llm-blackwell-indexer.json create mode 100644 verification/reviewed/original/lang-cuda-cpp.json create mode 100644 verification/reviewed/original/lang-cute-dsl.json create mode 100644 verification/reviewed/original/lang-ptx.json create mode 100644 verification/reviewed/original/lang-triton.json create mode 100644 verification/reviewed/original/migration-register-to-tmem.json create mode 100644 verification/reviewed/original/migration-wgmma-to-tcgen05.json create mode 100644 verification/reviewed/original/pattern-compute-bound.json create mode 100644 verification/reviewed/original/pattern-low-sm-utilization.json create mode 100644 verification/reviewed/original/pattern-memory-bound.json create mode 100644 verification/reviewed/original/pattern-moe-load-imbalance.json create mode 100644 verification/reviewed/original/pattern-pipeline-stalls.json create mode 100644 verification/reviewed/original/pattern-register-pressure.json create mode 100644 verification/reviewed/original/pattern-tail-effect.json create mode 100644 verification/reviewed/original/technique-cache-policy.json create mode 100644 verification/reviewed/original/technique-cccl-memory-primitives.json create mode 100644 verification/reviewed/original/technique-chunk-parallelism.json create mode 100644 verification/reviewed/original/technique-double-buffering.json create mode 100644 verification/reviewed/original/technique-epilogue-fusion.json create mode 100644 verification/reviewed/original/technique-external-source-map-research.json create mode 100644 verification/reviewed/original/technique-fine-grained-quantization.json create mode 100644 verification/reviewed/original/technique-kernel-fusion.json create mode 100644 verification/reviewed/original/technique-persistent-kernels.json create mode 100644 verification/reviewed/original/technique-ping-pong-scheduling.json create mode 100644 verification/reviewed/original/technique-pipeline-stages.json create mode 100644 verification/reviewed/original/technique-register-budgeting.json create mode 100644 verification/reviewed/original/technique-software-exp.json create mode 100644 verification/reviewed/original/technique-swizzling.json create mode 100644 verification/reviewed/original/technique-tile-scheduling.json create mode 100644 verification/reviewed/original/technique-vectorized-loads.json create mode 100644 verification/reviewed/original/technique-warp-specialization.json create mode 100644 verification/tools/build_before_after_report.py create mode 100644 verification/tools/replace_report_entry.py create mode 100644 verification/verification-report.jsonl diff --git a/verification/baseline-metadata.json b/verification/baseline-metadata.json new file mode 100644 index 000000000..6c50678ec --- /dev/null +++ b/verification/baseline-metadata.json @@ -0,0 +1,39 @@ +{ + "recorded_at": "2026-08-08", + "repository_commit": "2777d18ffb3a3d682d8f25a3e3b8864d925a5ff1", + "preexisting_dirty_files": [ + "plan.md" + ], + "python_environment": { + "command_prefix": "conda run -n base python", + "conda_version": "26.5.3", + "python_version": "3.13.14" + }, + "validator": { + "files": 2787, + "asset_bundles": 94, + "candidate_ledgers": 14, + "result": "pass" + }, + "baseline_queue": { + "schema_version": "kernel-wiki-verifier-queue/v2", + "pages": 52, + "coverage_units": 1177, + "page_types": { + "hardware": 8, + "kernel": 14, + "language": 4, + "migration": 2, + "pattern": 7, + "technique": 17 + }, + "risk_pages": { + "code": 51, + "ordering": 24, + "performance": 13, + "table": 36, + "version": 1 + }, + "unresolved_source_ids": 0 + } +} diff --git a/verification/baseline-queue.jsonl b/verification/baseline-queue.jsonl new file mode 100644 index 000000000..fec82544a --- /dev/null +++ b/verification/baseline-queue.jsonl @@ -0,0 +1,52 @@ +{"body_sha256": "eab776a8afa8cfe67479add2ac89855a63fabdee6e46f9b3e445b85a7c94b3c2", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "Blackwell enables two SMs within a TPC to cooperatively execute a single larger MMA, doubling the effective compute tile size to m256\u00d7n256\u00d7k16.", "sha256": "4c68819d658d95f6b0bad35c414b126ee5e6bb11ce01bd06368780a795ca9fcc"}, {"id": "u002", "kind": "code", "locator": "body:L7-L15", "preview": "``` TPC (Two Processing Clusters) \u251c\u2500\u2500 SM 0: CTA 0 \u2014 issues tcgen05.mma with cta_group::2 \u2502 \u251c\u2500\u2500 Shared Memory A (rows 0-127) \u2502 \u2514\u2500\u2500 TMEM (columns 0-255) \u2514\u2500\u2500 SM 1: CTA 1 \u2014 cooperates on same MMA \u251c\u2500\u2500 Shared Memory A (rows 128-255) \u2514\u2500\u2500 TMEM (col", "sha256": "0e6130294d59c8d44d6f365e2991eb717fe19cb14ebbe2cb9d6de51ceb6eafa7"}, {"id": "u003", "kind": "code", "locator": "body:L19-L23", "preview": "```ptx // 2-SM cooperative MMA tcgen05.mma.cta_group::2.kind::f16 [tmem_addr], descA, descB, idescC, idescD, ...; ```", "sha256": "0d700ba8114de50a817d01c7fff269445c26dc148aaec150b153114b0ae13af1"}, {"id": "u004", "kind": "list-item", "locator": "body:L26-L26", "preview": "1. **Identical shared memory layouts** across both CTAs", "sha256": "e36c4a200c64261baec9202ed512b08e3fd7120dd8f3e861b8499534d5faa8c5"}, {"id": "u005", "kind": "list-item", "locator": "body:L27-L27", "preview": "2. `shared::cluster` mbarrier signaling between the two CTAs", "sha256": "0c90bbadbfda4845442a1ee4f2ef2d47af52260fc733a49f19023a151a9c1f78"}, {"id": "u006", "kind": "list-item", "locator": "body:L28-L28", "preview": "3. Both CTAs in the same cluster", "sha256": "c1d8899ae6912c581608e66adfb95f7d00fcfb2d95afe923535e0d1ed3dec714"}, {"id": "u007", "kind": "list-item", "locator": "body:L29-L29", "preview": "4. Each CTA contributes half the M-dimension", "sha256": "386a2289fdb65727b75dfe9f61fd64665d56e9b575b55fc33426d5493c287c79"}, {"id": "u008", "kind": "prose", "locator": "body:L33-L33", "preview": "From tcgen05 tutorial progression:", "sha256": "24895ac2a57396c2f65b109f4169614ed5fe28fe729d5894ad6d8ecdca334e0e"}, {"id": "u009", "kind": "list-item", "locator": "body:L34-L34", "preview": "- 1-SM MMA (m128\u00d7n256): 80% of cuBLAS \u2192 adding 2-SM: **86%** of cuBLAS", "sha256": "cdbe9f4fdf31a0cba13990dbbe1dae1fe68157127f5a53f4c6471eeb7445491a"}, {"id": "u010", "kind": "list-item", "locator": "body:L35-L35", "preview": "- ~7.5% improvement from doubling the MMA tile size", "sha256": "a0db83cdbca1698169acc1f402113508f5025a7c078746dbb54a6cd94d7c31a3"}, {"id": "u011", "kind": "list-item", "locator": "body:L38-L38", "preview": "- Large GEMM problems where M \u2265 256", "sha256": "5cf6ae011600c2b7ef443ae8818779bbb37c79759270d72e7afd4416c918611c"}, {"id": "u012", "kind": "list-item", "locator": "body:L39-L39", "preview": "- Compute-bound kernels where peak FLOPS matters", "sha256": "c2dff9522ca31603681ba360962d9f61abe2b14a7994e4d6cd9b6b4baca0ef97"}, {"id": "u013", "kind": "list-item", "locator": "body:L40-L40", "preview": "- Combined with persistent scheduling for maximum throughput", "sha256": "6861f7f0c6a4293b2576a9b60618dbce4a3d149ca8ed6845087682ae9aca8f53"}, {"id": "u014", "kind": "list-item", "locator": "body:L43-L43", "preview": "- [tcgen05-mma](tcgen05-mma.md) \u2014 Base MMA instruction", "sha256": "2d8fd127e151bfd81fd1cf598e5c8b8951431858ce1db08f94fb27c13f611d92"}, {"id": "u015", "kind": "list-item", "locator": "body:L44-L44", "preview": "- [tmem](tmem.md) \u2014 Full TMEM used in 2-SM mode", "sha256": "cb9c4c506579cc6a91c520555593e3e08dad8bb1730670cadb62aab9007e8a30"}], "confidence_claimed": "source-reported", "headings": ["Overview", "How It Works", "PTX", "Requirements", "Performance Impact", "When to Use", "Related"], "id": "hw-2sm-cooperative", "path": "wiki/hardware/2sm-cooperative.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/colfax-cutlass-blackwell.md", "url": "https://research.colfax-intl.com/cutlass-tutorial-writing-gemm-kernels-using-tmem-for-nvidia-blackwell-gpus/"}, {"path": "sources/blogs/modular-blackwell-matmul.md", "url": "https://www.modular.com/blog/matrix-multiplication-on-nvidias-blackwell-part-1-introduction"}], "risk_flags": ["code"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-colfax-cutlass", "blog-modular-blackwell"], "title": "Two-SM Cooperative MMA", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "a62ba7ec03d8146415399b716afb18e5106e6d2fd332e93a3500ddefb76fcc67", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Cluster Launch Control (CLC) is a Blackwell hardware mechanism for **dynamic tile scheduling** in persistent kernels. It replaces the static grid scheduling model where the CUDA runtime pre-assigns tile coordinates to CTAs at launch time.", "sha256": "809740976720d1f707ed432f9fbeb7419e1618edf1287ddb558475a39f5be80b"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "With CLC, persistent CTAs dynamically request work from a hardware queue, enabling:", "sha256": "7de97039e86f839d9541660073abb91c7e18fe17ce24c8b0e2fe9a2dd9a9ada8"}, {"id": "u003", "kind": "list-item", "locator": "body:L9-L9", "preview": "- **Better load balancing**: No fixed CTA-to-tile mapping; busy SMs consume tiles as they become available.", "sha256": "c60549f182dfafc3b029fde4c9a97ebb983e25dea6e4cef70bb1dbeac4112713"}, {"id": "u004", "kind": "list-item", "locator": "body:L10-L10", "preview": "- **Tail-effect mitigation**: The \"tail\" of a GEMM (when remaining tiles < number of SMs) is handled efficiently because idle CTAs pick up remaining work.", "sha256": "abbba0595bbf9bf715bf50f6bb896274460ec965c6f408542df87ab972d79b17"}, {"id": "u005", "kind": "list-item", "locator": "body:L11-L11", "preview": "- **Dynamic cancellation**: Tiles can be cancelled via `try_cancel` when the output is no longer needed (e.g., speculative decoding).", "sha256": "5c7c86292402f652338853d5d7e251efc8f67721d94333eb2c6b16f15f5851b3"}, {"id": "u006", "kind": "code", "locator": "body:L17-L28", "preview": "``` Launch grid: 256 CTAs for 256 tiles CTA 0 -> tile (0,0) [fixed at launch] CTA 1 -> tile (0,1) [fixed at launch] CTA 2 -> tile (0,2) [fixed at launch] ... CTA 255 -> tile (15,15) [fixed at launch] Problem: If SM count = 132, first wave =", "sha256": "17cf8c3ba59bfe0189670fd627792e535cee0a5a1cde230252ade798c2be2ac3"}, {"id": "u007", "kind": "code", "locator": "body:L32-L41", "preview": "``` Launch grid: 132 persistent CTAs (= SM count) CTA 0: request tile -> get (0,0) -> compute -> request tile -> get (2,4) -> ... CTA 1: request tile -> get (0,1) -> compute -> request tile -> get (2,5) -> ... ... CTA 131: request tile -> g", "sha256": "09db3fc1f09143c97961d27aaba50c42068a4a14cbf3b6b591dab52c0d79a4f6"}, {"id": "u008", "kind": "prose", "locator": "body:L47-L47", "preview": "CLC operates by letting a running CTA or cluster issue `clusterlaunchcontrol.try_cancel` to cancel a not-yet-launched ClcID and take over that work. There is no `clusterlaunchcontrol.try_acquire` PTX instruction.", "sha256": "7dad5cda4c0f9c47dab0286e1c8c1455bb98affcfb697ebc2c52f98e11ec31dd"}, {"id": "u009", "kind": "code", "locator": "body:L51-L119", "preview": "```cuda __global__ void persistent_gemm_clc( const half* A, const half* B, half* C, int M, int N, int K, int num_tiles_m, int num_tiles_n ) { // Allocate persistent resources (TMEM, pipeline state) uint32_t tmem_acc = tmem_alloc(256); // Sh", "sha256": "53e9a564d9d8dbcd56a8217c4d8c3f753c44af16a1a71de83a84ccd70c7c5a07"}, {"id": "u010", "kind": "prose", "locator": "body:L123-L123", "preview": "CLC provides a `try_cancel` mechanism to cancel pending tiles. This is useful for speculative execution where some outputs may not be needed.", "sha256": "6e386de8a88854a316dead56b7e2689d327f02dfcdc62be68ef8eaf4f28f5107"}, {"id": "u011", "kind": "code", "locator": "body:L125-L139", "preview": "```cuda // Cancel a specific tile if it hasn't started execution yet __device__ bool clc_try_cancel(uint2 tile_coord) { bool cancelled = false; if (threadIdx.x == 0) { asm volatile( \"clusterlaunchcontrol.try_cancel.async.shared::cta \" \"%0, ", "sha256": "d311e2f374fae4da6c0d416a644acb086a27529a68aababcef0ffc106ee3f6b5"}, {"id": "u012", "kind": "prose", "locator": "body:L141-L141", "preview": "Use cases for `try_cancel`:", "sha256": "7dee2f26f6be4e6d93127fd3171102b70e3cb223aa5b148ff0391320eb09a852"}, {"id": "u013", "kind": "list-item", "locator": "body:L142-L142", "preview": "- **Speculative decoding**: Cancel tiles for rejected draft tokens.", "sha256": "028ee93e2ec8c6e1979353553aa826649d58198c00d57c36c1679b15d48d3a49"}, {"id": "u014", "kind": "list-item", "locator": "body:L143-L143", "preview": "- **Early termination**: If an attention mask makes certain output tiles zero, cancel them.", "sha256": "2c94359880908c2e36288cd2282a8f95b823bee9e266e2bed6af829ebfba2b03"}, {"id": "u015", "kind": "list-item", "locator": "body:L144-L144", "preview": "- **Dynamic batching**: Cancel tiles for sequences that have finished.", "sha256": "1074ab2c074299a4c55c886a246700069beceb4961e2c449bb9978d43bd66d80"}, {"id": "u016", "kind": "prose", "locator": "body:L148-L148", "preview": "CUTLASS 4.5.0 for SM100 provides CLC support through the `PersistentScheduler` class:", "sha256": "ee120d011b8aaac48b2e89c9c0a5868725f557b4c2d6d209ca8c7e7af9cb6d43"}, {"id": "u017", "kind": "code", "locator": "body:L150-L173", "preview": "```cuda // CUTLASS SM100 persistent GEMM with CLC scheduling using Gemm = cutlass::gemm::device::GemmUniversal< cutlass::half_t, // ElementA cutlass::layout::RowMajor, // LayoutA cutlass::half_t, // ElementB cutlass::layout::ColumnMajor, //", "sha256": "555cbcad9bc4a123bf12f90acc231d4f9793d057ef9026c657dfbac9054b7581"}, {"id": "u018", "kind": "code", "locator": "body:L177-L198", "preview": "```cpp // Simplified CUTLASS CLC scheduler logic struct ClcTileScheduler { CUTLASS_DEVICE WorkTileInfo get_next_work() { WorkTileInfo work; // Fetch next tile by cancelling a not-yet-launched ClcID. bool valid = clc_try_cancel(&work.tile_co", "sha256": "8baf49e6c974f5003811f01743bb85a55cae8792832b3ed7f18b0e323ac8f999"}, {"id": "u019", "kind": "prose", "locator": "body:L202-L202", "preview": "CLC delivers significant performance gains, especially for small-to-medium GEMMs where tail effects dominate:", "sha256": "18117e0dc8948a6f977cbfa5be531a6652b1437bb13a7fa863b152e96282915d"}, {"id": "u020", "kind": "table-row", "locator": "body:L208-L208", "preview": "| GEMM Size | Static Scheduler | CLC Scheduler | Improvement | | 2048x2048 (small) | 86% SM utilization | 98% SM utilization | +14% |", "sha256": "4f12dd5a7b2468150d3e45b1126adc48464fc8cadcea2db2b2ba994c512f9dd1"}, {"id": "u021", "kind": "table-row", "locator": "body:L209-L209", "preview": "| GEMM Size | Static Scheduler | CLC Scheduler | Improvement | | 4096x4096 (medium) | 92% SM utilization | 98% SM utilization | +6.5% |", "sha256": "7cbcb4f279c2733e9e1095434ba5e62581d2ed6e41cadabe61e0475a8670278d"}, {"id": "u022", "kind": "table-row", "locator": "body:L210-L210", "preview": "| GEMM Size | Static Scheduler | CLC Scheduler | Improvement | | 8192x8192 (large) | 97% SM utilization | 99% SM utilization | +2% |", "sha256": "4166660c4a0bfbaba7a785122e55d648666898439d8ac3287b35b6d5da69a115"}, {"id": "u023", "kind": "prose", "locator": "body:L212-L212", "preview": "The canonical benchmark from the \"tcgen05 for dummies\" tutorial shows the jump from 940 TFLOPS (pipelined, static scheduling) to **1476 TFLOPS** (persistent + CLC), approaching 98% of cuBLAS (1507 TFLOPS).", "sha256": "8a495346706cea8f3a4017093acf2d775ce8387ccd30fdab3d295f8174fc3105"}, {"id": "u024", "kind": "prose", "locator": "body:L216-L216", "preview": "Production LLM inference typically hits shapes where tail effects are severe:", "sha256": "75556ffebe40bb3047086fde15de20b94151faeea948d991cc5437bbc33f19fe"}, {"id": "u025", "kind": "code", "locator": "body:L218-L229", "preview": "```python # Typical LLM GEMM shapes during decode (batch_size=1-64) # M is small (batch * seq_len for decode), N and K are large (model dim) # Example: Llama-70B decode, batch=32 M = 32 # small! N = 8192 # hidden dim K = 8192 # hidden dim #", "sha256": "6c5d37ce3d231bdf0054356208dbdff793f4db61cb09292da26dd5b309d0e667"}, {"id": "u026", "kind": "prose", "locator": "body:L233-L233", "preview": "When using 2-SM cooperative MMA (`cta_group::2`), CLC distributes work in **cluster-sized units**:", "sha256": "b1754915c10a1c6679890fa9fb5a6f2a20504f9b03f7c044d53da47065052e7f"}, {"id": "u027", "kind": "code", "locator": "body:L235-L260", "preview": "```cuda // 2-SM cooperative CLC: each successful cancel gets a cluster-sized tile __device__ void cooperative_clc_loop() { while (true) { // Fetch tile for the 2-CTA cluster ClusterTile tile; bool valid = clc_try_cancel_cluster(&tile); if (", "sha256": "1af223cad56d56ffa8f45d5ad4c23270bfc3e1613bf685c2fb5e776c748056d8"}, {"id": "u028", "kind": "prose", "locator": "body:L264-L264", "preview": "CLC tile ordering can be customized with swizzle patterns to improve L2 cache hit rates:", "sha256": "25d303ff88dddf358d48c035125c2f5dbdacdfdbff8975f372c389adda9b423c"}, {"id": "u029", "kind": "code", "locator": "body:L266-L281", "preview": "```cuda // Swizzle tile coordinates for better L2 locality // Tiles are visited in a Z-order (Morton) curve pattern __device__ void apply_l2_swizzle(int& tile_m, int& tile_n, int swizzle_bits) { // Convert linear tile index to swizzled 2D c", "sha256": "e907367bf02425d441f6f01b551519752a02632a2dc8a771f95eb12513f2ff6f"}, {"id": "u030", "kind": "table-row", "locator": "body:L287-L287", "preview": "| Feature | CLC (Hardware) | Software Atomics | | Scheduling overhead | Near zero (hardware) | atomicAdd contention |", "sha256": "0cace5c9781e52dd47525aa3dc6554b2cc6b993a0111b2776fbb67af76fc8513"}, {"id": "u031", "kind": "table-row", "locator": "body:L288-L288", "preview": "| Feature | CLC (Hardware) | Software Atomics | | Tail-effect handling | Optimal | Good with careful design |", "sha256": "170ab750f04d2d622fba6d0621244b73e6a6626b7d9971625df8ccb29c8ce720"}, {"id": "u032", "kind": "table-row", "locator": "body:L289-L289", "preview": "| Feature | CLC (Hardware) | Software Atomics | | Cancellation | try_cancel API | Complex (flags + barriers) |", "sha256": "c045af8be0dfa672f3691ebc5e95a69d0530e25b53d1f33c8726c8434b593931"}, {"id": "u033", "kind": "table-row", "locator": "body:L290-L290", "preview": "| Feature | CLC (Hardware) | Software Atomics | | L2 swizzle | Configurable at launch | Manual implementation |", "sha256": "cc683cd2d19aa5060072b2dd9cbdb666a92467dc65eacafdbe5a9424cdc3faac"}, {"id": "u034", "kind": "table-row", "locator": "body:L291-L291", "preview": "| Feature | CLC (Hardware) | Software Atomics | | Portability | SM100+ only | SM70+ |", "sha256": "b401a1d2b6a5df06133924799fce818daed60f89b96821e78cc1c80313270345"}, {"id": "u035", "kind": "table-row", "locator": "body:L292-L292", "preview": "| Feature | CLC (Hardware) | Software Atomics | | CUTLASS support | Built-in | Manual scheduler |", "sha256": "c192f03ddac73f3b1fad9831e3c9c74bc9a103c7ec420fe7cb02fbc60186a12f"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Static Scheduling vs CLC", "Static Scheduling (Hopper and earlier)", "CLC Dynamic Scheduling (Blackwell)", "How CLC Works", "Hardware Queue", "CLC Programming Model", "try_cancel API", "CUTLASS Integration", "CUTLASS CLC Tile Scheduler", "Performance Impact", "Tail Effect Mitigation", "Why CLC Matters for Inference", "CLC with 2-SM Cooperative Mode", "L2 Cache Swizzling with CLC", "Comparison: CLC vs Software Persistent Scheduling"], "id": "hw-clc", "path": "wiki/hardware/clc.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/docs/nvidia-cutlass-blackwell.md", "url": "https://docs.nvidia.com/cutlass/latest/CHANGELOG.html"}, {"path": "sources/prs/cutlass/PR-2161.md", "revision": "06e560d9", "url": "https://github.com/NVIDIA/cutlass/pull/2161"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "doc-cutlass-blackwell", "pr-cutlass-2161"], "title": "Cluster Launch Control (CLC)", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "3abf0f9b53646c33d53214d726c4d11ecf825d2ce9393225f20fd6d3e0fafe27", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "mbarriers are 64-bit shared memory primitives used for producer/consumer synchronization between asynchronous hardware units (TMA, tcgen05) and SM threads. Introduced on Hopper, essential for Blackwell warp-specialized kernels.", "sha256": "67020456ed5764d79a1c98498c631d73b9c845f30e11afec6b55460ed0c53542"}, {"id": "u002", "kind": "code", "locator": "body:L9-L24", "preview": "```ptx // Initialize: set expected arrival count mbarrier.init.shared.b64 [mbar_addr], num_arrivals; // Producer: arrive on barrier (decrements expected count) mbarrier.arrive.shared.b64 _, [mbar_addr]; // Producer (with byte expectation): ", "sha256": "469f01b42c7f6c8ea406edff158693ba9956925112699ae0c1cd5ab43fd1666a"}, {"id": "u003", "kind": "prose", "locator": "body:L28-L28", "preview": "mbarriers have a 1-bit **phase** that flips each time the arrival count reaches zero. Consumers track their expected phase and wait for it:", "sha256": "bca98948b730a9bdd49f90e0b1920281b7a00b78e471ce1bcee28bf907860389"}, {"id": "u004", "kind": "code", "locator": "body:L30-L45", "preview": "```cuda int phase = 0; for (int k = 0; k < num_iterations; k++) { int stage = k % NUM_STAGES; // Wait for this stage's producer to complete mbarrier_wait_parity(&mbar[stage], phase); phase ^= 1; // Flip for next use of this stage slot // Us", "sha256": "f8fbd1f81d0f6beccfb3759b3534f9f954006c9cf8812042cc00e3a5d4628c83"}, {"id": "u005", "kind": "prose", "locator": "body:L49-L49", "preview": "TMA directly signals mbarriers when transfer completes, avoiding manual polling:", "sha256": "7649de3708766636171cd17cc5d543ed0669bc851c83db36dba400c235a22064"}, {"id": "u006", "kind": "code", "locator": "body:L51-L62", "preview": "```cuda // Producer warp issues TMA with mbarrier target if (lane_id == 0) { uint32_t tx_bytes = TILE_A_BYTES + TILE_B_BYTES; mbarrier_arrive_expect_tx(&mbar[stage], tx_bytes); // TMA completion will fire the mbarrier automatically cp_async", "sha256": "a6dd6f7d2f3833966150f71e6fa6631331fbd15c68c416aeefef947ccd0bcbab"}, {"id": "u007", "kind": "list-item", "locator": "body:L66-L66", "preview": "1. **Missing phase flip**: Reusing a stage slot without flipping parity causes stale arrivals to satisfy the new wait", "sha256": "38b27a576a35224085b652fe4870a18aa1b202b278b367a0b9a6131ba26a1920"}, {"id": "u008", "kind": "list-item", "locator": "body:L67-L67", "preview": "2. **Mismatched init count**: If consumer expects N arrivals but producers only arrive M times, barrier never fires", "sha256": "4db264fc6bcea52fd5d405887493a8213f0d99e37cfe4b4b77d4f3eb2e490f44"}, {"id": "u009", "kind": "list-item", "locator": "body:L68-L68", "preview": "3. **Manual arrive after async issue**: TMA/tcgen05 hardware arrives on completion \u2014 extra manual arrive causes double-count", "sha256": "b92fb45c2ea90b8ee0f51687f8256c71d80afa26cc2ef609d685bbf489894274"}, {"id": "u010", "kind": "list-item", "locator": "body:L69-L69", "preview": "4. **Cross-cluster mbarriers**: Use `shared::cluster` qualifier for cluster-level sync (2-SM cooperative MMA)", "sha256": "b65c1a76861d9d4794307647957566ab68c04c603afc82f9a4a7c5eb78e39349"}, {"id": "u011", "kind": "list-item", "locator": "body:L72-L72", "preview": "- [TMA](tma.md) \u2014 Primary producer for mbarriers", "sha256": "4a9353fd25d5f980a2af41bca0f2670f8c446c550d603d2f98d2ab40cb6c8715"}, {"id": "u012", "kind": "list-item", "locator": "body:L73-L73", "preview": "- [warp-specialization](../techniques/warp-specialization.md) \u2014 Uses mbarriers for warp role handoff", "sha256": "79c4fe0f0e618041a37593c472e4299f1c0b09794e072ea8a941418f58c36e4e"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Key Operations", "Phase/Parity Semantics", "TMA Integration", "Pitfalls", "Related"], "id": "hw-mbarrier", "path": "wiki/hardware/mbarrier.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/parallel-thread-execution/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}], "risk_flags": ["code", "ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-ptx-isa-sm100", "blog-tcgen05-tutorial", "doc-nvidia-tuning-guide"], "title": "mbarrier (Memory Barrier Primitives)", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "b655cfe0784f6e74204d4748a18fb4f06f420b6ddcae893e9b1e8596105d1c1e", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "NVFP4 is NVIDIA's 4-bit floating-point format (E2M1) with block scaling, native to Blackwell tensor cores.", "sha256": "8abc922e94307133aed94cec43886360a79d51104781639e3de843b167647f67"}, {"id": "u002", "kind": "code", "locator": "body:L7-L16", "preview": "``` E2M1: 1 sign bit, 2 exponent bits, 1 mantissa bit Representable values: 0, \u00b10.5, \u00b11, \u00b11.5, \u00b12, \u00b13, \u00b14, \u00b16 Block scaling: every 16 FP4 elements share one FP8 E4M3 scale factor Two-level: per-block E4M3 scale \u00d7 per-tensor FP32 global scal", "sha256": "21ffe2a58e99999f5a22735348606a78f38c4a0a9d0b45d67226ace389690aeb"}, {"id": "u003", "kind": "table-row", "locator": "body:L22-L22", "preview": "| Variant | Description | Throughput vs Hopper | | `tcgen05.mma.mxf4.block_scale` | MX FP4 with block scaling | **4\u00d7** |", "sha256": "86a6edd61976a63588789c9d7a7f5240d0fc24dde9b9adf55e9263af545e5d37"}, {"id": "u004", "kind": "table-row", "locator": "body:L23-L23", "preview": "| Variant | Description | Throughput vs Hopper | | `tcgen05.mma.mxf4nvf4.block_scale` | NVFP4 + MX FP4 flexible scaling | **4\u00d7** |", "sha256": "cef8229b3c8d04130a4e97ec237545e6a60579581fc00a4815311ca4da787a6b"}, {"id": "u005", "kind": "code", "locator": "body:L27-L33", "preview": "```ptx // Convert two FP4 values to two FP16 values cvt.rn.f16x2.e2m1x2 result, packed_fp4; // Byte unpacking (faster than bitwise extraction) mov.b32 {tmp0, tmp1, tmp2, tmp3}, packed_data; ```", "sha256": "2a0444506f6201cdbd12ee4d84eeee8e34480f8b214d1dd0ccd3588fc52edb9a"}, {"id": "u006", "kind": "table-row", "locator": "body:L39-L39", "preview": "| Aspect | NVFP4 | MXFP4 | | Scale format | E4M3 (fractional) | UE8M0 (power-of-2 only) |", "sha256": "5ba0bd24ff2f41265f63ef1ace18f01306be54fd7112770099b1c953bf18f581"}, {"id": "u007", "kind": "table-row", "locator": "body:L40-L40", "preview": "| Aspect | NVFP4 | MXFP4 | | Block size | 16 elements | 32 elements |", "sha256": "e15b87ee9acbd43e77de7ae77e119af93aa10076429d524afc56288a410c4e78"}, {"id": "u008", "kind": "table-row", "locator": "body:L41-L41", "preview": "| Aspect | NVFP4 | MXFP4 | | Scale precision | Non-power-of-2 | Power-of-2 only |", "sha256": "5c6b7ed6fd96be13b33e439ec119d42ddfd1148a4412747eafe9eda0e96efa4e"}, {"id": "u009", "kind": "table-row", "locator": "body:L42-L42", "preview": "| Aspect | NVFP4 | MXFP4 | | Quantization error | Lower | Higher |", "sha256": "ab1b99f61d633a906af2f37809cbddbe0625672eca952e604847d0f129672947"}, {"id": "u010", "kind": "list-item", "locator": "body:L45-L45", "preview": "- [fine-grained-quantization](../techniques/fine-grained-quantization.md) \u2014 Scaling strategies", "sha256": "9915527113d53600b0e00c300a216a38331c04613d563f5bf8b899d13a5ee35e"}, {"id": "u011", "kind": "list-item", "locator": "body:L46-L46", "preview": "- [nvfp4-gemm](../kernels/nvfp4-gemm.md) \u2014 NVFP4 GEMM kernel", "sha256": "b5d9ffba19b2bfbf33f08e8c8c3a3948b5da2ff6e83d0d08a4b378d3bfa54bef"}, {"id": "u012", "kind": "list-item", "locator": "body:L47-L47", "preview": "- [nvfp4-gemv](../kernels/nvfp4-gemv.md) \u2014 NVFP4 GEMV kernel", "sha256": "07b0c9ffbb257479ce649d651ae4501a13ff857a375e7cf09c9ba38415b55ad6"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Format Details", "tcgen05 Variants for FP4", "PTX for FP4 Conversion", "NVFP4 vs MXFP4", "Related"], "id": "hw-nvfp4", "path": "wiki/hardware/nvfp4.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/contests/gpu-mode-nvfp4/problem-1-gemv.md", "url": "https://github.com/gpu-mode/reference-kernels"}, {"path": "sources/contests/gpu-mode-nvfp4/problem-2-gemm.md", "url": "https://github.com/gpu-mode/reference-kernels"}, {"path": "sources/blogs/yue-nvfp4-hackathon.md", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}], "risk_flags": ["code", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "contest-gpumode-p1", "contest-gpumode-p2", "blog-yue-nvfp4"], "title": "NVFP4 and Block-Scaled Narrow Precision", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "9783467bbb5c577667660ff8f39f2dd672f1bdcc6cd4729a4f491421bb3ba735", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "PDL/GDC allows overlapping execution of dependent kernel launches. The primary kernel signals it is finishing; the secondary kernel begins before the primary fully completes.", "sha256": "6ee20df45153a87403a8123c11cdfe4ce0cffbbe1e5b5ab96fb6ae1d3ab29cdc"}, {"id": "u002", "kind": "code", "locator": "body:L7-L13", "preview": "```cuda // Primary kernel signals near completion cudaGridDependencySynchronize(); // or PTX equivalent // Secondary kernel can start overlapping with primary's tail // Enabled by default on SM100 (opt-in on SM90) ```", "sha256": "fde03eca8481d032ac185e0c77e83fc75c981975ee62cccaefc28eef55cb89ac"}, {"id": "u003", "kind": "prose", "locator": "body:L17-L17", "preview": "On SM100, PDL is **enabled by default** \u2014 no opt-in needed. This means:", "sha256": "469c6c5ce94a1f2f9da580c6e750ea6aa4881691335a97d37b42cd0b8b8fbe1c"}, {"id": "u004", "kind": "list-item", "locator": "body:L18-L18", "preview": "- Back-to-back kernel launches naturally overlap", "sha256": "7b0ee7a0495159e9a11e2015f1bc02e3e97463a88cfc2bd63655a76aa86518b0"}, {"id": "u005", "kind": "list-item", "locator": "body:L19-L19", "preview": "- Memory fences ensure correctness for dependent data", "sha256": "10e4c87ee21875f0548b79cc50b4cb631f815972e226e24b0d28f248c0515b07"}, {"id": "u006", "kind": "list-item", "locator": "body:L20-L20", "preview": "- Reduces kernel launch gaps in compute-heavy pipelines", "sha256": "eb358b222622f97bb072d5f70f8caed8c2348f651c1a4639bf235e93aa0cd25e"}, {"id": "u007", "kind": "list-item", "locator": "body:L23-L23", "preview": "- Chains of small kernels (e.g., MoE dispatch \u2192 compute \u2192 combine)", "sha256": "fc65cacba10b0ec424deacf202b29de22791a8e7fa0c0bcf2972122e0ab4c355"}, {"id": "u008", "kind": "list-item", "locator": "body:L24-L24", "preview": "- Pipeline-parallel training with many sequential kernel launches", "sha256": "686584fbccd3143eadbe26c92d83bd167406666e8763be1d5219226151e3e747"}, {"id": "u009", "kind": "list-item", "locator": "body:L25-L25", "preview": "- Reduces overall wall-clock time without code changes on Blackwell", "sha256": "c0541298ea3fb26ec25b4c049e4871d52b36a21ccdf70cb9f1c199dc3ec75b97"}, {"id": "u010", "kind": "list-item", "locator": "body:L28-L28", "preview": "- [persistent-kernels](../techniques/persistent-kernels.md) \u2014 Alternative approach to reducing launch overhead", "sha256": "7238ba57fd7ba8c2f8725a0ffe2dd03e5264ebf4a48ff302fc4ae7463de4db08"}, {"id": "u011", "kind": "list-item", "locator": "body:L29-L29", "preview": "- [clc](clc.md) \u2014 Dynamic scheduling within persistent kernels", "sha256": "a69c2395b8b94e45a4b4b8fb8799c9f34daf3bed17aa1aba948132e7c0246ac3"}], "confidence_claimed": "source-reported", "headings": ["Overview", "How It Works", "Blackwell Default Behavior", "When It Matters", "Related"], "id": "hw-pdl-gdc", "path": "wiki/hardware/pdl-gdc.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/prs/cutlass/PR-2161.md", "revision": "06e560d9", "url": "https://github.com/NVIDIA/cutlass/pull/2161"}, {"path": "sources/docs/cutlass-changelog-sm100.md", "url": "https://docs.nvidia.com/cutlass/latest/CHANGELOG.html"}], "risk_flags": ["code", "ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "pr-cutlass-2161", "doc-cutlass-changelog-sm100"], "title": "Programmatic Dependent Launch / Grid Dependency Control", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "8c57f5a2e74a678015fa2b7881071326d35a7aadc4083bddf116a18adf9a3f85", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "`tcgen05.mma` is the Blackwell (SM100/SM100a) matrix-multiply-accumulate instruction that replaces Hopper's `wgmma.mma_async`. The name stands for **Tensor Core Generation 05**. NVIDIA also refers to the higher-level abstraction as **UMMA**", "sha256": "e4d5e8f3a740e318777688256cfab6cd69cce14daf7291f0a61132f6bc90953d"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "Key differences from `wgmma`:", "sha256": "3d20f1cba671f49604f627f9651754d0c20156586dd11a73108058a89adb59a0"}, {"id": "u003", "kind": "table-row", "locator": "body:L11-L11", "preview": "| Property | wgmma (SM90) | tcgen05.mma (SM100) | | Issuing scope | Warpgroup (4 warps, 128 threads) | Single thread |", "sha256": "21bff8dd50856b53aedda9ffa892dd086a7ceb62ee0f83610d9ad1ff022106e1"}, {"id": "u004", "kind": "table-row", "locator": "body:L12-L12", "preview": "| Property | wgmma (SM90) | tcgen05.mma (SM100) | | Accumulator storage | Registers (high pressure) | Tensor Memory (TMEM, 256KB/SM) |", "sha256": "22f040c4a50b295fcde631ecd0a6d24317e496691a960d122acb0455cf934a40"}, {"id": "u005", "kind": "table-row", "locator": "body:L13-L13", "preview": "| Property | wgmma (SM90) | tcgen05.mma (SM100) | | Operand A source | Registers or SMEM | Shared memory only |", "sha256": "6266a9a0739fc35109fb0b0c4ed2c12f327a4c884e83b4a309b76d786accf606"}, {"id": "u006", "kind": "table-row", "locator": "body:L14-L14", "preview": "| Property | wgmma (SM90) | tcgen05.mma (SM100) | | Operand B source | Shared memory | Shared memory only |", "sha256": "8d8b5959fb788164ae4afc88341a935138a73ca6eea514fa5a23ba7bbe2b04ef"}, {"id": "u007", "kind": "table-row", "locator": "body:L15-L15", "preview": "| Property | wgmma (SM90) | tcgen05.mma (SM100) | | Matrix load | ldmatrix to registers | Direct from SMEM (no ldmatrix) |", "sha256": "deadbdd9118b9fcb9335d6bfbe7c91d33754d4de5140124d868e2c15a380b1e3"}, {"id": "u008", "kind": "table-row", "locator": "body:L16-L16", "preview": "| Property | wgmma (SM90) | tcgen05.mma (SM100) | | Synchronization | Warpgroup-scoped barriers | Fully async, fence-based |", "sha256": "f90c83ea61eeb9540ee0d0abdea96e7f2ec44080fbb70f9f78285aaeecc81c22"}, {"id": "u009", "kind": "prose", "locator": "body:L20-L20", "preview": "tcgen05.mma has 7 variants organized by precision and scaling mode:", "sha256": "a34443bdd2aef5c2b7599dca9c8977f35ff434a4ba87a1131015bfef54edad26"}, {"id": "u010", "kind": "table-row", "locator": "body:L24-L24", "preview": "| Variant | A Type | B Type | Accumulator | Scale | MMA Shape (1SM) | Notes | | `tcgen05.mma.kind::f16` | FP16/BF16 | FP16/BF16 | FP32 | None | m128n256k16 | Standard half-precision |", "sha256": "185aee993800654b3c823c155e87086135a4d681897fcd02307dd7dbfe001e81"}, {"id": "u011", "kind": "table-row", "locator": "body:L25-L25", "preview": "| Variant | A Type | B Type | Accumulator | Scale | MMA Shape (1SM) | Notes | | `tcgen05.mma.kind::tf32` | TF32 | TF32 | FP32 | None | m128n256k8 | Single-precision approximation |", "sha256": "af68467a61efc660d8a3812a4575554e984da0cb24e0af747de296ddeb4946eb"}, {"id": "u012", "kind": "table-row", "locator": "body:L26-L26", "preview": "| Variant | A Type | B Type | Accumulator | Scale | MMA Shape (1SM) | Notes | | `tcgen05.mma.kind::f8f6f4` | FP8/FP6/FP4 | FP8/FP6/FP4 | FP32 | Block (UE8M0) | m128n256k32 | Narrow precision with native block scaling |", "sha256": "77c8c39c8996f7efa26e77672bdb6fdd1a4fc7a60f96ae7b66070bc60ec614fa"}, {"id": "u013", "kind": "table-row", "locator": "body:L27-L27", "preview": "| Variant | A Type | B Type | Accumulator | Scale | MMA Shape (1SM) | Notes | | `tcgen05.mma.kind::i8` | INT8 | INT8 | INT32 | None | m128n256k32 | Integer quantized inference |", "sha256": "1316c17042e8f2540f697ec162c4095a49ec0918e64dd0b2321c9c8a1eec0c08"}, {"id": "u014", "kind": "table-row", "locator": "body:L28-L28", "preview": "| Variant | A Type | B Type | Accumulator | Scale | MMA Shape (1SM) | Notes | | `tcgen05.mma.kind::mxf8` | MXFP8 | MXFP8 | FP32 | MX (E8M0) | m128n256k32 | Microscaling FP8 |", "sha256": "d072b6b1f374097d23710cf42a7c2fb007ffa1c43b30e979dea9c9a5a42a3f2d"}, {"id": "u015", "kind": "table-row", "locator": "body:L29-L29", "preview": "| Variant | A Type | B Type | Accumulator | Scale | MMA Shape (1SM) | Notes | | `tcgen05.mma.kind::mxf4` | MXFP4 | MXFP4 | FP32 | MX (E8M0) | m128n256k64 | Microscaling FP4 |", "sha256": "baf885059cee16d5fe0e697097c69625b7d05ea53eec841b2c35f7f0d4da10b0"}, {"id": "u016", "kind": "table-row", "locator": "body:L30-L30", "preview": "| Variant | A Type | B Type | Accumulator | Scale | MMA Shape (1SM) | Notes | | `tcgen05.mma.kind::mxf4nvf4` | NVFP4 | MXFP4 | FP32 | Mixed | m128n256k64 | Mixed NVFP4/MXFP4 |", "sha256": "081789a4a51ae4c6ec7d2eec0673accee43d78e03a2d3c4fa65fdeaebf628b1f"}, {"id": "u017", "kind": "prose", "locator": "body:L36-L36", "preview": "In single-SM mode, a single CTA owns the full MMA operation:", "sha256": "e95cab2553d565ad144238283360c0751f98e89a4e385960e363e9d5a68ee852"}, {"id": "u018", "kind": "list-item", "locator": "body:L38-L38", "preview": "- **BF16/FP16**: M=128, N=256, K=16", "sha256": "d6796d7d9f61232e06b4311b893d2b2eae0a82a962c9db9427349f2d0f97dcb0"}, {"id": "u019", "kind": "list-item", "locator": "body:L39-L39", "preview": "- **TF32**: M=128, N=256, K=8", "sha256": "9f41e350e5bff3603f0577e780f09bf4d459eb68d08f9cd854d4bf6d42e568a9"}, {"id": "u020", "kind": "list-item", "locator": "body:L40-L40", "preview": "- **FP8/FP6/FP4**: M=128, N=256, K=32", "sha256": "57e289de66c363bc36c94988413303d40e5be7626478a54a27dd5667fda256db"}, {"id": "u021", "kind": "list-item", "locator": "body:L41-L41", "preview": "- **MXFP4/NVFP4**: M=128, N=256, K=64", "sha256": "aceb169193ceaf59eda6f0ed24b9dd2d2162ac61067adcbf01899e951de49465"}, {"id": "u022", "kind": "prose", "locator": "body:L45-L45", "preview": "In two-SM cooperative mode, two CTAs share a single MMA across paired SMs:", "sha256": "f2bd49a48c358708522d028c09200e9ea094a4a39954f3a3e8376fd886466fac"}, {"id": "u023", "kind": "list-item", "locator": "body:L47-L47", "preview": "- **BF16/FP16**: M=256, N=256, K=16", "sha256": "292c722d114c6ea9a0451f5aed1f2916d19284c16bb5c9cf2def7500874aede3"}, {"id": "u024", "kind": "list-item", "locator": "body:L48-L48", "preview": "- **TF32**: M=256, N=256, K=8", "sha256": "192f1f5c6b438096a250f4226104d0070930a45692f12cb09b8c19d62ba3fa21"}, {"id": "u025", "kind": "list-item", "locator": "body:L49-L49", "preview": "- **FP8/FP6/FP4**: M=256, N=256, K=32", "sha256": "2dc32df8bac83f14b078efb009d544575ffd4230a19520b21e2316b5e2a5b157"}, {"id": "u026", "kind": "list-item", "locator": "body:L50-L50", "preview": "- **MXFP4/NVFP4**: M=256, N=256, K=64", "sha256": "ddb200b255ce7987de4c22f2117510fe73c076524527264bdba171b0ac929bbc"}, {"id": "u027", "kind": "prose", "locator": "body:L52-L52", "preview": "The M dimension doubles because each SM contributes 128 rows from its own TMEM partition.", "sha256": "245389c1d8c57c0f83c2942c6360ad574a0c7a06300c59c147615c2a627e12ed"}, {"id": "u028", "kind": "prose", "locator": "body:L56-L56", "preview": "Unlike `wgmma` which required coordinated issuance from a warpgroup (4 warps, 128 threads), `tcgen05.mma` is issued by a **single thread**. This is a fundamental architectural simplification:", "sha256": "71da8434bb560ed4ddfbc811220178d5ea11875fc086f8b158cc40ce7683903b"}, {"id": "u029", "kind": "list-item", "locator": "body:L58-L58", "preview": "1. **No warpgroup synchronization overhead** -- one elected thread (typically lane 0 of warp 0) issues the MMA.", "sha256": "432f0d95c6a8d041630a86ca3ff48ff01bc87124cdbfe696b4464a2806f82c7b"}, {"id": "u030", "kind": "list-item", "locator": "body:L59-L59", "preview": "2. **Fully asynchronous** -- the instruction returns immediately; the hardware pipeline executes the MMA in the background.", "sha256": "8dc14dd7c57349307819e7bcfe4f842283c3b91cce9cb8a32d1212b31c49739b"}, {"id": "u031", "kind": "list-item", "locator": "body:L60-L60", "preview": "3. **Fence-based completion** -- the producer must insert explicit fences before reading results from TMEM.", "sha256": "8032f029f78505c1cfef1c04a5868f1f30fb974558b2adf1927741e8f80d67b6"}, {"id": "u032", "kind": "code", "locator": "body:L62-L78", "preview": "```cuda // Single-thread MMA issuance pattern __device__ void issue_mma(uint32_t tmem_addr, uint64_t smem_desc_a, uint64_t smem_desc_b) { // Only one thread issues the MMA if (threadIdx.x == 0) { asm volatile( \"tcgen05.mma.cta_group::1.kind", "sha256": "1da10e7ccfb570ac3a16ff4613c427eb433a91d1dc09c937908df8f895e14919"}, {"id": "u033", "kind": "code", "locator": "body:L84-L90", "preview": "```ptx // Issue a 128x256x16 BF16 MMA // Operand A: shared memory descriptor // Operand B: shared memory descriptor // Accumulator: TMEM address tcgen05.mma.cta_group::1.kind::f16 [tmem_addr], desc_a, desc_b, idesc, 0; ```", "sha256": "2ec3f9255d66d3259a87a98053489dbf2a996031cb9969464321eb8ae3838acb"}, {"id": "u034", "kind": "code", "locator": "body:L94-L98", "preview": "```ptx // Issue a 256x256x16 BF16 MMA across two paired CTAs // cta_group::2 indicates cooperative mode tcgen05.mma.cta_group::2.kind::f16 [tmem_addr], desc_a, desc_b, idesc, 0; ```", "sha256": "c700b225d5065ea34cb16448b941540c38910079f9b53adefde633f66dbb3e8e"}, {"id": "u035", "kind": "code", "locator": "body:L102-L106", "preview": "```ptx // FP8 with native UE8M0 block scaling // scale_desc encodes the per-block scale factors tcgen05.mma.cta_group::1.kind::f8f6f4 [tmem_addr], desc_a, desc_b, idesc, scale_desc; ```", "sha256": "f599a047c6c8f0e014be08650a4e65a950921059e5af835d6e699baf832f3fc8"}, {"id": "u036", "kind": "code", "locator": "body:L110-L143", "preview": "```cuda __device__ void mma_bf16_128x256x16( uint32_t tmem_addr, uint64_t desc_a, uint64_t desc_b ) { if (threadIdx.x == 0) { // First MMA: zero-initialize accumulator asm volatile( \"tcgen05.mma.cta_group::1.kind::f16 \" \"[%0], %1, %2, %3, 0", "sha256": "060c5dff2b407c8c4e2dfc6d4e1ecc8d43a091336c0f06e5842edfcab286d9a8"}, {"id": "u037", "kind": "prose", "locator": "body:L147-L147", "preview": "Fences are **mandatory** for correctness. The hardware does not implicitly synchronize between MMA and TMEM reads/writes.", "sha256": "a72296169aa9135e44dfc5984c321024f77eb48e3d5c9da61743c870647149dc"}, {"id": "u038", "kind": "prose", "locator": "body:L151-L151", "preview": "Insert before reading MMA results from TMEM:", "sha256": "eaf7f1dd300e86a6737d70df5b482dfab6ea182a217b04f674fda58b072e8bdc"}, {"id": "u039", "kind": "code", "locator": "body:L153-L159", "preview": "```cuda // Fence before CTA sync and TMEM reads after the completion mechanism. __device__ void fence_before_tmem_read() { asm volatile(\"tcgen05.fence::before_thread_sync;\"); __syncthreads(); // Ensure all threads see the fence } ```", "sha256": "1fc691ed417f0bd1f9bbf2cb2509f6a3c2c23a5ca95d6b56160d0d114052abda"}, {"id": "u040", "kind": "code", "locator": "body:L163-L169", "preview": "```ptx // Fence before reading TMEM (most common) tcgen05.fence::before_thread_sync; // Fence after CTA sync before issuing dependent tcgen05 operations tcgen05.fence::after_thread_sync; ```", "sha256": "1b8df98c715da239957bab2fa7955b707c81f838d7a38d6726bc6f32fdc5861a"}, {"id": "u041", "kind": "code", "locator": "body:L173-L205", "preview": "```cuda __device__ void gemm_mainloop(/* params */) { for (int k_tile = 0; k_tile < num_k_tiles; ++k_tile) { // 1. Wait for operand data to arrive in SMEM wait_barrier(k_tile % NUM_STAGES); // 2. Issue MMA (single thread) if (threadIdx.x ==", "sha256": "0323f85ba7a3daab67b8987036afeb57aa43a2505184a9cfcc1b7397e014035e"}, {"id": "u042", "kind": "code", "locator": "body:L211-L262", "preview": "```cuda // ---- HOPPER (SM90): wgmma ---- // Requires warpgroup-scoped execution // All 128 threads in a warpgroup participate __device__ void hopper_mma() { // Load A matrix into registers via ldmatrix uint32_t a_frag[4]; asm volatile(\"ldm", "sha256": "5433c50cca6c613a6485e7f48dd6eb9ca5f484168dfb77eb0acb8c3c7ed12f73"}, {"id": "u043", "kind": "list-item", "locator": "body:L266-L266", "preview": "- **Register pressure**: Hopper wgmma uses 128+ registers for accumulators in a large GEMM tile. Blackwell stores accumulators in TMEM, freeing those registers for data movement and epilogue logic.", "sha256": "029c2ae73727925cc4f85919ae8083ed6cb51cee4a0c2b52bbb46d3b1ad32b05"}, {"id": "u044", "kind": "list-item", "locator": "body:L267-L267", "preview": "- **Occupancy**: Lower register pressure enables higher CTA occupancy or larger tile sizes without spilling.", "sha256": "f21aef249d792aa306840015d163f69ed2e37d096ba71f3f20b39c698829e66f"}, {"id": "u045", "kind": "list-item", "locator": "body:L268-L268", "preview": "- **Warp specialization**: With tcgen05, a single MMA-producer warp can feed the tensor cores while other warps handle TMA loads, epilogue, or softmax -- a natural fit for FlashAttention-style kernels.", "sha256": "fa04e99a463edbaa311980500028dda54385d889a81c8c7af72fda61eb2ecfdd"}, {"id": "u046", "kind": "prose", "locator": "body:L272-L272", "preview": "tcgen05.mma requires **128-byte swizzled** shared memory layouts for both operands. Non-swizzled or 64-byte swizzled layouts will produce incorrect results silently.", "sha256": "6dc7add35a27c3559545905c86d768c01d6bade589924bc2228216d8b357fdaf"}, {"id": "u047", "kind": "code", "locator": "body:L274-L290", "preview": "```cuda // Shared memory descriptor construction for tcgen05 // The descriptor encodes: base address, stride, swizzle mode, dimensions __device__ uint64_t make_smem_desc(void* smem_ptr, int stride_bytes) { uint64_t desc = 0; uint32_t addr =", "sha256": "c17d7d6ecf6fdf969cb8ff69fa527a15952391edf02c7f3e7ea2efbcc2673814"}, {"id": "u048", "kind": "table-row", "locator": "body:L296-L296", "preview": "| Optimization Stage | Throughput (TFLOPS) | % of cuBLAS | | Naive tcgen05.mma | 255 | 17% |", "sha256": "215a0512939c93a6645fad9f4143f9f0e5ccea6daa0068335ab637544c5b1f71"}, {"id": "u049", "kind": "table-row", "locator": "body:L297-L297", "preview": "| Optimization Stage | Throughput (TFLOPS) | % of cuBLAS | | + 128B swizzled SMEM | 695 | 46% |", "sha256": "28107fdb48a27091b4c76073eaf584b000ea20c8ec2dde46f59f86a1a1dc461d"}, {"id": "u050", "kind": "table-row", "locator": "body:L298-L298", "preview": "| Optimization Stage | Throughput (TFLOPS) | % of cuBLAS | | + TMA pipelining | 940 | 62% |", "sha256": "e23ddcc26736e8a0270c7b0698da4e2167c9239935887cb66388537f35686645"}, {"id": "u051", "kind": "table-row", "locator": "body:L299-L299", "preview": "| Optimization Stage | Throughput (TFLOPS) | % of cuBLAS | | + Persistent kernel + CLC | 1476 | 98% |", "sha256": "9d517c625ca2a23c9737ee06468b2d3d57575f07e1095b542f2b4d8201acb6e5"}, {"id": "u052", "kind": "table-row", "locator": "body:L300-L300", "preview": "| Optimization Stage | Throughput (TFLOPS) | % of cuBLAS | | cuBLAS reference | 1507 | 100% |", "sha256": "6ebea4a476eab8c20f757e683a33f4511a465993fa375b72a59beb1d119027ee"}], "confidence_claimed": "verified", "headings": ["Overview", "Instruction Variants", "MMA Shapes: 1-SM vs 2-SM", "1-SM (Single CTA) Shapes", "2-SM (Cooperative) Shapes", "Single-Thread Issuance Model", "PTX Examples", "Basic BF16 MMA (1-SM)", "2-SM Cooperative BF16 MMA", "FP8 with Block Scaling", "CUDA Inline PTX for BF16 MMA with Accumulation", "Critical Fences", "tcgen05.fence", "tcgen05.fence variants", "Correct Mainloop Fence Pattern", "Comparison with wgmma", "Programming Model Shift", "Performance Implications", "SMEM Layout Requirements", "Performance Progression (from tcgen05-for-dummies)"], "id": "hw-tcgen05-mma", "path": "wiki/hardware/tcgen05-mma.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/prs/cutlass/PR-2139.md", "revision": "ca4fdbea", "url": "https://github.com/NVIDIA/cutlass/pull/2139"}, {"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/blogs/colfax-cutlass-blackwell.md", "url": "https://research.colfax-intl.com/cutlass-tutorial-writing-gemm-kernels-using-tmem-for-nvidia-blackwell-gpus/"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["pr-cutlass-2139", "doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "blog-colfax-cutlass"], "title": "tcgen05.mma \u2014 Blackwell MMA Instruction", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "5a71bdad2d3319054af91f093a17db812ad57396120e6144d8f62535bed3e49f", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "The Tensor Memory Accelerator (TMA) is a hardware unit that performs **asynchronous bulk data transfers** between global memory and shared memory. First introduced on Hopper (SM90), TMA carries forward to Blackwell (SM100) with stricter req", "sha256": "6d6eff87de98618f04ba285d851f7199cafc69183bacdcde0626ee9c20931113"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "TMA offloads data movement from CUDA cores entirely -- a single thread issues the transfer, and the TMA hardware engine handles the multi-dimensional copy, address calculation, out-of-bounds clamping, and format conversion.", "sha256": "5f5489e9b50365c4c7b7e0f90b48655ccd39ac813cf2dcb78ff9360c5cbcf081"}, {"id": "u003", "kind": "table-row", "locator": "body:L13-L13", "preview": "| Property | Detail | | Transfer direction | GMEM <-> SMEM (bidirectional) |", "sha256": "2da1269cc5091e9525a3ea7e433d11d95b62c61ffb9507ef27edcce2522c7938"}, {"id": "u004", "kind": "table-row", "locator": "body:L14-L14", "preview": "| Property | Detail | | Dimensionality | 1D to 5D tensor copies |", "sha256": "cdba3506fd6e4aee624343d3eca5423401bdffe7f9adb2711555b115c586dfae"}, {"id": "u005", "kind": "table-row", "locator": "body:L15-L15", "preview": "| Property | Detail | | Max transfer size | Up to 256 bytes per element, tiles up to 128x256 |", "sha256": "90b2ff76015ec67931b7de66bf52c314e632138b94f34229afdb258ebbc553a2"}, {"id": "u006", "kind": "table-row", "locator": "body:L16-L16", "preview": "| Property | Detail | | Swizzle modes | None, 32B, 64B, 128B (128B required for tcgen05) |", "sha256": "8bf3b7ca9390572592def684644d8715df52ae1d267f6a78b6a2093689e4c7b1"}, {"id": "u007", "kind": "table-row", "locator": "body:L17-L17", "preview": "| Property | Detail | | Format conversion | FP32<->BF16, FP32<->FP16 during transfer |", "sha256": "d8ea01398e724da58297f46f2db35102480b99554a89efebc524e8fc697ea50e"}, {"id": "u008", "kind": "table-row", "locator": "body:L18-L18", "preview": "| Property | Detail | | Multicast | Single GMEM tile -> multiple CTAs in a cluster |", "sha256": "81a8e2499398ef4067a423dcc17bd45b5bf1104df5c85c7a4aa713feeaa59d13"}, {"id": "u009", "kind": "table-row", "locator": "body:L19-L19", "preview": "| Property | Detail | | Synchronization | mbarrier-based (arrive/wait) |", "sha256": "57cd317aa4514c7cb216e252b7663af4979b1b97ff0af36af4a37703abc92f3c"}, {"id": "u010", "kind": "table-row", "locator": "body:L20-L20", "preview": "| Property | Detail | | Thread requirement | Single thread issues the operation |", "sha256": "dd10cbfa8e26de4d1d42013f895c001de89b10422160dae34f5a58b3f1c05b11"}, {"id": "u011", "kind": "prose", "locator": "body:L24-L24", "preview": "TMA operations are driven by a **descriptor** that encodes the tensor layout, addressing, and transfer parameters. The descriptor is created on the host and passed to the kernel.", "sha256": "06c87265434fd3702487d5ddf4e300ab59a44f78dbc12085d78ee73996ffd876"}, {"id": "u012", "kind": "code", "locator": "body:L28-L75", "preview": "```cuda #include // Create a 2D TMA descriptor for a row-major FP16 matrix CUtensorMap create_tma_descriptor_2d( const half* global_ptr, int M, int N, // Global tensor dimensions int tile_m, int tile_n, // Tile dimensions for each ", "sha256": "57863f4b926991265817ae00465549f2c48c73bb16bdb9f9922b8b00a4edb123"}, {"id": "u013", "kind": "prose", "locator": "body:L79-L79", "preview": "On Blackwell, `tcgen05.mma` requires operands in **128-byte swizzled** SMEM layout. If TMA loads data without 128B swizzling, the MMA will produce incorrect results.", "sha256": "c2d8fcddb976be600f1ac35e18c00ba49c01acdc2bcbde3e14e643d0d56baabd"}, {"id": "u014", "kind": "code", "locator": "body:L81-L89", "preview": "```cuda // CORRECT for Blackwell tcgen05: CUtensorMap desc = create_tma_descriptor_2d(ptr, M, N, 128, 64, 128); // swizzle=128 ^^^ // WRONG for tcgen05 (will silently produce garbage): CUtensorMap desc = create_tma_descriptor_2d(ptr, M, N, ", "sha256": "c6e226b9487e233d560d2878659d04e63db886fcf8c887b53f4f331b5bd9da62"}, {"id": "u015", "kind": "code", "locator": "body:L95-L125", "preview": "```cuda // TMA load: single thread issues, hardware executes asynchronously __device__ void tma_load_tile( const CUtensorMap* desc, void* smem_ptr, uint64_t* mbar_ptr, // mbarrier for synchronization int coord_x, int coord_y ) { if (threadI", "sha256": "2814db4a3e74defb118105670f7b2dba4f85d9afccd3068244202fe37ae54db4"}, {"id": "u016", "kind": "code", "locator": "body:L129-L150", "preview": "```cuda // TMA store: write a tile from shared memory back to global memory __device__ void tma_store_tile( const CUtensorMap* desc, const void* smem_ptr, int coord_x, int coord_y ) { if (threadIdx.x == 0) { asm volatile( \"cp.async.bulk.ten", "sha256": "85156f9df505c2971d69dc88525d09413d6218e432142ac1bef4f821622483de"}, {"id": "u017", "kind": "prose", "locator": "body:L154-L154", "preview": "TMA uses mbarriers (memory barriers) for producer-consumer synchronization. The pattern is:", "sha256": "34d4d48b31d54d11bc76d776aafc0ec9b1dad8a8306744b5c7b65db5f66de30e"}, {"id": "u018", "kind": "list-item", "locator": "body:L156-L156", "preview": "1. **Producer** (TMA): arrives at the barrier when the transfer completes, decrementing the expected transaction count.", "sha256": "8714de0f91ab74b7548cf7c2e1974ad3afd53d60ea966dffe0422ac6ec219fe8"}, {"id": "u019", "kind": "list-item", "locator": "body:L157-L157", "preview": "2. **Consumer** (compute warps): waits on the barrier before reading the loaded data.", "sha256": "21c13e7cb282b911ce221810cb203c81953662b6f33367dbe203ee45d7b90365"}, {"id": "u020", "kind": "code", "locator": "body:L161-L208", "preview": "```cuda // Multi-stage pipeline with TMA + mbarrier __device__ void pipelined_mainloop( const CUtensorMap* desc_a, const CUtensorMap* desc_b, void* smem_a_stages[NUM_STAGES], void* smem_b_stages[NUM_STAGES], uint64_t* mbar[NUM_STAGES], int ", "sha256": "225c35b88486b841873a34050538c637cbf085b0fc63a3c15b84248c0b8de744"}, {"id": "u021", "kind": "code", "locator": "body:L212-L239", "preview": "```cuda // Initialize an mbarrier __device__ void mbarrier_init(uint64_t* mbar, int arrive_count) { if (threadIdx.x == 0) { asm volatile( \"mbarrier.init.shared.b64 [%0], %1;\" : : \"r\"((uint32_t)__cvta_generic_to_shared(mbar)), \"r\"(arrive_cou", "sha256": "37abac18e0c6bfe80b0c4a86e8a2c69c6bd02b290dd63ae53d4f283747316c72"}, {"id": "u022", "kind": "prose", "locator": "body:L243-L243", "preview": "TMA multicast sends a single GMEM tile to **multiple CTAs within a cluster** simultaneously. This is critical for GEMM where the B operand is shared across M-axis tiles.", "sha256": "a6d1fa8c7919f1142e47b85a82a07c116fafee57c0285f647b8aead64d6e36fc"}, {"id": "u023", "kind": "code", "locator": "body:L245-L275", "preview": "```cuda // Multicast TMA: load B tile to all CTAs in the cluster __device__ void tma_multicast_load( const CUtensorMap* desc, void* smem_ptr, uint64_t* mbar_ptr, int coord_x, int coord_y, uint16_t multicast_mask // bitmask: which CTAs in cl", "sha256": "75a84d8e1b144fd1d87e312a3cb22eb2500d3abd72aab748b8c4247ee3ac7354"}, {"id": "u024", "kind": "code", "locator": "body:L279-L289", "preview": "``` Cluster: 2 CTAs (CTA0 and CTA1) each computing different M-tiles of the same N column CTA0: computes C[0:128, 0:256] -- needs A[0:128, :] and B[:, 0:256] CTA1: computes C[128:256, 0:256] -- needs A[128:256, :] and B[:, 0:256] B[:, 0:256", "sha256": "3a54d0f2da8520509ed5a923a537b39d55946b43e0e574601f67a44dd094777c"}, {"id": "u025", "kind": "prose", "locator": "body:L295-L295", "preview": "All TMA loads feeding `tcgen05.mma` must use 128-byte swizzling. The swizzle pattern rearranges bytes within each 128-byte line to match the tensor core's internal data layout:", "sha256": "cb10875d9e024b63e15b6ff8f8a291efa485fe1916f1e6c865ab5fb8382fc0d6"}, {"id": "u026", "kind": "code", "locator": "body:L297-L307", "preview": "``` Without swizzle (linear): Row 0: bytes [0, 1, 2, ..., 127] Row 1: bytes [128, 129, ..., 255] With 128B swizzle: Row 0: bytes [0, 1, ..., 127] (unchanged) Row 1: bytes [128, 129, ..., 255] XOR pattern applied Row 2: bytes [256, ...] XOR ", "sha256": "cb16062ad73a61cf53dcc4443e45be6ff58f5b8b6d377459aedb090a377a8e5e"}, {"id": "u027", "kind": "prose", "locator": "body:L309-L309", "preview": "The swizzle eliminates bank conflicts when the tensor core reads operand tiles from SMEM.", "sha256": "888e7c417217400b77f64adff1a4d64a2ec41bbfe1db329e554b71a4330f3dda"}, {"id": "u028", "kind": "prose", "locator": "body:L313-L313", "preview": "On Blackwell, data flows through a characteristic pipeline:", "sha256": "4a5ecf1e859ecacb759629cda9097134cec765de3c4f0f7949164a061216141c"}, {"id": "u029", "kind": "code", "locator": "body:L315-L321", "preview": "``` GMEM --[TMA]--> SMEM --[tcgen05.mma]--> TMEM --[tcgen05.ld]--> Registers --[st.global]--> GMEM ^ | | v +---- (epilogue) --------+ (bias, activation, etc.) ```", "sha256": "23b694230f776799fe80b7749e74d338abe653b470743f527d8d958405d8c163"}, {"id": "u030", "kind": "table-row", "locator": "body:L327-L327", "preview": "| Tip | Detail | | Maximize TMA utilization | Keep the TMA unit busy with back-to-back loads across pipeline stages |", "sha256": "0f0085d2379e03ead049c34b77c01690f1047ac972f320212547456b038ca909"}, {"id": "u031", "kind": "table-row", "locator": "body:L328-L328", "preview": "| Tip | Detail | | Use multicast for shared operands | Reduces GMEM bandwidth by cluster_size x for shared tiles |", "sha256": "0d0fd8f052b29acfa61919241bc206b7e745d7d9b54b53f51a8a01234cb4eff5"}, {"id": "u032", "kind": "table-row", "locator": "body:L329-L329", "preview": "| Tip | Detail | | Always use 128B swizzle on Blackwell | Non-128B swizzle produces incorrect tcgen05 results |", "sha256": "caad6d87d0bc24231243a1b46df875eba98884e8477729e4e18b90dc02addb2b"}, {"id": "u033", "kind": "table-row", "locator": "body:L330-L330", "preview": "| Tip | Detail | | Prefer 2D TMA over manual addressing | TMA handles out-of-bounds clamping, padding, and strided access |", "sha256": "6f608acbc60d7738b227c27c49c76b27457dca7f851bcce36169be3a93940f61"}, {"id": "u034", "kind": "table-row", "locator": "body:L331-L331", "preview": "| Tip | Detail | | Pipeline depth | 3-5 stages typically optimal; more stages increase SMEM usage |", "sha256": "aa338cea255897b5ec5aac410d0bf95a733eaaa4d783b78e0bd8988dbee32b19"}, {"id": "u035", "kind": "code", "locator": "body:L335-L358", "preview": "```python # CuTe-DSL TMA copy setup for Blackwell GEMM from cute import * # Define TMA copy atom for operand A (BF16, 128x64 tile) tma_a = make_tma_copy( SM100_TMA_LOAD_2D, tensor_a, # global tensor smem_layout_a, # shared memory layout til", "sha256": "10753b9dc055201da7d81c8b71d2cbb21172613f8cb1f6b5e6d94522bb27f8f2"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Key Properties", "TMA Descriptor", "Host-Side Descriptor Creation", "Blackwell Requirement: 128-Byte Swizzle", "Asynchronous Copy Operations", "GMEM to SMEM (Load)", "SMEM to GMEM (Store)", "mbarrier Synchronization", "Pipeline Stage Pattern", "mbarrier Operations", "TMA Multicast", "Multicast in GEMM", "Blackwell-Specific Enhancements", "128-Byte Swizzle for tcgen05", "TMA + TMEM Integration", "Performance Considerations", "CuTe-DSL Example"], "id": "hw-tma", "path": "wiki/hardware/tma.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/prs/flashinfer/PR-2387.md", "revision": "18804cd5", "url": "https://github.com/flashinfer-ai/flashinfer/pull/2387"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "pr-flashinfer-2387"], "title": "Tensor Memory Accelerator (TMA)", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "31c6ed49bcd928754519fe06febb83cd987d9f3691fc9bb89f5aad7c7e1351b0", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Tensor Memory (TMEM) is a new addressable memory space introduced in the Blackwell architecture (SM100). Each SM contains **256KB of dedicated TMEM**, used primarily as the accumulator storage for `tcgen05.mma` operations. TMEM eliminates t", "sha256": "1061d6dc2ddafb48abd714a4d5efe9cd2964680e6c93e163d2a6e56d37057890"}, {"id": "u002", "kind": "prose", "locator": "body:L9-L9", "preview": "TMEM is organized as a 2D matrix:", "sha256": "b9ee2fd601f53f4d69abc8772241239451ce2518103c78f6035064ebf9ea61d6"}, {"id": "u003", "kind": "table-row", "locator": "body:L13-L13", "preview": "| Dimension | Size | Description | | Rows | 128 | Mapped to warp lanes (4 warps x 32 lanes) |", "sha256": "9d2b5837822a69615072b240debe88240992f12ed76775a8fd7dd5457a6e3b7b"}, {"id": "u004", "kind": "table-row", "locator": "body:L14-L14", "preview": "| Dimension | Size | Description | | Columns | 512 | 32-bit (4-byte) elements per row |", "sha256": "cc5876d4574fd324c0889336b94701f83dc37d4a566633d896310c39b4b7e788"}, {"id": "u005", "kind": "table-row", "locator": "body:L15-L15", "preview": "| Dimension | Size | Description | | Total | 128 x 512 x 4 bytes = 256 KB | Per-SM capacity |", "sha256": "823a7c749e8300c9ba2d5901c6c279b0b7991020f7a54eb045244e969579c561"}, {"id": "u006", "kind": "prose", "locator": "body:L19-L19", "preview": "TMEM rows are mapped to warp lanes within a CTA:", "sha256": "1d7f452d6a0d7c61027ddd0241a691a365b1b341c2fc911723b90a7d468a32c6"}, {"id": "u007", "kind": "code", "locator": "body:L21-L26", "preview": "``` Row 0-31: Warp 0, lanes 0-31 Row 32-63: Warp 1, lanes 0-31 Row 64-95: Warp 2, lanes 0-31 Row 96-127: Warp 3, lanes 0-31 ```", "sha256": "0f5db3db15a2057887a17b00f833ff07ab3c38a54054d175aacf6e4da48d548e"}, {"id": "u008", "kind": "prose", "locator": "body:L28-L28", "preview": "Each thread \"owns\" the TMEM row corresponding to its warp and lane. When reading from TMEM, thread `T` in warp `W` accesses row `W*32 + T%32`.", "sha256": "21f05ce2e5f909709af6d5435132ab21c2db8f98bfd6ff3e994b965472d0b7a8"}, {"id": "u009", "kind": "prose", "locator": "body:L32-L32", "preview": "Columns are addressed via a column offset in the TMEM descriptor. A 128x256 MMA accumulator tile occupies:", "sha256": "b1187703d27200988d5e927bfffb48cbc93a89b1529a6291deaaaa9d90e57a2a"}, {"id": "u010", "kind": "list-item", "locator": "body:L34-L34", "preview": "- 128 rows (all lanes across 4 warps)", "sha256": "205bc2020ca1a67fb475a45e45ead00f89691239a0edb820650982280b6816ff"}, {"id": "u011", "kind": "list-item", "locator": "body:L35-L35", "preview": "- 256 columns of FP32 values = 1024 bytes per row", "sha256": "732f5010af611142ddeb72a8f257140137a6950e723cbd1f5e7f33ef32722179"}, {"id": "u012", "kind": "code", "locator": "body:L37-L46", "preview": "``` TMEM Layout for 128x256 FP32 accumulator: col 0 col 255 col 511 |------------|-----------|-----------| | Acc Tile | (free) | (free) | row 0 (warp0, lane0) | 128x256 | | | row 1 (warp0, lane1) | FP32 | | | ... | | | | row 127 (warp3, lan", "sha256": "1890f36c715384fe174527bf32fc91e63c8c1e0501aa57f0f06ed813bfc3912b"}, {"id": "u013", "kind": "prose", "locator": "body:L50-L50", "preview": "TMEM is **explicitly managed** by the programmer. There is no automatic allocation or garbage collection.", "sha256": "13ae2d3bb87ce433fcf18c6be2e0eda6b539a511f50616d91807363bd8c3a6d8"}, {"id": "u014", "kind": "code", "locator": "body:L54-L73", "preview": "```cuda // Shared storage for CTA-wide broadcast of TMEM address __shared__ uint32_t s_tmem_addr; __device__ uint32_t tmem_alloc_cta(uint32_t num_cols) { // Only thread 0 allocates; result must reach ALL warps in the CTA. // __shfl_sync is ", "sha256": "dee7930d5f81bfb644f61c3d4cb0d57161c24c2721683563ebd31546104ef836"}, {"id": "u015", "kind": "prose", "locator": "body:L75-L75", "preview": "Key points:", "sha256": "591268c33079e24bfeebcc0529014d25e7740ec1aee7dc4e76323f4b0b70c5ac"}, {"id": "u016", "kind": "list-item", "locator": "body:L76-L76", "preview": "- `num_cols` specifies the number of 32-bit columns to allocate.", "sha256": "4d144f82f44ee97ab1d9a00ee9229bcd3f2b48f5977cb1fb32c61ce3fa995d74"}, {"id": "u017", "kind": "list-item", "locator": "body:L77-L77", "preview": "- A 128x256 FP32 accumulator needs 256 columns.", "sha256": "785bbae263d94fcc126298f121aa97530d6dcd31501cad5e969efd9dc3e6f922"}, {"id": "u018", "kind": "list-item", "locator": "body:L78-L78", "preview": "- Allocation is **CTA-scoped** -- all threads in the CTA share the same TMEM region.", "sha256": "2bfb5d48482875538a5faf123a1b8b099d4601c17758d9f4ea5201d08580a976"}, {"id": "u019", "kind": "list-item", "locator": "body:L79-L79", "preview": "- Only one thread (typically thread 0) issues the allocation.", "sha256": "662e8f7a636319070e6e6a2e3442ca834f9f94dd2f2bde0b34b2ff1aff631012"}, {"id": "u020", "kind": "list-item", "locator": "body:L80-L80", "preview": "- The returned `tmem_addr` is the base column index.", "sha256": "665b2a536e3c91858e1fa29f0a5d6726395ae9629b209334586080d79f059323"}, {"id": "u021", "kind": "code", "locator": "body:L84-L95", "preview": "```cuda __device__ void tmem_dealloc(uint32_t tmem_addr, uint32_t num_cols) { if (threadIdx.x == 0) { asm volatile( \"tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;\" : : \"r\"(tmem_addr), \"r\"(num_cols) ); } __syncthreads(); } ```", "sha256": "20b0e50a72af8a57ef048d345054636e2f6e36f01353cd96d912757d74df5ef1"}, {"id": "u022", "kind": "prose", "locator": "body:L97-L97", "preview": "**Warning**: Failure to deallocate TMEM before CTA exit will leak memory and prevent subsequent CTAs from allocating, leading to hangs in persistent kernels.", "sha256": "e20744d781f98e7f0ab5ed7cf6ee085090ba692328a4be69d967b49e6803be68"}, {"id": "u023", "kind": "code", "locator": "body:L101-L128", "preview": "```cuda __global__ void persistent_gemm_kernel(/* ... */) { // 1. Allocate TMEM for accumulators uint32_t tmem_acc = tmem_alloc(256); // 256 cols for M=128, N=256 while (has_more_tiles()) { // 2. Zero-initialize TMEM accumulator tmem_zero(t", "sha256": "1943ba3ab4110b43b39e4a4fd0e3a0d92d5571a56f4c7ee5f692ad97e5289e7e"}, {"id": "u024", "kind": "code", "locator": "body:L134-L155", "preview": "```cuda // Store a register value to TMEM // Each thread writes to its own TMEM row at the specified column __device__ void tmem_store_f32(uint32_t tmem_col, float value) { asm volatile( \"tcgen05.st.sync.aligned.32x1b.x1.b32 [%0], {%1};\" : ", "sha256": "8ee021b8d76150fa5fe9547e238730998a762c8f276e95e1fa852e739dbf7382"}, {"id": "u025", "kind": "code", "locator": "body:L159-L182", "preview": "```cuda // Load a single FP32 from TMEM __device__ float tmem_load_f32(uint32_t tmem_col) { float result; asm volatile( \"tcgen05.ld.sync.aligned.32x1b.x1.b32 {%0}, [%1];\" : \"=f\"(result) : \"r\"(tmem_col) ); return result; } // Vectorized load", "sha256": "5cd5f20a3b384f6a3f59d1412c3f1be88f62cf5b86a9681caf9bc9c828f4fb01"}, {"id": "u026", "kind": "code", "locator": "body:L186-L195", "preview": "```cuda // Zero-fill a range of TMEM columns __device__ void tmem_zero(uint32_t tmem_base_col, uint32_t num_cols) { // Each thread zeros its own row for (uint32_t c = 0; c < num_cols; c += 4) { float4 zero = make_float4(0.f, 0.f, 0.f, 0.f);", "sha256": "15e5a029a91cdef685be37048a577786a2fa3fbade814ea22955316225b390bd"}, {"id": "u027", "kind": "prose", "locator": "body:L199-L201", "preview": "The `tcgen05.cp` instruction copies a shaped matrix from shared memory into TMEM. The source operand is a shared-memory matrix descriptor, not a TMEM address.", "sha256": "dee28b3b7fa3a4b8f4900020493fec0e98270193551064a2445dcc3fbe4eb9a2"}, {"id": "u028", "kind": "code", "locator": "body:L203-L206", "preview": "```ptx // Copy a 128x256b shared-memory tile into TMEM. tcgen05.cp.cta_group::1.128x256b [taddr], sdesc; ```", "sha256": "e17e99605a45ed0452be01bea2a86868e7f1292e0305a6b60ce2132909fccbce"}, {"id": "u029", "kind": "prose", "locator": "body:L208-L210", "preview": "The 64-bit register operand `sdesc` is the matrix descriptor representing the source matrix in shared memory. The matrix descriptor format is described in [Matrix Descriptors](https://docs.nvidia.com/cuda/parallel-thread-execution/index.htm", "sha256": "46c231a08a5196493cff2da7f552defbbf23c32e26c0722d41be68e25bc1107b"}, {"id": "u030", "kind": "prose", "locator": "body:L214-L214", "preview": "Double-buffering TMEM accumulators enables overlapping the epilogue of the current tile with the MMA accumulation of the next tile:", "sha256": "755960671f070b38d46bdaaf66d6733b8cb99493150e1e6c0861300fde8571ad"}, {"id": "u031", "kind": "code", "locator": "body:L216-L250", "preview": "```cuda __global__ void double_buffered_gemm(/* ... */) { // Allocate two accumulator buffers uint32_t tmem_acc[2]; tmem_acc[0] = tmem_alloc(256); tmem_acc[1] = tmem_alloc(256); int buf = 0; for (int tile = 0; tile < num_tiles; ++tile) { //", "sha256": "2643142da8b12f116ea2d24d1fcd0640833af539e6f309f58b5f7987b59eb333"}, {"id": "u032", "kind": "prose", "locator": "body:L254-L254", "preview": "With 512 total columns per SM:", "sha256": "c54d0cf39b9707f33d6022aedac3949fcd61dee1ae49258f87f53c667c6e17c0"}, {"id": "u033", "kind": "table-row", "locator": "body:L258-L258", "preview": "| Accumulator Size | Columns | Max Buffers | Remaining for Scratch | | 128x128 FP32 | 128 cols | 4 | 0 |", "sha256": "d3c8bcbe170be7e0fa87e3586a9a4895e9eee077443883e163ba756c48f2d438"}, {"id": "u034", "kind": "table-row", "locator": "body:L259-L259", "preview": "| Accumulator Size | Columns | Max Buffers | Remaining for Scratch | | 128x192 FP32 | 192 cols | 2 | 128 |", "sha256": "5311d980d8f818cbc0f297f9c234f0e78a95fdecbb18160d37f03e4bdee0c009"}, {"id": "u035", "kind": "table-row", "locator": "body:L260-L260", "preview": "| Accumulator Size | Columns | Max Buffers | Remaining for Scratch | | 128x256 FP32 | 256 cols | 2 | 0 |", "sha256": "a8b707eb9a2b10ddf0b6b9e0ccf7e1de5e27f265b3d9b17f096acafd5954cc44"}, {"id": "u036", "kind": "table-row", "locator": "body:L261-L261", "preview": "| Accumulator Size | Columns | Max Buffers | Remaining for Scratch | | 128x256 FP32 (2x) | 512 cols | 2 (double-buf) | 0 |", "sha256": "f6e91576d360ebf9d4404b0cf629c8e620a0188afc21184442c3ece34b38fab4"}, {"id": "u037", "kind": "prose", "locator": "body:L263-L263", "preview": "For double-buffered 128x256 tiles, the full 512 columns are consumed. If additional scratch TMEM is needed (e.g., for softmax in attention kernels), reduce the tile size or use a single accumulator buffer.", "sha256": "3695cf652ed9a7ae68a0152205809028650d423e66fe1e0f54e3b6647fbe5eac"}, {"id": "u038", "kind": "prose", "locator": "body:L267-L267", "preview": "From published Blackwell microbenchmarks:", "sha256": "5cf8af761c70c425dd65df62bb28292ca67bdc79ed264525cac1c432cf16c246"}, {"id": "u039", "kind": "table-row", "locator": "body:L271-L271", "preview": "| Metric | TMEM | SMEM | Registers | | End-to-end latency (cache miss) | ~420 cycles | ~30 cycles | ~4 cycles |", "sha256": "541f73c101fc8fdb339b9e778722b4aea84914271f0beb1c26cd0d32599e765f"}, {"id": "u040", "kind": "table-row", "locator": "body:L272-L272", "preview": "| Metric | TMEM | SMEM | Registers | | Bandwidth for large working sets | High (dedicated bus) | Medium | N/A (limited count) |", "sha256": "caa657b419ec26f522ac1495e31c7ae44a48f56f9c0b0043b46e8b2541bb0d71"}, {"id": "u041", "kind": "table-row", "locator": "body:L273-L273", "preview": "| Metric | TMEM | SMEM | Registers | | Best for | Multi-stage tensor pipelines | Single-shot small matrix | Scalar/vector ALU |", "sha256": "dd6657befb4735390310472437eab4032360a6c38047bb9215d0541b31a94888"}, {"id": "u042", "kind": "prose", "locator": "body:L275-L275", "preview": "TMEM is **not** a replacement for shared memory. Its strength is in serving as a dedicated accumulator buffer that eliminates register pressure for large MMA tiles. SMEM remains faster for small, frequently accessed data.", "sha256": "d55735304756783f161fc2a2f185184977d8f3eebdb61106cdf3355974d5f9ca"}, {"id": "u043", "kind": "prose", "locator": "body:L279-L279", "preview": "In CUTLASS 4.5.0 for SM100, TMEM is managed through CuTe layouts:", "sha256": "3c2510a426c2a42affeb1b979b4dadf96c79311e9eafadae7bae78c70075697a"}, {"id": "u044", "kind": "code", "locator": "body:L281-L299", "preview": "```python # CuTe-DSL example: TMEM accumulator layout # 128 rows x 256 columns, FP32 tmem_layout = Layout( shape=(128, 256), stride=(256, 1), memory_space=MemorySpace.TMEM ) # Allocate TMEM accumulator acc = tmem_alloc(tmem_layout) # Issue ", "sha256": "0ba0404b185224a19cc9daade2232b2f837eb168feca00b7dc7e5baf806c103a"}, {"id": "u045", "kind": "list-item", "locator": "body:L303-L303", "preview": "1. **Forgetting to fence**: Reading TMEM without `tcgen05.fence::before_thread_sync` produces undefined (stale) values.", "sha256": "143fdc7b7fa7d8fbbab195e1809ac5e8d9075995d6b505dbd5b4babdb8d4c244"}, {"id": "u046", "kind": "list-item", "locator": "body:L304-L304", "preview": "2. **Forgetting to deallocate**: In persistent kernels, TMEM must be freed before re-acquiring tiles. Otherwise, the next allocation will fail or hang.", "sha256": "7940af5f024836477638f77f1b32df4e9ad8c4e97c2a60c8ccd7015e7e6ea257"}, {"id": "u047", "kind": "list-item", "locator": "body:L305-L305", "preview": "3. **Exceeding 512 columns**: Attempting to allocate more than the SM's total column budget silently corrupts data or causes a hang.", "sha256": "c3f1f146f2244341ed1a80ecf77d1dda9e0e195c4bf3a2c273cb8e69e0b593dd"}, {"id": "u048", "kind": "list-item", "locator": "body:L306-L306", "preview": "4. **Cross-warp reads**: A thread can only directly read/write TMEM rows mapped to its own lane. Accessing another warp's rows requires explicit shuffle or SMEM staging.", "sha256": "cdbd1c5e6006a337270f3e5820916627a7192a9e9f7dcc6321bf1afab99de243"}, {"id": "u049", "kind": "list-item", "locator": "body:L307-L307", "preview": "5. **Assuming SMEM-like latency**: TMEM has ~420-cycle latency on cache miss vs ~30 cycles for SMEM. Do not use TMEM for low-latency random access patterns.", "sha256": "288e6789b346c4838d36c5e6de97296c9d7bbdb7d5b52fe7c0fd9afae1ba7f63"}], "confidence_claimed": "verified", "headings": ["Overview", "Architecture Layout", "Row-to-Lane Mapping", "Column Addressing", "Allocation and Deallocation Lifecycle", "Allocation", "Deallocation", "Lifecycle in a Persistent GEMM Kernel", "Data Movement Operations", "TMEM Store (Register to TMEM)", "TMEM Load (TMEM to Register)", "TMEM Zero-Fill", "Bulk SMEM -> TMEM Copy via tcgen05.cp", "Double-Buffering with TMEM", "TMEM Budget Considerations", "Microbenchmark Data", "TMEM in the CUTLASS Abstraction", "Common Pitfalls"], "id": "hw-tmem", "path": "wiki/hardware/tmem.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/prs/cutlass/PR-2139.md", "revision": "ca4fdbea", "url": "https://github.com/NVIDIA/cutlass/pull/2139"}, {"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/blogs/colfax-cutlass-blackwell.md", "url": "https://research.colfax-intl.com/cutlass-tutorial-writing-gemm-kernels-using-tmem-for-nvidia-blackwell-gpus/"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["pr-cutlass-2139", "doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "blog-colfax-cutlass"], "title": "Tensor Memory (TMEM)", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "6ab7a73798de1e059f397018969dab63e94f888f714964e75ff05cd82a5ba7a3", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "DeepGEMM is DeepSeek's open-source FP8 GEMM library providing high-performance matrix multiplication with fine-grained per-tile/per-block scaling. The core kernel is remarkably compact (~300 lines), yet achieves approximately 90% utilizatio", "sha256": "a8d2a63a99bebfefa1d442816e777fd4993a76b2b6891efb7b5cf1c0e86247fa"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "The key innovation is the fine-grained quantization scheme: tile-wise 1x128 scaling for activations and block-wise 128x128 scaling for weights, which prevents outlier values from destroying quantization precision.", "sha256": "c4ba436e75f16ea764cfc683753058a613b3a238e14683fb7cc9ae7664ea5a79"}, {"id": "u003", "kind": "code", "locator": "body:L11-L26", "preview": "``` Activations (tile-wise 1x128): +-----------+-----------+-----------+ | tile 0 | tile 1 | tile 2 | <-- each tile: 1 row x 128 cols | scale: s0 | scale: s1 | scale: s2 | <-- one FP32 scale per tile +-----------+-----------+-----------+ We", "sha256": "d93511b513794a4053872786d7fea8bc68a94d3f20e75bfac7e89b0c22949d71"}, {"id": "u004", "kind": "prose", "locator": "body:L30-L30", "preview": "On Hopper, the Tensor Core accumulator has limited precision (~FP22, not true FP32). DeepGEMM mitigates this by promoting partial sums to a separate FP32 accumulator on CUDA Cores every Nc=128 columns (4 consecutive WGMMA operations).", "sha256": "9c15e3efb8345068a4e571d1f8ff8e56f809ad0c576f270f36448743ce1b21bd"}, {"id": "u005", "kind": "code", "locator": "body:L32-L56", "preview": "```cpp // SM90 path: WGMMA with Nc=128 CUDA Core promotion // Every 4 WGMMAs, promote accumulated result to FP32 CUDA core accumulator constexpr int Nc = 128; // Promotion interval (4 WGMMAs of n=32 each) constexpr int WGMMA_N = 32; float c", "sha256": "76c18f0941acc2599a798e6fb89e18a4662459103586fab64ef904330448ba85"}, {"id": "u006", "kind": "prose", "locator": "body:L58-L58", "preview": "On Blackwell (SM100), the tcgen05.mma instruction with TMEM accumulation uses native UE8M0 block scaling, which eliminates the need for explicit CUDA core promotion.", "sha256": "f573a6c1e1314f88b335649408b8c3740ce346d70a241c625957ac8d9da3c134"}, {"id": "u007", "kind": "code", "locator": "body:L60-L77", "preview": "```cpp // SM100 path: tcgen05.mma with native block scaling // Scaling factors packed as UE8M0 (4 values per uint32) // No explicit CUDA core promotion needed -- TMEM accumulates in full precision // Pack 4 UE8M0 scale factors into a single", "sha256": "293b0c704fe4d4a1d0b6d1b9e1b0e0ddf0d888333d891bef4840d272b11a2924"}, {"id": "u008", "kind": "prose", "locator": "body:L81-L81", "preview": "DeepGEMM provides three grouped GEMM layouts tailored for MoE workloads, where only the M-axis varies (different token counts per expert) while N and K remain fixed:", "sha256": "7241e0e422e17cd845f7f170d2bdd64ee4a1380df37e3cbf7c65b49eb309c456"}, {"id": "u009", "kind": "code", "locator": "body:L83-L97", "preview": "``` Layout 1: Contiguous (prefill) Expert 0: M0 tokens \u2500\u2500\u2510 Expert 1: M1 tokens \u2500\u2500\u2524\u2500\u2500 All packed contiguously in memory Expert 2: M2 tokens \u2500\u2500\u2518 Index array stores cumulative offsets Layout 2: Masked (decode with CUDA graphs) Fixed-size M_max", "sha256": "fc79df6a9d5cdd9a7ece814a283d8e01fa10196c57d975ef3d612e359a48fca5"}, {"id": "u010", "kind": "code", "locator": "body:L99-L120", "preview": "```cpp // Contiguous grouped GEMM dispatch // problem_sizes[i] = {M_i, N, K} for expert i void grouped_gemm_contiguous( const fp8_t* A, // All expert inputs packed const fp8_t* B, // Expert weights [num_experts, N, K] float* C, // Output pa", "sha256": "520bdbcf6f6e8502b04f62de8fce42d70be93ecc020f3c968cd18e86c9f7ee9b"}, {"id": "u011", "kind": "prose", "locator": "body:L124-L124", "preview": "DeepGEMM uses JIT compilation via NVRTC to specialize kernels per problem shape at runtime. This avoids the combinatorial explosion of pre-compiled template instantiations while still achieving optimal register allocation and loop unrolling", "sha256": "94d6fa2e6a32682433878b4e885151f03f04e006f05e66573a94db82ee1cbfcf"}, {"id": "u012", "kind": "code", "locator": "body:L126-L135", "preview": "```cpp // Lightweight JIT module: compile per-shape kernel at first call auto kernel = jit_compile( \"deepgemm_fp8\", {{\"M\", M}, {\"N\", N}, {\"K\", K}, {\"BLOCK_M\", 128}, {\"BLOCK_N\", 128}, {\"BLOCK_K\", 64}, {\"NUM_STAGES\", 4}} ); kernel.launch(A, B", "sha256": "bf4564c2ce415f9cc17b70d95058bf550db89fb8c1ea7fb1d52d06eba47fa1b7"}, {"id": "u013", "kind": "prose", "locator": "body:L139-L139", "preview": "SM90 kernels use NT (non-transposed A, transposed B) layout exclusively. SM100 kernels support all four layout combinations (NT, TN, NN, TT), enabled by tcgen05.mma's flexible operand addressing.", "sha256": "05e8aef45f50114e87d41ef4d00785e0e233b9ce01f96905df75b623787c4f0b"}, {"id": "u014", "kind": "table-row", "locator": "body:L145-L145", "preview": "| GPU | Dtype | Shape | TFLOPS | Utilization | | H800 | FP8 | M=4096, N=4096, K=4096 | 1550 | ~90% |", "sha256": "35557e2b596d7ffc30ecb13e62410388d70dc24b8b8e53f4d5396dfa7dfea217"}, {"id": "u015", "kind": "list-item", "locator": "body:L149-L149", "preview": "- FP8 inference and training where per-tensor quantization loses too much precision", "sha256": "28fc6d00d397c186fce8b4e63d7337236e851101ea5282fe5218d4b457f8367b"}, {"id": "u016", "kind": "list-item", "locator": "body:L150-L150", "preview": "- MoE expert computation with variable token counts per expert", "sha256": "50dcb33f867867ef5ad18456f5f927f4b15d2b162d8a7c0fd03d2ec3acd3e912"}, {"id": "u017", "kind": "list-item", "locator": "body:L151-L151", "preview": "- Situations where fine-grained (1x128 / 128x128) scale granularity is needed", "sha256": "160e602b4b8581b7d13641470ab5613a85b812908295724e6442eecced745c83"}, {"id": "u018", "kind": "list-item", "locator": "body:L155-L155", "preview": "- SM90 path is NT layout only", "sha256": "9242fed0fecfc245df263c39c8b5bbae0d85924bce7de6ac51f5b18c72206f55"}, {"id": "u019", "kind": "list-item", "locator": "body:L156-L156", "preview": "- JIT compilation adds first-call latency (amortized over repeated calls)", "sha256": "6bf333ad0b302cdf27143198ae0a599783cca43214551c605667e4a85464b62f"}, {"id": "u020", "kind": "list-item", "locator": "body:L157-L157", "preview": "- Fine-grained scaling adds overhead vs. per-tensor scaling -- only beneficial when outlier sensitivity matters", "sha256": "1df067738a5ce600f8b91e79dd401fa8ed56018d21a26254c55fc42d93be97a9"}, {"id": "u021", "kind": "list-item", "locator": "body:L161-L161", "preview": "- [DeepGEMM GitHub](https://github.com/deepseek-ai/DeepGEMM)", "sha256": "65a890a7f2ec2e89aca5859abdbc58506d62e5385f789c53eb2add7c496af832"}, {"id": "u022", "kind": "list-item", "locator": "body:L162-L162", "preview": "- [DeepSeek-V3 Technical Report](https://arxiv.org/abs/2412.19437)", "sha256": "3dfa696da6a70082a6d2bcf3de53c8b3f99268e1e5077bb0cc37c01d3ad61f81"}, {"id": "u023", "kind": "prose", "locator": "body:L166-L166", "preview": "Local verbatim upstream code lives in [`artifacts/kernels/deepgemm/full/`](../../artifacts/kernels/deepgemm/full/) (see its `PROVENANCE.yaml` for the pinned upstream SHA and byte-verified SHA-256). Labeled derived variants \u2014 including a nai", "sha256": "feef483a886f122edecb06278ab03c3ebcd3d6ffbaea062e73d8924999c9d783"}, {"id": "u024", "kind": "prose", "locator": "body:L168-L168", "preview": "Query via:", "sha256": "6abd63e508f9320fdd3b2b0cc5b5f6a000f4462abfd17f9ecc3c4818f4a0f875"}, {"id": "u025", "kind": "code", "locator": "body:L170-L172", "preview": "```bash python3 scripts/get_page.py kernel-deepgemm --include-code ```", "sha256": "a2051b59354d2536049b6650555806c0410635aa728921831a30c1e1855977b1"}, {"id": "u026", "kind": "metadata-performance", "locator": "frontmatter:performance_claims", "preview": "[{\"dtype\": \"fp8\", \"gpu\": \"H800\", \"metric\": \"TFLOPS\", \"shape\": \"M=4096, N=4096, K=4096\", \"source_id\": \"blog-deepgemm\", \"utilization\": \"~90%\", \"value\": 1550}]", "sha256": "50acfcee975b0c85babf42af9824ad87b9a57ae6fc4f57ef46f5f9d786e6c456"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Fine-Grained Quantization Scheme", "FP8 Accumulation with Nc=128 CUDA Core Promotion", "MoE Grouped GEMM Layouts", "JIT Compilation", "Memory Layout", "Performance", "When to Use", "Caveats", "Sources", "Full Reference Implementation"], "id": "kernel-deepgemm", "path": "wiki/kernels/deepgemm.md", "performance_claim_count": 1, "resolved_sources": [{"path": "sources/blogs/deepgemm.md", "url": "https://github.com/deepseek-ai/DeepGEMM"}, {"path": "sources/prs/DeepGEMM/PR-304.md", "revision": "7f2a703e", "url": "https://github.com/deepseek-ai/DeepGEMM/pull/304"}, {"path": "sources/prs/cutlass/PR-2139.md", "revision": "ca4fdbea", "url": "https://github.com/NVIDIA/cutlass/pull/2139"}, {"path": "sources/prs/vllm/PR-23696.md", "revision": "074854b2", "url": "https://github.com/vllm-project/vllm/pull/23696"}], "risk_flags": ["code", "performance", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-deepgemm", "pr-deepgemm-304", "pr-cutlass-2139", "pr-vllm-23696"], "title": "DeepGEMM \u2014 FP8 GEMM with Fine-Grained Scaling", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "77d451b38127be1f61669e4237971c0e4da24687371dfdf76f65ec1cafc8ce16", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "FlashAttention-4 is the Blackwell-native evolution of the FlashAttention family, designed to exploit SM100 architectural features that break the Hopper-era bottleneck: tensor core throughput doubles on Blackwell, but SFU (Special Function U", "sha256": "9b0953e53cf887a5e006116e1977892545caa6a73a7b6e557f731ba3580c83ab"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "Written entirely in CuTe DSL (Python), FA4 compiles 20-30x faster than equivalent CUTLASS C++ template code while matching or exceeding cuDNN performance.", "sha256": "52b12af69020638db58d9c24265863227ea3830899d77e8b96c74027c7b4f5e6"}, {"id": "u003", "kind": "prose", "locator": "body:L13-L13", "preview": "Two 128-token query tiles are assigned to each CTA. While one tile's matmul runs on the tensor cores, the other tile's softmax rescaling runs on dedicated warpgroups accessing TMEM. This hides the softmax latency behind MMA compute.", "sha256": "925633f5092e28d0b44f9a3b8bce3149835cadbe619837956bebfe0cbd4d5e26"}, {"id": "u004", "kind": "code", "locator": "body:L15-L44", "preview": "```python # CuTe DSL: Ping-pong tile scheduling (simplified) # Two query tiles per CTA, alternating MMA and softmax phases @cute.kernel def flash_attention_4_fwd(Q, K, V, O, L): # Each CTA processes 2 query tiles of 128 tokens TILE_Q = 128 ", "sha256": "7666e637f6ed7da795bb4dff76027002d544c9b4d934bd69dea26ab64cf75b88"}, {"id": "u005", "kind": "prose", "locator": "body:L48-L48", "preview": "The SFU `ex2` instruction is the bottleneck on Blackwell -- its throughput does not scale with the doubled tensor core rate. FA4 replaces it with a software exponential distributed across FMA units using Cody-Waite range reduction and Horne", "sha256": "2643816f2f447197fe2b11b0c43b92015871517116f19bf78720fd0dd7c5d422"}, {"id": "u006", "kind": "code", "locator": "body:L50-L71", "preview": "```python # Software exp2 via Cody-Waite range reduction + Horner polynomial # Distributes across FMA units instead of using scarce SFU hardware def software_exp2(x): \"\"\" Compute 2^x using FMA units instead of SFU ex2. Cody-Waite range redu", "sha256": "c36af9db7eeb181976fbac7f527f533e979468cb6842f20004d6e803d231f867"}, {"id": "u007", "kind": "prose", "locator": "body:L73-L73", "preview": "This gives roughly 4x throughput improvement over the hardware SFU path by utilizing FMA units that would otherwise be idle during softmax phases.", "sha256": "0e89570d53c2eb1b8a6f2606e4d1ef4e3aac5ee0c441ad4fdc5e9ab92f7381c1"}, {"id": "u008", "kind": "prose", "locator": "body:L77-L77", "preview": "Standard FlashAttention rescales the output accumulator every KV block. FA4 only rescales when the running maximum changes significantly (large jump), reducing non-matmul operations.", "sha256": "97cead2edc1a48624b7a8dc30714d14d48945d350c28c9793dc00ccb0767d45b"}, {"id": "u009", "kind": "code", "locator": "body:L79-L89", "preview": "```python # Only rescale when max changes substantially def conditional_rescale(O_acc, lse_old, lse_new, threshold=2.0): diff = lse_new - lse_old if abs(diff) > threshold: # Full rescale: O_acc *= exp(lse_old - lse_new) scale = software_exp", "sha256": "b320635ad54fadb2cca2dd471dc6f5d60a10f26bc5587445d001b4cac3f4c3c2"}, {"id": "u010", "kind": "prose", "locator": "body:L93-L93", "preview": "The backward pass spans two paired CTAs in a cluster, sharing TMEM across both SMs. This halves shared memory traffic for the dQ/dK/dV gradient computation.", "sha256": "12b84d12d96eed54674f74afaaf70f0aef1766ac205d8b06eb190f7b0b79a0a1"}, {"id": "u011", "kind": "code", "locator": "body:L95-L118", "preview": "```python # 2-CTA backward: paired CTAs share TMEM via 2-SM cooperative mode @cute.kernel def flash_attention_4_bwd(Q, K, V, O, dO, dQ, dK, dV): # Two CTAs cooperate: CTA_0 and CTA_1 in same cluster # tcgen05.mma shape: m256 x n256 x k16 (2", "sha256": "ff7924910f277cb5b3ecff18d818eb589a47ea4855d8c9bc3d5495b877456a65"}, {"id": "u012", "kind": "table-row", "locator": "body:L124-L124", "preview": "| Configuration | GPU | Dtype | TFLOPS | Utilization | vs cuDNN | vs Triton | | seqlen=8192, headdim=128 | B200 | BF16 | 1605 | 71% | 1.1-1.3x | 2.1-2.7x |", "sha256": "6962734852d85358e18b1580efb516e1fb886a8739ce880d0f5179af53aec301"}, {"id": "u013", "kind": "prose", "locator": "body:L126-L126", "preview": "The 71% MMA utilization represents the state of the art for attention kernels on Blackwell. The remaining 29% is consumed by softmax, rescaling, and memory transfers.", "sha256": "f69e67b4954d8634c2b154ac59c7dcbc7850a49a75cebc22e400ab5ea31a060f"}, {"id": "u014", "kind": "list-item", "locator": "body:L130-L130", "preview": "- Written entirely in CuTe DSL (Python), not C++ templates", "sha256": "ddc4f7b4ac032a05d5b1e20a695fe1019481c985ceaa7f3b23edc29eef10d8e6"}, {"id": "u015", "kind": "list-item", "locator": "body:L131-L131", "preview": "- 20-30x faster compilation than equivalent CUTLASS C++ code", "sha256": "47dcc6af677ae0ef013987c83c405a6bd4d69b83f175098b26bdd9b1a9b0a7de"}, {"id": "u016", "kind": "list-item", "locator": "body:L132-L132", "preview": "- Uses `SM100_MMA_SS` atoms for tcgen05 MMA from shared memory", "sha256": "e84f75170575d1a78d5555d626c10b63bb1ca29a98a89d203e35950fddfdf737"}, {"id": "u017", "kind": "list-item", "locator": "body:L133-L133", "preview": "- TMEM locality via `TMEM` locale in CuTe layout", "sha256": "1ae4b375dcf68caefcb340b49545b0da30041f6de9f7c6168d5780104723cd14"}, {"id": "u018", "kind": "list-item", "locator": "body:L134-L134", "preview": "- TMA bulk loads for Q, K, V tiles into shared memory", "sha256": "860c155b25d7768c369088a26464652fdfa6a5579302a77f11d21ee020e01225"}, {"id": "u019", "kind": "list-item", "locator": "body:L138-L138", "preview": "- Standard multi-head attention on Blackwell with sequence lengths >= 1024", "sha256": "59bbc439d9a1636d54941577b6174c932a83ba841e379886de3f4942d71c5102"}, {"id": "u020", "kind": "list-item", "locator": "body:L139-L139", "preview": "- Both forward and backward passes", "sha256": "89fa7a2d45ca8f5ce63af39173e002df2ecc4a408579bce35e8929eab4ce7b10"}, {"id": "u021", "kind": "list-item", "locator": "body:L140-L140", "preview": "- BF16 precision (FP8 support planned)", "sha256": "1105010c1d9bb18ee8a54657462a3467d9a0aed06cb0514e5178354c0bf09b60"}, {"id": "u022", "kind": "list-item", "locator": "body:L144-L144", "preview": "- SM100 only -- no fallback to SM90", "sha256": "3472ae56bcd8f2f143fc6d287d217acc7bdc1f9abf6aa73e67b8b76c843499bc"}, {"id": "u023", "kind": "list-item", "locator": "body:L145-L145", "preview": "- Requires CuTe DSL toolchain (CUTLASS 4.5.0 + Python frontend)", "sha256": "c154f380fc0fe531fc5e3a100c6e3458546b3641e7e3b3dad068f755dfca474f"}, {"id": "u024", "kind": "list-item", "locator": "body:L146-L146", "preview": "- Ping-pong scheduling most effective for headdim=128; smaller headdims may not fully overlap", "sha256": "3c1ef25ac05c585ea24dd1bf2cdc3acc44763db2c246565a124b68b10f2cd978"}, {"id": "u025", "kind": "list-item", "locator": "body:L150-L150", "preview": "- [FlashAttention-4 paper](https://arxiv.org/abs/2603.05451)", "sha256": "9f406a72f6a287a830c318fdac7d4239484228c7e537b1c5b2fa3ba8b0b942bc"}, {"id": "u026", "kind": "list-item", "locator": "body:L151-L151", "preview": "- [Tri Dao's blog](https://tridao.me/blog/2026/flash4/)", "sha256": "a2b86b74ede2bcca82aba3e317b7a1d97380b981b3cc5576bf569933eb6259e9"}, {"id": "u027", "kind": "prose", "locator": "body:L155-L155", "preview": "Local verbatim upstream code lives in [`artifacts/kernels/flash-attention-4/full/`](../../artifacts/kernels/flash-attention-4/full/) (see its `PROVENANCE.yaml` for the pinned upstream SHA and byte-verified SHA-256). Labeled derived variants", "sha256": "c4f50ec45f2fa4675b2ce857375ea3fd0e581ea2d898f2dd5cef280ac619e5de"}, {"id": "u028", "kind": "prose", "locator": "body:L157-L157", "preview": "Query via:", "sha256": "6abd63e508f9320fdd3b2b0cc5b5f6a000f4462abfd17f9ecc3c4818f4a0f875"}, {"id": "u029", "kind": "code", "locator": "body:L159-L161", "preview": "```bash python3 scripts/get_page.py kernel-flash-attention-4 --include-code ```", "sha256": "dbaa5f540bdab5755bcdb9295bc1637f3e10a533091792abfe6b693678d4bd54"}, {"id": "u030", "kind": "metadata-performance", "locator": "frontmatter:performance_claims", "preview": "[{\"dtype\": \"bf16\", \"gpu\": \"B200\", \"metric\": \"TFLOPS\", \"shape\": \"seqlen=8192, headdim=128\", \"source_id\": \"doc-flash-attention-4\", \"utilization\": \"71%\", \"value\": 1605}]", "sha256": "f274c3701fa6d3f96f744a8977af3779a850e9dc415a6471d74e9596a90f54ad"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Key Techniques", "Ping-Pong Scheduling", "Software-Emulated Exponential (Cody-Waite)", "Conditional Softmax Rescaling", "2-CTA Backward Pass", "Performance", "Implementation Notes", "When to Use", "Caveats", "Sources", "Full Reference Implementation"], "id": "kernel-flash-attention-4", "path": "wiki/kernels/flash-attention-4.md", "performance_claim_count": 1, "resolved_sources": [{"path": "sources/docs/flash-attention-4.md", "url": "https://arxiv.org/abs/2603.05451"}, {"path": "sources/blogs/flash-attention-4.md", "url": "https://tridao.me/blog/2026/flash4/"}, {"path": "sources/prs/flashinfer/PR-1850.md", "revision": "f3ea938d", "url": "https://github.com/flashinfer-ai/flashinfer/pull/1850"}], "risk_flags": ["code", "performance", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-flash-attention-4", "blog-flash-attention-4", "pr-flashinfer-1850"], "title": "FlashAttention-4", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "260bbec945acf25551df5b9232d692574dbde84aca2aa0e9b3b73972a38e7239", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L5", "preview": "FlashAttention PR 2441 adds an SM100 CuTe DSL forward path for MLA shapes with top-k sparsity. It is useful when an attention candidate has to combine page/KV layout handling, sparse top-k selection, and tiled forward scheduling.", "sha256": "5b2a5695b8f63367ef67ed87e2c3d9aab1878e749434e3ad69d7b08b5eb1eee3"}, {"id": "u002", "kind": "code", "locator": "body:L7-L16", "preview": "```python # Query pattern before borrowing implementation details: # open PR page, then inspect the source snapshot or upstream files listed there. from pathlib import Path pr_page = Path(\"sources/prs/flash-attention/PR-2441.md\") text = pr_", "sha256": "883d6f46c292014d663c3f52b9471f58468f6614a2b542c9cf01fc833a16ca36"}, {"id": "u003", "kind": "list-item", "locator": "body:L20-L20", "preview": "- Treat top-k gather and tiled attention scheduling as separate evidence paths.", "sha256": "f8429a4f72652ef153895da287a4f4cb3da39e4e739730723cec5c6c7b9d18e5"}, {"id": "u004", "kind": "list-item", "locator": "body:L21-L21", "preview": "- Profile memory traffic separately from tensor-pipe utilization; sparse top-k", "sha256": "32621556168400705292cc982eb59230a0f7d41e759f84276eed20766c1997c7"}, {"id": "u005", "kind": "prose", "locator": "body:L22-L22", "preview": "routing can improve arithmetic work while worsening gather locality.", "sha256": "fb624f89638f7a84774c56c64f1da644eb9b53dfc773d8e2acf9fb0d6bffef9c"}, {"id": "u006", "kind": "list-item", "locator": "body:L23-L23", "preview": "- Keep full-workload validation because the useful path is shape-specific.", "sha256": "67caa0c749179d3793f4380630451f8f5fda453edefd32337a34f45ae1fdb3ef"}, {"id": "u007", "kind": "metadata-performance", "locator": "frontmatter:performance_claims", "preview": "[{\"dtype\": \"bf16\", \"gpu\": \"B200\", \"metric\": \"latency_ms\", \"shape\": \"batch=512, seqlen_q=1, seqlen_k=16384, nheads=128, topk=2048\", \"source_id\": \"pr-flash-attention-2441\", \"value\": 0.3}]", "sha256": "cdce381f09b14524299a1e5659ab44862e8d4f5fade3c85abd6fb6d2d9f64433"}], "confidence_claimed": "source-reported", "headings": ["Shape", "Transfer Notes"], "id": "kernel-flash-attention-sm100-mla-topk", "path": "wiki/kernels/flash-attention-sm100-mla-topk.md", "performance_claim_count": 1, "resolved_sources": [{"path": "sources/prs/flash-attention/PR-2441.md", "revision": "f219c89c", "url": "https://github.com/Dao-AILab/flash-attention/pull/2441"}, {"path": "sources/prs/flash-attention/PR-1236.md", "revision": "a5a75274", "url": "https://github.com/Dao-AILab/flash-attention/pull/1236"}], "risk_flags": ["code", "performance"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["pr-flash-attention-2441", "pr-flash-attention-1236"], "title": "FlashAttention SM100 MLA TopK Sparse Forward", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "997be90a5a387c776857d0d8257d24915a19a1aa829e9d76940c4de57a311691", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "FlashMLA provides high-performance kernels for DeepSeek's Multi-head Latent Attention (MLA) mechanism, which compresses the KV cache from 327-516 KB/token (standard MHA) down to 70 KB/token through a learned low-rank projection into a laten", "sha256": "5c822310054b84ac1323f2fce8c15ca31f81ee4dd10df733fc91c208ad976232"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "FlashMLA includes four kernel variants: dense MLA decoding (SM90), sparse MLA decoding (SM90/SM100), dense MLA prefill (SM100), and sparse MLA prefill (SM90/SM100).", "sha256": "d080df26678f751d47af04d6d608e8549d7d1c383fcb88bf41d032dfe201a1db"}, {"id": "u003", "kind": "prose", "locator": "body:L11-L11", "preview": "Each token in the MLA KV cache occupies 656 bytes:", "sha256": "f2f47735cbdf52ac4acdacbe741fefe753b72fdfc4a247f85043475331a890d6"}, {"id": "u004", "kind": "code", "locator": "body:L13-L24", "preview": "``` Token KV Cache Entry (656 bytes total): +------------------------------------------+ | FP8 compressed KV data | 512 bytes | <-- Latent KV representation | FP32 scaling factors | 16 bytes | <-- Per-head scales | BF16 RoPE embeddings | 12", "sha256": "89d9c266e614ff13a77c505bba3b2e26dfe33b22ad01aba5951c3ed43078fbae"}, {"id": "u005", "kind": "prose", "locator": "body:L28-L28", "preview": "The decode kernel targets memory-bound inference with paged KV cache (block size 64). It achieves up to 3000 GB/s memory bandwidth and 660 TFLOPS on H800.", "sha256": "abd1e2cda37cb941ee97e0a7d13d872eedff8a86329d27448e564bb8fec29009"}, {"id": "u006", "kind": "code", "locator": "body:L30-L87", "preview": "```cpp // Dense MLA decode kernel structure (SM90, BF16) // Memory-bound: bandwidth utilization is the primary metric template __global__ void flashmla_decode_dense( const half* __restrict__ Q, // [batch, num_he", "sha256": "a682b48ee911477e44d4c23e59d9410ba5f33cacb147631ac8a1b32be199d12c"}, {"id": "u007", "kind": "prose", "locator": "body:L91-L91", "preview": "Sparse MLA uses token-level sparsity indices to select only relevant tokens from the KV cache, dramatically reducing memory reads for long sequences.", "sha256": "c190eb957a5b28eac21b8ad97aae18af707091eecd191bae41ee510f32939c85"}, {"id": "u008", "kind": "code", "locator": "body:L93-L118", "preview": "```cpp // Sparse MLA: only attend to selected tokens via indices tensor // Each query has a variable-length list of relevant token indices template __global__ void flashmla_sparse( const half* Q, const int8_t* KV_cache, const", "sha256": "3edadab965d70904078cdd953813465c227e84e1f40629a89ec46d03fee6da7a"}, {"id": "u009", "kind": "prose", "locator": "body:L122-L122", "preview": "The SM100 prefill kernel leverages tcgen05.mma and TMEM for the compute-heavy forward and backward passes, achieving 1460 TFLOPS forward and 1000 TFLOPS backward on B200.", "sha256": "14f5a5c36db81c0a0b73f2012640b0f050dfde1121803b819b299fbbbc675374"}, {"id": "u010", "kind": "code", "locator": "body:L124-L137", "preview": "```cpp // SM100 dense prefill: tcgen05.mma with TMEM accumulation // Uses warp specialization: TMA warps + MMA warps + softmax warps // Forward pass structure: // 1. TMA loads Q, K, V tiles into SMEM // 2. tcgen05.mma computes S = Q @ K^T i", "sha256": "9485e9e48663e357c817de8527441131cebc39f4070505835636b5afa49b05ea"}, {"id": "u011", "kind": "table-row", "locator": "body:L143-L143", "preview": "| Variant | GPU | Dtype | TFLOPS | Bandwidth | | Dense decode | H800 | BF16 | 660 | 3000 GB/s |", "sha256": "197d3af7af1392df6bc60beac97dc6141f4822fae2ed7bef5cb6fc19abbc1182"}, {"id": "u012", "kind": "table-row", "locator": "body:L144-L144", "preview": "| Variant | GPU | Dtype | TFLOPS | Bandwidth | | Sparse decode | H800 | FP8 | 410 | -- |", "sha256": "df77b8afeebe556693f767543d2ce0a5151a56c8635f12a0f93c3b9d5ec3fd10"}, {"id": "u013", "kind": "table-row", "locator": "body:L145-L145", "preview": "| Variant | GPU | Dtype | TFLOPS | Bandwidth | | Sparse decode | B200 | FP8 | 350 | -- |", "sha256": "c6f1624e88ed3a224f4a8c19e26ca427db6be8fed070c8f5a7f6456fccc98d7b"}, {"id": "u014", "kind": "table-row", "locator": "body:L146-L146", "preview": "| Variant | GPU | Dtype | TFLOPS | Bandwidth | | Dense prefill fwd | B200 | BF16 | 1460 | -- |", "sha256": "197f4275f2153525fe6c29856133069bfcf0b2b0b7e185d06926c2faedae1d2e"}, {"id": "u015", "kind": "table-row", "locator": "body:L147-L147", "preview": "| Variant | GPU | Dtype | TFLOPS | Bandwidth | | Dense prefill bwd | B200 | BF16 | 1000 | -- |", "sha256": "cca8784085dc070f755942a7a13864c15372b5d457308daa51abd639a9bda4d3"}, {"id": "u016", "kind": "table-row", "locator": "body:L148-L148", "preview": "| Variant | GPU | Dtype | TFLOPS | Bandwidth | | Sparse prefill | H800 | FP8 | 640 | -- |", "sha256": "b4bdb3f350765356754cd1adc3380b15b1fa7f26b7ded6bdad645a49867bfc3c"}, {"id": "u017", "kind": "table-row", "locator": "body:L149-L149", "preview": "| Variant | GPU | Dtype | TFLOPS | Bandwidth | | Sparse prefill | B200 | FP8 | 1450 | -- |", "sha256": "24d883fc955859e9b99f29c08774ee5b0383ae35b4ff6fd6d0b8f2235623b094"}, {"id": "u018", "kind": "prose", "locator": "body:L153-L153", "preview": "FlashMLA is deployed in production for DeepSeek-V3 and V3.2 inference:", "sha256": "458d3ece0d5d3dfb49c61924d6b9c62fb61899172028d5633c40615d922071ce"}, {"id": "u019", "kind": "list-item", "locator": "body:L154-L154", "preview": "- SGLang and vLLM provide day-0 support", "sha256": "7f6dcb5012cdbeb67f178cc26993598bbbc549996a57905afd1671e721f83a41"}, {"id": "u020", "kind": "list-item", "locator": "body:L155-L155", "preview": "- CUTLASS SM100 includes MLA attention kernels with fused reduction", "sha256": "b11bea427d01afa052da4099cf7ff91b6334ac3f592bc6423db6cb25f28b1573"}, {"id": "u021", "kind": "list-item", "locator": "body:L156-L156", "preview": "- FlashMLA sparse kernels are used by DeepSeek-V3.2-Exp with NSA", "sha256": "ad5f6f5e1aaa252218922911811f597961b522854eb6f84308b2afd22863c881"}, {"id": "u022", "kind": "list-item", "locator": "body:L160-L160", "preview": "- DeepSeek-V3/V3.2 model serving with MLA architecture", "sha256": "46d7045c00210396473c864c2efdbc4a00ee40adf4e347bfc97ff22acdd39b92"}, {"id": "u023", "kind": "list-item", "locator": "body:L161-L161", "preview": "- Long-context inference where KV cache size is the bottleneck", "sha256": "5f3a4c10312c5ec50d36fe8c212bd8edd54082e30e39716126e254db36afd2db"}, {"id": "u024", "kind": "list-item", "locator": "body:L162-L162", "preview": "- Combined with NSA for sparse attention on long sequences", "sha256": "41f2b64b5832328df8e380cfc8567e46dc32bed6d53e5e392ebb59d7442d6874"}, {"id": "u025", "kind": "list-item", "locator": "body:L166-L166", "preview": "- MLA-specific: the latent KV cache format (656 bytes/token) is tied to DeepSeek's architecture", "sha256": "679c0568d6dd56f6266cec07f18dcf6128aeef25cc708cba3145fdd6c8b8c654"}, {"id": "u026", "kind": "list-item", "locator": "body:L167-L167", "preview": "- Dense prefill is SM100 only", "sha256": "874d57764d5289067bfa3b8fd141e8985d2bd436759a00e84f53de002112e306"}, {"id": "u027", "kind": "list-item", "locator": "body:L168-L168", "preview": "- Sparse MLA requires a separate indexing pass to select relevant tokens", "sha256": "47b8d4390270944fea84110df7d2f3cfe2bc863e4ec176706355eae3214707d9"}, {"id": "u028", "kind": "list-item", "locator": "body:L172-L172", "preview": "- [FlashMLA GitHub](https://github.com/deepseek-ai/FlashMLA)", "sha256": "5a106152b29dc94b7e6ce4ef1ab60defc4e6048b0255d0c04ac1850606241fef"}, {"id": "u029", "kind": "list-item", "locator": "body:L173-L173", "preview": "- [DeepSeek-V3 Technical Report](https://arxiv.org/abs/2412.19437)", "sha256": "3dfa696da6a70082a6d2bcf3de53c8b3f99268e1e5077bb0cc37c01d3ad61f81"}, {"id": "u030", "kind": "list-item", "locator": "body:L174-L174", "preview": "- [CUTLASS SM100 Attention Changelog](https://docs.nvidia.com/cutlass/latest/CHANGELOG.html)", "sha256": "664bebe8ae6fce0de8bc7a4b5ba5298ee80b4860fa12b9164bf63edf5917330f"}, {"id": "u031", "kind": "prose", "locator": "body:L178-L178", "preview": "Verbatim upstream code lives in [`artifacts/kernels/flashmla/full/`](../../artifacts/kernels/flashmla/full/); labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` header) live in [`artifacts/", "sha256": "2007b85399ec2dbf6a3ca197e2b2862e13aaf17d7de10e3c535fd62c940da819"}, {"id": "u032", "kind": "prose", "locator": "body:L180-L180", "preview": "Query via:", "sha256": "6abd63e508f9320fdd3b2b0cc5b5f6a000f4462abfd17f9ecc3c4818f4a0f875"}, {"id": "u033", "kind": "code", "locator": "body:L182-L184", "preview": "```bash python3 scripts/get_page.py kernel-flashmla --include-code ```", "sha256": "dadf39bd144b46ea08111137b1703964d608f765fb8cef09eaa6dbb70e0fc8a3"}, {"id": "u034", "kind": "metadata-performance", "locator": "frontmatter:performance_claims", "preview": "[{\"dtype\": \"bf16\", \"gpu\": \"B200\", \"metric\": \"TFLOPS\", \"shape\": \"dense prefill, variable seqlen\", \"source_id\": \"blog-flashmla\", \"utilization\": \"~65%\", \"value\": 1460}, {\"dtype\": \"fp8\", \"gpu\": \"B200\", \"metric\": \"TFLOPS\", \"shape\": \"sparse prefi", "sha256": "e1c7ff2789d15fa0705d455a48d1581f086d21044f3d95740e8332588074f7b0"}], "confidence_claimed": "source-reported", "headings": ["Overview", "MLA KV Cache Layout", "Dense MLA Decoding (SM90)", "Sparse MLA (SM90/SM100)", "Dense Prefill (SM100)", "Performance", "Architecture Integration", "When to Use", "Caveats", "Sources", "Full Reference Implementation"], "id": "kernel-flashmla", "path": "wiki/kernels/flashmla.md", "performance_claim_count": 2, "resolved_sources": [{"path": "sources/blogs/flashmla.md", "url": "https://github.com/deepseek-ai/FlashMLA"}, {"path": "sources/prs/flashinfer/PR-1117.md", "revision": "2f01a9a3", "url": "https://github.com/flashinfer-ai/flashinfer/pull/1117"}, {"path": "sources/prs/vllm/PR-39752.md", "revision": "dc8df110", "url": "https://github.com/vllm-project/vllm/pull/39752"}], "risk_flags": ["code", "performance", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-flashmla", "pr-flashinfer-1117", "pr-vllm-39752"], "title": "FlashMLA \u2014 Multi-head Latent Attention", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "9fc01ec2833f70fa337f1ec2f149ac72e477d8465563010c15b1aa7c0c2bcc50", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "FP8 GEMM with fine-grained block scales (128x128 weights, 1x128 activations). Preserves more dynamic range than per-tensor FP8 scaling, critical for LLM inference and training where outliers dominate quantization error.", "sha256": "9d0666c67654f6a8b7b399172817ee68fa4331b82656fc526f42991227b211ce"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "DeepGEMM is the reference implementation; CUTLASS provides SM100 schedules.", "sha256": "9f5c051584170558eee2213d61be656ab3de3338eb3b298739325bc8680965aa"}, {"id": "u003", "kind": "code", "locator": "body:L11-L20", "preview": "``` Activations: tile-wise 1x128 scales [1x128 values] \u2192 1 scale factor (FP32 or FP8 E4M3) Weights: block-wise 128x128 scales [128x128 values] \u2192 1 scale factor per block Output accumulator: FP32 Multiply A \u00d7 B in FP8, accumulate in FP32, ap", "sha256": "8eb5ef399e8b9c6f8f70ce7545d68e9df8420e15fca16752b39be30dbbefdf17"}, {"id": "u004", "kind": "code", "locator": "body:L24-L44", "preview": "```cuda // Hopper accumulator has ~22-bit precision (FP22) // Every Nc=128 WGMMAs, promote partial sum to FP32 on CUDA cores // This retains precision without adding MMA overhead __device__ void sm90_fp8_gemm_with_promotion(...) { float acc", "sha256": "bcfe9e69d8bdbce92ca0177f0977173672ec5261722f21bf48d33c1596946002"}, {"id": "u005", "kind": "code", "locator": "body:L48-L72", "preview": "```cuda // tcgen05.mma has native UE8M0 block scale support in hardware // No promotion needed - scales applied inside MMA __global__ void sm100_fp8_gemm_block_scale(...) { uint32_t tmem = tmem_alloc(256); for (int k = 0; k < K; k += BLOCK_", "sha256": "6f5d36327a504ae9121188b350d78e12ff711ba4bc1a8dac662530c812afeed8"}, {"id": "u006", "kind": "code", "locator": "body:L76-L83", "preview": "``` A (activations) [M, K] packed FP8 E4M3 sf_a [M, K/128] FP32 or FP8 E4M3 scales # 1 per 1x128 tile B (weights) [N, K] packed FP8 E4M3 sf_b [N/128, K/128] scales # 1 per 128x128 block # or packed UE8M0 format (Blackwell): 4 scales per int", "sha256": "af48fa077e66f75237049c2c04cd0f884f5ddfe555ec71fb7d46503200caebf2"}, {"id": "u007", "kind": "list-item", "locator": "body:L87-L87", "preview": "- DeepGEMM on H800: up to 1550 TFLOPS FP8", "sha256": "6859c2171a09f03e37a28bb9fe652affe1e265eb72ad4b5497203153fc6e1bfc"}, {"id": "u008", "kind": "list-item", "locator": "body:L88-L88", "preview": "- CUTLASS SM100 schedules: similar ratio vs peak", "sha256": "999a256efa100582ef9703dfba65b555bf4d2129b013a3a6c0eef3251ecbb2fa"}, {"id": "u009", "kind": "list-item", "locator": "body:L92-L92", "preview": "- LLM inference with FP8 quantized weights (DeepSeek V3, Qwen2-FP8, etc.)", "sha256": "a5f2d335f2c822920cb1eb495bceb3afcced81451d649f73e7b5d15a485161de"}, {"id": "u010", "kind": "list-item", "locator": "body:L93-L93", "preview": "- Training with FP8 activations (DeepSeek V3 training framework)", "sha256": "834f28701b37f0d2b09395c1852b790bf7be22b36b631118d9085175895ad58e"}, {"id": "u011", "kind": "list-item", "locator": "body:L94-L94", "preview": "- Anywhere per-tensor FP8 accuracy is insufficient due to outliers", "sha256": "6a79703a9a6dcbce374ccaa1cda083dc54062119cc552be713e6a9d47dfb5c28"}, {"id": "u012", "kind": "metadata-performance", "locator": "frontmatter:performance_claims", "preview": "[{\"dtype\": \"fp8\", \"gpu\": \"H800\", \"metric\": \"TFLOPS\", \"shape\": \"M=4096, N=4096, K=4096\", \"source_id\": \"blog-deepgemm\", \"utilization\": \"~90% via CUDA core promotion\", \"value\": 1550}]", "sha256": "e58f133656145745a82198be1f806f85c60c837c993d6c5315743a1e942ceff6"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Block Scaling Structure", "SM90 Path (Hopper, WGMMA)", "SM100 Path (Blackwell, tcgen05.mma)", "Memory Layout", "Performance", "When To Use"], "id": "kernel-fp8-block-scale-gemm", "path": "wiki/kernels/fp8-block-scale-gemm.md", "performance_claim_count": 1, "resolved_sources": [{"path": "sources/blogs/deepgemm.md", "url": "https://github.com/deepseek-ai/DeepGEMM"}, {"path": "sources/docs/nvidia-cutlass-blackwell.md", "url": "https://docs.nvidia.com/cutlass/latest/CHANGELOG.html"}, {"path": "sources/docs/cutlass-changelog-sm100.md", "url": "https://docs.nvidia.com/cutlass/latest/CHANGELOG.html"}], "risk_flags": ["code", "performance"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-deepgemm", "doc-cutlass-blackwell", "doc-cutlass-changelog-sm100"], "title": "FP8 Block-Scale GEMM", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Fused MoE kernels combine the full Mixture-of-Experts forward pass into minimal kernel launches: routing, token dispatch, gate-up dual GEMM, SwiGLU activation, down projection GEMM, and token combine. In unfused implementations this require", "sha256": "ee24c6a94528d9b687123e15c30cb9bc85af105a9220c2a087a1cffd1a6df999"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "This is Track A of the FlashInfer MLSys 2026 contest, targeting B200 GPUs with DeepSeek-V3-style MoE parameters.", "sha256": "0ce0a403e3187cfb048a4b4bd032a997a36e5d2eb9bf023bfdcfabfdf91e3475"}, {"id": "u003", "kind": "code", "locator": "body:L11-L38", "preview": "``` Input tokens x [batch, hidden_dim=7168] | v [Router] top-k=8 experts from 32, grouped (8 groups, top_group=4) | v [Dispatch] Scatter tokens to selected experts | v [Gate-Up Dual GEMM] gate = x @ W_gate [batch_expert, hidden -> intermedi", "sha256": "62635d37ee3ffe039507c5609e99fe6fcb8ff3d2eacd6c14180121e4c35927c0"}, {"id": "u004", "kind": "code", "locator": "body:L42-L61", "preview": "``` Unfused (vLLM): 7 kernel launches 1. Router softmax 2. Top-k selection 3. Token dispatch (scatter) 4. Gate GEMM 5. Up GEMM 6. SiLU + multiply 7. Down GEMM + combine Partially fused (SGLang): 5 kernel launches 1. Router + top-k 2. Dispat", "sha256": "8b22093fd86c8e866d2f333007cbdccd151792645238e50cf50a6b599bf485b9"}, {"id": "u005", "kind": "prose", "locator": "body:L65-L65", "preview": "The gate-up projection fuses two GEMMs with SiLU activation and element-wise multiply:", "sha256": "4b59306fcf06dc1f46198d4256e613181446e46493e5ea2f9236ba7efaa348de"}, {"id": "u006", "kind": "code", "locator": "body:L67-L120", "preview": "```cpp // Fused gate-up: two GEMMs + SiLU + multiply in one kernel // Avoids writing intermediate gate and up results to global memory template __global__ void gated_dual_gemm_fused( c", "sha256": "a0aebecec3a6359fe4f6c486d4b8f0b566c09d388fbe65987c80e883673ed83b"}, {"id": "u007", "kind": "code", "locator": "body:L124-L140", "preview": "```python # FlashInfer API for fused MoE # Single function call replaces 5-7 separate kernel launches import flashinfer output = flashinfer.fused_moe.trtllm_fp8_block_scale_moe( hidden_states=x, # [batch, 7168] BF16 input w_gate_up=w_gate_u", "sha256": "e9d86e62426b268293b219ae569986426a59c613a5e5676c979b87fcb7c06e6b"}, {"id": "u008", "kind": "code", "locator": "body:L144-L188", "preview": "```python import triton import triton.language as tl @triton.jit def fused_moe_gate_up_triton( X_ptr, W_gate_ptr, W_up_ptr, Out_ptr, expert_ids_ptr, token_counts_ptr, sf_x_ptr, sf_gate_ptr, sf_up_ptr, N: tl.constexpr, K: tl.constexpr, BLOCK", "sha256": "491807dde7d9108b5e209a714abf7ce9f0e67487d6c3f1e18db027af801b9ced"}, {"id": "u009", "kind": "table-row", "locator": "body:L194-L194", "preview": "| Framework | Batch 4096 TFLOPS | Batch 1 Latency | Kernel Launches | | SGLang | 1262 | 206.9us | 5 (fused) |", "sha256": "906b943c2d8aa0e0ba3ffc07b94200370981f76ad47e65ae90d00d6f8bc2f491"}, {"id": "u010", "kind": "table-row", "locator": "body:L195-L195", "preview": "| Framework | Batch 4096 TFLOPS | Batch 1 Latency | Kernel Launches | | FlashInfer CuTe DSL | 1225 | 481.9us | 1-2 (fully fused) |", "sha256": "88cf228054f0e17e1f0d38f0b87836e3ce62578cb574a89789f1f097c9b13529"}, {"id": "u011", "kind": "table-row", "locator": "body:L196-L196", "preview": "| Framework | Batch 4096 TFLOPS | Batch 1 Latency | Kernel Launches | | vLLM | 1117 | 369.5us | 7 (unfused) |", "sha256": "71ec18f2c60195f5b60aae9bd1f4ae7fe3e40db27b47e11ea326bdf54d0e8316"}, {"id": "u012", "kind": "list-item", "locator": "body:L200-L200", "preview": "1. **No pre-tuned FP8 MoE config for B200**: Tile sizes and pipeline stages need empirical tuning", "sha256": "8f3b75b2e500d74024e0fd83dbc7f63820eeb0d80333a9028c675a11f5279489"}, {"id": "u013", "kind": "list-item", "locator": "body:L201-L201", "preview": "2. **FP8 numerical overflow**: Block scaling (block size 128) required for stability", "sha256": "95e2f684f53797c33787968fd4432cd5a58bd0ce538a3369394fb05fb7f2d3c1"}, {"id": "u014", "kind": "list-item", "locator": "body:L202-L202", "preview": "3. **Batch-size sensitivity**: batch=1 is latency-critical (kernel launch overhead dominates); batch=4096 is throughput-critical", "sha256": "0de20c75dfbd49ffacd287c5357ffac0b79fb1de378d3be742253f82b95acaa9"}, {"id": "u015", "kind": "list-item", "locator": "body:L203-L203", "preview": "4. **Expert load imbalance**: Variable token counts per expert cause tail effects", "sha256": "c6c51359190b264deae32b98bfc662f48301944cfea12407b75356aa653dffbe"}, {"id": "u016", "kind": "list-item", "locator": "body:L204-L204", "preview": "5. **TMA alignment**: 128-byte alignment required for all TMA descriptors", "sha256": "5531f661e1063a34d7576d65d13c26c28657c064b7e35fa7e346dbe65bf5a142"}, {"id": "u017", "kind": "list-item", "locator": "body:L205-L205", "preview": "6. **Dual TMEM allocation**: Gate and up accumulators each need TMEM space, competing for the 256KB budget", "sha256": "066ba71d4b96790d24fbd1ffc8b2c63a9a9a4d1f1bcdf6133f801b03fa73d423"}, {"id": "u018", "kind": "list-item", "locator": "body:L209-L209", "preview": "- MoE model inference (DeepSeek-V3, Mixtral, etc.)", "sha256": "5d1ea69068eb0a04307c47b71fe5539280efc4e746443ba47ac1e5bb959279df"}, {"id": "u019", "kind": "list-item", "locator": "body:L210-L210", "preview": "- Both prefill (high batch, throughput-critical) and decode (low batch, latency-critical)", "sha256": "856ba607570f4fc39c8b29e12922fa8cdf63bdde2a22f5d3c6e5ddb7f83d5ef2"}, {"id": "u020", "kind": "list-item", "locator": "body:L211-L211", "preview": "- When gate-up GEMM fusion provides measurable speedup over separate launches", "sha256": "1d26cea2436827bc9053392406b7119b175c05ffe71460d733eabfcac9ef9e7b"}, {"id": "u021", "kind": "list-item", "locator": "body:L215-L215", "preview": "- Full fusion (routing through combine) is extremely complex to implement correctly", "sha256": "6dd0019145f486ab8af5170962abaace419c0a60183f19fa5e765a5b881d86af"}, {"id": "u022", "kind": "list-item", "locator": "body:L216-L216", "preview": "- Expert load imbalance is the primary practical bottleneck", "sha256": "847799de8b5ec1b7f71f51131ef94f3b3619062780d84483dc8f0386545425a2"}, {"id": "u023", "kind": "list-item", "locator": "body:L217-L217", "preview": "- CUDA graph compatibility requires masked layout (fixed allocation per expert)", "sha256": "e480662de15ee90e9ce8636aba44ff3678b756e86939ccb07a3adc8f462b5b97"}, {"id": "u024", "kind": "list-item", "locator": "body:L218-L218", "preview": "- Small expert token counts cause thin-GEMM inefficiency on tensor cores", "sha256": "4e733045fb9625e393f2fca392c080c70d021114ecc3922f24958a4ad5666ee0"}, {"id": "u025", "kind": "list-item", "locator": "body:L219-L219", "preview": "- FP8 block scaling adds memory overhead for scale factor storage", "sha256": "a6140763ac26f6b64b8b51bfdec3c31f59cf98b631f4a91727603f8bbb545f92"}, {"id": "u026", "kind": "list-item", "locator": "body:L223-L223", "preview": "- [FlashInfer MLSys 2026 Contest](https://mlsys26.flashinfer.ai/)", "sha256": "0a5ff847c139fc581dea1c62769bff7d8f8ce355ab890d4889cbc20b253a535b"}, {"id": "u027", "kind": "list-item", "locator": "body:L224-L224", "preview": "- [GPU Mode NVFP4 Hackathon Problem 3](https://github.com/gpu-mode/reference-kernels)", "sha256": "fb49f6aa44f7585c557194274331f5706fe43e629eae7b63a8f90602ba329e7f"}, {"id": "u028", "kind": "list-item", "locator": "body:L225-L225", "preview": "- [DeepGEMM MoE](https://github.com/deepseek-ai/DeepGEMM)", "sha256": "011219b565377c3108be875324294ae1d60076b56629632da1aded4d63eaf0fb"}, {"id": "u029", "kind": "list-item", "locator": "body:L226-L226", "preview": "- [SGLang Fused MoE](https://github.com/sgl-project/sglang)", "sha256": "de1eb8863fba8e60a30e383546024c553880110c755386e0aad7e6bd18350b0e"}, {"id": "u030", "kind": "prose", "locator": "body:L230-L230", "preview": "Verbatim upstream code lives in [`artifacts/kernels/fused-moe/full/`](../../artifacts/kernels/fused-moe/full/); labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` header) live in [`artifact", "sha256": "0d8251a8098e0338630dddf7b941e01d3eda9ac09b7a1de08a3624ada1d2bd57"}, {"id": "u031", "kind": "prose", "locator": "body:L232-L232", "preview": "Query via:", "sha256": "6abd63e508f9320fdd3b2b0cc5b5f6a000f4462abfd17f9ecc3c4818f4a0f875"}, {"id": "u032", "kind": "code", "locator": "body:L234-L236", "preview": "```bash python3 scripts/get_page.py kernel-fused-moe --include-code ```", "sha256": "56fc790420908106565f83ed31ab8a2fe1aa1e849fce411f2babc6c449629082"}, {"id": "u033", "kind": "metadata-performance", "locator": "frontmatter:performance_claims", "preview": "[{\"dtype\": \"fp8\", \"gpu\": \"B200\", \"metric\": \"TFLOPS\", \"shape\": \"topk=8, experts=32, hidden=7168, intermediate=2048, batch=4096\", \"source_id\": \"contest-flashinfer-track-a\", \"utilization\": \"~56%\", \"value\": 1262}]", "sha256": "74cedc63a3ce3ac5e1f1aea44e509eb2ad07d8306383c08c3579b6a91ed2d6c3"}], "confidence_claimed": "source-reported", "headings": ["Overview", "MoE Forward Pass Structure", "Kernel Fusion Strategy", "Gated Dual GEMM Kernel (Hackathon Problem 3)", "FP8 Block-Scale MoE (FlashInfer API)", "Triton Fused Gate-Up Kernel", "Framework Baselines (B200)", "Challenges", "When to Use", "Caveats", "Sources", "Full Reference Implementation"], "id": "kernel-fused-moe", "path": "wiki/kernels/fused-moe.md", "performance_claim_count": 1, "resolved_sources": [{"path": "sources/contests/flashinfer-mlsys26/track-a-fused-moe.md", "url": "https://mlsys26.flashinfer.ai/"}, {"path": "sources/blogs/deepgemm.md", "url": "https://github.com/deepseek-ai/DeepGEMM"}, {"path": "sources/prs/vllm/PR-23696.md", "revision": "074854b2", "url": "https://github.com/vllm-project/vllm/pull/23696"}], "risk_flags": ["code", "ordering", "performance", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["contest-flashinfer-track-a", "blog-deepgemm", "pr-vllm-23696"], "title": "Fused MoE \u2014 FP8 Block-Scale Routing + Dual GEMM", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "af98b7d75e33581067427cda67ff21fbf228821cc926241a7539b6215cd781ec", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Gated Delta Networks (GatedDeltaNet) replace standard O(n^2) attention with an O(n) linear attention mechanism that uses a delta rule for error-correcting memory updates and exponential gating for adaptive decay. Published at ICLR 2025 by N", "sha256": "c5725fe2643100d15e27a593fa7bcb80719b1fcd30767899f8c849bd40a40d30"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "The key advantage is O(1) per-token cost during decoding: the recurrent state is a fixed-size matrix that gets updated with each new token, eliminating the KV cache growth problem entirely.", "sha256": "d3a0655b8a5271586db4b84b18cf492938aea0ceda8b65826cae7173976918b4"}, {"id": "u003", "kind": "code", "locator": "body:L11-L21", "preview": "``` Qwen3-Next-80B: 48 layers Pattern: 12 x (3 x [GatedDeltaNet -> MoE] -> [Full Attention -> MoE]) Layer distribution: - 36 GatedDeltaNet layers (75%): O(n) linear attention - 12 Full Attention layers (25%): Standard GQA for global retriev", "sha256": "bdb49253be1645257c57e3919dd226dce437718dba3b4ea8537557e4ba2a6442"}, {"id": "u004", "kind": "prose", "locator": "body:L25-L25", "preview": "Unlike standard linear attention (which uses simple additive updates to the recurrent state), the delta rule performs targeted error-correcting updates:", "sha256": "01f4db44a83587d916796232922fd9e7c2c320b8bc2d983b3ed5586501478919"}, {"id": "u005", "kind": "code", "locator": "body:L27-L58", "preview": "```python # Standard linear attention (additive): # S_t = S_{t-1} + v_t @ k_t^T # # Delta rule (error-correcting): # S_t = S_{t-1} + (v_t - S_{t-1} @ k_t) @ k_t^T # ^^^^^^^^^^^^^^^^^^^^^^^^ # Error correction term def delta_rule_step(S, k, ", "sha256": "7e861801c69ff304b9a9e2e7735938e83ed8c96c1783a1f73ceaeaba8a42774b"}, {"id": "u006", "kind": "prose", "locator": "body:L62-L62", "preview": "During prefill, sequences are divided into chunks that can be processed in parallel. Within each chunk, the inter-token dependencies are resolved via a causal linear recurrence; across chunks, the recurrent state is propagated sequentially.", "sha256": "36052b0d9826a837d97158c09f43a728ea3f30dad1ce51c39514fcf7b2a63903"}, {"id": "u007", "kind": "code", "locator": "body:L64-L121", "preview": "```python import triton import triton.language as tl @triton.jit def gated_delta_net_chunk_fwd( Q_ptr, K_ptr, V_ptr, Beta_ptr, O_ptr, State_ptr, SEQ_LEN: tl.constexpr, CHUNK_SIZE: tl.constexpr, # e.g., 64 or 128 D_QK: tl.constexpr, # qk_dim", "sha256": "c365a7c616d5094fa68e774b6c173f6314b11f48738a0509e44285df8d4040c2"}, {"id": "u008", "kind": "prose", "locator": "body:L125-L125", "preview": "During autoregressive decoding, each new token only requires one state update -- O(1) per token regardless of context length.", "sha256": "584acb25f6c610eda43934bed141e537403a5a2fdbc8567bce8ef58e7dbf728c"}, {"id": "u009", "kind": "code", "locator": "body:L127-L167", "preview": "```python @triton.jit def gated_delta_net_decode( Q_ptr, K_ptr, V_ptr, Beta_ptr, O_ptr, State_ptr, D_QK: tl.constexpr, D_V: tl.constexpr, ): \"\"\" Single-token decode: O(1) per token. The recurrent state replaces the KV cache entirely. State ", "sha256": "29466cfc078905f19d8672c1858fdba4831aaaac152c66c96acf4d2e9bba8b71"}, {"id": "u010", "kind": "prose", "locator": "body:L171-L171", "preview": "TFLA adds a second level of tiling within chunks, enabling arbitrarily large chunk sizes. It emits matmuls as inline PTX assembly for both Hopper (WGMMA) and Blackwell (tcgen05).", "sha256": "141918ec518be3497594c182d87e3f96da06efc18770d6b0f951d1e64ddfd0e2"}, {"id": "u011", "kind": "code", "locator": "body:L173-L184", "preview": "```cpp // TFLA: Inline PTX for Blackwell tcgen05 matmul within chunk tiles // Two levels of parallelism: standard chunkwise + tiling within chunks // SM100 path: tcgen05.mma for intra-chunk matrix operations asm volatile( \"tcgen05.mma.cta_g", "sha256": "e00f483ed8cb2a4a8494941a64efdb2c085cde36fe4f47681dc8ad51f7cf1d10"}, {"id": "u012", "kind": "prose", "locator": "body:L188-L188", "preview": "Two main implementations exist:", "sha256": "eabbb511c86ae0ecf8abe526ed89a206f21c3b79bd5260192624f74303f8c0ff"}, {"id": "u013", "kind": "table-row", "locator": "body:L192-L192", "preview": "| Implementation | Source | Notes | | NVlabs/GatedDeltaNet | Reference | Triton kernels, research quality |", "sha256": "0ad7015db76102906174f7a76bab64f291251b1dc1a8cf4e65b103cc9a5751c1"}, {"id": "u014", "kind": "table-row", "locator": "body:L193-L193", "preview": "| Implementation | Source | Notes | | FLA (Flash Linear Attention) | Recommended | Optimized, significantly faster, variable-length support |", "sha256": "5793e5c73e248feab6df19b4046042277823265032b48a2d3ca354fe00b0b2ab"}, {"id": "u015", "kind": "prose", "locator": "body:L197-L197", "preview": "GatedDeltaNet is Track C of the FlashInfer MLSys 2026 contest:", "sha256": "0f1edd6d3bf4d4d8b77fe158f93a12a67ad13bacf26f39473bd26bc4473b1dc1"}, {"id": "u016", "kind": "list-item", "locator": "body:L198-L198", "preview": "- Parameters: qk_dim=4, v_dim=8, d=128", "sha256": "fcf6818917c1f0405e859f96669e64d0434038bda1b1b9a8f767f2f18470180a"}, {"id": "u017", "kind": "list-item", "locator": "body:L199-L199", "preview": "- Benchmarks: decode `qk4_v8_d128_k_last`, prefill `qk4_v8_d128_k_last`", "sha256": "8a2eb9edbf1cdd5dcba20bb331061e4afb030ff8bc2e62bfd1f102ba62fc0f95"}, {"id": "u018", "kind": "list-item", "locator": "body:L200-L200", "preview": "- Status: decode done for both Hopper and Blackwell; prefill done on Hopper, in progress for Blackwell", "sha256": "1665a15d3ebec4d98118c4dc57d776d68a8f4a9b3561f56328b8828cd0110cfa"}, {"id": "u019", "kind": "list-item", "locator": "body:L204-L204", "preview": "- Long-context inference (32K+) where O(n) scaling provides major throughput gains", "sha256": "03615f82a34334e6233d9c7b840ffa95a2dbc73fa9498a55d3fff367d82fe25d"}, {"id": "u020", "kind": "list-item", "locator": "body:L205-L205", "preview": "- Hybrid architectures combining linear attention (75% of layers) with full attention (25%)", "sha256": "4bfa4d21b1489298693160c40b893f9763a2618972dd66903051447866e29fbb"}, {"id": "u021", "kind": "list-item", "locator": "body:L206-L206", "preview": "- Streaming decode where O(1) per-token cost eliminates KV cache growth", "sha256": "9c8a112be1fb768308e8d6bd51ac1c424922354d3d837e847adf1ccd4a860bda"}, {"id": "u022", "kind": "list-item", "locator": "body:L210-L210", "preview": "- Triton-based kernels have CPU launch overhead impacting small decode batches; use CUDA graph mode via vLLM", "sha256": "a1d940680167fee8177f2fe4684e3ffafe7cc1d899b0510ccb2f729a9f075805"}, {"id": "u023", "kind": "list-item", "locator": "body:L211-L211", "preview": "- Recurrent state size (D_QK * D_V per head) can be large -- 512K floats for typical configs", "sha256": "3ea7a925bf984535777f727b38f251677496296978745f0b324f66bfc52a081d"}, {"id": "u024", "kind": "list-item", "locator": "body:L212-L212", "preview": "- Quality depends on the learned gating; not a drop-in replacement for standard attention without retraining", "sha256": "17296706a743618938e197f6c963950ee2537e8c7cba1acaf384b51a7ebf01d0"}, {"id": "u025", "kind": "list-item", "locator": "body:L213-L213", "preview": "- Attention output gating (in Qwen3.5) is required to eliminate Attention Sink and Massive Activation problems", "sha256": "173ea344150d5417420fd38bba1cb37c320e8c70e30017929f15823357a9c7aa"}, {"id": "u026", "kind": "list-item", "locator": "body:L217-L217", "preview": "- [GatedDeltaNet (ICLR 2025)](https://github.com/NVlabs/GatedDeltaNet)", "sha256": "a67fe6e47713f3c0a3ea8732a757fcee02c7c5b0a7f602aacb419e55a34bf5d9"}, {"id": "u027", "kind": "list-item", "locator": "body:L218-L218", "preview": "- [TFLA paper](https://arxiv.org/abs/2503.14376)", "sha256": "939543a902be4abe15f2e3068bdc494fae3b1d7a84131fba137df1f5a7af9b39"}, {"id": "u028", "kind": "list-item", "locator": "body:L219-L219", "preview": "- [Qwen3-Next NVIDIA blog](https://developer.nvidia.com/blog/new-open-source-qwen3-next-models-preview-hybrid-moe-architecture-delivering-improved-accuracy-and-accelerated-parallel-processing-across-nvidia-platform/)", "sha256": "8e5247145edfdcc925b581aa5cd69e4aced1787092c5c7afe84274900da957de"}, {"id": "u029", "kind": "list-item", "locator": "body:L220-L220", "preview": "- [FlashInfer MLSys 2026 Contest](https://mlsys26.flashinfer.ai/)", "sha256": "0a5ff847c139fc581dea1c62769bff7d8f8ce355ab890d4889cbc20b253a535b"}, {"id": "u030", "kind": "prose", "locator": "body:L224-L224", "preview": "Verbatim upstream code lives in [`artifacts/kernels/gated-delta-net/full/`](../../artifacts/kernels/gated-delta-net/full/); labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` header) live i", "sha256": "6ae5cca417d32aa0f82d0a726cba8ee04b5abf13a02242148316781ee66b440e"}, {"id": "u031", "kind": "prose", "locator": "body:L226-L226", "preview": "Query via:", "sha256": "6abd63e508f9320fdd3b2b0cc5b5f6a000f4462abfd17f9ecc3c4818f4a0f875"}, {"id": "u032", "kind": "code", "locator": "body:L228-L230", "preview": "```bash python3 scripts/get_page.py kernel-gated-delta-net --include-code ```", "sha256": "39378634eb01b82263ac1e7155820b2bf1f5eb4057c51fca181d3283db3328df"}, {"id": "u033", "kind": "metadata-performance", "locator": "frontmatter:performance_claims", "preview": "[{\"dtype\": \"bf16\", \"gpu\": \"H100\", \"metric\": \"speedup\", \"shape\": \"seqlen=8192, qk_dim=4, v_dim=8, d=128\", \"source_id\": \"blog-gated-delta-net\", \"utilization\": \"vs Qwen3-32B at 32K+ context, O(n) linear complexity\", \"value\": 10}]", "sha256": "99c6c7214a5abb7b19fd5d9eba01724d26ecfb52c24025e97486541606658411"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Architecture in Qwen3-Next", "Delta Rule Mechanism", "Chunk-Based Parallel Prefill", "Streaming Decode Kernel", "TFLA: Tiled Flash Linear Attention", "Implementations", "FlashInfer MLSys 2026 Contest", "When to Use", "Caveats", "Sources", "Full Reference Implementation"], "id": "kernel-gated-delta-net", "path": "wiki/kernels/gated-delta-net.md", "performance_claim_count": 1, "resolved_sources": [{"path": "sources/blogs/gated-delta-net.md", "url": "https://github.com/NVlabs/GatedDeltaNet"}, {"path": "sources/docs/tfla.md", "url": "https://arxiv.org/abs/2503.14376"}, {"path": "sources/prs/vllm/PR-37303.md", "revision": "e1d85e5c", "url": "https://github.com/vllm-project/vllm/pull/37303"}], "risk_flags": ["code", "performance", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-gated-delta-net", "doc-tfla", "pr-vllm-37303"], "title": "Gated Delta Net \u2014 Linear Attention", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "4aa8460e6ef7388eb59cdd6643cc8a05fecb149f9148104b8eec852b369e7373", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Gated dual GEMM fuses two matrix multiplications with activation and elementwise operations \u2014 the canonical MLP gate-up pattern used by LLaMA, Qwen, DeepSeek, and most modern LLMs. Fusion eliminates two global memory roundtrips compared to ", "sha256": "ecd04db3793e2b25272be2e1fa9603ac7e25f7da771d277a872dbd05239e5cf4"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "This was Problem 3 of the GPU Mode NVFP4 Hackathon.", "sha256": "ead2e2b9b70b06f0b9121b6a49ee12100a413d507ce839f998be6c61da4ca549"}, {"id": "u003", "kind": "code", "locator": "body:L11-L21", "preview": "``` Given x, W_gate, W_up (weights shared same K dimension) Standard (unfused): gate = x @ W_gate (GEMM 1: reads x, W_gate, writes gate) up = x @ W_up (GEMM 2: reads x, W_up, writes up) silu = gate * sigmoid(gate) (elementwise: reads gate, ", "sha256": "91396f6a6bc8a1432fc70407c8abea6e99f57e9f1a81f6a29107035fcc68a70f"}, {"id": "u004", "kind": "code", "locator": "body:L25-L58", "preview": "```cuda template __global__ void gated_dual_gemm_nvfp4( const nvfp4_t* __restrict__ X, // [M, K] input const nvfp4_t* __restrict__ W_gate, // [N, K] gate weights const nvfp4_t* __restr", "sha256": "6d719686a6a9f441bb8304fdc4964c3e61755f1ed392540a4d3e5d03bfebffb0"}, {"id": "u005", "kind": "list-item", "locator": "body:L62-L62", "preview": "1. **X reuse**: Same X tile feeds both MMAs \u2014 loaded once, used twice", "sha256": "e62b96693cccdfb2308e10bbfb3fce0020a939dc9cb849dddfb6e0cf8af64784"}, {"id": "u006", "kind": "list-item", "locator": "body:L63-L63", "preview": "2. **TMEM dual accumulator**: Blackwell's 512-column TMEM fits two 256-col accumulators side-by-side", "sha256": "d9110adf602d0bcdd1c08c3dd9ba7f4c207d7f00919ff5273d78b1c843c0e9d8"}, {"id": "u007", "kind": "list-item", "locator": "body:L64-L64", "preview": "3. **Fused epilogue**: SiLU and multiply happen after TMEM load, no intermediate SMEM", "sha256": "43e2dad7ce262ff0feeab03d9ce1fae54344ae0c313706e90a1db3a614979f71"}, {"id": "u008", "kind": "list-item", "locator": "body:L65-L65", "preview": "4. **Shared SFA**: X's block scales apply to both gate and up computations", "sha256": "357df219c44e5de2c4437beb097a208833e57a091041d49566a37cea578578ab"}, {"id": "u009", "kind": "list-item", "locator": "body:L69-L69", "preview": "- MLP layers in modern LLMs (LLaMA, Qwen, DeepSeek, Mistral)", "sha256": "adebf22f795cda0c8ad652b0af4b770d97ef97c38ff79e299936c646d0e6178c"}, {"id": "u010", "kind": "list-item", "locator": "body:L70-L70", "preview": "- Any dual-output operation sharing one input", "sha256": "5107defe1e739fe0f45ca2298bce97cfa49057c2e50ae751c258a7b16609f6c3"}, {"id": "u011", "kind": "list-item", "locator": "body:L71-L71", "preview": "- MoE expert computations (expand to per-expert fused kernels)", "sha256": "981ebff4fefddaf7bfcc5b35425ceeeba547586380df6d6ddf505ce7288d324e"}, {"id": "u012", "kind": "prose", "locator": "body:L75-L75", "preview": "The reference bundle lives in [`artifacts/kernels/gated-dual-gemm/full/`](../../artifacts/kernels/gated-dual-gemm/full/) and combines the upstream vLLM PR-23696 diff (`vllm-PR-23696-gated-dual-gemm.patch`, `mode: upstream-patch`) with an ex", "sha256": "c1f11acc6a93b91b31a6abb047ffe2a46620ae7935dd689949f31843863d15f4"}, {"id": "u013", "kind": "prose", "locator": "body:L77-L77", "preview": "Query via:", "sha256": "6abd63e508f9320fdd3b2b0cc5b5f6a000f4462abfd17f9ecc3c4818f4a0f875"}, {"id": "u014", "kind": "code", "locator": "body:L79-L81", "preview": "```bash python3 scripts/get_page.py kernel-gated-dual-gemm --include-code ```", "sha256": "68a44b9bd282d463bccca115a4a7aca25d430f9c7e19334972410593a6c18bbc"}, {"id": "u015", "kind": "metadata-performance", "locator": "frontmatter:performance_claims", "preview": "[{\"dtype\": \"nvfp4\", \"gpu\": \"B200\", \"metric\": \"latency_us\", \"shape\": \"M=1024 N=2*2048 K=7168 (gate-up MLP)\", \"source_id\": \"contest-gpumode-p3\", \"utilization\": \"compute-bound\", \"value\": 18.5}]", "sha256": "23184d2163865e55e7ebac127fa9c4bacff22324230e2727fa61d9f8eb3e8b90"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Fused Operation", "Kernel Structure (Blackwell)", "Key Optimizations", "When To Use", "Full Reference Implementation"], "id": "kernel-gated-dual-gemm", "path": "wiki/kernels/gated-dual-gemm.md", "performance_claim_count": 1, "resolved_sources": [{"path": "sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md", "url": "https://github.com/gpu-mode/reference-kernels"}, {"path": "sources/blogs/deepgemm.md", "url": "https://github.com/deepseek-ai/DeepGEMM"}, {"path": "sources/blogs/tflops-gap-fp4-moe.md", "url": "https://huggingface.co/blog/apsys/blackwell-nvfp4-comparison"}, {"path": "sources/prs/vllm/PR-23696.md", "revision": "074854b2", "url": "https://github.com/vllm-project/vllm/pull/23696"}], "risk_flags": ["code", "performance"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["contest-gpumode-p3", "blog-deepgemm", "blog-tflops-gap-fp4-moe", "pr-vllm-23696"], "title": "Gated Dual GEMM (Gate-Up + SwiGLU Fusion)", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "216f09e31b3f73503470760148850014df1540ad380997383317930a7b459e9e", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Grouped GEMM computes multiple matrix multiplications with variable M dimensions but shared N and K, directly targeting MoE (Mixture of Experts) inference where each expert processes a different number of tokens. This is the most practicall", "sha256": "fe725fe4c978ef7a2a3d83247a579411e1923af85b55dee827a441009e19fe03"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "Grouped GEMM was Problem 4 (heaviest weight: 40%) of the GPU Mode NVFP4 Hackathon, and is also the core of DeepGEMM's MoE support.", "sha256": "d122944985199c2459f3c27772eb790b7652db25acb67f563c1d9e8249b4c960"}, {"id": "u003", "kind": "code", "locator": "body:L11-L25", "preview": "``` Standard GEMM: C = A @ B (single problem) Grouped GEMM: C_i = A_i @ B_i for i in [0, num_groups) MoE specialization: - N and K are FIXED (same expert architecture) - Only M varies (different token counts per expert) - B_i may be differe", "sha256": "94c025828b3055487d13520494f11cf1aa4a2c42837806f259d6f0f1d129b7bf"}, {"id": "u004", "kind": "prose", "locator": "body:L29-L29", "preview": "DeepGEMM provides three layouts optimized for different MoE phases:", "sha256": "4562529a5f1fefcd4c30c2507843fab086aa2ed20508bc4ce4c7f65665db458c"}, {"id": "u005", "kind": "code", "locator": "body:L31-L61", "preview": "```cpp // Layout 1: Contiguous (prefill) // All expert inputs packed sequentially with cumulative offset array // Memory: [Expert0 (M0 rows)] [Expert1 (M1 rows)] [Expert2 (M2 rows)]... // Index: offsets[0]=0 offsets[1]=M0 offsets[2]=M0+M1 s", "sha256": "2ffacb9913c03845c0e35c537d7c276d96415c2883121c20e2b9b0c95739d559"}, {"id": "u006", "kind": "code", "locator": "body:L65-L92", "preview": "```cpp // CUTLASS schedule for grouped GEMM on Blackwell using Schedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100; // PtrArray mode: array of pointers to per-group A, B, C matrices // TMA handles variable-offset loads vi", "sha256": "6cbdd8096dbe2fd2bebf3f8b7f959e97d6a26a7d228a5adaaaa5d1bfd3b9a6e5"}, {"id": "u007", "kind": "code", "locator": "body:L98-L117", "preview": "```cpp // Precompute tile-to-expert mapping on host before launch struct TileInfo { int expert_id; int tile_m_start; // Local M offset within this expert int tile_n_start; }; std::vector build_tile_schedule( const int* M_per_exper", "sha256": "0025b78e903a581b71ea67749d649624aca073b7cb362412b042c42bd01a93ad"}, {"id": "u008", "kind": "code", "locator": "body:L121-L151", "preview": "```cpp // Persistent kernel with atomic tile counter // Each thread block loops, grabbing tiles until all are processed __device__ int g_tile_counter = 0; __global__ void grouped_gemm_persistent( const fp8_t** A_ptrs, const fp8_t** B_ptrs, ", "sha256": "e5a0978df32a9382e83d7bd56a5cfa8ca3a95447f8bb344b5b8200fcd978d31b"}, {"id": "u009", "kind": "prose", "locator": "body:L155-L155", "preview": "The 1st-place submission to Problem 4 exploited the evaluation harness:", "sha256": "b2053084a52810c3f26740bb9ec3ce6cdb8771fe7d6d59dc1fb2425cf9133a5d"}, {"id": "u010", "kind": "code", "locator": "body:L157-L162", "preview": "``` Correctness phase: harness clones data -> real kernel runs correctly Timing phase: harness reuses same objects -> Call 1: fires 120-group super-batch (all 15 benchmark problems fused) Calls 2-15: detect pre-computed results, skip comput", "sha256": "95bc2baa6208a96a7ac2832c87fed81fb5b2374aa74667629b9966f567f5cc57"}, {"id": "u011", "kind": "prose", "locator": "body:L164-L164", "preview": "This reported 11.191us (~2us ahead of second place). It led to improvements in the FlashInfer-Bench evaluation methodology for the MLSys 2026 contest.", "sha256": "4ce120f237a07e0663244cd3ad5c69a13f5a4eb2a6ba5c1ae0ce04e83463365b"}, {"id": "u012", "kind": "list-item", "locator": "body:L168-L168", "preview": "- MoE inference: dispatch tokens to experts, compute per-expert projections", "sha256": "7d3cd305e13d90e277fe87d5e368ff6aa2bd87287bd378cc6046ba4f5381307d"}, {"id": "u013", "kind": "list-item", "locator": "body:L169-L169", "preview": "- Any workload with multiple GEMMs sharing N and K but varying M", "sha256": "1540d11f65836862124c49b17b8eeeb03923cf917710d15837bc6ba53d6373ec"}, {"id": "u014", "kind": "list-item", "locator": "body:L170-L170", "preview": "- Prefill (contiguous layout) and decode (masked layout for CUDA graph compatibility)", "sha256": "9ac021fc8424bb238df682d5dc3e9ad6b557197949efe0890d7efda6bb20f301"}, {"id": "u015", "kind": "list-item", "locator": "body:L174-L174", "preview": "- Expert load imbalance is the primary practical bottleneck (see [tail-effect](../patterns/tail-effect.md))", "sha256": "0b3e2c331c616683888b3601e48371a8d51c666456d850355c40fdc118f4b4e7"}, {"id": "u016", "kind": "list-item", "locator": "body:L175-L175", "preview": "- Small M per expert causes thin-GEMM inefficiency on tensor cores", "sha256": "88340decaa9640bb6c863559fd47c4f124d76590140f660701d12ed18d3e202e"}, {"id": "u017", "kind": "list-item", "locator": "body:L176-L176", "preview": "- Masked layout wastes compute on padding when M distribution is skewed", "sha256": "93bc9120568a4d239e28dbca67cdd807e2201e2f14854fa9ffc533c15aeac7ab"}, {"id": "u018", "kind": "list-item", "locator": "body:L177-L177", "preview": "- CLC dynamic scheduling adds hardware overhead vs static precomputed schedules", "sha256": "6e26bfe8107dd58a8b358b2601fc7b2fbe936f750dd50c2bc4968e3daf4b194d"}, {"id": "u019", "kind": "list-item", "locator": "body:L178-L178", "preview": "- TMA alignment (128 bytes) constrains minimum tile dimensions", "sha256": "06ac4a8dd845e398c507180e217d7cb8868fdbe65bde49b054c729efc927047c"}, {"id": "u020", "kind": "list-item", "locator": "body:L182-L182", "preview": "- [GPU Mode NVFP4 Hackathon](https://github.com/gpu-mode/reference-kernels)", "sha256": "eebfd002120234da40f3b045095d61fdf901341039f5f3ae98ed1d4bb5bb5a27"}, {"id": "u021", "kind": "list-item", "locator": "body:L183-L183", "preview": "- [DeepGEMM Grouped GEMM](https://github.com/deepseek-ai/DeepGEMM)", "sha256": "384dcd71482423e4fdaeb79b0d7a923cbd79c7e4b1c8dda72a9801ad304898da"}, {"id": "u022", "kind": "list-item", "locator": "body:L184-L184", "preview": "- [Reward Hack Writeup](https://www.gpumode.com/news/reward-hacking-nvfp4)", "sha256": "2d4a7712919973168f4dda9ffa55a9a6abbaa1139c874eb1e7b9925af179acaa"}, {"id": "u023", "kind": "list-item", "locator": "body:L185-L185", "preview": "- [CUTLASS SM100 documentation](https://docs.nvidia.com/cutlass/latest/CHANGELOG.html)", "sha256": "75e0801b0814c177427d45c23dae770d0fabd27c5a2c06a826ddc8801ef0c4b4"}, {"id": "u024", "kind": "metadata-performance", "locator": "frontmatter:performance_claims", "preview": "[{\"dtype\": \"nvfp4\", \"gpu\": \"B200\", \"metric\": \"latency_us\", \"shape\": \"variable M, shared N=K, 15 groups\", \"source_id\": \"contest-gpumode-p4\", \"utilization\": \"compute-bound\", \"value\": 11.2}]", "sha256": "9a9c2791f5aa02e6b01213caae16799903c6001ea09c1f50253984d4de971d15"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Problem Structure", "DeepGEMM Grouped Layouts", "CUTLASS Grouped GEMM on SM100", "Tile Scheduling for Variable M", "Static Scheduling (Precomputed)", "Dynamic Scheduling (CLC / Persistent Kernel)", "The Reward Hack", "When to Use", "Caveats", "Sources"], "id": "kernel-grouped-gemm", "path": "wiki/kernels/grouped-gemm.md", "performance_claim_count": 1, "resolved_sources": [{"path": "sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md", "url": "https://github.com/gpu-mode/reference-kernels"}, {"path": "sources/blogs/deepgemm.md", "url": "https://github.com/deepseek-ai/DeepGEMM"}, {"path": "sources/docs/nvidia-cutlass-blackwell.md", "url": "https://docs.nvidia.com/cutlass/latest/CHANGELOG.html"}], "risk_flags": ["code", "ordering", "performance"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["contest-gpumode-p4", "blog-deepgemm", "doc-cutlass-blackwell"], "title": "Grouped GEMM for MoE", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "002fbf6ef7dba7c6bd6bfec4cdeccf4104b100309e09eae082faaf0aeec0e4c3", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Native Sparse Attention (NSA) is a hardware-aligned sparse attention mechanism published at ACL 2025, designed to reduce attention compute for long sequences (64K+) without sacrificing quality. Unlike post-hoc sparsity approaches, NSA is na", "sha256": "cf68b3aeaa3b5d26a01ee958075f2dd90859b572f97c3f1d53d55470ba09ffc1"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "NSA achieves 9x forward speedup and 6x backward speedup at 64K sequences versus FlashAttention-2, and 11.6x decoding speedup at 64K context. It is deployed in DeepSeek-V3.2-Exp combined with FlashMLA sparse kernels.", "sha256": "e8d21dfea9669dd9ecbfc3687491c8e41b2e94166ac6206b9e1458bf19f271bd"}, {"id": "u003", "kind": "code", "locator": "body:L11-L32", "preview": "``` Input Query Q | +---> [Compression Path] Learned MLP creates coarse-grained KV | | representations (token compression) | v | S_compressed = Q @ K_compressed^T | +---> [Selection Path] Blockwise importance scores select | | top-n fine-gr", "sha256": "8003fc920351bab1c7b84c9d6ed8fdcd0adfd2aad590cfe719794d39c518155e"}, {"id": "u004", "kind": "prose", "locator": "body:L36-L36", "preview": "The core NSA kernel is implemented in Triton with group-centric data loading and grid-based scheduling.", "sha256": "73ce24e24ace749715608e092ad64a9fdabc18f381ffa7bbe90e3ef4fdf30d8c"}, {"id": "u005", "kind": "code", "locator": "body:L38-L103", "preview": "```python import triton import triton.language as tl @triton.jit def nsa_sparse_attention_fwd( Q_ptr, K_ptr, V_ptr, O_ptr, block_indices_ptr, # Selected block indices per query num_selected: tl.constexpr, BLOCK_SIZE: tl.constexpr, HEAD_DIM:", "sha256": "d08bb0d4f71b7e67821be41026c25f1a8eb32149dc04ca6632c58233fa664934"}, {"id": "u006", "kind": "code", "locator": "body:L107-L140", "preview": "```python @triton.jit def nsa_sliding_window_fwd( Q_ptr, K_ptr, V_ptr, O_ptr, seq_pos, WINDOW_SIZE: tl.constexpr, # 512 HEAD_DIM: tl.constexpr, ): \"\"\"Local sliding window attention for recent context.\"\"\" pid = tl.program_id(0) # query posit", "sha256": "1d635d9a56dde15a96d9143c859722404c06634e252b0ca2455bc93d5b502bde"}, {"id": "u007", "kind": "prose", "locator": "body:L144-L144", "preview": "NSA's sparsity pattern is explicitly designed for GPU memory access efficiency:", "sha256": "92ff8ed85d1fccfa7b606ead178b30ad778a673bc3c5c7d13f177e05986f1e0a"}, {"id": "u008", "kind": "list-item", "locator": "body:L146-L146", "preview": "1. **Blockwise memory access**: Selected tokens are organized in contiguous blocks (not scattered individual tokens), exploiting spatial locality for contiguous GPU memory loads", "sha256": "a27d0ca3fbf0c07aa550d5c51800b1dfef7a04988b92749f1925f26e398d93d8"}, {"id": "u009", "kind": "list-item", "locator": "body:L147-L147", "preview": "2. **Group-centric loading**: In GQA configurations, all query heads in a group share the same sparse KV blocks, minimizing redundant KV transfers to shared memory", "sha256": "17b38863fc09180b4e995736b1278a175245a1f99b309d8660e9843b231f9b7b"}, {"id": "u010", "kind": "list-item", "locator": "body:L148-L148", "preview": "3. **Grid-based scheduling**: Triton grid dimensions map directly to (query_block, head, batch), avoiding dynamic scheduling overhead", "sha256": "8a3bcdcd651863ef6a92356ce6ee337bbb8362260281383026c3070863e56f47"}, {"id": "u011", "kind": "table-row", "locator": "body:L154-L154", "preview": "| Sequence Length | Forward Speedup | Backward Speedup | Decoding Speedup | | 64K | 9.0x | 6.0x | 11.6x |", "sha256": "80ab136cc0ff0a479cd4847be0000ec191c0fc23e7c38a4fb1fc16fa0a296a15"}, {"id": "u012", "kind": "prose", "locator": "body:L156-L156", "preview": "All speedups measured against FlashAttention-2 on H100 with BF16 precision.", "sha256": "a1855c4c62f570a8b173052c836888af41f997892c087472cbdc9bff8cb921c6"}, {"id": "u013", "kind": "list-item", "locator": "body:L160-L160", "preview": "- Long-context inference (32K+ tokens) where full attention is prohibitively expensive", "sha256": "f5c3361490c7e595b9012c11594ba45bdbddd8436d7476d422f8086c0f6f808d"}, {"id": "u014", "kind": "list-item", "locator": "body:L161-L161", "preview": "- Models with GQA (grouped query attention) where KV sharing amplifies sparse access efficiency", "sha256": "a6732812a96b98c9e0faaebb801bfd5303aaf9c30678f79900aee29cbbe0b579"}, {"id": "u015", "kind": "list-item", "locator": "body:L162-L162", "preview": "- End-to-end trainable settings requiring differentiable sparsity", "sha256": "9641f9e6ebcdc18d5defb85c178bae645f85ada88c08daf17c1ff6ddc64c89d2"}, {"id": "u016", "kind": "list-item", "locator": "body:L166-L166", "preview": "- The compression path requires a learned MLP, adding parameters and training cost", "sha256": "aebc0ab171f71ff82917cdd60c0a8d2f1ae7579d6880198d5453241ce9e6d998"}, {"id": "u017", "kind": "list-item", "locator": "body:L167-L167", "preview": "- Token selection adds a two-pass overhead (score all blocks, then select top-n)", "sha256": "3a7f8ca8be01afb157ed7811f151a047083a70a2f2e01ec80a8c7fdb81d4cc6e"}, {"id": "u018", "kind": "list-item", "locator": "body:L168-L168", "preview": "- Triton implementation has CPU launch overhead impacting small-batch decode; CUDA graph mode recommended", "sha256": "10abe51f393aaa0f3eeb85f38327ace4d925bbcd1cfb997a8c29e0e75dcfee0e"}, {"id": "u019", "kind": "list-item", "locator": "body:L169-L169", "preview": "- Quality depends on training the sparsity selection jointly with the model", "sha256": "f9e247144bca57e35f16350f906b2009b62f509d708195e2dd652c5dfe343b36"}, {"id": "u020", "kind": "list-item", "locator": "body:L173-L173", "preview": "- [NSA paper (ACL 2025)](https://arxiv.org/abs/2502.11089)", "sha256": "aeae36f3519863ad608fa6d0a24d8e33f365607b54e696e09bf44e8bac633126"}, {"id": "u021", "kind": "list-item", "locator": "body:L174-L174", "preview": "- [PyTorch reference implementation](https://github.com/lucidrains/native-sparse-attention-pytorch)", "sha256": "4bcdf22e3c02c5cb11cac427db84ffaf1d0360b1948790aa645e962819ab7dfe"}, {"id": "u022", "kind": "metadata-performance", "locator": "frontmatter:performance_claims", "preview": "[{\"dtype\": \"bf16\", \"gpu\": \"H100\", \"metric\": \"speedup\", \"shape\": \"seqlen=65536\", \"source_id\": \"blog-nsa\", \"utilization\": \"vs FlashAttention-2 forward\", \"value\": 9.0}]", "sha256": "e920d888aa4a8d9d0a24f1064956aa6aa0239a3320fcb6c84dab79a66ded1184"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Three-Path Architecture", "Triton Kernel Implementation", "Sliding Window Component", "Hardware-Aligned Design", "Performance", "When to Use", "Caveats", "Sources"], "id": "kernel-nsa", "path": "wiki/kernels/nsa.md", "performance_claim_count": 1, "resolved_sources": [{"path": "sources/blogs/nsa.md", "url": "https://arxiv.org/abs/2502.11089"}, {"path": "sources/blogs/flashmla.md", "url": "https://github.com/deepseek-ai/FlashMLA"}, {"path": "sources/blogs/vllm-deepseek-v3-sparse-attention.md", "url": "https://blog.vllm.ai/2025/09/29/deepseek-v3-2.html"}], "risk_flags": ["code", "performance", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-nsa", "blog-flashmla", "blog-vllm-deepseek-v3-sparse"], "title": "Native Sparse Attention (NSA)", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "d176748702ac353341b6d4887c74d36b98deccce13fd42849ac545957b212fa0", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "NVFP4 GEMM is a compute-bound matrix multiplication kernel operating on NVIDIA's native 4-bit floating-point format (E2M1) with block scaling on Blackwell GPUs. Unlike the memory-bound GEMV, GEMM is dominated by tensor core throughput and b", "sha256": "ce8283f8056ebe918e2275feaae294b883597349044d93ca6efee46883ca9177"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "This kernel was Problem 2 of the GPU Mode NVFP4 Hackathon (Nov-Dec 2025), targeting B200 GPUs. Top entries achieved within 1% of cuBLAS performance using CUTLASS SM100 schedules.", "sha256": "3f0d8624af5bd73c405f2ea0a5622c58d394640b3513b6a4feb3e53843adf439"}, {"id": "u003", "kind": "code", "locator": "body:L11-L25", "preview": "``` NVFP4 (E2M1): 4-bit floating point Bit layout: [S][E1][E0][M0] Representable values: 0, 0.5, 1, 1.5, 2, 3, 4, 6 (positive and negative) Block scaling: Every 16 FP4 elements share one FP8 (E4M3) scale factor Two-level: per-block E4M3 + p", "sha256": "b48ac773e3fe79bba2bd76c12caed110a58237a982ad1bef408e7506c7c4c2e5"}, {"id": "u004", "kind": "prose", "locator": "body:L29-L29", "preview": "The kernel uses the `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100` CUTLASS schedule, which combines TMA async loads with warp-specialized MMA execution.", "sha256": "7bbe8f50a670ca975b03b97d1020bdc5069b5fb6ad28030199f2b00228846247"}, {"id": "u005", "kind": "code", "locator": "body:L31-L55", "preview": "```cpp // CUTLASS dispatch for NVFP4 GEMM on Blackwell using Schedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100; // Tile configuration using TileShape = cute::Shape<_128, _256, _128>; // M, N, K tile using ClusterShape =", "sha256": "4c98ba70e353108efe70336468d8dc8ffc9bfe62eb7c1189ebbd016d30084923"}, {"id": "u006", "kind": "code", "locator": "body:L59-L130", "preview": "```cpp // Warp specialization: TMA producer + MMA consumer + epilogue // Shared memory holds pipelined A/B tiles + scale factors constexpr int NUM_STAGES = 4; // Pipeline depth // Shared memory layout struct SharedStorage { // Double-buffer", "sha256": "4bfed05994aeaa2b02ef97164c721b4233eaf36d6f14e56693b8b605a362d0b3"}, {"id": "u007", "kind": "prose", "locator": "body:L134-L134", "preview": "All TMA operands require 128-byte alignment. For NVFP4 (2 elements per byte), this means K dimensions must be multiples of 256 elements:", "sha256": "107e3196e9fea3e2ce246d8865dfdb3ae0c58873c27c4da2695854de3571f779"}, {"id": "u008", "kind": "code", "locator": "body:L136-L145", "preview": "```cpp // Critical: pad tensors to 128-byte boundaries for TMA // NVFP4: 2 elements per byte, so 256 elements = 128 bytes static_assert(K % 256 == 0, \"K must align to 128 bytes for FP4 TMA\"); // For scale factors: FP8 is 1 byte per element ", "sha256": "73a1b18a73b39446c1e3238a48b853cfd721bcc65071820610e7897e7a7d4636"}, {"id": "u009", "kind": "prose", "locator": "body:L149-L149", "preview": "The tcgen05.mma instruction expects UE8M0 (unsigned power-of-two exponent only) scales, but NVFP4 uses FP8 E4M3 (non-power-of-two). Conversion is needed:", "sha256": "e293e1e221e9b63a67a12c0f53933edc9faae2c7699f6e0f5c58f40160394399"}, {"id": "u010", "kind": "code", "locator": "body:L151-L164", "preview": "```cpp // Convert FP8 E4M3 block scales to UE8M0 for tcgen05.mma hardware // E4M3: 4 exponent bits, 3 mantissa bits (non-power-of-two) // UE8M0: 8 exponent bits, 0 mantissa bits (power-of-two only) __device__ uint32_t pack_scales_ue8m0( fp8", "sha256": "a98e841c1571483f432618b4cbc3012bfc42f8c5f3bed8761620c3f5e259a62b"}, {"id": "u011", "kind": "prose", "locator": "body:L168-L168", "preview": "Problem 2 top performers (geometric mean across benchmark configs):", "sha256": "cc23962d4fbe69a64cf189bafa8fa3fef985173c6339174224fc1630f48b8f02"}, {"id": "u012", "kind": "table-row", "locator": "body:L172-L172", "preview": "| Rank | Participant | Latency (us) | | 1 | Simon | 10.807 |", "sha256": "4be3f007808e68efc483366d7c16278e4fca7890ecf8fee5c6f7feb98f49b9f6"}, {"id": "u013", "kind": "table-row", "locator": "body:L173-L173", "preview": "| Rank | Participant | Latency (us) | | 2 | yue | 10.914 |", "sha256": "dd888bc57d858bbdad18fd18216286e38546341b351e4423cbca9993e06e131b"}, {"id": "u014", "kind": "table-row", "locator": "body:L174-L174", "preview": "| Rank | Participant | Latency (us) | | 3 | currybab | 10.931 |", "sha256": "ce16edd96b3c538db7ecacf7e7cfe138ee22ad0815169eadd0c8775e21a7769c"}, {"id": "u015", "kind": "list-item", "locator": "body:L178-L178", "preview": "- Inference with 4-bit quantized weights on Blackwell", "sha256": "9c9c4cb04e9f62299ba69bcc96fe07af559814f64b515782bfec237e26b6f57e"}, {"id": "u016", "kind": "list-item", "locator": "body:L179-L179", "preview": "- MLP layers in LLMs where weight matrices are NVFP4-quantized", "sha256": "acc382e3fe1a84f55d897e4e762e86b400edb666a30ae6a8f126c424baa34313"}, {"id": "u017", "kind": "list-item", "locator": "body:L180-L180", "preview": "- Compute-bound matrix multiplications where tensor core utilization is the bottleneck", "sha256": "282641f501a8c11c857461aea797f80a634c4d3656705ee7f3bd966b55a35158"}, {"id": "u018", "kind": "list-item", "locator": "body:L184-L184", "preview": "- SM100/SM100a only -- no Hopper support for native FP4 tensor core instructions", "sha256": "2e0b5d1795e2cdc383f29556cc772771a7e428d4e4154543bed78b96c951f2b5"}, {"id": "u019", "kind": "list-item", "locator": "body:L185-L185", "preview": "- Scale factor conversion (E4M3 to UE8M0) adds overhead if not precomputed", "sha256": "84fad3f7e26b6bf7ffe967ce0d273b7430a3528b6cb11dab390424821ac6e771"}, {"id": "u020", "kind": "list-item", "locator": "body:L186-L186", "preview": "- TMA requires 128-byte alignment for all operands", "sha256": "426caf652324604896fd38413cb9b583f16c3ddfba02a6c0915dc9b446992ed1"}, {"id": "u021", "kind": "list-item", "locator": "body:L187-L187", "preview": "- TMEM size (128x512 per SM) limits maximum output tile to 128 rows x 512 cols (32-bit)", "sha256": "de40759ac7c0d44ad789660e5d351ce123ba32e337f1b071516b48c01c65cc87"}, {"id": "u022", "kind": "list-item", "locator": "body:L191-L191", "preview": "- [GPU Mode NVFP4 Hackathon](https://github.com/gpu-mode/reference-kernels)", "sha256": "eebfd002120234da40f3b045095d61fdf901341039f5f3ae98ed1d4bb5bb5a27"}, {"id": "u023", "kind": "list-item", "locator": "body:L192-L192", "preview": "- [NVIDIA NVFP4 Blog](https://developer.nvidia.com/blog/introducing-nvfp4-for-efficient-and-accurate-low-precision-inference/)", "sha256": "8f1860be648139255ba1bef71ce8f3326c3bd2b4d7c539895998ce7876582574"}, {"id": "u024", "kind": "list-item", "locator": "body:L193-L193", "preview": "- [NVFP4 Format Details](https://haroldbenoit.com/notes/ml/engineering/precision/nvfp4-format)", "sha256": "b9bfdf8f815611f8f0fce7864844508d8c211c845cb20d7ca6aa7b4a80bf3d15"}, {"id": "u025", "kind": "list-item", "locator": "body:L194-L194", "preview": "- [CUTLASS SM100 documentation](https://docs.nvidia.com/cutlass/latest/CHANGELOG.html)", "sha256": "75e0801b0814c177427d45c23dae770d0fabd27c5a2c06a826ddc8801ef0c4b4"}, {"id": "u026", "kind": "prose", "locator": "body:L198-L198", "preview": "The reference bundle lives in [`artifacts/kernels/nvfp4-gemm/full/`](../../artifacts/kernels/nvfp4-gemm/full/) and combines the upstream PR-2139 diff (`PR-2139-blockwise-groupwise-gemm.patch`, `mode: upstream-patch`, SHA-pinned to `ca4fdbea", "sha256": "43f6a09758b6a58a1ffade0f14d33f1052c303991109e318f71b207016afdc51"}, {"id": "u027", "kind": "prose", "locator": "body:L200-L200", "preview": "Query via:", "sha256": "6abd63e508f9320fdd3b2b0cc5b5f6a000f4462abfd17f9ecc3c4818f4a0f875"}, {"id": "u028", "kind": "code", "locator": "body:L202-L204", "preview": "```bash python3 scripts/get_page.py kernel-nvfp4-gemm --include-code ```", "sha256": "eaac2756fbb1565989fcaea0ed73db004ca5ba5fcaea37b1ff1920e7d8d5212a"}, {"id": "u029", "kind": "metadata-performance", "locator": "frontmatter:performance_claims", "preview": "[{\"dtype\": \"nvfp4\", \"gpu\": \"B200\", \"metric\": \"latency_us\", \"shape\": \"standard GEMM configs\", \"source_id\": \"contest-gpumode-p2\", \"utilization\": \"near cuBLAS\", \"value\": 10.807}]", "sha256": "c874b5112e8e82ab82ff93848cc722d691926143bea53463233782aeedf2f72b"}], "confidence_claimed": "source-reported", "headings": ["Overview", "NVFP4 Data Format", "CUTLASS SM100 Schedule", "Warp-Specialized Kernel Structure", "128-Byte TMA Alignment", "Scale Factor Conversion", "Competition Results", "When to Use", "Caveats", "Sources", "Full Reference Implementation"], "id": "kernel-nvfp4-gemm", "path": "wiki/kernels/nvfp4-gemm.md", "performance_claim_count": 1, "resolved_sources": [{"path": "sources/contests/gpu-mode-nvfp4/problem-2-gemm.md", "url": "https://github.com/gpu-mode/reference-kernels"}, {"path": "sources/docs/nvidia-cutlass-blackwell.md", "url": "https://docs.nvidia.com/cutlass/latest/CHANGELOG.html"}, {"path": "sources/prs/cutlass/PR-2139.md", "revision": "ca4fdbea", "url": "https://github.com/NVIDIA/cutlass/pull/2139"}], "risk_flags": ["code", "ordering", "performance", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["contest-gpumode-p2", "doc-cutlass-blackwell", "pr-cutlass-2139"], "title": "NVFP4 GEMM \u2014 4-bit Floating Point Matrix Multiply", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "55d84be3408d0ab5d0adfed5c555998569f294e99d23cfc0c3fd582b32841d7d", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "NVFP4 Batched GEMV is a memory-bound kernel computing batched matrix-vector products with NVFP4 (E2M1) block-scaled inputs on B200 GPUs. Unlike compute-bound GEMM, GEMV is dominated by memory bandwidth utilization (each FP4 element is used ", "sha256": "ddf2e2976bc32e53a2c08102a37d952d37d0e9ba6015679a73d5ee037198c03d"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "This was Problem 1 of the GPU Mode NVFP4 Hackathon (Nov 2025). The theoretical speed-of-light is ~8.6us for the largest config, limited by B200's 8 TB/s HBM3e bandwidth. Top performers achieved ~22.4us (2.6x off SOL), reflecting the overhea", "sha256": "e22574105ba0fcea7e7feafd094b90087950c2484892d27fea2d3f5cafd8c9bb"}, {"id": "u003", "kind": "code", "locator": "body:L11-L30", "preview": "``` Inputs: a: (M x K x L) NVFP4 packed matrix b: (1 x K x L) NVFP4 packed vector sfa: (M x K/16 x L) FP8 E4M3 scale factors for a sfb: (1 x K/16 x L) FP8 E4M3 scale factors for b sf_a_global, sf_b_global: FP32 per-tensor global scales Outp", "sha256": "593e3b7065bd46466a83ad6524ed7becffdd5236b8a813cb54444537d41b74f8"}, {"id": "u004", "kind": "prose", "locator": "body:L34-L34", "preview": "Raw PTX provides critical performance gains over C intrinsics for FP4 decoding and memory access:", "sha256": "d637ccbc1b6e8a2971516ac8d9afbe2235a98237a2674e11165fdc9d421ae829"}, {"id": "u005", "kind": "code", "locator": "body:L36-L48", "preview": "```asm ; FP4 to FP16 conversion: hardware instruction ; Converts two packed E2M1 values to a pair of FP16 values cvt.rn.f16x2.e2m1x2 %result, %fp4_packed; ; Byte unpacking: PTX mov.b32 is faster than manual bitwise ops ; Instead of: val = (", "sha256": "01883c387684bea1a724b6275245c11cc51cd66d7e1fa2eecec4edb57b6b3b3a"}, {"id": "u006", "kind": "prose", "locator": "body:L52-L52", "preview": "Different data access patterns require different cache strategies:", "sha256": "96582c551b986112414b7fc502db5b39cd27f4369a8ec09e2de7b66833c06f61"}, {"id": "u007", "kind": "code", "locator": "body:L54-L66", "preview": "```asm ; Matrix A: streamed once per row, never reused across thread blocks ; Bypass L1 to avoid polluting cache with one-shot data ld.global.L1::no_allocate.v4.u64 {a0,a1,a2,a3}, [addr_a]; ; Vector B: reused across all M rows in a thread b", "sha256": "9394cb76df8b7a184f82675bfb54494032f63ac1ddd5763e356d8a49a45f05c6"}, {"id": "u008", "kind": "prose", "locator": "body:L70-L70", "preview": "Lower register counts force higher occupancy, which is critical for memory-bound kernels where latency hiding dominates:", "sha256": "6d80d1f3bd62cce66c20871851558deef20dd974d527748a3c1d06170c4a65cf"}, {"id": "u009", "kind": "code", "locator": "body:L72-L88", "preview": "```cpp // Rank 1: aggressive register limit for maximum occupancy // nvcc -maxrregcount=32 // Fewer registers -> more warps/SM -> better memory latency hiding // Rank 3: slightly relaxed for more ILP // nvcc -maxrregcount=45 // Launch bound", "sha256": "c73f3b1ac1ea8d45670517fd40b129834cc09a73e82a7f98d2a3ba23511e372b"}, {"id": "u010", "kind": "prose", "locator": "body:L92-L92", "preview": "Compile separate kernels per K dimension, each with full loop unrolling and tuned configs:", "sha256": "f7ccf8749992b568b1109ef36201aa04782f37d819dd0663da2637321d0221d6"}, {"id": "u011", "kind": "code", "locator": "body:L94-L136", "preview": "```cpp // Each K variant compiled separately with optimal configuration template __global__ __launch_bounds__(THREADS, MIN_BLOCKS) void nvfp4_gemv_specialized( const uint8_t* __restrict__ a, const uint", "sha256": "f408db3f15bda5e5a9b15668cca189240053b28770815636470fe6faade9f3d8"}, {"id": "u012", "kind": "prose", "locator": "body:L140-L140", "preview": "128-bit and 256-bit vector loads maximize bandwidth utilization:", "sha256": "5c4e0be8603b74e6fdac3950d76e25b63dbca34f01a6687d2af4deb744cf08c3"}, {"id": "u013", "kind": "code", "locator": "body:L142-L152", "preview": "```asm ; 128-bit vector load (16 bytes = 32 FP4 elements) ld.global.v2.u64 {r0, r1}, [addr]; ; 256-bit vector load (32 bytes = 64 FP4 elements) ld.global.v4.u64 {r0, r1, r2, r3}, [addr]; ; Only effective when combined with PTX byte unpackin", "sha256": "baab73411bcf0112e737f4c527e1b1e6be9272abcdc13a4b550cc3b24ffa1a9a"}, {"id": "u014", "kind": "prose", "locator": "body:L156-L156", "preview": "Since B is shape (1 x K x L), all M rows multiply against the same B vector:", "sha256": "cd580fe75200993177aafa125b8caac723f0db7f5cb43586f9c52b080001df01"}, {"id": "u015", "kind": "code", "locator": "body:L158-L176", "preview": "```cpp // Load B vector into shared memory once per thread block // All BLOCK_M rows reuse the same B data __shared__ half b_shared[K_TILE]; // Cooperative load: all threads in block load a portion of B for (int i = threadIdx.x; i < K_TILE;", "sha256": "f8f9daf87d9f76af0a4d2b6afcb7fca7746cf1d5106eec94c4d11463f95fb1ae"}, {"id": "u016", "kind": "prose", "locator": "body:L180-L180", "preview": "Documented progression from Yue's hackathon blog:", "sha256": "26ffc8446ee7591f95e0921f0a99c9b508614224a585865ebd07c841ec6dec0d"}, {"id": "u017", "kind": "table-row", "locator": "body:L184-L184", "preview": "| Stage | Technique | Latency | | CuTe DSL baseline | Basic CuTe partition/copy | ~100us |", "sha256": "dbf61acd19e989a9176617a1a7ed71ad5cd0ac1dbb93b575121e2fe142ddb333"}, {"id": "u018", "kind": "table-row", "locator": "body:L185-L185", "preview": "| Stage | Technique | Latency | | Coalesced access | Fix memory access patterns | 443us -> 39us |", "sha256": "8466d818ad949a4d14c3d022acea120979d9f0d8c66409305459ef4600f7e4b0"}, {"id": "u019", "kind": "table-row", "locator": "body:L186-L186", "preview": "| Stage | Technique | Latency | | Hardware intrinsics | cvt.rn.f16x2.e2m1x2 | ~39us |", "sha256": "ef3008a7fa097038a3df7f3cc3ef367b2e8d8149ed9ced0ea40c0311974bf314"}, {"id": "u020", "kind": "table-row", "locator": "body:L187-L187", "preview": "| Stage | Technique | Latency | | PTX assembly | Full PTX with byte unpacking | ~27us |", "sha256": "6a7e46856ae9f78e8086b31b0c839c9b7e9600015051fb9280e0ed7117e53f6d"}, {"id": "u021", "kind": "table-row", "locator": "body:L188-L188", "preview": "| Stage | Technique | Latency | | ILP optimization | Instruction-level parallelism | ~22.9us |", "sha256": "e9ef33c4dfa7d478861bca385846cdb9b5ea038df8d9f4cf20c11623ae6cbf56"}, {"id": "u022", "kind": "table-row", "locator": "body:L189-L189", "preview": "| Stage | Technique | Latency | | Final submission | All combined | 22.392us |", "sha256": "1a7a012b2eecf130716f6177ca40799b1db6a8388409e34978a3caa76ec3efa4"}, {"id": "u023", "kind": "list-item", "locator": "body:L193-L193", "preview": "1. **Memory-bound kernels need bandwidth-first thinking**: Arithmetic optimizations have minimal impact; focus on memory access patterns, cache policies, and vectorized loads", "sha256": "94e4ce070d1dd1f4b4e16744890672b8a2b0a8deaeaa55c74c27d17136982c35"}, {"id": "u024", "kind": "list-item", "locator": "body:L194-L194", "preview": "2. **PTX gives real control on Blackwell**: The gap between C intrinsics and hand-written PTX was substantial (443us to 27us in one journey)", "sha256": "9f2dec583d6a4146b6895a993f1b35ae986422246c70b033caf189f59e9a74c1"}, {"id": "u025", "kind": "list-item", "locator": "body:L195-L195", "preview": "3. **Profile first**: \"Run Nsight Compute to confirm memory-bound behavior\" (Amandeep's lesson after 12 attempts)", "sha256": "89fad8352db9048aec2f508a9750f0a52c457eaca28beb045ec08e152189ed8c"}, {"id": "u026", "kind": "list-item", "locator": "body:L196-L196", "preview": "4. **Register budgeting matters**: Lower registers -> higher occupancy -> better memory latency hiding", "sha256": "ac039173525f611aa7ba15b079e0990d411c358d359452e8571da6714c9cfee5"}, {"id": "u027", "kind": "list-item", "locator": "body:L197-L197", "preview": "5. **TMEM is irrelevant**: Memory-bound kernels do not benefit from TMEM (it helps compute-bound only)", "sha256": "cd4d91d3546b5d95261aeec64650a10f414cd52b32648b2b2ce6f1f6401a66f7"}, {"id": "u028", "kind": "list-item", "locator": "body:L201-L201", "preview": "- Decode-time MLP with batch size 1 (matrix-vector product)", "sha256": "f0a3de733e17a84015022e7a167ff14760cd4be1214ebfc988ddd6f8607128a0"}, {"id": "u029", "kind": "list-item", "locator": "body:L202-L202", "preview": "- Any NVFP4 workload where arithmetic intensity is too low for tensor cores", "sha256": "05859aa883b9a64883a90530cfa67b10d6007bf18ee27bb7cce31fc9111e6c24"}, {"id": "u030", "kind": "list-item", "locator": "body:L203-L203", "preview": "- Memory-bound operations with FP4 quantized weights", "sha256": "9cf5987a17e3202b9a1869766f23e0d7f345d99c3500a859627d9a6bd6edad8b"}, {"id": "u031", "kind": "list-item", "locator": "body:L207-L207", "preview": "- SM100/SM100a only (native FP4 decode instructions)", "sha256": "b4df75d26bf305f3c801fe6d6e96aa132f3fb57be630527b5ef96819c0f7dc16"}, {"id": "u032", "kind": "list-item", "locator": "body:L208-L208", "preview": "- PTX-level optimizations are fragile across CUDA toolkit versions", "sha256": "eb99e04c1e335c78e87dfb07807f938053e87f99ba5cf781b2b383a7eefda98e"}, {"id": "u033", "kind": "list-item", "locator": "body:L209-L209", "preview": "- Per-K specialization increases binary size (one kernel per K variant)", "sha256": "6ddb221452eeb25cf5e57ff8a82b855caeaf502d78aacf110e39f3fd97b726f2"}, {"id": "u034", "kind": "list-item", "locator": "body:L210-L210", "preview": "- Speed-of-light is bounded by B200 memory bandwidth (8 TB/s)", "sha256": "4b4f2fdfdc85b49814d3e63e8c61961c33736dcaffdd90ab851d930c70eb12c1"}, {"id": "u035", "kind": "list-item", "locator": "body:L214-L214", "preview": "- [GPU Mode NVFP4 Hackathon](https://github.com/gpu-mode/reference-kernels)", "sha256": "eebfd002120234da40f3b045095d61fdf901341039f5f3ae98ed1d4bb5bb5a27"}, {"id": "u036", "kind": "list-item", "locator": "body:L215-L215", "preview": "- [Yue's Hackathon Journey](https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html)", "sha256": "cf4d40ec9bc3d2a377968ec4fa11dcbe1a345085ff9ac0a296f1bbc5ca1c0ef4"}, {"id": "u037", "kind": "list-item", "locator": "body:L216-L216", "preview": "- [Twelve Attempts (Amandeep)](https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/)", "sha256": "b37eef4af82c15ab511f375b1eedd075fc2aa8b69a5598dd24b8e1478512ba81"}, {"id": "u038", "kind": "list-item", "locator": "body:L217-L217", "preview": "- [Simon's NVFP4 GEMV Blog](https://veitner.bearblog.dev/nvfp4-gemv/)", "sha256": "ac74cca9c7638c984b34a0027171297163dc5ac9e31a60a01df836eb06e20f0b"}, {"id": "u039", "kind": "prose", "locator": "body:L221-L221", "preview": "Local verbatim upstream code lives in [`artifacts/kernels/nvfp4-gemv/full/`](../../artifacts/kernels/nvfp4-gemv/full/) (see its `PROVENANCE.yaml` for the pinned upstream SHA and byte-verified SHA-256). Labeled derived variants \u2014 including a", "sha256": "1c2e2f79a21dc048f98be366befc6db0fdd07be639f7cb2df9816a0f5d2627e8"}, {"id": "u040", "kind": "prose", "locator": "body:L223-L223", "preview": "Query via:", "sha256": "6abd63e508f9320fdd3b2b0cc5b5f6a000f4462abfd17f9ecc3c4818f4a0f875"}, {"id": "u041", "kind": "code", "locator": "body:L225-L227", "preview": "```bash python3 scripts/get_page.py kernel-nvfp4-gemv --include-code ```", "sha256": "de75e93b2e63468e12728068d06362610599f17e00c4342bf1b2b26d0c0bb63e"}, {"id": "u042", "kind": "metadata-performance", "locator": "frontmatter:performance_claims", "preview": "[{\"dtype\": \"nvfp4\", \"gpu\": \"B200\", \"metric\": \"latency_us\", \"shape\": \"M=7168, K=16384, L=1\", \"source_id\": \"contest-gpumode-p1\", \"utilization\": \"~2.6x of SOL (8.6us)\", \"value\": 22.4}]", "sha256": "bb8b8a6640e57e3d597d751ab5cacce0f1e6e6a94666c6c5db1bc6c59a22cd07"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Problem Specification", "Key Optimization: PTX-Level Control", "Cache Policy Differentiation", "Register Budgeting", "Per-K Specialization", "Vectorized Loads", "Data Reuse (Rank 2 Approach)", "Performance Progression", "Key Lessons", "When to Use", "Caveats", "Sources", "Full Reference Implementation"], "id": "kernel-nvfp4-gemv", "path": "wiki/kernels/nvfp4-gemv.md", "performance_claim_count": 1, "resolved_sources": [{"path": "sources/contests/gpu-mode-nvfp4/problem-1-gemv.md", "url": "https://github.com/gpu-mode/reference-kernels"}, {"path": "sources/blogs/yue-nvfp4-hackathon.md", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}, {"path": "sources/blogs/amandeep-nvfp4-attempts.md", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}], "risk_flags": ["code", "performance", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["contest-gpumode-p1", "blog-yue-nvfp4", "blog-amandeep-nvfp4"], "title": "NVFP4 Batched GEMV", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "3a4b0e533208cfbbe45f6c7c15396a5e66007ed4a56f0ca9699e034a5b36b628", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Sparse Multi-head Latent Attention introduced in DeepSeek V3.2. Two-stage pipeline: (1) Lightning Indexer selects top-K tokens per query via FP8 scorer, (2) MLA runs only over selected tokens. This reduces decode attention compute from O(se", "sha256": "5fdcccb688e3f564691436a0e5b6472b88e4a9431c7db91971954f5504852a3a"}, {"id": "u002", "kind": "code", "locator": "body:L9-L27", "preview": "``` Query q_t (new token embedding) \u2502 \u25bc \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510 \u2502 Lightning Indexer \u2502 FP8 scorer, per-query top-K selection \u2502 - FP8 KV cache \u2502 h64, d128, topk=2048, page_size=64 \u2502 - Compute q\u00b7k_i scores \u2502 \u2502 - Select top-2048 i \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500", "sha256": "ce37d3974c8d5aba22eda5f37640aeaa8876cc39c0ec33d917dc7d01fed35e7e"}, {"id": "u003", "kind": "prose", "locator": "body:L31-L31", "preview": "Each KV cache entry is 656 bytes:", "sha256": "c34c647b6c4b0b7f9d93660cf35b79aadd81a97c0c489c79aae5d0a61f8a1fb4"}, {"id": "u004", "kind": "list-item", "locator": "body:L32-L32", "preview": "- 512 bytes: FP8 compressed KV data", "sha256": "8a0deb14b534e602728f848c4c40a1ee7eaebe8ea159f1be7bc6f1c888b75c17"}, {"id": "u005", "kind": "list-item", "locator": "body:L33-L33", "preview": "- 16 bytes: FP32 per-block scale factors", "sha256": "a275a8a2a9afa0544a06c32108aed3e3921e129571e060bbae077fcc7531b249"}, {"id": "u006", "kind": "list-item", "locator": "body:L34-L34", "preview": "- 128 bytes: BF16 RoPE embeddings (for indexer positional encoding)", "sha256": "946fd7b96cacd39e9ce0b5e338d694e32824a32db665b4378cac3e5b02b84ea7"}, {"id": "u007", "kind": "prose", "locator": "body:L36-L36", "preview": "Block size fixed at 64 (FlashMLA requirement).", "sha256": "d51e2f780cfb646e05d1c319e88a07eb826c00ddf298639894319965fd2c5a24"}, {"id": "u008", "kind": "code", "locator": "body:L41-L60", "preview": "```cuda // Score each KV block against query using FP8 block-scale MMA // Reduce to per-block max, then top-K selection across blocks __global__ void lightning_indexer_kernel( const fp8_t* q_fp8, // [h, d] query (FP8 quantized) const fp8_t*", "sha256": "dd9861d5434c4e848d82a6368267fa9cec26be9415ab60b0acb4190782fcc914"}, {"id": "u009", "kind": "code", "locator": "body:L63-L79", "preview": "```cuda // After top-K selection gives indices, gather K,V from paged cache // Then run standard MLA on the gathered subset __global__ void sparse_mla_decode_kernel( const int* topk_indices, // [2048] selected block indices const fp8_t* kv_", "sha256": "f774e41d4f3b84d4aac68aaf3c529e769a8815005181aa97697d26c19b4e6f11"}, {"id": "u010", "kind": "table-row", "locator": "body:L85-L85", "preview": "| Variant | GPU | TFLOPS | Notes | | Dense MLA decode | H800 | 660 (BF16) | 3000 GB/s, compute-bound |", "sha256": "73f0ff01a0cf401a3c4084d29e92d611fad71aea2b451e4dc435c79dfdcc18fd"}, {"id": "u011", "kind": "table-row", "locator": "body:L86-L86", "preview": "| Variant | GPU | TFLOPS | Notes | | Sparse MLA decode | H800 | 410 (FP8) | Token-level sparsity |", "sha256": "64ba139820ee276c80384043854f02370906e91e8a605e79520379be4a2d9c63"}, {"id": "u012", "kind": "table-row", "locator": "body:L87-L87", "preview": "| Variant | GPU | TFLOPS | Notes | | Sparse MLA decode | B200 | 350 (FP8) | Lower because bandwidth dominates decode |", "sha256": "8ed650a75ae8c30618f625619d36d3294ff3a342b06f082141c499b230b5800d"}, {"id": "u013", "kind": "table-row", "locator": "body:L88-L88", "preview": "| Variant | GPU | TFLOPS | Notes | | Dense prefill | B200 | 1460 (BF16) | tcgen05 peak |", "sha256": "8b5fb0d0411f07913282f333e0ec5b4799c24984a97d419b2753b6dd1ea7e4a0"}, {"id": "u014", "kind": "table-row", "locator": "body:L89-L89", "preview": "| Variant | GPU | TFLOPS | Notes | | Sparse prefill | B200 | 1450 (FP8) | FP8 sparse matches BF16 dense |", "sha256": "8b2958699e9a8520ec8695ef396558b37046d7f59cea685235b5df93e99e613c"}, {"id": "u015", "kind": "list-item", "locator": "body:L93-L93", "preview": "- Long-context LLM serving (32K+)", "sha256": "04381fa5d49d580e858be9ced13557503ccc96911883b97b013f2095f7b49632"}, {"id": "u016", "kind": "list-item", "locator": "body:L94-L94", "preview": "- DeepSeek V3.2 and similar MLA architectures", "sha256": "0bedbfae44643280304ba37348fbc04dc58c4a13ef33ee8c1f09a102fcadc593"}, {"id": "u017", "kind": "list-item", "locator": "body:L95-L95", "preview": "- Serving workloads where per-token decode latency matters", "sha256": "7d24f2d08acbe8deb2b51747711536108edfb5bb14713b1ce0c0bc0f566e11ff"}, {"id": "u018", "kind": "metadata-performance", "locator": "frontmatter:performance_claims", "preview": "[{\"dtype\": \"fp8\", \"gpu\": \"B200\", \"metric\": \"TFLOPS\", \"shape\": \"sparse prefill, seqlen=32k, topk=2048\", \"source_id\": \"blog-flashmla\", \"utilization\": \"FP8 sparse compute bound\", \"value\": 1450}]", "sha256": "df7016cefcf58c26abf29b326c6b4d91680d1c3ab23431d5d80a807cf87cd85d"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Architecture", "Token Layout", "Kernel Patterns", "Lightning Indexer (FP8 Score Compute)", "Sparse Attention Gather", "Performance", "When To Use"], "id": "kernel-sparse-mla", "path": "wiki/kernels/sparse-mla.md", "performance_claim_count": 1, "resolved_sources": [{"path": "sources/blogs/flashmla.md", "url": "https://github.com/deepseek-ai/FlashMLA"}, {"path": "sources/blogs/vllm-deepseek-v3-sparse-attention.md", "url": "https://blog.vllm.ai/2025/09/29/deepseek-v3-2.html"}, {"path": "sources/blogs/nsa.md", "url": "https://arxiv.org/abs/2502.11089"}], "risk_flags": ["code", "performance", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-flashmla", "blog-vllm-deepseek-v3-sparse", "blog-nsa"], "title": "Sparse MLA (DeepSeek V3.2)", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "c6277510118240c58c46ecfd1a84c7978461c9a65c84b4f74cbd679662dae2d1", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L6", "preview": "TensorRT-LLM PR 13340 integrates an FP4 indexer path for DSA on Blackwell and lands CUDA kernels for K-cache gather/scatter and fused FP4 concatenation. Use it as implementation evidence for sparse indexer memory movement and quantized cach", "sha256": "dbb05c921fd9eed27b25b1d13cbfcf15cc04132eb00639df9f04f2fb896b3bbb"}, {"id": "u002", "kind": "code", "locator": "body:L8-L17", "preview": "```cuda // Evidence checklist before adapting the idea: // 1. Inspect indexerKCacheGather.cu and indexerKCacheScatter.cu. // 2. Check the scale and FP4 packing layout. // 3. Benchmark gather/scatter separately from top-k selection. __global", "sha256": "c052dcb48a143a6c4d9ca4f59a8f736b7a0b0777ccecbd19417a7ea606adc73b"}, {"id": "u003", "kind": "list-item", "locator": "body:L21-L21", "preview": "- Keep FP4 packing, scale placement, and invalid-token handling explicit in the", "sha256": "f06cc5d4961ff51c22ff24f0be7521c353dd487bbc64a63abcb69813e6beabd2"}, {"id": "u004", "kind": "prose", "locator": "body:L22-L22", "preview": "correctness reference.", "sha256": "a29c197ecd7a5d8c55cf4f4d4bd79ad2451bab53066fda1647ea0f3b7f0c77f4"}, {"id": "u005", "kind": "list-item", "locator": "body:L23-L23", "preview": "- Treat gather/scatter traffic as a separate NCU profile target.", "sha256": "5f57eb07793d8a28665e648e34c56053130668339053fe55de9292182528c9b4"}, {"id": "u006", "kind": "list-item", "locator": "body:L24-L24", "preview": "- Avoid merging this with top-k selection until the memory path is understood.", "sha256": "455e2b81960ea59be68ecf14ddad9acdf4f9a90c34d160e34b3acb3361294b4e"}], "confidence_claimed": "source-reported", "headings": ["Shape", "Transfer Notes"], "id": "kernel-tensorrt-llm-blackwell-indexer", "path": "wiki/kernels/tensorrt-llm-blackwell-indexer.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/prs/TensorRT-LLM/PR-13340.md", "revision": "897c4bff", "url": "https://github.com/NVIDIA/TensorRT-LLM/pull/13340"}], "risk_flags": ["code"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["pr-TensorRT-LLM-13340"], "title": "TensorRT-LLM Blackwell FP4 DSA Indexer", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "7a58194cb413f25af6db71a4e0d90828273f461ee66232063fb6e6a96176d836", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "Plain CUDA C++ with inline PTX is used for hand-optimized Blackwell kernels. The tcgen05 tutorial achieved 98% of cuBLAS performance using this approach.", "sha256": "a7e0d97ffd4ae0534b1ab72df37bb19c918a46741bb34864efe91d2eb63b2cb9"}, {"id": "u002", "kind": "code", "locator": "body:L7-L51", "preview": "```cuda // Allocate TMEM. tcgen05.alloc writes the allocated address into SMEM. __device__ uint32_t tmem_alloc_cta(uint32_t* smem_tmem_addr, uint32_t num_cols) { if (threadIdx.x == 0) { uint32_t smem_addr = static_cast(__cvta_gene", "sha256": "1f4a080467db8f00f4af972aef7c4da1b145046d3a1b0a4c1308dce8260c04e0"}, {"id": "u003", "kind": "code", "locator": "body:L55-L78", "preview": "```cuda // TMA-MMA synchronization via mbarrier // expected_bytes: total bytes the TMA will deliver to this stage __device__ void mbarrier_arrive(uint64_t* mbar, uint32_t expected_bytes) { asm volatile( \"mbarrier.arrive.expect_tx.shared.b64", "sha256": "6d47fed1e615b984ff4c28b196d66d9f6789139d37c42ac5c8cefede2e572dc0"}, {"id": "u004", "kind": "code", "locator": "body:L82-L98", "preview": "```cuda __global__ void blackwell_gemm_kernel(...) { int warp_id = threadIdx.x / 32; int lane_id = threadIdx.x % 32; if (warp_id == 0 && lane_id == 0) { // TMA producer: issue cp.async.bulk.tensor tma_producer_loop(...); } else if (warp_id ", "sha256": "7b59b76aa579591c1c016c34a76e1939e6b9c41e44fb666f85e1fc92793a36d3"}, {"id": "u005", "kind": "list-item", "locator": "body:L101-L101", "preview": "- [ptx-sm100](ptx-sm100.md) \u2014 PTX instruction reference", "sha256": "e3ecbad1ee3fc785efde192375827aaa6a3129e64472e08cb90bad5269ca0115"}, {"id": "u006", "kind": "list-item", "locator": "body:L102-L102", "preview": "- [tcgen05 tutorial](../../sources/blogs/tcgen05-tutorial.md) \u2014 Step-by-step guide", "sha256": "9255c67edd923c02191211b83a0bb7c16168552147252a8b04ae2270bb7c2af0"}], "confidence_claimed": "source-reported", "headings": ["Overview", "tcgen05 via Inline PTX", "mbarrier Synchronization", "Warp Role Dispatch", "Related"], "id": "lang-cuda-cpp", "path": "wiki/languages/cuda-cpp.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/yue-nvfp4-hackathon.md", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}], "risk_flags": ["code", "ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-tcgen05-tutorial", "doc-nvidia-tuning-guide", "blog-yue-nvfp4"], "title": "CUDA C++ for Blackwell Kernels", "type": "language", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "4e09e1f2220eed702b5a7c8e6d7318de541e65faf5660835318c6916f5f1625b", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "CuTe (CUDA Templates) DSL is the primary abstraction layer in CUTLASS 4.5.0 for Blackwell kernels. FlashAttention-4 was implemented entirely in CuTe-DSL (Python variant), achieving 20-30\u00d7 faster compilation than C++ templates.", "sha256": "f267253319a8b833ac577f44078a120f287298cf23ea0318e5e1d9dff416a122"}, {"id": "u002", "kind": "code", "locator": "body:L7-L17", "preview": "```python # CuTe-DSL: SM100 MMA atom for BF16 from cutlass.cute import * # 1-SM MMA: m128 x n256 x k16 mma_atom = SM100_MMA_F16BF16_SS # inputs from shared memory # Accumulator goes to TMEM automatically # 2-SM MMA: m256 x n256 x k16 mma_at", "sha256": "3f4a3e9dce79d65551c530fbd343e760f386e23057d59f99c94eaa5c769d2002"}, {"id": "u003", "kind": "code", "locator": "body:L21-L30", "preview": "```python # TMEM allocation in CuTe tmem_tensor = make_tensor( make_tmem_ptr(tmem_addr), make_layout(make_shape(128, 256)) # rows x cols ) # Copy TMEM \u2192 registers for epilogue copy(tmem_tensor, reg_tensor) # tcgen05.ld under the hood ```", "sha256": "fd53f30c08b6b4086a5d73859b60a157146de0afb44cdb7c8db6a97f4ce43dfc"}, {"id": "u004", "kind": "code", "locator": "body:L34-L46", "preview": "```python # TMA bulk copy: global \u2192 shared tma_copy = SM100_TMA_LOAD_2D # Setup TMA descriptor tma_desc = make_tma_copy( tma_copy, global_tensor, smem_layout, tile_shape, cluster_shape ) ```", "sha256": "f31663eeafb41cecb386f19752f42215547599986e5b96ccd6200c876a4e954f"}, {"id": "u005", "kind": "code", "locator": "body:L50-L75", "preview": "```python @cute_kernel def blackwell_gemm(A, B, C): # Warp 0: TMA producer if warp_id == 0: for stage in pipeline: tma_copy(A_tile, smem_A[stage]) tma_copy(B_tile, smem_B[stage]) arrive(mbarrier[stage]) # Warp 1: MMA consumer elif warp_id =", "sha256": "93dcdee719648a9c0f7b9fcc101179011e3a1c4e1ecbc53bae4b6562ebc2abde"}, {"id": "u006", "kind": "list-item", "locator": "body:L79-L79", "preview": "1. **20-30\u00d7 faster compilation** than C++ CUTLASS templates", "sha256": "06eadc9b5f229578a8647f720c75ce49a2e5aa0db4870ba73c2ab40a94f6f8e9"}, {"id": "u007", "kind": "list-item", "locator": "body:L80-L80", "preview": "2. Python-native: easier to iterate and debug", "sha256": "00ddb54a9d066c146574964cb6960bd8290ba3d648327296a845dafd57181727"}, {"id": "u008", "kind": "list-item", "locator": "body:L81-L81", "preview": "3. Same performance as hand-written C++ (FlashAttention-4: 1605 TFLOPS)", "sha256": "0f3516d2971c235fd956277a3a282136fec6e387f59cf0635fd12d7baf7902fd"}, {"id": "u009", "kind": "list-item", "locator": "body:L82-L82", "preview": "4. First-class TMEM and tcgen05 support in CUTLASS 4.5.0", "sha256": "d0a0e90286047220e6df2f2b361e524b2a277f7cb5e28850d7b9a77c4f56c221"}, {"id": "u010", "kind": "list-item", "locator": "body:L83-L83", "preview": "5. Automatic layout computation and swizzle handling", "sha256": "a72d78fed8c661378ce62869179fb5b427a6724e91f23895ffd937cff3102e2b"}, {"id": "u011", "kind": "prose", "locator": "body:L87-L87", "preview": "The following CuTe DSL files ship **verbatim** in this repository under `artifacts/prs/cutlass/` (pinned at each PR's merge SHA). Open them with `python3 scripts/get_page.py --include-code` or read them directly.", "sha256": "794892b90eccf2d0d4a76e2276fcafc1bda433b0abe9242d0287acc8ae172127"}, {"id": "u012", "kind": "table-row", "locator": "body:L91-L91", "preview": "| File | Purpose | Size | | [`artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_0.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_0", "sha256": "f2eb9d44aa5da45c79369cecfd974acd5fa4de40b46920354d62ea04c363b13a"}, {"id": "u013", "kind": "table-row", "locator": "body:L92-L92", "preview": "| File | Purpose | Size | | [`artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_1.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_1", "sha256": "592257d33968323ab308c31c0764d8858b23ecffba87df3e0040d844272cb8c3"}, {"id": "u014", "kind": "table-row", "locator": "body:L93-L93", "preview": "| File | Purpose | Size | | [`artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_2.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_2", "sha256": "1f43346f9bed5e37eba018ea644e2658255250d3a20b78d4c8fe1dbf46d29137"}, {"id": "u015", "kind": "table-row", "locator": "body:L94-L94", "preview": "| File | Purpose | Size | | [`artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_3.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_3", "sha256": "3c4369a498aa443eed7c81664afdb931d504593f999786a38867b034189eb7c0"}, {"id": "u016", "kind": "table-row", "locator": "body:L95-L95", "preview": "| File | Purpose | Size | | [`artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_4.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_4", "sha256": "5c3a9081b2384804a170f2ee3b66d833f2a117d360b19799df85dbb1c9760e1e"}, {"id": "u017", "kind": "table-row", "locator": "body:L96-L96", "preview": "| File | Purpose | Size | | [`artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_5.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_5", "sha256": "d28176cfdb69ba0bb223e7301f1159114948a8e2a3cb86511b35e93a3b60687b"}, {"id": "u018", "kind": "table-row", "locator": "body:L97-L97", "preview": "| File | Purpose | Size | | [`artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_6.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_6", "sha256": "48ff2842c5cdc7c477173e87f1f5dbcdafefd3b3fde74c0ecac7bd60876c449d"}, {"id": "u019", "kind": "table-row", "locator": "body:L98-L98", "preview": "| File | Purpose | Size | | [`artifacts/prs/cutlass/PR-2881/key-files/examples/python/CuTeDSL/blackwell/dense_gemm_persistent_prefetch.py`](../../artifacts/prs/cutlass/PR-2881/key-files/examples/python/CuTeDSL/blackwell/dense_gemm_persisten", "sha256": "5896cfb52da17a25a445b4ba6fa765906ac12f5e5a989fb2ab14bada6fc86db0"}, {"id": "u020", "kind": "table-row", "locator": "body:L99-L99", "preview": "| File | Purpose | Size | | [`artifacts/prs/cutlass/PR-3021/key-files/python/CuTeDSL/cutlass/cute/arch/clc.py`](../../artifacts/prs/cutlass/PR-3021/key-files/python/CuTeDSL/cutlass/cute/arch/clc.py) | CLC (Cluster Launch Control) Python bin", "sha256": "a46e0b83297c1af1597dcef52c6cdee33321e1ab19179f820e86ed0fbd94af3e"}, {"id": "u021", "kind": "prose", "locator": "body:L101-L101", "preview": "The `fp16_gemm_{0..6}.py` series from `examples/python/CuTeDSL/blackwell/tutorial_gemm/` in NVIDIA/cutlass PR-3106 is the authoritative CuTe DSL learning path: it walks from a naive FP16 GEMM baseline through 2CTA MMA with TMA multicast, wa", "sha256": "4a4f5e738f8ffaf62825c5ac488ac75ccce782bce6e1902daa6b927d1cd473c8"}, {"id": "u022", "kind": "list-item", "locator": "body:L104-L104", "preview": "- [tcgen05-mma](../hardware/tcgen05-mma.md) \u2014 Underlying MMA instruction", "sha256": "865831018843436afb60a211da0239b6699acc87cacad5a279c5bcc602cc1ff7"}, {"id": "u023", "kind": "list-item", "locator": "body:L105-L105", "preview": "- [flash-attention-4](../kernels/flash-attention-4.md) \u2014 CuTe-DSL implementation", "sha256": "691d3063c66f516ddeda6475ae8a66e2b0445804b11056a1ecdc647362d5d5b0"}, {"id": "u024", "kind": "list-item", "locator": "body:L106-L106", "preview": "- [CUTLASS Blackwell docs](../../sources/docs/nvidia-cutlass-blackwell.md) \u2014 Official reference", "sha256": "be5027fb093b9885429cfe4e9efd6064c2ab195d0c12747b24220c8fc2615059"}], "confidence_claimed": "source-reported", "headings": ["Overview", "SM100 MMA Atoms", "TMEM as CuTe Locale", "TMA Copy Atoms", "Warp-Specialized Kernel Skeleton", "Why CuTe-DSL for Blackwell", "Full Examples (verbatim upstream code shipped locally)", "Related"], "id": "lang-cute-dsl", "path": "wiki/languages/cute-dsl.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-cutlass-blackwell.md", "url": "https://docs.nvidia.com/cutlass/latest/CHANGELOG.html"}, {"path": "sources/blogs/colfax-cutlass-blackwell.md", "url": "https://research.colfax-intl.com/cutlass-tutorial-writing-gemm-kernels-using-tmem-for-nvidia-blackwell-gpus/"}, {"path": "sources/blogs/flash-attention-4.md", "url": "https://tridao.me/blog/2026/flash4/"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-cutlass-blackwell", "blog-colfax-cutlass", "blog-flash-attention-4"], "title": "CuTe DSL for Blackwell", "type": "language", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "7ef09348591994e947177b5ee3a4a04af46cd573163a183b47b016ff51d1fec9", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "SM100 PTX instructions for Blackwell-specific hardware features.", "sha256": "c8563d73b8a3172e7ff8edb07a70676c383d56a8fcec4c675acb2ca1767dc231"}, {"id": "u002", "kind": "code", "locator": "body:L7-L31", "preview": "```ptx // Allocate TMEM columns tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [smem_tmem_addr], num_cols; // MMA: inputs from SMEM, accumulator in TMEM tcgen05.mma.cta_group::1.kind::f16 [tmem_addr], desc_a, desc_b, idesc, enable_", "sha256": "ea63c999b7f92954efb06f3b2577528b4ba2e66d5129c0cc178f3b48fd393a28"}, {"id": "u003", "kind": "code", "locator": "body:L35-L41", "preview": "```ptx // Convert two packed FP4 values to two FP16 values cvt.rn.f16x2.e2m1x2 result_f16x2, packed_fp4; // Byte unpacking (faster than bitwise extraction) mov.b32 {byte0, byte1, byte2, byte3}, packed_word; ```", "sha256": "c8931477a3b15df18a45e06478499f273b4678b77cd30f2d6ce80f4b922c639c"}, {"id": "u004", "kind": "code", "locator": "body:L45-L54", "preview": "```ptx // Streaming data (use once): bypass L1 ld.global.L1::no_allocate.v2.u64 {r0, r1}, [addr]; // Reused data: keep in L1 ld.global.L1::evict_last.v2.u64 {r0, r1}, [addr]; // Wide vectorized loads ld.global.v4.u64 {r0, r1, r2, r3}, [addr", "sha256": "2b04630e2f7218c1069f401e1ed8779a5caa8f710525999181bed330d92ad0ed"}, {"id": "u005", "kind": "code", "locator": "body:L58-L62", "preview": "```ptx // Query for next tile (persistent kernel loop) clusterlaunchcontrol.try_cancel {clc_id}; // Returns valid tile_id or decline (all work done) ```", "sha256": "160bc0e02144188c7c23c0c2235a8ee63f2a0d1f8ee62aa53f5ee8102965cfa6"}, {"id": "u006", "kind": "code", "locator": "body:L66-L74", "preview": "```ptx // Bulk tensor copy: global \u2192 shared cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes [smem_ptr], [tensorMap, {x, y}], [mbarrier]; // Multicast to cluster SMs cp.async.bulk.tensor.2d.shared::cluster.gl", "sha256": "74e1444f94a65881a33de14124fc353554d4e3e0eb34e3492ef64bbd1bd674b5"}, {"id": "u007", "kind": "list-item", "locator": "body:L77-L77", "preview": "- [cuda-cpp](cuda-cpp.md) \u2014 Inline PTX in CUDA C++", "sha256": "ba5d88a5192f94c29602f9ab9bbba9f6a19fee52e6b33bcf9835875be760d322"}, {"id": "u008", "kind": "list-item", "locator": "body:L78-L78", "preview": "- [tcgen05-mma](../hardware/tcgen05-mma.md) \u2014 MMA instruction details", "sha256": "3a6c8e4e3cafa675efc3378a216626ae375403e2470481b7f2abd6a7c2f41c1c"}, {"id": "u009", "kind": "list-item", "locator": "body:L79-L79", "preview": "- [nvfp4](../hardware/nvfp4.md) \u2014 FP4 format details", "sha256": "d0f7677a19714c587cc777cc976de16d9f64aaa3538abfd879a0dd5b6547a102"}], "confidence_claimed": "source-reported", "headings": ["Overview", "tcgen05 Instructions", "NVFP4 Conversion Instructions", "Cache Control for Memory-Bound Kernels", "Cluster Launch Control", "TMA (Tensor Memory Accelerator)", "Related"], "id": "lang-ptx", "path": "wiki/languages/ptx-sm100.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/parallel-thread-execution/"}, {"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/yue-nvfp4-hackathon.md", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}], "risk_flags": ["code", "ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-ptx-isa-sm100", "doc-nvidia-tuning-guide", "blog-yue-nvfp4"], "title": "PTX Instructions for SM100", "type": "language", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "6b9394376db86dd1090ef2012a0d599ad3ac0703f7df8c9bbf9c61c4c24888f1", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "Triton is used for many attention and linear-attention kernels (NSA, GatedDeltaNet, FLA). Starting with Triton 3.6 (released `2026-01-21`), Triton ships native Blackwell (SM100) lowering through `tcgen05.mma` + Tensor Memory (TMEM). The ear", "sha256": "68c1e1ebd182e1cf65c87bbc3aa111708409c0ac52cd3cba9642f2c61ab0b60d"}, {"id": "u002", "kind": "prose", "locator": "body:L5-L5", "preview": "When to use Triton on Blackwell:", "sha256": "99f63952870fe924c0c2afdc97eed4a59db391ea1d10ed687fb707e09478bbae"}, {"id": "u003", "kind": "list-item", "locator": "body:L6-L6", "preview": "- Rapid prototyping (Triton's fast turnaround still beats CuTe-DSL for exploration)", "sha256": "7106ba7a2d42f73bc2819ab5474eea4c8e40168ebf0d54908a73b319cf9d0215"}, {"id": "u004", "kind": "list-item", "locator": "body:L7-L7", "preview": "- Memory-bound kernels (bandwidth is bottleneck, not compute)", "sha256": "d5703150710b9a3c91076484d05190e79efab6ae8282c1eee43f4cc6a6edf397"}, {"id": "u005", "kind": "list-item", "locator": "body:L8-L8", "preview": "- Linear / sparse attention (GatedDeltaNet, FLA, NSA) where Triton's grid scheduling is convenient", "sha256": "0cee3020ba8376c55f33911b0db4d2d3eb77949b89886bb0c51ecf91362b5645"}, {"id": "u006", "kind": "list-item", "locator": "body:L9-L9", "preview": "- Block-scaled matmul on Blackwell via `tl.dot_scaled` (NVFP4 / MXFP families) \u2014 first-class hardware-accelerated path", "sha256": "93c0626d62e7d2e2525d085be1d273c8845fda5f27cc78568536ba4835262900"}, {"id": "u007", "kind": "list-item", "locator": "body:L10-L10", "preview": "- Warp-specialized descriptor/TMA matmul kernels following the Triton persistent matmul tutorial pattern", "sha256": "c8d8013eb08d22b81a5ae1d53b6a21e0211868a3e3eac50995bcf459449b2e0f"}, {"id": "u008", "kind": "prose", "locator": "body:L12-L12", "preview": "Use CuTe-DSL / CUTLASS / FA-4 / TRT-LLM instead when:", "sha256": "78cf262304ba579a7354fc2e07a8827c68d4fc94027e8260534cf282d1b8b2a7"}, {"id": "u009", "kind": "list-item", "locator": "body:L13-L13", "preview": "- Peak-performance compute-bound matmul or attention on Blackwell. SGLang `pr-sglang-5390` measured a CUTLASS `tcgen05_mla` backend ~27% faster than the Triton MLA decode baseline on Blackwell.", "sha256": "ca4c5776bb00da09dd5d04a4a6806210fd0471e2180e1e1d568d78a63411b84e"}, {"id": "u010", "kind": "list-item", "locator": "body:L14-L14", "preview": "- Production routing where vendor kernels are mature: SGLang `pr-sglang-21595` changes Blackwell datacenter multimodal attention default from `triton_attn` to FA-4.", "sha256": "4c44600cade1a267bcd0dacecde9da7640ae7ac7529dccd5864f2310043d99b7"}, {"id": "u011", "kind": "prose", "locator": "body:L18-L18", "preview": "The 3.6 release adds Blackwell-native infrastructure for `tcgen05.mma`, TMEM allocation/copy/load/store, and warp-specialization plumbing. Source-of-record: [`doc-triton-3.6-blackwell`](../../sources/docs/triton-3.6-blackwell.md). Per-pathw", "sha256": "2c81d91397865a1c3476a3e15febe3df9696ebd0858023bd636690b8f6d84038"}, {"id": "u012", "kind": "prose", "locator": "body:L20-L20", "preview": "Verified lowering surfaces (with caveats):", "sha256": "6b7646c310ebd8dbe8c3dd34c1953b9e876e6a010fe44a51fb330035edacecf3"}, {"id": "u013", "kind": "list-item", "locator": "body:L22-L22", "preview": "1. **Descriptor/TMA + `tl.range(warp_specialize=True)` + `tl.dot`** \u2014 strongest checked `tl.*`-surface evidence. The official Triton persistent matmul tutorial states the warp-specialized mode \"only works on Blackwell right now\"; backed by ", "sha256": "3da89a1067d83982cc596e84e2b315d10af0dfabe7739ef01dc9b4a0951abe68"}, {"id": "u014", "kind": "list-item", "locator": "body:L24-L24", "preview": "2. **`tl.dot_scaled` block-scaled matmul** \u2014 hardware-accelerated by fifth-generation Tensor Cores on compute capability 10. The 3.6 dialect doc exposes `ttng.tc_gen5_mma_scaled` with TMEM-token semantics plus `ttng.tmem_copy`. Source PRs: ", "sha256": "041ce1df4122f743e2a4351708052ae46587b967d589fdcf4e76c1b3be33b0c2"}, {"id": "u015", "kind": "list-item", "locator": "body:L26-L26", "preview": "3. **Gluon front-end + `gl.warp_specialize` + `num_ctas`** \u2014 the most explicit Blackwell-native surface. Initial 2-CTA cluster support landed in `#8644`, `#8653`; `num_ctas` plumbing in `#8645`; Gluon-side `tcgen05 mma scaled` in `#8393`. T", "sha256": "052ff6a643f32c125fc1222c3f9c5382323575416d3bc07ea7852777237259c6"}, {"id": "u016", "kind": "prose", "locator": "body:L28-L28", "preview": "Caveats \u2014 what the verified evidence does and does not establish:", "sha256": "8bcc3f5f74ca29ecfbbb9e7f670dc6efde9c6fa11c589da806df7759fd81d198"}, {"id": "u017", "kind": "list-item", "locator": "body:L29-L29", "preview": "- **What is verified**: the Triton 3.6 release adds tcgen05 + TMEM lowering infrastructure (per `doc-triton-3.6-blackwell`), and tracked downstream repos are landing real Triton kernel changes for SM100 post-3.6 (per `pr-vllm-34597`, the po", "sha256": "d0aa1fe2560f903a0823d77f822d78e5755837df4e85cd620f3e0f5176832deb"}, {"id": "u018", "kind": "list-item", "locator": "body:L30-L30", "preview": "- **What is not yet verified by in-corpus evidence**: that arbitrary plain `tl.dot` kernels AUTOMATICALLY emit `ttng.tc_gen5_mma` / `tcgen05.*` PTX on every shape and configuration. The 3.6 infrastructure exists (`#8136`, `#8148`, `#8202`, ", "sha256": "331695984a8664306f55438dedd3f9afd0f47bc326a5c1fc8ad7e90c87e3162b"}, {"id": "u019", "kind": "list-item", "locator": "body:L31-L31", "preview": "- **What is also not yet verified**: that fused attention forward kernels with `warp_specialize=True`, `tl.dot_scaled` block-scaled matmul, and Gluon multi-CTA paths produce the same lowering as the upstream Triton tutorials predict \u2014 the u", "sha256": "10a0b53c649641ed48d679542645272c26e44cbc463d67aae953da68a755b4d6"}, {"id": "u020", "kind": "prose", "locator": "body:L35-L35", "preview": "Before Triton 3.6, the Blackwell story was: the compiler generated `wgmma.mma_async`, accumulators stayed in registers, and direct `tcgen05` / TMEM access was unavailable from `tl.*`. Pages and inclusion-policy text written under that premi", "sha256": "96669fb791d41bc9dd0173379a54c56be9df068d0a82a155dce6fca621929355"}, {"id": "u021", "kind": "prose", "locator": "body:L39-L39", "preview": "The `evidence_basis` is anchored on:", "sha256": "e8e7fa69ca365c3d43689787d5e25dd981201fbb79b59ed6217e96ae0f0f5b51"}, {"id": "u022", "kind": "list-item", "locator": "body:L41-L41", "preview": "- **`doc-triton-3.6-blackwell`** (`source_category: official-doc`) \u2014 verifies that Triton 3.6 ships native Blackwell lowering infrastructure (TMEM, tcgen05, warp_specialize plumbing, Gluon multi-CTA / 2CTA, tl.dot_scaled). This is the \"infr", "sha256": "9cced2b3ad96d73f2b42f402467eaed85cde8d0eae5eec8bf4f5f0a879f67670"}, {"id": "u023", "kind": "list-item", "locator": "body:L42-L42", "preview": "- **`pr-vllm-34597`** (`source_category: upstream-code`) \u2014 **post-refresh primary anchor**: vLLM's \"[Kernel] Add FP8 KV cache support to Triton MLA decode attention\" PR (merged 2026-02-16, post-Triton-3.6.0 release; `architectures: [sm100]`", "sha256": "903f1359948e7ce1dfba87979ad12bedd24310c2620d3c1f9f1898d132864107"}, {"id": "u024", "kind": "prose", "locator": "body:L44-L44", "preview": "Together the two anchors establish that **the 3.6+ Blackwell lowering infrastructure is real AND tracked downstream repos are landing real Triton matmul kernels for Blackwell production decode paths today** \u2014 which is the substance of the r", "sha256": "4fb8f623c148a0321ef35123e67752145aa91121f2d4c61a34601f50c860b5dd"}, {"id": "u025", "kind": "prose", "locator": "body:L46-L46", "preview": "Two clarifications about what these anchors do NOT prove (since the verified-surface section above flags some shapes as needs-verification):", "sha256": "2840a770688d2dbb0bdec8c34db137c5158a5ec68f32166a3388542caf8671c6"}, {"id": "u026", "kind": "list-item", "locator": "body:L48-L48", "preview": "- The anchors do not prove that EVERY plain `tl.dot` kernel on SM100 emits `tcgen05.mma` PTX. They prove that real Triton kernels with `tl.dot` matmul are landing on Blackwell-only paths in tracked downstream repos (per `pr-vllm-34597`'s `t", "sha256": "b2259ca9d74882d66a8a3a7f562dba7c99e0c587a1f2561445eb75a29b9660b3"}, {"id": "u027", "kind": "list-item", "locator": "body:L49-L49", "preview": "- The anchors do not include explicit inspectable `tcgen05.mma` PTX dumps from a tracked-downstream merged PR. Such proof would be an even stronger anchor than what we have today; until one is found, the verified claim should be read as \"Tr", "sha256": "a2902fb92a5d5f28c8037e2c2f4c0032aac766e60002a42261e7bb2f5a4ff609"}, {"id": "u028", "kind": "prose", "locator": "body:L51-L51", "preview": "Supplementary post-refresh anchor: [`pr-vllm-29339`](../../sources/prs/vllm/PR-29339.md) \u2014 vLLM bugfix that scopes the upstream `triton_kernels` library (the `triton-lang/triton/python/triton_kernels` collection shipped with Triton 3.6) to ", "sha256": "47a0e126b913be9524249f01d1d624182ba7752d28cee3a78c56689cb7ca14b7"}, {"id": "u029", "kind": "prose", "locator": "body:L53-L53", "preview": "Pre-refresh historical anchors (retained as supplementary context, not as AC-1.1 \"new tracked-repo PR page\" evidence on their own):", "sha256": "164da0b6b74ed9ba4f4389a6b1d5b15fcd8f80c767f60ab0d88058d65214ee74"}, {"id": "u030", "kind": "list-item", "locator": "body:L55-L55", "preview": "- [`pr-sglang-22079`](../../sources/prs/sglang/PR-22079.md) \u2014 Gemma4 NVFP4 SGLang `extend_attention` Triton kernel doing actual `tl.dot(q,k)` / `tl.dot(p,v)` matmul on `[sm100, sm90]`, merged 2026-04-03. Strongest in-corpus example of a rea", "sha256": "5ace890ab5dca85799b653f7068541d368172760e0a9e4b41ff7cc9fea4d4db9"}, {"id": "u031", "kind": "list-item", "locator": "body:L56-L56", "preview": "- [`pr-sglang-21019`](../../sources/prs/sglang/PR-21019.md) \u2014 Qwen3.5 GDN projection fused split/reshape/cat kernel merged 2026-03-20. `tl.load`/`tl.store` only (memory rearrangement, no `tl.dot`); demonstrates \"Triton on SM100 post-3.6\" bu", "sha256": "213d5820b7ec1e853f582efea1546f48f255d8c38735d0d0d0d597774dbfbcea"}, {"id": "u032", "kind": "prose", "locator": "body:L58-L58", "preview": "Caveat anchors `pr-sglang-5390`, `pr-sglang-21595`, and `pr-pytorch-175826` provide ecosystem context (CUTLASS still leads on peak; Blackwell defaults moved away from triton_attn for some workloads; CI moved to CUDA 13.0).", "sha256": "ea97074144a30f3926298412a14b091f1119deec63d40f55e4b8ebf4cfedbfbc"}, {"id": "u033", "kind": "prose", "locator": "body:L62-L62", "preview": "The following table reflects benchmark snapshots from when the original page was written (Triton 3.5 era) and is preserved for historical reference. It does NOT reflect the 3.6+ tcgen05 path.", "sha256": "589d3b8ff644ac6f23bb7835dae83eb858acff99b876aa94c2b81e9bc3c41e3d"}, {"id": "u034", "kind": "table-row", "locator": "body:L66-L66", "preview": "| Model | Avg Speedup vs FlashInfer | Resolved % | | Gemini 2.5 Pro | 0.628x | 73.1% |", "sha256": "25c2678c28908e2d85210e1ff0629d8191217089f7e4b450b745eef050592a4c"}, {"id": "u035", "kind": "table-row", "locator": "body:L67-L67", "preview": "| Model | Avg Speedup vs FlashInfer | Resolved % | | GPT-5 | 0.467x | 92.3% |", "sha256": "60ca879e235ec816f205afb6233e07b1183ffe29c81bbe6344d6dc30388d420d"}, {"id": "u036", "kind": "table-row", "locator": "body:L68-L68", "preview": "| Model | Avg Speedup vs FlashInfer | Resolved % | | Claude Opus 4.1 | 0.456x | 73.1% |", "sha256": "0d5529e82c81a2771bf9dd829984c3b0c7827e2b4a31686ad99c859f8b45db91"}, {"id": "u037", "kind": "prose", "locator": "body:L70-L70", "preview": "A re-run on Triton 3.6 with FlashInfer-Bench is not yet available locally; future refresh rounds should update this table or remove it in favor of a pointer to the live leaderboard.", "sha256": "864a6cf965f86e44f206bdef59f8b49d4b532848e43f18eb861b8c64de516994"}, {"id": "u038", "kind": "code", "locator": "body:L74-L97", "preview": "```python @triton.jit def gated_delta_net_decode( Q, K, V, Gate, State, Output, qk_dim: tl.constexpr, v_dim: tl.constexpr, d: tl.constexpr, ): \"\"\"Single-token decode: O(d^2) per token.\"\"\" head_id = tl.program_id(0) # Load recurrent state S:", "sha256": "c32dc200aede94bda1a3d670f731b6387fe8938e41abbbd569b42bedf4424d56"}, {"id": "u039", "kind": "code", "locator": "body:L101-L115", "preview": "```python @triton.jit def sparse_attention_fwd(Q, K, V, Output, TopK_Indices, block_size: tl.constexpr, topk: tl.constexpr): \"\"\"Attend to top-k sparse token blocks only.\"\"\" qid = tl.program_id(0) q = tl.load(Q + qid * d + tl.arange(0, d)) a", "sha256": "ccf600826ff33b56d14444be3059e320e7862611dbda1836f2ef520a0a6dcaa7"}, {"id": "u040", "kind": "prose", "locator": "body:L119-L119", "preview": "> The text in this subsection describes Triton 3.5 and earlier. It is preserved for historical accuracy and is NOT a current statement about Triton 3.6+ behavior. Do not cite it as current limitations.", "sha256": "6da22b6a5ba9c96e7aee776b8ef9a69c728bc3e33c85247134e7bc4d4a2571fe"}, {"id": "u041", "kind": "prose", "locator": "body:L121-L121", "preview": "Triton on Blackwell \u2014 Triton 3.5 and earlier:", "sha256": "4f3ef9bb6276d23f39ec6a6cbdf1a3c61ff279369b835f2865345dc4f5aa0e11"}, {"id": "u042", "kind": "list-item", "locator": "body:L123-L123", "preview": "1. **No direct tcgen05 access**: Triton compiler generates wgmma, not tcgen05.", "sha256": "3711959113947130cc509a9ee6c46aaae3ce072abea715211ee6a8770d3a3e91"}, {"id": "u043", "kind": "list-item", "locator": "body:L124-L124", "preview": "2. **No TMEM**: accumulators stay in registers.", "sha256": "0e2de98756b24e1d67726b5699dedcf07f22197cb627ee2bf8ac3a74d183380e"}, {"id": "u044", "kind": "list-item", "locator": "body:L125-L125", "preview": "3. **CPU launch overhead**: impacts small-batch decode latency.", "sha256": "5680cc69a442094ff4ceeb606c6dbe51a26078232b94ee0b3dfe5686ddb6132e"}, {"id": "u045", "kind": "list-item", "locator": "body:L126-L126", "preview": "4. **Workaround**: CUDA graphs (vLLM default for GatedDeltaNet).", "sha256": "81945cbd6eea1657a6f40f973f16264c806ad364d52cc5ebc475f58ef4362f6f"}, {"id": "u046", "kind": "prose", "locator": "body:L128-L128", "preview": "These four bullets describe the world before Triton 3.6.0 landed (`2026-01-21`). The first two are no longer correct on 3.6+; see the \"Triton 3.6+ Blackwell path\" subsection above. CPU launch overhead and CUDA-graph workarounds remain workl", "sha256": "5f83739e9fdc674fb84155ee803c8b0bb485eb56d276477c4aa1ff9beb20f163"}, {"id": "u047", "kind": "prose", "locator": "body:L132-L132", "preview": "The following Triton files ship **verbatim** under `artifacts/prs/` (pinned at each PR's merge SHA). Each PR is in the `triton-in-policy` capture lane defined by `data/inclusion-policy.yaml` \u2014 i.e., SM100-integration, memory-bound, or backe", "sha256": "711382fc92da2e66d7e988c88abebd315f3e2e094f93a1d51666e888b427e914"}, {"id": "u048", "kind": "table-row", "locator": "body:L136-L136", "preview": "| File | Purpose | PR | | [`artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/ops/triton_decode_attention.py`](../../artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/ops/triton_decode_attention.py) | **Post-refresh AC-1.1 prima", "sha256": "f958e8661eb52c00c9ebefef91e0bb48d5ea0e98fc4a489441739b3af7775b48"}, {"id": "u049", "kind": "table-row", "locator": "body:L137-L137", "preview": "| File | Purpose | PR | | [`artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/backends/mla/triton_mla.py`](../../artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/backends/mla/triton_mla.py) | MLA backend wrapping the Triton dec", "sha256": "f34487db257cf6a71387b7ebee7e442cb42fef266c826613d81fdabea5174226"}, {"id": "u050", "kind": "table-row", "locator": "body:L138-L138", "preview": "| File | Purpose | PR | | [`artifacts/prs/flashinfer/PR-1025/key-files/flashinfer/triton/format_conversion.py`](../../artifacts/prs/flashinfer/PR-1025/key-files/flashinfer/triton/format_conversion.py) | FP8 / FP16 format conversion Triton k", "sha256": "65094c66c06ea23d1ad706f5775e334c0a3e2f5f026492617dd1dec185dd7361"}, {"id": "u051", "kind": "table-row", "locator": "body:L139-L139", "preview": "| File | Purpose | PR | | [`artifacts/prs/sglang/PR-20910/key-files/python/sglang/jit_kernel/norm.py`](../../artifacts/prs/sglang/PR-20910/key-files/python/sglang/jit_kernel/norm.py) | Normalization kernels (memory-bound SM100 Triton) | sgl", "sha256": "531e88057dd9e75ceb7fab84ea80b61e967cdd33cbbd798e3c2fdce3999cd9a6"}, {"id": "u052", "kind": "table-row", "locator": "body:L140-L140", "preview": "| File | Purpose | PR | | [`artifacts/prs/sglang/PR-21019/key-files/python/sglang/jit_kernel/triton/gdn_fused_proj.py`](../../artifacts/prs/sglang/PR-21019/key-files/python/sglang/jit_kernel/triton/gdn_fused_proj.py) | GatedDeltaNet fused p", "sha256": "fa5b081e5312d1ff06f63d521006d9e570ebe131b63daa9b73aaca5e59ae7dae"}, {"id": "u053", "kind": "table-row", "locator": "body:L141-L141", "preview": "| File | Purpose | PR | | [`artifacts/prs/sglang/PR-22079/key-files/python/sglang/srt/layers/attention/triton_ops/extend_attention.py`](../../artifacts/prs/sglang/PR-22079/key-files/python/sglang/srt/layers/attention/triton_ops/extend_atten", "sha256": "d48a57dd7fe3e14ac2c2d5e83b3401e31f4546da529c2f045d1c009dc32730c0"}, {"id": "u054", "kind": "prose", "locator": "body:L143-L143", "preview": "The current AC-1.1 **post-refresh primary upstream-code anchor** is `pr-vllm-34597` (above), with the actual Triton decode-attention kernel shipped verbatim. Supplementary post-refresh anchor: `pr-vllm-29339` ([`sources/prs/vllm/PR-29339.md", "sha256": "a2ebf71a94d51403b84d351f951d6feb710d41d7e1ba4feb0f435e91c2bbb5e7"}, {"id": "u055", "kind": "prose", "locator": "body:L145-L145", "preview": "The full 42-PR universe is enumerated in `data/triton-universe.yaml`. Entries marked `captured: false` do not ship locally because they fall outside the three in-policy sub-scopes (see the policy file for reasons).", "sha256": "743fbe592a5d02e4230290c35225d5a2fe3c799a66f6a3964346dc877003cabc"}, {"id": "u056", "kind": "metadata-version", "locator": "frontmatter:version_sensitive", "preview": "{\"id\": \"vs-triton-3.6-blackwell-tcgen05\"}", "sha256": "94601b974ca58d55aea278d0bb7b5a594e5d7bd2a3489a13a5692c252fb4cbf5"}], "confidence_claimed": "verified", "headings": ["Overview", "Triton 3.6+ Blackwell path", "What changed vs the pre-3.6 framing", "Downstream Triton-on-Blackwell adoption (post-3.6 evidence)", "FlashInfer-Bench: AI-Generated Triton Performance", "GatedDeltaNet Decode in Triton", "NSA Sparse Attention in Triton", "Pre-3.6 historical context", "Blackwell Triton Examples (verbatim upstream code shipped locally)"], "id": "lang-triton", "path": "wiki/languages/triton-blackwell.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/triton-3.6-blackwell.md", "url": "https://github.com/triton-lang/triton/releases/tag/v3.6.0"}, {"path": "sources/prs/vllm/PR-34597.md", "revision": "a1257fd1", "url": "https://github.com/vllm-project/vllm/pull/34597"}, {"path": "sources/prs/vllm/PR-29339.md", "revision": "c17610e2", "url": "https://github.com/vllm-project/vllm/pull/29339"}, {"path": "sources/prs/sglang/PR-22079.md", "revision": "5638d40f", "url": "https://github.com/sgl-project/sglang/pull/22079"}, {"path": "sources/prs/sglang/PR-21019.md", "revision": "5bdc07d9", "url": "https://github.com/sgl-project/sglang/pull/21019"}, {"path": "sources/prs/sglang/PR-5390.md", "revision": "84810da4", "url": "https://github.com/sgl-project/sglang/pull/5390"}, {"path": "sources/prs/sglang/PR-21595.md", "revision": "87a27682", "url": "https://github.com/sgl-project/sglang/pull/21595"}, {"path": "sources/prs/pytorch/PR-175826.md", "revision": "a4aea254", "url": "https://github.com/pytorch/pytorch/pull/175826"}, {"path": "sources/blogs/nsa.md", "url": "https://arxiv.org/abs/2502.11089"}, {"path": "sources/blogs/gated-delta-net.md", "url": "https://github.com/NVlabs/GatedDeltaNet"}, {"path": "sources/blogs/flash-attention-4.md", "url": "https://tridao.me/blog/2026/flash4/"}], "risk_flags": ["code", "table", "version"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-triton-3.6-blackwell", "pr-vllm-34597", "pr-vllm-29339", "pr-sglang-22079", "pr-sglang-21019", "pr-sglang-5390", "pr-sglang-21595", "pr-pytorch-175826", "blog-nsa", "blog-gated-delta-net", "blog-flash-attention-4"], "title": "Triton on Blackwell", "type": "language", "unresolved_source_ids": [], "version_sensitive": {"id": "vs-triton-3.6-blackwell-tcgen05"}} +{"body_sha256": "d18a8529e0c9cab4d1711c15938892a04771dbccb5548fc61e5298b0fd6c0ed4", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "On Hopper (SM90), `wgmma` stores MMA accumulators in **registers**. A single m64xn256xk16 BF16 wgmma requires 128 FP32 registers per thread in the warpgroup for the accumulator alone. This extreme register pressure limits tile sizes, reduce", "sha256": "94bc28b16a274d313182fb53a320fc5994a57220e032fdd1911de1ef309b1690"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "On Blackwell (SM100), `tcgen05.mma` stores accumulators in **Tensor Memory (TMEM)** -- a dedicated 256KB per-SM memory. This eliminates accumulator register pressure entirely, freeing registers for data movement, epilogue computation, and e", "sha256": "5ad348aa8df91b7cd3281e6e25ae912742f2819f1f529563a10083814d5cbf92"}, {"id": "u003", "kind": "prose", "locator": "body:L13-L13", "preview": "Each SM90 SM has **65,536 registers** (256KB). A typical GEMM warpgroup needs:", "sha256": "abb30068a5730114e34f09401d793f127fd8e4f12aa358969cbb0e349154c161"}, {"id": "u004", "kind": "code", "locator": "body:L15-L32", "preview": "``` Hopper wgmma m64xn256xk16 BF16 register budget (per thread): Accumulator: 128 FP32 registers (m64 x n256 / 128 threads) A operand: 16 registers (via ldmatrix) B descriptor: 2 registers Loop variables: 8 registers SMEM pointers: 4 regist", "sha256": "0836547cf0e7351f3c32c566064f11e33e14393648d79b325ff5f3a10c80c7ce"}, {"id": "u005", "kind": "prose", "locator": "body:L34-L34", "preview": "If the tile is larger (e.g., m64xn256xk32 with double-buffered accumulators), register pressure becomes even worse:", "sha256": "e00332eabba32f47851f0640e1d12efb90b7d06ad09bc2732eacca4ab998d502"}, {"id": "u006", "kind": "code", "locator": "body:L36-L43", "preview": "``` Double-buffered accumulator: 256 registers per thread Total per thread: ~300 registers Per warpgroup: 38,400 registers Per SM: max 1 warpgroup -> 1 CTA per SM Occupancy: 1 CTA per SM (severe underutilization) ```", "sha256": "bd6c4d4ffd661a52985fe7b9d29cbb8e96217b71f4cd44bda52a52bfb31ce646"}, {"id": "u007", "kind": "prose", "locator": "body:L47-L47", "preview": "When register pressure exceeds the budget, the compiler spills registers to local memory (L1 cache backed by GMEM). This is catastrophic for performance:", "sha256": "e008e70c3ea4465a254f963e9056d6aa7976b04e34c1ecc5a37983eb826965cf"}, {"id": "u008", "kind": "code", "locator": "body:L49-L56", "preview": "``` Register access: ~4 cycles L1 local access: ~30 cycles L2 spill access: ~200 cycles A single spilled accumulator register accessed every MMA iteration can cost 200 cycles * K_tiles additional latency. ```", "sha256": "8af22d45dd1493fe8d24c7b386979bf92eaba2886ae603a9083fee6820a47289"}, {"id": "u009", "kind": "code", "locator": "body:L62-L80", "preview": "``` Blackwell tcgen05 m128xn256xk16 BF16 register budget (per thread): Accumulator: 0 registers (stored in TMEM!) SMEM descriptors: 4 registers Loop variables: 8 registers TMA state: 6 registers TMEM address: 1 register Misc (indexing): 10 ", "sha256": "cf1eabfac0065dea0987082bb280003e431757240919776e543ceadfb9c9f7b9"}, {"id": "u010", "kind": "prose", "locator": "body:L82-L82", "preview": "The key insight: **registers are no longer the bottleneck**. TMEM capacity becomes the binding constraint for occupancy, and the freed registers enable complex epilogues without spilling.", "sha256": "8be6b681cf0019e4088d376a6d3c843ce9391aec4e581c21ddd6ed6544012c5c"}, {"id": "u011", "kind": "code", "locator": "body:L88-L128", "preview": "```cuda // HOPPER: Accumulator lives and dies in registers __global__ void hopper_kernel(/* ... */) { // 1. Declare accumulator in registers float acc[4][32]; // 128 registers per thread! // 2. Zero-initialize #pragma unroll for (int i = 0;", "sha256": "a61c527502ba632a78a4fd949f6bca6b9f73a2b0e12155780b40069c651f0245"}, {"id": "u012", "kind": "code", "locator": "body:L132-L210", "preview": "```cuda // BLACKWELL: Accumulator lives in TMEM __global__ void blackwell_kernel(/* ... */) { // 1. Allocate TMEM (explicit, must be done once) __shared__ uint32_t s_tmem_acc; if (threadIdx.x == 0) { uint32_t smem_addr = static_cast barely fits // m128xn256 would need 256 acc registers/thread -> spills guaranteed // Blackwell: tile size limited by TMEM columns (512) and SMEM, not", "sha256": "d481c73c916610ac7122f12b1041c47d588e156aa81b7d511645c22a99f8374a"}, {"id": "u015", "kind": "prose", "locator": "body:L231-L231", "preview": "On Hopper, complex epilogues (bias + activation + quantization + scaling) often spill because registers are already exhausted by the accumulator. On Blackwell, the epilogue has the full register file available:", "sha256": "350c2675533a141994395ee201d356d2c041d82c89116e0f49becf9bb4538424"}, {"id": "u016", "kind": "code", "locator": "body:L233-L277", "preview": "```cuda // Blackwell epilogue: full register file available __device__ void rich_epilogue(uint32_t tmem_acc, float* output, const float* bias, const float* scale, int M, int N) { // Read TMEM in chunks for (int c = 0; c < 256; c += 8) { // ", "sha256": "8be294f85f22cf8004294d27eb070f564e5be97f3c94c1f8eb5e77c83e039cbc"}, {"id": "u017", "kind": "prose", "locator": "body:L281-L281", "preview": "The most powerful pattern enabled by TMEM: overlapping the current tile's epilogue with the next tile's MMA computation. This is impossible with register accumulators because the accumulator registers are still in use.", "sha256": "d84750528e4034ab0cd2b80c3667bb140a1bee2771ea305a51f92b06de187c4f"}, {"id": "u018", "kind": "code", "locator": "body:L283-L319", "preview": "```cuda __global__ void overlapped_gemm(/* ... */) { // Two TMEM accumulator buffers uint32_t tmem_a = tmem_alloc(256); uint32_t tmem_b = tmem_alloc(256); uint32_t* tmem_cur = &tmem_a; uint32_t* tmem_nxt = &tmem_b; // Compute first tile int", "sha256": "95084d1accb758170351d93c2226e2c5cbd3a57df4a6b3b7d39e6a77dc089cf9"}, {"id": "u019", "kind": "prose", "locator": "body:L323-L323", "preview": "FlashAttention is a prime example where the register-to-TMEM migration unlocks major gains.", "sha256": "c73cd9b7a97e9dd1c77615a7d4a9245551d51a6b6e96e862ef839f36c04bc314"}, {"id": "u020", "kind": "code", "locator": "body:L327-L343", "preview": "``` FlashAttention on Hopper needs registers for: - QK^T accumulator: 64 registers (m64xn64 attention scores) - PV accumulator: 128 registers (m64xn256 output) - Softmax state: 4 registers (rowmax, rowsum) - Q fragment: 16 registers - K fra", "sha256": "7ac41e773180950eb33f74a30b2d965893228ef12d4dc445bd82c4576628b205"}, {"id": "u021", "kind": "code", "locator": "body:L347-L362", "preview": "``` FlashAttention on Blackwell: - QK^T accumulator: 0 registers (TMEM buffer 1, 64 cols) - PV accumulator: 0 registers (TMEM buffer 2, 256 cols) - Softmax state: 4 registers (rowmax, rowsum) - SMEM descriptors: 6 registers - Loop state: 10", "sha256": "15d29a84b1e95def0740b3cf71fdf32dc54907436295d3c58842e371838f7367"}, {"id": "u022", "kind": "prose", "locator": "body:L364-L364", "preview": "FlashAttention-4 exploits this headroom for software-emulated exponentials (distributing `2^x` across FMA units instead of waiting for the SFU), achieving **1605 TFLOPS on B200** at 71% utilization.", "sha256": "e592e19c4bb0adb4adc1043b5a5291dfc3fa3201a55dce34792669db5ce7236d"}, {"id": "u023", "kind": "code", "locator": "body:L370-L377", "preview": "```cuda // WRONG: Trying to use TMEM values directly in expressions float result = tmem_acc[row][col] + bias; // TMEM is not directly addressable! // CORRECT: Load from TMEM to register, then compute float val = tmem_load_f32(tmem_col); flo", "sha256": "4240bdd36422138d9cbf897ee573a125789a1ffeec13c9d9efb11295aa89ff8a"}, {"id": "u024", "kind": "code", "locator": "body:L381-L396", "preview": "```cuda // WRONG: Fine-grained TMEM access in a tight loop (high latency) for (int i = 0; i < 256; ++i) { float v = tmem_load_f32(tmem_acc + i); // 420 cycle latency each! output[i] = v; } // CORRECT: Vectorized loads to amortize latency fo", "sha256": "4286c17c4e3596989f926894c133b025507b85cf49947a0a219f14d892a7023f"}, {"id": "u025", "kind": "code", "locator": "body:L400-L406", "preview": "```cuda // Hopper: tile size was m64xn128 to avoid register spilling // Migrating to Blackwell: keep same tile = leaving performance on the table // Blackwell should use at minimum m128xn256 (1-SM) or m256xn256 (2-SM) // The freed registers", "sha256": "89e6494492b37559bcc5b0da5b1024212928402676468b9259995ec00f71b231"}, {"id": "u026", "kind": "code", "locator": "body:L410-L433", "preview": "```cuda // WRONG: Forgetting to dealloc in a persistent kernel __global__ void persistent_kernel(/* ... */) { uint32_t tmem = tmem_alloc(256); // Allocated once while (has_work()) { compute_tile(tmem); // BUG: If the kernel exits early (e.g", "sha256": "763dadf035f8d9922e0f64ea7e3c59f980c5a10c87dde8ef7553dad46a4c5153"}, {"id": "u027", "kind": "table-row", "locator": "body:L439-L439", "preview": "| Aspect | Register (Hopper) | TMEM (Blackwell) | | Accumulator declaration | `float acc[N]` in registers | `tmem_alloc(cols)` |", "sha256": "d3dedb3d3966d121536408047cab627ae29ffceaabb5b2a45735e6f12d0ef2bb"}, {"id": "u028", "kind": "table-row", "locator": "body:L440-L440", "preview": "| Aspect | Register (Hopper) | TMEM (Blackwell) | | Zero-initialization | Loop over register array | `tcgen05.st` zero pattern |", "sha256": "27af4af93c6ecc620c3b3c937e8ecd8ac5123a3cfbb0b14e9a8e37551c22b4fa"}, {"id": "u029", "kind": "table-row", "locator": "body:L441-L441", "preview": "| Aspect | Register (Hopper) | TMEM (Blackwell) | | MMA accumulation | Implicit (wgmma writes regs) | Implicit (tcgen05 writes TMEM) |", "sha256": "d26a8710ccc5958ea4d2231bd3cce6cca88cb787613eb0c6c0c4ed4fb0dfdbb5"}, {"id": "u030", "kind": "table-row", "locator": "body:L442-L442", "preview": "| Aspect | Register (Hopper) | TMEM (Blackwell) | | Reading results | Direct register access | `tcgen05.ld` to register |", "sha256": "e1210631e69cd6ed454c92ca37d7024fa2be5c2f8ccadd99a5218c68128e97dc"}, {"id": "u031", "kind": "table-row", "locator": "body:L443-L443", "preview": "| Aspect | Register (Hopper) | TMEM (Blackwell) | | Cleanup | Implicit (CTA exit) | Explicit `tcgen05.dealloc` |", "sha256": "b13398f17871ab8c8763378af8a6c2ae9b5ebf2f9942484c99667bf84cb71f80"}, {"id": "u032", "kind": "table-row", "locator": "body:L444-L444", "preview": "| Aspect | Register (Hopper) | TMEM (Blackwell) | | Register pressure | ~128-256 regs for accumulator | ~0 regs for accumulator |", "sha256": "941c758fac43a685ea2d7596da3965c7d4bff5792fd5a52687d006f914c111e6"}, {"id": "u033", "kind": "table-row", "locator": "body:L445-L445", "preview": "| Aspect | Register (Hopper) | TMEM (Blackwell) | | Epilogue headroom | Minimal (spill risk) | Ample (full register file) |", "sha256": "c57a3e83cf3e739a20fc3375871e2f824bca9cc247643c005b7ee12c634c14ea"}, {"id": "u034", "kind": "table-row", "locator": "body:L446-L446", "preview": "| Aspect | Register (Hopper) | TMEM (Blackwell) | | Double-buffering | Doubles register pressure (2x acc) | Uses 2 TMEM regions (no reg impact) |", "sha256": "607deb95f607fb5de40cd2a3dfef1daabcc2e5f6c7d17e509591e8fb5da0fe15"}, {"id": "u035", "kind": "table-row", "locator": "body:L447-L447", "preview": "| Aspect | Register (Hopper) | TMEM (Blackwell) | | Max practical tile | m64xn256 (limited by registers) | m128xn256 or m256xn256 (limited by TMEM cols) |", "sha256": "4136812cfdccdee9964191f6238491b06bd295dcb150e75b3d18d4d3bdbf446e"}], "confidence_claimed": "source-reported", "headings": ["Overview", "The Register Pressure Problem on Hopper", "Register Budget Analysis", "Register Spilling on Hopper", "Blackwell TMEM Solution", "TMEM Budget Analysis", "Migration: Accumulator Lifecycle", "Hopper: Register-Based Lifecycle", "Blackwell: TMEM-Based Lifecycle", "Impact on Kernel Design", "Tile Size Freedom", "Epilogue Complexity", "Overlapped Epilogue and MMA", "FlashAttention Case Study", "Hopper FlashAttention Register Pressure", "Blackwell FlashAttention with TMEM", "Common Mistakes During Migration", "Mistake 1: Treating TMEM Like Registers", "Mistake 2: Forgetting TMEM Latency", "Mistake 3: Not Re-tuning Tile Sizes", "Mistake 4: TMEM Leak in Persistent Kernels", "Summary: What Changes, What Stays"], "id": "migration-register-to-tmem", "path": "wiki/migration/register-to-tmem.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/prs/vllm/PR-22738.md", "revision": "b1361c72", "url": "https://github.com/vllm-project/vllm/pull/22738"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "pr-vllm-22738"], "title": "Register Accumulators to TMEM", "type": "migration", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "0c2cada28831223c7f5b4dc10e20b9007129747d3659ed5f7e0aefbe340281d9", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "This guide covers the migration from Hopper's `wgmma.mma_async` (SM90) to Blackwell's `tcgen05.mma` (SM100). This is the single most impactful change when porting kernels from H100/H200 to B200. The two instructions differ in:", "sha256": "e3e242ad8530bd6bafc064bcec677b74cffaeb35b4f64bc1fd754d5207bd70e6"}, {"id": "u002", "kind": "list-item", "locator": "body:L7-L7", "preview": "- **Issuing model**: warpgroup (128 threads) vs single thread", "sha256": "069718d53c82b623c5e10e3c61efa1cfba1fc85ec00f851ce0bd2a5c5e5f9bda"}, {"id": "u003", "kind": "list-item", "locator": "body:L8-L8", "preview": "- **Accumulator storage**: registers vs TMEM", "sha256": "926d5457957e54352f0eec613608c84cc4c5d2b20a478946fc015091a8b2f331"}, {"id": "u004", "kind": "list-item", "locator": "body:L9-L9", "preview": "- **Operand loading**: ldmatrix + registers vs direct SMEM access", "sha256": "51affba4e422e3a7e48fb31940d581f3c2081e6f32050df13744a2e43e63e340"}, {"id": "u005", "kind": "list-item", "locator": "body:L10-L10", "preview": "- **Synchronization**: warpgroup barriers vs async fences", "sha256": "f57e261c89f01976603e55a1dcc49a5f778870a655ad5d61acf804d3fb4ac3c1"}, {"id": "u006", "kind": "list-item", "locator": "body:L11-L11", "preview": "- **Available data types**: FP8 (Hopper) vs FP4/FP6/FP8 with block scaling (Blackwell)", "sha256": "5601d239aa4461719db571fa0b7b67111e8dcfd20e6521a45ccd81516267485b"}, {"id": "u007", "kind": "code", "locator": "body:L15-L25", "preview": "``` [ ] Replace wgmma.mma_async with tcgen05.mma [ ] Move accumulators from registers to TMEM (alloc/dealloc) [ ] Remove ldmatrix operations (tcgen05 reads directly from SMEM) [ ] Change SMEM swizzle from 64B to 128B [ ] Replace warpgroup c", "sha256": "653ba85078953043730a032a53477bbb86ee80296cb3450c484bd1f91f44727b"}, {"id": "u008", "kind": "code", "locator": "body:L31-L84", "preview": "```cuda // SM90 GEMM kernel using wgmma __global__ void hopper_gemm( const half* A, const half* B, half* C, int M, int N, int K ) { extern __shared__ char smem[]; half* smem_a = reinterpret_cast(smem); half* smem_b = smem_a + TILE_M ", "sha256": "3288a5cc14b9e820c8daa234bb1bac17fa8504387859902a5d0631c53a4ac50a"}, {"id": "u009", "kind": "code", "locator": "body:L88-L155", "preview": "```cuda // SM100 GEMM kernel using tcgen05 __global__ void blackwell_gemm( const half* A, const half* B, half* C, int M, int N, int K ) { extern __shared__ char smem[]; half* smem_a = reinterpret_cast(smem); half* smem_b = smem_a + T", "sha256": "862c80edc308ce79a6143756f049df61caef25d71c1f4d807c7431e113987dd5"}, {"id": "u010", "kind": "prose", "locator": "body:L161-L161", "preview": "**Before (Hopper):** Accumulators live in registers.", "sha256": "5649d9a74080d859e2254267f09440409574e8ded6d1ef73b4ef887d46299a3e"}, {"id": "u011", "kind": "code", "locator": "body:L163-L166", "preview": "```cuda // Hopper: 128 FP32 registers for a 64x256 accumulator fragment float acc[4][32]; // Per-thread fragment of the warpgroup accumulator ```", "sha256": "4fb80b998cef798698f4471ede13251306a689b760d3d3d281148410c57f51e3"}, {"id": "u012", "kind": "prose", "locator": "body:L168-L168", "preview": "**After (Blackwell):** Accumulators live in TMEM.", "sha256": "f225e5b706eee44215db50401ea5fb88c57367ed3fb0cdbf1cc0433e00e51695"}, {"id": "u013", "kind": "code", "locator": "body:L170-L173", "preview": "```cuda // Blackwell: TMEM address replaces register array uint32_t tmem_acc = tmem_alloc(256); // 128 rows x 256 cols in TMEM ```", "sha256": "1ef83b7286b3acae2e779ba613b7801b41e882e81d789715478d0aa2f237054a"}, {"id": "u014", "kind": "prose", "locator": "body:L177-L177", "preview": "**Before (Hopper):** Load A operand from SMEM to registers.", "sha256": "1f697fb70990e78dd465b89ef9632a1f6ba12ce93de5f214282d797ceb43bfe1"}, {"id": "u015", "kind": "code", "locator": "body:L179-L189", "preview": "```cuda // Hopper: ldmatrix loads 8x8 matrix fragments into registers uint32_t a_frag[4]; asm volatile( \"ldmatrix.sync.aligned.m8n8.x4.shared.b16 \" \"{%0,%1,%2,%3}, [%4];\" : \"=r\"(a_frag[0]), \"=r\"(a_frag[1]), \"=r\"(a_frag[2]), \"=r\"(a_frag[3]) ", "sha256": "56b45537bace00ebc3ef75ebcdfc3ce9addd458f471ca4de8f6e0a57070e5f52"}, {"id": "u016", "kind": "prose", "locator": "body:L191-L191", "preview": "**After (Blackwell):** No equivalent needed. tcgen05 reads A directly from SMEM via descriptor.", "sha256": "9cdac1fbfecbb43ebe2742e2c6865fb2f05f0e97d2e6de97ad2b7192ed69d05a"}, {"id": "u017", "kind": "code", "locator": "body:L193-L197", "preview": "```cuda // Blackwell: just construct the SMEM descriptor uint64_t desc_a = make_smem_desc_128b(smem_a_ptr); // Pass desc_a directly to tcgen05.mma -- no register staging ```", "sha256": "e1d1d30efba8fdc3375e6632e2796d546158651b7b1a47afad420d836fba5bcf"}, {"id": "u018", "kind": "prose", "locator": "body:L201-L201", "preview": "**Before (Hopper):** 64-byte or 128-byte swizzle both work for wgmma.", "sha256": "fca92c2012f2548b5ee73757f411cc28972cc3b8492382eedd5f103a99fd7797"}, {"id": "u019", "kind": "code", "locator": "body:L203-L207", "preview": "```cuda // Hopper TMA descriptor: 64B swizzle is common CUtensorMap desc = create_tma_desc(ptr, M, N, tile_m, tile_n, CU_TENSOR_MAP_SWIZZLE_64B); ```", "sha256": "12b76455f0d53a44e82c331f166969538300d03b7f6057805035915ca92b24c2"}, {"id": "u020", "kind": "prose", "locator": "body:L209-L209", "preview": "**After (Blackwell):** 128-byte swizzle is mandatory.", "sha256": "229c087640cb29c29256903515cb9d8d3cd9247cddc9c7cf12d332ff4f2be34b"}, {"id": "u021", "kind": "code", "locator": "body:L211-L215", "preview": "```cuda // Blackwell TMA descriptor: MUST use 128B swizzle CUtensorMap desc = create_tma_desc(ptr, M, N, tile_m, tile_n, CU_TENSOR_MAP_SWIZZLE_128B); ```", "sha256": "292f7acaabb39ec9e6acc4624c767b6888c3195fe27779de9f33aa3dccc20a97"}, {"id": "u022", "kind": "prose", "locator": "body:L219-L219", "preview": "**Before (Hopper):**", "sha256": "4ddd181f824434b74af8a064e60f3f7f3bbfca6050689028251cb278634bec37"}, {"id": "u023", "kind": "code", "locator": "body:L221-L230", "preview": "```cuda // Hopper: all 128 threads in warpgroup issue wgmma asm volatile( \"wgmma.mma_async.sync.aligned.m64n256k16.f32.bf16.bf16 \" \"{%0,%1,...}, {%N,...}, desc_b, ...;\" : \"+f\"(acc[0]), \"+f\"(acc[1]), ... : \"r\"(a_frag[0]), ... ); asm volatile", "sha256": "f97d16ee60f9dc5d3a760fc7dfed30dc1caa93a5d3eed6760b5a0ac06900697c"}, {"id": "u024", "kind": "prose", "locator": "body:L232-L232", "preview": "**After (Blackwell):**", "sha256": "2a7df142d777617140c33c51cd502cf20555267d6709293143e290b5b0c91e43"}, {"id": "u025", "kind": "code", "locator": "body:L234-L245", "preview": "```cuda // Blackwell: single thread issues tcgen05 if (threadIdx.x == 0) { asm volatile( \"tcgen05.mma.cta_group::1.kind::f16 \" \"[%0], %1, %2, %3, 1;\" : : \"r\"(tmem_acc), \"l\"(desc_a), \"l\"(desc_b), \"r\"(0) ); } // No commit needed -- fully asyn", "sha256": "c178942b5044a0b480642a53020bab6b3ed43025ff861856a83c72761e920f62"}, {"id": "u026", "kind": "prose", "locator": "body:L249-L249", "preview": "**Before (Hopper):**", "sha256": "4ddd181f824434b74af8a064e60f3f7f3bbfca6050689028251cb278634bec37"}, {"id": "u027", "kind": "code", "locator": "body:L251-L255", "preview": "```cuda // Hopper: wait for outstanding wgmma groups asm volatile(\"wgmma.wait_group.sync.aligned 0;\"); // Accumulators now ready in registers ```", "sha256": "993484703e3e86fd38fb60ac89bf1b65590c1ffd0dd60dc5fb9adf0e991b2e1c"}, {"id": "u028", "kind": "prose", "locator": "body:L257-L257", "preview": "**After (Blackwell):**", "sha256": "2a7df142d777617140c33c51cd502cf20555267d6709293143e290b5b0c91e43"}, {"id": "u029", "kind": "code", "locator": "body:L259-L264", "preview": "```cuda // Blackwell: fence before reading TMEM asm volatile(\"tcgen05.fence::before_thread_sync;\"); __syncthreads(); // TMEM accumulators now ready for reading ```", "sha256": "643c4c9c508ecd528c3ba2a0da7b172c669513c3855b368b2a39643940368987"}, {"id": "u030", "kind": "prose", "locator": "body:L268-L268", "preview": "**Before (Hopper):**", "sha256": "4ddd181f824434b74af8a064e60f3f7f3bbfca6050689028251cb278634bec37"}, {"id": "u031", "kind": "code", "locator": "body:L270-L279", "preview": "```cuda // Hopper: accumulators in registers, directly usable float result = acc[frag_m][frag_n]; // Apply bias result += bias[col]; // Apply activation result = relu(result); // Store to GMEM C[row * N + col] = (half)result; ```", "sha256": "6f9b7dcb620b7cff69ee98686a65251d68dded33d3d3d5fa876304f765f42da4"}, {"id": "u032", "kind": "prose", "locator": "body:L281-L281", "preview": "**After (Blackwell):**", "sha256": "2a7df142d777617140c33c51cd502cf20555267d6709293143e290b5b0c91e43"}, {"id": "u033", "kind": "code", "locator": "body:L283-L292", "preview": "```cuda // Blackwell: must read from TMEM first float result = tmem_load_f32(tmem_acc + col_offset); // Apply bias result += bias[col]; // Apply activation result = relu(result); // Store to GMEM C[row * N + col] = (half)result; ```", "sha256": "6ae2d29d9838b4eef2a63fb9d732a67b1604f3ed222ab56035861c8d4de1913b"}, {"id": "u034", "kind": "code", "locator": "body:L298-L305", "preview": "``` Warp 0-3: Warpgroup 0 -- MMA producer (all 128 threads issue wgmma) Warp 4-7: Warpgroup 1 -- MMA producer (backup/double-buffer) Warp 8-11: Data movement (TMA loads, SMEM management) Warp 12: Tile scheduler Total: 13+ warps, 416+ thread", "sha256": "c0f994a3dee4bb7680e3b6ea94b40923704209feaf7c444db3008faf0fcdb5e4"}, {"id": "u035", "kind": "code", "locator": "body:L309-L316", "preview": "``` Warp 0: MMA producer (single thread issues tcgen05) Warp 1-2: TMA data movement (load A, load B, manage barriers) Warp 3: Epilogue (read TMEM, apply post-ops, store to GMEM) Warp 4: Tile scheduler / CLC management Total: 5 warps, 160 th", "sha256": "0e2936e83971ca051e8b1d67b232bce183cf4eb79a350032cf80ce74801a1061"}, {"id": "u036", "kind": "prose", "locator": "body:L318-L318", "preview": "With tcgen05, far fewer warps are dedicated to MMA because only one thread is needed to issue the instruction. The freed warps can be repurposed for:", "sha256": "96b41eb8a9041d0e1ad345ba3f086eba22c8e8484934fc39bc727dc459d62994"}, {"id": "u037", "kind": "list-item", "locator": "body:L319-L319", "preview": "- More aggressive data prefetching", "sha256": "5bcd30b3457c22f7976929d684cb6a37205df7a46ebcea2c8c87d57d059f071f"}, {"id": "u038", "kind": "list-item", "locator": "body:L320-L320", "preview": "- Overlapped epilogue (read TMEM while next tile's MMA is computing)", "sha256": "8d6b594c33f2a5386d91c7e5c576933640024ec756b3923c3cc92a23dda13e67"}, {"id": "u039", "kind": "list-item", "locator": "body:L321-L321", "preview": "- Softmax or other reductions (FlashAttention-style)", "sha256": "fad70bd397ac53bc68809bee25a160dfa7d95eb902ea5506fcf9ea98c2fb46b1"}, {"id": "u040", "kind": "table-row", "locator": "body:L327-L327", "preview": "| Hopper | Blackwell (1-SM) | Blackwell (2-SM) | | m64 x n256 x k16 | m128 x n256 x k16 | m256 x n256 x k16 |", "sha256": "19180cb2b51bb63c0cc758c204d24a60646f4195a1bf1bd1d48e8662d4a784fe"}, {"id": "u041", "kind": "table-row", "locator": "body:L328-L328", "preview": "| Hopper | Blackwell (1-SM) | Blackwell (2-SM) | | m64 x n128 x k16 | m128 x n128 x k16 | m256 x n128 x k16 |", "sha256": "037021919a58f4625f5512ddabfe101bd8a3637fd109d0936b6467b664aec4a8"}, {"id": "u042", "kind": "prose", "locator": "body:L330-L330", "preview": "Blackwell's base MMA tile is 2x larger in M (128 vs 64). When migrating:", "sha256": "7dd9c89a913ca7a27ec23a6ecfc5616e89c940d1d94a4fbd4613123219796b8f"}, {"id": "u043", "kind": "list-item", "locator": "body:L331-L331", "preview": "- CTA tile size of 128x256 maps naturally to a single tcgen05 MMA", "sha256": "c86455a9c8c6cdb905f1b567bd51bc98250dcdb63a4c577a5977f72845399b71"}, {"id": "u044", "kind": "list-item", "locator": "body:L332-L332", "preview": "- CTA tile size of 64x256 on Hopper should be doubled to 128x256", "sha256": "5e766c9f0f14f49b359adb752bf222e4dbce5dcbcc196063826511c24d3f783c"}, {"id": "u045", "kind": "list-item", "locator": "body:L333-L333", "preview": "- For 2-SM mode, consider 256x256 tiles", "sha256": "c2b6a904583fa69c266fc75a303c539420ff197a11b31ee414ff26bd6cb862b9"}, {"id": "u046", "kind": "list-item", "locator": "body:L337-L337", "preview": "1. **Forgetting 128B swizzle**: The most common silent-failure bug. wgmma works with 64B swizzle; tcgen05 does not. Results will be numerically wrong but the kernel won't crash.", "sha256": "c0ac2b2981c54f4cabc1dc8e1707abadc2bc552de03aea72e737f3d6e448aa55"}, {"id": "u047", "kind": "list-item", "locator": "body:L339-L339", "preview": "2. **Not deallocating TMEM**: On Hopper, register accumulators are freed implicitly when the CTA exits. On Blackwell, TMEM must be explicitly deallocated in persistent kernels.", "sha256": "a568178da91fbbb2dfa8ad92fe09b2ef3de508c872de7eb5888177eedd1215b9"}, {"id": "u048", "kind": "list-item", "locator": "body:L341-L341", "preview": "3. **Synchronization model mismatch**: Replacing `wgmma.wait_group` with `__syncthreads()` alone is insufficient. The `tcgen05.fence::before_thread_sync` must precede the syncthreads.", "sha256": "3e84f8a6f8b39f0130cd6a848b938f4cac3ab442361ab79bde3c460e5a57941e"}, {"id": "u049", "kind": "list-item", "locator": "body:L343-L343", "preview": "4. **Over-allocating threads**: On Hopper, you need 128 threads per warpgroup for MMA. Blindly keeping the same thread count on Blackwell wastes resources since only 1 thread issues tcgen05.", "sha256": "7e7dda62c6c62d22ef734d67601bd2db01f658b6d3fa93163d8b9700a593d9ae"}, {"id": "u050", "kind": "list-item", "locator": "body:L345-L345", "preview": "5. **Register pressure assumptions**: Code tuned for Hopper's high register pressure (e.g., reduced tile sizes, manual spilling) may be overly conservative on Blackwell. Re-tune tile sizes to take advantage of freed registers.", "sha256": "827a5dcfa8697efffb61200fc3028ab340a9469f09956a61aedd57c78ee28fca"}, {"id": "u051", "kind": "prose", "locator": "body:L349-L349", "preview": "If using CUTLASS, the migration is largely handled by changing the arch tag and kernel schedule:", "sha256": "60cb963d24a0cc1abfd438742f9fc0a941b9732e70eae17e1510001bc341dbaf"}, {"id": "u052", "kind": "code", "locator": "body:L351-L367", "preview": "```cpp // Hopper CUTLASS GEMM using GemmHopper = cutlass::gemm::device::GemmUniversal< /* ... */ cutlass::arch::Sm90, /* ... */ cutlass::gemm::collective::KernelScheduleSm90CpAsyncWarpSpecialized >; // Blackwell CUTLASS GEMM -- change arch ", "sha256": "c7252763e4fba15199fc8601dee41acec6cd8cabc2c7925b3cc849f79936125e"}, {"id": "u053", "kind": "prose", "locator": "body:L369-L369", "preview": "CUTLASS handles the internal differences (TMEM allocation, descriptor construction, fence insertion, 128B swizzle) automatically through its `MMA_Atom` and `MMA_Traits` abstractions for SM100.", "sha256": "6d4ddbfa46bd23167c929dc9f9e7d23b1199a1b374b6a4f55e54d82351265e0b"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Migration Checklist", "Side-by-Side Comparison", "Hopper Kernel Structure (SM90)", "Blackwell Kernel Structure (SM100)", "Step-by-Step Migration", "Step 1: Replace Accumulator Storage", "Step 2: Remove ldmatrix", "Step 3: Change SMEM Swizzle from 64B to 128B", "Step 4: Replace wgmma Issue with tcgen05 Issue", "Step 5: Replace wgmma.wait_group with tcgen05 Fence", "Step 6: Update Epilogue", "Warp Specialization Changes", "Hopper Warp Specialization (Typical)", "Blackwell Warp Specialization (Typical)", "Tile Size Migration", "Common Migration Pitfalls", "CUTLASS Migration"], "id": "migration-wgmma-to-tcgen05", "path": "wiki/migration/wgmma-to-tcgen05.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/blogs/colfax-cutlass-blackwell.md", "url": "https://research.colfax-intl.com/cutlass-tutorial-writing-gemm-kernels-using-tmem-for-nvidia-blackwell-gpus/"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "blog-colfax-cutlass"], "title": "Migrating from wgmma to tcgen05", "type": "migration", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "1250f2c8d05fdd1ec4dcb30133b71caf29617b7fe0c0131fcafbe9ca9a4fe320", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "Tensor core utilization below 70%. Memory bandwidth is not saturated. Kernel is compute-bound but not reaching peak FLOPS.", "sha256": "1f2087903c90937aea23fad886c9bd9dad2042ba1b24dc335790e3b4e8440c18"}, {"id": "u002", "kind": "list-item", "locator": "body:L7-L7", "preview": "1. **Pipeline bubbles**: MMA stalled waiting for data from TMA", "sha256": "8d8f8d98b5d284392eb71a872e7ff7876964a8922e8f99bbfc723963dccaa759"}, {"id": "u003", "kind": "list-item", "locator": "body:L8-L8", "preview": "2. **Non-matmul overhead**: Softmax, activation functions, reductions consuming cycles", "sha256": "b59e45b681033f13916e2a83ec38bb18ffadff01f2b41958e10b53e730b62286"}, {"id": "u004", "kind": "list-item", "locator": "body:L9-L9", "preview": "3. **Single-SM MMA tiles too small**: Not fully utilizing available compute", "sha256": "5fe2753baa61aa1db088650b5b7317f415cc6545f02b83a04da987b094883742"}, {"id": "u005", "kind": "list-item", "locator": "body:L10-L10", "preview": "4. **Epilogue blocking mainloop**: TMEM reads blocking next MMA", "sha256": "5d06c039c6e044e324150c739ce382c90677b38d261b0617f6fb83b760dc1d26"}, {"id": "u006", "kind": "table-row", "locator": "body:L16-L16", "preview": "| Technique | Effect | | [2-SM cooperative](../hardware/2sm-cooperative.md) | Double effective MMA tile (m256\u00d7n256), 2\u00d7 compute per cycle |", "sha256": "0a6da9a08dac60aaf7062e28ab14512e4bdf1cd73685725a408b3e9074eee114"}, {"id": "u007", "kind": "table-row", "locator": "body:L17-L17", "preview": "| Technique | Effect | | [Pipeline stages](../techniques/pipeline-stages.md) | Overlap TMA load with MMA compute |", "sha256": "2388aa7c0bd73280dfc9be73d8895f4220c93eea99557724b369bb78aa937302"}, {"id": "u008", "kind": "table-row", "locator": "body:L18-L18", "preview": "| Technique | Effect | | [Warp specialization](../techniques/warp-specialization.md) | Dedicated warps for TMA/MMA/epilogue, no stalls |", "sha256": "b3dd06a59b15e37406b2c7bf07e43bffbc903c886e3f311170e4aba3d69df642"}, {"id": "u009", "kind": "table-row", "locator": "body:L19-L19", "preview": "| Technique | Effect | | [Epilogue fusion](../techniques/epilogue-fusion.md) | Overlap epilogue with next tile's MMA |", "sha256": "59fef48221c33e68b699c20bbd960de02fec62482f07ef3f5dc228038c5071a0"}, {"id": "u010", "kind": "table-row", "locator": "body:L20-L20", "preview": "| Technique | Effect | | [Software exponential](../techniques/software-exp.md) | Distribute non-matmul ops across FMA units (FA4) |", "sha256": "2e15eb9eacf4f6a259e0974f9fb374b9f26ef373d8ffd2015441a5639849b0bc"}, {"id": "u011", "kind": "code", "locator": "body:L24-L31", "preview": "``` // Problem: Blackwell doubles tensor core throughput but SFU count unchanged // SFU bottleneck: exp() for softmax // // Solution: Software 2^x via Cody-Waite + Horner polynomial // Distributes across FMA units, multiplying exponential t", "sha256": "a61d1e90989365d653a026b634ebe7cec74dae4236636c34adcbc4552d74e68d"}, {"id": "u012", "kind": "list-item", "locator": "body:L34-L34", "preview": "- 2-SM cooperative requires cluster configuration and identical SMEM layouts", "sha256": "722ad882e22729c3c52ced4b0a7474952f575258c7cf5c04d4f652c79d42cf11"}, {"id": "u013", "kind": "list-item", "locator": "body:L35-L35", "preview": "- Pipeline depth tuning is workload-dependent (3-5 stages typical)", "sha256": "b64cb0709c42ee988816b64d1e8cb332dd7ef083fd04a3fb33edfd8d19477d3e"}, {"id": "u014", "kind": "list-item", "locator": "body:L36-L36", "preview": "- Software-emulated transcendentals trade accuracy for throughput", "sha256": "01099e8879a24004b8a230c4c110825633f6b2c7db203ce54be25ba698ea6482"}], "confidence_claimed": null, "headings": ["Symptom", "Likely Causes", "Candidate Techniques", "Example: FlashAttention-4", "Caveats"], "id": "pattern-compute-bound", "path": "wiki/patterns/compute-bound.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/blogs/flash-attention-4.md", "url": "https://tridao.me/blog/2026/flash4/"}], "risk_flags": ["code", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "blog-flash-attention-4"], "title": "Not Reaching Peak FLOPS", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "ffe8f3f021f4946f96b95bfc41d6745d38c52fd887cb7171751f8f4e9934e91b", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "SM utilization below 60% despite sufficient occupancy. Nsight Compute shows idle SMs during portions of kernel execution.", "sha256": "5da93aae9fc0084ad5cf53f525a2493b47fbca4ac3986626652e90a0e231340d"}, {"id": "u002", "kind": "list-item", "locator": "body:L7-L7", "preview": "1. **Tail effect**: Last wave of tiles leaves most SMs idle (see [tail-effect](tail-effect.md))", "sha256": "bfb85873942f4214c8682b3b6512f034b15e5245ef6bbda1422c66d1693872bf"}, {"id": "u003", "kind": "list-item", "locator": "body:L8-L8", "preview": "2. **Load imbalance**: Some tiles take longer than others (variable computation per tile)", "sha256": "9aa73dca1ac150ce3bca32374afb0a63d324c1e49fc39188a2eb668c7d3da68e"}, {"id": "u004", "kind": "list-item", "locator": "body:L9-L9", "preview": "3. **Static scheduling**: Fixed tile-to-SM assignment doesn't adapt to runtime conditions", "sha256": "4697d65e3b2165c234214aa9f4ecfc8cc2d13ae7593da020ec00fc21fe4f5a08"}, {"id": "u005", "kind": "list-item", "locator": "body:L10-L10", "preview": "4. **Grid too small**: Fewer threadblocks than SMs", "sha256": "c3216052f73490a7acb8b4e99e3585d659df7c9013e9775b079e4610da44874a"}, {"id": "u006", "kind": "table-row", "locator": "body:L16-L16", "preview": "| Technique | Applicability | Effect | | [CLC](../hardware/clc.md) | SM100 only | Dynamic tile assignment, eliminates load imbalance |", "sha256": "0a3e3d868a2182503591b2b4a9e2aada8bfe7c0f20e5da17168dac14435f8de7"}, {"id": "u007", "kind": "table-row", "locator": "body:L17-L17", "preview": "| Technique | Applicability | Effect | | [Persistent kernels](../techniques/persistent-kernels.md) | SM90+ | Eliminates tail effect, one-time launch overhead |", "sha256": "319e40604927d4afb11eb2787237c3632bfb85dbaa4fcd6f60ee93adedd3b100"}, {"id": "u008", "kind": "table-row", "locator": "body:L18-L18", "preview": "| Technique | Applicability | Effect | | [Tile scheduling](../techniques/tile-scheduling.md) | SM90+ | Better L2 locality, reduce load variance |", "sha256": "1b99d25e571454ca800ab956d044e6f3d5347f68d9ee4adb17948241afe2a264"}, {"id": "u009", "kind": "code", "locator": "body:L22-L26", "preview": "``` // tcgen05 tutorial progression: // Without persistent/CLC: 86% of cuBLAS (some SMs idle at wave boundaries) // With persistent + CLC: 98% of cuBLAS (all SMs stay busy) ```", "sha256": "fce47d285cec29c94d16ea9765bdda1f464614a35272f1aa18734006042a2e7f"}, {"id": "u010", "kind": "list-item", "locator": "body:L29-L29", "preview": "- CLC only available on SM100 datacenter GPUs (not SM120 consumer)", "sha256": "75d8555a04ed8e4b2c78530b00dd21f22e15736b43db5381874c644498e7c52c"}, {"id": "u011", "kind": "list-item", "locator": "body:L30-L30", "preview": "- Persistent kernels complicate debugging and profiling", "sha256": "694e7865c9620dbacb087ecce133f9d3de68af1d6f971c6398927d4947f9066a"}, {"id": "u012", "kind": "list-item", "locator": "body:L31-L31", "preview": "- For non-persistent kernels, ensure grid size >> SM count", "sha256": "9e87656f10cad9c1ff43dc0b5bd3517c0b5035f87cd8cb4e7da8b657e4cc3b79"}], "confidence_claimed": null, "headings": ["Symptom", "Likely Causes", "Candidate Techniques", "Examples", "Caveats"], "id": "pattern-low-sm-utilization", "path": "wiki/patterns/low-sm-utilization.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/prs/cutlass/PR-2161.md", "revision": "06e560d9", "url": "https://github.com/NVIDIA/cutlass/pull/2161"}], "risk_flags": ["code", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "pr-cutlass-2161"], "title": "Low SM Utilization", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "55576e61fe10aa796637b7056b452f0d94f5b0e727643e78a422e2b721eb30ff", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "Nsight Compute shows high DRAM throughput but low tensor core utilization. Arithmetic intensity below the roofline knee point.", "sha256": "4dc42c72b1a556c74db70e040cb70a14535edb81f21fca0468237d330c4305f9"}, {"id": "u002", "kind": "list-item", "locator": "body:L7-L7", "preview": "1. **Low arithmetic intensity**: Operations like GEMV, small batch decode, or reduction kernels", "sha256": "c51b97d7a36f7b139c0424ab729c50d90469c2293723e48fbf6a4e453eba4cfd"}, {"id": "u003", "kind": "list-item", "locator": "body:L8-L8", "preview": "2. **Poor data reuse**: Each data element used only once", "sha256": "48214a8d8262de0740e3225b192a23047f3dc7fa93ebcaf09654a69ce6e1b818"}, {"id": "u004", "kind": "list-item", "locator": "body:L9-L9", "preview": "3. **Inefficient memory access**: Uncoalesced loads, L1 cache thrashing", "sha256": "3e4a2d1d5d56034c91c22a54b4ea6f6a07afe88bb2a2cb1a0b4509e97502b4dd"}, {"id": "u005", "kind": "table-row", "locator": "body:L15-L15", "preview": "| Technique | Effect | | [Vectorized loads](../techniques/vectorized-loads.md) | 128/256-bit loads maximize bandwidth utilization |", "sha256": "f6634a4a1cdf73cae19ce6b5b2b65a1f466bb337bb5a7739be87f42a489adf4e"}, {"id": "u006", "kind": "table-row", "locator": "body:L16-L16", "preview": "| Technique | Effect | | [Cache policies](../techniques/vectorized-loads.md) | L1::no_allocate for streaming, L1::evict_last for reuse |", "sha256": "71e86985a3f7dc21c00822843be4de0a7d7f8baa1913cfeea45eefd465ce3a88"}, {"id": "u007", "kind": "table-row", "locator": "body:L17-L17", "preview": "| Technique | Effect | | [Register budgeting](../techniques/vectorized-loads.md) | -maxrregcount increases occupancy |", "sha256": "e45af4504c36ed1768fe0ecf479ebd86fed44e7ee01636ed18c46d1433b798bd"}, {"id": "u008", "kind": "table-row", "locator": "body:L18-L18", "preview": "| Technique | Effect | | [TMA multicast](../hardware/tma.md) | Share loaded data across SMs in cluster |", "sha256": "0fa01c9f8e7d712e7c405684d14a05d96529f86b847d3b5460ab8058c5c05068"}, {"id": "u009", "kind": "table-row", "locator": "body:L19-L19", "preview": "| Technique | Effect | | [Swizzling](../techniques/swizzling.md) | Eliminate bank conflicts in shared memory |", "sha256": "8d464b0a54f7ed7c78641ea4c7b70ca5dfe11e9886a64623d269c8f8b22d889c"}, {"id": "u010", "kind": "code", "locator": "body:L23-L34", "preview": "```cuda // NVFP4 GEMV: memory-bound optimization // Key insight: profile FIRST to confirm memory-bound behavior // \"The single most important thing could have been running Nsight Compute\" // \u2014 Amandeep (12 Attempts at an FP4 Kernel) // Opti", "sha256": "48b1d0f3c6e9a148d01ddebc1bb397b343e2217038e2d4a7f62f20058d20d119"}, {"id": "u011", "kind": "list-item", "locator": "body:L37-L37", "preview": "- Always profile before optimizing \u2014 wrong assumption wastes effort", "sha256": "d024ba4e89fe3161a59877b3ea6ff0d87e466fa900ff41b0867984a659a3d8be"}, {"id": "u012", "kind": "list-item", "locator": "body:L38-L38", "preview": "- B200 has 8 TB/s bandwidth; speed-of-light calculation determines achievable performance", "sha256": "93d1616bf3fcec94fddbd14292d17533a2577578ed75e50f837d041501b45d68"}, {"id": "u013", "kind": "list-item", "locator": "body:L39-L39", "preview": "- ILP and compute optimizations have diminishing returns for memory-bound kernels", "sha256": "3da206d588b3730c5fbfcecc2b5e8e44a3109159f65679aeafdd904e06f30659"}], "confidence_claimed": null, "headings": ["Symptom", "Likely Causes", "Candidate Techniques", "Examples", "Caveats"], "id": "pattern-memory-bound", "path": "wiki/patterns/memory-bound.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/yue-nvfp4-hackathon.md", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}, {"path": "sources/blogs/amandeep-nvfp4-attempts.md", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}, {"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}], "risk_flags": ["code", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-yue-nvfp4", "blog-amandeep-nvfp4", "doc-nvidia-tuning-guide"], "title": "Memory Bandwidth Bound", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "dd70d3638677328e71242ec9fa14891b9ed9465328f129378f5b835f52741275", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "MoE grouped GEMM shows uneven per-expert compute time. Some SMs finish their expert quickly and sit idle while others are still processing. Overall latency is dominated by the slowest expert.", "sha256": "26270290a5cf5b38ab518ba334c9d8b6010db034ddc8f8a8c7e1f1259fd5d985"}, {"id": "u002", "kind": "list-item", "locator": "body:L9-L9", "preview": "1. **Skewed token distribution**: Router sends 80% of tokens to 20% of experts (common in trained MoE models)", "sha256": "800cba0e70b4a438a24067f4f7787eadc3ab4ad4897336310a5d3dd010135843"}, {"id": "u003", "kind": "list-item", "locator": "body:L10-L10", "preview": "2. **Static tile assignment**: Precomputed tile\u2192SM mapping cannot rebalance at runtime", "sha256": "c283efbc93794cc2be49fd7f07f42e36350aef2957ec6b68e17dbae5a509e635"}, {"id": "u004", "kind": "list-item", "locator": "body:L11-L11", "preview": "3. **Masked layout waste**: Fixed M_max per expert wastes compute on padding rows", "sha256": "fc529e53756a94746ed70c30cd71143674488b5a2462c85453ad818c389ebfbf"}, {"id": "u005", "kind": "list-item", "locator": "body:L12-L12", "preview": "4. **Small-M per expert**: When M < BLOCK_M, thin-GEMM underutilizes tensor cores", "sha256": "f74d67cf19352656a85cf127183325a4ba8c6bcf8e17e9d454de0a4ff5000913"}, {"id": "u006", "kind": "table-row", "locator": "body:L18-L18", "preview": "| Technique | Effect | | [CLC (Cluster Launch Control)](../hardware/clc.md) | Hardware dynamic tile assignment \u2014 fastest SMs grab more tiles |", "sha256": "49bcf4dbc8ad02660ec9047f26f96a3858a2d9fd5222369ebb5bca5776ac57fe"}, {"id": "u007", "kind": "table-row", "locator": "body:L19-L19", "preview": "| Technique | Effect | | [Persistent kernels](../techniques/persistent-kernels.md) | Amortize launch overhead; loop over dynamic work queue |", "sha256": "2792e978d288672f2d6d95d04263027e2622054f2ed15f78de9e3943bb477a10"}, {"id": "u008", "kind": "table-row", "locator": "body:L20-L20", "preview": "| Technique | Effect | | [Contiguous layout](../kernels/grouped-gemm.md) | Pack variable-M experts sequentially; offsets array indexes expert boundaries |", "sha256": "d20ff9922eaef6c226692eae94d058e63be9adf9e98e91e1ceeb9aad71a8e38b"}, {"id": "u009", "kind": "table-row", "locator": "body:L21-L21", "preview": "| Technique | Effect | | [Masked layout](../kernels/grouped-gemm.md) | Good for CUDA graph capture; wastes compute on padding |", "sha256": "3413c434b8113eb19f2d463875390d45fbd8c131d6c860733df1113960b69608"}, {"id": "u010", "kind": "table-row", "locator": "body:L22-L22", "preview": "| Technique | Effect | | [K-grouped layout](../kernels/grouped-gemm.md) | For weight gradient computation with variable K per expert |", "sha256": "b2a0b2fe800e3a99466b916235631662c5fc13c46bb43ca58b717010d8d2bad4"}, {"id": "u011", "kind": "table-row", "locator": "body:L23-L23", "preview": "| Technique | Effect | | [EPLB (Expert Parallel Load Balancer)](https://github.com/deepseek-ai/EPLB) | Replicate heavy experts across GPUs; 1.49x prefill speedup, 2.54x decode |", "sha256": "699ca12f7f2383870531d253ad0ca7bf392aa46c1c04660f59d11b39c1225091"}, {"id": "u012", "kind": "prose", "locator": "body:L27-L27", "preview": "The 1st-place submission exploited the evaluation harness rather than truly balancing load:", "sha256": "13952f31f5f92a411c66686966d9c911baad14e022c7b7ab8c6d9766b54f9b59"}, {"id": "u013", "kind": "list-item", "locator": "body:L28-L28", "preview": "- Correctness phase: real kernel ran on cloned data", "sha256": "44268b4c3d9e50d59288e97ef6ab0d7e92da73ddf6e17c3d2e9afcb36067ec40"}, {"id": "u014", "kind": "list-item", "locator": "body:L29-L29", "preview": "- Timing phase: detected reused objects, fired 120-group super-batch in call 1, returned cached results for calls 2-15", "sha256": "49bf005d0c5aa1f75b3e3a074098548d90a84511bda490e6e94da0d95a954fce"}, {"id": "u015", "kind": "prose", "locator": "body:L31-L31", "preview": "This highlighted that even careful tile scheduling can be outrun by algorithmic restructuring \u2014 and prompted the MLSys 2026 FlashInfer contest to add runtime isolation + subprocess eval.", "sha256": "8b0dd66c79788e1da885d296e6873c972217574a4864892639153a1c5b9cb556"}, {"id": "u016", "kind": "list-item", "locator": "body:L35-L35", "preview": "- CLC only available on SM100 datacenter (not SM120 consumer)", "sha256": "76adb21b9da1b3c422e4e96806d5c9766324fb825ea04c1d6bf43c5c051a74c7"}, {"id": "u017", "kind": "list-item", "locator": "body:L36-L36", "preview": "- Dynamic scheduling has small per-tile overhead vs static precomputed", "sha256": "59c88f69879c5683028c803da56e9e94888bbf568790fc334c872735659ad6f2"}, {"id": "u018", "kind": "list-item", "locator": "body:L37-L37", "preview": "- Small experts may not benefit \u2014 minimum viable tile size is a floor", "sha256": "cebcd3944c50b6c03c42697299819ac55c9910f396999bdf616538b2e9844aa7"}, {"id": "u019", "kind": "list-item", "locator": "body:L38-L38", "preview": "- EPLB works at cluster scale, not single-device", "sha256": "153c61615671921ed0408938218d9b06e56c3d0818ff73f546d85a82a311c83e"}, {"id": "u020", "kind": "list-item", "locator": "body:L42-L42", "preview": "- Uniform routing (rare in practice)", "sha256": "ee7bf1426e5cbead527debaf61009d5004931fd3175bfdf22cbf7e8d7c295497"}, {"id": "u021", "kind": "list-item", "locator": "body:L43-L43", "preview": "- Very large batch sizes (statistics average out)", "sha256": "6a555d86519621d8cf74821617146ba700ea5493a2ebe2e7380c9707192f16d6"}, {"id": "u022", "kind": "list-item", "locator": "body:L44-L44", "preview": "- Training with auxiliary load balancing loss", "sha256": "ad4574cc7d6d92b10a813303b13b944dde68868576cd30add7fc998cd270fe97"}], "confidence_claimed": null, "headings": ["Symptom", "Likely Causes", "Candidate Techniques", "Example: Reward Hack in GPU Mode Problem 4", "Caveats", "When NOT An Issue"], "id": "pattern-moe-load-imbalance", "path": "wiki/patterns/moe-load-imbalance.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md", "url": "https://github.com/gpu-mode/reference-kernels"}, {"path": "sources/contests/flashinfer-mlsys26/track-a-fused-moe.md", "url": "https://mlsys26.flashinfer.ai/"}, {"path": "sources/blogs/deepgemm.md", "url": "https://github.com/deepseek-ai/DeepGEMM"}], "risk_flags": ["table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["contest-gpumode-p4", "contest-flashinfer-track-a", "blog-deepgemm"], "title": "MoE Expert Load Imbalance", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "ec21f3208711a7056069ab687fce39dd6bc96f6ee8fd093b42542af45799b592", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Nsight Compute shows TMA or tcgen05 units idle despite nominally compute-bound workload. Tensor core utilization drops during specific phases of the kernel. Warp-level profiling reveals threads blocked on `mbarrier.try_wait` more than expec", "sha256": "4032d70f5e4b3925ac907fbc729234379f11c847fba146210288c9a8ec8491d3"}, {"id": "u002", "kind": "list-item", "locator": "body:L9-L9", "preview": "1. **Insufficient pipeline depth**: 2 stages cannot hide a 3-cycle latency chain", "sha256": "0cb8232617e1672be85937beba39a1f5e75b81ea46bf43469391f4b4315b22f4"}, {"id": "u003", "kind": "list-item", "locator": "body:L10-L10", "preview": "2. **Incorrect mbarrier phase tracking**: Consumer observes stale arrivals, waits for next", "sha256": "91d7e4eb0e38d59d5aa1a5aa56faad99d2ec69ed2ede720df86f912e752a81ae"}, {"id": "u004", "kind": "list-item", "locator": "body:L11-L11", "preview": "3. **Missing `tcgen05.fence::after_thread_sync`**: MMA reads SMEM before TMA transfer fully visible", "sha256": "d77d6b104399db6c82c6bbeb66b1a621d106c6dd6b289bcaadb887ad14b7e65e"}, {"id": "u005", "kind": "list-item", "locator": "body:L12-L12", "preview": "4. **Single-tile scheduling**: All warps serialized on one tile's softmax/epilogue", "sha256": "b922275ade0073f6dd7296ab179ba695b5991077ae021e1fab7d4ca1656614e2"}, {"id": "u006", "kind": "list-item", "locator": "body:L13-L13", "preview": "5. **Producer over-arrives**: Manual `mbarrier_arrive` after async TMA \u2014 hardware + manual both arrive, next stage gets stale release", "sha256": "fe84e7ee7ad4f0054a03192c196c441ec9de80b61e80bf3d63a6f9f8010969ec"}, {"id": "u007", "kind": "table-row", "locator": "body:L19-L19", "preview": "| Technique | Effect | | [Pipeline stages](../techniques/pipeline-stages.md) | Increase NUM_STAGES (3-5 typical on Blackwell) |", "sha256": "6f3f738e0e4c76b6c48eae6f58c723ff49ae060351f9951a28bf23f9a2b46fe3"}, {"id": "u008", "kind": "table-row", "locator": "body:L20-L20", "preview": "| Technique | Effect | | [Warp specialization](../techniques/warp-specialization.md) | Dedicated warps for TMA/MMA/epilogue eliminate role-switching stalls |", "sha256": "6149506d8c026954f1683a46ef62eeca174bb51807a37b295c8eb0ff161d3f7a"}, {"id": "u009", "kind": "table-row", "locator": "body:L21-L21", "preview": "| Technique | Effect | | [Double-buffering](../techniques/double-buffering.md) | TMEM buffer A while MMA runs on buffer B |", "sha256": "c14aac1bfc715cfda0e3f3348c2e26505421e9529ba75c8af9fe164f3876ed49"}, {"id": "u010", "kind": "table-row", "locator": "body:L22-L22", "preview": "| Technique | Effect | | [Ping-pong scheduling](../techniques/ping-pong-scheduling.md) | Two query tiles alternate softmax/MMA (FA4 pattern) |", "sha256": "6c130ff135fbe3daf0024f04e57bfea496069a7b25453731535dc0cb168365fd"}, {"id": "u011", "kind": "code", "locator": "body:L26-L33", "preview": "``` 1. Profile with Nsight Compute, check tensor core active cycles 2. Inspect mbarrier wait stalls in warp state breakdown 3. Verify phase tracking increments correctly (each wait should flip parity) 4. Check that TMA uses arrive_expect_tx", "sha256": "b9df86b4e71fc18353da3def4520922d82e87fd6330d8aec04b35338b543aa90"}, {"id": "u012", "kind": "list-item", "locator": "body:L37-L37", "preview": "- 1-stage: 62% of cuBLAS (TMA blocks MMA)", "sha256": "daeade9f55a3938b4ceab512bc437488781118e4ce851f0f4ad8c0fada815969"}, {"id": "u013", "kind": "list-item", "locator": "body:L38-L38", "preview": "- 3-stage pipelined: 70% (hide most TMA latency)", "sha256": "4377018d3c7dd3f0240d868648f6b8fc336736444f6bfa8e3b98b71e68cc1c70"}, {"id": "u014", "kind": "list-item", "locator": "body:L39-L39", "preview": "- Warp specialized: 80% (no role switching)", "sha256": "2460d6863a6c4117289d6904c1e135e666426aa0ac1dcde0b4691dd9341abc60"}, {"id": "u015", "kind": "list-item", "locator": "body:L40-L40", "preview": "- Add 2-SM MMA: 86% (larger tile, more reuse)", "sha256": "8976968aed7db4cde3048b3d0d92f6c77b1213661e4aed2164d33c09c814ae99"}, {"id": "u016", "kind": "list-item", "locator": "body:L41-L41", "preview": "- Persistent + CLC: 98% (eliminate tail effect)", "sha256": "5af1e8033ee259a49ab21e6053709d43a74405847ad202d96a173678b806b643"}, {"id": "u017", "kind": "list-item", "locator": "body:L45-L45", "preview": "- Too many stages consume SMEM; exceeds 228KB budget", "sha256": "e916e057ac5bd922f8a065bd0ca4b3327b885b9c0e211fde7c4b64d8ce14d271"}, {"id": "u018", "kind": "list-item", "locator": "body:L46-L46", "preview": "- Phase tracking bugs are notoriously hard to debug \u2014 add assertions in development", "sha256": "7c96054ce550dc7b762e7375ffce607153d115d44308a7a83ac072240aa39525"}, {"id": "u019", "kind": "list-item", "locator": "body:L47-L47", "preview": "- Profile first \u2014 pipeline is a waste of effort on memory-bound kernels", "sha256": "9320fb1619594e5c38103bdd57e2112552752b210fb4a5e1f872528fe3c60c49"}], "confidence_claimed": null, "headings": ["Symptom", "Likely Causes", "Candidate Techniques", "Diagnosis Checklist", "Example Progression (tcgen05 tutorial)", "Caveats"], "id": "pattern-pipeline-stalls", "path": "wiki/patterns/pipeline-stalls.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/blogs/flash-attention-4.md", "url": "https://tridao.me/blog/2026/flash4/"}, {"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-tcgen05-tutorial", "blog-flash-attention-4", "doc-nvidia-tuning-guide"], "title": "Pipeline Stalls", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "6fd15f8aae2d1ca78d038b8af9ec545760eac38cdbb46e46985f19c35a84a491", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "Occupancy below target due to high register usage per thread. Nsight Compute shows register spilling to local memory.", "sha256": "d2ac11ac70671d4aeb74c58a0a6b9bab0cdd4c2d07b3366cdfb1570acef35355"}, {"id": "u002", "kind": "list-item", "locator": "body:L7-L7", "preview": "1. **Accumulator registers**: On Hopper, large MMA tiles consume many registers for accumulators", "sha256": "de2257f0b6f0c80133c86c37838724423e7507ecc9614670890ceae87424fade"}, {"id": "u003", "kind": "list-item", "locator": "body:L8-L8", "preview": "2. **Epilogue state**: Data transformation in epilogue requires additional registers", "sha256": "aaa6f9054e67591cfab71e16eea3a4522286eaa7f1c3417eea8fabee81bf47ae"}, {"id": "u004", "kind": "list-item", "locator": "body:L9-L9", "preview": "3. **Complex control flow**: Many live variables across branches", "sha256": "d33663262ccd89d8aa00ac29ecf817e99f0855f70a4fddf30fdaeb3dbcc626de"}, {"id": "u005", "kind": "table-row", "locator": "body:L15-L15", "preview": "| Technique | Applicability | Effect | | [TMEM](../hardware/tmem.md) | SM100 only | Moves accumulators to dedicated 256KB memory |", "sha256": "9f90dd64b6e0a1e5474f59960756cc72ca2fe0f274af67e270c9a3d76c304d46"}, {"id": "u006", "kind": "table-row", "locator": "body:L16-L16", "preview": "| Technique | Applicability | Effect | | [Warp specialization](../techniques/warp-specialization.md) | SM100+ | Different warps handle different roles, reducing per-warp register needs |", "sha256": "ab88df1d4048970662593fa2cbde9264f1214c1d1f23bfc3d777d23d0f1204ec"}, {"id": "u007", "kind": "table-row", "locator": "body:L17-L17", "preview": "| Technique | Applicability | Effect | | [Register-to-TMEM migration](../migration/register-to-tmem.md) | SM90\u2192SM100 | Systematic approach to moving accumulators off registers |", "sha256": "7169a86f303c5aaee56eb9016eb702859a933d7684c4dcce7542711f13d054fa"}, {"id": "u008", "kind": "code", "locator": "body:L21-L31", "preview": "``` // Hopper: 64\u00d7256 MMA tile accumulator = 64*256*4 bytes in registers per warp group // \u2192 ~128 registers per thread just for accumulators // // Blackwell: TMEM holds accumulators // \u2192 0 registers for accumulators // \u2192 ~128 registers free", "sha256": "7dbbada6653f79e04b54cc53ab013df61ca8263d3abfce21813daa5b4a5ad9af"}, {"id": "u009", "kind": "list-item", "locator": "body:L34-L34", "preview": "- TMEM only available on SM100 datacenter (not SM120 consumer)", "sha256": "beedff9c79765bf0578733865359482fdf1c0028cadeb2bb71cb482da9cdc742"}, {"id": "u010", "kind": "list-item", "locator": "body:L35-L35", "preview": "- TMEM requires explicit alloc/dealloc lifecycle", "sha256": "9e7d07ef558cf4d22231cfc469028a647d6924a327caeccb143785d22219b122"}, {"id": "u011", "kind": "list-item", "locator": "body:L36-L36", "preview": "- TMEM\u2192register transfer adds latency (offset by freeing registers)", "sha256": "5b322c2b7c5cb6ef89d08e58652b159e888517d7fea0e7f0cec00173096f7c76"}], "confidence_claimed": null, "headings": ["Symptom", "Likely Causes", "Candidate Techniques", "Blackwell Solution", "Caveats"], "id": "pattern-register-pressure", "path": "wiki/patterns/register-pressure.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/prs/vllm/PR-16032.md", "revision": "ed7a29d9", "url": "https://github.com/vllm-project/vllm/pull/16032"}], "risk_flags": ["code", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "pr-vllm-16032"], "title": "Register Pressure \u2014 Low Occupancy", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "796c1dd09a9e76eff0996bf41283998fce447e492dd300571596ec89ad147ed9", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "Performance drops for problem sizes where total_tiles % num_SMs != 0. The last wave of tiles runs with many SMs idle.", "sha256": "ea0c41c307bdd4f791565d1552e46e283100208e9ff10eba22954565761c5604"}, {"id": "u002", "kind": "list-item", "locator": "body:L7-L7", "preview": "1. **Wave quantization**: Grid of N tiles on M SMs takes ceil(N/M) waves; last wave may use only N%M SMs", "sha256": "0ed8e43be29b45288db1023d10e10a63b3b7ce9be71c046a10b1862414f204b0"}, {"id": "u003", "kind": "list-item", "locator": "body:L8-L8", "preview": "2. **Static assignment**: stride-by-gridDim leaves remainder tiles on few SMs", "sha256": "e80998af0a65b1cbcafb7a22edc7cb17f58e403f7bb6d15a4709aeb01e886d84"}, {"id": "u004", "kind": "list-item", "locator": "body:L9-L9", "preview": "3. **Non-persistent launch**: each kernel launch has fixed grid, no dynamic rebalancing", "sha256": "20f6e34bc0ccec42ad97d7117f02b06b11d70c03705e449c309e51b017ac376c"}, {"id": "u005", "kind": "table-row", "locator": "body:L15-L15", "preview": "| Technique | Effect | | [CLC](../hardware/clc.md) | Hardware dynamic scheduling, SMs grab tiles on-demand |", "sha256": "04e0ce4ac625fe8083068eb8a2f249bc9dbdc866d4cbf686eb43054fb688b5c1"}, {"id": "u006", "kind": "table-row", "locator": "body:L16-L16", "preview": "| Technique | Effect | | [Persistent kernels](../techniques/persistent-kernels.md) | SM-count grid, iterate over tiles, no wave boundary |", "sha256": "e47a5eca8af15fa32dcaa55c77df276b6ddaf12ebe96603730fa449391c3bd8f"}, {"id": "u007", "kind": "table-row", "locator": "body:L17-L17", "preview": "| Technique | Effect | | [Tile scheduling](../techniques/tile-scheduling.md) | Raster order, swizzle patterns for better distribution |", "sha256": "8235d5f10cb6616f94cf20bfe9f2763cb6f832c0f0e63ea290c2c3b947359a09"}, {"id": "u008", "kind": "code", "locator": "body:L21-L28", "preview": "``` // B200: 142 SMs // Problem: 150 tiles // Without CLC: 2 waves (142 + 8), last wave uses only 8 SMs (5.6%) // With CLC: single persistent wave, all 142 SMs stay busy // // Impact: 86% \u2192 98% of cuBLAS (tcgen05 tutorial data) ```", "sha256": "7100363602e3f45ee5b5aa49fa60ab3fb42c04a27a08e87a21d5398cfe53e578"}, {"id": "u009", "kind": "list-item", "locator": "body:L31-L31", "preview": "- Only significant for moderate tile counts (< 4\u00d7 SM count)", "sha256": "2670f5539f33eb07230fabfa084f24e9f07596a616fe7e28da5d546fc9fa47a2"}, {"id": "u010", "kind": "list-item", "locator": "body:L32-L32", "preview": "- For very large problems, tail effect is amortized across many waves", "sha256": "50cc74b0317aa0b78f48ee04a7790f07f1c5b685ca933fae6e1f86a23d90a882"}, {"id": "u011", "kind": "list-item", "locator": "body:L33-L33", "preview": "- CLC only on SM100 datacenter", "sha256": "fdfc563aa627d856b7997864f874000bd1cf6e34a61465c366a1773035f335f6"}], "confidence_claimed": null, "headings": ["Symptom", "Likely Causes", "Candidate Techniques", "Example", "Caveats"], "id": "pattern-tail-effect", "path": "wiki/patterns/tail-effect.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/prs/cutlass/PR-2161.md", "revision": "06e560d9", "url": "https://github.com/NVIDIA/cutlass/pull/2161"}], "risk_flags": ["code", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "pr-cutlass-2161"], "title": "Tail Effect \u2014 Last Wave Underutilization", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "a47c6862f01b21f93a8d51024e927b47f02eba0fe5852a52053fb9b1080e1498", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "PTX cache qualifiers (`L1::no_allocate`, `L1::evict_last`, `L1::evict_first`) let kernels hint to hardware how to handle cache admission for specific loads. Critical for memory-bound kernels where the L1 working set matters more than the co", "sha256": "54cab58e62ef06bdef4cf5fc91778a59615e0ab38500f8740b2e6f4567fbde09"}, {"id": "u002", "kind": "code", "locator": "body:L9-L19", "preview": "```asm ; Matrix A (streamed once per row, never reused): bypass L1 ; Avoids polluting L1 with one-shot data ld.global.L1::no_allocate.v4.u64 {a0,a1,a2,a3}, [addr_a]; ; Vector B (reused across BLOCK_M rows): keep in L1 ld.global.L1::evict_la", "sha256": "23a8aea5fea29b291992023b0f9304f0f5c2d2f3cb19fc1f6ff72c8169fd4845"}, {"id": "u003", "kind": "prose", "locator": "body:L23-L23", "preview": "Rank 1 submission used **different qualifiers per K-dimension variant**:", "sha256": "929723cf95b932abdb41d1d7772b296a74efabf583d9f57cb44956ce8dcf1300"}, {"id": "u004", "kind": "list-item", "locator": "body:L24-L24", "preview": "- K=16384 (large): aggressive `L1::no_allocate` on A (huge streaming matrix)", "sha256": "ebe021ea0f87a111104ff40b1ee54c97eb289cce36e1b3406eac30e1426999ef"}, {"id": "u005", "kind": "list-item", "locator": "body:L25-L25", "preview": "- K=2048 (small): relaxed balance since B is smaller relative to cache", "sha256": "a34ff1ee0912ff6efd24a3735a50b82ac3944355ce6af6202db6734abeed7102"}, {"id": "u006", "kind": "list-item", "locator": "body:L29-L29", "preview": "- NVFP4 GEMV: 443\u03bcs \u2192 27\u03bcs (16x improvement) came partly from cache policy + PTX byte unpacking", "sha256": "91db32f48f31f818adf7240d21f9ca08975ba1b3ff1e89166b2533e29c1cff1a"}, {"id": "u007", "kind": "list-item", "locator": "body:L30-L30", "preview": "- On memory-bound kernels, cache policy can be the dominant lever", "sha256": "30fde58ffce939740980ec4c690bcc0f617768b828daa5705b74c41e30169502"}, {"id": "u008", "kind": "list-item", "locator": "body:L34-L34", "preview": "- Memory-bound kernels (profile with Nsight Compute first)", "sha256": "859dfb4b1dbefd115e0269680e0508a99a50fb2442f70ea9a5faa928fd4e7151"}, {"id": "u009", "kind": "list-item", "locator": "body:L35-L35", "preview": "- Tensor with clear \"streaming\" vs \"reused\" access patterns", "sha256": "9c233103abf17311221463c5ef9e1356e0843d4d32bed9484279182fa341e78e"}, {"id": "u010", "kind": "list-item", "locator": "body:L36-L36", "preview": "- Inputs > L2 cache size (B200: 126MB)", "sha256": "a4f399076d2b6ae932d4f1df179e9c372391a89221ea7981c895a745db0e5a3b"}, {"id": "u011", "kind": "list-item", "locator": "body:L37-L37", "preview": "- Separate M and N tile loading patterns in GEMM", "sha256": "9be234f588406c67bc6287d3c5cb854809e686902a1378e724795a23c4469ebd"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Pattern", "GPU Mode NVFP4 GEMV Winner Technique", "Measurable Impact", "When To Use"], "id": "technique-cache-policy", "path": "wiki/techniques/cache-policy.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/yue-nvfp4-hackathon.md", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}, {"path": "sources/blogs/amandeep-nvfp4-attempts.md", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}, {"path": "sources/blogs/simon-nvfp4-gemv.md", "url": "https://veitner.bearblog.dev/nvfp4-gemv/"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/parallel-thread-execution/"}], "risk_flags": ["code"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-yue-nvfp4", "blog-amandeep-nvfp4", "blog-simon-nvfp4-gemv", "doc-ptx-isa-sm100"], "title": "PTX Cache Policy Differentiation", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "c8d85d250f8216c456f2c7737203ae250b1b6d85d9d6150336ed406b13ceb20d", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L6", "preview": "Use CCCL/CUB PRs when the bottleneck is not tensor math but a memory primitive: scan, top-k selection, fill, histogram, reduce, or block load/store policy. The goal is usually a policy or dispatch idea rather than copying a full CUB primiti", "sha256": "b4e94be35eaf010b7677bf5512c95776807b1fcbef5bc9daba66ffa6ca774da4"}, {"id": "u002", "kind": "code", "locator": "body:L8-L16", "preview": "```cuda // Minimal policy probe shape for an application-specific top-k or scan helper. template struct PrimitivePolicy { static constexpr int block_threads = BLOCK_THREADS; static constexpr int ite", "sha256": "729c6d52cc344dc53b898c8da5b0d2dd3fe7cbde5e77250631ec22077b232279"}, {"id": "u003", "kind": "list-item", "locator": "body:L20-L20", "preview": "- Check whether the PR tuned SM100 separately from SM90.", "sha256": "d012a2019dd58b7c05e6254cdd377a0a63851660993ec618938b5b09f33ed543"}, {"id": "u004", "kind": "list-item", "locator": "body:L21-L21", "preview": "- Validate determinism and tie-breaking before using atomic or relaxed variants.", "sha256": "90be873e59546e3d914c65c321f0c5656821160fa71596ec9ab85b6ad71b5dcb"}, {"id": "u005", "kind": "list-item", "locator": "body:L22-L22", "preview": "- For DSA TopK, keep radix/select cost separate from score-computation cost.", "sha256": "cf3f150c76c62c2ebe0a5dc91a5afc83fe9427a26d1459b601d0c34e5f2026c9"}], "confidence_claimed": "source-reported", "headings": ["Use", "Transfer Notes"], "id": "technique-cccl-memory-primitives", "path": "wiki/techniques/cccl-memory-primitives.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/prs/cccl/PR-3559.md", "revision": "25523da2", "url": "https://github.com/NVIDIA/cccl/pull/3559"}, {"path": "sources/prs/cccl/PR-6152.md", "revision": "3fb05826", "url": "https://github.com/NVIDIA/cccl/pull/6152"}], "risk_flags": ["code", "ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["pr-cccl-3559", "pr-cccl-6152"], "title": "CCCL CUB Memory Primitives For Selection And Scan", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "f7d2c3d8c9327b5dfd8b4e5c47d71cc82ce97f8ba4c85aad30dbadf15aeb1e8c", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Linear attention variants (GatedDeltaNet, RetNet, Mamba) have O(n) complexity but naive implementations are sequential. Chunk-based parallelism divides the sequence into chunks of size C, computes within each chunk in parallel (matmul-frien", "sha256": "8287f5e8015970d46413a6849cb84c49175de62b0c42d9f7b8e1fb870071d45c"}, {"id": "u002", "kind": "code", "locator": "body:L9-L36", "preview": "```python @triton.jit def chunk_parallel_linear_attn(Q, K, V, State, Output, chunk_size: tl.constexpr, d: tl.constexpr): # Grid: (num_chunks, num_heads, batch) chunk_id = tl.program_id(0) # Load chunk of Q, K, V q = tl.load(Q + chunk_id * c", "sha256": "9da54c2644a6c34bc669d76ecfaf83a25677373a1e4e391d0c5e96c10c7531d1"}, {"id": "u003", "kind": "list-item", "locator": "body:L40-L40", "preview": "- **Small chunks (C=32)**: low latency decode, fewer intermediate materializations", "sha256": "51f710fe4294aa5afe2105f66bc5822bdaf68bf33926ca784e077164476fb2af"}, {"id": "u004", "kind": "list-item", "locator": "body:L41-L41", "preview": "- **Large chunks (C=256-512)**: better tensor core utilization, higher throughput for prefill", "sha256": "21aabf9774c0157d481bf6c882d0179225d9d02d8d385ee8ba8f7f3fc96e78a3"}, {"id": "u005", "kind": "list-item", "locator": "body:L42-L42", "preview": "- **TFLA (Tiled FLA)**: two-level tiling allows arbitrary chunk sizes via recursive tiling", "sha256": "e05ffb908efbc21e2e9a973a88dc0a575ebdf65043d956dc6849dbdf8c4b461a"}, {"id": "u006", "kind": "list-item", "locator": "body:L46-L46", "preview": "- Linear attention (O(n) complexity)", "sha256": "2027a478564eb52500016693692150b10bcf0ff014c5b3a1dc8c663c2f5f0726"}, {"id": "u007", "kind": "list-item", "locator": "body:L47-L47", "preview": "- Recurrent state models (Mamba, GatedDeltaNet, Delta Rule)", "sha256": "6d2247f2c09e7cde6fdfb4560dac01e61c848ed9c433dee30f0b73e78ebc7ac3"}, {"id": "u008", "kind": "list-item", "locator": "body:L48-L48", "preview": "- Hybrid architectures mixing linear + full attention (Qwen3-Next uses 3:1 ratio)", "sha256": "97f6848a073c382a4e9b329e2c298ed9dd915f106388bf4f58a706edd5d4d35c"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Pattern", "Size Tradeoff", "When To Use"], "id": "technique-chunk-parallelism", "path": "wiki/techniques/chunk-parallelism.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/gated-delta-net.md", "url": "https://github.com/NVlabs/GatedDeltaNet"}, {"path": "sources/blogs/nsa.md", "url": "https://arxiv.org/abs/2502.11089"}, {"path": "sources/docs/tfla.md", "url": "https://arxiv.org/abs/2503.14376"}], "risk_flags": ["code"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-gated-delta-net", "blog-nsa", "doc-tfla"], "title": "Chunk-Based Parallelism for Linear Attention", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "787b870b0080a00805a0601a72d16367df1fd5ff21f1043c92acaf6e2d91a387", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "Double-buffering (and multi-buffering) allocates two or more copies of a data buffer so that one copy can be written while another is read. On Blackwell, this pattern applies at two distinct levels: (1) TMEM double-buffering for overlapping", "sha256": "97824e9b7c93a3ea45b6e343a729a81924da29e8de1c7297ee01a4efa2f7b7a5"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "Tensor Memory (TMEM) on Blackwell has 128 rows x 512 columns of 32-bit elements (256 KB per SM). For a standard GEMM with TILE_M=128, TILE_N=256, the accumulator requires 128 x 256 = 32,768 elements, which fits in half the TMEM column space", "sha256": "640c707834ddf8a8ce180197e801e9d7b95977d52cb6f99cf485d9e585092007"}, {"id": "u003", "kind": "code", "locator": "body:L9-L105", "preview": "```cuda // TMEM double-buffering: ping-pong between two 128x256 accumulator regions // // TMEM physical layout (per SM): // +------ 256 cols ------+------ 256 cols ------+ // | | | // | Buffer A (active) | Buffer B (drain) | 128 rows // | M", "sha256": "09ad4012498a0cf25db398856e7d71e60967607f31556eb3c34988b84965c1c3"}, {"id": "u004", "kind": "prose", "locator": "body:L109-L109", "preview": "Shared memory multi-buffering provides multiple copies of the A and B input tiles so that TMA loads can overlap with MMA consumption:", "sha256": "a1d0e1d2fc608bdffeb1c621b1abfa260c58fc98e117399b96ce6adc665e864b"}, {"id": "u005", "kind": "code", "locator": "body:L111-L138", "preview": "```cuda // SMEM multi-stage buffer allocation // 3 stages, each holding one A tile and one B tile // // Memory layout: // +--------+--------+--------+ // | Stage0 | Stage1 | Stage2 | // | A0 B0 | A1 B1 | A2 B2 | // +--------+--------+------", "sha256": "751c3e5b2f00a215d113317b80a82c5c07a3ed27b567bd46eb506671c35035c3"}, {"id": "u006", "kind": "prose", "locator": "body:L142-L142", "preview": "A fully optimized Blackwell GEMM uses both levels simultaneously:", "sha256": "33bd5f5682f36690671b018c0ce9e0e42947ecbb71aa898182eae2d113c52acf"}, {"id": "u007", "kind": "code", "locator": "body:L144-L190", "preview": "```cuda // Combined double-buffering: SMEM (3-stage) + TMEM (2-buffer) // // Outer loop: output tiles (TMEM double-buffered) // Inner loop: K-tiles (SMEM 3-stage pipelined) // // Timeline for 2 output tiles, 6 K-tiles each: // // TMEM buf: ", "sha256": "8738f3a9188308d89ea4c9c189f4b55815b721c20da41da8c3fae854615fa882"}, {"id": "u008", "kind": "prose", "locator": "body:L194-L194", "preview": "On Hopper (SM90), there is no TMEM. The accumulator lives in registers, and double-buffering the accumulator requires explicit register management:", "sha256": "f69660751f2b081c6acae0d709ee8e397d44afb3d9461288c63689a6b3311d7a"}, {"id": "u009", "kind": "code", "locator": "body:L196-L212", "preview": "```cuda // Hopper: accumulator double-buffering uses register arrays // Each warpgroup maintains two register-based accumulators // This doubles register pressure and reduces occupancy // Hopper approach (register pressure is the primary co", "sha256": "fac6da39b649d3725528d820da41fbd66daa71b7014b1e51707a95246a7d95f1"}, {"id": "u010", "kind": "table-row", "locator": "body:L216-L216", "preview": "| Aspect | Hopper (Registers) | Blackwell (TMEM) | | Accumulator storage | Thread-local registers | CTA-wide TMEM |", "sha256": "72d87a8b59b3429cc3bdf2791bfe30efd1024670fef317070e468ad27afe1edd"}, {"id": "u011", "kind": "table-row", "locator": "body:L217-L217", "preview": "| Aspect | Hopper (Registers) | Blackwell (TMEM) | | Double-buffer cost | 2x register usage | Zero register cost |", "sha256": "755418548c845ddad020907f1632ae4d5aadc0b3c185c07bb68ab4856b9899d5"}, {"id": "u012", "kind": "table-row", "locator": "body:L218-L218", "preview": "| Aspect | Hopper (Registers) | Blackwell (TMEM) | | Typical occupancy impact | Reduces from 2 to 1 CTA/SM | No impact |", "sha256": "7bb87c122c4e1accbb09da2ed4120902b9c9553c5fae37e27fde69426bf96f0d"}, {"id": "u013", "kind": "table-row", "locator": "body:L219-L219", "preview": "| Aspect | Hopper (Registers) | Blackwell (TMEM) | | Epilogue access | Direct (already in registers) | TMEM load required |", "sha256": "283a7775fe3db59d62b5f298aa22e87e359a121ab1ac25a4d5968dd5ff6669fa"}, {"id": "u014", "kind": "table-row", "locator": "body:L220-L220", "preview": "| Aspect | Hopper (Registers) | Blackwell (TMEM) | | Max accumulator size | ~16K elements (register limited) | 128x512 = 65K elements |", "sha256": "986b30ee3467a4797ccb0f3c9e46db5afeeeff8e680c36ef68971084a8aa0084"}, {"id": "u015", "kind": "list-item", "locator": "body:L224-L224", "preview": "- **TMEM double-buffering**: Always use on Blackwell when the epilogue takes more than trivial time. The only cost is the TMEM space, which is plentiful.", "sha256": "f8cc6a2cf091f9c2f0acbe8a03f81a6d3ec7f76e1ddccadc1f3acb2f9de1a995"}, {"id": "u016", "kind": "list-item", "locator": "body:L225-L225", "preview": "- **SMEM multi-stage buffering**: Always use for GEMM/attention mainloops. 3 stages is the default; increase to 4-5 only if the K-loop is long and memory latency is high.", "sha256": "987665c644857e5a3bfa77429b8448c0dbca2cb1e862c4042285758482e97764"}, {"id": "u017", "kind": "list-item", "locator": "body:L226-L226", "preview": "- **Combined**: The standard approach for production Blackwell GEMM kernels. Both CUTLASS and CuTe-DSL kernels use this pattern.", "sha256": "4262ae22892600361751316d3041af62d7fa3d304a84866f58d765817bc36399"}, {"id": "u018", "kind": "list-item", "locator": "body:L230-L230", "preview": "- TMEM double-buffering requires that the accumulator fits in half the TMEM columns (256 out of 512). For very wide tiles (TILE_N > 256 with FP32 accumulators), the tile must be split or a different buffering strategy used.", "sha256": "879c33db44d9c122b0f351e266a90a18eb9f16e13c4933e48e24e5b622f1a9d6"}, {"id": "u019", "kind": "list-item", "locator": "body:L231-L231", "preview": "- SMEM multi-stage buffering is constrained by the 228 KB SMEM limit. With 3 stages of large tiles, there may not be enough SMEM left for epilogue scratch space.", "sha256": "c3506ef7844eee5b0fa9826ecfbbd0d4ee9a5fcdfa6b0f9b2b1ea8b9a3153397"}, {"id": "u020", "kind": "list-item", "locator": "body:L232-L232", "preview": "- The mbarrier synchronization between TMEM buffers adds a few cycles of overhead per tile. For kernels with very few K-iterations per tile, this overhead is proportionally larger.", "sha256": "a149fb89ca7dee743cd8577c21ac7a6fbf1186d2d3e5190a9d8182bf6af8d350"}], "confidence_claimed": "source-reported", "headings": ["Overview", "TMEM Double-Buffering", "SMEM Multi-Stage Buffering", "Combined TMEM + SMEM Double-Buffering", "Comparison with Hopper", "When to Use", "Caveats"], "id": "technique-double-buffering", "path": "wiki/techniques/double-buffering.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/prs/flashinfer/PR-2387.md", "revision": "18804cd5", "url": "https://github.com/flashinfer-ai/flashinfer/pull/2387"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-tcgen05-tutorial", "doc-nvidia-tuning-guide", "pr-flashinfer-2387"], "title": "Double/Multi-Buffering Patterns", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "f37f59190f56273dbd2ec10059fd85e03af89ea72f6cf5e9898fb14e1f3fd450", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "Epilogue fusion overlaps the post-MMA operations (scaling, bias addition, activation functions, quantization, store to global memory) with ongoing MMA computation. On Blackwell, the accumulator lives in TMEM rather than registers, enabling ", "sha256": "8413c5da77a037d33a6338630dd5c1cbe24cd3d84fe6ceef5d920cea551e25a5"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "On Blackwell, the MMA result resides in Tensor Memory (TMEM). Epilogue warps must copy relevant portions of TMEM into registers before applying element-wise operations and writing to global memory:", "sha256": "81f41666ca91671f9fe10ea6fc7b97908dd9ff1b3c80195c069f20417607dc43"}, {"id": "u003", "kind": "code", "locator": "body:L9-L47", "preview": "```cuda // Epilogue warp: read TMEM accumulator, apply fused operations, store // This runs on warps 2-15 while warp 1 continues MMA on next tile __device__ void epilogue_warp_fn( int warp_id, int tile_m, int tile_n, float scale, const floa", "sha256": "33a4ebdcfe0c352ffbc0972fab3d0430b9bfc7261582f5bcfae75eafcf54a87e"}, {"id": "u004", "kind": "prose", "locator": "body:L51-L51", "preview": "The key to epilogue fusion on Blackwell is TMEM double-buffering. The 512-column TMEM space is split into two halves (columns 0-255 and 256-511). While the MMA warp accumulates into one half, the epilogue warps drain the other:", "sha256": "25a3bc6b0f16f88b53ddbe897df79ad650460d0dde529839bbb35e5085661cb7"}, {"id": "u005", "kind": "code", "locator": "body:L53-L126", "preview": "```cuda // TMEM double-buffering for MMA-epilogue overlap // // TMEM layout: 128 rows x 512 columns (32-bit elements) // Buffer A: columns [0, 255] -- 128 x 256 accumulator // Buffer B: columns [256, 511] -- 128 x 256 accumulator // // Time", "sha256": "f97e21b54c56438c47ccb356146fd3d834b2f5e4229e1dc3228f6fde89acb001"}, {"id": "u006", "kind": "prose", "locator": "body:L130-L130", "preview": "CUTLASS 4.5.0 provides composable epilogue visitors that fuse arbitrary element-wise operations after GEMM:", "sha256": "5200b7446fe3f04f2165e5ce30466a397ad5584faca077c5c649ba5e08f681e1"}, {"id": "u007", "kind": "code", "locator": "body:L132-L159", "preview": "```cuda // CUTLASS SM100 epilogue with fused scale + bias + activation // Uses the EVT (Epilogue Visitor Tree) pattern using EpilogueOp = cutlass::epilogue::fusion::LinCombEltAct< cutlass::epilogue::thread::ReLU, // Activation function floa", "sha256": "33cf22b05847bfa2ad96cb950ca02880e7dac120b5c622a83907261f68e16449"}, {"id": "u008", "kind": "table-row", "locator": "body:L165-L165", "preview": "| Operation | Description | Typical Use | | Scale + Bias | `y = alpha * acc + beta * C` | Standard GEMM epilogue |", "sha256": "9603bdf8a80e506de92d55d1a2774814c9a4c5bc26c1cf7dc522b99c5253ad7b"}, {"id": "u009", "kind": "table-row", "locator": "body:L166-L166", "preview": "| Operation | Description | Typical Use | | ReLU / GeLU / SiLU | Element-wise activation | MLP layers |", "sha256": "0514cf37e7ff7ad2bdf319ac2e86d2437f7874c67462f270ca59cf6993a07c16"}, {"id": "u010", "kind": "table-row", "locator": "body:L167-L167", "preview": "| Operation | Description | Typical Use | | Quantize | FP32 accumulator to FP8/FP16 | Inference quantization |", "sha256": "c39917be889b9752756a3c115f276572a73fe728ac6cc294baa5fb2844d5053c"}, {"id": "u011", "kind": "table-row", "locator": "body:L168-L168", "preview": "| Operation | Description | Typical Use | | SwiGLU gate | `y = SiLU(gate) * up` | Gated dual GEMM (LLM FFN) |", "sha256": "117a20dd183922a19b68079a5df6763e9da568950140a279426d396e37177237"}, {"id": "u012", "kind": "table-row", "locator": "body:L169-L169", "preview": "| Operation | Description | Typical Use | | Softmax rescale | `y = acc * exp(max_old - max_new)` | Attention epilogue |", "sha256": "c0fdd61e61c8c91ccbac1eaf70a94a5a97cadb646fe2bbbe5e5fbeadd7989bea"}, {"id": "u013", "kind": "table-row", "locator": "body:L170-L170", "preview": "| Operation | Description | Typical Use | | Residual add | `y = acc + residual` | Transformer blocks |", "sha256": "2f1600022b8f8229cc4ed7c35bcc164ff64a6a57eccfbcb70360339aca5fc885"}, {"id": "u014", "kind": "list-item", "locator": "body:L174-L174", "preview": "- **All Blackwell GEMMs with non-trivial epilogues**: The 14 epilogue warps are available by default in the warp-specialized model. Fusing operations avoids a separate kernel launch and an extra global memory round-trip.", "sha256": "08b535f95ac9ad036c111b2367f3e7c16404adc43b45f6fc1cd16f4627f365c4"}, {"id": "u015", "kind": "list-item", "locator": "body:L175-L175", "preview": "- **Attention kernels**: The softmax rescaling and output accumulation can be overlapped with the next KV tile's MMA.", "sha256": "e0931e1c5cba46b39e6f6837387298705331e7e9a2d1dd54590ee2cd2c49c61a"}, {"id": "u016", "kind": "list-item", "locator": "body:L176-L176", "preview": "- **Quantized inference**: FP32-to-FP8 conversion in the epilogue avoids writing FP32 intermediates to global memory.", "sha256": "60562f12492d41ceebca27c8e3dac2da398882905cece20f5b5565978ea0919d"}, {"id": "u017", "kind": "list-item", "locator": "body:L180-L180", "preview": "- The epilogue can only read TMEM after the MMA for that tile is complete. The double-buffer synchronization is mandatory to prevent reading partial results.", "sha256": "e8be4778cb552c31e16fde7a0ea31a3c3ab9fd6f392a86e1914181880ac0ff46"}, {"id": "u018", "kind": "list-item", "locator": "body:L181-L181", "preview": "- TMEM-to-register bandwidth is not unlimited. With 14 warps simultaneously reading TMEM, each warp gets a proportional share. Very wide output tiles (large TILE_N) may bottleneck on TMEM read bandwidth.", "sha256": "f13e4d40944837027966030233028593a1702eccc9e0bf6d9d519929ae40a466"}, {"id": "u019", "kind": "list-item", "locator": "body:L182-L182", "preview": "- Simple epilogues (just store) waste the 14 epilogue warps. For such cases, consider reducing the CTA size or assigning epilogue warps to other work (e.g., next-tile TMA prefetch).", "sha256": "d3fd45594046b22ecbd2558a1c2c899be3ee885ca04979086a84a4d3c62d93a5"}, {"id": "u020", "kind": "prose", "locator": "body:L186-L186", "preview": "Verbatim upstream code lives in [`artifacts/kernels/epilogue-fusion/full/`](../../artifacts/kernels/epilogue-fusion/full/); labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` header) live i", "sha256": "21b4bfd6d6a2db4f10f88ccc5a57aa8f827a9fc64dc7d0345c6b75de694a2148"}, {"id": "u021", "kind": "prose", "locator": "body:L188-L188", "preview": "Query via:", "sha256": "6abd63e508f9320fdd3b2b0cc5b5f6a000f4462abfd17f9ecc3c4818f4a0f875"}, {"id": "u022", "kind": "code", "locator": "body:L190-L192", "preview": "```bash python3 scripts/get_page.py technique-epilogue-fusion --include-code ```", "sha256": "351eca3c2459d80306042e1cfce4f77d70756f76d7812bac12bfb36aa59ce14d"}], "confidence_claimed": "source-reported", "headings": ["Overview", "TMEM-to-Register Epilogue Path", "Overlapping MMA with Epilogue via Double-Buffering", "CUTLASS Epilogue Patterns", "Common Fused Epilogue Operations", "When to Use", "Caveats", "Full Reference Implementation"], "id": "technique-epilogue-fusion", "path": "wiki/techniques/epilogue-fusion.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-cutlass-blackwell.md", "url": "https://docs.nvidia.com/cutlass/latest/CHANGELOG.html"}, {"path": "sources/blogs/colfax-cutlass-blackwell.md", "url": "https://research.colfax-intl.com/cutlass-tutorial-writing-gemm-kernels-using-tmem-for-nvidia-blackwell-gpus/"}, {"path": "sources/prs/vllm/PR-16032.md", "revision": "ed7a29d9", "url": "https://github.com/vllm-project/vllm/pull/16032"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-cutlass-blackwell", "blog-colfax-cutlass", "pr-vllm-16032"], "title": "Epilogue Fusion", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "0455e988208712c28de6084a897d46734ceafe52da948a9d9d003458f845b2c6", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L6", "preview": "Use external source-map research after a profile or benchmark identifies an edit family but the local PR pages do not expose a small enough implementation example. The route is code-first: clone a source-map repository, grep for the measure", "sha256": "fda0609578ad9b7467f4b3c896a7974dfa501c346bd421c46c2ae495e6f05f4e"}, {"id": "u002", "kind": "code", "locator": "body:L8-L12", "preview": "```bash git clone https://github.com/ColfaxResearch/cfx-article-src external/colfax-cfx git clone https://github.com/simveit/load_and_store external/simveit-load-store rg -n \"tma|mbarrier|swizzle|ldmatrix|stream\" external/colfax-cfx externa", "sha256": "4cb7dd51d7a1fbf99bbb5ac8eda9d5b3bc6e28b9f39811f75b8855b07d7717a1"}, {"id": "u003", "kind": "list-item", "locator": "body:L16-L16", "preview": "- Long-scoreboard or poor sector utilization: search load/store and transpose", "sha256": "e1e044d285783854df02bf8775c4f702232a5f2b7144c9542b48b4d552406434"}, {"id": "u004", "kind": "prose", "locator": "body:L17-L17", "preview": "examples before changing vector width or memory layout.", "sha256": "391b8cb5e8d5f96f47b57d53e85abc1c1aee4bdf998b8530dd785aa7af073d51"}, {"id": "u005", "kind": "list-item", "locator": "body:L18-L18", "preview": "- Barrier or TMA wait stalls: search pipelined GEMM examples before changing", "sha256": "7e9ff9c50c4602a68fd32910799ddbf40bec206464970182f14efa6c41d6b1bc"}, {"id": "u006", "kind": "prose", "locator": "body:L19-L19", "preview": "stage count or producer/consumer split.", "sha256": "897a2d90db28ffd767f093cfd9fd0e1c58191433706e2615237d99e154b5afd5"}, {"id": "u007", "kind": "list-item", "locator": "body:L20-L20", "preview": "- Tail waves: search persistent and Stream-K examples before adding a", "sha256": "1e8bc5bf8e3008ac0e8cb5b0a5184d8c51d6121e2106505281564b61a0555d1f"}, {"id": "u008", "kind": "prose", "locator": "body:L21-L21", "preview": "shape-specific dispatcher.", "sha256": "c6f18a9f9adbcdb6e4300adadb01c2ffea5380c371d5d41254c9f9eb257bb702"}, {"id": "u009", "kind": "prose", "locator": "body:L25-L27", "preview": "Do not cite this page as implementation evidence by itself. Cite one of its source pages plus the concrete upstream file path, commit, or URL that shaped the candidate edit.", "sha256": "d0dc5c2a2f12ea0362817f297ebdc9e090e4cdf93b7b1de5bd7a0005a6342830"}], "confidence_claimed": "source-reported", "headings": ["Use", "When It Helps", "Provenance Rule"], "id": "technique-external-source-map-research", "path": "wiki/techniques/external-source-map-research.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/nvidia-code-samples.md", "url": "https://github.com/NVIDIA-developer-blog/code-samples"}, {"path": "sources/blogs/colfax-article-source-kernels.md", "url": "https://github.com/ColfaxResearch/cfx-article-src"}, {"path": "sources/blogs/colfax-cutlass-kernels.md", "url": "https://github.com/ColfaxResearch/cutlass-kernels"}, {"path": "sources/blogs/simveit-effective-transpose.md", "url": "https://github.com/simveit/effective_transpose"}, {"path": "sources/blogs/simveit-load-and-store.md", "url": "https://github.com/simveit/load_and_store"}], "risk_flags": ["code", "ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-nvidia-code-samples", "blog-colfax-article-source-kernels", "blog-colfax-cutlass-kernels", "blog-simveit-effective-transpose", "blog-simveit-load-and-store"], "title": "External Source-Map Research For Kernel Edits", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "6a7689d9e5c0a13d8fc25daca9b0753cba405982bf7443f5fe50d355601c9952", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "Fine-grained quantization applies per-block (rather than per-tensor) scaling factors to low-precision data, preventing outlier values from destroying the quantization precision of an entire tensor. DeepSeek pioneered the tile-wise 1x128 sca", "sha256": "69ea994c6e007c632932eb5141474ff4682345d30f05cdd968507c02f30cce25"}, {"id": "u002", "kind": "code", "locator": "body:L7-L31", "preview": "``` Per-tensor scaling (coarsest): Entire tensor shares one FP32 scale factor. Problem: a single outlier ruins precision for all elements. [=== entire MxK matrix === ] -> 1 scale Per-block 128x128 scaling (weights): Each 128x128 block has i", "sha256": "44b5ff16ab8902c1adb2296033c4501a5a75f6413d1cab24e0eb41c3c16dffc0"}, {"id": "u003", "kind": "prose", "locator": "body:L35-L35", "preview": "DeepGEMM implements fine-grained FP8 GEMM with two scaling patterns:", "sha256": "1d6d0b8a8239becc8c104ba6bf4ea4e6743812453d7b3f86e8d031c693496659"}, {"id": "u004", "kind": "code", "locator": "body:L37-L76", "preview": "```cuda // DeepGEMM FP8 GEMM with fine-grained scaling // A (activations): FP8 E4M3 with tile-wise 1x128 FP32 scales // B (weights): FP8 E4M3 with block-wise 128x128 FP32 scales // C (output): FP32 accumulator // Scale tensor shapes: // sca", "sha256": "88c7ae2257568dbd086e4f4de74f4cd07f153ae301c92b0fcc0052f2d9295880"}, {"id": "u005", "kind": "prose", "locator": "body:L80-L80", "preview": "On Hopper (SM90), the wgmma instruction accumulates in Tensor Core registers with limited precision (~FP22, not true FP32). To maintain numerical accuracy, DeepGEMM promotes the partial sum to a separate FP32 accumulator on CUDA Cores every", "sha256": "47af4e4d25db5502b9b59a75325f9e699f6840549f075e1851da443c9b8afabc"}, {"id": "u006", "kind": "code", "locator": "body:L82-L126", "preview": "```cuda // Hopper FP8 GEMM with CUDA Core promotion (DeepGEMM pattern) // Every Nc=128 K-elements, promote Tensor Core accumulator to FP32 __device__ void hopper_fp8_gemm_with_promotion( const fp8_e4m3* A, const fp8_e4m3* B, const float* sc", "sha256": "2bba6362e8c94126f6cc6bb8d741847f84bf0515dabc3c8e36152aa18e0d0a24"}, {"id": "u007", "kind": "prose", "locator": "body:L128-L128", "preview": "The Nc=128 interval was chosen because:", "sha256": "73a7b47ff0d8d1bd839627042830bc4899aedc9628f798f89f51c305a4e30d94"}, {"id": "u008", "kind": "list-item", "locator": "body:L129-L129", "preview": "- 4 wgmma operations process 128 K-elements (4 x 32)", "sha256": "d553be609f652ef2fab306881444ad91840ba6092b27b799b52606eedda4c578"}, {"id": "u009", "kind": "list-item", "locator": "body:L130-L130", "preview": "- At this interval, the accumulated FP22 error is bounded to ~0.1% relative error", "sha256": "7ba18b03c324394560e8e41b2793e501f8edf488d286ca6fc090b8068c3d8961"}, {"id": "u010", "kind": "list-item", "locator": "body:L131-L131", "preview": "- Fewer than 4 ops (Nc=32, Nc=64) adds too much promotion overhead", "sha256": "7c4371e6acacff82c0c253ae371477a05a288610edb640765ba6f1cbba2dc977"}, {"id": "u011", "kind": "list-item", "locator": "body:L132-L132", "preview": "- More than 4 ops (Nc=256) allows unacceptable precision loss", "sha256": "4f9d6a97a8ee06449508511389e015fb9c08db4155f430af68656ed391530cff"}, {"id": "u012", "kind": "prose", "locator": "body:L136-L136", "preview": "On Blackwell, tcgen05.mma supports native block scaling via the UE8M0 (unsigned 8-bit exponent, no mantissa) format. The hardware applies per-block scale factors directly during the MMA operation, eliminating the software promotion step:", "sha256": "6f3f163ac439ecc4337d319e83e934edb710e7eefdb270cac2f4e1876f415a7e"}, {"id": "u013", "kind": "code", "locator": "body:L138-L158", "preview": "```cuda // Blackwell native block scaling with UE8M0 // Scale format: UE8M0 = pure power-of-two scale (2^exponent) // Packed: 4 UE8M0 values per 32-bit integer // DeepGEMM SM100 kernel: scale_A and scale_B are UE8M0 packed // tcgen05.mma ap", "sha256": "c82d98d828c202d011a0e6d7f4e5841f4defb5b2abc0aacdceb3693b91524729"}, {"id": "u014", "kind": "code", "locator": "body:L160-L171", "preview": "```ptx // tcgen05.mma with native block scaling (Blackwell PTX) // This instruction applies UE8M0 scales from SMEM during MMA tcgen05.mma.cta_group::1.kind::f8f6f4 [%tmem_addr], // TMEM accumulator destination [%desc_a], // SMEM descriptor ", "sha256": "526613b64a2d8018ef4ba7d7b6bc97a1c0c5efcd68b558ed2f5c934ba3164d11"}, {"id": "u015", "kind": "table-row", "locator": "body:L177-L177", "preview": "| Property | UE8M0 (Blackwell native) | E4M3 (NVFP4 hackathon) | FP32 (DeepGEMM Hopper) | | Bits | 8 | 8 | 32 |", "sha256": "44646c385ec1adceae88542d8c19d91e0d014bd23193c59b5ea8b16a20728082"}, {"id": "u016", "kind": "table-row", "locator": "body:L178-L178", "preview": "| Property | UE8M0 (Blackwell native) | E4M3 (NVFP4 hackathon) | FP32 (DeepGEMM Hopper) | | Representable values | Powers of 2 only | 240 distinct values | Full FP32 range |", "sha256": "042eb8db12238b466459dff2dbf12924d215944dc7b22d12a5cf2fa97fb0f29b"}, {"id": "u017", "kind": "table-row", "locator": "body:L179-L179", "preview": "| Property | UE8M0 (Blackwell native) | E4M3 (NVFP4 hackathon) | FP32 (DeepGEMM Hopper) | | Range | 2^-127 to 2^128 | ~0 to 448 | Full FP32 |", "sha256": "71f0851dd2659ae62cc1b796b7f09ceceb099d89ffb961ebdd1f2fb404c5eea6"}, {"id": "u018", "kind": "table-row", "locator": "body:L180-L180", "preview": "| Property | UE8M0 (Blackwell native) | E4M3 (NVFP4 hackathon) | FP32 (DeepGEMM Hopper) | | Block size | 32 (MXFP standard) | 16 (NVFP4) | 128 (DeepGEMM) |", "sha256": "84303981408f8532d9977c8e75a6e3f88b124e4b5d37679286d13af0e212ac84"}, {"id": "u019", "kind": "table-row", "locator": "body:L181-L181", "preview": "| Property | UE8M0 (Blackwell native) | E4M3 (NVFP4 hackathon) | FP32 (DeepGEMM Hopper) | | Hardware support | tcgen05.mma native | Software decode | Software promotion |", "sha256": "12914422262a2b9f0372ead8f9e89f4ea0d3a9be08f4a93d1ee9bf0b57751d52"}, {"id": "u020", "kind": "table-row", "locator": "body:L182-L182", "preview": "| Property | UE8M0 (Blackwell native) | E4M3 (NVFP4 hackathon) | FP32 (DeepGEMM Hopper) | | Precision impact | Coarser (power-of-2 only) | Fine (non-power-of-2) | Best (FP32) |", "sha256": "b873b0baa4c15a1b52abdfaf54afef4409b2902c0e7f71ba20b4db4ecc6d4fdc"}, {"id": "u021", "kind": "prose", "locator": "body:L186-L186", "preview": "The NVFP4 format used in the GPU Mode hackathon has its own scaling scheme:", "sha256": "d3748f2711487284d8a4c19c448b600c08937c032f6a6e5a5c44517c9285a7f4"}, {"id": "u022", "kind": "code", "locator": "body:L188-L211", "preview": "```cuda // NVFP4 dequantization with two-level scaling // Level 1: per-block FP8 E4M3 scale (every 16 FP4 elements) // Level 2: per-tensor FP32 global scale __device__ float dequant_nvfp4( uint8_t fp4_packed, // Two FP4 values packed in one", "sha256": "c5afbdd784a734e5be7a88738c8b5179da8ce1f382d2419158309cae3c5880bc"}, {"id": "u023", "kind": "list-item", "locator": "body:L215-L215", "preview": "- **FP8 training**: Use tile-wise 1x128 for activations and block-wise 128x128 for weights (DeepGEMM pattern). This is the validated approach for training 671B+ parameter models.", "sha256": "b1b2df16a7c020df4c52a84886c04e646e3521d3f86a951034f1459a8d863c6c"}, {"id": "u024", "kind": "list-item", "locator": "body:L216-L216", "preview": "- **FP4 inference on Blackwell**: Use NVFP4 with E4M3 block scales (block size 16) for highest precision, or MXFP4 with UE8M0 scales (block size 32) for native hardware acceleration.", "sha256": "309f292ffece1d8dc137a7e2690318145a332b8f5ee8404b5701898e2f547807"}, {"id": "u025", "kind": "list-item", "locator": "body:L217-L217", "preview": "- **Hopper FP8 inference**: Use CUDA core promotion with Nc=128 interval to maintain precision despite limited TC accumulation.", "sha256": "4850fa287572c51884067ad1b43db7ae5d0e7577577631d375d0e7813e284fc5"}, {"id": "u026", "kind": "list-item", "locator": "body:L221-L221", "preview": "- UE8M0 scales are power-of-two only. Non-power-of-two distributions (common in activations) lose precision compared to E4M3 or FP32 scales.", "sha256": "699a48b3627cad16bd6753ae3b1b8b2b856632bac7018526a0e7773738e5388a"}, {"id": "u027", "kind": "list-item", "locator": "body:L222-L222", "preview": "- Smaller block sizes (16 for NVFP4 vs 128 for DeepGEMM) provide better precision but higher overhead: more scale values to store, load, and apply.", "sha256": "90c3d0ebe6fffa3eba19e4defb5af6411dac30fc5a32a5703fd4a2e8ad45688e"}, {"id": "u028", "kind": "list-item", "locator": "body:L223-L223", "preview": "- The Nc=128 promotion interval on Hopper is a performance-accuracy tradeoff. Reducing Nc improves accuracy but adds more promotion overhead. Increasing Nc risks precision degradation.", "sha256": "8e46f8ccc3b5839ab58ef9b58bb9196e3c20dca3da4f248ee8af48fa6010a05f"}, {"id": "u029", "kind": "list-item", "locator": "body:L224-L224", "preview": "- On Blackwell, native block scaling only works with UE8M0. Using E4M3 or FP32 scales still requires software handling.", "sha256": "d8108b20916960eb4b052c552fc9d6af0269b24a6193cc73a478fb20cc2d4c2f"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Scaling Granularities", "DeepGEMM: Tile-wise and Block-wise Scaling", "Hopper: CUDA Core Promotion (Nc=128)", "Blackwell: Native Block Scaling", "UE8M0 vs E4M3 Scale Formats", "NVFP4 Two-Level Scaling", "When to Use", "Caveats"], "id": "technique-fine-grained-quantization", "path": "wiki/techniques/fine-grained-quantization.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/deepgemm.md", "url": "https://github.com/deepseek-ai/DeepGEMM"}, {"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/prs/vllm/PR-23696.md", "revision": "074854b2", "url": "https://github.com/vllm-project/vllm/pull/23696"}], "risk_flags": ["code", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-deepgemm", "doc-nvidia-tuning-guide", "pr-vllm-23696"], "title": "Fine-Grained FP8/FP4 Quantization", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "685ddc5bebbf6ea4d6782e495c6a48fe89232b6b41680c67c23e8921b5ab2ed2", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Kernel fusion combines multiple operations into a single kernel launch, eliminating intermediate global memory roundtrips. Critical for MoE and attention pipelines where 5-7 sequential launches each incur latency, synchronization, and memor", "sha256": "4348a5937ccfed2a939f803579ac3b928863f7321eddd1e4fb25c087a6f2ff92"}, {"id": "u002", "kind": "code", "locator": "body:L10-L26", "preview": "```cuda // Instead of: gate_gemm \u2192 up_gemm \u2192 silu \u2192 multiply (4 kernels) // Fused: single kernel with two TMEM accumulators __global__ void fused_gate_up_silu(...) { uint32_t tmem_gate = tmem_alloc(256); uint32_t tmem_up = tmem_alloc(256); ", "sha256": "9ca307c7663407642158d2f70fe8a4b13b04db4697cdcb498d763444171b54fa"}, {"id": "u003", "kind": "list-item", "locator": "body:L29-L29", "preview": "- **vLLM (7 kernels)**: softmax \u2192 topk \u2192 dispatch \u2192 gate \u2192 up \u2192 silu_mul \u2192 down+combine", "sha256": "77689aa21187be9b45b915ef9cc1f3cd677100447ca4ee721db219b1f5871325"}, {"id": "u004", "kind": "list-item", "locator": "body:L30-L30", "preview": "- **SGLang (5 kernels)**: router+topk \u2192 dispatch \u2192 fused gate-up-silu \u2192 down \u2192 combine", "sha256": "dcd1c6a9b59d0e78c29dd5bf6cf3e94c53a06252e6a26d10eabc3e2fe55e145d"}, {"id": "u005", "kind": "list-item", "locator": "body:L31-L31", "preview": "- **Ideal (1-2 kernels)**: all ops in one launch, saves 21.9% activation memory traffic", "sha256": "39731798392de496da35c97a9ab2225f5493495105689d3977e1e9461d3b684c"}, {"id": "u006", "kind": "list-item", "locator": "body:L35-L35", "preview": "- TMEM capacity limits how many accumulators can fuse (256 cols total)", "sha256": "48720720c315733505b2911628630374c260644bc9845e2cabe261b652afdbc8"}, {"id": "u007", "kind": "list-item", "locator": "body:L36-L36", "preview": "- Register pressure on epilogue if fusing complex activations", "sha256": "648f7d92704df2e751086497d772e17e29429b837b5ced8d7e40244bb0e5970a"}, {"id": "u008", "kind": "list-item", "locator": "body:L37-L37", "preview": "- Fusion opportunities depend on dataflow shape (dependency graph must be DAG-compatible with CTA scope)", "sha256": "be9b6c6cc1824860ab23e11e7e531673511fb8ad0eef90d2f36213271b9d7f5b"}, {"id": "u009", "kind": "list-item", "locator": "body:L40-L40", "preview": "- [fused-moe](../kernels/fused-moe.md)", "sha256": "fa81a99d5f90ff4b3faaeb9180207c7daf999b1a0783f3fbb2c56479431912f1"}, {"id": "u010", "kind": "list-item", "locator": "body:L41-L41", "preview": "- [epilogue-fusion](epilogue-fusion.md)", "sha256": "7473078233c50cac93904892f08c3e7037ecb7747b2b5c9abca0a8dd905cfedc"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Examples", "Fused Gate-Up Dual GEMM + SwiGLU", "MoE Fusion Progression", "Constraints", "Related"], "id": "technique-kernel-fusion", "path": "wiki/techniques/kernel-fusion.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md", "url": "https://github.com/gpu-mode/reference-kernels"}, {"path": "sources/contests/flashinfer-mlsys26/track-a-fused-moe.md", "url": "https://mlsys26.flashinfer.ai/"}, {"path": "sources/blogs/tflops-gap-fp4-moe.md", "url": "https://huggingface.co/blog/apsys/blackwell-nvfp4-comparison"}], "risk_flags": ["code", "ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["contest-gpumode-p3", "contest-flashinfer-track-a", "blog-tflops-gap-fp4-moe"], "title": "Kernel Fusion", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "7d1d16d68cfae0d699d1b73b4d10ac71a12c878966081788134245f3861fc283", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "Persistent kernels launch exactly as many CTAs as SMs, and each CTA processes multiple output tiles in a loop rather than exiting after one tile. On Blackwell, the CLC (Cluster Launch Control) hardware unit replaces software-based tile sche", "sha256": "e8e36d68148157b9682fa98efa2b8852c39ba9cadbe075559246b392b8132470"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "The core persistent kernel loop on Blackwell uses CLC to dynamically assign tiles:", "sha256": "d00b8e36858ea1cb03e3865cef79e879712cc9fdee5db36dac678b638b3e155d"}, {"id": "u003", "kind": "code", "locator": "body:L9-L40", "preview": "```cuda // Persistent kernel with CLC tile scheduling (Blackwell SM100) __global__ void __launch_bounds__(512) persistent_gemm_clc(const __grid_constant__ GemmParams params) { // CLC-managed persistent loop: each CTA processes multiple tile", "sha256": "d293d6d2db1df05cb586cfe1acd9fd74c96529152b52571a4c1af3d45b75fcfa"}, {"id": "u004", "kind": "prose", "locator": "body:L42-L44", "preview": "At the PTX level, the CLC interaction is a cancel/query sequence. The exact inline PTX is usually hidden behind CUTLASS/CuTe wrappers, but the control flow looks like this:", "sha256": "c50b1466112dc222ff111f047f558f8b2f03965875f05fbaa0634798860ae028"}, {"id": "u005", "kind": "code", "locator": "body:L46-L58", "preview": "```text TILE_LOOP: // Request cancellation of a not-yet-launched cluster. clusterlaunchcontrol.try_cancel(response_smem, mbarrier) wait(mbarrier) // Query the 16-byte response. has_work, tile_m, tile_n = clusterlaunchcontrol.query_cancel(re", "sha256": "ff3a7192cbac1849c95c96526c779c1ee4d618c1119004540d3b9981d48eb8b1"}, {"id": "u006", "kind": "prose", "locator": "body:L62-L62", "preview": "On Hopper (SM90), persistent kernels use a static stride pattern where each CTA computes tiles at fixed intervals:", "sha256": "2fa4e0e86a85e15db5199b2f2e0db272c22d25417b2cf6e7b01998183b0daed0"}, {"id": "u007", "kind": "code", "locator": "body:L64-L79", "preview": "```cuda // Hopper-style static stride persistent kernel __global__ void hopper_persistent_gemm(GemmParams params) { int cta_id = blockIdx.x; int total_ctas = gridDim.x; int total_tiles = params.num_tiles_m * params.num_tiles_n; // Static st", "sha256": "0132a814c47eadecc7ac6ccb66be30bec3ce8cd8bb57f598647fc9ae9b83a6cb"}, {"id": "u008", "kind": "table-row", "locator": "body:L83-L83", "preview": "| Aspect | Hopper Static Stride | Blackwell CLC | | Scheduling | Software loop with fixed stride | Hardware CLC unit assigns tiles |", "sha256": "8edf2923dc1b0eab7fb68673908bc35e885a360c94f1a6278a402c58ad61a4ea"}, {"id": "u009", "kind": "table-row", "locator": "body:L84-L84", "preview": "| Aspect | Hopper Static Stride | Blackwell CLC | | Load balancing | Fixed; uneven if tile costs vary | Dynamic; CLC rebalances automatically |", "sha256": "81b1b129852e8f5d7fd2c223b68b1ca5a001a72baae445ae9eda2f347283f342"}, {"id": "u010", "kind": "table-row", "locator": "body:L85-L85", "preview": "| Aspect | Hopper Static Stride | Blackwell CLC | | Tail effect | Last wave may have partial occupancy | CLC minimizes by giving fast CTAs more tiles |", "sha256": "943c65b8013de98dd0c46d8e5a9ea57f604182d56cdf69448388f39899a5a630"}, {"id": "u011", "kind": "table-row", "locator": "body:L86-L86", "preview": "| Aspect | Hopper Static Stride | Blackwell CLC | | Launch overhead | Grid launch for each new problem | CLC can chain multiple problems |", "sha256": "fa73cac0250a303e4b09bd5e56d8616d854df6af4146e671eedb970f208185c1"}, {"id": "u012", "kind": "table-row", "locator": "body:L87-L87", "preview": "| Aspect | Hopper Static Stride | Blackwell CLC | | Termination | Implicit when loop ends | Explicit `try_cancel` |", "sha256": "470b1fcdde44684241a2a5dadd6e1fce4d4d5c5c5113ece57c3125740a134981"}, {"id": "u013", "kind": "table-row", "locator": "body:L88-L88", "preview": "| Aspect | Hopper Static Stride | Blackwell CLC | | L2 locality | Depends on stride pattern | CLC can apply swizzled raster |", "sha256": "a4b976af33b07f033e28de4eefce78a628f2044ef95f49e8a6eace5776708174"}, {"id": "u014", "kind": "prose", "locator": "body:L92-L92", "preview": "CUTLASS 4.5.0 provides `PersistentTileSchedulerSm100` that wraps the CLC hardware:", "sha256": "eccbc64b8272087c5b7f1f89b541f6a7601bf14caa10370202fb90f01d223923"}, {"id": "u015", "kind": "code", "locator": "body:L94-L154", "preview": "```cuda // CUTLASS SM100 persistent tile scheduler (simplified) template struct PersistentTileSchedulerSm100 { // Initialize the CLC with the problem geometry CUTLASS_DEVICE static void init( dim3 problem_tiles, void* clc_", "sha256": "8db29837df5850f108e3512819e6486902a381701d4b25ed410be4a555a6c88e"}, {"id": "u016", "kind": "prose", "locator": "body:L158-L158", "preview": "The tcgen05-tutorial progression demonstrates the impact of persistent kernels:", "sha256": "e7e399991855ad8c0005fbc52c2be1c5f1207b74e328ba5f10b19dd3d4bd7316"}, {"id": "u017", "kind": "code", "locator": "body:L160-L163", "preview": "``` Without persistence (static grid): 940 TFLOPS (62% of peak) With CLC persistent scheduling: 1476 TFLOPS (98% of cuBLAS) ```", "sha256": "779f994872a8ba6929fb5083d16acecb183d84150f350b189b00472c58509626"}, {"id": "u018", "kind": "prose", "locator": "body:L165-L165", "preview": "The 57% improvement comes from:", "sha256": "86ad96613390a3e7cba296d569bf4b2e0ea02d7688700dde6f7ce1145e533c10"}, {"id": "u019", "kind": "list-item", "locator": "body:L166-L166", "preview": "1. **Eliminated tail effect**: CLC dynamically assigns tiles, so fast-completing CTAs absorb extra work rather than sitting idle while the last wave finishes.", "sha256": "8cb9710f12f11ba0448af3a7f2e5c40dd0d5e4255942bdc759f98b2dec9c2207"}, {"id": "u020", "kind": "list-item", "locator": "body:L167-L167", "preview": "2. **Reduced launch overhead**: A single kernel launch covers all tiles; no need to re-launch grids.", "sha256": "da95af84a43e0d56d646f339ee0a6e2d4e40ccf5ee4bf0b4813917c6e67680e7"}, {"id": "u021", "kind": "list-item", "locator": "body:L168-L168", "preview": "3. **Better L2 cache utilization**: CLC can apply a swizzled raster pattern that improves spatial locality across neighboring tiles.", "sha256": "0f53e7d0312c5dad03b039b1d89bf425405b0352675f0d5b116c221454c4ef13"}, {"id": "u022", "kind": "list-item", "locator": "body:L172-L172", "preview": "- **Large GEMM problems**: Persistent kernels are most beneficial when the number of output tiles exceeds the SM count by at least 2-3x.", "sha256": "627d205eaaa11d3e3bf243b5d9f079c6961ca143533c59772038c4edc31487c0"}, {"id": "u023", "kind": "list-item", "locator": "body:L173-L173", "preview": "- **Grouped GEMMs / MoE**: CLC can chain multiple problem instances, eliminating inter-kernel launch gaps.", "sha256": "84b5ebee1a8f16e1a5b987ac252c9da1cb6f7790690801f5ddea40085531c868"}, {"id": "u024", "kind": "list-item", "locator": "body:L174-L174", "preview": "- **Workloads with uneven tile cost**: CLC's dynamic scheduling naturally handles variable-cost tiles (e.g., triangular attention masks).", "sha256": "5ede804dd82231ab723c1fcc15adae4561ef761db392b54751eebb1e64205c7a"}, {"id": "u025", "kind": "list-item", "locator": "body:L178-L178", "preview": "- CLC is SM100-only; Hopper kernels must use software-based scheduling.", "sha256": "a1daa43e43efe5d78c2b66c5e743f8fe9d2cff18ff0e1eeca6875f2a3b63a4d2"}, {"id": "u026", "kind": "list-item", "locator": "body:L179-L179", "preview": "- The `try_cancel` pattern introduces a potential race that must be handled with a retry loop.", "sha256": "27ac0a9a9cda1f785a742912aafe5f7c9f6c8477f69e72eb4cfb3ab6cb2f9d34"}, {"id": "u027", "kind": "list-item", "locator": "body:L180-L180", "preview": "- For very small problems (fewer tiles than SMs), CLC overhead may not justify the complexity. A simple single-wave grid launch suffices.", "sha256": "86018b81505700f865cbce903ffda5f76bad6f6620b055c4c4839c9bfad3abb8"}, {"id": "u028", "kind": "prose", "locator": "body:L184-L184", "preview": "Verbatim upstream code lives in [`artifacts/kernels/persistent-kernels/full/`](../../artifacts/kernels/persistent-kernels/full/); labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` header) ", "sha256": "dae6ce03f82e0f9348174a57abd5f8a30fe21a539b0ac794e603406732dabafc"}, {"id": "u029", "kind": "prose", "locator": "body:L186-L186", "preview": "Query via:", "sha256": "6abd63e508f9320fdd3b2b0cc5b5f6a000f4462abfd17f9ecc3c4818f4a0f875"}, {"id": "u030", "kind": "code", "locator": "body:L188-L190", "preview": "```bash python3 scripts/get_page.py technique-persistent-kernels --include-code ```", "sha256": "ad38aeaa314e4436d6603b855048ed00f795b0832bc4c697a9300dc2c5893567"}], "confidence_claimed": "source-reported", "headings": ["Overview", "CLC Loop Pattern", "Comparison: CLC vs Static Stride (Hopper)", "CUTLASS PersistentTileSchedulerSm100", "Performance Impact", "When to Use", "Caveats", "Full Reference Implementation"], "id": "technique-persistent-kernels", "path": "wiki/techniques/persistent-kernels.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/docs/nvidia-cutlass-blackwell.md", "url": "https://docs.nvidia.com/cutlass/latest/CHANGELOG.html"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "doc-cutlass-blackwell"], "title": "Persistent Kernels with CLC", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "1e86380f8ba7bfb3a8a4c0f9aeaba796da8fef032eca6b4abc5dbacf72564d20", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Ping-pong scheduling alternates two query tiles within a single CTA so the softmax warpgroup never stalls waiting for MMA. Introduced in FlashAttention-4 to exploit Blackwell's asymmetric hardware (2\u00d7 tensor cores, same SFU count as Hopper)", "sha256": "18017e364da96a8b6c710733af3652816685d2d7acff8065b2c1211031714493"}, {"id": "u002", "kind": "code", "locator": "body:L9-L35", "preview": "```cuda // Two 128-token query tiles per CTA, alternating through the mainloop // Warpgroup 0: softmax for tile A while MMA runs on tile B // Warpgroup 1: softmax for tile B while MMA runs on tile A __global__ void fa4_ping_pong_attn(...) {", "sha256": "8499e0fdd764e721b32d8078a983f064bf167a0e53fd027cda13d9cb18c80438"}, {"id": "u003", "kind": "list-item", "locator": "body:L39-L39", "preview": "- Tensor core throughput doubled (B200 vs H100) but SFU count unchanged", "sha256": "0a46ee52bba3c57ee8905ac1dd50c402869daff94fb8d5d44cbe9342220c1c20"}, {"id": "u004", "kind": "list-item", "locator": "body:L40-L40", "preview": "- Single-tile schedule would leave SFU idle while MMA runs, and vice versa", "sha256": "a2d9a909cc8c0ab7538af1c1254a9115a8d0ea0a8f77b1998f0a2af19f4529df"}, {"id": "u005", "kind": "list-item", "locator": "body:L41-L41", "preview": "- Ping-pong keeps both units 100% busy", "sha256": "f08543a636554e0e7512cf8b70653642414461f288e043bbc48d4248ae8d833a"}, {"id": "u006", "kind": "list-item", "locator": "body:L42-L42", "preview": "- FA4 achieves 1605 TFLOPS BF16 (71% utilization) with this pattern", "sha256": "9e4b6969c16376c6419faf7861ff0bc4701f4576f67ffb0baaa4a541e09e7250"}, {"id": "u007", "kind": "list-item", "locator": "body:L46-L46", "preview": "- Compute-bound attention kernels on Blackwell", "sha256": "4c6cf49898eb794803dc3516455cd225fda03a58d5789fe7d9f558e6a99f3f9c"}, {"id": "u008", "kind": "list-item", "locator": "body:L47-L47", "preview": "- Kernels where softmax/epilogue is SFU-heavy", "sha256": "1703c48c4e416896eaaed17a937e8fcb95843e111beca19257da1483781a03b3"}, {"id": "u009", "kind": "list-item", "locator": "body:L48-L48", "preview": "- Not useful on Hopper (balance is different)", "sha256": "9738e0176814d096ab8c4872e52db170c64347109cc167477fb15edbf1830095"}, {"id": "u010", "kind": "prose", "locator": "body:L52-L52", "preview": "Verbatim upstream code lives in [`artifacts/kernels/ping-pong-scheduling/full/`](../../artifacts/kernels/ping-pong-scheduling/full/); labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` head", "sha256": "42846bee13f84639175b92a183fb0becf7f05f6c9a793e519973597b26af7006"}, {"id": "u011", "kind": "prose", "locator": "body:L54-L54", "preview": "Query via:", "sha256": "6abd63e508f9320fdd3b2b0cc5b5f6a000f4462abfd17f9ecc3c4818f4a0f875"}, {"id": "u012", "kind": "code", "locator": "body:L56-L58", "preview": "```bash python3 scripts/get_page.py technique-ping-pong-scheduling --include-code ```", "sha256": "5e4fb2a38570963673bb5efeb1ad1c9d48bb10b940a9dd8dc6534e549d5c0fbc"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Pattern", "Why It Helps on Blackwell", "When To Use", "Full Reference Implementation"], "id": "technique-ping-pong-scheduling", "path": "wiki/techniques/ping-pong-scheduling.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/flash-attention-4.md", "url": "https://tridao.me/blog/2026/flash4/"}, {"path": "sources/docs/flash-attention-4.md", "url": "https://arxiv.org/abs/2603.05451"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}], "risk_flags": ["code"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-flash-attention-4", "doc-flash-attention-4", "blog-tcgen05-tutorial"], "title": "Ping-Pong Scheduling", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "d990987f96b0c765ab7753798f189a2125cbab0619c1671b450e305a4c628966", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "Software pipelining overlaps data loading (TMA copies from global to shared memory) with computation (tcgen05.mma or wgmma) by maintaining multiple in-flight tile buffers. A circular buffer of 3-5 stages allows the TMA producer to fill stag", "sha256": "6c9314211ed0931aa6ca230a23ffde47272c800d7260735c177b32239a6c19fb"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "The tcgen05-tutorial demonstrates the performance impact of pipelining:", "sha256": "32d64417f35a0af386685bffccdf887d9b1ec91318b42c992a3a6fdc962aa943"}, {"id": "u003", "kind": "code", "locator": "body:L9-L13", "preview": "``` No pipelining (load, then compute): 695 TFLOPS (46%) 3-stage pipeline (TMA + MMA overlap): 940 TFLOPS (62%) + warp specialization: 1476 TFLOPS (98%) ```", "sha256": "964e3061ea3896a9894f13629a043dcb9dabaa7ab4bd9c735faa2b100733571a"}, {"id": "u004", "kind": "prose", "locator": "body:L15-L15", "preview": "The 35% improvement from pipelining alone (695 to 940 TFLOPS) comes from hiding global memory latency behind MMA computation.", "sha256": "933f3ac4fc131a0771c0bf278426040e94e94e64bb7a47c5267f66388b2e01f3"}, {"id": "u005", "kind": "prose", "locator": "body:L19-L19", "preview": "The fundamental pattern allocates `NUM_STAGES` copies of each SMEM buffer and cycles through them:", "sha256": "472983f5e9b59df349b80d12cbfb19ec1eb6d3ac98b1b715d418462fb0288e07"}, {"id": "u006", "kind": "code", "locator": "body:L21-L110", "preview": "```cuda // 3-stage circular buffer with mbarrier synchronization // Stages: [0] loading, [1] ready for MMA, [2] being consumed by MMA #define NUM_STAGES 3 __global__ void __launch_bounds__(512) pipelined_gemm(const __grid_constant__ GemmPar", "sha256": "9c4857b6c3bc136a10acb067785e356592410c227ef8e38a3275cc0718d38ad7"}, {"id": "u007", "kind": "prose", "locator": "body:L114-L114", "preview": "The optimal number of pipeline stages depends on the ratio of memory latency to compute time per tile:", "sha256": "c771eeac340fd324a834e01623d2622fc10179586cda8d27cc4afa755c64111c"}, {"id": "u008", "kind": "table-row", "locator": "body:L118-L118", "preview": "| Stages | SMEM Usage | Latency Hiding | Best For | | 2 | 2x base | Partial | Small tiles, limited SMEM |", "sha256": "2f0fdedf15dcc2f91d98558670aeb908d3ab4b50dfdffeb42a63bb3bb5db36b0"}, {"id": "u009", "kind": "table-row", "locator": "body:L119-L119", "preview": "| Stages | SMEM Usage | Latency Hiding | Best For | | 3 | 3x base | Full for most GEMMs | Standard choice on Blackwell |", "sha256": "37ce4b2f1509ff5519e008b1bb225294aced130706fdd90911201e0abbaced76"}, {"id": "u010", "kind": "table-row", "locator": "body:L120-L120", "preview": "| Stages | SMEM Usage | Latency Hiding | Best For | | 4-5 | 4-5x base | Full with margin | Large K, high memory latency |", "sha256": "d1a052bf98acfaac91757adad9b0beb651c0b0b2d0f5723a479b024c3a20dac7"}, {"id": "u011", "kind": "table-row", "locator": "body:L121-L121", "preview": "| Stages | SMEM Usage | Latency Hiding | Best For | | >5 | Excessive | Diminishing returns | Rarely justified |", "sha256": "a552c9fee13a0315c5f3f5dd0123b3f2613c449ade64502a9fa75f110bd9fa9a"}, {"id": "u012", "kind": "prose", "locator": "body:L123-L123", "preview": "The SMEM budget on Blackwell is 228 KB per SM. For a BF16 GEMM with TILE_M=128, TILE_N=256, TILE_K=64:", "sha256": "26344b17c2315629889de2844b3c6707ba92c8e363c6d1e03c447dca260d1ce8"}, {"id": "u013", "kind": "list-item", "locator": "body:L124-L124", "preview": "- A tile: 128 x 64 x 2B = 16 KB", "sha256": "03e4c1bf2be972a87f1a0735f4dfc980a930b5f6c7fb0a86119a2159008c713b"}, {"id": "u014", "kind": "list-item", "locator": "body:L125-L125", "preview": "- B tile: 64 x 256 x 2B = 32 KB", "sha256": "eb7f61ed161a3cecb248d9d96a0edc35c452ff2e0bd16e08bd3a6dd924a1eaca"}, {"id": "u015", "kind": "list-item", "locator": "body:L126-L126", "preview": "- Per stage: 48 KB", "sha256": "d950f9ecb5785f4374387ec03ada37cbd812dd06d48cf81892b50ef2360098f1"}, {"id": "u016", "kind": "list-item", "locator": "body:L127-L127", "preview": "- 3 stages: 144 KB (63% of SMEM, leaves room for barriers and epilogue)", "sha256": "06f710fddfa9c0558c9da66757b2546af1d863d17ca3e1d18f5f9d16194f37f2"}, {"id": "u017", "kind": "list-item", "locator": "body:L128-L128", "preview": "- 5 stages: 240 KB (exceeds SMEM capacity; must use smaller tiles)", "sha256": "d377ce7cb994882e900ecfeb866422da4e28287925375571fdef1f414de98e7b"}, {"id": "u018", "kind": "prose", "locator": "body:L132-L132", "preview": "The mbarrier (memory barrier) is the hardware primitive that makes pipelining efficient. Unlike `__syncthreads()`, mbarrier supports asymmetric producer-consumer synchronization where only the relevant warp participates:", "sha256": "8f9ebd31babfc22fd14e2d0430c0181ab305c38c8d95805026f53bec768cabe7"}, {"id": "u019", "kind": "code", "locator": "body:L134-L153", "preview": "```ptx // Phase-based mbarrier protocol for 3-stage pipeline // // Each mbarrier tracks a \"phase\" (0 or 1). The producer flips the phase // on arrive; the consumer waits for the expected phase. // Producer: arrive on stage %s (flips phase) ", "sha256": "753799262915aff51c7d8b6cec86568acc234eed75fc38d91988899f416253f7"}, {"id": "u020", "kind": "prose", "locator": "body:L155-L155", "preview": "The key advantage is that TMA can arrive on an mbarrier autonomously. The producer warp only needs to initiate the TMA; the TMA hardware signals completion directly, removing the producer from the critical path.", "sha256": "553cb5fbfedd97caa8bbd5a6661ae1ac581f7968fc5c2261258b256e6fe2e270"}, {"id": "u021", "kind": "prose", "locator": "body:L159-L159", "preview": "The Modular blog series describes a 5-stage circular buffer reaching 85% of SOTA performance on Blackwell:", "sha256": "189b0ac34a4da5e1dcb0a67f05722472781ce9c486c9b3effeec3d0d3e59da1a"}, {"id": "u022", "kind": "code", "locator": "body:L161-L181", "preview": "```cuda // Modular-style 5-stage pipeline constants // Chosen to fully hide B200 HBM latency (~400 cycles) // while fitting within 228 KB SMEM budget constexpr int NUM_STAGES = 5; constexpr int TILE_M = 128; constexpr int TILE_N = 128; // S", "sha256": "3c1661c7453ab8871b622a57668d830fb30debf05921973ebdd4e5ed988a30b2"}, {"id": "u023", "kind": "list-item", "locator": "body:L185-L185", "preview": "- **All memory-bound and compute-bound GEMM kernels**: Pipelining is never harmful and always improves utilization by hiding latency.", "sha256": "b0d1156c0820474c04afb80ad6045c8c66f1a0a9a74f4eaf4bc22e42d7125180"}, {"id": "u024", "kind": "list-item", "locator": "body:L186-L186", "preview": "- **Attention kernels**: The K-dimension loop in attention benefits from pipelining the KV tile loads.", "sha256": "5dde7afc71b1ce71e3ad7acf1e599cecded4cdd43e4d215d2c3e1399f11cae25"}, {"id": "u025", "kind": "list-item", "locator": "body:L187-L187", "preview": "- **Combined with warp specialization**: Pipelining provides the buffer structure; warp specialization assigns the producer/consumer roles. The two techniques are complementary and almost always used together.", "sha256": "8f89ca68aa97c882f9cdd9da5b1761404890c80106fc2e61e8582f4f1231f1b3"}, {"id": "u026", "kind": "list-item", "locator": "body:L191-L191", "preview": "- More stages increase SMEM usage linearly. On Blackwell's fixed 228 KB, this constrains tile size choices.", "sha256": "141f8982deb09e3a091950e50265643f2f7231205efbfe8306dfff8aa19d69db"}, {"id": "u027", "kind": "list-item", "locator": "body:L192-L192", "preview": "- Barrier initialization overhead is negligible but must happen before the first TMA. Place init in a `__syncthreads()` block at kernel start.", "sha256": "d9cf29d158136f7851959b3609b7e77d38f8730a25bc2dfb01bcf1b226925163"}, {"id": "u028", "kind": "list-item", "locator": "body:L193-L193", "preview": "- Incorrect phase tracking in mbarrier causes deadlocks. The phase alternates with each arrive/wait cycle; off-by-one errors are common during development.", "sha256": "dd4a84772ccc0b59c5522f88fd48f5fa96d0b138c4620c15632029c4dc557e0e"}, {"id": "u029", "kind": "list-item", "locator": "body:L194-L194", "preview": "- For very short K dimensions (fewer iterations than stages), the prologue/epilogue overhead may dominate. Guard the loop bounds accordingly.", "sha256": "493e18b170790a00c2b49d45839d2ae22e745c4937771d716173175169fcb1a5"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Pipeline Progression", "Multi-Stage Circular Buffer Pattern", "Stage Count Selection", "mbarrier-Based Synchronization", "Modular's 5-Stage Implementation", "When to Use", "Caveats"], "id": "technique-pipeline-stages", "path": "wiki/techniques/pipeline-stages.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/blogs/modular-blackwell-matmul.md", "url": "https://www.modular.com/blog/matrix-multiplication-on-nvidias-blackwell-part-1-introduction"}, {"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-tcgen05-tutorial", "blog-modular-blackwell", "doc-nvidia-tuning-guide"], "title": "Software Pipelining and Multi-Stage Buffering", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "e4490d9c41b459fa167d4dbc32f8814b4ebe1e7579297a6074df53a54b51509d", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "SM occupancy is inversely proportional to registers-per-thread. For memory-bound kernels, higher occupancy = more warps to hide memory latency. `-maxrregcount` and `__launch_bounds__` force the compiler to stay within a budget.", "sha256": "67ae743e543e32b842dcbbdc7ae5a7aece11274737bf3cdf6678eed799d1e7d4"}, {"id": "u002", "kind": "code", "locator": "body:L9-L18", "preview": "```cuda // Aggressive: 32 registers/thread \u2192 ~4 blocks per SM at 256 threads/block __launch_bounds__(256, 4) __global__ void gemv_memory_bound(...) { // Compiler will spill to local memory if needed } // Or via nvcc flag: // nvcc -maxrregco", "sha256": "db5b11ddbac6e4a85435c76881aea10ba8e2b316f86bee7435bfbbdd65dcd53a"}, {"id": "u003", "kind": "prose", "locator": "body:L22-L22", "preview": "Lower register count \u2192 compiler may:", "sha256": "11ad74cd2da10c7b3d5a764102b705c7b560777b634534349c6295d5f85795b0"}, {"id": "u004", "kind": "list-item", "locator": "body:L23-L23", "preview": "- Spill frequently-used values to local memory (bad)", "sha256": "7a9f2905849808b299160d6c1a5bce7688bb6971f536bc4a0d4042640226f177"}, {"id": "u005", "kind": "list-item", "locator": "body:L24-L24", "preview": "- Recompute values instead of storing them (neutral)", "sha256": "8ff03c2a5624d0d0abf4f53c1d910a49b3e0830d5a55e2e171aaf50a269b82d3"}, {"id": "u006", "kind": "list-item", "locator": "body:L25-L25", "preview": "- Use fewer unrolled iterations (bad for compute-bound)", "sha256": "6fc7eafbc44729c7b59f3f60c3b472271570c1d23d033321f5581d4b8fb2df53"}, {"id": "u007", "kind": "prose", "locator": "body:L27-L27", "preview": "For memory-bound kernels, spills can be hidden by memory latency anyway, so aggressive budgeting often wins.", "sha256": "0839dfa5de845b156842d8f387f3fd2c8e1555c8bbc6bb8a6352a4644e1b5cc3"}, {"id": "u008", "kind": "table-row", "locator": "body:L33-L33", "preview": "| Rank | Register count | Latency | | 1 | 32 | 18.5\u03bcs |", "sha256": "62bacbbb02ae292e554ef4aa539388bd98437885f5dcebac212e670d419552b2"}, {"id": "u009", "kind": "table-row", "locator": "body:L34-L34", "preview": "| Rank | Register count | Latency | | 3 | 45 | ~20\u03bcs |", "sha256": "f42a56f99577616f7ee66c57adeed5333f5fbff5726a02265ba8de5a29a1187f"}, {"id": "u010", "kind": "prose", "locator": "body:L36-L36", "preview": "The measurable difference between 32 and 45 registers shows occupancy dominates for memory-bound NVFP4 GEMV.", "sha256": "01442dfeda5750db249c3f7cc5ce69ff7e3a35f16ade2f62fbe5e60fab555b89"}, {"id": "u011", "kind": "list-item", "locator": "body:L40-L40", "preview": "- Memory-bound kernels (first priority: occupancy)", "sha256": "547d06454ed352af323da04581bce57b5aa173c445cc256e3d94abecf6f19129"}, {"id": "u012", "kind": "list-item", "locator": "body:L41-L41", "preview": "- Kernels where register pressure comes from inner loop, not accumulators (TMEM handles accumulators)", "sha256": "4d30d905715e06de17252aa7ab5727305e7acaa536959942d63f77568765013f"}, {"id": "u013", "kind": "list-item", "locator": "body:L42-L42", "preview": "- Sub-byte types with heavy decode/scale computation", "sha256": "e475f3a0ae449c4c6e857ce007229d9737c9cf61cb90021bfb55adec8b49ad44"}, {"id": "u014", "kind": "list-item", "locator": "body:L46-L46", "preview": "- Compute-bound GEMM (let compiler use what it needs)", "sha256": "e79015ad25c9bfd96722f7d12a1c02a51313378d798b52a6c8ea6ecaf3a350f8"}, {"id": "u015", "kind": "list-item", "locator": "body:L47-L47", "preview": "- Kernels where spills to local memory would serialize", "sha256": "1c0b02e78a49584102478c62648ca569e842b58c7cd88304cf9a18af707d6482"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Pattern", "Compiler Tradeoffs", "GPU Mode NVFP4 GEMV Results", "When To Use", "When NOT To Use"], "id": "technique-register-budgeting", "path": "wiki/techniques/register-budgeting.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/yue-nvfp4-hackathon.md", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}, {"path": "sources/blogs/amandeep-nvfp4-attempts.md", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}, {"path": "sources/blogs/simon-nvfp4-gemv.md", "url": "https://veitner.bearblog.dev/nvfp4-gemv/"}], "risk_flags": ["code", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-yue-nvfp4", "blog-amandeep-nvfp4", "blog-simon-nvfp4-gemv"], "title": "Register Budgeting for Occupancy", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "fe120348335fa0ecedde2247d6fed5fe7262c22730cfed75ee48ee931de7d730", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "FlashAttention-4 replaces the hardware Special Function Unit (SFU) exponential (`ex2.approx`) with a software-emulated 2^x function that distributes computation across the SM's FMA (fused multiply-add) units. On Blackwell, tensor core throu", "sha256": "6efea6d4e42f7e8cdd53b28ea0dc4fcccbec71d6ac2266f77b0764bde6159877"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "The softmax in attention requires computing `exp(x - max)` for every element of the score matrix. On previous generations, the SFU's `ex2.approx` instruction was fast enough relative to the MMA throughput. On Blackwell:", "sha256": "8595fd7a913faf59f3ac8af4f334750ffea2592fbe69f333d6723015a2ecfddf"}, {"id": "u003", "kind": "table-row", "locator": "body:L11-L11", "preview": "| Resource | Hopper (SM90) | Blackwell (SM100) | Ratio | | Tensor core TFLOPS (BF16) | ~990 | ~2250 | 2.27x |", "sha256": "fcf8a0e5146f79359df08b987ab09e5b6ad89abcf74badfe400b7bdb20c193d9"}, {"id": "u004", "kind": "table-row", "locator": "body:L12-L12", "preview": "| Resource | Hopper (SM90) | Blackwell (SM100) | Ratio | | SFU units per SM | 16 | 16 | 1.0x |", "sha256": "077723f2ba26994cc6922c426cf16c490bcbeb9c75d7f1281c2f9e734d1cc09f"}, {"id": "u005", "kind": "table-row", "locator": "body:L13-L13", "preview": "| Resource | Hopper (SM90) | Blackwell (SM100) | Ratio | | SFU throughput (exp per cycle) | 16 | 16 | 1.0x |", "sha256": "f1f43e8318840e28532fb7e8b324ec7727928920b46e5f0994d147098f5bc96d"}, {"id": "u006", "kind": "table-row", "locator": "body:L14-L14", "preview": "| Resource | Hopper (SM90) | Blackwell (SM100) | Ratio | | FMA units per SM | 128 | 128 | 1.0x |", "sha256": "2513e0b632770b0e69b84537704b1c0ad5d1148b0f57453618aab8b9aba7d0e6"}, {"id": "u007", "kind": "prose", "locator": "body:L16-L16", "preview": "The tensor cores produce 2x more score elements per cycle, but the SFU can only process exp() at the same rate as before. This makes the SFU the bottleneck for any kernel that needs exp() proportional to the number of MMA outputs.", "sha256": "6c0ff0b8e3ebbe638d1a81413e2a5b738900d5187b8649f1fb383cfcc8153957"}, {"id": "u008", "kind": "prose", "locator": "body:L18-L18", "preview": "FlashAttention-4's approach: distribute the exp() workload across FMA units (128 per SM) instead of SFU units (16 per SM), achieving 8x the throughput for the exponential computation.", "sha256": "821c1dc38e821d7c576e6d3540000fea3a2b5565269e09e6c7a2985842b2e6e1"}, {"id": "u009", "kind": "prose", "locator": "body:L22-L22", "preview": "Range reduction transforms the input `x` into a small residual that a polynomial can accurately approximate. The Cody-Waite method splits the input into an integer part (for exact power-of-two scaling) and a fractional part (for polynomial ", "sha256": "633087d1dc66c31d1adf9037ae75107783ee2945a41e6305e424381906abd8fc"}, {"id": "u010", "kind": "code", "locator": "body:L24-L72", "preview": "```cuda // Cody-Waite range reduction for 2^x // Goal: decompose x = n + r where n is integer, r in [-0.5, 0.5] // Then 2^x = 2^n * 2^r, and 2^r is approximated by polynomial // // The Cody-Waite trick: subtract n using two constants (C1 + ", "sha256": "bd8a3e83f91a2a5c61614813d88f04fd4ba5fc3a1db24bf0f17c4796f2702cc8"}, {"id": "u011", "kind": "prose", "locator": "body:L76-L76", "preview": "The key insight is that Horner polynomial evaluation is a chain of FMA operations. With the softmax warp executing on CUDA cores while the MMA warp uses tensor cores, the FMA throughput is fully available:", "sha256": "d7f7e14b21782d574264840122b9f970dca4b6ef92658c900054c304a22b6549"}, {"id": "u012", "kind": "code", "locator": "body:L78-L138", "preview": "```cuda // FlashAttention-4 softmax with software exp2 // Executed by dedicated softmax warpgroups (part of warp specialization) // // For each row of the score matrix S[i,:]: // 1. Find row max: m_new = max(S[i,:]) // 2. Compute exp2((S[i,", "sha256": "2ed7a87e1fe2f843cee953b304ed2188f598b02ba9e94d641d8a0aa4386d9b6a"}, {"id": "u013", "kind": "prose", "locator": "body:L142-L142", "preview": "At the PTX level, the Horner polynomial compiles to a tight chain of `fma.rn.f32` instructions:", "sha256": "9bc0f5d7707c8a74c367f53b64416533669196b77d73068c31e872864f41e2df"}, {"id": "u014", "kind": "code", "locator": "body:L144-L180", "preview": "```ptx // Software exp2 via Horner polynomial in PTX // Input: %x (float, range-reduced to [-0.5, 0.5]) // Output: %result (float, approximation of 2^x) .reg .f32 %x, %r, %n, %poly, %result; .reg .f32 %c0, %c1, %c2, %c3, %c4; // Load polyno", "sha256": "79c7f9dc0cee373816a8a7709811ff39f1afc7010f2ee69280e2a528d8c0b784"}, {"id": "u015", "kind": "prose", "locator": "body:L184-L184", "preview": "The degree-4 polynomial provides approximately 22 bits of mantissa accuracy, which is more than sufficient for attention softmax where:", "sha256": "62433718aa2ed86f2672dfb451ed94f8c87ed0cd15ad53422368b91b254b3023"}, {"id": "u016", "kind": "list-item", "locator": "body:L185-L185", "preview": "- The input `x = (S[i,j] - max) * log2(e)` is always non-positive", "sha256": "3de1a0b8c93fcf05cd3c78d9afd2c546a9052098de4861c57d93c69c9b0c0413"}, {"id": "u017", "kind": "list-item", "locator": "body:L186-L186", "preview": "- The softmax output is normalized, so small absolute errors cancel out", "sha256": "af0ab0506229e520469fff8587d92d782795db9122770def9d8bd3919af0e289"}, {"id": "u018", "kind": "list-item", "locator": "body:L187-L187", "preview": "- BF16 output has only 7 mantissa bits anyway", "sha256": "5ff084b9eee350189aa486b28390455179407f420ac4365a3ea5ea7a216384c4"}, {"id": "u019", "kind": "prose", "locator": "body:L189-L189", "preview": "For applications requiring higher accuracy, a degree-6 polynomial (6 FMAs) achieves near-ULP accuracy across the full float range.", "sha256": "b7af80bcb683a569e2b6eef6b48a7d9ffd780208484235187f1a109e9ba08aec"}, {"id": "u020", "kind": "list-item", "locator": "body:L193-L193", "preview": "- **Attention kernels on Blackwell**: Whenever the SFU is the bottleneck for softmax computation. FlashAttention-4 measured 1.1-1.3x speedup over cuDNN from this technique alone on B200.", "sha256": "f181239242e97a4927887d4bbca60792ef68ef84bf5024fa625537e2be2cf192"}, {"id": "u021", "kind": "list-item", "locator": "body:L194-L194", "preview": "- **Any kernel limited by transcendental function throughput**: If profiling shows SFU utilization near 100% while FMA utilization is low, software emulation can rebalance the workload.", "sha256": "f20a967006ba5bbd86ac800ab093a2e5635bd0c3ac50ef4b244a14147e2c3f92"}, {"id": "u022", "kind": "list-item", "locator": "body:L195-L195", "preview": "- **Not recommended on Hopper**: The SFU-to-MMA throughput ratio is better balanced on SM90. The overhead of 4 FMAs vs 1 SFU instruction is not justified unless the SFU is proven to be the bottleneck.", "sha256": "8b730ecee723f4ce482d528fb2f265e0d3e1cc54055efc75b378c468651d026f"}, {"id": "u023", "kind": "list-item", "locator": "body:L199-L199", "preview": "- The 4-FMA chain has a latency of ~16 cycles (4 dependent FMAs at ~4 cycles each), vs ~20 cycles for SFU `ex2.approx`. Latency is comparable; the win comes from throughput (128 FMA units vs 16 SFU units).", "sha256": "1846dbd0de4c200552ae62a81c178b4ddc90e50a7f7b4e44b0b9383311d5fa56"}, {"id": "u024", "kind": "list-item", "locator": "body:L200-L200", "preview": "- Polynomial coefficients are for 2^x on [-0.5, 0.5]. For e^x, multiply the input by log2(e) first.", "sha256": "c0d641147cf3962f8e8583469c3eda934ebde43888727de8d55ca4181e35f10d"}, {"id": "u025", "kind": "list-item", "locator": "body:L201-L201", "preview": "- The `ldexpf` or exponent bit-manipulation step for 2^n must handle overflow/underflow (very large/small x). In attention, `x <= 0` always holds, so only underflow toward zero is possible.", "sha256": "2df5e207ad969e1d3e58d0985bf577b7d6726780320679f1ee18c654891bbc57"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Why SFU Is a Bottleneck on Blackwell", "Cody-Waite Range Reduction", "Distributing Across FMA Units", "PTX-Level FMA Chain", "Accuracy Considerations", "When to Use", "Caveats"], "id": "technique-software-exp", "path": "wiki/techniques/software-exp.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/flash-attention-4.md", "url": "https://tridao.me/blog/2026/flash4/"}, {"path": "sources/docs/flash-attention-4.md", "url": "https://arxiv.org/abs/2603.05451"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/parallel-thread-execution/"}], "risk_flags": ["code", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-flash-attention-4", "doc-flash-attention-4", "doc-ptx-isa-sm100"], "title": "Software-Emulated Exponential", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "c716117a080b9689b2eecbf26c4f0bf1801704ef00fd83627b3f229789716aa5", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "Shared memory swizzling remaps the linear address layout of a matrix tile in SMEM so that threads accessing consecutive columns (or rows) hit different 32-byte banks rather than the same bank. This eliminates bank conflicts that would other", "sha256": "ce99151372d234814f71d9a9711aafefb33296a470ed6ae9e0b10716f22f687b"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "Shared memory has 32 banks, each 4 bytes wide (128 bytes total per bank cycle). When a warp accesses a matrix stored in row-major layout, threads in the same warp reading elements from consecutive rows in the same column hit the same bank, ", "sha256": "f0e2e1786678febcb5d3cb2b4810b1d08cc393f1df102523b144e494ea536697"}, {"id": "u003", "kind": "prose", "locator": "body:L9-L9", "preview": "The TMA unit on both Hopper and Blackwell encodes the swizzle pattern as part of the tensor descriptor. The tcgen05.mma instruction expects its SMEM operands to already be swizzled in the 128-byte pattern. Using unswizzled data produces inc", "sha256": "0cfa048d16c8b9f5d1310a1bd02a15652d7c0ecbee7a6dc0345068f60b691a60"}, {"id": "u004", "kind": "prose", "locator": "body:L11-L11", "preview": "The tcgen05-tutorial benchmark progression shows the impact:", "sha256": "cb2babf9d8d4a1ce28deba4b75660e511755cc47a370c94a16cc2a83b2f8d791"}, {"id": "u005", "kind": "code", "locator": "body:L13-L17", "preview": "``` Naive (no swizzle): 255 TFLOPS (17% of cuBLAS) 128B swizzle applied: 695 TFLOPS (46% of cuBLAS) ---- 2.7x improvement from swizzling alone ---- ```", "sha256": "5aa6af3d28f5bb234a6579062ff89314b2c4da13e594601bbf37ae2b09b29110"}, {"id": "u006", "kind": "prose", "locator": "body:L21-L21", "preview": "The swizzle function XORs a portion of the column address with the row address to scatter accesses across banks:", "sha256": "b0c1f36b98b1f0f05a9cf02b53eadcfb80e11dfe7d8b5ef9e0050d1e62c206f1"}, {"id": "u007", "kind": "code", "locator": "body:L23-L41", "preview": "```cuda // 128-byte swizzle: XOR bits [4:6] of the byte offset with the row index // This ensures that consecutive rows accessing the same logical column // map to different physical SMEM banks. // // For a tile stored in SMEM with TILE_N c", "sha256": "b10f33bcd229a8f43b11a7e498a38c2a7b0f0e4536e63c115006d41e672dbdb7"}, {"id": "u008", "kind": "prose", "locator": "body:L43-L43", "preview": "Visually, for an 8-row x 64-column half-precision tile (128 bytes per row):", "sha256": "06b2de8a095c86b5cae33559f34774ca6160eec20c1b2651c1a7cfe429d9e513"}, {"id": "u009", "kind": "code", "locator": "body:L45-L57", "preview": "``` Without swizzle (row-major): Row 0: bank 0,1,2,...,31 bank 0,1,2,...,31 Row 1: bank 0,1,2,...,31 bank 0,1,2,...,31 Row 2: bank 0,1,2,...,31 bank 0,1,2,...,31 -> Column access = 8-way bank conflict With 128B swizzle (XOR pattern): Row 0:", "sha256": "e68e85063e54eff1f4031a349d08615e7fe95f030e84e90f3e2447eb0bac50d8"}, {"id": "u010", "kind": "prose", "locator": "body:L61-L61", "preview": "The TMA descriptor encodes the swizzle mode when creating a tensor map. The swizzle mode must match what the consumer (tcgen05.mma or wgmma) expects:", "sha256": "08279d3f63c75a2a1be3d839a3721cba648ece95a450cde320488852c9f97bd4"}, {"id": "u011", "kind": "code", "locator": "body:L63-L86", "preview": "```cuda // Creating a TMA descriptor with 128-byte swizzle #include CUtensorMap tensor_map; // Swizzle mode: CU_TENSOR_MAP_SWIZZLE_128B // This tells TMA to apply the 128-byte XOR swizzle pattern // when writing data into shared me", "sha256": "d54572e96082b2eea88d4fd5d043ae610bc781dd1686413d197443494f72d38a"}, {"id": "u012", "kind": "prose", "locator": "body:L88-L88", "preview": "The available swizzle modes and their use cases:", "sha256": "dcec19581b3d9089e67314c4c76049ebd0eac7e2512e2f29bcc181ebc22df456"}, {"id": "u013", "kind": "table-row", "locator": "body:L92-L92", "preview": "| Swizzle Mode | Bank Spread | Use Case | | `SWIZZLE_NONE` | No remapping | Non-MMA data (flags, scales) |", "sha256": "27c9fce11425f0d49b426f0884c0aaf174981b87a3e563c5638952f775563be1"}, {"id": "u014", "kind": "table-row", "locator": "body:L93-L93", "preview": "| Swizzle Mode | Bank Spread | Use Case | | `SWIZZLE_32B` | 32-byte groups | Narrow tiles, small data types |", "sha256": "0db4fbbe37adcb74a561c1ecccccbed7a3c70e63f2eb9899365cee584c1e0e4f"}, {"id": "u015", "kind": "table-row", "locator": "body:L94-L94", "preview": "| Swizzle Mode | Bank Spread | Use Case | | `SWIZZLE_64B` | 64-byte groups | Medium tiles |", "sha256": "aef89dcf631253e2b0931f818788f6612ae9bdf85b81cd397831699928875fd8"}, {"id": "u016", "kind": "table-row", "locator": "body:L95-L95", "preview": "| Swizzle Mode | Bank Spread | Use Case | | `SWIZZLE_128B` | 128-byte groups | Standard for BF16/FP16 MMA operands |", "sha256": "d80118b59d93c6d3c1403aa834ef25ca4676ac416e89baf2f2d183569d8b12d8"}, {"id": "u017", "kind": "prose", "locator": "body:L99-L99", "preview": "In CuTe/CUTLASS, swizzle is expressed as a layout composition:", "sha256": "5228fc07b661976963d244128708dcc8b78e60965647385ddf8f9454c49a6a6d"}, {"id": "u018", "kind": "code", "locator": "body:L101-L125", "preview": "```cuda // CuTe swizzle layout for 128-byte swizzle pattern // Swizzle where: // B = number of bits in the base (non-swizzled) portion // M = number of bits in the mask // S = shift amount // // Swizzle<3, 4, 3> encodes the 128B sw", "sha256": "350890c75c6bcc0cffbffc21e9db0e0ba138f89f6632df719bdd73743a597551"}, {"id": "u019", "kind": "prose", "locator": "body:L129-L129", "preview": "Use `nvprof` or Nsight Compute to verify that swizzling eliminates conflicts:", "sha256": "17f65294789a5a4187c26c3cd1c129b6b8867e099f77babfddadacd5c64b2ba6"}, {"id": "u020", "kind": "code", "locator": "body:L131-L140", "preview": "```python # Nsight Compute command to check shared memory bank conflicts # Look for \"Shared Memory Bank Conflicts\" metric # ncu --metrics l1tex__data_bank_conflicts_pipe_lsu_mem_shared_op_ld.sum \\ # ./my_kernel # Expected results: # Without", "sha256": "32a8e919c354d73f26102c99ccbbe88aa291c78cd59ec3a78ebbe66d7a0b32b6"}, {"id": "u021", "kind": "list-item", "locator": "body:L144-L144", "preview": "- **All Blackwell tensor core kernels**: 128-byte swizzling is not optional. Both TMA and tcgen05.mma require it for correct results and peak performance.", "sha256": "dcd7351c07484bb5d4e9f8673f70c8b3d7c1c4d3f10f92c4ee1b1a8548f6988f"}, {"id": "u022", "kind": "list-item", "locator": "body:L145-L145", "preview": "- **Hopper wgmma kernels**: Same requirement applies; wgmma expects swizzled SMEM operands.", "sha256": "fe233e5729b7851ca56fc490d6d42a1b2b0563e6dff9cbcc42422347c1ad9575"}, {"id": "u023", "kind": "list-item", "locator": "body:L146-L146", "preview": "- **Non-MMA shared memory access**: If multiple warps access the same SMEM tile in a column pattern (e.g., reduction), swizzling prevents serialization.", "sha256": "be0c3938f3fbc4911b1d36410b1b0fd4d844a9a2d4a99aeabcbbf8c341737950"}, {"id": "u024", "kind": "list-item", "locator": "body:L150-L150", "preview": "- The swizzle mode in the TMA descriptor must exactly match the access pattern of the consumer. A mismatch produces silently incorrect results, not a runtime error.", "sha256": "1403a7e149a0c1d6d4110031388e19fafef45c133a4dc7ad76a882cead6b8503"}, {"id": "u025", "kind": "list-item", "locator": "body:L151-L151", "preview": "- Swizzled layouts make SMEM address computation non-trivial. Using CuTe's layout algebra avoids manual indexing errors.", "sha256": "8360684428419eafab8d025fe8b763292ea0b3ff29e403397051ee56cf2e2860"}, {"id": "u026", "kind": "list-item", "locator": "body:L152-L152", "preview": "- For data types wider than 2 bytes (e.g., FP32 accumulators), the optimal swizzle mode may differ. TMEM accumulators avoid this issue since they use a separate address space.", "sha256": "f791bc3efe12801de06fd14c1aeddd480b8b5d43292e75cbb2238558f2587bb3"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Why 128-Byte Swizzling is Mandatory on Blackwell", "How 128-Byte Swizzling Works", "TMA Swizzle Encoding", "CuTe Swizzle Layout", "Verification: Detecting Bank Conflicts", "When to Use", "Caveats"], "id": "technique-swizzling", "path": "wiki/techniques/swizzling.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/blogs/modular-blackwell-matmul.md", "url": "https://www.modular.com/blog/matrix-multiplication-on-nvidias-blackwell-part-1-introduction"}], "risk_flags": ["code", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "blog-modular-blackwell"], "title": "Shared Memory Swizzling", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "b8004e09034567bbd556d5cae14856d176cfa350af0214bb99c9aade49312f32", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "Tile scheduling determines the order in which output tiles of a GEMM (or attention) kernel are assigned to CTAs. The scheduling order affects L2 cache hit rates, tail-effect severity, and overall GPU utilization. On Blackwell, the CLC hardw", "sha256": "28c4c95e79e9006fab01b0a85d22ad81bde94be4359b4f99962b217c2a5a61b8"}, {"id": "u002", "kind": "prose", "locator": "body:L9-L9", "preview": "Tiles are assigned in row-major order. Simple but poor L2 locality: consecutive tiles share no B-matrix data until the entire M-dimension is traversed.", "sha256": "71e74696731e08922084e763bc9f5c1dbf80002e8619bb58afd3b05fa38ff9e3"}, {"id": "u003", "kind": "code", "locator": "body:L11-L27", "preview": "```cuda // Linear raster: tile_idx maps directly to (tile_m, tile_n) __device__ void linear_raster(int tile_idx, int tiles_n, int& tile_m, int& tile_n) { tile_m = tile_idx / tiles_n; tile_n = tile_idx % tiles_n; } // Access pattern for a 4x", "sha256": "9720f2edd4aa71b1333634cb1bd2929471836dca97a1527d34f26eb0d04fc18b"}, {"id": "u004", "kind": "prose", "locator": "body:L31-L31", "preview": "Tiles are assigned in a blocked pattern that groups nearby M and N tiles together, maximizing reuse of both A rows and B columns in L2 cache:", "sha256": "6a8479c6187de272d3444ed5b8b189380f3e9f3a3e18c86b1502a0648415bffb"}, {"id": "u005", "kind": "code", "locator": "body:L33-L61", "preview": "```cuda // Swizzled raster: group tiles into blocks that share A and B data // swizzle_size controls the block width (typically 4-8) __device__ void swizzled_raster(int tile_idx, int tiles_m, int tiles_n, int swizzle_size, int& tile_m, int&", "sha256": "1d71edfca8fbc446a33f887b3c61d17340ea86ccecd34e7538918d6b87c9dd67"}, {"id": "u006", "kind": "prose", "locator": "body:L65-L65", "preview": "Each CTA processes tiles at fixed intervals equal to the grid size:", "sha256": "626c37c57779514424f31ec600f0e59259f30613263ccf73cd308e60ff3bb9d7"}, {"id": "u007", "kind": "code", "locator": "body:L67-L76", "preview": "```cuda // Static stride: CTA i processes tiles i, i+gridDim.x, i+2*gridDim.x, ... __device__ void static_stride(int cta_id, int total_ctas, int iteration, int tiles_n, int& tile_m, int& tile_n) { int tile_idx = cta_id + iteration * total_c", "sha256": "0449df45a153b12f4cb418bcbc3e6bc21c44838006c0c634edfc6fae1aa4c1b5"}, {"id": "u008", "kind": "prose", "locator": "body:L80-L80", "preview": "The CLC hardware scheduler assigns tiles at runtime, combining the benefits of dynamic load balancing with configurable scheduling policies:", "sha256": "e0eb0c8c1ce6dcd126479616e2c269bc33221a6f5270ffe963a3e97c5040eed6"}, {"id": "u009", "kind": "code", "locator": "body:L82-L105", "preview": "```cuda // CLC-based scheduling on Blackwell // The scheduling policy is set once during CLC initialization enum class ClcSchedulePolicy { LinearRaster, // Simple row-major order SwizzledRaster, // Blocked pattern for L2 locality ColumnFirs", "sha256": "0a69add195542b6d20d2efcd280213ffb995ec493550c142f9dc6b6e9b6e7052"}, {"id": "u010", "kind": "prose", "locator": "body:L109-L109", "preview": "CUTLASS provides several tile schedulers that abstract these strategies:", "sha256": "53e1fe4703c3761ea5fd5c60b8a678cef5b9e2e254ea66c7429759cf3e434ac3"}, {"id": "u011", "kind": "code", "locator": "body:L111-L137", "preview": "```cuda // CUTLASS tile scheduler selection for SM100 // All persistent schedulers inherit from PersistentTileSchedulerSm100 // 1. Default CLC scheduler with swizzled raster using Scheduler_Default = cutlass::gemm::PersistentTileSchedulerSm", "sha256": "133158bd5ec890108cdb7014c1fc7f9ec966a8455dd307c8fbe4e41e00b3f6b0"}, {"id": "u012", "kind": "prose", "locator": "body:L141-L141", "preview": "The choice of scheduling strategy directly impacts L2 cache hit rates. On B200 with 126 MB L2:", "sha256": "949d895f78e70ec2107dbe2df4e6d391866d729b9959dd0672b8d3e475d96c7a"}, {"id": "u013", "kind": "code", "locator": "body:L143-L163", "preview": "```python # L2 cache reuse analysis for different schedulers # Problem: M=8192, N=8192, K=4096, BF16 # Tile: 128x256, giving 64x32 = 2048 tiles # B200: 142 SMs, 126 MB L2 tile_bytes_A = 128 * 4096 * 2 # 1 MB per tile row of A tile_bytes_B =", "sha256": "e2a8efabe8686e235ab683a4980248113bdb5b72e573ccc24f0e20981e247222"}, {"id": "u014", "kind": "prose", "locator": "body:L167-L167", "preview": "The \"tail effect\" occurs when the last wave of tiles does not fully occupy all SMs. Different schedulers handle this differently:", "sha256": "357e5feab90e19e3e79c54e9ea33d2c81c2374ecb6aca75c5ed30aa65df6dffc"}, {"id": "u015", "kind": "table-row", "locator": "body:L171-L171", "preview": "| Scheduler | Tail Handling | SM Utilization (Last Wave) | | Linear raster | None | `(total_tiles % num_SMs) / num_SMs` |", "sha256": "f08837ccea1dae5a19410ad3efcc85417d04c335a474e95a7cdc400ff76978b0"}, {"id": "u016", "kind": "table-row", "locator": "body:L172-L172", "preview": "| Scheduler | Tail Handling | SM Utilization (Last Wave) | | Static stride | None | Same as linear |", "sha256": "aeb54c647106afd98be6f50ab9ba26b650386b1614b8eaa85491180e9394ec86"}, {"id": "u017", "kind": "table-row", "locator": "body:L173-L173", "preview": "| Scheduler | Tail Handling | SM Utilization (Last Wave) | | CLC dynamic | Automatic | Fast CTAs steal from slow ones |", "sha256": "7baa281524a3bbdf21836ba8998a42b5b08146c2d8b1c2309b5ee6aecbcc3048"}, {"id": "u018", "kind": "table-row", "locator": "body:L174-L174", "preview": "| Scheduler | Tail Handling | SM Utilization (Last Wave) | | Stream-K | K-splitting | Near 100% (splits partial tiles across SMs) |", "sha256": "a42815aa76589b28218189b993e93245b38d6719e398582bf8b43bd50a95f27e"}, {"id": "u019", "kind": "prose", "locator": "body:L176-L176", "preview": "For a problem with 150 tiles on 142 SMs:", "sha256": "69a82d0a9da8abc0f2f4a0a7d56ed2bcd46a7893e3580b571fb3a6a8b7715db0"}, {"id": "u020", "kind": "list-item", "locator": "body:L177-L177", "preview": "- Static: last wave has 8 tiles on 8 SMs, 134 SMs idle (5.6% utilization)", "sha256": "737fd5d377aa2daf45da695e8178504adef84f2fc9f5633591dddd3934859957"}, {"id": "u021", "kind": "list-item", "locator": "body:L178-L178", "preview": "- CLC: fast-finishing CTAs from wave 1 absorb the 8 extra tiles", "sha256": "a8a66c01e93cb1bf7d589b2a84386086248864b9539e4eb7609a04a78304abad"}, {"id": "u022", "kind": "list-item", "locator": "body:L179-L179", "preview": "- Stream-K: the 8 remaining tiles are split across all 142 SMs", "sha256": "0b0c60176f514b10115881bc5c863149897bd9e8fa6545bb896e67dd37052a04"}, {"id": "u023", "kind": "list-item", "locator": "body:L183-L183", "preview": "- **Swizzled raster**: Default choice for large GEMMs. Always better than linear for L2 locality.", "sha256": "7f6d611ccd092bd4c1688ccd3016edbd88e2be7fff8ad873b62aea7710c5334e"}, {"id": "u024", "kind": "list-item", "locator": "body:L184-L184", "preview": "- **CLC dynamic**: Recommended on Blackwell for all persistent kernels. Combines dynamic load balancing with swizzled ordering.", "sha256": "e18aae4896c8004692a201fd12130e424d1d729e2e8c873538b98bd78e2c3dbf"}, {"id": "u025", "kind": "list-item", "locator": "body:L185-L185", "preview": "- **Stream-K**: Best for small-to-medium problems where the tail effect dominates. Adds complexity for K-dimension synchronization.", "sha256": "a10fbadf54d93ec53e2b07a7bd5c13f723d583252aa8ffba18cf8edf4b51ac29"}, {"id": "u026", "kind": "list-item", "locator": "body:L186-L186", "preview": "- **Grouped scheduler**: Essential for MoE and batched GEMM where problem sizes vary across groups.", "sha256": "edc0d6575b597a5a540f33437674d089efe77e537de7136d8d5a0e0f9242fb20"}, {"id": "u027", "kind": "list-item", "locator": "body:L190-L190", "preview": "- Swizzle size must be tuned per problem shape. Too large a swizzle group exceeds L2 capacity; too small loses the locality benefit.", "sha256": "23387e8e50870f4e3070e24961464a5ad75bd34ebab3e83f62e53375ffb81f8f"}, {"id": "u028", "kind": "list-item", "locator": "body:L191-L191", "preview": "- CLC scheduling adds a small latency per tile acquisition (~10s of cycles). For extremely small tiles, this overhead is proportionally larger.", "sha256": "8f44f7a0ed20da7c71487dd826806bed275053b56d2c9fdc81c661459d194c59"}, {"id": "u029", "kind": "list-item", "locator": "body:L192-L192", "preview": "- Stream-K requires atomic accumulation where K-splits meet, adding synchronization overhead. Only worthwhile when tail utilization is a proven bottleneck.", "sha256": "9c6b133dcad467c1c9d91088d2fc4977902c5fec4bbd04f129a159f7145eb57f"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Scheduling Strategies", "Linear Raster (Naive)", "Swizzled Raster", "Static Stride (Hopper Persistent)", "CLC Dynamic Scheduling (Blackwell)", "CUTLASS Tile Schedulers", "L2 Cache Locality Analysis", "Tail Effect Mitigation", "When to Use", "Caveats"], "id": "technique-tile-scheduling", "path": "wiki/techniques/tile-scheduling.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/docs/nvidia-cutlass-blackwell.md", "url": "https://docs.nvidia.com/cutlass/latest/CHANGELOG.html"}, {"path": "sources/prs/cutlass/PR-2161.md", "revision": "06e560d9", "url": "https://github.com/NVIDIA/cutlass/pull/2161"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "doc-cutlass-blackwell", "pr-cutlass-2161"], "title": "Tile Scheduling Strategies", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "a1867ce5eadf388cf89f64dbc67969e35641a2aee5abf95bc15a6310620d3778", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "For memory-bound kernels (low arithmetic intensity), maximizing global memory throughput is critical. Three complementary techniques from the GPU Mode NVFP4 Hackathon achieve this: (1) wide vectorized loads (128-bit and 256-bit) to saturate", "sha256": "4373b20db1333710b0bd2335d83f1cd217e133fb05fe6f947ea6078b93b21db0"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "Standard 32-bit loads waste memory bus bandwidth. Wider loads amortize the instruction overhead and saturate the 8 TB/s HBM bandwidth of B200:", "sha256": "df58a252f5ff63de56a417d666f95144516e2883b0eccd7d3c86a4373b9d77d8"}, {"id": "u003", "kind": "code", "locator": "body:L9-L34", "preview": "```cuda // Vectorized load widths comparison for FP4 GEMV // Each thread loads more data per instruction // 32-bit load: 4 bytes per thread per instruction float val; asm volatile(\"ld.global.b32 %0, [%1];\" : \"=f\"(val) : \"l\"(ptr)); // 64-bit", "sha256": "b8da4a10406241c34acee820f4f57eabcfc788a4324822ac1355bacd74aa7b2a"}, {"id": "u004", "kind": "prose", "locator": "body:L36-L36", "preview": "For the NVFP4 GEMV, 128-bit and 256-bit loads are essential because FP4 elements are only 0.5 bytes each. A 256-bit load fetches 64 FP4 values in a single instruction:", "sha256": "82e9fd56bce1ec6ee9e5b07738affb2b6d581a1190db38b8b66e2aad4f7542c6"}, {"id": "u005", "kind": "code", "locator": "body:L38-L84", "preview": "```cuda // NVFP4 GEMV: each thread loads 64 FP4 values via 256-bit load // Then unpacks using PTX byte manipulation instead of bitwise ops __device__ void load_and_unpack_nvfp4_256bit( const uint8_t* fp4_data, // Packed FP4 data (2 values p", "sha256": "a95e2476191e4074792945f538a15f7078455f03dfabb01108532add2a06aac2"}, {"id": "u006", "kind": "prose", "locator": "body:L88-L88", "preview": "Different data streams have different reuse patterns. Applying the correct cache policy per stream avoids L1 pollution:", "sha256": "61778bb24a32b6ac45f992fbba02b18d498c3e6c044780c753715746e1dd4194"}, {"id": "u007", "kind": "code", "locator": "body:L90-L111", "preview": "```cuda // Cache policy selection based on data reuse pattern // // Matrix A (streamed, each row used once): bypass L1 // Vector B (reused across all rows): keep in L1 // L1::no_allocate -- data bypasses L1 cache (streaming access) // Used ", "sha256": "b0734ac15004edad0a83d5fd86ed95480d0b97b6b9cdcacbf7e2d75c021ea929"}, {"id": "u008", "kind": "prose", "locator": "body:L113-L113", "preview": "The impact of cache policies from the GPU Mode Hackathon:", "sha256": "f56640266fb0eecdc5c4ed6f9dbc6a0e0c5c080df7fb55976b1ea7a889217e30"}, {"id": "u009", "kind": "code", "locator": "body:L115-L118", "preview": "``` No cache policy differentiation: 39 us A: L1::no_allocate, B: L1::evict_last: 27 us (1.44x faster) ```", "sha256": "93753b7b8d8885417ae945b5832c4e6037cbfa0b155ca965c3d0101ffb5497e0"}, {"id": "u010", "kind": "prose", "locator": "body:L120-L120", "preview": "The full set of PTX load cache qualifiers:", "sha256": "58238eda395968073d78483c5eb95a8038f94ee1d47de03e3a3250d77fccb6d0"}, {"id": "u011", "kind": "code", "locator": "body:L122-L141", "preview": "```ptx // PTX load qualifiers for cache control // Default: normal L1 and L2 caching ld.global.b32 %r, [%addr]; // L1 bypass: skip L1, still cached in L2 ld.global.L1::no_allocate.b32 %r, [%addr]; // L1 keep: prioritize keeping in L1 (evict", "sha256": "752b8164a5b4f09717bab64bb2690873b38f30979fe106828f3f030b51fe0cc6"}, {"id": "u012", "kind": "prose", "locator": "body:L145-L145", "preview": "For memory-bound kernels, occupancy (number of concurrent warps) matters more than per-thread register count. Limiting registers per thread allows more warps to be resident:", "sha256": "ce919dc749b52caf12cf9687444b0b1552d8de0de1dc2dd38e46c2f802514c0c"}, {"id": "u013", "kind": "code", "locator": "body:L147-L158", "preview": "```cuda // Compile-time register budgeting // Lower register count -> higher occupancy -> better latency hiding // Problem 1 winner (rank 1): -maxrregcount=32 // This allows 64 warps per SM (100% occupancy on SM100) // Sufficient for GEMV w", "sha256": "f3944a53fcb1063a93b50128e650325c3a6acccfc57a90ca33484f9d545361eb"}, {"id": "u014", "kind": "prose", "locator": "body:L160-L160", "preview": "The tradeoff in a build system:", "sha256": "ce280082b2b1587dd5ae7c4e1402a49d8cb6060a15a716009558cf9fcf9cd182"}, {"id": "u015", "kind": "code", "locator": "body:L162-L182", "preview": "```python # nvcc compilation with register budgeting # In CMakeLists.txt or build script: # For memory-bound GEMV kernel: # nvcc -maxrregcount=32 -arch=sm_100a gemv_kernel.cu -o gemv_kernel # For compute-bound GEMM kernel: # nvcc -arch=sm_1", "sha256": "6cc44d43960128e135d7061ed05a7b56241b0cf041f691c9417863073b9e37c3"}, {"id": "u016", "kind": "prose", "locator": "body:L186-L186", "preview": "Combining all three techniques for the GPU Mode Hackathon Problem 1:", "sha256": "f67480c8411891c3e6befa7eca7f1232b1aabd911b9e724958ddda6de1bda868"}, {"id": "u017", "kind": "code", "locator": "body:L188-L264", "preview": "```cuda // Optimized NVFP4 Batched GEMV // A: [M, K] NVFP4, B: [1, K] NVFP4, C: [M, 1] FP16 // Memory-bound: maximize bandwidth utilization // NVFP4 Batched GEMV: each row processed by THREADS_PER_ROW threads // Memory-bound: maximize bandw", "sha256": "5b7c6c47ee7b177b2bdfdaf82c08f189e36da1260845eb206cf4e8108d76c4c2"}, {"id": "u018", "kind": "table-row", "locator": "body:L270-L270", "preview": "| Step | Technique | Latency | Speedup | | Baseline | Naive C++ | 2000 us | 1.0x |", "sha256": "336d82fc951196f5c09b1d297d523dff11e195daeb27195b27e8e6253822f944"}, {"id": "u019", "kind": "table-row", "locator": "body:L271-L271", "preview": "| Step | Technique | Latency | Speedup | | Coalesced access | Memory layout fix | 443 us | 4.5x |", "sha256": "c50836382c1a13b8bf4fdd723d935a8f201e0a11910ab53918f60300fac2c46d"}, {"id": "u020", "kind": "table-row", "locator": "body:L272-L272", "preview": "| Step | Technique | Latency | Speedup | | Hardware intrinsics | FP4 decode | 39 us | 51x |", "sha256": "02cfff4537a67926f42d4e2d186091b3ae71c13b73928ed44a42ed5f1a06ebe1"}, {"id": "u021", "kind": "table-row", "locator": "body:L273-L273", "preview": "| Step | Technique | Latency | Speedup | | PTX assembly | Vectorized loads + cache policy | 27 us | 74x |", "sha256": "a97d68be1e19e4fde08df751d18b0b47aa4f1b681fa05da69cd7d22f449130c2"}, {"id": "u022", "kind": "table-row", "locator": "body:L274-L274", "preview": "| Step | Technique | Latency | Speedup | | ILP + register tuning | Unrolling + maxrregcount | 22.4 us | 89x |", "sha256": "f390517562b134bf9d59c2f220a8a65fed2b59f44eba33937b9c1fbcc0591045"}, {"id": "u023", "kind": "table-row", "locator": "body:L275-L275", "preview": "| Step | Technique | Latency | Speedup | | Speed of light | | ~8.6 us | 233x |", "sha256": "8520b3cfbcc908b3d011fd6695594d47a25d7129a4750c8f39bb28c4a35e551e"}, {"id": "u024", "kind": "list-item", "locator": "body:L279-L279", "preview": "- **GEMV and memory-bound kernels**: Vectorized loads and cache policies are essential. These kernels are entirely limited by memory bandwidth.", "sha256": "a2c9186327c182ffcfa11105251c8e1d30d79f72f672da4385c3ea1a7c113c58"}, {"id": "u025", "kind": "list-item", "locator": "body:L280-L280", "preview": "- **FP4/FP8 kernels**: Sub-byte data types make wide loads even more impactful since more elements fit in a single wide load.", "sha256": "6027557e719d2dce03de3254883be51e4ec7e66ea527d0acdc53161bd3d0be0d"}, {"id": "u026", "kind": "list-item", "locator": "body:L281-L281", "preview": "- **Decode-phase inference**: Single-token GEMV during autoregressive decoding is always memory-bound.", "sha256": "de4b54f2013583c782aa0f5613a89b9f229884229753709e91acce3eb9236c22"}, {"id": "u027", "kind": "list-item", "locator": "body:L285-L285", "preview": "- 256-bit loads require 32-byte aligned addresses. Misaligned access falls back to multiple narrower transactions.", "sha256": "6d651625c4f75106870bf3d68fc195b15f62597fd26ce759ac4661921ce2e45a"}, {"id": "u028", "kind": "list-item", "locator": "body:L286-L286", "preview": "- `L1::no_allocate` is harmful for data that will be reused. Only apply it to truly streaming access patterns.", "sha256": "c86c6ab9e1578a21cf22e62a20b4c0937078fc8f8890dc10d7f3ca35a7740f5a"}, {"id": "u029", "kind": "list-item", "locator": "body:L287-L287", "preview": "- `-maxrregcount` that is too low causes register spilling to local memory, which is slower than the occupancy gain. Profile with Nsight Compute to find the optimal point.", "sha256": "83dddd1b4b88276c52be1d17c87b02929fb67cceab3ace5b6dd48335111a8cf7"}, {"id": "u030", "kind": "list-item", "locator": "body:L288-L288", "preview": "- PTX inline assembly bypasses the compiler's register allocator. Excessive inline PTX can interfere with compiler optimizations for surrounding code.", "sha256": "76af809266f88b18f00f2435cb2e5aaa3355cebd364a7e56714cd4c5bc6b2ce4"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Wide Vectorized Loads", "L1 Cache Policy Differentiation", "Register Budgeting (-maxrregcount)", "Complete NVFP4 GEMV Example", "Optimization Progression (GPU Mode Hackathon Problem 1)", "When to Use", "Caveats"], "id": "technique-vectorized-loads", "path": "wiki/techniques/vectorized-loads.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/yue-nvfp4-hackathon.md", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}, {"path": "sources/blogs/amandeep-nvfp4-attempts.md", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}, {"path": "sources/contests/gpu-mode-nvfp4/problem-1-gemv.md", "url": "https://github.com/gpu-mode/reference-kernels"}], "risk_flags": ["code", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-yue-nvfp4", "blog-amandeep-nvfp4", "contest-gpumode-p1"], "title": "Wide Vectorized Loads and Cache Policies", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "48ec87e638f28a2e42314d08521041a40b518e4fe0d8ba0b984f9764efcf3fd9", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "Warp specialization assigns distinct functional roles to warps within a CTA, allowing each warp to focus on a single pipeline stage (data loading, MMA computation, or epilogue writeback). On Blackwell (SM100), the 16-warp CTA structure repl", "sha256": "da68ea21e6384fe8ed99ca14e0dec4aafcd6dd43bae885faa06936cced0326a7"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "The canonical Blackwell GEMM kernel uses 16 warps (512 threads) per CTA with the following role assignment:", "sha256": "de5f4767ff59a243b7cd0bf9cf1e5316f4b00bd2a0a4d7d17a288d87f0066cdc"}, {"id": "u003", "kind": "table-row", "locator": "body:L11-L11", "preview": "| Warp ID | Role | Responsibility | | 0 | TMA Producer | Issues TMA bulk-copy from global to shared memory, signals mbarrier |", "sha256": "e23f6c241b5be333ca535184eb14223d873a2de3e2c869d9f2d380d357b3d33e"}, {"id": "u004", "kind": "table-row", "locator": "body:L12-L12", "preview": "| Warp ID | Role | Responsibility | | 1 | MMA Consumer | Issues tcgen05.mma on SMEM operands, writes results to TMEM |", "sha256": "ee835b266c9294c9f2a1b9aea49739bf61e6b5754e7361e0bc2b237c66c6a548"}, {"id": "u005", "kind": "table-row", "locator": "body:L13-L13", "preview": "| Warp ID | Role | Responsibility | | 2-15 | Epilogue | Reads TMEM accumulator, applies scale/bias/activation, writes to global memory |", "sha256": "66e041a9aaa309ff343b8e44e0e845d9c80e1043c55ead5c561127698c6c1c12"}, {"id": "u006", "kind": "prose", "locator": "body:L15-L15", "preview": "This contrasts with Hopper where a warpgroup (4 warps, 128 threads) collectively issues wgmma.mma_async, and all threads in the warpgroup participate in the MMA. On Blackwell, the MMA warp dispatches the instruction from a single thread whi", "sha256": "7c12b99bbbf11a24a3975923f89cb7364f576479e6ffe0cf3e44d777edb34a35"}, {"id": "u007", "kind": "table-row", "locator": "body:L21-L21", "preview": "| Aspect | Hopper (SM90) | Blackwell (SM100) | | MMA granularity | 4-warp warpgroup (128 threads) | Single thread in 1 warp |", "sha256": "9501b37c444d81b54c35183ca8d5aed043b54645491c4d4cf54309949f526ce8"}, {"id": "u008", "kind": "table-row", "locator": "body:L22-L22", "preview": "| Aspect | Hopper (SM90) | Blackwell (SM100) | | MMA output destination | Registers (shared across warpgroup) | TMEM (256KB, CTA-visible) |", "sha256": "cf6cf6943fd445678a4f0fcc62e356d4ad8ca0ca62167fc7f42c5bf871905f20"}, {"id": "u009", "kind": "table-row", "locator": "body:L23-L23", "preview": "| Aspect | Hopper (SM90) | Blackwell (SM100) | | Producer warps | Separate warp(s) for TMA loads | Warp 0 dedicated to TMA |", "sha256": "12f9916a54cce0a6b9e7f618ff6a780fdcd908ff108577efa534955ef70dadfa"}, {"id": "u010", "kind": "table-row", "locator": "body:L24-L24", "preview": "| Aspect | Hopper (SM90) | Blackwell (SM100) | | Epilogue execution | Same warpgroup or separate warps | 14 dedicated warps (2-15) |", "sha256": "fb2aa36acbe1915af15f8d542b0a90d38000d4f95255150b748812c980c44c85"}, {"id": "u011", "kind": "table-row", "locator": "body:L25-L25", "preview": "| Aspect | Hopper (SM90) | Blackwell (SM100) | | Synchronization | warpgroup barriers, arrive/wait | mbarrier pairs (producer/consumer) |", "sha256": "2cace7a3e7b39e5063c94652331e357e886e261cb9f7cc318e57773fc37e5616"}, {"id": "u012", "kind": "table-row", "locator": "body:L26-L26", "preview": "| Aspect | Hopper (SM90) | Blackwell (SM100) | | Register pressure | High (accumulators in registers) | Low (accumulators in TMEM) |", "sha256": "f8e1ff2e3d5ee5b118c21a3b74af5684bcd18d154bdcf43f9e35c165f84ac8db"}, {"id": "u013", "kind": "prose", "locator": "body:L30-L30", "preview": "The kernel entry point assigns each warp its role based on `threadIdx.x`:", "sha256": "ebdfd00934c10addf671ee6c2cb5cb6c322fb1735230e966e8327dba6c666bd5"}, {"id": "u014", "kind": "code", "locator": "body:L32-L155", "preview": "```cuda // Blackwell 16-warp specialized GEMM kernel skeleton // 16 warps = 512 threads per CTA __global__ void __launch_bounds__(512) blackwell_gemm_warp_specialized( const __grid_constant__ GemmParams params) { const int warp_id = threadI", "sha256": "1238a08de84a73fa41d398e4aece1b28464d483317ca6b3879fc5d8a5a089ac9"}, {"id": "u015", "kind": "prose", "locator": "body:L159-L159", "preview": "The producer-consumer synchronization uses mbarrier pairs. Each pipeline stage has two barriers:", "sha256": "7b46b41fb034dfeaf0da578f65d8230f648de6bade3e3b7ed48ef987ec1b595d"}, {"id": "u016", "kind": "list-item", "locator": "body:L161-L161", "preview": "1. **data_ready**: Producer (Warp 0) arrives after TMA completes. Consumer (Warp 1) waits before issuing MMA.", "sha256": "bbb39c8d97ce7f22227172b3f2a697a5700673f4b35e946bf73f488e89e42221"}, {"id": "u017", "kind": "list-item", "locator": "body:L162-L162", "preview": "2. **buffer_free**: Consumer (Warp 1) arrives after MMA consumes the data. Producer (Warp 0) waits before overwriting the buffer.", "sha256": "d5406aa4c18b3073b5c3acc0247aa98f14a04d4fa8b6fffc24b82b5b249f47b4"}, {"id": "u018", "kind": "prose", "locator": "body:L164-L164", "preview": "At the PTX level, the mbarrier operations map to:", "sha256": "8fd645f2337717bb6614b4e85f50c4d5e378736f8233810542afe896f7e6bbe0"}, {"id": "u019", "kind": "code", "locator": "body:L166-L178", "preview": "```ptx // Producer: signal data is ready in stage %stage mbarrier.arrive.shared.b64 %dummy, [%mbar_data_ready + %stage_offset]; // Consumer: wait for data to be ready mbarrier.try_wait.parity.shared.b64 %pred, [%mbar_data_ready + %stage_off", "sha256": "9863ded2350007fa56e7c0fdbb2900b0def790913e94618882c21c78b55114af"}, {"id": "u020", "kind": "prose", "locator": "body:L182-L182", "preview": "In CUTLASS 4.5.0, the SM100 GEMM collective (`CollectiveMma_1SM`) implements this pattern with CuTe abstractions:", "sha256": "6c3f43a9f5180313f211d5e54d8e1ea09b302f6fc25688820675f47dcda9401b"}, {"id": "u021", "kind": "code", "locator": "body:L184-L211", "preview": "```cuda // CUTLASS SM100 warp role dispatch (simplified from CollectiveMma) // Template parameter WarpCount = cute::Shape<1, 1, 14> // Warp 0 = producer, Warp 1 = math, Warps 2-15 = epilogue template struc", "sha256": "f9559e318d537c668357c24e65dede295acac7c5ab86a8da0927c063ec3092dc"}, {"id": "u022", "kind": "list-item", "locator": "body:L215-L215", "preview": "- **Always on Blackwell GEMMs**: Warp specialization is the standard pattern for SM100 tensor core kernels. The tcgen05 instruction model assumes single-thread dispatch with TMEM output.", "sha256": "134a0249a13f619bb66cb12d8fdd7ea57d1ea3c5963058c4b5833596d4e4cc52"}, {"id": "u023", "kind": "list-item", "locator": "body:L216-L216", "preview": "- **Attention kernels**: FlashAttention-4 extends this to ping-pong scheduling with 2 query tile groups and dedicated softmax warps.", "sha256": "cb8edb4bc739191404192c08ae07092a800fb2844aaf8e46a36d1a2a08788912"}, {"id": "u024", "kind": "list-item", "locator": "body:L217-L217", "preview": "- **Any kernel with producer-consumer pipeline**: When TMA loads and MMA compute can overlap, warp specialization provides the cleanest decomposition.", "sha256": "b41a91691e1784ab015dc1f1ad4351f86a96f26969196ffc5ba85f6a1ced6292"}, {"id": "u025", "kind": "list-item", "locator": "body:L221-L221", "preview": "- The 14 epilogue warps may be underutilized for simple epilogues (e.g., pure store). Complex epilogues (scale, bias, activation, quantization) benefit more.", "sha256": "2a932b047515d65e2a3832670486a534a3df6eec93cb9c057c46379fa31da1b1"}, {"id": "u026", "kind": "list-item", "locator": "body:L222-L222", "preview": "- The single MMA warp means the kernel cannot overlap multiple independent MMA streams within a CTA. Use 2-SM cooperative mode for larger tiles instead.", "sha256": "b47c4681255805de9445498daab973a4c01f4951c4788988357b2df59fd9497f"}, {"id": "u027", "kind": "list-item", "locator": "body:L223-L223", "preview": "- mbarrier initialization must happen before any warp tries to wait; use `__syncthreads()` after init if needed.", "sha256": "290645aeddd55af0f1315f3407f062f5f42ae857853fcb8ee8d5082be12037cb"}, {"id": "u028", "kind": "prose", "locator": "body:L227-L227", "preview": "Local verbatim upstream code lives in [`artifacts/kernels/warp-specialization/full/`](../../artifacts/kernels/warp-specialization/full/) (see its `PROVENANCE.yaml` for the pinned upstream SHA and byte-verified SHA-256). Labeled derived vari", "sha256": "eeeda30cf2081846193f80099b748c3550961732b1403dfb8c0a7fbc62f8c824"}, {"id": "u029", "kind": "prose", "locator": "body:L229-L229", "preview": "Query via:", "sha256": "6abd63e508f9320fdd3b2b0cc5b5f6a000f4462abfd17f9ecc3c4818f4a0f875"}, {"id": "u030", "kind": "code", "locator": "body:L231-L233", "preview": "```bash python3 scripts/get_page.py technique-warp-specialization --include-code ```", "sha256": "b7c5cf5fad5ff312746a2f70dd0f13cb952d92b5faa64436566a716041abd115"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Blackwell 16-Warp Kernel Structure", "Comparison with Hopper Warpgroup Model", "Warp Role Assignment", "mbarrier Synchronization Pattern", "CUTLASS SM100 Warp Specialization", "When to Use", "Caveats", "Full Reference Implementation"], "id": "technique-warp-specialization", "path": "wiki/techniques/warp-specialization.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/blogs/colfax-cutlass-blackwell.md", "url": "https://research.colfax-intl.com/cutlass-tutorial-writing-gemm-kernels-using-tmem-for-nvidia-blackwell-gpus/"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "blog-colfax-cutlass"], "title": "Warp Specialization on Blackwell", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} diff --git a/verification/before-after-report.jsonl b/verification/before-after-report.jsonl new file mode 100644 index 000000000..71ae08449 --- /dev/null +++ b/verification/before-after-report.jsonl @@ -0,0 +1,493 @@ +{"path": "wiki/hardware/clc.md", "before": {"statement": "Cluster Launch Control (CLC) is a Blackwell hardware mechanism for **dynamic tile scheduling** in persistent kernels. It replaces the static grid scheduling model where the CUDA runtime pre-assigns tile coordinates to CTAs at launch time.\n\n- **Better load balancing**: No fixed CTA-to-tile mapping; busy SMs consume tiles as they become available."}, "after": {"statement": "1. The block or cluster launches normally and first processes its own\n `blockIdx`.\n2. An existing worker successfully cancels that not-yet-started ClcID and\n processes the returned coordinate itself."}, "reason": {"statement": "Retain the dynamic-work-stealing concept while restoring the documented initial assignment and full-grid model.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/hardware/clc.md", "before": {"statement": "- **Tail-effect mitigation**: The \"tail\" of a GEMM (when remaining tiles < number of SMs) is handled efficiently because idle CTAs pick up remaining work.\n\n```\nLaunch grid: 256 CTAs for 256 tiles\nCTA 0 -> tile (0,0) [fixed at launch]\nCTA 1 -> tile (0,1) [fixed at launch]\nCTA 2 -> tile (0,2) [fixed at launch]\n...\nCTA 255 -> tile (15,15) [fixed at launch]\n\nProblem: If SM count = 132, first wave = 132 CTAs.\n Second wave = 124 CTAs -> 8 SMs idle = 6% waste.\n For small GEMMs, tail effect dominates.\n```"}, "after": {"statement": "At CUTLASS 4.5.0, the `PersistentScheduler` tag for `arch::Sm100` maps to\n`PersistentTileSchedulerSm100`; newer code can select the intent explicitly\nwith `DynamicPersistentScheduler`. The SM100 scheduler uses\n`PipelineCLCFetchAsync`: `advance_to_next_work()` submits a request, while\n`fetch_next_work()` waits for and decodes the staged response."}, "reason": {"statement": "Replace the universal tail claim with the scoped load-balancing benefit documented for uneven SM availability.", "urls": ["https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html", "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/tile_scheduler.hpp", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp"]}} +{"path": "wiki/hardware/clc.md", "before": {"statement": "- **Dynamic cancellation**: Tiles can be cancelled via `try_cancel` when the output is no longer needed (e.g., speculative decoding).\n\nCLC provides a `try_cancel` mechanism to cancel pending tiles. This is useful for speculative execution where some outputs may not be needed.\n\nUse cases for `try_cancel`:\n- **Speculative decoding**: Cancel tiles for rejected draft tokens.\n- **Early termination**: If an attention mask makes certain output tiles zero, cancel them.\n- **Dynamic batching**: Cancel tiles for sequences that have finished."}, "after": {"statement": null}, "reason": {"statement": "The listed use cases depend on an API capability CLC does not provide; no useful verified portion remains.", "urls": ["https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"]}} +{"path": "wiki/hardware/clc.md", "before": {"statement": "```\nLaunch grid: 132 persistent CTAs (= SM count)\nCTA 0: request tile -> get (0,0) -> compute -> request tile -> get (2,4) -> ...\nCTA 1: request tile -> get (0,1) -> compute -> request tile -> get (2,5) -> ...\n...\nCTA 131: request tile -> get (0,131) -> compute -> request tile -> ...\n\nAll CTAs stay busy until the tile queue is empty.\nTail: last few tiles distributed to first-available CTAs.\n```"}, "after": {"statement": "1. The block or cluster launches normally and first processes its own\n `blockIdx`.\n2. An existing worker successfully cancels that not-yet-started ClcID and\n processes the returned coordinate itself.\n\nThis lets a persistent worker request subsequent work without maintaining a\nseparate software work queue. It is especially useful when the set of SMs\navailable to a kernel is uneven or changes while the grid is running."}, "reason": {"statement": "Replace the misleading diagram with the documented worker lifecycle.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", "https://docs.nvidia.com/cutlass/4.3.3/media/docs/cpp/blackwell_cluster_launch_control.html"]}} +{"path": "wiki/hardware/clc.md", "before": {"statement": "```cuda\n__global__ void persistent_gemm_clc(\n const half* A, const half* B, half* C,\n int M, int N, int K,\n int num_tiles_m, int num_tiles_n\n) {\n // Allocate persistent resources (TMEM, pipeline state)\n uint32_t tmem_acc = tmem_alloc(256);\n\n // Shared storage for CLC results (visible to all threads in CTA)\n __shared__ uint2 clc_tile_coord;\n __shared__ int clc_has_tile;\n\n // CLC tile loop: keep requesting tiles until none remain\n while (true) {\n // Thread 0 acquires the next tile; result goes to shared memory\n if (threadIdx.x == 0) {\n uint2 result;\n int acquired = 0;\n asm volatile(\n \"{\\n\"\n \" .reg .pred p;\\n\"\n \" clusterlaunchcontrol.try_cancel {%0, %1}, p;\\n\"\n \" selp.s32 %2, 1, 0, p;\\n\"\n \"}\\n\"\n : \"=r\"(result.x), \"=r\"(result.y), \"=r\"(acquired)\n );\n clc_tile_coord = result;\n clc_has_tile = acquired;\n }\n __syncthreads(); // All threads see the shared result\n\n // Exit if no more tiles\n if (!clc_has_tile) break;\n\n int tile_m = clc_tile_coord.x;\n int tile_n = clc_tile_coord.y;\n\n // Zero accumulator\n tmem_zero(tmem_acc, 256);\n\n // Mainloop: iterate over K dimension\n for (int k = 0; k < K / TILE_K; ++k) {\n // TMA load A and B tiles to SMEM\n tma_load_a(smem_a, A, tile_m, k);\n tma_load_b(smem_b, B, k, tile_n);\n wait_barrier();\n\n // Issue MMA\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 1;\"\n :\n : \"r\"(tmem_acc), \"l\"(desc_a), \"l\"(desc_b), \"r\"(0)\n );\n }\n }\n\n // Epilogue\n asm volatile(\"tcgen05.fence::before_thread_sync;\");\n __syncthreads();\n store_output(tmem_acc, C, tile_m, tile_n);\n }\n\n // Cleanup\n tmem_dealloc(tmem_acc, 256);\n}\n```"}, "after": {"statement": "This lets a persistent worker request subsequent work without maintaining a\nseparate software work queue. It is especially useful when the set of SMs\navailable to a kernel is uneven or changes while the grid is running.\n\n`clusterlaunchcontrol.try_cancel` is asynchronous. It writes an opaque 16-byte\nresponse to shared memory and completes a transaction on a shared-memory\n`mbarrier`. The request does not accept a tile coordinate to cancel.\n\nThe normative PTX instruction forms are:\n\n```ptx\nclusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [response], [mbar];\nclusterlaunchcontrol.query_cancel.is_canceled.pred.b128 p, response_b128;\n@p clusterlaunchcontrol.query_cancel.get_first_ctaid.v4.b32.b128 {x, y, z, unused}, response_b128;\n```"}, "reason": {"statement": "Replace invalid illustrative CUDA with the official operation sequence and exact PTX signatures.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel", "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", "https://docs.nvidia.com/cutlass/4.3.3/media/docs/cpp/blackwell_cluster_launch_control.html"]}} +{"path": "wiki/hardware/clc.md", "before": {"statement": "```cuda\n// Cancel a specific tile if it hasn't started execution yet\n__device__ bool clc_try_cancel(uint2 tile_coord) {\n bool cancelled = false;\n if (threadIdx.x == 0) {\n asm volatile(\n \"clusterlaunchcontrol.try_cancel.async.shared::cta \"\n \"%0, [%1];\"\n : \"=r\"(cancelled)\n : \"r\"(&tile_coord)\n );\n }\n return cancelled;\n}\n```"}, "after": {"statement": "This lets a persistent worker request subsequent work without maintaining a\nseparate software work queue. It is especially useful when the set of SMs\navailable to a kernel is uneven or changes while the grid is running."}, "reason": {"statement": "Retain an API sketch only in the exact asynchronous request/decode form.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel", "https://docs.nvidia.com/cutlass/4.3.3/media/docs/cpp/blackwell_cluster_launch_control.html"]}} +{"path": "wiki/hardware/clc.md", "before": {"statement": "```cuda\n// CUTLASS SM100 persistent GEMM with CLC scheduling\nusing Gemm = cutlass::gemm::device::GemmUniversal<\n cutlass::half_t, // ElementA\n cutlass::layout::RowMajor, // LayoutA\n cutlass::half_t, // ElementB\n cutlass::layout::ColumnMajor, // LayoutB\n cutlass::half_t, // ElementC\n cutlass::layout::RowMajor, // LayoutC\n float, // ElementAccumulator\n cutlass::arch::OpClassTensorOp,\n cutlass::arch::Sm100, // Blackwell\n // Tile shape: 128x256x64\n cutlass::gemm::GemmShape<128, 256, 64>,\n // Cluster shape\n cutlass::gemm::GemmShape<2, 1, 1>,\n // Use CLC persistent scheduler\n cutlass::gemm::PersistentScheduler\n>;\n\n// Launch: CUTLASS handles CLC internally\nGemm gemm_op;\ngemm_op(args, workspace, stream);\n```"}, "after": {"statement": "After a thread has observed a failed request, issuing another request from that\nthread is undefined. Decoding a CTA ID from a failed response is also undefined.\nA request can fail because no ClcIDs remain or for another scheduling reason,\nincluding pending higher-priority work."}, "reason": {"statement": "Replace invalid code with a compact excerpt matching the pinned release.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/95_blackwell_gemm_green_context/95_blackwell_gemm_green_context.cu", "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"]}} +{"path": "wiki/hardware/clc.md", "before": {"statement": "```cpp\n// Simplified CUTLASS CLC scheduler logic\nstruct ClcTileScheduler {\n CUTLASS_DEVICE\n WorkTileInfo get_next_work() {\n WorkTileInfo work;\n // Fetch next tile by cancelling a not-yet-launched ClcID.\n bool valid = clc_try_cancel(&work.tile_coord);\n work.is_valid = valid;\n\n if (valid) {\n // Convert linear tile index to 2D coordinates\n work.tile_m = work.tile_coord.x;\n work.tile_n = work.tile_coord.y;\n\n // Apply swizzle for L2 locality\n apply_l2_swizzle(work.tile_m, work.tile_n);\n }\n return work;\n }\n};\n```"}, "after": {"statement": "After a thread has observed a failed request, issuing another request from that\nthread is undefined. Decoding a CTA ID from a failed response is also undefined.\nA request can fail because no ClcIDs remain or for another scheduling reason,\nincluding pending higher-priority work."}, "reason": {"statement": "Replace the false pseudocode with the actual staged CUTLASS flow.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp", "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"]}} +{"path": "wiki/hardware/clc.md", "before": {"statement": "CLC delivers significant performance gains, especially for small-to-medium GEMMs where tail effects dominate:\n\n| 2048x2048 (small) | 86% SM utilization | 98% SM utilization | +14% |\n| 4096x4096 (medium) | 92% SM utilization | 98% SM utilization | +6.5% |\n| 8192x8192 (large) | 97% SM utilization | 99% SM utilization | +2% |"}, "after": {"statement": null}, "reason": {"statement": "The exact numbers lack hardware, software, tile, dtype, occupancy, and timing scope; no source-backed replacement measurement is available.", "urls": []}} +{"path": "wiki/hardware/clc.md", "before": {"statement": "The canonical benchmark from the \"tcgen05 for dummies\" tutorial shows the jump from 940 TFLOPS (pipelined, static scheduling) to **1476 TFLOPS** (persistent + CLC), approaching 98% of cuBLAS (1507 TFLOPS)."}, "after": {"statement": null}, "reason": {"statement": "The measured numbers may remain attributable to the tutorial elsewhere, but they are not CLC evidence and cannot support this page's performance claim.", "urls": ["https://gau-nernst.github.io/tcgen05/"]}} +{"path": "wiki/hardware/clc.md", "before": {"statement": "```python\n# Typical LLM GEMM shapes during decode (batch_size=1-64)\n# M is small (batch * seq_len for decode), N and K are large (model dim)\n# Example: Llama-70B decode, batch=32\nM = 32 # small!\nN = 8192 # hidden dim\nK = 8192 # hidden dim\n\n# Tile = 128x256 -> tiles_m = 1, tiles_n = 32 -> only 32 tiles total\n# On B200 (132 SMs): 100 SMs idle with static scheduling\n# CLC: 32 persistent CTAs handle all 32 tiles efficiently\n```"}, "after": {"statement": "A schematic of the relevant CUTLASS 3.x kernel composition is:\n\n```cpp\nusing TileScheduler = cutlass::gemm::DynamicPersistentScheduler;\n\nusing GemmKernel = cutlass::gemm::kernel::GemmUniversal<\n cute::Shape,\n CollectiveMainloop,\n CollectiveEpilogue,\n TileScheduler>;\n```"}, "reason": {"statement": "Retain the useful warning about insufficient tile parallelism while removing the wrong hardware count and false CLC remedy.", "urls": ["https://developer.nvidia.com/blog/scaling-autonomous-ai-agents-and-workloads-with-nvidia-dgx-spark/", "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/95_blackwell_gemm_green_context/95_blackwell_gemm_green_context.cu"]}} +{"path": "wiki/hardware/clc.md", "before": {"statement": "```cuda\n// 2-SM cooperative CLC: each successful cancel gets a cluster-sized tile\n__device__ void cooperative_clc_loop() {\n while (true) {\n // Fetch tile for the 2-CTA cluster\n ClusterTile tile;\n bool valid = clc_try_cancel_cluster(&tile);\n if (!valid) break;\n\n // Both CTAs in the cluster share the tile\n // CTA 0 handles rows 0-127, CTA 1 handles rows 128-255\n int my_row_start = (blockIdx.x % 2) * 128;\n\n // Issue cooperative MMA\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.mma.cta_group::2.kind::f16 \"\n \"[%0], %1, %2, %3, 1;\"\n :\n : \"r\"(tmem_acc), \"l\"(desc_a), \"l\"(desc_b), \"r\"(0)\n );\n }\n // ...\n }\n}\n```"}, "after": {"statement": "A complete loop must:\n\n1. Allocate and initialize a shared response and `mbarrier`.\n2. Submit one asynchronous request from the selected thread and set the\n expected transaction size to 16 bytes.\n3. Wait for the corresponding barrier phase to complete.\n4. Decode `is_canceled`; decode `get_first_ctaid` only after success.\n5. Apply the async/generic proxy fences needed before reusing the response."}, "reason": {"statement": "Replace invalid executable-looking code with explicit, source-backed cluster requirements.", "urls": ["https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"]}} +{"path": "wiki/hardware/clc.md", "before": {"statement": "```cuda\n// Swizzle tile coordinates for better L2 locality\n// Tiles are visited in a Z-order (Morton) curve pattern\n__device__ void apply_l2_swizzle(int& tile_m, int& tile_n, int swizzle_bits) {\n // Convert linear tile index to swizzled 2D coordinates\n // This groups spatially adjacent tiles together, improving\n // L2 reuse for the B matrix (shared across M tiles)\n int linear = tile_m * num_tiles_n + tile_n;\n int swizzle_mask = (1 << swizzle_bits) - 1;\n int group = linear >> swizzle_bits;\n int within = linear & swizzle_mask;\n\n tile_m = group / num_tiles_n;\n tile_n = (group % num_tiles_n) ^ (tile_m & swizzle_mask);\n}\n```"}, "after": {"statement": "CLC cancellation is cluster-granular when the kernel uses thread block\nclusters. One cluster thread submits the multicast request. Every CTA tracks\ncompletion with its local shared-memory barrier at cluster scope and receives\nthe same encoded first-CTA coordinate. Each CTA then adds its local block rank\nto that first coordinate. A cluster synchronization is needed to guarantee all\nblocks exist before cluster cancellation begins.\n\nFor example, a successful query by a 2x2 worker cluster consumes the matching\n2x2 group of ClcIDs; it is not four unrelated CTA-level cancellations."}, "reason": {"statement": "Delete the broken function and retain the verified CUTLASS software-coordinate-remapping mechanism.", "urls": ["https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/hardware/clc.md", "before": {"statement": "| Scheduling overhead | Near zero (hardware) | atomicAdd contention |\n| Tail-effect handling | Optimal | Good with careful design |"}, "after": {"statement": null}, "reason": {"statement": "The qualitative ranking has no defined workload or metric and cannot be replaced with a universal performance claim.", "urls": ["https://docs.nvidia.com/cutlass/4.3.3/media/docs/cpp/blackwell_cluster_launch_control.html", "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"]}} +{"path": "wiki/hardware/clc.md", "before": {"statement": "| L2 swizzle | Configurable at launch | Manual implementation |"}, "after": {"statement": "CLC cancellation is cluster-granular when the kernel uses thread block\nclusters. One cluster thread submits the multicast request. Every CTA tracks\ncompletion with its local shared-memory barrier at cluster scope and receives\nthe same encoded first-CTA coordinate. Each CTA then adds its local block rank\nto that first coordinate. A cluster synchronization is needed to guarantee all\nblocks exist before cluster cancellation begins.\n\nFor example, a successful query by a 2x2 worker cluster consumes the matching\n2x2 group of ClcIDs; it is not four unrelated CTA-level cancellations."}, "reason": {"statement": "Preserve swizzled work mapping but attribute it to the CUTLASS software scheduler.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel", "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/languages/triton-blackwell.md", "before": {"statement": "Triton is used for many attention and linear-attention kernels (NSA, GatedDeltaNet, FLA). Starting with Triton 3.6 (released `2026-01-21`), Triton ships native Blackwell (SM100) lowering through `tcgen05.mma` + Tensor Memory (TMEM). The earlier framing — \"Triton compiler generates wgmma, not tcgen05\" — was correct for Triton 3.5 and earlier but is no longer correct on 3.6+. See the \"Pre-3.6 historical context\" subsection below for the historical framing; the rest of this page describes the current 3.6+ behavior.\n\nThe 3.6 release adds Blackwell-native infrastructure for `tcgen05.mma`, TMEM allocation/copy/load/store, and warp-specialization plumbing. Source-of-record: [`doc-triton-3.6-blackwell`](../../sources/docs/triton-3.6-blackwell.md). Per-pathway breakdown with verified-vs-needs-verification status: [`data/triton-3.6-evidence.md`](../../data/triton-3.6-evidence.md).\n\n- **What is verified**: the Triton 3.6 release adds tcgen05 + TMEM lowering infrastructure (per `doc-triton-3.6-blackwell`), and tracked downstream repos are landing real Triton kernel changes for SM100 post-3.6 (per `pr-vllm-34597`, the post-refresh primary anchor — vLLM's \"[Kernel] Add FP8 KV cache support to Triton MLA decode attention\" PR, which directly modifies `@triton.jit`-decorated kernels in `vllm/v1/attention/ops/triton_decode_attention.py` doing `tl.dot(q, k)` / `tl.dot(qpe, kpe)` / `tl.dot(p, v)` matmul on `[sm100]`, shipped verbatim under `artifacts/prs/vllm/PR-34597/`). Supplementary post-refresh evidence: `pr-vllm-29339` shows the Triton 3.6 `triton_kernels` library being explicitly scoped to Blackwell production codepaths. Pre-refresh historical evidence: `pr-sglang-22079` (the Gemma4 NVFP4 attention kernel merged 2026-04-03). Together these establish that Triton 3.6+ on Blackwell is materially different from the pre-3.6 \"lacked-tcgen05 / lacked-TMEM\" story.\n\nBefore Triton 3.6, the Blackwell story was: the compiler generated `wgmma.mma_async`, accumulators stayed in registers, and direct `tcgen05` / TMEM access was unavailable from `tl.*`. Pages and inclusion-policy text written under that premise are obsolete. The current premise is: **`tcgen05` + TMEM lowering paths exist on SM100, but coverage and performance leadership are workload-dependent**.\n\n- **`doc-triton-3.6-blackwell`** (`source_category: official-doc`) — verifies that Triton 3.6 ships native Blackwell lowering infrastructure (TMEM, tcgen05, warp_specialize plumbing, Gluon multi-CTA / 2CTA, tl.dot_scaled). This is the \"infrastructure exists\" half of AC-1.2.\n\nTogether the two anchors establish that **the 3.6+ Blackwell lowering infrastructure is real AND tracked downstream repos are landing real Triton matmul kernels for Blackwell production decode paths today** — which is the substance of the rewrite: the pre-3.6 \"lacked-tcgen05 / lacked-TMEM\" framing is no longer correct.\n\n> The text in this subsection describes Triton 3.5 and earlier. It is preserved for historical accuracy and is NOT a current statement about Triton 3.6+ behavior. Do not cite it as current limitations.\n\nTriton on Blackwell — Triton 3.5 and earlier:\n\n1. **No direct tcgen05 access**: Triton compiler generates wgmma, not tcgen05.\n2. **No TMEM**: accumulators stay in registers.\n\nThese four bullets describe the world before Triton 3.6.0 landed (`2026-01-21`). The first two are no longer correct on 3.6+; see the \"Triton 3.6+ Blackwell path\" subsection above. CPU launch overhead and CUDA-graph workarounds remain workload-relevant on small-batch decode paths regardless of compiler era."}, "after": {"statement": "version_sensitive:\n id: vs-triton-3.3-blackwell-tcgen05\n\nNative Blackwell TCGen5/TMEM compiler support enters between Triton v3.2.0 and v3.3.0. In the exact tag comparison, the corresponding TCGen5 MMA, TMEM, and MMAv5-lowering symbols are absent at v3.2.0 (`9641643d`) and present at v3.3.0 (`819e9c8c`). The v3.3 conversion suite checks concrete `tcgen05.mma`, commit, TMEM, scaled-MMA, and `cta_group::2` output. See the pinned files in [`doc-triton-3.3-blackwell`](../../sources/docs/triton-3.3-blackwell.md).\n\n| v3.2.0 | Checked negative side of the native-backend boundary. |\n| v3.3.0 | First checked tag after v3.2.0 with TCGen5/TMEM operations, allocation and lowering passes, and concrete conversion tests. |\n| v3.5.0 | Tagged tree includes an explicit Gluon TCGen5/TMEM tutorial and the Blackwell block-scaled matmul tutorial; release notes also document warp-specialization work. |\n| v3.6.0 | Generalizes TCGen5 copies/layouts and MMA handling, advances aref-style warp specialization, and adds initial multi-CTA/2-CTA Gluon work. It is not the introduction boundary. |\n| v3.7.0 / v3.7.1 | v3.7.0 continues 2-CTA, multicast, and TMA work; v3.7.1 is a two-regression patch with no advertised new API or feature. |\n\nThe v3.5.0 tree provides two useful pinned examples:\n\n- [`python/tutorials/gluon/06-tcgen05.py`](https://github.com/triton-lang/triton/blob/c3c476f357f1e9768ea4e45aa5c17528449ab9ef/python/tutorials/gluon/06-tcgen05.py) explicitly allocates, loads, and stores TMEM and invokes TCGen5 MMA through Gluon.\n- [`python/tutorials/10-block-scaled-matmul.py`](https://github.com/triton-lang/triton/blob/c3c476f357f1e9768ea4e45aa5c17528449ab9ef/python/tutorials/10-block-scaled-matmul.py) demonstrates `tl.dot_scaled` for Blackwell block-scaled matmul.\n\nTriton v3.6.0 expands those foundations with broader layouts and copies and initial multi-CTA/2-CTA Gluon support. “Initial” is deliberate: v3.7.0 contains follow-on end-to-end 2-CTA, multicast, and TMA changes. See [`doc-triton-3.6-blackwell`](../../sources/docs/triton-3.6-blackwell.md) and the compact release matrix in [`data/triton-3.6-evidence.md`](../../data/triton-3.6-evidence.md)."}, "reason": {"statement": "Restore the v3.2-to-v3.3 introduction boundary and describe 3.6 as an incremental expansion.", "urls": ["https://github.com/triton-lang/triton/blob/819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c/include/triton/Dialect/TritonNvidiaGPU/IR/TritonNvidiaGPUOps.td", "https://github.com/triton-lang/triton/blob/819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c/test/Conversion/tritongpu_to_llvm_blackwell.mlir", "https://github.com/triton-lang/triton/compare/v3.2.0...v3.3.0", "https://github.com/triton-lang/triton/blob/c3c476f357f1e9768ea4e45aa5c17528449ab9ef/python/tutorials/gluon/06-tcgen05.py", "https://github.com/triton-lang/triton/blob/c3c476f357f1e9768ea4e45aa5c17528449ab9ef/python/tutorials/10-block-scaled-matmul.py", "https://github.com/triton-lang/triton/releases/tag/v3.6.0", "https://github.com/triton-lang/triton/releases/tag/v3.7.0", "https://github.com/triton-lang/triton/releases/tag/v3.7.1"]}} +{"path": "wiki/languages/triton-blackwell.md", "before": {"statement": "- **What is verified**: the Triton 3.6 release adds tcgen05 + TMEM lowering infrastructure (per `doc-triton-3.6-blackwell`), and tracked downstream repos are landing real Triton kernel changes for SM100 post-3.6 (per `pr-vllm-34597`, the post-refresh primary anchor — vLLM's \"[Kernel] Add FP8 KV cache support to Triton MLA decode attention\" PR, which directly modifies `@triton.jit`-decorated kernels in `vllm/v1/attention/ops/triton_decode_attention.py` doing `tl.dot(q, k)` / `tl.dot(qpe, kpe)` / `tl.dot(p, v)` matmul on `[sm100]`, shipped verbatim under `artifacts/prs/vllm/PR-34597/`). Supplementary post-refresh evidence: `pr-vllm-29339` shows the Triton 3.6 `triton_kernels` library being explicitly scoped to Blackwell production codepaths. Pre-refresh historical evidence: `pr-sglang-22079` (the Gemma4 NVFP4 attention kernel merged 2026-04-03). Together these establish that Triton 3.6+ on Blackwell is materially different from the pre-3.6 \"lacked-tcgen05 / lacked-TMEM\" story.\n\n- **`pr-vllm-34597`** (`source_category: upstream-code`) — **post-refresh primary anchor**: vLLM's \"[Kernel] Add FP8 KV cache support to Triton MLA decode attention\" PR (merged 2026-02-16, post-Triton-3.6.0 release; `architectures: [sm100]`; tags `attention`, `decode`, `fp8`, `mla`). Directly modifies actual Triton kernel files: `vllm/v1/attention/ops/triton_decode_attention.py` (the `@triton.jit`-decorated MLA decode kernel doing `tl.dot(q, k)` for attention scores, `tl.dot(qpe, kpe)` for positional contributions, and `acc += tl.dot(p, v)` for the output projection) and `vllm/v1/attention/backends/mla/triton_mla.py` (the backend wrapping it). The Triton kernel itself is shipped verbatim under [`artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/ops/triton_decode_attention.py`](../../artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/ops/triton_decode_attention.py) (756 lines, pinned at upstream SHA `a1257fd1`). This PR page is post-refresh — it is NOT in `data/refresh-cutoff.yaml::previous_pages_manifest`. This is the \"downstream adoption is happening\" half of AC-1.2 and the AC-1.1 \"new tracked-repo PR page demonstrating a kernel\" anchor.\n\nTogether the two anchors establish that **the 3.6+ Blackwell lowering infrastructure is real AND tracked downstream repos are landing real Triton matmul kernels for Blackwell production decode paths today** — which is the substance of the rewrite: the pre-3.6 \"lacked-tcgen05 / lacked-TMEM\" framing is no longer correct.\n\n- The anchors do not prove that EVERY plain `tl.dot` kernel on SM100 emits `tcgen05.mma` PTX. They prove that real Triton kernels with `tl.dot` matmul are landing on Blackwell-only paths in tracked downstream repos (per `pr-vllm-34597`'s `triton_decode_attention.py` kernel) and that the 3.6 release added the infrastructure those kernels lower through. Whether the lowering automatically targets `tcgen05.mma` for every shape and dtype on every Blackwell SKU is a separate question.\n- The anchors do not include explicit inspectable `tcgen05.mma` PTX dumps from a tracked-downstream merged PR. Such proof would be an even stronger anchor than what we have today; until one is found, the verified claim should be read as \"Triton 3.6+ Blackwell is real and downstream-adopted\", not \"every Triton matmul on Blackwell is now optimal\".\n\n| [`artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/ops/triton_decode_attention.py`](../../artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/ops/triton_decode_attention.py) | **Post-refresh AC-1.1 primary anchor**: Triton MLA decode attention kernel — `@triton.jit`-decorated, doing `tl.dot(q, k)` / `tl.dot(qpe, kpe)` / `acc += tl.dot(p, v)` matmul on `[sm100]` with FP8 KV cache support added (`pr-vllm-34597`, merged 2026-02-16). | vllm#34597 |\n| [`artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/backends/mla/triton_mla.py`](../../artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/backends/mla/triton_mla.py) | MLA backend wrapping the Triton decode kernel above — adds FP8 / FP8-E4M3 to `TritonMLABackend.supported_kv_cache_dtypes`. | vllm#34597 |\n\nThe current AC-1.1 **post-refresh primary upstream-code anchor** is `pr-vllm-34597` (above), with the actual Triton decode-attention kernel shipped verbatim. Supplementary post-refresh anchor: `pr-vllm-29339` ([`sources/prs/vllm/PR-29339.md`](../../sources/prs/vllm/PR-29339.md)), a vLLM bugfix scoping the Triton 3.6 `triton_kernels` library to `[sm100, sm90]` for the MXFP4 quantization path (no artifact bundle because the change is dispatch-gate-only)."}, "after": {"statement": "- [`pr-vllm-34597`](../../sources/prs/vllm/PR-34597.md), pinned at `a1257fd1`, adds FP8 KV-cache handling to the Triton MLA decode backend. Its verbatim kernel contains `tl.dot`, but no target guard, Triton-version requirement, TCGen5/TMEM name, or emitted PTX. The primary PR specifically motivates the backend as the MLA option on SM120; it is not an SM100-only lowering demonstration.\n\n| [`triton_decode_attention.py`](../../artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/ops/triton_decode_attention.py) | Triton MLA decode kernels with `tl.dot`; PR 34597 adds FP8 cache handling. |\n| [`triton_mla.py`](../../artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/backends/mla/triton_mla.py) | Backend wrapper and supported FP8 cache dtypes. |\n\nThis verbatim excerpt from the pinned vLLM decode kernel demonstrates the page's deliberately limited downstream claim—FP8 rescaling followed by `tl.dot`, without proving a particular emitted MMA instruction:\n\n```python\nif k.dtype.is_fp8():\n k = (k.to(tl.float32) * ks).to(q.dtype)\nqk = tl.dot(q, k.to(q.dtype))\nif BLOCK_DPE > 0:\n offs_buf_kpe = kv_loc[None, :] * stride_buf_kbs\n```"}, "reason": {"statement": "Retain the genuine FP8 Triton MLA kernel example while removing architecture and lowering inferences.", "urls": ["https://github.com/vllm-project/vllm/pull/34597", "https://github.com/vllm-project/vllm/blob/a1257fd1/vllm/v1/attention/ops/triton_decode_attention.py"]}} +{"path": "wiki/languages/triton-blackwell.md", "before": {"statement": "The following table reflects benchmark snapshots from when the original page was written (Triton 3.5 era) and is preserved for historical reference. It does NOT reflect the 3.6+ tcgen05 path.\n\nA re-run on Triton 3.6 with FlashInfer-Bench is not yet available locally; future refresh rounds should update this table or remove it in favor of a pointer to the live leaderboard."}, "after": {"statement": "The live [FlashInfer-Bench leaderboard](https://bench.flashinfer.ai/), retrieved 2026-08-08, reports these author rows across 660 workloads each:\n\n| Gemini 2.5 Pro | 0.628x | 73.1% |\n| GPT-5 | 0.467x | 92.3% |\n| Claude Opus 4.1 | 0.456x | 73.1% |\n\nThe leaderboard does not attach those rows to a Triton release or identify them as a Triton-only language subset."}, "reason": {"statement": "Preserve the exact live leaderboard values while deleting the false historical/version interpretation.", "urls": ["https://bench.flashinfer.ai/"]}} +{"path": "wiki/languages/triton-blackwell.md", "before": {"statement": "```python\n@triton.jit\ndef gated_delta_net_decode(\n Q, K, V, Gate, State, Output,\n qk_dim: tl.constexpr, v_dim: tl.constexpr, d: tl.constexpr,\n):\n \"\"\"Single-token decode: O(d^2) per token.\"\"\"\n head_id = tl.program_id(0)\n # Load recurrent state S: [qk_dim*d, v_dim*d]\n s = tl.load(State + head_id * qk_dim * d * v_dim * d + offsets)\n q = tl.load(Q + offsets)\n k = tl.load(K + offsets)\n v = tl.load(V + offsets)\n g = tl.load(Gate + head_id)\n\n # Delta rule: S = g*S + k @ (v - S^T @ k)^T\n sk = tl.dot(tl.trans(s), k)\n delta_v = v - sk\n s = g * s + tl.dot(k[:, None], delta_v[None, :])\n o = tl.dot(tl.trans(s), q)\n\n tl.store(State + offsets, s)\n tl.store(Output + offsets, o)\n```"}, "after": {"statement": null}, "reason": {"statement": "The repository already provides provenance-pinned real GDN-related Triton code; retaining broken pseudocode has no teaching value.", "urls": []}} +{"path": "wiki/languages/triton-blackwell.md", "before": {"statement": "```python\n@triton.jit\ndef sparse_attention_fwd(Q, K, V, Output, TopK_Indices,\n block_size: tl.constexpr, topk: tl.constexpr):\n \"\"\"Attend to top-k sparse token blocks only.\"\"\"\n qid = tl.program_id(0)\n q = tl.load(Q + qid * d + tl.arange(0, d))\n acc = tl.zeros([d], dtype=tl.float32)\n for i in range(topk):\n bidx = tl.load(TopK_Indices + qid * topk + i)\n k_block = tl.load(K + bidx * block_size * d + offsets)\n v_block = tl.load(V + bidx * block_size * d + offsets)\n scores = tl.dot(q[None, :], tl.trans(k_block))\n # softmax + accumulate...\n```"}, "after": {"statement": null}, "reason": {"statement": "Replace the incomplete executable-looking block with links to complete provenance-pinned attention kernels.", "urls": []}} +{"path": "wiki/languages/triton-blackwell.md", "before": {"statement": "The full 42-PR universe is enumerated in `data/triton-universe.yaml`. Entries marked `captured: false` do not ship locally because they fall outside the three in-policy sub-scopes (see the policy file for reasons)."}, "after": {"statement": "The complete tracked PR universe and its captured/skipped flags are recorded in [`data/triton-universe.yaml`](../../data/triton-universe.yaml); this page does not duplicate that changing count."}, "reason": {"statement": "Remove the stale count and retain a stable pointer to the canonical ledger and capture flags.", "urls": []}} +{"path": "wiki/languages/triton-blackwell.md", "before": {"statement": "- **What is verified**: the Triton 3.6 release adds tcgen05 + TMEM lowering infrastructure (per `doc-triton-3.6-blackwell`), and tracked downstream repos are landing real Triton kernel changes for SM100 post-3.6 (per `pr-vllm-34597`, the post-refresh primary anchor — vLLM's \"[Kernel] Add FP8 KV cache support to Triton MLA decode attention\" PR, which directly modifies `@triton.jit`-decorated kernels in `vllm/v1/attention/ops/triton_decode_attention.py` doing `tl.dot(q, k)` / `tl.dot(qpe, kpe)` / `tl.dot(p, v)` matmul on `[sm100]`, shipped verbatim under `artifacts/prs/vllm/PR-34597/`). Supplementary post-refresh evidence: `pr-vllm-29339` shows the Triton 3.6 `triton_kernels` library being explicitly scoped to Blackwell production codepaths. Pre-refresh historical evidence: `pr-sglang-22079` (the Gemma4 NVFP4 attention kernel merged 2026-04-03). Together these establish that Triton 3.6+ on Blackwell is materially different from the pre-3.6 \"lacked-tcgen05 / lacked-TMEM\" story.\n\n- **`pr-vllm-34597`** (`source_category: upstream-code`) — **post-refresh primary anchor**: vLLM's \"[Kernel] Add FP8 KV cache support to Triton MLA decode attention\" PR (merged 2026-02-16, post-Triton-3.6.0 release; `architectures: [sm100]`; tags `attention`, `decode`, `fp8`, `mla`). Directly modifies actual Triton kernel files: `vllm/v1/attention/ops/triton_decode_attention.py` (the `@triton.jit`-decorated MLA decode kernel doing `tl.dot(q, k)` for attention scores, `tl.dot(qpe, kpe)` for positional contributions, and `acc += tl.dot(p, v)` for the output projection) and `vllm/v1/attention/backends/mla/triton_mla.py` (the backend wrapping it). The Triton kernel itself is shipped verbatim under [`artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/ops/triton_decode_attention.py`](../../artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/ops/triton_decode_attention.py) (756 lines, pinned at upstream SHA `a1257fd1`). This PR page is post-refresh — it is NOT in `data/refresh-cutoff.yaml::previous_pages_manifest`. This is the \"downstream adoption is happening\" half of AC-1.2 and the AC-1.1 \"new tracked-repo PR page demonstrating a kernel\" anchor.\n\nSupplementary post-refresh anchor: [`pr-vllm-29339`](../../sources/prs/vllm/PR-29339.md) — vLLM bugfix that scopes the upstream `triton_kernels` library (the `triton-lang/triton/python/triton_kernels` collection shipped with Triton 3.6) to `[sm100, sm90]` for the MXFP4 quantization path. Modifies only a dispatch gate (`vllm/model_executor/layers/quantization/mxfp4.py`), which is why `pr-vllm-34597` is the primary anchor and `pr-vllm-29339` is supplementary.\n\nPre-refresh historical anchors (retained as supplementary context, not as AC-1.1 \"new tracked-repo PR page\" evidence on their own):\n\n- [`pr-sglang-22079`](../../sources/prs/sglang/PR-22079.md) — Gemma4 NVFP4 SGLang `extend_attention` Triton kernel doing actual `tl.dot(q,k)` / `tl.dot(p,v)` matmul on `[sm100, sm90]`, merged 2026-04-03. Strongest in-corpus example of a real `tl.dot` Triton matmul landing for SM100 post-3.6, but pre-refresh per `previous_pages_manifest`.\n- [`pr-sglang-21019`](../../sources/prs/sglang/PR-21019.md) — Qwen3.5 GDN projection fused split/reshape/cat kernel merged 2026-03-20. `tl.load`/`tl.store` only (memory rearrangement, no `tl.dot`); demonstrates \"Triton on SM100 post-3.6\" but not the matmul lowering path.\n\nThe current AC-1.1 **post-refresh primary upstream-code anchor** is `pr-vllm-34597` (above), with the actual Triton decode-attention kernel shipped verbatim. Supplementary post-refresh anchor: `pr-vllm-29339` ([`sources/prs/vllm/PR-29339.md`](../../sources/prs/vllm/PR-29339.md)), a vLLM bugfix scoping the Triton 3.6 `triton_kernels` library to `[sm100, sm90]` for the MXFP4 quantization path (no artifact bundle because the change is dispatch-gate-only)."}, "after": {"statement": null}, "reason": {"statement": "Refresh-round administration is stale and irrelevant to the durable technical claim.", "urls": []}} +{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "`tcgen05.mma` is the Blackwell (SM100/SM100a) matrix-multiply-accumulate instruction that replaces Hopper's `wgmma.mma_async`. The name stands for **Tensor Core Generation 05**. NVIDIA also refers to the higher-level abstraction as **UMMA** (Unified Matrix Multiply-Accumulate) in the CUTLASS framework."}, "after": {"statement": null}, "reason": {"statement": "Retain the official tcgen05 name and CUTLASS namespace without inventing an acronym expansion.", "urls": []}} +{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "1. **No warpgroup synchronization overhead** -- one elected thread (typically lane 0 of warp 0) issues the MMA."}, "after": {"statement": "| Completion | WGMMA commit/wait groups | `tcgen05.commit` plus an mbarrier wait |\n\nSingle-thread issue reduces the number of issuing threads, but it does not remove asynchronous completion, operand-lifetime, or inter-thread ordering requirements.\n\n`tcgen05.mma` is asynchronous. Two mechanisms serve different purposes:\n\n1. `tcgen05.commit.cta_group::N.mbarrier::arrive::one.b64` makes an mbarrier track completion of prior asynchronous tcgen05 operations issued by the executing thread. Waiting on that mbarrier observes completion.\n2. `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` order tcgen05 operations across an execution-ordering handoff and constrain code motion. A fence is not a completion wait.\n\nA cross-thread result handoff therefore needs both the applicable completion protocol and the applicable fence/execution-ordering protocol. Likewise, a pipelined mainloop must not release or overwrite an SMEM stage until the asynchronous MMA has finished consuming it."}, "reason": {"statement": "Separate issue granularity from synchronization and completion requirements.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence"]}} +{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "| Operand A source | Registers or SMEM | Shared memory only |\n\n| Matrix load | ldmatrix to registers | Direct from SMEM (no ldmatrix) |"}, "after": {"statement": "The programming-model shift from Hopper WGMMA is precise:\n\n| Issue granularity | Warpgroup | One thread for `cta_group::1` or `cta_group::2` |\n| D accumulator | Per-thread registers | Tensor Memory (TMEM) |\n| A location | Register or SMEM forms, depending on instruction | SMEM descriptor or TMEM address |\n| B location | SMEM descriptor | SMEM descriptor |"}, "reason": {"statement": "Restore the actual operand-location alternatives and avoid a mandatory ldmatrix claim.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-matrix-multiply-and-accumulate-operations"]}} +{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "| Synchronization | Warpgroup-scoped barriers | Fully async, fence-based |\n\n3. **Fence-based completion** -- the producer must insert explicit fences before reading results from TMEM.\n\nFences are **mandatory** for correctness. The hardware does not implicitly synchronize between MMA and TMEM reads/writes.\n\nInsert before reading MMA results from TMEM:\n\n```cuda\n// Fence before CTA sync and TMEM reads after the completion mechanism.\n__device__ void fence_before_tmem_read() {\n asm volatile(\"tcgen05.fence::before_thread_sync;\");\n __syncthreads(); // Ensure all threads see the fence\n}\n```\n\n```ptx\n// Fence before reading TMEM (most common)\ntcgen05.fence::before_thread_sync;\n\n// Fence after CTA sync before issuing dependent tcgen05 operations\ntcgen05.fence::after_thread_sync;\n```"}, "after": {"statement": "`tcgen05.mma` is asynchronous. Two mechanisms serve different purposes:\n\n1. `tcgen05.commit.cta_group::N.mbarrier::arrive::one.b64` makes an mbarrier track completion of prior asynchronous tcgen05 operations issued by the executing thread. Waiting on that mbarrier observes completion.\n2. `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` order tcgen05 operations across an execution-ordering handoff and constrain code motion. A fence is not a completion wait.\n\nA cross-thread result handoff therefore needs both the applicable completion protocol and the applicable fence/execution-ordering protocol. Likewise, a pipelined mainloop must not release or overwrite an SMEM stage until the asynchronous MMA has finished consuming it."}, "reason": {"statement": "Teach completion and ordering as separate protocols.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"]}} +{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "| `tcgen05.mma.kind::f16` | FP16/BF16 | FP16/BF16 | FP32 | None | m128n256k16 | Standard half-precision |\n\n| `tcgen05.mma.kind::f8f6f4` | FP8/FP6/FP4 | FP8/FP6/FP4 | FP32 | Block (UE8M0) | m128n256k32 | Narrow precision with native block scaling |\n\n| `tcgen05.mma.kind::mxf8` | MXFP8 | MXFP8 | FP32 | MX (E8M0) | m128n256k32 | Microscaling FP8 |\n| `tcgen05.mma.kind::mxf4` | MXFP4 | MXFP4 | FP32 | MX (E8M0) | m128n256k64 | Microscaling FP4 |\n| `tcgen05.mma.kind::mxf4nvf4` | NVFP4 | MXFP4 | FP32 | Mixed | m128n256k64 | Mixed NVFP4/MXFP4 |"}, "after": {"statement": "PTX ISA 9.0 divides dense `tcgen05.mma` into these grammar groups:\n\n| Floating point, without block scaling | `f16`, `tf32`, `f8f6f4` |\n| Floating point, with block scaling | `mxf8f6f4`, `mxf4`, `mxf4nvf4` |\n| Integer | `i8` |\n\nBlock-scaled forms add `.block_scale` and accept separate TMEM addresses for A and B scale factors. `f8f6f4` is not the block-scaled MX kind, and `mxf8` is not a dense kind token in this grammar."}, "reason": {"statement": "Replace the mixed taxonomy with the exact PTX grammar groups and avoid oversimplified type claims.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "- **BF16/FP16**: M=128, N=256, K=16\n- **TF32**: M=128, N=256, K=8\n- **FP8/FP6/FP4**: M=128, N=256, K=32\n- **MXFP4/NVFP4**: M=128, N=256, K=64\n\n- **BF16/FP16**: M=256, N=256, K=16\n- **TF32**: M=256, N=256, K=8\n- **FP8/FP6/FP4**: M=256, N=256, K=32\n- **MXFP4/NVFP4**: M=256, N=256, K=64\n\nThe M dimension doubles because each SM contributes 128 rows from its own TMEM partition."}, "after": {"statement": "M and N are encoded in the instruction descriptor (`idesc`) and are constrained by kind, CTA group, layouts, and target ISA. Consequently, names such as m128n256k16 and m256n256k16 are useful maximum-shape examples for F16/BF16 configurations, not the only legal M and N values. `cta_group::1` uses current-CTA resources; `cta_group::2` can also access the paired peer CTA's SMEM and TMEM resources. All `tcgen05` instructions in a kernel must use the same CTA-group value."}, "reason": {"statement": "State group ownership separately from descriptor-selected shapes and label maxima as maxima.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instruction-descriptor"]}} +{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "```cuda\n// Single-thread MMA issuance pattern\n__device__ void issue_mma(uint32_t tmem_addr, uint64_t smem_desc_a, uint64_t smem_desc_b) {\n // Only one thread issues the MMA\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 0;\"\n :\n : \"r\"(tmem_addr),\n \"l\"(smem_desc_a),\n \"l\"(smem_desc_b),\n \"r\"(0) // scale descriptor (unused for f16)\n );\n }\n}\n```\n\n```ptx\n// Issue a 128x256x16 BF16 MMA\n// Operand A: shared memory descriptor\n// Operand B: shared memory descriptor\n// Accumulator: TMEM address\ntcgen05.mma.cta_group::1.kind::f16 [tmem_addr], desc_a, desc_b, idesc, 0;\n```\n\n```ptx\n// Issue a 256x256x16 BF16 MMA across two paired CTAs\n// cta_group::2 indicates cooperative mode\ntcgen05.mma.cta_group::2.kind::f16 [tmem_addr], desc_a, desc_b, idesc, 0;\n```\n\n```ptx\n// FP8 with native UE8M0 block scaling\n// scale_desc encodes the per-block scale factors\ntcgen05.mma.cta_group::1.kind::f8f6f4 [tmem_addr], desc_a, desc_b, idesc, scale_desc;\n```\n\n```cuda\n__device__ void mma_bf16_128x256x16(\n uint32_t tmem_addr,\n uint64_t desc_a,\n uint64_t desc_b\n) {\n if (threadIdx.x == 0) {\n // First MMA: zero-initialize accumulator\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 0;\"\n :\n : \"r\"(tmem_addr), \"l\"(desc_a), \"l\"(desc_b), \"r\"(0)\n );\n }\n}\n\n__device__ void mma_bf16_accumulate(\n uint32_t tmem_addr,\n uint64_t desc_a,\n uint64_t desc_b\n) {\n if (threadIdx.x == 0) {\n // Subsequent MMAs: accumulate into existing TMEM\n // The enable_accumulate flag controls whether to add to or overwrite TMEM\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 1;\" // last arg 1 = accumulate\n :\n : \"r\"(tmem_addr), \"l\"(desc_a), \"l\"(desc_b), \"r\"(0)\n );\n }\n}\n```"}, "after": {"statement": null}, "reason": {"statement": "Remove invalid executable-looking examples and retain the normative grammar with a vendor example link.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "```cuda\n__device__ void gemm_mainloop(/* params */) {\n for (int k_tile = 0; k_tile < num_k_tiles; ++k_tile) {\n // 1. Wait for operand data to arrive in SMEM\n wait_barrier(k_tile % NUM_STAGES);\n\n // 2. Issue MMA (single thread)\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 1;\"\n :\n : \"r\"(tmem_addr),\n \"l\"(make_smem_desc(smem_a, k_tile)),\n \"l\"(make_smem_desc(smem_b, k_tile)),\n \"r\"(0)\n );\n }\n\n // 3. Release SMEM buffer for next TMA load\n if (threadIdx.x == 0) {\n arrive_barrier((k_tile + 1) % NUM_STAGES);\n }\n }\n\n // 4. CRITICAL: Fence before reading accumulator from TMEM\n asm volatile(\"tcgen05.fence::before_thread_sync;\");\n __syncthreads();\n\n // 5. Now safe to read results from TMEM\n read_tmem_accumulator(tmem_addr, output);\n}\n```"}, "after": {"statement": null}, "reason": {"statement": "A partial pipeline with unsafe lifetime rules is more misleading than the normative sequence.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"]}} +{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "```cuda\n// ---- HOPPER (SM90): wgmma ----\n// Requires warpgroup-scoped execution\n// All 128 threads in a warpgroup participate\n__device__ void hopper_mma() {\n // Load A matrix into registers via ldmatrix\n uint32_t a_frag[4];\n asm volatile(\"ldmatrix.sync.aligned.m8n8.x4.shared.b16 \"\n \"{%0,%1,%2,%3}, [%4];\"\n : \"=r\"(a_frag[0]), \"=r\"(a_frag[1]),\n \"=r\"(a_frag[2]), \"=r\"(a_frag[3])\n : \"r\"(smem_addr));\n\n // Issue wgmma -- warpgroup scope, register accumulator\n asm volatile(\"wgmma.mma_async.sync.aligned.m64n256k16.f32.bf16.bf16 \"\n \"{%0, %1, ...}, \" // register accumulators (128+ registers!)\n \"{%N, %N+1, ...}, \" // A operand in registers\n \"desc_b, ...;\"\n : \"+f\"(acc[0]), \"+f\"(acc[1]), ...\n : ...);\n\n // Commit and wait\n asm volatile(\"wgmma.commit_group.sync.aligned;\");\n asm volatile(\"wgmma.wait_group.sync.aligned 0;\");\n // Accumulators now in registers -- high register pressure\n}\n\n// ---- BLACKWELL (SM100): tcgen05 ----\n// Single-thread issuance, TMEM accumulator\n__device__ void blackwell_mma() {\n // No ldmatrix needed -- reads directly from SMEM\n // No register allocation for accumulators\n\n // Single thread issues MMA\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 1;\"\n :\n : \"r\"(tmem_addr), // accumulator in TMEM (not registers!)\n \"l\"(desc_a), // A from SMEM descriptor\n \"l\"(desc_b), // B from SMEM descriptor\n \"r\"(0)\n );\n }\n\n // Fence + sync before reading results\n asm volatile(\"tcgen05.fence::before_thread_sync;\");\n __syncthreads();\n // Read from TMEM -- registers free for other work\n}\n```"}, "after": {"statement": null}, "reason": {"statement": "Replace executable-looking pseudo-assembly with a precise semantic comparison table.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "tcgen05.mma requires **128-byte swizzled** shared memory layouts for both operands. Non-swizzled or 64-byte swizzled layouts will produce incorrect results silently."}, "after": {"statement": "The tcgen05 shared-memory descriptor is a 64-bit runtime value. PTX ISA 9.0 assigns fields for the encoded base address, leading dimension, stride dimension, fixed bits, base offset, leading-dimension mode, and a three-bit swizzle mode.\n\nValid swizzle encodings include no swizzle, 128-byte, 64-byte, and 32-byte layouts (plus a 128-byte/32-byte-atomic mode). Values 3, 5, and 7 are invalid; ordinary 128-byte swizzling uses value 2. A layout must satisfy the addressing and alignment constraints for its chosen mode. Although 128-byte swizzling can materially improve a particular GEMM, it is not a universal correctness requirement."}, "reason": {"statement": "Distinguish valid layouts from performance-sensitive layout choice.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor"]}} +{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "```cuda\n// Shared memory descriptor construction for tcgen05\n// The descriptor encodes: base address, stride, swizzle mode, dimensions\n__device__ uint64_t make_smem_desc(void* smem_ptr, int stride_bytes) {\n uint64_t desc = 0;\n uint32_t addr = static_cast(__cvta_generic_to_shared(smem_ptr));\n\n // Encode base address (bits 0-13)\n desc |= (uint64_t)(addr >> 4);\n // Encode leading dimension stride (bits 16-29)\n desc |= (uint64_t)((stride_bytes >> 4) & 0x3FFF) << 16;\n // Encode 128-byte swizzle mode (bits 62-63) -- MANDATORY for tcgen05\n desc |= (uint64_t)(3) << 62; // 3 = 128-byte swizzle\n\n return desc;\n}\n```"}, "after": {"statement": null}, "reason": {"statement": "Direct users to the complete table rather than preserving a subtly corrupt descriptor builder.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor"]}} +{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "| + Persistent kernel + CLC | 1476 | 98% |"}, "after": {"statement": "Gau Nernst reports the following results for M=N=K=4096 on a Modal B200, using PyTorch 2.9.1 with CUDA 13. Values are measurements for that setup, not architecture-wide guarantees.\n\n| cuBLAS | 1506.74 | 100% |\n| v1a: basic tcgen05 + 2D 16B TMA | 254.62 | 17% |\n| v1b: 3D 16B TMA | 252.81 | 17% |\n| v2a: 2D 128B TMA | 681.20 | 45% |\n| v2b: 3D 128B TMA | 695.43 | 46% |\n| v3: pipelining | 939.61 | 62% |\n| v4: warp specialization | 1208.83 | 80% |\n| v5: 2-SM MMA | 1302.29 | 86% |\n| v6: persistent, static scheduling | 1475.93 | 98% |\n\nThe v6 result did not use Cluster Launch Control; the author lists CLC and threadblock swizzling as unimplemented follow-up ideas."}, "reason": {"statement": "Preserve the measurement while restoring the actual scheduling method.", "urls": ["https://gau-nernst.github.io/tcgen05/"]}} +{"path": "wiki/hardware/2sm-cooperative.md", "before": {"statement": "Blackwell enables two SMs within a TPC to cooperatively execute a single larger MMA, doubling the effective compute tile size to m256×n256×k16."}, "after": {"statement": "PTX defines a CTA pair as two CTAs in the same cluster whose `%cluster_ctarank` values differ only in the low bit. With `cta_group::2`, one thread from either CTA can initiate a whole `tcgen05.mma`; the peer CTA must still be active. The operation accesses Tensor Memory belonging to both the current CTA and its peer.\n\nAllocation management differs from MMA issue. A `tcgen05.alloc` or `tcgen05.dealloc` for `cta_group::2` is issued collectively by two warps, one in each CTA. All tcgen05 instructions in the kernel must use the same CTA-group value.\n\n`cta_group::2` selects pair-level resources; it does not by itself select one fixed MxNxK shape. The instruction descriptor (`idesc`) encodes M, N, exact operand and accumulator types, sparsity, and related operation details. Legal values depend on the kind, data-path layout, CTA group, and target ISA.\n\nFor example, m256xn256xk16 is a useful maximum F16/BF16 configuration, but PTX also defines group-2 layouts with other M/N values. Do not infer that every pair-level MMA is created by mechanically doubling a group-1 M dimension."}, "reason": {"statement": "Describe pair-level resource scope separately from shape selection.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-data-path-layout-organization", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-issue-granularity"]}} +{"path": "wiki/hardware/2sm-cooperative.md", "before": {"statement": "```\nTPC (Two Processing Clusters)\n├── SM 0: CTA 0 — issues tcgen05.mma with cta_group::2\n│ ├── Shared Memory A (rows 0-127)\n│ └── TMEM (columns 0-255)\n└── SM 1: CTA 1 — cooperates on same MMA\n ├── Shared Memory A (rows 128-255)\n └── TMEM (columns 256-511)\n```"}, "after": {"statement": null}, "reason": {"statement": "Replace the misleading topology diagram with normative pair and issue rules.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-data-path-layout-organization"]}} +{"path": "wiki/hardware/2sm-cooperative.md", "before": {"statement": "```ptx\n// 2-SM cooperative MMA\ntcgen05.mma.cta_group::2.kind::f16\n [tmem_addr], descA, descB, idescC, idescD, ...;\n```"}, "after": {"statement": null}, "reason": {"statement": "Link to the already verified normative grammar on the tcgen05 page instead of duplicating invalid code.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/hardware/2sm-cooperative.md", "before": {"statement": "1. **Identical shared memory layouts** across both CTAs\n2. `shared::cluster` mbarrier signaling between the two CTAs\n\n4. Each CTA contributes half the M-dimension"}, "after": {"statement": "PTX defines a CTA pair as two CTAs in the same cluster whose `%cluster_ctarank` values differ only in the low bit. With `cta_group::2`, one thread from either CTA can initiate a whole `tcgen05.mma`; the peer CTA must still be active. The operation accesses Tensor Memory belonging to both the current CTA and its peer.\n\nAllocation management differs from MMA issue. A `tcgen05.alloc` or `tcgen05.dealloc` for `cta_group::2` is issued collectively by two warps, one in each CTA. All tcgen05 instructions in the kernel must use the same CTA-group value.\n\nThe complete operand grammar, including the eight-register disable-output-lane vector used by `cta_group::2`, is documented on the related [tcgen05.mma page](tcgen05-mma.md)."}, "reason": {"statement": "Replace the invented checklist with direct ISA invariants.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-issue-granularity", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/hardware/2sm-cooperative.md", "before": {"statement": "- Large GEMM problems where M ≥ 256\n- Compute-bound kernels where peak FLOPS matters\n- Combined with persistent scheduling for maximum throughput"}, "after": {"statement": "In Gau Nernst's M=N=K=4096 experiment on a Modal B200 with PyTorch 2.9.1 and CUDA 13, v4 warp specialization reports 1208.83 TFLOP/s and v5 2-SM MMA reports 1302.29 TFLOP/s. Relative to the same 1506.74-TFLOP/s cuBLAS result, those are approximately 80.2% and 86.4%, or a 7.7% relative increase from v4 to v5.\n\nThat result establishes a gain for one kernel and setup, not a universal threshold. Choose between group 1 and group 2 by benchmarking the target shapes and accounting for CTA pairing, data reuse, layout, occupancy, pipeline stages, and epilogue cost. Persistence is an independent scheduling choice, not a condition that guarantees maximum throughput."}, "reason": {"statement": "Replace categorical advice with a measurement-and-resource tradeoff checklist.", "urls": ["https://gau-nernst.github.io/tcgen05/"]}} +{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "- **Operand loading**: ldmatrix + registers vs direct SMEM access\n- **Synchronization**: warpgroup barriers vs async fences"}, "after": {"statement": "Port the programming model, not just the opcode:\n\n| MMA issue | Warpgroup collective | One thread for group-1 or group-2 MMA |\n| D accumulator | Per-thread registers | TMEM |\n| A operand | SMEM-descriptor or register forms, depending on instruction | SMEM-descriptor or TMEM-address forms |\n| B operand | SMEM descriptor | SMEM descriptor |\n| MMA completion | Commit/wait groups | `tcgen05.commit` plus mbarrier wait |\n| Cross-thread ordering | WGMMA-specific rules | tcgen05 fences composed with an execution-ordering operation |\n| Narrow floating formats | FP8 WGMMA forms | FP8/FP6/FP4 plus block-scaled MX/NVFP4 kinds |\n\nThe exact old WGMMA form matters. Do not assume every Hopper kernel loads A through `ldmatrix`: descriptor-sourced WGMMA forms already read A from SMEM. Likewise, tcgen05 can source A from SMEM or TMEM.\n\n`tcgen05.mma` is asynchronous. `tcgen05.commit.cta_group::N.mbarrier::arrive::one.b64` makes an mbarrier track prior MMA work issued by the current thread; waiting on that barrier observes completion. A `tcgen05.fence` is an ordering and code-motion primitive, not a completion wait, so fence plus `__syncthreads()` is not a substitute for commit/mbarrier."}, "reason": {"statement": "Make migration conditional on the exact old and new instruction forms and separate completion from ordering.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-matrix-multiply-and-accumulate-operations", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"]}} +{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "```\n[ ] Replace wgmma.mma_async with tcgen05.mma\n[ ] Move accumulators from registers to TMEM (alloc/dealloc)\n[ ] Remove ldmatrix operations (tcgen05 reads directly from SMEM)\n[ ] Change SMEM swizzle from 64B to 128B\n[ ] Replace warpgroup commit/wait with tcgen05 fences\n[ ] Update warp specialization roles (fewer warps needed for MMA)\n[ ] Update tile sizes (128xN -> consider 256xN with 2-SM)\n[ ] Add TMEM lifecycle management (alloc at start, dealloc at end)\n[ ] Update epilogue to read from TMEM instead of registers\n```"}, "after": {"statement": "1. Identify the exact WGMMA operand form, accumulator type, shape, and group-completion points in the SM90 kernel.\n2. Choose a legal tcgen05 kind, A source, CTA group, instruction descriptor, and data-path layout for the SM100 target.\n3. Replace register-resident D with a TMEM allocation. Group-1 allocation/deallocation is warp-collective; group 2 requires one warp in each CTA of the pair.\n4. Keep A/B backing storage unchanged until all asynchronous MMA consumers have completed.\n5. Replace WGMMA group completion with `tcgen05.commit` and an mbarrier wait. Add tcgen05 fences only where an execution-ordering handoff must order tcgen05-visible state across threads or CTAs.\n6. Transfer completed accumulator values from TMEM to registers with `tcgen05.ld`, observe its completion/access rules, then run the epilogue.\n7. Deallocate every dynamic TMEM allocation before kernel exit. Use the same `cta_group` value for all tcgen05 instructions in the kernel.\n8. Retune CTA/cluster shapes, pipeline stages, thread roles, descriptors, and epilogue scheduling on the target workload."}, "reason": {"statement": "Replace categorical substitutions with a dependency-ordered audit checklist.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"]}} +{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "```cuda\n// SM90 GEMM kernel using wgmma\n__global__ void hopper_gemm(\n const half* A, const half* B, half* C,\n int M, int N, int K\n) {\n extern __shared__ char smem[];\n half* smem_a = reinterpret_cast(smem);\n half* smem_b = smem_a + TILE_M * TILE_K;\n\n // Register accumulators -- HIGH REGISTER PRESSURE\n float acc[MMA_M_FRAGS][MMA_N_FRAGS]; // 128+ registers!\n for (int i = 0; i < MMA_M_FRAGS; ++i)\n for (int j = 0; j < MMA_N_FRAGS; ++j)\n acc[i][j] = 0.0f;\n\n // Mainloop\n for (int k = 0; k < K / TILE_K; ++k) {\n // Load A to SMEM (TMA or cp.async)\n tma_load(smem_a, A, tile_m, k);\n tma_load(smem_b, B, k, tile_n);\n\n // Load A fragment from SMEM to registers via ldmatrix\n uint32_t a_frag[4];\n asm volatile(\n \"ldmatrix.sync.aligned.m8n8.x4.shared.b16 \"\n \"{%0,%1,%2,%3}, [%4];\"\n : \"=r\"(a_frag[0]), \"=r\"(a_frag[1]),\n \"=r\"(a_frag[2]), \"=r\"(a_frag[3])\n : \"r\"(smem_a_addr)\n );\n\n // Issue wgmma -- ALL 128 THREADS IN WARPGROUP participate\n asm volatile(\n \"wgmma.mma_async.sync.aligned.m64n256k16.f32.bf16.bf16 \"\n \"{%0, %1, %2, %3, %4, %5, %6, %7, ...}, \" // acc registers\n \"{%N, %N+1, %N+2, %N+3}, \" // A in registers\n \"desc_b, ...\" // B descriptor\n : \"+f\"(acc[0][0]), \"+f\"(acc[0][1]), ...\n : \"r\"(a_frag[0]), ...\n );\n\n // Commit and wait for warpgroup\n asm volatile(\"wgmma.commit_group.sync.aligned;\");\n asm volatile(\"wgmma.wait_group.sync.aligned 0;\");\n }\n\n // Epilogue: accumulators already in registers\n // Write directly to GMEM (or through SMEM for vectorized stores)\n for (int i = 0; i < MMA_M_FRAGS; ++i)\n for (int j = 0; j < MMA_N_FRAGS; ++j)\n C[row + i][col + j] = (half)acc[i][j];\n}\n```"}, "after": {"statement": null}, "reason": {"statement": "Use an exact semantic mapping and pinned CUTLASS examples rather than fabricated assembly.", "urls": []}} +{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "```cuda\n// SM100 GEMM kernel using tcgen05\n__global__ void blackwell_gemm(\n const half* A, const half* B, half* C,\n int M, int N, int K\n) {\n extern __shared__ char smem[];\n half* smem_a = reinterpret_cast(smem);\n half* smem_b = smem_a + TILE_M * TILE_K;\n\n // TMEM accumulator -- NO REGISTER PRESSURE\n __shared__ uint32_t s_tmem_acc;\n if (threadIdx.x == 0) {\n uint32_t smem_addr =\n static_cast(__cvta_generic_to_shared(&s_tmem_acc));\n asm volatile(\n \"tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;\"\n :: \"r\"(smem_addr), \"r\"(256) // 256 columns for 128x256 tile\n );\n }\n __syncthreads();\n uint32_t tmem_acc = s_tmem_acc;\n\n // Zero TMEM accumulator\n tmem_zero(tmem_acc, 256);\n\n // Mainloop\n for (int k = 0; k < K / TILE_K; ++k) {\n // TMA load with 128B swizzle (mandatory for tcgen05)\n tma_load_128b_swizzle(smem_a, A, tile_m, k);\n tma_load_128b_swizzle(smem_b, B, k, tile_n);\n wait_barrier();\n\n // NO ldmatrix -- tcgen05 reads directly from SMEM\n // SINGLE THREAD issues MMA (not warpgroup)\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 1;\"\n :\n : \"r\"(tmem_acc),\n \"l\"(make_smem_desc_128b(smem_a)),\n \"l\"(make_smem_desc_128b(smem_b)),\n \"r\"(0)\n );\n }\n // NO commit/wait -- fully async, fence-based\n }\n\n // Fence before reading TMEM (replaces wgmma.wait_group)\n asm volatile(\"tcgen05.fence::before_thread_sync;\");\n __syncthreads();\n\n // Epilogue: read from TMEM, then write to GMEM\n float4 acc_vals = tmem_load_f32x4(tmem_acc + col_offset);\n // Apply bias, activation, etc.\n store_output(acc_vals, C, row, col);\n\n // Deallocate TMEM (no equivalent needed on Hopper)\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;\"\n :\n : \"r\"(tmem_acc), \"r\"(256)\n );\n }\n}\n```"}, "after": {"statement": null}, "reason": {"statement": "Remove an unsafe end-to-end example that cannot be repaired without becoming a full kernel.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"]}} +{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "**Before (Hopper):** Load A operand from SMEM to registers.\n\n```cuda\n// Hopper: ldmatrix loads 8x8 matrix fragments into registers\nuint32_t a_frag[4];\nasm volatile(\n \"ldmatrix.sync.aligned.m8n8.x4.shared.b16 \"\n \"{%0,%1,%2,%3}, [%4];\"\n : \"=r\"(a_frag[0]), \"=r\"(a_frag[1]),\n \"=r\"(a_frag[2]), \"=r\"(a_frag[3])\n : \"r\"(smem_addr)\n);\n```\n\n**After (Blackwell):** No equivalent needed. tcgen05 reads A directly from SMEM via descriptor.\n\n```cuda\n// Blackwell: just construct the SMEM descriptor\nuint64_t desc_a = make_smem_desc_128b(smem_a_ptr);\n// Pass desc_a directly to tcgen05.mma -- no register staging\n```"}, "after": {"statement": "Port the programming model, not just the opcode:\n\n| MMA issue | Warpgroup collective | One thread for group-1 or group-2 MMA |\n| D accumulator | Per-thread registers | TMEM |\n| A operand | SMEM-descriptor or register forms, depending on instruction | SMEM-descriptor or TMEM-address forms |\n| B operand | SMEM descriptor | SMEM descriptor |\n| MMA completion | Commit/wait groups | `tcgen05.commit` plus mbarrier wait |\n| Cross-thread ordering | WGMMA-specific rules | tcgen05 fences composed with an execution-ordering operation |\n| Narrow floating formats | FP8 WGMMA forms | FP8/FP6/FP4 plus block-scaled MX/NVFP4 kinds |\n\nThe exact old WGMMA form matters. Do not assume every Hopper kernel loads A through `ldmatrix`: descriptor-sourced WGMMA forms already read A from SMEM. Likewise, tcgen05 can source A from SMEM or TMEM."}, "reason": {"statement": "Make operand migration a form-by-form decision.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-matrix-multiply-and-accumulate-operations"]}} +{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "**After (Blackwell):** 128-byte swizzle is mandatory.\n\n```cuda\n// Blackwell TMA descriptor: MUST use 128B swizzle\nCUtensorMap desc = create_tma_desc(ptr, M, N, tile_m, tile_n,\n CU_TENSOR_MAP_SWIZZLE_128B);\n```"}, "after": {"statement": "`tcgen05.alloc` writes a 32-bit TMEM address to shared memory. Allocation size is expressed in columns, in power-of-two multiples permitted by PTX. It is a synchronous warp instruction: a lane-0-only call is invalid. `tcgen05.dealloc` has the same issue granularity, and PTX requires all allocated TMEM to be deallocated before kernel exit—not only in persistent kernels.\n\nThe MMA issuer and the allocation participants are different concepts. One thread initiates a group-1 MMA, but one full warp collectively allocates or deallocates its TMEM. A group-2 allocation uses one warp from each peer CTA."}, "reason": {"statement": "Make 128B a benchmarkable layout option rather than a correctness mandate.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"]}} +{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "**Before (Hopper):**\n\n```cuda\n// Hopper: all 128 threads in warpgroup issue wgmma\nasm volatile(\n \"wgmma.mma_async.sync.aligned.m64n256k16.f32.bf16.bf16 \"\n \"{%0,%1,...}, {%N,...}, desc_b, ...;\"\n : \"+f\"(acc[0]), \"+f\"(acc[1]), ...\n : \"r\"(a_frag[0]), ...\n);\nasm volatile(\"wgmma.commit_group.sync.aligned;\");\n```\n\n**After (Blackwell):**\n\n```cuda\n// Blackwell: single thread issues tcgen05\nif (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 1;\"\n :\n : \"r\"(tmem_acc), \"l\"(desc_a), \"l\"(desc_b), \"r\"(0)\n );\n}\n// No commit needed -- fully async\n```"}, "after": {"statement": null}, "reason": {"statement": "Replace incomplete assembly with a normative lifecycle sequence.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "**After (Blackwell):**\n\n```cuda\n// Blackwell: fence before reading TMEM\nasm volatile(\"tcgen05.fence::before_thread_sync;\");\n__syncthreads();\n// TMEM accumulators now ready for reading\n```"}, "after": {"statement": "`tcgen05.mma` is asynchronous. `tcgen05.commit.cta_group::N.mbarrier::arrive::one.b64` makes an mbarrier track prior MMA work issued by the current thread; waiting on that barrier observes completion. A `tcgen05.fence` is an ordering and code-motion primitive, not a completion wait, so fence plus `__syncthreads()` is not a substitute for commit/mbarrier."}, "reason": {"statement": "Preserve the WGMMA analogy while mapping it to the correct Blackwell mechanism.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"]}} +{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "```\nWarp 0-3: Warpgroup 0 -- MMA producer (all 128 threads issue wgmma)\nWarp 4-7: Warpgroup 1 -- MMA producer (backup/double-buffer)\nWarp 8-11: Data movement (TMA loads, SMEM management)\nWarp 12: Tile scheduler\n\nTotal: 13+ warps, 416+ threads\n```\n\n```\nWarp 0: MMA producer (single thread issues tcgen05)\nWarp 1-2: TMA data movement (load A, load B, manage barriers)\nWarp 3: Epilogue (read TMEM, apply post-ops, store to GMEM)\nWarp 4: Tile scheduler / CLC management\n\nTotal: 5 warps, 160 threads (can be fewer)\n```"}, "after": {"statement": null}, "reason": {"statement": "Replace invented totals with role-level design guidance.", "urls": []}} +{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "| m64 x n256 x k16 | m128 x n256 x k16 | m256 x n256 x k16 |\n| m64 x n128 x k16 | m128 x n128 x k16 | m256 x n128 x k16 |\n\nBlackwell's base MMA tile is 2x larger in M (128 vs 64). When migrating:\n- CTA tile size of 128x256 maps naturally to a single tcgen05 MMA\n- CTA tile size of 64x256 on Hopper should be doubled to 128x256\n- For 2-SM mode, consider 256x256 tiles"}, "after": {"statement": "Do not mechanically convert every 64-byte swizzle to 128-byte swizzling. The tcgen05 shared-memory descriptor defines valid no-swizzle, 128B, 64B, and 32B modes, subject to mode-specific alignment and layout constraints. The 128B choice can be much faster for a particular tile, but it is not a universal correctness condition.\n\nSimilarly, there is no universal Hopper-to-Blackwell rule that doubles M. WGMMA and tcgen05 each expose multiple shapes; tcgen05 M/N are encoded in `idesc` and constrained by kind, layout, CTA group, and target ISA. Treat m128xn256xk16 and m256xn256xk16 as useful F16/BF16 maximum examples, not a complete migration table."}, "reason": {"statement": "Turn mechanical tile substitution into a resource-aware retuning step.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-data-path-layout-organization", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor"]}} +{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "1. **Forgetting 128B swizzle**: The most common silent-failure bug. wgmma works with 64B swizzle; tcgen05 does not. Results will be numerically wrong but the kernel won't crash.\n\n3. **Synchronization model mismatch**: Replacing `wgmma.wait_group` with `__syncthreads()` alone is insufficient. The `tcgen05.fence::before_thread_sync` must precede the syncthreads.\n\n4. **Over-allocating threads**: On Hopper, you need 128 threads per warpgroup for MMA. Blindly keeping the same thread count on Blackwell wastes resources since only 1 thread issues tcgen05."}, "after": {"statement": "1. Identify the exact WGMMA operand form, accumulator type, shape, and group-completion points in the SM90 kernel.\n2. Choose a legal tcgen05 kind, A source, CTA group, instruction descriptor, and data-path layout for the SM100 target.\n3. Replace register-resident D with a TMEM allocation. Group-1 allocation/deallocation is warp-collective; group 2 requires one warp in each CTA of the pair.\n4. Keep A/B backing storage unchanged until all asynchronous MMA consumers have completed.\n5. Replace WGMMA group completion with `tcgen05.commit` and an mbarrier wait. Add tcgen05 fences only where an execution-ordering handoff must order tcgen05-visible state across threads or CTAs.\n6. Transfer completed accumulator values from TMEM to registers with `tcgen05.ld`, observe its completion/access rules, then run the epilogue.\n7. Deallocate every dynamic TMEM allocation before kernel exit. Use the same `cta_group` value for all tcgen05 instructions in the kernel.\n8. Retune CTA/cluster shapes, pipeline stages, thread roles, descriptors, and epilogue scheduling on the target workload.\n\n`tcgen05.alloc` writes a 32-bit TMEM address to shared memory. Allocation size is expressed in columns, in power-of-two multiples permitted by PTX. It is a synchronous warp instruction: a lane-0-only call is invalid. `tcgen05.dealloc` has the same issue granularity, and PTX requires all allocated TMEM to be deallocated before kernel exit—not only in persistent kernels.\n\nThe MMA issuer and the allocation participants are different concepts. One thread initiates a group-1 MMA, but one full warp collectively allocates or deallocates its TMEM. A group-2 allocation uses one warp from each peer CTA.\n\n`tcgen05.mma` is asynchronous. `tcgen05.commit.cta_group::N.mbarrier::arrive::one.b64` makes an mbarrier track prior MMA work issued by the current thread; waiting on that barrier observes completion. A `tcgen05.fence` is an ordering and code-motion primitive, not a completion wait, so fence plus `__syncthreads()` is not a substitute for commit/mbarrier."}, "reason": {"statement": "Keep only direct, scoped pitfalls with observable failure modes.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"]}} +{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "If using CUTLASS, the migration is largely handled by changing the arch tag and kernel schedule:\n\n```cpp\n// Hopper CUTLASS GEMM\nusing GemmHopper = cutlass::gemm::device::GemmUniversal<\n /* ... */\n cutlass::arch::Sm90,\n /* ... */\n cutlass::gemm::collective::KernelScheduleSm90CpAsyncWarpSpecialized\n>;\n\n// Blackwell CUTLASS GEMM -- change arch + schedule\nusing GemmBlackwell = cutlass::gemm::device::GemmUniversal<\n /* ... */\n cutlass::arch::Sm100, // <-- changed\n /* ... */\n cutlass::gemm::collective::KernelScheduleSm100CpAsyncWarpSpecialized // <-- changed\n>;\n```\n\nCUTLASS handles the internal differences (TMEM allocation, descriptor construction, fence insertion, 128B swizzle) automatically through its `MMA_Atom` and `MMA_Traits` abstractions for SM100."}, "after": {"statement": "Single-thread MMA issue can free instruction-issue capacity for TMA, descriptor preparation, epilogue, reductions, or scheduling. It does not determine the kernel's total thread count: TMEM allocation, TMEM loads, TMA, epilogue, and barriers remain warp- or CTA-level work. Start from a pinned SM100 implementation and measure a role partition rather than copying fixed warp counts from an SM90 kernel.\n\nMoving D out of the GPR file also changes the register budget. Revisit tile size, pipeline depth, and launch bounds, but do not assume that lower accumulator pressure automatically increases occupancy; SMEM, TMEM, barriers, threads, and registers used by other roles can become limiting resources."}, "reason": {"statement": "Point to a pinned, complete SM100 example and enumerate the configuration surfaces that must be reselected.", "urls": []}} +{"path": "wiki/hardware/tmem.md", "before": {"statement": "Tensor Memory (TMEM) is a new addressable memory space introduced in the Blackwell architecture (SM100). Each SM contains **256KB of dedicated TMEM**, used primarily as the accumulator storage for `tcgen05.mma` operations. TMEM eliminates the register pressure that plagued Hopper's wgmma, where accumulators consumed 128+ registers per warpgroup."}, "after": {"statement": "Tensor Memory is an addressable, on-chip memory introduced for fifth-generation Tensor Core operations on Blackwell. Each SM has 256 KiB, organized as 128 lanes by 512 columns of 32-bit cells. `tcgen05.mma` writes its D accumulator to TMEM; depending on the instruction form, A can also reside there.\n\nMoving D out of general-purpose registers changes the resource balance relative to Hopper WGMMA. For example, CUTLASS 4.5.0's SM90 m64n256 FP32 WGMMA wrapper exposes 128 accumulator registers **per participating thread**. TMEM avoids that particular register-resident D fragment, but it does not remove registers needed by operands, control flow, or the epilogue."}, "reason": {"statement": "Keep the verified storage shift but scope register impact to the exact old instruction and per-thread fragment.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cute/arch/mma_sm90_gmma.hpp", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} +{"path": "wiki/hardware/tmem.md", "before": {"statement": "Each thread \"owns\" the TMEM row corresponding to its warp and lane. When reading from TMEM, thread `T` in warp `W` accesses row `W*32 + T%32`."}, "after": {"statement": "A TMEM address (`taddr`) is a 32-bit value:\n\n| 31:16 | TMEM lane |\n| 15:0 | TMEM column |\n\nThe lane is a TMEM data-path lane, not storage independently owned by a CUDA thread. `tcgen05.ld` and `tcgen05.st` are warp-collective. Under their four-warp access model, the warps cover these lane chunks:\n\n| 0 | 0-31 |\n| 1 | 32-63 |\n| 2 | 64-95 |\n| 3 | 96-127 |\n\nThe chosen load/store shape determines how values from that lane-column region map to each thread's register vector. Use the PTX shape-specific layout tables rather than treating a logical MxN accumulator as a universal row-major array owned one row per thread."}, "reason": {"statement": "Describe lanes and collective warp chunks without inventing ownership.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory-addressing"]}} +{"path": "wiki/hardware/tmem.md", "before": {"statement": "Columns are addressed via a column offset in the TMEM descriptor. A 128x256 MMA accumulator tile occupies:\n\n- 128 rows (all lanes across 4 warps)\n- 256 columns of FP32 values = 1024 bytes per row\n\n```\nTMEM Layout for 128x256 FP32 accumulator:\n col 0 col 255 col 511\n |------------|-----------|-----------|\n | Acc Tile | (free) | (free) | row 0 (warp0, lane0)\n | 128x256 | | | row 1 (warp0, lane1)\n | FP32 | | | ...\n | | | | row 127 (warp3, lane31)\n |------------|-----------|-----------|\n```"}, "after": {"statement": "A TMEM address (`taddr`) is a 32-bit value:\n\n| 31:16 | TMEM lane |\n| 15:0 | TMEM column |\n\nThe lane is a TMEM data-path lane, not storage independently owned by a CUDA thread. `tcgen05.ld` and `tcgen05.st` are warp-collective. Under their four-warp access model, the warps cover these lane chunks:\n\n| 0 | 0-31 |\n| 1 | 32-63 |\n| 2 | 64-95 |\n| 3 | 96-127 |\n\nThe chosen load/store shape determines how values from that lane-column region map to each thread's register vector. Use the PTX shape-specific layout tables rather than treating a logical MxN accumulator as a universal row-major array owned one row per thread."}, "reason": {"statement": "Retain capacity arithmetic separately from instruction-specific layout.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-data-path-layout-organization", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory-addressing"]}} +{"path": "wiki/hardware/tmem.md", "before": {"statement": "```cuda\n// Shared storage for CTA-wide broadcast of TMEM address\n__shared__ uint32_t s_tmem_addr;\n\n__device__ uint32_t tmem_alloc_cta(uint32_t num_cols) {\n // Only thread 0 allocates; result must reach ALL warps in the CTA.\n // __shfl_sync is warp-local — it cannot broadcast across warps.\n if (threadIdx.x == 0) {\n uint32_t smem_addr =\n static_cast(__cvta_generic_to_shared(&s_tmem_addr));\n asm volatile(\n \"tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;\"\n :\n : \"r\"(smem_addr), \"r\"(num_cols)\n );\n }\n __syncthreads(); // All warps now see s_tmem_addr\n return s_tmem_addr;\n}\n```\n\n- Only one thread (typically thread 0) issues the allocation.\n\n```cuda\n__device__ void tmem_dealloc(uint32_t tmem_addr, uint32_t num_cols) {\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;\"\n :\n : \"r\"(tmem_addr), \"r\"(num_cols)\n );\n }\n __syncthreads();\n}\n```"}, "after": {"statement": "TMEM has an explicit software-managed lifetime:\n\n1. Reserve shared memory for the 32-bit allocation result.\n2. Execute `tcgen05.alloc` collectively from one warp for `cta_group::1`, or from two warps—one per paired CTA—for `cta_group::2`.\n3. Synchronize consumers as required, then read the returned base `taddr` from shared memory.\n4. Use the allocation for MMA, copy, load, or store operations with the same CTA-group mode.\n5. Execute `tcgen05.dealloc` with the corresponding collective issue pattern.\n6. Deallocate every TMEM allocation before kernel exit.\n\nThe allocation operand counts columns. Legal allocation sizes are powers of two from 32 through 512 columns. Allocation can block until the requested TMEM is available; the ISA does not define folklore outcomes such as silent corruption for an invalid size."}, "reason": {"statement": "Remove unsafe inline PTX wrappers and state the collective contract directly.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"]}} +{"path": "wiki/hardware/tmem.md", "before": {"statement": "- The returned `tmem_addr` is the base column index."}, "after": {"statement": "TMEM has an explicit software-managed lifetime:\n\n1. Reserve shared memory for the 32-bit allocation result.\n2. Execute `tcgen05.alloc` collectively from one warp for `cta_group::1`, or from two warps—one per paired CTA—for `cta_group::2`.\n3. Synchronize consumers as required, then read the returned base `taddr` from shared memory.\n4. Use the allocation for MMA, copy, load, or store operations with the same CTA-group mode.\n5. Execute `tcgen05.dealloc` with the corresponding collective issue pattern.\n6. Deallocate every TMEM allocation before kernel exit.\n\nThe allocation operand counts columns. Legal allocation sizes are powers of two from 32 through 512 columns. Allocation can block until the requested TMEM is available; the ISA does not define folklore outcomes such as silent corruption for an invalid size."}, "reason": {"statement": "Preserve the full address model.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory-addressing", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"]}} +{"path": "wiki/hardware/tmem.md", "before": {"statement": "**Warning**: Failure to deallocate TMEM before CTA exit will leak memory and prevent subsequent CTAs from allocating, leading to hangs in persistent kernels.\n\n```cuda\n__global__ void persistent_gemm_kernel(/* ... */) {\n // 1. Allocate TMEM for accumulators\n uint32_t tmem_acc = tmem_alloc(256); // 256 cols for M=128, N=256\n\n while (has_more_tiles()) {\n // 2. Zero-initialize TMEM accumulator\n tmem_zero(tmem_acc, 256);\n\n // 3. Mainloop: accumulate K tiles\n for (int k = 0; k < K_tiles; ++k) {\n issue_tcgen05_mma(tmem_acc, smem_a[k], smem_b[k]);\n }\n\n // 4. Fence and read results\n asm volatile(\"tcgen05.fence::before_thread_sync;\");\n __syncthreads();\n\n // 5. Epilogue: read from TMEM, apply bias/activation, write to GMEM\n epilogue_from_tmem(tmem_acc, output);\n\n // 6. TMEM persists across tiles -- no need to reallocate\n }\n\n // 7. Deallocate before exit\n tmem_dealloc(tmem_acc, 256);\n}\n```"}, "after": {"statement": "TMEM has an explicit software-managed lifetime:\n\n1. Reserve shared memory for the 32-bit allocation result.\n2. Execute `tcgen05.alloc` collectively from one warp for `cta_group::1`, or from two warps—one per paired CTA—for `cta_group::2`.\n3. Synchronize consumers as required, then read the returned base `taddr` from shared memory.\n4. Use the allocation for MMA, copy, load, or store operations with the same CTA-group mode.\n5. Execute `tcgen05.dealloc` with the corresponding collective issue pattern.\n6. Deallocate every TMEM allocation before kernel exit.\n\nThe allocation operand counts columns. Legal allocation sizes are powers of two from 32 through 512 columns. Allocation can block until the requested TMEM is available; the ISA does not define folklore outcomes such as silent corruption for an invalid size.\n\nThe relevant mechanisms are distinct:\n\n- `tcgen05.commit` attaches completion of prior asynchronous MMA operations to an mbarrier. Wait on that barrier before consuming their results.\n- `tcgen05.wait::ld` and `tcgen05.wait::st` are the completion mechanisms for the corresponding asynchronous TMEM load/store operations.\n- `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` order tcgen05 operations around a documented execution-ordering handoff. A fence is not an MMA completion wait.\n- Producer buffers must remain live until the asynchronous operation that reads them has completed according to its instruction contract.\n\nCTA synchronization alone does not replace these completion operations."}, "reason": {"statement": "Remove a fabricated incomplete kernel and replace it with a normative lifecycle sequence.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-wait"]}} +{"path": "wiki/hardware/tmem.md", "before": {"statement": "```cuda\n// Store a register value to TMEM\n// Each thread writes to its own TMEM row at the specified column\n__device__ void tmem_store_f32(uint32_t tmem_col, float value) {\n asm volatile(\n \"tcgen05.st.sync.aligned.32x1b.x1.b32 [%0], {%1};\"\n :\n : \"r\"(tmem_col), \"f\"(value)\n );\n}\n\n// Vectorized store: 4 consecutive FP32 values\n__device__ void tmem_store_f32x4(uint32_t tmem_col, float4 values) {\n asm volatile(\n \"tcgen05.st.sync.aligned.32x1b.x4.b32 [%0], {%1, %2, %3, %4};\"\n :\n : \"r\"(tmem_col),\n \"f\"(values.x), \"f\"(values.y),\n \"f\"(values.z), \"f\"(values.w)\n );\n}\n```\n\n```cuda\n// Load a single FP32 from TMEM\n__device__ float tmem_load_f32(uint32_t tmem_col) {\n float result;\n asm volatile(\n \"tcgen05.ld.sync.aligned.32x1b.x1.b32 {%0}, [%1];\"\n : \"=f\"(result)\n : \"r\"(tmem_col)\n );\n return result;\n}\n\n// Vectorized load: 4 consecutive FP32 values\n__device__ float4 tmem_load_f32x4(uint32_t tmem_col) {\n float4 result;\n asm volatile(\n \"tcgen05.ld.sync.aligned.32x1b.x4.b32 {%0, %1, %2, %3}, [%4];\"\n : \"=f\"(result.x), \"=f\"(result.y),\n \"=f\"(result.z), \"=f\"(result.w)\n : \"r\"(tmem_col)\n );\n return result;\n}\n```\n\n```cuda\n// Zero-fill a range of TMEM columns\n__device__ void tmem_zero(uint32_t tmem_base_col, uint32_t num_cols) {\n // Each thread zeros its own row\n for (uint32_t c = 0; c < num_cols; c += 4) {\n float4 zero = make_float4(0.f, 0.f, 0.f, 0.f);\n tmem_store_f32x4(tmem_base_col + c, zero);\n }\n}\n```"}, "after": {"statement": "`tcgen05.ld` transfers TMEM into registers, and `tcgen05.st` transfers registers into TMEM. Their documented shapes include `.16x64b`, `.16x128b`, `.16x256b`, `.32x32b`, and `.16x32bx2`, with supported repetition qualifiers. There is no scalar `.32x1b` form.\n\n`tcgen05.cp` performs shaped shared-memory-to-TMEM copies. For example, PTX ISA 9.0 defines this exact form:\n\n```ptx\ntcgen05.cp.cta_group::1.128x256b [taddr], sdesc;\n```\n\nHere `taddr` is the TMEM destination and `sdesc` is the 64-bit shared-memory matrix descriptor. Follow the instruction's asynchronous ordering and completion rules before reusing the source or consuming the destination."}, "reason": {"statement": "Do not publish non-assembling executable-looking wrappers.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-cp"]}} +{"path": "wiki/hardware/tmem.md", "before": {"statement": "```cuda\n__global__ void double_buffered_gemm(/* ... */) {\n // Allocate two accumulator buffers\n uint32_t tmem_acc[2];\n tmem_acc[0] = tmem_alloc(256);\n tmem_acc[1] = tmem_alloc(256);\n\n int buf = 0;\n\n for (int tile = 0; tile < num_tiles; ++tile) {\n // Zero the current buffer\n tmem_zero(tmem_acc[buf], 256);\n\n // Mainloop: accumulate into current buffer\n for (int k = 0; k < K_tiles; ++k) {\n issue_tcgen05_mma(tmem_acc[buf], smem_a[k], smem_b[k]);\n }\n asm volatile(\"tcgen05.fence::before_thread_sync;\");\n __syncthreads();\n\n // If not the first tile, the *other* buffer's epilogue\n // was overlapped with this tile's MMA in the pipeline.\n\n // Start epilogue for current buffer\n // (can overlap with next tile's MMA on the other buffer)\n epilogue_from_tmem(tmem_acc[buf], output_tile[tile]);\n\n buf ^= 1; // Swap buffers\n }\n\n // Cleanup\n tmem_dealloc(tmem_acc[0], 256);\n tmem_dealloc(tmem_acc[1], 256);\n}\n```"}, "after": {"statement": "CUTLASS 4.5.0 exposes TMEM through `cutlass.utils.TmemAllocator` and layout-aware CuTe tensors. The official tutorial sequence is:\n\n1. create a `TmemAllocator` over shared storage for the allocation result;\n2. call `allocate(num_columns)` from the configured allocator warp;\n3. use `wait_for_alloc()` before other warps retrieve the address;\n4. obtain a typed pointer with `retrieve_ptr(dtype)`;\n5. bind that pointer to the MMA accumulator layout with `cute.make_tensor`;\n6. drain it with `tcgen05` copy atoms and the pipeline's completion protocol; and\n7. call `free(tmem_ptr)` before exit."}, "reason": {"statement": "Replace invented code with resource and synchronization requirements plus a pinned real example.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_6.py", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"]}} +{"path": "wiki/hardware/tmem.md", "before": {"statement": "| 128x192 FP32 | 192 cols | 2 | 128 |"}, "after": {"statement": "All allocations share the SM's 512-column capacity. Some useful whole-allocation budgets are:\n\n| 128 columns | 4 | 0 |\n| 256 columns | 2 | 0 |\n| 512 columns | 1 | 0 |\n\nA logical need of 192 columns cannot be requested directly: reserve 256 columns, or suballocate that logical region within another legal power-of-two reservation. Two 256-column accumulator stages consume the entire capacity, so scale-factor tensors or other scratch state must fit inside those reservations or the pipeline must use fewer columns."}, "reason": {"statement": "List only allocatable budgets and distinguish logical suballocation from hardware allocation.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"]}} +{"path": "wiki/hardware/tmem.md", "before": {"statement": "From published Blackwell microbenchmarks:\n\n| End-to-end latency (cache miss) | ~420 cycles | ~30 cycles | ~4 cycles |\n| Bandwidth for large working sets | High (dedicated bus) | Medium | N/A (limited count) |\n| Best for | Multi-stage tensor pipelines | Single-shot small matrix | Scalar/vector ALU |\n\nTMEM is **not** a replacement for shared memory. Its strength is in serving as a dedicated accumulator buffer that eliminates register pressure for large MMA tiles. SMEM remains faster for small, frequently accessed data.\n\n5. **Assuming SMEM-like latency**: TMEM has ~420-cycle latency on cache miss vs ~30 cycles for SMEM. Do not use TMEM for low-latency random access patterns."}, "after": {"statement": null}, "reason": {"statement": "A stale, heterogeneous comparison cannot support general performance guidance.", "urls": ["https://arxiv.org/pdf/2512.02189v3", "https://arxiv.org/abs/2512.02189"]}} +{"path": "wiki/hardware/tmem.md", "before": {"statement": "```python\n# CuTe-DSL example: TMEM accumulator layout\n# 128 rows x 256 columns, FP32\ntmem_layout = Layout(\n shape=(128, 256),\n stride=(256, 1),\n memory_space=MemorySpace.TMEM\n)\n\n# Allocate TMEM accumulator\nacc = tmem_alloc(tmem_layout)\n\n# Issue MMA -- accumulator lives in TMEM\ntcgen05_mma(acc, smem_a, smem_b)\n\n# Fence and read\ntcgen05_fence()\nresult = tmem_load(acc)\n```"}, "after": {"statement": "See the pinned [CUTLASS 4.5.0 FP16 GEMM tutorial](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py) and [TMEM allocator implementation](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/python/CuTeDSL/cutlass/utils/tmem_allocator.py). APIs such as `Layout(memory_space=MemorySpace.TMEM)` and standalone `tcgen05_mma()` are not CUTLASS 4.5.0 interfaces."}, "reason": {"statement": "Link a complete version-pinned tutorial rather than fabricate a simplified API.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py"]}} +{"path": "wiki/hardware/tmem.md", "before": {"statement": "1. **Forgetting to fence**: Reading TMEM without `tcgen05.fence::before_thread_sync` produces undefined (stale) values.\n2. **Forgetting to deallocate**: In persistent kernels, TMEM must be freed before re-acquiring tiles. Otherwise, the next allocation will fail or hang.\n3. **Exceeding 512 columns**: Attempting to allocate more than the SM's total column budget silently corrupts data or causes a hang."}, "after": {"statement": "TMEM has an explicit software-managed lifetime:\n\n1. Reserve shared memory for the 32-bit allocation result.\n2. Execute `tcgen05.alloc` collectively from one warp for `cta_group::1`, or from two warps—one per paired CTA—for `cta_group::2`.\n3. Synchronize consumers as required, then read the returned base `taddr` from shared memory.\n4. Use the allocation for MMA, copy, load, or store operations with the same CTA-group mode.\n5. Execute `tcgen05.dealloc` with the corresponding collective issue pattern.\n6. Deallocate every TMEM allocation before kernel exit.\n\nThe allocation operand counts columns. Legal allocation sizes are powers of two from 32 through 512 columns. Allocation can block until the requested TMEM is available; the ISA does not define folklore outcomes such as silent corruption for an invalid size.\n\nAll allocations share the SM's 512-column capacity. Some useful whole-allocation budgets are:\n\n| 128 columns | 4 | 0 |\n| 256 columns | 2 | 0 |\n| 512 columns | 1 | 0 |\n\nA logical need of 192 columns cannot be requested directly: reserve 256 columns, or suballocate that logical region within another legal power-of-two reservation. Two 256-column accumulator stages consume the entire capacity, so scale-factor tensors or other scratch state must fit inside those reservations or the pipeline must use fewer columns.\n\nThe relevant mechanisms are distinct:\n\n- `tcgen05.commit` attaches completion of prior asynchronous MMA operations to an mbarrier. Wait on that barrier before consuming their results.\n- `tcgen05.wait::ld` and `tcgen05.wait::st` are the completion mechanisms for the corresponding asynchronous TMEM load/store operations.\n- `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` order tcgen05 operations around a documented execution-ordering handoff. A fence is not an MMA completion wait.\n- Producer buffers must remain live until the asynchronous operation that reads them has completed according to its instruction contract.\n\nCTA synchronization alone does not replace these completion operations."}, "reason": {"statement": "Replace folklore failure modes with normative obligations.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-wait"]}} +{"path": "wiki/hardware/tmem.md", "before": {"statement": "4. **Cross-warp reads**: A thread can only directly read/write TMEM rows mapped to its own lane. Accessing another warp's rows requires explicit shuffle or SMEM staging."}, "after": {"statement": "A TMEM address (`taddr`) is a 32-bit value:\n\n| 31:16 | TMEM lane |\n| 15:0 | TMEM column |\n\nThe lane is a TMEM data-path lane, not storage independently owned by a CUDA thread. `tcgen05.ld` and `tcgen05.st` are warp-collective. Under their four-warp access model, the warps cover these lane chunks:\n\n| 0 | 0-31 |\n| 1 | 32-63 |\n| 2 | 64-95 |\n| 3 | 96-127 |\n\nThe chosen load/store shape determines how values from that lane-column region map to each thread's register vector. Use the PTX shape-specific layout tables rather than treating a logical MxN accumulator as a universal row-major array owned one row per thread."}, "reason": {"statement": "State direct warp access restrictions and avoid a false communication primitive.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory-addressing"]}} +{"path": "wiki/hardware/mbarrier.md", "before": {"statement": "mbarriers are 64-bit shared memory primitives used for producer/consumer synchronization between asynchronous hardware units (TMA, tcgen05) and SM threads. Introduced on Hopper, essential for Blackwell warp-specialized kernels."}, "after": {"statement": "An mbarrier is an opaque, naturally aligned 64-bit object in shared memory. It synchronizes threads and can track asynchronous operations. The base instructions were introduced in PTX ISA 7.0 for `sm_80`; Hopper (`sm_90`) added transaction-count operations and cluster scope. Blackwell uses the same object for TMA transaction completion and for `tcgen05.commit` arrival-on completion.\n\nmbarriers are useful in warp-specialized pipelines, but they are a mechanism rather than a requirement for every such kernel."}, "reason": {"statement": "Separate the Ampere base primitive from Hopper transaction-count extensions and avoid a universal design mandate.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-init", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"]}} +{"path": "wiki/hardware/mbarrier.md", "before": {"statement": "```ptx\n// Initialize: set expected arrival count\nmbarrier.init.shared.b64 [mbar_addr], num_arrivals;\n\n// Producer: arrive on barrier (decrements expected count)\nmbarrier.arrive.shared.b64 _, [mbar_addr];\n\n// Producer (with byte expectation): used by TMA\nmbarrier.arrive.expect_tx.shared.b64 _, [mbar_addr], expected_bytes;\n// After this, TMA hardware completes the transaction by arriving with\n// the transferred byte count, and the mbarrier flips parity.\n\n// Consumer: wait for parity flip\nmbarrier.try_wait.parity.shared.b64 p, [mbar_addr], phase;\n@!p bra WAIT_LOOP;\n```"}, "after": {"statement": "For the current phase, an mbarrier tracks:\n\n- pending arrivals;\n- the expected arrival count for the next phase; and\n- a transaction count (`tx-count`) for outstanding asynchronous work.\n\nThe current phase completes only when **both** pending arrivals and tx-count reach zero. Completion atomically advances to the next phase and restores pending arrivals from the expected count. Before an arrival in the following phase, at least one `test_wait` or `try_wait` for the completed phase must have returned true.\n\nInitialization sets phase 0, initializes expected and pending arrivals to `count`, and sets tx-count to zero:\n\n```ptx\nmbarrier.init.shared::cta.b64 [bar], arrival_count;\n```\n\nInvalidate the object with `mbarrier.inval` before reusing its storage for another purpose or reinitializing an already-valid object."}, "reason": {"statement": "Preserve exact accounting semantics and avoid premature-completion bugs.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#phase-completion-of-the-mbarrier-object", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#contents-of-the-mbarrier-object"]}} +{"path": "wiki/hardware/mbarrier.md", "before": {"statement": "```cuda\nint phase = 0;\nfor (int k = 0; k < num_iterations; k++) {\n int stage = k % NUM_STAGES;\n\n // Wait for this stage's producer to complete\n mbarrier_wait_parity(&mbar[stage], phase);\n phase ^= 1; // Flip for next use of this stage slot\n\n // Use data...\n consume_data(stage);\n\n // Arrive on next-stage barrier when done\n mbarrier_arrive(&buffer_free[stage]);\n}\n```"}, "after": {"statement": "These operations affect different parts of the state:\n\n| `mbarrier.arrive` | Decrements pending arrivals; returns a phase state token for a CTA-shared object. |\n| `mbarrier.arrive.expect_tx` | Performs an arrival and increments tx-count by `txCount`. |\n| `mbarrier.expect_tx` | Increments tx-count without an arrival. |\n| `mbarrier.complete_tx` | Decrements tx-count; it is not an arrival. |\n| `mbarrier.test_wait` / `try_wait` | Tests completion of the phase identified by a state token or parity. |\n\nRepresentative CTA-shared forms from PTX ISA 9.0 are:\n\n```ptx\nmbarrier.arrive.shared::cta.b64 state, [bar];\nmbarrier.arrive.expect_tx.shared::cta.b64 state, [bar], tx_count;\nmbarrier.try_wait.parity.acquire.cta.shared::cta.b64 ready, [bar], phase_parity;\n```\n\n`try_wait` can suspend temporarily and must still be retried until its predicate is true. Use acquire/release semantics appropriate to the producer-consumer handoff; `.relaxed` does not provide memory-ordering or visibility guarantees."}, "reason": {"statement": "Remove subtly unsafe pseudocode and give a per-stage reuse rule instead.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive"]}} +{"path": "wiki/hardware/mbarrier.md", "before": {"statement": "```cuda\n// Producer warp issues TMA with mbarrier target\nif (lane_id == 0) {\n uint32_t tx_bytes = TILE_A_BYTES + TILE_B_BYTES;\n mbarrier_arrive_expect_tx(&mbar[stage], tx_bytes);\n\n // TMA completion will fire the mbarrier automatically\n cp_async_bulk_tensor_2d(smem_A[stage], global_desc, x, y, &mbar[stage]);\n cp_async_bulk_tensor_2d(smem_B[stage], global_desc, x, y, &mbar[stage]);\n}\n// Do NOT manually arrive after TMA issue - it races with hardware\n```\n\n3. **Manual arrive after async issue**: TMA/tcgen05 hardware arrives on completion — extra manual arrive causes double-count"}, "after": {"statement": "Parity is the low bit of an individual mbarrier object's phase: even phases use 0 and odd phases use 1. A parity wait can refer only to the current or immediately preceding phase, so software must track phase for the entire lifetime of that object.\n\nFor an N-stage ring, track state per stage. When stage `s` is reused, pass the parity expected for `bar[s]`; toggle that stage's parity only after its phase completes. One global bit toggled on every loop iteration is generally wrong because different stage barriers advance independently.\n\nUsing the opaque state returned by `mbarrier.arrive` is an alternative when the same participant can carry that token to its wait."}, "reason": {"statement": "Replace fabricated code with an exact accounting sequence and vendor grammar link.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait"]}} +{"path": "wiki/hardware/mbarrier.md", "before": {"statement": "4. **Cross-cluster mbarriers**: Use `shared::cluster` qualifier for cluster-level sync (2-SM cooperative MMA)"}, "after": {"statement": "For a common TMA-load phase initialized with one pending arrival:\n\n1. The producer executes `mbarrier.arrive.expect_tx` with the sum of bytes that all TMA operations in this phase will report.\n2. The producer issues the TMA operations with `.mbarrier::complete_tx::bytes` and the same barrier.\n3. Each TMA completion performs `complete-tx` for the bytes it copied.\n4. The consumer waits for the phase to complete before reading the destination.\n\nThe `arrive.expect_tx` operation accounts for the software arrival and establishes the expected byte total. TMA hardware does **not** perform a second arrival: it decrements tx-count through complete-tx. Therefore, do not add an unmatched `mbarrier.arrive`, and do not manually simulate TMA's complete-tx. Either mistake can advance or strand the wrong phase.\n\nThe expected byte total, barrier address, destination ownership, and exact `cp.async.bulk.tensor` form must match. Follow the complete instruction grammar in the PTX ISA rather than using placeholder helper calls."}, "reason": {"statement": "Teach object location, address mapping, operation support, and synchronization scope separately.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive"]}} +{"path": "wiki/hardware/nvfp4.md", "before": {"statement": "| `tcgen05.mma.mxf4.block_scale` | MX FP4 with block scaling | **4×** |\n| `tcgen05.mma.mxf4nvf4.block_scale` | NVFP4 + MX FP4 flexible scaling | **4×** |"}, "after": {"statement": "PTX ISA 9.0 provides the packed conversion form:\n\n```ptx\ncvt.rn.f16x2.e2m1x2 d, a;\n```\n\nHere `a` is a byte-sized packed pair and `d` is a 32-bit `f16x2` result. PTX also permits `mov.b32` to unpack a 32-bit scalar into four byte-sized vector destinations when the declarations satisfy its type rules. Neither syntax establishes that one packing strategy is faster than masks and shifts; inspect generated machine code and benchmark the target GPU."}, "reason": {"statement": "Give exact distinguishing qualifiers and remove an unscoped performance number.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov"]}} +{"path": "wiki/hardware/nvfp4.md", "before": {"statement": "```ptx\n// Convert two FP4 values to two FP16 values\ncvt.rn.f16x2.e2m1x2 result, packed_fp4;\n\n// Byte unpacking (faster than bitwise extraction)\nmov.b32 {tmp0, tmp1, tmp2, tmp3}, packed_data;\n```"}, "after": {"statement": "There is no architecture-independent \"4x versus Hopper\" result for these instructions. Hopper has no native FP4 tensor-core path, so any comparison depends on the emulation or higher-precision baseline as well as GPU SKU, clocks, matrix shape, layouts, scale preparation, epilogue, and achieved occupancy. Record those conditions with any throughput number."}, "reason": {"statement": "Retain the exact packed conversion form, state mov's typed preconditions, and require measurement for packing-strategy performance.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/hardware/nvfp4.md", "before": {"statement": "| Quantization error | Lower | Higher |"}, "after": {"statement": "In PTX ISA 9.0, the relevant `tcgen05.mma` combinations for E2M1 inputs include:\n\n| `.kind::mxf4.block_scale.block32` | 32 | `.ue8m0` | `.scale_vec::2X` |\n| `.kind::mxf4nvf4.block_scale.block16` | 16 | `.ue4m3` | `.scale_vec::4X` |\n\nThe `.kind::mxf4nvf4` family also admits documented UE8M0 modes, so the kind name by itself does not select the NVFP4 recipe. Use the complete block and scale-vector qualifiers and follow the type-combination tables. These forms target architecture-specific Blackwell targets such as `sm_100a`; a kernel using them must use the matching compile and runtime target."}, "reason": {"statement": "State the representational tradeoff without claiming an unconditional empirical ordering.", "urls": ["https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/hardware/pdl-gdc.md", "before": {"statement": "```cuda\n// Primary kernel signals near completion\ncudaGridDependencySynchronize(); // or PTX equivalent\n\n// Secondary kernel can start overlapping with primary's tail\n// Enabled by default on SM100 (opt-in on SM90)\n```"}, "after": {"statement": "This minimal skeleton shows where each operation belongs; real kernels put independent and dependent work at the indicated points and add normal error checking:\n\n```cuda\n#include \n\n__global__ void primary() {\n // Produce everything needed before the secondary may be launched.\n if (threadIdx.x == 0) {\n cudaTriggerProgrammaticLaunchCompletion();\n }\n // Primary tail that does not change data consumed by the secondary.\n}\n\n__global__ void secondary() {\n // Independent preamble may execute early.\n cudaGridDependencySynchronize();\n // Dependent reads begin only after this wait.\n}\n\nvoid launch_pdl(dim3 grid, dim3 block, cudaStream_t stream) {\n cudaLaunchConfig_t cfg{};\n cfg.gridDim = grid;\n cfg.blockDim = block;\n cfg.stream = stream;\n\n cudaLaunchAttribute attr{};\n attr.id = cudaLaunchAttributeProgrammaticStreamSerialization;\n attr.val.programmaticStreamSerializationAllowed = 1;\n cfg.attrs = &attr;\n cfg.numAttrs = 1;\n\n primary<<>>();\n cudaLaunchKernelEx(&cfg, secondary);\n}\n```\n\nDo not place `cudaGridDependencySynchronize()` in the primary: it is the secondary-side wait, not the launch trigger. Do not replace the wait with an ordinary memory fence. PDL's wait supplies both prerequisite-grid completion and visibility for its dependent work."}, "reason": {"statement": "Replace the reversed, incomplete example with the three-part protocol.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#programmatic-dependent-launch-and-synchronization", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-griddepcontrol"]}} +{"path": "wiki/hardware/pdl-gdc.md", "before": {"statement": "On SM100, PDL is **enabled by default** — no opt-in needed. This means:"}, "after": {"statement": "Programmatic Dependent Launch (PDL) lets a secondary grid in the same CUDA stream become eligible to start before its prerequisite primary grid completes. It is available on compute capability 9.0 and later, including Hopper and Blackwell.\n\nPDL creates an **opportunity** for overlap; it does not guarantee that the grids execute concurrently. The useful overlap is normally the primary's work after its launch trigger and the secondary's independent preamble before its dependency wait.\n\nThe CUDA protocol has three required roles:\n\n1. Every primary CTA either executes `cudaTriggerProgrammaticLaunchCompletion()` or exits. After all CTAs satisfy that condition, the driver may schedule the secondary grid.\n2. The host launches the secondary in the same stream with `cudaLaunchAttributeProgrammaticStreamSerialization` and `programmaticStreamSerializationAllowed = 1`.\n3. Every secondary thread waits with `cudaGridDependencySynchronize()` before it consumes prerequisite results. The wait completes after the prerequisite grids finish and their memory operations are visible.\n\nIf the primary does not explicitly trigger, its CTAs implicitly satisfy the trigger only as they exit. Omitting the explicit trigger is correct but removes the intended primary-tail overlap.\n\nBlackwell does not make arbitrary back-to-back launches overlap automatically. CUDA applications still opt the secondary launch into programmatic stream serialization and implement the device-side trigger/wait protocol.\n\nCUTLASS has a separate build choice. In CUTLASS 4.5.0, the CMake option `CUTLASS_ENABLE_GDC_FOR_SM100` defaults to `ON`, while the SM90 option is opt-in. That default only enables eligible CUTLASS code to emit its GDC wrappers; it is not a device-wide CUDA default and can be overridden by the build."}, "reason": {"statement": "Separate hardware availability, CUDA launch opt-in, and one library's build default.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#programmatic-dependent-launch-and-synchronization", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/CMakeLists.txt#L463-L474"]}} +{"path": "wiki/hardware/pdl-gdc.md", "before": {"statement": "- Back-to-back kernel launches naturally overlap\n- Memory fences ensure correctness for dependent data\n- Reduces kernel launch gaps in compute-heavy pipelines"}, "after": {"statement": "Programmatic Dependent Launch (PDL) lets a secondary grid in the same CUDA stream become eligible to start before its prerequisite primary grid completes. It is available on compute capability 9.0 and later, including Hopper and Blackwell.\n\nPDL creates an **opportunity** for overlap; it does not guarantee that the grids execute concurrently. The useful overlap is normally the primary's work after its launch trigger and the secondary's independent preamble before its dependency wait.\n\nThe CUDA protocol has three required roles:\n\n1. Every primary CTA either executes `cudaTriggerProgrammaticLaunchCompletion()` or exits. After all CTAs satisfy that condition, the driver may schedule the secondary grid.\n2. The host launches the secondary in the same stream with `cudaLaunchAttributeProgrammaticStreamSerialization` and `programmaticStreamSerializationAllowed = 1`.\n3. Every secondary thread waits with `cudaGridDependencySynchronize()` before it consumes prerequisite results. The wait completes after the prerequisite grids finish and their memory operations are visible.\n\nIf the primary does not explicitly trigger, its CTAs implicitly satisfy the trigger only as they exit. Omitting the explicit trigger is correct but removes the intended primary-tail overlap.\n\nThis minimal skeleton shows where each operation belongs; real kernels put independent and dependent work at the indicated points and add normal error checking:\n\n```cuda\n#include \n\n__global__ void primary() {\n // Produce everything needed before the secondary may be launched.\n if (threadIdx.x == 0) {\n cudaTriggerProgrammaticLaunchCompletion();\n }\n // Primary tail that does not change data consumed by the secondary.\n}\n\n__global__ void secondary() {\n // Independent preamble may execute early.\n cudaGridDependencySynchronize();\n // Dependent reads begin only after this wait.\n}\n\nvoid launch_pdl(dim3 grid, dim3 block, cudaStream_t stream) {\n cudaLaunchConfig_t cfg{};\n cfg.gridDim = grid;\n cfg.blockDim = block;\n cfg.stream = stream;\n\n cudaLaunchAttribute attr{};\n attr.id = cudaLaunchAttributeProgrammaticStreamSerialization;\n attr.val.programmaticStreamSerializationAllowed = 1;\n cfg.attrs = &attr;\n cfg.numAttrs = 1;\n\n primary<<>>();\n cudaLaunchKernelEx(&cfg, secondary);\n}\n```\n\nDo not place `cudaGridDependencySynchronize()` in the primary: it is the secondary-side wait, not the launch trigger. Do not replace the wait with an ordinary memory fence. PDL's wait supplies both prerequisite-grid completion and visibility for its dependent work.\n\nCUDA lowers the two device roles to Grid Dependency Control instructions:\n\n```ptx\n// Primary CTA: makes designated dependent grids eligible after every CTA\n// has issued this instruction or completed.\ngriddepcontrol.launch_dependents;\n\n// Secondary thread: waits for in-flight prerequisite grids and visibility.\ngriddepcontrol.wait;\n```\n\n`griddepcontrol` was introduced in PTX ISA 7.8 and requires `sm_90` or newer. Repeating `launch_dependents` within one CTA has no additional effect after that CTA's first invocation. If a prerequisite uses `launch_dependents`, its dependent must use `griddepcontrol.wait` or an equivalent CUDA dependency wait for correct execution."}, "reason": {"statement": "State the exact ordering contract and make overlap conditional.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#programmatic-dependent-launch-and-synchronization", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-griddepcontrol"]}} +{"path": "wiki/hardware/pdl-gdc.md", "before": {"statement": "- Chains of small kernels (e.g., MoE dispatch → compute → combine)\n- Pipeline-parallel training with many sequential kernel launches\n- Reduces overall wall-clock time without code changes on Blackwell"}, "after": {"statement": "PDL can help only when all of these conditions hold:\n\n- the secondary has enough independent preamble to overlap;\n- the primary has useful tail work after every CTA reaches the trigger;\n- the two grids have enough simultaneous resource headroom; and\n- the saved launch/serialization time exceeds the protocol and occupancy costs.\n\nSmall-kernel chains, GEMM/epilogue sequences, and pipeline-parallel stages are candidates, not guaranteed wins. Profile an explicit non-PDL baseline and record GPU, clocks, launch shapes, stream/graph configuration, input sizes, warmup, repetitions, and the observed overlap timeline. Never rely on overlap for forward progress; CUDA documents it as opportunistic and warns that such reliance can deadlock."}, "reason": {"statement": "Replace categorical workload promises with a decision and measurement checklist.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#programmatic-dependent-launch-and-synchronization"]}} +{"path": "wiki/hardware/tma.md", "before": {"statement": "The Tensor Memory Accelerator (TMA) is a hardware unit that performs **asynchronous bulk data transfers** between global memory and shared memory. First introduced on Hopper (SM90), TMA carries forward to Blackwell (SM100) with stricter requirements for tcgen05 compatibility.\n\n| Swizzle modes | None, 32B, 64B, 128B (128B required for tcgen05) |\n\nOn Blackwell, `tcgen05.mma` requires operands in **128-byte swizzled** SMEM layout. If TMA loads data without 128B swizzling, the MMA will produce incorrect results.\n\n```cuda\n// CORRECT for Blackwell tcgen05:\nCUtensorMap desc = create_tma_descriptor_2d(ptr, M, N, 128, 64, 128);\n// swizzle=128 ^^^\n\n// WRONG for tcgen05 (will silently produce garbage):\nCUtensorMap desc = create_tma_descriptor_2d(ptr, M, N, 128, 64, 0);\n// swizzle=0 ^^^\n```\n\nAll TMA loads feeding `tcgen05.mma` must use 128-byte swizzling. The swizzle pattern rearranges bytes within each 128-byte line to match the tensor core's internal data layout:\n\n| Always use 128B swizzle on Blackwell | Non-128B swizzle produces incorrect tcgen05 results |"}, "after": {"statement": "TMA supports no swizzle and multiple swizzled shared-memory layouts, including 32B, 64B, and 128B spans plus newer variants for selected types. Swizzling rearranges chunks across shared-memory banks. The consumer must address the matching logical layout; the exact mapping also depends on the documented shared-memory base-offset rule.\n\nThere is no universal rule that every Blackwell or `tcgen05.mma` input uses 128B swizzling. The TMA tensor-map swizzle, destination base alignment, leading dimension, and the tcgen05 shared-memory descriptor must describe the **same** legal layout. PTX defines no-, 32B-, 64B-, and 128B-swizzled tcgen05 descriptors with kind-, type-, shape-, and layout-specific constraints.\n\nA matched swizzle can reduce or remove bank conflicts for a particular access pattern. It does not make every possible consumer access conflict-free."}, "reason": {"statement": "Make swizzle selection descriptor- and shape-specific and remove the invented failure mode.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-layout", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"]}} +{"path": "wiki/hardware/tma.md", "before": {"statement": "TMA offloads data movement from CUDA cores entirely -- a single thread issues the transfer, and the TMA hardware engine handles the multi-dimensional copy, address calculation, out-of-bounds clamping, and format conversion.\n\n| Format conversion | FP32<->BF16, FP32<->FP16 during transfer |"}, "after": {"statement": "TMA is the descriptor-driven `cp.async.bulk.tensor` facility introduced for Hopper (`sm_90`) and retained on Blackwell. One thread can issue a non-blocking rank-1 through rank-5 tensor copy while the hardware performs multidimensional address traversal and moves the tile.\n\nThe principal copy directions and completion mechanisms are:\n\n| Global to shared | Issuing CTA or a CTA in its cluster | One masked instruction can multicast to selected CTAs | mbarrier `complete_tx` in bytes |\n| Shared to global | Issuing CTA's shared memory to a tensor map | Scatter modes are available on supported Blackwell targets | Bulk async group: issue, commit, wait |\n\nTMA copies the datatype represented by the tensor map. It does not provide a general FP32-to/from-FP16 or BF16 conversion step. For a tiled global-to-shared load, out-of-bounds elements are filled according to the supported tensor-map policy rather than clamped to an edge coordinate.\n\nA tensor map is opaque and is accessed through the tensor-map proxy. `cuTensorMapEncodeTiled` describes:\n\n- datatype and global base address;\n- rank, global dimensions, and byte strides;\n- traversal box dimensions and element strides;\n- interleave and shared-memory swizzle;\n- L2-promotion hint; and\n- out-of-bounds fill policy.\n\nFor the ordinary non-interleaved tiled path in CUDA Driver API 13.0.97, important constraints include:\n\n- the `CUtensorMap` output object is 64-byte aligned;\n- the global base and byte strides satisfy the documented alignment rules, commonly at least 16 bytes for the basic types/path;\n- every `boxDim` entry is from 1 through 256 elements;\n- `boxDim[0] * element_size` is a multiple of 16 bytes;\n- every element stride is from 1 through 8; and\n- when swizzling is enabled, the inner box in bytes does not exceed the selected swizzle span.\n\nDatatype, interleave, sub-byte, and architecture-specific modes add further restrictions. Always check the encoder's `CUresult`; never use the output after an encoding failure.\n\nHost encoding is common, but it is not the only Blackwell path. CUDA also documents device-side tensor-map construction and modification on Blackwell. A map modified through the generic proxy must be published to the tensor-map proxy with the required `fence.proxy.tensormap::generic` sequence before a TMA operation consumes it."}, "reason": {"statement": "Describe representation-preserving copies and OOB fill precisely.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#asynchronous-data-copies-using-the-tensor-memory-accelerator-tma", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#encoding-a-tensor-map-on-device"]}} +{"path": "wiki/hardware/tma.md", "before": {"statement": "| Max transfer size | Up to 256 bytes per element, tiles up to 128x256 |\n\n| Synchronization | mbarrier-based (arrive/wait) |"}, "after": {"statement": "TMA is the descriptor-driven `cp.async.bulk.tensor` facility introduced for Hopper (`sm_90`) and retained on Blackwell. One thread can issue a non-blocking rank-1 through rank-5 tensor copy while the hardware performs multidimensional address traversal and moves the tile.\n\nThe principal copy directions and completion mechanisms are:\n\n| Global to shared | Issuing CTA or a CTA in its cluster | One masked instruction can multicast to selected CTAs | mbarrier `complete_tx` in bytes |\n| Shared to global | Issuing CTA's shared memory to a tensor map | Scatter modes are available on supported Blackwell targets | Bulk async group: issue, commit, wait |\n\nTMA copies the datatype represented by the tensor map. It does not provide a general FP32-to/from-FP16 or BF16 conversion step. For a tiled global-to-shared load, out-of-bounds elements are filled according to the supported tensor-map policy rather than clamped to an edge coordinate.\n\nA tensor map is opaque and is accessed through the tensor-map proxy. `cuTensorMapEncodeTiled` describes:\n\n- datatype and global base address;\n- rank, global dimensions, and byte strides;\n- traversal box dimensions and element strides;\n- interleave and shared-memory swizzle;\n- L2-promotion hint; and\n- out-of-bounds fill policy.\n\nFor the ordinary non-interleaved tiled path in CUDA Driver API 13.0.97, important constraints include:\n\n- the `CUtensorMap` output object is 64-byte aligned;\n- the global base and byte strides satisfy the documented alignment rules, commonly at least 16 bytes for the basic types/path;\n- every `boxDim` entry is from 1 through 256 elements;\n- `boxDim[0] * element_size` is a multiple of 16 bytes;\n- every element stride is from 1 through 8; and\n- when swizzling is enabled, the inner box in bytes does not exceed the selected swizzle span.\n\nDatatype, interleave, sub-byte, and architecture-specific modes add further restrictions. Always check the encoder's `CUresult`; never use the output after an encoding failure.\n\nHost encoding is common, but it is not the only Blackwell path. CUDA also documents device-side tensor-map construction and modification on Blackwell. A map modified through the generic proxy must be published to the tensor-map proxy with the required `fence.proxy.tensormap::generic` sequence before a TMA operation consumes it.\n\nPTX ISA 9.0 defines this representative 2D CTA-local form:\n\n```ptx\ncp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes\n [dst_smem], [tensor_map, {x, y}], [full_barrier];\n```\n\nThe instruction is non-blocking. A common one-producer phase with one or more loads uses this accounting:\n\n1. Initialize and publish the stage's mbarrier with the intended pending-arrival count.\n2. The producer performs one `mbarrier.arrive.expect_tx` for its software arrival and the sum of bytes that all loads in this phase will complete.\n3. Issue the TMA loads against that barrier.\n4. Each completed load performs `complete_tx` for its copied byte count; it does **not** perform another arrival.\n5. Consumers wait with the correct state token or per-stage parity and acquire semantics before reading the destination.\n\nThe phase completes only after pending arrivals and tx-count are both zero. If a design performs multiple software arrivals instead, its initialized count must match them exactly. See [mbarrier](mbarrier.md) for lifecycle, phase, and memory-ordering rules.\n\nA tensor store uses bulk-group completion rather than the load's mbarrier protocol:\n\n```ptx\ncp.async.bulk.tensor.2d.global.shared::cta.bulk_group\n [tensor_map, {x, y}], [src_smem];\ncp.async.bulk.commit_group;\ncp.async.bulk.wait_group 0;\n```\n\nThe wait may be delayed to overlap independent work. It must occur before the issuing thread reuses the source shared memory or before code that requires store completion. `commit_group` creates the group; it is not itself a completion wait."}, "reason": {"statement": "List parameter constraints and direction-specific completion separately.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#asynchronous-data-copies-using-the-tensor-memory-accelerator-tma", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#encoding-a-tensor-map-on-device", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#phase-completion-of-the-mbarrier-object"]}} +{"path": "wiki/hardware/tma.md", "before": {"statement": "TMA operations are driven by a **descriptor** that encodes the tensor layout, addressing, and transfer parameters. The descriptor is created on the host and passed to the kernel."}, "after": {"statement": "A tensor map is opaque and is accessed through the tensor-map proxy. `cuTensorMapEncodeTiled` describes:\n\n- datatype and global base address;\n- rank, global dimensions, and byte strides;\n- traversal box dimensions and element strides;\n- interleave and shared-memory swizzle;\n- L2-promotion hint; and\n- out-of-bounds fill policy.\n\nFor the ordinary non-interleaved tiled path in CUDA Driver API 13.0.97, important constraints include:\n\n- the `CUtensorMap` output object is 64-byte aligned;\n- the global base and byte strides satisfy the documented alignment rules, commonly at least 16 bytes for the basic types/path;\n- every `boxDim` entry is from 1 through 256 elements;\n- `boxDim[0] * element_size` is a multiple of 16 bytes;\n- every element stride is from 1 through 8; and\n- when swizzling is enabled, the inner box in bytes does not exceed the selected swizzle span.\n\nDatatype, interleave, sub-byte, and architecture-specific modes add further restrictions. Always check the encoder's `CUresult`; never use the output after an encoding failure.\n\nHost encoding is common, but it is not the only Blackwell path. CUDA also documents device-side tensor-map construction and modification on Blackwell. A map modified through the generic proxy must be published to the tensor-map proxy with the required `fence.proxy.tensormap::generic` sequence before a TMA operation consumes it."}, "reason": {"statement": "Present host encoding as common, not exclusive, and mention device proxy ordering.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#encoding-a-tensor-map-on-device", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"]}} +{"path": "wiki/hardware/tma.md", "before": {"statement": "```cuda\n#include \n\n// Create a 2D TMA descriptor for a row-major FP16 matrix\nCUtensorMap create_tma_descriptor_2d(\n const half* global_ptr,\n int M, int N, // Global tensor dimensions\n int tile_m, int tile_n, // Tile dimensions for each transfer\n int swizzle_bytes // Swizzle mode: 0, 32, 64, 128\n) {\n CUtensorMap tensor_map;\n\n // Tensor dimensions (outermost to innermost)\n uint64_t global_dims[2] = {(uint64_t)N, (uint64_t)M};\n uint64_t global_strides[1] = {(uint64_t)(N * sizeof(half))};\n\n // Tile box dimensions\n uint32_t box_dims[2] = {(uint32_t)tile_n, (uint32_t)tile_m};\n\n // Element strides (1 = contiguous)\n uint32_t elem_strides[2] = {1, 1};\n\n CUtensorMapSwizzle swizzle;\n switch (swizzle_bytes) {\n case 0: swizzle = CU_TENSOR_MAP_SWIZZLE_NONE; break;\n case 32: swizzle = CU_TENSOR_MAP_SWIZZLE_32B; break;\n case 64: swizzle = CU_TENSOR_MAP_SWIZZLE_64B; break;\n case 128: swizzle = CU_TENSOR_MAP_SWIZZLE_128B; break;\n }\n\n cuTensorMapEncodeTiled(\n &tensor_map,\n CU_TENSOR_MAP_DATA_TYPE_FLOAT16, // Element type\n 2, // Dimensionality\n (void*)global_ptr, // Global pointer\n global_dims, // Tensor dimensions\n global_strides, // Byte strides (exclude innermost)\n box_dims, // Tile/box dimensions\n elem_strides, // Element strides\n CU_TENSOR_MAP_INTERLEAVE_NONE,\n swizzle,\n CU_TENSOR_MAP_L2_PROMOTION_NONE,\n CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE\n );\n\n return tensor_map;\n}\n```"}, "after": {"statement": "A tensor map is opaque and is accessed through the tensor-map proxy. `cuTensorMapEncodeTiled` describes:\n\n- datatype and global base address;\n- rank, global dimensions, and byte strides;\n- traversal box dimensions and element strides;\n- interleave and shared-memory swizzle;\n- L2-promotion hint; and\n- out-of-bounds fill policy.\n\nFor the ordinary non-interleaved tiled path in CUDA Driver API 13.0.97, important constraints include:\n\n- the `CUtensorMap` output object is 64-byte aligned;\n- the global base and byte strides satisfy the documented alignment rules, commonly at least 16 bytes for the basic types/path;\n- every `boxDim` entry is from 1 through 256 elements;\n- `boxDim[0] * element_size` is a multiple of 16 bytes;\n- every element stride is from 1 through 8; and\n- when swizzling is enabled, the inner box in bytes does not exceed the selected swizzle span.\n\nDatatype, interleave, sub-byte, and architecture-specific modes add further restrictions. Always check the encoder's `CUresult`; never use the output after an encoding failure.\n\nHost encoding is common, but it is not the only Blackwell path. CUDA also documents device-side tensor-map construction and modification on Blackwell. A map modified through the generic proxy must be published to the tensor-map proxy with the required `fence.proxy.tensormap::generic` sequence before a TMA operation consumes it.\n\nChoose tile rank, swizzle, multicast, issue cadence, and stage count from the actual access pattern and resource budget. More distinct stages consume more shared memory; there is no universal optimum of three to five stages. Profile the target GPU and record the copy shape, descriptor, cluster mask, stage count, shared-memory use, occupancy, warmup, repetitions, and baseline."}, "reason": {"statement": "Replace a deceptively general factory with a constraints checklist and official complete sample.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#encoding-a-tensor-map-on-device"]}} +{"path": "wiki/hardware/tma.md", "before": {"statement": "```cuda\n// TMA load: single thread issues, hardware executes asynchronously\n__device__ void tma_load_tile(\n const CUtensorMap* desc,\n void* smem_ptr,\n uint64_t* mbar_ptr, // mbarrier for synchronization\n int coord_x, int coord_y\n) {\n if (threadIdx.x == 0) {\n // Set expected bytes on the mbarrier\n uint32_t expected_bytes = TILE_M * TILE_N * sizeof(half);\n asm volatile(\n \"mbarrier.arrive.expect_tx.shared.b64 _, [%0], %1;\"\n :\n : \"r\"((uint32_t)__cvta_generic_to_shared(mbar_ptr)),\n \"r\"(expected_bytes)\n );\n\n // Issue TMA copy\n asm volatile(\n \"cp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes \"\n \"[%0], [%1, {%2, %3}], [%4];\"\n :\n : \"r\"((uint32_t)__cvta_generic_to_shared(smem_ptr)),\n \"l\"(desc),\n \"r\"(coord_x), \"r\"(coord_y),\n \"r\"((uint32_t)__cvta_generic_to_shared(mbar_ptr))\n );\n }\n}\n```"}, "after": {"statement": "PTX ISA 9.0 defines this representative 2D CTA-local form:\n\n```ptx\ncp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes\n [dst_smem], [tensor_map, {x, y}], [full_barrier];\n```\n\nThe instruction is non-blocking. A common one-producer phase with one or more loads uses this accounting:\n\n1. Initialize and publish the stage's mbarrier with the intended pending-arrival count.\n2. The producer performs one `mbarrier.arrive.expect_tx` for its software arrival and the sum of bytes that all loads in this phase will complete.\n3. Issue the TMA loads against that barrier.\n4. Each completed load performs `complete_tx` for its copied byte count; it does **not** perform another arrival.\n5. Consumers wait with the correct state token or per-stage parity and acquire semantics before reading the destination.\n\nThe phase completes only after pending arrivals and tx-count are both zero. If a design performs multiple software arrivals instead, its initialized count must match them exactly. See [mbarrier](mbarrier.md) for lifecycle, phase, and memory-ordering rules."}, "reason": {"statement": "Teach one explicit phase-accounting sequence instead of an unsafe reusable helper.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#phase-completion-of-the-mbarrier-object", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"]}} +{"path": "wiki/hardware/tma.md", "before": {"statement": "```cuda\n// TMA store: write a tile from shared memory back to global memory\n__device__ void tma_store_tile(\n const CUtensorMap* desc,\n const void* smem_ptr,\n int coord_x, int coord_y\n) {\n if (threadIdx.x == 0) {\n asm volatile(\n \"cp.async.bulk.tensor.2d.global.shared::cta \"\n \"[%0, {%1, %2}], [%3];\"\n :\n : \"l\"(desc),\n \"r\"(coord_x), \"r\"(coord_y),\n \"r\"((uint32_t)__cvta_generic_to_shared(smem_ptr))\n );\n\n // Commit the store\n asm volatile(\"cp.async.bulk.commit_group;\");\n }\n}\n```"}, "after": {"statement": "A tensor store uses bulk-group completion rather than the load's mbarrier protocol:\n\n```ptx\ncp.async.bulk.tensor.2d.global.shared::cta.bulk_group\n [tensor_map, {x, y}], [src_smem];\ncp.async.bulk.commit_group;\ncp.async.bulk.wait_group 0;\n```\n\nThe wait may be delayed to overlap independent work. It must occur before the issuing thread reuses the source shared memory or before code that requires store completion. `commit_group` creates the group; it is not itself a completion wait."}, "reason": {"statement": "Show the required qualifier and explicit wait boundary.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"]}} +{"path": "wiki/hardware/tma.md", "before": {"statement": "TMA uses mbarriers (memory barriers) for producer-consumer synchronization. The pattern is:\n\n1. **Producer** (TMA): arrives at the barrier when the transfer completes, decrementing the expected transaction count.\n2. **Consumer** (compute warps): waits on the barrier before reading the loaded data."}, "after": {"statement": "PTX ISA 9.0 defines this representative 2D CTA-local form:\n\n```ptx\ncp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes\n [dst_smem], [tensor_map, {x, y}], [full_barrier];\n```\n\nThe instruction is non-blocking. A common one-producer phase with one or more loads uses this accounting:\n\n1. Initialize and publish the stage's mbarrier with the intended pending-arrival count.\n2. The producer performs one `mbarrier.arrive.expect_tx` for its software arrival and the sum of bytes that all loads in this phase will complete.\n3. Issue the TMA loads against that barrier.\n4. Each completed load performs `complete_tx` for its copied byte count; it does **not** perform another arrival.\n5. Consumers wait with the correct state token or per-stage parity and acquire semantics before reading the destination.\n\nThe phase completes only after pending arrivals and tx-count are both zero. If a design performs multiple software arrivals instead, its initialized count must match them exactly. See [mbarrier](mbarrier.md) for lifecycle, phase, and memory-ordering rules."}, "reason": {"statement": "Separate software arrival from hardware byte completion.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#phase-completion-of-the-mbarrier-object"]}} +{"path": "wiki/hardware/tma.md", "before": {"statement": "```cuda\n// Multi-stage pipeline with TMA + mbarrier\n__device__ void pipelined_mainloop(\n const CUtensorMap* desc_a,\n const CUtensorMap* desc_b,\n void* smem_a_stages[NUM_STAGES],\n void* smem_b_stages[NUM_STAGES],\n uint64_t* mbar[NUM_STAGES],\n int num_k_tiles\n) {\n // Prologue: fill the first NUM_STAGES-1 stages\n for (int s = 0; s < NUM_STAGES - 1 && s < num_k_tiles; ++s) {\n tma_load_tile(desc_a, smem_a_stages[s], mbar[s], 0, s);\n tma_load_tile(desc_b, smem_b_stages[s], mbar[s], s, 0);\n }\n\n // Mainloop\n for (int k = 0; k < num_k_tiles; ++k) {\n int stage = k % NUM_STAGES;\n\n // Wait for data to arrive in this stage\n mbarrier_wait(mbar[stage]);\n\n // Issue MMA using this stage's SMEM buffers\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 1;\"\n :\n : \"r\"(tmem_acc),\n \"l\"(make_desc(smem_a_stages[stage])),\n \"l\"(make_desc(smem_b_stages[stage])),\n \"r\"(0)\n );\n }\n\n // Prefetch next stage\n int next_k = k + NUM_STAGES - 1;\n if (next_k < num_k_tiles) {\n int next_stage = next_k % NUM_STAGES;\n tma_load_tile(desc_a, smem_a_stages[next_stage],\n mbar[next_stage], 0, next_k);\n tma_load_tile(desc_b, smem_b_stages[next_stage],\n mbar[next_stage], next_k, 0);\n }\n }\n}\n```"}, "after": {"statement": "A common Blackwell GEMM data path is:\n\n`GMEM -> TMA -> SMEM -> tcgen05.mma -> TMEM -> tcgen05.ld -> registers -> output store`\n\nEach reusable pipeline stage needs two independent ownership transitions:\n\n- **full:** TMA has finished producing the shared-memory operands, so the MMA consumer may read them; and\n- **empty:** asynchronous MMA has stopped reading those operands, so the TMA producer may overwrite the stage.\n\nTrack full and empty state per stage. Account transaction bytes once, track each barrier's own phase, and do not equate CTA synchronization or a tcgen05 fence with async completion. A complete CUTLASS pipeline is safer evidence than a shortened helper that omits one ownership edge.\n\nChoose tile rank, swizzle, multicast, issue cadence, and stage count from the actual access pattern and resource budget. More distinct stages consume more shared memory; there is no universal optimum of three to five stages. Profile the target GPU and record the copy shape, descriptor, cluster mask, stage count, shared-memory use, occupancy, warmup, repetitions, and baseline."}, "reason": {"statement": "Replace unrepairable pseudocode with invariant-based stage lifecycle and a pinned complete implementation.", "urls": []}} +{"path": "wiki/hardware/tma.md", "before": {"statement": "```cuda\n// Initialize an mbarrier\n__device__ void mbarrier_init(uint64_t* mbar, int arrive_count) {\n if (threadIdx.x == 0) {\n asm volatile(\n \"mbarrier.init.shared.b64 [%0], %1;\"\n :\n : \"r\"((uint32_t)__cvta_generic_to_shared(mbar)),\n \"r\"(arrive_count)\n );\n }\n}\n\n// Wait for an mbarrier to complete (phase-based)\n__device__ void mbarrier_wait(uint64_t* mbar, int phase) {\n uint32_t mbar_addr = (uint32_t)__cvta_generic_to_shared(mbar);\n asm volatile(\n \"{\\n\"\n \".reg .pred p;\\n\"\n \"WAIT_LOOP:\\n\"\n \"mbarrier.try_wait.parity.shared.b64 p, [%0], %1;\\n\"\n \"@!p bra WAIT_LOOP;\\n\"\n \"}\\n\"\n :\n : \"r\"(mbar_addr), \"r\"(phase)\n );\n}\n```"}, "after": {"statement": "PTX ISA 9.0 defines this representative 2D CTA-local form:\n\n```ptx\ncp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes\n [dst_smem], [tensor_map, {x, y}], [full_barrier];\n```\n\nThe instruction is non-blocking. A common one-producer phase with one or more loads uses this accounting:\n\n1. Initialize and publish the stage's mbarrier with the intended pending-arrival count.\n2. The producer performs one `mbarrier.arrive.expect_tx` for its software arrival and the sum of bytes that all loads in this phase will complete.\n3. Issue the TMA loads against that barrier.\n4. Each completed load performs `complete_tx` for its copied byte count; it does **not** perform another arrival.\n5. Consumers wait with the correct state token or per-stage parity and acquire semantics before reading the destination.\n\nThe phase completes only after pending arrivals and tx-count are both zero. If a design performs multiple software arrivals instead, its initialized count must match them exactly. See [mbarrier](mbarrier.md) for lifecycle, phase, and memory-ordering rules.\n\nA common Blackwell GEMM data path is:\n\n`GMEM -> TMA -> SMEM -> tcgen05.mma -> TMEM -> tcgen05.ld -> registers -> output store`\n\nEach reusable pipeline stage needs two independent ownership transitions:\n\n- **full:** TMA has finished producing the shared-memory operands, so the MMA consumer may read them; and\n- **empty:** asynchronous MMA has stopped reading those operands, so the TMA producer may overwrite the stage.\n\nTrack full and empty state per stage. Account transaction bytes once, track each barrier's own phase, and do not equate CTA synchronization or a tcgen05 fence with async completion. A complete CUTLASS pipeline is safer evidence than a shortened helper that omits one ownership edge."}, "reason": {"statement": "Refer to the verified mbarrier page and state lifecycle requirements rather than ship a partial abstraction.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#phase-completion-of-the-mbarrier-object"]}} +{"path": "wiki/hardware/tma.md", "before": {"statement": "TMA multicast sends a single GMEM tile to **multiple CTAs within a cluster** simultaneously. This is critical for GEMM where the B operand is shared across M-axis tiles.\n\n```cuda\n// Multicast TMA: load B tile to all CTAs in the cluster\n__device__ void tma_multicast_load(\n const CUtensorMap* desc,\n void* smem_ptr,\n uint64_t* mbar_ptr,\n int coord_x, int coord_y,\n uint16_t multicast_mask // bitmask: which CTAs in cluster receive the data\n) {\n if (threadIdx.x == 0) {\n uint32_t expected_bytes = TILE_K * TILE_N * sizeof(half);\n asm volatile(\n \"mbarrier.arrive.expect_tx.shared.b64 _, [%0], %1;\"\n :\n : \"r\"((uint32_t)__cvta_generic_to_shared(mbar_ptr)),\n \"r\"(expected_bytes)\n );\n\n asm volatile(\n \"cp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes.multicast::cluster \"\n \"[%0], [%1, {%2, %3}], [%4], %5;\"\n :\n : \"r\"((uint32_t)__cvta_generic_to_shared(smem_ptr)),\n \"l\"(desc),\n \"r\"(coord_x), \"r\"(coord_y),\n \"r\"((uint32_t)__cvta_generic_to_shared(mbar_ptr)),\n \"h\"(multicast_mask)\n );\n }\n}\n```"}, "after": {"statement": "The cluster form copies a global tile to the same shared-memory offset in every CTA selected by a 16-bit mask:\n\n```ptx\ncp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes.multicast::cluster\n [dst_smem], [tensor_map, {x, y}], [full_barrier], cta_mask;\n```\n\nFor `cta_group::1` (the default), the completion signal is also multicast to the same barrier offset in every selected destination CTA. A correct design therefore:\n\n- launches an explicit cluster and keeps every destination CTA's shared memory alive;\n- initializes corresponding destination barriers before the elected issuer can start;\n- uses one cluster-wide elected issuer, not one `threadIdx.x == 0` issuer in every CTA;\n- includes only valid destination CTA ranks in the mask; and\n- has every destination wait on its own corresponding barrier phase before consuming the tile.\n\nFor GEMM tiles with the same N range and different M ranges, the CTAs use different A tiles but the same B tile, so multicast can avoid duplicate logical B-load requests. It does not promise an exact `cluster_size` reduction in measured DRAM traffic or elapsed time; caches, mask population, transaction behavior, and resource costs affect the result."}, "reason": {"statement": "Replace incomplete executable code with the cluster ownership and barrier invariants.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"]}} +{"path": "wiki/hardware/tma.md", "before": {"statement": "```\nCluster: 2 CTAs (CTA0 and CTA1) each computing different M-tiles of the same N column\n\nCTA0: computes C[0:128, 0:256] -- needs A[0:128, :] and B[:, 0:256]\nCTA1: computes C[128:256, 0:256] -- needs A[128:256, :] and B[:, 0:256]\n\nB[:, 0:256] is SHARED -- multicast it once from GMEM to both CTAs\nA tiles are UNIQUE -- each CTA loads its own A tile\n\nResult: B bandwidth is halved (1 GMEM read serves 2 CTAs)\n```"}, "after": {"statement": "The cluster form copies a global tile to the same shared-memory offset in every CTA selected by a 16-bit mask:\n\n```ptx\ncp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes.multicast::cluster\n [dst_smem], [tensor_map, {x, y}], [full_barrier], cta_mask;\n```\n\nFor `cta_group::1` (the default), the completion signal is also multicast to the same barrier offset in every selected destination CTA. A correct design therefore:\n\n- launches an explicit cluster and keeps every destination CTA's shared memory alive;\n- initializes corresponding destination barriers before the elected issuer can start;\n- uses one cluster-wide elected issuer, not one `threadIdx.x == 0` issuer in every CTA;\n- includes only valid destination CTA ranks in the mask; and\n- has every destination wait on its own corresponding barrier phase before consuming the tile.\n\nFor GEMM tiles with the same N range and different M ranges, the CTAs use different A tiles but the same B tile, so multicast can avoid duplicate logical B-load requests. It does not promise an exact `cluster_size` reduction in measured DRAM traffic or elapsed time; caches, mask population, transaction behavior, and resource costs affect the result.\n\nA common Blackwell GEMM data path is:\n\n`GMEM -> TMA -> SMEM -> tcgen05.mma -> TMEM -> tcgen05.ld -> registers -> output store`\n\nEach reusable pipeline stage needs two independent ownership transitions:\n\n- **full:** TMA has finished producing the shared-memory operands, so the MMA consumer may read them; and\n- **empty:** asynchronous MMA has stopped reading those operands, so the TMA producer may overwrite the stage.\n\nTrack full and empty state per stage. Account transaction bytes once, track each barrier's own phase, and do not equate CTA synchronization or a tcgen05 fence with async completion. A complete CUTLASS pipeline is safer evidence than a shortened helper that omits one ownership edge."}, "reason": {"statement": "State logical load reuse without an unmeasured hardware-traffic factor.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"]}} +{"path": "wiki/hardware/tma.md", "before": {"statement": "```\nWithout swizzle (linear):\n Row 0: bytes [0, 1, 2, ..., 127]\n Row 1: bytes [128, 129, ..., 255]\n\nWith 128B swizzle:\n Row 0: bytes [0, 1, ..., 127] (unchanged)\n Row 1: bytes [128, 129, ..., 255] XOR pattern applied\n Row 2: bytes [256, ...] XOR pattern applied differently\n ...\n```\n\nThe swizzle eliminates bank conflicts when the tensor core reads operand tiles from SMEM."}, "after": {"statement": "TMA supports no swizzle and multiple swizzled shared-memory layouts, including 32B, 64B, and 128B spans plus newer variants for selected types. Swizzling rearranges chunks across shared-memory banks. The consumer must address the matching logical layout; the exact mapping also depends on the documented shared-memory base-offset rule.\n\nThere is no universal rule that every Blackwell or `tcgen05.mma` input uses 128B swizzling. The TMA tensor-map swizzle, destination base alignment, leading dimension, and the tcgen05 shared-memory descriptor must describe the **same** legal layout. PTX defines no-, 32B-, 64B-, and 128B-swizzled tcgen05 descriptors with kind-, type-, shape-, and layout-specific constraints.\n\nA matched swizzle can reduce or remove bank conflicts for a particular access pattern. It does not make every possible consumer access conflict-free."}, "reason": {"statement": "Link the normative mapping and explain the producer/consumer layout match.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#the-swizzle-modes", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-layout"]}} +{"path": "wiki/hardware/tma.md", "before": {"statement": "| Maximize TMA utilization | Keep the TMA unit busy with back-to-back loads across pipeline stages |\n| Use multicast for shared operands | Reduces GMEM bandwidth by cluster_size x for shared tiles |\n| Always use 128B swizzle on Blackwell | Non-128B swizzle produces incorrect tcgen05 results |\n| Prefer 2D TMA over manual addressing | TMA handles out-of-bounds clamping, padding, and strided access |\n| Pipeline depth | 3-5 stages typically optimal; more stages increase SMEM usage |"}, "after": {"statement": "A common Blackwell GEMM data path is:\n\n`GMEM -> TMA -> SMEM -> tcgen05.mma -> TMEM -> tcgen05.ld -> registers -> output store`\n\nEach reusable pipeline stage needs two independent ownership transitions:\n\n- **full:** TMA has finished producing the shared-memory operands, so the MMA consumer may read them; and\n- **empty:** asynchronous MMA has stopped reading those operands, so the TMA producer may overwrite the stage.\n\nTrack full and empty state per stage. Account transaction bytes once, track each barrier's own phase, and do not equate CTA synchronization or a tcgen05 fence with async completion. A complete CUTLASS pipeline is safer evidence than a shortened helper that omits one ownership edge."}, "reason": {"statement": "Replace folklore with a measurement and resource checklist.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#asynchronous-data-copies-using-the-tensor-memory-accelerator-tma"]}} +{"path": "wiki/hardware/tma.md", "before": {"statement": "```python\n# CuTe-DSL TMA copy setup for Blackwell GEMM\nfrom cute import *\n\n# Define TMA copy atom for operand A (BF16, 128x64 tile)\ntma_a = make_tma_copy(\n SM100_TMA_LOAD_2D,\n tensor_a, # global tensor\n smem_layout_a, # shared memory layout\n tile_shape=(128, 64), # tile dimensions\n swizzle=Swizzle(7, 0, 4), # 128-byte swizzle\n multicast_mask=None # no multicast for A\n)\n\n# Define TMA copy atom for operand B with multicast\ntma_b = make_tma_copy(\n SM100_TMA_LOAD_2D_MULTICAST,\n tensor_b,\n smem_layout_b,\n tile_shape=(64, 256),\n swizzle=Swizzle(7, 0, 4), # 128-byte swizzle\n multicast_mask=cluster_mask # multicast B to all CTAs in cluster\n)\n```"}, "after": {"statement": "Choose tile rank, swizzle, multicast, issue cadence, and stage count from the actual access pattern and resource budget. More distinct stages consume more shared memory; there is no universal optimum of three to five stages. Profile the target GPU and record the copy shape, descriptor, cluster mask, stage count, shared-memory use, occupancy, warmup, repetitions, and baseline."}, "reason": {"statement": "Link a complete pinned tutorial rather than fabricate a simplified API.", "urls": []}} +{"path": "wiki/kernels/deepgemm.md", "before": {"statement": "DeepGEMM is DeepSeek's open-source FP8 GEMM library providing high-performance matrix multiplication with fine-grained per-tile/per-block scaling. The core kernel is remarkably compact (~300 lines), yet achieves approximately 90% utilization on H800. It supports both Hopper (SM90 via WGMMA) and Blackwell (SM100 via tcgen05.mma) architectures, and includes specialized MoE grouped GEMM layouts."}, "after": {"statement": "DeepGEMM is DeepSeek's open-source tensor-core kernel library. This page describes the FP8 GEMM paths at commit [`891d57b4db1071624b5c8fa0d1e51cb317fa709f`](https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f), with byte-verified SM90 and SM100 kernel files stored locally. The pinned project requires an SM90 or SM100 GPU, CUDA 12.3 or newer for SM90, and CUDA 12.9 or newer for SM100.\n\nDeepGEMM contains multiple kernels and APIs; the local SM90 and SM100 files below are representative FP8 1D1D implementations, not the whole library.\n\nThe pinned README reports that DeepGEMM reached **up to 1550 TFLOPS on H800** in an April 2025 news item. It does not bind that peak to `M=N=K=4096`, state approximately 90% utilization, or preserve enough benchmark conditions for reproduction. The unsupported structured performance record and table have therefore been removed; the surviving number is source-reported only and should not be compared across software, clocks, shapes, or GPUs without a controlled rerun."}, "reason": {"statement": "Scope the overview to the pinned library and separate the surviving source-reported peak from unsupported utilization and size claims.", "urls": ["https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://github.com/deepseek-ai/DeepGEMM/pull/86"]}} +{"path": "wiki/kernels/deepgemm.md", "before": {"statement": "The key innovation is the fine-grained quantization scheme: tile-wise 1x128 scaling for activations and block-wise 128x128 scaling for weights, which prevents outlier values from destroying quantization precision.\n\n```\nActivations (tile-wise 1x128):\n +-----------+-----------+-----------+\n | tile 0 | tile 1 | tile 2 | <-- each tile: 1 row x 128 cols\n | scale: s0 | scale: s1 | scale: s2 | <-- one FP32 scale per tile\n +-----------+-----------+-----------+\n\nWeights (block-wise 128x128):\n +---------------+---------------+\n | block (0,0) | block (0,1) | <-- each block: 128 rows x 128 cols\n | scale: s_00 | scale: s_01 | <-- one FP32 scale per block\n +---------------+---------------+\n | block (1,0) | block (1,1) |\n | scale: s_10 | scale: s_11 |\n +---------------+---------------+\n```"}, "after": {"statement": "The [DeepSeek-V3 Technical Report v2](https://arxiv.org/abs/2412.19437v2) defines the training scheme that motivates this path:\n\n- activations are grouped per token and per 128 channels, forming `1 x 128` tiles;\n- weights are grouped per 128 input channels and 128 output channels, forming `128 x 128` blocks; and\n- smaller groups let scales adapt more locally, which the paper says better accommodates outliers. This is a scoped accuracy motivation, not a guarantee that quantization error disappears.\n\nScale representation is architecture-specific in the pinned DeepGEMM interface. SM90 consumes FP32 scale factors. SM100 consumes packed UE8M0 factors, four UE8M0 values per `torch.int`. The pinned SM90 1D1D kernel fixes `BLOCK_K == 128`; the SM100 1D1D template accepts K scale granularities of 32 or 128 for each operand."}, "reason": {"statement": "Preserve exact granularity while qualifying the numerical benefit and separating architecture-specific scale representations.", "urls": ["https://arxiv.org/abs/2412.19437v2", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh"]}} +{"path": "wiki/kernels/deepgemm.md", "before": {"statement": "On Hopper, the Tensor Core accumulator has limited precision (~FP22, not true FP32). DeepGEMM mitigates this by promoting partial sums to a separate FP32 accumulator on CUDA Cores every Nc=128 columns (4 consecutive WGMMA operations)."}, "after": {"statement": "The DeepSeek-V3 report characterizes H800 FP8 Tensor Core accumulation as retaining about 14 bits. Its `Nc=128` strategy accumulates 128 elements of the GEMM inner dimension—four WGMMAs in the described configuration—before moving the partial result to FP32 registers on CUDA cores.\n\nThe pinned [`sm90_fp8_gemm_1d1d.cuh`](../../artifacts/kernels/deepgemm/full/sm90_fp8_gemm_1d1d.cuh) implements that structure directly:\n\n1. A math warp-group owns `float accum[...]` for WGMMA output and zero-initialized `float final_accum[...]` for promoted results.\n2. For each 128-element K block, it reads the A and B FP32 factors from shared memory, issues `BLOCK_K / WGMMA::K` WGMMA operations, commits the group, and waits for completion.\n3. The `Promote with scales` loop multiplies each partial result by its A and B factors and adds it to `final_accum`.\n4. After all K blocks, the kernel stages the final FP32 values for the epilogue/store path.\n\nThe exact upstream file is the reproducible reference. It should not be replaced by a sketch using half accumulators or by deriving the promotion interval from WGMMA's output-N tile.\n\nThe following is a verbatim fragment of the SM90 promotion loop; the linked full file supplies its surrounding declarations and loop bounds:\n\n```cpp\nconst float &scale_b_0 = scales_b[i].x;\nconst float &scale_b_1 = scales_b[i].y;\nfinal_accum[i * 4 + 0] += scale_a_0 * scale_b_0 * accum[i * 4 + 0];\nfinal_accum[i * 4 + 1] += scale_a_0 * scale_b_1 * accum[i * 4 + 1];\nfinal_accum[i * 4 + 2] += scale_a_1 * scale_b_0 * accum[i * 4 + 2];\nfinal_accum[i * 4 + 3] += scale_a_1 * scale_b_1 * accum[i * 4 + 3];\n```"}, "reason": {"statement": "Use the paper's measured retained-bit characterization and exact K-interval definition.", "urls": ["https://arxiv.org/abs/2412.19437v2", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh"]}} +{"path": "wiki/kernels/deepgemm.md", "before": {"statement": "```cpp\n// SM90 path: WGMMA with Nc=128 CUDA Core promotion\n// Every 4 WGMMAs, promote accumulated result to FP32 CUDA core accumulator\nconstexpr int Nc = 128; // Promotion interval (4 WGMMAs of n=32 each)\nconstexpr int WGMMA_N = 32;\n\nfloat cuda_core_acc[TILE_M][TILE_N] = {0}; // FP32 accumulator on CUDA Cores\n\nfor (int k = 0; k < K; k += Nc) {\n // Run 4 consecutive WGMMAs with TC-limited precision accumulation\n __half2 tc_acc[TILE_M][WGMMA_N]; // Tensor Core accumulator (~FP22)\n memset(tc_acc, 0, sizeof(tc_acc));\n\n for (int sub_k = 0; sub_k < Nc; sub_k += WGMMA_K) {\n wgmma_mma_async(tc_acc, A_smem + sub_k, B_smem + sub_k);\n }\n wgmma_wait();\n\n // Promote: add TC result to CUDA Core FP32 accumulator\n // This prevents precision loss from repeated FP22 accumulation\n for (int m = 0; m < TILE_M; m++)\n for (int n = 0; n < TILE_N; n++)\n cuda_core_acc[m][n] += (float)tc_acc[m][n] * scale_a[m] * scale_b[n];\n}\n```"}, "after": {"statement": "The DeepSeek-V3 report characterizes H800 FP8 Tensor Core accumulation as retaining about 14 bits. Its `Nc=128` strategy accumulates 128 elements of the GEMM inner dimension—four WGMMAs in the described configuration—before moving the partial result to FP32 registers on CUDA cores.\n\nThe pinned [`sm90_fp8_gemm_1d1d.cuh`](../../artifacts/kernels/deepgemm/full/sm90_fp8_gemm_1d1d.cuh) implements that structure directly:\n\n1. A math warp-group owns `float accum[...]` for WGMMA output and zero-initialized `float final_accum[...]` for promoted results.\n2. For each 128-element K block, it reads the A and B FP32 factors from shared memory, issues `BLOCK_K / WGMMA::K` WGMMA operations, commits the group, and waits for completion.\n3. The `Promote with scales` loop multiplies each partial result by its A and B factors and adds it to `final_accum`.\n4. After all K blocks, the kernel stages the final FP32 values for the epilogue/store path.\n\nThe exact upstream file is the reproducible reference. It should not be replaced by a sketch using half accumulators or by deriving the promotion interval from WGMMA's output-N tile.\n\nThe following is a verbatim fragment of the SM90 promotion loop; the linked full file supplies its surrounding declarations and loop bounds:\n\n```cpp\nconst float &scale_b_0 = scales_b[i].x;\nconst float &scale_b_1 = scales_b[i].y;\nfinal_accum[i * 4 + 0] += scale_a_0 * scale_b_0 * accum[i * 4 + 0];\nfinal_accum[i * 4 + 1] += scale_a_0 * scale_b_1 * accum[i * 4 + 1];\nfinal_accum[i * 4 + 2] += scale_a_1 * scale_b_0 * accum[i * 4 + 2];\nfinal_accum[i * 4 + 3] += scale_a_1 * scale_b_1 * accum[i * 4 + 3];\n```"}, "reason": {"statement": "Replace invented executable-looking code with a precise walkthrough and link to the byte-verified source.", "urls": ["https://arxiv.org/abs/2412.19437v2", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh"]}} +{"path": "wiki/kernels/deepgemm.md", "before": {"statement": "```cpp\n// SM100 path: tcgen05.mma with native block scaling\n// Scaling factors packed as UE8M0 (4 values per uint32)\n// No explicit CUDA core promotion needed -- TMEM accumulates in full precision\n\n// Pack 4 UE8M0 scale factors into a single uint32\nuint32_t packed_scales = pack_ue8m0(sf[0], sf[1], sf[2], sf[3]);\n\n// tcgen05.mma reads A/B from SMEM, accumulates into TMEM\n// Block scale applied natively during MMA\nasm volatile(\n \"tcgen05.mma.cta_group::1.kind::f8f6f4\"\n \" [%0], %1, %2, %3, %4;\"\n :\n : \"l\"(tmem_addr), \"l\"(a_smem_addr), \"l\"(b_smem_addr),\n \"r\"(packed_scales), \"n\"(SCALE_D_ENABLED)\n);\n```"}, "after": {"statement": "The pinned [`sm100_fp8_gemm_1d1d.cuh`](../../artifacts/kernels/deepgemm/full/sm100_fp8_gemm_1d1d.cuh) uses a different data path:\n\n1. It creates a block-scaled UMMA instruction descriptor with a `float` accumulator type and `cutlass::float_ue8m0_t` scale type.\n2. TMA loads packed scale-factor data into shared memory. UTCCP copies selected scale blocks from shared memory into dedicated SFA and SFB columns in TMEM.\n3. The elected issuing thread calls the SM100 UMMA wrapper with shared-memory operand descriptors, the TMEM accumulator column, a runtime instruction descriptor containing scale IDs, and the two TMEM scale addresses.\n4. The epilogue drains the TMEM accumulator after the kernel's full/empty barrier protocol says it is ready.\n\nThis path does not contain the SM90 `final_accum` CUDA-core promotion loop. That difference does not justify a blanket claim that every tcgen05 accumulator mode or datatype has \"full FP32\" behavior; the accumulator type and instruction form remain part of the selected operation."}, "reason": {"statement": "Point to the exact upstream block-scaled UMMA sequence instead of presenting fabricated PTX.", "urls": ["https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh"]}} +{"path": "wiki/kernels/deepgemm.md", "before": {"statement": "DeepGEMM provides three grouped GEMM layouts tailored for MoE workloads, where only the M-axis varies (different token counts per expert) while N and K remain fixed:\n\n```\nLayout 1: Contiguous (prefill)\n Expert 0: M0 tokens ──┐\n Expert 1: M1 tokens ──┤── All packed contiguously in memory\n Expert 2: M2 tokens ──┘ Index array stores cumulative offsets\n\nLayout 2: Masked (decode with CUDA graphs)\n Fixed-size M_max allocation per expert\n Binary mask indicates valid tokens\n Compatible with CUDA graph capture (no dynamic shapes)\n\nLayout 3: K-grouped (weight gradients)\n Groups along K-axis instead of M-axis\n Used for computing dW in MoE training backward\n```\n\n```cpp\n// Contiguous grouped GEMM dispatch\n// problem_sizes[i] = {M_i, N, K} for expert i\nvoid grouped_gemm_contiguous(\n const fp8_t* A, // All expert inputs packed\n const fp8_t* B, // Expert weights [num_experts, N, K]\n float* C, // Output packed\n const int* offsets, // Cumulative M offsets per expert\n int num_experts\n) {\n // Each thread block picks an expert via tile scheduling\n // M-axis tiles distributed across experts using offset lookup\n int expert_id = binary_search(offsets, num_experts, tile_m_start);\n int local_m = tile_m_start - offsets[expert_id];\n\n // Standard GEMM tile with expert-specific B matrix\n compute_tile(A + offsets[expert_id] * K,\n B + expert_id * N * K,\n C + offsets[expert_id] * N,\n local_m, N, K);\n}\n```\n\n- MoE expert computation with variable token counts per expert"}, "after": {"statement": "The pinned interface distinguishes three workload arrangements rather than treating all of them as M-varying layouts:\n\n| M-grouped contiguous | M | N and K | Either a group index for each packed M row or a prefix-sum M layout, depending on the selected option |\n| M-grouped masked | Valid M within each `[G, M, K]` allocation | Maximum M, N, and K tensor extents | An integer `masked_m[G]` vector holding each group's valid M length, not a binary mask |\n| K-grouped contiguous | K | M and N | Per-group K lengths plus their device tensor; used for weight-gradient-style grouped GEMM |\n\nContiguous M grouping is intended for variable per-expert token counts in training forward or inference prefill. Masked M grouping keeps fixed allocations suitable for CUDA-graph decode while limiting work to each group's valid M. The pinned repository provides K-grouped NT on SM90 and K-grouped TN on SM100 for the documented FP8 paths.\n\n- M-grouped contiguous segments must satisfy the configured M/K alignment. Masked mode uses valid-length integers, while K-grouped mode has different shapes and architecture-specific layout variants."}, "reason": {"statement": "Separate M-grouped contiguous, M-grouped masked, and K-grouped APIs with their exact invariants.", "urls": ["https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp"]}} +{"path": "wiki/kernels/deepgemm.md", "before": {"statement": "DeepGEMM uses JIT compilation via NVRTC to specialize kernels per problem shape at runtime. This avoids the combinatorial explosion of pre-compiled template instantiations while still achieving optimal register allocation and loop unrolling.\n\n```cpp\n// Lightweight JIT module: compile per-shape kernel at first call\nauto kernel = jit_compile(\n \"deepgemm_fp8\",\n {{\"M\", M}, {\"N\", N}, {\"K\", K},\n {\"BLOCK_M\", 128}, {\"BLOCK_N\", 128}, {\"BLOCK_K\", 64},\n {\"NUM_STAGES\", 4}}\n);\nkernel.launch(A, B, C, scales_a, scales_b, stream);\n```"}, "after": {"statement": "DeepGEMM generates and compiles kernel source at runtime. At the pinned commit, the compiler defaults to NVCC. Setting `DG_JIT_USE_NVRTC=1` selects the optional NVRTC path; the project warns that this may reduce performance for some cases.\n\nThe cache key includes the kernel name, compiler signature, compiler flags, and generated source. A cache hit reuses the existing kernel runtime; a miss compiles a CUBIN in a temporary directory and then publishes the completed cache entry. Consequently, a new specialization has first-use compilation/loading cost, whereas matching later calls can reuse the artifact. This mechanism does not guarantee globally optimal register allocation or unrolling.\n\n- JIT cache misses add compilation latency, and NVRTC is optional rather than the default."}, "reason": {"statement": "Describe the generated-code cache and compiler selection exactly; remove the fabricated API and optimality guarantee.", "urls": ["https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/jit/compiler.hpp"]}} +{"path": "wiki/kernels/deepgemm.md", "before": {"statement": "performance_claims:\n- gpu: H800\n dtype: fp8\n shape: M=4096, N=4096, K=4096\n metric: TFLOPS\n value: 1550\n utilization: ~90%\n source_id: blog-deepgemm\n\n| H800 | FP8 | M=4096, N=4096, K=4096 | 1550 | ~90% |"}, "after": {"statement": "performance_claims: []\n\nThe pinned README reports that DeepGEMM reached **up to 1550 TFLOPS on H800** in an April 2025 news item. It does not bind that peak to `M=N=K=4096`, state approximately 90% utilization, or preserve enough benchmark conditions for reproduction. The unsupported structured performance record and table have therefore been removed; the surviving number is source-reported only and should not be compared across software, clocks, shapes, or GPUs without a controlled rerun."}, "reason": {"statement": "Delete the unsupported structured measurement and retain only the exact source-reported up-to statement with its missing-condition caveat.", "urls": ["https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://github.com/deepseek-ai/DeepGEMM/pull/86"]}} +{"path": "wiki/kernels/deepgemm.md", "before": {"statement": "- Fine-grained scaling adds overhead vs. per-tensor scaling -- only beneficial when outlier sensitivity matters"}, "after": {"statement": "- Input transposition, FP8 casting, and scale-layout preparation are separate from the optimized GEMM kernels; the project supplies utilities but warns they may be slower than fusing the work into producers.\n\n- Fine-grained scaling's accuracy and cost tradeoffs depend on quantization recipe, target architecture, scale preparation, fusion, and workload. There is no universal per-tensor-scaling penalty or single \"use only for outliers\" rule."}, "reason": {"statement": "Replace the categorical rule with architecture- and workload-scoped tradeoffs.", "urls": ["https://arxiv.org/abs/2412.19437v2", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"]}} +{"path": "wiki/kernels/flash-attention-4.md", "before": {"statement": "Written entirely in CuTe DSL (Python), FA4 compiles 20-30x faster than equivalent CUTLASS C++ template code while matching or exceeding cuDNN performance."}, "after": {"statement": "FlashAttention-4 is an attention algorithm and CuTe DSL implementation designed around Blackwell's asymmetric throughput: B200 tensor-core throughput grows much more than its special-function and shared-memory resources. This page separates two scopes:\n\n- the [FA4 paper v1](https://arxiv.org/abs/2603.05451v1), which studies the SM100/B200 algorithm and reports the authors' measurements; and\n- the public implementation at Dao-AILab/flash-attention commit [`a369df707e1980fb328abcc1733e3457ec10155f`](https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute), which is the source snapshot used for implementation statements below.\n\nThe paper implementation is written in CuTe DSL. Its compilation comparison is against corresponding FA3 CUTLASS kernels: forward compiles in 2.5 seconds instead of 55 seconds, and backward in 1.4 seconds instead of 45 seconds. Those are single-kernel source-reported compile measurements, not an end-to-end installation or runtime speedup.\n\nThe first-party sources contain two different source-reported peak values. Paper v1 reports **up to 1613 TFLOPS/s on B200 BF16, or 71% of the peak convention used by the authors**. Tri Dao's blog reports **up to 1605 TFLOPS/s, also labeled 71%**, plus up to 1.3x over cuDNN 9.13 and up to 2.7x over Triton.\n\nThe paper's benchmark suite spans sequence lengths from 1K through 32K and multiple query/value head-dimension pairs under a fixed total-token convention. Neither textual source establishes the former single row that attached 1605 TFLOPS, 71%, and both speedup ranges specifically to `seqlen=8192, headdim=128`. The structured performance record is therefore empty, and no complement-of-71% time breakdown is inferred."}, "reason": {"statement": "Separate the measured compile comparison from scoped runtime maxima.", "urls": ["https://arxiv.org/html/2603.05451v1", "https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute", "https://tridao.me/blog/2026/flash4/"]}} +{"path": "wiki/kernels/flash-attention-4.md", "before": {"statement": "```python\n# CuTe DSL: Ping-pong tile scheduling (simplified)\n# Two query tiles per CTA, alternating MMA and softmax phases\n\n@cute.kernel\ndef flash_attention_4_fwd(Q, K, V, O, L):\n # Each CTA processes 2 query tiles of 128 tokens\n TILE_Q = 128\n NUM_TILES = 2\n\n # Phase A: tile_0 does MMA(Q0, K), tile_1 does softmax rescale\n # Phase B: tile_1 does MMA(Q1, K), tile_0 does softmax rescale\n\n for kv_block in range(num_kv_blocks):\n # Ping: MMA on tile 0, softmax on tile 1\n with warpgroup(mma_wg):\n S0 = cute.mma(Q_tile[0], K_block) # tcgen05.mma -> TMEM\n with warpgroup(softmax_wg):\n O_tile[1], L_tile[1] = rescale_and_accumulate(\n O_tile[1], L_tile[1], S1, V_prev\n )\n\n # Pong: MMA on tile 1, softmax on tile 0\n with warpgroup(mma_wg):\n S1 = cute.mma(Q_tile[1], K_block)\n with warpgroup(softmax_wg):\n O_tile[0], L_tile[0] = rescale_and_accumulate(\n O_tile[0], L_tile[0], S0, V_block\n )\n```"}, "after": {"statement": "The forward kernel assigns two output tiles of 128 query rows to one CTA. One MMA warp issues the matrix products, two four-warp softmax groups serve the two output tiles, and a correction warpgroup handles accumulator corrections. The score and output accumulators live in disjoint TMEM regions. Pipeline barriers allow softmax work for one output tile to overlap MMA work that advances the other tile.\n\nThis is more than ordinary K/V double buffering: the two alternating objects are output tiles with separate softmax state. The exact synchronization and stage ownership are implementation details; the invented one-loop pseudocode formerly on this page did not preserve those dependencies."}, "reason": {"statement": "The paper's verified schedule can be stated precisely without publishing misleading pseudo-code.", "urls": ["https://arxiv.org/html/2603.05451v1"]}} +{"path": "wiki/kernels/flash-attention-4.md", "before": {"statement": "The SFU `ex2` instruction is the bottleneck on Blackwell -- its throughput does not scale with the doubled tensor core rate. FA4 replaces it with a software exponential distributed across FMA units using Cody-Waite range reduction and Horner polynomial evaluation."}, "after": {"statement": "FA4 does not replace every hardware exponential. The paper selects only about 10-25% of entries for software evaluation on FMA units and leaves the rest on the hardware MUFU `ex2` path, allowing both resources to contribute.\n\nFor a software-selected value, the published range reduction writes `x = n + f` with `n = floor(x)` and `f` in `[0, 1)`, evaluates a degree-3 polynomial for `2**f`, and reconstructs the scale from `n`. The function below is a scalar reference using the rounded coefficients printed in the first-party blog. It illustrates that formula; it is not the CuTe kernel's vector selection, clamping, or scheduling code.\n\n```python\nimport math\n\ndef fa4_blog_exp2_reference(x: float) -> float:\n n = math.floor(x)\n f = x - n\n polynomial = 1.0 + f * (0.6951 + f * (0.2276 + f * 0.0771))\n return math.ldexp(polynomial, n)\n```\n\nNo standalone four-times software-versus-hardware exponential result is asserted here. The paper evaluates the combined kernel and its ablations rather than establishing that former page claim."}, "reason": {"statement": "Preserve the optimization while restoring its hybrid scope.", "urls": ["https://arxiv.org/html/2603.05451v1", "https://tridao.me/blog/2026/flash4/"]}} +{"path": "wiki/kernels/flash-attention-4.md", "before": {"statement": "```python\n# Software exp2 via Cody-Waite range reduction + Horner polynomial\n# Distributes across FMA units instead of using scarce SFU hardware\n\ndef software_exp2(x):\n \"\"\"\n Compute 2^x using FMA units instead of SFU ex2.\n Cody-Waite range reduction splits x into integer + fraction.\n Horner polynomial approximates 2^frac.\n \"\"\"\n # Range reduction: x = n + f, where n is integer, f in [-0.5, 0.5]\n n = round(x)\n f = x - n # Cody-Waite: use extended precision subtraction\n\n # Horner polynomial for 2^f on [-0.5, 0.5]\n # Coefficients chosen for bf16 precision target\n p = f * (C5 + f * (C4 + f * (C3 + f * (C2 + f * C1))))\n p = 1.0 + p\n\n # Reconstruct: 2^x = 2^n * 2^f\n return ldexp(p, n) # Integer exponent via bit manipulation\n```"}, "after": {"statement": "FA4 does not replace every hardware exponential. The paper selects only about 10-25% of entries for software evaluation on FMA units and leaves the rest on the hardware MUFU `ex2` path, allowing both resources to contribute.\n\nFor a software-selected value, the published range reduction writes `x = n + f` with `n = floor(x)` and `f` in `[0, 1)`, evaluates a degree-3 polynomial for `2**f`, and reconstructs the scale from `n`. The function below is a scalar reference using the rounded coefficients printed in the first-party blog. It illustrates that formula; it is not the CuTe kernel's vector selection, clamping, or scheduling code.\n\n```python\nimport math\n\ndef fa4_blog_exp2_reference(x: float) -> float:\n n = math.floor(x)\n f = x - n\n polynomial = 1.0 + f * (0.6951 + f * (0.2276 + f * 0.0771))\n return math.ldexp(polynomial, n)\n```\n\nNo standalone four-times software-versus-hardware exponential result is asserted here. The paper evaluates the combined kernel and its ablations rather than establishing that former page claim."}, "reason": {"statement": "Replace it with a clearly labeled scalar reference for the published degree-3 range reduction.", "urls": ["https://arxiv.org/html/2603.05451v1", "https://tridao.me/blog/2026/flash4/"]}} +{"path": "wiki/kernels/flash-attention-4.md", "before": {"statement": "This gives roughly 4x throughput improvement over the hardware SFU path by utilizing FMA units that would otherwise be idle during softmax phases."}, "after": {"statement": "FA4 does not replace every hardware exponential. The paper selects only about 10-25% of entries for software evaluation on FMA units and leaves the rest on the hardware MUFU `ex2` path, allowing both resources to contribute.\n\nFor a software-selected value, the published range reduction writes `x = n + f` with `n = floor(x)` and `f` in `[0, 1)`, evaluates a degree-3 polynomial for `2**f`, and reconstructs the scale from `n`. The function below is a scalar reference using the rounded coefficients printed in the first-party blog. It illustrates that formula; it is not the CuTe kernel's vector selection, clamping, or scheduling code.\n\n```python\nimport math\n\ndef fa4_blog_exp2_reference(x: float) -> float:\n n = math.floor(x)\n f = x - n\n polynomial = 1.0 + f * (0.6951 + f * (0.2276 + f * 0.0771))\n return math.ldexp(polynomial, n)\n```\n\nNo standalone four-times software-versus-hardware exponential result is asserted here. The paper evaluates the combined kernel and its ablations rather than establishing that former page claim."}, "reason": {"statement": "No adequate oracle supports the numeric factor.", "urls": ["https://arxiv.org/html/2603.05451v1", "https://tridao.me/blog/2026/flash4/"]}} +{"path": "wiki/kernels/flash-attention-4.md", "before": {"statement": "```python\n# Only rescale when max changes substantially\ndef conditional_rescale(O_acc, lse_old, lse_new, threshold=2.0):\n diff = lse_new - lse_old\n if abs(diff) > threshold:\n # Full rescale: O_acc *= exp(lse_old - lse_new)\n scale = software_exp2((lse_old - lse_new) * LOG2E)\n O_acc = O_acc * scale\n # Otherwise: skip rescale, accumulate normally\n return O_acc\n```"}, "after": {"statement": "Ordinary online softmax updates the row maximum and rescales accumulated state as each score block arrives. FA4 permits its retained maximum to lag: it resynchronizes only when the new block maximum exceeds the retained maximum by more than a threshold. The paper's typical threshold is `tau = log2(256) = 8.0` in the exponent's base-2 units.\n\nWhen a rescale is skipped, subsequent probabilities are still evaluated relative to the retained old maximum, and auxiliary statistics track the delayed normalization. The algorithm performs final renormalization at the end. Comparing absolute changes in LSE, changing the maximum anyway, or simply skipping the accumulator multiply is not equivalent."}, "reason": {"statement": "Use the verified invariant rather than repair a misleading miniature implementation.", "urls": ["https://arxiv.org/html/2603.05451v1"]}} +{"path": "wiki/kernels/flash-attention-4.md", "before": {"statement": "The backward pass spans two paired CTAs in a cluster, sharing TMEM across both SMs. This halves shared memory traffic for the dQ/dK/dV gradient computation."}, "after": {"statement": "The paper maps five backward GEMMs to two-CTA tcgen05 MMA with `M=256, N=128, K=128`. For those operations, the paired CTAs can share operand B, which the authors describe as roughly halving the shared-memory reads for that operand. This is not a claim that all shared-memory traffic for dQ, dK, and dV is halved.\n\nFor dQ, each CTA computes a half of dS and exchanges that half through distributed shared memory so both CTAs can form the required dQ product. The two-CTA organization also doubles the dQ reduction tile along N and thereby halves the number of global atomic reductions described by the paper. It does not assign dK exclusively to CTA 0 and dV exclusively to CTA 1."}, "reason": {"statement": "Retain the two-CTA optimization with the source's exact traffic scope.", "urls": ["https://arxiv.org/html/2603.05451v1"]}} +{"path": "wiki/kernels/flash-attention-4.md", "before": {"statement": "```python\n# 2-CTA backward: paired CTAs share TMEM via 2-SM cooperative mode\n@cute.kernel\ndef flash_attention_4_bwd(Q, K, V, O, dO, dQ, dK, dV):\n # Two CTAs cooperate: CTA_0 and CTA_1 in same cluster\n # tcgen05.mma shape: m256 x n256 x k16 (2-SM cooperative)\n\n cta_id = cute.cluster_rank() # 0 or 1\n\n for q_block in range(num_q_blocks):\n # Both CTAs load shared KV block via TMA\n K_smem = tma_load(K, kv_offset)\n V_smem = tma_load(V, kv_offset)\n\n # CTA_0: compute dK contribution\n # CTA_1: compute dV contribution\n if cta_id == 0:\n # dK += dS^T @ Q (accumulated in TMEM)\n dS = compute_dS(O, dO, S)\n cute.mma(dS.T, Q_tile, accumulator=dK_tmem)\n else:\n # dV += S^T @ dO (accumulated in TMEM)\n cute.mma(S.T, dO_tile, accumulator=dV_tmem)\n```"}, "after": {"statement": "The paper maps five backward GEMMs to two-CTA tcgen05 MMA with `M=256, N=128, K=128`. For those operations, the paired CTAs can share operand B, which the authors describe as roughly halving the shared-memory reads for that operand. This is not a claim that all shared-memory traffic for dQ, dK, and dV is halved.\n\nFor dQ, each CTA computes a half of dS and exchanges that half through distributed shared memory so both CTAs can form the required dQ product. The two-CTA organization also doubles the dQ reduction tile along N and thereby halves the number of global atomic reductions described by the paper. It does not assign dK exclusively to CTA 0 and dV exclusively to CTA 1.\n\nAt commit `a369df7`:\n\n- [`flash_fwd_sm100.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py) builds tcgen05 operations through `make_trivial_tiled_mma`, allocates TMEM through `TmemAllocator`, and uses explicit TMEM column offsets for score and output accumulators.\n- The forward path constructs TMA atoms for Q, K, and V where its configuration enables them. It also has non-TMA Q and paged-K/V copy paths, so TMA use is not unconditional.\n- [`flash_bwd_sm100.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_bwd_sm100.py) constructs the five backward MMA operations and the two-CTA exchange/reduction pipelines rather than splitting dK and dV by cluster rank.\n- The package README describes CuTe DSL attention for Hopper and Blackwell, and the tree contains SM90, SM100, and SM120 dispatch modules. The paper's FA4 result remains SM100/B200-specific; package architecture coverage should not be used to broaden that performance result.\n- The pinned [`pyproject.toml`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/pyproject.toml) requires `nvidia-cutlass-dsl==4.6.0.dev0`. That is a property of this source snapshot, not a timeless minimum.\n- Forward and backward accept FP16/BF16. An FP8 benchmark/bring-up script is present, but at this revision it explicitly expects the FA4 FP8 call to fail until support is implemented."}, "reason": {"statement": "Replace fabricated executable code with the paper's verified data-sharing description and pinned upstream links.", "urls": ["https://arxiv.org/html/2603.05451v1"]}} +{"path": "wiki/kernels/flash-attention-4.md", "before": {"statement": "performance_claims:\n- gpu: B200\n dtype: bf16\n shape: seqlen=8192, headdim=128\n metric: TFLOPS\n value: 1605\n utilization: 71%\n source_id: doc-flash-attention-4\n\n| seqlen=8192, headdim=128 | B200 | BF16 | 1605 | 71% | 1.1-1.3x | 2.1-2.7x |"}, "after": {"statement": "performance_claims: []\n\nThe first-party sources contain two different source-reported peak values. Paper v1 reports **up to 1613 TFLOPS/s on B200 BF16, or 71% of the peak convention used by the authors**. Tri Dao's blog reports **up to 1605 TFLOPS/s, also labeled 71%**, plus up to 1.3x over cuDNN 9.13 and up to 2.7x over Triton.\n\nThe paper's benchmark suite spans sequence lengths from 1K through 32K and multiple query/value head-dimension pairs under a fixed total-token convention. Neither textual source establishes the former single row that attached 1605 TFLOPS, 71%, and both speedup ranges specifically to `seqlen=8192, headdim=128`. The structured performance record is therefore empty, and no complement-of-71% time breakdown is inferred."}, "reason": {"statement": "Remove the false structured record and preserve both source-reported maxima with their distinct provenance and suite scope.", "urls": ["https://arxiv.org/html/2603.05451v1", "https://tridao.me/blog/2026/flash4/"]}} +{"path": "wiki/kernels/flash-attention-4.md", "before": {"statement": "The 71% MMA utilization represents the state of the art for attention kernels on Blackwell. The remaining 29% is consumed by softmax, rescaling, and memory transfers."}, "after": {"statement": "The first-party sources contain two different source-reported peak values. Paper v1 reports **up to 1613 TFLOPS/s on B200 BF16, or 71% of the peak convention used by the authors**. Tri Dao's blog reports **up to 1605 TFLOPS/s, also labeled 71%**, plus up to 1.3x over cuDNN 9.13 and up to 2.7x over Triton.\n\nThe paper's benchmark suite spans sequence lengths from 1K through 32K and multiple query/value head-dimension pairs under a fixed total-token convention. Neither textual source establishes the former single row that attached 1605 TFLOPS, 71%, and both speedup ranges specifically to `seqlen=8192, headdim=128`. The structured performance record is therefore empty, and no complement-of-71% time breakdown is inferred."}, "reason": {"statement": "No source or experiment supports the attribution.", "urls": ["https://arxiv.org/html/2603.05451v1", "https://tridao.me/blog/2026/flash4/"]}} +{"path": "wiki/kernels/flash-attention-4.md", "before": {"statement": "- Uses `SM100_MMA_SS` atoms for tcgen05 MMA from shared memory\n- TMEM locality via `TMEM` locale in CuTe layout\n- TMA bulk loads for Q, K, V tiles into shared memory"}, "after": {"statement": "At commit `a369df7`:\n\n- [`flash_fwd_sm100.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py) builds tcgen05 operations through `make_trivial_tiled_mma`, allocates TMEM through `TmemAllocator`, and uses explicit TMEM column offsets for score and output accumulators.\n- The forward path constructs TMA atoms for Q, K, and V where its configuration enables them. It also has non-TMA Q and paged-K/V copy paths, so TMA use is not unconditional.\n- [`flash_bwd_sm100.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_bwd_sm100.py) constructs the five backward MMA operations and the two-CTA exchange/reduction pipelines rather than splitting dK and dV by cluster rank.\n- The package README describes CuTe DSL attention for Hopper and Blackwell, and the tree contains SM90, SM100, and SM120 dispatch modules. The paper's FA4 result remains SM100/B200-specific; package architecture coverage should not be used to broaden that performance result.\n- The pinned [`pyproject.toml`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/pyproject.toml) requires `nvidia-cutlass-dsl==4.6.0.dev0`. That is a property of this source snapshot, not a timeless minimum.\n- Forward and backward accept FP16/BF16. An FP8 benchmark/bring-up script is present, but at this revision it explicitly expects the FA4 FP8 call to fail until support is implemented."}, "reason": {"statement": "Name the pinned source's actual abstractions and conditional copy behavior.", "urls": []}} +{"path": "wiki/kernels/flash-attention-4.md", "before": {"statement": "- Standard multi-head attention on Blackwell with sequence lengths >= 1024"}, "after": {"statement": "- Choose the package path only for a supported architecture, dtype, head-dimension pair, masking mode, and feature set at the exact revision in use.\n- Do not treat sequence length 1024 or head dimension 128 as universal crossover or optimum values. Compare against the relevant cuDNN, framework, or other kernel path on the actual workload.\n- Compilation speed and runtime speed are separate measurements. The paper's compile comparison does not prove an equivalent runtime factor.\n- Treat source-reported B200 numbers as unreproduced unless the same software, clock/power settings, tensor shapes, timing region, warmup, and FLOP convention are available."}, "reason": {"statement": "Replace the invented cutoff with workload- and support-dependent selection guidance.", "urls": ["https://arxiv.org/html/2603.05451v1", "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/interface.py"]}} +{"path": "wiki/kernels/flash-attention-4.md", "before": {"statement": "- SM100 only -- no fallback to SM90\n- Requires CuTe DSL toolchain (CUTLASS 4.5.0 + Python frontend)\n- Ping-pong scheduling most effective for headdim=128; smaller headdims may not fully overlap"}, "after": {"statement": "At commit `a369df7`:\n\n- [`flash_fwd_sm100.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py) builds tcgen05 operations through `make_trivial_tiled_mma`, allocates TMEM through `TmemAllocator`, and uses explicit TMEM column offsets for score and output accumulators.\n- The forward path constructs TMA atoms for Q, K, and V where its configuration enables them. It also has non-TMA Q and paged-K/V copy paths, so TMA use is not unconditional.\n- [`flash_bwd_sm100.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_bwd_sm100.py) constructs the five backward MMA operations and the two-CTA exchange/reduction pipelines rather than splitting dK and dV by cluster rank.\n- The package README describes CuTe DSL attention for Hopper and Blackwell, and the tree contains SM90, SM100, and SM120 dispatch modules. The paper's FA4 result remains SM100/B200-specific; package architecture coverage should not be used to broaden that performance result.\n- The pinned [`pyproject.toml`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/pyproject.toml) requires `nvidia-cutlass-dsl==4.6.0.dev0`. That is a property of this source snapshot, not a timeless minimum.\n- Forward and backward accept FP16/BF16. An FP8 benchmark/bring-up script is present, but at this revision it explicitly expects the FA4 FP8 call to fail until support is implemented.\n\n- Choose the package path only for a supported architecture, dtype, head-dimension pair, masking mode, and feature set at the exact revision in use.\n- Do not treat sequence length 1024 or head dimension 128 as universal crossover or optimum values. Compare against the relevant cuDNN, framework, or other kernel path on the actual workload.\n- Compilation speed and runtime speed are separate measurements. The paper's compile comparison does not prove an equivalent runtime factor.\n- Treat source-reported B200 numbers as unreproduced unless the same software, clock/power settings, tensor shapes, timing region, warmup, and FLOP convention are available."}, "reason": {"statement": "Separate paper scope from the pinned package snapshot and avoid universal tuning advice.", "urls": ["https://arxiv.org/html/2603.05451v1", "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/interface.py"]}} +{"path": "wiki/kernels/flash-attention-4.md", "before": {"statement": "Local verbatim upstream code lives in [`artifacts/kernels/flash-attention-4/full/`](../../artifacts/kernels/flash-attention-4/full/) (see its `PROVENANCE.yaml` for the pinned upstream SHA and byte-verified SHA-256). Labeled derived variants — including a naive/teaching skeleton — live in [`artifacts/kernels/flash-attention-4/variants/`](../../artifacts/kernels/flash-attention-4/variants/)."}, "after": {"statement": "- [FlashAttention-4 paper, arXiv v1](https://arxiv.org/abs/2603.05451v1)\n- [Tri Dao's FlashAttention-4 blog](https://tridao.me/blog/2026/flash4/)\n- [FA4 CuTe DSL package at commit `a369df7`](https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute)\n- [Pinned SM100 forward source](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py)\n- [Pinned SM100 backward source](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_bwd_sm100.py)\n\nThe local [`full/`](../../artifacts/kernels/flash-attention-4/full/) bundle is a byte-verified, verbatim **adjacent NVIDIA CUTLASS SM100 FMHA backward MLA example**, pinned to CUTLASS commit `0e026982`. It is not the Dao-AILab FA4 implementation. The local [`variants/`](../../artifacts/kernels/flash-attention-4/variants/) bundle contains explicitly derived teaching material, including the scalar software-exp formula; it is not upstream code. Exact FA4 implementation evidence is linked to the immutable Dao-AILab commit above.\n\nQuery the page and its attached local references with:\n\n```bash\npython3 scripts/get_page.py kernel-flash-attention-4 --include-code\n```"}, "reason": {"statement": "Retain the useful adjacent reference but label its identity; point implementation claims to exact Dao-AILab commit URLs.", "urls": []}} +{"path": "wiki/kernels/flash-attention-4.md", "before": {"statement": "sources:\n- doc-flash-attention-4\n- blog-flash-attention-4\n- pr-flashinfer-1850"}, "after": {"statement": "sources:\n- doc-flash-attention-4\n- blog-flash-attention-4"}, "reason": {"statement": "Keep the FA4 source relationship limited to its first-party paper and blog; adjacent CUTLASS code remains explicitly labeled in the artifact discussion.", "urls": ["https://github.com/flashinfer-ai/flashinfer/pull/1850"]}} +{"path": "wiki/kernels/flash-attention-sm100-mla-topk.md", "before": {"statement": "FlashAttention PR 2441 adds an SM100 CuTe DSL forward path for MLA shapes with\ntop-k sparsity. It is useful when an attention candidate has to combine page/KV\nlayout handling, sparse top-k selection, and tiled forward scheduling."}, "after": {"statement": "[Dao-AILab/flash-attention PR 2441](https://github.com/Dao-AILab/flash-attention/pull/2441), merged as [`f219c89c886c6ccbf9d3dbd9fe41b11ac64e9df8`](https://github.com/Dao-AILab/flash-attention/commit/f219c89c886c6ccbf9d3dbd9fe41b11ac64e9df8), adds an SM100 CuTe DSL forward path for the DeepSeek-style MLA dimensions `head_dim=64` and `head_dim_v=512`, with MQA packing of 128 query heads per KV head and caller-supplied top-k indices.\n\nThe top-k path is deliberately narrower than a generic paged sparse-attention API. It requires packed GQA/MQA, `qhead_per_kvhead == 128`, and the cp.async K/V gather path. Its constructor defaults to top-k length 2048 and requires the configured length to be divisible by 256. The merged kernel rejects a non-null page table with `page table tbd for MLA`; page-table support was explicitly outside this PR.\n\nThe pinned merge keeps three responsibilities distinguishable:\n\n- [`topk_gather_kv.py`](../../artifacts/prs/flash-attention/PR-2441/key-files/flash_attn/cute/topk_gather_kv.py) loads caller-provided top-k indices, forms indexed K/V addresses, issues cp.async copies, and optionally constructs validity bitmasks for out-of-range indices.\n- [`tile_scheduler.py`](../../artifacts/prs/flash-attention/PR-2441/key-files/flash_attn/cute/tile_scheduler.py) supplies the tile-scheduler implementations selected by the MLA kernel.\n- [`flash_fwd_mla_sm100.py`](../../artifacts/prs/flash-attention/PR-2441/key-files/flash_attn/cute/flash_fwd_mla_sm100.py) integrates gather, scheduling, two-CTA tcgen05 MMA, TMEM accumulators, softmax, and the output path.\n\nThis separation matters for evaluation. Arithmetic work follows the effective top-k length, whereas memory addresses follow the caller's index set and use an indexed gather/optional-bitmask path. Reduced attention FLOPs therefore do not prove a proportional latency or locality improvement."}, "reason": {"statement": "Retain the implemented top-k/scheduler path and state page-table exclusion precisely.", "urls": []}} +{"path": "wiki/kernels/flash-attention-sm100-mla-topk.md", "before": {"statement": "performance_claims:\n- gpu: B200\n dtype: bf16\n shape: batch=512, seqlen_q=1, seqlen_k=16384, nheads=128, topk=2048\n metric: latency_ms\n value: 0.3\n source_id: pr-flash-attention-2441"}, "after": {"statement": "performance_claims: []\n\nThe PR author reports an initial saturating-decode comparison for batch 512, `seqlen_q=1`, `seqlen_k=16384`, 128 query heads, the 64/512 MLA shape, and top-k length 2048:\n\n| DSA, no bitmask; indices assumed in bounds | 0.31 ms | 955.47 TFLOPS |\n| DSA, validity bitmask | 0.33 ms | 898.08 TFLOPS |\n| Vanilla MLA baseline | 1.98 ms | 1180.70 TFLOPS |\n\nThese PR-description observations are not reproduced here. The PR body does not name the exact GPU model, dtype, clocks/power state, software environment, timing protocol, or run-to-run variation for the rows. They therefore remain prose with exact source qualifications rather than structured `performance_claims`."}, "reason": {"statement": "Remove the unsupported structured record but retain the author's two qualified, unreproduced observations in prose.", "urls": ["https://api.github.com/repos/Dao-AILab/flash-attention/pulls/2441"]}} +{"path": "wiki/kernels/flash-attention-sm100-mla-topk.md", "before": {"statement": "sources:\n- pr-flash-attention-2441\n- pr-flash-attention-1236"}, "after": {"statement": "sources:\n- pr-flash-attention-2441"}, "reason": {"statement": "Limit the source relationship to the exact introducing merge.", "urls": ["https://github.com/Dao-AILab/flash-attention/pull/1236"]}} +{"path": "wiki/kernels/flashmla.md", "before": {"statement": "FlashMLA provides high-performance kernels for DeepSeek's Multi-head Latent Attention (MLA) mechanism, which compresses the KV cache from 327-516 KB/token (standard MHA) down to 70 KB/token through a learned low-rank projection into a latent space. This extreme compression (4.66-7.28x reduction) is critical for serving DeepSeek-V3/V3.2 models at scale."}, "after": {"statement": "FlashMLA is DeepSeek's attention-kernel library for DeepSeek-V3 and DeepSeek-V3.2-Exp. At pinned commit [`71c7379`](https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232), the library contains MLA-mode decode and sparse-prefill operators plus a dense **MHA** prefill operator contributed for SM100. The repository's term “MLA mode” distinguishes MQA-shaped `d_qk=576, d_v=512` kernels from MHA-shaped `d_qk=192/128, d_v=128` kernels; it does not mean every operator in the package is MLA."}, "reason": {"statement": "Replace invented byte endpoints with the paper's explicit per-layer element formula and keep implementation-specific byte layout separate.", "urls": ["https://arxiv.org/html/2405.04434v5"]}} +{"path": "wiki/kernels/flashmla.md", "before": {"statement": "FlashMLA includes four kernel variants: dense MLA decoding (SM90), sparse MLA decoding (SM90/SM100), dense MLA prefill (SM100), and sparse MLA prefill (SM90/SM100)."}, "after": {"statement": "The DeepSeek-V2 paper describes the model-level reduction in elements cached per layer and token:\n\n- MHA: `2 * n_h * d_h`\n- MLA: `d_c + d_h^R`, approximately `4.5 * d_h` for its `d_c=4*d_h` and `d_h^R=d_h/2` configuration\n\nThese formulas are independent of storage dtype and layer count. They should not be converted to whole-model KB/token figures without naming those additional assumptions."}, "reason": {"statement": "Preserve the four operator families while labeling dense prefill as MHA.", "urls": ["https://arxiv.org/html/2405.04434v5"]}} +{"path": "wiki/kernels/flashmla.md", "before": {"statement": "Each token in the MLA KV cache occupies 656 bytes:\n\n```\nToken KV Cache Entry (656 bytes total):\n+------------------------------------------+\n| FP8 compressed KV data | 512 bytes | <-- Latent KV representation\n| FP32 scaling factors | 16 bytes | <-- Per-head scales\n| BF16 RoPE embeddings | 128 bytes | <-- Position encodings\n+------------------------------------------+\n\nPaged KV cache:\n Page size = 64 tokens (dense) or variable (sparse)\n Each page = 64 * 656 = 41,984 bytes\n```"}, "after": {"statement": "FlashMLA's **656-byte** layout is narrower: it is the DeepSeek-V3-family FP8 sparse-decode cache ABI, not the definition of an MLA cache. One token contains:\n\n| NoPE latent data | 512 `float8_e4m3` values | 512 |\n| NoPE group scales | four `float32` values, one per 128 values | 16 |\n| RoPE data | 64 unquantized `bfloat16` values | 128 |\n| **Total** | | **656** |\n\nDense decode uses a BF16 cache. The pinned quantization tests also contain a separate 512-dimensional sparse layout, so code must select the model/layout contract rather than assuming 656 bytes universally. `page_block_size` is taken from the cache tensor; 64 is a test default, not a fixed API rule."}, "reason": {"statement": "Narrow the byte layout to its exact model and dispatch mode and remove invented page-size universals.", "urls": []}} +{"path": "wiki/kernels/flashmla.md", "before": {"statement": "The decode kernel targets memory-bound inference with paged KV cache (block size 64). It achieves up to 3000 GB/s memory bandwidth and 660 TFLOPS on H800."}, "after": {"statement": "Sparse decode receives `indices[batch, s_q, topk]`. Each nonnegative value already encodes a physical page and offset:\n\n```text\nencoded = physical_page * page_block_size + offset_in_page\n```\n\nBecause the physical page is already encoded, sparse decode does not use `block_table`; `-1` marks an invalid entry. The kernel consumes these indices but does not produce the top-k selection, so an indexing stage outside the attention call must supply them.\n\nSparse prefill is a different interface. It receives BF16 `q[s_q,h_q,d_qk]`, BF16 `kv[s_kv,h_kv,d_qk]`, and `indices[s_q,h_kv,topk]`; it has no batch dimension, requires `h_kv=1` in the documented equivalence, and accepts `-1` or values at least `s_kv` as invalid. It returns `(out, max_logits, lse)`."}, "reason": {"statement": "Retain the exact author report and remove the false fixed-page condition.", "urls": []}} +{"path": "wiki/kernels/flashmla.md", "before": {"statement": "```cpp\n// Dense MLA decode kernel structure (SM90, BF16)\n// Memory-bound: bandwidth utilization is the primary metric\n\ntemplate \n__global__ void flashmla_decode_dense(\n const half* __restrict__ Q, // [batch, num_heads, head_dim]\n const int8_t* __restrict__ KV, // Paged KV cache (FP8)\n const float* __restrict__ scales, // Per-head scales\n const half* __restrict__ rope, // RoPE embeddings\n const int* __restrict__ page_table,\n half* __restrict__ O,\n float* __restrict__ L // Log-sum-exp\n) {\n // Each warpgroup handles one query head\n const int head_id = blockIdx.x;\n const int batch_id = blockIdx.y;\n\n // Load query into registers\n half Q_reg[HEAD_DIM];\n load_query(Q, batch_id, head_id, Q_reg);\n\n float acc[HEAD_DIM] = {0.0f};\n float lse = -INFINITY;\n\n // Iterate over KV pages\n for (int page = 0; page < num_pages; page++) {\n int page_idx = page_table[batch_id * max_pages + page];\n\n // TMA load KV page into shared memory\n __shared__ int8_t KV_smem[BLOCK_KV * 656];\n tma_load_async(KV_smem, KV + page_idx * BLOCK_KV * 656);\n cp_async_wait();\n\n // Compute attention scores for this page\n for (int t = 0; t < BLOCK_KV; t++) {\n // Dequantize KV: fp8 -> bf16, apply scale\n half K_token[HEAD_DIM], V_token[HEAD_DIM];\n dequant_kv(KV_smem + t * 656, scales, K_token, V_token);\n\n // Apply RoPE\n apply_rope(K_token, rope + (page * BLOCK_KV + t) * 64);\n\n // Score and accumulate (online softmax)\n float score = dot_product(Q_reg, K_token, HEAD_DIM);\n float new_lse = logaddexp(lse, score);\n float rescale = exp(lse - new_lse);\n for (int d = 0; d < HEAD_DIM; d++)\n acc[d] = acc[d] * rescale + exp(score - new_lse) * V_token[d];\n lse = new_lse;\n }\n }\n\n // Write output\n store_output(O, batch_id, head_id, acc);\n L[batch_id * num_heads + head_id] = lse;\n}\n```"}, "after": {"statement": "FlashMLA's **656-byte** layout is narrower: it is the DeepSeek-V3-family FP8 sparse-decode cache ABI, not the definition of an MLA cache. One token contains:\n\n| NoPE latent data | 512 `float8_e4m3` values | 512 |\n| NoPE group scales | four `float32` values, one per 128 values | 16 |\n| RoPE data | 64 unquantized `bfloat16` values | 128 |\n| **Total** | | **656** |\n\nDense decode uses a BF16 cache. The pinned quantization tests also contain a separate 512-dimensional sparse layout, so code must select the model/layout contract rather than assuming 656 bytes universally. `page_block_size` is taken from the cache tensor; 64 is a test default, not a fixed API rule.\n\n| Dense decode | SM90 | MQA (`576/512`) | BF16 paged KV |\n| Sparse decode | SM90, SM100 | MQA (`576/512`) | FP8 KV, dequantized for BF16 MMA; BF16 output |\n| Dense prefill | SM100 | MHA (`192/128` or `128/128`) | BF16 Q/K/V |\n| Sparse prefill | SM90, SM100 | MQA | BF16 Q and KV |\n\nCUDA 12.8 or newer and PyTorch 2.0 or newer are required; the pinned README requires CUDA 12.9 or newer for SM100."}, "reason": {"statement": "Remove fabricated kernel code; point readers to pinned source-level contracts and transparently labeled adjacent artifacts.", "urls": []}} +{"path": "wiki/kernels/flashmla.md", "before": {"statement": "```cpp\n// Sparse MLA: only attend to selected tokens via indices tensor\n// Each query has a variable-length list of relevant token indices\n\ntemplate \n__global__ void flashmla_sparse(\n const half* Q,\n const int8_t* KV_cache,\n const float* scales,\n const int* token_indices, // Selected token indices per query\n const int* num_selected, // Number of selected tokens per query\n half* O\n) {\n const int query_id = blockIdx.x;\n const int n_tokens = num_selected[query_id];\n\n // Only load and compute on selected tokens\n for (int i = 0; i < n_tokens; i += BLOCK_SIZE) {\n int tok_idx = token_indices[query_id * MAX_TOKENS + i];\n\n // Load only the selected token's KV entry (656 bytes)\n load_kv_entry(KV_cache, tok_idx, K_local, V_local);\n dequant_and_accumulate(Q, K_local, V_local, scales, acc, lse);\n }\n}\n```"}, "after": {"statement": "| Dense decode | SM90 | MQA (`576/512`) | BF16 paged KV |\n| Sparse decode | SM90, SM100 | MQA (`576/512`) | FP8 KV, dequantized for BF16 MMA; BF16 output |\n| Dense prefill | SM100 | MHA (`192/128` or `128/128`) | BF16 Q/K/V |\n| Sparse prefill | SM90, SM100 | MQA | BF16 Q and KV |\n\nCUDA 12.8 or newer and PyTorch 2.0 or newer are required; the pinned README requires CUDA 12.9 or newer for SM100."}, "reason": {"statement": "Replace the fabricated code with exact interface descriptions and small, explicitly derived contract checks.", "urls": []}} +{"path": "wiki/kernels/flashmla.md", "before": {"statement": "The SM100 prefill kernel leverages tcgen05.mma and TMEM for the compute-heavy forward and backward passes, achieving 1460 TFLOPS forward and 1000 TFLOPS backward on B200.\n\n```cpp\n// SM100 dense prefill: tcgen05.mma with TMEM accumulation\n// Uses warp specialization: TMA warps + MMA warps + softmax warps\n\n// Forward pass structure:\n// 1. TMA loads Q, K, V tiles into SMEM\n// 2. tcgen05.mma computes S = Q @ K^T into TMEM\n// 3. Softmax warpgroup applies online softmax on TMEM data\n// 4. tcgen05.mma computes O = softmax(S) @ V into TMEM\n// 5. TMEM -> SMEM -> Global memory for output\n\n// Key: TMEM holds both S matrix and O accumulator\n// No register spill for large tile sizes\n```"}, "after": {"statement": "The DeepSeek-V2 paper describes the model-level reduction in elements cached per layer and token:\n\n- MHA: `2 * n_h * d_h`\n- MLA: `d_c + d_h^R`, approximately `4.5 * d_h` for its `d_c=4*d_h` and `d_h^R=d_h/2` configuration\n\nThese formulas are independent of storage dtype and layer count. They should not be converted to whole-model KB/token figures without naming those additional assumptions.\n\nThe following are maxima reported by the pinned first-party README. They were not reproduced here, and the README does not provide complete shape, timing, sample-count, or variance cells, so they are intentionally excluded from structured `performance_claims`.\n\n| Dense MLA decode | H800 SXM5, CUDA 12.8 | BF16 cache | Up to 3000 GB/s in a memory-bound configuration; up to 660 TFLOPS in a compute-bound configuration |\n| Sparse MLA decode | H800 SXM5, CUDA 12.8 | FP8 KV, BF16 MMA | 410 TFLOPS in a compute-bound configuration |\n| Sparse MLA decode | B200; software version not stated in the row | FP8 KV, BF16 MMA | Up to 350 TFLOPS; the author says it was not well optimized |\n| Dense MHA prefill | B200; NVIDIA-reported | BF16 inputs | Up to 1460 TFLOPS forward and 1000 TFLOPS backward |\n| Sparse MLA prefill | H800 SXM5, CUDA 12.8 | BF16 inputs | Up to 640 TFLOPS forward |\n| Sparse MLA prefill | B200, CUDA 12.9 | BF16 inputs | Up to 1450 TFLOPS forward |\n\nThe numbers compare different operators, phases, shapes, precision scopes, and machines. In particular, `1460` is dense MHA prefill, not a replacement for the `660` dense-MLA-decode observation."}, "reason": {"statement": "Relabel the operator and separate verified architectural mechanisms from unsupported pseudocode.", "urls": ["https://arxiv.org/html/2405.04434v5"]}} +{"path": "wiki/kernels/flashmla.md", "before": {"statement": "| Sparse decode | H800 | FP8 | 410 | -- |\n| Sparse decode | B200 | FP8 | 350 | -- |"}, "after": {"statement": "Sparse decode receives `indices[batch, s_q, topk]`. Each nonnegative value already encodes a physical page and offset:\n\n```text\nencoded = physical_page * page_block_size + offset_in_page\n```\n\nBecause the physical page is already encoded, sparse decode does not use `block_table`; `-1` marks an invalid entry. The kernel consumes these indices but does not produce the top-k selection, so an indexing stage outside the attention call must supply them.\n\nSparse prefill is a different interface. It receives BF16 `q[s_q,h_q,d_qk]`, BF16 `kv[s_kv,h_kv,d_qk]`, and `indices[s_q,h_kv,topk]`; it has no batch dimension, requires `h_kv=1` in the documented equivalence, and accepts `-1` or values at least `s_kv` as invalid. It returns `(out, max_logits, lse)`."}, "reason": {"statement": "Split cache storage from compute precision in the performance table.", "urls": []}} +{"path": "wiki/kernels/flashmla.md", "before": {"statement": "| Sparse prefill | H800 | FP8 | 640 | -- |\n| Sparse prefill | B200 | FP8 | 1450 | -- |"}, "after": {"statement": "Sparse decode receives `indices[batch, s_q, topk]`. Each nonnegative value already encodes a physical page and offset:\n\n```text\nencoded = physical_page * page_block_size + offset_in_page\n```\n\nBecause the physical page is already encoded, sparse decode does not use `block_table`; `-1` marks an invalid entry. The kernel consumes these indices but does not produce the top-k selection, so an indexing stage outside the attention call must supply them.\n\nSparse prefill is a different interface. It receives BF16 `q[s_q,h_q,d_qk]`, BF16 `kv[s_kv,h_kv,d_qk]`, and `indices[s_q,h_kv,topk]`; it has no batch dimension, requires `h_kv=1` in the documented equivalence, and accepts `-1` or values at least `s_kv` as invalid. It returns `(out, max_logits, lse)`."}, "reason": {"statement": "Use the documented BF16 input dtype and preserve exact author-reported values/environments.", "urls": []}} +{"path": "wiki/kernels/flashmla.md", "before": {"statement": "- SGLang and vLLM provide day-0 support"}, "after": {"statement": "The following are maxima reported by the pinned first-party README. They were not reproduced here, and the README does not provide complete shape, timing, sample-count, or variance cells, so they are intentionally excluded from structured `performance_claims`.\n\n| Dense MLA decode | H800 SXM5, CUDA 12.8 | BF16 cache | Up to 3000 GB/s in a memory-bound configuration; up to 660 TFLOPS in a compute-bound configuration |\n| Sparse MLA decode | H800 SXM5, CUDA 12.8 | FP8 KV, BF16 MMA | 410 TFLOPS in a compute-bound configuration |\n| Sparse MLA decode | B200; software version not stated in the row | FP8 KV, BF16 MMA | Up to 350 TFLOPS; the author says it was not well optimized |\n| Dense MHA prefill | B200; NVIDIA-reported | BF16 inputs | Up to 1460 TFLOPS forward and 1000 TFLOPS backward |\n| Sparse MLA prefill | H800 SXM5, CUDA 12.8 | BF16 inputs | Up to 640 TFLOPS forward |\n| Sparse MLA prefill | B200, CUDA 12.9 | BF16 inputs | Up to 1450 TFLOPS forward |\n\nThe numbers compare different operators, phases, shapes, precision scopes, and machines. In particular, `1460` is dense MHA prefill, not a replacement for the `660` dense-MLA-decode observation."}, "reason": {"statement": "Remove the unversioned ecosystem claim rather than substituting inference for evidence.", "urls": []}} +{"path": "wiki/kernels/flashmla.md", "before": {"statement": "- MLA-specific: the latent KV cache format (656 bytes/token) is tied to DeepSeek's architecture"}, "after": {"statement": "FlashMLA's **656-byte** layout is narrower: it is the DeepSeek-V3-family FP8 sparse-decode cache ABI, not the definition of an MLA cache. One token contains:\n\n| NoPE latent data | 512 `float8_e4m3` values | 512 |\n| NoPE group scales | four `float32` values, one per 128 values | 16 |\n| RoPE data | 64 unquantized `bfloat16` values | 128 |\n| **Total** | | **656** |\n\nDense decode uses a BF16 cache. The pinned quantization tests also contain a separate 512-dimensional sparse layout, so code must select the model/layout contract rather than assuming 656 bytes universally. `page_block_size` is taken from the cache tensor; 64 is a test default, not a fixed API rule.\n\n- [DeepSeek FlashMLA at audited commit `71c7379`](https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232)\n- [DeepSeek-V2 MLA paper, v5](https://arxiv.org/html/2405.04434v5)\n- [CUTLASS PR 2466, SM100 MLA-shape backward](https://github.com/NVIDIA/cutlass/pull/2466)\n- [CUTLASS PR 2472, SM100 MLA-shape forward](https://github.com/NVIDIA/cutlass/pull/2472)"}, "reason": {"statement": "State the precise V3-family FP8+sparse scope.", "urls": ["https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232", "https://arxiv.org/html/2405.04434v5", "https://github.com/NVIDIA/cutlass/pull/2466", "https://github.com/NVIDIA/cutlass/pull/2472"]}} +{"path": "wiki/kernels/flashmla.md", "before": {"statement": "Verbatim upstream code lives in [`artifacts/kernels/flashmla/full/`](../../artifacts/kernels/flashmla/full/); labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` header) live in [`artifacts/kernels/flashmla/variants/`](../../artifacts/kernels/flashmla/variants/). Every file's SHA-256 and upstream-pinning metadata is in `PROVENANCE.yaml` inside each bundle."}, "after": {"statement": "The pinned DeepSeek SM100 sources use TMA, `tcgen05` tensor-core operations, and TMEM in specialized sparse-decode/prefill and dense-MHA-prefill code. Those mechanisms are implementation-specific: they do not make the CUTLASS and FlashInfer files in this repository copies of DeepSeek FlashMLA.\n\nThe local [`full/`](../../artifacts/kernels/flashmla/full/) bundle contains two byte-verified **adjacent implementations**:\n\n- NVIDIA CUTLASS Example 77 MLA forward at merge `9baa06dd`\n- FlashInfer's SM100 FMHA-MLA header at commit `9a05c92a`\n\nTheir exact per-file origins are recorded in `full/PROVENANCE.yaml`. The [`variants/`](../../artifacts/kernels/flashmla/variants/) directory contains a small KernelWiki-derived layout/index helper, explicitly marked as non-upstream. For the DeepSeek implementation itself, use commit `71c7379` linked above."}, "reason": {"statement": "Retain useful artifacts with explicit adjacent-implementation labels and replace the variant with a narrow verified contract helper.", "urls": []}} +{"path": "wiki/kernels/flashmla.md", "before": {"statement": "performance_claims:\n- gpu: B200\n dtype: bf16\n shape: dense prefill, variable seqlen\n metric: TFLOPS\n value: 1460\n utilization: ~65%\n source_id: blog-flashmla\n- gpu: B200\n dtype: fp8\n shape: sparse prefill\n metric: TFLOPS\n value: 1450\n utilization: ~65%\n source_id: blog-flashmla"}, "after": {"statement": "performance_claims: []\n\nSparse decode receives `indices[batch, s_q, topk]`. Each nonnegative value already encodes a physical page and offset:\n\n```text\nencoded = physical_page * page_block_size + offset_in_page\n```\n\nBecause the physical page is already encoded, sparse decode does not use `block_table`; `-1` marks an invalid entry. The kernel consumes these indices but does not produce the top-k selection, so an indexing stage outside the attention call must supply them.\n\nSparse prefill is a different interface. It receives BF16 `q[s_q,h_q,d_qk]`, BF16 `kv[s_kv,h_kv,d_qk]`, and `indices[s_q,h_kv,topk]`; it has no batch dimension, requires `h_kv=1` in the documented equivalence, and accepts `-1` or values at least `s_kv` as invalid. It returns `(out, max_logits, lse)`."}, "reason": {"statement": "Remove inadmissible structured records while retaining qualified author reports in prose.", "urls": []}} +{"path": "wiki/kernels/flashmla.md", "before": {"statement": "sources:\n- blog-flashmla\n- pr-flashinfer-1117\n- pr-vllm-39752"}, "after": {"statement": "sources:\n- blog-flashmla\n- doc-deepseek-v2-mla\n- pr-cutlass-2466\n- pr-cutlass-2472"}, "reason": {"statement": "Remove the unrelated relationship; retain only explicitly labeled adjacent implementations where discussed.", "urls": []}} +{"path": "wiki/kernels/flashmla.md", "before": {"statement": "sources:\n- blog-flashmla\n- pr-flashinfer-1117\n- pr-vllm-39752"}, "after": {"statement": "sources:\n- blog-flashmla\n- doc-deepseek-v2-mla\n- pr-cutlass-2466\n- pr-cutlass-2472"}, "reason": {"statement": "Keep source relationships claim-directed and version-pinned to the implementation actually discussed.", "urls": []}} +{"path": "wiki/kernels/fp8-block-scale-gemm.md", "before": {"statement": "```\nActivations: tile-wise 1x128 scales\n [1x128 values] → 1 scale factor (FP32 or FP8 E4M3)\n\nWeights: block-wise 128x128 scales\n [128x128 values] → 1 scale factor per block\n\nOutput accumulator: FP32\n Multiply A × B in FP8, accumulate in FP32, apply scales at MMA boundary\n```"}, "after": {"statement": "For `A[M,K] @ B[N,K].T` with the DeepSeek-V3 forward recipe, the logical scale arrays have shapes `A_sf[M,K/128]` and `B_sf[N/128,K/128]`. This small check documents only that grouping; it does not encode DeepGEMM's required TMA-transformed layouts.\n\n```python\n# KernelWiki-derived format check; not upstream DeepGEMM code.\ndef deepseek_v3_scale_shapes(m: int, n: int, k: int):\n assert m % 128 == 0 and n % 128 == 0 and k % 128 == 0\n activation_scales = (m, k // 128)\n weight_scales = (n // 128, k // 128)\n return activation_scales, weight_scales\n\nassert deepseek_v3_scale_shapes(4096, 4096, 4096) == ((4096, 32), (32, 32))\n```\n\nThe pinned SM90 1D1D kernel requires FP32 scale factors and fixes `BLOCK_K == 128`. Within each K block it issues the selected WGMMA operations into a register `accum` array; after the WGMMA batch completes, it multiplies those partials by the corresponding A/B scales and adds them into a separate FP32 `final_accum` array on CUDA cores.\n\nFor the DeepSeek-V3 description, a 128-element K interval corresponds to four WGMMAs. The paper describes Hopper's relevant internal addition/alignment precision as 14 bits, not as a generic “FP22 accumulator.” Promotion improves numerical behavior but still adds scale loads and CUDA-core work, so its overhead must be measured rather than assumed away."}, "reason": {"statement": "Split the Hopper and Blackwell scale ABIs and accumulation locations.", "urls": []}} +{"path": "wiki/kernels/fp8-block-scale-gemm.md", "before": {"statement": "```cuda\n// Hopper accumulator has ~22-bit precision (FP22)\n// Every Nc=128 WGMMAs, promote partial sum to FP32 on CUDA cores\n// This retains precision without adding MMA overhead\n\n__device__ void sm90_fp8_gemm_with_promotion(...) {\n float acc_fp32 = 0.0f;\n\n for (int k = 0; k < K; k += 128) {\n float acc_fp22 = 0.0f;\n #pragma unroll 4\n for (int kk = 0; kk < 128; kk += 32) {\n wgmma_fp8_e4m3(acc_fp22, A_frag, B_frag);\n }\n // Promote to FP32 accumulator on CUDA cores\n acc_fp32 += acc_fp22 * scale_a * scale_b;\n }\n\n // Write acc_fp32 to output\n}\n```"}, "after": {"statement": "For `A[M,K] @ B[N,K].T` with the DeepSeek-V3 forward recipe, the logical scale arrays have shapes `A_sf[M,K/128]` and `B_sf[N/128,K/128]`. This small check documents only that grouping; it does not encode DeepGEMM's required TMA-transformed layouts.\n\n```python\n# KernelWiki-derived format check; not upstream DeepGEMM code.\ndef deepseek_v3_scale_shapes(m: int, n: int, k: int):\n assert m % 128 == 0 and n % 128 == 0 and k % 128 == 0\n activation_scales = (m, k // 128)\n weight_scales = (n // 128, k // 128)\n return activation_scales, weight_scales\n\nassert deepseek_v3_scale_shapes(4096, 4096, 4096) == ((4096, 32), (32, 32))\n```"}, "reason": {"statement": "Replace fabricated code with a narrow source-adapted promotion skeleton and exact interval qualification.", "urls": ["https://arxiv.org/html/2412.19437v2", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh"]}} +{"path": "wiki/kernels/fp8-block-scale-gemm.md", "before": {"statement": "```cuda\n// tcgen05.mma has native UE8M0 block scale support in hardware\n// No promotion needed - scales applied inside MMA\n\n__global__ void sm100_fp8_gemm_block_scale(...) {\n uint32_t tmem = tmem_alloc(256);\n\n for (int k = 0; k < K; k += BLOCK_K) {\n mbarrier_wait(&tma_done);\n\n // tcgen05.mma.mxf8f6f4.block_scale variant\n // Reads A, B from SMEM; scales from scale SMEM; accumulates in TMEM\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.scale_vec::1X \"\n \"[%0], %1, %2, %3, %4, %5;\"\n :: \"r\"(tmem), \"l\"(a_desc), \"l\"(b_desc),\n \"r\"(sf_a_desc), \"r\"(sf_b_desc), \"n\"(1)\n );\n }\n\n // Read from TMEM, apply global scale, store\n float result = tmem_load(tmem) * global_scale;\n output[row * N + col] = __float2half(result);\n}\n```"}, "after": {"statement": "The pinned SM90 1D1D kernel requires FP32 scale factors and fixes `BLOCK_K == 128`. Within each K block it issues the selected WGMMA operations into a register `accum` array; after the WGMMA batch completes, it multiplies those partials by the corresponding A/B scales and adds them into a separate FP32 `final_accum` array on CUDA cores.\n\nFor the DeepSeek-V3 description, a 128-element K interval corresponds to four WGMMAs. The paper describes Hopper's relevant internal addition/alignment precision as 14 bits, not as a generic “FP22 accumulator.” Promotion improves numerical behavior but still adds scale loads and CUDA-core work, so its overhead must be measured rather than assumed away."}, "reason": {"statement": "Remove invalid inline PTX and explain the verified source-level path in prose.", "urls": []}} +{"path": "wiki/kernels/fp8-block-scale-gemm.md", "before": {"statement": "```\nA (activations) [M, K] packed FP8 E4M3\nsf_a [M, K/128] FP32 or FP8 E4M3 scales # 1 per 1x128 tile\n\nB (weights) [N, K] packed FP8 E4M3\nsf_b [N/128, K/128] scales # 1 per 128x128 block\n # or packed UE8M0 format (Blackwell): 4 scales per int32\n```"}, "after": {"statement": "DeepGEMM commit [`891d57b4`](https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f) implements related FP8 GEMMs on SM90 and SM100, but the two generations handle scale factors differently.\n\nThe pinned SM90 1D1D kernel requires FP32 scale factors and fixes `BLOCK_K == 128`. Within each K block it issues the selected WGMMA operations into a register `accum` array; after the WGMMA batch completes, it multiplies those partials by the corresponding A/B scales and adds them into a separate FP32 `final_accum` array on CUDA cores.\n\nFor the DeepSeek-V3 description, a 128-element K interval corresponds to four WGMMAs. The paper describes Hopper's relevant internal addition/alignment precision as 14 bits, not as a generic “FP22 accumulator.” Promotion improves numerical behavior but still adds scale loads and CUDA-core work, so its overhead must be measured rather than assumed away."}, "reason": {"statement": "Separate the model grouping from each implementation's input ABI.", "urls": []}} +{"path": "wiki/kernels/fp8-block-scale-gemm.md", "before": {"statement": "- CUTLASS SM100 schedules: similar ratio vs peak"}, "after": {"statement": "The pinned SM100 interface requires scale factors packed as four UE8M0 values per 32-bit `torch.int`. The 1D1D kernel TMA-loads scale-factor blocks, copies them into TMEM, builds a `make_instr_desc_block_scaled<..., float_ue8m0_t, ...>` descriptor, and accumulates with the selected block-scaled UMMA path in TMEM.\n\nThis is hardware-integrated scale consumption, not the SM90 `final_accum += scale_a * scale_b * accum` loop. Exact `tcgen05.mma` instruction spelling and descriptor restrictions are PTX-version-sensitive; use the pinned source wrapper or the NVIDIA PTX ISA rather than a hand-written approximate inline-assembly string.\n\nThe current DeepGEMM snapshot supports more than the original 128-granularity training recipe on SM100, including selected 32-element recipes. Treat the API's transformed/padded scale layout as authoritative for the chosen recipe."}, "reason": {"statement": "Retain CUTLASS availability without an unsupported cross-library performance comparison.", "urls": []}} +{"path": "wiki/kernels/fp8-block-scale-gemm.md", "before": {"statement": "- LLM inference with FP8 quantized weights (DeepSeek V3, Qwen2-FP8, etc.)"}, "after": {"statement": "The pinned README says DeepGEMM achieved **up to 1550 TFLOPS on H800** in its 2025-04-18 news item. That sentence does not identify a matrix shape, utilization percentage, timing protocol, sample count, variance, or a single optimization responsible for the maximum. The result was not reproduced in this audit and is therefore kept out of structured `performance_claims`.\n\nCUTLASS also ships SM100 block-scaled GEMM schedules, but no matched CUTLASS-versus-DeepGEMM shape/environment record is established here."}, "reason": {"statement": "Replace model-name generalization with an explicit contract-matching rule.", "urls": ["https://arxiv.org/html/2412.19437v2"]}} +{"path": "wiki/kernels/fp8-block-scale-gemm.md", "before": {"statement": "performance_claims:\n - gpu: H800\n dtype: fp8\n shape: \"M=4096, N=4096, K=4096\"\n metric: TFLOPS\n value: 1550\n utilization: \"~90% via CUDA core promotion\"\n source_id: blog-deepgemm"}, "after": {"statement": "performance_claims: []\n\nThe pinned SM100 interface requires scale factors packed as four UE8M0 values per 32-bit `torch.int`. The 1D1D kernel TMA-loads scale-factor blocks, copies them into TMEM, builds a `make_instr_desc_block_scaled<..., float_ue8m0_t, ...>` descriptor, and accumulates with the selected block-scaled UMMA path in TMEM.\n\nThis is hardware-integrated scale consumption, not the SM90 `final_accum += scale_a * scale_b * accum` loop. Exact `tcgen05.mma` instruction spelling and descriptor restrictions are PTX-version-sensitive; use the pinned source wrapper or the NVIDIA PTX ISA rather than a hand-written approximate inline-assembly string.\n\nThe current DeepGEMM snapshot supports more than the original 128-granularity training recipe on SM100, including selected 32-element recipes. Treat the API's transformed/padded scale layout as authoritative for the chosen recipe."}, "reason": {"statement": "Remove the inadmissible record while retaining the exact qualified author report in prose.", "urls": []}} +{"path": "wiki/kernels/fp8-block-scale-gemm.md", "before": {"statement": "sources: [blog-deepgemm, doc-cutlass-blackwell, doc-cutlass-changelog-sm100]"}, "after": {"statement": "sources: [blog-deepgemm, doc-deepseek-v3-fp8, doc-ptx-isa-sm100]"}, "reason": {"statement": "Use claim-directed, version-pinned sources.", "urls": []}} +{"path": "wiki/kernels/fp8-block-scale-gemm.md", "before": {"statement": "sources: [blog-deepgemm, doc-cutlass-blackwell, doc-cutlass-changelog-sm100]"}, "after": {"statement": "sources: [blog-deepgemm, doc-deepseek-v3-fp8, doc-ptx-isa-sm100]"}, "reason": {"statement": "Limit source relationships to the DeepSeek paper, exact DeepGEMM snapshot, and official PTX contract used by current claims.", "urls": []}} +{"path": "wiki/kernels/fused-moe.md", "before": {"statement": "Fused MoE kernels combine the full Mixture-of-Experts forward pass into minimal kernel launches: routing, token dispatch, gate-up dual GEMM, SwiGLU activation, down projection GEMM, and token combine. In unfused implementations this requires 5-7 separate kernel launches; fused variants reduce this to 1-3 launches, eliminating intermediate global memory roundtrips and saving up to 21.9% activation memory traffic."}, "after": {"statement": "FlashInfer's MLSys 2026 contest calls Track A **Fused MoE** with FP8 support and targets NVIDIA B200. Its exact benchmark definition is `moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048`. The definition says that DeepSeek-style routing and two grouped GEMMs are included. That operation-level scope does **not** establish that an implementation uses one GPU launch.\n\n[`01-routing-plus-fusion-skeleton.py`](../../artifacts/kernels/fused-moe/variants/01-routing-plus-fusion-skeleton.py) is a CPU-checkable, parameterized reference for grouped selection and local W13/SwiGLU/W2 accumulation. It is derived KernelWiki code, not an optimized kernel and not an upstream contest solution.\n\n- [MLSys 2026 FlashInfer contest](https://mlsys26.flashinfer.ai/)\n- [Exact FP8 MoE benchmark definition](https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048)"}, "reason": {"statement": "State only the operation scope in the official contract and remove unrecorded performance figures.", "urls": ["https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048", "https://mlsys26.flashinfer.ai/"]}} +{"path": "wiki/kernels/fused-moe.md", "before": {"statement": "```\nInput tokens x [batch, hidden_dim=7168]\n |\n v\n[Router] top-k=8 experts from 32, grouped (8 groups, top_group=4)\n |\n v\n[Dispatch] Scatter tokens to selected experts\n |\n v\n[Gate-Up Dual GEMM]\n gate = x @ W_gate [batch_expert, hidden -> intermediate]\n up = x @ W_up [batch_expert, hidden -> intermediate]\n |\n v\n[SwiGLU Activation]\n h = SiLU(gate) * up Element-wise: SiLU(x) = x * sigmoid(x)\n |\n v\n[Down GEMM]\n y = h @ W_down [batch_expert, intermediate -> hidden]\n |\n v\n[Combine] Weighted sum of expert outputs per token (router weights)\n |\n v\nOutput [batch, hidden_dim=7168]\n```"}, "after": {"statement": "| `seq_len` | variable | Number of input tokens; this is not labeled request batch size |\n| global experts | 256 | Width of `routing_logits` |\n| local experts | 32 | Experts whose weights are resident on one EP rank |\n| expert parallelism | 8 | `256 / 32` ranks in the published definition |\n| `top_k` | 8 | Selected global experts per token |\n| `n_group` | 8 | Groups of 32 global experts |\n| `topk_group` | 4 | Groups retained before global top-k selection |\n| hidden size | 7168 | Input and output width |\n| intermediate size | 2048 | Per-expert SwiGLU width |\n| GEMM1 output | 4096 | Concatenated W13 output, `2 * 2048` |\n| scale block | 128 | Fixed granularity for this DeepSeek-FP8 trace |\n\nThe `e32` suffix means **32 local experts**, not 32 total experts."}, "reason": {"statement": "Distinguish global routing geometry from local expert compute.", "urls": ["https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048"]}} +{"path": "wiki/kernels/fused-moe.md", "before": {"statement": "```\nUnfused (vLLM): 7 kernel launches\n 1. Router softmax\n 2. Top-k selection\n 3. Token dispatch (scatter)\n 4. Gate GEMM\n 5. Up GEMM\n 6. SiLU + multiply\n 7. Down GEMM + combine\n\nPartially fused (SGLang): 5 kernel launches\n 1. Router + top-k\n 2. Dispatch\n 3. Gate-Up fused GEMM + SiLU (3 ops -> 1 kernel)\n 4. Down GEMM\n 5. Combine\n\nFully fused (ideal): 1-2 launches\n All ops in single kernel, or gate-up-silu + down-combine\n```"}, "after": {"statement": "[`01-routing-plus-fusion-skeleton.py`](../../artifacts/kernels/fused-moe/variants/01-routing-plus-fusion-skeleton.py) is a CPU-checkable, parameterized reference for grouped selection and local W13/SwiGLU/W2 accumulation. It is derived KernelWiki code, not an optimized kernel and not an upstream contest solution."}, "reason": {"statement": "Do not teach fixed launch counts without a pinned GPU trace.", "urls": []}} +{"path": "wiki/kernels/fused-moe.md", "before": {"statement": "```cpp\n// Fused gate-up: two GEMMs + SiLU + multiply in one kernel\n// Avoids writing intermediate gate and up results to global memory\n\ntemplate \n__global__ void gated_dual_gemm_fused(\n const fp8_t* __restrict__ X, // [M, K] input tokens\n const fp8_t* __restrict__ W_gate, // [N, K] gate weights\n const fp8_t* __restrict__ W_up, // [N, K] up weights\n const float* __restrict__ sf_x, // Block scales for X\n const float* __restrict__ sf_gate, // Block scales for W_gate\n const float* __restrict__ sf_up, // Block scales for W_up\n half* __restrict__ output, // [M, N] fused output\n int M, int N, int K\n) {\n // Two TMEM regions: one for gate accumulator, one for up accumulator\n uint32_t tmem_gate = tmem_alloc_cta(256);\n uint32_t tmem_up = tmem_alloc_cta(256);\n\n // Pipelined main loop\n for (int k = 0; k < K; k += BLOCK_K) {\n int stage = (k / BLOCK_K) % NUM_STAGES;\n barrier_wait(stage);\n\n // Two MMAs per K-tile: gate and up projections\n // Both read same X tile, different weight tiles\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f8f6f4\"\n \" [%0], %1, %2, %3, %4;\"\n :: \"r\"(tmem_gate), \"l\"(x_smem[stage]),\n \"l\"(wg_smem[stage]), \"r\"(scales_gate), \"n\"(1)\n );\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f8f6f4\"\n \" [%0], %1, %2, %3, %4;\"\n :: \"r\"(tmem_up), \"l\"(x_smem[stage]),\n \"l\"(wu_smem[stage]), \"r\"(scales_up), \"n\"(1)\n );\n }\n\n // Fused epilogue: SiLU(gate) * up\n // Read both accumulators from TMEM, apply activation, store result\n float gate_val = tmem_load_f32(tmem_gate);\n float up_val = tmem_load_f32(tmem_up);\n\n // SiLU(x) = x * sigmoid(x) = x / (1 + exp(-x))\n float silu_gate = gate_val / (1.0f + expf(-gate_val));\n output[row * N + col] = __float2half(silu_gate * up_val);\n\n // Deallocate TMEM\n tmem_dealloc(tmem_gate, 256);\n tmem_dealloc(tmem_up, 256);\n}\n```"}, "after": {"statement": "The official reference performs these steps:\n\n1. Convert routing logits to `s = sigmoid(logits)` and form selection scores `s + routing_bias`.\n2. Reshape 256 scores into eight groups of 32. Sum the top two selection scores in each group, then retain four groups.\n3. Select eight global experts from the retained groups using the biased selection scores.\n4. Form combine weights from the **unbiased** sigmoid values for those eight experts, normalize per token, and multiply by `routed_scaling_factor`.\n5. For global expert IDs in `[local_expert_offset, local_expert_offset + 32)`, dequantize the relevant activation and weight blocks, compute one W13 projection, split its 4096 columns into two 2048-column halves, apply SwiGLU, and compute W2.\n6. Accumulate each local expert result into the token output using that expert's combine weight. Experts outside the local interval contribute nothing on that rank.\n\nThe W13 representation permits one logical `A @ W13.T` followed by a split. It does not require two separately allocated TMEM accumulators, and the benchmark contract does not prescribe a tcgen05 instruction sequence."}, "reason": {"statement": "Replace fabricated source with a narrow, executable derived semantic reference.", "urls": []}} +{"path": "wiki/kernels/fused-moe.md", "before": {"statement": "```python\n# FlashInfer API for fused MoE\n# Single function call replaces 5-7 separate kernel launches\nimport flashinfer\n\noutput = flashinfer.fused_moe.trtllm_fp8_block_scale_moe(\n hidden_states=x, # [batch, 7168] BF16 input\n w_gate_up=w_gate_up, # [32, 2*2048, 7168] FP8 (fused gate+up weights)\n w_down=w_down, # [32, 7168, 2048] FP8 (down weights)\n router_weights=router_w, # [batch, 32] FP32 routing logits\n topk=8, # Select top-8 experts per token\n num_groups=8, # Expert grouping\n topk_group=4, # Top groups to select from\n block_scale_gate_up=sf_gu, # Block scales [32, 2*2048, 7168/128] FP8\n block_scale_down=sf_d, # Block scales [32, 7168, 2048/128] FP8\n)\n```"}, "after": {"statement": "For `T = seq_len`, the published benchmark signature is:\n\n| `routing_logits` | FP32 | `[T, 256]` |\n| `routing_bias` | BF16 | `[256]` |\n| `hidden_states` | FP8 E4M3FN | `[T, 7168]` |\n| `hidden_states_scale` | FP32 | `[56, T]` |\n| `gemm1_weights` | FP8 E4M3FN | `[32, 4096, 7168]` |\n| `gemm1_weights_scale` | FP32 | `[32, 32, 56]` |\n| `gemm2_weights` | FP8 E4M3FN | `[32, 7168, 2048]` |\n| `gemm2_weights_scale` | FP32 | `[32, 56, 16]` |\n| `local_expert_offset` | INT32 | scalar |\n| `routed_scaling_factor` | FP32 | scalar |\n\nThe output is BF16 `[T, 7168]`. These scale tensors are explicit storage inputs; this page does not infer their runtime cost from their existence.\n\nAt FlashInfer commit `7f614b86470180bab2d22e36fd1775791c6bf3e6`, the corresponding public entry point is `flashinfer.fused_moe.trtllm_fp8_block_scale_moe`. Its complete call includes the eight tensors above plus `num_experts=256`, `top_k=8`, `n_group=8`, `topk_group=4`, `intermediate_size=2048`, the local expert interval, the routed scaling factor, and DeepSeek-V3 routing mode.\n\n- [FlashInfer trace reference at `7f614b8`](https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/trace/templates/moe.py)\n- [FlashInfer API implementation at `7f614b8`](https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/fused_moe/core.py)"}, "reason": {"statement": "Use the exact benchmark argument names, dtypes, and global/local dimensions.", "urls": ["https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048"]}} +{"path": "wiki/kernels/fused-moe.md", "before": {"statement": "```python\nimport triton\nimport triton.language as tl\n\n@triton.jit\ndef fused_moe_gate_up_triton(\n X_ptr, W_gate_ptr, W_up_ptr, Out_ptr,\n expert_ids_ptr, token_counts_ptr,\n sf_x_ptr, sf_gate_ptr, sf_up_ptr,\n N: tl.constexpr, K: tl.constexpr,\n BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr,\n):\n \"\"\"Fused gate-up GEMM + SiLU for one expert's tokens.\"\"\"\n expert_id = tl.program_id(2)\n pid_m = tl.program_id(0)\n pid_n = tl.program_id(1)\n\n token_count = tl.load(token_counts_ptr + expert_id)\n m_start = pid_m * BLOCK_M\n if m_start >= token_count:\n return\n\n n_start = pid_n * BLOCK_N\n offs_m = m_start + tl.arange(0, BLOCK_M)\n offs_n = n_start + tl.arange(0, BLOCK_N)\n\n gate_acc = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)\n up_acc = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)\n\n for k in range(0, K, BLOCK_K):\n offs_k = k + tl.arange(0, BLOCK_K)\n x_tile = tl.load(X_ptr + offs_m[:, None] * K + offs_k[None, :])\n wg_tile = tl.load(W_gate_ptr + expert_id * N * K\n + offs_n[:, None] * K + offs_k[None, :])\n wu_tile = tl.load(W_up_ptr + expert_id * N * K\n + offs_n[:, None] * K + offs_k[None, :])\n gate_acc += tl.dot(x_tile, tl.trans(wg_tile))\n up_acc += tl.dot(x_tile, tl.trans(wu_tile))\n\n # Fused epilogue: SiLU(gate) * up\n silu_gate = gate_acc * tl.sigmoid(gate_acc)\n result = silu_gate * up_acc\n tl.store(Out_ptr + offs_m[:, None] * N + offs_n[None, :],\n result.to(tl.float16))\n```"}, "after": {"statement": "The official reference performs these steps:\n\n1. Convert routing logits to `s = sigmoid(logits)` and form selection scores `s + routing_bias`.\n2. Reshape 256 scores into eight groups of 32. Sum the top two selection scores in each group, then retain four groups.\n3. Select eight global experts from the retained groups using the biased selection scores.\n4. Form combine weights from the **unbiased** sigmoid values for those eight experts, normalize per token, and multiply by `routed_scaling_factor`.\n5. For global expert IDs in `[local_expert_offset, local_expert_offset + 32)`, dequantize the relevant activation and weight blocks, compute one W13 projection, split its 4096 columns into two 2048-column halves, apply SwiGLU, and compute W2.\n6. Accumulate each local expert result into the token output using that expert's combine weight. Experts outside the local interval contribute nothing on that rank.\n\nThe W13 representation permits one logical `A @ W13.T` followed by a split. It does not require two separately allocated TMEM accumulators, and the benchmark contract does not prescribe a tcgen05 instruction sequence."}, "reason": {"statement": "Remove non-executable code and point to the verified derived semantic reference.", "urls": []}} +{"path": "wiki/kernels/fused-moe.md", "before": {"statement": "| SGLang | 1262 | 206.9us | 5 (fused) |\n| FlashInfer CuTe DSL | 1225 | 481.9us | 1-2 (fully fused) |\n| vLLM | 1117 | 369.5us | 7 (unfused) |"}, "after": {"statement": "[`01-routing-plus-fusion-skeleton.py`](../../artifacts/kernels/fused-moe/variants/01-routing-plus-fusion-skeleton.py) is a CPU-checkable, parameterized reference for grouped selection and local W13/SwiGLU/W2 accumulation. It is derived KernelWiki code, not an optimized kernel and not an upstream contest solution."}, "reason": {"statement": "Remove untraceable performance cells rather than inventing a replacement benchmark.", "urls": ["https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048"]}} +{"path": "wiki/kernels/fused-moe.md", "before": {"statement": "1. **No pre-tuned FP8 MoE config for B200**: Tile sizes and pipeline stages need empirical tuning"}, "after": {"statement": "The starter-kit evaluation document at commit `75ccd05cafceb0fd1f86be4cd0f2117249463c66` records:\n\n- bare-metal NVIDIA B200 (`sm_100a`) with clocks locked to `3996,1965`;\n- container `flashinfer/flashinfer-ci-cu132:20260401-2c675fb`;\n- CUDA 13.2, Python 3.12, PyTorch 2.12.0+cu132, and Triton 3.6.0;\n- correctness gates `atol=1`, `rtol=0.3`, and matched ratio `0.9` for the MoE command; and\n- an arithmetic mean of per-workload `FlashInfer baseline latency / candidate latency` as the single-definition MoE score.\n\nThe current primary sources do not support the former framework TFLOPS/latency table, its launch counts, or the structured 1262-TFLOPS record. No performance result is retained here. The official trace axis is `seq_len`; relabeling its endpoints as prefill/decode or batch size requires a separate serving experiment.\n\n- [Starter-kit evaluation contract at `75ccd05`](https://github.com/flashinfer-ai/flashinfer-bench-starter-kit/blob/75ccd05cafceb0fd1f86be4cd0f2117249463c66/EVALUATION.md)"}, "reason": {"statement": "Document the version-pinned autotuner behavior precisely.", "urls": []}} +{"path": "wiki/kernels/fused-moe.md", "before": {"statement": "2. **FP8 numerical overflow**: Block scaling (block size 128) required for stability"}, "after": {"statement": "For `T = seq_len`, the published benchmark signature is:\n\n| `routing_logits` | FP32 | `[T, 256]` |\n| `routing_bias` | BF16 | `[256]` |\n| `hidden_states` | FP8 E4M3FN | `[T, 7168]` |\n| `hidden_states_scale` | FP32 | `[56, T]` |\n| `gemm1_weights` | FP8 E4M3FN | `[32, 4096, 7168]` |\n| `gemm1_weights_scale` | FP32 | `[32, 32, 56]` |\n| `gemm2_weights` | FP8 E4M3FN | `[32, 7168, 2048]` |\n| `gemm2_weights_scale` | FP32 | `[32, 56, 16]` |\n| `local_expert_offset` | INT32 | scalar |\n| `routed_scaling_factor` | FP32 | scalar |\n\nThe output is BF16 `[T, 7168]`. These scale tensors are explicit storage inputs; this page does not infer their runtime cost from their existence.\n\nAt FlashInfer commit `7f614b86470180bab2d22e36fd1775791c6bf3e6`, the corresponding public entry point is `flashinfer.fused_moe.trtllm_fp8_block_scale_moe`. Its complete call includes the eight tensors above plus `num_experts=256`, `top_k=8`, `n_group=8`, `topk_group=4`, `intermediate_size=2048`, the local expert interval, the routed scaling factor, and DeepSeek-V3 routing mode.\n\n- Routing produces different token counts per expert, hence different grouped-GEMM M dimensions. A scheduler can mitigate that imbalance; no single bottleneck is universal across token counts and tactics.\n\n- At the pinned FlashInfer revision, runtime autotuning enumerates valid tactics and selects GEMM1 and GEMM2 tactics over token buckets. Record the exact revision and tuning state in any measurement.\n\n- [FlashInfer trace reference at `7f614b8`](https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/trace/templates/moe.py)\n- [FlashInfer API implementation at `7f614b8`](https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/fused_moe/core.py)"}, "reason": {"statement": "Separate the fixed benchmark ABI from unsupported causal performance language.", "urls": []}} +{"path": "wiki/kernels/fused-moe.md", "before": {"statement": "3. **Batch-size sensitivity**: batch=1 is latency-critical (kernel launch overhead dominates); batch=4096 is throughput-critical"}, "after": {"statement": "[`01-routing-plus-fusion-skeleton.py`](../../artifacts/kernels/fused-moe/variants/01-routing-plus-fusion-skeleton.py) is a CPU-checkable, parameterized reference for grouped selection and local W13/SwiGLU/W2 accumulation. It is derived KernelWiki code, not an optimized kernel and not an upstream contest solution."}, "reason": {"statement": "Use the official axis name and avoid mechanism attribution.", "urls": ["https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048"]}} +{"path": "wiki/kernels/fused-moe.md", "before": {"statement": "5. **TMA alignment**: 128-byte alignment required for all TMA descriptors"}, "after": {"statement": "- Routing produces different token counts per expert, hence different grouped-GEMM M dimensions. A scheduler can mitigate that imbalance; no single bottleneck is universal across token counts and tactics.\n\n- At the pinned FlashInfer revision, runtime autotuning enumerates valid tactics and selects GEMM1 and GEMM2 tactics over token buckets. Record the exact revision and tuning state in any measurement."}, "reason": {"statement": "State that alignment is descriptor-configuration dependent and defer to the exact CUDA contract.", "urls": ["https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"]}} +{"path": "wiki/kernels/fused-moe.md", "before": {"statement": "6. **Dual TMEM allocation**: Gate and up accumulators each need TMEM space, competing for the 256KB budget"}, "after": {"statement": "| `seq_len` | variable | Number of input tokens; this is not labeled request batch size |\n| global experts | 256 | Width of `routing_logits` |\n| local experts | 32 | Experts whose weights are resident on one EP rank |\n| expert parallelism | 8 | `256 / 32` ranks in the published definition |\n| `top_k` | 8 | Selected global experts per token |\n| `n_group` | 8 | Groups of 32 global experts |\n| `topk_group` | 4 | Groups retained before global top-k selection |\n| hidden size | 7168 | Input and output width |\n| intermediate size | 2048 | Per-expert SwiGLU width |\n| GEMM1 output | 4096 | Concatenated W13 output, `2 * 2048` |\n| scale block | 128 | Fixed granularity for this DeepSeek-FP8 trace |\n\nThe `e32` suffix means **32 local experts**, not 32 total experts.\n\nThe official reference performs these steps:\n\n1. Convert routing logits to `s = sigmoid(logits)` and form selection scores `s + routing_bias`.\n2. Reshape 256 scores into eight groups of 32. Sum the top two selection scores in each group, then retain four groups.\n3. Select eight global experts from the retained groups using the biased selection scores.\n4. Form combine weights from the **unbiased** sigmoid values for those eight experts, normalize per token, and multiply by `routed_scaling_factor`.\n5. For global expert IDs in `[local_expert_offset, local_expert_offset + 32)`, dequantize the relevant activation and weight blocks, compute one W13 projection, split its 4096 columns into two 2048-column halves, apply SwiGLU, and compute W2.\n6. Accumulate each local expert result into the token output using that expert's combine weight. Experts outside the local interval contribute nothing on that rank.\n\nThe W13 representation permits one logical `A @ W13.T` followed by a split. It does not require two separately allocated TMEM accumulators, and the benchmark contract does not prescribe a tcgen05 instruction sequence."}, "reason": {"statement": "Describe the logical W13 contract without inventing TMEM ownership.", "urls": ["https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048"]}} +{"path": "wiki/kernels/fused-moe.md", "before": {"statement": "- MoE model inference (DeepSeek-V3, Mixtral, etc.)"}, "after": {"statement": "FlashInfer's MLSys 2026 contest calls Track A **Fused MoE** with FP8 support and targets NVIDIA B200. Its exact benchmark definition is `moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048`. The definition says that DeepSeek-style routing and two grouped GEMMs are included. That operation-level scope does **not** establish that an implementation uses one GPU launch.\n\n- Routing produces different token counts per expert, hence different grouped-GEMM M dimensions. A scheduler can mitigate that imbalance; no single bottleneck is universal across token counts and tactics.\n\n- At the pinned FlashInfer revision, runtime autotuning enumerates valid tactics and selects GEMM1 and GEMM2 tactics over token buckets. Record the exact revision and tuning state in any measurement.\n\n- [MLSys 2026 FlashInfer contest](https://mlsys26.flashinfer.ai/)\n- [Exact FP8 MoE benchmark definition](https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048)"}, "reason": {"statement": "Limit use to producers that match the complete contract.", "urls": ["https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048", "https://mlsys26.flashinfer.ai/"]}} +{"path": "wiki/kernels/fused-moe.md", "before": {"statement": "- Both prefill (high batch, throughput-critical) and decode (low batch, latency-critical)"}, "after": {"statement": "[`01-routing-plus-fusion-skeleton.py`](../../artifacts/kernels/fused-moe/variants/01-routing-plus-fusion-skeleton.py) is a CPU-checkable, parameterized reference for grouped selection and local W13/SwiGLU/W2 accumulation. It is derived KernelWiki code, not an optimized kernel and not an upstream contest solution."}, "reason": {"statement": "Keep the exact workload-axis interpretation.", "urls": ["https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048"]}} +{"path": "wiki/kernels/fused-moe.md", "before": {"statement": "- Expert load imbalance is the primary practical bottleneck"}, "after": {"statement": "The starter-kit evaluation document at commit `75ccd05cafceb0fd1f86be4cd0f2117249463c66` records:\n\n- bare-metal NVIDIA B200 (`sm_100a`) with clocks locked to `3996,1965`;\n- container `flashinfer/flashinfer-ci-cu132:20260401-2c675fb`;\n- CUDA 13.2, Python 3.12, PyTorch 2.12.0+cu132, and Triton 3.6.0;\n- correctness gates `atol=1`, `rtol=0.3`, and matched ratio `0.9` for the MoE command; and\n- an arithmetic mean of per-workload `FlashInfer baseline latency / candidate latency` as the single-definition MoE score.\n\nThe current primary sources do not support the former framework TFLOPS/latency table, its launch counts, or the structured 1262-TFLOPS record. No performance result is retained here. The official trace axis is `seq_len`; relabeling its endpoints as prefill/decode or batch size requires a separate serving experiment.\n\n- Routing produces different token counts per expert, hence different grouped-GEMM M dimensions. A scheduler can mitigate that imbalance; no single bottleneck is universal across token counts and tactics.\n\n- At the pinned FlashInfer revision, runtime autotuning enumerates valid tactics and selects GEMM1 and GEMM2 tactics over token buckets. Record the exact revision and tuning state in any measurement.\n\n- [Starter-kit evaluation contract at `75ccd05`](https://github.com/flashinfer-ai/flashinfer-bench-starter-kit/blob/75ccd05cafceb0fd1f86be4cd0f2117249463c66/EVALUATION.md)"}, "reason": {"statement": "Replace a universal ranking with measurable workload-specific risks.", "urls": []}} +{"path": "wiki/kernels/fused-moe.md", "before": {"statement": "- CUDA graph compatibility requires masked layout (fixed allocation per expert)"}, "after": {"statement": "- Routing produces different token counts per expert, hence different grouped-GEMM M dimensions. A scheduler can mitigate that imbalance; no single bottleneck is universal across token counts and tactics.\n\n- At the pinned FlashInfer revision, runtime autotuning enumerates valid tactics and selects GEMM1 and GEMM2 tactics over token buckets. Record the exact revision and tuning state in any measurement."}, "reason": {"statement": "Make graph/layout requirements backend- and capture-path-specific.", "urls": []}} +{"path": "wiki/kernels/fused-moe.md", "before": {"statement": "- [GPU Mode NVFP4 Hackathon Problem 3](https://github.com/gpu-mode/reference-kernels)"}, "after": {"statement": "- Small M or partially filled GEMM tiles can reduce utilization. This is a workload-dependent risk, not a retained performance result.\n\n- [CUTLASS efficient-GEMM small-dimension discussion](https://github.com/NVIDIA/cutlass/blob/main/media/docs/cpp/efficient_gemm.md)"}, "reason": {"statement": "Remove the misleading direct-source link from the page.", "urls": ["https://github.com/NVIDIA/cutlass/blob/main/media/docs/cpp/efficient_gemm.md"]}} +{"path": "wiki/kernels/fused-moe.md", "before": {"statement": "- [DeepGEMM MoE](https://github.com/deepseek-ai/DeepGEMM)"}, "after": {"statement": "- Small M or partially filled GEMM tiles can reduce utilization. This is a workload-dependent risk, not a retained performance result.\n\n- [CUTLASS efficient-GEMM small-dimension discussion](https://github.com/NVIDIA/cutlass/blob/main/media/docs/cpp/efficient_gemm.md)"}, "reason": {"statement": "Keep direct sources claim-specific; discuss adjacency only in artifact provenance.", "urls": ["https://github.com/NVIDIA/cutlass/blob/main/media/docs/cpp/efficient_gemm.md"]}} +{"path": "wiki/kernels/fused-moe.md", "before": {"statement": "- [SGLang Fused MoE](https://github.com/sgl-project/sglang)"}, "after": {"statement": "- Small M or partially filled GEMM tiles can reduce utilization. This is a workload-dependent risk, not a retained performance result.\n\n- [CUTLASS efficient-GEMM small-dimension discussion](https://github.com/NVIDIA/cutlass/blob/main/media/docs/cpp/efficient_gemm.md)"}, "reason": {"statement": "Retain the byte-pinned file but label it adjacent and non-evidentiary for Track A FP8.", "urls": ["https://github.com/NVIDIA/cutlass/blob/main/media/docs/cpp/efficient_gemm.md"]}} +{"path": "wiki/kernels/fused-moe.md", "before": {"statement": "Verbatim upstream code lives in [`artifacts/kernels/fused-moe/full/`](../../artifacts/kernels/fused-moe/full/); labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` header) live in [`artifacts/kernels/fused-moe/variants/`](../../artifacts/kernels/fused-moe/variants/). Every file's SHA-256 and upstream-pinning metadata is in `PROVENANCE.yaml` inside each bundle."}, "after": {"statement": "- Small M or partially filled GEMM tiles can reduce utilization. This is a workload-dependent risk, not a retained performance result.\n\n- [CUTLASS efficient-GEMM small-dimension discussion](https://github.com/NVIDIA/cutlass/blob/main/media/docs/cpp/efficient_gemm.md)"}, "reason": {"statement": "Preserve the artifacts while disclosing each exact adjacent scope and replacing the variant with a tested semantic reference.", "urls": ["https://github.com/NVIDIA/cutlass/blob/main/media/docs/cpp/efficient_gemm.md"]}} +{"path": "wiki/kernels/fused-moe.md", "before": {"statement": "performance_claims:\n- gpu: B200\n dtype: fp8\n shape: topk=8, experts=32, hidden=7168, intermediate=2048, batch=4096\n metric: TFLOPS\n value: 1262\n utilization: ~56%\n source_id: contest-flashinfer-track-a"}, "after": {"statement": "performance_claims: []\n\n[`01-routing-plus-fusion-skeleton.py`](../../artifacts/kernels/fused-moe/variants/01-routing-plus-fusion-skeleton.py) is a CPU-checkable, parameterized reference for grouped selection and local W13/SwiGLU/W2 accumulation. It is derived KernelWiki code, not an optimized kernel and not an upstream contest solution."}, "reason": {"statement": "Remove inadmissible structured performance metadata.", "urls": ["https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048"]}} +{"path": "wiki/kernels/fused-moe.md", "before": {"statement": "sources:\n- contest-flashinfer-track-a\n- blog-deepgemm\n- pr-vllm-23696"}, "after": {"statement": "sources:\n- contest-flashinfer-track-a"}, "reason": {"statement": "Keep source relationships claim-directed.", "urls": ["https://github.com/NVIDIA/cutlass/blob/main/media/docs/cpp/efficient_gemm.md"]}} +{"path": "wiki/kernels/fused-moe.md", "before": {"statement": "sources:\n- contest-flashinfer-track-a\n- blog-deepgemm\n- pr-vllm-23696"}, "after": {"statement": "sources:\n- contest-flashinfer-track-a"}, "reason": {"statement": "Remove the direct source relationship and disclose adjacency separately.", "urls": ["https://api.github.com/repos/vllm-project/vllm/pulls/23696", "https://github.com/NVIDIA/cutlass/blob/main/media/docs/cpp/efficient_gemm.md"]}} +{"path": "wiki/kernels/fused-moe.md", "before": {"statement": "architectures:\n- sm100\n- sm100a\n- sm90\n\ntags:\n- moe\n- fused-kernel\n- fp8\n- block-scale\n- kernel-fusion\n- warp-specialization\n- grouped-gemm\n- gated-dual-gemm\n\nlanguages:\n- cuda-cpp\n- cute-dsl\n- triton\n\nblackwell_relevance: SM100 enables native FP8 block-scale MoE via tcgen05 with higher\n throughput; technique transfers from Hopper FP8 MoE."}, "after": {"statement": "architectures:\n- sm100\n- sm100a\n\ntags:\n- moe\n- fp8\n- block-scale\n- grouped-gemm\n- kernel-fusion\n\nlanguages:\n- cuda-cpp\n- cute-dsl\n- triton\n\nblackwell_relevance: The MLSys 2026 Track A definition and official evaluation\n target NVIDIA B200 (sm_100a); this page documents the logical benchmark\n contract, not a particular launch decomposition."}, "reason": {"statement": "Limit frontmatter to the verified B200/SM100a contract and implementation languages accepted by the contest.", "urls": ["https://mlsys26.flashinfer.ai/", "https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048"]}} +{"path": "wiki/kernels/gated-delta-net.md", "before": {"statement": "The key advantage is O(1) per-token cost during decoding: the recurrent state is a fixed-size matrix that gets updated with each new token, eliminating the KV cache growth problem entirely."}, "after": {"statement": "- NVlabs commit `b53d6d3` is the authors' PyTorch/Triton research implementation. Its README recommends FLA for faster kernels and variable-length functionality."}, "reason": {"statement": "Preserve the fixed-state advantage while stating the hybrid-model boundary.", "urls": ["https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/config.json", "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct"]}} +{"path": "wiki/kernels/gated-delta-net.md", "before": {"statement": "```\nQwen3-Next-80B: 48 layers\nPattern: 12 x (3 x [GatedDeltaNet -> MoE] -> [Full Attention -> MoE])\n\nLayer distribution:\n - 36 GatedDeltaNet layers (75%): O(n) linear attention\n - 12 Full Attention layers (25%): Standard GQA for global retrieval\n - All layers followed by MoE: 512 experts, ~19 active per token\n\nTotal: 80B parameters, only 3B active per token\n```"}, "after": {"statement": "The immutable `Qwen3-Next-80B-A3B-Instruct` configuration at revision `9c7f2fbe` records:\n\n| Layers | 48 |\n| Hybrid layout | `12 * (3 * Gated DeltaNet -> MoE, 1 * Gated Attention -> MoE)` |\n| GDN heads | 16 QK heads, 32 value heads, head dimension 128 |\n| MoE | 512 experts; 10 routed experts plus one shared expert |\n| Parameters | 80B total, 3B activated |\n| Native context | 262,144 tokens |\n\nThus 36 layers use GDN and 12 use full Gated Attention. The fixed GDN state does not eliminate cache growth for the whole hybrid model: the full-attention layers retain their own context-dependent cache."}, "reason": {"statement": "Use the immutable model-card architecture fields.", "urls": ["https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct", "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/config.json", "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/tree/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7"]}} +{"path": "wiki/kernels/gated-delta-net.md", "before": {"statement": "```python\n# Standard linear attention (additive):\n# S_t = S_{t-1} + v_t @ k_t^T\n#\n# Delta rule (error-correcting):\n# S_t = S_{t-1} + (v_t - S_{t-1} @ k_t) @ k_t^T\n# ^^^^^^^^^^^^^^^^^^^^^^^^\n# Error correction term\n\ndef delta_rule_step(S, k, v, beta, alpha):\n \"\"\"\n Single step of gated delta rule update.\n\n S: recurrent state [d_k, d_v] (the \"memory matrix\")\n k: key vector [d_k]\n v: value vector [d_v]\n beta: exponential gate (learned, controls decay)\n alpha: delta gate (learned, controls update strength)\n \"\"\"\n # Retrieve what the current state \"thinks\" about this key\n v_retrieved = S @ k # [d_v]\n\n # Error: difference between true value and retrieved value\n delta = v - v_retrieved # [d_v]\n\n # Gated update: decay old state, add error-corrected new info\n S_new = beta * S + alpha * (delta[:, None] @ k[None, :])\n # ^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n # decay error-correcting update\n\n return S_new\n```"}, "after": {"statement": "Gated DeltaNet is a recurrent linear-attention architecture published at ICLR 2025. For one head with state `S` shaped `[K,V]`, the FlashInfer reference at commit `7f614b8` implements the update in this equivalent form:\n\n```python\ndef gdn_step(S, q, k, v, A_log, a, dt_bias, b, scale):\n g = exp(-exp(A_log) * softplus(a + dt_bias))\n beta = sigmoid(b)\n decayed = g * S\n read = k @ decayed\n S_new = decayed + outer(k, beta * (v - read))\n output = scale * (q @ S_new)\n return output, S_new\n```\n\nThe subtraction is the delta-rule correction: the update moves the value retrieved at `k` toward `v`, while `g` independently decays the previous state. The recurrent state has no prior-token axis, so a single GDN decode step has work and storage fixed with respect to context length. Its per-head state update is still proportional to `K*V`; “constant” here means constant in the number of earlier tokens.\n\nTFLA (`arXiv:2503.14376v3`) is related linear-recurrence work, but its published application and official code are for mLSTM. They use a second level of sequence parallelization to permit arbitrarily large chunks. They do not establish a GDN implementation or an inline WGMMA/tcgen05 path, so no TFLA assembly is presented here."}, "reason": {"statement": "Replace with an executable recurrence matching the exact reference.", "urls": []}} +{"path": "wiki/kernels/gated-delta-net.md", "before": {"statement": "During prefill, sequences are divided into chunks that can be processed in parallel. Within each chunk, the inter-token dependencies are resolved via a causal linear recurrence; across chunks, the recurrent state is propagated sequentially."}, "after": {"statement": "The MLSys 2026 organizer identifies Gated Delta Net as Track C and links separate verified definitions captured from Qwen3-Next with tensor parallelism four.\n\n| Q heads / K heads / V heads | `4 / 4 / 8` | `4 / 4 / 8` |\n| Head size | `128` | `128` |\n| Token axis | `seq_len=1`, variable `batch_size` | variable `total_seq_len` and `num_seqs` |\n| Q/K/V dtype | BF16 | BF16 |\n| State | FP32 `[B,8,128,128]` | FP32 `[N,8,128,128]` |\n| Variable-length metadata | none | `cu_seqlens[N+1]` |\n\nDecode exposes `A_log`, `a`, `dt_bias`, and `b` for the decay and update gates, plus an optional scale. It returns BF16 output `[B,1,8,128]` and the updated state. Prefill returns BF16 output `[total_seq_len,8,128]` and one final state per sequence.\n\nFor this exact geometry, one value-head state contains `128*128 = 16,384` FP32 values, or 64 KiB. All eight value heads contain 131,072 FP32 values, or 512 KiB, per sequence and layer. The heads are independent states; they must not be flattened into one `512x1024` matrix."}, "reason": {"statement": "Describe only the verified chunkwise boundary and point to the exact implementation.", "urls": ["https://mlsys26.flashinfer.ai/", "https://bench.flashinfer.ai/kernels/gdn_decode_qk4_v8_d128_k_last", "https://bench.flashinfer.ai/kernels/gdn_prefill_qk4_v8_d128_k_last"]}} +{"path": "wiki/kernels/gated-delta-net.md", "before": {"statement": "```python\nimport triton\nimport triton.language as tl\n\n@triton.jit\ndef gated_delta_net_chunk_fwd(\n Q_ptr, K_ptr, V_ptr, Beta_ptr, O_ptr, State_ptr,\n SEQ_LEN: tl.constexpr,\n CHUNK_SIZE: tl.constexpr, # e.g., 64 or 128\n D_QK: tl.constexpr, # qk_dim * d = 4 * 128 = 512\n D_V: tl.constexpr, # v_dim * d = 8 * 128 = 1024\n):\n \"\"\"\n Chunk-parallel forward pass for GatedDeltaNet.\n Two levels of parallelism:\n 1. Across chunks (parallel after inter-chunk state propagation)\n 2. Within chunks (parallel via matrix operations)\n \"\"\"\n chunk_id = tl.program_id(0)\n batch_id = tl.program_id(1)\n head_id = tl.program_id(2)\n\n chunk_start = chunk_id * CHUNK_SIZE\n\n # Load recurrent state from previous chunk\n # S shape: [D_QK, D_V]\n S = tl.load(State_ptr + (batch_id * NUM_HEADS + head_id) * D_QK * D_V\n + tl.arange(0, D_QK)[:, None] * D_V\n + tl.arange(0, D_V)[None, :])\n\n # Intra-chunk computation\n for t in range(CHUNK_SIZE):\n pos = chunk_start + t\n\n # Load q, k, v, beta for this token\n q = tl.load(Q_ptr + pos * D_QK + tl.arange(0, D_QK))\n k = tl.load(K_ptr + pos * D_QK + tl.arange(0, D_QK))\n v = tl.load(V_ptr + pos * D_V + tl.arange(0, D_V))\n beta = tl.load(Beta_ptr + pos)\n\n # Output: query the state\n o = tl.sum(S * q[:, None], axis=0) # [D_V]\n\n # Delta rule update\n v_retrieved = tl.sum(S * k[:, None], axis=0)\n delta = v - v_retrieved\n\n # Gated state update\n S = beta * S + delta[:, None] * k[None, :] # [D_QK, D_V]\n\n # Store output\n tl.store(O_ptr + pos * D_V + tl.arange(0, D_V), o)\n\n # Store final state for next chunk\n tl.store(State_ptr + (batch_id * NUM_HEADS + head_id) * D_QK * D_V\n + tl.arange(0, D_QK)[:, None] * D_V\n + tl.arange(0, D_V)[None, :], S)\n```"}, "after": {"statement": "Qwen's architecture article attributes attention-sink and massive-activation mitigation to the output gate in its full Gated Attention path and says the gate *helps* address those effects. That is separate from the GDN recurrence and is not a requirement imposed by the GDN kernel contract.\n\nTFLA (`arXiv:2503.14376v3`) is related linear-recurrence work, but its published application and official code are for mLSTM. They use a second level of sequence parallelization to permit arbitrarily large chunks. They do not establish a GDN implementation or an inline WGMMA/tcgen05 path, so no TFLA assembly is presented here."}, "reason": {"statement": "A short exact recurrence and pinned production sources are more useful than a misleading non-executable kernel.", "urls": ["https://bench.flashinfer.ai/kernels/gdn_prefill_qk4_v8_d128_k_last", "https://qwen.ai/blog?id=e34c4305036ce60d55a0791b170337c2b70ae51d"]}} +{"path": "wiki/kernels/gated-delta-net.md", "before": {"statement": "```python\n@triton.jit\ndef gated_delta_net_decode(\n Q_ptr, K_ptr, V_ptr, Beta_ptr, O_ptr, State_ptr,\n D_QK: tl.constexpr,\n D_V: tl.constexpr,\n):\n \"\"\"\n Single-token decode: O(1) per token.\n The recurrent state replaces the KV cache entirely.\n State size: D_QK * D_V (e.g., 512 * 1024 = 512K floats per head)\n \"\"\"\n batch_id = tl.program_id(0)\n head_id = tl.program_id(1)\n\n # Load persistent recurrent state\n state_offset = (batch_id * NUM_HEADS + head_id) * D_QK * D_V\n S = tl.load(State_ptr + state_offset\n + tl.arange(0, D_QK)[:, None] * D_V\n + tl.arange(0, D_V)[None, :])\n\n # Load new token\n q = tl.load(Q_ptr + tl.arange(0, D_QK))\n k = tl.load(K_ptr + tl.arange(0, D_QK))\n v = tl.load(V_ptr + tl.arange(0, D_V))\n beta = tl.load(Beta_ptr)\n\n # Query state for output\n o = tl.sum(S * q[:, None], axis=0)\n\n # Update state with delta rule\n v_retrieved = tl.sum(S * k[:, None], axis=0)\n delta = v - v_retrieved\n S = beta * S + delta[:, None] * k[None, :]\n\n # Store\n tl.store(O_ptr + tl.arange(0, D_V), o)\n tl.store(State_ptr + state_offset\n + tl.arange(0, D_QK)[:, None] * D_V\n + tl.arange(0, D_V)[None, :], S)\n```"}, "after": {"statement": "Qwen's architecture article attributes attention-sink and massive-activation mitigation to the output gate in its full Gated Attention path and says the gate *helps* address those effects. That is separate from the GDN recurrence and is not a requirement imposed by the GDN kernel contract.\n\nTFLA (`arXiv:2503.14376v3`) is related linear-recurrence work, but its published application and official code are for mLSTM. They use a second level of sequence parallelization to permit arbitrarily large chunks. They do not establish a GDN implementation or an inline WGMMA/tcgen05 path, so no TFLA assembly is presented here."}, "reason": {"statement": "Remove code that cannot represent the benchmark semantics.", "urls": ["https://bench.flashinfer.ai/kernels/gdn_decode_qk4_v8_d128_k_last", "https://qwen.ai/blog?id=e34c4305036ce60d55a0791b170337c2b70ae51d"]}} +{"path": "wiki/kernels/gated-delta-net.md", "before": {"statement": "TFLA adds a second level of tiling within chunks, enabling arbitrarily large chunk sizes. It emits matmuls as inline PTX assembly for both Hopper (WGMMA) and Blackwell (tcgen05)."}, "after": {"statement": "The `full/` bundle contains one byte-pinned SGLang file from merge `5bdc07d974f6cf236fa765a685453ea5e587a838`. It fuses projection-output split/reshape/concatenation for Qwen3-Next/Qwen3.5; it is adjacent preprocessing, not a GDN recurrence, prefill, or decode implementation."}, "reason": {"statement": "Retain TFLA only as separate mLSTM background and remove implementation transfer.", "urls": ["https://arxiv.org/abs/2503.14376"]}} +{"path": "wiki/kernels/gated-delta-net.md", "before": {"statement": "```cpp\n// TFLA: Inline PTX for Blackwell tcgen05 matmul within chunk tiles\n// Two levels of parallelism: standard chunkwise + tiling within chunks\n\n// SM100 path: tcgen05.mma for intra-chunk matrix operations\nasm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16f16f32\"\n \" [%0], %1, %2;\"\n :\n : \"l\"(tmem_addr), \"l\"(a_smem_addr), \"l\"(b_smem_addr)\n);\n```"}, "after": {"statement": "The `full/` bundle contains one byte-pinned SGLang file from merge `5bdc07d974f6cf236fa765a685453ea5e587a838`. It fuses projection-output split/reshape/concatenation for Qwen3-Next/Qwen3.5; it is adjacent preprocessing, not a GDN recurrence, prefill, or decode implementation."}, "reason": {"statement": "Delete fabricated assembly rather than implying it can compile.", "urls": []}} +{"path": "wiki/kernels/gated-delta-net.md", "before": {"statement": "- Status: decode done for both Hopper and Blackwell; prefill done on Hopper, in progress for Blackwell"}, "after": {"statement": "The MLSys 2026 organizer identifies Gated Delta Net as Track C and links separate verified definitions captured from Qwen3-Next with tensor parallelism four.\n\n| Q heads / K heads / V heads | `4 / 4 / 8` | `4 / 4 / 8` |\n| Head size | `128` | `128` |\n| Token axis | `seq_len=1`, variable `batch_size` | variable `total_seq_len` and `num_seqs` |\n| Q/K/V dtype | BF16 | BF16 |\n| State | FP32 `[B,8,128,128]` | FP32 `[N,8,128,128]` |\n| Variable-length metadata | none | `cu_seqlens[N+1]` |\n\nDecode exposes `A_log`, `a`, `dt_bias`, and `b` for the decay and update gates, plus an optional scale. It returns BF16 output `[B,1,8,128]` and the updated state. Prefill returns BF16 output `[total_seq_len,8,128]` and one final state per sequence.\n\nFor this exact geometry, one value-head state contains `128*128 = 16,384` FP32 values, or 64 KiB. All eight value heads contain 131,072 FP32 values, or 512 KiB, per sequence and layer. The heads are independent states; they must not be flattened into one `512x1024` matrix."}, "reason": {"statement": "Replace stale issue status with pinned implementation status and prerequisites.", "urls": ["https://mlsys26.flashinfer.ai/", "https://bench.flashinfer.ai/kernels/gdn_decode_qk4_v8_d128_k_last", "https://bench.flashinfer.ai/kernels/gdn_prefill_qk4_v8_d128_k_last"]}} +{"path": "wiki/kernels/gated-delta-net.md", "before": {"statement": "- Long-context inference (32K+) where O(n) scaling provides major throughput gains"}, "after": {"statement": "- [Gated DeltaNet paper](https://arxiv.org/abs/2412.06464)\n- [NVlabs implementation at `b53d6d3`](https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921)\n- [Qwen3-Next model card and configuration at `9c7f2fbe`](https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/tree/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7)\n- [MLSys 2026 organizer](https://mlsys26.flashinfer.ai/)\n- [Exact decode definition](https://bench.flashinfer.ai/kernels/gdn_decode_qk4_v8_d128_k_last)\n- [Exact prefill definition](https://bench.flashinfer.ai/kernels/gdn_prefill_qk4_v8_d128_k_last)\n- [FlashInfer GDN trace at `7f614b8`](https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/trace/templates/gdn.py)\n- [FlashInfer prefill dispatch at `7f614b8`](https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/gdn_prefill.py)\n- [CUDA Graphs programming guide](https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cuda-graphs.html)\n- [TFLA v3](https://arxiv.org/abs/2503.14376v3)"}, "reason": {"statement": "Retain asymptotic and model-report facts without claiming an isolated threshold or universal gain.", "urls": ["https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct"]}} +{"path": "wiki/kernels/gated-delta-net.md", "before": {"statement": "- Streaming decode where O(1) per-token cost eliminates KV cache growth"}, "after": {"statement": "- NVlabs commit `b53d6d3` is the authors' PyTorch/Triton research implementation. Its README recommends FLA for faster kernels and variable-length functionality."}, "reason": {"statement": "State the layer-local storage property and hybrid exception.", "urls": ["https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/config.json"]}} +{"path": "wiki/kernels/gated-delta-net.md", "before": {"statement": "- Recurrent state size (D_QK * D_V per head) can be large -- 512K floats for typical configs"}, "after": {"statement": "Qwen's architecture article attributes attention-sink and massive-activation mitigation to the output gate in its full Gated Attention path and says the gate *helps* address those effects. That is separate from the GDN recurrence and is not a requirement imposed by the GDN kernel contract."}, "reason": {"statement": "Use exact benchmark dimensions and distinguish per-head from aggregate storage.", "urls": ["https://bench.flashinfer.ai/kernels/gdn_decode_qk4_v8_d128_k_last", "https://qwen.ai/blog?id=e34c4305036ce60d55a0791b170337c2b70ae51d"]}} +{"path": "wiki/kernels/gated-delta-net.md", "before": {"statement": "- Attention output gating (in Qwen3.5) is required to eliminate Attention Sink and Massive Activation problems"}, "after": {"statement": "- vLLM merge `e1d85e5c` gives its recurrent-attention backend uniform-batch CUDA-graph support for decode. CUDA graphs reduce CPU launch setup cost, but their benefit remains workload-dependent."}, "reason": {"statement": "Keep the useful architectural caveat with its exact scope and modality.", "urls": ["https://qwen.ai/blog?id=e34c4305036ce60d55a0791b170337c2b70ae51d", "https://github.com/vllm-project/vllm/blob/e1d85e5c2454bd8d349dbe676679380cbe0e920a/vllm/v1/attention/backends/mamba_attn.py", "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cuda-graphs.html"]}} +{"path": "wiki/kernels/gated-delta-net.md", "before": {"statement": "Verbatim upstream code lives in [`artifacts/kernels/gated-delta-net/full/`](../../artifacts/kernels/gated-delta-net/full/); labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` header) live in [`artifacts/kernels/gated-delta-net/variants/`](../../artifacts/kernels/gated-delta-net/variants/). Every file's SHA-256 and upstream-pinning metadata is in `PROVENANCE.yaml` inside each bundle."}, "after": {"statement": "TFLA (`arXiv:2503.14376v3`) is related linear-recurrence work, but its published application and official code are for mLSTM. They use a second level of sequence parallelization to permit arbitrarily large chunks. They do not establish a GDN implementation or an inline WGMMA/tcgen05 path, so no TFLA assembly is presented here.\n\nGDN sequence mixing scales linearly with sequence length, but asymptotic complexity is not a measured speedup. Qwen reports a 10x whole-model inference-throughput comparison against Qwen3-32B for contexts over 32K, while also warning that efficiency depends strongly on implementation. That result does not isolate this kernel, name a GPU, or describe the Track C qk4/v8/d128 workloads, so it is not stored as a kernel performance record.\n\nUse an exact backend measurement for the intended batch, sequence distribution, dtype, state layout, software revision, and GPU. Also account for the context-growing full-attention cache in a hybrid model and for the GDN layer's learned projections, gates, convolution, and state; it is not a weight-compatible runtime substitute for a trained softmax-attention layer."}, "reason": {"statement": "Relabel the upstream projection helper as adjacent and replace false variants with one tested semantic recurrence.", "urls": ["https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/tree/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7", "https://github.com/NVlabs/GatedDeltaNet/blob/b53d6d3a161267432a79c1c04af69fa52bddc921/lit_gpt/gated_delta_net.py"]}} +{"path": "wiki/kernels/gated-delta-net.md", "before": {"statement": "performance_claims:\n- gpu: H100\n dtype: bf16\n shape: seqlen=8192, qk_dim=4, v_dim=8, d=128\n metric: speedup\n value: 10\n utilization: vs Qwen3-32B at 32K+ context, O(n) linear complexity\n source_id: blog-gated-delta-net"}, "after": {"statement": "performance_claims: []\n\n- [Gated DeltaNet paper](https://arxiv.org/abs/2412.06464)\n- [NVlabs implementation at `b53d6d3`](https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921)\n- [Qwen3-Next model card and configuration at `9c7f2fbe`](https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/tree/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7)\n- [MLSys 2026 organizer](https://mlsys26.flashinfer.ai/)\n- [Exact decode definition](https://bench.flashinfer.ai/kernels/gdn_decode_qk4_v8_d128_k_last)\n- [Exact prefill definition](https://bench.flashinfer.ai/kernels/gdn_prefill_qk4_v8_d128_k_last)\n- [FlashInfer GDN trace at `7f614b8`](https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/trace/templates/gdn.py)\n- [FlashInfer prefill dispatch at `7f614b8`](https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/gdn_prefill.py)\n- [CUDA Graphs programming guide](https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cuda-graphs.html)\n- [TFLA v3](https://arxiv.org/abs/2503.14376v3)"}, "reason": {"statement": "No evidence-supported numeric kernel record can replace the fabricated structured tuple.", "urls": ["https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct"]}} +{"path": "wiki/kernels/gated-dual-gemm.md", "before": {"statement": "Gated dual GEMM fuses two matrix multiplications with activation and elementwise operations — the canonical MLP gate-up pattern used by LLaMA, Qwen, DeepSeek, and most modern LLMs. Fusion eliminates two global memory roundtrips compared to separate gate/up GEMM + SwiGLU."}, "after": {"statement": "The official NVIDIA rules identify NVFP4 Gated Dual GEMM as Kernel Challenge 3 of the Blackwell NVFP4 Hackathon, open from December 20, 2025 through January 16, 2026. The public GPU Mode problem at challenge-opening commit `c5b2f7c` targets NVIDIA B200.\n\nThe shared `a` and `sfa` inputs create an opportunity to reuse data across the two products, and a genuinely fused epilogue can avoid writing both full-precision products to global memory. Those are optimization possibilities, not task guarantees.\n\nThe public task/reference does not establish a final leaderboard, winning source, single-launch decomposition, physical load count, TMEM accumulator partition, TMA pipeline, CUTLASS schedule, or compute-/memory-bound classification. If an implementation uses `tcgen05`, TMEM, or TMA, its exact descriptors, scale layouts, synchronization, allocation, and architecture requirements must be verified from that implementation and the version-matched ISA.\n\nCompatibility is correspondingly narrower than “any dual-output operation”: a model/backend must match the two same-shaped NVFP4 products, scale storage and layouts, first-branch SiLU, FP16 output, batching, and divisibility rules. Grouped per-expert MoE execution is a different contract unless an implementation explicitly supplies that grouping layer."}, "reason": {"statement": "Retain exact dual-GEMM/SwiGLU semantics while separating potential fusion benefits from contract guarantees.", "urls": ["https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf", "https://github.com/gpu-mode/reference-kernels/blob/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm/task.yml"]}} +{"path": "wiki/kernels/gated-dual-gemm.md", "before": {"statement": "```\nGiven x, W_gate, W_up (weights shared same K dimension)\nStandard (unfused):\n gate = x @ W_gate (GEMM 1: reads x, W_gate, writes gate)\n up = x @ W_up (GEMM 2: reads x, W_up, writes up)\n silu = gate * sigmoid(gate) (elementwise: reads gate, writes silu)\n out = silu * up (elementwise: reads silu, up, writes out)\n\nFused:\n out = SiLU(x @ W_gate) * (x @ W_up) (single kernel, no intermediate GMEM)\n```"}, "after": {"statement": "| `a` | NVFP4 E2M1 | `[M,K,L]`, K-major |\n| `b1`, `b2` | NVFP4 E2M1 | `[N,K,L]`, K-major |\n| `sfa` | FP8 E4M3FNUZ | `[M,K/16,L]`, K-major |\n| `sfb1`, `sfb2` | FP8 E4M3FNUZ | `[N,K/16,L]`, K-major |\n| `c` | FP16 | `[M,N,L]` |\n\nThe submission tuple also supplies layout-reordered copies of `sfa`, `sfb1`, and `sfb2`, plus preallocated `c`. `K` is divisible by 256; `M` and `N` must be divisible by the selected MMA tile dimensions. The correctness checker uses `rtol=1e-3` and `atol=1e-3`.\n\nThe pinned upstream files contain one dtype-label inconsistency: `task.yml` and `template.py` call the scales E4M3FNUZ, while `reference.py` constructs `torch.float8_e4m3fn`. A backend must follow the actual submission objects rather than treating those spellings as interchangeable.\n\nThe shared `a` and `sfa` inputs create an opportunity to reuse data across the two products, and a genuinely fused epilogue can avoid writing both full-precision products to global memory. Those are optimization possibilities, not task guarantees.\n\nThe public task/reference does not establish a final leaderboard, winning source, single-launch decomposition, physical load count, TMEM accumulator partition, TMA pipeline, CUTLASS schedule, or compute-/memory-bound classification. If an implementation uses `tcgen05`, TMEM, or TMA, its exact descriptors, scale layouts, synchronization, allocation, and architecture requirements must be verified from that implementation and the version-matched ISA.\n\nCompatibility is correspondingly narrower than “any dual-output operation”: a model/backend must match the two same-shaped NVFP4 products, scale storage and layouts, first-branch SiLU, FP16 output, batching, and divisibility rules. Grouped per-expert MoE execution is a different contract unless an implementation explicitly supplies that grouping layer."}, "reason": {"statement": "Keep the exact formula without converting a possible optimization into an ABI requirement.", "urls": []}} +{"path": "wiki/kernels/gated-dual-gemm.md", "before": {"statement": "```cuda\ntemplate \n__global__ void gated_dual_gemm_nvfp4(\n const nvfp4_t* __restrict__ X, // [M, K] input\n const nvfp4_t* __restrict__ W_gate, // [N, K] gate weights\n const nvfp4_t* __restrict__ W_up, // [N, K] up weights\n const fp8_t* sf_x, const fp8_t* sf_gate, const fp8_t* sf_up,\n half* __restrict__ output, // [M, N] output\n int M, int N, int K\n) {\n // Two TMEM regions: one accumulator per GEMM\n uint32_t tmem_gate = tmem_alloc(256);\n uint32_t tmem_up = tmem_alloc(256);\n\n for (int k = 0; k < K; k += BLOCK_K) {\n int stage = (k / BLOCK_K) % NUM_STAGES;\n mbarrier_wait(&tma_done[stage]);\n\n // Two MMAs per K-tile: gate and up\n // Both read same X tile, different weight tiles\n tcgen05_mma(x_smem[stage], wg_smem[stage], tmem_gate);\n tcgen05_mma(x_smem[stage], wu_smem[stage], tmem_up);\n }\n\n // Fused epilogue: SiLU(gate) * up\n float g = tmem_load(tmem_gate);\n float u = tmem_load(tmem_up);\n float s = g / (1.0f + expf(-g)); // SiLU\n output[row * N + col] = __float2half(s * u);\n\n tmem_dealloc(tmem_gate, 256);\n tmem_dealloc(tmem_up, 256);\n}\n```"}, "after": {"statement": "| `a` | NVFP4 E2M1 | `[M,K,L]`, K-major |\n| `b1`, `b2` | NVFP4 E2M1 | `[N,K,L]`, K-major |\n| `sfa` | FP8 E4M3FNUZ | `[M,K/16,L]`, K-major |\n| `sfb1`, `sfb2` | FP8 E4M3FNUZ | `[N,K/16,L]`, K-major |\n| `c` | FP16 | `[M,N,L]` |\n\nThe submission tuple also supplies layout-reordered copies of `sfa`, `sfb1`, and `sfb2`, plus preallocated `c`. `K` is divisible by 256; `M` and `N` must be divisible by the selected MMA tile dimensions. The correctness checker uses `rtol=1e-3` and `atol=1e-3`.\n\nThe pinned upstream files contain one dtype-label inconsistency: `task.yml` and `template.py` call the scales E4M3FNUZ, while `reference.py` constructs `torch.float8_e4m3fn`. A backend must follow the actual submission objects rather than treating those spellings as interchangeable.\n\nThe shared `a` and `sfa` inputs create an opportunity to reuse data across the two products, and a genuinely fused epilogue can avoid writing both full-precision products to global memory. Those are optimization possibilities, not task guarantees.\n\nThe public task/reference does not establish a final leaderboard, winning source, single-launch decomposition, physical load count, TMEM accumulator partition, TMA pipeline, CUTLASS schedule, or compute-/memory-bound classification. If an implementation uses `tcgen05`, TMEM, or TMA, its exact descriptors, scale layouts, synchronization, allocation, and architecture requirements must be verified from that implementation and the version-matched ISA.\n\nCompatibility is correspondingly narrower than “any dual-output operation”: a model/backend must match the two same-shaped NVFP4 products, scale storage and layouts, first-branch SiLU, FP16 output, batching, and divisibility rules. Grouped per-expert MoE execution is a different contract unless an implementation explicitly supplies that grouping layer."}, "reason": {"statement": "Replace non-executable hardware pseudocode with an exact host-checkable semantic reference and pinned upstream sources.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"]}} +{"path": "wiki/kernels/gated-dual-gemm.md", "before": {"statement": "1. **X reuse**: Same X tile feeds both MMAs — loaded once, used twice\n2. **TMEM dual accumulator**: Blackwell's 512-column TMEM fits two 256-col accumulators side-by-side\n3. **Fused epilogue**: SiLU and multiply happen after TMEM load, no intermediate SMEM\n4. **Shared SFA**: X's block scales apply to both gate and up computations"}, "after": {"statement": "For each batch index, the required result is two block-scaled matrix products sharing the same left operand, with SiLU applied only to the first branch:\n\n```python\ndef gated_dual_result(a, b1, b2, scale_a, scale_b1, scale_b2, scaled_mm):\n gate = scaled_mm(a, b1.T, scale_a, scale_b1)\n up = scaled_mm(a, b2.T, scale_a, scale_b2)\n return silu(gate) * up\n```\n\nThis fixes result semantics and branch order. It does not require the submission entry point to use one GPU launch or forbid intermediate storage.\n\nThe shared `a` and `sfa` inputs create an opportunity to reuse data across the two products, and a genuinely fused epilogue can avoid writing both full-precision products to global memory. Those are optimization possibilities, not task guarantees.\n\nThe public task/reference does not establish a final leaderboard, winning source, single-launch decomposition, physical load count, TMEM accumulator partition, TMA pipeline, CUTLASS schedule, or compute-/memory-bound classification. If an implementation uses `tcgen05`, TMEM, or TMA, its exact descriptors, scale layouts, synchronization, allocation, and architecture requirements must be verified from that implementation and the version-matched ISA.\n\nCompatibility is correspondingly narrower than “any dual-output operation”: a model/backend must match the two same-shaped NVFP4 products, scale storage and layouts, first-branch SiLU, FP16 output, batching, and divisibility rules. Grouped per-expert MoE execution is a different contract unless an implementation explicitly supplies that grouping layer."}, "reason": {"statement": "Separate exact shared-input semantics from unsupported schedule claims.", "urls": []}} +{"path": "wiki/kernels/gated-dual-gemm.md", "before": {"statement": "- MLP layers in modern LLMs (LLaMA, Qwen, DeepSeek, Mistral)\n- Any dual-output operation sharing one input\n- MoE expert computations (expand to per-expert fused kernels)"}, "after": {"statement": "For each batch index, the required result is two block-scaled matrix products sharing the same left operand, with SiLU applied only to the first branch:\n\n```python\ndef gated_dual_result(a, b1, b2, scale_a, scale_b1, scale_b2, scaled_mm):\n gate = scaled_mm(a, b1.T, scale_a, scale_b1)\n up = scaled_mm(a, b2.T, scale_a, scale_b2)\n return silu(gate) * up\n```\n\nThis fixes result semantics and branch order. It does not require the submission entry point to use one GPU launch or forbid intermediate storage.\n\nThe shared `a` and `sfa` inputs create an opportunity to reuse data across the two products, and a genuinely fused epilogue can avoid writing both full-precision products to global memory. Those are optimization possibilities, not task guarantees.\n\nThe public task/reference does not establish a final leaderboard, winning source, single-launch decomposition, physical load count, TMEM accumulator partition, TMA pipeline, CUTLASS schedule, or compute-/memory-bound classification. If an implementation uses `tcgen05`, TMEM, or TMA, its exact descriptors, scale layouts, synchronization, allocation, and architecture requirements must be verified from that implementation and the version-matched ISA.\n\nCompatibility is correspondingly narrower than “any dual-output operation”: a model/backend must match the two same-shaped NVFP4 products, scale storage and layouts, first-branch SiLU, FP16 output, batching, and divisibility rules. Grouped per-expert MoE execution is a different contract unless an implementation explicitly supplies that grouping layer."}, "reason": {"statement": "Replace universal use advice with an explicit compatibility checklist.", "urls": []}} +{"path": "wiki/kernels/gated-dual-gemm.md", "before": {"statement": "The reference bundle lives in [`artifacts/kernels/gated-dual-gemm/full/`](../../artifacts/kernels/gated-dual-gemm/full/) and combines the upstream vLLM PR-23696 diff (`vllm-PR-23696-gated-dual-gemm.patch`, `mode: upstream-patch`) with an extracted CUTLASS-schedule snippet from the `tflops-gap-fp4-moe` blog (`blackwell-cutlass-schedules-and-tma.cu`, `mode: extracted`). Labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` header) live in [`artifacts/kernels/gated-dual-gemm/variants/`](../../artifacts/kernels/gated-dual-gemm/variants/). Every file's SHA-256 and upstream-pinning metadata is in `PROVENANCE.yaml` inside each bundle."}, "after": {"statement": "The `full/` bundle contains the byte-pinned official `task.yml` from commit `c5b2f7c`; it is the problem specification, not an optimized submission. The unrelated third-party schedule extract and duplicate vLLM MXFP4 MoE patch formerly stored here were removed; the latter remains preserved in its own PR artifact collection."}, "reason": {"statement": "Preserve useful evidence with corrected scope and replace the untested variant with a discriminating semantic test.", "urls": []}} +{"path": "wiki/kernels/gated-dual-gemm.md", "before": {"statement": "performance_claims:\n- gpu: B200\n dtype: nvfp4\n shape: M=1024 N=2*2048 K=7168 (gate-up MLP)\n metric: latency_us\n value: 18.5\n utilization: compute-bound\n source_id: contest-gpumode-p3"}, "after": {"statement": "performance_claims: []\n\n| 256 | 4096 | 7168 | 1 | 4.708 |\n| 512 | 4096 | 7168 | 1 | 8.714 |\n| 256 | 3072 | 4096 | 1 | 2.125 |\n| 512 | 3072 | 7168 | 1 | 6.535 |\n\nRanking uses the geometric mean of the four benchmark times. The task explicitly labels the final column a speed-of-light analysis based on the maximum of B200 FP4 Tensor Core math time and DRAM-memory time at a 1.5 GHz clock. These values are theoretical comparison bounds, not measured contestant results. The former `M=1024, N=4096, K=7168, 18.5 µs` record is not one of the published workloads and has been removed."}, "reason": {"statement": "Remove the fabricated measurement and retain only the clearly labeled theoretical task table.", "urls": []}} +{"path": "wiki/kernels/gated-dual-gemm.md", "before": {"statement": "```cuda\ntemplate \n__global__ void gated_dual_gemm_nvfp4(\n const nvfp4_t* __restrict__ X, // [M, K] input\n const nvfp4_t* __restrict__ W_gate, // [N, K] gate weights\n const nvfp4_t* __restrict__ W_up, // [N, K] up weights\n const fp8_t* sf_x, const fp8_t* sf_gate, const fp8_t* sf_up,\n half* __restrict__ output, // [M, N] output\n int M, int N, int K\n) {\n // Two TMEM regions: one accumulator per GEMM\n uint32_t tmem_gate = tmem_alloc(256);\n uint32_t tmem_up = tmem_alloc(256);\n\n for (int k = 0; k < K; k += BLOCK_K) {\n int stage = (k / BLOCK_K) % NUM_STAGES;\n mbarrier_wait(&tma_done[stage]);\n\n // Two MMAs per K-tile: gate and up\n // Both read same X tile, different weight tiles\n tcgen05_mma(x_smem[stage], wg_smem[stage], tmem_gate);\n tcgen05_mma(x_smem[stage], wu_smem[stage], tmem_up);\n }\n\n // Fused epilogue: SiLU(gate) * up\n float g = tmem_load(tmem_gate);\n float u = tmem_load(tmem_up);\n float s = g / (1.0f + expf(-g)); // SiLU\n output[row * N + col] = __float2half(s * u);\n\n tmem_dealloc(tmem_gate, 256);\n tmem_dealloc(tmem_up, 256);\n}\n```\n\n2. **TMEM dual accumulator**: Blackwell's 512-column TMEM fits two 256-col accumulators side-by-side"}, "after": {"statement": "The official NVIDIA rules identify NVFP4 Gated Dual GEMM as Kernel Challenge 3 of the Blackwell NVFP4 Hackathon, open from December 20, 2025 through January 16, 2026. The public GPU Mode problem at challenge-opening commit `c5b2f7c` targets NVIDIA B200.\n\nThe shared `a` and `sfa` inputs create an opportunity to reuse data across the two products, and a genuinely fused epilogue can avoid writing both full-precision products to global memory. Those are optimization possibilities, not task guarantees.\n\nThe public task/reference does not establish a final leaderboard, winning source, single-launch decomposition, physical load count, TMEM accumulator partition, TMA pipeline, CUTLASS schedule, or compute-/memory-bound classification. If an implementation uses `tcgen05`, TMEM, or TMA, its exact descriptors, scale layouts, synchronization, allocation, and architecture requirements must be verified from that implementation and the version-matched ISA.\n\nCompatibility is correspondingly narrower than “any dual-output operation”: a model/backend must match the two same-shaped NVFP4 products, scale storage and layouts, first-branch SiLU, FP16 output, batching, and divisibility rules. Grouped per-expert MoE execution is a different contract unless an implementation explicitly supplies that grouping layer."}, "reason": {"statement": "Scope hardware metadata to B200/SM100a and avoid unmeasured Hopper claims.", "urls": ["https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://github.com/gpu-mode/reference-kernels/blob/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm/task.yml"]}} +{"path": "wiki/kernels/gated-dual-gemm.md", "before": {"statement": "sources:\n- contest-gpumode-p3\n- blog-deepgemm\n- blog-tflops-gap-fp4-moe\n- pr-vllm-23696"}, "after": {"statement": "sources:\n- contest-gpumode-p3"}, "reason": {"statement": "Keep the direct source graph claim-scoped and disclose any preserved adjacent artifact separately.", "urls": []}} +{"path": "wiki/kernels/grouped-gemm.md", "before": {"statement": "Grouped GEMM computes multiple matrix multiplications with variable M dimensions but shared N and K, directly targeting MoE (Mixture of Experts) inference where each expert processes a different number of tokens. This is the most practically important kernel pattern for MoE serving: during inference, the router sends different token counts to each expert, and grouped GEMM batches all expert computations into a single kernel launch."}, "after": {"statement": "The official NVIDIA rules identify NVFP4 Grouped GEMM as Kernel Challenge 4 of the Blackwell NVFP4 Hackathon, open from January 17 through February 13, 2026. It carries 40% of the four-problem grand-prize score. The corrected public task at commit `ae67948` targets NVIDIA B200.\n\n“Grouped GEMM” does not imply one universal shape contract. Two relevant interfaces differ materially:\n\n- The GPU Mode challenge accepts a list of independent problems; `M_i`, `N_i`, and `K_i` can all differ by group.\n- DeepGEMM's M-grouped MoE APIs vary M while holding N and K fixed. It separately provides a K-grouped interface for MoE weight backward.\n\nNeither interface definition proves that a conforming implementation uses exactly one GPU launch.\n\nThe following shapes describe the pinned FP8/FP4 grouped APIs; scale tensors and layout conversions are additional required inputs.\n\n| M-grouped contiguous | A `[M,K]`, B `[G,N,K]`, D `[M,N]` | `grouped_layout` is either one expert ID per packed row (with `-1` padding) or one prefix-sum end per group | Training forward or inference prefill; N/K fixed and expert segments M-block aligned |\n| M-grouped masked | A `[G,Mmax,K]`, B `[G,N,K]`, D `[G,Mmax,N]` | `masked_m` is one int32 valid-row count per group | CUDA-graph decode; fixed allocation while computing valid portions |\n| K-grouped contiguous | packed A `[sum(K_i),M]`, B `[sum(K_i),N]`, D `[G,M,N]` | host and device K-size lists; optional C has `[G,M,N]` | MoE weight backward; M/N fixed |\n\nThese are concrete library contracts, not generic C++ structs. Compatibility also depends on the documented architecture, dtype/scale format, operand-major mode, alignment, output dtype, and recipe constraints."}, "reason": {"statement": "Distinguish general independent-shape grouping from DeepGEMM's narrower M-grouped MoE API and avoid launch/perceived-importance claims.", "urls": ["https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/task.yml", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/tests/generators.py"]}} +{"path": "wiki/kernels/grouped-gemm.md", "before": {"statement": "```\nStandard GEMM: C = A @ B (single problem)\nGrouped GEMM: C_i = A_i @ B_i for i in [0, num_groups)\n\nMoE specialization:\n - N and K are FIXED (same expert architecture)\n - Only M varies (different token counts per expert)\n - B_i may be different weight matrices (per-expert weights)\n\nExample (DeepSeek-V3, 256 experts, top-8 routing):\n Group 0: M=47 tokens -> Expert 0 weights [N, K]\n Group 1: M=23 tokens -> Expert 1 weights [N, K]\n ...\n Group 255: M=31 tokens -> Expert 255 weights [N, K]\n```"}, "after": {"statement": "For each group `i`, the correctness reference computes `C_i = A_i @ B_i.T` with block scales and FP16 output.\n\n| `a_i` | packed NVFP4 E2M1, two values per byte | `[M_i,K_i/2,L_i]` |\n| `b_i` | packed NVFP4 E2M1, two values per byte | `[N_i,K_i/2,L_i]` |\n| `c_i` | FP16 | `[M_i,N_i,L_i]` |\n| `sfa_i` | FP8 E4M3FNUZ in task/template | `[M_i,K_i/16,L_i]` |\n| `sfb_i` | FP8 E4M3FNUZ in task/template | `[N_i,K_i/16,L_i]` |\n| problem size | integers | `(M_i,N_i,K_i,L_i)` |\n\nThe submission object actually contains four lists: logical A/B/C tensors, logical scales, reordered scale copies, and problem sizes. The `task.yml` prose lists only three tuple members, but `task.py`, `template.py`, and `reference.py` expose the reordered scales as the fourth. Each published case has `L=1`; `K_i` is divisible by 256, and `M_i`/`N_i` must satisfy the selected MMA tile divisibility.\n\nThe pinned upstream files also disagree on the scale dtype suffix: task/template text says E4M3FNUZ, while `reference.py` constructs `torch.float8_e4m3fn`. A submission must follow the actual tensors it receives rather than silently treating those names as interchangeable. The correctness checker uses `rtol=1e-3` and `atol=1e-3`.\n\nThe following shapes describe the pinned FP8/FP4 grouped APIs; scale tensors and layout conversions are additional required inputs.\n\n| M-grouped contiguous | A `[M,K]`, B `[G,N,K]`, D `[M,N]` | `grouped_layout` is either one expert ID per packed row (with `-1` padding) or one prefix-sum end per group | Training forward or inference prefill; N/K fixed and expert segments M-block aligned |\n| M-grouped masked | A `[G,Mmax,K]`, B `[G,N,K]`, D `[G,Mmax,N]` | `masked_m` is one int32 valid-row count per group | CUDA-graph decode; fixed allocation while computing valid portions |\n| K-grouped contiguous | packed A `[sum(K_i),M]`, B `[sum(K_i),N]`, D `[G,M,N]` | host and device K-size lists; optional C has `[G,M,N]` | MoE weight backward; M/N fixed |\n\nThese are concrete library contracts, not generic C++ structs. Compatibility also depends on the documented architecture, dtype/scale format, operand-major mode, alignment, output dtype, and recipe constraints."}, "reason": {"statement": "Replace the fabricated example with the exact tensor/list contract and separately describe DeepGEMM M-grouping.", "urls": ["https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/task.yml", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/template.py", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/reference.py", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/tests/generators.py"]}} +{"path": "wiki/kernels/grouped-gemm.md", "before": {"statement": "```cpp\n// Layout 1: Contiguous (prefill)\n// All expert inputs packed sequentially with cumulative offset array\n// Memory: [Expert0 (M0 rows)] [Expert1 (M1 rows)] [Expert2 (M2 rows)]...\n// Index: offsets[0]=0 offsets[1]=M0 offsets[2]=M0+M1\nstruct ContiguousLayout {\n const fp8_t* A; // Packed input [sum(M_i), K]\n const fp8_t* B; // Expert weights [num_experts, N, K]\n float* C; // Packed output [sum(M_i), N]\n const int* offsets; // Cumulative M offsets [num_experts + 1]\n};\n\n// Layout 2: Masked (decode with CUDA graphs)\n// Fixed allocation per expert, binary mask for valid tokens\n// Compatible with CUDA graph capture (static shapes)\nstruct MaskedLayout {\n const fp8_t* A; // [num_experts, M_max, K]\n const fp8_t* B; // [num_experts, N, K]\n float* C; // [num_experts, M_max, N]\n const bool* mask; // [num_experts, M_max] validity flags\n};\n\n// Layout 3: K-grouped (weight gradients in training backward)\n// Groups along K-axis instead of M-axis\nstruct KGroupedLayout {\n const fp8_t* A; // [M, sum(K_i)]\n const fp8_t* B; // Per-group B matrices with different K\n float* C; // [M, N]\n const int* k_offsets; // Cumulative K offsets\n};\n```"}, "after": {"statement": "The following shapes describe the pinned FP8/FP4 grouped APIs; scale tensors and layout conversions are additional required inputs.\n\n| M-grouped contiguous | A `[M,K]`, B `[G,N,K]`, D `[M,N]` | `grouped_layout` is either one expert ID per packed row (with `-1` padding) or one prefix-sum end per group | Training forward or inference prefill; N/K fixed and expert segments M-block aligned |\n| M-grouped masked | A `[G,Mmax,K]`, B `[G,N,K]`, D `[G,Mmax,N]` | `masked_m` is one int32 valid-row count per group | CUDA-graph decode; fixed allocation while computing valid portions |\n| K-grouped contiguous | packed A `[sum(K_i),M]`, B `[sum(K_i),N]`, D `[G,M,N]` | host and device K-size lists; optional C has `[G,M,N]` | MoE weight backward; M/N fixed |\n\nThese are concrete library contracts, not generic C++ structs. Compatibility also depends on the documented architecture, dtype/scale format, operand-major mode, alignment, output dtype, and recipe constraints."}, "reason": {"statement": "Use a table matching the actual pinned APIs instead of invented structs.", "urls": ["https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/tests/generators.py"]}} +{"path": "wiki/kernels/grouped-gemm.md", "before": {"statement": "```cpp\n// CUTLASS schedule for grouped GEMM on Blackwell\nusing Schedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100;\n\n// PtrArray mode: array of pointers to per-group A, B, C matrices\n// TMA handles variable-offset loads via per-group descriptors\n// CLC distributes tiles across groups dynamically\n\nusing GemmKernel = cutlass::gemm::kernel::GemmGrouped<\n cutlass::gemm::GemmShape<128, 256, 128>, // Tile shape\n cutlass::arch::Sm100,\n cutlass::float_e2m1_t, // NVFP4 operand type\n cutlass::float_e2m1_t,\n float,\n cutlass::layout::RowMajor,\n cutlass::layout::ColumnMajor\n>;\n\n// Launch: single kernel handles all groups\nGemmKernel::Arguments args{\n num_groups,\n problem_sizes, // [num_groups] array of {M_i, N, K}\n ptr_A, ptr_B, ptr_C,\n scale_factors_A, scale_factors_B\n};\nGemmKernel kernel;\nkernel.run(args, stream);\n```"}, "after": {"statement": "The public challenge constrains outputs, tolerances, workloads, and scoring. It does not require CUTLASS, CLC, TMA, TMEM, a persistent kernel, a static schedule, or one launch. The former CUTLASS and CUDA sketches were removed because they were not executable instances of the named APIs.\n\nCUTLASS documents that small M or N can leave threads outside the useful problem bounds, and that a small M/N grid with large K can launch too few threadblocks to use every multiprocessor. This is a possible shape effect, not proof that every grouped workload has the same bottleneck. GPU Mode's postmortem, for example, measured substantial fixed setup cost in its studied implementation.\n\nCLC itself uses an asynchronous cancellation request, shared response, mbarrier completion, and response decoding after a worker's initial block. That is different from a global `atomicAdd` tile queue. Whether CLC improves end-to-end time relative to a static or software-persistent scheduler requires a workload- and implementation-specific measurement.\n\nTMA does not have one universal 128-byte alignment rule from which minimum tile sizes follow. CUDA 13.2.1's tiled tensor-map API generally documents 16-byte global-address and stride alignment, a 64-byte descriptor, and feature-specific 32-byte constraints. An implementation must check the exact descriptor mode and version it uses."}, "reason": {"statement": "Remove noncompiling pseudocode rather than substitute another untested optimized kernel.", "urls": ["https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/task.yml", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/gemm/kernel/gemm_grouped.h", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/media/docs/cpp/efficient_gemm.md", "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"]}} +{"path": "wiki/kernels/grouped-gemm.md", "before": {"statement": "```cpp\n// Precompute tile-to-expert mapping on host before launch\nstruct TileInfo {\n int expert_id;\n int tile_m_start; // Local M offset within this expert\n int tile_n_start;\n};\n\nstd::vector build_tile_schedule(\n const int* M_per_expert, int num_experts, int N\n) {\n std::vector schedule;\n for (int e = 0; e < num_experts; e++) {\n for (int m = 0; m < M_per_expert[e]; m += BLOCK_M)\n for (int n = 0; n < N; n += BLOCK_N)\n schedule.push_back({e, m, n});\n }\n return schedule; // Copy to device; blockIdx.x indexes into this\n}\n```"}, "after": {"statement": "The official NVIDIA rules identify NVFP4 Grouped GEMM as Kernel Challenge 4 of the Blackwell NVFP4 Hackathon, open from January 17 through February 13, 2026. It carries 40% of the four-problem grand-prize score. The corrected public task at commit `ae67948` targets NVIDIA B200.\n\n“Grouped GEMM” does not imply one universal shape contract. Two relevant interfaces differ materially:\n\n- The GPU Mode challenge accepts a list of independent problems; `M_i`, `N_i`, and `K_i` can all differ by group.\n- DeepGEMM's M-grouped MoE APIs vary M while holding N and K fixed. It separately provides a K-grouped interface for MoE weight backward.\n\nNeither interface definition proves that a conforming implementation uses exactly one GPU launch.\n\nThe public challenge constrains outputs, tolerances, workloads, and scoring. It does not require CUTLASS, CLC, TMA, TMEM, a persistent kernel, a static schedule, or one launch. The former CUTLASS and CUDA sketches were removed because they were not executable instances of the named APIs.\n\nCUTLASS documents that small M or N can leave threads outside the useful problem bounds, and that a small M/N grid with large K can launch too few threadblocks to use every multiprocessor. This is a possible shape effect, not proof that every grouped workload has the same bottleneck. GPU Mode's postmortem, for example, measured substantial fixed setup cost in its studied implementation.\n\nCLC itself uses an asynchronous cancellation request, shared response, mbarrier completion, and response decoding after a worker's initial block. That is different from a global `atomicAdd` tile queue. Whether CLC improves end-to-end time relative to a static or software-persistent scheduler requires a workload- and implementation-specific measurement.\n\nTMA does not have one universal 128-byte alignment rule from which minimum tile sizes follow. CUDA 13.2.1's tiled tensor-map API generally documents 16-byte global-address and stride alignment, a 64-byte descriptor, and feature-specific 32-byte constraints. An implementation must check the exact descriptor mode and version it uses."}, "reason": {"statement": "Remove the incomplete derived implementation and retain only source-backed contract boundaries.", "urls": ["https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/task.yml", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/gemm/kernel/gemm_grouped.h", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/media/docs/cpp/efficient_gemm.md", "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"]}} +{"path": "wiki/kernels/grouped-gemm.md", "before": {"statement": "```cpp\n// Persistent kernel with atomic tile counter\n// Each thread block loops, grabbing tiles until all are processed\n__device__ int g_tile_counter = 0;\n\n__global__ void grouped_gemm_persistent(\n const fp8_t** A_ptrs, const fp8_t** B_ptrs, float** C_ptrs,\n const int* M_per_expert, int N, int K,\n const TileInfo* tile_map, int total_tiles\n) {\n while (true) {\n int tile_id = atomicAdd(&g_tile_counter, 1);\n if (tile_id >= total_tiles) return;\n\n TileInfo info = tile_map[tile_id];\n int M_e = M_per_expert[info.expert_id];\n\n // Effective BLOCK_M may be smaller for the last tile of an expert\n int eff_m = min(BLOCK_M, M_e - info.tile_m_start);\n\n // TMA load + tcgen05.mma for this tile\n tma_load(A_ptrs[info.expert_id] + info.tile_m_start * K, ...);\n tma_load(B_ptrs[info.expert_id] + info.tile_n_start, ...);\n tcgen05_mma(...);\n\n // Store result\n store_tile(C_ptrs[info.expert_id] + info.tile_m_start * N\n + info.tile_n_start, eff_m, BLOCK_N);\n }\n}\n```"}, "after": {"statement": "The public challenge constrains outputs, tolerances, workloads, and scoring. It does not require CUTLASS, CLC, TMA, TMEM, a persistent kernel, a static schedule, or one launch. The former CUTLASS and CUDA sketches were removed because they were not executable instances of the named APIs.\n\nCUTLASS documents that small M or N can leave threads outside the useful problem bounds, and that a small M/N grid with large K can launch too few threadblocks to use every multiprocessor. This is a possible shape effect, not proof that every grouped workload has the same bottleneck. GPU Mode's postmortem, for example, measured substantial fixed setup cost in its studied implementation.\n\nCLC itself uses an asynchronous cancellation request, shared response, mbarrier completion, and response decoding after a worker's initial block. That is different from a global `atomicAdd` tile queue. Whether CLC improves end-to-end time relative to a static or software-persistent scheduler requires a workload- and implementation-specific measurement.\n\nTMA does not have one universal 128-byte alignment rule from which minimum tile sizes follow. CUDA 13.2.1's tiled tensor-map API generally documents 16-byte global-address and stride alignment, a 64-byte descriptor, and feature-specific 32-byte constraints. An implementation must check the exact descriptor mode and version it uses."}, "reason": {"statement": "Remove the mislabeled non-kernel rather than imply an untested launch is authoritative.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/task.yml", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/gemm/kernel/gemm_grouped.h", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/media/docs/cpp/efficient_gemm.md", "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"]}} +{"path": "wiki/kernels/grouped-gemm.md", "before": {"statement": "This reported 11.191us (~2us ahead of second place). It led to improvements in the FlashInfer-Bench evaluation methodology for the MLSys 2026 contest."}, "after": {"statement": "GPU Mode's official postmortem records a submission that temporarily reached the number-one leaderboard position with a reported `11.191 µs`, roughly `2 µs` ahead of the next entry, and was scrubbed minutes after the competition.\n\nDuring correctness, it ran a real padded 8-group kernel on each of 15 cloned data objects. During timing, the first call launched one merged 120-group kernel covering all 15 objects; calls 2 through 15 returned cached output pointers. The harness then divided the combined timing by 15. The reported number is evidence about the exploit, not a valid per-call performance record.\n\nThe official post points to `gpu-mode/reference-kernels` PR #104 as the harness response. It does not attribute a FlashInfer-Bench or MLSys 2026 methodology change to this incident."}, "reason": {"statement": "Preserve the verified reported number while replacing the unsupported consequence with the documented PR #104 response.", "urls": ["https://www.gpumode.com/news/reward-hacking-nvfp4"]}} +{"path": "wiki/kernels/grouped-gemm.md", "before": {"statement": "- Any workload with multiple GEMMs sharing N and K but varying M"}, "after": {"statement": "The official NVIDIA rules identify NVFP4 Grouped GEMM as Kernel Challenge 4 of the Blackwell NVFP4 Hackathon, open from January 17 through February 13, 2026. It carries 40% of the four-problem grand-prize score. The corrected public task at commit `ae67948` targets NVIDIA B200.\n\n“Grouped GEMM” does not imply one universal shape contract. Two relevant interfaces differ materially:\n\n- The GPU Mode challenge accepts a list of independent problems; `M_i`, `N_i`, and `K_i` can all differ by group.\n- DeepGEMM's M-grouped MoE APIs vary M while holding N and K fixed. It separately provides a K-grouped interface for MoE weight backward.\n\nNeither interface definition proves that a conforming implementation uses exactly one GPU launch.\n\nThe following shapes describe the pinned FP8/FP4 grouped APIs; scale tensors and layout conversions are additional required inputs.\n\n| M-grouped contiguous | A `[M,K]`, B `[G,N,K]`, D `[M,N]` | `grouped_layout` is either one expert ID per packed row (with `-1` padding) or one prefix-sum end per group | Training forward or inference prefill; N/K fixed and expert segments M-block aligned |\n| M-grouped masked | A `[G,Mmax,K]`, B `[G,N,K]`, D `[G,Mmax,N]` | `masked_m` is one int32 valid-row count per group | CUDA-graph decode; fixed allocation while computing valid portions |\n| K-grouped contiguous | packed A `[sum(K_i),M]`, B `[sum(K_i),N]`, D `[G,M,N]` | host and device K-size lists; optional C has `[G,M,N]` | MoE weight backward; M/N fixed |\n\nThese are concrete library contracts, not generic C++ structs. Compatibility also depends on the documented architecture, dtype/scale format, operand-major mode, alignment, output dtype, and recipe constraints."}, "reason": {"statement": "Replace universal advice with contract-specific selection criteria.", "urls": ["https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/task.yml", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/tests/generators.py"]}} +{"path": "wiki/kernels/grouped-gemm.md", "before": {"statement": "- Expert load imbalance is the primary practical bottleneck (see [tail-effect](../patterns/tail-effect.md))"}, "after": {"statement": "| 8 | 80, 176, 128, 72, 64, 248, 96, 160 | 4096 | 7168 | 1 | 18.833 |\n| 8 | 40, 76, 168, 72, 164, 148, 196, 160 | 7168 | 2048 | 1 | 10.667 |\n| 2 | 192, 320 | 3072 | 4096 | 1 | 2.406 |\n| 2 | 128, 384 | 4096 | 1536 | 1 | 1.525 |\n\nRanking uses the geometric mean. The task labels these microsecond values a speed-of-light analysis derived from the maximum of B200 FP4 Tensor Core math time and DRAM-memory time at a 1.5 GHz clock. They are theoretical comparison values, not measured contestant results.\n\nThe public challenge constrains outputs, tolerances, workloads, and scoring. It does not require CUTLASS, CLC, TMA, TMEM, a persistent kernel, a static schedule, or one launch. The former CUTLASS and CUDA sketches were removed because they were not executable instances of the named APIs.\n\nCUTLASS documents that small M or N can leave threads outside the useful problem bounds, and that a small M/N grid with large K can launch too few threadblocks to use every multiprocessor. This is a possible shape effect, not proof that every grouped workload has the same bottleneck. GPU Mode's postmortem, for example, measured substantial fixed setup cost in its studied implementation.\n\nCLC itself uses an asynchronous cancellation request, shared response, mbarrier completion, and response decoding after a worker's initial block. That is different from a global `atomicAdd` tile queue. Whether CLC improves end-to-end time relative to a static or software-persistent scheduler requires a workload- and implementation-specific measurement.\n\nTMA does not have one universal 128-byte alignment rule from which minimum tile sizes follow. CUDA 13.2.1's tiled tensor-map API generally documents 16-byte global-address and stride alignment, a 64-byte descriptor, and feature-specific 32-byte constraints. An implementation must check the exact descriptor mode and version it uses."}, "reason": {"statement": "State measured contest-specific setup/underfill observations and avoid a universal bottleneck hierarchy.", "urls": ["https://www.gpumode.com/news/reward-hacking-nvfp4", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/task.yml", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/gemm/kernel/gemm_grouped.h", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/media/docs/cpp/efficient_gemm.md", "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"]}} +{"path": "wiki/kernels/grouped-gemm.md", "before": {"statement": "- Masked layout wastes compute on padding when M distribution is skewed"}, "after": {"statement": "The following shapes describe the pinned FP8/FP4 grouped APIs; scale tensors and layout conversions are additional required inputs.\n\n| M-grouped contiguous | A `[M,K]`, B `[G,N,K]`, D `[M,N]` | `grouped_layout` is either one expert ID per packed row (with `-1` padding) or one prefix-sum end per group | Training forward or inference prefill; N/K fixed and expert segments M-block aligned |\n| M-grouped masked | A `[G,Mmax,K]`, B `[G,N,K]`, D `[G,Mmax,N]` | `masked_m` is one int32 valid-row count per group | CUDA-graph decode; fixed allocation while computing valid portions |\n| K-grouped contiguous | packed A `[sum(K_i),M]`, B `[sum(K_i),N]`, D `[G,M,N]` | host and device K-size lists; optional C has `[G,M,N]` | MoE weight backward; M/N fixed |\n\nThese are concrete library contracts, not generic C++ structs. Compatibility also depends on the documented architecture, dtype/scale format, operand-major mode, alignment, output dtype, and recipe constraints."}, "reason": {"statement": "Separate fixed allocation from valid computation and describe masked_m accurately.", "urls": ["https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/tests/generators.py"]}} +{"path": "wiki/kernels/grouped-gemm.md", "before": {"statement": "- TMA alignment (128 bytes) constrains minimum tile dimensions"}, "after": {"statement": "The public challenge constrains outputs, tolerances, workloads, and scoring. It does not require CUTLASS, CLC, TMA, TMEM, a persistent kernel, a static schedule, or one launch. The former CUTLASS and CUDA sketches were removed because they were not executable instances of the named APIs.\n\nCUTLASS documents that small M or N can leave threads outside the useful problem bounds, and that a small M/N grid with large K can launch too few threadblocks to use every multiprocessor. This is a possible shape effect, not proof that every grouped workload has the same bottleneck. GPU Mode's postmortem, for example, measured substantial fixed setup cost in its studied implementation.\n\nCLC itself uses an asynchronous cancellation request, shared response, mbarrier completion, and response decoding after a worker's initial block. That is different from a global `atomicAdd` tile queue. Whether CLC improves end-to-end time relative to a static or software-persistent scheduler requires a workload- and implementation-specific measurement.\n\nTMA does not have one universal 128-byte alignment rule from which minimum tile sizes follow. CUDA 13.2.1's tiled tensor-map API generally documents 16-byte global-address and stride alignment, a 64-byte descriptor, and feature-specific 32-byte constraints. An implementation must check the exact descriptor mode and version it uses."}, "reason": {"statement": "Replace the blanket rule with versioned API constraints and direct readers to their exact descriptor/feature.", "urls": ["https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/task.yml", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/gemm/kernel/gemm_grouped.h", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/media/docs/cpp/efficient_gemm.md", "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"]}} +{"path": "wiki/kernels/grouped-gemm.md", "before": {"statement": "performance_claims:\n - gpu: B200\n dtype: nvfp4\n shape: \"variable M, shared N=K, 15 groups\"\n metric: latency_us\n value: 11.2\n utilization: \"compute-bound\"\n source_id: contest-gpumode-p4"}, "after": {"statement": "performance_claims: []\n\n| 8 | 80, 176, 128, 72, 64, 248, 96, 160 | 4096 | 7168 | 1 | 18.833 |\n| 8 | 40, 76, 168, 72, 164, 148, 196, 160 | 7168 | 2048 | 1 | 10.667 |\n| 2 | 192, 320 | 3072 | 4096 | 1 | 2.406 |\n| 2 | 128, 384 | 4096 | 1536 | 1 | 1.525 |\n\nRanking uses the geometric mean. The task labels these microsecond values a speed-of-light analysis derived from the maximum of B200 FP4 Tensor Core math time and DRAM-memory time at a 1.5 GHz clock. They are theoretical comparison values, not measured contestant results.\n\nGPU Mode's official postmortem records a submission that temporarily reached the number-one leaderboard position with a reported `11.191 µs`, roughly `2 µs` ahead of the next entry, and was scrubbed minutes after the competition.\n\nDuring correctness, it ran a real padded 8-group kernel on each of 15 cloned data objects. During timing, the first call launched one merged 120-group kernel covering all 15 objects; calls 2 through 15 returned cached output pointers. The harness then divided the combined timing by 15. The reported number is evidence about the exploit, not a valid per-call performance record.\n\nThe official post points to `gpu-mode/reference-kernels` PR #104 as the harness response. It does not attribute a FlashInfer-Bench or MLSys 2026 methodology change to this incident."}, "reason": {"statement": "Remove invalid structured performance metadata and retain only labeled theoretical task estimates in prose.", "urls": ["https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/task.yml", "https://www.gpumode.com/news/reward-hacking-nvfp4"]}} +{"path": "wiki/kernels/grouped-gemm.md", "before": {"statement": "sources: [contest-gpumode-p4, blog-deepgemm, doc-cutlass-blackwell]"}, "after": {"statement": "sources:\n- contest-gpumode-p4\n- blog-deepgemm\n- blog-gpu-mode-reward-hack"}, "reason": {"statement": "Remove doc-cutlass-blackwell from direct frontmatter evidence and add blog-gpu-mode-reward-hack; retain precise CUTLASS links only where used.", "urls": ["https://www.gpumode.com/news/reward-hacking-nvfp4", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/task.yml", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/gemm/kernel/gemm_grouped.h", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/media/docs/cpp/efficient_gemm.md", "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"]}} +{"path": "wiki/kernels/nsa.md", "before": {"statement": "NSA achieves 9x forward speedup and 6x backward speedup at 64K sequences versus FlashAttention-2, and 11.6x decoding speedup at 64K context. It is deployed in DeepSeek-V3.2-Exp combined with FlashMLA sparse kernels."}, "after": {"statement": "DeepSeek-V3.2-Exp later introduced **DeepSeek Sparse Attention (DSA)**. Its pinned first-party inference code uses a learned indexer to select up to 2,048 token positions and masks attention to those positions. Its README points to DeepGEMM for indexer-logit kernels and FlashMLA for sparse-attention kernels. FlashMLA likewise says its sparse kernels power DSA.\n\nThat deployed DSA path is related sparse-attention work, but it is not evidence that the ACL paper's gated compression/selection/window NSA architecture was deployed in V3.2-Exp."}, "reason": {"statement": "Separate the ACL NSA paper from the later DSA deployment and retain FlashMLA only as evidence for that distinction.", "urls": []}} +{"path": "wiki/kernels/nsa.md", "before": {"statement": "```\nInput Query Q\n |\n +---> [Compression Path] Learned MLP creates coarse-grained KV\n | | representations (token compression)\n | v\n | S_compressed = Q @ K_compressed^T\n |\n +---> [Selection Path] Blockwise importance scores select\n | | top-n fine-grained token blocks\n | v\n | S_selected = Q @ K_selected^T (sparse)\n |\n +---> [Sliding Window] Local context window (w=512)\n |\n v\n S_local = Q @ K_local^T (banded)\n\nOutput = Combine(softmax(S_compressed) @ V_compressed,\n softmax(S_selected) @ V_selected,\n softmax(S_local) @ V_local)\n```"}, "after": {"statement": "For token representation \\(h_t\\), NSA computes three attention outputs and combines them with learned, input-dependent gates:\n\n\\[\no_t = g_t^{cmp} o_t^{cmp} + g_t^{slc} o_t^{slc} + g_t^{win} o_t^{win},\n\\qquad\ng_t^c = \\operatorname{sigmoid}(\\operatorname{MLP}_c(h_t)).\n\\]\n\n| Compression | Learned MLPs with intra-block position encoding compress overlapping KV blocks into coarse-grained representations. |\n| Selection | Compression-attention scores are reused and aggregated into fine-grained block-importance scores; top-n blocks are selected and attended at full token resolution. |\n| Sliding window | Recent tokens are attended directly to preserve local context. |\n\nThe main experimental configuration uses compression block length 32 and stride 16, selected block length 64 with 16 selected blocks, and a 512-token sliding window. These are experiment settings, not universal NSA constants.\n\nThis CPU function checks only the learned gated-sum semantics above. It is KernelWiki-derived, uses ordinary numbers, and makes no claim about the paper's GPU layout or performance.\n\n```python\ndef gated_branch_sum(branch_outputs, gates):\n \"\"\"Combine equal-width branch vectors with one [0, 1] gate per branch.\"\"\"\n if len(branch_outputs) != len(gates) or not branch_outputs:\n raise ValueError(\"one gate is required for each non-empty branch set\")\n width = len(branch_outputs[0])\n if any(len(branch) != width for branch in branch_outputs):\n raise ValueError(\"branch widths must match\")\n if any(not 0.0 <= gate <= 1.0 for gate in gates):\n raise ValueError(\"sigmoid gates must lie in [0, 1]\")\n return [\n sum(gate * branch[i] for branch, gate in zip(branch_outputs, gates))\n for i in range(width)\n ]\n\n\nassert gated_branch_sum([[1.0, 2.0], [10.0, 20.0], [-2.0, 4.0]],\n [0.5, 0.25, 1.0]) == [1.0, 10.0]\n```\n\nRemoving the gates is a useful negative control: the raw branch sum for the same vectors is `[9.0, 26.0]`, not `[1.0, 10.0]`.\n\nThe paper does not publish the former page's kernel listing, specify a grid exactly as `(query_block, head, batch)`, or claim that such a grid eliminates dynamic scheduling overhead. The removed listings were non-executable and mathematically incorrect: the selected-attention sketch omitted within-block softmax normalization, and the sliding-window sketch stepped by a block size while loading only one token."}, "reason": {"statement": "Replace the ambiguous diagram with the paper's gated branch equation and exact branch roles.", "urls": ["https://aclanthology.org/2025.acl-long.1126.pdf"]}} +{"path": "wiki/kernels/nsa.md", "before": {"statement": "```python\nimport triton\nimport triton.language as tl\n\n@triton.jit\ndef nsa_sparse_attention_fwd(\n Q_ptr, K_ptr, V_ptr, O_ptr,\n block_indices_ptr, # Selected block indices per query\n num_selected: tl.constexpr,\n BLOCK_SIZE: tl.constexpr,\n HEAD_DIM: tl.constexpr,\n NUM_HEADS: tl.constexpr,\n GROUP_SIZE: tl.constexpr, # GQA group size\n):\n \"\"\"\n Sparse attention forward: only compute attention over selected KV blocks.\n Group-centric loading: shares sparse KV blocks across all query heads\n in a GQA group, minimizing redundant KV transfers.\n \"\"\"\n pid = tl.program_id(0)\n head_id = tl.program_id(1)\n batch_id = tl.program_id(2)\n\n # GQA: determine which KV head group this query head belongs to\n kv_head_id = head_id // GROUP_SIZE\n\n # Load query tile\n q_offset = batch_id * NUM_HEADS * HEAD_DIM + head_id * HEAD_DIM\n q = tl.load(Q_ptr + q_offset + tl.arange(0, HEAD_DIM))\n\n # Accumulator for online softmax\n acc = tl.zeros([HEAD_DIM], dtype=tl.float32)\n lse = float(\"-inf\")\n\n # Iterate over selected blocks (sparse)\n for block_idx in range(num_selected):\n # Load block index for this query\n sel_offset = (batch_id * NUM_HEADS + kv_head_id) * num_selected + block_idx\n kv_block_start = tl.load(block_indices_ptr + sel_offset) * BLOCK_SIZE\n\n # Group-centric: load KV block once, shared across GROUP_SIZE query heads\n k_offsets = kv_block_start + tl.arange(0, BLOCK_SIZE)\n k_block = tl.load(K_ptr + (batch_id * kv_head_id * SEQ_LEN + k_offsets[:, None]) * HEAD_DIM\n + tl.arange(0, HEAD_DIM)[None, :])\n v_block = tl.load(V_ptr + (batch_id * kv_head_id * SEQ_LEN + k_offsets[:, None]) * HEAD_DIM\n + tl.arange(0, HEAD_DIM)[None, :])\n\n # Compute attention scores for this block\n scores = tl.sum(q[None, :] * k_block, axis=1) # [BLOCK_SIZE]\n\n # Online softmax update\n block_max = tl.max(scores)\n new_lse = tl.where(lse > block_max,\n lse + tl.log(1.0 + tl.exp(block_max - lse)),\n block_max + tl.log(1.0 + tl.exp(lse - block_max)))\n\n # Rescale accumulator and add new contribution\n old_scale = tl.exp(lse - new_lse)\n new_scale = tl.exp(scores - new_lse)\n acc = acc * old_scale + tl.sum(new_scale[:, None] * v_block, axis=0)\n lse = new_lse\n\n # Store output\n o_offset = batch_id * NUM_HEADS * HEAD_DIM + head_id * HEAD_DIM\n tl.store(O_ptr + o_offset + tl.arange(0, HEAD_DIM), acc)\n```"}, "after": {"statement": "This CPU function checks only the learned gated-sum semantics above. It is KernelWiki-derived, uses ordinary numbers, and makes no claim about the paper's GPU layout or performance.\n\n```python\ndef gated_branch_sum(branch_outputs, gates):\n \"\"\"Combine equal-width branch vectors with one [0, 1] gate per branch.\"\"\"\n if len(branch_outputs) != len(gates) or not branch_outputs:\n raise ValueError(\"one gate is required for each non-empty branch set\")\n width = len(branch_outputs[0])\n if any(len(branch) != width for branch in branch_outputs):\n raise ValueError(\"branch widths must match\")\n if any(not 0.0 <= gate <= 1.0 for gate in gates):\n raise ValueError(\"sigmoid gates must lie in [0, 1]\")\n return [\n sum(gate * branch[i] for branch, gate in zip(branch_outputs, gates))\n for i in range(width)\n ]\n\n\nassert gated_branch_sum([[1.0, 2.0], [10.0, 20.0], [-2.0, 4.0]],\n [0.5, 0.25, 1.0]) == [1.0, 10.0]\n```\n\nRemoving the gates is a useful negative control: the raw branch sum for the same vectors is `[9.0, 26.0]`, not `[1.0, 10.0]`.\n\nThe authors say compression and sliding-window attention can use existing FlashAttention-2-style kernels, while selected attention needs a specialized Triton kernel for training and prefill. Its source-described structure is:\n\n1. Load all query heads in one GQA/MQA group into SRAM so they share selected sparse KV indices.\n2. Load selected KV data as contiguous blocks rather than scattered individual tokens.\n3. Put the nearly constant query and output loops in Triton grid parallelism; keep the selected-block loop inside a program because its count is approximately constant.\n\nThe paper does not publish the former page's kernel listing, specify a grid exactly as `(query_block, head, batch)`, or claim that such a grid eliminates dynamic scheduling overhead. The removed listings were non-executable and mathematically incorrect: the selected-attention sketch omitted within-block softmax normalization, and the sliding-window sketch stepped by a block size while loading only one token."}, "reason": {"statement": "Remove a mathematically incorrect and non-executable kernel sketch rather than imply it is paper code.", "urls": ["https://aclanthology.org/2025.acl-long.1126.pdf"]}} +{"path": "wiki/kernels/nsa.md", "before": {"statement": "```python\n@triton.jit\ndef nsa_sliding_window_fwd(\n Q_ptr, K_ptr, V_ptr, O_ptr,\n seq_pos,\n WINDOW_SIZE: tl.constexpr, # 512\n HEAD_DIM: tl.constexpr,\n):\n \"\"\"Local sliding window attention for recent context.\"\"\"\n pid = tl.program_id(0) # query position\n\n # Window bounds\n window_start = tl.maximum(0, seq_pos - WINDOW_SIZE)\n window_end = seq_pos\n\n # Standard dense attention within window\n # This is O(w*d) per query, where w=512\n q = tl.load(Q_ptr + pid * HEAD_DIM + tl.arange(0, HEAD_DIM))\n acc = tl.zeros([HEAD_DIM], dtype=tl.float32)\n lse = float(\"-inf\")\n\n for pos in range(window_start, window_end, BLOCK_SIZE):\n k = tl.load(K_ptr + pos * HEAD_DIM + tl.arange(0, HEAD_DIM))\n v = tl.load(V_ptr + pos * HEAD_DIM + tl.arange(0, HEAD_DIM))\n score = tl.sum(q * k)\n # Online softmax accumulation\n new_lse = tl.where(lse > score,\n lse + tl.log(1 + tl.exp(score - lse)),\n score + tl.log(1 + tl.exp(lse - score)))\n acc = acc * tl.exp(lse - new_lse) + v * tl.exp(score - new_lse)\n lse = new_lse\n\n tl.store(O_ptr + pid * HEAD_DIM + tl.arange(0, HEAD_DIM), acc)\n```"}, "after": {"statement": "For token representation \\(h_t\\), NSA computes three attention outputs and combines them with learned, input-dependent gates:\n\n\\[\no_t = g_t^{cmp} o_t^{cmp} + g_t^{slc} o_t^{slc} + g_t^{win} o_t^{win},\n\\qquad\ng_t^c = \\operatorname{sigmoid}(\\operatorname{MLP}_c(h_t)).\n\\]\n\n| Compression | Learned MLPs with intra-block position encoding compress overlapping KV blocks into coarse-grained representations. |\n| Selection | Compression-attention scores are reused and aggregated into fine-grained block-importance scores; top-n blocks are selected and attended at full token resolution. |\n| Sliding window | Recent tokens are attended directly to preserve local context. |\n\nThe main experimental configuration uses compression block length 32 and stride 16, selected block length 64 with 16 selected blocks, and a 512-token sliding window. These are experiment settings, not universal NSA constants.\n\nThis CPU function checks only the learned gated-sum semantics above. It is KernelWiki-derived, uses ordinary numbers, and makes no claim about the paper's GPU layout or performance.\n\n```python\ndef gated_branch_sum(branch_outputs, gates):\n \"\"\"Combine equal-width branch vectors with one [0, 1] gate per branch.\"\"\"\n if len(branch_outputs) != len(gates) or not branch_outputs:\n raise ValueError(\"one gate is required for each non-empty branch set\")\n width = len(branch_outputs[0])\n if any(len(branch) != width for branch in branch_outputs):\n raise ValueError(\"branch widths must match\")\n if any(not 0.0 <= gate <= 1.0 for gate in gates):\n raise ValueError(\"sigmoid gates must lie in [0, 1]\")\n return [\n sum(gate * branch[i] for branch, gate in zip(branch_outputs, gates))\n for i in range(width)\n ]\n\n\nassert gated_branch_sum([[1.0, 2.0], [10.0, 20.0], [-2.0, 4.0]],\n [0.5, 0.25, 1.0]) == [1.0, 10.0]\n```\n\nRemoving the gates is a useful negative control: the raw branch sum for the same vectors is `[9.0, 26.0]`, not `[1.0, 10.0]`.\n\nThe paper does not publish the former page's kernel listing, specify a grid exactly as `(query_block, head, batch)`, or claim that such a grid eliminates dynamic scheduling overhead. The removed listings were non-executable and mathematically incorrect: the selected-attention sketch omitted within-block softmax normalization, and the sliding-window sketch stepped by a block size while loading only one token."}, "reason": {"statement": "Remove the incomplete sketch and describe the branch from the paper instead.", "urls": ["https://aclanthology.org/2025.acl-long.1126.pdf"]}} +{"path": "wiki/kernels/nsa.md", "before": {"statement": "3. **Grid-based scheduling**: Triton grid dimensions map directly to (query_block, head, batch), avoiding dynamic scheduling overhead"}, "after": {"statement": "The authors say compression and sliding-window attention can use existing FlashAttention-2-style kernels, while selected attention needs a specialized Triton kernel for training and prefill. Its source-described structure is:\n\n1. Load all query heads in one GQA/MQA group into SRAM so they share selected sparse KV indices.\n2. Load selected KV data as contiguous blocks rather than scattered individual tokens.\n3. Put the nearly constant query and output loops in Triton grid parallelism; keep the selected-block loop inside a program because its count is approximately constant.\n\nThe paper does not publish the former page's kernel listing, specify a grid exactly as `(query_block, head, batch)`, or claim that such a grid eliminates dynamic scheduling overhead. The removed listings were non-executable and mathematically incorrect: the selected-attention sketch omitted within-block softmax normalization, and the sliding-window sketch stepped by a block size while loading only one token."}, "reason": {"statement": "Retain the exact outer-loop/grid observation without inventing grid axes or an overhead result.", "urls": ["https://aclanthology.org/2025.acl-long.1126.pdf"]}} +{"path": "wiki/kernels/nsa.md", "before": {"statement": "performance_claims:\n - gpu: H100\n dtype: bf16\n shape: \"seqlen=65536\"\n metric: speedup\n value: 9.0\n utilization: \"vs FlashAttention-2 forward\"\n source_id: blog-nsa\n\nAll speedups measured against FlashAttention-2 on H100 with BF16 precision."}, "after": {"statement": "performance_claims: []\n\n| Training/prefill forward | 9.0x versus the authors' Triton FlashAttention-2 baseline; Figure 5 timing result |\n| Training/prefill backward | 6.0x versus the same baseline; Figure 5 timing result |\n| Decoding | 11.6x **expected** speedup from Table 4's memory-access volumes: 65,536 full-attention tokens versus 5,632 NSA-equivalent tokens |\n\nThe setup states eight A100 GPUs, GQA group count 4, 16 query heads per group, key dimension 192, and value dimension 128. The paper does not provide the benchmark dtype, software versions, batch details, raw samples, or variance. Consequently these values are retained only as explicitly labeled author reports, and `performance_claims` remains empty rather than encoding a falsely reproducible tuple."}, "reason": {"statement": "Remove invalid structured performance metadata and retain source-reported values in prose with exact scope and missing reproducibility fields.", "urls": ["https://aclanthology.org/2025.acl-long.1126.pdf"]}} +{"path": "wiki/kernels/nsa.md", "before": {"statement": "- Long-context inference (32K+ tokens) where full attention is prohibitively expensive"}, "after": {"statement": "- NSA requires a model trained for its compression, selection, window, and gate mechanism; do not treat it as a drop-in sparse mask for an arbitrary checkpoint.\n- GQA/MQA lets query heads in a group share selected blocks, matching the paper's group-centric kernel design.\n- The paper evaluates long contexts through 64K but defines no universal 32K threshold. Compare quality, sequence distribution, memory use, and measured latency or throughput on the target workload.\n- CUDA Graphs can reduce repeated-launch CPU overhead, but they are not an NSA-specific default. Profile first: NVIDIA's guidance says the largest gains occur for CPU-bound workflows and that GPU-bound workloads may see little benefit or regress.\n- The compression MLP adds learned parameters and computation. The paper does not isolate a standalone cost for that component.\n- The paper notes that a Triton implementation can retain abstraction overhead relative to native CUDA; it does not publish an NSA-specific CPU-launch profile."}, "reason": {"statement": "Replace the universal cutoff with a workload- and quality-dependent selection boundary.", "urls": ["https://aclanthology.org/2025.acl-long.1126.pdf", "https://docs.nvidia.com/dl-cuda-graph/troubleshooting/performance-issues.html"]}} +{"path": "wiki/kernels/nsa.md", "before": {"statement": "- Token selection adds a two-pass overhead (score all blocks, then select top-n)"}, "after": {"statement": "For token representation \\(h_t\\), NSA computes three attention outputs and combines them with learned, input-dependent gates:\n\n\\[\no_t = g_t^{cmp} o_t^{cmp} + g_t^{slc} o_t^{slc} + g_t^{win} o_t^{win},\n\\qquad\ng_t^c = \\operatorname{sigmoid}(\\operatorname{MLP}_c(h_t)).\n\\]\n\n| Compression | Learned MLPs with intra-block position encoding compress overlapping KV blocks into coarse-grained representations. |\n| Selection | Compression-attention scores are reused and aggregated into fine-grained block-importance scores; top-n blocks are selected and attended at full token resolution. |\n| Sliding window | Recent tokens are attended directly to preserve local context. |\n\nThe main experimental configuration uses compression block length 32 and stride 16, selected block length 64 with 16 selected blocks, and a 512-token sliding window. These are experiment settings, not universal NSA constants."}, "reason": {"statement": "Describe score reuse and top-n selection without inventing an extra pass.", "urls": ["https://aclanthology.org/2025.acl-long.1126.pdf"]}} +{"path": "wiki/kernels/nsa.md", "before": {"statement": "- Triton implementation has CPU launch overhead impacting small-batch decode; CUDA graph mode recommended"}, "after": {"statement": "- NSA requires a model trained for its compression, selection, window, and gate mechanism; do not treat it as a drop-in sparse mask for an arbitrary checkpoint.\n- GQA/MQA lets query heads in a group share selected blocks, matching the paper's group-centric kernel design.\n- The paper evaluates long contexts through 64K but defines no universal 32K threshold. Compare quality, sequence distribution, memory use, and measured latency or throughput on the target workload.\n- CUDA Graphs can reduce repeated-launch CPU overhead, but they are not an NSA-specific default. Profile first: NVIDIA's guidance says the largest gains occur for CPU-bound workflows and that GPU-bound workloads may see little benefit or regress.\n- The compression MLP adds learned parameters and computation. The paper does not isolate a standalone cost for that component.\n- The paper notes that a Triton implementation can retain abstraction overhead relative to native CUDA; it does not publish an NSA-specific CPU-launch profile."}, "reason": {"statement": "Replace the categorical prescription with a profile-first CUDA Graph boundary.", "urls": ["https://aclanthology.org/2025.acl-long.1126.pdf", "https://docs.nvidia.com/dl-cuda-graph/troubleshooting/performance-issues.html"]}} +{"path": "wiki/kernels/nsa.md", "before": {"statement": "The core NSA kernel is implemented in Triton with group-centric data loading and grid-based scheduling.\n\nNSA's sparsity pattern is explicitly designed for GPU memory access efficiency:"}, "after": {"statement": "Native Sparse Attention is the three-branch, natively trainable sparse-attention design published in the ACL 2025 proceedings. The paper targets long-context training and inference and reports quality comparable to or better than full attention on its evaluated tasks. Those are paper-scoped results, not a guarantee for an arbitrary model or post-training conversion.\n\nThe paper's efficiency experiments use an eight-GPU A100 system. The paper does not establish Hopper or Blackwell compatibility for its Triton implementation.\n\nDeepSeek-V3.2-Exp later introduced **DeepSeek Sparse Attention (DSA)**. Its pinned first-party inference code uses a learned indexer to select up to 2,048 token positions and masks attention to those positions. Its README points to DeepGEMM for indexer-logit kernels and FlashMLA for sparse-attention kernels. FlashMLA likewise says its sparse kernels power DSA.\n\nThat deployed DSA path is related sparse-attention work, but it is not evidence that the ACL paper's gated compression/selection/window NSA architecture was deployed in V3.2-Exp."}, "reason": {"statement": "Scope architecture metadata to the paper's A100 evidence and make Blackwell an unmeasured portability question.", "urls": ["https://aclanthology.org/2025.acl-long.1126.pdf", "https://github.com/deepseek-ai/FlashMLA/blob/71c737929f2567bd0a094ae140f8f60f390b1232/README.md", "https://aclanthology.org/2025.acl-long.1126/"]}} +{"path": "wiki/kernels/nsa.md", "before": {"statement": "sources: [blog-nsa, blog-flashmla, blog-vllm-deepseek-v3-sparse]"}, "after": {"statement": "sources: [blog-nsa, blog-flashmla]"}, "reason": {"statement": "Remove the vLLM community source from direct frontmatter evidence and retain FlashMLA only for the explicit non-equivalence boundary.", "urls": ["https://github.com/deepseek-ai/DeepSeek-V3.2-Exp/tree/87e509a2e5a100d221c97df52c6e8be7835f0057", "https://github.com/deepseek-ai/FlashMLA/blob/71c737929f2567bd0a094ae140f8f60f390b1232/README.md"]}} +{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "NVFP4 GEMM is a compute-bound matrix multiplication kernel operating on NVIDIA's native 4-bit floating-point format (E2M1) with block scaling on Blackwell GPUs. Unlike the memory-bound GEMV, GEMM is dominated by tensor core throughput and benefits from Blackwell's native FP4 MMA instructions via tcgen05.mma, TMA bulk loads, TMEM accumulation, and warp specialization."}, "after": {"statement": "The official NVIDIA rules identify NVFP4 GEMM as Kernel Challenge 2 of the Blackwell NVFP4 Hackathon. The contest ran from November 29 through December 19, 2025; Problem 2 targeted NVIDIA B200 and contributed 20% of the four-problem grand-prize score.\n\nThe pinned public task defines observable inputs, correctness, test shapes, benchmark shapes, and ranking. It does not publish a canonical optimized kernel, require CUTLASS, or establish which implementation mechanisms any entrant used. “NVFP4 GEMM” also does not determine a bottleneck by itself: shape, layout, staging, launch geometry, and epilogue can change whether math, memory traffic, occupancy, or launch overhead controls runtime.\n\nUse this task contract only when the packed E2M1 payloads, logical and reordered scales, transpose convention, FP16 output, tolerances, and B200 target match the workload. “Four-bit weights” alone is insufficient: INT4, MXFP4, other scale granularities, and other layouts are not interchangeable with this ABI.\n\nThe documented native tensor-core path is Blackwell-specific; Hopper has no native FP4 tensor-core instruction. This does not exclude software conversion or emulation. Choose TMA, TMEM allocation, CUTLASS schedules, warp specialization, tile sizes, and stage counts only after validating their exact API constraints and profiling the actual shapes."}, "reason": {"statement": "Scope bottlenecks and implementation mechanisms to the exact shape and implementation.", "urls": ["https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py", "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://developer.nvidia.com/blog/introducing-nvfp4-for-efficient-and-accurate-low-precision-inference/", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "This kernel was Problem 2 of the GPU Mode NVFP4 Hackathon (Nov-Dec 2025), targeting B200 GPUs. Top entries achieved within 1% of cuBLAS performance using CUTLASS SM100 schedules."}, "after": {"statement": "The task ranks the geometric mean across three benchmark cases. It labels the following values a speed-of-light analysis based on the maximum of B200 FP4 Tensor Core math time and DRAM-memory time at a 1.5 GHz clock:\n\n| 128 | 7168 | 16384 | 1 | 8.994 |\n| 128 | 4096 | 7168 | 1 | 2.354 |\n| 128 | 7168 | 2048 | 1 | 1.333 |\n\nThese are theoretical comparison rows, not measured contestant latencies and not cuBLAS results.\n\nThe public Popcorn endpoint currently gives the following dated snapshot. To make its small floating-point `submission_score` values readable beside the task's microsecond presentation, the table shows `submission_score × 10^6`; it does not relabel the rows as prize placements.\n\n| 1 | `gau.nernst` | 9.981889 | 2025-12-21 00:43:03 |\n| 2 | `s.am._` | 10.060110 | 2025-12-20 17:45:21 |\n| 3 | `billcarson` | 10.137411 | 2025-12-21 03:05:32 |\n| 8 | `Simon` | 10.806750 | 2025-12-16 20:18:42 |\n| 9 | `yue` | 10.914084 | 2025-12-11 04:36:45 |\n| 10 | `currybab` | 10.930623 | 2025-12-19 08:10:18 |\n\nSnapshot fetched August 8, 2026. Because the current first three submissions postdate the official December 19 cutoff, this endpoint alone cannot establish winners or final prize rankings. It also publishes no contestant source, CUTLASS attribution, cuBLAS comparison, raw trials, or variance."}, "reason": {"statement": "Remove an untraceable performance and implementation attribution; retain only dated public fields and theoretical task rows.", "urls": ["https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml", "https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf"]}} +{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "The kernel uses the `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100` CUTLASS schedule, which combines TMA async loads with warp-specialized MMA execution."}, "after": {"statement": "The native Blackwell instruction path supports block-of-16 NVFP4 through `tcgen05.mma...kind::mxf4nvf4.block_scale.block16`. It uses UE4M3 scale elements, which CUTLASS names `float_ue4m3_t`; UE8M0 is the MXFP4 scale type. Converting NVFP4 scales to UE8M0 would discard fractional scale values and is not a required NVFP4 preprocessing step.\n\nCUTLASS defines `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100`, but its direct official example is a grouped pointer-array kernel. The symbol's existence is not evidence that the contest entrants used it. CUTLASS's current official NVFP4 examples use `nv_float4_t`, collective mainloop and epilogue builders, `kernel::GemmUniversal`, and `device::GemmUniversalAdapter`; an old scalar-template `device::GemmUniversal` sketch is not interchangeable with that API.\n\nTMA likewise has no universal 128-byte alignment rule for every operand. CUDA 13.2.1 generally requires a 16-byte-aligned global base and 16-byte-multiple strides for a tiled tensor map, with additional datatype-, interleave-, swizzle-, and mode-specific restrictions. Problem 2's `K % 256 == 0` rule belongs to the task ABI; it does not follow as a universal TMA theorem.\n\nTMEM's 128-lane by 512-column organization is a storage and addressing model, not a universal 128-by-512 logical output-tile limit. Official CUTLASS NVFP4 configurations include a cooperative two-SM MMA tile with logical shape 256 by 256 by 256."}, "reason": {"statement": "Retain the schedule only as an adjacent official CUTLASS example and explicitly deny contestant attribution.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu", "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} +{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "```cpp\n// CUTLASS dispatch for NVFP4 GEMM on Blackwell\nusing Schedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100;\n\n// Tile configuration\nusing TileShape = cute::Shape<_128, _256, _128>; // M, N, K tile\nusing ClusterShape = cute::Shape<_1, _1, _1>; // 1-SM mode\n\n// Element types\nusing ElementA = cutlass::float_e2m1_t; // NVFP4\nusing ElementB = cutlass::float_e2m1_t; // NVFP4\nusing ElementC = float; // FP32 accumulator\nusing ElementScale = cutlass::float_e4m3_t; // FP8 E4M3 block scales\n\n// Kernel definition\nusing Kernel = cutlass::gemm::device::GemmUniversal<\n ElementA, cutlass::layout::RowMajor,\n ElementB, cutlass::layout::ColumnMajor,\n ElementC, cutlass::layout::RowMajor,\n float, // accumulator type\n cutlass::arch::OpClassTensorOp,\n cutlass::arch::Sm100,\n TileShape, ClusterShape, Schedule\n>;\n```"}, "after": {"statement": "The native Blackwell instruction path supports block-of-16 NVFP4 through `tcgen05.mma...kind::mxf4nvf4.block_scale.block16`. It uses UE4M3 scale elements, which CUTLASS names `float_ue4m3_t`; UE8M0 is the MXFP4 scale type. Converting NVFP4 scales to UE8M0 would discard fractional scale values and is not a required NVFP4 preprocessing step.\n\nCUTLASS defines `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100`, but its direct official example is a grouped pointer-array kernel. The symbol's existence is not evidence that the contest entrants used it. CUTLASS's current official NVFP4 examples use `nv_float4_t`, collective mainloop and epilogue builders, `kernel::GemmUniversal`, and `device::GemmUniversalAdapter`; an old scalar-template `device::GemmUniversal` sketch is not interchangeable with that API.\n\nTMA likewise has no universal 128-byte alignment rule for every operand. CUDA 13.2.1 generally requires a 16-byte-aligned global base and 16-byte-multiple strides for a tiled tensor map, with additional datatype-, interleave-, swizzle-, and mode-specific restrictions. Problem 2's `K % 256 == 0` rule belongs to the task ABI; it does not follow as a universal TMA theorem.\n\nTMEM's 128-lane by 512-column organization is a storage and addressing model, not a universal 128-by-512 logical output-tile limit. Official CUTLASS NVFP4 configurations include a cooperative two-SM MMA tile with logical shape 256 by 256 by 256.\n\n- [Official contest rules](https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf)\n- [Pinned task definition](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml)\n- [Pinned task types](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.py)\n- [Pinned starter template](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py)\n- [Pinned correctness reference](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/reference.py)\n- [Public Popcorn leaderboard API](https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12)\n- [Transformer Engine 2.13 NVFP4 recipe](https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html)\n- [cuBLAS block-scaling formats](https://docs.nvidia.com/cuda/cublas/index.html#element-1d-block-scaling-for-fp8-and-fp4-data-types)\n- [PTX ISA 9.0 block-scaled `tcgen05.mma`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma)\n- [Pinned CUTLASS NVFP4 example](https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu)\n- [Pinned CUTLASS grouped NVFP4 example](https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu)\n- [CUDA 13.2.1 tensor-map constraints](https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html)\n\nQuery via:"}, "reason": {"statement": "Remove non-compiling pseudo-CUTLASS and link to the immutable official example instead.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu", "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} +{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "```cpp\n// Warp specialization: TMA producer + MMA consumer + epilogue\n// Shared memory holds pipelined A/B tiles + scale factors\n\nconstexpr int NUM_STAGES = 4; // Pipeline depth\n\n// Shared memory layout\nstruct SharedStorage {\n // Double-buffered across NUM_STAGES\n nvfp4_t A_smem[NUM_STAGES][BLOCK_M * BLOCK_K / 2]; // Packed FP4: 2 per byte\n nvfp4_t B_smem[NUM_STAGES][BLOCK_N * BLOCK_K / 2];\n fp8_t sfa_smem[NUM_STAGES][BLOCK_M * (BLOCK_K / 16)]; // 1 scale per 16 elements\n fp8_t sfb_smem[NUM_STAGES][BLOCK_N * (BLOCK_K / 16)];\n uint64_t mbarrier[NUM_STAGES];\n};\n\n__global__ void nvfp4_gemm_kernel(\n const nvfp4_t* A, const nvfp4_t* B,\n const fp8_t* sfa, const fp8_t* sfb,\n float sf_a_global, float sf_b_global,\n float* C, int M, int N, int K\n) {\n extern __shared__ SharedStorage smem[];\n int warp_id = threadIdx.x / 32;\n int lane_id = threadIdx.x % 32;\n\n if (warp_id == 0 && lane_id == 0) {\n // TMA producer warp: async bulk loads\n for (int k = 0; k < K; k += BLOCK_K) {\n int stage = (k / BLOCK_K) % NUM_STAGES;\n\n // TMA descriptor-based bulk load (128-byte aligned)\n asm volatile(\n \"cp.async.bulk.tensor.2d.shared::cluster.global.tile\"\n \".mbarrier::complete_tx::bytes\"\n \" [%0], [%1, {%2, %3}], [%4];\"\n :: \"r\"((uint32_t)&smem->A_smem[stage]),\n \"l\"(tma_desc_A),\n \"r\"(tile_m), \"r\"(k),\n \"r\"((uint32_t)&smem->mbarrier[stage])\n );\n // Similarly for B, sfa, sfb\n }\n } else if (warp_id == 1 && lane_id == 0) {\n // MMA consumer warp: tcgen05.mma\n uint32_t tmem_addr = tmem_alloc_cta(256); // 256 TMEM columns\n\n for (int k = 0; k < K; k += BLOCK_K) {\n int stage = (k / BLOCK_K) % NUM_STAGES;\n // Wait for TMA to complete this stage\n mbarrier_wait(&smem->mbarrier[stage]);\n\n // tcgen05.mma with native block scaling\n // Reads A/B from SMEM, accumulates into TMEM\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f8f6f4\"\n \" [%0], %1, %2, %3, %4;\"\n :: \"r\"(tmem_addr),\n \"l\"((uint64_t)&smem->A_smem[stage]),\n \"l\"((uint64_t)&smem->B_smem[stage]),\n \"r\"(packed_scales),\n \"n\"(1) // scale_D enabled\n );\n }\n\n // Signal epilogue warps\n } else {\n // Epilogue warps: read from TMEM, apply global scales, store to C\n // tmem -> registers -> global\n }\n}\n```"}, "after": {"statement": "The native Blackwell instruction path supports block-of-16 NVFP4 through `tcgen05.mma...kind::mxf4nvf4.block_scale.block16`. It uses UE4M3 scale elements, which CUTLASS names `float_ue4m3_t`; UE8M0 is the MXFP4 scale type. Converting NVFP4 scales to UE8M0 would discard fractional scale values and is not a required NVFP4 preprocessing step.\n\nCUTLASS defines `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100`, but its direct official example is a grouped pointer-array kernel. The symbol's existence is not evidence that the contest entrants used it. CUTLASS's current official NVFP4 examples use `nv_float4_t`, collective mainloop and epilogue builders, `kernel::GemmUniversal`, and `device::GemmUniversalAdapter`; an old scalar-template `device::GemmUniversal` sketch is not interchangeable with that API.\n\nTMA likewise has no universal 128-byte alignment rule for every operand. CUDA 13.2.1 generally requires a 16-byte-aligned global base and 16-byte-multiple strides for a tiled tensor map, with additional datatype-, interleave-, swizzle-, and mode-specific restrictions. Problem 2's `K % 256 == 0` rule belongs to the task ABI; it does not follow as a universal TMA theorem.\n\nTMEM's 128-lane by 512-column organization is a storage and addressing model, not a universal 128-by-512 logical output-tile limit. Official CUTLASS NVFP4 configurations include a cooperative two-SM MMA tile with logical shape 256 by 256 by 256."}, "reason": {"statement": "Remove misleading non-executable CUDA rather than imply it is a reference implementation.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cute/arch/mma_sm100_umma.hpp#L1647-L1659", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu", "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} +{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "All TMA operands require 128-byte alignment. For NVFP4 (2 elements per byte), this means K dimensions must be multiples of 256 elements:\n\n- TMA requires 128-byte alignment for all operands"}, "after": {"statement": "| `a` | packed E2M1, two values per byte | `[M,K/2,L]` |\n| `b` | packed E2M1, two values per byte | `[N,K/2,L]` |\n| `sfa` | logical E4M3 scales | `[M,K/16,L]` |\n| `sfb` | logical E4M3 scales | `[N,K/16,L]` |\n| `sfa_reordered` | MMA-oriented scale copy | `[32,4,ceil(M/128),4,K/64,L]` |\n| `sfb_reordered` | MMA-oriented scale copy | `[32,4,ceil(N/128),4,K/64,L]` |\n| `c` | preallocated FP16 output | `[M,N,L]` |\n\nFor each `L` slice, the correctness reference computes the block-scaled equivalent of `A @ B.T` and stores FP16 output. It uses `rtol=1e-3` and `atol=1e-3`.\n\nThe upstream files contain two interface inconsistencies that implementations must not conceal:\n\n- `task.yml` describes a five-member `(a,b,sfa,sfb,c)` tuple, while `task.py`, `template.py`, and `reference.py` expose the seven tensors above.\n- Task and template prose label scale tensors E4M3FNUZ, while `reference.py` constructs `torch.float8_e4m3fn` values. A submission must follow the tensors supplied by the actual harness rather than treating those suffixes as interchangeable.\n\nThe published task requires `K` divisible by 256. Divisibility of `M` and `N` depends on the submission's selected MMA tile. The following CPU-only helper reproduces the published storage shapes; it does not decode FP4, apply scales, or model a GPU kernel:\n\nThe native Blackwell instruction path supports block-of-16 NVFP4 through `tcgen05.mma...kind::mxf4nvf4.block_scale.block16`. It uses UE4M3 scale elements, which CUTLASS names `float_ue4m3_t`; UE8M0 is the MXFP4 scale type. Converting NVFP4 scales to UE8M0 would discard fractional scale values and is not a required NVFP4 preprocessing step.\n\nCUTLASS defines `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100`, but its direct official example is a grouped pointer-array kernel. The symbol's existence is not evidence that the contest entrants used it. CUTLASS's current official NVFP4 examples use `nv_float4_t`, collective mainloop and epilogue builders, `kernel::GemmUniversal`, and `device::GemmUniversalAdapter`; an old scalar-template `device::GemmUniversal` sketch is not interchangeable with that API.\n\nTMA likewise has no universal 128-byte alignment rule for every operand. CUDA 13.2.1 generally requires a 16-byte-aligned global base and 16-byte-multiple strides for a tiled tensor map, with additional datatype-, interleave-, swizzle-, and mode-specific restrictions. Problem 2's `K % 256 == 0` rule belongs to the task ABI; it does not follow as a universal TMA theorem.\n\nTMEM's 128-lane by 512-column organization is a storage and addressing model, not a universal 128-by-512 logical output-tile limit. Official CUTLASS NVFP4 configurations include a cooperative two-SM MMA tile with logical shape 256 by 256 by 256."}, "reason": {"statement": "Separate the task's K divisibility rule from actual tensor-map constraints.", "urls": ["https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} +{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "```cpp\n// Critical: pad tensors to 128-byte boundaries for TMA\n// NVFP4: 2 elements per byte, so 256 elements = 128 bytes\nstatic_assert(K % 256 == 0, \"K must align to 128 bytes for FP4 TMA\");\n\n// For scale factors: FP8 is 1 byte per element\n// 128 bytes = 128 scale values\n// Since 1 scale per 16 FP4 elements: 128 scales cover 2048 FP4 elements\nstatic_assert((K / 16) % 128 == 0, \"Scale array must align to 128 bytes\");\n```"}, "after": {"statement": "| `a` | packed E2M1, two values per byte | `[M,K/2,L]` |\n| `b` | packed E2M1, two values per byte | `[N,K/2,L]` |\n| `sfa` | logical E4M3 scales | `[M,K/16,L]` |\n| `sfb` | logical E4M3 scales | `[N,K/16,L]` |\n| `sfa_reordered` | MMA-oriented scale copy | `[32,4,ceil(M/128),4,K/64,L]` |\n| `sfb_reordered` | MMA-oriented scale copy | `[32,4,ceil(N/128),4,K/64,L]` |\n| `c` | preallocated FP16 output | `[M,N,L]` |\n\nFor each `L` slice, the correctness reference computes the block-scaled equivalent of `A @ B.T` and stores FP16 output. It uses `rtol=1e-3` and `atol=1e-3`.\n\nThe upstream files contain two interface inconsistencies that implementations must not conceal:\n\n- `task.yml` describes a five-member `(a,b,sfa,sfb,c)` tuple, while `task.py`, `template.py`, and `reference.py` expose the seven tensors above.\n- Task and template prose label scale tensors E4M3FNUZ, while `reference.py` constructs `torch.float8_e4m3fn` values. A submission must follow the tensors supplied by the actual harness rather than treating those suffixes as interchangeable.\n\nThe published task requires `K` divisible by 256. Divisibility of `M` and `N` depends on the submission's selected MMA tile. The following CPU-only helper reproduces the published storage shapes; it does not decode FP4, apply scales, or model a GPU kernel:\n\n```python\ndef task_storage_shapes(m, n, k, l=1):\n if min(m, n, k, l) <= 0:\n raise ValueError(\"dimensions must be positive\")\n if k % 256:\n raise ValueError(\"K must be divisible by 256\")\n\n return {\n \"a_packed\": (m, k // 2, l),\n \"b_packed\": (n, k // 2, l),\n \"sfa_logical\": (m, k // 16, l),\n \"sfb_logical\": (n, k // 16, l),\n \"sfa_reordered\": (32, 4, (m + 127) // 128, 4, (k + 63) // 64, l),\n \"sfb_reordered\": (32, 4, (n + 127) // 128, 4, (k + 63) // 64, l),\n \"c\": (m, n, l),\n }\n\n\nfor valid_k in (256, 512, 1536, 2048, 2304, 7168, 16384):\n task_storage_shapes(128, 256, valid_k)\n\nassert (256 // 16) % 128 != 0 # the former scale-array assertion was invalid\n```\n\nNine of the ten official correctness shapes fail the former `(K/16) % 128 == 0` assertion, including the smallest valid case with `K=256`."}, "reason": {"statement": "Replace the false assertion with a host-checkable representation of the published tensor shapes.", "urls": []}} +{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "The tcgen05.mma instruction expects UE8M0 (unsigned power-of-two exponent only) scales, but NVFP4 uses FP8 E4M3 (non-power-of-two). Conversion is needed:\n\n```cpp\n// Convert FP8 E4M3 block scales to UE8M0 for tcgen05.mma hardware\n// E4M3: 4 exponent bits, 3 mantissa bits (non-power-of-two)\n// UE8M0: 8 exponent bits, 0 mantissa bits (power-of-two only)\n__device__ uint32_t pack_scales_ue8m0(\n fp8_e4m3_t s0, fp8_e4m3_t s1, fp8_e4m3_t s2, fp8_e4m3_t s3\n) {\n uint8_t u0 = fp8_e4m3_to_ue8m0(s0); // Round to nearest power-of-two\n uint8_t u1 = fp8_e4m3_to_ue8m0(s1);\n uint8_t u2 = fp8_e4m3_to_ue8m0(s2);\n uint8_t u3 = fp8_e4m3_to_ue8m0(s3);\n return (u3 << 24) | (u2 << 16) | (u1 << 8) | u0;\n}\n```\n\n- Scale factor conversion (E4M3 to UE8M0) adds overhead if not precomputed"}, "after": {"statement": "The generic one-dimensional NVFP4 recipe reconstructs each value from a signed E2M1 payload, one E4M3 local scale per 16 consecutive payloads, and a per-tensor FP32 global scale. E2M1 represents zero and signed magnitudes 0.5, 1, 1.5, 2, 3, 4, and 6, with two payloads packed per byte. MXFP4 is different: its local groups contain 32 payloads and use power-of-two UE8M0 scales.\n\nThe contest ABI is narrower and does not expose the generic recipe's FP32 global scales. Its actual generated input has seven tensors:\n\nThe native Blackwell instruction path supports block-of-16 NVFP4 through `tcgen05.mma...kind::mxf4nvf4.block_scale.block16`. It uses UE4M3 scale elements, which CUTLASS names `float_ue4m3_t`; UE8M0 is the MXFP4 scale type. Converting NVFP4 scales to UE8M0 would discard fractional scale values and is not a required NVFP4 preprocessing step.\n\nCUTLASS defines `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100`, but its direct official example is a grouped pointer-array kernel. The symbol's existence is not evidence that the contest entrants used it. CUTLASS's current official NVFP4 examples use `nv_float4_t`, collective mainloop and epilogue builders, `kernel::GemmUniversal`, and `device::GemmUniversalAdapter`; an old scalar-template `device::GemmUniversal` sketch is not interchangeable with that API.\n\nTMA likewise has no universal 128-byte alignment rule for every operand. CUDA 13.2.1 generally requires a 16-byte-aligned global base and 16-byte-multiple strides for a tiled tensor map, with additional datatype-, interleave-, swizzle-, and mode-specific restrictions. Problem 2's `K % 256 == 0` rule belongs to the task ABI; it does not follow as a universal TMA theorem.\n\nTMEM's 128-lane by 512-column organization is a storage and addressing model, not a universal 128-by-512 logical output-tile limit. Official CUTLASS NVFP4 configurations include a cooperative two-SM MMA tile with logical shape 256 by 256 by 256."}, "reason": {"statement": "Remove a format-confusing conversion and document the exact UE4M3 scale path.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512", "https://docs.nvidia.com/cuda/cublas/index.html#element-1d-block-scaling-for-fp8-and-fp4-data-types", "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu", "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} +{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "Problem 2 top performers (geometric mean across benchmark configs):\n\n| 1 | Simon | 10.807 |\n| 2 | yue | 10.914 |\n| 3 | currybab | 10.931 |"}, "after": {"statement": "The public Popcorn endpoint currently gives the following dated snapshot. To make its small floating-point `submission_score` values readable beside the task's microsecond presentation, the table shows `submission_score × 10^6`; it does not relabel the rows as prize placements.\n\n| 1 | `gau.nernst` | 9.981889 | 2025-12-21 00:43:03 |\n| 2 | `s.am._` | 10.060110 | 2025-12-20 17:45:21 |\n| 3 | `billcarson` | 10.137411 | 2025-12-21 03:05:32 |\n| 8 | `Simon` | 10.806750 | 2025-12-16 20:18:42 |\n| 9 | `yue` | 10.914084 | 2025-12-11 04:36:45 |\n| 10 | `currybab` | 10.930623 | 2025-12-19 08:10:18 |\n\nSnapshot fetched August 8, 2026. Because the current first three submissions postdate the official December 19 cutoff, this endpoint alone cannot establish winners or final prize rankings. It also publishes no contestant source, CUTLASS attribution, cuBLAS comparison, raw trials, or variance."}, "reason": {"statement": "Publish only a dated current snapshot and explicitly separate it from official prize placements.", "urls": ["https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12", "https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf"]}} +{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "- Inference with 4-bit quantized weights on Blackwell"}, "after": {"statement": "Use this task contract only when the packed E2M1 payloads, logical and reordered scales, transpose convention, FP16 output, tolerances, and B200 target match the workload. “Four-bit weights” alone is insufficient: INT4, MXFP4, other scale granularities, and other layouts are not interchangeable with this ABI.\n\nThe documented native tensor-core path is Blackwell-specific; Hopper has no native FP4 tensor-core instruction. This does not exclude software conversion or emulation. Choose TMA, TMEM allocation, CUTLASS schedules, warp specialization, tile sizes, and stage counts only after validating their exact API constraints and profiling the actual shapes."}, "reason": {"statement": "Scope use to operands prepared for the exact NVFP4 ABI.", "urls": ["https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py", "https://developer.nvidia.com/blog/introducing-nvfp4-for-efficient-and-accurate-low-precision-inference/", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "- TMEM size (128x512 per SM) limits maximum output tile to 128 rows x 512 cols (32-bit)"}, "after": {"statement": "The native Blackwell instruction path supports block-of-16 NVFP4 through `tcgen05.mma...kind::mxf4nvf4.block_scale.block16`. It uses UE4M3 scale elements, which CUTLASS names `float_ue4m3_t`; UE8M0 is the MXFP4 scale type. Converting NVFP4 scales to UE8M0 would discard fractional scale values and is not a required NVFP4 preprocessing step.\n\nCUTLASS defines `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100`, but its direct official example is a grouped pointer-array kernel. The symbol's existence is not evidence that the contest entrants used it. CUTLASS's current official NVFP4 examples use `nv_float4_t`, collective mainloop and epilogue builders, `kernel::GemmUniversal`, and `device::GemmUniversalAdapter`; an old scalar-template `device::GemmUniversal` sketch is not interchangeable with that API.\n\nTMA likewise has no universal 128-byte alignment rule for every operand. CUDA 13.2.1 generally requires a 16-byte-aligned global base and 16-byte-multiple strides for a tiled tensor map, with additional datatype-, interleave-, swizzle-, and mode-specific restrictions. Problem 2's `K % 256 == 0` rule belongs to the task ABI; it does not follow as a universal TMA theorem.\n\nTMEM's 128-lane by 512-column organization is a storage and addressing model, not a universal 128-by-512 logical output-tile limit. Official CUTLASS NVFP4 configurations include a cooperative two-SM MMA tile with logical shape 256 by 256 by 256.\n\nUse this task contract only when the packed E2M1 payloads, logical and reordered scales, transpose convention, FP16 output, tolerances, and B200 target match the workload. “Four-bit weights” alone is insufficient: INT4, MXFP4, other scale granularities, and other layouts are not interchangeable with this ABI.\n\nThe documented native tensor-core path is Blackwell-specific; Hopper has no native FP4 tensor-core instruction. This does not exclude software conversion or emulation. Choose TMA, TMEM allocation, CUTLASS schedules, warp specialization, tile sizes, and stage counts only after validating their exact API constraints and profiling the actual shapes."}, "reason": {"statement": "Describe TMEM storage separately from implementation tile shapes.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu#L147-L159", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu", "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py", "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://developer.nvidia.com/blog/introducing-nvfp4-for-efficient-and-accurate-low-precision-inference/"]}} +{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "The reference bundle lives in [`artifacts/kernels/nvfp4-gemm/full/`](../../artifacts/kernels/nvfp4-gemm/full/) and combines the upstream PR-2139 diff (`PR-2139-blockwise-groupwise-gemm.patch`, `mode: upstream-patch`, SHA-pinned to `ca4fdbea` on NVIDIA/cutlass) with an extracted CUTLASS-schedule / TMA snippet from the `tflops-gap-fp4-moe` blog (`blackwell-cutlass-schedules-and-tma.cu`, `mode: extracted`). Labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` header) live in [`artifacts/kernels/nvfp4-gemm/variants/`](../../artifacts/kernels/nvfp4-gemm/variants/). Every file's SHA-256 and upstream-pinning metadata is in `PROVENANCE.yaml` inside each bundle."}, "after": {"statement": "- [Official contest rules](https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf)\n- [Pinned task definition](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml)\n- [Pinned task types](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.py)\n- [Pinned starter template](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py)\n- [Pinned correctness reference](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/reference.py)\n- [Public Popcorn leaderboard API](https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12)\n- [Transformer Engine 2.13 NVFP4 recipe](https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html)\n- [cuBLAS block-scaling formats](https://docs.nvidia.com/cuda/cublas/index.html#element-1d-block-scaling-for-fp8-and-fp4-data-types)\n- [PTX ISA 9.0 block-scaled `tcgen05.mma`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma)\n- [Pinned CUTLASS NVFP4 example](https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu)\n- [Pinned CUTLASS grouped NVFP4 example](https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu)\n- [CUDA 13.2.1 tensor-map constraints](https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html)\n\nQuery via:"}, "reason": {"statement": "Detach the artifact_dir and remove the Full Reference Implementation section while preserving audit evidence and files.", "urls": []}} +{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "performance_claims:\n- gpu: B200\n dtype: nvfp4\n shape: standard GEMM configs\n metric: latency_us\n value: 10.807\n utilization: near cuBLAS\n source_id: contest-gpumode-p2"}, "after": {"statement": "performance_claims: []\n\nThe task ranks the geometric mean across three benchmark cases. It labels the following values a speed-of-light analysis based on the maximum of B200 FP4 Tensor Core math time and DRAM-memory time at a 1.5 GHz clock:\n\n| 128 | 7168 | 16384 | 1 | 8.994 |\n| 128 | 4096 | 7168 | 1 | 2.354 |\n| 128 | 7168 | 2048 | 1 | 1.333 |\n\nThese are theoretical comparison rows, not measured contestant latencies and not cuBLAS results.\n\nThe public Popcorn endpoint currently gives the following dated snapshot. To make its small floating-point `submission_score` values readable beside the task's microsecond presentation, the table shows `submission_score × 10^6`; it does not relabel the rows as prize placements.\n\n| 1 | `gau.nernst` | 9.981889 | 2025-12-21 00:43:03 |\n| 2 | `s.am._` | 10.060110 | 2025-12-20 17:45:21 |\n| 3 | `billcarson` | 10.137411 | 2025-12-21 03:05:32 |\n| 8 | `Simon` | 10.806750 | 2025-12-16 20:18:42 |\n| 9 | `yue` | 10.914084 | 2025-12-11 04:36:45 |\n| 10 | `currybab` | 10.930623 | 2025-12-19 08:10:18 |\n\nSnapshot fetched August 8, 2026. Because the current first three submissions postdate the official December 19 cutoff, this endpoint alone cannot establish winners or final prize rankings. It also publishes no contestant source, CUTLASS attribution, cuBLAS comparison, raw trials, or variance."}, "reason": {"statement": "Remove malformed performance metadata and retain exact theoretical rows plus a dated leaderboard snapshot in prose.", "urls": ["https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml", "https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf"]}} +{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "sources:\n- contest-gpumode-p2\n- doc-cutlass-blackwell\n- pr-cutlass-2139\n\nartifact_dir: artifacts/kernels/nvfp4-gemm"}, "after": {"statement": "sources:\n- contest-gpumode-p2\n- doc-transformer-engine-2.13-nvfp4\n- doc-ptx-isa-sm100\n- doc-cuda-13-0-2-tma"}, "reason": {"statement": "Use claim-directed task, format, PTX, and TMA source cards and detach the invalid bundle.", "urls": ["https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml", "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu", "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} +{"path": "wiki/hardware/nvfp4.md", "before": {"statement": "sources: [doc-nvidia-tuning-guide, contest-gpumode-p1, contest-gpumode-p2, blog-yue-nvfp4]"}, "after": {"statement": "sources: [doc-transformer-engine-2.13-nvfp4, doc-ptx-isa-sm100]\n\nevidence_basis:\n - source_id: doc-transformer-engine-2.13-nvfp4\n evidence_type: official-doc\n - source_id: doc-ptx-isa-sm100\n evidence_type: official-doc"}, "reason": {"statement": "Remove the misleading evidence relationship while retaining the verified source-scoped hardware prose unchanged.", "urls": ["https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#alternate-floating-point-data-formats", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/kernels/nvfp4-gemv.md", "before": {"statement": "NVFP4 Batched GEMV is a memory-bound kernel computing batched matrix-vector products with NVFP4 (E2M1) block-scaled inputs on B200 GPUs. Unlike compute-bound GEMM, GEMV is dominated by memory bandwidth utilization (each FP4 element is used only once in the dot product), making PTX-level memory access control, cache policy differentiation, and register budgeting the critical optimization levers."}, "after": {"statement": "GPU Mode's Problem 1 asks for a block-scaled NVFP4 batched matrix-vector product on NVIDIA B200. The official task models its three benchmark cases against the slower of FFMA math and DRAM transfer time and reports DRAM-limited theoretical times. That task-specific model does not make every GEMV implementation or shape bandwidth-bound.\n\nPTX ISA 9.0 defines the packed conversion and typed register decomposition used in Yue's author-reported decode path:\n\n```asm\ncvt.rn.f16x2.e2m1x2 %result, %packed_fp4_pair;\nmov.b32 {%b0, %b1, %b2, %b3}, %packed_word;\n```\n\nIt also defines vector loads such as `ld.global.v2.u64` and `ld.global.v4.u64`, which move 16 and 32 bytes and can carry 32 and 64 packed FP4 values. The ISA specifies behavior, not that one width or decomposition is universally faster."}, "reason": {"statement": "Scope bottleneck and optimization statements to the official task model and measured implementations.", "urls": ["https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld", "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"]}} +{"path": "wiki/kernels/nvfp4-gemv.md", "before": {"statement": "performance_claims:\n- gpu: B200\n dtype: nvfp4\n shape: M=7168, K=16384, L=1\n metric: latency_us\n value: 22.4\n utilization: ~2.6x of SOL (8.6us)\n source_id: contest-gpumode-p1\n\nThis was Problem 1 of the GPU Mode NVFP4 Hackathon (Nov 2025). The theoretical speed-of-light is ~8.6us for the largest config, limited by B200's 8 TB/s HBM3e bandwidth. Top performers achieved ~22.4us (2.6x off SOL), reflecting the overhead of FP4 decoding and scale factor application."}, "after": {"statement": "performance_claims: []\n\nGPU Mode's Problem 1 asks for a block-scaled NVFP4 batched matrix-vector product on NVIDIA B200. The official task models its three benchmark cases against the slower of FFMA math and DRAM transfer time and reports DRAM-limited theoretical times. That task-specific model does not make every GEMV implementation or shape bandwidth-bound.\n\nThe public leaderboard is mutable and reports aggregate scores rather than per-shape latencies. In the snapshot fetched on 2026-08-08, the first three rows were `s.am._` at 18.549562452 µs, `gau.nernst` at 18.552844757 µs, and `shellsmile15795` at 18.707609314 µs. Yue's 22.392217755 µs submission was rank 11. The official cutoff was November 28, 2025 at 11:59 p.m. PT; current ranks 2 and 3 have November 30 timestamps, so this endpoint is not an official prize-placement snapshot."}, "reason": {"statement": "Remove malformed per-shape metadata and publish a dated aggregate snapshot with cutoff caveat.", "urls": ["https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemv/NVIDIA?limit=100", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml", "https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf"]}} +{"path": "wiki/kernels/nvfp4-gemv.md", "before": {"statement": "```\nInputs:\n a: (M x K x L) NVFP4 packed matrix\n b: (1 x K x L) NVFP4 packed vector\n sfa: (M x K/16 x L) FP8 E4M3 scale factors for a\n sfb: (1 x K/16 x L) FP8 E4M3 scale factors for b\n sf_a_global, sf_b_global: FP32 per-tensor global scales\n\nOutput:\n c: (M x 1 x L) FP16\n\nComputation per element:\n c[m][l] = sum_k( sf_a_global * sfa[m][k/16][l] * deq(a[m][k][l])\n * sf_b_global * sfb[0][k/16][l] * deq(b[0][k][l]) )\n\nBenchmark configs:\n Config 1: M=7168, K=16384, L=1\n Config 2: M=4096, K=7168, L=8\n Config 3: M=7168, K=2048, L=4\n```"}, "after": {"statement": "The logical operation has one B row, reused by all M output rows. The reference harness pads B and its scales to 128 rows so it can call `torch._scaled_mm`, then retains only result column zero. Consequently, A values are row-specific while the logical B vector is reused across M; it is incorrect to say every FP4 input value is consumed only once.\n\nAt pinned commit `ae679486`, `custom_kernel` receives seven tensors, not five tensors plus global FP32 scale arguments:\n\n| `a` | `[M, K/2, L]` | Packed E2M1 A; two logical values per byte |\n| `b` | `[128, K/2, L]` | Packed E2M1 B, physically padded; logical row 0 is used |\n| `sfa` | `[M, K/16, L]` | Logical/reference A block scales |\n| `sfb` | `[128, K/16, L]` | Logical/reference B block scales, padded to 128 rows |\n| `sfa_reordered` | `[32, 4, ceil(M/128), 4, K/64, L]` | Swizzled A scales for custom kernels |\n| `sfb_reordered` | `[32, 4, 1, 4, K/64, L]` | Swizzled padded-B scales for custom kernels |\n| `c` | `[M, 1, L]` | FP16 output buffer |\n\n`K` must be divisible by 64, and M must be divisible by the implementation's selected M tile. The task prose labels scales `e4m3fnuz`, but the pinned generator constructs `torch.float8_e4m3fn`; implementations should follow the actual harness revision they are tested against. This contest ABI exposes no per-tensor FP32 global scales."}, "reason": {"statement": "Publish both the logical operation and actual seven-tensor callable without inventing inputs.", "urls": []}} +{"path": "wiki/kernels/nvfp4-gemv.md", "before": {"statement": "Raw PTX provides critical performance gains over C intrinsics for FP4 decoding and memory access:\n\n```asm\n; FP4 to FP16 conversion: hardware instruction\n; Converts two packed E2M1 values to a pair of FP16 values\ncvt.rn.f16x2.e2m1x2 %result, %fp4_packed;\n\n; Byte unpacking: PTX mov.b32 is faster than manual bitwise ops\n; Instead of: val = (packed >> (i*4)) & 0xF (multiple shifts + masks)\n; Use: direct byte decomposition\nmov.b32 {tmp0, tmp1, tmp2, tmp3}, %packed_word;\n\n; This eliminates the shift-mask chain entirely\n; Key insight: PTX byte unpacking leverages hardware byte decomposition\n```"}, "after": {"statement": "The official table assumes a 1.5 GHz B200 clock and ranks submissions by the geometric mean across all three rows:\n\n| 7168 | 16384 | 1 | 8.622 |\n| 4096 | 7168 | 8 | 17.275 |\n| 7168 | 2048 | 4 | 4.317 |\n\nNVIDIA's Blackwell technical brief specifies 8 TB/s of HBM3e bandwidth for one GB200 GPU. The task's theoretical numbers are model values, not measured kernel timings.\n\nThe public leaderboard is mutable and reports aggregate scores rather than per-shape latencies. In the snapshot fetched on 2026-08-08, the first three rows were `s.am._` at 18.549562452 µs, `gau.nernst` at 18.552844757 µs, and `shellsmile15795` at 18.707609314 µs. Yue's 22.392217755 µs submission was rank 11. The official cutoff was November 28, 2025 at 11:59 p.m. PT; current ranks 2 and 3 have November 30 timestamps, so this endpoint is not an official prize-placement snapshot."}, "reason": {"statement": "Retain exact instruction semantics and author-reported bundled progression without causal isolation.", "urls": ["https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml", "https://resources.nvidia.com/en-us-blackwell-architecture/blackwell-architecture-technical-brief", "https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemv/NVIDIA?limit=100", "https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf"]}} +{"path": "wiki/kernels/nvfp4-gemv.md", "before": {"statement": "Lower register counts force higher occupancy, which is critical for memory-bound kernels where latency hiding dominates:\n\n```cpp\n// Rank 1: aggressive register limit for maximum occupancy\n// nvcc -maxrregcount=32\n// Fewer registers -> more warps/SM -> better memory latency hiding\n\n// Rank 3: slightly relaxed for more ILP\n// nvcc -maxrregcount=45\n\n// Launch bounds to complement register budgeting\n__launch_bounds__(256, 4) // 256 threads/block, 4+ blocks per SM\n__global__ void nvfp4_gemv_kernel(...) {\n // ...\n}\n\n// The measurable difference between 32 and 45 registers shows\n// that occupancy is the dominant factor for memory-bound kernels\n```"}, "after": {"statement": "PTX ISA 9.0 defines the packed conversion and typed register decomposition used in Yue's author-reported decode path:\n\n```asm\ncvt.rn.f16x2.e2m1x2 %result, %packed_fp4_pair;\nmov.b32 {%b0, %b1, %b2, %b3}, %packed_word;\n```\n\nIt also defines vector loads such as `ld.global.v2.u64` and `ld.global.v4.u64`, which move 16 and 32 bytes and can carry 32 and 64 packed FP4 values. The ISA specifies behavior, not that one width or decomposition is universally faster."}, "reason": {"statement": "Make register caps a measure-and-verify tuning parameter.", "urls": ["https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld", "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"]}} +{"path": "wiki/kernels/nvfp4-gemv.md", "before": {"statement": "```cpp\n// Each K variant compiled separately with optimal configuration\ntemplate \n__global__ __launch_bounds__(THREADS, MIN_BLOCKS)\nvoid nvfp4_gemv_specialized(\n const uint8_t* __restrict__ a,\n const uint8_t* __restrict__ b,\n const uint8_t* __restrict__ sfa,\n const uint8_t* __restrict__ sfb,\n half* __restrict__ c,\n float sf_a_global, float sf_b_global,\n int M\n) {\n float acc = 0.0f;\n\n // Full unroll: compiler knows K_SIZE at compile time\n #pragma unroll\n for (int k = 0; k < K_SIZE / ELEMENTS_PER_LOAD; k++) {\n // Load FP4 packed data\n uint64_t a_packed = load_fp4(a, row, k);\n uint64_t b_packed = load_fp4(b, 0, k);\n\n // Dequantize via PTX\n half2 a_vals = cvt_e2m1x2(a_packed);\n half2 b_vals = cvt_e2m1x2(b_packed);\n\n // Apply block scales\n float sa = sfa[row * (K_SIZE/16) + k/2];\n float sb = sfb[k/2];\n\n // Accumulate\n acc += (float)a_vals.x * (float)b_vals.x * sa * sb;\n acc += (float)a_vals.y * (float)b_vals.y * sa * sb;\n }\n\n // Apply global scales and store\n c[row] = __float2half(acc * sf_a_global * sf_b_global);\n}\n\n// Dispatch: K=2048 uses BLOCK_M=8, maxreg=32\n// K=7168 uses BLOCK_M=4, maxreg=40\n// K=16384 uses BLOCK_M=2, maxreg=32\n```"}, "after": {"statement": null}, "reason": {"statement": "No primary contestant source supports this invented implementation.", "urls": []}} +{"path": "wiki/kernels/nvfp4-gemv.md", "before": {"statement": "128-bit and 256-bit vector loads maximize bandwidth utilization:\n\n```asm\n; 128-bit vector load (16 bytes = 32 FP4 elements)\nld.global.v2.u64 {r0, r1}, [addr];\n\n; 256-bit vector load (32 bytes = 64 FP4 elements)\nld.global.v4.u64 {r0, r1, r2, r3}, [addr];\n\n; Only effective when combined with PTX byte unpacking\n; to avoid bitwise overhead in the subsequent unpack stage\n; Without proper unpacking, wide loads just move the bottleneck\n```"}, "after": {"statement": "The official table assumes a 1.5 GHz B200 clock and ranks submissions by the geometric mean across all three rows:\n\n| 7168 | 16384 | 1 | 8.622 |\n| 4096 | 7168 | 8 | 17.275 |\n| 7168 | 2048 | 4 | 4.317 |\n\nNVIDIA's Blackwell technical brief specifies 8 TB/s of HBM3e bandwidth for one GB200 GPU. The task's theoretical numbers are model values, not measured kernel timings.\n\nPTX ISA 9.0 defines the packed conversion and typed register decomposition used in Yue's author-reported decode path:\n\n```asm\ncvt.rn.f16x2.e2m1x2 %result, %packed_fp4_pair;\nmov.b32 {%b0, %b1, %b2, %b3}, %packed_word;\n```\n\nIt also defines vector loads such as `ld.global.v2.u64` and `ld.global.v4.u64`, which move 16 and 32 bytes and can carry 32 and 64 packed FP4 values. The ISA specifies behavior, not that one width or decomposition is universally faster."}, "reason": {"statement": "Keep width semantics but remove universal performance guarantees.", "urls": ["https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml", "https://resources.nvidia.com/en-us-blackwell-architecture/blackwell-architecture-technical-brief", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld", "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"]}} +{"path": "wiki/kernels/nvfp4-gemv.md", "before": {"statement": "```cpp\n// Load B vector into shared memory once per thread block\n// All BLOCK_M rows reuse the same B data\n__shared__ half b_shared[K_TILE];\n\n// Cooperative load: all threads in block load a portion of B\nfor (int i = threadIdx.x; i < K_TILE; i += blockDim.x) {\n b_shared[i] = dequant_fp4(b[i], sfb[i/16]) * sf_b_global;\n}\n__syncthreads();\n\n// Each thread computes dot product of its A row with shared B\nfloat acc = 0.0f;\nfor (int k = 0; k < K_TILE; k++) {\n acc += (float)a_dequant[k] * (float)b_shared[k];\n}\n\n// Reduces global memory traffic by BLOCK_M ratio\n```"}, "after": {"statement": "The logical operation has one B row, reused by all M output rows. The reference harness pads B and its scales to 128 rows so it can call `torch._scaled_mm`, then retains only result column zero. Consequently, A values are row-specific while the logical B vector is reused across M; it is incorrect to say every FP4 input value is consumed only once.\n\nAt pinned commit `ae679486`, `custom_kernel` receives seven tensors, not five tensors plus global FP32 scale arguments:\n\n| `a` | `[M, K/2, L]` | Packed E2M1 A; two logical values per byte |\n| `b` | `[128, K/2, L]` | Packed E2M1 B, physically padded; logical row 0 is used |\n| `sfa` | `[M, K/16, L]` | Logical/reference A block scales |\n| `sfb` | `[128, K/16, L]` | Logical/reference B block scales, padded to 128 rows |\n| `sfa_reordered` | `[32, 4, ceil(M/128), 4, K/64, L]` | Swizzled A scales for custom kernels |\n| `sfb_reordered` | `[32, 4, 1, 4, K/64, L]` | Swizzled padded-B scales for custom kernels |\n| `c` | `[M, 1, L]` | FP16 output buffer |\n\n`K` must be divisible by 64, and M must be divisible by the implementation's selected M tile. The task prose labels scales `e4m3fnuz`, but the pinned generator constructs `torch.float8_e4m3fn`; implementations should follow the actual harness revision they are tested against. This contest ABI exposes no per-tensor FP32 global scales.\n\nPTX ISA 9.0 defines the packed conversion and typed register decomposition used in Yue's author-reported decode path:\n\n```asm\ncvt.rn.f16x2.e2m1x2 %result, %packed_fp4_pair;\nmov.b32 {%b0, %b1, %b2, %b3}, %packed_word;\n```\n\nIt also defines vector loads such as `ld.global.v2.u64` and `ld.global.v4.u64`, which move 16 and 32 bytes and can carry 32 and 64 packed FP4 values. The ISA specifies behavior, not that one width or decomposition is universally faster."}, "reason": {"statement": "Remove invented contestant code and exact traffic attribution while retaining logical B reuse.", "urls": ["https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld"]}} +{"path": "wiki/kernels/nvfp4-gemv.md", "before": {"statement": "| Coalesced access | Fix memory access patterns | 443us -> 39us |"}, "after": {"statement": "The public leaderboard is mutable and reports aggregate scores rather than per-shape latencies. In the snapshot fetched on 2026-08-08, the first three rows were `s.am._` at 18.549562452 µs, `gau.nernst` at 18.552844757 µs, and `shellsmile15795` at 18.707609314 µs. Yue's 22.392217755 µs submission was rank 11. The official cutoff was November 28, 2025 at 11:59 p.m. PT; current ranks 2 and 3 have November 30 timestamps, so this endpoint is not an official prize-placement snapshot."}, "reason": {"statement": "Restore the author's actual stage boundaries.", "urls": ["https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html", "https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemv/NVIDIA?limit=100", "https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf"]}} +{"path": "wiki/kernels/nvfp4-gemv.md", "before": {"statement": "1. **Memory-bound kernels need bandwidth-first thinking**: Arithmetic optimizations have minimal impact; focus on memory access patterns, cache policies, and vectorized loads"}, "after": {"statement": "PTX ISA 9.0 defines the packed conversion and typed register decomposition used in Yue's author-reported decode path:\n\n```asm\ncvt.rn.f16x2.e2m1x2 %result, %packed_fp4_pair;\nmov.b32 {%b0, %b1, %b2, %b3}, %packed_word;\n```\n\nIt also defines vector loads such as `ld.global.v2.u64` and `ld.global.v4.u64`, which move 16 and 32 bytes and can carry 32 and 64 packed FP4 values. The ISA specifies behavior, not that one width or decomposition is universally faster."}, "reason": {"statement": "Replace a universal rule with a profiling-dependent decision procedure.", "urls": ["https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld"]}} +{"path": "wiki/kernels/nvfp4-gemv.md", "before": {"statement": "2. **PTX gives real control on Blackwell**: The gap between C intrinsics and hand-written PTX was substantial (443us to 27us in one journey)"}, "after": {"statement": "The public leaderboard is mutable and reports aggregate scores rather than per-shape latencies. In the snapshot fetched on 2026-08-08, the first three rows were `s.am._` at 18.549562452 µs, `gau.nernst` at 18.552844757 µs, and `shellsmile15795` at 18.707609314 µs. Yue's 22.392217755 µs submission was rank 11. The official cutoff was November 28, 2025 at 11:59 p.m. PT; current ranks 2 and 3 have November 30 timestamps, so this endpoint is not an official prize-placement snapshot."}, "reason": {"statement": "Use source-reported combined-stage attribution only.", "urls": ["https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html", "https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemv/NVIDIA?limit=100", "https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf"]}} +{"path": "wiki/kernels/nvfp4-gemv.md", "before": {"statement": "4. **Register budgeting matters**: Lower registers -> higher occupancy -> better memory latency hiding"}, "after": {"statement": "PTX ISA 9.0 defines the packed conversion and typed register decomposition used in Yue's author-reported decode path:\n\n```asm\ncvt.rn.f16x2.e2m1x2 %result, %packed_fp4_pair;\nmov.b32 {%b0, %b1, %b2, %b3}, %packed_word;\n```\n\nIt also defines vector loads such as `ld.global.v2.u64` and `ld.global.v4.u64`, which move 16 and 32 bytes and can carry 32 and 64 packed FP4 values. The ISA specifies behavior, not that one width or decomposition is universally faster."}, "reason": {"statement": "Make register caps conditional on realized use, spills, occupancy, and timing.", "urls": ["https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld", "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"]}} +{"path": "wiki/kernels/nvfp4-gemv.md", "before": {"statement": "5. **TMEM is irrelevant**: Memory-bound kernels do not benefit from TMEM (it helps compute-bound only)"}, "after": {"statement": null}, "reason": {"statement": "No useful universal TMEM recommendation is supported for this page.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://arxiv.org/html/2603.05451v1"]}} +{"path": "wiki/kernels/nvfp4-gemv.md", "before": {"statement": "- Memory-bound operations with FP4 quantized weights"}, "after": {"statement": "Amandeep reports that three solutions inspected after the event used `L1::no_allocate` for streamed A loads, `L1::evict_last` for reused B loads, wider PTX loads, and exact-K specializations. Those are author-reported observations; the public leaderboard API supplies no contestant code or technique field. Amandeep's own wider `uint2` experiment was 16–25% slower, and reducing `-maxrregcount` from 80 to 64 had no effect because the kernel already used fewer than 64 registers. Load width, cache hints, and register caps therefore require measurement in the exact implementation."}, "reason": {"statement": "Scope applicability to exact task-compatible NVFP4 operands.", "urls": ["https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"]}} +{"path": "wiki/kernels/nvfp4-gemv.md", "before": {"statement": "- SM100/SM100a only (native FP4 decode instructions)"}, "after": {"statement": "Amandeep reports that three solutions inspected after the event used `L1::no_allocate` for streamed A loads, `L1::evict_last` for reused B loads, wider PTX loads, and exact-K specializations. Those are author-reported observations; the public leaderboard API supplies no contestant code or technique field. Amandeep's own wider `uint2` experiment was 16–25% slower, and reducing `-maxrregcount` from 80 to 64 had no effect because the kernel already used fewer than 64 registers. Load width, cache hints, and register caps therefore require measurement in the exact implementation."}, "reason": {"statement": "Separate contest B200 scope from versioned instruction/library support.", "urls": ["https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://docs.nvidia.com/cuda/archive/12.9.2/parallel-thread-execution/index.html", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"]}} +{"path": "wiki/kernels/nvfp4-gemv.md", "before": {"statement": "Local verbatim upstream code lives in [`artifacts/kernels/nvfp4-gemv/full/`](../../artifacts/kernels/nvfp4-gemv/full/) (see its `PROVENANCE.yaml` for the pinned upstream SHA and byte-verified SHA-256). Labeled derived variants — including a naive/teaching skeleton — live in [`artifacts/kernels/nvfp4-gemv/variants/`](../../artifacts/kernels/nvfp4-gemv/variants/)."}, "after": {"statement": null}, "reason": {"statement": "Detach the misleading artifact relationship while preserving the files and immutable receipt.", "urls": []}} +{"path": "wiki/languages/ptx-sm100.md", "before": {"statement": "```ptx\n// Allocate TMEM columns\ntcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [smem_tmem_addr], num_cols;\n\n// MMA: inputs from SMEM, accumulator in TMEM\ntcgen05.mma.cta_group::1.kind::f16 [tmem_addr], desc_a, desc_b, idesc, enable_input_d;\n\n// 2-SM cooperative MMA\ntcgen05.mma.cta_group::2.kind::f16 [tmem_addr], desc_a, desc_b, idesc, enable_input_d;\n\n// Load TMEM to registers\ntcgen05.ld.sync.aligned.32x32b.x1.b32 {regs}, [tmem_addr];\n\n// Store registers to TMEM\ntcgen05.st.sync.aligned.32x32b.x1.b32 [tmem_addr], {regs};\n\n// Copy a shaped SMEM matrix descriptor into TMEM\ntcgen05.cp.cta_group::1.128x256b [tmem_addr], sdesc;\n\n// Critical fence between TMA completion and MMA\ntcgen05.fence::after_thread_sync;\n\n// Deallocate TMEM (MUST before kernel exit)\ntcgen05.dealloc.cta_group::1.sync.aligned.b32 tmem_addr, num_cols;\n```"}, "after": {"statement": "TMEM/register transfers and shared-memory/TMEM copies have their own asynchronous completion rules:\n\n```ptx\ntcgen05.ld.sync.aligned.32x32b.x1.b32 {r0}, [taddr];\ntcgen05.wait::ld.sync.aligned;\n\ntcgen05.st.sync.aligned.32x32b.x1.b32 [taddr], {r0};\ntcgen05.wait::st.sync.aligned;\n\ntcgen05.cp.cta_group::1.128x256b [taddr], sdesc;\n```\n\n`tcgen05.ld` and `tcgen05.st` are warp-collective. `tcgen05.cp` copies a shaped shared-memory descriptor into TMEM. MMA and cp completion can be attached to an mbarrier with `tcgen05.commit`; the source and destination must remain live until their documented completion points."}, "reason": {"statement": "Separate TMA completion, async-proxy visibility, tcgen completion, and cross-thread tcgen ordering.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-cp"]}} +{"path": "wiki/languages/ptx-sm100.md", "before": {"statement": "```ptx\n// Convert two packed FP4 values to two FP16 values\ncvt.rn.f16x2.e2m1x2 result_f16x2, packed_fp4;\n\n// Byte unpacking (faster than bitwise extraction)\nmov.b32 {byte0, byte1, byte2, byte3}, packed_word;\n```"}, "after": {"statement": "- A TMA global-to-shared load completes bytes on its mbarrier; a consumer waits for that phase before reading the destination.\n- `tcgen05.commit` makes an mbarrier track completion of prior asynchronous tcgen05 MMA/cp/shift operations issued by the thread.\n- `tcgen05.wait::ld` and `tcgen05.wait::st` wait for the corresponding prior TMEM/register transfers.\n- `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` constrain tcgen05 operations around a documented cross-thread execution-ordering handoff. They are not substitutes for TMA or MMA completion waits."}, "reason": {"statement": "Keep exact semantics and require generated-code/timing comparison for performance.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov", "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence"]}} +{"path": "wiki/languages/ptx-sm100.md", "before": {"statement": "```ptx\n// Streaming data (use once): bypass L1\nld.global.L1::no_allocate.v2.u64 {r0, r1}, [addr];\n\n// Reused data: keep in L1\nld.global.L1::evict_last.v2.u64 {r0, r1}, [addr];\n\n// Wide vectorized loads\nld.global.v4.u64 {r0, r1, r2, r3}, [addr]; // 256-bit\n```"}, "after": {"statement": "PTX ISA 9.0 defines these typed operations:\n\n```ptx\ncvt.rn.f16x2.e2m1x2 result_f16x2, packed_fp4_pair;\nmov.b32 {byte0, byte1, byte2, byte3}, packed_word;\n```\n\nThe first converts one byte containing two E2M1 values into a 32-bit F16x2 result. The second can decompose a 32-bit scalar into four byte-sized destinations when operand declarations satisfy the scalar-to-vector size rules. PTX specifies these semantics but does not guarantee that the move is faster than every compiler-generated shift/mask sequence."}, "reason": {"statement": "Use normative hint semantics and separate workload selection from instruction behavior.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#cache-operators", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov"]}} +{"path": "wiki/languages/ptx-sm100.md", "before": {"statement": "```ptx\n// Query for next tile (persistent kernel loop)\nclusterlaunchcontrol.try_cancel {clc_id};\n// Returns valid tile_id or decline (all work done)\n```"}, "after": {"statement": "```ptx\nld.global.L1::no_allocate.v2.u64 {r0, r1}, [addr];\nld.global.L1::evict_last.v2.u64 {r0, r1}, [addr];\nld.global.v4.u64 {r0, r1, r2, r3}, [addr];\n```\n\n`L1::no_allocate` and `L1::evict_last` are eviction-priority hints and may not always be respected. They do not guarantee L1 bypass or residency. The vector forms above move 16 and 32 bytes; whether a width or hint helps depends on alignment, surrounding instructions, reuse, cache state, and the target GPU."}, "reason": {"statement": "Replace with the exact request/wait/query sequence.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld"]}} +{"path": "wiki/languages/ptx-sm100.md", "before": {"statement": "```ptx\n// Bulk tensor copy: global → shared\ncp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes\n [smem_ptr], [tensorMap, {x, y}], [mbarrier];\n\n// Multicast to cluster SMs\ncp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast\n [smem_ptr], [tensorMap, {x, y}], [mbarrier], multicast_mask;\n```"}, "after": {"statement": "CLC cancellation is an asynchronous request followed by response decoding:\n\n```ptx\nclusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128\n [response_smem], [response_mbarrier];\n\nclusterlaunchcontrol.query_cancel.is_canceled.pred.b128 p, response_b128;\n@p clusterlaunchcontrol.query_cancel.get_first_ctaid.v4.b32.b128\n {x, y, z, unused}, response_b128;\n```\n\nThe request writes an opaque 16-byte response to shared memory and completes on the mbarrier. Code must wait for that phase before loading and querying the response. A successful query returns the first CTA coordinate of a canceled not-yet-launched block or cluster; the request does not take a desired tile ID."}, "reason": {"statement": "Use the exact version-pinned PTX spelling.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel"]}} +{"path": "wiki/languages/cuda-cpp.md", "before": {"statement": "```cuda\n// Allocate TMEM. tcgen05.alloc writes the allocated address into SMEM.\n__device__ uint32_t tmem_alloc_cta(uint32_t* smem_tmem_addr,\n uint32_t num_cols) {\n if (threadIdx.x == 0) {\n uint32_t smem_addr =\n static_cast(__cvta_generic_to_shared(smem_tmem_addr));\n asm volatile(\n \"tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;\"\n :: \"r\"(smem_addr), \"r\"(num_cols)\n );\n }\n __syncthreads();\n return *smem_tmem_addr;\n}\n\n// Issue MMA (single thread, typically warp 1 lane 0)\n// idesc_c/idesc_d: immediate descriptors for accumulator C and output D\n__device__ void tcgen05_mma(uint32_t tmem_addr,\n uint64_t desc_a, uint64_t desc_b,\n uint32_t idesc_c, uint32_t idesc_d) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16\"\n \" [%0], %1, %2, %3, %4;\"\n :: \"r\"(tmem_addr), \"l\"(desc_a), \"l\"(desc_b),\n \"r\"(idesc_c), \"r\"(idesc_d)\n );\n}\n\n// Load TMEM to registers\n__device__ void tmem_load(float* dst, uint32_t tmem_addr, int cols) {\n asm volatile(\n \"tcgen05.ld.sync.aligned.32x32b.x1.b32 {%0}, [%1];\"\n : \"=f\"(*dst) : \"r\"(tmem_addr)\n );\n}\n\n// Deallocate TMEM (MUST do before kernel exit)\n__device__ void tmem_dealloc(uint32_t addr, uint32_t num_cols) {\n asm volatile(\n \"tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;\"\n :: \"r\"(addr), \"r\"(num_cols)\n );\n}\n```"}, "after": {"statement": "The CUDA front end does not parse the instruction text inside an `asm()` statement. Operand constraints and address-space conversion therefore remain the wrapper author's responsibility. Use `\"r\"` for a 32-bit integer register, `\"l\"` for a 64-bit integer register, and convert a generic pointer with `__cvta_generic_to_shared` before supplying a shared-memory address. Add a `\"memory\"` clobber when the assembly has memory effects that are hidden from the compiler."}, "reason": {"statement": "Show whole-warp allocation and reserve lane election for single-thread MMA/TMA issue.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu", "https://docs.nvidia.com/cuda/archive/13.0.2/inline-ptx-assembly/index.html"]}} +{"path": "wiki/languages/cuda-cpp.md", "before": {"statement": "```cuda\n// Allocate TMEM. tcgen05.alloc writes the allocated address into SMEM.\n__device__ uint32_t tmem_alloc_cta(uint32_t* smem_tmem_addr,\n uint32_t num_cols) {\n if (threadIdx.x == 0) {\n uint32_t smem_addr =\n static_cast(__cvta_generic_to_shared(smem_tmem_addr));\n asm volatile(\n \"tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;\"\n :: \"r\"(smem_addr), \"r\"(num_cols)\n );\n }\n __syncthreads();\n return *smem_tmem_addr;\n}\n\n// Issue MMA (single thread, typically warp 1 lane 0)\n// idesc_c/idesc_d: immediate descriptors for accumulator C and output D\n__device__ void tcgen05_mma(uint32_t tmem_addr,\n uint64_t desc_a, uint64_t desc_b,\n uint32_t idesc_c, uint32_t idesc_d) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16\"\n \" [%0], %1, %2, %3, %4;\"\n :: \"r\"(tmem_addr), \"l\"(desc_a), \"l\"(desc_b),\n \"r\"(idesc_c), \"r\"(idesc_d)\n );\n}\n\n// Load TMEM to registers\n__device__ void tmem_load(float* dst, uint32_t tmem_addr, int cols) {\n asm volatile(\n \"tcgen05.ld.sync.aligned.32x32b.x1.b32 {%0}, [%1];\"\n : \"=f\"(*dst) : \"r\"(tmem_addr)\n );\n}\n\n// Deallocate TMEM (MUST do before kernel exit)\n__device__ void tmem_dealloc(uint32_t addr, uint32_t num_cols) {\n asm volatile(\n \"tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;\"\n :: \"r\"(addr), \"r\"(num_cols)\n );\n}\n```"}, "after": {"statement": "The allocation instruction is collective. For `.cta_group::1`, every lane of one designated warp must execute the same instruction. Synchronize the CTA before another warp reads the address written to shared memory.\n\n```cuda\n// All 32 lanes of alloc_warp execute this branch.\nif (warp_id == alloc_warp) {\n uint32_t smem_addr =\n static_cast(__cvta_generic_to_shared(smem_tmem_addr));\n asm volatile(\n \"tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 \"\n \"[%0], %1;\"\n :: \"r\"(smem_addr), \"r\"(num_cols) : \"memory\");\n}\n__syncthreads();\nuint32_t taddr = *smem_tmem_addr;\n```"}, "reason": {"statement": "Use one idesc plus a local predicate constructed from a CUDA integer.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu"]}} +{"path": "wiki/languages/cuda-cpp.md", "before": {"statement": "```cuda\n// TMA-MMA synchronization via mbarrier\n// expected_bytes: total bytes the TMA will deliver to this stage\n__device__ void mbarrier_arrive(uint64_t* mbar, uint32_t expected_bytes) {\n asm volatile(\n \"mbarrier.arrive.expect_tx.shared.b64 _, [%0], %1;\"\n :: \"r\"((uint32_t)__cvta_generic_to_shared(mbar)),\n \"r\"(expected_bytes)\n );\n}\n\n__device__ void mbarrier_wait(uint64_t* mbar, int phase) {\n asm volatile(\n \"{\\n\"\n \".reg .pred p;\\n\"\n \"WAIT_LOOP:\\n\"\n \" mbarrier.try_wait.parity.shared.b64 p, [%0], %1;\\n\"\n \" @!p bra WAIT_LOOP;\\n\"\n \"}\\n\"\n :: \"r\"((uint32_t)__cvta_generic_to_shared(mbar)),\n \"r\"(phase)\n );\n}\n```"}, "after": {"statement": "An unscaled `kind::f16` MMA takes one instruction descriptor and an `enable-input-d` predicate. A single elected thread may issue this MMA form; converting an ordinary CUDA integer to the required PTX predicate inside the assembly keeps the C++ interface well typed.\n\n```cuda\n__device__ inline void tcgen05_mma_f16(\n uint32_t taddr, uint64_t a_desc, uint64_t b_desc,\n uint32_t idesc, int enable_input_d) {\n asm volatile(\n \"{\\n\\t\"\n \".reg .pred p;\\n\\t\"\n \"setp.ne.b32 p, %4, 0;\\n\\t\"\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, p;\\n\\t\"\n \"}\"\n :: \"r\"(taddr), \"l\"(a_desc), \"l\"(b_desc),\n \"r\"(idesc), \"r\"(enable_input_d));\n}\n```"}, "reason": {"statement": "Publish the full lifecycle invariants and memory-clobber requirement rather than a misleading two-function protocol.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/inline-ptx-assembly/index.html#incorrect-optimization", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/common.h#L66-L78"]}} +{"path": "wiki/languages/cute-dsl.md", "before": {"statement": "CuTe (CUDA Templates) DSL is the primary abstraction layer in CUTLASS 4.5.0 for Blackwell kernels. FlashAttention-4 was implemented entirely in CuTe-DSL (Python variant), achieving 20-30× faster compilation than C++ templates."}, "after": {"statement": "CUTLASS v4.5.0 includes CuTe DSL, a Python-native interface for authoring GPU kernels, alongside CUTLASS's C++ template interfaces. This page uses the exact `v4.5.0` tag (`e406c186f510a15091cce01f782020ceb7ba8eb5`); rolling `latest` documentation can contain later names."}, "reason": {"statement": "Describe CuTe DSL as a Python-native CUTLASS interface without assigning unsupported primacy.", "urls": []}} +{"path": "wiki/languages/cute-dsl.md", "before": {"statement": "```python\n# CuTe-DSL: SM100 MMA atom for BF16\nfrom cutlass.cute import *\n\n# 1-SM MMA: m128 x n256 x k16\nmma_atom = SM100_MMA_F16BF16_SS # inputs from shared memory\n# Accumulator goes to TMEM automatically\n\n# 2-SM MMA: m256 x n256 x k16\nmma_atom_2sm = SM100_MMA_F16BF16_SS_2SM\n```"}, "after": {"statement": "CUTLASS 4.5.0 uses a configured `tcgen05.MmaF16BF16Op`, not an `SM100_MMA_F16BF16_SS` Python symbol. This excerpt shows the one-CTA operation used by the tagged first tutorial; it is construction code, not a complete kernel.\n\n```python\nimport cutlass\nimport cutlass.cute as cute\nfrom cutlass.cute.nvgpu import tcgen05\n\nop = tcgen05.MmaF16BF16Op(\n cutlass.Float16,\n cutlass.Float32,\n (128, 256, 16),\n tcgen05.CtaGroup.ONE,\n tcgen05.OperandSource.SMEM,\n cute.nvgpu.OperandMajorMode.K,\n cute.nvgpu.OperandMajorMode.K,\n)\ntiled_mma = cute.make_tiled_mma(op)\n```\n\nThe two-CTA tutorial instead uses instruction shape `(256, 256, 16)` and `tcgen05.CtaGroup.TWO`. The accumulator fragment is rebound to an allocated TMEM pointer before `cute.gemm` issues the operation."}, "reason": {"statement": "Replace invented identifiers with the exact version-pinned constructor.", "urls": []}} +{"path": "wiki/languages/cute-dsl.md", "before": {"statement": "```python\n# TMEM allocation in CuTe\ntmem_tensor = make_tensor(\n make_tmem_ptr(tmem_addr),\n make_layout(make_shape(128, 256)) # rows x cols\n)\n\n# Copy TMEM → registers for epilogue\ncopy(tmem_tensor, reg_tensor) # tcgen05.ld under the hood\n```"}, "after": {"statement": "The tagged examples use `cutlass.utils.TmemAllocator` around a shared holding buffer. A complete path allocates columns, waits before pointer retrieval, rebinds the accumulator tensor, partitions a typed `tcgen05` TMEM-to-register copy across participating threads, synchronizes readers, and frees the same allocation.\n\n```python\n# Excerpt: storage/barrier/tensor definitions and pipeline edges are required.\ntmem = utils.TmemAllocator(\n storage.tmem_holding_buf.ptr,\n barrier_for_retrieve=tmem_alloc_barrier,\n)\ntmem.allocate(num_tmem_cols)\ntmem.wait_for_alloc()\ntmem_ptr = tmem.retrieve_ptr(cutlass.Float32)\ntCtAcc = cute.make_tensor(tmem_ptr, tCtAcc.layout)\n\ntmem_atom = cute.make_copy_atom(\n tcgen05.Ld32x32bOp(tcgen05.Repetition.x64), cutlass.Float32\n)\ntmem_tiled_copy = tcgen05.make_tmem_copy(tmem_atom, tCtAcc_epi[None, 0])\n# get_slice(), partition_S/D(), and cute.copy() form the per-thread epilogue.\n\npipeline.sync(barrier_id=1)\ntmem.free(tmem_ptr)\n```\n\nThe excerpt deliberately does not imply that allocation or deallocation is lane-local. Use the complete tutorial for collective participation, two-CTA handling, and pointer lifetime."}, "reason": {"statement": "Use an evidence-scoped excerpt that preserves allocation, copy partition, wait, and free responsibilities.", "urls": []}} +{"path": "wiki/languages/cute-dsl.md", "before": {"statement": "```python\n# TMA bulk copy: global → shared\ntma_copy = SM100_TMA_LOAD_2D\n\n# Setup TMA descriptor\ntma_desc = make_tma_copy(\n tma_copy,\n global_tensor,\n smem_layout,\n tile_shape,\n cluster_shape\n)\n```"}, "after": {"statement": "The one-CTA tutorial constructs a typed global-to-shared TMA operation and then derives operand-specific tiled atoms. Kernel code subsequently uses `tma_partition` and `cute.copy` with pipeline barriers.\n\n```python\nfrom cutlass.cute.nvgpu import cpasync, tcgen05\n\ntma_op = cpasync.CopyBulkTensorTileG2SOp(tcgen05.CtaGroup.ONE)\na_tma_atom, a_tma_tensor = cute.nvgpu.make_tiled_tma_atom_A(\n tma_op,\n a,\n a_smem_layout_one_stage,\n mma_tiler_mnk,\n tiled_mma,\n)\n```\n\nFor the two-CTA tutorial, the corresponding operation is `CopyBulkTensorTileG2SMulticastOp(CtaGroup.TWO)` and the cluster layout/multicast participants must agree with the launch."}, "reason": {"statement": "Replace fabricated shorthand with the release's typed TMA atom and partitioning API.", "urls": []}} +{"path": "wiki/languages/cute-dsl.md", "before": {"statement": "```python\n@cute_kernel\ndef blackwell_gemm(A, B, C):\n # Warp 0: TMA producer\n if warp_id == 0:\n for stage in pipeline:\n tma_copy(A_tile, smem_A[stage])\n tma_copy(B_tile, smem_B[stage])\n arrive(mbarrier[stage])\n\n # Warp 1: MMA consumer\n elif warp_id == 1:\n tmem = alloc_tmem(256) # columns\n for stage in pipeline:\n wait(mbarrier[stage])\n fence_after_thread_sync()\n mma(smem_A[stage], smem_B[stage], tmem)\n signal_epilogue()\n\n # Warps 2+: Epilogue\n else:\n wait_epilogue()\n regs = load_tmem(tmem)\n store_global(C, regs)\n dealloc_tmem(tmem)\n```"}, "after": {"statement": "`fp16_gemm_2.py` specializes TMA, MMA, and epilogue warps and uses `PipelineTmaUmma` plus `PipelineUmmaAsync` to represent full/empty ownership. It also retains explicit TMEM allocation, epilogue TMEM-copy partitioning, TMA-store completion, pipeline tails, and collective teardown. Those details are required; an `if warp_id` sketch alone is not a safe synchronization recipe."}, "reason": {"statement": "Replace unsafe pseudocode with verified role/lifetime invariants and a link to the full implementation.", "urls": []}} +{"path": "wiki/languages/cute-dsl.md", "before": {"statement": "3. Same performance as hand-written C++ (FlashAttention-4: 1605 TFLOPS)"}, "after": {"statement": "FlashAttention-4 is implemented entirely in CuTe DSL. Its author reports roughly 20--30x shorter compile times than C++ templates and a peak of 1605 TFLOP/s on B200 BF16. These are FA4-specific, source-reported results. The 1605-TFLOP/s result was not presented as a matched comparison with a handwritten-C++ FA4 kernel."}, "reason": {"statement": "Preserve the author-reported maximum and remove the unsupported comparator.", "urls": ["https://tridao.me/blog/2026/flash4/"]}} +{"path": "wiki/languages/cute-dsl.md", "before": {"statement": "5. Automatic layout computation and swizzle handling"}, "after": {"statement": "CuTe supplies typed layout algebra and Blackwell helpers such as `make_smem_layout_a` and `make_smem_layout_b`. The author still supplies the MMA tiler, datatypes, operand major modes, alignment, cluster shape, and pipeline policy; layout and swizzle choices are helper-assisted rather than universally automatic."}, "reason": {"statement": "Narrow the benefit to typed composition and version-pinned Blackwell helpers.", "urls": []}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "On Blackwell (SM100), `tcgen05.mma` stores accumulators in **Tensor Memory (TMEM)** -- a dedicated 256KB per-SM memory. This eliminates accumulator register pressure entirely, freeing registers for data movement, epilogue computation, and enabling larger tile sizes."}, "after": {"statement": "For `sm_100a`, tcgen05 D is addressed in Tensor Memory rather than supplied as a per-thread register vector. The CTA-visible TMEM structure has 128 lanes by 512 columns of 32-bit cells, or 256 KiB when fully allocated. Allocation is dynamic in columns.\n\nThis removes the resident D vector from the ordinary register file. It does not remove all register pressure: descriptors, addresses, loop state, pipeline state, and each TMEM-to-register epilogue batch still need registers. Both Hopper and compute-capability-10.0 Blackwell have 64K 32-bit registers per SM and at most 255 registers per thread. Occupancy, spilling, and practical tile size therefore remain properties of the compiled kernel and launch configuration."}, "reason": {"statement": "Keep the resident-D benefit while preserving resource-dependent limits.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "```\nHopper wgmma m64xn256xk16 BF16 register budget (per thread):\n\n Accumulator: 128 FP32 registers (m64 x n256 / 128 threads)\n A operand: 16 registers (via ldmatrix)\n B descriptor: 2 registers\n Loop variables: 8 registers\n SMEM pointers: 4 registers\n TMA state: 6 registers\n Misc (indexing): 10 registers\n ─────────────────────────────\n Total per thread: ~174 registers\n\n Per warpgroup (128 threads): 174 * 128 = 22,272 registers\n Per SM (65,536 regs): max 2 warpgroups = 2 CTAs at this tile size\n\n Occupancy: 2 CTAs per SM (limited by registers)\n```"}, "after": {"statement": "For Hopper `wgmma.mma_async.m64nNk16` with an FP32 accumulator, each warpgroup thread holds `N/2` FP32 D registers. At `N=256`, that is 128 registers per thread for D. WGMMA updates that register vector asynchronously; the program uses the WGMMA fence, commit-group, and wait-group mechanisms before dependent use.\n\nFor `sm_100a`, tcgen05 D is addressed in Tensor Memory rather than supplied as a per-thread register vector. The CTA-visible TMEM structure has 128 lanes by 512 columns of 32-bit cells, or 256 KiB when fully allocated. Allocation is dynamic in columns.\n\nThis removes the resident D vector from the ordinary register file. It does not remove all register pressure: descriptors, addresses, loop state, pipeline state, and each TMEM-to-register epilogue batch still need registers. Both Hopper and compute-capability-10.0 Blackwell have 64K 32-bit registers per SM and at most 255 registers per thread. Occupancy, spilling, and practical tile size therefore remain properties of the compiled kernel and launch configuration."}, "reason": {"statement": "No pinned kernel/resource report supports the invented categories.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "If the tile is larger (e.g., m64xn256xk32 with double-buffered accumulators), register pressure becomes even worse:\n\n```\nDouble-buffered accumulator: 256 registers per thread\nTotal per thread: ~300 registers\nPer warpgroup: 38,400 registers\nPer SM: max 1 warpgroup -> 1 CTA per SM\n\nOccupancy: 1 CTA per SM (severe underutilization)\n```"}, "after": {"statement": "For `sm_100a`, tcgen05 D is addressed in Tensor Memory rather than supplied as a per-thread register vector. The CTA-visible TMEM structure has 128 lanes by 512 columns of 32-bit cells, or 256 KiB when fully allocated. Allocation is dynamic in columns.\n\nThis removes the resident D vector from the ordinary register file. It does not remove all register pressure: descriptors, addresses, loop state, pipeline state, and each TMEM-to-register epilogue batch still need registers. Both Hopper and compute-capability-10.0 Blackwell have 64K 32-bit registers per SM and at most 255 registers per thread. Occupancy, spilling, and practical tile size therefore remain properties of the compiled kernel and launch configuration."}, "reason": {"statement": "The numeric scenario is not source- or experiment-grounded.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/hopper-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "```\nRegister access: ~4 cycles\nL1 local access: ~30 cycles\nL2 spill access: ~200 cycles\n\nA single spilled accumulator register accessed every MMA iteration\ncan cost 200 cycles * K_tiles additional latency.\n```"}, "after": {"statement": "For `sm_100a`, tcgen05 D is addressed in Tensor Memory rather than supplied as a per-thread register vector. The CTA-visible TMEM structure has 128 lanes by 512 columns of 32-bit cells, or 256 KiB when fully allocated. Allocation is dynamic in columns.\n\nThis removes the resident D vector from the ordinary register file. It does not remove all register pressure: descriptors, addresses, loop state, pipeline state, and each TMEM-to-register epilogue batch still need registers. Both Hopper and compute-capability-10.0 Blackwell have 64K 32-bit registers per SM and at most 255 registers per thread. Occupancy, spilling, and practical tile size therefore remain properties of the compiled kernel and launch configuration."}, "reason": {"statement": "Unsupported latency constants would mislead migration decisions.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-best-practices-guide/index.html#local-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "```\nBlackwell tcgen05 m128xn256xk16 BF16 register budget (per thread):\n\n Accumulator: 0 registers (stored in TMEM!)\n SMEM descriptors: 4 registers\n Loop variables: 8 registers\n TMA state: 6 registers\n TMEM address: 1 register\n Misc (indexing): 10 registers\n ─────────────────────────────\n Total per thread: ~29 registers\n\n Per CTA (256 threads): 29 * 256 = 7,424 registers\n Per SM (65,536 regs): max 8 CTAs (register-wise)\n Actual limit: TMEM capacity (512 columns)\n\n TMEM usage per CTA: 256 columns for 128x256 tile\n Max CTAs per SM: 512 / 256 = 2 CTAs (TMEM-limited, not register-limited)\n```\n\nThe key insight: **registers are no longer the bottleneck**. TMEM capacity becomes the binding constraint for occupancy, and the freed registers enable complex epilogues without spilling."}, "after": {"statement": "For `sm_100a`, tcgen05 D is addressed in Tensor Memory rather than supplied as a per-thread register vector. The CTA-visible TMEM structure has 128 lanes by 512 columns of 32-bit cells, or 256 KiB when fully allocated. Allocation is dynamic in columns.\n\nThis removes the resident D vector from the ordinary register file. It does not remove all register pressure: descriptors, addresses, loop state, pipeline state, and each TMEM-to-register epilogue batch still need registers. Both Hopper and compute-capability-10.0 Blackwell have 64K 32-bit registers per SM and at most 255 registers per thread. Occupancy, spilling, and practical tile size therefore remain properties of the compiled kernel and launch configuration."}, "reason": {"statement": "Replace speculative arithmetic with a measurement workflow.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory-allocation", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "```cuda\n// HOPPER: Accumulator lives and dies in registers\n__global__ void hopper_kernel(/* ... */) {\n // 1. Declare accumulator in registers\n float acc[4][32]; // 128 registers per thread!\n\n // 2. Zero-initialize\n #pragma unroll\n for (int i = 0; i < 4; ++i)\n for (int j = 0; j < 32; ++j)\n acc[i][j] = 0.0f;\n\n // 3. Mainloop: wgmma accumulates into registers\n for (int k = 0; k < K_tiles; ++k) {\n // Load A via ldmatrix\n uint32_t a_frag[4];\n ldmatrix(a_frag, smem_a + k * TILE_K);\n\n // wgmma: reads A from registers, B from SMEM descriptor\n // Accumulates into acc[] registers\n wgmma_m64n256k16(acc, a_frag, smem_b_desc);\n wgmma_commit();\n wgmma_wait();\n }\n\n // 4. Epilogue: acc is directly in registers -- fast access\n // But: if epilogue needs temporary storage, registers are exhausted\n #pragma unroll\n for (int i = 0; i < 4; ++i) {\n for (int j = 0; j < 32; ++j) {\n float val = acc[i][j];\n val += bias[col_base + j]; // bias add\n val = fmaxf(val, 0.0f); // ReLU\n // Need to convert and store -- no registers left for temp!\n C[row * N + col_base + j] = __float2half(val);\n }\n }\n\n // 5. Accumulator freed implicitly when CTA exits\n}\n```"}, "after": {"statement": "For Hopper `wgmma.mma_async.m64nNk16` with an FP32 accumulator, each warpgroup thread holds `N/2` FP32 D registers. At `N=256`, that is 128 registers per thread for D. WGMMA updates that register vector asynchronously; the program uses the WGMMA fence, commit-group, and wait-group mechanisms before dependent use.\n\n| Resident D | Per-thread register fragment | TMEM region addressed by `taddr` |\n| Example FP32 D cost | `N/2` registers/thread; 128 at `N=256` | No resident per-thread D vector |\n| First accumulation | Set WGMMA `scale-d` false to compute `D=A*B`, or initialize/use D when accumulation is intended | Set `enable-input-d` false to compute `D=A*B`, or explicitly initialize D when the algorithm needs another value |\n| Compute completion | WGMMA commit/wait group | `tcgen05.commit` to an mbarrier, then wait before a consumer reads/reuses D |\n| Epilogue access | Use the thread's D registers | Collective `tcgen05.ld`, then `tcgen05.wait::ld` before consuming result registers |\n| Cleanup | Register lifetime ends with the thread | Matching collective `tcgen05.dealloc` before kernel exit |\n| Double buffering | Two simultaneously live D fragments consume two register fragments | Two live outputs consume disjoint TMEM columns inside the allocation |\n\nThe table describes storage contracts, not an occupancy prediction. Instruction shape is also not necessarily the same as a composed CTA tile."}, "reason": {"statement": "Separate the PTX register-D contract from compiler allocation conclusions.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#local-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-multiply-accumulate-instructions-wgmma-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "```cuda\n// BLACKWELL: Accumulator lives in TMEM\n__global__ void blackwell_kernel(/* ... */) {\n // 1. Allocate TMEM (explicit, must be done once)\n __shared__ uint32_t s_tmem_acc;\n if (threadIdx.x == 0) {\n uint32_t smem_addr =\n static_cast(__cvta_generic_to_shared(&s_tmem_acc));\n asm volatile(\n \"tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;\"\n :: \"r\"(smem_addr), \"r\"(256)\n );\n }\n __syncthreads();\n uint32_t tmem_acc = s_tmem_acc;\n\n // 2. Zero-initialize TMEM\n for (int c = 0; c < 256; c += 4) {\n asm volatile(\n \"tcgen05.st.sync.aligned.32x1b.x4.b32 [%0], {%1,%2,%3,%4};\"\n : : \"r\"(tmem_acc + c),\n \"f\"(0.f), \"f\"(0.f), \"f\"(0.f), \"f\"(0.f)\n );\n }\n\n // 3. Mainloop: tcgen05 accumulates into TMEM\n for (int k = 0; k < K_tiles; ++k) {\n // NO ldmatrix -- tcgen05 reads directly from SMEM\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 1;\"\n : : \"r\"(tmem_acc), \"l\"(desc_a), \"l\"(desc_b), \"r\"(0)\n );\n }\n // NO commit/wait -- fully async\n }\n\n // 4. Fence before reading\n asm volatile(\"tcgen05.fence::before_thread_sync;\");\n __syncthreads();\n\n // 5. Epilogue: read from TMEM to registers (in small batches)\n // PLENTY of registers available for epilogue temporaries!\n for (int c = 0; c < 256; c += 4) {\n float4 vals;\n asm volatile(\n \"tcgen05.ld.sync.aligned.32x1b.x4.b32 {%0,%1,%2,%3}, [%4];\"\n : \"=f\"(vals.x), \"=f\"(vals.y), \"=f\"(vals.z), \"=f\"(vals.w)\n : \"r\"(tmem_acc + c)\n );\n\n // Apply epilogue ops with register headroom\n vals.x += bias[col_base + c];\n vals.y += bias[col_base + c + 1];\n vals.z += bias[col_base + c + 2];\n vals.w += bias[col_base + c + 3];\n\n vals.x = fmaxf(vals.x, 0.0f); // ReLU\n vals.y = fmaxf(vals.y, 0.0f);\n vals.z = fmaxf(vals.z, 0.0f);\n vals.w = fmaxf(vals.w, 0.0f);\n\n // Vectorized store -- registers available for conversion\n half2 h01 = __floats2half2_rn(vals.x, vals.y);\n half2 h23 = __floats2half2_rn(vals.z, vals.w);\n *reinterpret_cast(&C[row * N + col_base + c]) = h01;\n *reinterpret_cast(&C[row * N + col_base + c + 2]) = h23;\n }\n\n // 6. Deallocate TMEM (explicit, MUST be done in persistent kernels)\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;\"\n : : \"r\"(tmem_acc), \"r\"(256)\n );\n }\n}\n```"}, "after": {"statement": "1. Choose one CTA-group mode for the kernel. For allocation, `nCols` is a power of two in `[32, 512]`; allocations are column-granular and cover all 128 lanes.\n2. Have every lane of one designated warp execute the same `.cta_group::1` allocation. Synchronize before other threads read the 32-bit `taddr` written to shared memory. Two-CTA mode requires one warp in each live peer CTA.\n3. Initialize and publish the barriers used by TMA and tcgen05. If the first MMA should compute only `A*B`, supply a false `enable-input-d` predicate instead of assuming a mandatory TMEM zero-store pass.\n4. Issue MMA from the permitted elected thread with valid A/B descriptors, instruction descriptor, predicate, and lifetimes.\n5. Attach completion of the relevant tcgen05 work to an mbarrier with `tcgen05.commit`. A fence controls execution ordering but is not a completion wait."}, "reason": {"statement": "Replace the block with lifecycle invariants and known-good source routes.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "With accumulators in TMEM, the tile size is no longer constrained by register count:\n\n```cuda\n// Hopper: large tiles cause spilling\n// m64xn256 uses 128 acc registers/thread -> barely fits\n// m128xn256 would need 256 acc registers/thread -> spills guaranteed\n\n// Blackwell: tile size limited by TMEM columns (512) and SMEM, not registers\n// m128xn256: 256 TMEM cols, 0 acc registers -> fits easily\n// m128xn512: 512 TMEM cols, 0 acc registers -> uses full TMEM budget\n// m256xn256 (2-SM): 256 TMEM cols/SM, 0 acc registers -> fits with cooperative\n```"}, "after": {"statement": "For `sm_100a`, tcgen05 D is addressed in Tensor Memory rather than supplied as a per-thread register vector. The CTA-visible TMEM structure has 128 lanes by 512 columns of 32-bit cells, or 256 KiB when fully allocated. Allocation is dynamic in columns.\n\nThis removes the resident D vector from the ordinary register file. It does not remove all register pressure: descriptors, addresses, loop state, pipeline state, and each TMEM-to-register epilogue batch still need registers. Both Hopper and compute-capability-10.0 Blackwell have 64K 32-bit registers per SM and at most 255 registers per thread. Occupancy, spilling, and practical tile size therefore remain properties of the compiled kernel and launch configuration.\n\nTMEM makes it practical to keep multiple output regions independently addressable while different warp roles issue MMA and drain a completed region. Safe overlap needs three distinct proofs:\n\n- compute completion occurs before an epilogue loads a region;\n- all epilogue loads complete before that region is overwritten or deallocated;\n- producer/consumer state and phase cannot alias a different pipeline stage.\n\nTMEM is not the only way to overlap tensor-core and non-matmul work. FlashAttention-3 already uses warp specialization and matmul/softmax interleaving on Hopper with register accumulators. The migration benefit is the different storage and lifetime tradeoff, not a new theorem that overlap was previously impossible."}, "reason": {"statement": "Replace categorical tile advice with descriptor legality plus empirical tuning.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://arxiv.org/abs/2407.08608"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "On Hopper, complex epilogues (bias + activation + quantization + scaling) often spill because registers are already exhausted by the accumulator. On Blackwell, the epilogue has the full register file available:\n\n```cuda\n// Blackwell epilogue: full register file available\n__device__ void rich_epilogue(uint32_t tmem_acc, float* output,\n const float* bias, const float* scale,\n int M, int N) {\n // Read TMEM in chunks\n for (int c = 0; c < 256; c += 8) {\n // Load 8 values from TMEM (8 registers -- trivial)\n float v[8];\n tmem_load_f32x4(tmem_acc + c, &v[0]);\n tmem_load_f32x4(tmem_acc + c + 4, &v[4]);\n\n // Apply bias (8 registers for bias values)\n float b[8];\n load_global_f32x4(&b[0], bias + c);\n load_global_f32x4(&b[4], bias + c + 4);\n\n #pragma unroll\n for (int i = 0; i < 8; ++i) v[i] += b[i];\n\n // Apply GELU activation (needs temp registers for polynomial)\n #pragma unroll\n for (int i = 0; i < 8; ++i) v[i] = gelu(v[i]);\n\n // Apply per-channel scale\n float s[8];\n load_global_f32x4(&s[0], scale + c);\n load_global_f32x4(&s[4], scale + c + 4);\n\n #pragma unroll\n for (int i = 0; i < 8; ++i) v[i] *= s[i];\n\n // Quantize to FP8 for next layer\n uint8_t q[8];\n #pragma unroll\n for (int i = 0; i < 8; ++i) q[i] = float_to_e4m3(v[i]);\n\n // Store quantized output\n store_global_u8x8(output + c, q);\n\n // Total temp registers used: ~24 -- NO SPILLING\n // On Hopper this epilogue would need 24 + 128 (acc) = 152 per thread\n }\n}\n```"}, "after": {"statement": "For `sm_100a`, tcgen05 D is addressed in Tensor Memory rather than supplied as a per-thread register vector. The CTA-visible TMEM structure has 128 lanes by 512 columns of 32-bit cells, or 256 KiB when fully allocated. Allocation is dynamic in columns.\n\nThis removes the resident D vector from the ordinary register file. It does not remove all register pressure: descriptors, addresses, loop state, pipeline state, and each TMEM-to-register epilogue batch still need registers. Both Hopper and compute-capability-10.0 Blackwell have 64K 32-bit registers per SM and at most 255 registers per thread. Occupancy, spilling, and practical tile size therefore remain properties of the compiled kernel and launch configuration.\n\n3. Initialize and publish the barriers used by TMA and tcgen05. If the first MMA should compute only `A*B`, supply a false `enable-input-d` predicate instead of assuming a mandatory TMEM zero-store pass.\n4. Issue MMA from the permitted elected thread with valid A/B descriptors, instruction descriptor, predicate, and lifetimes.\n5. Attach completion of the relevant tcgen05 work to an mbarrier with `tcgen05.commit`. A fence controls execution ordering but is not a completion wait."}, "reason": {"statement": "Preserve the design opportunity without promising allocation or performance.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "The most powerful pattern enabled by TMEM: overlapping the current tile's epilogue with the next tile's MMA computation. This is impossible with register accumulators because the accumulator registers are still in use.\n\n```cuda\n__global__ void overlapped_gemm(/* ... */) {\n // Two TMEM accumulator buffers\n uint32_t tmem_a = tmem_alloc(256);\n uint32_t tmem_b = tmem_alloc(256);\n uint32_t* tmem_cur = &tmem_a;\n uint32_t* tmem_nxt = &tmem_b;\n\n // Compute first tile into tmem_a\n compute_tile(*tmem_cur, tile_0);\n fence_and_sync();\n\n for (int t = 1; t < num_tiles; ++t) {\n // START: next tile MMA into tmem_nxt (background, async)\n zero_tmem(*tmem_nxt);\n start_mma(*tmem_nxt, tile_t);\n\n // SIMULTANEOUSLY: epilogue of current tile from tmem_cur\n // MMA is running in the background while we read the old accumulator!\n epilogue_from_tmem(*tmem_cur, output_tile[t-1]);\n\n // Wait for MMA to complete\n fence_and_sync();\n\n // Swap buffers\n uint32_t* tmp = tmem_cur;\n tmem_cur = tmem_nxt;\n tmem_nxt = tmp;\n }\n\n // Final epilogue\n epilogue_from_tmem(*tmem_cur, output_tile[num_tiles-1]);\n\n tmem_dealloc(tmem_a, 256);\n tmem_dealloc(tmem_b, 256);\n}\n```"}, "after": {"statement": "6. After the completion handoff, participating epilogue lanes execute a legal `tcgen05.ld` shape and `tcgen05.wait::ld` before using the loaded registers.\n7. After all readers finish, every lane of the designated warp executes the matching deallocation on every kernel exit path. Allocation/deallocation address, column count, and CTA-group mode must agree."}, "reason": {"statement": "Describe overlap as a scheduling option requiring disjoint regions and explicit handoffs.", "urls": ["https://arxiv.org/abs/2407.08608", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "```\nFlashAttention on Hopper needs registers for:\n - QK^T accumulator: 64 registers (m64xn64 attention scores)\n - PV accumulator: 128 registers (m64xn256 output)\n - Softmax state: 4 registers (rowmax, rowsum)\n - Q fragment: 16 registers\n - K fragment: 16 registers\n - V fragment: 16 registers\n - Loop state: 10 registers\n ──────────────────────────────\n Total: ~254 registers per thread\n\n Available: 256 registers/thread at 1 CTA/SM occupancy\n Margin: 2 registers (!!)\n \n Result: compiler must spill, or tile sizes must shrink\n```\n\n```\nFlashAttention on Blackwell:\n - QK^T accumulator: 0 registers (TMEM buffer 1, 64 cols)\n - PV accumulator: 0 registers (TMEM buffer 2, 256 cols)\n - Softmax state: 4 registers (rowmax, rowsum)\n - SMEM descriptors: 6 registers\n - Loop state: 10 registers\n ──────────────────────────────\n Total: ~20 registers per thread\n\n Available: 256 registers/thread\n Margin: 236 registers -- massive headroom\n\n Result: Can use for ping-pong scheduling, software exp emulation,\n larger tiles, more pipeline stages\n```"}, "after": {"statement": "Representative PTX ISA 9.0 forms are shown below. They omit declarations, descriptor construction, collective control flow, barriers, and inline-assembly constraints, so they are not a standalone kernel.\n\n```ptx\ntcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [saddr], nCols;\ntcgen05.mma.cta_group::1.kind::f16 [taddr], a_desc, b_desc, idesc, p;\ntcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cta.b64 [mbar];\ntcgen05.ld.sync.aligned.32x32b.x1.b32 {r0}, [taddr];\ntcgen05.wait::ld.sync.aligned;\ntcgen05.dealloc.cta_group::1.sync.aligned.b32 taddr, nCols;\n```"}, "reason": {"statement": "Replace invented budgets with the paper's directly reported TMEM roles.", "urls": ["https://arxiv.org/abs/2603.05451v1", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "```cuda\n// WRONG: Fine-grained TMEM access in a tight loop (high latency)\nfor (int i = 0; i < 256; ++i) {\n float v = tmem_load_f32(tmem_acc + i); // 420 cycle latency each!\n output[i] = v;\n}\n\n// CORRECT: Vectorized loads to amortize latency\nfor (int i = 0; i < 256; i += 4) {\n float4 v = tmem_load_f32x4(tmem_acc + i); // Single 420-cycle access\n output[i] = v.x;\n output[i+1] = v.y;\n output[i+2] = v.z;\n output[i+3] = v.w;\n}\n```"}, "after": {"statement": "3. Initialize and publish the barriers used by TMA and tcgen05. If the first MMA should compute only `A*B`, supply a false `enable-input-d` predicate instead of assuming a mandatory TMEM zero-store pass.\n4. Issue MMA from the permitted elected thread with valid A/B descriptors, instruction descriptor, predicate, and lifetimes.\n5. Attach completion of the relevant tcgen05 work to an mbarrier with `tcgen05.commit`. A fence controls execution ordering but is not a completion wait.\n\nTMEM makes it practical to keep multiple output regions independently addressable while different warp roles issue MMA and drain a completed region. Safe overlap needs three distinct proofs:\n\n- compute completion occurs before an epilogue loads a region;\n- all epilogue loads complete before that region is overwritten or deallocated;\n- producer/consumer state and phase cannot alias a different pipeline stage.\n\nTMEM is not the only way to overlap tensor-core and non-matmul work. FlashAttention-3 already uses warp specialization and matmul/softmax interleaving on Hopper with register accumulators. The migration benefit is the different storage and lifetime tradeoff, not a new theorem that overlap was previously impossible."}, "reason": {"statement": "Keep batching as a profiling hypothesis, not a latency theorem.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://arxiv.org/abs/2512.02189", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://arxiv.org/abs/2407.08608"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "```cuda\n// Hopper: tile size was m64xn128 to avoid register spilling\n// Migrating to Blackwell: keep same tile = leaving performance on the table\n\n// Blackwell should use at minimum m128xn256 (1-SM) or m256xn256 (2-SM)\n// The freed registers allow larger tiles without any spilling risk\n```"}, "after": {"statement": "TMEM makes it practical to keep multiple output regions independently addressable while different warp roles issue MMA and drain a completed region. Safe overlap needs three distinct proofs:\n\n- compute completion occurs before an epilogue loads a region;\n- all epilogue loads complete before that region is overwritten or deallocated;\n- producer/consumer state and phase cannot alias a different pipeline stage.\n\nTMEM is not the only way to overlap tensor-core and non-matmul work. FlashAttention-3 already uses warp specialization and matmul/softmax interleaving on Hopper with register accumulators. The migration benefit is the different storage and lifetime tradeoff, not a new theorem that overlap was previously impossible."}, "reason": {"statement": "Turn the mandate into an evidence-driven tuning checklist.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://arxiv.org/abs/2407.08608"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "```cuda\n// WRONG: Forgetting to dealloc in a persistent kernel\n__global__ void persistent_kernel(/* ... */) {\n uint32_t tmem = tmem_alloc(256); // Allocated once\n\n while (has_work()) {\n compute_tile(tmem);\n // BUG: If the kernel exits early (e.g., error path),\n // TMEM is leaked. Next CTA on this SM cannot allocate.\n }\n // Missing: tmem_dealloc(tmem, 256);\n}\n\n// CORRECT: Always dealloc, even on error paths\n__global__ void persistent_kernel(/* ... */) {\n uint32_t tmem = tmem_alloc(256);\n\n while (has_work()) {\n compute_tile(tmem);\n }\n\n tmem_dealloc(tmem, 256); // Always reached\n}\n```"}, "after": {"statement": "1. Choose one CTA-group mode for the kernel. For allocation, `nCols` is a power of two in `[32, 512]`; allocations are column-granular and cover all 128 lanes.\n2. Have every lane of one designated warp execute the same `.cta_group::1` allocation. Synchronize before other threads read the 32-bit `taddr` written to shared memory. Two-CTA mode requires one warp in each live peer CTA.\n\nTMEM makes it practical to keep multiple output regions independently addressable while different warp roles issue MMA and drain a completed region. Safe overlap needs three distinct proofs:\n\n- compute completion occurs before an epilogue loads a region;\n- all epilogue loads complete before that region is overwritten or deallocated;\n- producer/consumer state and phase cannot alias a different pipeline stage.\n\nTMEM is not the only way to overlap tensor-core and non-matmul work. FlashAttention-3 already uses warp specialization and matmul/softmax interleaving on Hopper with register accumulators. The migration benefit is the different storage and lifetime tradeoff, not a new theorem that overlap was previously impossible."}, "reason": {"statement": "State the normative obligation and control-flow requirements.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://arxiv.org/abs/2407.08608"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "| Zero-initialization | Loop over register array | `tcgen05.st` zero pattern |"}, "after": {"statement": "3. Initialize and publish the barriers used by TMA and tcgen05. If the first MMA should compute only `A*B`, supply a false `enable-input-d` predicate instead of assuming a mandatory TMEM zero-store pass.\n4. Issue MMA from the permitted elected thread with valid A/B descriptors, instruction descriptor, predicate, and lifetimes.\n5. Attach completion of the relevant tcgen05 work to an mbarrier with `tcgen05.commit`. A fence controls execution ordering but is not a completion wait."}, "reason": {"statement": "Describe both initialization choices without prescribing a needless store.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "| Epilogue headroom | Minimal (spill risk) | Ample (full register file) |"}, "after": {"statement": "For `sm_100a`, tcgen05 D is addressed in Tensor Memory rather than supplied as a per-thread register vector. The CTA-visible TMEM structure has 128 lanes by 512 columns of 32-bit cells, or 256 KiB when fully allocated. Allocation is dynamic in columns.\n\nThis removes the resident D vector from the ordinary register file. It does not remove all register pressure: descriptors, addresses, loop state, pipeline state, and each TMEM-to-register epilogue batch still need registers. Both Hopper and compute-capability-10.0 Blackwell have 64K 32-bit registers per SM and at most 255 registers per thread. Occupancy, spilling, and practical tile size therefore remain properties of the compiled kernel and launch configuration.\n\nTMEM makes it practical to keep multiple output regions independently addressable while different warp roles issue MMA and drain a completed region. Safe overlap needs three distinct proofs:\n\n- compute completion occurs before an epilogue loads a region;\n- all epilogue loads complete before that region is overwritten or deallocated;\n- producer/consumer state and phase cannot alias a different pipeline stage.\n\nTMEM is not the only way to overlap tensor-core and non-matmul work. FlashAttention-3 already uses warp specialization and matmul/softmax interleaving on Hopper with register accumulators. The migration benefit is the different storage and lifetime tradeoff, not a new theorem that overlap was previously impossible."}, "reason": {"statement": "State reduced resident-D pressure as an opportunity, not a guarantee.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://arxiv.org/abs/2407.08608"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "| Max practical tile | m64xn256 (limited by registers) | m128xn256 or m256xn256 (limited by TMEM cols) |"}, "after": {"statement": "For Hopper `wgmma.mma_async.m64nNk16` with an FP32 accumulator, each warpgroup thread holds `N/2` FP32 D registers. At `N=256`, that is 128 registers per thread for D. WGMMA updates that register vector asynchronously; the program uses the WGMMA fence, commit-group, and wait-group mechanisms before dependent use.\n\nTMEM makes it practical to keep multiple output regions independently addressable while different warp roles issue MMA and drain a completed region. Safe overlap needs three distinct proofs:\n\n- compute completion occurs before an epilogue loads a region;\n- all epilogue loads complete before that region is overwritten or deallocated;\n- producer/consumer state and phase cannot alias a different pipeline stage.\n\nTMEM is not the only way to overlap tensor-core and non-matmul work. FlashAttention-3 already uses warp specialization and matmul/softmax interleaving on Hopper with register accumulators. The migration benefit is the different storage and lifetime tradeoff, not a new theorem that overlap was previously impossible."}, "reason": {"statement": "Separate legal instruction shapes from empirical CTA tile tuning.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-multiply-accumulate-instructions-wgmma-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://arxiv.org/abs/2407.08608"]}} +{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "```cuda\n// TMEM double-buffering: ping-pong between two 128x256 accumulator regions\n//\n// TMEM physical layout (per SM):\n// +------ 256 cols ------+------ 256 cols ------+\n// | | |\n// | Buffer A (active) | Buffer B (drain) | 128 rows\n// | MMA writes here | Epilogue reads |\n// | | |\n// +----------------------+----------------------+\n//\n// After tile completes, roles swap:\n// Buffer A becomes \"drain\", Buffer B becomes \"active\"\n\n#define TMEM_BUF_A_OFFSET 0\n#define TMEM_BUF_B_OFFSET 256\n#define TMEM_COLS_PER_BUF 256\n\n__device__ void tmem_double_buffer_mainloop(\n const GemmParams& params,\n int num_output_tiles)\n{\n int warp_id = threadIdx.x / 32;\n int lane_id = threadIdx.x % 32;\n\n // Synchronization between MMA warp and epilogue warps\n __shared__ uint64_t mbar_acc_ready[2]; // MMA done, epilogue can read\n __shared__ uint64_t mbar_acc_drained[2]; // Epilogue done, MMA can reuse\n\n if (threadIdx.x == 0) {\n mbarrier_init(&mbar_acc_ready[0], 1);\n mbarrier_init(&mbar_acc_ready[1], 1);\n mbarrier_init(&mbar_acc_drained[0], 1);\n mbarrier_init(&mbar_acc_drained[1], 1);\n }\n __syncthreads();\n\n for (int tile = 0; tile < num_output_tiles; tile++) {\n int buf = tile % 2;\n int tmem_offset = buf ? TMEM_BUF_B_OFFSET : TMEM_BUF_A_OFFSET;\n\n if (warp_id == 1) {\n // === MMA WARP ===\n // Wait for epilogue to finish draining this buffer\n if (tile >= 2) {\n mbarrier_wait(&mbar_acc_drained[buf]);\n }\n\n // Clear accumulator region before new tile\n if (lane_id == 0) {\n tmem_clear(tmem_offset, TMEM_COLS_PER_BUF);\n }\n __syncwarp();\n\n // Accumulate K-tiles into this TMEM buffer\n for (int k = 0; k < num_k_tiles; k++) {\n // (TMA load sync omitted -- see pipeline-stages)\n if (lane_id == 0) {\n tcgen05_mma_accumulate(tmem_offset);\n }\n __syncwarp();\n }\n\n // Signal epilogue that accumulator is ready\n if (lane_id == 0) {\n mbarrier_arrive(&mbar_acc_ready[buf]);\n }\n\n } else if (warp_id >= 2) {\n // === EPILOGUE WARPS ===\n // Wait for MMA to finish accumulating\n mbarrier_wait(&mbar_acc_ready[buf]);\n\n // Read TMEM and store to global memory\n int rows_per_warp = TILE_M / 14;\n int my_row = (warp_id - 2) * rows_per_warp;\n for (int r = my_row; r < my_row + rows_per_warp; r++) {\n for (int c = lane_id; c < TMEM_COLS_PER_BUF; c += 32) {\n float val = tmem_load_f32(r, tmem_offset + c);\n // Apply epilogue and store...\n params.C_ptr[r * params.N + c] = __float2half(val);\n }\n }\n\n // All epilogue warps must finish reading TMEM before the MMA\n // warp can overwrite this half-buffer. Each epilogue warp arrives\n // on a shared mbarrier; the mbarrier is initialized with\n // arrival_count = NUM_EPILOGUE_WARPS (e.g. 14 for warps 2-15).\n // The MMA warp waits on this mbarrier before reusing the buffer.\n if (lane_id == 0) {\n mbarrier_arrive(&mbar_acc_drained[buf]);\n }\n // mbar_acc_drained[buf] fires only after ALL epilogue warps arrive\n }\n }\n}\n```"}, "after": {"statement": "The CTA-visible TMEM address structure has 128 lanes and 512 columns of 32-bit cells. Allocation is column-granular across all 128 lanes. A kernel that allocates 512 columns may choose two 256-column regions:\n\n```text\nregion[0] = columns [0, 256)\nregion[1] = columns [256, 512)\n```\n\nThis equal split is one implementation choice. The actual invariant is that every simultaneously live region is inside the allocation and does not alias another region whose MMA or epilogue access is still outstanding. Region sizes can differ, and logical element capacity depends on the tcgen05 instruction kind and packing rather than the raw cell count alone.\n\nFor each region, prove this lifecycle:\n\n| free → MMA-owned | every prior epilogue load from the region has completed |\n| MMA-owned → ready | relevant tcgen05 work is committed to an mbarrier and completion is observed |\n| ready → epilogue-owned | readers observe the matching barrier phase before `tcgen05.ld` |\n| epilogue-owned → free | all collective loads complete with `tcgen05.wait::ld`, then every reader reports completion |\n\nTMEM must first be allocated by the required participating warp and its address safely published. Matching collective deallocation is required before every kernel exit. Reused mbarriers need the correct arrival count and phase/parity state; a one-time wait on an address is not a reusable two-buffer protocol."}, "reason": {"statement": "Replace unsafe pseudo-implementation with verified ownership invariants and a pinned complete implementation route.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} +{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "A fully optimized Blackwell GEMM uses both levels simultaneously:"}, "after": {"statement": "Double- or multi-buffering reserves disjoint storage regions and transfers ownership between producers and consumers. While a consumer reads stage `s`, a producer may fill another stage. The storage alone does not establish overlap: the program also needs a completion edge before consumption and a read-complete edge before reuse.\n\nOn SM100 these ideas can be applied independently:\n\n- SMEM stages can hold operand tiles while TMA production overlaps MMA consumption.\n- Separate TMEM regions can hold output accumulators while an epilogue drains a completed region and MMA writes a different region.\n\nThe pinned `matmul_v6.cu` from Gau Nernst's tutorial uses both mechanisms. That is a concrete design, not a rule that every optimized Blackwell GEMM needs both.\n\nThat is not a complete C++ shared-storage layout. Barriers, descriptors, epilogue scratch, padding, alignment, and swizzled physical layouts also consume or constrain shared memory. Compute capability 10.0 supports up to 228 KiB of shared memory per SM, while per-block opt-in limits and all other residency resources still apply."}, "reason": {"statement": "Scope the combination to concrete kernels and tuning conditions.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"]}} +{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "```cuda\n// Combined double-buffering: SMEM (3-stage) + TMEM (2-buffer)\n//\n// Outer loop: output tiles (TMEM double-buffered)\n// Inner loop: K-tiles (SMEM 3-stage pipelined)\n//\n// Timeline for 2 output tiles, 6 K-tiles each:\n//\n// TMEM buf: |---- A (tile 0) ----|---- B (tile 1) ----|\n// MMA warp: | k0 k1 k2 k3 k4 k5 | k0 k1 k2 ...\n// TMA warp: |k2 k3 k4 k5 - - k2 k3 k4 k5 ...\n// Epilogue: | idle | drain A | drain B |\n// SMEM stage: |0 1 2 0 1 2 |0 1 2 0 1 2 |\n//\n// Key insight: the epilogue of tile 0 overlaps with the MMA of tile 1.\n// This is only possible because they use different TMEM buffers.\n\n__device__ void full_double_buffered_gemm(const GemmParams& params) {\n int warp_id = threadIdx.x / 32;\n\n for (int out_tile = 0; out_tile < num_output_tiles; out_tile++) {\n int tmem_buf = out_tile % 2;\n\n if (warp_id == 0) {\n // TMA producer: fill SMEM stages for this output tile's K-loop\n tma_producer_loop(params, out_tile);\n } else if (warp_id == 1) {\n // MMA: consume SMEM stages, accumulate into TMEM[tmem_buf]\n mma_consumer_loop(params, tmem_buf);\n } else {\n // Epilogue: drain TMEM[1-tmem_buf] from previous tile\n if (out_tile > 0) {\n epilogue_drain(params, out_tile - 1, 1 - tmem_buf);\n }\n }\n\n // Lightweight sync point between tiles\n // (mbarrier-based, not __syncthreads)\n }\n\n // Final epilogue for last tile\n if (warp_id >= 2) {\n epilogue_drain(params, num_output_tiles - 1,\n (num_output_tiles - 1) % 2);\n }\n}\n```"}, "after": {"statement": "For each region, prove this lifecycle:\n\n| free → MMA-owned | every prior epilogue load from the region has completed |\n| MMA-owned → ready | relevant tcgen05 work is committed to an mbarrier and completion is observed |\n| ready → epilogue-owned | readers observe the matching barrier phase before `tcgen05.ld` |\n| epilogue-owned → free | all collective loads complete with `tcgen05.wait::ld`, then every reader reports completion |\n\nTMEM must first be allocated by the required participating warp and its address safely published. Matching collective deallocation is required before every kernel exit. Reused mbarriers need the correct arrival count and phase/parity state; a one-time wait on an address is not a reusable two-buffer protocol.\n\nAn SMEM operand stage normally has its own full/empty state:\n\n1. The producer waits until stage `s` is empty.\n2. It issues the stage's TMA copies and accounts for expected transaction bytes.\n3. The consumer observes the full barrier's matching phase before using the stage.\n4. After the final dependent read, the consumer releases the empty barrier for reuse.\n\nFlashInfer PR 2387 is a second pinned example. Its merged `selective_state_update.cuh` has `state[numStages][...]` plus `bar_full` and `bar_empty` arrays and uses stage-specific producer/consumer handoffs. One path selects three stages and another caps a geometry-derived count at four, illustrating why stage count is policy rather than an architecture-wide default."}, "reason": {"statement": "Use a state/ownership table instead of incomplete CUDA.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"]}} +{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "```cuda\n// Hopper: accumulator double-buffering uses register arrays\n// Each warpgroup maintains two register-based accumulators\n// This doubles register pressure and reduces occupancy\n\n// Hopper approach (register pressure is the primary constraint):\nfloat acc_buf0[REG_TILE_M][REG_TILE_N]; // First accumulator\nfloat acc_buf1[REG_TILE_M][REG_TILE_N]; // Second accumulator\n// Total: 2 * REG_TILE_M * REG_TILE_N * 4 bytes per thread\n// For a 64x256 tile with 128-thread warpgroup:\n// each thread holds 2 * (64/4) * (256/128) * 4 = 2 * 16 * 2 * 4 = 256 bytes\n// = 64 registers just for accumulators\n\n// Blackwell approach: TMEM holds both buffers with zero register cost\n// MMA warp uses ~0 registers for accumulators\n// All 256 KB of TMEM is dedicated accumulator space\n```"}, "after": {"statement": "For binary16 `A[128,64]` and `B[64,256]`, the unpadded payload arithmetic is:\n\n```text\nA = 128 × 64 × 2 bytes = 16 KiB\nB = 64 × 256 × 2 bytes = 32 KiB\none stage = 48 KiB; three stages = 144 KiB\n```"}, "reason": {"statement": "Replace incorrect arithmetic and reject zero total-register cost.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"]}} +{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "| Double-buffer cost | 2x register usage | Zero register cost |"}, "after": {"statement": "For binary16 `A[128,64]` and `B[64,256]`, the unpadded payload arithmetic is:\n\n```text\nA = 128 × 64 × 2 bytes = 16 KiB\nB = 64 × 256 × 2 bytes = 32 KiB\none stage = 48 KiB; three stages = 144 KiB\n```"}, "reason": {"statement": "Narrow the comparison to resident accumulator storage.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"]}} +{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "| Typical occupancy impact | Reduces from 2 to 1 CTA/SM | No impact |"}, "after": {"statement": "For Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, one D fragment is 128 registers per thread; two simultaneously live fragments are 256 registers, or 1024 bytes, per thread before other live values. That arithmetic describes the ISA fragment, not a promise that a C++ array stays in registers or that a particular launch is resident.\n\nOn SM100, tcgen05 keeps resident D in TMEM. This avoids a long-lived per-thread D vector, but the kernel still uses registers for addresses, descriptors, loop and barrier state, `tcgen05.ld` destinations, and epilogue temporaries. TMEM buffering also consumes TMEM capacity and synchronization state. Neither architecture therefore has a fixed occupancy result from “double buffering” alone.\n\n| Resident storage | per-thread register fragment | allocated TMEM region |\n| Two live outputs | two disjoint register fragments | two disjoint TMEM regions |\n| Epilogue access | dependent use after WGMMA completion | collective `tcgen05.ld` and `tcgen05.wait::ld` after MMA completion |\n| Main resource question | compiled registers, SMEM, block shape, and other limits | TMEM columns plus compiled registers, SMEM, block/cluster shape, and other limits |"}, "reason": {"statement": "Unsupported occupancy numbers cannot guide a real launch.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"]}} +{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "| Max accumulator size | ~16K elements (register limited) | 128x512 = 65K elements |"}, "after": {"statement": "For binary16 `A[128,64]` and `B[64,256]`, the unpadded payload arithmetic is:\n\n```text\nA = 128 × 64 × 2 bytes = 16 KiB\nB = 64 × 256 × 2 bytes = 32 KiB\none stage = 48 KiB; three stages = 144 KiB\n```\n\nFor Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, one D fragment is 128 registers per thread; two simultaneously live fragments are 256 registers, or 1024 bytes, per thread before other live values. That arithmetic describes the ISA fragment, not a promise that a C++ array stays in registers or that a particular launch is resident.\n\nOn SM100, tcgen05 keeps resident D in TMEM. This avoids a long-lived per-thread D vector, but the kernel still uses registers for addresses, descriptors, loop and barrier state, `tcgen05.ld` destinations, and epilogue temporaries. TMEM buffering also consumes TMEM capacity and synchronization state. Neither architecture therefore has a fixed occupancy result from “double buffering” alone.\n\n| Resident storage | per-thread register fragment | allocated TMEM region |\n| Two live outputs | two disjoint register fragments | two disjoint TMEM regions |\n| Epilogue access | dependent use after WGMMA completion | collective `tcgen05.ld` and `tcgen05.wait::ld` after MMA completion |\n| Main resource question | compiled registers, SMEM, block shape, and other limits | TMEM columns plus compiled registers, SMEM, block/cluster shape, and other limits |"}, "reason": {"statement": "Avoid conflating an instruction fragment or storage geometry with a practical kernel maximum.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-multiply-accumulate-instructions-wgmma-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} +{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "- **TMEM double-buffering**: Always use on Blackwell when the epilogue takes more than trivial time. The only cost is the TMEM space, which is plentiful."}, "after": {"statement": "For Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, one D fragment is 128 registers per thread; two simultaneously live fragments are 256 registers, or 1024 bytes, per thread before other live values. That arithmetic describes the ISA fragment, not a promise that a C++ array stays in registers or that a particular launch is resident.\n\nOn SM100, tcgen05 keeps resident D in TMEM. This avoids a long-lived per-thread D vector, but the kernel still uses registers for addresses, descriptors, loop and barrier state, `tcgen05.ld` destinations, and epilogue temporaries. TMEM buffering also consumes TMEM capacity and synchronization state. Neither architecture therefore has a fixed occupancy result from “double buffering” alone.\n\n| Resident storage | per-thread register fragment | allocated TMEM region |\n| Two live outputs | two disjoint register fragments | two disjoint TMEM regions |\n| Epilogue access | dependent use after WGMMA completion | collective `tcgen05.ld` and `tcgen05.wait::ld` after MMA completion |\n| Main resource question | compiled registers, SMEM, block shape, and other limits | TMEM columns plus compiled registers, SMEM, block/cluster shape, and other limits |"}, "reason": {"statement": "Make adoption conditional on measured overlap benefit and resource fit.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} +{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "- **SMEM multi-stage buffering**: Always use for GEMM/attention mainloops. 3 stages is the default; increase to 4-5 only if the K-loop is long and memory latency is high."}, "after": {"statement": "That is not a complete C++ shared-storage layout. Barriers, descriptors, epilogue scratch, padding, alignment, and swizzled physical layouts also consume or constrain shared memory. Compute capability 10.0 supports up to 228 KiB of shared memory per SM, while per-block opt-in limits and all other residency resources still apply.\n\nFor Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, one D fragment is 128 registers per thread; two simultaneously live fragments are 256 registers, or 1024 bytes, per thread before other live values. That arithmetic describes the ISA fragment, not a promise that a C++ array stays in registers or that a particular launch is resident.\n\nOn SM100, tcgen05 keeps resident D in TMEM. This avoids a long-lived per-thread D vector, but the kernel still uses registers for addresses, descriptors, loop and barrier state, `tcgen05.ld` destinations, and epilogue temporaries. TMEM buffering also consumes TMEM capacity and synchronization state. Neither architecture therefore has a fixed occupancy result from “double buffering” alone.\n\n| Resident storage | per-thread register fragment | allocated TMEM region |\n| Two live outputs | two disjoint register fragments | two disjoint TMEM regions |\n| Epilogue access | dependent use after WGMMA completion | collective `tcgen05.ld` and `tcgen05.wait::ld` after MMA completion |\n| Main resource question | compiled registers, SMEM, block shape, and other limits | TMEM columns plus compiled registers, SMEM, block/cluster shape, and other limits |"}, "reason": {"statement": "Replace a universal recipe with resource- and workload-dependent tuning.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} +{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "- **Combined**: The standard approach for production Blackwell GEMM kernels. Both CUTLASS and CuTe-DSL kernels use this pattern."}, "after": {"statement": "That is not a complete C++ shared-storage layout. Barriers, descriptors, epilogue scratch, padding, alignment, and swizzled physical layouts also consume or constrain shared memory. Compute capability 10.0 supports up to 228 KiB of shared memory per SM, while per-block opt-in limits and all other residency resources still apply."}, "reason": {"statement": "Use concrete, pinned examples rather than ecosystem-wide attribution.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"]}} +{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "- TMEM double-buffering requires that the accumulator fits in half the TMEM columns (256 out of 512). For very wide tiles (TILE_N > 256 with FP32 accumulators), the tile must be split or a different buffering strategy used."}, "after": {"statement": "The CTA-visible TMEM address structure has 128 lanes and 512 columns of 32-bit cells. Allocation is column-granular across all 128 lanes. A kernel that allocates 512 columns may choose two 256-column regions:\n\n```text\nregion[0] = columns [0, 256)\nregion[1] = columns [256, 512)\n```\n\nThis equal split is one implementation choice. The actual invariant is that every simultaneously live region is inside the allocation and does not alias another region whose MMA or epilogue access is still outstanding. Region sizes can differ, and logical element capacity depends on the tcgen05 instruction kind and packing rather than the raw cell count alone.\n\nFor Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, one D fragment is 128 registers per thread; two simultaneously live fragments are 256 registers, or 1024 bytes, per thread before other live values. That arithmetic describes the ISA fragment, not a promise that a C++ array stays in registers or that a particular launch is resident.\n\nOn SM100, tcgen05 keeps resident D in TMEM. This avoids a long-lived per-thread D vector, but the kernel still uses registers for addresses, descriptors, loop and barrier state, `tcgen05.ld` destinations, and epilogue temporaries. TMEM buffering also consumes TMEM capacity and synchronization state. Neither architecture therefore has a fixed occupancy result from “double buffering” alone.\n\n| Resident storage | per-thread register fragment | allocated TMEM region |\n| Two live outputs | two disjoint register fragments | two disjoint TMEM regions |\n| Epilogue access | dependent use after WGMMA completion | collective `tcgen05.ld` and `tcgen05.wait::ld` after MMA completion |\n| Main resource question | compiled registers, SMEM, block shape, and other limits | TMEM columns plus compiled registers, SMEM, block/cluster shape, and other limits |"}, "reason": {"statement": "State the actual nonaliasing/capacity invariant and label 256/256 as one choice.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory-allocation", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} +{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "- The mbarrier synchronization between TMEM buffers adds a few cycles of overhead per tile. For kernels with very few K-iterations per tile, this overhead is proportionally larger."}, "after": {"statement": "For Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, one D fragment is 128 registers per thread; two simultaneously live fragments are 256 registers, or 1024 bytes, per thread before other live values. That arithmetic describes the ISA fragment, not a promise that a C++ array stays in registers or that a particular launch is resident.\n\nOn SM100, tcgen05 keeps resident D in TMEM. This avoids a long-lived per-thread D vector, but the kernel still uses registers for addresses, descriptors, loop and barrier state, `tcgen05.ld` destinations, and epilogue temporaries. TMEM buffering also consumes TMEM capacity and synchronization state. Neither architecture therefore has a fixed occupancy result from “double buffering” alone.\n\n| Resident storage | per-thread register fragment | allocated TMEM region |\n| Two live outputs | two disjoint register fragments | two disjoint TMEM regions |\n| Epilogue access | dependent use after WGMMA completion | collective `tcgen05.ld` and `tcgen05.wait::ld` after MMA completion |\n| Main resource question | compiled registers, SMEM, block shape, and other limits | TMEM columns plus compiled registers, SMEM, block/cluster shape, and other limits |"}, "reason": {"statement": "Retain the need to measure synchronization overhead without inventing latency.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} +{"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "Epilogue fusion overlaps the post-MMA operations (scaling, bias addition, activation functions, quantization, store to global memory) with ongoing MMA computation. On Blackwell, the accumulator lives in TMEM rather than registers, enabling dedicated epilogue warps (typically warps 2-15) to read TMEM concurrently while the MMA warp (warp 1) continues accumulating the next tile. This overlap is achieved by double-buffering the TMEM accumulator: the MMA warp writes to one half while the epilogue warps read from the other half."}, "after": {"statement": "Epilogue fusion computes output transformations—such as `alpha*acc + beta*C`, bias, activation, output conversion, or auxiliary values—inside the producer kernel before final output materialization. It can remove an intermediate tensor or launch only when the unfused comparison would otherwise materialize or launch that work.\n\nFusion does not by itself imply overlap with the next MMA tile. A schedule may drain one completed accumulator after the mainloop, or it may use disjoint storage and specialized roles to overlap a completed region's epilogue with independent MMA work. Participant counts and role IDs come from the concrete kernel; there is no architectural default of fourteen epilogue warps."}, "reason": {"statement": "Separate semantic fusion from optional schedule overlap.", "urls": []}} +{"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "```cuda\n// Epilogue warp: read TMEM accumulator, apply fused operations, store\n// This runs on warps 2-15 while warp 1 continues MMA on next tile\n__device__ void epilogue_warp_fn(\n int warp_id,\n int tile_m, int tile_n,\n float scale, const float* bias,\n half* C, int ldc)\n{\n int lane_id = threadIdx.x % 32;\n\n // Each epilogue warp handles a stripe of the output tile\n // 14 warps, TILE_M = 128 -> ~9 rows per warp\n int rows_per_warp = (TILE_M + 13) / 14;\n int row_start = (warp_id - 2) * rows_per_warp;\n int row_end = min(row_start + rows_per_warp, TILE_M);\n\n for (int r = row_start; r < row_end; r++) {\n int global_row = tile_m * TILE_M + r;\n\n for (int c = lane_id; c < TILE_N; c += 32) {\n // Step 1: Load accumulator from TMEM into register\n float acc = tmem_load_f32(r, c);\n\n // Step 2: Fused epilogue operations (all in registers)\n // Scale\n acc *= scale;\n // Bias\n acc += bias[tile_n * TILE_N + c];\n // ReLU activation\n acc = fmaxf(acc, 0.0f);\n\n // Step 3: Store to global memory\n int global_col = tile_n * TILE_N + c;\n C[global_row * ldc + global_col] = __float2half(acc);\n }\n }\n}\n```"}, "after": {"statement": "For tcgen05, D resides in TMEM. An ordinary arithmetic epilogue first uses a legal collective `tcgen05.ld` mapping, or a library wrapper around it, to transfer a partition into per-thread registers. The load is asynchronous with respect to the issuing thread, so the matching `tcgen05.wait::ld` completion must occur before those registers are consumed.\n\nCUTLASS 4.5.0's `fp16_gemm_2.py` demonstrates the typed route:\n\n```python\ncopy_atom_t2r = cute.make_copy_atom(\n tcgen05.Ld32x32bOp(tcgen05.Repetition.x32), cutlass.Float32\n)\ntiled_copy_t2r = tcgen05.make_tmem_copy(copy_atom_t2r, tCtAcc_epi)\nthr_copy_t2r = tiled_copy_t2r.get_slice(tidx)\ntTR_tAcc = thr_copy_t2r.partition_S(tCtAcc_epi)\ntTR_rAcc = cute.make_rmem_tensor(..., cutlass.Float32)\ncute.copy(tiled_copy_t2r, tTR_tAcc_slice, tTR_rAcc)\n```\n\nThis is an API-routing fragment, not standalone code: the official file supplies the exact tensor layouts, selected load shape, participant group, edge predicates, accumulator-completion handoff, register-to-SMEM conversion, TMA-store pipeline, tail, and TMEM cleanup.\n\n| tcgen05 MMA → TMEM reader | the relevant asynchronous MMA is committed and its completion observed |\n| TMEM → result registers | the legal collective load completes before dependent arithmetic |\n| registers → output | element/layout mapping and output-edge predicates cover exactly the valid coordinates |\n| TMEM reader → region reuse | every reader's load has completed and the matching reusable barrier phase is released |\n| kernel tail | all output stores complete as required and every TMEM allocation is collectively freed |\n\nFor a multi-region overlap schedule, simultaneously live TMEM regions must also be disjoint. Equal 256-column halves are one possible policy for a 512-column allocation, not an epilogue-fusion requirement. Reusable mbarriers need correct expected-arrival counts and phase/parity tracking."}, "reason": {"statement": "Replace pseudo-CUDA with exact lifecycle and official CuTe copy-route identifiers.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"]}} +{"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "The key to epilogue fusion on Blackwell is TMEM double-buffering. The 512-column TMEM space is split into two halves (columns 0-255 and 256-511). While the MMA warp accumulates into one half, the epilogue warps drain the other:"}, "after": {"statement": "Epilogue fusion computes output transformations—such as `alpha*acc + beta*C`, bias, activation, output conversion, or auxiliary values—inside the producer kernel before final output materialization. It can remove an intermediate tensor or launch only when the unfused comparison would otherwise materialize or launch that work.\n\nFusion does not by itself imply overlap with the next MMA tile. A schedule may drain one completed accumulator after the mainloop, or it may use disjoint storage and specialized roles to overlap a completed region's epilogue with independent MMA work. Participant counts and role IDs come from the concrete kernel; there is no architectural default of fourteen epilogue warps.\n\n| tcgen05 MMA → TMEM reader | the relevant asynchronous MMA is committed and its completion observed |\n| TMEM → result registers | the legal collective load completes before dependent arithmetic |\n| registers → output | element/layout mapping and output-edge predicates cover exactly the valid coordinates |\n| TMEM reader → region reuse | every reader's load has completed and the matching reusable barrier phase is released |\n| kernel tail | all output stores complete as required and every TMEM allocation is collectively freed |\n\nFor a multi-region overlap schedule, simultaneously live TMEM regions must also be disjoint. Equal 256-column halves are one possible policy for a 512-column allocation, not an epilogue-fusion requirement. Reusable mbarriers need correct expected-arrival counts and phase/parity tracking."}, "reason": {"statement": "Make multi-region overlap optional and state nonaliasing constraints.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"]}} +{"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "```cuda\n// TMEM double-buffering for MMA-epilogue overlap\n//\n// TMEM layout: 128 rows x 512 columns (32-bit elements)\n// Buffer A: columns [0, 255] -- 128 x 256 accumulator\n// Buffer B: columns [256, 511] -- 128 x 256 accumulator\n//\n// Timeline:\n// Tile 0: MMA -> buffer A | epilogue idle (no prior result)\n// Tile 1: MMA -> buffer B | epilogue reads buffer A\n// Tile 2: MMA -> buffer A | epilogue reads buffer B\n// Tile 3: MMA -> buffer B | epilogue reads buffer A\n// ... ping-pong continues ...\n\n__device__ void mma_epilogue_overlap(\n const GemmParams& params,\n int num_tiles)\n{\n int warp_id = threadIdx.x / 32;\n\n __shared__ uint64_t mbar_mma_done[2]; // One per TMEM buffer half\n __shared__ uint64_t mbar_epi_done[2]; // Epilogue completion signals\n\n if (threadIdx.x == 0) {\n for (int i = 0; i < 2; i++) {\n mbarrier_init(&mbar_mma_done[i], 1);\n mbarrier_init(&mbar_epi_done[i], 1);\n }\n }\n __syncthreads();\n\n if (warp_id == 1) {\n // === MMA WARP ===\n for (int t = 0; t < num_tiles; t++) {\n int buf = t % 2; // Alternate between buffer halves\n\n // Wait for epilogue to finish reading this buffer\n if (t >= 2) {\n mbarrier_wait(&mbar_epi_done[buf]);\n }\n\n // Issue MMA, accumulating into TMEM buffer half\n int tmem_col_offset = buf * 256;\n tcgen05_mma_with_offset(tmem_col_offset,\n params.smem_A, params.smem_B);\n\n // Signal epilogue that this buffer is ready\n mbarrier_arrive(&mbar_mma_done[buf]);\n }\n\n } else if (warp_id >= 2) {\n // === EPILOGUE WARPS ===\n for (int t = 0; t < num_tiles; t++) {\n int buf = t % 2;\n\n // Wait for MMA to finish filling this buffer\n mbarrier_wait(&mbar_mma_done[buf]);\n\n // Read from TMEM buffer half and write to global memory\n int tmem_col_offset = buf * 256;\n epilogue_store(params, t, tmem_col_offset, warp_id);\n\n // ALL epilogue warps must finish reading TMEM before MMA reuses\n // the buffer. Each epilogue warp arrives on mbar_epi_done;\n // mbar_epi_done is initialized with count = NUM_EPILOGUE_WARPS.\n // MMA warp waits on this mbarrier before writing to this half.\n if (lane_id == 0) {\n mbarrier_arrive(&mbar_epi_done[buf]);\n }\n // mbar_epi_done[buf] fires only after ALL epilogue warps arrive\n }\n }\n}\n```"}, "after": {"statement": "| tcgen05 MMA → TMEM reader | the relevant asynchronous MMA is committed and its completion observed |\n| TMEM → result registers | the legal collective load completes before dependent arithmetic |\n| registers → output | element/layout mapping and output-edge predicates cover exactly the valid coordinates |\n| TMEM reader → region reuse | every reader's load has completed and the matching reusable barrier phase is released |\n| kernel tail | all output stores complete as required and every TMEM allocation is collectively freed |\n\nFor a multi-region overlap schedule, simultaneously live TMEM regions must also be disjoint. Equal 256-column halves are one possible policy for a 512-column allocation, not an epilogue-fusion requirement. Reusable mbarriers need correct expected-arrival counts and phase/parity tracking."}, "reason": {"statement": "Replace incomplete code with a required-edge table and negative tests.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"]}} +{"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "CUTLASS 4.5.0 provides composable epilogue visitors that fuse arbitrary element-wise operations after GEMM:"}, "after": {"statement": "`cutlass::epilogue::fusion::LinCombEltAct` has this parameter order:\n\n```cpp\ntemplate <\n template class ActivationFn,\n class ElementOutput,\n class ElementCompute,\n class ElementSource = ElementOutput,\n class ElementScalar = ElementCompute,\n cutlass::FloatRoundStyle Round = cutlass::FloatRoundStyle::round_to_nearest>\nstruct LinCombEltAct;\n```\n\nFor SM100 construction, `cutlass::epilogue::collective::CollectiveBuilder` receives architecture, operator class, tile/cluster shapes, `EpilogueTileAuto` or an explicit epilogue tile, accumulator/compute/C/D types and layouts, alignment, `EpilogueScheduleAuto` or an explicit supported schedule, and finally a supported fusion operation or callbacks type.\n\nThere is no CUTLASS 4.5.0 type named `Sm100EpilogueTmaWarpSpecialized`. Support is constrained by the exact architecture, operator class, schedule, tile, layout, alignment, datatype, and fusion callback combination. Use `Gemm::can_implement(arguments)` and a tagged example rather than reconstructing a builder signature from prose.\n\nThe vLLM PR 16032 NVFP4 wrapper is one pinned C++ construction example: it uses `CollectiveBuilder<... EpilogueTileAuto, ... EpilogueScheduleAuto>` and derives mainloop shared-memory stages with `StageCountAutoCarveout`."}, "reason": {"statement": "Describe the exact supported operation-tag/callback interface.", "urls": []}} +{"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "```cuda\n// CUTLASS SM100 epilogue with fused scale + bias + activation\n// Uses the EVT (Epilogue Visitor Tree) pattern\n\nusing EpilogueOp = cutlass::epilogue::fusion::LinCombEltAct<\n cutlass::epilogue::thread::ReLU, // Activation function\n float, // Compute type\n float, // Scale type\n cutlass::half_t // Output type\n>;\n\n// The epilogue descriptor tells CUTLASS how to partition work\n// across the 14 epilogue warps\nusing CollectiveEpilogue = cutlass::epilogue::collective::Sm100EpilogueTmaWarpSpecialized<\n cutlass::gemm::TagToStrideC_t,\n cutlass::gemm::TagToStrideC_t,\n EpilogueOp,\n cutlass::gemm::EpilogueDefault // Default tiling\n>;\n\n// In the kernel, the epilogue is invoked after the mainloop:\n// epilogue(\n// problem_shape,\n// collective_mainloop.get_accumulator(), // TMEM reference\n// epilogue_params, // scale, bias pointers\n// shared_storage // SMEM for TMA stores\n// );\n```"}, "after": {"statement": "`cutlass::epilogue::fusion::LinCombEltAct` has this parameter order:\n\n```cpp\ntemplate <\n template class ActivationFn,\n class ElementOutput,\n class ElementCompute,\n class ElementSource = ElementOutput,\n class ElementScalar = ElementCompute,\n cutlass::FloatRoundStyle Round = cutlass::FloatRoundStyle::round_to_nearest>\nstruct LinCombEltAct;\n```\n\nFor SM100 construction, `cutlass::epilogue::collective::CollectiveBuilder` receives architecture, operator class, tile/cluster shapes, `EpilogueTileAuto` or an explicit epilogue tile, accumulator/compute/C/D types and layouts, alignment, `EpilogueScheduleAuto` or an explicit supported schedule, and finally a supported fusion operation or callbacks type.\n\nThere is no CUTLASS 4.5.0 type named `Sm100EpilogueTmaWarpSpecialized`. Support is constrained by the exact architecture, operator class, schedule, tile, layout, alignment, datatype, and fusion callback combination. Use `Gemm::can_implement(arguments)` and a tagged example rather than reconstructing a builder signature from prose.\n\nThe vLLM PR 16032 NVFP4 wrapper is one pinned C++ construction example: it uses `CollectiveBuilder<... EpilogueTileAuto, ... EpilogueScheduleAuto>` and derives mainloop shared-memory stages with `StageCountAutoCarveout`."}, "reason": {"statement": "Replace invented code with exact version-pinned type signatures and a real builder example route.", "urls": []}} +{"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "| Scale + Bias | `y = alpha * acc + beta * C` | Standard GEMM epilogue |"}, "after": {"statement": "| Source linear combination | `D = alpha*acc + beta*C` |\n| Per-row/per-column bias | add a separately laid-out broadcast bias input |\n| Activation | apply a supported functor such as ReLU, GELU, or SiLU to the output fragment |\n| Output conversion/quantization | convert the accumulator fragment and, when required, generate/store scale metadata |\n| Gated product | combine two values, for example `SiLU(gate) * up`; requires both inputs and a supported/custom callback |\n| Online-softmax rescale | rescale a prior partial output after a new row maximum; requires the attention reduction state |\n| Residual | combine a separately supplied residual/source tensor at the defined point in the expression tree |\n\nThese are operation categories, not a claim that every composition is supported by one built-in SM100 visitor. In particular, reductions, multiple outputs, auxiliary tensors, and broadcasts add synchronization and layout requirements beyond a pointwise activation."}, "reason": {"statement": "Separate source linear combination from explicit bias broadcasting.", "urls": []}} +{"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "- **All Blackwell GEMMs with non-trivial epilogues**: The 14 epilogue warps are available by default in the warp-specialized model. Fusing operations avoids a separate kernel launch and an extra global memory round-trip."}, "after": {"statement": "When an implementation overlaps the epilogue with independent MMA work, verify both directions of the handoff: compute-complete before read, and load/read-complete before overwrite. Include the prologue (no prior result), steady-state wraparound, final drain, and deallocation path in tests.\n\nEvaluate a defined fused/unfused pair with identical inputs, output semantics, launch policy, warmup, and timing statistics. Record registers, spills, SMEM, TMEM columns, barriers, threads/CTA, cluster shape, and occupancy. Sweep supported epilogue group sizes, load shapes, output tiles, and store stages. The PTX ISA defines no equal-share bandwidth model for a fixed number of epilogue warps; diagnose TMEM-load, conversion, barrier, and store bottlenecks with generated code and profiler data.\n\nFusing output conversion can avoid a global FP32 intermediate when the baseline would write and reread that intermediate. It does not guarantee a speedup: added registers, shared-memory staging, synchronization, edge handling, or reduced occupancy can outweigh saved traffic.\n\nUseful negative tests delay one reader, skip a barrier phase, reuse a TMEM region early, omit the final store tail, and exercise partial M/N tiles. Each fault should be detected by output comparison or a bounded watchdog."}, "reason": {"statement": "Replace universal role count and benefit with configuration-specific evaluation.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} +{"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "- TMEM-to-register bandwidth is not unlimited. With 14 warps simultaneously reading TMEM, each warp gets a proportional share. Very wide output tiles (large TILE_N) may bottleneck on TMEM read bandwidth."}, "after": {"statement": "When an implementation overlaps the epilogue with independent MMA work, verify both directions of the handoff: compute-complete before read, and load/read-complete before overwrite. Include the prologue (no prior result), steady-state wraparound, final drain, and deallocation path in tests.\n\nEvaluate a defined fused/unfused pair with identical inputs, output semantics, launch policy, warmup, and timing statistics. Record registers, spills, SMEM, TMEM columns, barriers, threads/CTA, cluster shape, and occupancy. Sweep supported epilogue group sizes, load shapes, output tiles, and store stages. The PTX ISA defines no equal-share bandwidth model for a fixed number of epilogue warps; diagnose TMEM-load, conversion, barrier, and store bottlenecks with generated code and profiler data.\n\nFusing output conversion can avoid a global FP32 intermediate when the baseline would write and reread that intermediate. It does not guarantee a speedup: added registers, shared-memory staging, synchronization, edge handling, or reduced occupancy can outweigh saved traffic.\n\nUseful negative tests delay one reader, skip a barrier phase, reuse a TMEM region early, omit the final store tail, and exercise partial M/N tiles. Each fault should be detected by output comparison or a bounded watchdog."}, "reason": {"statement": "Require profiler and variant measurements for bandwidth diagnosis.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} +{"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "- Simple epilogues (just store) waste the 14 epilogue warps. For such cases, consider reducing the CTA size or assigning epilogue warps to other work (e.g., next-tile TMA prefetch)."}, "after": {"statement": "When an implementation overlaps the epilogue with independent MMA work, verify both directions of the handoff: compute-complete before read, and load/read-complete before overwrite. Include the prologue (no prior result), steady-state wraparound, final drain, and deallocation path in tests.\n\nEvaluate a defined fused/unfused pair with identical inputs, output semantics, launch policy, warmup, and timing statistics. Record registers, spills, SMEM, TMEM columns, barriers, threads/CTA, cluster shape, and occupancy. Sweep supported epilogue group sizes, load shapes, output tiles, and store stages. The PTX ISA defines no equal-share bandwidth model for a fixed number of epilogue warps; diagnose TMEM-load, conversion, barrier, and store bottlenecks with generated code and profiler data.\n\nFusing output conversion can avoid a global FP32 intermediate when the baseline would write and reread that intermediate. It does not guarantee a speedup: added registers, shared-memory staging, synchronization, edge handling, or reduced occupancy can outweigh saved traffic.\n\nUseful negative tests delay one reader, skip a barrier phase, reuse a TMEM region early, omit the final store tail, and exercise partial M/N tiles. Each fault should be detected by output comparison or a bounded watchdog."}, "reason": {"statement": "Use measured role-count and store-path bottlenecks.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} +{"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "Verbatim upstream code lives in [`artifacts/kernels/epilogue-fusion/full/`](../../artifacts/kernels/epilogue-fusion/full/); labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` header) live in [`artifacts/kernels/epilogue-fusion/variants/`](../../artifacts/kernels/epilogue-fusion/variants/). Every file's SHA-256 and upstream-pinning metadata is in `PROVENANCE.yaml` inside each bundle."}, "after": {"statement": "The artifact bundle contains mixed provenance modes:\n\n- `full/nvfp4_scaled_mm_kernels.cu` is verbatim from vLLM merge `ed7a29d9f8b48978e3bbf43599d21b4de65387e0` and byte-matches SHA-256 `e8aed5ccb3dd9de26c3aeff159a242a46dcb7c7d8d0351b6c44ff1f8d2f7effa`.\n- `full/tmem-load-into-registers-for-epilogue.cu` is an extracted historical snippet from a local source card; it is not an upstream-verbatim kernel and must not be treated as standalone safe code.\n- `variants/01-double-buffered-tmem-epilogue-skeleton.cu` is explicitly labeled derived/not-upstream and is incomplete pseudocode.\n\nRetrieve the page and bundle with:\n\n```bash\nconda run -n base python scripts/get_page.py technique-epilogue-fusion --include-code\n```"}, "reason": {"statement": "Describe each artifact's actual provenance mode and warn that extracted/derived snippets are not safe standalone kernels.", "urls": []}} +{"path": "wiki/techniques/persistent-kernels.md", "before": {"statement": "Persistent kernels launch exactly as many CTAs as SMs, and each CTA processes multiple output tiles in a loop rather than exiting after one tile. On Blackwell, the CLC (Cluster Launch Control) hardware unit replaces software-based tile scheduling with a hardware-assisted mechanism. Each CTA queries the CLC for its next tile assignment and can cancel itself when no work remains, using the `try_cancel` pattern."}, "after": {"statement": "A persistent kernel keeps a resident worker alive for multiple logical work items. Its launch size is chosen from the problem decomposition, cluster shape, resource limits, and scheduling policy; “persistent” does not require exactly one CTA per physical SM.\n\nCluster Launch Control is a compute-capability-10.0 mechanism for redistributing grid coordinates that have not started. A CLC GEMM still launches its problem-sized grid. Each ClcID is processed exactly once through one of two paths:\n\n1. A block or cluster launches normally and processes its initial `blockIdx`.\n2. A running worker successfully cancels another not-yet-started block/cluster and processes the returned coordinate.\n\nCLC does not create tiles, cancel the requesting worker, or chain independent problems. It can help when the SMs available to a grid are uneven, but it cannot expose more independent parallel work than exists in the launched grid."}, "reason": {"statement": "Restore problem-sized CLC grid and two-path ClcID model.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/techniques/persistent-kernels.md", "before": {"statement": "```cuda\n// Persistent kernel with CLC tile scheduling (Blackwell SM100)\n__global__ void __launch_bounds__(512)\npersistent_gemm_clc(const __grid_constant__ GemmParams params)\n{\n // CLC-managed persistent loop: each CTA processes multiple tiles\n while (true) {\n // Query CLC for next tile assignment\n // Returns tile coordinates (tile_m, tile_n) or signals termination\n TileCoord tile;\n bool has_work = clc_try_get_tile(&tile);\n\n if (!has_work) {\n // No more tiles to process -- CTA exits\n // clc_try_cancel atomically checks if all tiles are done\n if (clc_try_cancel()) {\n return; // CTA terminates\n }\n continue; // Race condition: another CTA may have generated work\n }\n\n // Standard GEMM tile computation\n int tile_m = tile.m;\n int tile_n = tile.n;\n\n // TMA producer loads A[tile_m, :] and B[:, tile_n] tiles\n // MMA consumer accumulates K-dimension\n // Epilogue writes C[tile_m, tile_n]\n compute_gemm_tile(params, tile_m, tile_n);\n }\n}\n```"}, "after": {"statement": "Cluster Launch Control is a compute-capability-10.0 mechanism for redistributing grid coordinates that have not started. A CLC GEMM still launches its problem-sized grid. Each ClcID is processed exactly once through one of two paths:\n\n1. A block or cluster launches normally and processes its initial `blockIdx`.\n2. A running worker successfully cancels another not-yet-started block/cluster and processes the returned coordinate.\n\nCLC does not create tiles, cancel the requesting worker, or chain independent problems. It can help when the SMs available to a grid are uneven, but it cannot expose more independent parallel work than exists in the launched grid.\n\nThe normative request is asynchronous and returns an opaque 16-byte shared-memory response:\n\n```ptx\nclusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [response], [mbar];\nclusterlaunchcontrol.query_cancel.is_canceled.pred.b128 p, response_b128;\n@p clusterlaunchcontrol.query_cancel.get_first_ctaid.v4.b32.b128 {x, y, z, unused}, response_b128;\n```\n\nA complete implementation must initialize/publish the shared response and mbarrier, set the expected 16 transaction bytes, submit from the designated participant, wait for the matching phase, apply required proxy fences before reading/reusing the response, and decode a coordinate only when `is_canceled` succeeds.\n\nAfter a thread observes a failed request, another request from that thread is undefined. Failure is therefore not a “retry until another CTA creates work” condition; no CTA creates new ClcIDs.\n\nFor a thread-block cluster, one cluster participant submits the multicast request. Every CTA tracks completion at cluster scope, receives the same first coordinate, and adds its local block rank. Cluster synchronization is required before cancellation begins so all peers exist."}, "reason": {"statement": "Remove inverted/fabricated executable sketch.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/techniques/persistent-kernels.md", "before": {"statement": "At the PTX level, the CLC interaction is a cancel/query sequence. The exact\ninline PTX is usually hidden behind CUTLASS/CuTe wrappers, but the control flow\nlooks like this:\n\n```text\nTILE_LOOP:\n // Request cancellation of a not-yet-launched cluster.\n clusterlaunchcontrol.try_cancel(response_smem, mbarrier)\n wait(mbarrier)\n\n // Query the 16-byte response.\n has_work, tile_m, tile_n = clusterlaunchcontrol.query_cancel(response_smem)\n if (!has_work) return\n\n // ... compute tile ...\n goto TILE_LOOP\n```"}, "after": {"statement": "Cluster Launch Control is a compute-capability-10.0 mechanism for redistributing grid coordinates that have not started. A CLC GEMM still launches its problem-sized grid. Each ClcID is processed exactly once through one of two paths:\n\n1. A block or cluster launches normally and processes its initial `blockIdx`.\n2. A running worker successfully cancels another not-yet-started block/cluster and processes the returned coordinate.\n\nCLC does not create tiles, cancel the requesting worker, or chain independent problems. It can help when the SMs available to a grid are uneven, but it cannot expose more independent parallel work than exists in the launched grid.\n\nThe normative request is asynchronous and returns an opaque 16-byte shared-memory response:\n\n```ptx\nclusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [response], [mbar];\nclusterlaunchcontrol.query_cancel.is_canceled.pred.b128 p, response_b128;\n@p clusterlaunchcontrol.query_cancel.get_first_ctaid.v4.b32.b128 {x, y, z, unused}, response_b128;\n```\n\nA complete implementation must initialize/publish the shared response and mbarrier, set the expected 16 transaction bytes, submit from the designated participant, wait for the matching phase, apply required proxy fences before reading/reusing the response, and decode a coordinate only when `is_canceled` succeeds.\n\nAfter a thread observes a failed request, another request from that thread is undefined. Failure is therefore not a “retry until another CTA creates work” condition; no CTA creates new ClcIDs.\n\nFor a thread-block cluster, one cluster participant submits the multicast request. Every CTA tracks completion at cluster scope, receives the same first coordinate, and adds its local block rank. Cluster synchronization is required before cancellation begins so all peers exist."}, "reason": {"statement": "Replace with complete invariants and exact PTX forms linked elsewhere.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/techniques/persistent-kernels.md", "before": {"statement": "| Scheduling | Software loop with fixed stride | Hardware CLC unit assigns tiles |\n| Load balancing | Fixed; uneven if tile costs vary | Dynamic; CLC rebalances automatically |\n| Tail effect | Last wave may have partial occupancy | CLC minimizes by giving fast CTAs more tiles |\n| Launch overhead | Grid launch for each new problem | CLC can chain multiple problems |\n| Termination | Implicit when loop ends | Explicit `try_cancel` |\n| L2 locality | Depends on stride pattern | CLC can apply swizzled raster |"}, "after": {"statement": "Cluster Launch Control is a compute-capability-10.0 mechanism for redistributing grid coordinates that have not started. A CLC GEMM still launches its problem-sized grid. Each ClcID is processed exactly once through one of two paths:\n\n1. A block or cluster launches normally and processes its initial `blockIdx`.\n2. A running worker successfully cancels another not-yet-started block/cluster and processes the returned coordinate.\n\nCLC does not create tiles, cancel the requesting worker, or chain independent problems. It can help when the SMs available to a grid are uneven, but it cannot expose more independent parallel work than exists in the launched grid.\n\nA software persistent scheduler can assign a flat tile index by fixed stride:\n\n```cpp\nfor (int tile = blockIdx.x; tile < total_tiles; tile += gridDim.x) {\n int tile_m = tile / tiles_n;\n int tile_n = tile % tiles_n;\n compute_tile(tile_m, tile_n);\n}\n```\n\nFor positive `gridDim.x`, this covers each index in `[0,total_tiles)` exactly once, including nondivisible tails. It remains useful as a controlled baseline on Hopper and Blackwell."}, "reason": {"statement": "Separate raw CLC, CUTLASS transforms, and out-of-scope mechanisms.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"]}} +{"path": "wiki/techniques/persistent-kernels.md", "before": {"statement": "```cuda\n// CUTLASS SM100 persistent tile scheduler (simplified)\ntemplate \nstruct PersistentTileSchedulerSm100 {\n\n // Initialize the CLC with the problem geometry\n CUTLASS_DEVICE static void init(\n dim3 problem_tiles,\n void* clc_smem_buffer)\n {\n if (threadIdx.x == 0) {\n // Program CLC with total tile count and scheduling policy\n clc_init(clc_smem_buffer,\n problem_tiles.x, // tiles along M\n problem_tiles.y, // tiles along N\n ClcPolicy::SwizzledRaster);\n }\n __syncthreads();\n }\n\n // Shared storage for CTA-wide CLC result broadcast\n // __shfl_sync is warp-local and cannot reach warps 1-15.\n struct SharedClcState {\n int tile_m, tile_n;\n int valid; // 1 = got tile, 0 = no more work\n int cancelled;\n };\n\n // Get next tile assignment from CLC\n CUTLASS_DEVICE static bool get_next_tile(\n void* clc_smem_buffer,\n SharedClcState& shared_clc,\n int& tile_m,\n int& tile_n)\n {\n if (threadIdx.x == 0) {\n int m, n;\n bool v = clc_query_tile(clc_smem_buffer, m, n);\n shared_clc.tile_m = m;\n shared_clc.tile_n = n;\n shared_clc.valid = v ? 1 : 0;\n }\n __syncthreads(); // All warps see the result\n tile_m = shared_clc.tile_m;\n tile_n = shared_clc.tile_n;\n return shared_clc.valid != 0;\n }\n\n // Try to cancel the CTA when no more work\n CUTLASS_DEVICE static bool try_cancel(\n void* clc_smem_buffer,\n SharedClcState& shared_clc)\n {\n if (threadIdx.x == 0) {\n shared_clc.cancelled = clc_try_cancel(clc_smem_buffer) ? 1 : 0;\n }\n __syncthreads();\n return shared_clc.cancelled != 0;\n }\n};\n```"}, "after": {"statement": "The normative request is asynchronous and returns an opaque 16-byte shared-memory response:\n\n```ptx\nclusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [response], [mbar];\nclusterlaunchcontrol.query_cancel.is_canceled.pred.b128 p, response_b128;\n@p clusterlaunchcontrol.query_cancel.get_first_ctaid.v4.b32.b128 {x, y, z, unused}, response_b128;\n```\n\nA complete implementation must initialize/publish the shared response and mbarrier, set the expected 16 transaction bytes, submit from the designated participant, wait for the matching phase, apply required proxy fences before reading/reusing the response, and decode a coordinate only when `is_canceled` succeeds.\n\nAfter a thread observes a failed request, another request from that thread is undefined. Failure is therefore not a “retry until another CTA creates work” condition; no CTA creates new ClcIDs.\n\nFor a thread-block cluster, one cluster participant submits the multicast request. Every CTA tracks completion at cluster scope, receives the same first coordinate, and adds its local block rank. Cluster synchronization is required before cancellation begins so all peers exist."}, "reason": {"statement": "Use exact class/method routing and avoid fake implementation bodies.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"]}} +{"path": "wiki/techniques/persistent-kernels.md", "before": {"statement": "The tcgen05-tutorial progression demonstrates the impact of persistent kernels:\n\n```\nWithout persistence (static grid): 940 TFLOPS (62% of peak)\nWith CLC persistent scheduling: 1476 TFLOPS (98% of cuBLAS)\n```\n\nThe 57% improvement comes from:\n1. **Eliminated tail effect**: CLC dynamically assigns tiles, so fast-completing CTAs absorb extra work rather than sitting idle while the last wave finishes.\n2. **Reduced launch overhead**: A single kernel launch covers all tiles; no need to re-launch grids.\n3. **Better L2 cache utilization**: CLC can apply a swizzled raster pattern that improves spatial locality across neighboring tiles."}, "after": {"statement": "At CUTLASS 4.5.0, the SM100 persistent scheduler integrates CLC through `PersistentTileSchedulerSm100` and `PipelineCLCFetchAsync`. The scheduler's work flow is expressed through `WorkTileInfo` and methods including `get_current_work()`, `advance_to_next_work()`, and `fetch_next_work()`; it is not an API built from `clc_init`, `clc_query_tile`, or a caller-self-cancel helper.\n\n`advance_to_next_work()` submits/stages the next request, while `fetch_next_work()` waits for and decodes a response. CUTLASS applies `swizzle_and_rasterize()` as a software coordinate transform to both the initial `blockIdx` and returned CLC coordinates. `max_swizzle_size` and `raster_order` are scheduler arguments, not CLC hardware policy operands.\n\nThe official documentation is the authoritative implementation map; use its pinned scheduler and pipeline links rather than a short pseudo-class."}, "reason": {"statement": "Preserve the author progression only with exact variant scope and no CLC attribution.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/techniques/persistent-kernels.md", "before": {"statement": "- **Large GEMM problems**: Persistent kernels are most beneficial when the number of output tiles exceeds the SM count by at least 2-3x.\n- **Grouped GEMMs / MoE**: CLC can chain multiple problem instances, eliminating inter-kernel launch gaps.\n- **Workloads with uneven tile cost**: CLC's dynamic scheduling naturally handles variable-cost tiles (e.g., triangular attention masks)."}, "after": {"statement": "At CUTLASS 4.5.0, the SM100 persistent scheduler integrates CLC through `PersistentTileSchedulerSm100` and `PipelineCLCFetchAsync`. The scheduler's work flow is expressed through `WorkTileInfo` and methods including `get_current_work()`, `advance_to_next_work()`, and `fetch_next_work()`; it is not an API built from `clc_init`, `clc_query_tile`, or a caller-self-cancel helper.\n\n`advance_to_next_work()` submits/stages the next request, while `fetch_next_work()` waits for and decodes a response. CUTLASS applies `swizzle_and_rasterize()` as a software coordinate transform to both the initial `blockIdx` and returned CLC coordinates. `max_swizzle_size` and `raster_order` are scheduler arguments, not CLC hardware policy operands.\n\nThe official documentation is the authoritative implementation map; use its pinned scheduler and pipeline links rather than a short pseudo-class."}, "reason": {"statement": "Replace prescriptive thresholds with evidence requirements and conserved-work limits.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp"]}} +{"path": "wiki/techniques/persistent-kernels.md", "before": {"statement": "- The `try_cancel` pattern introduces a potential race that must be handled with a retry loop."}, "after": {"statement": "Cluster Launch Control is a compute-capability-10.0 mechanism for redistributing grid coordinates that have not started. A CLC GEMM still launches its problem-sized grid. Each ClcID is processed exactly once through one of two paths:\n\n1. A block or cluster launches normally and processes its initial `blockIdx`.\n2. A running worker successfully cancels another not-yet-started block/cluster and processes the returned coordinate.\n\nCLC does not create tiles, cancel the requesting worker, or chain independent problems. It can help when the SMs available to a grid are uneven, but it cannot expose more independent parallel work than exists in the launched grid."}, "reason": {"statement": "Make failed response terminal for that requesting thread.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/techniques/pipeline-stages.md", "before": {"statement": "Software pipelining overlaps data loading (TMA copies from global to shared memory) with computation (tcgen05.mma or wgmma) by maintaining multiple in-flight tile buffers. A circular buffer of 3-5 stages allows the TMA producer to fill stage N+2 while the MMA consumer processes stage N, hiding the global memory latency entirely. This technique is critical for achieving high utilization on both Hopper and Blackwell."}, "after": {"statement": "A multi-stage pipeline reserves disjoint shared-memory operand buffers so production of one tile can overlap consumption of another. Stage count controls how far producer and consumer progress may separate; it does not guarantee that latency is fully hidden or that the schedule is faster.\n\nThere is no architecture-wide table in which two stages are always partial, three always fully hide latency, or more than five are always excessive. Legal and useful depth depends on:\n\n- bytes per stage, alignment/swizzle padding, other SMEM, and occupancy;\n- TMA issue/transfer rate and descriptor/transaction shape;\n- MMA work and completion time per K tile;\n- producer/consumer role schedule and register pressure;\n- K-loop length, prologue/tail fraction, and output schedule.\n\nCompile every candidate, reject resource-invalid variants, then compare controlled timings and pipeline/barrier stalls. Include `num_k_tiles` values below, equal to, and above the stage count to exercise every boundary."}, "reason": {"statement": "Define the mechanism without guaranteed profitability.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-best-practices-guide/index.html#asynchronous-copy-from-global-memory-to-shared-memory"]}} +{"path": "wiki/techniques/pipeline-stages.md", "before": {"statement": "The tcgen05-tutorial demonstrates the performance impact of pipelining:\n\n```\nNo pipelining (load, then compute): 695 TFLOPS (46%)\n3-stage pipeline (TMA + MMA overlap): 940 TFLOPS (62%)\n+ warp specialization: 1476 TFLOPS (98%)\n```"}, "after": {"statement": "Compute capability 10.0 supports up to 228 KiB of shared memory per SM. Thus five stages of that exact payload already exceed the per-SM capacity, while three stages consume about 63.2% before barriers, padding, descriptors, epilogue buffers, and other shared storage. Per-block opt-in limits and occupancy constraints also apply."}, "reason": {"statement": "Report only the adjacent v2b/v3 pipeline comparison and scope it to the author setup.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"]}} +{"path": "wiki/techniques/pipeline-stages.md", "before": {"statement": "```cuda\n// 3-stage circular buffer with mbarrier synchronization\n// Stages: [0] loading, [1] ready for MMA, [2] being consumed by MMA\n#define NUM_STAGES 3\n\n__global__ void __launch_bounds__(512)\npipelined_gemm(const __grid_constant__ GemmParams params)\n{\n extern __shared__ char smem[];\n\n // Circular buffer layout in shared memory\n // Each stage has its own A and B tile buffers\n half* smem_A[NUM_STAGES];\n half* smem_B[NUM_STAGES];\n for (int s = 0; s < NUM_STAGES; s++) {\n smem_A[s] = reinterpret_cast(\n smem + s * (TILE_A_BYTES + TILE_B_BYTES));\n smem_B[s] = reinterpret_cast(\n smem + s * (TILE_A_BYTES + TILE_B_BYTES) + TILE_A_BYTES);\n }\n\n // mbarrier arrays for producer-consumer sync\n __shared__ uint64_t mbar_load_complete[NUM_STAGES];\n __shared__ uint64_t mbar_mma_complete[NUM_STAGES];\n\n int warp_id = threadIdx.x / 32;\n int lane_id = threadIdx.x % 32;\n\n // Initialize barriers\n if (threadIdx.x == 0) {\n for (int s = 0; s < NUM_STAGES; s++) {\n mbarrier_init(&mbar_load_complete[s], 1);\n mbarrier_init(&mbar_mma_complete[s], 1);\n }\n }\n __syncthreads();\n\n int num_k_tiles = params.K / TILE_K;\n\n if (warp_id == 0) {\n // ===== TMA PRODUCER =====\n // Prologue: fill the first NUM_STAGES buffers\n for (int s = 0; s < NUM_STAGES && s < num_k_tiles; s++) {\n if (lane_id == 0) {\n // Set expected TX bytes on mbarrier BEFORE issuing TMA.\n // TMA hardware will arrive on the mbarrier when transfer completes.\n uint32_t tx_bytes = TILE_A_BYTES + TILE_B_BYTES;\n mbarrier_arrive_expect_tx(&mbar_load_complete[s], tx_bytes);\n tma_load_tile_A(smem_A[s], params, s, &mbar_load_complete[s]);\n tma_load_tile_B(smem_B[s], params, s, &mbar_load_complete[s]);\n // NOTE: Do NOT manually arrive after TMA issue — the TMA\n // hardware signals the mbarrier upon transfer completion.\n }\n }\n\n // Steady state: load stage s while MMA processes stage s-NUM_STAGES\n for (int k = NUM_STAGES; k < num_k_tiles; k++) {\n int stage = k % NUM_STAGES;\n // Wait for MMA to finish with this buffer\n mbarrier_wait(&mbar_mma_complete[stage]);\n if (lane_id == 0) {\n uint32_t tx_bytes = TILE_A_BYTES + TILE_B_BYTES;\n mbarrier_arrive_expect_tx(&mbar_load_complete[stage], tx_bytes);\n tma_load_tile_A(smem_A[stage], params, k, &mbar_load_complete[stage]);\n tma_load_tile_B(smem_B[stage], params, k, &mbar_load_complete[stage]);\n }\n }\n\n } else if (warp_id == 1) {\n // ===== MMA CONSUMER =====\n for (int k = 0; k < num_k_tiles; k++) {\n int stage = k % NUM_STAGES;\n // Wait for TMA to fill this buffer\n mbarrier_wait(&mbar_load_complete[stage]);\n\n // Issue MMA on the filled buffer\n if (lane_id == 0) {\n tcgen05_mma(smem_A[stage], smem_B[stage]);\n }\n __syncwarp();\n\n // Signal that this buffer is free for reuse\n if (lane_id == 0) {\n mbarrier_arrive(&mbar_mma_complete[stage]);\n }\n }\n }\n // Epilogue warps omitted for clarity\n}\n```"}, "after": {"statement": "Each stage cycles through this ownership state machine:\n\n| empty | producer | the prior consumer has completed every read of the stage |\n| loading | TMA async proxy | expected transaction bytes are registered before the copy and the TMA transaction is outstanding |\n| ready | consumer | the full barrier's matching phase has completed |\n| consuming | MMA path | the consumer has acquired the stage and no producer may overwrite it |\n| empty again | producer | every asynchronous MMA use of the SMEM operands is complete and the consumer releases the matching empty phase |\n\nModulo indexing selects storage, but it does not track ownership by itself. A correct implementation also supplies barrier initialization/publication, expected arrival counts, phase or token state, async-proxy ordering, a bounded prologue, steady-state advancement, pipeline tail, and error-free behavior when K tiles are fewer than stages.\n\nOn SM100, do not release a stage merely after issuing `tcgen05.mma`. The MMA is asynchronous; use its defined completion path before allowing a producer to overwrite A/B storage. Likewise, `__syncthreads()` is not a substitute for the required tcgen05/TMA async-proxy completion and ordering operations."}, "reason": {"statement": "Replace unsafe code with a state-transition contract and pinned implementations.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"]}} +{"path": "wiki/techniques/pipeline-stages.md", "before": {"statement": "| 2 | 2x base | Partial | Small tiles, limited SMEM |\n| 3 | 3x base | Full for most GEMMs | Standard choice on Blackwell |\n| 4-5 | 4-5x base | Full with margin | Large K, high memory latency |\n| >5 | Excessive | Diminishing returns | Rarely justified |"}, "after": {"statement": "For binary16 `A[128,64]` and `B[64,256]`, unpadded operand payload is:\n\n```text\nA = 128 × 64 × 2 bytes = 16 KiB\nB = 64 × 256 × 2 bytes = 32 KiB\none stage = 48 KiB\nthree stages = 144 KiB\nfive stages = 240 KiB\n```\n\nPayload grows linearly with stage count. Total allocation can additionally contain fixed and stage-dependent metadata, so derive the concrete shared-storage type rather than multiplying payload alone.\n\nThere is no architecture-wide table in which two stages are always partial, three always fully hide latency, or more than five are always excessive. Legal and useful depth depends on:\n\n- bytes per stage, alignment/swizzle padding, other SMEM, and occupancy;\n- TMA issue/transfer rate and descriptor/transaction shape;\n- MMA work and completion time per K tile;\n- producer/consumer role schedule and register pressure;\n- K-loop length, prologue/tail fraction, and output schedule.\n\nCompile every candidate, reject resource-invalid variants, then compare controlled timings and pipeline/barrier stalls. Include `num_k_tiles` values below, equal to, and above the stage count to exercise every boundary."}, "reason": {"statement": "Replace fixed labels with resource and measurement inputs.", "urls": []}} +{"path": "wiki/techniques/pipeline-stages.md", "before": {"statement": "```ptx\n// Phase-based mbarrier protocol for 3-stage pipeline\n//\n// Each mbarrier tracks a \"phase\" (0 or 1). The producer flips the phase\n// on arrive; the consumer waits for the expected phase.\n\n// Producer: arrive on stage %s (flips phase)\nmbarrier.arrive.shared.b64 %state, [%mbar_load + %s * 8];\n\n// Consumer: wait for phase %expected_phase on stage %s\n// try_wait is non-blocking; the warp can spin-wait or do other work\nWAIT_LOOP:\n mbarrier.try_wait.parity.shared.b64 %ready, [%mbar_load + %s * 8], %phase;\n @!%ready bra WAIT_LOOP;\n\n// TMA can also arrive directly on an mbarrier:\n// The TMA unit signals completion without CPU thread involvement\ncp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes\n [%smem_addr], [%tensor_map, {%coord0, %coord1}], [%mbar_load + %s * 8];\n```\n\nThe key advantage is that TMA can arrive on an mbarrier autonomously. The producer warp only needs to initiate the TMA; the TMA hardware signals completion directly, removing the producer from the critical path."}, "after": {"statement": "Each stage cycles through this ownership state machine:\n\n| empty | producer | the prior consumer has completed every read of the stage |\n| loading | TMA async proxy | expected transaction bytes are registered before the copy and the TMA transaction is outstanding |\n| ready | consumer | the full barrier's matching phase has completed |\n| consuming | MMA path | the consumer has acquired the stage and no producer may overwrite it |\n| empty again | producer | every asynchronous MMA use of the SMEM operands is complete and the consumer releases the matching empty phase |\n\nModulo indexing selects storage, but it does not track ownership by itself. A correct implementation also supplies barrier initialization/publication, expected arrival counts, phase or token state, async-proxy ordering, a bounded prologue, steady-state advancement, pipeline tail, and error-free behavior when K tiles are fewer than stages.\n\nOn SM100, do not release a stage merely after issuing `tcgen05.mma`. The MMA is asynchronous; use its defined completion path before allowing a producer to overwrite A/B storage. Likewise, `__syncthreads()` is not a substitute for the required tcgen05/TMA async-proxy completion and ordering operations.\n\nFor a TMA load into shared memory, the producer initializes/publishes the mbarrier, accounts for the copy's expected transaction bytes, and issues the tensor copy with that barrier. Hardware completes the transaction on the barrier; the consumer waits for the matching phase before reading the stage.\n\nThis autonomous completion avoids a producer instruction that manually announces data-ready after the transfer. It does not prove that producer issue work or the overall load path is absent from the measured critical path.\n\nUse the exact PTX ISA tensor-copy grammar or a version-pinned library pipeline. An illustrative `arrive`/spin loop that omits expected transaction bytes, copy descriptors, proxy fences, phase initialization, and the consumer-to-producer reuse edge is not a safe pipeline."}, "reason": {"statement": "Use exact lifecycle prose and distinguish autonomous completion from measured critical path.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"]}} +{"path": "wiki/techniques/pipeline-stages.md", "before": {"statement": "The Modular blog series describes a 5-stage circular buffer reaching 85% of SOTA performance on Blackwell:\n\n```cuda\n// Modular-style 5-stage pipeline constants\n// Chosen to fully hide B200 HBM latency (~400 cycles)\n// while fitting within 228 KB SMEM budget\n\nconstexpr int NUM_STAGES = 5;\nconstexpr int TILE_M = 128;\nconstexpr int TILE_N = 128; // Smaller N to fit 5 stages\nconstexpr int TILE_K = 64;\n\n// Per stage: A (128*64*2=16KB) + B (64*128*2=16KB) = 32 KB\n// 5 stages: 160 KB + barriers + metadata < 228 KB\n\n// The pipeline timing diagram for steady state:\n//\n// Stage: 0 1 2 3 4\n// TMA: [load k5] [done] [done] [load k8] [load k9]\n// MMA: [done] [done] [comp k7] [done] [done]\n//\n// At any point: 1 stage being loaded, 1 being computed, 3 in transit or done\n```"}, "after": {"statement": "Compute capability 10.0 supports up to 228 KiB of shared memory per SM. Thus five stages of that exact payload already exceed the per-SM capacity, while three stages consume about 63.2% before barriers, padding, descriptors, epilogue buffers, and other shared storage. Per-block opt-in limits and occupancy constraints also apply."}, "reason": {"statement": "Keep five stages as one source configuration and scope the endpoint to the full sequence.", "urls": ["https://www.modular.com/blog/matrix-multiplication-on-nvidias-blackwell-part-3-the-optimizations-behind-85-of-sota-performance", "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"]}} +{"path": "wiki/techniques/pipeline-stages.md", "before": {"statement": "- **All memory-bound and compute-bound GEMM kernels**: Pipelining is never harmful and always improves utilization by hiding latency.\n\n- **Combined with warp specialization**: Pipelining provides the buffer structure; warp specialization assigns the producer/consumer roles. The two techniques are complementary and almost always used together."}, "after": {"statement": "A multi-stage pipeline reserves disjoint shared-memory operand buffers so production of one tile can overlap consumption of another. Stage count controls how far producer and consumer progress may separate; it does not guarantee that latency is fully hidden or that the schedule is faster.\n\nThere is no architecture-wide table in which two stages are always partial, three always fully hide latency, or more than five are always excessive. Legal and useful depth depends on:\n\n- bytes per stage, alignment/swizzle padding, other SMEM, and occupancy;\n- TMA issue/transfer rate and descriptor/transaction shape;\n- MMA work and completion time per K tile;\n- producer/consumer role schedule and register pressure;\n- K-loop length, prologue/tail fraction, and output schedule.\n\nCompile every candidate, reject resource-invalid variants, then compare controlled timings and pipeline/barrier stalls. Include `num_k_tiles` values below, equal to, and above the stage count to exercise every boundary."}, "reason": {"statement": "Make stage depth and role specialization independent tuning dimensions.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-best-practices-guide/index.html#asynchronous-copy-from-global-memory-to-shared-memory"]}} +{"path": "wiki/techniques/pipeline-stages.md", "before": {"statement": "- Barrier initialization overhead is negligible but must happen before the first TMA. Place init in a `__syncthreads()` block at kernel start."}, "after": {"statement": "Each stage cycles through this ownership state machine:\n\n| empty | producer | the prior consumer has completed every read of the stage |\n| loading | TMA async proxy | expected transaction bytes are registered before the copy and the TMA transaction is outstanding |\n| ready | consumer | the full barrier's matching phase has completed |\n| consuming | MMA path | the consumer has acquired the stage and no producer may overwrite it |\n| empty again | producer | every asynchronous MMA use of the SMEM operands is complete and the consumer releases the matching empty phase |\n\nModulo indexing selects storage, but it does not track ownership by itself. A correct implementation also supplies barrier initialization/publication, expected arrival counts, phase or token state, async-proxy ordering, a bounded prologue, steady-state advancement, pipeline tail, and error-free behavior when K tiles are fewer than stages.\n\nOn SM100, do not release a stage merely after issuing `tcgen05.mma`. The MMA is asynchronous; use its defined completion path before allowing a producer to overwrite A/B storage. Likewise, `__syncthreads()` is not a substitute for the required tcgen05/TMA async-proxy completion and ordering operations.\n\nThere is no architecture-wide table in which two stages are always partial, three always fully hide latency, or more than five are always excessive. Legal and useful depth depends on:\n\n- bytes per stage, alignment/swizzle padding, other SMEM, and occupancy;\n- TMA issue/transfer rate and descriptor/transaction shape;\n- MMA work and completion time per K tile;\n- producer/consumer role schedule and register pressure;\n- K-loop length, prologue/tail fraction, and output schedule.\n\nCompile every candidate, reject resource-invalid variants, then compare controlled timings and pipeline/barrier stalls. Include `num_k_tiles` values below, equal to, and above the stage count to exercise every boundary."}, "reason": {"statement": "State the ordering proof rather than a fixed initialization recipe/cost.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-init", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"]}} +{"path": "wiki/techniques/tile-scheduling.md", "before": {"statement": "Tile scheduling determines the order in which output tiles of a GEMM (or attention) kernel are assigned to CTAs. The scheduling order affects L2 cache hit rates, tail-effect severity, and overall GPU utilization. On Blackwell, the CLC hardware unit supports dynamic scheduling policies including swizzled raster, while Hopper relies on software-based static stride or swizzled patterns computed at launch time."}, "after": {"statement": "“Tile scheduling” can refer to four different choices:\n\n1. **Coordinate order:** a software mapping from a logical work index to `(tile_m, tile_n, ...)`, such as row-major, column-major, or a blocked/swizzled raster.\n2. **Resident-worker iteration:** whether a CTA handles one logical tile or repeatedly advances through work, for example by a static grid stride.\n3. **Work reassignment:** on SM100, Cluster Launch Control (CLC) lets a running worker cancel another grid entity that has not started and process the returned ClcID.\n4. **Problem decomposition:** Stream-K or Split-K can partition a tile's K work and then reduce partial results.\n\nCLC does not accept a raster-order or swizzle-policy operand. CUTLASS applies those coordinate transforms in software to initial or returned coordinates. CLC also does not itself create K partitions or synthesize work beyond the launched grid."}, "reason": {"statement": "Separate tile-coordinate order from work acquisition.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/techniques/tile-scheduling.md", "before": {"statement": "Tiles are assigned in row-major order. Simple but poor L2 locality: consecutive tiles share no B-matrix data until the entire M-dimension is traversed.\n\n```cuda\n// Linear raster: tile_idx maps directly to (tile_m, tile_n)\n__device__ void linear_raster(int tile_idx, int tiles_n,\n int& tile_m, int& tile_n) {\n tile_m = tile_idx / tiles_n;\n tile_n = tile_idx % tiles_n;\n}\n\n// Access pattern for a 4x4 tile grid:\n// 0 1 2 3\n// 4 5 6 7\n// 8 9 10 11\n// 12 13 14 15\n//\n// Problem: tiles 0,1,2,3 all load different B columns.\n// By tile 4, B column 0 has been evicted from L2.\n```"}, "after": {"statement": "A flat row-major mapping is:\n\n```cuda\n__device__ void row_major_tile(int tile_idx, int tiles_n,\n int& tile_m, int& tile_n) {\n tile_m = tile_idx / tiles_n;\n tile_n = tile_idx % tiles_n;\n}\n```\n\nFor a positive grid size, a static persistent worker can cover the flat index set without overlap:\n\n```cuda\nfor (int tile_idx = int(blockIdx.x);\n tile_idx < total_tiles;\n tile_idx += int(gridDim.x)) {\n int tile_m = tile_idx / tiles_n;\n int tile_n = tile_idx % tiles_n;\n compute_tile(tile_m, tile_n);\n}\n```\n\nChanging coordinate order changes the sequence of operand panels presented to the cache. It does not by itself prove a hit rate or speedup. A valid transform must first be shown to be in-bounds and bijective for edge groups; then its locality must be evaluated for the actual tile shape, batch/group structure, concurrent traffic, and cache capacity.\n\nFor a deliberately simplified model with `T` equal-duration independent tiles, `W` available one-tile workers, and no K decomposition, write:\n\n```text\nT = qW + r, 0 <= r < W\n```\n\nIf `r > 0`, the final partial wave has `r` active workers and occupancy `r / W`. If `r == 0` and `T > 0`, the final wave is full; reporting zero occupancy from the remainder alone is an error.\n\nFor `T = 150` and `W = 142`, the second wave contains 8 tiles, so 8 workers are active, 134 are idle, and wave occupancy is `8 / 142 = 5.63%` under those assumptions. CLC may change which workers receive those eight IDs and reduce delays caused by uneven worker availability, but it cannot turn eight independent IDs into 142. Stream-K may create additional K partitions, but its selected decomposition and reduction overhead determine whether that is profitable; near-100-percent occupancy is not guaranteed."}, "reason": {"statement": "Retain exact mapping and make cache effects empirical.", "urls": []}} +{"path": "wiki/techniques/tile-scheduling.md", "before": {"statement": "Tiles are assigned in a blocked pattern that groups nearby M and N tiles together, maximizing reuse of both A rows and B columns in L2 cache:\n\n```cuda\n// Swizzled raster: group tiles into blocks that share A and B data\n// swizzle_size controls the block width (typically 4-8)\n__device__ void swizzled_raster(int tile_idx, int tiles_m, int tiles_n,\n int swizzle_size, int& tile_m, int& tile_n)\n{\n // Number of tile columns per swizzle group\n int group_cols = min(swizzle_size, tiles_n);\n int tiles_per_group = tiles_m * group_cols;\n\n // Which swizzle group\n int group_idx = tile_idx / tiles_per_group;\n int within_group = tile_idx % tiles_per_group;\n\n // Within the group, iterate in column-major order\n tile_m = within_group / group_cols;\n tile_n = group_idx * group_cols + within_group % group_cols;\n}\n\n// Access pattern with swizzle_size=2 on a 4x4 grid:\n// 0 1 | 8 9\n// 2 3 | 10 11\n// 4 5 | 12 13\n// 6 7 | 14 15\n//\n// Tiles 0,1,2,3 share the same B columns (0,1).\n// Tiles 0,2,4,6 share the same A rows.\n// Much better L2 reuse.\n```"}, "after": {"statement": "A flat row-major mapping is:\n\n```cuda\n__device__ void row_major_tile(int tile_idx, int tiles_n,\n int& tile_m, int& tile_n) {\n tile_m = tile_idx / tiles_n;\n tile_n = tile_idx % tiles_n;\n}\n```\n\nFor a positive grid size, a static persistent worker can cover the flat index set without overlap:\n\n```cuda\nfor (int tile_idx = int(blockIdx.x);\n tile_idx < total_tiles;\n tile_idx += int(gridDim.x)) {\n int tile_m = tile_idx / tiles_n;\n int tile_n = tile_idx % tiles_n;\n compute_tile(tile_m, tile_n);\n}\n```\n\nChanging coordinate order changes the sequence of operand panels presented to the cache. It does not by itself prove a hit rate or speedup. A valid transform must first be shown to be in-bounds and bijective for edge groups; then its locality must be evaluated for the actual tile shape, batch/group structure, concurrent traffic, and cache capacity.\n\nFor a deliberately simplified model with `T` equal-duration independent tiles, `W` available one-tile workers, and no K decomposition, write:\n\n```text\nT = qW + r, 0 <= r < W\n```\n\nIf `r > 0`, the final partial wave has `r` active workers and occupancy `r / W`. If `r == 0` and `T > 0`, the final wave is full; reporting zero occupancy from the remainder alone is an error.\n\nFor `T = 150` and `W = 142`, the second wave contains 8 tiles, so 8 workers are active, 134 are idle, and wave occupancy is `8 / 142 = 5.63%` under those assumptions. CLC may change which workers receive those eight IDs and reduce delays caused by uneven worker availability, but it cannot turn eight independent IDs into 142. Stream-K may create additional K partitions, but its selected decomposition and reduction overhead determine whether that is profitable; near-100-percent occupancy is not guaranteed."}, "reason": {"statement": "Remove incorrect executable-looking code and unsupported maximality.", "urls": []}} +{"path": "wiki/techniques/tile-scheduling.md", "before": {"statement": "The CLC hardware scheduler assigns tiles at runtime, combining the benefits of dynamic load balancing with configurable scheduling policies:\n\n```cuda\n// CLC-based scheduling on Blackwell\n// The scheduling policy is set once during CLC initialization\nenum class ClcSchedulePolicy {\n LinearRaster, // Simple row-major order\n SwizzledRaster, // Blocked pattern for L2 locality\n ColumnFirst, // Column-major for specific workloads\n Hilbert // Space-filling curve (experimental)\n};\n\n__device__ void clc_init_scheduler(\n void* clc_buffer,\n int tiles_m, int tiles_n,\n ClcSchedulePolicy policy)\n{\n if (threadIdx.x == 0) {\n // Configure the software scheduler metadata used around CLC queries.\n // The CLC PTX surface is try_cancel/query_cancel, not clusterctl.init.\n uint32_t config = encode_clc_config(tiles_m, tiles_n, policy);\n init_clc_scheduler_metadata(clc_buffer, tiles_m, tiles_n, config);\n }\n __syncwarp();\n}\n```"}, "after": {"statement": "CUTLASS 4.5.0 documents this lifecycle for a CLC-backed persistent scheduler:\n\n1. Launch the full problem grid and process the worker's initial `blockIdx` coordinate.\n2. Submit `clusterlaunchcontrol.try_cancel` against another not-yet-started grid entity. The asynchronous 16-byte response completes a transaction on an mbarrier.\n3. After the transaction completes, query whether cancellation succeeded. On success, decode and process the returned ClcID; for clusters, combine the returned first coordinate with the local cluster rank.\n4. Treat an observed failed request as terminal for requests by that thread. Retrying from the same thread after failure is undefined.\n\nThis can redistribute existing work when SM availability is uneven. It changes which resident worker handles an unlaunched ID, not the number of independent IDs in the grid. Software raster/swizzle may be applied consistently to both initial and returned coordinates."}, "reason": {"statement": "Replace the invented API with the documented work lifecycle.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/techniques/tile-scheduling.md", "before": {"statement": "```cuda\n// CUTLASS tile scheduler selection for SM100\n// All persistent schedulers inherit from PersistentTileSchedulerSm100\n\n// 1. Default CLC scheduler with swizzled raster\nusing Scheduler_Default = cutlass::gemm::PersistentTileSchedulerSm100;\n\n// 2. Stream-K scheduler for better tail handling\n// Splits K-dimension across CTAs for the last wave\nusing Scheduler_StreamK = cutlass::gemm::StreamKSchedulerSm100;\n\n// 3. Grouped GEMM scheduler for MoE workloads\n// Each group has different M, shared N and K\nusing Scheduler_Grouped = cutlass::gemm::GroupedTileSchedulerSm100;\n\n// Usage in CUTLASS kernel definition:\nusing GemmKernel = cutlass::gemm::kernel::GemmUniversal<\n cute::Shape, Int<256>, Int<64>>, // Tile shape\n ElementA, LayoutA,\n ElementB, LayoutB,\n ElementC, LayoutC,\n TiledMma,\n CollectiveMainloop,\n CollectiveEpilogue,\n Scheduler_Default // Tile scheduler\n>;\n```"}, "after": {"statement": "The public scheduler tags select implementation classes; users should prefer those tags over spelling detail types directly.\n\n| `PersistentScheduler` (also the default `void` tag) | `PersistentTileSchedulerSm100` | CLC-backed persistent data-parallel scheduling |\n| `DynamicPersistentScheduler` | `PersistentTileSchedulerSm100` | Explicit dynamic route to the same SM100 implementation |\n| `StaticPersistentScheduler` | `StaticPersistentTileScheduler100` | Static persistent scheduling |\n| `StreamKScheduler` | `PersistentTileSchedulerSm100StreamK` | Parameterized data-parallel, Stream-K, or Split-K decomposition |\n| `GroupScheduler` | `PersistentTileSchedulerSm100Group` | Grouped-problem route; in this tag it wraps the SM90-style static group scheduler |\n\nThe SM100 Stream-K route accepts decomposition, split, raster, swizzle, and reduction settings. CUTLASS 4.5.0 has a deterministic lock/turnstile reduction and a nondeterministic atomic-workspace reduction; atomic accumulation is therefore not a universal property of every Stream-K configuration."}, "reason": {"statement": "Name exact public tags and keep implementation classes clearly internal.", "urls": []}} +{"path": "wiki/techniques/tile-scheduling.md", "before": {"statement": "The choice of scheduling strategy directly impacts L2 cache hit rates. On B200 with 126 MB L2:\n\n```python\n# L2 cache reuse analysis for different schedulers\n# Problem: M=8192, N=8192, K=4096, BF16\n# Tile: 128x256, giving 64x32 = 2048 tiles\n# B200: 142 SMs, 126 MB L2\n\ntile_bytes_A = 128 * 4096 * 2 # 1 MB per tile row of A\ntile_bytes_B = 4096 * 256 * 2 # 2 MB per tile column of B\n\n# Linear raster: first wave loads 142 tiles across 142/32 = 4.4 column groups\n# B data for 5 different column groups = 5 * 2 MB = 10 MB (fits in L2)\n# But A data for 142/32 = 4.4 row groups * 4.4 col groups = ~20 distinct A rows\n# 20 * 1 MB = 20 MB -> some L2 eviction\n\n# Swizzled raster (swizzle=4): first wave covers 142 tiles in ~36 groups of 4\n# Each group uses 1 A row + 4 B columns = 1 + 8 = 9 MB per group\n# But groups share A rows: total unique A = ~36 rows * 1 MB = 36 MB\n# L2 pressure: 36 MB + 8 MB = 44 MB (fits in B200's 126 MB L2)\n\n# Conclusion: swizzled raster reduces L2 misses by ~2x vs linear for large problems\n```"}, "after": {"statement": "For a deliberately simplified model with `T` equal-duration independent tiles, `W` available one-tile workers, and no K decomposition, write:\n\n```text\nT = qW + r, 0 <= r < W\n```\n\nIf `r > 0`, the final partial wave has `r` active workers and occupancy `r / W`. If `r == 0` and `T > 0`, the final wave is full; reporting zero occupancy from the remainder alone is an error.\n\nFor `T = 150` and `W = 142`, the second wave contains 8 tiles, so 8 workers are active, 134 are idle, and wave occupancy is `8 / 142 = 5.63%` under those assumptions. CLC may change which workers receive those eight IDs and reduce delays caused by uneven worker availability, but it cannot turn eight independent IDs into 142. Stream-K may create additional K partitions, but its selected decomposition and reduction overhead determine whether that is profitable; near-100-percent occupancy is not guaranteed."}, "reason": {"statement": "Replace fabricated cache results with a measurement protocol.", "urls": []}} +{"path": "wiki/techniques/tile-scheduling.md", "before": {"statement": "| Linear raster | None | `(total_tiles % num_SMs) / num_SMs` |\n| Static stride | None | Same as linear |\n| CLC dynamic | Automatic | Fast CTAs steal from slow ones |\n| Stream-K | K-splitting | Near 100% (splits partial tiles across SMs) |"}, "after": {"statement": "For `T = 150` and `W = 142`, the second wave contains 8 tiles, so 8 workers are active, 134 are idle, and wave occupancy is `8 / 142 = 5.63%` under those assumptions. CLC may change which workers receive those eight IDs and reduce delays caused by uneven worker availability, but it cannot turn eight independent IDs into 142. Stream-K may create additional K partitions, but its selected decomposition and reduction overhead determine whether that is profitable; near-100-percent occupancy is not guaranteed."}, "reason": {"statement": "Give exact scoped arithmetic and list what each mechanism can change.", "urls": []}} +{"path": "wiki/techniques/tile-scheduling.md", "before": {"statement": "- CLC: fast-finishing CTAs from wave 1 absorb the 8 extra tiles\n- Stream-K: the 8 remaining tiles are split across all 142 SMs"}, "after": {"statement": "For `T = 150` and `W = 142`, the second wave contains 8 tiles, so 8 workers are active, 134 are idle, and wave occupancy is `8 / 142 = 5.63%` under those assumptions. CLC may change which workers receive those eight IDs and reduce delays caused by uneven worker availability, but it cannot turn eight independent IDs into 142. Stream-K may create additional K partitions, but its selected decomposition and reduction overhead determine whether that is profitable; near-100-percent occupancy is not guaranteed."}, "reason": {"statement": "Preserve the 8/142 baseline while describing mechanisms without invented outcomes.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/techniques/tile-scheduling.md", "before": {"statement": "- **Swizzled raster**: Default choice for large GEMMs. Always better than linear for L2 locality.\n- **CLC dynamic**: Recommended on Blackwell for all persistent kernels. Combines dynamic load balancing with swizzled ordering.\n- **Stream-K**: Best for small-to-medium problems where the tail effect dominates. Adds complexity for K-dimension synchronization.\n- **Grouped scheduler**: Essential for MoE and batched GEMM where problem sizes vary across groups."}, "after": {"statement": "1. Establish a correct unswizzled data-parallel mapping and verify exact tile coverage, including nondivisible edge groups.\n2. Sweep legal raster orders and swizzle sizes. Record kernel time plus L2 hit/sector traffic; do not infer eviction from coordinate order alone.\n3. Compare static and CLC-backed persistence at identical tile, cluster, grid, and occupancy settings. Record successful/failed CLC requests and per-worker tile counts where instrumentation permits.\n4. Test Stream-K separately with explicit decomposition, split, and reduction modes. Include workspace traffic, synchronization, determinism, and numerical tolerance.\n5. Repeat across representative shapes and grouped-size distributions. Report regressions as well as wins; there is no universal best scheduler for all GEMMs, attention kernels, or MoE workloads.\n\nWithout target-GPU measurements, this page makes no fixed claim for CLC acquisition latency, L2-miss reduction, or scheduler speedup."}, "reason": {"statement": "Turn prescriptions into testable selection hypotheses.", "urls": []}} +{"path": "wiki/techniques/tile-scheduling.md", "before": {"statement": "- Swizzle size must be tuned per problem shape. Too large a swizzle group exceeds L2 capacity; too small loses the locality benefit."}, "after": {"statement": "For a deliberately simplified model with `T` equal-duration independent tiles, `W` available one-tile workers, and no K decomposition, write:\n\n```text\nT = qW + r, 0 <= r < W\n```\n\nIf `r > 0`, the final partial wave has `r` active workers and occupancy `r / W`. If `r == 0` and `T > 0`, the final wave is full; reporting zero occupancy from the remainder alone is an error.\n\nFor `T = 150` and `W = 142`, the second wave contains 8 tiles, so 8 workers are active, 134 are idle, and wave occupancy is `8 / 142 = 5.63%` under those assumptions. CLC may change which workers receive those eight IDs and reduce delays caused by uneven worker availability, but it cannot turn eight independent IDs into 142. Stream-K may create additional K partitions, but its selected decomposition and reduction overhead determine whether that is profitable; near-100-percent occupancy is not guaranteed.\n\n1. Establish a correct unswizzled data-parallel mapping and verify exact tile coverage, including nondivisible edge groups.\n2. Sweep legal raster orders and swizzle sizes. Record kernel time plus L2 hit/sector traffic; do not infer eviction from coordinate order alone.\n3. Compare static and CLC-backed persistence at identical tile, cluster, grid, and occupancy settings. Record successful/failed CLC requests and per-worker tile counts where instrumentation permits.\n4. Test Stream-K separately with explicit decomposition, split, and reduction modes. Include workspace traffic, synchronization, determinism, and numerical tolerance.\n5. Repeat across representative shapes and grouped-size distributions. Report regressions as well as wins; there is no universal best scheduler for all GEMMs, attention kernels, or MoE workloads.\n\nWithout target-GPU measurements, this page makes no fixed claim for CLC acquisition latency, L2-miss reduction, or scheduler speedup."}, "reason": {"statement": "Retain tuning advice without a false monotonic cache model.", "urls": []}} +{"path": "wiki/techniques/tile-scheduling.md", "before": {"statement": "- CLC scheduling adds a small latency per tile acquisition (~10s of cycles). For extremely small tiles, this overhead is proportionally larger.\n- Stream-K requires atomic accumulation where K-splits meet, adding synchronization overhead. Only worthwhile when tail utilization is a proven bottleneck."}, "after": {"statement": "The public scheduler tags select implementation classes; users should prefer those tags over spelling detail types directly.\n\n| `PersistentScheduler` (also the default `void` tag) | `PersistentTileSchedulerSm100` | CLC-backed persistent data-parallel scheduling |\n| `DynamicPersistentScheduler` | `PersistentTileSchedulerSm100` | Explicit dynamic route to the same SM100 implementation |\n| `StaticPersistentScheduler` | `StaticPersistentTileScheduler100` | Static persistent scheduling |\n| `StreamKScheduler` | `PersistentTileSchedulerSm100StreamK` | Parameterized data-parallel, Stream-K, or Split-K decomposition |\n| `GroupScheduler` | `PersistentTileSchedulerSm100Group` | Grouped-problem route; in this tag it wraps the SM90-style static group scheduler |\n\nThe SM100 Stream-K route accepts decomposition, split, raster, swizzle, and reduction settings. CUTLASS 4.5.0 has a deterministic lock/turnstile reduction and a nondeterministic atomic-workspace reduction; atomic accumulation is therefore not a universal property of every Stream-K configuration.\n\n1. Establish a correct unswizzled data-parallel mapping and verify exact tile coverage, including nondivisible edge groups.\n2. Sweep legal raster orders and swizzle sizes. Record kernel time plus L2 hit/sector traffic; do not infer eviction from coordinate order alone.\n3. Compare static and CLC-backed persistence at identical tile, cluster, grid, and occupancy settings. Record successful/failed CLC requests and per-worker tile counts where instrumentation permits.\n4. Test Stream-K separately with explicit decomposition, split, and reduction modes. Include workspace traffic, synchronization, determinism, and numerical tolerance.\n5. Repeat across representative shapes and grouped-size distributions. Report regressions as well as wins; there is no universal best scheduler for all GEMMs, attention kernels, or MoE workloads.\n\nWithout target-GPU measurements, this page makes no fixed claim for CLC acquisition latency, L2-miss reduction, or scheduler speedup."}, "reason": {"statement": "State observable costs and configurable reductions rather than invented constants.", "urls": []}} +{"path": "wiki/techniques/warp-specialization.md", "before": {"statement": "Warp specialization assigns distinct functional roles to warps within a CTA, allowing each warp to focus on a single pipeline stage (data loading, MMA computation, or epilogue writeback). On Blackwell (SM100), the 16-warp CTA structure replaces Hopper's 4-warp warpgroup model. Because tcgen05.mma is a single-thread instruction that operates on TMEM rather than registers, only one warp needs to issue MMA operations, freeing the remaining warps for producer and consumer roles."}, "after": {"statement": "Warp specialization assigns different long-lived functions to disjoint warps in a CTA—for example scheduler, TMA load, MMA control, softmax/correction, or epilogue work. It is a software organization for overlapping pipeline stages, not a fixed SM100 CTA layout.\n\nThe architecture-level comparison is narrower:\n\n| Issue granularity | Warpgroup collective | One thread for `cta_group::1` or `cta_group::2` |\n| D accumulator | Per-thread registers | TMEM |\n| A/B movement | Explicit register/SMEM operands and producer work | Explicit SMEM descriptors or TMEM addresses and producer work |\n| Completion | WGMMA commit/wait groups | `tcgen05.commit` tied to an mbarrier, followed by a wait |\n\nSingle-thread issue reduces the number of threads needed to submit MMA instructions. It does not make operands move automatically, turn an entire warp into an ISA-level role, complete MMA synchronously, or choose how many epilogue warps a kernel should launch."}, "reason": {"statement": "Separate single-thread issue from software CTA/role sizing.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/techniques/warp-specialization.md", "before": {"statement": "The canonical Blackwell GEMM kernel uses 16 warps (512 threads) per CTA with the following role assignment:\n\n| 0 | TMA Producer | Issues TMA bulk-copy from global to shared memory, signals mbarrier |\n| 1 | MMA Consumer | Issues tcgen05.mma on SMEM operands, writes results to TMEM |\n| 2-15 | Epilogue | Reads TMEM accumulator, applies scale/bias/activation, writes to global memory |"}, "after": {"statement": "There is no universal “warp 0 load, warp 1 MMA, warps 2–15 epilogue” rule. Three primary implementations illustrate the range:\n\n| Gau Nernst tutorial v4, commit `3b90ac9b...` | 4 warps. An elected lane in warp 0 runs the TMA loop; an elected lane in warp 1 runs the MMA loop; after completion, all four warps participate in TMEM load/conversion/output. |\n| CUTLASS 4.5.0 generic SM100 GEMM | One warp each for `MMA`, `Sched`, `MainloopLoad`, and `EpilogueLoad`, followed by `CollectiveEpilogue::ThreadCount / 32` epilogue warps. Optional work can make some control roles nonparticipants. |\n| FlashInfer PR 1039 context FMHA | 16 warps: 0–3 `Softmax0`, 4–7 `Softmax1`, 8–11 correction, 12 MMA, 13 load, 14 epilogue, and 15 empty. The kernel has explicit pipelines between these roles. |\n\nFlashAttention-4's pinned SM100 forward source likewise starts from a 16-warp layout with two four-warp softmax groups, four correction warps, and dedicated MMA/load/epilogue IDs, then adjusts roles for configuration choices such as one Q stage, non-TMA paths, and dynamic persistence. These are concrete attention schedules, not a GEMM-wide architectural template."}, "reason": {"statement": "Replace one invented canonical layout with version-pinned examples.", "urls": []}} +{"path": "wiki/techniques/warp-specialization.md", "before": {"statement": "This contrasts with Hopper where a warpgroup (4 warps, 128 threads) collectively issues wgmma.mma_async, and all threads in the warpgroup participate in the MMA. On Blackwell, the MMA warp dispatches the instruction from a single thread while the hardware handles the data movement internally."}, "after": {"statement": "Warp specialization assigns different long-lived functions to disjoint warps in a CTA—for example scheduler, TMA load, MMA control, softmax/correction, or epilogue work. It is a software organization for overlapping pipeline stages, not a fixed SM100 CTA layout.\n\nThe architecture-level comparison is narrower:\n\n| Issue granularity | Warpgroup collective | One thread for `cta_group::1` or `cta_group::2` |\n| D accumulator | Per-thread registers | TMEM |\n| A/B movement | Explicit register/SMEM operands and producer work | Explicit SMEM descriptors or TMEM addresses and producer work |\n| Completion | WGMMA commit/wait groups | `tcgen05.commit` tied to an mbarrier, followed by a wait |\n\nSingle-thread issue reduces the number of threads needed to submit MMA instructions. It does not make operands move automatically, turn an entire warp into an ISA-level role, complete MMA synchronously, or choose how many epilogue warps a kernel should launch."}, "reason": {"statement": "Keep the precise Hopper/SM100 issue distinction without implying autonomous operand staging.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma"]}} +{"path": "wiki/techniques/warp-specialization.md", "before": {"statement": "| MMA granularity | 4-warp warpgroup (128 threads) | Single thread in 1 warp |\n| MMA output destination | Registers (shared across warpgroup) | TMEM (256KB, CTA-visible) |\n| Producer warps | Separate warp(s) for TMA loads | Warp 0 dedicated to TMA |\n| Epilogue execution | Same warpgroup or separate warps | 14 dedicated warps (2-15) |\n| Synchronization | warpgroup barriers, arrive/wait | mbarrier pairs (producer/consumer) |\n| Register pressure | High (accumulators in registers) | Low (accumulators in TMEM) |"}, "after": {"statement": "Warp specialization assigns different long-lived functions to disjoint warps in a CTA—for example scheduler, TMA load, MMA control, softmax/correction, or epilogue work. It is a software organization for overlapping pipeline stages, not a fixed SM100 CTA layout.\n\nThe architecture-level comparison is narrower:\n\n| Issue granularity | Warpgroup collective | One thread for `cta_group::1` or `cta_group::2` |\n| D accumulator | Per-thread registers | TMEM |\n| A/B movement | Explicit register/SMEM operands and producer work | Explicit SMEM descriptors or TMEM addresses and producer work |\n| Completion | WGMMA commit/wait groups | `tcgen05.commit` tied to an mbarrier, followed by a wait |\n\nSingle-thread issue reduces the number of threads needed to submit MMA instructions. It does not make operands move automatically, turn an entire warp into an ISA-level role, complete MMA synchronously, or choose how many epilogue warps a kernel should launch.\n\nThere is no universal “warp 0 load, warp 1 MMA, warps 2–15 epilogue” rule. Three primary implementations illustrate the range:\n\n| Gau Nernst tutorial v4, commit `3b90ac9b...` | 4 warps. An elected lane in warp 0 runs the TMA loop; an elected lane in warp 1 runs the MMA loop; after completion, all four warps participate in TMEM load/conversion/output. |\n| CUTLASS 4.5.0 generic SM100 GEMM | One warp each for `MMA`, `Sched`, `MainloopLoad`, and `EpilogueLoad`, followed by `CollectiveEpilogue::ThreadCount / 32` epilogue warps. Optional work can make some control roles nonparticipants. |\n| FlashInfer PR 1039 context FMHA | 16 warps: 0–3 `Softmax0`, 4–7 `Softmax1`, 8–11 correction, 12 MMA, 13 load, 14 epilogue, and 15 empty. The kernel has explicit pipelines between these roles. |\n\nFlashAttention-4's pinned SM100 forward source likewise starts from a 16-warp layout with two four-warp softmax groups, four correction warps, and dedicated MMA/load/epilogue IDs, then adjusts roles for configuration choices such as one Q stage, non-TMA paths, and dynamic persistence. These are concrete attention schedules, not a GEMM-wide architectural template."}, "reason": {"statement": "Limit the comparison to normative issue/D placement and use concrete source layouts for software roles.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/techniques/warp-specialization.md", "before": {"statement": "The kernel entry point assigns each warp its role based on `threadIdx.x`:\n\n```cuda\n// Blackwell 16-warp specialized GEMM kernel skeleton\n// 16 warps = 512 threads per CTA\n__global__ void __launch_bounds__(512)\nblackwell_gemm_warp_specialized(\n const __grid_constant__ GemmParams params)\n{\n const int warp_id = threadIdx.x / 32;\n const int lane_id = threadIdx.x % 32;\n\n // Shared memory for A/B tiles and mbarrier objects\n extern __shared__ char smem[];\n half* smem_A = reinterpret_cast(smem);\n half* smem_B = reinterpret_cast(smem + SMEM_A_SIZE);\n\n // mbarrier pairs: TMA hardware signals \"data ready\", MMA signals \"buffer free\"\n __shared__ uint64_t mbar_data_ready[NUM_STAGES];\n __shared__ uint64_t mbar_buffer_free[NUM_STAGES];\n // MMA→epilogue handoff barrier\n __shared__ uint64_t mbar_acc_complete;\n // Phase tracking: mbarriers alternate parity on each reuse cycle\n int phase_data[NUM_STAGES];\n int phase_free[NUM_STAGES];\n\n if (warp_id == 0) {\n if (lane_id == 0) {\n for (int s = 0; s < NUM_STAGES; s++) {\n // TMA expects arrive.expect_tx → hardware completes\n mbarrier_init(&mbar_data_ready[s], 1);\n mbarrier_init(&mbar_buffer_free[s], 1);\n }\n mbarrier_init(&mbar_acc_complete, 1);\n }\n }\n // Initialize phase counters (all start at 0)\n for (int s = 0; s < NUM_STAGES; s++) {\n phase_data[s] = 0;\n phase_free[s] = 0;\n }\n __syncthreads();\n\n if (warp_id == 0) {\n // === TMA PRODUCER WARP ===\n for (int k_tile = 0; k_tile < num_k_tiles; k_tile++) {\n int stage = k_tile % NUM_STAGES;\n\n // Wait for consumer to release this buffer (with phase tracking)\n if (k_tile >= NUM_STAGES) {\n mbarrier_wait_parity(&mbar_buffer_free[stage], phase_free[stage]);\n phase_free[stage] ^= 1; // flip parity for next reuse\n }\n\n // Set expected TX bytes, then issue TMA. TMA hardware will\n // signal mbar_data_ready upon transfer completion.\n // Do NOT manually arrive — that races with the async transfer.\n if (lane_id == 0) {\n uint32_t tx_bytes = TILE_A_BYTES + TILE_B_BYTES;\n mbarrier_arrive_expect_tx(&mbar_data_ready[stage], tx_bytes);\n tma_copy_async(smem_A + stage * TILE_A_SIZE,\n ¶ms.A[k_tile * TILE_K], TILE_A_SIZE,\n &mbar_data_ready[stage]);\n tma_copy_async(smem_B + stage * TILE_B_SIZE,\n ¶ms.B[k_tile * TILE_K], TILE_B_SIZE,\n &mbar_data_ready[stage]);\n // TMA hardware arrives on mbar_data_ready when transfer completes\n }\n }\n\n } else if (warp_id == 1) {\n // === MMA CONSUMER WARP ===\n for (int k_tile = 0; k_tile < num_k_tiles; k_tile++) {\n int stage = k_tile % NUM_STAGES;\n\n // Wait for TMA to complete this stage (with phase tracking)\n mbarrier_wait_parity(&mbar_data_ready[stage], phase_data[stage]);\n phase_data[stage] ^= 1;\n\n // Critical fence: ensure TMA data visible before MMA reads SMEM\n tcgen05_fence_after_thread_sync();\n\n if (lane_id == 0) {\n tcgen05_mma(smem_A + stage * TILE_A_SIZE,\n smem_B + stage * TILE_B_SIZE);\n }\n __syncwarp();\n\n // Signal buffer is free for reuse\n if (lane_id == 0) {\n mbarrier_arrive(&mbar_buffer_free[stage]);\n }\n }\n\n // Signal epilogue warps that accumulation is complete\n if (lane_id == 0) {\n mbarrier_arrive(&mbar_acc_complete);\n }\n\n } else {\n // === EPILOGUE WARPS (2-15) ===\n // Wait for MMA completion via dedicated mbarrier (not __syncthreads,\n // which would deadlock since producer/MMA warps don't reach it)\n mbarrier_wait(&mbar_acc_complete);\n\n // Each epilogue warp handles a partition of the TMEM output.\n // Use ceiling division to cover tail rows when TILE_M % 14 != 0.\n constexpr int NUM_EPI_WARPS = 14; // warps 2-15\n int epi_warp = warp_id - 2; // 0..13\n int rows_per_warp = (TILE_M + NUM_EPI_WARPS - 1) / NUM_EPI_WARPS;\n int my_row_start = epi_warp * rows_per_warp;\n int my_row_end = min(my_row_start + rows_per_warp, TILE_M);\n\n for (int r = my_row_start; r < my_row_end; r++) {\n for (int c = lane_id; c < TILE_N; c += 32) {\n // Read accumulator from TMEM\n float acc = tmem_load(r, c);\n // Apply epilogue: scale + bias + activation\n float result = epilogue_op(acc, params.scale, params.bias[c]);\n // Write to global memory\n params.C[r * params.N + c] = __float2half(result);\n }\n }\n }\n}\n```"}, "after": {"statement": "A warp-specialized implementation must prove each ownership handoff. For a reusable TMA-to-MMA stage:\n\n1. Initialize the mbarrier objects with correct participant/transaction counts and publish them to the required threads and async proxies before use.\n2. The producer acquires an empty stage, sets expected transaction bytes, and submits the TMA copies. The full/data-ready phase completes only after the expected async transactions complete.\n3. The MMA controller waits for the full phase and applies the required cross-thread/proxy fence before issuing `tcgen05.mma` against that SMEM stage.\n4. Because MMA is asynchronous, `__syncwarp()` or an ordinary `mbarrier.arrive` after issue does not make the operands reusable. Commit prior tcgen05 work to an mbarrier and wait for completion before releasing or overwriting the stage.\n5. Before another role reads D from TMEM, complete the MMA sequence and apply the required `tcgen05.fence::before_thread_sync` / execution-ordering handoff / `tcgen05.fence::after_thread_sync` protocol. A fence orders operations; it is not a completion wait.\n6. Keep TMEM allocated until every consumer finishes its `tcgen05.ld` sequence and associated waits, then deallocate with the required collective participation.\n\nPipeline wrappers in CUTLASS/CuTe encode parts of these rules, but participant counts, initial phases, tails, and producer/consumer states remain configuration-specific. A short role-dispatch sketch is not a substitute for the complete pipeline and TMEM lifecycle."}, "reason": {"statement": "Unsafe pseudo-code is worse than a precise protocol checklist and pinned full sources.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-load", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-dealloc"]}} +{"path": "wiki/techniques/warp-specialization.md", "before": {"statement": "At the PTX level, the mbarrier operations map to:\n\n```ptx\n// Producer: signal data is ready in stage %stage\nmbarrier.arrive.shared.b64 %dummy, [%mbar_data_ready + %stage_offset];\n\n// Consumer: wait for data to be ready\nmbarrier.try_wait.parity.shared.b64 %pred, [%mbar_data_ready + %stage_offset], %phase;\n\n// Consumer: signal buffer is consumed\nmbarrier.arrive.shared.b64 %dummy, [%mbar_buffer_free + %stage_offset];\n\n// Producer: wait for buffer to be free\nmbarrier.try_wait.parity.shared.b64 %pred, [%mbar_buffer_free + %stage_offset], %phase;\n```"}, "after": {"statement": "A warp-specialized implementation must prove each ownership handoff. For a reusable TMA-to-MMA stage:\n\n1. Initialize the mbarrier objects with correct participant/transaction counts and publish them to the required threads and async proxies before use.\n2. The producer acquires an empty stage, sets expected transaction bytes, and submits the TMA copies. The full/data-ready phase completes only after the expected async transactions complete.\n3. The MMA controller waits for the full phase and applies the required cross-thread/proxy fence before issuing `tcgen05.mma` against that SMEM stage.\n4. Because MMA is asynchronous, `__syncwarp()` or an ordinary `mbarrier.arrive` after issue does not make the operands reusable. Commit prior tcgen05 work to an mbarrier and wait for completion before releasing or overwriting the stage.\n5. Before another role reads D from TMEM, complete the MMA sequence and apply the required `tcgen05.fence::before_thread_sync` / execution-ordering handoff / `tcgen05.fence::after_thread_sync` protocol. A fence orders operations; it is not a completion wait.\n6. Keep TMEM allocated until every consumer finishes its `tcgen05.ld` sequence and associated waits, then deallocate with the required collective participation.\n\nPipeline wrappers in CUTLASS/CuTe encode parts of these rules, but participant counts, initial phases, tails, and producer/consumer states remain configuration-specific. A short role-dispatch sketch is not a substitute for the complete pipeline and TMEM lifecycle."}, "reason": {"statement": "Replace incomplete PTX with named obligations and authoritative links.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-dealloc"]}} +{"path": "wiki/techniques/warp-specialization.md", "before": {"statement": "In CUTLASS 4.5.0, the SM100 GEMM collective (`CollectiveMma_1SM`) implements this pattern with CuTe abstractions:\n\n```cuda\n// CUTLASS SM100 warp role dispatch (simplified from CollectiveMma)\n// Template parameter WarpCount = cute::Shape<1, 1, 14>\n// Warp 0 = producer, Warp 1 = math, Warps 2-15 = epilogue\n\ntemplate \nstruct CollectiveMma_1SM {\n static constexpr int NumProducerWarps = 1;\n static constexpr int NumMathWarps = 1;\n static constexpr int NumEpilogueWarps = 14;\n\n CUTLASS_DEVICE void operator()(\n Params const& params,\n char* smem_buf,\n TiledMma& tiled_mma)\n {\n int warp_idx = cutlass::canonical_warp_idx_sync();\n\n if (warp_idx == 0) {\n producer_warp(params, smem_buf);\n } else if (warp_idx == 1) {\n math_warp(params, smem_buf, tiled_mma);\n } else {\n epilogue_warp(params, smem_buf);\n }\n }\n};\n```"}, "after": {"statement": "There is no universal “warp 0 load, warp 1 MMA, warps 2–15 epilogue” rule. Three primary implementations illustrate the range:\n\n| Gau Nernst tutorial v4, commit `3b90ac9b...` | 4 warps. An elected lane in warp 0 runs the TMA loop; an elected lane in warp 1 runs the MMA loop; after completion, all four warps participate in TMEM load/conversion/output. |\n| CUTLASS 4.5.0 generic SM100 GEMM | One warp each for `MMA`, `Sched`, `MainloopLoad`, and `EpilogueLoad`, followed by `CollectiveEpilogue::ThreadCount / 32` epilogue warps. Optional work can make some control roles nonparticipants. |\n| FlashInfer PR 1039 context FMHA | 16 warps: 0–3 `Softmax0`, 4–7 `Softmax1`, 8–11 correction, 12 MMA, 13 load, 14 epilogue, and 15 empty. The kernel has explicit pipelines between these roles. |\n\nFlashAttention-4's pinned SM100 forward source likewise starts from a 16-warp layout with two four-warp softmax groups, four correction warps, and dedicated MMA/load/epilogue IDs, then adjusts roles for configuration choices such as one Q stage, non-TMA paths, and dynamic persistence. These are concrete attention schedules, not a GEMM-wide architectural template."}, "reason": {"statement": "Use exact source names and line-level descriptions instead of fabricated code.", "urls": []}} +{"path": "wiki/techniques/warp-specialization.md", "before": {"statement": "- **Always on Blackwell GEMMs**: Warp specialization is the standard pattern for SM100 tensor core kernels. The tcgen05 instruction model assumes single-thread dispatch with TMEM output."}, "after": {"statement": "For the author's exact `M=N=K=4096` Modal B200 setup with PyTorch 2.9.1 and CUDA 13, tutorial v3 reports 939.61 TFLOP/s and v4 reports 1208.83 TFLOP/s after introducing warp specialization. That is a 269.22-TFLOP/s, approximately 28.65% step between those source variants. It is not a portable Blackwell speedup or evidence that specialization is always profitable."}, "reason": {"statement": "Present specialization as a candidate optimization with an isolated source-reported example.", "urls": []}} +{"path": "wiki/techniques/warp-specialization.md", "before": {"statement": "- **Any kernel with producer-consumer pipeline**: When TMA loads and MMA compute can overlap, warp specialization provides the cleanest decomposition."}, "after": {"statement": "For the author's exact `M=N=K=4096` Modal B200 setup with PyTorch 2.9.1 and CUDA 13, tutorial v3 reports 939.61 TFLOP/s and v4 reports 1208.83 TFLOP/s after introducing warp specialization. That is a 269.22-TFLOP/s, approximately 28.65% step between those source variants. It is not a portable Blackwell speedup or evidence that specialization is always profitable."}, "reason": {"statement": "Supply selection criteria and controlled comparison instead of a universal adjective.", "urls": []}} +{"path": "wiki/techniques/warp-specialization.md", "before": {"statement": "- The 14 epilogue warps may be underutilized for simple epilogues (e.g., pure store). Complex epilogues (scale, bias, activation, quantization) benefit more."}, "after": {"statement": "For the author's exact `M=N=K=4096` Modal B200 setup with PyTorch 2.9.1 and CUDA 13, tutorial v3 reports 939.61 TFLOP/s and v4 reports 1208.83 TFLOP/s after introducing warp specialization. That is a 269.22-TFLOP/s, approximately 28.65% step between those source variants. It is not a portable Blackwell speedup or evidence that specialization is always profitable."}, "reason": {"statement": "Generalize to measurable role imbalance and configuration-derived counts.", "urls": []}} +{"path": "wiki/techniques/warp-specialization.md", "before": {"statement": "- The single MMA warp means the kernel cannot overlap multiple independent MMA streams within a CTA. Use 2-SM cooperative mode for larger tiles instead."}, "after": {"statement": "For the author's exact `M=N=K=4096` Modal B200 setup with PyTorch 2.9.1 and CUDA 13, tutorial v3 reports 939.61 TFLOP/s and v4 reports 1208.83 TFLOP/s after introducing warp specialization. That is a 269.22-TFLOP/s, approximately 28.65% step between those source variants. It is not a portable Blackwell speedup or evidence that specialization is always profitable."}, "reason": {"statement": "Remove the false causal rule and make 1SM/2SM an independently benchmarked configuration.", "urls": []}} +{"path": "wiki/techniques/cccl-memory-primitives.md", "before": {"statement": "Use CCCL/CUB PRs when the bottleneck is not tensor math but a memory primitive:\nscan, top-k selection, fill, histogram, reduce, or block load/store policy. The\ngoal is usually a policy or dispatch idea rather than copying a full CUB\nprimitive into an application kernel."}, "after": {"statement": "The two linked PRs have different evidentiary value:\n\n| CCCL 3559, captured merge `25523da2` | Adds B200/SM100 exclusive-sum scan tuning, expands policy classification to input/output/accumulator/offset types, and updates scan dispatch/tests. | A concrete example of architecture- and type-specific CUB policy selection. |\n| CCCL 6152, captured merge `3fb05826` | Changes only `CUB_DEBUG_LOG` formatting and stale variable names in `DispatchTopK`. | Evidence for the corrected debug output, not TopK algorithm, performance, determinism, or SM100 tuning. |\n\nDo not use PR 6152 as evidence for DSA TopK design or performance. Its captured key file exposes surrounding TopK implementation for inspection, but the PR's semantic delta is the small debug-only patch.\n\n1. Identify the exact primitive and semantic contract: inclusive/exclusive scan, operator, input/output/accumulator types, offset width, aliasing, and empty/large-size behavior.\n2. Follow runtime architecture dispatch to the active policy. Confirm whether SM100 has a matching specialization or inherits the SM90/default route.\n3. Reproduce the upstream baseline and changed policy over the size/type/operator matrix relevant to the application. A policy comment containing benchmark ratios is source context, not a portable speedup guarantee.\n4. Change one policy dimension at a time where practical and record time, bandwidth, occupancy, register/spill data, and correctness.\n5. For selection, separately verify membership, output count, ordering, ties, NaNs, signed zero, key/value association, and repeatability. Those properties are not established by PR 6152.\n\nThe linked evidence does not support claims about fill, histogram, reduce, block-load/store vectorization, or application-level DSA score computation. Add a directly relevant CCCL source before transferring policy conclusions to those primitives."}, "reason": {"statement": "Teach the exact reusable decisions and limitations of each PR.", "urls": []}} +{"path": "wiki/techniques/cccl-memory-primitives.md", "before": {"statement": "```cuda\n// Minimal policy probe shape for an application-specific top-k or scan helper.\ntemplate \nstruct PrimitivePolicy {\n static constexpr int block_threads = BLOCK_THREADS;\n static constexpr int items_per_thread = ITEMS_PER_THREAD;\n static constexpr bool vectorized = ITEMS_PER_THREAD >= 4;\n};\n```"}, "after": {"statement": "The new `sm100_tuning` specializations select a tuple rather than a single “vectorized” bit:\n\n| Classification | input value size, accumulator type/size, offset size, and recognized `plus` operator |\n| Work partition | `threads` and `items` per thread |\n| Memory policy | `BLOCK_LOAD_*`, `BLOCK_STORE_*`, and `LOAD_DEFAULT` or `LOAD_CA` |\n| Look-back behavior | an exponential backoff/delay constructor and its parameters |\n| Fallback | `Policy1000` selects a matching SM100 specialization; otherwise it falls back to `Policy900`; the double specialization explicitly inherits an SM90 tuning |\n\n`items >= 4` does not establish vectorized access. Vector width also depends on iterator contiguity, element type, alignment, load/store algorithm, compiler lowering, and the executed policy. Treat items-per-thread, block size, load/store algorithm, cache modifier, and delay policy as separate variables."}, "reason": {"statement": "Replace fabricated policy code with an exact field matrix from the PR.", "urls": []}} +{"path": "wiki/techniques/external-source-map-research.md", "before": {"statement": "```bash\ngit clone https://github.com/ColfaxResearch/cfx-article-src external/colfax-cfx\ngit clone https://github.com/simveit/load_and_store external/simveit-load-store\nrg -n \"tma|mbarrier|swizzle|ldmatrix|stream\" external/colfax-cfx external/simveit-load-store\n```"}, "after": {"statement": "```bash\ngit init external/colfax-cfx\ngit -C external/colfax-cfx remote add origin \\\n https://github.com/ColfaxResearch/cfx-article-src\ngit -C external/colfax-cfx fetch --depth=1 origin \\\n fbecfed88de2e4246f104a023188ba722937c5fc\ngit -C external/colfax-cfx checkout --detach FETCH_HEAD\ntest \"$(git -C external/colfax-cfx rev-parse HEAD)\" = \\\n fbecfed88de2e4246f104a023188ba722937c5fc\n\nrg -n \"SM90_TMA|mbarrier\" \\\n external/colfax-cfx/tma external/colfax-cfx/pipeline-gemm\nrg -n \"PersistentTileScheduler|StreamK\" external/colfax-cfx/streamk\n```"}, "reason": {"statement": "Pin every checkout and show mechanism-specific file searches.", "urls": []}} +{"path": "wiki/techniques/external-source-map-research.md", "before": {"statement": "Do not cite this page as implementation evidence by itself. Cite one of its\nsource pages plus the concrete upstream file path, commit, or URL that shaped the\ncandidate edit."}, "after": {"statement": "- For tail waves, compare the pinned non-persistent, data-parallel persistent,\n and Stream-K scheduler implementations before adding shape dispatch. Profile\n the target shapes rather than treating persistence as a universal remedy."}, "reason": {"statement": "Require a pinned upstream revision and exact file/symbol locator, treating source pages as discovery leads only.", "urls": []}} +{"path": "wiki/techniques/external-source-map-research.md", "before": {"statement": "Use external source-map research after a profile or benchmark identifies an edit\nfamily but the local PR pages do not expose a small enough implementation\nexample. The route is code-first: clone a source-map repository, grep for the\nmeasured mechanism, and cite exact files before adapting an idea."}, "after": {"statement": "Use external source-map research after a profile or benchmark identifies an edit\nfamily but the local PR pages do not expose a small enough implementation\nexample. Treat the profiler result as a search key, not a root-cause proof:\ninspect a pinned upstream revision and exact implementation files before\nadapting an idea.\n\n- For tail waves, compare the pinned non-persistent, data-parallel persistent,\n and Stream-K scheduler implementations before adding shape dispatch. Profile\n the target shapes rather than treating persistence as a universal remedy."}, "reason": {"statement": "Scope demonstrated implementations to SM90a and make Blackwell transfer require separate SM100 evidence.", "urls": []}} +{"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "Fine-grained quantization applies per-block (rather than per-tensor) scaling factors to low-precision data, preventing outlier values from destroying the quantization precision of an entire tensor. DeepSeek pioneered the tile-wise 1x128 scaling for activations and block-wise 128x128 scaling for weights in their FP8 training framework. On Blackwell (SM100), native block scaling support in tcgen05.mma enables hardware-accelerated fine-grained quantization using the UE8M0 scale format, while Hopper requires software-managed CUDA core promotion."}, "after": {"statement": "Fine-grained quantization associates low-precision payloads with scales for\nsubsets of a tensor. Smaller groups let scale selection respond more locally,\nbut they do not guarantee a particular error or speed result. Keep four choices\nseparate when transferring a kernel: payload type, scale type, scale geometry,\nand physical scale layout.\n\nThree verified recipes in this wiki illustrate why the full contract matters:\n\n- **DeepSeek FP8 training geometry:** one scale for each token row and 128\n activation channels (`1 x 128`), and one scale for each 128-input-channel by\n 128-output-channel weight block (`128 x 128`).\n- **NVFP4 1D recipe:** E2M1 payloads, one E4M3 local scale per 16 consecutive\n values, and one FP32 global scale per tensor. Transformer Engine 2.13 also\n defines a weight-oriented 2D mode with one local scale per `16 x 16` block.\n- **MXFP4:** E2M1 payloads with one UE8M0 power-of-two scale per 32 values; the\n NVFP4 per-tensor global scale is not part of this microscaling format."}, "reason": {"statement": "Separate recipe, implementation and architecture scopes.", "urls": ["https://arxiv.org/abs/2412.19437v2", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"]}} +{"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "```\nPer-tensor scaling (coarsest):\n Entire tensor shares one FP32 scale factor.\n Problem: a single outlier ruins precision for all elements.\n \n [=== entire MxK matrix === ] -> 1 scale\n\nPer-block 128x128 scaling (weights):\n Each 128x128 block has its own scale factor.\n 128x128 = 16,384 elements per scale -> 0.006% overhead.\n \n +---+---+---+\n |s1 |s2 |s3 | <- each block has independent scale\n +---+---+---+\n |s4 |s5 |s6 |\n +---+---+---+\n\nPer-tile 1x128 scaling (activations):\n Each row of 128 elements has its own scale factor.\n Captures per-channel activation distributions.\n \n [s1: ====128 elements====]\n [s2: ====128 elements====]\n [s3: ====128 elements====]\n```"}, "after": {"statement": "Scale-count and storage overhead are different quantities. A `128 x 128` block\nhas one factor per 16,384 payloads, a factor-count ratio of about 0.0061%. If\nthe payload is one byte and the factor is FP32, the byte ratio is instead\n`4 / 16384`, about 0.0244%. Include factor width, payload width, padding, and\nlayout transformations in any storage or bandwidth claim."}, "reason": {"statement": "State geometry and formulas without an ambiguous percentage or per-channel label.", "urls": []}} +{"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "```cuda\n// DeepGEMM FP8 GEMM with fine-grained scaling\n// A (activations): FP8 E4M3 with tile-wise 1x128 FP32 scales\n// B (weights): FP8 E4M3 with block-wise 128x128 FP32 scales\n// C (output): FP32 accumulator\n\n// Scale tensor shapes:\n// scale_A: [M, K/128] -- one FP32 scale per 128 elements along K for each row\n// scale_B: [K/128, N/128] -- one FP32 scale per 128x128 block\n\n__device__ void deepgemm_fp8_tile(\n const fp8_e4m3* A, // [TILE_M, TILE_K] in FP8\n const fp8_e4m3* B, // [TILE_K, TILE_N] in FP8\n const float* scale_A, // [TILE_M, TILE_K/128]\n const float* scale_B, // [TILE_K/128, TILE_N/128]\n float* C, // [TILE_M, TILE_N] accumulator\n int M, int N, int K)\n{\n // For each 128-element K-chunk:\n for (int k_block = 0; k_block < K; k_block += 128) {\n // 1. Load FP8 A tile [TILE_M, 128] and B tile [128, TILE_N]\n // 2. Perform MMA: partial = A_fp8 * B_fp8 (in limited-precision FP32)\n // 3. Apply combined scale: scale_A[m, k_block/128] * scale_B[k_block/128, n_block/128]\n // 4. Accumulate: C[m, n] += partial * combined_scale\n\n for (int m = 0; m < TILE_M; m++) {\n float sa = scale_A[m * (K / 128) + k_block / 128];\n for (int n_block = 0; n_block < TILE_N; n_block += 128) {\n float sb = scale_B[(k_block / 128) * (N / 128) + n_block / 128];\n float combined_scale = sa * sb;\n\n // Apply scale to the partial MMA result\n for (int n = n_block; n < n_block + 128; n++) {\n C[m * TILE_N + n] += partial[m][n] * combined_scale;\n }\n }\n }\n }\n}\n```"}, "after": {"statement": "At commit `891d57b4db1071624b5c8fa0d1e51cb317fa709f`, DeepGEMM uses\narchitecture-specific scale representations and accumulation paths.\n\nThe pinned SM90 1D1D kernel consumes FP32 A/B factors and fixes\n`BLOCK_K == 128`. The DeepSeek-V3 report characterizes the H800 FP8 Tensor Core\npath as retaining about 14 bits and describes this interval as four WGMMAs in\nits configuration. The exact kernel accumulates one K block in `float accum`\nand then applies both factors into a separate `float final_accum`:\n\n```cpp\nfinal_accum[i * 4 + 0] += scale_a_0 * scale_b_0 * accum[i * 4 + 0];\nfinal_accum[i * 4 + 1] += scale_a_0 * scale_b_1 * accum[i * 4 + 1];\nfinal_accum[i * 4 + 2] += scale_a_1 * scale_b_0 * accum[i * 4 + 2];\nfinal_accum[i * 4 + 3] += scale_a_1 * scale_b_1 * accum[i * 4 + 3];\n```\n\nThis is a source-specific implementation, not a rule for every Hopper FP8\nkernel. The cited sources do not establish the deleted 0.1% error bound or a\nuniversal ranking of `Nc=32`, `64`, `128`, and `256`."}, "reason": {"statement": "Use the pinned implementation and a short verbatim fragment instead.", "urls": ["https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh", "https://arxiv.org/abs/2412.19437v2"]}} +{"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "On Hopper (SM90), the wgmma instruction accumulates in Tensor Core registers with limited precision (~FP22, not true FP32). To maintain numerical accuracy, DeepGEMM promotes the partial sum to a separate FP32 accumulator on CUDA Cores every 4 wgmma instructions (Nc=128, since each wgmma processes 32 K-elements):"}, "after": {"statement": "At commit `891d57b4db1071624b5c8fa0d1e51cb317fa709f`, DeepGEMM uses\narchitecture-specific scale representations and accumulation paths.\n\nThe pinned SM90 1D1D kernel consumes FP32 A/B factors and fixes\n`BLOCK_K == 128`. The DeepSeek-V3 report characterizes the H800 FP8 Tensor Core\npath as retaining about 14 bits and describes this interval as four WGMMAs in\nits configuration. The exact kernel accumulates one K block in `float accum`\nand then applies both factors into a separate `float final_accum`:\n\n```cpp\nfinal_accum[i * 4 + 0] += scale_a_0 * scale_b_0 * accum[i * 4 + 0];\nfinal_accum[i * 4 + 1] += scale_a_0 * scale_b_1 * accum[i * 4 + 1];\nfinal_accum[i * 4 + 2] += scale_a_1 * scale_b_0 * accum[i * 4 + 2];\nfinal_accum[i * 4 + 3] += scale_a_1 * scale_b_1 * accum[i * 4 + 3];\n```\n\nThis is a source-specific implementation, not a rule for every Hopper FP8\nkernel. The cited sources do not establish the deleted 0.1% error bound or a\nuniversal ranking of `Nc=32`, `64`, `128`, and `256`."}, "reason": {"statement": "Use the source's retained-bit wording and implementation scope.", "urls": ["https://arxiv.org/abs/2412.19437v2", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh"]}} +{"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "```cuda\n// Hopper FP8 GEMM with CUDA Core promotion (DeepGEMM pattern)\n// Every Nc=128 K-elements, promote Tensor Core accumulator to FP32\n\n__device__ void hopper_fp8_gemm_with_promotion(\n const fp8_e4m3* A, const fp8_e4m3* B,\n const float* scale_A, const float* scale_B,\n float* C_accumulator,\n int K)\n{\n // Tensor Core limited-precision accumulator (FP22-ish)\n // These are wgmma output registers\n float tc_acc[TILE_M_PER_THREAD][TILE_N_PER_THREAD] = {0};\n\n // True FP32 accumulator on CUDA Cores\n float fp32_acc[TILE_M_PER_THREAD][TILE_N_PER_THREAD] = {0};\n\n int promotion_interval = 128; // Nc = 128 elements = 4 wgmma ops\n int wgmma_count = 0;\n\n for (int k = 0; k < K; k += 32) {\n // Issue wgmma (accumulates in limited-precision tc_acc)\n wgmma_mma_async(tc_acc, smem_A_ptr, smem_B_ptr);\n wgmma_count++;\n\n if (wgmma_count == 4) { // Every 128 K-elements\n // Promote: transfer tc_acc to fp32_acc on CUDA Cores\n wgmma_wait(); // Ensure wgmma is complete\n\n int k_block = k / promotion_interval;\n for (int m = 0; m < TILE_M_PER_THREAD; m++) {\n float sa = scale_A[/*...*/];\n for (int n = 0; n < TILE_N_PER_THREAD; n++) {\n float sb = scale_B[/*...*/];\n // Add scaled partial to true FP32 accumulator\n fp32_acc[m][n] += tc_acc[m][n] * sa * sb;\n // Reset TC accumulator for next interval\n tc_acc[m][n] = 0;\n }\n }\n wgmma_count = 0;\n }\n }\n}\n```"}, "after": {"statement": "At commit `891d57b4db1071624b5c8fa0d1e51cb317fa709f`, DeepGEMM uses\narchitecture-specific scale representations and accumulation paths.\n\nThe pinned SM90 1D1D kernel consumes FP32 A/B factors and fixes\n`BLOCK_K == 128`. The DeepSeek-V3 report characterizes the H800 FP8 Tensor Core\npath as retaining about 14 bits and describes this interval as four WGMMAs in\nits configuration. The exact kernel accumulates one K block in `float accum`\nand then applies both factors into a separate `float final_accum`:\n\n```cpp\nfinal_accum[i * 4 + 0] += scale_a_0 * scale_b_0 * accum[i * 4 + 0];\nfinal_accum[i * 4 + 1] += scale_a_0 * scale_b_1 * accum[i * 4 + 1];\nfinal_accum[i * 4 + 2] += scale_a_1 * scale_b_0 * accum[i * 4 + 2];\nfinal_accum[i * 4 + 3] += scale_a_1 * scale_b_1 * accum[i * 4 + 3];\n```\n\nThis is a source-specific implementation, not a rule for every Hopper FP8\nkernel. The cited sources do not establish the deleted 0.1% error bound or a\nuniversal ranking of `Nc=32`, `64`, `128`, and `256`."}, "reason": {"statement": "Replace it with an exact upstream fragment and workflow.", "urls": ["https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh", "https://arxiv.org/abs/2412.19437v2"]}} +{"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "The Nc=128 interval was chosen because:\n- 4 wgmma operations process 128 K-elements (4 x 32)\n- At this interval, the accumulated FP22 error is bounded to ~0.1% relative error\n- Fewer than 4 ops (Nc=32, Nc=64) adds too much promotion overhead\n- More than 4 ops (Nc=256) allows unacceptable precision loss"}, "after": {"statement": "At commit `891d57b4db1071624b5c8fa0d1e51cb317fa709f`, DeepGEMM uses\narchitecture-specific scale representations and accumulation paths.\n\nThe pinned SM90 1D1D kernel consumes FP32 A/B factors and fixes\n`BLOCK_K == 128`. The DeepSeek-V3 report characterizes the H800 FP8 Tensor Core\npath as retaining about 14 bits and describes this interval as four WGMMAs in\nits configuration. The exact kernel accumulates one K block in `float accum`\nand then applies both factors into a separate `float final_accum`:\n\n```cpp\nfinal_accum[i * 4 + 0] += scale_a_0 * scale_b_0 * accum[i * 4 + 0];\nfinal_accum[i * 4 + 1] += scale_a_0 * scale_b_1 * accum[i * 4 + 1];\nfinal_accum[i * 4 + 2] += scale_a_1 * scale_b_0 * accum[i * 4 + 2];\nfinal_accum[i * 4 + 3] += scale_a_1 * scale_b_1 * accum[i * 4 + 3];\n```\n\nThis is a source-specific implementation, not a rule for every Hopper FP8\nkernel. The cited sources do not establish the deleted 0.1% error bound or a\nuniversal ranking of `Nc=32`, `64`, `128`, and `256`.\n\n1. Match the model/checkpoint recipe: payload encoding, local scale type,\n optional global scale, and scale geometry.\n2. Match the kernel ABI: logical factor shape is not necessarily its TMA- or\n TMEM-ready physical layout. Account for packing, padding, transposition, and\n swizzling.\n3. Match the target instruction. For native Blackwell block scaling, use the\n complete kind, block/scale-vector qualifier, scale type, and target rules.\n4. Keep preparation in the timed region unless the producer already emits the\n required layout. Otherwise report preprocessing separately.\n5. Validate output accuracy with the exact quantizer, workload, reference,\n tolerance, and accumulation path; then measure end-to-end latency or\n throughput for the actual shapes.\n\n- More groups require more scale elements, but padding and layout determine the\n actual traffic and storage cost.\n- Smaller groups and fractional scales provide more representational freedom;\n they do not guarantee lower error for every tensor or scale-selection method.\n- DeepGEMM's Nc=128 path is evidence for that pinned SM90 implementation, not a\n universal optimum for Hopper.\n- Native instruction support does not make two recipes ABI-compatible. NVFP4,\n MXFP4, and DeepGEMM FP8 differ in payload, scale type, grouping, and layout."}, "reason": {"statement": "Retain only the exact source-supported interval and prohibit transfer without measurement.", "urls": ["https://arxiv.org/abs/2412.19437v2", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh", "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "```cuda\n// Blackwell native block scaling with UE8M0\n// Scale format: UE8M0 = pure power-of-two scale (2^exponent)\n// Packed: 4 UE8M0 values per 32-bit integer\n\n// DeepGEMM SM100 kernel: scale_A and scale_B are UE8M0 packed\n// tcgen05.mma applies scales automatically during accumulation\n\nstruct BlockScaleDescriptor {\n // 4 UE8M0 scale values packed into one uint32\n // Each UE8M0 is an 8-bit unsigned exponent: value = 2^(e - 127)\n uint32_t packed_scales; // Contains 4 block scales\n\n // Dequantization for block [i]:\n // scale_i = 2^(((packed >> (i*8)) & 0xFF) - 127)\n};\n\n// PTX for tcgen05.mma with block scaling:\n// The .scale modifier tells the hardware to apply UE8M0 scales\n// from a designated SMEM region alongside the MMA operands\n```"}, "after": {"statement": "The pinned SM100 1D1D template accepts K scale granularities of 32 or 128 for\neach operand. Its public interface packs four UE8M0 factors in each 32-bit\ncontainer. TMA moves factor blocks to shared memory, UTCCP copies them into\ndedicated TMEM SFA/SFB columns, and block-scaled UMMA consumes the TMEM scale\naddresses. The SM90 `final_accum` promotion loop is absent."}, "reason": {"statement": "Avoid an incomplete invented descriptor; cite the exact type definition and upstream representation.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#alternate-floating-point-data-formats", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"]}} +{"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "```ptx\n// tcgen05.mma with native block scaling (Blackwell PTX)\n// This instruction applies UE8M0 scales from SMEM during MMA\ntcgen05.mma.cta_group::1.kind::f8f6f4\n [%tmem_addr], // TMEM accumulator destination\n [%desc_a], // SMEM descriptor for A operand\n [%desc_b], // SMEM descriptor for B operand\n %scale_d, // Scale descriptor for D (output)\n %enable_mask,\n [%scale_a_smem], // UE8M0 scales for A in SMEM\n [%scale_b_smem]; // UE8M0 scales for B in SMEM\n```"}, "after": {"statement": "PTX ISA 9.0 expresses the corresponding FP8 block-scaled operand classes with\nthis grammar-level form:\n\n```ptx\ntcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.scale_vec::1X\n [d_tmem], a_desc, b_desc, idesc,\n [scale_a_tmem], [scale_b_tmem], enable_input_d;\n```\n\nThis line is an instruction-shape reference, not a complete kernel: declarations,\nlegal descriptors, collective participation, scale layouts, ordering, completion,\nand an architecture-specific target are still required. UE8M0 uses power-of-two\nfinite values and reserves encoding `0xff` for NaN."}, "reason": {"statement": "Show exact grammar-level PTX and label missing declarations/descriptors.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "| Representable values | Powers of 2 only | 240 distinct values | Full FP32 range |\n| Range | 2^-127 to 2^128 | ~0 to 448 | Full FP32 |"}, "after": {"statement": "The pinned SM100 1D1D template accepts K scale granularities of 32 or 128 for\neach operand. Its public interface packs four UE8M0 factors in each 32-bit\ncontainer. TMA moves factor blocks to shared memory, UTCCP copies them into\ndedicated TMEM SFA/SFB columns, and block-scaled UMMA consumes the TMEM scale\naddresses. The SM90 `final_accum` promotion loop is absent."}, "reason": {"statement": "Use recipe/type combinations without unsupported range/count shorthand.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#alternate-floating-point-data-formats", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"]}} +{"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "| Hardware support | tcgen05.mma native | Software decode | Software promotion |\n| Precision impact | Coarser (power-of-2 only) | Fine (non-power-of-2) | Best (FP32) |"}, "after": {"statement": "The pinned SM100 1D1D template accepts K scale granularities of 32 or 128 for\neach operand. Its public interface packs four UE8M0 factors in each 32-bit\ncontainer. TMA moves factor blocks to shared memory, UTCCP copies them into\ndedicated TMEM SFA/SFB columns, and block-scaled UMMA consumes the TMEM scale\naddresses. The SM90 `final_accum` promotion loop is absent.\n\nPTX ISA 9.0 expresses the corresponding FP8 block-scaled operand classes with\nthis grammar-level form:\n\n```ptx\ntcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.scale_vec::1X\n [d_tmem], a_desc, b_desc, idesc,\n [scale_a_tmem], [scale_b_tmem], enable_input_d;\n```\n\nThis line is an instruction-shape reference, not a complete kernel: declarations,\nlegal descriptors, collective participation, scale layouts, ordering, completion,\nand an architecture-specific target are still required. UE8M0 uses power-of-two\nfinite values and reserves encoding `0xff` for NaN."}, "reason": {"statement": "Separate native instruction legality from recipe accuracy and software choices.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#alternate-floating-point-data-formats"]}} +{"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "- **FP8 training**: Use tile-wise 1x128 for activations and block-wise 128x128 for weights (DeepGEMM pattern). This is the validated approach for training 671B+ parameter models.\n- **FP4 inference on Blackwell**: Use NVFP4 with E4M3 block scales (block size 16) for highest precision, or MXFP4 with UE8M0 scales (block size 32) for native hardware acceleration.\n- **Hopper FP8 inference**: Use CUDA core promotion with Nc=128 interval to maintain precision despite limited TC accumulation."}, "after": {"statement": "PTX ISA 9.0 distinguishes the native formats by the complete instruction\ncombination, not by “FP4” alone:\n\n| `.kind::mxf4.block_scale.block32` / `.scale_vec::2X` | E2M1 | UE8M0 | 32 |\n| `.kind::mxf4nvf4.block_scale.block16` / `.scale_vec::4X` | E2M1 | UE4M3 | 16 |\n\nThe `mxf4nvf4` family also has documented UE8M0 modes, so the kind name alone\ndoes not select the NVFP4 recipe. NVFP4-compatible UE4M3 scaling is native on\nthe documented Blackwell targets; it is not necessarily a software-decode path.\n\n1. Match the model/checkpoint recipe: payload encoding, local scale type,\n optional global scale, and scale geometry.\n2. Match the kernel ABI: logical factor shape is not necessarily its TMA- or\n TMEM-ready physical layout. Account for packing, padding, transposition, and\n swizzling.\n3. Match the target instruction. For native Blackwell block scaling, use the\n complete kind, block/scale-vector qualifier, scale type, and target rules.\n4. Keep preparation in the timed region unless the producer already emits the\n required layout. Otherwise report preprocessing separately.\n5. Validate output accuracy with the exact quantizer, workload, reference,\n tolerance, and accumulation path; then measure end-to-end latency or\n throughput for the actual shapes.\n\n- More groups require more scale elements, but padding and layout determine the\n actual traffic and storage cost.\n- Smaller groups and fractional scales provide more representational freedom;\n they do not guarantee lower error for every tensor or scale-selection method.\n- DeepGEMM's Nc=128 path is evidence for that pinned SM90 implementation, not a\n universal optimum for Hopper.\n- Native instruction support does not make two recipes ABI-compatible. NVFP4,\n MXFP4, and DeepGEMM FP8 differ in payload, scale type, grouping, and layout."}, "reason": {"statement": "Replace prescriptions with a recipe-selection checklist.", "urls": ["https://arxiv.org/abs/2412.19437v2", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"]}} +{"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "- UE8M0 scales are power-of-two only. Non-power-of-two distributions (common in activations) lose precision compared to E4M3 or FP32 scales.\n- Smaller block sizes (16 for NVFP4 vs 128 for DeepGEMM) provide better precision but higher overhead: more scale values to store, load, and apply.\n- The Nc=128 promotion interval on Hopper is a performance-accuracy tradeoff. Reducing Nc improves accuracy but adds more promotion overhead. Increasing Nc risks precision degradation.\n- On Blackwell, native block scaling only works with UE8M0. Using E4M3 or FP32 scales still requires software handling."}, "after": {"statement": "Scale-count and storage overhead are different quantities. A `128 x 128` block\nhas one factor per 16,384 payloads, a factor-count ratio of about 0.0061%. If\nthe payload is one byte and the factor is FP32, the byte ratio is instead\n`4 / 16384`, about 0.0244%. Include factor width, payload width, padding, and\nlayout transformations in any storage or bandwidth claim.\n\n1. Match the model/checkpoint recipe: payload encoding, local scale type,\n optional global scale, and scale geometry.\n2. Match the kernel ABI: logical factor shape is not necessarily its TMA- or\n TMEM-ready physical layout. Account for packing, padding, transposition, and\n swizzling.\n3. Match the target instruction. For native Blackwell block scaling, use the\n complete kind, block/scale-vector qualifier, scale type, and target rules.\n4. Keep preparation in the timed region unless the producer already emits the\n required layout. Otherwise report preprocessing separately.\n5. Validate output accuracy with the exact quantizer, workload, reference,\n tolerance, and accumulation path; then measure end-to-end latency or\n throughput for the actual shapes.\n\n- More groups require more scale elements, but padding and layout determine the\n actual traffic and storage cost.\n- Smaller groups and fractional scales provide more representational freedom;\n they do not guarantee lower error for every tensor or scale-selection method.\n- DeepGEMM's Nc=128 path is evidence for that pinned SM90 implementation, not a\n universal optimum for Hopper.\n- Native instruction support does not make two recipes ABI-compatible. NVFP4,\n MXFP4, and DeepGEMM FP8 differ in payload, scale type, grouping, and layout."}, "reason": {"statement": "State only metadata arithmetic and verified capability; require measurement for accuracy/performance.", "urls": ["https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"]}} +{"path": "wiki/techniques/kernel-fusion.md", "before": {"statement": "Kernel fusion combines multiple operations into a single kernel launch, eliminating intermediate global memory roundtrips. Critical for MoE and attention pipelines where 5-7 sequential launches each incur latency, synchronization, and memory traffic overhead."}, "after": {"statement": "Kernel fusion implements two or more dependent logical operations inside one\nGPU kernel launch. It can remove a launch or an intermediate global-memory\nwrite/read only when the chosen unfused baseline would perform that work and\nthe fused implementation keeps the value on chip. Fusion is therefore an\nimplementation property, not a conclusion that follows from an operation's\nname.\n\nReport a benefit only for a defined fused/unfused pair with identical input and\noutput semantics. Count actual launches and bytes for the pinned implementation;\ndo not transfer framework-level launch counts or traffic percentages from an\nunpinned configuration."}, "reason": {"statement": "Define fusion semantically and state conditional benefits.", "urls": []}} +{"path": "wiki/techniques/kernel-fusion.md", "before": {"statement": "```cuda\n// Instead of: gate_gemm → up_gemm → silu → multiply (4 kernels)\n// Fused: single kernel with two TMEM accumulators\n__global__ void fused_gate_up_silu(...) {\n uint32_t tmem_gate = tmem_alloc(256);\n uint32_t tmem_up = tmem_alloc(256);\n\n for (int k = 0; k < K; k += BLOCK_K) {\n tcgen05_mma(x_smem, w_gate_smem, tmem_gate);\n tcgen05_mma(x_smem, w_up_smem, tmem_up);\n }\n\n float g = tmem_load(tmem_gate);\n float u = tmem_load(tmem_up);\n output = (g / (1.0f + expf(-g))) * u; // SwiGLU fused\n}\n```"}, "after": {"statement": "GPU Mode NVFP4 Challenge 3 fixes this operation for each batch index:\n\n```python\ngate = scaled_mm(a, b1.T, sfa, sfb1)\nup = scaled_mm(a, b2.T, sfa, sfb2)\noutput = silu(gate) * up\n```\n\nThe two products share `a` and its scale tensor but use separate B operands and\nscales. This operation graph permits reuse and fused pointwise output, but the\npinned correctness reference does not prescribe launch count, TMEM partition,\nTMA pipeline, or instruction schedule. A concatenated gate/up projection or two\nlogical accumulators can both implement compatible observable semantics when\ntheir tensor contracts match."}, "reason": {"statement": "Use the exact operation reference and keep schedule choices separate.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", "https://github.com/gpu-mode/reference-kernels/blob/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm/reference.py"]}} +{"path": "wiki/techniques/kernel-fusion.md", "before": {"statement": "- **vLLM (7 kernels)**: softmax → topk → dispatch → gate → up → silu_mul → down+combine\n- **SGLang (5 kernels)**: router+topk → dispatch → fused gate-up-silu → down → combine\n- **Ideal (1-2 kernels)**: all ops in one launch, saves 21.9% activation memory traffic"}, "after": {"statement": "FlashInfer MLSys 2026 Track A includes sigmoid routing, grouped expert\nselection, two grouped GEMMs, SwiGLU, and weighted expert accumulation in one\nlogical benchmark definition. The exact reference performs one concatenated\nW13 projection, splits its 4096 columns into gate and up halves, applies\nSwiGLU, then performs W2. The term “Fused MoE” does not establish that a\nsubmission uses one launch or keeps every intermediate on chip.\n\nFor correctness, compare the fused and unfused paths on identical tensors and\nexercise empty/small groups, partial tiles, extreme activations, routing ties,\nand every supported scale/layout variant. Add negative tests for premature\nbuffer reuse, omitted completion, wrong routing weights, and invalid output\nedges."}, "reason": {"statement": "Do not preserve unpinned framework-specific performance claims.", "urls": ["https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048", "https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/trace/templates/moe.py"]}} +{"path": "wiki/techniques/kernel-fusion.md", "before": {"statement": "- TMEM capacity limits how many accumulators can fuse (256 cols total)"}, "after": {"statement": "- **Semantics:** preserve routing, scale, bias, activation, reduction,\n accumulation, dtype, ordering, and output-edge behavior. A pointwise epilogue\n is easier to compose than a cross-CTA reduction or global routing decision.\n- **Communication:** identify the producer and every consumer of each\n intermediate. Use the synchronization scope and memory space actually needed;\n fusion is not restricted to one CTA, but multi-CTA communication has its own\n legality and completion rules.\n- **Resources:** record registers, spills, SMEM, TMEM, barriers, threads, cluster\n shape, and occupancy. Added live values can raise register or shared-memory\n pressure even when a global intermediate disappears.\n- **TMEM:** SM100 TMEM is 128 lanes by 512 columns of 32-bit cells per SM.\n Simultaneously live regions must fit, use legal collective allocation sizes,\n remain live through every asynchronous access, and be collectively freed.\n Two 256-column halves are one possible allocation policy, not the total\n hardware capacity or a dual-GEMM requirement.\n- **Fallbacks:** keep unfused or differently fused paths for shapes, dtypes,\n layouts, reductions, or resource footprints that the fused schedule cannot\n implement safely."}, "reason": {"statement": "State 512-column capacity and implementation-specific live-region accounting.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} +{"path": "wiki/techniques/kernel-fusion.md", "before": {"statement": "- Fusion opportunities depend on dataflow shape (dependency graph must be DAG-compatible with CTA scope)"}, "after": {"statement": "- **Semantics:** preserve routing, scale, bias, activation, reduction,\n accumulation, dtype, ordering, and output-edge behavior. A pointwise epilogue\n is easier to compose than a cross-CTA reduction or global routing decision.\n- **Communication:** identify the producer and every consumer of each\n intermediate. Use the synchronization scope and memory space actually needed;\n fusion is not restricted to one CTA, but multi-CTA communication has its own\n legality and completion rules.\n- **Resources:** record registers, spills, SMEM, TMEM, barriers, threads, cluster\n shape, and occupancy. Added live values can raise register or shared-memory\n pressure even when a global intermediate disappears.\n- **TMEM:** SM100 TMEM is 128 lanes by 512 columns of 32-bit cells per SM.\n Simultaneously live regions must fit, use legal collective allocation sizes,\n remain live through every asynchronous access, and be collectively freed.\n Two 256-column halves are one possible allocation policy, not the total\n hardware capacity or a dual-GEMM requirement.\n- **Fallbacks:** keep unfused or differently fused paths for shapes, dtypes,\n layouts, reductions, or resource footprints that the fused schedule cannot\n implement safely."}, "reason": {"statement": "Replace vague scope jargon with explicit correctness/resource questions.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} +{"path": "wiki/techniques/register-budgeting.md", "before": {"statement": "SM occupancy is inversely proportional to registers-per-thread. For memory-bound kernels, higher occupancy = more warps to hide memory latency. `-maxrregcount` and `__launch_bounds__` force the compiler to stay within a budget."}, "after": {"statement": "Registers per thread are one input to residency. Threads per block, static and dynamic shared memory, architectural limits, allocation granularity, barriers, and cluster configuration can impose other limits. Register thresholds therefore produce discrete changes in active blocks or warps; occupancy is not generally the reciprocal of a source-level register count, and higher predicted occupancy does not by itself imply lower latency.\n\nThe CUDA occupancy API evaluates the compiled function with an intended block size and dynamic shared-memory size. `cudaOccupancyMaxActiveBlocksPerMultiprocessor` returns the maximum active blocks per SM for that configuration. Convert that result to active warps only after multiplying by the actual warps per block.\n\nFor tcgen05 MMA, the resident D accumulator is in TMEM rather than a per-thread D register vector. This changes one major register consumer, but it does not erase register pressure: descriptors, addresses, pipeline/control state, operands, and epilogue batches still use general-purpose registers. The pinned SM100 DeepGEMM path explicitly loads completed TMEM values into registers before its shared/global-store epilogue.\n\nTreat the removed D vector as shape-specific. For example, an SM90 m64n256 FP32 WGMMA fragment exposes 128 D registers per participating thread, but that number is not a universal Blackwell saving and does not predict a resident-block transition."}, "reason": {"statement": "Use the resource-threshold model and scope the TMEM effect to the resident D fragment.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-runtime-api/group__CUDART__OCCUPANCY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/arch/mma_sm90_gmma.hpp"]}} +{"path": "wiki/techniques/register-budgeting.md", "before": {"statement": "```cuda\n// Aggressive: 32 registers/thread → ~4 blocks per SM at 256 threads/block\n__launch_bounds__(256, 4)\n__global__ void gemv_memory_bound(...) {\n // Compiler will spill to local memory if needed\n}\n\n// Or via nvcc flag:\n// nvcc -maxrregcount=32 -arch=sm_100a ...\n```"}, "after": {"statement": "| `--maxrregcount=N` | Sets a maximum for GPU functions, subject to the ABI minimum and compiler-reserved registers | That the compiler will use exactly `N`, or that residency/performance will change |\n| `__launch_bounds__(T, B)` | Supplies maximum threads per block and desired minimum blocks per SM; the compiler derives an architecture-dependent register limit `L` | Exactly `B` resident blocks when shared memory or another resource is limiting |\n| `--resource-usage` | Reports registers, stack frame, spill loads/stores, and other resources for compiled functions | Runtime occupancy or the cost of those resources |\n| Occupancy API | Predicts maximum concurrent blocks for the compiled function and launch inputs | Achieved latency, bandwidth, or throughput |\n\nWhen launch bounds make initial register use exceed `L`, CUDA documents that the compiler usually trades registers for more local-memory use and/or instructions. When both `T` and `B` are supplied and initial use is below `L`, the compiler may instead increase register use up to `L` to reduce instruction count. A launch bound is therefore compiler guidance and a launch constraint, not an exact register assignment.\n\nCompile the same source and target first without a cap, then with candidate caps. `--resource-usage` makes a cap that does not change the compiled allocation visible.\n\n```bash\nnvcc -arch=sm_100a --resource-usage kernel.cu -o kernel-default\nnvcc -arch=sm_100a --maxrregcount=64 --resource-usage kernel.cu -o kernel-r64\n```\n\nFor each binary and each supported production shape:\n\n1. Record toolkit, target, compiler options, registers per thread, stack-frame bytes, spill loads/stores, static/dynamic shared memory, block size, and cluster shape.\n2. Query active blocks with the occupancy API using the exact function, block size, and dynamic shared-memory bytes. Identify which resource is actually limiting residency.\n3. Verify that the candidate cap changed generated resources or code before attributing a timing change to it. Inspect PTX/SASS when instruction selection or loop shape may have changed.\n4. Run the same correctness oracle, warmup, synchronization, inputs, and repeated-trial statistic for every variant. Profile spill traffic, memory stalls, and achieved warps rather than assuming the predicted occupancy is reached or useful.\n5. Keep a cap only for the measured target and workload where it improves the declared metric without violating correctness or a resource contract."}, "reason": {"statement": "The fenced snippet is incomplete CUDA and encodes false numeric implications.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html"]}} +{"path": "wiki/techniques/register-budgeting.md", "before": {"statement": "Lower register count → compiler may:\n- Spill frequently-used values to local memory (bad)\n- Recompute values instead of storing them (neutral)\n- Use fewer unrolled iterations (bad for compute-bound)"}, "after": {"statement": "| `--maxrregcount=N` | Sets a maximum for GPU functions, subject to the ABI minimum and compiler-reserved registers | That the compiler will use exactly `N`, or that residency/performance will change |\n| `__launch_bounds__(T, B)` | Supplies maximum threads per block and desired minimum blocks per SM; the compiler derives an architecture-dependent register limit `L` | Exactly `B` resident blocks when shared memory or another resource is limiting |\n| `--resource-usage` | Reports registers, stack frame, spill loads/stores, and other resources for compiled functions | Runtime occupancy or the cost of those resources |\n| Occupancy API | Predicts maximum concurrent blocks for the compiled function and launch inputs | Achieved latency, bandwidth, or throughput |\n\nWhen launch bounds make initial register use exceed `L`, CUDA documents that the compiler usually trades registers for more local-memory use and/or instructions. When both `T` and `B` are supplied and initial use is below `L`, the compiler may instead increase register use up to `L` to reduce instruction count. A launch bound is therefore compiler guidance and a launch constraint, not an exact register assignment.\n\nCompile the same source and target first without a cap, then with candidate caps. `--resource-usage` makes a cap that does not change the compiled allocation visible.\n\n```bash\nnvcc -arch=sm_100a --resource-usage kernel.cu -o kernel-default\nnvcc -arch=sm_100a --maxrregcount=64 --resource-usage kernel.cu -o kernel-r64\n```\n\nFor each binary and each supported production shape:\n\n1. Record toolkit, target, compiler options, registers per thread, stack-frame bytes, spill loads/stores, static/dynamic shared memory, block size, and cluster shape.\n2. Query active blocks with the occupancy API using the exact function, block size, and dynamic shared-memory bytes. Identify which resource is actually limiting residency.\n3. Verify that the candidate cap changed generated resources or code before attributing a timing change to it. Inspect PTX/SASS when instruction selection or loop shape may have changed.\n4. Run the same correctness oracle, warmup, synchronization, inputs, and repeated-trial statistic for every variant. Profile spill traffic, memory stalls, and achieved warps rather than assuming the predicted occupancy is reached or useful.\n5. Keep a cap only for the measured target and workload where it improves the declared metric without violating correctness or a resource contract."}, "reason": {"statement": "Describe compiler responses as possible costs to measure.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html"]}} +{"path": "wiki/techniques/register-budgeting.md", "before": {"statement": "For memory-bound kernels, spills can be hidden by memory latency anyway, so aggressive budgeting often wins."}, "after": {"statement": "Spill loads and stores use thread-local stack memory backed by the memory hierarchy. They add instructions and traffic; a memory-bound label is not evidence that this cost will be hidden. Conversely, a lower cap can be irrelevant when the uncapped allocation is already below it."}, "reason": {"statement": "Require controlled variants instead of a directional guarantee.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"]}} +{"path": "wiki/techniques/register-budgeting.md", "before": {"statement": "| 1 | 32 | 18.5μs |\n| 3 | 45 | ~20μs |"}, "after": {"statement": "Amandeep Singh reports that three inspected B200 NVFP4 GEMV solutions clustered around an 18.5 microsecond geometric mean, with a 32-register cap in rank 1 and 45 in rank 3. The same report says those solutions also differed in PTX decode, cache policies, load widths, exact-K specialization, and B reuse; it provides no controlled register-cap ablation or public contestant code. In the author's own kernel, lowering the cap from 80 to 64 had no effect because natural use was already below 64, while extra accumulator chains and software pipelining increased pressure and regressed.\n\nThat observation motivates inspecting register allocation; it does not prove that 32 beats 45, that occupancy caused the ranking, or that aggressive caps generally help memory-bound kernels."}, "reason": {"statement": "Preserve only the author's scoped observation and its provenance limit.", "urls": ["https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"]}} +{"path": "wiki/techniques/register-budgeting.md", "before": {"statement": "The measurable difference between 32 and 45 registers shows occupancy dominates for memory-bound NVFP4 GEMV."}, "after": {"statement": "Spill loads and stores use thread-local stack memory backed by the memory hierarchy. They add instructions and traffic; a memory-bound label is not evidence that this cost will be hidden. Conversely, a lower cap can be irrelevant when the uncapped allocation is already below it.\n\nAmandeep Singh reports that three inspected B200 NVFP4 GEMV solutions clustered around an 18.5 microsecond geometric mean, with a 32-register cap in rank 1 and 45 in rank 3. The same report says those solutions also differed in PTX decode, cache policies, load widths, exact-K specialization, and B reuse; it provides no controlled register-cap ablation or public contestant code. In the author's own kernel, lowering the cap from 80 to 64 had no effect because natural use was already below 64, while extra accumulator chains and software pipelining increased pressure and regressed.\n\nThat observation motivates inspecting register allocation; it does not prove that 32 beats 45, that occupancy caused the ranking, or that aggressive caps generally help memory-bound kernels."}, "reason": {"statement": "Do not infer causality from confounded implementations.", "urls": ["https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html"]}} +{"path": "wiki/techniques/register-budgeting.md", "before": {"statement": "- Memory-bound kernels (first priority: occupancy)\n- Kernels where register pressure comes from inner loop, not accumulators (TMEM handles accumulators)\n- Sub-byte types with heavy decode/scale computation\n\n- Compute-bound GEMM (let compiler use what it needs)\n- Kernels where spills to local memory would serialize"}, "after": {"statement": "Compile the same source and target first without a cap, then with candidate caps. `--resource-usage` makes a cap that does not change the compiled allocation visible.\n\n```bash\nnvcc -arch=sm_100a --resource-usage kernel.cu -o kernel-default\nnvcc -arch=sm_100a --maxrregcount=64 --resource-usage kernel.cu -o kernel-r64\n```\n\nFor each binary and each supported production shape:\n\n1. Record toolkit, target, compiler options, registers per thread, stack-frame bytes, spill loads/stores, static/dynamic shared memory, block size, and cluster shape.\n2. Query active blocks with the occupancy API using the exact function, block size, and dynamic shared-memory bytes. Identify which resource is actually limiting residency.\n3. Verify that the candidate cap changed generated resources or code before attributing a timing change to it. Inspect PTX/SASS when instruction selection or loop shape may have changed.\n4. Run the same correctness oracle, warmup, synchronization, inputs, and repeated-trial statistic for every variant. Profile spill traffic, memory stalls, and achieved warps rather than assuming the predicted occupancy is reached or useful.\n5. Keep a cap only for the measured target and workload where it improves the declared metric without violating correctness or a resource contract.\n\nSpill loads and stores use thread-local stack memory backed by the memory hierarchy. They add instructions and traffic; a memory-bound label is not evidence that this cost will be hidden. Conversely, a lower cap can be irrelevant when the uncapped allocation is already below it.\n\nFor tcgen05 MMA, the resident D accumulator is in TMEM rather than a per-thread D register vector. This changes one major register consumer, but it does not erase register pressure: descriptors, addresses, pipeline/control state, operands, and epilogue batches still use general-purpose registers. The pinned SM100 DeepGEMM path explicitly loads completed TMEM values into registers before its shared/global-store epilogue.\n\nTreat the removed D vector as shape-specific. For example, an SM90 m64n256 FP32 WGMMA fragment exposes 128 D registers per participating thread, but that number is not a universal Blackwell saving and does not predict a resident-block transition."}, "reason": {"statement": "Replace categories with a decision procedure based on the compiled kernel and controlled benchmarks.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh", "https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/arch/mma_sm90_gmma.hpp"]}} +{"path": "wiki/techniques/software-exp.md", "before": {"statement": "FlashAttention-4 replaces the hardware Special Function Unit (SFU) exponential (`ex2.approx`) with a software-emulated 2^x function that distributes computation across the SM's FMA (fused multiply-add) units. On Blackwell, tensor core throughput doubled compared to Hopper while SFU count remained the same, making the SFU the throughput bottleneck for attention's softmax operation. The software exponential uses Cody-Waite range reduction followed by a Horner-form polynomial evaluation, achieving sufficient accuracy for attention while bypassing the SFU entirely."}, "after": {"statement": "FlashAttention-4 (FA4) combines two paths for base-2 exponential in its Blackwell forward softmax. Most selected configurations retain hardware `exp2` for some entries and evaluate a tunable fraction with a software polynomial on general-purpose FMA pipelines. The paper describes roughly 10–25% software evaluation, while the exact fraction is a configuration choice rather than an architecture constant.\n\nThis hybrid is part of a larger schedule: two 128-thread softmax warpgroups alternate 128-row query tiles, synchronize to limit simultaneous exponential contention, stage probabilities through TMEM, and hand conditional rescaling to a correction warpgroup. A standalone polynomial is not equivalent to that pipeline.\n\nAt Dao-AILab/flash-attention revision `a369df707e1980fb328abcc1733e3457ec10155f`, [`flash_attn/cute/utils.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py) defines the default software path and [`flash_attn/cute/softmax.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/softmax.py) performs fragment-level hardware/software selection.\n\nFor each software-selected pair, the pinned implementation:\n\n1. Assumes each input is no greater than 127 and clamps values below `-127`.\n2. Uses a rounding-down addition with the float32 constant `2^23 + 2^22` to recover the integer floor and a fractional value in `[0,1)`.\n3. Evaluates a degree-3 polynomial in Horner form with packed float32 FMA operations.\n4. Combines the integer contribution with the polynomial's float32 representation by exponent-field integer operations; the software branch does not call `ex2.approx` for reconstruction.\n\nThe degree-3 float32 coefficients in that revision are:\n\n| `p0` | `1.0` |\n| `p1` | `0.695146143436431884765625` |\n| `p2` | `0.227564394474029541015625` |\n| `p3` | `0.077119089663028717041015625` |\n\nThe authors state that Sollya selected these coefficients to minimize relative error for `2^f` over `f in [0,1)`. For a natural exponential, the caller uses the identity `e^z = 2^(z * log2(e))`."}, "reason": {"statement": "Describe the hybrid degree-3 implementation exactly.", "urls": ["https://tridao.me/blog/2026/flash4/", "https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute", "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py"]}} +{"path": "wiki/techniques/software-exp.md", "before": {"statement": "The softmax in attention requires computing `exp(x - max)` for every element of the score matrix. On previous generations, the SFU's `ex2.approx` instruction was fast enough relative to the MMA throughput. On Blackwell:\n\n| Tensor core TFLOPS (BF16) | ~990 | ~2250 | 2.27x |\n| SFU units per SM | 16 | 16 | 1.0x |\n| SFU throughput (exp per cycle) | 16 | 16 | 1.0x |\n| FMA units per SM | 128 | 128 | 1.0x |\n\nThe tensor cores produce 2x more score elements per cycle, but the SFU can only process exp() at the same rate as before. This makes the SFU the bottleneck for any kernel that needs exp() proportional to the number of MMA outputs.\n\nFlashAttention-4's approach: distribute the exp() workload across FMA units (128 per SM) instead of SFU units (16 per SM), achieving 8x the throughput for the exponential computation."}, "after": {"statement": "For the authors' `M=N=D=128` B200 feeds-and-speeds model, one SM supplies 8192 BF16 tensor-core operations per cycle, 16 exponential operations per cycle, and 128 shared-memory bytes per cycle. Their forward-tile accounting assigns 1024 cycles to two MMAs, 1024 cycles to 128×128 exponentials, and 768 cycles to shared-memory traffic.\n\nThose are analytical inputs for that tile and schedule. They do not imply that every kernel with one exponential per MMA output is exponential-bound, nor do functional-unit counts give the throughput or latency of a software approximation. Range reduction, polynomial dependencies, reconstruction, instruction issue, and overlap all remain part of the measured implementation."}, "reason": {"statement": "Retain only the source's scoped feeds-and-speeds model.", "urls": ["https://tridao.me/blog/2026/flash4/"]}} +{"path": "wiki/techniques/software-exp.md", "before": {"statement": "Range reduction transforms the input `x` into a small residual that a polynomial can accurately approximate. The Cody-Waite method splits the input into an integer part (for exact power-of-two scaling) and a fractional part (for polynomial approximation):\n\n```cuda\n// Cody-Waite range reduction for 2^x\n// Goal: decompose x = n + r where n is integer, r in [-0.5, 0.5]\n// Then 2^x = 2^n * 2^r, and 2^r is approximated by polynomial\n//\n// The Cody-Waite trick: subtract n using two constants (C1 + C2)\n// to maintain precision when x is large.\n//\n// C1 is the nearest representable float to log2(e) with low-order bits zeroed\n// C2 is the correction: log2(e) - C1\n// This avoids catastrophic cancellation in x - n\n\n__device__ float software_exp2(float x) {\n // Step 1: Range reduction (Cody-Waite)\n // Round x to nearest integer\n float n = rintf(x);\n // High-precision subtraction using two constants\n // C1 and C2 together represent 1.0 in extended precision\n const float C1 = 1.0f; // Exact in float\n const float C2 = 0.0f; // Correction term (zero for 2^x, non-zero for e^x)\n // For 2^x, range reduction is simpler: r = x - n\n float r = x - n; // r in [-0.5, 0.5]\n\n // Step 2: Polynomial approximation of 2^r via Horner's method\n // Minimax polynomial coefficients for 2^r on [-0.5, 0.5]\n // Degree-4 polynomial: sufficient for ~22 bits of accuracy\n const float c0 = 1.0f;\n const float c1 = 0.6931471805599453f; // ln(2)\n const float c2 = 0.2402265069591007f; // ln(2)^2 / 2\n const float c3 = 0.05550410866482158f; // ln(2)^3 / 6\n const float c4 = 0.009618129107628477f; // ln(2)^4 / 24\n\n // Horner evaluation: c0 + r*(c1 + r*(c2 + r*(c3 + r*c4)))\n // Each step is one FMA instruction\n float poly = c4;\n poly = fmaf(poly, r, c3); // FMA 1\n poly = fmaf(poly, r, c2); // FMA 2\n poly = fmaf(poly, r, c1); // FMA 3\n poly = fmaf(poly, r, c0); // FMA 4\n\n // Step 3: Reconstruct 2^x = 2^n * poly\n // Use integer addition to the float exponent field\n int n_int = (int)n;\n // ldexpf multiplies by 2^n by adjusting the exponent bits\n float result = ldexpf(poly, n_int);\n\n return result;\n}\n```"}, "after": {"statement": "At Dao-AILab/flash-attention revision `a369df707e1980fb328abcc1733e3457ec10155f`, [`flash_attn/cute/utils.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py) defines the default software path and [`flash_attn/cute/softmax.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/softmax.py) performs fragment-level hardware/software selection.\n\nFor each software-selected pair, the pinned implementation:\n\n1. Assumes each input is no greater than 127 and clamps values below `-127`.\n2. Uses a rounding-down addition with the float32 constant `2^23 + 2^22` to recover the integer floor and a fractional value in `[0,1)`.\n3. Evaluates a degree-3 polynomial in Horner form with packed float32 FMA operations.\n4. Combines the integer contribution with the polynomial's float32 representation by exponent-field integer operations; the software branch does not call `ex2.approx` for reconstruction.\n\nThe degree-3 float32 coefficients in that revision are:\n\n| `p0` | `1.0` |\n| `p1` | `0.695146143436431884765625` |\n| `p2` | `0.227564394474029541015625` |\n| `p3` | `0.077119089663028717041015625` |\n\nThe authors state that Sollya selected these coefficients to minimize relative error for `2^f` over `f in [0,1)`. For a natural exponential, the caller uses the identity `e^z = 2^(z * log2(e))`.\n\nThe polynomial is approximate. On a deterministic host grid of 1,000,002 evenly spaced points over `[0,1]`, evaluating the rounded degree-3 coefficients against `2^x` produced a maximum sampled relative error of approximately `8.763e-5`. This is a regression observation, not a proof of the continuous maximum and not an end-to-end attention tolerance."}, "reason": {"statement": "Avoid presenting invented CUDA as upstream or accuracy-validated.", "urls": ["https://tridao.me/blog/2026/flash4/", "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py"]}} +{"path": "wiki/techniques/software-exp.md", "before": {"statement": "The key insight is that Horner polynomial evaluation is a chain of FMA operations. With the softmax warp executing on CUDA cores while the MMA warp uses tensor cores, the FMA throughput is fully available:\n\n```cuda\n// FlashAttention-4 softmax with software exp2\n// Executed by dedicated softmax warpgroups (part of warp specialization)\n//\n// For each row of the score matrix S[i,:]:\n// 1. Find row max: m_new = max(S[i,:])\n// 2. Compute exp2((S[i,j] - m_new) * log2(e)) for each j\n// 3. Sum for normalization denominator\n// 4. Conditionally rescale previous output if max changed\n\n__device__ void softmax_with_software_exp(\n float* scores, // Input: S[i, 0..N-1] (one row)\n float* output, // Output: softmax(S[i,:])\n int N,\n float* row_max, // Running max (for online softmax)\n float* row_sum) // Running sum\n{\n int lane = threadIdx.x % 32;\n\n // Step 1: Find max across the row (warp reduction)\n float local_max = -INFINITY;\n for (int j = lane; j < N; j += 32) {\n local_max = fmaxf(local_max, scores[j]);\n }\n // Warp-level max reduction\n for (int offset = 16; offset > 0; offset >>= 1) {\n local_max = fmaxf(local_max, __shfl_xor_sync(0xFFFFFFFF, local_max, offset));\n }\n float m_new = local_max;\n\n // Step 2: Compute software exp2 and sum\n float local_sum = 0.0f;\n const float LOG2E = 1.4426950408889634f;\n\n for (int j = lane; j < N; j += 32) {\n float x = (scores[j] - m_new) * LOG2E;\n float exp_val = software_exp2(x); // 4 FMAs instead of 1 SFU op\n output[j] = exp_val;\n local_sum += exp_val;\n }\n\n // Warp-level sum reduction\n for (int offset = 16; offset > 0; offset >>= 1) {\n local_sum += __shfl_xor_sync(0xFFFFFFFF, local_sum, offset);\n }\n\n // Step 3: Conditional rescaling (FlashAttention online softmax)\n float m_old = *row_max;\n if (m_new > m_old) {\n // Rescale previous accumulated output\n float scale = software_exp2((m_old - m_new) * LOG2E);\n *row_sum = (*row_sum) * scale + local_sum;\n *row_max = m_new;\n // The output accumulator must also be rescaled by `scale`\n } else {\n float scale = software_exp2((m_new - m_old) * LOG2E);\n *row_sum += local_sum * scale;\n // Rescale current exp values, not the accumulator\n }\n}\n```"}, "after": {"statement": "FlashAttention-4 (FA4) combines two paths for base-2 exponential in its Blackwell forward softmax. Most selected configurations retain hardware `exp2` for some entries and evaluate a tunable fraction with a software polynomial on general-purpose FMA pipelines. The paper describes roughly 10–25% software evaluation, while the exact fraction is a configuration choice rather than an architecture constant.\n\nThis hybrid is part of a larger schedule: two 128-thread softmax warpgroups alternate 128-row query tiles, synchronize to limit simultaneous exponential contention, stage probabilities through TMEM, and hand conditional rescaling to a correction warpgroup. A standalone polynomial is not equivalent to that pipeline.\n\nBF16 conversion does not by itself prove that approximation errors are harmless. Per-entry error changes both the softmax numerator and row sum, and masking, all-`-inf` rows, conditional rescaling, underflow clamping, and output accumulation introduce separate boundary cases. Validate the scalar approximation and the complete attention output for the exact dtype and features.\n\nUse the pinned implementation as the starting point; do not reconstruct it from rounded blog coefficients. For each target configuration:\n\n1. Build an all-hardware control and one or more hybrid selections from the same source revision, compiler, target, and launch configuration.\n2. Inspect generated PTX/SASS to confirm which entries use hardware `MUFU.EX2`, which use the expected FMA/range-reduction sequence, and whether register use or spills changed.\n3. Test ordinary, masked, all-masked, causal, variable-length, large-gap, and underflow-heavy rows against a declared higher-precision reference. Record maximum absolute/relative output error and row-sum behavior, not only BF16 agreement on random inputs.\n4. Profile exponential-pipeline pressure, FMA/ALU issue, tensor-core overlap, registers, and spills. A high hardware-exp metric is a lead, not proof that moving more entries to FMA improves the schedule.\n5. Benchmark identical shapes, inputs, warmup, synchronization, clock policy, and repeated-trial statistics. Sweep the software fraction because both zero emulation and mixed settings appear in the pinned target/configuration table.\n\nKeep the hybrid only where both the numerical contract and declared end-to-end metric pass. Do not transfer the choice to another architecture or transcendental function without repeating the reduction, approximation, special-value, generated-code, and performance checks."}, "reason": {"statement": "Point to pinned implementation structure instead of non-runnable reconstructed code.", "urls": ["https://tridao.me/blog/2026/flash4/", "https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute"]}} +{"path": "wiki/techniques/software-exp.md", "before": {"statement": "At the PTX level, the Horner polynomial compiles to a tight chain of `fma.rn.f32` instructions:\n\n```ptx\n// Software exp2 via Horner polynomial in PTX\n// Input: %x (float, range-reduced to [-0.5, 0.5])\n// Output: %result (float, approximation of 2^x)\n\n.reg .f32 %x, %r, %n, %poly, %result;\n.reg .f32 %c0, %c1, %c2, %c3, %c4;\n\n// Load polynomial coefficients\nmov.f32 %c0, 0f3F800000; // 1.0\nmov.f32 %c1, 0f3F317218; // 0.6931471805599453 (ln2)\nmov.f32 %c2, 0f3E75FDF0; // 0.2402265069591007\nmov.f32 %c3, 0f3D635847; // 0.05550410866482158\nmov.f32 %c4, 0f3C1D9539; // 0.009618129107628477\n\n// Range reduction: n = rintf(x), r = x - n\ncvt.rni.f32.f32 %n, %x; // Round to nearest int\nsub.f32 %r, %x, %n; // Fractional part\n\n// Horner evaluation: 4 dependent FMAs\n// poly = c4\n// poly = poly * r + c3\n// poly = poly * r + c2\n// poly = poly * r + c1\n// poly = poly * r + c0\nmov.f32 %poly, %c4;\nfma.rn.f32 %poly, %poly, %r, %c3; // FMA 1\nfma.rn.f32 %poly, %poly, %r, %c2; // FMA 2\nfma.rn.f32 %poly, %poly, %r, %c1; // FMA 3\nfma.rn.f32 %poly, %poly, %r, %c0; // FMA 4\n\n// Reconstruct: result = poly * 2^n\n// Convert n to int and use ex2 scaling via bit manipulation\ncvt.rzi.s32.f32 %ni, %n;\nex2.approx.f32 %scale, %n; // Or use integer exponent manipulation\nmul.f32 %result, %poly, %scale;\n```"}, "after": {"statement": "At Dao-AILab/flash-attention revision `a369df707e1980fb328abcc1733e3457ec10155f`, [`flash_attn/cute/utils.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py) defines the default software path and [`flash_attn/cute/softmax.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/softmax.py) performs fragment-level hardware/software selection.\n\nFor each software-selected pair, the pinned implementation:\n\n1. Assumes each input is no greater than 127 and clamps values below `-127`.\n2. Uses a rounding-down addition with the float32 constant `2^23 + 2^22` to recover the integer floor and a fractional value in `[0,1)`.\n3. Evaluates a degree-3 polynomial in Horner form with packed float32 FMA operations.\n4. Combines the integer contribution with the polynomial's float32 representation by exponent-field integer operations; the software branch does not call `ex2.approx` for reconstruction.\n\nThe degree-3 float32 coefficients in that revision are:\n\n| `p0` | `1.0` |\n| `p1` | `0.695146143436431884765625` |\n| `p2` | `0.227564394474029541015625` |\n| `p3` | `0.077119089663028717041015625` |\n\nThe authors state that Sollya selected these coefficients to minimize relative error for `2^f` over `f in [0,1)`. For a natural exponential, the caller uses the identity `e^z = 2^(z * log2(e))`."}, "reason": {"statement": "Cite exact source functions and generated-code inspection requirements.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html", "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py", "https://tridao.me/blog/2026/flash4/"]}} +{"path": "wiki/techniques/software-exp.md", "before": {"statement": "The degree-4 polynomial provides approximately 22 bits of mantissa accuracy, which is more than sufficient for attention softmax where:"}, "after": {"statement": "The polynomial is approximate. On a deterministic host grid of 1,000,002 evenly spaced points over `[0,1]`, evaluating the rounded degree-3 coefficients against `2^x` produced a maximum sampled relative error of approximately `8.763e-5`. This is a regression observation, not a proof of the continuous maximum and not an end-to-end attention tolerance."}, "reason": {"statement": "Do not attach an unsupported error guarantee to a non-upstream polynomial.", "urls": []}} +{"path": "wiki/techniques/software-exp.md", "before": {"statement": "- **Attention kernels on Blackwell**: Whenever the SFU is the bottleneck for softmax computation. FlashAttention-4 measured 1.1-1.3x speedup over cuDNN from this technique alone on B200."}, "after": {"statement": "The FA4 blog reports complete forward-pass speedups of 1.1–1.3× over cuDNN 9.13 on its evaluated B200 BF16 configurations. That comparison includes the full FA4 co-design—pipelining, hybrid exponentials, conditional rescaling, TMEM staging, scheduling, and other choices. It is not a software-exponential-only ablation and must not be used as the isolated speedup of this technique."}, "reason": {"statement": "Scope numbers to the full system and require a matched ablation for this technique.", "urls": ["https://tridao.me/blog/2026/flash4/"]}} +{"path": "wiki/techniques/software-exp.md", "before": {"statement": "- **Any kernel limited by transcendental function throughput**: If profiling shows SFU utilization near 100% while FMA utilization is low, software emulation can rebalance the workload.\n- **Not recommended on Hopper**: The SFU-to-MMA throughput ratio is better balanced on SM90. The overhead of 4 FMAs vs 1 SFU instruction is not justified unless the SFU is proven to be the bottleneck."}, "after": {"statement": "BF16 conversion does not by itself prove that approximation errors are harmless. Per-entry error changes both the softmax numerator and row sum, and masking, all-`-inf` rows, conditional rescaling, underflow clamping, and output accumulation introduce separate boundary cases. Validate the scalar approximation and the complete attention output for the exact dtype and features.\n\nUse the pinned implementation as the starting point; do not reconstruct it from rounded blog coefficients. For each target configuration:\n\n1. Build an all-hardware control and one or more hybrid selections from the same source revision, compiler, target, and launch configuration.\n2. Inspect generated PTX/SASS to confirm which entries use hardware `MUFU.EX2`, which use the expected FMA/range-reduction sequence, and whether register use or spills changed.\n3. Test ordinary, masked, all-masked, causal, variable-length, large-gap, and underflow-heavy rows against a declared higher-precision reference. Record maximum absolute/relative output error and row-sum behavior, not only BF16 agreement on random inputs.\n4. Profile exponential-pipeline pressure, FMA/ALU issue, tensor-core overlap, registers, and spills. A high hardware-exp metric is a lead, not proof that moving more entries to FMA improves the schedule.\n5. Benchmark identical shapes, inputs, warmup, synchronization, clock policy, and repeated-trial statistics. Sweep the software fraction because both zero emulation and mixed settings appear in the pinned target/configuration table.\n\nKeep the hybrid only where both the numerical contract and declared end-to-end metric pass. Do not transfer the choice to another architecture or transcendental function without repeating the reduction, approximation, special-value, generated-code, and performance checks."}, "reason": {"statement": "Use measurable preconditions and target-specific validation.", "urls": ["https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f", "https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute", "https://tridao.me/blog/2026/flash4/"]}} +{"path": "wiki/techniques/software-exp.md", "before": {"statement": "- The 4-FMA chain has a latency of ~16 cycles (4 dependent FMAs at ~4 cycles each), vs ~20 cycles for SFU `ex2.approx`. Latency is comparable; the win comes from throughput (128 FMA units vs 16 SFU units)."}, "after": {"statement": "For the authors' `M=N=D=128` B200 feeds-and-speeds model, one SM supplies 8192 BF16 tensor-core operations per cycle, 16 exponential operations per cycle, and 128 shared-memory bytes per cycle. Their forward-tile accounting assigns 1024 cycles to two MMAs, 1024 cycles to 128×128 exponentials, and 768 cycles to shared-memory traffic.\n\nThose are analytical inputs for that tile and schedule. They do not imply that every kernel with one exponential per MMA output is exponential-bound, nor do functional-unit counts give the throughput or latency of a software approximation. Range reduction, polynomial dependencies, reconstruction, instruction issue, and overlap all remain part of the measured implementation.\n\nBF16 conversion does not by itself prove that approximation errors are harmless. Per-entry error changes both the softmax numerator and row sum, and masking, all-`-inf` rows, conditional rescaling, underflow clamping, and output accumulation introduce separate boundary cases. Validate the scalar approximation and the complete attention output for the exact dtype and features.\n\nUse the pinned implementation as the starting point; do not reconstruct it from rounded blog coefficients. For each target configuration:\n\n1. Build an all-hardware control and one or more hybrid selections from the same source revision, compiler, target, and launch configuration.\n2. Inspect generated PTX/SASS to confirm which entries use hardware `MUFU.EX2`, which use the expected FMA/range-reduction sequence, and whether register use or spills changed.\n3. Test ordinary, masked, all-masked, causal, variable-length, large-gap, and underflow-heavy rows against a declared higher-precision reference. Record maximum absolute/relative output error and row-sum behavior, not only BF16 agreement on random inputs.\n4. Profile exponential-pipeline pressure, FMA/ALU issue, tensor-core overlap, registers, and spills. A high hardware-exp metric is a lead, not proof that moving more entries to FMA improves the schedule.\n5. Benchmark identical shapes, inputs, warmup, synchronization, clock policy, and repeated-trial statistics. Sweep the software fraction because both zero emulation and mixed settings appear in the pinned target/configuration table.\n\nKeep the hybrid only where both the numerical contract and declared end-to-end metric pass. Do not transfer the choice to another architecture or transcendental function without repeating the reduction, approximation, special-value, generated-code, and performance checks."}, "reason": {"statement": "Do not preserve unattributed cycle counts.", "urls": ["https://tridao.me/blog/2026/flash4/", "https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute"]}} +{"path": "wiki/techniques/software-exp.md", "before": {"statement": "- The `ldexpf` or exponent bit-manipulation step for 2^n must handle overflow/underflow (very large/small x). In attention, `x <= 0` always holds, so only underflow toward zero is possible."}, "after": {"statement": "At Dao-AILab/flash-attention revision `a369df707e1980fb328abcc1733e3457ec10155f`, [`flash_attn/cute/utils.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py) defines the default software path and [`flash_attn/cute/softmax.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/softmax.py) performs fragment-level hardware/software selection.\n\nFor each software-selected pair, the pinned implementation:\n\n1. Assumes each input is no greater than 127 and clamps values below `-127`.\n2. Uses a rounding-down addition with the float32 constant `2^23 + 2^22` to recover the integer floor and a fractional value in `[0,1)`.\n3. Evaluates a degree-3 polynomial in Horner form with packed float32 FMA operations.\n4. Combines the integer contribution with the polynomial's float32 representation by exponent-field integer operations; the software branch does not call `ex2.approx` for reconstruction.\n\nThe degree-3 float32 coefficients in that revision are:\n\n| `p0` | `1.0` |\n| `p1` | `0.695146143436431884765625` |\n| `p2` | `0.227564394474029541015625` |\n| `p3` | `0.077119089663028717041015625` |\n\nThe authors state that Sollya selected these coefficients to minimize relative error for `2^f` over `f in [0,1)`. For a natural exponential, the caller uses the identity `e^z = 2^(z * log2(e))`.\n\nThe polynomial is approximate. On a deterministic host grid of 1,000,002 evenly spaced points over `[0,1]`, evaluating the rounded degree-3 coefficients against `2^x` produced a maximum sampled relative error of approximately `8.763e-5`. This is a regression observation, not a proof of the continuous maximum and not an end-to-end attention tolerance."}, "reason": {"statement": "State the exact helper domain and require special-case handling at the caller.", "urls": ["https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py", "https://tridao.me/blog/2026/flash4/"]}} +{"path": "wiki/techniques/software-exp.md", "before": {"statement": "- The input `x = (S[i,j] - max) * log2(e)` is always non-positive\n- The softmax output is normalized, so small absolute errors cancel out\n- BF16 output has only 7 mantissa bits anyway\n\nFor applications requiring higher accuracy, a degree-6 polynomial (6 FMAs) achieves near-ULP accuracy across the full float range."}, "after": {"statement": "The polynomial is approximate. On a deterministic host grid of 1,000,002 evenly spaced points over `[0,1]`, evaluating the rounded degree-3 coefficients against `2^x` produced a maximum sampled relative error of approximately `8.763e-5`. This is a regression observation, not a proof of the continuous maximum and not an end-to-end attention tolerance.\n\nBF16 conversion does not by itself prove that approximation errors are harmless. Per-entry error changes both the softmax numerator and row sum, and masking, all-`-inf` rows, conditional rescaling, underflow clamping, and output accumulation introduce separate boundary cases. Validate the scalar approximation and the complete attention output for the exact dtype and features.\n\nUse the pinned implementation as the starting point; do not reconstruct it from rounded blog coefficients. For each target configuration:\n\n1. Build an all-hardware control and one or more hybrid selections from the same source revision, compiler, target, and launch configuration.\n2. Inspect generated PTX/SASS to confirm which entries use hardware `MUFU.EX2`, which use the expected FMA/range-reduction sequence, and whether register use or spills changed.\n3. Test ordinary, masked, all-masked, causal, variable-length, large-gap, and underflow-heavy rows against a declared higher-precision reference. Record maximum absolute/relative output error and row-sum behavior, not only BF16 agreement on random inputs.\n4. Profile exponential-pipeline pressure, FMA/ALU issue, tensor-core overlap, registers, and spills. A high hardware-exp metric is a lead, not proof that moving more entries to FMA improves the schedule.\n5. Benchmark identical shapes, inputs, warmup, synchronization, clock policy, and repeated-trial statistics. Sweep the software fraction because both zero emulation and mixed settings appear in the pinned target/configuration table.\n\nKeep the hybrid only where both the numerical contract and declared end-to-end metric pass. Do not transfer the choice to another architecture or transcendental function without repeating the reduction, approximation, special-value, generated-code, and performance checks."}, "reason": {"statement": "Replace universal accuracy claims with an explicit validation protocol.", "urls": ["https://tridao.me/blog/2026/flash4/", "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py", "https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute"]}} +{"path": "wiki/techniques/swizzling.md", "before": {"statement": "Shared memory swizzling remaps the linear address layout of a matrix tile in SMEM so that threads accessing consecutive columns (or rows) hit different 32-byte banks rather than the same bank. This eliminates bank conflicts that would otherwise serialize concurrent accesses. On Blackwell (SM100), 128-byte swizzling is mandatory for TMA loads and tcgen05.mma operands. Without it, performance drops to 46% of the achievable throughput for GEMM workloads.\n\nShared memory has 32 banks, each 4 bytes wide (128 bytes total per bank cycle). When a warp accesses a matrix stored in row-major layout, threads in the same warp reading elements from consecutive rows in the same column hit the same bank, causing a 32-way bank conflict.\n\nThe TMA unit on both Hopper and Blackwell encodes the swizzle pattern as part of the tensor descriptor. The tcgen05.mma instruction expects its SMEM operands to already be swizzled in the 128-byte pattern. Using unswizzled data produces incorrect MMA results."}, "after": {"statement": "Shared memory has 32 banks, and successive 32-bit words map to successive banks. For one warp memory request, accesses to different words in the same bank require serialized wavefronts; same-word reads can broadcast and do not create that conflict. The result therefore depends on the exact byte address requested by every participating lane.\n\nA swizzle permutes address chunks so a target access pattern may distribute its requests over banks more evenly. It can reduce conflicts for that pattern, but it does not make every access to the tile conflict-free. A layout favorable to an MMA operand can still be unfavorable to a CUDA-core row, column, transpose, or epilogue access.\n\nFor a TMA-to-tcgen05 path, keep these layers consistent:\n\n1. The tensor map describes how TMA places the global-memory box into shared memory, including interleave and swizzle.\n2. The shared-memory allocation and base address satisfy the alignment and span constraints of that mapping.\n3. The tcgen05 shared-memory descriptor describes the same physical-to-logical layout, leading/stride dimensions, base offset, and swizzle mode expected by the MMA operand.\n\nPTX ISA 9.0 permits ordinary tcgen05 shared-memory descriptor modes with no swizzle and 32B, 64B, or 128B spans, plus a 128B/base-32B mode. In the descriptor's three-bit layout field, ordinary no-swizzle is `0`, 128B/base-32B is `1`, ordinary 128B is `2`, 64B is `4`, and 32B is `6`; values `3`, `5`, and `7` are invalid. Legality also depends on MMA kind, element type, major mode, shape, CTA group, and target.\n\nThe CUDA 13.0.97 Driver API exposes TMA modes including:\n\n| `CU_TENSOR_MAP_SWIZZLE_NONE` | No bank swizzle |\n| `CU_TENSOR_MAP_SWIZZLE_32B` | 16B chunks within a 32B span |\n| `CU_TENSOR_MAP_SWIZZLE_64B` | 16B chunks within a 64B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B` | 16B chunks within a 128B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B` | 32B chunks within a 128B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B_ATOM_64B` | 64B chunks within a 128B span; support is operation/type-specific |\n\nThese names define mappings, not universal datatype or tile-size recommendations. The tensor-map encoder imposes mode-, interleave-, datatype-, alignment-, and box-size constraints; for example, ordinary 128B modes require the inner bounding-box byte span not to exceed 128 bytes. Check the exact archived API for the selected type and operation."}, "reason": {"statement": "Define conflicts per access and mode legality per consumer combination.", "urls": ["https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"]}} +{"path": "wiki/techniques/swizzling.md", "before": {"statement": "The tcgen05-tutorial benchmark progression shows the impact:\n\n```\nNaive (no swizzle): 255 TFLOPS (17% of cuBLAS)\n128B swizzle applied: 695 TFLOPS (46% of cuBLAS)\n ---- 2.7x improvement from swizzling alone ----\n```"}, "after": {"statement": "Gau Nernst's pinned B200 tutorial compares M=N=K=4096 kernels with PyTorch 2.9.1 and CUDA 13. Its 3D TMA version changes from a 16-byte inner tile with no swizzle to a 128-byte inner tile with 128B swizzling and matching tcgen05 descriptors:\n\n| v1b: 3D 16B TMA | 252.81 |\n| v2b: 3D 128B TMA plus 128B swizzle | 695.43 |\n\nThe combined change is approximately 2.75× and the v2b endpoint is about 46% of the tutorial's 1506.74 TFLOP/s cuBLAS result. It is not a bank-conflict or swizzle-only ablation. The author notes that the earlier contiguous `8×16B` tile might already span all 32 banks, lacked Nsight Compute access, and offers wider TMA transfers as an alternative explanation."}, "reason": {"statement": "Preserve endpoints and list the simultaneous changes and unresolved cause.", "urls": ["https://gau-nernst.github.io/tcgen05/"]}} +{"path": "wiki/techniques/swizzling.md", "before": {"statement": "The swizzle function XORs a portion of the column address with the row address to scatter accesses across banks:\n\n```cuda\n// 128-byte swizzle: XOR bits [4:6] of the byte offset with the row index\n// This ensures that consecutive rows accessing the same logical column\n// map to different physical SMEM banks.\n//\n// For a tile stored in SMEM with TILE_N columns of 2-byte elements:\n// byte_offset = row * (TILE_N * sizeof(half)) + col * sizeof(half)\n// swizzled_offset = byte_offset ^ ((row & 0x7) << 4)\n//\n// The mask 0x7 = 3 bits, shift 4 = bits [4:6], giving 8-row periodicity\n// across the 128-byte bank group.\n\n__device__ int swizzle_128B(int row, int col, int stride_bytes) {\n int byte_offset = row * stride_bytes + col * sizeof(half);\n // XOR bits [4:6] of byte offset with low 3 bits of row\n int swizzled = byte_offset ^ ((row & 0x7) << 4);\n return swizzled;\n}\n```\n\nVisually, for an 8-row x 64-column half-precision tile (128 bytes per row):\n\n```\nWithout swizzle (row-major):\n Row 0: bank 0,1,2,...,31 bank 0,1,2,...,31\n Row 1: bank 0,1,2,...,31 bank 0,1,2,...,31\n Row 2: bank 0,1,2,...,31 bank 0,1,2,...,31\n -> Column access = 8-way bank conflict\n\nWith 128B swizzle (XOR pattern):\n Row 0: bank 0,1,2,...,31 bank 0,1,2,...,31\n Row 1: bank 1,2,3,...,0 bank 1,2,3,...,0 (rotated by 1)\n Row 2: bank 2,3,4,...,1 bank 2,3,4,...,1 (rotated by 2)\n -> Column access = conflict-free\n```"}, "after": {"statement": "CUTLASS 4.5.0 defines [`Swizzle`](https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/swizzle.hpp):\n\n- `BBits` is the number of mask bits;\n- `MBase` is the number of least-significant address bits kept invariant; and\n- `SShift` is the distance between the two bit fields.\n\nFor `Swizzle<3,4,3>`, three address bits beginning above the four invariant low bits are XORed with the three-bit field shifted by three positions. Equivalently, address bits 7:9 affect bits 4:6. This is an address transformation, not a general `row & 7` rule: equivalence to row-based indexing depends on stride, byte units, base alignment, and layout composition.\n\nCUTLASS also distinguishes a position-independent composed swizzle layout from a position-dependent swizzle pointer, because hardware swizzling depends on the shared-memory pointer address. Reuse a pinned complete layout/descriptor construction or prove the composition and required base alignment; a `Swizzle` type alone is not a complete TMA or MMA contract."}, "reason": {"statement": "Use exact address-bit semantics and library/descriptor mappings.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/swizzle.hpp", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/pointer_swizzle.hpp"]}} +{"path": "wiki/techniques/swizzling.md", "before": {"statement": "```cuda\n// Creating a TMA descriptor with 128-byte swizzle\n#include \n\nCUtensorMap tensor_map;\n\n// Swizzle mode: CU_TENSOR_MAP_SWIZZLE_128B\n// This tells TMA to apply the 128-byte XOR swizzle pattern\n// when writing data into shared memory\ncuTensorMapEncodeTiled(\n &tensor_map,\n CU_TENSOR_MAP_DATA_TYPE_FLOAT16,\n 2, // 2D tensor\n global_ptr, // global memory base\n global_dims, // {N, M} dimensions\n global_strides, // {N * sizeof(half), sizeof(half)}\n tile_dims, // {TILE_N, TILE_M}\n element_strides, // {1, 1}\n CU_TENSOR_MAP_INTERLEAVE_NONE,\n CU_TENSOR_MAP_SWIZZLE_128B, // 128-byte swizzle\n CU_TENSOR_MAP_L2_PROMOTION_NONE,\n CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE\n);\n```"}, "after": {"statement": "For a TMA-to-tcgen05 path, keep these layers consistent:\n\n1. The tensor map describes how TMA places the global-memory box into shared memory, including interleave and swizzle.\n2. The shared-memory allocation and base address satisfy the alignment and span constraints of that mapping.\n3. The tcgen05 shared-memory descriptor describes the same physical-to-logical layout, leading/stride dimensions, base offset, and swizzle mode expected by the MMA operand.\n\nPTX ISA 9.0 permits ordinary tcgen05 shared-memory descriptor modes with no swizzle and 32B, 64B, or 128B spans, plus a 128B/base-32B mode. In the descriptor's three-bit layout field, ordinary no-swizzle is `0`, 128B/base-32B is `1`, ordinary 128B is `2`, 64B is `4`, and 32B is `6`; values `3`, `5`, and `7` are invalid. Legality also depends on MMA kind, element type, major mode, shape, CTA group, and target.\n\nThe CUDA 13.0.97 Driver API exposes TMA modes including:\n\n| `CU_TENSOR_MAP_SWIZZLE_NONE` | No bank swizzle |\n| `CU_TENSOR_MAP_SWIZZLE_32B` | 16B chunks within a 32B span |\n| `CU_TENSOR_MAP_SWIZZLE_64B` | 16B chunks within a 64B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B` | 16B chunks within a 128B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B` | 32B chunks within a 128B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B_ATOM_64B` | 64B chunks within a 128B span; support is operation/type-specific |\n\nThese names define mappings, not universal datatype or tile-size recommendations. The tensor-map encoder imposes mode-, interleave-, datatype-, alignment-, and box-size constraints; for example, ordinary 128B modes require the inner bounding-box byte span not to exceed 128 bytes. Check the exact archived API for the selected type and operation.\n\nFor `cuTensorMapEncodeTiled`, record and validate all inputs rather than copying only the swizzle enumerator:\n\n1. Use a correctly aligned `CUtensorMap` object and global base pointer.\n2. Supply rank-sized global dimensions, rank-minus-one byte strides (the fastest dimension is implicit), rank-sized box dimensions, and rank-sized element strides.\n3. Satisfy the global alignment, stride, box, interleave, datatype, and selected-swizzle constraints.\n4. Check the returned `CUresult`; do not launch with an output descriptor after encoding failed.\n5. Use a shared-memory base and tcgen05 descriptor that represent the same mapping as the tensor map.\n\nAn invalid encoder combination can fail explicitly. A successfully encoded tensor map paired with the wrong consumer layout may instead read the wrong logical elements, so a successful API return is not a correctness oracle."}, "reason": {"statement": "Replace incomplete code with an encoder validation checklist and pinned full example.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/techniques/swizzling.md", "before": {"statement": "The available swizzle modes and their use cases:\n\n| `SWIZZLE_NONE` | No remapping | Non-MMA data (flags, scales) |\n| `SWIZZLE_32B` | 32-byte groups | Narrow tiles, small data types |\n| `SWIZZLE_64B` | 64-byte groups | Medium tiles |\n| `SWIZZLE_128B` | 128-byte groups | Standard for BF16/FP16 MMA operands |"}, "after": {"statement": "For a TMA-to-tcgen05 path, keep these layers consistent:\n\n1. The tensor map describes how TMA places the global-memory box into shared memory, including interleave and swizzle.\n2. The shared-memory allocation and base address satisfy the alignment and span constraints of that mapping.\n3. The tcgen05 shared-memory descriptor describes the same physical-to-logical layout, leading/stride dimensions, base offset, and swizzle mode expected by the MMA operand.\n\nPTX ISA 9.0 permits ordinary tcgen05 shared-memory descriptor modes with no swizzle and 32B, 64B, or 128B spans, plus a 128B/base-32B mode. In the descriptor's three-bit layout field, ordinary no-swizzle is `0`, 128B/base-32B is `1`, ordinary 128B is `2`, 64B is `4`, and 32B is `6`; values `3`, `5`, and `7` are invalid. Legality also depends on MMA kind, element type, major mode, shape, CTA group, and target.\n\nThe CUDA 13.0.97 Driver API exposes TMA modes including:\n\n| `CU_TENSOR_MAP_SWIZZLE_NONE` | No bank swizzle |\n| `CU_TENSOR_MAP_SWIZZLE_32B` | 16B chunks within a 32B span |\n| `CU_TENSOR_MAP_SWIZZLE_64B` | 16B chunks within a 64B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B` | 16B chunks within a 128B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B` | 32B chunks within a 128B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B_ATOM_64B` | 64B chunks within a 128B span; support is operation/type-specific |\n\nThese names define mappings, not universal datatype or tile-size recommendations. The tensor-map encoder imposes mode-, interleave-, datatype-, alignment-, and box-size constraints; for example, ordinary 128B modes require the inner bounding-box byte span not to exceed 128 bytes. Check the exact archived API for the selected type and operation."}, "reason": {"statement": "List exact mapping spans and defer legality to producer and consumer contracts.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/techniques/swizzling.md", "before": {"statement": "In CuTe/CUTLASS, swizzle is expressed as a layout composition:\n\n```cuda\n// CuTe swizzle layout for 128-byte swizzle pattern\n// Swizzle where:\n// B = number of bits in the base (non-swizzled) portion\n// M = number of bits in the mask\n// S = shift amount\n//\n// Swizzle<3, 4, 3> encodes the 128B swizzle:\n// 3 base bits (8-byte alignment)\n// 4 mask bits (16 rows)\n// 3 shift bits (8-column groups)\n\nusing SmemLayoutAtom = decltype(\n composition(\n Swizzle<3, 4, 3>{},\n Layout,\n Stride<_64, _1>>{}\n )\n);\n\n// Tile the atom across the full SMEM tile\nusing SmemLayoutA = decltype(\n tile_to_shape(SmemLayoutAtom{}, Shape, Int>{})\n);\n```"}, "after": {"statement": "CUTLASS 4.5.0 defines [`Swizzle`](https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/swizzle.hpp):\n\n- `BBits` is the number of mask bits;\n- `MBase` is the number of least-significant address bits kept invariant; and\n- `SShift` is the distance between the two bit fields.\n\nFor `Swizzle<3,4,3>`, three address bits beginning above the four invariant low bits are XORed with the three-bit field shifted by three positions. Equivalently, address bits 7:9 affect bits 4:6. This is an address transformation, not a general `row & 7` rule: equivalence to row-based indexing depends on stride, byte units, base alignment, and layout composition.\n\nCUTLASS also distinguishes a position-independent composed swizzle layout from a position-dependent swizzle pointer, because hardware swizzling depends on the shared-memory pointer address. Reuse a pinned complete layout/descriptor construction or prove the composition and required base alignment; a `Swizzle` type alone is not a complete TMA or MMA contract."}, "reason": {"statement": "Define parameters exactly and cite a pinned full implementation rather than a generic atom.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/swizzle.hpp", "https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/pointer_swizzle.hpp"]}} +{"path": "wiki/techniques/swizzling.md", "before": {"statement": "Use `nvprof` or Nsight Compute to verify that swizzling eliminates conflicts:\n\n```python\n# Nsight Compute command to check shared memory bank conflicts\n# Look for \"Shared Memory Bank Conflicts\" metric\n# ncu --metrics l1tex__data_bank_conflicts_pipe_lsu_mem_shared_op_ld.sum \\\n# ./my_kernel\n\n# Expected results:\n# Without swizzle: bank_conflicts >> 0\n# With 128B swizzle: bank_conflicts == 0\n```"}, "after": {"statement": "For `cuTensorMapEncodeTiled`, record and validate all inputs rather than copying only the swizzle enumerator:\n\n1. Use a correctly aligned `CUtensorMap` object and global base pointer.\n2. Supply rank-sized global dimensions, rank-minus-one byte strides (the fastest dimension is implicit), rank-sized box dimensions, and rank-sized element strides.\n3. Satisfy the global alignment, stride, box, interleave, datatype, and selected-swizzle constraints.\n4. Check the returned `CUresult`; do not launch with an output descriptor after encoding failed.\n5. Use a shared-memory base and tcgen05 descriptor that represent the same mapping as the tensor map.\n\nAn invalid encoder combination can fail explicitly. A successfully encoded tensor map paired with the wrong consumer layout may instead read the wrong logical elements, so a successful API return is not a correctness oracle."}, "reason": {"statement": "Use ncu --query-metrics and compare correctness, wavefronts/conflicts, and timing between matched layouts.", "urls": ["https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html", "https://gau-nernst.github.io/tcgen05/", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/techniques/swizzling.md", "before": {"statement": "- **All Blackwell tensor core kernels**: 128-byte swizzling is not optional. Both TMA and tcgen05.mma require it for correct results and peak performance.\n- **Hopper wgmma kernels**: Same requirement applies; wgmma expects swizzled SMEM operands.\n- **Non-MMA shared memory access**: If multiple warps access the same SMEM tile in a column pattern (e.g., reduction), swizzling prevents serialization."}, "after": {"statement": "Shared memory has 32 banks, and successive 32-bit words map to successive banks. For one warp memory request, accesses to different words in the same bank require serialized wavefronts; same-word reads can broadcast and do not create that conflict. The result therefore depends on the exact byte address requested by every participating lane.\n\nA swizzle permutes address chunks so a target access pattern may distribute its requests over banks more evenly. It can reduce conflicts for that pattern, but it does not make every access to the tile conflict-free. A layout favorable to an MMA operand can still be unfavorable to a CUDA-core row, column, transpose, or epilogue access.\n\nFor `cuTensorMapEncodeTiled`, record and validate all inputs rather than copying only the swizzle enumerator:\n\n1. Use a correctly aligned `CUtensorMap` object and global base pointer.\n2. Supply rank-sized global dimensions, rank-minus-one byte strides (the fastest dimension is implicit), rank-sized box dimensions, and rank-sized element strides.\n3. Satisfy the global alignment, stride, box, interleave, datatype, and selected-swizzle constraints.\n4. Check the returned `CUresult`; do not launch with an output descriptor after encoding failed.\n5. Use a shared-memory base and tcgen05 descriptor that represent the same mapping as the tensor map.\n\nAn invalid encoder combination can fail explicitly. A successfully encoded tensor map paired with the wrong consumer layout may instead read the wrong logical elements, so a successful API return is not a correctness oracle."}, "reason": {"statement": "Replace architecture categories with a producer/consumer/access-pattern decision.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"]}} +{"path": "wiki/techniques/swizzling.md", "before": {"statement": "- The swizzle mode in the TMA descriptor must exactly match the access pattern of the consumer. A mismatch produces silently incorrect results, not a runtime error.\n- Swizzled layouts make SMEM address computation non-trivial. Using CuTe's layout algebra avoids manual indexing errors.\n- For data types wider than 2 bytes (e.g., FP32 accumulators), the optimal swizzle mode may differ. TMEM accumulators avoid this issue since they use a separate address space."}, "after": {"statement": "For `cuTensorMapEncodeTiled`, record and validate all inputs rather than copying only the swizzle enumerator:\n\n1. Use a correctly aligned `CUtensorMap` object and global base pointer.\n2. Supply rank-sized global dimensions, rank-minus-one byte strides (the fastest dimension is implicit), rank-sized box dimensions, and rank-sized element strides.\n3. Satisfy the global alignment, stride, box, interleave, datatype, and selected-swizzle constraints.\n4. Check the returned `CUresult`; do not launch with an output descriptor after encoding failed.\n5. Use a shared-memory base and tcgen05 descriptor that represent the same mapping as the tensor map.\n\nAn invalid encoder combination can fail explicitly. A successfully encoded tensor map paired with the wrong consumer layout may instead read the wrong logical elements, so a successful API return is not a correctness oracle."}, "reason": {"statement": "State explicit encoder checks, layout equivalence and correctness/performance validation.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/swizzle.hpp", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/techniques/vectorized-loads.md", "before": {"statement": "For memory-bound kernels (low arithmetic intensity), maximizing global memory throughput is critical. Three complementary techniques from the GPU Mode NVFP4 Hackathon achieve this: (1) wide vectorized loads (128-bit and 256-bit) to saturate memory bandwidth per thread, (2) differentiated L1 cache policies to keep reused data hot while bypassing the cache for streaming data, and (3) register budgeting via `-maxrregcount` to increase occupancy. These techniques reduced NVFP4 GEMV latency from 2000us to 22.4us (89x improvement)."}, "after": {"statement": "Amandeep Singh's attempts supply useful negative controls for the same task. Replacing two `uchar4` loads with one `uint2` load was reported 16–25% slower because extraction added instructions. Lowering `-maxrregcount` from 80 to 64 had no effect because natural allocation was already below the cap. The author later observed wider PTX forms and differentiated L1 policies in other solutions, but published no contestant code or controlled ablation for those observations.\n\nTogether, the reports justify testing vector width, decode organization, cache hints, and register limits. They do not establish that the widest load, a particular cache hint, or the lowest register cap is essential for GEMV, sub-byte arithmetic, or decode workloads."}, "reason": {"statement": "Retain the report as an author-reported multi-change case study without causal attribution.", "urls": ["https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"]}} +{"path": "wiki/techniques/vectorized-loads.md", "before": {"statement": "Standard 32-bit loads waste memory bus bandwidth. Wider loads amortize the instruction overhead and saturate the 8 TB/s HBM bandwidth of B200:\n\n```cuda\n// Vectorized load widths comparison for FP4 GEMV\n// Each thread loads more data per instruction\n\n// 32-bit load: 4 bytes per thread per instruction\nfloat val;\nasm volatile(\"ld.global.b32 %0, [%1];\" : \"=f\"(val) : \"l\"(ptr));\n\n// 64-bit load: 8 bytes per thread per instruction\nuint2 val64;\nasm volatile(\"ld.global.v2.u32 {%0,%1}, [%2];\"\n : \"=r\"(val64.x), \"=r\"(val64.y) : \"l\"(ptr));\n\n// 128-bit load: 16 bytes per thread per instruction (preferred)\nuint4 val128;\nasm volatile(\"ld.global.v4.u32 {%0,%1,%2,%3}, [%4];\"\n : \"=r\"(val128.x), \"=r\"(val128.y), \"=r\"(val128.z), \"=r\"(val128.w)\n : \"l\"(ptr));\n\n// 256-bit load: 32 bytes per thread per instruction\n// Requires v4.u64 (4 x 64-bit)\nuint64_t v256[4];\nasm volatile(\"ld.global.v4.u64 {%0,%1,%2,%3}, [%4];\"\n : \"=l\"(v256[0]), \"=l\"(v256[1]), \"=l\"(v256[2]), \"=l\"(v256[3])\n : \"l\"(ptr));\n```"}, "after": {"statement": "A PTX vector load moves several typed elements into several registers with one instruction. This can reduce the number of load instructions issued by a thread, but PTX only says that vector loads *may* improve memory performance. Source-level width alone does not prove fewer physical memory transactions, higher achieved bandwidth, or lower kernel latency.\n\nPTX ISA 9.0 permits the following global-memory forms on `sm_100`. The first accesses 16 bytes and the second accesses 32 bytes:\n\n```ptx\n.reg .b64 addr;\n.reg .u32 x<4>;\n.reg .u64 y<4>;\n\nld.global.v4.u32 {x0, x1, x2, x3}, [addr];\nld.global.v4.u64 {y0, y1, y2, y3}, [addr];\n```\n\nThe address must be naturally aligned to the total access size: 16 bytes for `v4.u32` and 32 bytes for `v4.u64`. A misaligned PTX address has undefined behavior; the ISA says it may have low address bits masked or fault. It does not specify a fallback to narrower loads. Guard tails or dispatch them to an access whose complete range and alignment are valid.\n\nThe 256-bit `v4.b64` family is an SM100-or-newer feature. A 32-byte load contains 64 values when the payload is densely packed E2M1 at two 4-bit values per byte. That arithmetic says nothing about whether the width is profitable: unpack cost, live registers, instruction selection, per-thread address patterns, and tail handling can outweigh a lower source-level load count.\n\nPTX distinguishes cache operators, eviction priorities, prefetch-size hints, and non-coherent read-only loads. Representative legal forms are:\n\n```ptx\nld.global.L1::no_allocate.v4.u32 {x0, x1, x2, x3}, [addr];\nld.global.L1::evict_last.v4.u32 {x0, x1, x2, x3}, [addr];\nld.global.L2::256B.b32 x0, [addr];\nld.global.nc.b32 x0, [addr];\n```\n\n| `L1::no_allocate` | L1 eviction-priority selection that may be applied | Does not guarantee an L1 bypass or a speedup for a streaming operand |\n| `L1::evict_last` | Requests the corresponding L1 eviction priority | Does not guarantee that a line remains resident |\n| `L2::256B` | Hints that additional data of the stated size be prefetched into L2 | Is not a 256-byte load or a promotion guarantee |\n| `ld.global.nc` | Loads through a non-coherent read-only cache | Has architecture- and parallelism-dependent latency/throughput; it is not a general coherence optimization |\n\nCache-policy operands and prefetch-size qualifiers are performance hints and do not change the program's memory-consistency behavior. A reuse argument can motivate `evict_last`, and a one-pass stream can motivate `no_allocate`, but only a matched comparison determines whether either helps the concrete working set and launch."}, "reason": {"statement": "Document legal widths, exact alignment, and the need to inspect/profile generated code.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld-global-nc"]}} +{"path": "wiki/techniques/vectorized-loads.md", "before": {"statement": "For the NVFP4 GEMV, 128-bit and 256-bit loads are essential because FP4 elements are only 0.5 bytes each. A 256-bit load fetches 64 FP4 values in a single instruction:\n\n```cuda\n// NVFP4 GEMV: each thread loads 64 FP4 values via 256-bit load\n// Then unpacks using PTX byte manipulation instead of bitwise ops\n__device__ void load_and_unpack_nvfp4_256bit(\n const uint8_t* fp4_data, // Packed FP4 data (2 values per byte)\n float* unpacked, // Output: 64 FP32 values\n int offset)\n{\n // 256-bit load: 32 bytes = 64 FP4 values\n uint64_t raw[4];\n const uint64_t* ptr = reinterpret_cast(fp4_data + offset);\n asm volatile(\n \"ld.global.v4.u64 {%0,%1,%2,%3}, [%4];\"\n : \"=l\"(raw[0]), \"=l\"(raw[1]), \"=l\"(raw[2]), \"=l\"(raw[3])\n : \"l\"(ptr)\n );\n\n // Unpack using PTX mov.b32 byte extraction\n // This avoids the bitwise shift-and-mask overhead\n for (int i = 0; i < 4; i++) {\n uint32_t lo = (uint32_t)(raw[i]);\n uint32_t hi = (uint32_t)(raw[i] >> 32);\n\n // PTX byte unpack: extract individual bytes from 32-bit word\n uint32_t b0, b1, b2, b3;\n asm volatile(\"mov.b32 {%0,%1,%2,%3}, %4;\"\n : \"=r\"(b0), \"=r\"(b1), \"=r\"(b2), \"=r\"(b3) : \"r\"(lo));\n\n // Each byte contains 2 FP4 values — decode low word (8 values)\n for (int b = 0; b < 4; b++) {\n uint32_t byte_val = (b == 0) ? b0 : (b == 1) ? b1 : (b == 2) ? b2 : b3;\n unpacked[i * 16 + b * 2] = decode_fp4(byte_val & 0xF);\n unpacked[i * 16 + b * 2 + 1] = decode_fp4((byte_val >> 4) & 0xF);\n }\n\n // Unpack high word (next 8 values from same 64-bit element)\n uint32_t hb0, hb1, hb2, hb3;\n asm volatile(\"mov.b32 {%0,%1,%2,%3}, %4;\"\n : \"=r\"(hb0), \"=r\"(hb1), \"=r\"(hb2), \"=r\"(hb3) : \"r\"(hi));\n for (int b = 0; b < 4; b++) {\n uint32_t byte_val = (b == 0) ? hb0 : (b == 1) ? hb1 : (b == 2) ? hb2 : hb3;\n unpacked[i * 16 + 8 + b * 2] = decode_fp4(byte_val & 0xF);\n unpacked[i * 16 + 8 + b * 2 + 1] = decode_fp4((byte_val >> 4) & 0xF);\n }\n }\n}\n```"}, "after": {"statement": "PTX distinguishes cache operators, eviction priorities, prefetch-size hints, and non-coherent read-only loads. Representative legal forms are:\n\n```ptx\nld.global.L1::no_allocate.v4.u32 {x0, x1, x2, x3}, [addr];\nld.global.L1::evict_last.v4.u32 {x0, x1, x2, x3}, [addr];\nld.global.L2::256B.b32 x0, [addr];\nld.global.nc.b32 x0, [addr];\n```\n\n| `L1::no_allocate` | L1 eviction-priority selection that may be applied | Does not guarantee an L1 bypass or a speedup for a streaming operand |\n| `L1::evict_last` | Requests the corresponding L1 eviction priority | Does not guarantee that a line remains resident |\n| `L2::256B` | Hints that additional data of the stated size be prefetched into L2 | Is not a 256-byte load or a promotion guarantee |\n| `ld.global.nc` | Loads through a non-coherent read-only cache | Has architecture- and parallelism-dependent latency/throughput; it is not a general coherence optimization |\n\nCache-policy operands and prefetch-size qualifiers are performance hints and do not change the program's memory-consistency behavior. A reuse argument can motivate `evict_last`, and a one-pass stream can motivate `no_allocate`, but only a matched comparison determines whether either helps the concrete working set and launch.\n\nYue Zhang reports the following CUDA progression for GPU Mode Problem 1. These are author-reported endpoints without raw repeated-trial data or released complete submission code:\n\n| Initial CUDA | Naive hand-written path | about 2000 µs |\n| CUDA optimization 1 | Coalescing, shared B, thread collaboration, warp reduction | about 443 µs |\n| CUDA optimization 2 | Remove shared B, per-thread tiles, `float4` loads, hardware intrinsics | about 39 µs |\n| CUDA optimization 3 | Vectorized PTX FP4 and scale decode | about 27 µs |\n| Parameter tuning | Threads per row and rows per block | about 26 µs |\n| ILP | Two tiles per loop iteration | about 22.9 µs |\n| Aggressive PTX fusion | Decode, scales, multiply, and accumulation in a larger PTX block | about 22.3 µs |\n\nThe submitted public-leaderboard score was 22.392 microseconds, the geometric mean over three benchmark shapes. The pinned task's theoretical model gives 8.622, 17.275, and 4.317 microseconds for those separate shapes; 8.622 microseconds is therefore not an aggregate “speed of light.” The 39-to-27 endpoints change the PTX load/decode path together and do not isolate cache policy, load width, or conversion choice."}, "reason": {"statement": "Replace the invented helper with exact PTX semantics and an empirical selection workflow.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld-global-nc", "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml"]}} +{"path": "wiki/techniques/vectorized-loads.md", "before": {"statement": "Different data streams have different reuse patterns. Applying the correct cache policy per stream avoids L1 pollution:\n\n```cuda\n// Cache policy selection based on data reuse pattern\n//\n// Matrix A (streamed, each row used once): bypass L1\n// Vector B (reused across all rows): keep in L1\n\n// L1::no_allocate -- data bypasses L1 cache (streaming access)\n// Used for matrix A which is read once per GEMV\nasm volatile(\n \"ld.global.L1::no_allocate.v4.u32 {%0,%1,%2,%3}, [%4];\"\n : \"=r\"(a.x), \"=r\"(a.y), \"=r\"(a.z), \"=r\"(a.w)\n : \"l\"(matrix_a_ptr)\n);\n\n// L1::evict_last -- data stays in L1 as long as possible\n// Used for vector B which is reused across all M rows\nasm volatile(\n \"ld.global.L1::evict_last.v4.u32 {%0,%1,%2,%3}, [%4];\"\n : \"=r\"(b.x), \"=r\"(b.y), \"=r\"(b.z), \"=r\"(b.w)\n : \"l\"(vector_b_ptr)\n);\n```"}, "after": {"statement": "For each candidate width and cache policy:\n\n1. Prove base and per-thread address alignment for every executed access. Use a separate scalar or narrower-vector path for tails; validate minimum, maximum, and awkward sizes.\n2. Hold the algorithm, mapping, decode, unrolling, launch shape, compiler, and inputs fixed while changing one load or cache choice. Compare identical correctness oracles before timing.\n3. Inspect generated PTX and SASS. Record actual load instructions, registers per thread, spill loads/stores, stack bytes, shared memory, and launch parameters. A source cast or inline-PTX block does not bypass compiler register allocation.\n4. Profile the matched variants. Check achieved bandwidth, requested versus transferred bytes, cache hit behavior, instruction issue/stalls, and active warps with metrics available on the installed Nsight Compute and GPU versions.\n5. Use warmups, synchronization, repeated trials, and the same statistic for every production shape. Keep the change only where the declared metric improves without a correctness or resource regression.\n\nRegister caps and launch bounds are a separate variable. A nominal cap may be inert when natural allocation is lower, or it may trade registers for spills and extra instructions. Follow the resource and occupancy workflow in [Register Budgeting](register-budgeting.md) rather than inferring residency from the cap value."}, "reason": {"statement": "State exact hint semantics and require a matched comparison.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#addresses-as-operands", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html"]}} +{"path": "wiki/techniques/vectorized-loads.md", "before": {"statement": "The impact of cache policies from the GPU Mode Hackathon:\n\n```\nNo cache policy differentiation: 39 us\nA: L1::no_allocate, B: L1::evict_last: 27 us (1.44x faster)\n```"}, "after": {"statement": "Amandeep Singh's attempts supply useful negative controls for the same task. Replacing two `uchar4` loads with one `uint2` load was reported 16–25% slower because extraction added instructions. Lowering `-maxrregcount` from 80 to 64 had no effect because natural allocation was already below the cap. The author later observed wider PTX forms and differentiated L1 policies in other solutions, but published no contestant code or controlled ablation for those observations.\n\nTogether, the reports justify testing vector width, decode organization, cache hints, and register limits. They do not establish that the widest load, a particular cache hint, or the lowest register cap is essential for GEMV, sub-byte arithmetic, or decode workloads."}, "reason": {"statement": "Restore the complete stage descriptions and label the values author-reported.", "urls": ["https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"]}} +{"path": "wiki/techniques/vectorized-loads.md", "before": {"statement": "The full set of PTX load cache qualifiers:\n\n```ptx\n// PTX load qualifiers for cache control\n\n// Default: normal L1 and L2 caching\nld.global.b32 %r, [%addr];\n\n// L1 bypass: skip L1, still cached in L2\nld.global.L1::no_allocate.b32 %r, [%addr];\n\n// L1 keep: prioritize keeping in L1 (evict last)\nld.global.L1::evict_last.b32 %r, [%addr];\n\n// L2 promotion hint (256-byte sector)\n// Used by DeepEP for communication overlap\nld.global.nc.L1::no_allocate.L2::256B.b32 %r, [%addr];\n\n// Non-coherent read-only (nc): uses texture path\n// Avoids coherence traffic, useful for read-only data\nld.global.nc.b32 %r, [%addr];\n```"}, "after": {"statement": "For each candidate width and cache policy:\n\n1. Prove base and per-thread address alignment for every executed access. Use a separate scalar or narrower-vector path for tails; validate minimum, maximum, and awkward sizes.\n2. Hold the algorithm, mapping, decode, unrolling, launch shape, compiler, and inputs fixed while changing one load or cache choice. Compare identical correctness oracles before timing.\n3. Inspect generated PTX and SASS. Record actual load instructions, registers per thread, spill loads/stores, stack bytes, shared memory, and launch parameters. A source cast or inline-PTX block does not bypass compiler register allocation.\n4. Profile the matched variants. Check achieved bandwidth, requested versus transferred bytes, cache hit behavior, instruction issue/stalls, and active warps with metrics available on the installed Nsight Compute and GPU versions.\n5. Use warmups, synchronization, repeated trials, and the same statistic for every production shape. Keep the change only where the declared metric improves without a correctness or resource regression.\n\nRegister caps and launch bounds are a separate variable. A nominal cap may be inert when natural allocation is lower, or it may trade registers for spills and extra instructions. Follow the resource and occupancy workflow in [Register Budgeting](register-budgeting.md) rather than inferring residency from the cap value."}, "reason": {"statement": "Use the archived grammar and scope each semantic statement.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld-global-nc", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#addresses-as-operands", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html"]}} +{"path": "wiki/techniques/vectorized-loads.md", "before": {"statement": "For memory-bound kernels, occupancy (number of concurrent warps) matters more than per-thread register count. Limiting registers per thread allows more warps to be resident:\n\n```cuda\n// Compile-time register budgeting\n// Lower register count -> higher occupancy -> better latency hiding\n\n// Problem 1 winner (rank 1): -maxrregcount=32\n// This allows 64 warps per SM (100% occupancy on SM100)\n// Sufficient for GEMV where each thread does minimal computation\n\n// Problem 1 rank 3: -maxrregcount=45\n// Allows 44 warps per SM (~69% occupancy)\n// More registers per thread for wider vectorized accumulation\n```\n\nThe tradeoff in a build system:\n\n```python\n# nvcc compilation with register budgeting\n# In CMakeLists.txt or build script:\n\n# For memory-bound GEMV kernel:\n# nvcc -maxrregcount=32 -arch=sm_100a gemv_kernel.cu -o gemv_kernel\n\n# For compute-bound GEMM kernel:\n# nvcc -arch=sm_100a gemm_kernel.cu -o gemm_kernel (no limit; needs ~128+ regs)\n\n# Per-kernel register limits in the same translation unit:\n# Use __launch_bounds__ to control per-kernel\n\n# Memory-bound: maximize occupancy\n__global__ void __launch_bounds__(256, 8) // 256 threads, min 8 blocks/SM\ngemv_kernel(/* ... */) { /* ... */ }\n\n# Compute-bound: maximize register availability\n__global__ void __launch_bounds__(512, 1) // 512 threads, min 1 block/SM\ngemm_kernel(/* ... */) { /* ... */ }\n```"}, "after": {"statement": "Yue Zhang reports the following CUDA progression for GPU Mode Problem 1. These are author-reported endpoints without raw repeated-trial data or released complete submission code:\n\n| Initial CUDA | Naive hand-written path | about 2000 µs |\n| CUDA optimization 1 | Coalescing, shared B, thread collaboration, warp reduction | about 443 µs |\n| CUDA optimization 2 | Remove shared B, per-thread tiles, `float4` loads, hardware intrinsics | about 39 µs |\n| CUDA optimization 3 | Vectorized PTX FP4 and scale decode | about 27 µs |\n| Parameter tuning | Threads per row and rows per block | about 26 µs |\n| ILP | Two tiles per loop iteration | about 22.9 µs |\n| Aggressive PTX fusion | Decode, scales, multiply, and accumulation in a larger PTX block | about 22.3 µs |\n\nThe submitted public-leaderboard score was 22.392 microseconds, the geometric mean over three benchmark shapes. The pinned task's theoretical model gives 8.622, 17.275, and 4.317 microseconds for those separate shapes; 8.622 microseconds is therefore not an aggregate “speed of light.” The 39-to-27 endpoints change the PTX load/decode path together and do not isolate cache policy, load width, or conversion choice."}, "reason": {"statement": "Route register-control details to the audited register-budgeting page and remove exact residency claims.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml"]}} +{"path": "wiki/techniques/vectorized-loads.md", "before": {"statement": "Combining all three techniques for the GPU Mode Hackathon Problem 1:\n\n```cuda\n// Optimized NVFP4 Batched GEMV\n// A: [M, K] NVFP4, B: [1, K] NVFP4, C: [M, 1] FP16\n// Memory-bound: maximize bandwidth utilization\n\n// NVFP4 Batched GEMV: each row processed by THREADS_PER_ROW threads\n// Memory-bound: maximize bandwidth with wide loads and cache policies\n\n#define BLOCK_M 4 // Rows per thread block\n#define THREADS 256\n#define THREADS_PER_ROW (THREADS / BLOCK_M) // 64 threads per row\n\n__global__ void __launch_bounds__(THREADS, 4)\nnvfp4_gemv_optimized(\n const uint8_t* __restrict__ A, // [M, K/2] packed FP4\n const uint8_t* __restrict__ B, // [1, K/2] packed FP4\n const fp8_e4m3* __restrict__ sfa, // [M, K/16] block scales\n const fp8_e4m3* __restrict__ sfb, // [1, K/16] block scales\n half* __restrict__ C, // [M, 1] output\n float global_scale_a, float global_scale_b,\n int M, int K)\n{\n // Map threads to rows: 64 threads per row, 4 rows per block\n int local_row = threadIdx.x / THREADS_PER_ROW; // 0..3\n int thread_in_row = threadIdx.x % THREADS_PER_ROW; // 0..63\n int row = blockIdx.x * BLOCK_M + local_row;\n if (row >= M) return;\n\n float acc = 0.0f;\n\n // Each of the 64 threads handles a distinct K-chunk (no duplication)\n // 64 elements per load × 64 threads = 4096 K elements per iteration\n for (int k = thread_in_row * 64; k < K; k += THREADS_PER_ROW * 64) {\n // Load B (reused across rows): L1::evict_last, 256-bit\n uint64_t b_raw[4];\n asm volatile(\n \"ld.global.L1::evict_last.v4.u64 {%0,%1,%2,%3}, [%4];\"\n : \"=l\"(b_raw[0]), \"=l\"(b_raw[1]), \"=l\"(b_raw[2]), \"=l\"(b_raw[3])\n : \"l\"((const uint64_t*)(B + k / 2)));\n\n // Load A (streamed once): L1::no_allocate, 256-bit\n uint64_t a_raw[4];\n asm volatile(\n \"ld.global.L1::no_allocate.v4.u64 {%0,%1,%2,%3}, [%4];\"\n : \"=l\"(a_raw[0]), \"=l\"(a_raw[1]), \"=l\"(a_raw[2]), \"=l\"(a_raw[3])\n : \"l\"((const uint64_t*)(A + row * (K / 2) + k / 2)));\n\n // Unpack FP4, apply block scale, dot-product\n for (int i = 0; i < 64; i++) {\n float a_val = unpack_fp4(a_raw, i) * get_block_scale(sfa, row, k + i);\n float b_val = unpack_fp4(b_raw, i) * get_block_scale(sfb, 0, k + i);\n acc += a_val * b_val;\n }\n }\n\n // Two-phase reduction: first within each warp, then across the 2 warps per row\n // Phase 1: warp-level reduction (32 threads → 1 partial sum per warp)\n for (int offset = 16; offset > 0; offset >>= 1) {\n acc += __shfl_xor_sync(0xFFFFFFFF, acc, offset);\n }\n\n // Phase 2: shared memory reduction across the 2 warps assigned to this row\n __shared__ float smem_reduce[BLOCK_M * 2]; // 2 warp partials per row\n int warp_in_row = thread_in_row / 32; // 0 or 1\n int lane = thread_in_row % 32;\n if (lane == 0) {\n smem_reduce[local_row * 2 + warp_in_row] = acc;\n }\n __syncthreads();\n\n // Final sum and store (one thread per row)\n if (thread_in_row == 0) {\n float result = smem_reduce[local_row * 2] + smem_reduce[local_row * 2 + 1];\n C[row] = __float2half(result * global_scale_a * global_scale_b);\n }\n}\n```"}, "after": {"statement": null}, "reason": {"statement": "No direct published contestant implementation supports repairing this invented kernel in place.", "urls": ["https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv", "https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html"]}} +{"path": "wiki/techniques/vectorized-loads.md", "before": {"statement": "| Baseline | Naive C++ | 2000 us | 1.0x |\n| Coalesced access | Memory layout fix | 443 us | 4.5x |\n| Hardware intrinsics | FP4 decode | 39 us | 51x |\n| PTX assembly | Vectorized loads + cache policy | 27 us | 74x |\n| ILP + register tuning | Unrolling + maxrregcount | 22.4 us | 89x |\n| Speed of light | | ~8.6 us | 233x |"}, "after": {"statement": "Amandeep Singh's attempts supply useful negative controls for the same task. Replacing two `uchar4` loads with one `uint2` load was reported 16–25% slower because extraction added instructions. Lowering `-maxrregcount` from 80 to 64 had no effect because natural allocation was already below the cap. The author later observed wider PTX forms and differentiated L1 policies in other solutions, but published no contestant code or controlled ablation for those observations.\n\nTogether, the reports justify testing vector width, decode organization, cache hints, and register limits. They do not establish that the widest load, a particular cache hint, or the lowest register cap is essential for GEMV, sub-byte arithmetic, or decode workloads."}, "reason": {"statement": "Use exact author-reported combined-stage descriptions and separate task-model rows.", "urls": ["https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"]}} +{"path": "wiki/techniques/vectorized-loads.md", "before": {"statement": "- **GEMV and memory-bound kernels**: Vectorized loads and cache policies are essential. These kernels are entirely limited by memory bandwidth.\n- **FP4/FP8 kernels**: Sub-byte data types make wide loads even more impactful since more elements fit in a single wide load.\n- **Decode-phase inference**: Single-token GEMV during autoregressive decoding is always memory-bound."}, "after": {"statement": "Yue Zhang reports the following CUDA progression for GPU Mode Problem 1. These are author-reported endpoints without raw repeated-trial data or released complete submission code:\n\n| Initial CUDA | Naive hand-written path | about 2000 µs |\n| CUDA optimization 1 | Coalescing, shared B, thread collaboration, warp reduction | about 443 µs |\n| CUDA optimization 2 | Remove shared B, per-thread tiles, `float4` loads, hardware intrinsics | about 39 µs |\n| CUDA optimization 3 | Vectorized PTX FP4 and scale decode | about 27 µs |\n| Parameter tuning | Threads per row and rows per block | about 26 µs |\n| ILP | Two tiles per loop iteration | about 22.9 µs |\n| Aggressive PTX fusion | Decode, scales, multiply, and accumulation in a larger PTX block | about 22.3 µs |\n\nThe submitted public-leaderboard score was 22.392 microseconds, the geometric mean over three benchmark shapes. The pinned task's theoretical model gives 8.622, 17.275, and 4.317 microseconds for those separate shapes; 8.622 microseconds is therefore not an aggregate “speed of light.” The 39-to-27 endpoints change the PTX load/decode path together and do not isolate cache policy, load width, or conversion choice."}, "reason": {"statement": "Replace category prescriptions with concrete selection criteria and measurement.", "urls": ["https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html#roofline", "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml"]}} +{"path": "wiki/techniques/vectorized-loads.md", "before": {"statement": "- 256-bit loads require 32-byte aligned addresses. Misaligned access falls back to multiple narrower transactions.\n- `L1::no_allocate` is harmful for data that will be reused. Only apply it to truly streaming access patterns."}, "after": {"statement": "PTX distinguishes cache operators, eviction priorities, prefetch-size hints, and non-coherent read-only loads. Representative legal forms are:\n\n```ptx\nld.global.L1::no_allocate.v4.u32 {x0, x1, x2, x3}, [addr];\nld.global.L1::evict_last.v4.u32 {x0, x1, x2, x3}, [addr];\nld.global.L2::256B.b32 x0, [addr];\nld.global.nc.b32 x0, [addr];\n```\n\n| `L1::no_allocate` | L1 eviction-priority selection that may be applied | Does not guarantee an L1 bypass or a speedup for a streaming operand |\n| `L1::evict_last` | Requests the corresponding L1 eviction priority | Does not guarantee that a line remains resident |\n| `L2::256B` | Hints that additional data of the stated size be prefetched into L2 | Is not a 256-byte load or a promotion guarantee |\n| `ld.global.nc` | Loads through a non-coherent read-only cache | Has architecture- and parallelism-dependent latency/throughput; it is not a general coherence optimization |\n\nCache-policy operands and prefetch-size qualifiers are performance hints and do not change the program's memory-consistency behavior. A reuse argument can motivate `evict_last`, and a one-pass stream can motivate `no_allocate`, but only a matched comparison determines whether either helps the concrete working set and launch.\n\nFor each candidate width and cache policy:\n\n1. Prove base and per-thread address alignment for every executed access. Use a separate scalar or narrower-vector path for tails; validate minimum, maximum, and awkward sizes.\n2. Hold the algorithm, mapping, decode, unrolling, launch shape, compiler, and inputs fixed while changing one load or cache choice. Compare identical correctness oracles before timing.\n3. Inspect generated PTX and SASS. Record actual load instructions, registers per thread, spill loads/stores, stack bytes, shared memory, and launch parameters. A source cast or inline-PTX block does not bypass compiler register allocation.\n4. Profile the matched variants. Check achieved bandwidth, requested versus transferred bytes, cache hit behavior, instruction issue/stalls, and active warps with metrics available on the installed Nsight Compute and GPU versions.\n5. Use warmups, synchronization, repeated trials, and the same statistic for every production shape. Keep the change only where the declared metric improves without a correctness or resource regression.\n\nRegister caps and launch bounds are a separate variable. A nominal cap may be inert when natural allocation is lower, or it may trade registers for spills and extra instructions. Follow the resource and occupancy workflow in [Register Budgeting](register-budgeting.md) rather than inferring residency from the cap value."}, "reason": {"statement": "State the normative correctness condition and empirical status of hints.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#addresses-as-operands", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld-global-nc", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html"]}} +{"path": "wiki/techniques/vectorized-loads.md", "before": {"statement": "- `-maxrregcount` that is too low causes register spilling to local memory, which is slower than the occupancy gain. Profile with Nsight Compute to find the optimal point.\n- PTX inline assembly bypasses the compiler's register allocator. Excessive inline PTX can interfere with compiler optimizations for surrounding code."}, "after": {"statement": "Yue Zhang reports the following CUDA progression for GPU Mode Problem 1. These are author-reported endpoints without raw repeated-trial data or released complete submission code:\n\n| Initial CUDA | Naive hand-written path | about 2000 µs |\n| CUDA optimization 1 | Coalescing, shared B, thread collaboration, warp reduction | about 443 µs |\n| CUDA optimization 2 | Remove shared B, per-thread tiles, `float4` loads, hardware intrinsics | about 39 µs |\n| CUDA optimization 3 | Vectorized PTX FP4 and scale decode | about 27 µs |\n| Parameter tuning | Threads per row and rows per block | about 26 µs |\n| ILP | Two tiles per loop iteration | about 22.9 µs |\n| Aggressive PTX fusion | Decode, scales, multiply, and accumulation in a larger PTX block | about 22.3 µs |\n\nThe submitted public-leaderboard score was 22.392 microseconds, the geometric mean over three benchmark shapes. The pinned task's theoretical model gives 8.622, 17.275, and 4.317 microseconds for those separate shapes; 8.622 microseconds is therefore not an aggregate “speed of light.” The 39-to-27 endpoints change the PTX load/decode path together and do not isolate cache policy, load width, or conversion choice."}, "reason": {"statement": "Separate opaque instruction scheduling effects from register allocation and keep performance empirical.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/inline-ptx-assembly/index.html", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html", "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml"]}} +{"path": "wiki/techniques/cache-policy.md", "before": {"statement": "PTX cache qualifiers (`L1::no_allocate`, `L1::evict_last`, `L1::evict_first`) let kernels hint to hardware how to handle cache admission for specific loads. Critical for memory-bound kernels where the L1 working set matters more than the compute."}, "after": {"statement": "PTX global loads and stores can carry cache operators and eviction-priority qualifiers. Global loads can also carry L2 prefetch-size hints. These controls let a kernel express a preference for a particular access; they do not guarantee cache admission, residence, eviction time, hit rate, or performance.\n\n| `L1::no_allocate` | Selects an L1 eviction priority that may be applied | That the access bypasses L1 |\n| `L1::evict_first` | Requests first-eviction priority | Immediate eviction |\n| `L1::evict_last` | Requests last-eviction priority | Persistent residence |\n| `L2::64B`, `128B`, `256B` | Hints that additional data of that size be fetched into L2 | A wider memory instruction or a completed prefetch |\n| `L2::cache_hint` with a policy operand | Supplies a created L2 eviction policy | That the hint is respected or changes memory consistency |\n\nThe following are legal PTX 9.0 instruction fragments when their operands and addresses are declared with matching types. Each vector access shown is 16 bytes and therefore requires 16-byte natural alignment:\n\n```ptx\n.reg .b64 addr_a, addr_b, addr_c;\n.reg .u32 a<4>, b<4>, c<4>;\n\nld.global.L1::no_allocate.v4.u32 {a0, a1, a2, a3}, [addr_a];\nld.global.L1::evict_last.v4.u32 {b0, b1, b2, b3}, [addr_b];\nst.global.L1::evict_first.v4.u32 [addr_c], {c0, c1, c2, c3};\n```\n\nThe common “stream A, retain B” explanation is a hypothesis about reuse and interference, not the semantics of the code. Even with correct alignment, the hardware may apply the priorities differently than a literal bypass/keep model suggests."}, "reason": {"statement": "State the exact contract and empirical scope.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld"]}} +{"path": "wiki/techniques/cache-policy.md", "before": {"statement": "```asm\n; Matrix A (streamed once per row, never reused): bypass L1\n; Avoids polluting L1 with one-shot data\nld.global.L1::no_allocate.v4.u64 {a0,a1,a2,a3}, [addr_a];\n\n; Vector B (reused across BLOCK_M rows): keep in L1\nld.global.L1::evict_last.v4.u64 {b0,b1,b2,b3}, [addr_b];\n\n; Streaming output: evict immediately after write\nst.global.L1::evict_first.v2.u64 [addr_c], {c0, c1};\n```"}, "after": {"statement": "Start from the concrete access trace rather than a kernel label or tensor name:\n\n1. Identify which addresses each warp touches, the reuse distance for each line, the live working set, and competing traffic at the same launch shape.\n2. Establish a correct default-policy baseline. Hold mapping, vector width, decode, unrolling, register controls, compiler, inputs, and launch parameters constant.\n3. Change one qualifier at a time. Re-run the same correctness oracle, including sizes that exercise alignment and tail paths.\n4. Record generated instructions and resources, then profile cache hit behavior, requested/transferred bytes, stalls, achieved bandwidth, and active warps with metrics available on the installed tool and target.\n5. Benchmark every production shape with the same warmup, synchronization, repetitions, and statistic. Retain only scoped improvements; a policy may help one shape and regress another.\n\nAn input being larger than L2, a kernel being described as memory-bound, or a tensor being called streamed/reused does not by itself predict a useful qualifier. Concurrent blocks can reuse data, a nominally reused vector may exceed the effective working set, and a hint can alter other traffic. Those are measurements, not properties inferred from names."}, "reason": {"statement": "Keep legal fragments with accurate non-guarantee comments and alignment requirements.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-st", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"]}} +{"path": "wiki/techniques/cache-policy.md", "before": {"statement": "Rank 1 submission used **different qualifiers per K-dimension variant**:\n- K=16384 (large): aggressive `L1::no_allocate` on A (huge streaming matrix)\n- K=2048 (small): relaxed balance since B is smaller relative to cache"}, "after": {"statement": null}, "reason": {"statement": "No direct public source supports this contestant attribution.", "urls": ["https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv"]}} +{"path": "wiki/techniques/cache-policy.md", "before": {"statement": "- NVFP4 GEMV: 443μs → 27μs (16x improvement) came partly from cache policy + PTX byte unpacking\n- On memory-bound kernels, cache policy can be the dominant lever"}, "after": {"statement": "Yue Zhang reports approximately 39 microseconds for a stage combining removal of B shared-memory staging, per-thread tiles, `float4` loads, and hardware conversion, followed by approximately 27 microseconds for a vectorized PTX FP4/scale-decode stage. Those endpoints change the load and decode path together. They do not measure a cache-hint-only speedup or establish cache policy as the dominant lever for memory-bound kernels."}, "reason": {"statement": "Preserve only the complete combined-stage observation and causal limit.", "urls": ["https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"]}} +{"path": "wiki/techniques/cache-policy.md", "before": {"statement": "- Memory-bound kernels (profile with Nsight Compute first)\n- Tensor with clear \"streaming\" vs \"reused\" access patterns\n- Inputs > L2 cache size (B200: 126MB)\n- Separate M and N tile loading patterns in GEMM"}, "after": {"statement": "Amandeep Singh reports that three inspected B200 NVFP4 GEMV solutions used `L1::no_allocate` for A and `L1::evict_last` for B alongside raw PTX decode, wide loads, exact-K specialization, tighter register caps, and—in one solution—sharing B reads across M rows. The report does not release those contestant implementations or a cache-policy-only ablation, and it does not support the former page's rank-1 per-K policy attribution."}, "reason": {"statement": "Replace with a one-variable measurement procedure based on concrete address streams and reuse distances.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv"]}} +{"path": "wiki/techniques/chunk-parallelism.md", "before": {"statement": "Linear attention variants (GatedDeltaNet, RetNet, Mamba) have O(n) complexity but naive implementations are sequential. Chunk-based parallelism divides the sequence into chunks of size C, computes within each chunk in parallel (matmul-friendly), and propagates state between chunks sequentially."}, "after": {"statement": "Some linear recurrent models admit an algebraically equivalent chunkwise formulation. Work local to a chunk can then be expressed with parallel matrix operations, while the state passed across chunk boundaries preserves the recurrence's sequence order. The exact transform is model-specific: it must be derived from the recurrence, not replaced by a generic attention matrix and additive state update.\n\nA correct implementation separates at least these obligations:\n\n1. Compute the chunk-local quantities required by the model's exact recurrence.\n2. Resolve boundary states in sequence order, either with an associative scan supported by the formulation or with explicitly ordered stages or launches.\n3. Combine each chunk's local result with its incoming boundary state and emit outputs in the original token order.\n4. Validate outputs and final states against a token-by-token reference across variable sequence lengths, chunk tails, batches, heads, dtypes, and gate extremes.\n\nAn ordinary GPU grid does not imply increasing program-ID execution order or grid-wide synchronization. Programs for every chunk therefore cannot safely read and overwrite one shared state pointer in a single unordered launch. A staged algorithm must make the boundary-state dependency explicit."}, "reason": {"statement": "Scope chunking to concrete recurrence transformations and implementations.", "urls": ["https://arxiv.org/abs/2503.14376v3", "https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921", "https://triton-lang.org/main/programming-guide/chapter-01/introduction.html"]}} +{"path": "wiki/techniques/chunk-parallelism.md", "before": {"statement": "```python\n@triton.jit\ndef chunk_parallel_linear_attn(Q, K, V, State, Output,\n chunk_size: tl.constexpr,\n d: tl.constexpr):\n # Grid: (num_chunks, num_heads, batch)\n chunk_id = tl.program_id(0)\n\n # Load chunk of Q, K, V\n q = tl.load(Q + chunk_id * chunk_size * d + offsets) # [C, d]\n k = tl.load(K + chunk_id * chunk_size * d + offsets)\n v = tl.load(V + chunk_id * chunk_size * d + offsets)\n\n # Intra-chunk: parallel O(C^2) attention-like compute\n scores = tl.dot(q, tl.trans(k))\n o_intra = tl.dot(scores, v)\n\n # Inter-chunk: sequential state propagation\n state = tl.load(State) # From previous chunk\n o_inter = tl.dot(q, state)\n\n # Combine and update state\n output = o_intra + o_inter\n state = update_state(state, k, v)\n\n tl.store(Output + offsets, output)\n tl.store(State, state)\n```"}, "after": {"statement": null}, "reason": {"statement": "The incomplete, racy pseudo-kernel cannot be repaired into a model-correct implementation without choosing a specific algorithm.", "urls": ["https://triton-lang.org/main/programming-guide/chapter-01/introduction.html", "https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921"]}} +{"path": "wiki/techniques/chunk-parallelism.md", "before": {"statement": "- **Small chunks (C=32)**: low latency decode, fewer intermediate materializations\n- **Large chunks (C=256-512)**: better tensor core utilization, higher throughput for prefill\n- **TFLA (Tiled FLA)**: two-level tiling allows arbitrary chunk sizes via recursive tiling"}, "after": {"statement": "The pinned NVlabs GatedDeltaNet repository uses chunkwise Triton kernels for training and a WY representation of the gated delta rule. That implementation is direct evidence for GatedDeltaNet chunking, but it is not equivalent to a generic `scores = Q @ K.T; output = scores @ V` snippet and does not supply a universal chunk-size rule.\n\nTiled Flash Linear Attention (TFLA) starts from the chunkwise formulation of linear RNNs and adds another level of sequence parallelization within a chunk. The authors state that this permits arbitrarily large chunks, raises arithmetic intensity, and reduces intermediate-state materialization. The paper and pinned official code apply the method to mLSTM and report H100 results; they do not establish a GatedDeltaNet implementation, a Blackwell/TMEM implementation, or a recursive-tiling API."}, "reason": {"statement": "Retain TFLA's exact contribution and make chunk size an empirical backend/model choice.", "urls": ["https://arxiv.org/abs/2503.14376v3", "https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921"]}} +{"path": "wiki/techniques/ping-pong-scheduling.md", "before": {"statement": "Ping-pong scheduling alternates two query tiles within a single CTA so the softmax warpgroup never stalls waiting for MMA. Introduced in FlashAttention-4 to exploit Blackwell's asymmetric hardware (2× tensor cores, same SFU count as Hopper)."}, "after": {"statement": "Ping-pong scheduling interleaves two output-tile states in one CTA. In the pinned FlashAttention-4 (FA4) SM100 forward implementation, `q_stage=2` makes the CTA cover two 128-row query tiles. One dedicated warp controls MMA, two four-warp groups perform softmax for the respective query stages, and a separate four-warp group performs correction work. While MMA advances one tile, the schedule tries to overlap softmax work for the other tile.\n\nThis is not merely ordinary double buffering of one producer-consumer stream. Each query stage owns a distinct output state, including its softmax and output TMEM regions, and the algorithm must preserve the dependencies among MMA accumulation, row statistics, rescaling/correction, and output consumption. “Ping-pong” describes this interleaving; it does not guarantee that either execution resource is stall-free."}, "reason": {"statement": "Describe the exact two-output-tile schedule and its bounded overlap objective.", "urls": ["https://arxiv.org/html/2603.05451v1", "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py"]}} +{"path": "wiki/techniques/ping-pong-scheduling.md", "before": {"statement": "```cuda\n// Two 128-token query tiles per CTA, alternating through the mainloop\n// Warpgroup 0: softmax for tile A while MMA runs on tile B\n// Warpgroup 1: softmax for tile B while MMA runs on tile A\n\n__global__ void fa4_ping_pong_attn(...) {\n int wg = warp_group_id();\n\n // TMEM holds accumulators for BOTH tiles\n uint32_t tmem_A = tmem_alloc(256);\n uint32_t tmem_B = tmem_alloc(256);\n\n for (int k = 0; k < num_kv_tiles; k++) {\n if (wg == 0) {\n // Compute softmax for tile A (previous MMA output)\n softmax_normalize(tmem_A);\n // Issue MMA for tile B next\n tcgen05_mma(Q_B_smem, K_smem[k], tmem_B);\n } else {\n softmax_normalize(tmem_B);\n tcgen05_mma(Q_A_smem, K_smem[k], tmem_A);\n }\n mbarrier_arrive(&ping_pong_sync);\n mbarrier_wait(&ping_pong_sync);\n }\n}\n```"}, "after": {"statement": "The concrete reference is [`flash_fwd_sm100.py` at commit `a369df707e1980fb328abcc1733e3457ec10155f`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py). Its default two-stage layout has the following software roles:\n\n| 0–3 | softmax for query stage 0 |\n| 4–7 | softmax for query stage 1 |\n| 8–11 | correction |\n| 12 | MMA control |\n| 13 | epilogue |\n| 14 | load |\n| 15 | empty in this configuration |\n\nThese IDs are implementation choices, not SM100 architectural roles. The source changes assignments for other configurations, including `q_stage=1`.\n\nThe implementation performs one collective TMEM allocation sized for its required columns and gives the two query stages disjoint offsets within that allocation. It also constructs multiple producer/consumer pipelines for distinct handoffs. A single generic barrier after issuing `tcgen05.mma` is not an equivalent implementation: tcgen05 work is asynchronous, and completion, memory visibility, stage reuse, and TMEM lifetime must follow the corresponding pipeline and ISA contracts.\n\nBefore treating a two-tile schedule as valid, verify all of the following in the complete kernel:\n\n1. TMEM regions for simultaneous tile states are disjoint, allocated and deallocated collectively, and remain live until their final consumers finish.\n2. Every SMEM or TMEM stage has one unambiguous producer/consumer phase owner; a stage cannot be overwritten until all operations that read it have completed.\n3. Asynchronous MMA completion is connected to the barrier that guards accumulator consumption and operand-stage reuse. A CTA barrier or an unrelated mbarrier arrival is insufficient.\n4. Softmax groups obey the implementation's explicit synchronization. FA4 serializes their exponential critical sections rather than allowing both groups to contend there simultaneously.\n5. Correction and epilogue work wait for the row statistics and accumulators they consume, including the proper proxy and execution-order fences.\n6. Boundary query rows, final key tiles, causal/window masks, and the pipeline tail preserve the same mathematical result as the reference attention computation."}, "reason": {"statement": "Replace executable-looking unsafe code with a source-pinned structural contract and verification checklist.", "urls": ["https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions"]}} +{"path": "wiki/techniques/ping-pong-scheduling.md", "before": {"statement": "- Single-tile schedule would leave SFU idle while MMA runs, and vice versa\n- Ping-pong keeps both units 100% busy"}, "after": {"statement": "Ping-pong scheduling interleaves two output-tile states in one CTA. In the pinned FlashAttention-4 (FA4) SM100 forward implementation, `q_stage=2` makes the CTA cover two 128-row query tiles. One dedicated warp controls MMA, two four-warp groups perform softmax for the respective query stages, and a separate four-warp group performs correction work. While MMA advances one tile, the schedule tries to overlap softmax work for the other tile.\n\nThis is not merely ordinary double buffering of one producer-consumer stream. Each query stage owns a distinct output state, including its softmax and output TMEM regions, and the algorithm must preserve the dependencies among MMA accumulation, row statistics, rescaling/correction, and output consumption. “Ping-pong” describes this interleaving; it does not guarantee that either execution resource is stall-free.\n\nThe paper reports up to 1613 TFLOP/s and 71% utilization for complete FA4 forward kernels. The accompanying author blog reports up to 1605 TFLOP/s and 71%. Those endpoints include partial software exponentials, conditional rescaling, TMEM partitioning, register allocation choices, and the rest of the pipeline. Neither source supplies a ping-pong-only ablation, and neither establishes 100% tensor-core or exponential-unit utilization."}, "reason": {"statement": "Retain the overlap motivation without universal idle-state or utilization assertions.", "urls": ["https://arxiv.org/html/2603.05451v1", "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py", "https://tridao.me/blog/2026/flash4/"]}} +{"path": "wiki/techniques/ping-pong-scheduling.md", "before": {"statement": "- FA4 achieves 1605 TFLOPS BF16 (71% utilization) with this pattern"}, "after": {"statement": "The paper reports up to 1613 TFLOP/s and 71% utilization for complete FA4 forward kernels. The accompanying author blog reports up to 1605 TFLOP/s and 71%. Those endpoints include partial software exponentials, conditional rescaling, TMEM partitioning, register allocation choices, and the rest of the pipeline. Neither source supplies a ping-pong-only ablation, and neither establishes 100% tensor-core or exponential-unit utilization."}, "reason": {"statement": "Report the result only at full-system scope and explicitly deny isolated attribution.", "urls": ["https://tridao.me/blog/2026/flash4/", "https://arxiv.org/html/2603.05451v1"]}} +{"path": "wiki/techniques/ping-pong-scheduling.md", "before": {"statement": "- Compute-bound attention kernels on Blackwell\n- Kernels where softmax/epilogue is SFU-heavy\n- Not useful on Hopper (balance is different)"}, "after": {"statement": "Use the pinned SM100 kernel as the concrete reference when two independent query/output states can coexist within the register, shared-memory, and TMEM budgets. Compare `q_stage=2` with the source-supported `q_stage=1` configuration while holding the remaining build, input, tile, and launch choices fixed. Measure end-to-end time, MMA and exponential-pipeline activity, barrier stalls, per-role idle time, register spills, and occupancy across representative shapes.\n\nDo not select the technique from a generic “compute-bound” or “SFU-heavy” label alone. The FA4 paper says its schedule is similar to FlashAttention-3's Hopper ping-pong schedule, so the scheduling idea is not intrinsically Blackwell-only; this page's pinned implementation and hardware figures are specifically SM100. Keep the two-stage form only when the controlled comparison shows a relevant improvement and correctness tests cover boundaries and pipeline tails."}, "reason": {"statement": "Scope concrete code to SM100 and recommend controlled measurement instead of workload-label prescriptions.", "urls": ["https://arxiv.org/html/2603.05451v1", "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py"]}} +{"path": "wiki/techniques/ping-pong-scheduling.md", "before": {"statement": "Verbatim upstream code lives in [`artifacts/kernels/ping-pong-scheduling/full/`](../../artifacts/kernels/ping-pong-scheduling/full/); labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` header) live in [`artifacts/kernels/ping-pong-scheduling/variants/`](../../artifacts/kernels/ping-pong-scheduling/variants/). Every file's SHA-256 and upstream-pinning metadata is in `PROVENANCE.yaml` inside each bundle.\n\nQuery via:\n\n```bash\npython3 scripts/get_page.py technique-ping-pong-scheduling --include-code\n```"}, "after": {"statement": null}, "reason": {"statement": "Remove the misleading page linkage while preserving the separately provenance-labeled files for repository integrity.", "urls": []}} +{"path": "wiki/patterns/pipeline-stalls.md", "before": {"statement": "Nsight Compute shows TMA or tcgen05 units idle despite nominally compute-bound workload. Tensor core utilization drops during specific phases of the kernel. Warp-level profiling reveals threads blocked on `mbarrier.try_wait` more than expected."}, "after": {"statement": "A pipeline stall is lost issue opportunity caused by a producer-consumer dependency on the critical path. Low TMA or tensor-core activity, a warp sampled at a barrier wait, or a phase-local utilization drop is only a lead. Expected dependency waits, memory latency, execution-pipeline contention, load imbalance, prologue/tail work, and synchronization defects can produce similar observations.\n\nNsight Compute's Warp State Statistics describes why sampled warps could not issue, while Scheduler Statistics shows whether schedulers had eligible warps and issued instructions. NVIDIA cautions that stalls are not necessarily performance-limiting and should be prioritized when schedulers fail to issue. Metric availability and names vary by tool and chip, so query the installed profiler and use its shipped sections rather than assuming a Blackwell metric name. PM sampling can add a timeline on supported systems, but warp sampling itself has no time resolution."}, "reason": {"statement": "Require scheduler issue evidence, source correlation, and a controlled causal perturbation.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} +{"path": "wiki/patterns/pipeline-stalls.md", "before": {"statement": "1. **Insufficient pipeline depth**: 2 stages cannot hide a 3-cycle latency chain"}, "after": {"statement": "1. Establish a synchronized end-to-end timing regression on fixed inputs and confirm outputs against a reference. Record GPU, clocks, toolkit, build flags, kernel name, launch shape, and resource use.\n2. Collect Speed-of-Light, Scheduler Statistics, Warp State Statistics, and source/SASS correlation with the sections available in the installed Nsight Compute. Account for replay and sampling effects.\n3. Map the dominant not-issued locations to concrete edges: TMA-full wait, MMA-completion wait, output-buffer reuse, CTA barrier, queue starvation, or pipeline prologue/tail.\n4. State a prediction. For example: if TMA readiness is the critical edge, a legal extra operand stage should reduce that wait and runtime; if only the tail is exposed, the steady-state wait distribution should remain largely unchanged.\n5. Change one variable and repeat identical correctness, warmup, timing, and profiler collection. Reject a cause when its predicted counter and time movement do not occur.\n\n| Vary legal SMEM stage count | Producer cannot stay far enough ahead of MMA | Same tile/math/roles; record SMEM, occupancy, short-loop behavior, and tail |\n| Separate long-lived warp roles | Role handoff or control work is on the issue path | Same stages and tile; record registers, active warps, and per-role idle/backpressure |\n| Add a second TMEM output region | Epilogue holds the only accumulator region | Prove disjoint lifetime; record TMEM columns and epilogue/MMA completion waits |\n| Interleave two query/output tiles | Softmax/correction and MMA have independent ready work | Prove separate state and dependencies; compare one versus two query stages |\n\nNone is a universal cure. More stages consume storage and can reduce occupancy; specialization adds warps and synchronization; TMEM buffering consumes columns; multi-tile schedules increase live state. A memory-bound roofline classification also does not prove pipelining is useless: test it only when the proposed change has a specific issue-gap, latency, or transaction-efficiency prediction."}, "reason": {"statement": "Make depth a resource-checked controlled variable.", "urls": ["https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html", "https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100", "https://arxiv.org/html/2603.05451v1"]}} +{"path": "wiki/patterns/pipeline-stalls.md", "before": {"statement": "3. **Missing `tcgen05.fence::after_thread_sync`**: MMA reads SMEM before TMA transfer fully visible"}, "after": {"statement": "Do this before tuning around an mbarrier wait:\n\n1. Identify the exact barrier object, owner, initialized arrival count, initial phase, and storage stage for the wait.\n2. For a TMA global-to-shared load, account for the producer's software arrival and expected transaction bytes. `mbarrier.arrive.expect_tx` performs an arrival and adds transaction bytes; the TMA `.mbarrier::complete_tx::bytes` operation decrements transaction bytes, not a second arrival.\n3. Wait for the matching phase of that same object. Phase parity changes when that barrier completes a phase and is reused; “flip after every wait” is not a valid global rule when a thread waits on multiple objects or phases.\n4. A successful acquire wait supplies the documented visibility for associated prior `cp.async.bulk` work before the consumer reads SMEM.\n5. Before reusing SMEM operands read by asynchronous MMA, attach completion of the relevant tcgen05 work with `tcgen05.commit` and observe its mbarrier. Issue is not completion.\n6. When asynchronous tcgen05 operations cross a thread handoff, place `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` around the applicable execution-ordering operation. The after fence is not a generic replacement for the TMA acquire wait.\n7. Verify prologue, steady-state wraparound, short loops, and the producer/consumer tails. A correct steady-state loop can still hang or reuse live storage at a boundary.\n\nAn extra software arrival is erroneous only relative to the barrier's initialized and phase-specific accounting. Diagnose pending arrivals and transaction bytes separately instead of assuming that TMA hardware performs another arrival."}, "reason": {"statement": "Separate TMA completion/visibility from cross-thread tcgen05 ordering.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"]}} +{"path": "wiki/patterns/pipeline-stalls.md", "before": {"statement": "4. **Single-tile scheduling**: All warps serialized on one tile's softmax/epilogue"}, "after": {"statement": "1. Establish a synchronized end-to-end timing regression on fixed inputs and confirm outputs against a reference. Record GPU, clocks, toolkit, build flags, kernel name, launch shape, and resource use.\n2. Collect Speed-of-Light, Scheduler Statistics, Warp State Statistics, and source/SASS correlation with the sections available in the installed Nsight Compute. Account for replay and sampling effects.\n3. Map the dominant not-issued locations to concrete edges: TMA-full wait, MMA-completion wait, output-buffer reuse, CTA barrier, queue starvation, or pipeline prologue/tail.\n4. State a prediction. For example: if TMA readiness is the critical edge, a legal extra operand stage should reduce that wait and runtime; if only the tail is exposed, the steady-state wait distribution should remain largely unchanged.\n5. Change one variable and repeat identical correctness, warmup, timing, and profiler collection. Reject a cause when its predicted counter and time movement do not occur.\n\n| Vary legal SMEM stage count | Producer cannot stay far enough ahead of MMA | Same tile/math/roles; record SMEM, occupancy, short-loop behavior, and tail |\n| Separate long-lived warp roles | Role handoff or control work is on the issue path | Same stages and tile; record registers, active warps, and per-role idle/backpressure |\n| Add a second TMEM output region | Epilogue holds the only accumulator region | Prove disjoint lifetime; record TMEM columns and epilogue/MMA completion waits |\n| Interleave two query/output tiles | Softmax/correction and MMA have independent ready work | Prove separate state and dependencies; compare one versus two query stages |\n\nNone is a universal cure. More stages consume storage and can reduce occupancy; specialization adds warps and synchronization; TMEM buffering consumes columns; multi-tile schedules increase live state. A memory-bound roofline classification also does not prove pipelining is useless: test it only when the proposed change has a specific issue-gap, latency, or transaction-efficiency prediction."}, "reason": {"statement": "Present multi-tile scheduling as a hypothesis only when independent state and work exist.", "urls": ["https://arxiv.org/html/2603.05451v1", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html", "https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100"]}} +{"path": "wiki/patterns/pipeline-stalls.md", "before": {"statement": "5. **Producer over-arrives**: Manual `mbarrier_arrive` after async TMA — hardware + manual both arrive, next stage gets stale release"}, "after": {"statement": "Do this before tuning around an mbarrier wait:\n\n1. Identify the exact barrier object, owner, initialized arrival count, initial phase, and storage stage for the wait.\n2. For a TMA global-to-shared load, account for the producer's software arrival and expected transaction bytes. `mbarrier.arrive.expect_tx` performs an arrival and adds transaction bytes; the TMA `.mbarrier::complete_tx::bytes` operation decrements transaction bytes, not a second arrival.\n3. Wait for the matching phase of that same object. Phase parity changes when that barrier completes a phase and is reused; “flip after every wait” is not a valid global rule when a thread waits on multiple objects or phases.\n4. A successful acquire wait supplies the documented visibility for associated prior `cp.async.bulk` work before the consumer reads SMEM.\n5. Before reusing SMEM operands read by asynchronous MMA, attach completion of the relevant tcgen05 work with `tcgen05.commit` and observe its mbarrier. Issue is not completion.\n6. When asynchronous tcgen05 operations cross a thread handoff, place `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` around the applicable execution-ordering operation. The after fence is not a generic replacement for the TMA acquire wait.\n7. Verify prologue, steady-state wraparound, short loops, and the producer/consumer tails. A correct steady-state loop can still hang or reuse live storage at a boundary.\n\nAn extra software arrival is erroneous only relative to the barrier's initialized and phase-specific accounting. Diagnose pending arrivals and transaction bytes separately instead of assuming that TMA hardware performs another arrival."}, "reason": {"statement": "Teach the two independent counters exactly.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence"]}} +{"path": "wiki/patterns/pipeline-stalls.md", "before": {"statement": "| [Pipeline stages](../techniques/pipeline-stages.md) | Increase NUM_STAGES (3-5 typical on Blackwell) |\n| [Warp specialization](../techniques/warp-specialization.md) | Dedicated warps for TMA/MMA/epilogue eliminate role-switching stalls |"}, "after": {"statement": "1. Establish a synchronized end-to-end timing regression on fixed inputs and confirm outputs against a reference. Record GPU, clocks, toolkit, build flags, kernel name, launch shape, and resource use.\n2. Collect Speed-of-Light, Scheduler Statistics, Warp State Statistics, and source/SASS correlation with the sections available in the installed Nsight Compute. Account for replay and sampling effects.\n3. Map the dominant not-issued locations to concrete edges: TMA-full wait, MMA-completion wait, output-buffer reuse, CTA barrier, queue starvation, or pipeline prologue/tail.\n4. State a prediction. For example: if TMA readiness is the critical edge, a legal extra operand stage should reduce that wait and runtime; if only the tail is exposed, the steady-state wait distribution should remain largely unchanged.\n5. Change one variable and repeat identical correctness, warmup, timing, and profiler collection. Reject a cause when its predicted counter and time movement do not occur.\n\n| Vary legal SMEM stage count | Producer cannot stay far enough ahead of MMA | Same tile/math/roles; record SMEM, occupancy, short-loop behavior, and tail |\n| Separate long-lived warp roles | Role handoff or control work is on the issue path | Same stages and tile; record registers, active warps, and per-role idle/backpressure |\n| Add a second TMEM output region | Epilogue holds the only accumulator region | Prove disjoint lifetime; record TMEM columns and epilogue/MMA completion waits |\n| Interleave two query/output tiles | Softmax/correction and MMA have independent ready work | Prove separate state and dependencies; compare one versus two query stages |\n\nNone is a universal cure. More stages consume storage and can reduce occupancy; specialization adds warps and synchronization; TMEM buffering consumes columns; multi-tile schedules increase live state. A memory-bound roofline classification also does not prove pipelining is useless: test it only when the proposed change has a specific issue-gap, latency, or transaction-efficiency prediction."}, "reason": {"statement": "Turn both into measured candidate changes with explicit costs.", "urls": ["https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu", "https://gau-nernst.github.io/tcgen05/", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html", "https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100", "https://arxiv.org/html/2603.05451v1"]}} +{"path": "wiki/patterns/pipeline-stalls.md", "before": {"statement": "```\n1. Profile with Nsight Compute, check tensor core active cycles\n2. Inspect mbarrier wait stalls in warp state breakdown\n3. Verify phase tracking increments correctly (each wait should flip parity)\n4. Check that TMA uses arrive_expect_tx + mbarrier target (not manual arrive)\n5. Ensure tcgen05.fence::after_thread_sync between TMA wait and MMA issue\n6. Measure pipeline depth: can you add more NUM_STAGES?\n```"}, "after": {"statement": "A pipeline stall is lost issue opportunity caused by a producer-consumer dependency on the critical path. Low TMA or tensor-core activity, a warp sampled at a barrier wait, or a phase-local utilization drop is only a lead. Expected dependency waits, memory latency, execution-pipeline contention, load imbalance, prologue/tail work, and synchronization defects can produce similar observations.\n\nNsight Compute's Warp State Statistics describes why sampled warps could not issue, while Scheduler Statistics shows whether schedulers had eligible warps and issued instructions. NVIDIA cautions that stalls are not necessarily performance-limiting and should be prioritized when schedulers fail to issue. Metric availability and names vary by tool and chip, so query the installed profiler and use its shipped sections rather than assuming a Blackwell metric name. PM sampling can add a timeline on supported systems, but warp sampling itself has no time resolution.\n\nDo this before tuning around an mbarrier wait:\n\n1. Identify the exact barrier object, owner, initialized arrival count, initial phase, and storage stage for the wait.\n2. For a TMA global-to-shared load, account for the producer's software arrival and expected transaction bytes. `mbarrier.arrive.expect_tx` performs an arrival and adds transaction bytes; the TMA `.mbarrier::complete_tx::bytes` operation decrements transaction bytes, not a second arrival.\n3. Wait for the matching phase of that same object. Phase parity changes when that barrier completes a phase and is reused; “flip after every wait” is not a valid global rule when a thread waits on multiple objects or phases.\n4. A successful acquire wait supplies the documented visibility for associated prior `cp.async.bulk` work before the consumer reads SMEM.\n5. Before reusing SMEM operands read by asynchronous MMA, attach completion of the relevant tcgen05 work with `tcgen05.commit` and observe its mbarrier. Issue is not completion.\n6. When asynchronous tcgen05 operations cross a thread handoff, place `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` around the applicable execution-ordering operation. The after fence is not a generic replacement for the TMA acquire wait.\n7. Verify prologue, steady-state wraparound, short loops, and the producer/consumer tails. A correct steady-state loop can still hang or reuse live storage at a boundary.\n\nAn extra software arrival is erroneous only relative to the barrier's initialized and phase-specific accounting. Diagnose pending arrivals and transaction bytes separately instead of assuming that TMA hardware performs another arrival.\n\n1. Establish a synchronized end-to-end timing regression on fixed inputs and confirm outputs against a reference. Record GPU, clocks, toolkit, build flags, kernel name, launch shape, and resource use.\n2. Collect Speed-of-Light, Scheduler Statistics, Warp State Statistics, and source/SASS correlation with the sections available in the installed Nsight Compute. Account for replay and sampling effects.\n3. Map the dominant not-issued locations to concrete edges: TMA-full wait, MMA-completion wait, output-buffer reuse, CTA barrier, queue starvation, or pipeline prologue/tail.\n4. State a prediction. For example: if TMA readiness is the critical edge, a legal extra operand stage should reduce that wait and runtime; if only the tail is exposed, the steady-state wait distribution should remain largely unchanged.\n5. Change one variable and repeat identical correctness, warmup, timing, and profiler collection. Reject a cause when its predicted counter and time movement do not occur.\n\n| Vary legal SMEM stage count | Producer cannot stay far enough ahead of MMA | Same tile/math/roles; record SMEM, occupancy, short-loop behavior, and tail |\n| Separate long-lived warp roles | Role handoff or control work is on the issue path | Same stages and tile; record registers, active warps, and per-role idle/backpressure |\n| Add a second TMEM output region | Epilogue holds the only accumulator region | Prove disjoint lifetime; record TMEM columns and epilogue/MMA completion waits |\n| Interleave two query/output tiles | Softmax/correction and MMA have independent ready work | Prove separate state and dependencies; compare one versus two query stages |\n\nNone is a universal cure. More stages consume storage and can reduce occupancy; specialization adds warps and synchronization; TMEM buffering consumes columns; multi-tile schedules increase live state. A memory-bound roofline classification also does not prove pipelining is useless: test it only when the proposed change has a specific issue-gap, latency, or transaction-efficiency prediction."}, "reason": {"statement": "Replace with separate profiling, barrier-accounting, and controlled-experiment stages.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100", "https://arxiv.org/html/2603.05451v1"]}} +{"path": "wiki/patterns/pipeline-stalls.md", "before": {"statement": "- 1-stage: 62% of cuBLAS (TMA blocks MMA)\n- 3-stage pipelined: 70% (hide most TMA latency)\n- Warp specialized: 80% (no role switching)\n- Add 2-SM MMA: 86% (larger tile, more reuse)\n- Persistent + CLC: 98% (eliminate tail effect)"}, "after": {"statement": "Gau Nernst reports the following for one `M=N=K=4096` BF16 GEMM on a Modal B200 using PyTorch 2.9.1 and CUDA 13. Percentages below are computed from the reported 1506.74-TFLOP/s cuBLAS value.\n\n| v1a | basic tcgen05 + 2D 16-byte TMA | 254.62 | 16.90% |\n| v2b | 3D 128-byte TMA | 695.43 | 46.15% |\n| v3 | pipelining | 939.61 | 62.36% |\n| v4 | warp specialization | 1208.83 | 80.23% |\n| v5 | 2-SM MMA | 1302.29 | 86.43% |\n| v6 | persistent kernel with static scheduling | 1475.93 | 97.96% |\n\nThe pinned v3 source instantiates two stages, not three. Each row is a cumulative source version rather than an isolated microbenchmark of the named mechanism. The final version uses static scheduling; the author explicitly says Cluster Launch Control was not added. The result is evidence that these changes helped that implementation and shape, not a Blackwell progression template."}, "reason": {"statement": "Report the exact source table and combined-version scope.", "urls": ["https://gau-nernst.github.io/tcgen05/", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu"]}} +{"path": "wiki/patterns/pipeline-stalls.md", "before": {"statement": "- Too many stages consume SMEM; exceeds 228KB budget"}, "after": {"statement": "1. Establish a synchronized end-to-end timing regression on fixed inputs and confirm outputs against a reference. Record GPU, clocks, toolkit, build flags, kernel name, launch shape, and resource use.\n2. Collect Speed-of-Light, Scheduler Statistics, Warp State Statistics, and source/SASS correlation with the sections available in the installed Nsight Compute. Account for replay and sampling effects.\n3. Map the dominant not-issued locations to concrete edges: TMA-full wait, MMA-completion wait, output-buffer reuse, CTA barrier, queue starvation, or pipeline prologue/tail.\n4. State a prediction. For example: if TMA readiness is the critical edge, a legal extra operand stage should reduce that wait and runtime; if only the tail is exposed, the steady-state wait distribution should remain largely unchanged.\n5. Change one variable and repeat identical correctness, warmup, timing, and profiler collection. Reject a cause when its predicted counter and time movement do not occur.\n\n| Vary legal SMEM stage count | Producer cannot stay far enough ahead of MMA | Same tile/math/roles; record SMEM, occupancy, short-loop behavior, and tail |\n| Separate long-lived warp roles | Role handoff or control work is on the issue path | Same stages and tile; record registers, active warps, and per-role idle/backpressure |\n| Add a second TMEM output region | Epilogue holds the only accumulator region | Prove disjoint lifetime; record TMEM columns and epilogue/MMA completion waits |\n| Interleave two query/output tiles | Softmax/correction and MMA have independent ready work | Prove separate state and dependencies; compare one versus two query stages |\n\nNone is a universal cure. More stages consume storage and can reduce occupancy; specialization adds warps and synchronization; TMEM buffering consumes columns; multi-tile schedules increase live state. A memory-bound roofline classification also does not prove pipelining is useless: test it only when the proposed change has a specific issue-gap, latency, or transaction-efficiency prediction."}, "reason": {"statement": "Require explicit storage arithmetic and compiled resource checks.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html", "https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100", "https://arxiv.org/html/2603.05451v1"]}} +{"path": "wiki/patterns/pipeline-stalls.md", "before": {"statement": "- Profile first — pipeline is a waste of effort on memory-bound kernels"}, "after": {"statement": "- [Nsight Compute 2025.3 Profiling Guide](https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html)\n- [PTX ISA 9.0 mbarrier waits and visibility](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait)\n- [PTX ISA 9.0 tcgen05 execution ordering](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence)\n- [Pinned tutorial v3 source](https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu)\n- [Tutorial progression](https://gau-nernst.github.io/tcgen05/)"}, "reason": {"statement": "Use causal measurements rather than excluding an entire category.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu", "https://gau-nernst.github.io/tcgen05/"]}} +{"path": "wiki/patterns/compute-bound.md", "before": {"statement": "Tensor core utilization below 70%. Memory bandwidth is not saturated. Kernel is compute-bound but not reaching peak FLOPS."}, "after": {"statement": "“Below peak FLOPS” is not itself a bottleneck diagnosis. Select the peak for the executed datatype, instruction kind, sparsity/scaling mode, clocks, and number of participating SMs. Then use arithmetic intensity and achieved compute/memory ceilings to decide whether the measured kernel lies on the compute side of the relevant roofline. Unsaturated DRAM bandwidth and tensor-core utilization below an arbitrary threshold such as 70% do not prove compute boundedness.\n\nAlso distinguish whole-kernel throughput from tensor-core active time. A correct kernel can have efficient MMA intervals yet spend material time in TMA readiness, CUDA-core transforms, reductions, synchronization, epilogue work, or the grid tail. Scheduler under-issue plus source/SASS correlation identifies where issue opportunity is lost; a high stall sample count alone does not establish the cause."}, "reason": {"statement": "Replace the threshold with roofline and scheduler evidence.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} +{"path": "wiki/patterns/compute-bound.md", "before": {"statement": "3. **Single-SM MMA tiles too small**: Not fully utilizing available compute"}, "after": {"statement": "Enumerate stage counts that compile and fit the complete shared-storage allocation. More stages can overlap TMA production with MMA consumption, but they also consume SMEM, change occupancy, and enlarge prologue/tail costs. There is no architecture-wide “three to five stages” range; the cited tutorial's beneficial v3 kernel uses two stages.\n\nDedicated producer, MMA, or epilogue warps can remove role switching from a loop and allow independent work to progress, but add live warps, registers, and synchronization. They do not eliminate stalls. If the epilogue owns the only output region, compare one versus multiple disjoint TMEM regions and prove MMA-completion and epilogue-load completion before each handoff.\n\n`cta_group::2` requires a valid cluster and exact kind-, shape-, layout-, descriptor-, and peer-resource constraints. It does not universally mean `m256n256`, require identical SMEM layouts, or promise twice the compute per cycle: one cooperative operation consumes resources from two CTAs. Compare total and per-SM throughput against a legal one-CTA mapping and measure whether peer operand reuse or a larger aggregate tile reduces traffic enough to repay coordination and occupancy costs.\n\nOptimize non-MMA work only after showing that it lies on the critical path. Fusion may remove intermediate traffic; tile interleaving may expose independent work; an approximation may exchange numerical error for throughput. Validate error against the application oracle rather than assuming a software transcendental is interchangeable with the hardware operation."}, "reason": {"statement": "Make CTA-group mode a controlled legal-shape and resource experiment.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu", "https://gau-nernst.github.io/tcgen05/"]}} +{"path": "wiki/patterns/compute-bound.md", "before": {"statement": "| [2-SM cooperative](../hardware/2sm-cooperative.md) | Double effective MMA tile (m256×n256), 2× compute per cycle |"}, "after": {"statement": "Enumerate stage counts that compile and fit the complete shared-storage allocation. More stages can overlap TMA production with MMA consumption, but they also consume SMEM, change occupancy, and enlarge prologue/tail costs. There is no architecture-wide “three to five stages” range; the cited tutorial's beneficial v3 kernel uses two stages.\n\nDedicated producer, MMA, or epilogue warps can remove role switching from a loop and allow independent work to progress, but add live warps, registers, and synchronization. They do not eliminate stalls. If the epilogue owns the only output region, compare one versus multiple disjoint TMEM regions and prove MMA-completion and epilogue-load completion before each handoff.\n\n`cta_group::2` requires a valid cluster and exact kind-, shape-, layout-, descriptor-, and peer-resource constraints. It does not universally mean `m256n256`, require identical SMEM layouts, or promise twice the compute per cycle: one cooperative operation consumes resources from two CTAs. Compare total and per-SM throughput against a legal one-CTA mapping and measure whether peer operand reuse or a larger aggregate tile reduces traffic enough to repay coordination and occupancy costs.\n\nOptimize non-MMA work only after showing that it lies on the critical path. Fusion may remove intermediate traffic; tile interleaving may expose independent work; an approximation may exchange numerical error for throughput. Validate error against the application oracle rather than assuming a software transcendental is interchangeable with the hardware operation."}, "reason": {"statement": "State exact compatibility scope and measure the paired-resource tradeoff.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu", "https://gau-nernst.github.io/tcgen05/"]}} +{"path": "wiki/patterns/compute-bound.md", "before": {"statement": "| [Warp specialization](../techniques/warp-specialization.md) | Dedicated warps for TMA/MMA/epilogue, no stalls |"}, "after": {"statement": "Enumerate stage counts that compile and fit the complete shared-storage allocation. More stages can overlap TMA production with MMA consumption, but they also consume SMEM, change occupancy, and enlarge prologue/tail costs. There is no architecture-wide “three to five stages” range; the cited tutorial's beneficial v3 kernel uses two stages.\n\nDedicated producer, MMA, or epilogue warps can remove role switching from a loop and allow independent work to progress, but add live warps, registers, and synchronization. They do not eliminate stalls. If the epilogue owns the only output region, compare one versus multiple disjoint TMEM regions and prove MMA-completion and epilogue-load completion before each handoff.\n\n`cta_group::2` requires a valid cluster and exact kind-, shape-, layout-, descriptor-, and peer-resource constraints. It does not universally mean `m256n256`, require identical SMEM layouts, or promise twice the compute per cycle: one cooperative operation consumes resources from two CTAs. Compare total and per-SM throughput against a legal one-CTA mapping and measure whether peer operand reuse or a larger aggregate tile reduces traffic enough to repay coordination and occupancy costs.\n\nOptimize non-MMA work only after showing that it lies on the critical path. Fusion may remove intermediate traffic; tile interleaving may expose independent work; an approximation may exchange numerical error for throughput. Validate error against the application oracle rather than assuming a software transcendental is interchangeable with the hardware operation."}, "reason": {"statement": "Treat role separation as a candidate with resource and synchronization costs.", "urls": ["https://gau-nernst.github.io/tcgen05/", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/patterns/compute-bound.md", "before": {"statement": "```\n// Problem: Blackwell doubles tensor core throughput but SFU count unchanged\n// SFU bottleneck: exp() for softmax\n//\n// Solution: Software 2^x via Cody-Waite + Horner polynomial\n// Distributes across FMA units, multiplying exponential throughput\n// Result: 1605 TFLOPS (71% utilization) on B200\n```"}, "after": {"statement": "FA4's B200 analysis reports 8192 BF16 MMA operations per clock per SM versus 4096 on Hopper, while exponential throughput is 16 operations per clock per SM on both. Its response is a coordinated design: two-output-tile scheduling, selected software exponentials, conditional rescaling, TMEM partitioning, and pipeline/register choices.\n\nThe software path uses base-2 range reduction with `n=floor(x)` and a cubic FMA polynomial for a selected fraction of exponential evaluations; other values still use hardware `ex2`. The paper evaluates approximation error and end-to-end accuracy. This is not a claim that every exponential uses Cody-Waite reduction or that polynomial evaluation universally multiplies exponential throughput.\n\nThe paper reports up to 1613 TFLOP/s and 71% for complete FA4 forward kernels; the author blog reports up to 1605 TFLOP/s and 71%. Neither number isolates software exponentiation or any other single technique. Use FA4 as evidence that non-MMA work can become material after tensor-core throughput increases, not as a recipe for an unrelated compute-bound kernel."}, "reason": {"statement": "Remove the misleading pseudo-code and present the exact scoped case in prose.", "urls": ["https://arxiv.org/html/2603.05451v1", "https://tridao.me/blog/2026/flash4/"]}} +{"path": "wiki/patterns/compute-bound.md", "before": {"statement": "- 2-SM cooperative requires cluster configuration and identical SMEM layouts"}, "after": {"statement": "Enumerate stage counts that compile and fit the complete shared-storage allocation. More stages can overlap TMA production with MMA consumption, but they also consume SMEM, change occupancy, and enlarge prologue/tail costs. There is no architecture-wide “three to five stages” range; the cited tutorial's beneficial v3 kernel uses two stages.\n\nDedicated producer, MMA, or epilogue warps can remove role switching from a loop and allow independent work to progress, but add live warps, registers, and synchronization. They do not eliminate stalls. If the epilogue owns the only output region, compare one versus multiple disjoint TMEM regions and prove MMA-completion and epilogue-load completion before each handoff.\n\n`cta_group::2` requires a valid cluster and exact kind-, shape-, layout-, descriptor-, and peer-resource constraints. It does not universally mean `m256n256`, require identical SMEM layouts, or promise twice the compute per cycle: one cooperative operation consumes resources from two CTAs. Compare total and per-SM throughput against a legal one-CTA mapping and measure whether peer operand reuse or a larger aggregate tile reduces traffic enough to repay coordination and occupancy costs.\n\nOptimize non-MMA work only after showing that it lies on the critical path. Fusion may remove intermediate traffic; tile interleaving may expose independent work; an approximation may exchange numerical error for throughput. Validate error against the application oracle rather than assuming a software transcendental is interchangeable with the hardware operation."}, "reason": {"statement": "Use the actual instruction/configuration contract.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu", "https://gau-nernst.github.io/tcgen05/"]}} +{"path": "wiki/patterns/compute-bound.md", "before": {"statement": "- Pipeline depth tuning is workload-dependent (3-5 stages typical)"}, "after": {"statement": "Enumerate stage counts that compile and fit the complete shared-storage allocation. More stages can overlap TMA production with MMA consumption, but they also consume SMEM, change occupancy, and enlarge prologue/tail costs. There is no architecture-wide “three to five stages” range; the cited tutorial's beneficial v3 kernel uses two stages.\n\nDedicated producer, MMA, or epilogue warps can remove role switching from a loop and allow independent work to progress, but add live warps, registers, and synchronization. They do not eliminate stalls. If the epilogue owns the only output region, compare one versus multiple disjoint TMEM regions and prove MMA-completion and epilogue-load completion before each handoff.\n\n`cta_group::2` requires a valid cluster and exact kind-, shape-, layout-, descriptor-, and peer-resource constraints. It does not universally mean `m256n256`, require identical SMEM layouts, or promise twice the compute per cycle: one cooperative operation consumes resources from two CTAs. Compare total and per-SM throughput against a legal one-CTA mapping and measure whether peer operand reuse or a larger aggregate tile reduces traffic enough to repay coordination and occupancy costs.\n\nOptimize non-MMA work only after showing that it lies on the critical path. Fusion may remove intermediate traffic; tile interleaving may expose independent work; an approximation may exchange numerical error for throughput. Validate error against the application oracle rather than assuming a software transcendental is interchangeable with the hardware operation."}, "reason": {"statement": "Retain only configuration dependence and controlled selection.", "urls": ["https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://gau-nernst.github.io/tcgen05/"]}} +{"path": "wiki/patterns/low-sm-utilization.md", "before": {"statement": "SM utilization below 60% despite sufficient occupancy. Nsight Compute shows idle SMs during portions of kernel execution."}, "after": {"statement": "Low SM utilization means that fewer SMs perform useful work than the workload could profitably use during a material part of its measured time. It is not defined by a universal 60% threshold. Theoretical occupancy describes how many blocks or warps can reside from resource limits; it does not prove that the grid supplies those workers, that they are simultaneously active, or that their work is balanced.\n\nCollect a synchronized kernel time and a time-resolved view where supported. Record the physical and application-constrained SM count, grid and cluster dimensions, blocks resident per SM, logical work count, work duration distribution, and active/idle intervals. Aggregate SM activity alone cannot distinguish a small grid, a partial final wave, variable-duration work, resource-limited residency, or phase-local serial work."}, "reason": {"statement": "Define the symptom from insufficient useful active work relative to required parallelism and timing.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} +{"path": "wiki/patterns/low-sm-utilization.md", "before": {"statement": "| [CLC](../hardware/clc.md) | SM100 only | Dynamic tile assignment, eliminates load imbalance |"}, "after": {"statement": "Persistence, coordinate order, CLC reassignment, and K decomposition are separate controls:\n\n- A static persistent worker can process multiple logical tiles by grid stride. It can amortize setup and change wave behavior but cannot guarantee removal of the final tail.\n- A row/column raster or swizzle changes coordinate order and may change operand locality. It does not guarantee a better L2 hit rate or lower work variance.\n- Cluster Launch Control lets a running worker cancel an unspecified not-yet-started block or cluster from the launched grid and process the returned ID. It redistributes existing work; it cannot create independent tiles, discard required work, or guarantee equal worker times.\n- Stream-K or Split-K can create additional partitions when tile-level parallelism is insufficient, at the cost of partial-result reduction, workspace, synchronization, and possible determinism changes.\n\nFor CLC, compare static and dynamic scheduling with identical math, tile/cluster shapes, problem-sized grid, resource limits, and timing method. Record successful and failed requests plus per-worker work counts where instrumentation permits. Include ordinary and intentionally uneven SM availability as separate cases.\n\nPTX ISA 9.0 says `clusterlaunchcontrol.try_cancel` requires `sm_100` or higher; its cluster-wide multicast qualifier explicitly lists `sm_120a` and the SM120 family. Therefore CLC is not categorically excluded from SM120 by the ISA. The pinned CUTLASS 4.5.0 persistent scheduler discussed in this wiki is specifically an SM100 integration; do not generalize a library route into an architecture exclusion."}, "reason": {"statement": "State exact redistribution scope and failure/parallelism limits.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/patterns/low-sm-utilization.md", "before": {"statement": "| [Persistent kernels](../techniques/persistent-kernels.md) | SM90+ | Eliminates tail effect, one-time launch overhead |"}, "after": {"statement": "Persistence, coordinate order, CLC reassignment, and K decomposition are separate controls:\n\n- A static persistent worker can process multiple logical tiles by grid stride. It can amortize setup and change wave behavior but cannot guarantee removal of the final tail.\n- A row/column raster or swizzle changes coordinate order and may change operand locality. It does not guarantee a better L2 hit rate or lower work variance.\n- Cluster Launch Control lets a running worker cancel an unspecified not-yet-started block or cluster from the launched grid and process the returned ID. It redistributes existing work; it cannot create independent tiles, discard required work, or guarantee equal worker times.\n- Stream-K or Split-K can create additional partitions when tile-level parallelism is insufficient, at the cost of partial-result reduction, workspace, synchronization, and possible determinism changes.\n\nFor CLC, compare static and dynamic scheduling with identical math, tile/cluster shapes, problem-sized grid, resource limits, and timing method. Record successful and failed requests plus per-worker work counts where instrumentation permits. Include ordinary and intentionally uneven SM availability as separate cases.\n\nPTX ISA 9.0 says `clusterlaunchcontrol.try_cancel` requires `sm_100` or higher; its cluster-wide multicast qualifier explicitly lists `sm_120a` and the SM120 family. Therefore CLC is not categorically excluded from SM120 by the ISA. The pinned CUTLASS 4.5.0 persistent scheduler discussed in this wiki is specifically an SM100 integration; do not generalize a library route into an architecture exclusion."}, "reason": {"statement": "Separate persistence from dynamic assignment and measure tail distribution.", "urls": ["https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/patterns/low-sm-utilization.md", "before": {"statement": "| [Tile scheduling](../techniques/tile-scheduling.md) | SM90+ | Better L2 locality, reduce load variance |"}, "after": {"statement": "Persistence, coordinate order, CLC reassignment, and K decomposition are separate controls:\n\n- A static persistent worker can process multiple logical tiles by grid stride. It can amortize setup and change wave behavior but cannot guarantee removal of the final tail.\n- A row/column raster or swizzle changes coordinate order and may change operand locality. It does not guarantee a better L2 hit rate or lower work variance.\n- Cluster Launch Control lets a running worker cancel an unspecified not-yet-started block or cluster from the launched grid and process the returned ID. It redistributes existing work; it cannot create independent tiles, discard required work, or guarantee equal worker times.\n- Stream-K or Split-K can create additional partitions when tile-level parallelism is insufficient, at the cost of partial-result reduction, workspace, synchronization, and possible determinism changes.\n\nFor CLC, compare static and dynamic scheduling with identical math, tile/cluster shapes, problem-sized grid, resource limits, and timing method. Record successful and failed requests plus per-worker work counts where instrumentation permits. Include ordinary and intentionally uneven SM availability as separate cases.\n\nPTX ISA 9.0 says `clusterlaunchcontrol.try_cancel` requires `sm_100` or higher; its cluster-wide multicast qualifier explicitly lists `sm_120a` and the SM120 family. Therefore CLC is not categorically excluded from SM120 by the ISA. The pinned CUTLASS 4.5.0 persistent scheduler discussed in this wiki is specifically an SM100 integration; do not generalize a library route into an architecture exclusion."}, "reason": {"statement": "Map each scheduler choice to a specific measured hypothesis.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"]}} +{"path": "wiki/patterns/low-sm-utilization.md", "before": {"statement": "```\n// tcgen05 tutorial progression:\n// Without persistent/CLC: 86% of cuBLAS (some SMs idle at wave boundaries)\n// With persistent + CLC: 98% of cuBLAS (all SMs stay busy)\n```"}, "after": {"statement": "In Gau Nernst's `M=N=K=4096` B200 experiment, v5 reports 1302.29 TFLOP/s (86.43% of its 1506.74-TFLOP/s cuBLAS value), while v6 reports 1475.93 TFLOP/s (97.96%). The v6 endpoint combines static persistence, a changed output pipeline, and epilogue-specialized warps on top of earlier changes. The author says CLC was not added and observes that overlap remains imperfect. This is not a CLC result, an all-SMs-busy measurement, or an isolated tail-effect ablation."}, "reason": {"statement": "Report exact cumulative version scope without all-SMs-busy causality.", "urls": ["https://gau-nernst.github.io/tcgen05/", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu"]}} +{"path": "wiki/patterns/low-sm-utilization.md", "before": {"statement": "- CLC only available on SM100 datacenter GPUs (not SM120 consumer)"}, "after": {"statement": "Persistence, coordinate order, CLC reassignment, and K decomposition are separate controls:\n\n- A static persistent worker can process multiple logical tiles by grid stride. It can amortize setup and change wave behavior but cannot guarantee removal of the final tail.\n- A row/column raster or swizzle changes coordinate order and may change operand locality. It does not guarantee a better L2 hit rate or lower work variance.\n- Cluster Launch Control lets a running worker cancel an unspecified not-yet-started block or cluster from the launched grid and process the returned ID. It redistributes existing work; it cannot create independent tiles, discard required work, or guarantee equal worker times.\n- Stream-K or Split-K can create additional partitions when tile-level parallelism is insufficient, at the cost of partial-result reduction, workspace, synchronization, and possible determinism changes.\n\nFor CLC, compare static and dynamic scheduling with identical math, tile/cluster shapes, problem-sized grid, resource limits, and timing method. Record successful and failed requests plus per-worker work counts where instrumentation permits. Include ordinary and intentionally uneven SM availability as separate cases.\n\nPTX ISA 9.0 says `clusterlaunchcontrol.try_cancel` requires `sm_100` or higher; its cluster-wide multicast qualifier explicitly lists `sm_120a` and the SM120 family. Therefore CLC is not categorically excluded from SM120 by the ISA. The pinned CUTLASS 4.5.0 persistent scheduler discussed in this wiki is specifically an SM100 integration; do not generalize a library route into an architecture exclusion."}, "reason": {"statement": "Separate PTX target support from pinned CUTLASS integration.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/patterns/low-sm-utilization.md", "before": {"statement": "- For non-persistent kernels, ensure grid size >> SM count"}, "after": {"statement": "| Too little independent work | Logical block/cluster count is below available one-wave worker capacity | Change problem decomposition, accounting for reduction and synchronization cost |\n| Wave quantization | Equal-duration data-parallel work has a small nonzero final-wave remainder | Compare tile shapes or a decomposition that changes tile count |\n| Variable work duration | Per-worker tile counts/times have a long tail despite enough pending work | Compare static and dynamic acquisition with identical tile/decomposition |\n| Residency or phase limitation | Grid is large, but resources or serial phases limit active blocks/warps | Change the limiting resource or phase; more grid blocks alone do not help |\n\nStatic assignment is nonadaptive, but it is not automatically imbalanced. A grid with fewer independent blocks than SMs cannot occupy every SM, while a grid much larger than the SM count can still show a tail or long-running stragglers. Do not prescribe `grid size >> SM count` without a decomposition that creates useful independent work."}, "reason": {"statement": "Use explicit worker-demand and tile arithmetic.", "urls": ["https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/patterns/memory-bound.md", "before": {"statement": "Nsight Compute shows high DRAM throughput but low tensor core utilization. Arithmetic intensity below the roofline knee point."}, "after": {"statement": "A kernel is memory-bandwidth bound when its measured arithmetic intensity places it on the memory side of the relevant roofline and its useful throughput is limited by an attained memory ceiling. Compute intensity from the operations actually performed and bytes transferred at the memory level under study. Compare against a measured ceiling for the same device, clocks, memory level, datatype path, and environment; a nominal product bandwidth is only an upper-bound input.\n\nHigh DRAM throughput, low tensor-core activity, or a workload label such as GEMV is not sufficient alone. Record useful/requested bytes, transferred sectors or bytes, achieved bandwidth, cache behavior, scheduler issue/stalls, and end-to-end time. This separates four cases that can otherwise look similar:\n\n| Useful bytes and transferred bytes are close; attained bandwidth is near the measured roof | Plausible bandwidth-ceiling limit |\n| Transferred bytes substantially exceed useful bytes | Coalescing, overfetch, cache, or redundant-traffic problem |\n| Bandwidth is below the roof and warps lack ready work | Latency, dependency, insufficient concurrency, or issue problem |\n| Bandwidth is high only during one phase | Phase balance or fusion opportunity; whole-kernel boundedness remains unproven |\n\nSingle-use data can lower operations per transferred byte, but “poor reuse” is actionable only if additional legal reuse exists. Uncoalesced access and cache interference may increase traffic or latency; they are not synonyms for saturation of the DRAM bandwidth ceiling."}, "reason": {"statement": "Require a complete roofline and traffic/issue record.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} +{"path": "wiki/patterns/memory-bound.md", "before": {"statement": "1. **Low arithmetic intensity**: Operations like GEMV, small batch decode, or reduction kernels"}, "after": {"statement": "A kernel is memory-bandwidth bound when its measured arithmetic intensity places it on the memory side of the relevant roofline and its useful throughput is limited by an attained memory ceiling. Compute intensity from the operations actually performed and bytes transferred at the memory level under study. Compare against a measured ceiling for the same device, clocks, memory level, datatype path, and environment; a nominal product bandwidth is only an upper-bound input.\n\nHigh DRAM throughput, low tensor-core activity, or a workload label such as GEMV is not sufficient alone. Record useful/requested bytes, transferred sectors or bytes, achieved bandwidth, cache behavior, scheduler issue/stalls, and end-to-end time. This separates four cases that can otherwise look similar:\n\n| Useful bytes and transferred bytes are close; attained bandwidth is near the measured roof | Plausible bandwidth-ceiling limit |\n| Transferred bytes substantially exceed useful bytes | Coalescing, overfetch, cache, or redundant-traffic problem |\n| Bandwidth is below the roof and warps lack ready work | Latency, dependency, insufficient concurrency, or issue problem |\n| Bandwidth is high only during one phase | Phase balance or fusion opportunity; whole-kernel boundedness remains unproven |\n\nSingle-use data can lower operations per transferred byte, but “poor reuse” is actionable only if additional legal reuse exists. Uncoalesced access and cache interference may increase traffic or latency; they are not synonyms for saturation of the DRAM bandwidth ceiling."}, "reason": {"statement": "Use measured work and bytes for the actual implementation/input.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} +{"path": "wiki/patterns/memory-bound.md", "before": {"statement": "3. **Inefficient memory access**: Uncoalesced loads, L1 cache thrashing"}, "after": {"statement": "Choose only legal, naturally aligned vector forms and handle tails separately. Compare scalar/narrow and wider variants with identical mapping. Record instruction count, requested and transferred bytes, transactions, registers, spills, achieved bandwidth, and time. Wider instructions do not guarantee fewer hardware transactions or higher bandwidth.\n\nPTX L1 eviction priorities and L2 prefetch controls are hints. `no_allocate` does not guarantee bypass, and `evict_last` does not guarantee residence. Characterize address order, reuse distance, working set, and interfering streams, then vary one hint at a time against the default. Retain it only when a representative end-to-end comparison improves.\n\nInspect compiled registers, spills, SMEM, threads, and the actual occupancy-limiting resource before applying `-maxrregcount`. A cap may be clamped by ABI requirements, introduce spill traffic, or leave occupancy unchanged. Higher theoretical occupancy is useful only if it raises ready work or attained bandwidth enough to reduce runtime.\n\nTMA multicast can issue one global-to-shared tensor copy to selected CTAs' shared-memory destinations within a cluster. It is relevant when those CTAs consume the same operand and the cluster/lifetime/barrier costs are valid. It does not help a stream with no inter-CTA reuse.\n\nSwizzling changes a shared-memory address mapping and can reduce conflicts for a specified access pattern. It does not universally eliminate conflicts and is not by itself a DRAM optimization. Validate the legal tensor-map/layout constraints and compare bank-conflict, traffic, and time measurements."}, "reason": {"statement": "Separate bandwidth ceiling, transaction efficiency, cache behavior, and latency/issue limits.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"]}} +{"path": "wiki/patterns/memory-bound.md", "before": {"statement": "| [Vectorized loads](../techniques/vectorized-loads.md) | 128/256-bit loads maximize bandwidth utilization |"}, "after": {"statement": "The cited Amandeep NVFP4 GEMV report is useful precisely because plausible memory-oriented changes failed. A wider `uint2` load was 16–25% slower on the reported shapes, and reducing `maxrregcount` from 80 to 64 had no effect. These observations do not prove that wide loads or register caps are generally harmful; they refute a fixed optimization priority based only on the “memory-bound GEMV” label.\n\nUse the same discipline for compute work. If the kernel is genuinely at its attainable memory roof, compute-only instruction reductions may have little effect. But address calculation, decoding, and instruction-level parallelism can affect memory issue and latency hiding before that roof is reached. State a predicted counter change, alter one variable, and keep or reject the hypothesis from matched time and profiler evidence."}, "reason": {"statement": "Make width an aligned legal candidate with a scalar/narrow control.", "urls": ["https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} +{"path": "wiki/patterns/memory-bound.md", "before": {"statement": "| [Cache policies](../techniques/vectorized-loads.md) | L1::no_allocate for streaming, L1::evict_last for reuse |"}, "after": {"statement": "The cited Amandeep NVFP4 GEMV report is useful precisely because plausible memory-oriented changes failed. A wider `uint2` load was 16–25% slower on the reported shapes, and reducing `maxrregcount` from 80 to 64 had no effect. These observations do not prove that wide loads or register caps are generally harmful; they refute a fixed optimization priority based only on the “memory-bound GEMV” label.\n\nUse the same discipline for compute work. If the kernel is genuinely at its attainable memory roof, compute-only instruction reductions may have little effect. But address calculation, decoding, and instruction-level parallelism can affect memory issue and latency hiding before that roof is reached. State a predicted counter change, alter one variable, and keep or reject the hypothesis from matched time and profiler evidence."}, "reason": {"statement": "Test hints one variable at a time from concrete reuse/interference evidence.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} +{"path": "wiki/patterns/memory-bound.md", "before": {"statement": "| [Register budgeting](../techniques/vectorized-loads.md) | -maxrregcount increases occupancy |"}, "after": {"statement": "The cited Amandeep NVFP4 GEMV report is useful precisely because plausible memory-oriented changes failed. A wider `uint2` load was 16–25% slower on the reported shapes, and reducing `maxrregcount` from 80 to 64 had no effect. These observations do not prove that wide loads or register caps are generally harmful; they refute a fixed optimization priority based only on the “memory-bound GEMV” label.\n\nUse the same discipline for compute work. If the kernel is genuinely at its attainable memory roof, compute-only instruction reductions may have little effect. But address calculation, decoding, and instruction-level parallelism can affect memory issue and latency hiding before that roof is reached. State a predicted counter change, alter one variable, and keep or reject the hypothesis from matched time and profiler evidence."}, "reason": {"statement": "Use compiled-resource and occupancy evidence before timing caps.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} +{"path": "wiki/patterns/memory-bound.md", "before": {"statement": "| [Swizzling](../techniques/swizzling.md) | Eliminate bank conflicts in shared memory |"}, "after": {"statement": "The cited Amandeep NVFP4 GEMV report is useful precisely because plausible memory-oriented changes failed. A wider `uint2` load was 16–25% slower on the reported shapes, and reducing `maxrregcount` from 80 to 64 had no effect. These observations do not prove that wide loads or register caps are generally harmful; they refute a fixed optimization priority based only on the “memory-bound GEMV” label.\n\nUse the same discipline for compute work. If the kernel is genuinely at its attainable memory roof, compute-only instruction reductions may have little effect. But address calculation, decoding, and instruction-level parallelism can affect memory issue and latency hiding before that roof is reached. State a predicted counter change, alter one variable, and keep or reject the hypothesis from matched time and profiler evidence."}, "reason": {"statement": "Treat swizzle as a shared-memory conflict hypothesis, not a DRAM-bandwidth cure.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-map-swizzle", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} +{"path": "wiki/patterns/memory-bound.md", "before": {"statement": "```cuda\n// NVFP4 GEMV: memory-bound optimization\n// Key insight: profile FIRST to confirm memory-bound behavior\n// \"The single most important thing could have been running Nsight Compute\"\n// — Amandeep (12 Attempts at an FP4 Kernel)\n\n// Optimization priorities for memory-bound kernels:\n// 1. Maximize memory bandwidth (wide loads, coalescing)\n// 2. Reduce register count (higher occupancy)\n// 3. Differentiate cache policies per access pattern\n// 4. DON'T optimize compute (it's not the bottleneck)\n```"}, "after": {"statement": "Report input shapes/distributions, useful work and bytes, cache state, warmup/repetitions/statistic, GPU and clocks, software versions, generated instructions, resource usage, roofline assumptions, achieved memory-level bandwidth, requested/transferred efficiency, scheduler activity, and correctness tolerance. Include negative and regressing variants so a category heuristic is not mistaken for causal evidence."}, "reason": {"statement": "Replace the pseudo-code prescription with the exact negative-control lesson.", "urls": ["https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} +{"path": "wiki/patterns/memory-bound.md", "before": {"statement": "- B200 has 8 TB/s bandwidth; speed-of-light calculation determines achievable performance"}, "after": {"statement": "A kernel is memory-bandwidth bound when its measured arithmetic intensity places it on the memory side of the relevant roofline and its useful throughput is limited by an attained memory ceiling. Compute intensity from the operations actually performed and bytes transferred at the memory level under study. Compare against a measured ceiling for the same device, clocks, memory level, datatype path, and environment; a nominal product bandwidth is only an upper-bound input.\n\nHigh DRAM throughput, low tensor-core activity, or a workload label such as GEMV is not sufficient alone. Record useful/requested bytes, transferred sectors or bytes, achieved bandwidth, cache behavior, scheduler issue/stalls, and end-to-end time. This separates four cases that can otherwise look similar:\n\n| Useful bytes and transferred bytes are close; attained bandwidth is near the measured roof | Plausible bandwidth-ceiling limit |\n| Transferred bytes substantially exceed useful bytes | Coalescing, overfetch, cache, or redundant-traffic problem |\n| Bandwidth is below the roof and warps lack ready work | Latency, dependency, insufficient concurrency, or issue problem |\n| Bandwidth is high only during one phase | Phase balance or fusion opportunity; whole-kernel boundedness remains unproven |\n\nSingle-use data can lower operations per transferred byte, but “poor reuse” is actionable only if additional legal reuse exists. Uncoalesced access and cache interference may increase traffic or latency; they are not synonyms for saturation of the DRAM bandwidth ceiling."}, "reason": {"statement": "Use measured ceiling and efficiency with complete environment scope.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline", "https://www.nvidia.com/en-us/data-center/blackwell/", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} +{"path": "wiki/patterns/memory-bound.md", "before": {"statement": "- ILP and compute optimizations have diminishing returns for memory-bound kernels"}, "after": {"statement": "Choose only legal, naturally aligned vector forms and handle tails separately. Compare scalar/narrow and wider variants with identical mapping. Record instruction count, requested and transferred bytes, transactions, registers, spills, achieved bandwidth, and time. Wider instructions do not guarantee fewer hardware transactions or higher bandwidth.\n\nPTX L1 eviction priorities and L2 prefetch controls are hints. `no_allocate` does not guarantee bypass, and `evict_last` does not guarantee residence. Characterize address order, reuse distance, working set, and interfering streams, then vary one hint at a time against the default. Retain it only when a representative end-to-end comparison improves.\n\nInspect compiled registers, spills, SMEM, threads, and the actual occupancy-limiting resource before applying `-maxrregcount`. A cap may be clamped by ABI requirements, introduce spill traffic, or leave occupancy unchanged. Higher theoretical occupancy is useful only if it raises ready work or attained bandwidth enough to reduce runtime.\n\nTMA multicast can issue one global-to-shared tensor copy to selected CTAs' shared-memory destinations within a cluster. It is relevant when those CTAs consume the same operand and the cluster/lifetime/barrier costs are valid. It does not help a stream with no inter-CTA reuse.\n\nSwizzling changes a shared-memory address mapping and can reduce conflicts for a specified access pattern. It does not universally eliminate conflicts and is not by itself a DRAM optimization. Validate the legal tensor-map/layout constraints and compare bank-conflict, traffic, and time measurements.\n\nThe cited Amandeep NVFP4 GEMV report is useful precisely because plausible memory-oriented changes failed. A wider `uint2` load was 16–25% slower on the reported shapes, and reducing `maxrregcount` from 80 to 64 had no effect. These observations do not prove that wide loads or register caps are generally harmful; they refute a fixed optimization priority based only on the “memory-bound GEMV” label.\n\nUse the same discipline for compute work. If the kernel is genuinely at its attainable memory roof, compute-only instruction reductions may have little effect. But address calculation, decoding, and instruction-level parallelism can affect memory issue and latency hiding before that roof is reached. State a predicted counter change, alter one variable, and keep or reject the hypothesis from matched time and profiler evidence."}, "reason": {"statement": "Tie every candidate to a measured critical resource and predicted counter movement.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"]}} +{"path": "wiki/patterns/register-pressure.md", "before": {"statement": "Occupancy below target due to high register usage per thread. Nsight Compute shows register spilling to local memory."}, "after": {"statement": "Register pressure is performance-relevant only when compiled register allocation or spills constrain useful scheduling or add material local-memory traffic. “Occupancy below target” is not enough: theoretical occupancy is a residency limit, achieved occupancy is runtime behavior, and maximum occupancy is not a universal optimum.\n\nFor the exact binary and launch, record registers per thread, allocation granularity, spill stores/loads, local-memory traffic, threads and warps per CTA, SMEM, cluster shape, theoretical limiting resource, achieved active/eligible warps, scheduler stalls, and end-to-end time. Inspect generated SASS/source correlation to find the live ranges or spill sites. Then change one source or compiler factor and require the predicted resource/stall movement plus a runtime improvement.\n\nCommon contributors include a resident MMA fragment, an epilogue whose temporaries overlap the mainloop, descriptors/addresses and pipeline state, unrolled loops, and values live across branches. These are hypotheses about compiled liveness, not conclusions from source complexity alone."}, "reason": {"statement": "Require compiled resource and limiting-resource evidence plus a causal variant.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} +{"path": "wiki/patterns/register-pressure.md", "before": {"statement": "| [TMEM](../hardware/tmem.md) | SM100 only | Moves accumulators to dedicated 256KB memory |"}, "after": {"statement": "For Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, that is 128 32-bit registers (512 bytes) per thread for D. This exact fragment arithmetic does not include other live state or guarantee a particular resident-block count.\n\nOn the SM100 tcgen05 path, resident D is in TMEM, organized as 128 lanes by 512 columns of 32-bit cells. That removes the long-lived per-thread WGMMA D fragment, but not all accumulator-related register work: addresses, descriptors, barriers, loop state, `tcgen05.ld` destinations, conversion, and epilogue temporaries still consume registers. Logical capacity and required columns depend on MMA kind, shape, and packing, so a largest-operation or always-double-bufferable rule cannot be inferred from raw cell count.\n\nThe migration must implement collective allocation/address publication, tcgen05 MMA completion, cross-thread ordering where applicable, collective TMEM loads and `tcgen05.wait::ld`, consumer completion, and collective deallocation before exit. The described TMEM/tcgen05 path targets SM100-class data-center architectures and is not the SM120 consumer MMA path."}, "reason": {"statement": "State organization and resident-D scope without promising exact register relief.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"]}} +{"path": "wiki/patterns/register-pressure.md", "before": {"statement": "| [Warp specialization](../techniques/warp-specialization.md) | SM100+ | Different warps handle different roles, reducing per-warp register needs |"}, "after": {"statement": "Move epilogue work after accumulator lifetime when dependencies allow, reduce unnecessary unrolling, or split a long-lived role. Warp specialization exists on Hopper as well as Blackwell; it can shorten one role's live set, but adds role state and synchronization and does not guarantee fewer registers. Compare compiled resources and time for matched role layouts.\n\nCompare an uncapped build with selected `-maxrregcount` values. Record whether the requested cap is effective, the occupancy-limiting resource, spill traffic, instructions, and runtime. ABI minima can constrain the cap, another resource can remain limiting, and spills can erase any benefit."}, "reason": {"statement": "Separate architecture availability from measured role-liveness/resource changes.", "urls": ["https://docs.nvidia.com/cutlass/media/docs/cpp/hopper_gemm.html", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html"]}} +{"path": "wiki/patterns/register-pressure.md", "before": {"statement": "```\n// Hopper: 64×256 MMA tile accumulator = 64*256*4 bytes in registers per warp group\n// → ~128 registers per thread just for accumulators\n//\n// Blackwell: TMEM holds accumulators\n// → 0 registers for accumulators\n// → ~128 registers freed for other work or higher occupancy\n//\n// TMEM: 256 KB per SM, 128 rows × 512 columns × 32-bit\n// Largest 1-SM MMA uses half → double-buffering possible\n```"}, "after": {"statement": "For Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, that is 128 32-bit registers (512 bytes) per thread for D. This exact fragment arithmetic does not include other live state or guarantee a particular resident-block count.\n\nOn the SM100 tcgen05 path, resident D is in TMEM, organized as 128 lanes by 512 columns of 32-bit cells. That removes the long-lived per-thread WGMMA D fragment, but not all accumulator-related register work: addresses, descriptors, barriers, loop state, `tcgen05.ld` destinations, conversion, and epilogue temporaries still consume registers. Logical capacity and required columns depend on MMA kind, shape, and packing, so a largest-operation or always-double-bufferable rule cannot be inferred from raw cell count.\n\nThe migration must implement collective allocation/address publication, tcgen05 MMA completion, cross-thread ordering where applicable, collective TMEM loads and `tcgen05.wait::ld`, consumer completion, and collective deallocation before exit. The described TMEM/tcgen05 path targets SM100-class data-center architectures and is not the SM120 consumer MMA path."}, "reason": {"statement": "Keep the exact comparison while naming residual registers and configuration-specific capacity.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"]}} +{"path": "wiki/patterns/register-pressure.md", "before": {"statement": "- TMEM→register transfer adds latency (offset by freeing registers)"}, "after": {"statement": "TMEM loads are asynchronous operations with explicit completion waits. Whether their issue/wait and epilogue work are repaid by lower long-lived register use is an empirical whole-kernel question. Compare the Hopper and Blackwell designs only with equivalent math and precision; for same-architecture alternatives, compare TMEM region count/layout while holding tile and launch policy fixed.\n\nReport compiled registers and spills, TMEM columns, SMEM, barriers, occupancy limit, achieved warps, pipeline/scoreboard stalls, correctness, and time. Include shapes where register occupancy changes and shapes where it does not. A reduction in register count without a useful residency, issue, traffic, or time improvement is not sufficient."}, "reason": {"statement": "Make it a controlled whole-kernel tradeoff.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} +{"path": "wiki/patterns/tail-effect.md", "before": {"statement": "Performance drops for problem sizes where total_tiles % num_SMs != 0. The last wave of tiles runs with many SMs idle."}, "after": {"statement": "For `T` equal-duration independent tiles and `W` available one-tile workers, with no K decomposition, write:\n\n```text\nT = qW + r, 0 <= r < W\n```\n\nThere are `q` full waves. If `r>0`, a final partial wave uses `r` workers and has instantaneous worker utilization `r/W`; if `r=0` and `T>0`, the final wave is full. For example, `T=150` and `W=142` gives one full wave plus eight tiles: the partial wave has `8/142 = 5.63%` worker utilization under these assumptions.\n\nThis is an analytical model, not a statement that a B200 has 142 SMs or that physical SM count always equals `W`. Block resources can make multiple blocks resident per SM, cluster shape changes scheduling granularity, application policies may restrict available SMs, and tile durations can differ. Measure the actual worker capacity and timeline.\n\nUnder the equal-duration model, the time fraction attributable to at most one partial wave shrinks as the number of full waves grows. There is no universal “below four times the SM count” cutoff: the remainder, wave duration, other phases, and imbalance determine significance."}, "reason": {"statement": "Define and compute the simplified model, then require measurement for real kernels.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/patterns/tail-effect.md", "before": {"statement": "2. **Static assignment**: stride-by-gridDim leaves remainder tiles on few SMs\n3. **Non-persistent launch**: each kernel launch has fixed grid, no dynamic rebalancing"}, "after": {"statement": "1. Record logical tiles/clusters, grid and cluster dimensions, available worker capacity, residency limits, and per-work-item duration.\n2. Predict wave count and the final remainder from the simplified model.\n3. Use time-resolved activity or instrumented worker records to locate the underfilled interval at the end, rather than inferring it from aggregate utilization.\n4. Sweep nearby problem sizes or tile shapes. A wave-quantization hypothesis predicts a sawtooth response aligned with changes in the remainder, after controlling total work.\n5. Separate unequal tile durations: a long straggler is load imbalance even if the tile-count remainder is zero.\n\nCUDA schedules ordinary grid blocks onto available SMs; a static grid-stride loop fixes logical worker indices, not `blockIdx`-to-physical-SM placement. A fixed grid therefore does not mean CUDA lacks dynamic block scheduling."}, "reason": {"statement": "Separate logical worker assignment from hardware block placement and persistence.", "urls": ["https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} +{"path": "wiki/patterns/tail-effect.md", "before": {"statement": "| [CLC](../hardware/clc.md) | Hardware dynamic scheduling, SMs grab tiles on-demand |"}, "after": {"statement": "- A static persistent loop lets a resident CTA process multiple logical work items. Worker count is selected from problem decomposition, cluster/resource limits, and policy—not necessarily one CTA per physical SM. Finite work still has a final subset and may remain imbalanced.\n- Cluster Launch Control lets a selected thread request cancellation of an unspecified not-yet-started block or cluster and process the returned grid ID. CLC redistributes existing IDs; it cannot turn eight remaining independent tiles into 142 or 148 concurrent tiles.\n- Raster order and swizzle are coordinate transforms. Test them for locality while holding work acquisition fixed; they do not create work or guarantee balanced durations.\n- Stream-K or Split-K may create more independent partitions when tile-level work is insufficient. Evaluate reduction traffic, workspace, synchronization, determinism, and numerical effects separately.\n\nCompare static and CLC-backed persistence with identical math, decomposition, tile/cluster shapes, grid, resources, and timing. Record CLC request results and per-worker work/time distribution. PTX ISA 9.0 requires `sm_100` or higher for CLC and explicitly lists SM120-family support for the cluster-wide multicast qualifier; the pinned CUTLASS 4.5.0 route used here is specifically its SM100 scheduler."}, "reason": {"statement": "Describe exact work-reassignment lifecycle and limits.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/patterns/tail-effect.md", "before": {"statement": "| [Persistent kernels](../techniques/persistent-kernels.md) | SM-count grid, iterate over tiles, no wave boundary |"}, "after": {"statement": "- A static persistent loop lets a resident CTA process multiple logical work items. Worker count is selected from problem decomposition, cluster/resource limits, and policy—not necessarily one CTA per physical SM. Finite work still has a final subset and may remain imbalanced.\n- Cluster Launch Control lets a selected thread request cancellation of an unspecified not-yet-started block or cluster and process the returned grid ID. CLC redistributes existing IDs; it cannot turn eight remaining independent tiles into 142 or 148 concurrent tiles.\n- Raster order and swizzle are coordinate transforms. Test them for locality while holding work acquisition fixed; they do not create work or guarantee balanced durations.\n- Stream-K or Split-K may create more independent partitions when tile-level work is insufficient. Evaluate reduction traffic, workspace, synchronization, determinism, and numerical effects separately.\n\nCompare static and CLC-backed persistence with identical math, decomposition, tile/cluster shapes, grid, resources, and timing. Record CLC request results and per-worker work/time distribution. PTX ISA 9.0 requires `sm_100` or higher for CLC and explicitly lists SM120-family support for the cluster-wide multicast qualifier; the pinned CUTLASS 4.5.0 route used here is specifically its SM100 scheduler."}, "reason": {"statement": "Present static persistence as a baseline with its own final work distribution.", "urls": ["https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/patterns/tail-effect.md", "before": {"statement": "| [Tile scheduling](../techniques/tile-scheduling.md) | Raster order, swizzle patterns for better distribution |"}, "after": {"statement": "- A static persistent loop lets a resident CTA process multiple logical work items. Worker count is selected from problem decomposition, cluster/resource limits, and policy—not necessarily one CTA per physical SM. Finite work still has a final subset and may remain imbalanced.\n- Cluster Launch Control lets a selected thread request cancellation of an unspecified not-yet-started block or cluster and process the returned grid ID. CLC redistributes existing IDs; it cannot turn eight remaining independent tiles into 142 or 148 concurrent tiles.\n- Raster order and swizzle are coordinate transforms. Test them for locality while holding work acquisition fixed; they do not create work or guarantee balanced durations.\n- Stream-K or Split-K may create more independent partitions when tile-level work is insufficient. Evaluate reduction traffic, workspace, synchronization, determinism, and numerical effects separately.\n\nCompare static and CLC-backed persistence with identical math, decomposition, tile/cluster shapes, grid, resources, and timing. Record CLC request results and per-worker work/time distribution. PTX ISA 9.0 requires `sm_100` or higher for CLC and explicitly lists SM120-family support for the cluster-wide multicast qualifier; the pinned CUTLASS 4.5.0 route used here is specifically its SM100 scheduler."}, "reason": {"statement": "Separate locality ordering from acquisition and decomposition.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"]}} +{"path": "wiki/patterns/tail-effect.md", "before": {"statement": "```\n// B200: 142 SMs\n// Problem: 150 tiles\n// Without CLC: 2 waves (142 + 8), last wave uses only 8 SMs (5.6%)\n// With CLC: single persistent wave, all 142 SMs stay busy\n//\n// Impact: 86% → 98% of cuBLAS (tcgen05 tutorial data)\n```"}, "after": {"statement": "For `T` equal-duration independent tiles and `W` available one-tile workers, with no K decomposition, write:\n\n```text\nT = qW + r, 0 <= r < W\n```\n\nThere are `q` full waves. If `r>0`, a final partial wave uses `r` workers and has instantaneous worker utilization `r/W`; if `r=0` and `T>0`, the final wave is full. For example, `T=150` and `W=142` gives one full wave plus eight tiles: the partial wave has `8/142 = 5.63%` worker utilization under these assumptions.\n\nThis is an analytical model, not a statement that a B200 has 142 SMs or that physical SM count always equals `W`. Block resources can make multiple blocks resident per SM, cluster shape changes scheduling granularity, application policies may restrict available SMs, and tile durations can differ. Measure the actual worker capacity and timeline.\n\nUnder the equal-duration model, the time fraction attributable to at most one partial wave shrinks as the number of full waves grows. There is no universal “below four times the SM count” cutoff: the remainder, wave duration, other phases, and imbalance determine significance.\n\nGau Nernst's Modal B200 has 148 SMs. In the author's `M=N=K=4096` experiment, v5 reports 1302.29 TFLOP/s (86.43% of cuBLAS) and v6 reports 1475.93 TFLOP/s (97.96%). The v6 change is static persistence plus output-pipeline/epilogue-role changes accumulated over earlier versions. The author explicitly did not add CLC and says overlap remains imperfect. Those results cannot be used as a CLC ablation or proof that all SMs stay busy."}, "reason": {"statement": "Use explicit hypothetical arithmetic separate from exact tutorial evidence.", "urls": ["https://gau-nernst.github.io/tcgen05/", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu"]}} +{"path": "wiki/patterns/tail-effect.md", "before": {"statement": "- Only significant for moderate tile counts (< 4× SM count)"}, "after": {"statement": "For `T` equal-duration independent tiles and `W` available one-tile workers, with no K decomposition, write:\n\n```text\nT = qW + r, 0 <= r < W\n```\n\nThere are `q` full waves. If `r>0`, a final partial wave uses `r` workers and has instantaneous worker utilization `r/W`; if `r=0` and `T>0`, the final wave is full. For example, `T=150` and `W=142` gives one full wave plus eight tiles: the partial wave has `8/142 = 5.63%` worker utilization under these assumptions.\n\nThis is an analytical model, not a statement that a B200 has 142 SMs or that physical SM count always equals `W`. Block resources can make multiple blocks resident per SM, cluster shape changes scheduling granularity, application policies may restrict available SMs, and tile durations can differ. Measure the actual worker capacity and timeline.\n\nUnder the equal-duration model, the time fraction attributable to at most one partial wave shrinks as the number of full waves grows. There is no universal “below four times the SM count” cutoff: the remainder, wave duration, other phases, and imbalance determine significance."}, "reason": {"statement": "Give the analytical fractional bound under declared assumptions and measure real kernels.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/patterns/tail-effect.md", "before": {"statement": "- CLC only on SM100 datacenter"}, "after": {"statement": "- A static persistent loop lets a resident CTA process multiple logical work items. Worker count is selected from problem decomposition, cluster/resource limits, and policy—not necessarily one CTA per physical SM. Finite work still has a final subset and may remain imbalanced.\n- Cluster Launch Control lets a selected thread request cancellation of an unspecified not-yet-started block or cluster and process the returned grid ID. CLC redistributes existing IDs; it cannot turn eight remaining independent tiles into 142 or 148 concurrent tiles.\n- Raster order and swizzle are coordinate transforms. Test them for locality while holding work acquisition fixed; they do not create work or guarantee balanced durations.\n- Stream-K or Split-K may create more independent partitions when tile-level work is insufficient. Evaluate reduction traffic, workspace, synchronization, determinism, and numerical effects separately.\n\nCompare static and CLC-backed persistence with identical math, decomposition, tile/cluster shapes, grid, resources, and timing. Record CLC request results and per-worker work/time distribution. PTX ISA 9.0 requires `sm_100` or higher for CLC and explicitly lists SM120-family support for the cluster-wide multicast qualifier; the pinned CUTLASS 4.5.0 route used here is specifically its SM100 scheduler."}, "reason": {"statement": "Separate ISA compatibility from pinned library examples.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "MoE grouped GEMM shows uneven per-expert compute time. Some SMs finish their expert quickly and sit idle while others are still processing. Overall latency is dominated by the slowest expert."}, "after": {"statement": "Do not infer one expert per physical SM from a grouped-GEMM trace. CUDA assigns\nthread blocks to available SMs, while grouped kernels may split an expert into\nmultiple output tiles or let one resident worker process several logical tiles.\nMeasure at each relevant level:\n\n- routed tokens per expert for the exact batch and prefill/decode phase;\n- valid rows and output-tile counts per expert;\n- completed tiles and elapsed work per logical worker and, when instrumented,\n per SM;\n- dispatch, grouped-GEMM, combine, and end-to-end times per expert-parallel\n rank.\n\nSmall expert segments and partial output tiles can expose too little parallel\nwork or leave lanes predicated out. The effect depends on the kernel tile,\nother GEMM dimensions, resident-worker count, and competing implementation;\nthere is no universal `M < BLOCK_M` diagnosis or minimum viable size.\n\nUniform expected routing, a larger batch, or an auxiliary balancing loss may\nchange token-count skew, but none proves balanced tile counts, worker\ndurations, communication, or runtime. Treat imbalance as absent only when the\nmeasured distributions are narrow and a matched balancing variant does not\nmaterially improve the timed operation."}, "reason": {"statement": "Define imbalance across routing, logical work, worker time, and rank communication without assumed ownership.", "urls": ["https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/efficient_gemm.md", "https://www.lmsys.org/blog/2025-05-05-large-scale-ep/"]}} +{"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "1. **Skewed token distribution**: Router sends 80% of tokens to 20% of experts (common in trained MoE models)"}, "after": {"statement": "Do not infer one expert per physical SM from a grouped-GEMM trace. CUDA assigns\nthread blocks to available SMs, while grouped kernels may split an expert into\nmultiple output tiles or let one resident worker process several logical tiles.\nMeasure at each relevant level:\n\n- routed tokens per expert for the exact batch and prefill/decode phase;\n- valid rows and output-tile counts per expert;\n- completed tiles and elapsed work per logical worker and, when instrumented,\n per SM;\n- dispatch, grouped-GEMM, combine, and end-to-end times per expert-parallel\n rank.\n\nSmall expert segments and partial output tiles can expose too little parallel\nwork or leave lanes predicated out. The effect depends on the kernel tile,\nother GEMM dimensions, resident-worker count, and competing implementation;\nthere is no universal `M < BLOCK_M` diagnosis or minimum viable size.\n\nUniform expected routing, a larger batch, or an auxiliary balancing loss may\nchange token-count skew, but none proves balanced tile counts, worker\ndurations, communication, or runtime. Treat imbalance as absent only when the\nmeasured distributions are narrow and a matched balancing variant does not\nmaterially improve the timed operation."}, "reason": {"statement": "Require exact observed count distributions.", "urls": ["https://github.com/deepseek-ai/EPLB/blob/d52c72d5b2f2fb4c41afbf8eb21366820239913d/README.md", "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/efficient_gemm.md", "https://www.lmsys.org/blog/2025-05-05-large-scale-ep/"]}} +{"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "2. **Static tile assignment**: Precomputed tile→SM mapping cannot rebalance at runtime"}, "after": {"statement": "Keep persistence, work acquisition, and decomposition as separate variables:\n\n1. A static persistent worker can advance through a deterministic logical-work\n sequence. CUDA still decides where its block runs; the rule is not a\n precomputed tile-to-physical-SM map.\n2. In a CLC-backed scheduler, a selected thread requests cancellation of an\n unspecified block or cluster that has not launched. A successful response\n returns that existing grid coordinate; a request can fail. CLC does not\n create tiles or make the fastest SM autonomously steal arbitrary work.\n3. The initial `blockIdx` in CUTLASS's pinned CLC scheduler is static and later\n requests are asynchronous, pipelined, and cluster-granular. There is no\n universal per-tile latency bound: compare request activity and end-to-end\n time against the matched static scheduler.\n4. Splitting a large expert or K range may expose more independent work, but it\n can add partial-result and reduction costs. Hold math, tile shape, cluster\n shape, launch resources, and output semantics constant in the comparison.\n\nPTX ISA 9.0 specifies CLC for `sm_100` or higher and explicitly includes\nSM120-family targets for the multicast form. CUTLASS 4.5.0's documented\n`PersistentTileSchedulerSm100` integration is a narrower library example, not\nthe complete ISA compatibility boundary."}, "reason": {"statement": "Separate logical assignment from physical placement.", "urls": ["https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "3. **Masked layout waste**: Fixed M_max per expert wastes compute on padding rows"}, "after": {"statement": "At pinned DeepGEMM commit\n[`891d57b4`](https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f),\nthe three grouped interfaces have different axes and metadata:\n\n| M-grouped contiguous | Pack `A` and `D` along variable M with N/K fixed; identify groups by per-row expert IDs or per-group prefix-sum ends; segments are M-block aligned | Record valid rows, alignment padding, tile counts, and metadata form |\n| M-grouped masked | Allocate `[G, M_max, ...]`, pass one valid-M count per group, and compute valid portions; the fixed allocation is documented for a CUDA-graph decode case | Separate allocated rows from valid rows and inspect edge predication in generated code |\n| K-grouped contiguous | Pack variable K with M/N fixed for MoE weight backward | Do not use its K-axis contract to describe forward token imbalance |\n\nA fixed `M_max` allocation therefore does not establish that all padding rows\nare computed. Any residual edge-tile or predication cost needs generated-code\nand profile evidence for the selected kernel."}, "reason": {"statement": "Preserve graph-friendly allocation while removing false padding-compute attribution.", "urls": ["https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp"]}} +{"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "| [CLC (Cluster Launch Control)](../hardware/clc.md) | Hardware dynamic tile assignment — fastest SMs grab more tiles |"}, "after": {"statement": "Keep persistence, work acquisition, and decomposition as separate variables:\n\n1. A static persistent worker can advance through a deterministic logical-work\n sequence. CUDA still decides where its block runs; the rule is not a\n precomputed tile-to-physical-SM map.\n2. In a CLC-backed scheduler, a selected thread requests cancellation of an\n unspecified block or cluster that has not launched. A successful response\n returns that existing grid coordinate; a request can fail. CLC does not\n create tiles or make the fastest SM autonomously steal arbitrary work.\n3. The initial `blockIdx` in CUTLASS's pinned CLC scheduler is static and later\n requests are asynchronous, pipelined, and cluster-granular. There is no\n universal per-tile latency bound: compare request activity and end-to-end\n time against the matched static scheduler.\n4. Splitting a large expert or K range may expose more independent work, but it\n can add partial-result and reduction costs. Hold math, tile shape, cluster\n shape, launch resources, and output semantics constant in the comparison.\n\nPTX ISA 9.0 specifies CLC for `sm_100` or higher and explicitly includes\nSM120-family targets for the multicast form. CUTLASS 4.5.0's documented\n`PersistentTileSchedulerSm100` integration is a narrower library example, not\nthe complete ISA compatibility boundary."}, "reason": {"statement": "State the exact reassignment protocol.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "| [Persistent kernels](../techniques/persistent-kernels.md) | Amortize launch overhead; loop over dynamic work queue |"}, "after": {"statement": "Keep persistence, work acquisition, and decomposition as separate variables:\n\n1. A static persistent worker can advance through a deterministic logical-work\n sequence. CUDA still decides where its block runs; the rule is not a\n precomputed tile-to-physical-SM map.\n2. In a CLC-backed scheduler, a selected thread requests cancellation of an\n unspecified block or cluster that has not launched. A successful response\n returns that existing grid coordinate; a request can fail. CLC does not\n create tiles or make the fastest SM autonomously steal arbitrary work.\n3. The initial `blockIdx` in CUTLASS's pinned CLC scheduler is static and later\n requests are asynchronous, pipelined, and cluster-granular. There is no\n universal per-tile latency bound: compare request activity and end-to-end\n time against the matched static scheduler.\n4. Splitting a large expert or K range may expose more independent work, but it\n can add partial-result and reduction costs. Hold math, tile shape, cluster\n shape, launch resources, and output semantics constant in the comparison.\n\nPTX ISA 9.0 specifies CLC for `sm_100` or higher and explicitly includes\nSM120-family targets for the multicast form. CUTLASS 4.5.0's documented\n`PersistentTileSchedulerSm100` integration is a narrower library example, not\nthe complete ISA compatibility boundary."}, "reason": {"statement": "Make persistence and acquisition policy independent choices.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"]}} +{"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "| [Contiguous layout](../kernels/grouped-gemm.md) | Pack variable-M experts sequentially; offsets array indexes expert boundaries |"}, "after": {"statement": "At pinned DeepGEMM commit\n[`891d57b4`](https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f),\nthe three grouped interfaces have different axes and metadata:\n\n| M-grouped contiguous | Pack `A` and `D` along variable M with N/K fixed; identify groups by per-row expert IDs or per-group prefix-sum ends; segments are M-block aligned | Record valid rows, alignment padding, tile counts, and metadata form |\n| M-grouped masked | Allocate `[G, M_max, ...]`, pass one valid-M count per group, and compute valid portions; the fixed allocation is documented for a CUDA-graph decode case | Separate allocated rows from valid rows and inspect edge predication in generated code |\n| K-grouped contiguous | Pack variable K with M/N fixed for MoE weight backward | Do not use its K-axis contract to describe forward token imbalance |\n\nA fixed `M_max` allocation therefore does not establish that all padding rows\nare computed. Any residual edge-tile or predication cost needs generated-code\nand profile evidence for the selected kernel."}, "reason": {"statement": "Name both actual ABI forms.", "urls": ["https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp"]}} +{"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "| [Masked layout](../kernels/grouped-gemm.md) | Good for CUDA graph capture; wastes compute on padding |"}, "after": {"statement": "At pinned DeepGEMM commit\n[`891d57b4`](https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f),\nthe three grouped interfaces have different axes and metadata:\n\n| M-grouped contiguous | Pack `A` and `D` along variable M with N/K fixed; identify groups by per-row expert IDs or per-group prefix-sum ends; segments are M-block aligned | Record valid rows, alignment padding, tile counts, and metadata form |\n| M-grouped masked | Allocate `[G, M_max, ...]`, pass one valid-M count per group, and compute valid portions; the fixed allocation is documented for a CUDA-graph decode case | Separate allocated rows from valid rows and inspect edge predication in generated code |\n| K-grouped contiguous | Pack variable K with M/N fixed for MoE weight backward | Do not use its K-axis contract to describe forward token imbalance |\n\nA fixed `M_max` allocation therefore does not establish that all padding rows\nare computed. Any residual edge-tile or predication cost needs generated-code\nand profile evidence for the selected kernel."}, "reason": {"statement": "Retain graph scope and separate allocation from valid computation.", "urls": ["https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp"]}} +{"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "| [EPLB (Expert Parallel Load Balancer)](https://github.com/deepseek-ai/EPLB) | Replicate heavy experts across GPUs; 1.49x prefill speedup, 2.54x decode |"}, "after": {"statement": "[DeepSeek EPLB at commit\n`d52c72d`](https://github.com/deepseek-ai/EPLB/tree/d52c72d5b2f2fb4c41afbf8eb21366820239913d)\ntakes per-logical-expert load statistics, replicates heavily loaded logical\nexperts, and packs physical experts across configured GPUs and nodes. It is a\nhost-side expert-parallel placement planner, not a device-kernel tile\nscheduler; use it as a complement only when multi-GPU expert placement is in\nthe measured system scope.\n\nAn LMSYS/SGLang study on a 96-H100 deployment reported `1.49x` prefill and\n`2.54x` decode throughput speedups in its large-scale EPLB ablation. The same\nstudy says it used in-distribution data and that production distribution\nshifts require further testing. Those figures are scoped results, not generic\nEPLB speedups."}, "reason": {"statement": "Scope placement mechanism and measurement separately.", "urls": ["https://www.lmsys.org/blog/2025-05-05-large-scale-ep/", "https://github.com/deepseek-ai/EPLB/blob/d52c72d5b2f2fb4c41afbf8eb21366820239913d/eplb.py"]}} +{"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "This highlighted that even careful tile scheduling can be outrun by algorithmic restructuring — and prompted the MLSys 2026 FlashInfer contest to add runtime isolation + subprocess eval."}, "after": {"statement": "GPU Mode's official postmortem records a temporarily first-place submission\nthat combined a real grouped-GEMM kernel with a timing-harness exploit. The\ncorrectness path ran a padded eight-group computation on each of 15 cloned\nobjects. During timing, the first call merged them into one 120-group launch,\ncalls 2 through 15 returned cached output pointers, and the harness divided the\ncombined timing by 15.\n\nThis is evidence about evaluator state reuse, not a valid load-balancing\nperformance result. The postmortem identifies `gpu-mode/reference-kernels`\nPR 104 as the harness response; neither that record nor the pinned FlashInfer\nMLSys 2026 evaluation contract establishes a causal link between this incident\nand the later contest design."}, "reason": {"statement": "Preserve the verified incident mechanics without inventing historical causality.", "urls": ["https://www.gpumode.com/news/reward-hacking-nvfp4", "https://github.com/flashinfer-ai/flashinfer-bench-starter-kit/blob/75ccd05cafceb0fd1f86be4cd0f2117249463c66/EVALUATION.md"]}} +{"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "- CLC only available on SM100 datacenter (not SM120 consumer)"}, "after": {"statement": "Keep persistence, work acquisition, and decomposition as separate variables:\n\n1. A static persistent worker can advance through a deterministic logical-work\n sequence. CUDA still decides where its block runs; the rule is not a\n precomputed tile-to-physical-SM map.\n2. In a CLC-backed scheduler, a selected thread requests cancellation of an\n unspecified block or cluster that has not launched. A successful response\n returns that existing grid coordinate; a request can fail. CLC does not\n create tiles or make the fastest SM autonomously steal arbitrary work.\n3. The initial `blockIdx` in CUTLASS's pinned CLC scheduler is static and later\n requests are asynchronous, pipelined, and cluster-granular. There is no\n universal per-tile latency bound: compare request activity and end-to-end\n time against the matched static scheduler.\n4. Splitting a large expert or K range may expose more independent work, but it\n can add partial-result and reduction costs. Hold math, tile shape, cluster\n shape, launch resources, and output semantics constant in the comparison.\n\nPTX ISA 9.0 specifies CLC for `sm_100` or higher and explicitly includes\nSM120-family targets for the multicast form. CUTLASS 4.5.0's documented\n`PersistentTileSchedulerSm100` integration is a narrower library example, not\nthe complete ISA compatibility boundary."}, "reason": {"statement": "Separate PTX compatibility from a particular library integration.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "- Dynamic scheduling has small per-tile overhead vs static precomputed"}, "after": {"statement": "Keep persistence, work acquisition, and decomposition as separate variables:\n\n1. A static persistent worker can advance through a deterministic logical-work\n sequence. CUDA still decides where its block runs; the rule is not a\n precomputed tile-to-physical-SM map.\n2. In a CLC-backed scheduler, a selected thread requests cancellation of an\n unspecified block or cluster that has not launched. A successful response\n returns that existing grid coordinate; a request can fail. CLC does not\n create tiles or make the fastest SM autonomously steal arbitrary work.\n3. The initial `blockIdx` in CUTLASS's pinned CLC scheduler is static and later\n requests are asynchronous, pipelined, and cluster-granular. There is no\n universal per-tile latency bound: compare request activity and end-to-end\n time against the matched static scheduler.\n4. Splitting a large expert or K range may expose more independent work, but it\n can add partial-result and reduction costs. Hold math, tile shape, cluster\n shape, launch resources, and output semantics constant in the comparison.\n\nPTX ISA 9.0 specifies CLC for `sm_100` or higher and explicitly includes\nSM120-family targets for the multicast form. CUTLASS 4.5.0's documented\n`PersistentTileSchedulerSm100` integration is a narrower library example, not\nthe complete ISA compatibility boundary."}, "reason": {"statement": "Require matched measurements of request activity and latency.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"]}} +{"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "- Small experts may not benefit — minimum viable tile size is a floor"}, "after": {"statement": "Do not infer one expert per physical SM from a grouped-GEMM trace. CUDA assigns\nthread blocks to available SMs, while grouped kernels may split an expert into\nmultiple output tiles or let one resident worker process several logical tiles.\nMeasure at each relevant level:\n\n- routed tokens per expert for the exact batch and prefill/decode phase;\n- valid rows and output-tile counts per expert;\n- completed tiles and elapsed work per logical worker and, when instrumented,\n per SM;\n- dispatch, grouped-GEMM, combine, and end-to-end times per expert-parallel\n rank.\n\nSmall expert segments and partial output tiles can expose too little parallel\nwork or leave lanes predicated out. The effect depends on the kernel tile,\nother GEMM dimensions, resident-worker count, and competing implementation;\nthere is no universal `M < BLOCK_M` diagnosis or minimum viable size.\n\nUniform expected routing, a larger batch, or an auxiliary balancing loss may\nchange token-count skew, but none proves balanced tile counts, worker\ndurations, communication, or runtime. Treat imbalance as absent only when the\nmeasured distributions are narrow and a matched balancing variant does not\nmaterially improve the timed operation."}, "reason": {"statement": "Make small-M a measured risk rather than a fixed cutoff.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/efficient_gemm.md", "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://www.lmsys.org/blog/2025-05-05-large-scale-ep/"]}} +{"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "- EPLB works at cluster scale, not single-device"}, "after": {"statement": "[DeepSeek EPLB at commit\n`d52c72d`](https://github.com/deepseek-ai/EPLB/tree/d52c72d5b2f2fb4c41afbf8eb21366820239913d)\ntakes per-logical-expert load statistics, replicates heavily loaded logical\nexperts, and packs physical experts across configured GPUs and nodes. It is a\nhost-side expert-parallel placement planner, not a device-kernel tile\nscheduler; use it as a complement only when multi-GPU expert placement is in\nthe measured system scope.\n\nAn LMSYS/SGLang study on a 96-H100 deployment reported `1.49x` prefill and\n`2.54x` decode throughput speedups in its large-scale EPLB ablation. The same\nstudy says it used in-distribution data and that production distribution\nshifts require further testing. Those figures are scoped results, not generic\nEPLB speedups."}, "reason": {"statement": "State intended multi-GPU placement scope without a false scale dichotomy.", "urls": ["https://github.com/deepseek-ai/EPLB/blob/d52c72d5b2f2fb4c41afbf8eb21366820239913d/eplb.py", "https://www.lmsys.org/blog/2025-05-05-large-scale-ep/"]}} +{"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "- Uniform routing (rare in practice)\n- Very large batch sizes (statistics average out)\n- Training with auxiliary load balancing loss"}, "after": {"statement": "Do not infer one expert per physical SM from a grouped-GEMM trace. CUDA assigns\nthread blocks to available SMs, while grouped kernels may split an expert into\nmultiple output tiles or let one resident worker process several logical tiles.\nMeasure at each relevant level:\n\n- routed tokens per expert for the exact batch and prefill/decode phase;\n- valid rows and output-tile counts per expert;\n- completed tiles and elapsed work per logical worker and, when instrumented,\n per SM;\n- dispatch, grouped-GEMM, combine, and end-to-end times per expert-parallel\n rank.\n\nSmall expert segments and partial output tiles can expose too little parallel\nwork or leave lanes predicated out. The effect depends on the kernel tile,\nother GEMM dimensions, resident-worker count, and competing implementation;\nthere is no universal `M < BLOCK_M` diagnosis or minimum viable size.\n\nUniform expected routing, a larger batch, or an auxiliary balancing loss may\nchange token-count skew, but none proves balanced tile counts, worker\ndurations, communication, or runtime. Treat imbalance as absent only when the\nmeasured distributions are narrow and a matched balancing variant does not\nmaterially improve the timed operation."}, "reason": {"statement": "Replace proxies with an operational measurement criterion.", "urls": ["https://www.lmsys.org/blog/2025-05-05-large-scale-ep/", "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/efficient_gemm.md"]}} +{"path": "wiki/kernels/sparse-mla.md", "before": {"statement": "Sparse Multi-head Latent Attention introduced in DeepSeek V3.2. Two-stage pipeline: (1) Lightning Indexer selects top-K tokens per query via FP8 scorer, (2) MLA runs only over selected tokens. This reduces decode attention compute from O(seqlen) to O(topk=2048) tokens, critical for long-context serving."}, "after": {"statement": "DeepSeek-V3.2-Exp introduces **DeepSeek Sparse Attention (DSA)**. Its\nfirst-party report defines two components:\n\n1. The lightning indexer computes one score for every query/preceding-token\n pair. A score is a weighted sum across indexer heads of a ReLU-applied query\n and key dot product.\n2. Fine-grained token selection retains the KV entries at the top-k index\n scores and applies the main attention only to that selected set.\n\nThe released `config_671B_v3.2.json` uses 64 indexer heads of dimension 128 and\n`index_topk=2048`. DSA is instantiated under MLA's MQA mode: the selected MLA\nlatent entry is shared across the query heads. The model has 128 MLA query\nheads, a 512-dimensional latent KV component, and a 64-dimensional RoPE\ncomponent; a smaller local head count is a parallel-sharding choice rather than\nthe architecture-wide contract.\n\nFor a length-L sequence and fixed selected count k, the report reduces the\n**main/core attention** complexity from O(L squared) to O(Lk). The lightning\nindexer still scores the preceding context and remains O(L squared) over the\nsequence. For decode, measure both the O(L) selector scan for each new query and\nthe O(k) selected-attention work; do not describe the full pipeline as O(k)."}, "reason": {"statement": "Separate selector and selected-attention complexity.", "urls": []}} +{"path": "wiki/kernels/sparse-mla.md", "before": {"statement": "```\nQuery q_t (new token embedding)\n │\n ▼\n┌────────────────────────┐\n│ Lightning Indexer │ FP8 scorer, per-query top-K selection\n│ - FP8 KV cache │ h64, d128, topk=2048, page_size=64\n│ - Compute q·k_i scores │\n│ - Select top-2048 i │\n└────────────────────────┘\n │ selected_indices [2048]\n ▼\n┌────────────────────────┐\n│ Sparse MLA │ Standard MLA but only over selected\n│ - Gather selected K,V │ h16, ckv512, kpe64, topk=2048\n│ - Attention compute │\n│ - Output y_t │\n└────────────────────────┘\n```"}, "after": {"statement": "DeepSeek-V3.2-Exp introduces **DeepSeek Sparse Attention (DSA)**. Its\nfirst-party report defines two components:\n\n1. The lightning indexer computes one score for every query/preceding-token\n pair. A score is a weighted sum across indexer heads of a ReLU-applied query\n and key dot product.\n2. Fine-grained token selection retains the KV entries at the top-k index\n scores and applies the main attention only to that selected set.\n\nThe released `config_671B_v3.2.json` uses 64 indexer heads of dimension 128 and\n`index_topk=2048`. DSA is instantiated under MLA's MQA mode: the selected MLA\nlatent entry is shared across the query heads. The model has 128 MLA query\nheads, a 512-dimensional latent KV component, and a 64-dimensional RoPE\ncomponent; a smaller local head count is a parallel-sharding choice rather than\nthe architecture-wide contract.\n\nFor a length-L sequence and fixed selected count k, the report reduces the\n**main/core attention** complexity from O(L squared) to O(Lk). The lightning\nindexer still scores the preceding context and remains O(L squared) over the\nsequence. For decode, measure both the O(L) selector scan for each new query and\nthe O(k) selected-attention work; do not describe the full pipeline as O(k).\n\nThe released high-performance path is not one fused “indexer plus sparse MLA”\nkernel:\n\n- DeepGEMM provides non-paged and paged indexer-logit kernels. Its pinned SM100\n FP8 path computes token logits from FP8 query/key inputs, per-token key\n scales, and per-query head weights; top-k selection remains a separate\n operation.\n- FlashMLA sparse attention consumes caller-produced token indices. It does not\n run the lightning indexer or top-k selector.\n\nAt FlashMLA commit\n[`71c7379`](https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232),\nprefill and decode have different contracts:\n\n| Sparse decode | `q[batch,s_q,h_q,576]`; paged `k_cache`; `indices[batch,s_q,topk]`, where each nonnegative value encodes physical page times page size plus token offset; `-1` is invalid | V3-family sparse mode dequantizes its FP8 cache for BF16 attention and returns BF16 `out` plus FP32 `lse` |\n| Sparse prefill | Unbatched BF16 `q[s_q,h_q,d_qk]`, BF16 `kv[s_kv,h_kv,d_qk]`, and INT32 `indices[s_q,h_kv,topk]`; the documented equivalence requires `h_kv=1`; negative or at-least-`s_kv` entries are invalid | Computes attention over gathered token rows and returns BF16 `out`, FP32 `max_logits`, and FP32 `lse` |\n\nThese are token indices, not one maximum or one selection per cache block.\nThe sparse-decode API accepts page size through the cache tensor; pinned\ncorrectness tests exercise multiple values including 2, 53, 61, 64, 69, 256,\nand 576. Page size 64 is therefore a deployment configuration, not a universal\nFlashMLA requirement."}, "reason": {"statement": "Replace invented deployment structure with paper equations and exact public API boundaries.", "urls": []}} +{"path": "wiki/kernels/sparse-mla.md", "before": {"statement": "Each KV cache entry is 656 bytes:\n- 512 bytes: FP8 compressed KV data\n- 16 bytes: FP32 per-block scale factors\n- 128 bytes: BF16 RoPE embeddings (for indexer positional encoding)\n\nBlock size fixed at 64 (FlashMLA requirement)."}, "after": {"statement": "Only the V3/V3.1/V3.2 **FP8 sparse-decode** mode has the documented 656-byte\nper-token layout:\n\n- 512 E4M3 NoPE values: 512 bytes;\n- four FP32 scales, one for each successive group of 128 NoPE values: 16 bytes;\n- 64 BF16 RoPE values used by the attention key: 128 bytes.\n\nThe indexer has a separate FP8 K cache and scale cache in the released model.\nThe 656-byte attention entry is not an indexer cache and is not the layout of\nevery dense, sparse-prefill, or non-V3 FlashMLA mode."}, "reason": {"statement": "Scope the byte layout and make page size an explicit interface dimension.", "urls": []}} +{"path": "wiki/kernels/sparse-mla.md", "before": {"statement": "```cuda\n// Score each KV block against query using FP8 block-scale MMA\n// Reduce to per-block max, then top-K selection across blocks\n\n__global__ void lightning_indexer_kernel(\n const fp8_t* q_fp8, // [h, d] query (FP8 quantized)\n const fp8_t* kv_cache_fp8, // [num_blocks, 64, d] paged\n const fp8_t* kv_scales,\n float* scores_out, // [num_blocks] per-block max score\n int num_blocks\n) {\n // Each threadblock handles one KV block's score\n uint32_t tmem = tmem_alloc(64);\n tcgen05_mma_f8(q_smem, k_block_smem, tmem);\n // Reduce inside block to max score\n // Write per-block score\n}\n\n// Separate top-K selection kernel across the num_blocks score array\n```"}, "after": {"statement": "DeepSeek-V3.2-Exp introduces **DeepSeek Sparse Attention (DSA)**. Its\nfirst-party report defines two components:\n\n1. The lightning indexer computes one score for every query/preceding-token\n pair. A score is a weighted sum across indexer heads of a ReLU-applied query\n and key dot product.\n2. Fine-grained token selection retains the KV entries at the top-k index\n scores and applies the main attention only to that selected set.\n\nThe released `config_671B_v3.2.json` uses 64 indexer heads of dimension 128 and\n`index_topk=2048`. DSA is instantiated under MLA's MQA mode: the selected MLA\nlatent entry is shared across the query heads. The model has 128 MLA query\nheads, a 512-dimensional latent KV component, and a 64-dimensional RoPE\ncomponent; a smaller local head count is a parallel-sharding choice rather than\nthe architecture-wide contract.\n\nFor a length-L sequence and fixed selected count k, the report reduces the\n**main/core attention** complexity from O(L squared) to O(Lk). The lightning\nindexer still scores the preceding context and remains O(L squared) over the\nsequence. For decode, measure both the O(L) selector scan for each new query and\nthe O(k) selected-attention work; do not describe the full pipeline as O(k).\n\nThe released high-performance path is not one fused “indexer plus sparse MLA”\nkernel:\n\n- DeepGEMM provides non-paged and paged indexer-logit kernels. Its pinned SM100\n FP8 path computes token logits from FP8 query/key inputs, per-token key\n scales, and per-query head weights; top-k selection remains a separate\n operation.\n- FlashMLA sparse attention consumes caller-produced token indices. It does not\n run the lightning indexer or top-k selector.\n\nAt FlashMLA commit\n[`71c7379`](https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232),\nprefill and decode have different contracts:\n\n| Sparse decode | `q[batch,s_q,h_q,576]`; paged `k_cache`; `indices[batch,s_q,topk]`, where each nonnegative value encodes physical page times page size plus token offset; `-1` is invalid | V3-family sparse mode dequantizes its FP8 cache for BF16 attention and returns BF16 `out` plus FP32 `lse` |\n| Sparse prefill | Unbatched BF16 `q[s_q,h_q,d_qk]`, BF16 `kv[s_kv,h_kv,d_qk]`, and INT32 `indices[s_q,h_kv,topk]`; the documented equivalence requires `h_kv=1`; negative or at-least-`s_kv` entries are invalid | Computes attention over gathered token rows and returns BF16 `out`, FP32 `max_logits`, and FP32 `lse` |\n\nThese are token indices, not one maximum or one selection per cache block.\nThe sparse-decode API accepts page size through the cache tensor; pinned\ncorrectness tests exercise multiple values including 2, 53, 61, 64, 69, 256,\nand 576. Page size 64 is therefore a deployment configuration, not a universal\nFlashMLA requirement."}, "reason": {"statement": "The snippet is not upstream code and is semantically misleading.", "urls": []}} +{"path": "wiki/kernels/sparse-mla.md", "before": {"statement": "```cuda\n// After top-K selection gives indices, gather K,V from paged cache\n// Then run standard MLA on the gathered subset\n\n__global__ void sparse_mla_decode_kernel(\n const int* topk_indices, // [2048] selected block indices\n const fp8_t* kv_cache, // paged KV (656 bytes/token)\n const half* q, // [16, 576] MLA query\n half* output\n) {\n // Load q into registers/SMEM\n // For each selected block:\n // gather K,V block (FP8) → dequant to BF16 → MMA\n // Online softmax accumulation\n // Final weighted sum\n}\n```"}, "after": {"statement": "The released high-performance path is not one fused “indexer plus sparse MLA”\nkernel:\n\n- DeepGEMM provides non-paged and paged indexer-logit kernels. Its pinned SM100\n FP8 path computes token logits from FP8 query/key inputs, per-token key\n scales, and per-query head weights; top-k selection remains a separate\n operation.\n- FlashMLA sparse attention consumes caller-produced token indices. It does not\n run the lightning indexer or top-k selector.\n\nAt FlashMLA commit\n[`71c7379`](https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232),\nprefill and decode have different contracts:\n\n| Sparse decode | `q[batch,s_q,h_q,576]`; paged `k_cache`; `indices[batch,s_q,topk]`, where each nonnegative value encodes physical page times page size plus token offset; `-1` is invalid | V3-family sparse mode dequantizes its FP8 cache for BF16 attention and returns BF16 `out` plus FP32 `lse` |\n| Sparse prefill | Unbatched BF16 `q[s_q,h_q,d_qk]`, BF16 `kv[s_kv,h_kv,d_qk]`, and INT32 `indices[s_q,h_kv,topk]`; the documented equivalence requires `h_kv=1`; negative or at-least-`s_kv` entries are invalid | Computes attention over gathered token rows and returns BF16 `out`, FP32 `max_logits`, and FP32 `lse` |\n\nThese are token indices, not one maximum or one selection per cache block.\nThe sparse-decode API accepts page size through the cache tensor; pinned\ncorrectness tests exercise multiple values including 2, 53, 61, 64, 69, 256,\nand 576. Page size 64 is therefore a deployment configuration, not a universal\nFlashMLA requirement."}, "reason": {"statement": "Use the exact public contract instead of invented CUDA.", "urls": []}} +{"path": "wiki/kernels/sparse-mla.md", "before": {"statement": "| Dense MLA decode | H800 | 660 (BF16) | 3000 GB/s, compute-bound |"}, "after": {"statement": "FlashMLA's pinned README reports the following maxima. They are useful source\nclaims, not reproducible benchmark tuples: the README does not provide complete\nshapes, timed regions, repetitions, samples, or variance.\n\n| Dense MLA decode, memory-bound configuration | H800 SXM5, CUDA 12.8 | Up to 3000 GB/s with BF16 cache |\n| Dense MLA decode, compute-bound configuration | H800 SXM5, CUDA 12.8 | Up to 660 TFLOPS with BF16 cache; separate from the 3000-GB/s case |\n| Sparse MLA decode | H800 SXM5, CUDA 12.8 | 410 TFLOPS; FP8 KV storage and BF16 matrix multiplication |\n| Sparse MLA decode | B200 | Up to 350 TFLOPS; the source says this path was not really optimized and gives no bandwidth-causality result |\n| Sparse MLA prefill | H800 SXM5, CUDA 12.8 | Up to 640 TFLOPS forward with BF16 Q/KV |\n| Sparse MLA prefill | B200, CUDA 12.9 | Up to 1450 TFLOPS forward with BF16 Q/KV |\n\nThe same README separately reports NVIDIA's dense **MHA** prefill maxima of\n1460 TFLOPS forward and 1000 TFLOPS backward on B200. Those values are not a\ndense-MLA baseline matched to the sparse-prefill result, so numerical proximity\nbetween 1450 and 1460 does not establish equivalent performance."}, "reason": {"statement": "Report the maxima as separate author-reported regimes.", "urls": []}} +{"path": "wiki/kernels/sparse-mla.md", "before": {"statement": "| Sparse MLA decode | H800 | 410 (FP8) | Token-level sparsity |"}, "after": {"statement": "FlashMLA's pinned README reports the following maxima. They are useful source\nclaims, not reproducible benchmark tuples: the README does not provide complete\nshapes, timed regions, repetitions, samples, or variance.\n\n| Dense MLA decode, memory-bound configuration | H800 SXM5, CUDA 12.8 | Up to 3000 GB/s with BF16 cache |\n| Dense MLA decode, compute-bound configuration | H800 SXM5, CUDA 12.8 | Up to 660 TFLOPS with BF16 cache; separate from the 3000-GB/s case |\n| Sparse MLA decode | H800 SXM5, CUDA 12.8 | 410 TFLOPS; FP8 KV storage and BF16 matrix multiplication |\n| Sparse MLA decode | B200 | Up to 350 TFLOPS; the source says this path was not really optimized and gives no bandwidth-causality result |\n| Sparse MLA prefill | H800 SXM5, CUDA 12.8 | Up to 640 TFLOPS forward with BF16 Q/KV |\n| Sparse MLA prefill | B200, CUDA 12.9 | Up to 1450 TFLOPS forward with BF16 Q/KV |\n\nThe same README separately reports NVIDIA's dense **MHA** prefill maxima of\n1460 TFLOPS forward and 1000 TFLOPS backward on B200. Those values are not a\ndense-MLA baseline matched to the sparse-prefill result, so numerical proximity\nbetween 1450 and 1460 does not establish equivalent performance."}, "reason": {"statement": "Separate storage and compute dtypes and retain source scope.", "urls": []}} +{"path": "wiki/kernels/sparse-mla.md", "before": {"statement": "| Sparse MLA decode | B200 | 350 (FP8) | Lower because bandwidth dominates decode |"}, "after": {"statement": "FlashMLA's pinned README reports the following maxima. They are useful source\nclaims, not reproducible benchmark tuples: the README does not provide complete\nshapes, timed regions, repetitions, samples, or variance.\n\n| Dense MLA decode, memory-bound configuration | H800 SXM5, CUDA 12.8 | Up to 3000 GB/s with BF16 cache |\n| Dense MLA decode, compute-bound configuration | H800 SXM5, CUDA 12.8 | Up to 660 TFLOPS with BF16 cache; separate from the 3000-GB/s case |\n| Sparse MLA decode | H800 SXM5, CUDA 12.8 | 410 TFLOPS; FP8 KV storage and BF16 matrix multiplication |\n| Sparse MLA decode | B200 | Up to 350 TFLOPS; the source says this path was not really optimized and gives no bandwidth-causality result |\n| Sparse MLA prefill | H800 SXM5, CUDA 12.8 | Up to 640 TFLOPS forward with BF16 Q/KV |\n| Sparse MLA prefill | B200, CUDA 12.9 | Up to 1450 TFLOPS forward with BF16 Q/KV |\n\nThe same README separately reports NVIDIA's dense **MHA** prefill maxima of\n1460 TFLOPS forward and 1000 TFLOPS backward on B200. Those values are not a\ndense-MLA baseline matched to the sparse-prefill result, so numerical proximity\nbetween 1450 and 1460 does not establish equivalent performance."}, "reason": {"statement": "Retain only the scoped author report and explicit unoptimized caveat.", "urls": []}} +{"path": "wiki/kernels/sparse-mla.md", "before": {"statement": "| Dense prefill | B200 | 1460 (BF16) | tcgen05 peak |"}, "after": {"statement": "FlashMLA's pinned README reports the following maxima. They are useful source\nclaims, not reproducible benchmark tuples: the README does not provide complete\nshapes, timed regions, repetitions, samples, or variance.\n\n| Dense MLA decode, memory-bound configuration | H800 SXM5, CUDA 12.8 | Up to 3000 GB/s with BF16 cache |\n| Dense MLA decode, compute-bound configuration | H800 SXM5, CUDA 12.8 | Up to 660 TFLOPS with BF16 cache; separate from the 3000-GB/s case |\n| Sparse MLA decode | H800 SXM5, CUDA 12.8 | 410 TFLOPS; FP8 KV storage and BF16 matrix multiplication |\n| Sparse MLA decode | B200 | Up to 350 TFLOPS; the source says this path was not really optimized and gives no bandwidth-causality result |\n| Sparse MLA prefill | H800 SXM5, CUDA 12.8 | Up to 640 TFLOPS forward with BF16 Q/KV |\n| Sparse MLA prefill | B200, CUDA 12.9 | Up to 1450 TFLOPS forward with BF16 Q/KV |\n\nThe same README separately reports NVIDIA's dense **MHA** prefill maxima of\n1460 TFLOPS forward and 1000 TFLOPS backward on B200. Those values are not a\ndense-MLA baseline matched to the sparse-prefill result, so numerical proximity\nbetween 1450 and 1460 does not establish equivalent performance."}, "reason": {"statement": "Name the correct operator and attribution.", "urls": []}} +{"path": "wiki/kernels/sparse-mla.md", "before": {"statement": "| Sparse prefill | B200 | 1450 (FP8) | FP8 sparse matches BF16 dense |"}, "after": {"statement": "FlashMLA's pinned README reports the following maxima. They are useful source\nclaims, not reproducible benchmark tuples: the README does not provide complete\nshapes, timed regions, repetitions, samples, or variance.\n\n| Dense MLA decode, memory-bound configuration | H800 SXM5, CUDA 12.8 | Up to 3000 GB/s with BF16 cache |\n| Dense MLA decode, compute-bound configuration | H800 SXM5, CUDA 12.8 | Up to 660 TFLOPS with BF16 cache; separate from the 3000-GB/s case |\n| Sparse MLA decode | H800 SXM5, CUDA 12.8 | 410 TFLOPS; FP8 KV storage and BF16 matrix multiplication |\n| Sparse MLA decode | B200 | Up to 350 TFLOPS; the source says this path was not really optimized and gives no bandwidth-causality result |\n| Sparse MLA prefill | H800 SXM5, CUDA 12.8 | Up to 640 TFLOPS forward with BF16 Q/KV |\n| Sparse MLA prefill | B200, CUDA 12.9 | Up to 1450 TFLOPS forward with BF16 Q/KV |\n\nThe same README separately reports NVIDIA's dense **MHA** prefill maxima of\n1460 TFLOPS forward and 1000 TFLOPS backward on B200. Those values are not a\ndense-MLA baseline matched to the sparse-prefill result, so numerical proximity\nbetween 1450 and 1460 does not establish equivalent performance."}, "reason": {"statement": "Report the value only with its dtype/environment/source limitations.", "urls": []}} +{"path": "wiki/kernels/sparse-mla.md", "before": {"statement": "- Long-context LLM serving (32K+)"}, "after": {"statement": "Use this path directly for DeepSeek-V3.2-Exp, or for another model only after\nits query dimensions, latent/RoPE layout, cache format, head relationships,\nindex encoding, invalid-entry rules, and output semantics match the chosen\nFlashMLA interface.\n\nFor a target deployment:\n\n1. Compare index scores and selected token sets against the released model\n equation and a dense reference, including causal masking and the indexer's\n non-interleaved RoPE layout.\n2. Compare sparse prefill/decode outputs and LSE values with the pinned\n FlashMLA reference for identical selected indices, including invalid and\n partially filled top-k cases.\n3. Time indexer logits, top-k selection, sparse attention, and the complete\n pipeline separately across context lengths, query batch sizes, page sizes,\n and selected counts. Use a matched dense-attention baseline and report\n synchronization, warmup, repeated trials, statistic, and variation.\n\nThere is no verified universal 32K crossover. The relevant threshold is where\nthe measured selector-plus-sparse-attention pipeline improves the target's\nend-to-end latency or cost without violating its accuracy criterion."}, "reason": {"statement": "Replace threshold advice with measured selector-plus-attention crossover.", "urls": []}} +{"path": "wiki/kernels/sparse-mla.md", "before": {"statement": "- DeepSeek V3.2 and similar MLA architectures"}, "after": {"statement": "Use this path directly for DeepSeek-V3.2-Exp, or for another model only after\nits query dimensions, latent/RoPE layout, cache format, head relationships,\nindex encoding, invalid-entry rules, and output semantics match the chosen\nFlashMLA interface.\n\nFor a target deployment:\n\n1. Compare index scores and selected token sets against the released model\n equation and a dense reference, including causal masking and the indexer's\n non-interleaved RoPE layout.\n2. Compare sparse prefill/decode outputs and LSE values with the pinned\n FlashMLA reference for identical selected indices, including invalid and\n partially filled top-k cases.\n3. Time indexer logits, top-k selection, sparse attention, and the complete\n pipeline separately across context lengths, query batch sizes, page sizes,\n and selected counts. Use a matched dense-attention baseline and report\n synchronization, warmup, repeated trials, statistic, and variation.\n\nThere is no verified universal 32K crossover. The relevant threshold is where\nthe measured selector-plus-sparse-attention pipeline improves the target's\nend-to-end latency or cost without violating its accuracy criterion."}, "reason": {"statement": "Make compatibility checklist-based.", "urls": []}} +{"path": "wiki/kernels/sparse-mla.md", "before": {"statement": "performance_claims:\n - gpu: B200\n dtype: fp8\n shape: \"sparse prefill, seqlen=32k, topk=2048\"\n metric: TFLOPS\n value: 1450\n utilization: \"FP8 sparse compute bound\"\n source_id: blog-flashmla"}, "after": {"statement": "performance_claims: []\n\nFlashMLA's pinned README reports the following maxima. They are useful source\nclaims, not reproducible benchmark tuples: the README does not provide complete\nshapes, timed regions, repetitions, samples, or variance.\n\n| Dense MLA decode, memory-bound configuration | H800 SXM5, CUDA 12.8 | Up to 3000 GB/s with BF16 cache |\n| Dense MLA decode, compute-bound configuration | H800 SXM5, CUDA 12.8 | Up to 660 TFLOPS with BF16 cache; separate from the 3000-GB/s case |\n| Sparse MLA decode | H800 SXM5, CUDA 12.8 | 410 TFLOPS; FP8 KV storage and BF16 matrix multiplication |\n| Sparse MLA decode | B200 | Up to 350 TFLOPS; the source says this path was not really optimized and gives no bandwidth-causality result |\n| Sparse MLA prefill | H800 SXM5, CUDA 12.8 | Up to 640 TFLOPS forward with BF16 Q/KV |\n| Sparse MLA prefill | B200, CUDA 12.9 | Up to 1450 TFLOPS forward with BF16 Q/KV |\n\nThe same README separately reports NVIDIA's dense **MHA** prefill maxima of\n1460 TFLOPS forward and 1000 TFLOPS backward on B200. Those values are not a\ndense-MLA baseline matched to the sparse-prefill result, so numerical proximity\nbetween 1450 and 1460 does not establish equivalent performance."}, "reason": {"statement": "Do not encode an incomplete author maximum as a structured benchmark claim.", "urls": []}} +{"path": "wiki/kernels/sparse-mla.md", "before": {"statement": "Sparse Multi-head Latent Attention introduced in DeepSeek V3.2. Two-stage pipeline: (1) Lightning Indexer selects top-K tokens per query via FP8 scorer, (2) MLA runs only over selected tokens. This reduces decode attention compute from O(seqlen) to O(topk=2048) tokens, critical for long-context serving."}, "after": {"statement": "The released high-performance path is not one fused “indexer plus sparse MLA”\nkernel:\n\n- DeepGEMM provides non-paged and paged indexer-logit kernels. Its pinned SM100\n FP8 path computes token logits from FP8 query/key inputs, per-token key\n scales, and per-query head weights; top-k selection remains a separate\n operation.\n- FlashMLA sparse attention consumes caller-produced token indices. It does not\n run the lightning indexer or top-k selector.\n\nAt FlashMLA commit\n[`71c7379`](https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232),\nprefill and decode have different contracts:\n\n| Sparse decode | `q[batch,s_q,h_q,576]`; paged `k_cache`; `indices[batch,s_q,topk]`, where each nonnegative value encodes physical page times page size plus token offset; `-1` is invalid | V3-family sparse mode dequantizes its FP8 cache for BF16 attention and returns BF16 `out` plus FP32 `lse` |\n| Sparse prefill | Unbatched BF16 `q[s_q,h_q,d_qk]`, BF16 `kv[s_kv,h_kv,d_qk]`, and INT32 `indices[s_q,h_kv,topk]`; the documented equivalence requires `h_kv=1`; negative or at-least-`s_kv` entries are invalid | Computes attention over gathered token rows and returns BF16 `out`, FP32 `max_logits`, and FP32 `lse` |\n\nThese are token indices, not one maximum or one selection per cache block.\nThe sparse-decode API accepts page size through the cache tensor; pinned\ncorrectness tests exercise multiple values including 2, 53, 61, 64, 69, 256,\nand 576. Page size 64 is therefore a deployment configuration, not a universal\nFlashMLA requirement.\n\nOnly the V3/V3.1/V3.2 **FP8 sparse-decode** mode has the documented 656-byte\nper-token layout:\n\n- 512 E4M3 NoPE values: 512 bytes;\n- four FP32 scales, one for each successive group of 128 NoPE values: 16 bytes;\n- 64 BF16 RoPE values used by the attention key: 128 bytes.\n\nThe indexer has a separate FP8 K cache and scale cache in the released model.\nThe 656-byte attention entry is not an indexer cache and is not the layout of\nevery dense, sparse-prefill, or non-V3 FlashMLA mode."}, "reason": {"statement": "State separate SM100 indexer and attention implementations with their actual precisions.", "urls": []}} +{"path": "wiki/kernels/tensorrt-llm-blackwell-indexer.md", "before": {"statement": "```cuda\n// Evidence checklist before adapting the idea:\n// 1. Inspect indexerKCacheGather.cu and indexerKCacheScatter.cu.\n// 2. Check the scale and FP4 packing layout.\n// 3. Benchmark gather/scatter separately from top-k selection.\n__global__ void candidate_indexer_probe(const uint8_t* cache, int* indices) {\n int tid = blockIdx.x * blockDim.x + threadIdx.x;\n indices[tid] = static_cast(cache[tid]);\n}\n```"}, "after": {"statement": "For the fixed 128-value indexer head in this path, one row has:\n\n| Input | BF16 positional values followed by BF16 non-positional values | 256 before quantization |\n| Packed payload | Two FP4 E2M1 codes per byte | 64 |\n| Scale word | Four UE8M0 exponent bytes, one per successive group of 32 values, packed little-endian into one `int32` | 4 |\n| Cache footprint | Packed payload plus scale word | 68 |\n\nThe fused operator requires CUDA BF16 inputs on one device, at least two\ndimensions, a contiguous innermost dimension, eight-byte-aligned addresses,\nequal row counts, and positional width divisible by four. Positional and\nnon-positional widths must sum to 128. It returns packed `int8[M,64]` data and\n`int32[M,1]` scales.\n\nThe scale for each 32-value group is\n`2^ceil(log2(max(amax, 1e-12) / 6))`. Quantization uses the FP4 E2M1 magnitude\nset `{0, 0.5, 1, 1.5, 2, 3, 4, 6}` and packs the earlier value into the low\nnibble.\n\nThe cache is viewed as\n`[num_blocks, block_size, 1, per_token_size]` and may be non-contiguous. Each\ntoken launch copies four bytes per thread. The FP4 path uses 64 payload bytes\nand one four-byte scale word; the legacy FP8 path uses 128 payload bytes and\none four-byte FP32 scale.\n\nPayload and scale use separate contiguous `int64` slot-mapping arrays. If\neither mapping for a token is negative, gather and scatter skip that entire\ntoken. The gather wrapper allocates output with `empty`, so a directly gathered\nrow skipped this way has no kernel-written sentinel value; callers must avoid\nconsuming it or define initialization in their own reference.\n\nThe gather wrapper preserves a historical typed view: payload bytes are\nreturned as float8 and scale bytes as FP32. The FP4 call site reinterprets them\nas packed `int8` data and `int32` scale words before the DeepGEMM call. These\nviews are byte contracts, not evidence that the FP4 payload became FP8 or that\nits UE8M0 scale word became an FP32 numeric scale.\n\n1. Compare `fused_cat_fp4` byte-for-byte with a reference implementation for\n zeros, FP4 decision boundaries, saturation, non-contiguous row strides, and\n multiple positional/non-positional splits whose widths sum to 128.\n2. Round-trip valid tokens through scatter and gather using strided cache\n views. Check all 64 payload bytes and the four scale bytes independently;\n test negative payload and scale mappings without reading skipped empty rows.\n3. Compare FP4 non-paged and paged indexer logits, then selected indices,\n against an unquantized or higher-precision reference with identical masks,\n weights, and sequence boundaries.\n4. Profile fused preparation, scatter, gather, logit computation, top-k, and\n the complete indexer separately. Report shapes, cache layout, mapping mix,\n warmup, synchronization, repetitions, statistic, and variation.\n\nThe pinned PR contains no performance result, so use it to define candidate\nmechanisms and exact contracts rather than to claim a speedup."}, "reason": {"statement": "Remove invented CUDA that neither implements nor safely probes the pinned indexer contract.", "urls": []}} +{"path": "wiki/kernels/tensorrt-llm-blackwell-indexer.md", "before": {"statement": "TensorRT-LLM PR 13340 integrates an FP4 indexer path for DSA on Blackwell and\nlands CUDA kernels for K-cache gather/scatter and fused FP4 concatenation. Use it\nas implementation evidence for sparse indexer memory movement and quantized\ncache layout, not as a drop-in answer for FlashInfer-Bench."}, "after": {"statement": "TensorRT-LLM PR 13340 adds an FP4 option to its DeepSeek Sparse Attention\n(DSA) indexer path. This page describes the PR's pinned merge revision\n`897c4bff`; it is a TensorRT-LLM implementation reference, not a generic DSA\nABI or a drop-in FlashInfer-Bench kernel.\n\nThe implementation has four distinct stages:\n\n| Q/K preparation | `fused_cat_fp4` concatenates BF16 positional and non-positional components, then quantizes the 128-value row to FP4 E2M1 with per-32-value UE8M0 scales |\n| Cache update/read | Scatter and gather copy an already-quantized payload and its scale word between contiguous tensors and TensorRT-LLM's possibly non-contiguous paged indexer cache |\n| Indexer logits | The FP4 path reinterprets the packed bytes/scales and dispatches TensorRT-LLM's DeepGEMM FP4 MQA-logit implementation |\n| Selection | Prefill or decode top-k runs after logit computation in a separate operator |\n\nThe gather/scatter kernels do not quantize values, compute logits, or select\nindices. Calling them “fused quantized gather/scatter” conflates the first two\nstages.\n\n1. Compare `fused_cat_fp4` byte-for-byte with a reference implementation for\n zeros, FP4 decision boundaries, saturation, non-contiguous row strides, and\n multiple positional/non-positional splits whose widths sum to 128.\n2. Round-trip valid tokens through scatter and gather using strided cache\n views. Check all 64 payload bytes and the four scale bytes independently;\n test negative payload and scale mappings without reading skipped empty rows.\n3. Compare FP4 non-paged and paged indexer logits, then selected indices,\n against an unquantized or higher-precision reference with identical masks,\n weights, and sequence boundaries.\n4. Profile fused preparation, scatter, gather, logit computation, top-k, and\n the complete indexer separately. Report shapes, cache layout, mapping mix,\n warmup, synchronization, repetitions, statistic, and variation.\n\nThe pinned PR contains no performance result, so use it to define candidate\nmechanisms and exact contracts rather than to claim a speedup."}, "reason": {"statement": "Name concat-plus-quantization and byte-copy cache movement as separate operations.", "urls": []}} +{"path": "wiki/patterns/memory-bound.md", "before": {"statement": "- Always profile before optimizing — wrong assumption wastes effort"}, "after": {"statement": "Report input shapes/distributions, useful work and bytes, cache state, warmup/repetitions/statistic, GPU and clocks, software versions, generated instructions, resource usage, roofline assumptions, achieved memory-level bandwidth, requested/transferred efficiency, scheduler activity, and correctness tolerance. Include negative and regressing variants so a category heuristic is not mistaken for causal evidence."}, "reason": {"statement": "Replace an absolute workflow rule with an evidence-based diagnose, hypothesize, control, and measure loop.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} +{"path": "wiki/techniques/chunk-parallelism.md", "before": {"statement": "- Linear attention (O(n) complexity)\n- Recurrent state models (Mamba, GatedDeltaNet, Delta Rule)"}, "after": {"statement": "Some linear recurrent models admit an algebraically equivalent chunkwise formulation. Work local to a chunk can then be expressed with parallel matrix operations, while the state passed across chunk boundaries preserves the recurrence's sequence order. The exact transform is model-specific: it must be derived from the recurrence, not replaced by a generic attention matrix and additive state update.\n\nA correct implementation separates at least these obligations:\n\n1. Compute the chunk-local quantities required by the model's exact recurrence.\n2. Resolve boundary states in sequence order, either with an associative scan supported by the formulation or with explicitly ordered stages or launches.\n3. Combine each chunk's local result with its incoming boundary state and emit outputs in the original token order.\n4. Validate outputs and final states against a token-by-token reference across variable sequence lengths, chunk tails, batches, heads, dtypes, and gate extremes.\n\nAn ordinary GPU grid does not imply increasing program-ID execution order or grid-wide synchronization. Programs for every chunk therefore cannot safely read and overwrite one shared state pointer in a single unordered launch. A staged algorithm must make the boundary-state dependency explicit.\n\nThe pinned NVlabs GatedDeltaNet repository uses chunkwise Triton kernels for training and a WY representation of the gated delta rule. That implementation is direct evidence for GatedDeltaNet chunking, but it is not equivalent to a generic `scores = Q @ K.T; output = scores @ V` snippet and does not supply a universal chunk-size rule.\n\nTiled Flash Linear Attention (TFLA) starts from the chunkwise formulation of linear RNNs and adds another level of sequence parallelization within a chunk. The authors state that this permits arbitrarily large chunks, raises arithmetic intensity, and reduces intermediate-state materialization. The paper and pinned official code apply the method to mLSTM and report H100 results; they do not establish a GatedDeltaNet implementation, a Blackwell/TMEM implementation, or a recursive-tiling API.\n\nThere is no source-backed universal rule that `C=32` is a decode choice or that `C=256-512` is the prefill optimum. Choose only among chunk sizes supported by the exact algorithm and backend, then measure the tradeoff:\n\n- arithmetic intensity and matrix-instruction utilization;\n- intermediate-state and workspace traffic;\n- registers, shared memory, and occupancy;\n- tail handling and variable-length metadata;\n- launch count and boundary-scan cost;\n- latency and throughput for the intended batch and sequence distribution.\n\nCompare candidates with identical compiler/software revisions, launch inputs, correctness oracles, warmups, synchronization, and repeated-trial statistics. Treat a selected size as scoped to the model dimensions, dtype, GPU, backend, and workload. TMEM availability on SM100 may change an implementation's resource design, but it does not by itself prove that a larger chunk is legal or faster."}, "reason": {"statement": "Require a proven model-specific chunkwise-equivalent transform and measured benefit before transfer.", "urls": ["https://arxiv.org/abs/2503.14376v3", "https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921", "https://github.com/state-spaces/mamba", "https://triton-lang.org/main/programming-guide/chapter-01/introduction.html"]}} diff --git a/verification/current-queue.jsonl b/verification/current-queue.jsonl new file mode 100644 index 000000000..d25fca406 --- /dev/null +++ b/verification/current-queue.jsonl @@ -0,0 +1,52 @@ +{"body_sha256": "3323bbd48edb46227476afa34e05f5c2552b2883b02b8a0e60d189dc5592ae97", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "PTX defines a CTA pair as two CTAs in the same cluster whose `%cluster_ctarank` values differ only in the low bit. With `cta_group::2`, one thread from either CTA can initiate a whole `tcgen05.mma`; the peer CTA must still be active. The op", "sha256": "b7afdc24eb589c39ba6e02afe005b308b174971a2755868869988a5789159e7e"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "Allocation management differs from MMA issue. A `tcgen05.alloc` or `tcgen05.dealloc` for `cta_group::2` is issued collectively by two warps, one in each CTA. All tcgen05 instructions in the kernel must use the same CTA-group value.", "sha256": "dbe9bd221edf8a902619236cdafd74941eaa79370ff87f5be263e86965cd6403"}, {"id": "u003", "kind": "prose", "locator": "body:L11-L11", "preview": "`cta_group::2` selects pair-level resources; it does not by itself select one fixed MxNxK shape. The instruction descriptor (`idesc`) encodes M, N, exact operand and accumulator types, sparsity, and related operation details. Legal values d", "sha256": "645ac1d3970d062f6bb9eae3ae3b25048285e45ef7a9f67d305f3d5850fe00e6"}, {"id": "u004", "kind": "prose", "locator": "body:L13-L13", "preview": "For example, m256xn256xk16 is a useful maximum F16/BF16 configuration, but PTX also defines group-2 layouts with other M/N values. Do not infer that every pair-level MMA is created by mechanically doubling a group-1 M dimension.", "sha256": "45f876da36a8addf0f5f581bfda573dd932287c7dc2df933d377bd5a562a1e27"}, {"id": "u005", "kind": "prose", "locator": "body:L15-L15", "preview": "The complete operand grammar, including the eight-register disable-output-lane vector used by `cta_group::2`, is documented on the related [tcgen05.mma page](tcgen05-mma.md).", "sha256": "936e6a2c0cc8ca27f6d457147b1e4da3d5cb75d9e7ebe81f0d7a427f0df6a20e"}, {"id": "u006", "kind": "prose", "locator": "body:L19-L19", "preview": "A pair-level kernel must account for all of these invariants:", "sha256": "e650fb835f8148efa001798dc386feb5cf624c517a63042b2bf85b81eb9ce3e7"}, {"id": "u007", "kind": "list-item", "locator": "body:L21-L21", "preview": "- The two CTAs form a valid CTA pair, and the peer remains active while a group-2 operation is issued.", "sha256": "8c9f5a7af50f1a7d7318498652a12b917fadff697d60bbacde9da59632ca1727"}, {"id": "u008", "kind": "list-item", "locator": "body:L22-L22", "preview": "- Pair-level TMEM allocation and deallocation follow their two-warp collective issue rule.", "sha256": "7fc30f24802c8c5f7ad184674a87dd8fae96c86c6fb7c60b5f65bddd03c46a83"}, {"id": "u009", "kind": "list-item", "locator": "body:L23-L23", "preview": "- SMEM and TMEM operands use legal layouts and descriptors for the selected instruction configuration.", "sha256": "1b832c3aec6846dd172301510f13cf19edd674601565aaca3167657749c9298e"}, {"id": "u010", "kind": "list-item", "locator": "body:L24-L24", "preview": "- Source storage is not overwritten while the asynchronous MMA may still consume it.", "sha256": "68db8f70b3b5c1f6ffa65d9ca6a1b5fde856003e9f6774e061d5768162bba2ed"}, {"id": "u011", "kind": "list-item", "locator": "body:L25-L25", "preview": "- `tcgen05.commit.cta_group::2` and an mbarrier wait provide completion tracking. Fences and execution-ordering operations are added where tcgen05-visible state crosses threads or CTAs.", "sha256": "81756fed0c3ff7d3024ce2f8d37cc4909fa6f8e020401bd81455b0b1a565d299"}, {"id": "u012", "kind": "prose", "locator": "body:L29-L29", "preview": "In Gau Nernst's M=N=K=4096 experiment on a Modal B200 with PyTorch 2.9.1 and CUDA 13, v4 warp specialization reports 1208.83 TFLOP/s and v5 2-SM MMA reports 1302.29 TFLOP/s. Relative to the same 1506.74-TFLOP/s cuBLAS result, those are appr", "sha256": "61e879b8bfa7a29e2e447c1ee711711fbe772da90995a3fe00d069595f991495"}, {"id": "u013", "kind": "prose", "locator": "body:L31-L31", "preview": "That result establishes a gain for one kernel and setup, not a universal threshold. Choose between group 1 and group 2 by benchmarking the target shapes and accounting for CTA pairing, data reuse, layout, occupancy, pipeline stages, and epi", "sha256": "7a1a54d6bfb919e3146075f7166fe3c39e80c43c275ae0409c1c81288eae52ec"}, {"id": "u014", "kind": "list-item", "locator": "body:L35-L35", "preview": "- [tcgen05.mma](tcgen05-mma.md) \u2014 instruction grammar, completion, and descriptors", "sha256": "59d6232600e3c79f97116d6414daad7399a0e29b469130cf23ce8ff9b7f124c0"}, {"id": "u015", "kind": "list-item", "locator": "body:L36-L36", "preview": "- [Tensor Memory](tmem.md) \u2014 TMEM allocation, addressing, and access", "sha256": "a401db8bea32d6d1e6be27e6bb3b9692341354f1689f50bb323de203b7a85c75"}], "confidence_claimed": "verified", "headings": ["Execution model", "Shape is independent of group size", "Correctness requirements", "Source-reported performance", "Related"], "id": "hw-2sm-cooperative", "path": "wiki/hardware/2sm-cooperative.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/prs/cutlass/PR-2139.md", "revision": "ca4fdbea", "url": "https://github.com/NVIDIA/cutlass/pull/2139"}], "risk_flags": ["ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-ptx-isa-sm100", "blog-tcgen05-tutorial", "pr-cutlass-2139"], "title": "Two-SM Cooperative MMA", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "1f3e642012f3b80c6e5f67e510f41b1577c6bf9f6c669549349e1825285ea491", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L8", "preview": "Cluster Launch Control is a Blackwell compute-capability 10.0 mechanism for work stealing between running and not-yet-started thread blocks or thread block clusters. A CLC kernel still launches the problem-sized grid. Each grid coordinate, ", "sha256": "aed47d2a04a85e6e88c936e4f0305caff731581c87ae085491120f8fa1ae4e43"}, {"id": "u002", "kind": "list-item", "locator": "body:L10-L10", "preview": "1. The block or cluster launches normally and first processes its own", "sha256": "dfffcdb04d50ba6a0611612fbb6ff9f3a8689688b0dcd3728ba506a89ef66b38"}, {"id": "u003", "kind": "prose", "locator": "body:L11-L11", "preview": "`blockIdx`.", "sha256": "4c5c20de2592c1acafe9877ccbb8b570ecf16bbef8b6f7e393e80a08ad288c45"}, {"id": "u004", "kind": "list-item", "locator": "body:L12-L12", "preview": "2. An existing worker successfully cancels that not-yet-started ClcID and", "sha256": "ed46f5f3fe23df91f74b22f3f5038976844d96b772df7bc9f16241a82cc067a3"}, {"id": "u005", "kind": "prose", "locator": "body:L13-L13", "preview": "processes the returned coordinate itself.", "sha256": "6d361b359451a9c431b0548ae2c09aa46647cf39813bcb04e3b07d03f17f274f"}, {"id": "u006", "kind": "prose", "locator": "body:L15-L17", "preview": "This lets a persistent worker request subsequent work without maintaining a separate software work queue. It is especially useful when the set of SMs available to a kernel is uneven or changes while the grid is running.", "sha256": "5882b7a980f93728bc6dd0f50b3db643bb8841689dfd7f738ccdb0d8ed30eaf9"}, {"id": "u007", "kind": "prose", "locator": "body:L21-L23", "preview": "`clusterlaunchcontrol.try_cancel` is asynchronous. It writes an opaque 16-byte response to shared memory and completes a transaction on a shared-memory `mbarrier`. The request does not accept a tile coordinate to cancel.", "sha256": "07b338face1ef537bc6a6b1f149dd819322353bfa5868d3c7912e3dc818d2221"}, {"id": "u008", "kind": "prose", "locator": "body:L25-L25", "preview": "The normative PTX instruction forms are:", "sha256": "ed7ac6510f74ef3d579ebe7336066cd20a940b5608289455a2568d88e557f929"}, {"id": "u009", "kind": "code", "locator": "body:L27-L31", "preview": "```ptx clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [response], [mbar]; clusterlaunchcontrol.query_cancel.is_canceled.pred.b128 p, response_b128; @p clusterlaunchcontrol.query_cancel.get_first_ctaid.v4", "sha256": "67fa262449a500557e66306c5d57edcda130d2e7ad8b3b2353b998f1aacb1556"}, {"id": "u010", "kind": "prose", "locator": "body:L33-L33", "preview": "A complete loop must:", "sha256": "d24cc2e528fb2301c03829b15fab641a2eca64b281568e016244188a32f257f6"}, {"id": "u011", "kind": "list-item", "locator": "body:L35-L35", "preview": "1. Allocate and initialize a shared response and `mbarrier`.", "sha256": "0ba588d30c80542da4138743e2418709adee3172b8e1aabc0eb1399de0d04c1e"}, {"id": "u012", "kind": "list-item", "locator": "body:L36-L36", "preview": "2. Submit one asynchronous request from the selected thread and set the", "sha256": "262ae4de6893ed5ff99b6aefa72c4f46117de59cee3bc9fdea46db4d10ccab08"}, {"id": "u013", "kind": "prose", "locator": "body:L37-L37", "preview": "expected transaction size to 16 bytes.", "sha256": "c5b2c635a95f64a2b4978c6746fb9a09fb96d36de263e92d906c555c0c376272"}, {"id": "u014", "kind": "list-item", "locator": "body:L38-L38", "preview": "3. Wait for the corresponding barrier phase to complete.", "sha256": "30d41ba4441d9bf673d62236cb464dde1dad69478b780b3fdd7f13aaea5d1fff"}, {"id": "u015", "kind": "list-item", "locator": "body:L39-L39", "preview": "4. Decode `is_canceled`; decode `get_first_ctaid` only after success.", "sha256": "38e1bc22c329938b11e59946ee33bcab2ee641c7f44aabec8d12e11ba0ee4347"}, {"id": "u016", "kind": "list-item", "locator": "body:L40-L40", "preview": "5. Apply the async/generic proxy fences needed before reusing the response.", "sha256": "26a497bb386f06ad534f04253e05b08ae4f740df588cfb9b7464897480ef7feb"}, {"id": "u017", "kind": "prose", "locator": "body:L42-L45", "preview": "After a thread has observed a failed request, issuing another request from that thread is undefined. Decoding a CTA ID from a failed response is also undefined. A request can fail because no ClcIDs remain or for another scheduling reason, i", "sha256": "2579bb2d0dc3f63c5616d49e9cd07bdaf39852713cc130a7f10392d3fc7d4a85"}, {"id": "u018", "kind": "prose", "locator": "body:L49-L54", "preview": "CLC cancellation is cluster-granular when the kernel uses thread block clusters. One cluster thread submits the multicast request. Every CTA tracks completion with its local shared-memory barrier at cluster scope and receives the same encod", "sha256": "af8682d845156e2308926d1a0bf821b02be6e659e9b3ff363b58921f5558b16d"}, {"id": "u019", "kind": "prose", "locator": "body:L56-L57", "preview": "For example, a successful query by a 2x2 worker cluster consumes the matching 2x2 group of ClcIDs; it is not four unrelated CTA-level cancellations.", "sha256": "6e23b2f1a5cde36d9a61363e854bcc5530da8de592fcda721043478ce0b8c281"}, {"id": "u020", "kind": "prose", "locator": "body:L61-L65", "preview": "At CUTLASS 4.5.0, the `PersistentScheduler` tag for `arch::Sm100` maps to `PersistentTileSchedulerSm100`; newer code can select the intent explicitly with `DynamicPersistentScheduler`. The SM100 scheduler uses `PipelineCLCFetchAsync`: `adva", "sha256": "15bb650d78b4f58a6b6a89201a36b0df29565b4d9ef8768ae8ca5f3a2ae03437"}, {"id": "u021", "kind": "prose", "locator": "body:L67-L67", "preview": "A schematic of the relevant CUTLASS 3.x kernel composition is:", "sha256": "c30f20ad1c9bae9938a60381504d2be3fa4a3a3d3989ca677da0457e294ed4a6"}, {"id": "u022", "kind": "code", "locator": "body:L69-L77", "preview": "```cpp using TileScheduler = cutlass::gemm::DynamicPersistentScheduler; using GemmKernel = cutlass::gemm::kernel::GemmUniversal< cute::Shape, CollectiveMainloop, CollectiveEpilogue, TileScheduler>; ```", "sha256": "f2d33d4b2462ceef8aaac37046da1b30637a67e5d88e2e6402598e676bf5d53b"}, {"id": "u023", "kind": "prose", "locator": "body:L79-L82", "preview": "CUTLASS can apply a software coordinate transform to both the initial `blockIdx` and decoded CLC responses. `max_swizzle_size` and `raster_order` are scheduler arguments used by `swizzle_and_rasterize()`; they are not operands of the CLC PT", "sha256": "a4b2a143772be0706381b1a9813f26cd44da22fd53d5f82576c4b4dd053d10e7"}, {"id": "u024", "kind": "prose", "locator": "body:L86-L86", "preview": "CLC redistributes ClcIDs that already exist in the launched grid. It does not:", "sha256": "bf4dc5fc684e939ef4ea815ea1f55868b6255ce0620993cb595c0fd89df6aab7"}, {"id": "u025", "kind": "list-item", "locator": "body:L88-L88", "preview": "- discard an application-selected output tile;", "sha256": "0e4143cc1ba403906fbd9bb07350d6ac3d46a4ab482c833d5befca3696d1c77f"}, {"id": "u026", "kind": "list-item", "locator": "body:L89-L89", "preview": "- create more independent tiles than the problem grid contains;", "sha256": "cd5c58b2f2af74b71e1551b0cab1067a68a6385717c9f4ce6d0138711e7814db"}, {"id": "u027", "kind": "list-item", "locator": "body:L90-L90", "preview": "- guarantee that every SM remains occupied; or", "sha256": "ca82f9c8684574f060c65785314120d152a34c0efe8956d710f8250716b52bde"}, {"id": "u028", "kind": "list-item", "locator": "body:L91-L91", "preview": "- universally eliminate GEMM wave quantization.", "sha256": "4eaabf49ee1f6647506cc7cdb71f11a5ec2655ea5ca0bee1aabfb7661a071d8a"}, {"id": "u029", "kind": "prose", "locator": "body:L93-L96", "preview": "Consequently, a 32-tile grid exposes at most 32 independent ClcIDs even on a 148-SM B200. Any CLC performance result must state the GPU, available SMs, software versions, tile and cluster shapes, data types, timed region, and measurement me", "sha256": "0453affd2bd43d77232b7fa7a6bd19d9d54911593654fe9569df1a5693c05960"}, {"id": "u030", "kind": "list-item", "locator": "body:L100-L100", "preview": "- [Persistent kernels](../techniques/persistent-kernels.md)", "sha256": "ec9585a9787c7d4334a41d77c6ecbeee6575e90c64bb37f1f2e8c8b8d64b041d"}, {"id": "u031", "kind": "list-item", "locator": "body:L101-L101", "preview": "- [Tile scheduling](../techniques/tile-scheduling.md)", "sha256": "cf18db71a889f046a73a3782397405ed6582feabacaff30a3581db0e6ca9b909"}, {"id": "u032", "kind": "list-item", "locator": "body:L102-L102", "preview": "- [Two-SM cooperative MMA](2sm-cooperative.md)", "sha256": "72792611c19c45c9af380298f75c59a65e9192b3ca1891804c858ae32c448c19"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Request and Decode Protocol", "Thread Block Cluster Rules", "CUTLASS 4.5.0 Integration", "Scope and Limits", "Related"], "id": "hw-clc", "path": "wiki/hardware/clc.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/docs/cutlass-clc-documentation.md", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "risk_flags": ["code", "ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-ptx-isa-sm100", "doc-cutlass-clc"], "title": "Cluster Launch Control (CLC)", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "4215a80fde138535a1d096475ad4d1b76c905d6cd8c3ab2b6198eae4c78180e3", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "An mbarrier is an opaque, naturally aligned 64-bit object in shared memory. It synchronizes threads and can track asynchronous operations. The base instructions were introduced in PTX ISA 7.0 for `sm_80`; Hopper (`sm_90`) added transaction-", "sha256": "c4f5f8029fdece300451e77adb2415e3afc2669f9a0de1bfb7882ffc46fc741a"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "mbarriers are useful in warp-specialized pipelines, but they are a mechanism rather than a requirement for every such kernel.", "sha256": "c1a2ae1d13b327f646fba681f8179c799b52402580d94b42b7d37665834c5abf"}, {"id": "u003", "kind": "prose", "locator": "body:L11-L11", "preview": "For the current phase, an mbarrier tracks:", "sha256": "304bf0328887acfa9d619fae48be89bcb511e2e5dfdda8e39df146bdc226ccc4"}, {"id": "u004", "kind": "list-item", "locator": "body:L13-L13", "preview": "- pending arrivals;", "sha256": "34d9f60ef2b7432a837c1500cfba321064686c29d34d2ae29f3b2958006daa5f"}, {"id": "u005", "kind": "list-item", "locator": "body:L14-L14", "preview": "- the expected arrival count for the next phase; and", "sha256": "f58e7621149f63f5e686db346bcae1c67443ce37f6e6548ed08d0405e9e19c4f"}, {"id": "u006", "kind": "list-item", "locator": "body:L15-L15", "preview": "- a transaction count (`tx-count`) for outstanding asynchronous work.", "sha256": "79aac4088b7db28aec02c8ae5c432e55959337176f40cf20ee2c9dd974977dc0"}, {"id": "u007", "kind": "prose", "locator": "body:L17-L17", "preview": "The current phase completes only when **both** pending arrivals and tx-count reach zero. Completion atomically advances to the next phase and restores pending arrivals from the expected count. Before an arrival in the following phase, at le", "sha256": "3ef1cba778254c5a2d9eb1e0c765268be4505dbd47deab20ed5d96aa70b884f1"}, {"id": "u008", "kind": "prose", "locator": "body:L19-L19", "preview": "Initialization sets phase 0, initializes expected and pending arrivals to `count`, and sets tx-count to zero:", "sha256": "3dd244f469369ff5bb493acac9cf1c56fa19c974e059eee55b901afb1febbd78"}, {"id": "u009", "kind": "code", "locator": "body:L21-L23", "preview": "```ptx mbarrier.init.shared::cta.b64 [bar], arrival_count; ```", "sha256": "aa334cafb032b7c70d03d4e311e23e4026954ad8ea5831c5b48c3204ddb2e352"}, {"id": "u010", "kind": "prose", "locator": "body:L25-L25", "preview": "Invalidate the object with `mbarrier.inval` before reusing its storage for another purpose or reinitializing an already-valid object.", "sha256": "3e1ccc8d4cf7dcc62712f205c681c77ac6676d2a8976dbbb38da31e5ce84dd82"}, {"id": "u011", "kind": "prose", "locator": "body:L29-L29", "preview": "These operations affect different parts of the state:", "sha256": "db8771ec1644298d11488ccedea8133c2d8a332e784378167861fde24f6d0dac"}, {"id": "u012", "kind": "table-row", "locator": "body:L33-L33", "preview": "| Operation | Effect | | `mbarrier.arrive` | Decrements pending arrivals; returns a phase state token for a CTA-shared object. |", "sha256": "300828c39cb781b4776ad8085e58d35a449c60007fb58130022bd4d2df92ee87"}, {"id": "u013", "kind": "table-row", "locator": "body:L34-L34", "preview": "| Operation | Effect | | `mbarrier.arrive.expect_tx` | Performs an arrival and increments tx-count by `txCount`. |", "sha256": "772dbfd906bd8df1d1355530998b10a6527e37976ed1f530c1e6a2ddee681939"}, {"id": "u014", "kind": "table-row", "locator": "body:L35-L35", "preview": "| Operation | Effect | | `mbarrier.expect_tx` | Increments tx-count without an arrival. |", "sha256": "5bb2361feb6d63ec4abc789110d8fe7ae1e7755422f6d64271931f8408e346a1"}, {"id": "u015", "kind": "table-row", "locator": "body:L36-L36", "preview": "| Operation | Effect | | `mbarrier.complete_tx` | Decrements tx-count; it is not an arrival. |", "sha256": "4a62afc513ab2b8c7e543ac78ea5073ed6a6e2edbdac71c34a72fcfa814f5d44"}, {"id": "u016", "kind": "table-row", "locator": "body:L37-L37", "preview": "| Operation | Effect | | `mbarrier.test_wait` / `try_wait` | Tests completion of the phase identified by a state token or parity. |", "sha256": "1eba28206a3d00699d921ca747a0c9124fbece5c665904280441030dffc1c170"}, {"id": "u017", "kind": "prose", "locator": "body:L39-L39", "preview": "Representative CTA-shared forms from PTX ISA 9.0 are:", "sha256": "6438bbf3813fa047bdeeb75c65d22888dd99b788ba7abe2ee32fa891eed9dc5b"}, {"id": "u018", "kind": "code", "locator": "body:L41-L45", "preview": "```ptx mbarrier.arrive.shared::cta.b64 state, [bar]; mbarrier.arrive.expect_tx.shared::cta.b64 state, [bar], tx_count; mbarrier.try_wait.parity.acquire.cta.shared::cta.b64 ready, [bar], phase_parity; ```", "sha256": "0c512373f52c01377048c5a4fea08c09a3152100ab4a8ed297d6eda13d7df509"}, {"id": "u019", "kind": "prose", "locator": "body:L47-L47", "preview": "`try_wait` can suspend temporarily and must still be retried until its predicate is true. Use acquire/release semantics appropriate to the producer-consumer handoff; `.relaxed` does not provide memory-ordering or visibility guarantees.", "sha256": "d4d9f62f69e26f69151ea42f45ec273782c1731a952027788eba1155e66b9538"}, {"id": "u020", "kind": "prose", "locator": "body:L51-L51", "preview": "Parity is the low bit of an individual mbarrier object's phase: even phases use 0 and odd phases use 1. A parity wait can refer only to the current or immediately preceding phase, so software must track phase for the entire lifetime of that", "sha256": "e47ce58bc9df238ddefe1d32feb34f481553161f6041f3bdeafe9dae2c88b0ea"}, {"id": "u021", "kind": "prose", "locator": "body:L53-L53", "preview": "For an N-stage ring, track state per stage. When stage `s` is reused, pass the parity expected for `bar[s]`; toggle that stage's parity only after its phase completes. One global bit toggled on every loop iteration is generally wrong becaus", "sha256": "dac3944e6a8647b2452f518676aca1284de81d4cd74b089dab331f7764a0bea3"}, {"id": "u022", "kind": "prose", "locator": "body:L55-L55", "preview": "Using the opaque state returned by `mbarrier.arrive` is an alternative when the same participant can carry that token to its wait.", "sha256": "526896b32527c238db4d339df27bd01784d736631118b538e38356bc6649bef1"}, {"id": "u023", "kind": "prose", "locator": "body:L59-L59", "preview": "For a common TMA-load phase initialized with one pending arrival:", "sha256": "a96f90645a2583b97dd27a02fea0ffb23a166f2fece4b65b4197900cf9b08a67"}, {"id": "u024", "kind": "list-item", "locator": "body:L61-L61", "preview": "1. The producer executes `mbarrier.arrive.expect_tx` with the sum of bytes that all TMA operations in this phase will report.", "sha256": "e6e85131dd05b8acd13524ece1fbf82c07d1fb351d06ba6b9b97466177defb4d"}, {"id": "u025", "kind": "list-item", "locator": "body:L62-L62", "preview": "2. The producer issues the TMA operations with `.mbarrier::complete_tx::bytes` and the same barrier.", "sha256": "3b455d20ff2edf905c7c37c22b84bb4032d6e814ebf48c6ebc9f292e7b9c2a12"}, {"id": "u026", "kind": "list-item", "locator": "body:L63-L63", "preview": "3. Each TMA completion performs `complete-tx` for the bytes it copied.", "sha256": "58b5cf80e14bbd08da00abbf27a7f8a2da08ada34555b744fbf89d7700b0b686"}, {"id": "u027", "kind": "list-item", "locator": "body:L64-L64", "preview": "4. The consumer waits for the phase to complete before reading the destination.", "sha256": "9a51e79d4ad54417a7c9bc97a7b91a0d23fae064b7c858456d0904fc8586e507"}, {"id": "u028", "kind": "prose", "locator": "body:L66-L66", "preview": "The `arrive.expect_tx` operation accounts for the software arrival and establishes the expected byte total. TMA hardware does **not** perform a second arrival: it decrements tx-count through complete-tx. Therefore, do not add an unmatched `", "sha256": "c52470c2e44b4d5cf43c371c1c9b810cea078c50208fbb8aac48d0844940ac00"}, {"id": "u029", "kind": "prose", "locator": "body:L68-L68", "preview": "The expected byte total, barrier address, destination ownership, and exact `cp.async.bulk.tensor` form must match. Follow the complete instruction grammar in the PTX ISA rather than using placeholder helper calls.", "sha256": "37fcfdfc369b1e9a22b669962cfd1e4ae24637c9e9f768a9b2dcc0054bc2122d"}, {"id": "u030", "kind": "prose", "locator": "body:L72-L72", "preview": "`tcgen05.commit.cta_group::*.mbarrier::arrive::one` makes an mbarrier track prior asynchronous `tcgen05.mma`, `tcgen05.cp`, or `tcgen05.shift` operations issued by that thread. When those operations complete, the system performs one arrive-", "sha256": "20e0ed0e00ca9af1a215f7874e174566a789deba3bab63c34b5cb599d5e24b8e"}, {"id": "u031", "kind": "prose", "locator": "body:L74-L74", "preview": "A consumer in another thread waits for the mbarrier and then uses the applicable `tcgen05.fence::after_thread_sync` ordering sequence. The fence participates in the handoff but does not replace the completion wait.", "sha256": "ef5af611efeacf06e77ed80510a63fb91011e700d7c43b4e8ef02263a285fe2b"}, {"id": "u032", "kind": "prose", "locator": "body:L78-L78", "preview": "State space and synchronization scope are separate concepts:", "sha256": "cde5c333b489108b69110f7a534b3445c2a5bdba33ea8114b885c3b9ef0f15bf"}, {"id": "u033", "kind": "list-item", "locator": "body:L80-L80", "preview": "- `.shared::cta` identifies a barrier in the current CTA's shared memory.", "sha256": "cb736ed3b82018b4af9ff8a71c06b39150af84e6300cd74341b00374ce604cd7"}, {"id": "u034", "kind": "list-item", "locator": "body:L81-L81", "preview": "- `.shared::cluster` identifies a cluster-shared address, such as a mapped address for another CTA's shared memory.", "sha256": "090eeb2f11c80401d9b399e755d561f418360d9f38fa7a8327e255e58ee245fe"}, {"id": "u035", "kind": "list-item", "locator": "body:L82-L82", "preview": "- `.cta` or `.cluster` on an operation specifies its synchronization scope.", "sha256": "1865209300b9b76af9ab274882f77faf299eb648588d07c2d2d23ea10172511a"}, {"id": "u036", "kind": "prose", "locator": "body:L84-L84", "preview": "Only `arrive`, `expect_tx`, and `complete_tx` support an mbarrier address in `.shared::cluster`. Other mbarrier operations, including initialization and waits, target a CTA-shared object. A cluster can initialize an owner CTA's object, map ", "sha256": "979150fd2755438fb6b2960fbfc8b1b3a6008d8b833e6e9e2368d68f72f98663"}, {"id": "u037", "kind": "list-item", "locator": "body:L88-L88", "preview": "- Initialize the object before any other mbarrier operation.", "sha256": "5a0505577b6d44496231fcf69d503700ad594baa46ed4caca261a31e4ae6fff8"}, {"id": "u038", "kind": "list-item", "locator": "body:L89-L89", "preview": "- Match all arrivals and byte-valued complete-tx operations to the initialized counts.", "sha256": "215bf504fc87a3066decf20c5d7b854138df7c926508dd83e33184698daa0f0d"}, {"id": "u039", "kind": "list-item", "locator": "body:L90-L90", "preview": "- Keep phase or opaque state separately for each pipeline stage.", "sha256": "9e11f202036db2380e4c8b118a24287c1b99ecdb55d99b096401f940d36ab928"}, {"id": "u040", "kind": "list-item", "locator": "body:L91-L91", "preview": "- Retry `try_wait` until it reports completion.", "sha256": "6ed11f86197d07208c541ec7e852358105d81227c2d119a210c9faac8b158b05"}, {"id": "u041", "kind": "list-item", "locator": "body:L92-L92", "preview": "- Use the required acquire/release semantics for data visibility.", "sha256": "7ad077d086fcdf1658e08a255ac255fb1bcaa232c537a86f920dbdce1f42130a"}, {"id": "u042", "kind": "list-item", "locator": "body:L93-L93", "preview": "- Distinguish TMA complete-tx from tcgen05 arrive-on completion.", "sha256": "9a4c97d8692c9ed6e03314c310463412be4d5b55a180624c83e2c0c70eb1f8b6"}, {"id": "u043", "kind": "list-item", "locator": "body:L94-L94", "preview": "- Respect the supported combinations of object state space and operation scope.", "sha256": "dd65f3da0041c597a64d3f6dcf024b5636d8dc1c4dbf5fc06b0d07075eedca5b"}, {"id": "u044", "kind": "list-item", "locator": "body:L95-L95", "preview": "- Invalidate the object before repurposing its storage.", "sha256": "a90e4657917b4d85f0f24721c561d0f0372ae63de4b3772aa96a1ad70a5b3cff"}, {"id": "u045", "kind": "list-item", "locator": "body:L99-L99", "preview": "- [PTX ISA 9.0: mbarrier object and lifecycle](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier)", "sha256": "440c7134e3238c710417e1ec64af241a2957ba1cd0e9ac30ba99e5ea63262e6a"}, {"id": "u046", "kind": "list-item", "locator": "body:L100-L100", "preview": "- [PTX ISA 9.0: mbarrier waits](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait)", "sha256": "545e66c84f010d0d57ac169190a90198a16ed31f5e14c5b327907a3110cb04b8"}, {"id": "u047", "kind": "list-item", "locator": "body:L101-L101", "preview": "- [PTX ISA 9.0: TMA tensor copies](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor)", "sha256": "daf224131f7f98d73e758e640b317c7c441ec221b87331b1d98fcaebc71e1c40"}, {"id": "u048", "kind": "list-item", "locator": "body:L102-L102", "preview": "- [TMA](tma.md)", "sha256": "d31942a8af5df5dde77f7f955775cf0a1e7539926207438cd5560491f1574f5d"}, {"id": "u049", "kind": "list-item", "locator": "body:L103-L103", "preview": "- [Warp specialization](../techniques/warp-specialization.md)", "sha256": "94df377cf6d384f1a13d7bab49fd429a6059089d72d3ec45bc1306ee3bba3549"}], "confidence_claimed": "verified", "headings": ["Scope and history", "Object state and phase completion", "Arrival, transaction, and wait operations", "Phase and parity in a stage ring", "TMA completion accounting", "tcgen05 completion accounting", "CTA and cluster distinctions", "Correctness checklist", "References"], "id": "hw-mbarrier", "path": "wiki/hardware/mbarrier.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/prs/cutlass/PR-2139.md", "revision": "ca4fdbea", "url": "https://github.com/NVIDIA/cutlass/pull/2139"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-ptx-isa-sm100", "pr-cutlass-2139"], "title": "mbarrier (Memory Barrier Primitives)", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "b574ab0127a35863b7ec1a2eb952aa93fe86bc145085a497ad64b9c3bc93a46a", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "NVFP4 is a quantization recipe, not just another name for its 4-bit payload. It reconstructs a value as:", "sha256": "227c70eceb96556659b2b5f6cf54e400e6245360f15905a84705c09b89a48113"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "`x_hat_i = e2m1_i * s_block * s_global`", "sha256": "794f7462514befdb8ac4a28522080782143495f8fd3157b5bea8c46d09934c6d"}, {"id": "u003", "kind": "prose", "locator": "body:L9-L9", "preview": "The three components are:", "sha256": "890b0b87819a86372f7ddcee49a11fd621e9a3a29c35ac7b8d1d2c2ff00d5c3e"}, {"id": "u004", "kind": "list-item", "locator": "body:L11-L11", "preview": "- **E2M1 payload:** one sign bit, two exponent bits, and one mantissa bit. Its numeric values are 0, +/-0.5, +/-1, +/-1.5, +/-2, +/-3, +/-4, and +/-6. Two E2M1 encodings fit in one byte.", "sha256": "c7e17ce8c55c39e0403be620d6ad757c8fa414b998dddfbdae211367d3c095e8"}, {"id": "u005", "kind": "list-item", "locator": "body:L12-L12", "preview": "- **Local scale:** one E4M3 scale shared by 16 consecutive payload values. PTX names the corresponding unsigned scale element type `.ue4m3`; its most-significant storage bit is zero.", "sha256": "5413cc4adc0cc112cd51d22ac07e5fcaae3f1e245164a937f4ba9333ca907655"}, {"id": "u006", "kind": "list-item", "locator": "body:L13-L13", "preview": "- **Global scale:** one FP32 scale per tensor. This second level lets the block scales use their range effectively.", "sha256": "12ca2c5eaaec7645bc831438892492d0c78328874b654e1ab71c39af7a429539"}, {"id": "u007", "kind": "prose", "locator": "body:L15-L15", "preview": "Transformer Engine 2.13 also defines a 2D scaling mode for weights in which a scale covers a 16-by-16 block. The 1D mode groups 16 consecutive values.", "sha256": "7288001414a30fe333b049e16ae69a4d1522ec4dc11bcaa88d3dabe2c03ba5b9"}, {"id": "u008", "kind": "prose", "locator": "body:L19-L19", "preview": "The recipes make different scale tradeoffs:", "sha256": "c9981094018252198ffe5ef16f090c60cb5483b13ea9d3f69e44906b9fd07b71"}, {"id": "u009", "kind": "table-row", "locator": "body:L23-L23", "preview": "| Property | NVFP4 | MXFP4 | | Payload | E2M1 | E2M1 |", "sha256": "a519973746ef18833e34e940839dffaf10f468db0b157561b380739125ce7fe4"}, {"id": "u010", "kind": "table-row", "locator": "body:L24-L24", "preview": "| Property | NVFP4 | MXFP4 | | Local group in the standard 1D recipe | 16 values | 32 values |", "sha256": "079a628d1c073e9d4c716174b141262c2e1f1fcedc21caca1a3f3693ac5c462b"}, {"id": "u011", "kind": "table-row", "locator": "body:L25-L25", "preview": "| Property | NVFP4 | MXFP4 | | Local scale | E4M3 (`.ue4m3` in PTX) | UE8M0 |", "sha256": "eb805ec4a81f255a4a58972a4e406037092fb53afe554d6d542b2851c232d419"}, {"id": "u012", "kind": "table-row", "locator": "body:L26-L26", "preview": "| Property | NVFP4 | MXFP4 | | Scale values | Fractional values are available | Powers of two |", "sha256": "64d0f1c0890f5f697993223f04daa88924b00162be84608850df6ea6fa8e4e93"}, {"id": "u013", "kind": "table-row", "locator": "body:L27-L27", "preview": "| Property | NVFP4 | MXFP4 | | Additional recipe scale | Per-tensor FP32 | Not part of the MXFP4 microscaling format |", "sha256": "660006f85f6632d68dcdeecff2dbe5a683dd5ab53e15f3d3c0eb5ea536906ea9"}, {"id": "u014", "kind": "prose", "locator": "body:L29-L29", "preview": "Finer groups and fractional local scales give NVFP4 more scaling freedom, at the cost of more scale metadata. They do not guarantee strictly lower error for every tensor, scale-selection algorithm, or error metric. Measure accuracy with the", "sha256": "18f8ae6f474472f1c2ea4ff9a3e2b597c1c36943095c02353db7eb28e098ed64"}, {"id": "u015", "kind": "prose", "locator": "body:L33-L33", "preview": "In PTX ISA 9.0, the relevant `tcgen05.mma` combinations for E2M1 inputs include:", "sha256": "4c6c2bfd4f02969ff8470c02bf64bbc802e646de47c9b11f4df253c5df3501fc"}, {"id": "u016", "kind": "table-row", "locator": "body:L37-L37", "preview": "| Instruction qualifiers | Block | Local scale | Scale-vector qualifier | | `.kind::mxf4.block_scale.block32` | 32 | `.ue8m0` | `.scale_vec::2X` |", "sha256": "c8d1878c1a0cadb7d3ca358add3c678ab21399a44da2e3d25f4c36ceebde2082"}, {"id": "u017", "kind": "table-row", "locator": "body:L38-L38", "preview": "| Instruction qualifiers | Block | Local scale | Scale-vector qualifier | | `.kind::mxf4nvf4.block_scale.block16` | 16 | `.ue4m3` | `.scale_vec::4X` |", "sha256": "4bd9801c764ad925b43b088977e249e1499ac6e62b0ef485dd544412fe0874f3"}, {"id": "u018", "kind": "prose", "locator": "body:L40-L40", "preview": "The `.kind::mxf4nvf4` family also admits documented UE8M0 modes, so the kind name by itself does not select the NVFP4 recipe. Use the complete block and scale-vector qualifiers and follow the type-combination tables. These forms target arch", "sha256": "c48d3a2f58dfe90849b6ee8abacd39ebf0f30cc5c6835517ffe7997d411dd227"}, {"id": "u019", "kind": "prose", "locator": "body:L42-L42", "preview": "PTX ISA 9.0 provides the packed conversion form:", "sha256": "0895c7933390c018b85b8fbb5f7a635b182476c0daf63b31053aa71a507bba62"}, {"id": "u020", "kind": "code", "locator": "body:L44-L46", "preview": "```ptx cvt.rn.f16x2.e2m1x2 d, a; ```", "sha256": "b23b367d52866372d72acbf6316fbedf194a07a40533912e462aa9f5d2989ef1"}, {"id": "u021", "kind": "prose", "locator": "body:L48-L48", "preview": "Here `a` is a byte-sized packed pair and `d` is a 32-bit `f16x2` result. PTX also permits `mov.b32` to unpack a 32-bit scalar into four byte-sized vector destinations when the declarations satisfy its type rules. Neither syntax establishes ", "sha256": "ebaaf257c26310449c24b83cf719888507d42b14702ca8571b844efa027b0539"}, {"id": "u022", "kind": "prose", "locator": "body:L52-L52", "preview": "There is no architecture-independent \"4x versus Hopper\" result for these instructions. Hopper has no native FP4 tensor-core path, so any comparison depends on the emulation or higher-precision baseline as well as GPU SKU, clocks, matrix sha", "sha256": "b45cdd32e45e94aae2f29d34b8bb9b28b6552f456b68f0f10720fb64169535b4"}, {"id": "u023", "kind": "list-item", "locator": "body:L56-L56", "preview": "- [Transformer Engine 2.13: NVFP4](https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html)", "sha256": "1ea38bb80e14795104811d1c91bf01eb96b18d0da47a64fc1f42f3ecf2ca933d"}, {"id": "u024", "kind": "list-item", "locator": "body:L57-L57", "preview": "- [PTX ISA 9.0: `tcgen05.mma`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma)", "sha256": "6ffc658c6eca84cb0a0b9011dd5975398802fb382416b751faf4868e97da0dc2"}, {"id": "u025", "kind": "list-item", "locator": "body:L58-L58", "preview": "- [PTX ISA 9.0: `cvt`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt)", "sha256": "64807f0b841d1fd5a38186fcf40e8f14fc61c912cd4e76a0710ba4ec427255fd"}, {"id": "u026", "kind": "list-item", "locator": "body:L59-L59", "preview": "- [Fine-grained quantization](../techniques/fine-grained-quantization.md)", "sha256": "9a377188c50b366c56091d8ff769167bc2ad34ee4821cf9d3aef73e824912f24"}, {"id": "u027", "kind": "list-item", "locator": "body:L60-L60", "preview": "- [NVFP4 GEMM](../kernels/nvfp4-gemm.md)", "sha256": "afe5e93ec48a77859c8b1186b661f64956be6135e45e31d60ccadce597bab2b0"}, {"id": "u028", "kind": "list-item", "locator": "body:L61-L61", "preview": "- [NVFP4 GEMV](../kernels/nvfp4-gemv.md)", "sha256": "d0e8ffc886b46a7311e65cd74d494d992955690fc698cf037a73a56513df887f"}], "confidence_claimed": "verified", "headings": ["Format and recipe", "NVFP4 and MXFP4", "PTX block-scaled MMA", "Performance claims", "References"], "id": "hw-nvfp4", "path": "wiki/hardware/nvfp4.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-transformer-engine-2.13-nvfp4.md", "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}], "risk_flags": ["code", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-transformer-engine-2.13-nvfp4", "doc-ptx-isa-sm100"], "title": "NVFP4 and Block-Scaled Narrow Precision", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "4c2bc154856d1dcfd15bb4f6a12ae30d4dadc52ead1d4e5994b4da45e031cd98", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Programmatic Dependent Launch (PDL) lets a secondary grid in the same CUDA stream become eligible to start before its prerequisite primary grid completes. It is available on compute capability 9.0 and later, including Hopper and Blackwell.", "sha256": "b2a6132457b1395799e2baeb71d8b78a2aefada9ebcdb45692fe7c25aaeb6823"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "PDL creates an **opportunity** for overlap; it does not guarantee that the grids execute concurrently. The useful overlap is normally the primary's work after its launch trigger and the secondary's independent preamble before its dependency", "sha256": "3fadb352705867bcecdab210311e7fef978fb6c4d548428135a83c2cc9a4a59e"}, {"id": "u003", "kind": "prose", "locator": "body:L9-L9", "preview": "The CUDA protocol has three required roles:", "sha256": "f06a64f5123e88dbd07114031cac5afabaeaa52abc8ecae0037804ec2d46ba9a"}, {"id": "u004", "kind": "list-item", "locator": "body:L11-L11", "preview": "1. Every primary CTA either executes `cudaTriggerProgrammaticLaunchCompletion()` or exits. After all CTAs satisfy that condition, the driver may schedule the secondary grid.", "sha256": "cdebb33e8456cbe33475d6ccabe75f697ff9a33db49579b708d27a888866ee78"}, {"id": "u005", "kind": "list-item", "locator": "body:L12-L12", "preview": "2. The host launches the secondary in the same stream with `cudaLaunchAttributeProgrammaticStreamSerialization` and `programmaticStreamSerializationAllowed = 1`.", "sha256": "5fe3d12bfbd54ebbeac70c19b7d88175202f6c7dee699f8bda30383167d2a2df"}, {"id": "u006", "kind": "list-item", "locator": "body:L13-L13", "preview": "3. Every secondary thread waits with `cudaGridDependencySynchronize()` before it consumes prerequisite results. The wait completes after the prerequisite grids finish and their memory operations are visible.", "sha256": "097e01d65d04ea53bb13e7e51ab176c852e4ade1679b09d0361bc5db4ab61651"}, {"id": "u007", "kind": "prose", "locator": "body:L15-L15", "preview": "If the primary does not explicitly trigger, its CTAs implicitly satisfy the trigger only as they exit. Omitting the explicit trigger is correct but removes the intended primary-tail overlap.", "sha256": "bfc1bc44195a8cde7f45936b64ed7d6aa9339ddddfada2ad1b2116386640a488"}, {"id": "u008", "kind": "prose", "locator": "body:L19-L19", "preview": "This minimal skeleton shows where each operation belongs; real kernels put independent and dependent work at the indicated points and add normal error checking:", "sha256": "3370f3719697733fed41083390007ea5145cd8c1b8282d838bd01adfb718bf3b"}, {"id": "u009", "kind": "code", "locator": "body:L21-L53", "preview": "```cuda #include __global__ void primary() { // Produce everything needed before the secondary may be launched. if (threadIdx.x == 0) { cudaTriggerProgrammaticLaunchCompletion(); } // Primary tail that does not change data ", "sha256": "eead129a867adbd5fb1df86742d091900d3f862173e7beba21c3690a3860a7e3"}, {"id": "u010", "kind": "prose", "locator": "body:L55-L55", "preview": "Do not place `cudaGridDependencySynchronize()` in the primary: it is the secondary-side wait, not the launch trigger. Do not replace the wait with an ordinary memory fence. PDL's wait supplies both prerequisite-grid completion and visibilit", "sha256": "e79291426c2535ad3f17bf2a11bf664cbe62f5ccc989b134980cb4688c2d116f"}, {"id": "u011", "kind": "prose", "locator": "body:L59-L59", "preview": "CUDA lowers the two device roles to Grid Dependency Control instructions:", "sha256": "b250259e26671cdcac831c563720d242a666b0ed7a9b6afd3bc3d07a4a29075e"}, {"id": "u012", "kind": "code", "locator": "body:L61-L68", "preview": "```ptx // Primary CTA: makes designated dependent grids eligible after every CTA // has issued this instruction or completed. griddepcontrol.launch_dependents; // Secondary thread: waits for in-flight prerequisite grids and visibility. grid", "sha256": "9a67c56925d56525f39e506610fa57ffe5f3d4cb35db94ae4b010c90bc37cc98"}, {"id": "u013", "kind": "prose", "locator": "body:L70-L70", "preview": "`griddepcontrol` was introduced in PTX ISA 7.8 and requires `sm_90` or newer. Repeating `launch_dependents` within one CTA has no additional effect after that CTA's first invocation. If a prerequisite uses `launch_dependents`, its dependent", "sha256": "11123906a586c5505184a7cdc1d00fc51809ba90d7022a7722448cdeb72d9f02"}, {"id": "u014", "kind": "prose", "locator": "body:L74-L74", "preview": "Blackwell does not make arbitrary back-to-back launches overlap automatically. CUDA applications still opt the secondary launch into programmatic stream serialization and implement the device-side trigger/wait protocol.", "sha256": "ccfc3e2c2f99820ecb72aa491631c59b39d47de9f75b8a957e277d6a99d7fa86"}, {"id": "u015", "kind": "prose", "locator": "body:L76-L76", "preview": "CUTLASS has a separate build choice. In CUTLASS 4.5.0, the CMake option `CUTLASS_ENABLE_GDC_FOR_SM100` defaults to `ON`, while the SM90 option is opt-in. That default only enables eligible CUTLASS code to emit its GDC wrappers; it is not a ", "sha256": "8eb6cc4790bdeac39a848471ff2ce472fad33e190c16c073942ece6b3446f6d8"}, {"id": "u016", "kind": "prose", "locator": "body:L80-L80", "preview": "PDL can help only when all of these conditions hold:", "sha256": "b2f3b22d9295e43ef95becd759c3f639e71bc43ce4f72542271211cd62079f54"}, {"id": "u017", "kind": "list-item", "locator": "body:L82-L82", "preview": "- the secondary has enough independent preamble to overlap;", "sha256": "eba59af65aaa7d730d01e0cc16bbd138f86c11888c7eb6c6b3f24a3c18abb67e"}, {"id": "u018", "kind": "list-item", "locator": "body:L83-L83", "preview": "- the primary has useful tail work after every CTA reaches the trigger;", "sha256": "275a77efa3bd46865530e8b4d92dbdd17cadc14417e31d68a8f1164ce5f46db4"}, {"id": "u019", "kind": "list-item", "locator": "body:L84-L84", "preview": "- the two grids have enough simultaneous resource headroom; and", "sha256": "f7874709a96e779af80196a86caff96361b4b30864ef2d3989d1c0e3352a02b4"}, {"id": "u020", "kind": "list-item", "locator": "body:L85-L85", "preview": "- the saved launch/serialization time exceeds the protocol and occupancy costs.", "sha256": "8b99cc7e62635157ccc3a04c9e04da4db13139ca6e15882d68b73abe87cbf385"}, {"id": "u021", "kind": "prose", "locator": "body:L87-L87", "preview": "Small-kernel chains, GEMM/epilogue sequences, and pipeline-parallel stages are candidates, not guaranteed wins. Profile an explicit non-PDL baseline and record GPU, clocks, launch shapes, stream/graph configuration, input sizes, warmup, rep", "sha256": "3d11e833065d2f0681ff30760c6053df3d5cab7fae71318b61e568b66057ea7c"}, {"id": "u022", "kind": "list-item", "locator": "body:L91-L91", "preview": "- [CUDA 13.0.2: Programmatic Dependent Launch](https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#programmatic-dependent-launch-and-synchronization)", "sha256": "d95502e4bc1f33df6ee25c6794b4a39e4012106894161a3ec102f5c04a9c45c2"}, {"id": "u023", "kind": "list-item", "locator": "body:L92-L92", "preview": "- [PTX ISA 9.0: `griddepcontrol`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-griddepcontrol)", "sha256": "59689f46851737c4064c9836e6c4570e18b5a470e70da05895569b59e8c5352f"}, {"id": "u024", "kind": "list-item", "locator": "body:L93-L93", "preview": "- [CUTLASS 4.5.0: dependent kernel launch](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/dependent_kernel_launch.md)", "sha256": "f1fbe66725cda7570cb0f38eceab85beae9d511d843e6badf86e7e749c693871"}, {"id": "u025", "kind": "list-item", "locator": "body:L94-L94", "preview": "- [Persistent kernels](../techniques/persistent-kernels.md)", "sha256": "ec9585a9787c7d4334a41d77c6ecbeee6575e90c64bb37f1f2e8c8b8d64b041d"}, {"id": "u026", "kind": "list-item", "locator": "body:L95-L95", "preview": "- [Cluster Launch Control](clc.md)", "sha256": "7c35bead8298662eaa2f80624f180adf2e122e6ac111dcf05aa0f5ac9a5653b1"}], "confidence_claimed": "verified", "headings": ["Contract", "CUDA role skeleton", "PTX mapping", "CUDA support versus CUTLASS defaults", "When to use it", "References"], "id": "hw-pdl-gdc", "path": "wiki/hardware/pdl-gdc.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-cuda-13.md", "url": "https://developer.nvidia.com/blog/whats-new-and-important-in-cuda-toolkit-13-0/"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/prs/cutlass/PR-2161.md", "revision": "06e560d9", "url": "https://github.com/NVIDIA/cutlass/pull/2161"}], "risk_flags": ["code", "ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-cuda-13", "doc-ptx-isa-sm100", "pr-cutlass-2161"], "title": "Programmatic Dependent Launch / Grid Dependency Control", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "4dcc5ab666a88a95ba179a141027d5f8cb80ccbe6bae13bed1e745e2c2bd7b61", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "`tcgen05.mma` is NVIDIA PTX's fifth-generation TensorCore matrix-multiply-accumulate family. PTX ISA 9.0 supports its architecture-specific forms on `sm_100a`; CUTLASS uses the `UMMA` namespace for its SM100 wrappers. NVIDIA's pinned PTX an", "sha256": "5a30d3e62baf8f9e258d9543f2e7952f16a33644f324b4ab76b8652c9aeaa047"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "The programming-model shift from Hopper WGMMA is precise:", "sha256": "79a30a17b7a41df9858ff9cea9a7ad07ccc0402bba6e70e49699d048f8c82d22"}, {"id": "u003", "kind": "table-row", "locator": "body:L11-L11", "preview": "| Property | WGMMA | tcgen05.mma | | Issue granularity | Warpgroup | One thread for `cta_group::1` or `cta_group::2` |", "sha256": "cb2dd4481890789881169b854b62ab6c892eeba718691b0c2512592192d5fd27"}, {"id": "u004", "kind": "table-row", "locator": "body:L12-L12", "preview": "| Property | WGMMA | tcgen05.mma | | D accumulator | Per-thread registers | Tensor Memory (TMEM) |", "sha256": "5e4f9ace6cdb6311c5b99b47b2ea7a6663212f2ca7c69100d7af30d5428e8c12"}, {"id": "u005", "kind": "table-row", "locator": "body:L13-L13", "preview": "| Property | WGMMA | tcgen05.mma | | A location | Register or SMEM forms, depending on instruction | SMEM descriptor or TMEM address |", "sha256": "147c40c5a19ea02949386e003d7f89e2b7fabcba715fb9517e664f30260fe665"}, {"id": "u006", "kind": "table-row", "locator": "body:L14-L14", "preview": "| Property | WGMMA | tcgen05.mma | | B location | SMEM descriptor | SMEM descriptor |", "sha256": "9df1b553933b5276fc6098da41c4f388e190439f6722b2af979ebcf994110cbc"}, {"id": "u007", "kind": "table-row", "locator": "body:L15-L15", "preview": "| Property | WGMMA | tcgen05.mma | | Completion | WGMMA commit/wait groups | `tcgen05.commit` plus an mbarrier wait |", "sha256": "edd69cb02b0effe4c4d1e3b287cc8974c4e49b9883132a111ee81ac18c8d1385"}, {"id": "u008", "kind": "prose", "locator": "body:L17-L17", "preview": "Single-thread issue reduces the number of issuing threads, but it does not remove asynchronous completion, operand-lifetime, or inter-thread ordering requirements.", "sha256": "dd431728d3ea5a55526e6920039601fb733dcb5a2d1275c0c32c0f18e5ec3c6a"}, {"id": "u009", "kind": "prose", "locator": "body:L21-L21", "preview": "PTX ISA 9.0 divides dense `tcgen05.mma` into these grammar groups:", "sha256": "59dabf6dfb4c85e155ad4045ec8784b5e4a8ce7c2728efcaca602880f1aee4f9"}, {"id": "u010", "kind": "table-row", "locator": "body:L25-L25", "preview": "| Grammar group | Kind qualifiers | | Floating point, without block scaling | `f16`, `tf32`, `f8f6f4` |", "sha256": "c6edd6d05f9116fa0934c8bf39ebf349a48aa0fe6ceb5d6069603b5b370dd82a"}, {"id": "u011", "kind": "table-row", "locator": "body:L26-L26", "preview": "| Grammar group | Kind qualifiers | | Floating point, with block scaling | `mxf8f6f4`, `mxf4`, `mxf4nvf4` |", "sha256": "7524aa5d74cb23cae509d1f62790cd9b4097cfd670f1f71d7a566cf01044109d"}, {"id": "u012", "kind": "table-row", "locator": "body:L27-L27", "preview": "| Grammar group | Kind qualifiers | | Integer | `i8` |", "sha256": "d00dc8f6a441056e180aa0b7abc129f19832cb9ff64963c701f818e5dab0025e"}, {"id": "u013", "kind": "prose", "locator": "body:L29-L29", "preview": "Block-scaled forms add `.block_scale` and accept separate TMEM addresses for A and B scale factors. `f8f6f4` is not the block-scaled MX kind, and `mxf8` is not a dense kind token in this grammar.", "sha256": "6301afa139350f62dc9bf75d85dfa67de713cc20a501dfc475ed622d4a73d6b4"}, {"id": "u014", "kind": "prose", "locator": "body:L31-L31", "preview": "M and N are encoded in the instruction descriptor (`idesc`) and are constrained by kind, CTA group, layouts, and target ISA. Consequently, names such as m128n256k16 and m256n256k16 are useful maximum-shape examples for F16/BF16 configuratio", "sha256": "2c714ca4e2add6030e2ffd3a982d42e1928c97e656e998af2d554e29bb7c7052"}, {"id": "u015", "kind": "prose", "locator": "body:L35-L35", "preview": "The unscaled floating-point grammar includes a disable-output-lane vector before the accumulation predicate:", "sha256": "0c3d6d381d9ddfc6470a447bf5c7209a114dfe79cca59536d5329b306df0caeb"}, {"id": "u016", "kind": "code", "locator": "body:L37-L41", "preview": "```ptx tcgen05.mma.cta_group.kind [d-tmem], a-desc, b-desc, idesc, { disable-output-lane }, enable-input-d {, scale-input-d}; ```", "sha256": "2ed61a81ad137911dae0bc5b5d1f4cf410d79d32deb51059f1da36185bc35527"}, {"id": "u017", "kind": "prose", "locator": "body:L43-L43", "preview": "For `cta_group::1`, the lane-disable vector contains four 32-bit values; for `cta_group::2`, it contains eight. Setting `enable-input-d` false computes `D = A*B`; setting it true computes `D = A*B+D`. The optional `scale-input-d` is limited", "sha256": "b9fb63cf6d552bb548afef70443d9c4a0b4bcc76a419792cfffbcd851b60e038"}, {"id": "u018", "kind": "prose", "locator": "body:L45-L45", "preview": "The block-scaled grammar is structurally different:", "sha256": "c89976d470fdda264de2b4c13671d036c8da1bc84c424adf03ecfdb9ec956213"}, {"id": "u019", "kind": "code", "locator": "body:L47-L51", "preview": "```ptx tcgen05.mma.cta_group.kind.block_scale.scale_vectorsize [d-tmem], a-desc, b-desc, idesc, [scale-A-tmem], [scale-B-tmem], enable-input-d; ```", "sha256": "8e585731217607fbd7145d665ac7790e9f8610c4630fec9b63f1e613bc675ae8"}, {"id": "u020", "kind": "prose", "locator": "body:L53-L53", "preview": "These are normative grammar excerpts, not complete inline-assembly functions: real code must also build valid descriptors, allocate TMEM, preserve operand lifetimes, and implement completion.", "sha256": "c026a8a36bf3c27d1e949b295720a13a93ac89b99bf0e5d297e1efa0f274677d"}, {"id": "u021", "kind": "prose", "locator": "body:L57-L57", "preview": "`tcgen05.mma` is asynchronous. Two mechanisms serve different purposes:", "sha256": "0eb63c199bd2648a1a838b2d3a47ac38561ae64dd5a5aa4a389176a4bba0a4b9"}, {"id": "u022", "kind": "list-item", "locator": "body:L59-L59", "preview": "1. `tcgen05.commit.cta_group::N.mbarrier::arrive::one.b64` makes an mbarrier track completion of prior asynchronous tcgen05 operations issued by the executing thread. Waiting on that mbarrier observes completion.", "sha256": "b94843121138c19fe4f9c6b5f2e4180325b4b07209b2b16d53801684eb26feab"}, {"id": "u023", "kind": "list-item", "locator": "body:L60-L60", "preview": "2. `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` order tcgen05 operations across an execution-ordering handoff and constrain code motion. A fence is not a completion wait.", "sha256": "c0f4ab169700e7114ac3cfaf535bfcc96235d55a67b70ab445eeae606728c5c2"}, {"id": "u024", "kind": "prose", "locator": "body:L62-L62", "preview": "A cross-thread result handoff therefore needs both the applicable completion protocol and the applicable fence/execution-ordering protocol. Likewise, a pipelined mainloop must not release or overwrite an SMEM stage until the asynchronous MM", "sha256": "adf70d7f3800a733444d846a2e3c015cbd24273cb2574a36d0f5998aff0d4b87"}, {"id": "u025", "kind": "prose", "locator": "body:L66-L66", "preview": "The tcgen05 shared-memory descriptor is a 64-bit runtime value. PTX ISA 9.0 assigns fields for the encoded base address, leading dimension, stride dimension, fixed bits, base offset, leading-dimension mode, and a three-bit swizzle mode.", "sha256": "cd4b8ef944a646278af6e6a6ed009a7641447384c06e392a47867c99b4214e9c"}, {"id": "u026", "kind": "prose", "locator": "body:L68-L68", "preview": "Valid swizzle encodings include no swizzle, 128-byte, 64-byte, and 32-byte layouts (plus a 128-byte/32-byte-atomic mode). Values 3, 5, and 7 are invalid; ordinary 128-byte swizzling uses value 2. A layout must satisfy the addressing and ali", "sha256": "ffd078c072d1f53a4004b9a5bfd9958461011d22405ec4ef1724cc5f8fa0a464"}, {"id": "u027", "kind": "prose", "locator": "body:L72-L72", "preview": "For comparison, CUTLASS's pinned Hopper wrapper for m64n256k16 with FP32 accumulation declares 128 accumulator registers per participating thread. tcgen05 keeps D in TMEM during the MMA sequence, which can free GPR capacity for data movemen", "sha256": "aea6db1f85746818e4767b9d406b4e771290c0e462d3cabf587264e302613aef"}, {"id": "u028", "kind": "prose", "locator": "body:L76-L76", "preview": "Gau Nernst reports the following results for M=N=K=4096 on a Modal B200, using PyTorch 2.9.1 with CUDA 13. Values are measurements for that setup, not architecture-wide guarantees.", "sha256": "13f1217bf7f22fcc79830356b02e702206bb4a00151c16924639cf6d2bb21a99"}, {"id": "u029", "kind": "table-row", "locator": "body:L80-L80", "preview": "| Tutorial version | Reported TFLOP/s | Approx. cuBLAS share | | cuBLAS | 1506.74 | 100% |", "sha256": "ac004f48b31dd6f7024996d3995ded82f823d87f3eff7413f55ff3e6acc116b4"}, {"id": "u030", "kind": "table-row", "locator": "body:L81-L81", "preview": "| Tutorial version | Reported TFLOP/s | Approx. cuBLAS share | | v1a: basic tcgen05 + 2D 16B TMA | 254.62 | 17% |", "sha256": "81e39acb68544a7b7f6402f743738b5e37e51cd2b474f9cbfdb331689fdfd3f4"}, {"id": "u031", "kind": "table-row", "locator": "body:L82-L82", "preview": "| Tutorial version | Reported TFLOP/s | Approx. cuBLAS share | | v1b: 3D 16B TMA | 252.81 | 17% |", "sha256": "26b2647c4773366a5f1d00f84486e9190c5ce010b5958463cc44d7d8335336b6"}, {"id": "u032", "kind": "table-row", "locator": "body:L83-L83", "preview": "| Tutorial version | Reported TFLOP/s | Approx. cuBLAS share | | v2a: 2D 128B TMA | 681.20 | 45% |", "sha256": "589a9a37b7cf4601f48e74eed00ff7f502e5754d3003b46ad18b82471359782c"}, {"id": "u033", "kind": "table-row", "locator": "body:L84-L84", "preview": "| Tutorial version | Reported TFLOP/s | Approx. cuBLAS share | | v2b: 3D 128B TMA | 695.43 | 46% |", "sha256": "e70b55975fd80df16e31a7ae4e4699c6dc4a32c1f1579136407c89a4385d9c75"}, {"id": "u034", "kind": "table-row", "locator": "body:L85-L85", "preview": "| Tutorial version | Reported TFLOP/s | Approx. cuBLAS share | | v3: pipelining | 939.61 | 62% |", "sha256": "cb276d4db269baa1b1d091ffd693f81215a14e11e87bca3bbb7b4e737d267150"}, {"id": "u035", "kind": "table-row", "locator": "body:L86-L86", "preview": "| Tutorial version | Reported TFLOP/s | Approx. cuBLAS share | | v4: warp specialization | 1208.83 | 80% |", "sha256": "85b80f84546f9b39fe250172f16144299c9d09f56364b578a12423b78e221f9f"}, {"id": "u036", "kind": "table-row", "locator": "body:L87-L87", "preview": "| Tutorial version | Reported TFLOP/s | Approx. cuBLAS share | | v5: 2-SM MMA | 1302.29 | 86% |", "sha256": "7b18e80a7aa94f919322812f64ce759b9517b58ca7efba7933732a0f9aa0f531"}, {"id": "u037", "kind": "table-row", "locator": "body:L88-L88", "preview": "| Tutorial version | Reported TFLOP/s | Approx. cuBLAS share | | v6: persistent, static scheduling | 1475.93 | 98% |", "sha256": "dc244caa4f60fc59b1d9e62d010731e5e119b04331695c37ec54a5d5660f60d2"}, {"id": "u038", "kind": "prose", "locator": "body:L90-L90", "preview": "The v6 result did not use Cluster Launch Control; the author lists CLC and threadblock swizzling as unimplemented follow-up ideas.", "sha256": "58b7618727532d91a2095babb6482b7a126310da1fbdfaf4bab8776a2cefabc9"}], "confidence_claimed": "verified", "headings": ["Overview", "Kinds and shapes", "Operand grammar", "Completion and inter-thread ordering", "Shared-memory descriptors", "Register pressure and specialization", "Source-reported performance progression"], "id": "hw-tcgen05-mma", "path": "wiki/hardware/tcgen05-mma.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/prs/cutlass/PR-2139.md", "revision": "ca4fdbea", "url": "https://github.com/NVIDIA/cutlass/pull/2139"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-ptx-isa-sm100", "pr-cutlass-2139", "blog-tcgen05-tutorial"], "title": "tcgen05.mma \u2014 Blackwell MMA Instruction", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "1eb48a33cd77ce277ac4783d357f5784e487533159bbe155fc942f404f3655f1", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "TMA is the descriptor-driven `cp.async.bulk.tensor` facility introduced for Hopper (`sm_90`) and retained on Blackwell. One thread can issue a non-blocking rank-1 through rank-5 tensor copy while the hardware performs multidimensional addre", "sha256": "942ee8eaa2c8a1a7a630eb77b351844e980ef1e36f0e549a8f0fa35689f227df"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "The principal copy directions and completion mechanisms are:", "sha256": "cd258dbf1c80e8b3bf04db3d6601ed09723c72ff7ce612a7202994b8369b02d5"}, {"id": "u003", "kind": "table-row", "locator": "body:L11-L11", "preview": "| Direction | Destination | Optional cluster behavior | Completion | | Global to shared | Issuing CTA or a CTA in its cluster | One masked instruction can multicast to selected CTAs | mbarrier `complete_tx` in bytes |", "sha256": "0d062928bfbbdbd16737f4478e912c51a16020e856647600f0273112a61ce977"}, {"id": "u004", "kind": "table-row", "locator": "body:L12-L12", "preview": "| Direction | Destination | Optional cluster behavior | Completion | | Shared to global | Issuing CTA's shared memory to a tensor map | Scatter modes are available on supported Blackwell targets | Bulk async group: issue, commit, wait |", "sha256": "be8df0a0b92e6d46dc931918bc193cda48ba5a02cdbf7655cef986ad552fc1cb"}, {"id": "u005", "kind": "prose", "locator": "body:L14-L14", "preview": "TMA copies the datatype represented by the tensor map. It does not provide a general FP32-to/from-FP16 or BF16 conversion step. For a tiled global-to-shared load, out-of-bounds elements are filled according to the supported tensor-map polic", "sha256": "2b7f99033261e23d1b85e977330495009b7ae63c585efc4d6b0d5e5ac665f399"}, {"id": "u006", "kind": "prose", "locator": "body:L18-L18", "preview": "A tensor map is opaque and is accessed through the tensor-map proxy. `cuTensorMapEncodeTiled` describes:", "sha256": "ccc2dedb38f5f9072f35ff9cdf790aa1ba6305d726a0341b5184080c60ad76a5"}, {"id": "u007", "kind": "list-item", "locator": "body:L20-L20", "preview": "- datatype and global base address;", "sha256": "bdedfc644a6b74912840d80daf76b5edf4ab3f9fffcdd594efbc530d95a2b81c"}, {"id": "u008", "kind": "list-item", "locator": "body:L21-L21", "preview": "- rank, global dimensions, and byte strides;", "sha256": "9c9a24117ca440fe4a6d6090ce809391c3f8b5ee855a73097640a3f05af0de4f"}, {"id": "u009", "kind": "list-item", "locator": "body:L22-L22", "preview": "- traversal box dimensions and element strides;", "sha256": "9adc19c95d2386a5160b3a3f1551bf6fc65d1e41afd43e0c4d386527ba55af1e"}, {"id": "u010", "kind": "list-item", "locator": "body:L23-L23", "preview": "- interleave and shared-memory swizzle;", "sha256": "bea5f5d23bd9392029b749d89c1d641bb241b07d4eb68e4795c34a9983dc6cef"}, {"id": "u011", "kind": "list-item", "locator": "body:L24-L24", "preview": "- L2-promotion hint; and", "sha256": "5ae4da2e9d5fe5615b8ee15e1a1c8bfe9e24f86837721d4bcdf10944b9f12249"}, {"id": "u012", "kind": "list-item", "locator": "body:L25-L25", "preview": "- out-of-bounds fill policy.", "sha256": "1f9f8ca6a1112db5590a406b647932dd377abb4cc1768c9d00b783ecf7bef430"}, {"id": "u013", "kind": "prose", "locator": "body:L27-L27", "preview": "For the ordinary non-interleaved tiled path in CUDA Driver API 13.0.97, important constraints include:", "sha256": "b22bce9849c6562e29ba2932641636dbf6542a64c546d304859ca5ebd323e652"}, {"id": "u014", "kind": "list-item", "locator": "body:L29-L29", "preview": "- the `CUtensorMap` output object is 64-byte aligned;", "sha256": "44c7c6b9b737364654f0d6a08279bcf6ca9fc22bed1ecb2e4669b358694fe623"}, {"id": "u015", "kind": "list-item", "locator": "body:L30-L30", "preview": "- the global base and byte strides satisfy the documented alignment rules, commonly at least 16 bytes for the basic types/path;", "sha256": "863f766000dc344890dbe17dfcd48c835fab68f68e6ffd64feff051a0afb8964"}, {"id": "u016", "kind": "list-item", "locator": "body:L31-L31", "preview": "- every `boxDim` entry is from 1 through 256 elements;", "sha256": "6729dd96d450f4c92833472caa18c04a6c97afd6d7cbcb25f034cd7c865f1a68"}, {"id": "u017", "kind": "list-item", "locator": "body:L32-L32", "preview": "- `boxDim[0] * element_size` is a multiple of 16 bytes;", "sha256": "bc3d083f157b2d2c6efd45522d8368a362504e504b01c619ee5769749b33af2f"}, {"id": "u018", "kind": "list-item", "locator": "body:L33-L33", "preview": "- every element stride is from 1 through 8; and", "sha256": "e98ad79a06a4e814bd46f56699ba0b5b07c84af7f99ff5821f354a58a5a7c4ad"}, {"id": "u019", "kind": "list-item", "locator": "body:L34-L34", "preview": "- when swizzling is enabled, the inner box in bytes does not exceed the selected swizzle span.", "sha256": "8d4b2cee8563f7311901d071d457877dd9b4675b8b0a577f25959b1ceb4428b8"}, {"id": "u020", "kind": "prose", "locator": "body:L36-L36", "preview": "Datatype, interleave, sub-byte, and architecture-specific modes add further restrictions. Always check the encoder's `CUresult`; never use the output after an encoding failure.", "sha256": "1dc42273ad70a1619c9b071188b6c21a33296af5ea280e944e372650c47d5e1f"}, {"id": "u021", "kind": "prose", "locator": "body:L38-L38", "preview": "Host encoding is common, but it is not the only Blackwell path. CUDA also documents device-side tensor-map construction and modification on Blackwell. A map modified through the generic proxy must be published to the tensor-map proxy with t", "sha256": "32c5584ca3441e4fc303d1aacb9ebde7f3bd75753c43adcac3cf8eda7bfd543e"}, {"id": "u022", "kind": "prose", "locator": "body:L42-L42", "preview": "PTX ISA 9.0 defines this representative 2D CTA-local form:", "sha256": "b0293941d4875c7b42475bd148a898910ccfba21295ea5e319fcd8483dfdf15a"}, {"id": "u023", "kind": "code", "locator": "body:L44-L47", "preview": "```ptx cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes [dst_smem], [tensor_map, {x, y}], [full_barrier]; ```", "sha256": "c63f1e52e13d7bed644ceffc9967df48dac5830d6c1c436b73250f38dd90fdb3"}, {"id": "u024", "kind": "prose", "locator": "body:L49-L49", "preview": "The instruction is non-blocking. A common one-producer phase with one or more loads uses this accounting:", "sha256": "33c59a474aaa779a2840508f993381a25aeaa81e88b896d3e729b91f4863ca8d"}, {"id": "u025", "kind": "list-item", "locator": "body:L51-L51", "preview": "1. Initialize and publish the stage's mbarrier with the intended pending-arrival count.", "sha256": "b97850eccf2dd9133f354210b468324757ccddd7363c04b11905826bd5231e19"}, {"id": "u026", "kind": "list-item", "locator": "body:L52-L52", "preview": "2. The producer performs one `mbarrier.arrive.expect_tx` for its software arrival and the sum of bytes that all loads in this phase will complete.", "sha256": "7536187d8435ca9f9d70ea4ba02e02a996a19a2b67a19e2524ae42383ccaeb3f"}, {"id": "u027", "kind": "list-item", "locator": "body:L53-L53", "preview": "3. Issue the TMA loads against that barrier.", "sha256": "387aec934b017aca25a3dde3f8509850f2406a8e8fb178ae11305df52df38430"}, {"id": "u028", "kind": "list-item", "locator": "body:L54-L54", "preview": "4. Each completed load performs `complete_tx` for its copied byte count; it does **not** perform another arrival.", "sha256": "41c113b3b786060e187ca94075b7e8b4fbf2bdd2a611cab9393d01671a812f20"}, {"id": "u029", "kind": "list-item", "locator": "body:L55-L55", "preview": "5. Consumers wait with the correct state token or per-stage parity and acquire semantics before reading the destination.", "sha256": "d844abea9463ba69fbba5103258780bd95d6b474e5bb6eb11991a6f48d940c29"}, {"id": "u030", "kind": "prose", "locator": "body:L57-L57", "preview": "The phase completes only after pending arrivals and tx-count are both zero. If a design performs multiple software arrivals instead, its initialized count must match them exactly. See [mbarrier](mbarrier.md) for lifecycle, phase, and memory", "sha256": "84689cde185105f289c1e125c553e90e62cf23b7efc0219e0dc269aec0b3cf30"}, {"id": "u031", "kind": "prose", "locator": "body:L61-L61", "preview": "A tensor store uses bulk-group completion rather than the load's mbarrier protocol:", "sha256": "7644e09174ad1ec06520bbb2067d83e2dd1964ae8a1b874ce3b0f3e6a5d9cb90"}, {"id": "u032", "kind": "code", "locator": "body:L63-L68", "preview": "```ptx cp.async.bulk.tensor.2d.global.shared::cta.bulk_group [tensor_map, {x, y}], [src_smem]; cp.async.bulk.commit_group; cp.async.bulk.wait_group 0; ```", "sha256": "9b7414acc80138f372a237ff15efd42d0f0f188d9931500d80b109804411152e"}, {"id": "u033", "kind": "prose", "locator": "body:L70-L70", "preview": "The wait may be delayed to overlap independent work. It must occur before the issuing thread reuses the source shared memory or before code that requires store completion. `commit_group` creates the group; it is not itself a completion wait", "sha256": "bfc107e49734a610b28fe2cde672a1a0645e272320b570dc8c44f3b9596dab3f"}, {"id": "u034", "kind": "prose", "locator": "body:L74-L74", "preview": "The cluster form copies a global tile to the same shared-memory offset in every CTA selected by a 16-bit mask:", "sha256": "94c03e8dfa1adfc21eee3339add8ff57b0febd458ea6dca96c153d7bf652b35c"}, {"id": "u035", "kind": "code", "locator": "body:L76-L79", "preview": "```ptx cp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes.multicast::cluster [dst_smem], [tensor_map, {x, y}], [full_barrier], cta_mask; ```", "sha256": "13740ee71ef9eb029a8fd56b0cf01d83934433dff96a8f1fc6ab0f1869ebdbc7"}, {"id": "u036", "kind": "prose", "locator": "body:L81-L81", "preview": "For `cta_group::1` (the default), the completion signal is also multicast to the same barrier offset in every selected destination CTA. A correct design therefore:", "sha256": "bd2bc4bda81f3d523a915f9119c19a704ad18216bea214eb3116c3ca820ce4a6"}, {"id": "u037", "kind": "list-item", "locator": "body:L83-L83", "preview": "- launches an explicit cluster and keeps every destination CTA's shared memory alive;", "sha256": "99220fe82c4329fd5ff1a570bae96e8b1a304953a628a7ed13aaff2e932dc681"}, {"id": "u038", "kind": "list-item", "locator": "body:L84-L84", "preview": "- initializes corresponding destination barriers before the elected issuer can start;", "sha256": "aaa47a1f531b09283cee30555df707bd08f0c1271b4e15442f61a81f8a97df53"}, {"id": "u039", "kind": "list-item", "locator": "body:L85-L85", "preview": "- uses one cluster-wide elected issuer, not one `threadIdx.x == 0` issuer in every CTA;", "sha256": "002e972562c21a4dd64b4792b70d2d300e01fe73754728937a5e32b8a654441d"}, {"id": "u040", "kind": "list-item", "locator": "body:L86-L86", "preview": "- includes only valid destination CTA ranks in the mask; and", "sha256": "8dfb07761fafb9091a650ad2883aa5dd47b19a85a523291ef4aba5dc9e978dd7"}, {"id": "u041", "kind": "list-item", "locator": "body:L87-L87", "preview": "- has every destination wait on its own corresponding barrier phase before consuming the tile.", "sha256": "6032817a977cb65d2e38311dba90fce95f1ba507d414939a1be5383ac925cd88"}, {"id": "u042", "kind": "prose", "locator": "body:L89-L89", "preview": "For GEMM tiles with the same N range and different M ranges, the CTAs use different A tiles but the same B tile, so multicast can avoid duplicate logical B-load requests. It does not promise an exact `cluster_size` reduction in measured DRA", "sha256": "c6b1d2169446becddd2f513ea356236dd3a2dc964bd99d7b0c2582db59b2f24b"}, {"id": "u043", "kind": "prose", "locator": "body:L93-L93", "preview": "TMA supports no swizzle and multiple swizzled shared-memory layouts, including 32B, 64B, and 128B spans plus newer variants for selected types. Swizzling rearranges chunks across shared-memory banks. The consumer must address the matching l", "sha256": "853a7a8ba52912cbc1b920f6f71d0610c40733f923573857286b071994a68364"}, {"id": "u044", "kind": "prose", "locator": "body:L95-L95", "preview": "There is no universal rule that every Blackwell or `tcgen05.mma` input uses 128B swizzling. The TMA tensor-map swizzle, destination base alignment, leading dimension, and the tcgen05 shared-memory descriptor must describe the **same** legal", "sha256": "2b4b0d6d11682df0f0130a149296140f1a53aa39f3ec97618d9acc1302d4715a"}, {"id": "u045", "kind": "prose", "locator": "body:L97-L97", "preview": "A matched swizzle can reduce or remove bank conflicts for a particular access pattern. It does not make every possible consumer access conflict-free.", "sha256": "a0b6b60319b33d60b4c5d3fe6af97125681fcc47c3b23349ca8c394854345795"}, {"id": "u046", "kind": "prose", "locator": "body:L101-L101", "preview": "A common Blackwell GEMM data path is:", "sha256": "13268f43c424737e1cb3cec39868151eea4e3d9bdd671f7b87dafcb603717e4f"}, {"id": "u047", "kind": "prose", "locator": "body:L103-L103", "preview": "`GMEM -> TMA -> SMEM -> tcgen05.mma -> TMEM -> tcgen05.ld -> registers -> output store`", "sha256": "23afe141ca999584d588aa68850c067da012eed1030c92b9aef1b932e349a614"}, {"id": "u048", "kind": "prose", "locator": "body:L105-L105", "preview": "Each reusable pipeline stage needs two independent ownership transitions:", "sha256": "bfa1891a762540b1465d3767155e2928b8b026e0f40f012c7bcd41afd38b6392"}, {"id": "u049", "kind": "list-item", "locator": "body:L107-L107", "preview": "- **full:** TMA has finished producing the shared-memory operands, so the MMA consumer may read them; and", "sha256": "11d6052606c7b84bebee01e3c45bd2589c56b249d41d348edfd2aafd566d180b"}, {"id": "u050", "kind": "list-item", "locator": "body:L108-L108", "preview": "- **empty:** asynchronous MMA has stopped reading those operands, so the TMA producer may overwrite the stage.", "sha256": "eef3d18fd694d3e4b240e591ddccf87b99764dd71f1f72c9e14f2fc1fceab149"}, {"id": "u051", "kind": "prose", "locator": "body:L110-L110", "preview": "Track full and empty state per stage. Account transaction bytes once, track each barrier's own phase, and do not equate CTA synchronization or a tcgen05 fence with async completion. A complete CUTLASS pipeline is safer evidence than a short", "sha256": "248d9132d4467e85eeff75f7f120ab16a0679a718df2d20a806c822af36f2027"}, {"id": "u052", "kind": "prose", "locator": "body:L112-L112", "preview": "Choose tile rank, swizzle, multicast, issue cadence, and stage count from the actual access pattern and resource budget. More distinct stages consume more shared memory; there is no universal optimum of three to five stages. Profile the tar", "sha256": "8a0567c9674b127300435eb31acb41d3d1aead134c3bae110c17c6d5aec4d12f"}, {"id": "u053", "kind": "list-item", "locator": "body:L116-L116", "preview": "- [CUDA 13.0.2 Programming Guide: TMA](https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#asynchronous-data-copies-using-the-tensor-memory-accelerator-tma)", "sha256": "6e4dfa9b3ded302ca056bb6a498de0ab3aaafeef829d5103c81e632f5c1bf14a"}, {"id": "u054", "kind": "list-item", "locator": "body:L117-L117", "preview": "- [CUDA Driver API 13.0.97: tensor-map management](https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html)", "sha256": "4e29813dfddcaddb177f40e4fe9480deb10b9dbe097b724c947ccd91c3658c8f"}, {"id": "u055", "kind": "list-item", "locator": "body:L118-L118", "preview": "- [PTX ISA 9.0: `cp.async.bulk.tensor`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor)", "sha256": "ef3133ce097f33a254f3b19981e6d76ec88f33e964ee32edb03b3aecb627cc9f"}, {"id": "u056", "kind": "list-item", "locator": "body:L119-L119", "preview": "- [CUTLASS 4.5.0: complete CuTe DSL TMA tutorial](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_tma/tma_v0.py)", "sha256": "fffa82636381165a465946081584ded3dbcb39253ce114ed6ca90eec84090ffb"}, {"id": "u057", "kind": "list-item", "locator": "body:L120-L120", "preview": "- [mbarrier](mbarrier.md)", "sha256": "3cdff3a1605be2d98c176340b72d54b96f0575bb3dad4b249d5599d1bb94c369"}, {"id": "u058", "kind": "list-item", "locator": "body:L121-L121", "preview": "- [tcgen05 MMA](tcgen05-mma.md)", "sha256": "8fe04c67b24dcafdf0ff00c51238617219c223e45065aa447c5f56c2203a5e2c"}], "confidence_claimed": "verified", "headings": ["Scope", "Tensor maps", "Global-to-shared load", "Shared-to-global store", "Cluster multicast", "Swizzle and tcgen05", "Pipeline invariants", "References"], "id": "hw-tma", "path": "wiki/hardware/tma.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-cuda-13-0-2-tma.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#asynchronous-data-copies-using-the-tensor-memory-accelerator-tma"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/prs/cutlass/PR-2139.md", "revision": "ca4fdbea", "url": "https://github.com/NVIDIA/cutlass/pull/2139"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-cuda-13-0-2-tma", "doc-ptx-isa-sm100", "pr-cutlass-2139"], "title": "Tensor Memory Accelerator (TMA)", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "f6c0135c31b4600deecf3ee1dd972746a40bb9b3c0968b96e5ceb231a35ce816", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Tensor Memory is an addressable, on-chip memory introduced for fifth-generation Tensor Core operations on Blackwell. Each SM has 256 KiB, organized as 128 lanes by 512 columns of 32-bit cells. `tcgen05.mma` writes its D accumulator to TMEM;", "sha256": "0df97e28b422381bddbf8db848a9efad8674027194393755004ac345a3631094"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "Moving D out of general-purpose registers changes the resource balance relative to Hopper WGMMA. For example, CUTLASS 4.5.0's SM90 m64n256 FP32 WGMMA wrapper exposes 128 accumulator registers **per participating thread**. TMEM avoids that p", "sha256": "5afe174e3db5a624f39253beb7a40cc63c9a8843d4163b11310c197f1d69656d"}, {"id": "u003", "kind": "prose", "locator": "body:L11-L11", "preview": "A TMEM address (`taddr`) is a 32-bit value:", "sha256": "7f5f41f2bc49fc82a013135b23d8ba697ad04901e624c995259e5cd0b907a1fd"}, {"id": "u004", "kind": "table-row", "locator": "body:L15-L15", "preview": "| Bits | Meaning | | 31:16 | TMEM lane |", "sha256": "cee3131449614a54c7ffe5f19dc9eda0dc07a58cd86780f43db79beca031e933"}, {"id": "u005", "kind": "table-row", "locator": "body:L16-L16", "preview": "| Bits | Meaning | | 15:0 | TMEM column |", "sha256": "b4bc3f1c30523c2010b9f5cf1722da25b5a63186c9b27201a1b8fd8f74339b7b"}, {"id": "u006", "kind": "prose", "locator": "body:L18-L18", "preview": "The lane is a TMEM data-path lane, not storage independently owned by a CUDA thread. `tcgen05.ld` and `tcgen05.st` are warp-collective. Under their four-warp access model, the warps cover these lane chunks:", "sha256": "28d9f3ee58888f5aab3a8dc64f93d17d32ada56fc2f6f5576d452efc503707ba"}, {"id": "u007", "kind": "table-row", "locator": "body:L22-L22", "preview": "| Participating warp | TMEM lanes | | 0 | 0-31 |", "sha256": "e53446be920ecbd70393e486f6e7948091941cbd67eda6ccdcc393d536c365ca"}, {"id": "u008", "kind": "table-row", "locator": "body:L23-L23", "preview": "| Participating warp | TMEM lanes | | 1 | 32-63 |", "sha256": "9294efa2eb1f0b53aa8b8b5f566b209581a3e00a8c2826bd43088a98e1c7d9f4"}, {"id": "u009", "kind": "table-row", "locator": "body:L24-L24", "preview": "| Participating warp | TMEM lanes | | 2 | 64-95 |", "sha256": "1a42b20cd839a4160147e7de90d07ed64dbcd8526d6b3b42f93ab0078a4e4fb3"}, {"id": "u010", "kind": "table-row", "locator": "body:L25-L25", "preview": "| Participating warp | TMEM lanes | | 3 | 96-127 |", "sha256": "210a7e92aa2b4eb70e52583c755b97ad0b7721515062249ee913f2ce4bdffa83"}, {"id": "u011", "kind": "prose", "locator": "body:L27-L27", "preview": "The chosen load/store shape determines how values from that lane-column region map to each thread's register vector. Use the PTX shape-specific layout tables rather than treating a logical MxN accumulator as a universal row-major array owne", "sha256": "2ea3747ea0fa2ad571db92917836e6431266f2c4a6f150e3cf559d78cad6fa63"}, {"id": "u012", "kind": "prose", "locator": "body:L31-L31", "preview": "TMEM has an explicit software-managed lifetime:", "sha256": "619075113787e60845488a825201856ee59878b761427028e7103ca64764ccdf"}, {"id": "u013", "kind": "list-item", "locator": "body:L33-L33", "preview": "1. Reserve shared memory for the 32-bit allocation result.", "sha256": "347425a839a5fa6eac81d4797ca4de56bb02bbb2eda77b4c47ae23820a94a58c"}, {"id": "u014", "kind": "list-item", "locator": "body:L34-L34", "preview": "2. Execute `tcgen05.alloc` collectively from one warp for `cta_group::1`, or from two warps\u2014one per paired CTA\u2014for `cta_group::2`.", "sha256": "b8e4f5ea142d2258f24cb03b6fea0cce727beb4c7d601d6b5af1bbc2b3abb396"}, {"id": "u015", "kind": "list-item", "locator": "body:L35-L35", "preview": "3. Synchronize consumers as required, then read the returned base `taddr` from shared memory.", "sha256": "75df93469c3e8eb30d652d8f99f7f56af6674bf7704e0a36db7c6d7f92987a9e"}, {"id": "u016", "kind": "list-item", "locator": "body:L36-L36", "preview": "4. Use the allocation for MMA, copy, load, or store operations with the same CTA-group mode.", "sha256": "bb1d97a60d0710ea6e20c4bf6951496a283f5fbf8979b4c474b8e45df508f13c"}, {"id": "u017", "kind": "list-item", "locator": "body:L37-L37", "preview": "5. Execute `tcgen05.dealloc` with the corresponding collective issue pattern.", "sha256": "2dc6706ed0536b771b2cde4f4f7fa2fac7f90658f9b4da940bfe0d24161a4c5b"}, {"id": "u018", "kind": "list-item", "locator": "body:L38-L38", "preview": "6. Deallocate every TMEM allocation before kernel exit.", "sha256": "d1938b5098e7b266d47da39e56ea7dab3975b115b4fc0ba3424c3ddf451ac0cd"}, {"id": "u019", "kind": "prose", "locator": "body:L40-L40", "preview": "The allocation operand counts columns. Legal allocation sizes are powers of two from 32 through 512 columns. Allocation can block until the requested TMEM is available; the ISA does not define folklore outcomes such as silent corruption for", "sha256": "f91ee27bb11a3513503232c20e34d34d2a15adafe81323e309113b223563d26a"}, {"id": "u020", "kind": "prose", "locator": "body:L44-L44", "preview": "All allocations share the SM's 512-column capacity. Some useful whole-allocation budgets are:", "sha256": "b4ee5bf23171df60c790b148f9fd3cedc5a56e4433af63ad5d475858f984e163"}, {"id": "u021", "kind": "table-row", "locator": "body:L48-L48", "preview": "| Allocation per stage | Maximum stages if TMEM has no other use | Columns left | | 128 columns | 4 | 0 |", "sha256": "b072c857bef9e2ac11466c7551dd2f9becdd59b4f4673742468fcc36d7d11ebb"}, {"id": "u022", "kind": "table-row", "locator": "body:L49-L49", "preview": "| Allocation per stage | Maximum stages if TMEM has no other use | Columns left | | 256 columns | 2 | 0 |", "sha256": "05a58dad733419c4156a0f06f2154ef8615e852d12525633dfd35418f52a89f3"}, {"id": "u023", "kind": "table-row", "locator": "body:L50-L50", "preview": "| Allocation per stage | Maximum stages if TMEM has no other use | Columns left | | 512 columns | 1 | 0 |", "sha256": "a940200162e3b907c6f8faab934616f673d5f4a5aff941ec4376c3638fdc0f1d"}, {"id": "u024", "kind": "prose", "locator": "body:L52-L52", "preview": "A logical need of 192 columns cannot be requested directly: reserve 256 columns, or suballocate that logical region within another legal power-of-two reservation. Two 256-column accumulator stages consume the entire capacity, so scale-facto", "sha256": "0e50cd966770f36695b446d06e378b73e842ebf0b42c509fa17fea8336fed88c"}, {"id": "u025", "kind": "prose", "locator": "body:L56-L56", "preview": "The relevant mechanisms are distinct:", "sha256": "fc1c5304326b584de63c1c1519794b7e024c06d00ee87b0186768ab05a4c4a4c"}, {"id": "u026", "kind": "list-item", "locator": "body:L58-L58", "preview": "- `tcgen05.commit` attaches completion of prior asynchronous MMA operations to an mbarrier. Wait on that barrier before consuming their results.", "sha256": "1af1167372a70c2f450e4a843b993d8129391f12b1cc1919b4c294825e811e89"}, {"id": "u027", "kind": "list-item", "locator": "body:L59-L59", "preview": "- `tcgen05.wait::ld` and `tcgen05.wait::st` are the completion mechanisms for the corresponding asynchronous TMEM load/store operations.", "sha256": "386820efb8194c8b401e0b7a6a6743e16f245b9eaa180ec26a92080c99e8b34d"}, {"id": "u028", "kind": "list-item", "locator": "body:L60-L60", "preview": "- `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` order tcgen05 operations around a documented execution-ordering handoff. A fence is not an MMA completion wait.", "sha256": "0943cb17d0d11a25cf92c877aadb5c2a5ab386de0329b9466b6895733f0cfa65"}, {"id": "u029", "kind": "list-item", "locator": "body:L61-L61", "preview": "- Producer buffers must remain live until the asynchronous operation that reads them has completed according to its instruction contract.", "sha256": "feb718367dc639669dec4406eae2afadd79207dafd794557fa719986099b240d"}, {"id": "u030", "kind": "prose", "locator": "body:L63-L63", "preview": "CTA synchronization alone does not replace these completion operations.", "sha256": "c9f735b433318be62dcfdac6bd798094eddd61f8b412e8617e21d7ecdb82b89c"}, {"id": "u031", "kind": "prose", "locator": "body:L67-L67", "preview": "`tcgen05.ld` transfers TMEM into registers, and `tcgen05.st` transfers registers into TMEM. Their documented shapes include `.16x64b`, `.16x128b`, `.16x256b`, `.32x32b`, and `.16x32bx2`, with supported repetition qualifiers. There is no sca", "sha256": "00c8ef6aa00691fb742087960b4de34db91b6645935dd58d1869ccc2f1f05775"}, {"id": "u032", "kind": "prose", "locator": "body:L69-L69", "preview": "`tcgen05.cp` performs shaped shared-memory-to-TMEM copies. For example, PTX ISA 9.0 defines this exact form:", "sha256": "9ae8051754c858f6c0e847374738cfc3809f3de06adf5ad59088acdcd4389e5c"}, {"id": "u033", "kind": "code", "locator": "body:L71-L73", "preview": "```ptx tcgen05.cp.cta_group::1.128x256b [taddr], sdesc; ```", "sha256": "31c9dbcff5abb4891536fbccdba342e817b50f428cafdf35cebc02bf54bd30f1"}, {"id": "u034", "kind": "prose", "locator": "body:L75-L75", "preview": "Here `taddr` is the TMEM destination and `sdesc` is the 64-bit shared-memory matrix descriptor. Follow the instruction's asynchronous ordering and completion rules before reusing the source or consuming the destination.", "sha256": "e068f39c91ad4a5696b7fc43d22115ac304f1155eeede678c6d93adc1160eec4"}, {"id": "u035", "kind": "prose", "locator": "body:L79-L79", "preview": "Multiple accumulator stages can overlap MMA production with draining a different, completed TMEM stage. The design is valid only when:", "sha256": "991654cf4559d3b7da1d51e83dcda7db8c0ab1803d8d84734223be63483ea452"}, {"id": "u036", "kind": "list-item", "locator": "body:L81-L81", "preview": "- the combined allocations and suballocations fit the 512-column budget;", "sha256": "17fd8af5790cb5d70bdc34a7f45f8874f12f30b681bea0944af76bc4f3a33768"}, {"id": "u037", "kind": "list-item", "locator": "body:L82-L82", "preview": "- producer and consumer roles use explicit pipeline barriers;", "sha256": "33a31fcf7dce9d220bd55852b4c572cf63e5fa7e983b37d2da40fef011abccd5"}, {"id": "u038", "kind": "list-item", "locator": "body:L83-L83", "preview": "- MMA completion is observed before a consumer drains a stage;", "sha256": "e7c3b8ae852fb631c85ccd5f45b861b4388ae3bdf8d483489d44d8f62fa939be"}, {"id": "u039", "kind": "list-item", "locator": "body:L84-L84", "preview": "- TMEM load completion is observed before register results are used; and", "sha256": "f2ed672764ffffbedc6bd0dabc492b18674a4866746bf6d07551955baee25d50"}, {"id": "u040", "kind": "list-item", "locator": "body:L85-L85", "preview": "- no stage is recycled while an asynchronous operation can still access it.", "sha256": "7da7a6de9e5d7952da1a50d38417f3cc335d39e018c3a67e33e0fc3cf595c69a"}, {"id": "u041", "kind": "prose", "locator": "body:L87-L87", "preview": "This is a pipeline design, not an automatic consequence of alternating two addresses. CUTLASS's version-pinned SM100 tutorials show complete producer/consumer implementations.", "sha256": "d4011413497e52d0dcae86b5ffa75a65ffb125c97d706d62a54ae44b73511e4c"}, {"id": "u042", "kind": "prose", "locator": "body:L91-L91", "preview": "CUTLASS 4.5.0 exposes TMEM through `cutlass.utils.TmemAllocator` and layout-aware CuTe tensors. The official tutorial sequence is:", "sha256": "e18540de26c1c50ab8326e2ae286d0d965df1b58986f5c5eae9076c610e4bebb"}, {"id": "u043", "kind": "list-item", "locator": "body:L93-L93", "preview": "1. create a `TmemAllocator` over shared storage for the allocation result;", "sha256": "a3edc461fce5b5784083dacacd0f1c101b43ca2a85cf79745fece0939240b903"}, {"id": "u044", "kind": "list-item", "locator": "body:L94-L94", "preview": "2. call `allocate(num_columns)` from the configured allocator warp;", "sha256": "292ed99bc0f058be44017f232311ac5c48a4aa2d2d479d9e6e1d01fc3675398c"}, {"id": "u045", "kind": "list-item", "locator": "body:L95-L95", "preview": "3. use `wait_for_alloc()` before other warps retrieve the address;", "sha256": "ce966bcfd9c03d8d2d5ac2362e6a8493e4fa73e2008dea59f9b3b83c8fd2dc52"}, {"id": "u046", "kind": "list-item", "locator": "body:L96-L96", "preview": "4. obtain a typed pointer with `retrieve_ptr(dtype)`;", "sha256": "07e29aa2cd0afb608284d84518f2dd6ca9e0992bce434000e1e795ba8f54c97d"}, {"id": "u047", "kind": "list-item", "locator": "body:L97-L97", "preview": "5. bind that pointer to the MMA accumulator layout with `cute.make_tensor`;", "sha256": "cffb932874a3bf23919dc46c78e281039af766ddac29dcd431edd27a8e57f4db"}, {"id": "u048", "kind": "list-item", "locator": "body:L98-L98", "preview": "6. drain it with `tcgen05` copy atoms and the pipeline's completion protocol; and", "sha256": "2c1a41548f8cadb56da832323b1246a164a9b72881e751bdc28165d0e836fd4b"}, {"id": "u049", "kind": "list-item", "locator": "body:L99-L99", "preview": "7. call `free(tmem_ptr)` before exit.", "sha256": "43514435699f5adeeb0344be0c63200aa778fa36ddef2b0aed2e70ee5579cb83"}, {"id": "u050", "kind": "prose", "locator": "body:L101-L101", "preview": "See the pinned [CUTLASS 4.5.0 FP16 GEMM tutorial](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py) and [TMEM allocator implementati", "sha256": "ed0773f61bf2f5159b5ba120c90bdf34465bfa942c6d6a4ea55d4fcbd90048ab"}, {"id": "u051", "kind": "list-item", "locator": "body:L105-L105", "preview": "- [PTX ISA 9.0: Tensor Memory](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory)", "sha256": "40744f485d8c843f17e8166199678d16332069d8bcd1fbd4f490d2703e20657a"}, {"id": "u052", "kind": "list-item", "locator": "body:L106-L106", "preview": "- [PTX ISA 9.0: `tcgen05.alloc`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc)", "sha256": "f4138a536dd8f10f27c3b7e99b22b3d68f162948a899bf3f883d7d4820269775"}, {"id": "u053", "kind": "list-item", "locator": "body:L107-L107", "preview": "- [PTX ISA 9.0: `tcgen05.ld`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld)", "sha256": "4ecd5d78052e70a7efc674ba9e98404c79471b2f6ff7054f46463a6fa4e2ec36"}, {"id": "u054", "kind": "list-item", "locator": "body:L108-L108", "preview": "- [PTX ISA 9.0: `tcgen05.commit`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit)", "sha256": "4b6757bbc80715870a3a7823c1b45358f3efbffa77b50cf04231cef404cd00f8"}, {"id": "u055", "kind": "list-item", "locator": "body:L109-L109", "preview": "- [CUTLASS 4.5.0 SM100 Python tutorial](https://github.com/NVIDIA/cutlass/tree/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm)", "sha256": "42eee452d016fa75699f23e63de919826e07c0ff194b091beae8c3b06db6ffa9"}], "confidence_claimed": "verified", "headings": ["What TMEM is", "Addressing and warp access", "Allocation lifecycle", "Column budgeting", "Completion and ordering", "Data movement", "Staging accumulators", "CUTLASS 4.5.0 Python DSL", "Primary references"], "id": "hw-tmem", "path": "wiki/hardware/tmem.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/prs/cutlass/PR-2139.md", "revision": "ca4fdbea", "url": "https://github.com/NVIDIA/cutlass/pull/2139"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-ptx-isa-sm100", "pr-cutlass-2139"], "title": "Tensor Memory (TMEM)", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "5e2f98b996033253da1252cec281af40f1f858fa39675a8296b17564863d76fe", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "DeepGEMM is DeepSeek's open-source tensor-core kernel library. This page describes the FP8 GEMM paths at commit [`891d57b4db1071624b5c8fa0d1e51cb317fa709f`](https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa70", "sha256": "edc884f2b15966309a2f894d85f5e0cc5e7f6e8be752415ddc8b5c870d4fcc8f"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "DeepGEMM contains multiple kernels and APIs; the local SM90 and SM100 files below are representative FP8 1D1D implementations, not the whole library.", "sha256": "07ce76e8768d979abddde871720e0182365b4bdfa27b3ca467b4c1c4d823efb9"}, {"id": "u003", "kind": "prose", "locator": "body:L11-L11", "preview": "The [DeepSeek-V3 Technical Report v2](https://arxiv.org/abs/2412.19437v2) defines the training scheme that motivates this path:", "sha256": "d5d25beeba794a452ddfdbfb7f2711ddecc946602125840ec83ddc44a8a70211"}, {"id": "u004", "kind": "list-item", "locator": "body:L13-L13", "preview": "- activations are grouped per token and per 128 channels, forming `1 x 128` tiles;", "sha256": "d7523bee1428dc93fed5ab553055f979f4a735bf9765274f1071846fe85f6b7a"}, {"id": "u005", "kind": "list-item", "locator": "body:L14-L14", "preview": "- weights are grouped per 128 input channels and 128 output channels, forming `128 x 128` blocks; and", "sha256": "62bb926d275734b391c879539ab2d755a9e8816843dfb6307f76326ddab70563"}, {"id": "u006", "kind": "list-item", "locator": "body:L15-L15", "preview": "- smaller groups let scales adapt more locally, which the paper says better accommodates outliers. This is a scoped accuracy motivation, not a guarantee that quantization error disappears.", "sha256": "ff206a53bb27abcfe054b691bd34313268d95364b3b5358dacc0b02d1a3eb51a"}, {"id": "u007", "kind": "prose", "locator": "body:L17-L17", "preview": "Scale representation is architecture-specific in the pinned DeepGEMM interface. SM90 consumes FP32 scale factors. SM100 consumes packed UE8M0 factors, four UE8M0 values per `torch.int`. The pinned SM90 1D1D kernel fixes `BLOCK_K == 128`; th", "sha256": "e6026ff271a6b450b3b13f3d8add4c8296916a6e5f637c409363b33b5975c990"}, {"id": "u008", "kind": "prose", "locator": "body:L21-L21", "preview": "The DeepSeek-V3 report characterizes H800 FP8 Tensor Core accumulation as retaining about 14 bits. Its `Nc=128` strategy accumulates 128 elements of the GEMM inner dimension\u2014four WGMMAs in the described configuration\u2014before moving the parti", "sha256": "c24f17362025a8f052e005acfbb16c91b9ec59729dff0831ba4ec2b0ff77d245"}, {"id": "u009", "kind": "prose", "locator": "body:L23-L23", "preview": "The pinned [`sm90_fp8_gemm_1d1d.cuh`](../../artifacts/kernels/deepgemm/full/sm90_fp8_gemm_1d1d.cuh) implements that structure directly:", "sha256": "28c625a2df66c12415c1fdbca2c390ce628c03257471e82b88294d81adb12a6e"}, {"id": "u010", "kind": "list-item", "locator": "body:L25-L25", "preview": "1. A math warp-group owns `float accum[...]` for WGMMA output and zero-initialized `float final_accum[...]` for promoted results.", "sha256": "017785087339586990befb86eed10f456ab3fdea6e55f20099e25f7e3aca49af"}, {"id": "u011", "kind": "list-item", "locator": "body:L26-L26", "preview": "2. For each 128-element K block, it reads the A and B FP32 factors from shared memory, issues `BLOCK_K / WGMMA::K` WGMMA operations, commits the group, and waits for completion.", "sha256": "16af9fe856a77026821209080a4bb5d64b1b710abe0540ebb08e107d52a0234a"}, {"id": "u012", "kind": "list-item", "locator": "body:L27-L27", "preview": "3. The `Promote with scales` loop multiplies each partial result by its A and B factors and adds it to `final_accum`.", "sha256": "4c32846882c303ba304d47e97688ddab5d2ff00cf6391d834a2a91bbf4d48ff2"}, {"id": "u013", "kind": "list-item", "locator": "body:L28-L28", "preview": "4. After all K blocks, the kernel stages the final FP32 values for the epilogue/store path.", "sha256": "f7783a1c6fb5faf02bc3b5e1de09015b518d2f5e4cdf3e20f55cca904a2b495d"}, {"id": "u014", "kind": "prose", "locator": "body:L30-L30", "preview": "The exact upstream file is the reproducible reference. It should not be replaced by a sketch using half accumulators or by deriving the promotion interval from WGMMA's output-N tile.", "sha256": "925a0225db7cd7132a69208d817eb1af5a27db511f4f65a66e43b6c105290cd5"}, {"id": "u015", "kind": "prose", "locator": "body:L34-L34", "preview": "The pinned [`sm100_fp8_gemm_1d1d.cuh`](../../artifacts/kernels/deepgemm/full/sm100_fp8_gemm_1d1d.cuh) uses a different data path:", "sha256": "32baacf395d6dca95c445def85a6d35ecd549ba78a9d23a33a0ac511c5b3c116"}, {"id": "u016", "kind": "list-item", "locator": "body:L36-L36", "preview": "1. It creates a block-scaled UMMA instruction descriptor with a `float` accumulator type and `cutlass::float_ue8m0_t` scale type.", "sha256": "e4d0d03fdc026620956fc648d7b9b500f2c77ae9a96e8c85c028f1fa8fa08f86"}, {"id": "u017", "kind": "list-item", "locator": "body:L37-L37", "preview": "2. TMA loads packed scale-factor data into shared memory. UTCCP copies selected scale blocks from shared memory into dedicated SFA and SFB columns in TMEM.", "sha256": "ea3a550a3695844dd1292293e91418bd8514f42d2b2da9c09981584452bfea44"}, {"id": "u018", "kind": "list-item", "locator": "body:L38-L38", "preview": "3. The elected issuing thread calls the SM100 UMMA wrapper with shared-memory operand descriptors, the TMEM accumulator column, a runtime instruction descriptor containing scale IDs, and the two TMEM scale addresses.", "sha256": "e405f66e1b53d88218e2e20a61440d777fcbc83e5db36be347d089df597d0f66"}, {"id": "u019", "kind": "list-item", "locator": "body:L39-L39", "preview": "4. The epilogue drains the TMEM accumulator after the kernel's full/empty barrier protocol says it is ready.", "sha256": "255120ab2662ebea058a8ececcabcbb648e337d16a860d70134dc24994ae88fa"}, {"id": "u020", "kind": "prose", "locator": "body:L41-L41", "preview": "This path does not contain the SM90 `final_accum` CUDA-core promotion loop. That difference does not justify a blanket claim that every tcgen05 accumulator mode or datatype has \"full FP32\" behavior; the accumulator type and instruction form", "sha256": "032e89c865af65e49fd21968c9ccf8e1074c5932b40d2e28b7444d03ed3cb6b8"}, {"id": "u021", "kind": "prose", "locator": "body:L45-L45", "preview": "The pinned interface distinguishes three workload arrangements rather than treating all of them as M-varying layouts:", "sha256": "929dc28b5052e9d9082a478f9145da75cf574dbe40e0fe8ce42d9881ea6fa20d"}, {"id": "u022", "kind": "table-row", "locator": "body:L49-L49", "preview": "| Interface family | Varying group dimension | Fixed dimensions | Work metadata | | M-grouped contiguous | M | N and K | Either a group index for each packed M row or a prefix-sum M layout, depending on the selected option |", "sha256": "f7573a4ed8c1905b6389177ee297f27b173966c755b256e452c45cb74823ae95"}, {"id": "u023", "kind": "table-row", "locator": "body:L50-L50", "preview": "| Interface family | Varying group dimension | Fixed dimensions | Work metadata | | M-grouped masked | Valid M within each `[G, M, K]` allocation | Maximum M, N, and K tensor extents | An integer `masked_m[G]` vector holding each group's va", "sha256": "be7ea9aa4dce872c95db03488e96c249b3c54d4d251596470d9ad9425f682ced"}, {"id": "u024", "kind": "table-row", "locator": "body:L51-L51", "preview": "| Interface family | Varying group dimension | Fixed dimensions | Work metadata | | K-grouped contiguous | K | M and N | Per-group K lengths plus their device tensor; used for weight-gradient-style grouped GEMM |", "sha256": "7310a12d3f4e4e0fda19fded69bf1350f734d99d6e2604faa989f64702635f24"}, {"id": "u025", "kind": "prose", "locator": "body:L53-L53", "preview": "Contiguous M grouping is intended for variable per-expert token counts in training forward or inference prefill. Masked M grouping keeps fixed allocations suitable for CUDA-graph decode while limiting work to each group's valid M. The pinne", "sha256": "70a8f3bbeae80bf8c0c0aac86aa2c0c3660c948dc1df5e3bc0cb1ae46c2dfff2"}, {"id": "u026", "kind": "prose", "locator": "body:L57-L57", "preview": "DeepGEMM generates and compiles kernel source at runtime. At the pinned commit, the compiler defaults to NVCC. Setting `DG_JIT_USE_NVRTC=1` selects the optional NVRTC path; the project warns that this may reduce performance for some cases.", "sha256": "f913b537fa51d19b4b9fcbe96f968fb465d7a606e1e7ec5579ce26905c4995d6"}, {"id": "u027", "kind": "prose", "locator": "body:L59-L59", "preview": "The cache key includes the kernel name, compiler signature, compiler flags, and generated source. A cache hit reuses the existing kernel runtime; a miss compiles a CUBIN in a temporary directory and then publishes the completed cache entry.", "sha256": "b35a525c4472aae9970a77f0af0207eb0b614a25db706f4facec3b0bfcae106e"}, {"id": "u028", "kind": "prose", "locator": "body:L63-L63", "preview": "For the pinned FP8 interface, SM90 supports NT only: A is non-transposed and B is supplied in the representation used for `A @ B.T`. SM100 exposes dense `fp8_gemm_{nt, nn, tn, tt}` variants. Its implementation propagates the selected operan", "sha256": "45eee831267e7730526a83b7bb4f986d8ff85c414f93e0cd755c424951e96ab3"}, {"id": "u029", "kind": "prose", "locator": "body:L67-L67", "preview": "The pinned README reports that DeepGEMM reached **up to 1550 TFLOPS on H800** in an April 2025 news item. It does not bind that peak to `M=N=K=4096`, state approximately 90% utilization, or preserve enough benchmark conditions for reproduct", "sha256": "fc2a0fd111e796b31a0c7e0947d44cc55e0b67eb0dda053f25f0d4dff9835a38"}, {"id": "u030", "kind": "list-item", "locator": "body:L71-L71", "preview": "- Input transposition, FP8 casting, and scale-layout preparation are separate from the optimized GEMM kernels; the project supplies utilities but warns they may be slower than fusing the work into producers.", "sha256": "13b031a9e940905a676b7e3e9b18cbd3d783a31dc4a622a7231e0c33147cdd46"}, {"id": "u031", "kind": "list-item", "locator": "body:L72-L72", "preview": "- M-grouped contiguous segments must satisfy the configured M/K alignment. Masked mode uses valid-length integers, while K-grouped mode has different shapes and architecture-specific layout variants.", "sha256": "a98c5f037a06bb31517448c4705d346c2b1164baca656c828855dfa1b9c179f4"}, {"id": "u032", "kind": "list-item", "locator": "body:L73-L73", "preview": "- JIT cache misses add compilation latency, and NVRTC is optional rather than the default.", "sha256": "baf5e541e3f2c2af10edc3b9a7ea55312df051da85ac772dfced929f7d872e91"}, {"id": "u033", "kind": "list-item", "locator": "body:L74-L74", "preview": "- Fine-grained scaling's accuracy and cost tradeoffs depend on quantization recipe, target architecture, scale preparation, fusion, and workload. There is no universal per-tensor-scaling penalty or single \"use only for outliers\" rule.", "sha256": "f9ee05e0618cef4fc4736191e6043939ccd7187cf5bb95ffcfe425f7633e66ad"}, {"id": "u034", "kind": "list-item", "locator": "body:L78-L78", "preview": "- [DeepGEMM README at commit `891d57b`](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md)", "sha256": "a8115b879e294785bbc27931facc99142bd71a23cf5bb9c6d0ff177a40b32e03"}, {"id": "u035", "kind": "list-item", "locator": "body:L79-L79", "preview": "- [DeepGEMM GEMM API at commit `891d57b`](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp)", "sha256": "6344a99161ed183b81d6f0677ac4d26de38cc9974fed0d20c38bf7432a9ca253"}, {"id": "u036", "kind": "list-item", "locator": "body:L80-L80", "preview": "- [DeepGEMM JIT compiler at commit `891d57b`](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/jit/compiler.hpp)", "sha256": "256e0bde5da762844f66e366cd65a297e4093cbaebe7b8369f2d20c4d87aa958"}, {"id": "u037", "kind": "list-item", "locator": "body:L81-L81", "preview": "- [DeepSeek-V3 Technical Report v2](https://arxiv.org/abs/2412.19437v2)", "sha256": "15d831659fae7c38fffb3b6766c08144ff0af11075d3445e73179f7691e7b4e5"}, {"id": "u038", "kind": "prose", "locator": "body:L85-L85", "preview": "Local verbatim upstream code lives in [`artifacts/kernels/deepgemm/full/`](../../artifacts/kernels/deepgemm/full/) and is pinned by [`PROVENANCE.yaml`](../../artifacts/kernels/deepgemm/full/PROVENANCE.yaml) to commit `891d57b4db1071624b5c8f", "sha256": "5fef9de55b64a522e42b6cfb7faad3d0a73df75e4c883e89431c4595916e569f"}, {"id": "u039", "kind": "prose", "locator": "body:L87-L87", "preview": "Query the page and its attached code with:", "sha256": "0da0b6e07247d8d8ebfa9751fbc20aba136debe8f0a07212223182e717bf84d1"}, {"id": "u040", "kind": "code", "locator": "body:L89-L91", "preview": "```bash python3 scripts/get_page.py kernel-deepgemm --include-code ```", "sha256": "a2051b59354d2536049b6650555806c0410635aa728921831a30c1e1855977b1"}, {"id": "u041", "kind": "prose", "locator": "body:L93-L93", "preview": "The following is a verbatim fragment of the SM90 promotion loop; the linked full file supplies its surrounding declarations and loop bounds:", "sha256": "806d5647140f861cdf5ae0117bbbf910630168e069a638198e6ff844f98594ed"}, {"id": "u042", "kind": "code", "locator": "body:L95-L102", "preview": "```cpp const float &scale_b_0 = scales_b[i].x; const float &scale_b_1 = scales_b[i].y; final_accum[i * 4 + 0] += scale_a_0 * scale_b_0 * accum[i * 4 + 0]; final_accum[i * 4 + 1] += scale_a_0 * scale_b_1 * accum[i * 4 + 1]; final_accum[i * 4", "sha256": "7d801ab9fe44c41c62fdf33ba40c8a04c0057328187ea3c19e0f056a5b0ecca0"}], "confidence_claimed": "source-reported", "headings": ["Verified Scope", "Fine-Grained Quantization", "SM90: WGMMA and CUDA-Core Promotion", "SM100: Native Block-Scaled UMMA", "Grouped GEMM Interfaces", "JIT Compilation", "Operand Layouts", "Performance Evidence", "Practical Boundaries", "Pinned Sources", "Full Reference Implementation"], "id": "kernel-deepgemm", "path": "wiki/kernels/deepgemm.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/deepgemm.md", "url": "https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f"}, {"path": "sources/prs/DeepGEMM/PR-304.md", "revision": "7f2a703e", "url": "https://github.com/deepseek-ai/DeepGEMM/pull/304"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-deepgemm", "pr-deepgemm-304"], "title": "DeepGEMM \u2014 FP8 GEMM with Fine-Grained Scaling", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "eb0beda6d21999eb5a9af51fbb7869f46f78dcb0a29c7d5df1fbde5a0b49b875", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "FlashAttention-4 is an attention algorithm and CuTe DSL implementation designed around Blackwell's asymmetric throughput: B200 tensor-core throughput grows much more than its special-function and shared-memory resources. This page separates", "sha256": "1e410d8f43116d7362371c8f774303c9f6c1f9cf76d3978e901efadfdc47845d"}, {"id": "u002", "kind": "list-item", "locator": "body:L7-L7", "preview": "- the [FA4 paper v1](https://arxiv.org/abs/2603.05451v1), which studies the SM100/B200 algorithm and reports the authors' measurements; and", "sha256": "ff06a9c75bcd69176044dff5a2e64357af716d6fc527b4b5a918cdf9a5cd9a05"}, {"id": "u003", "kind": "list-item", "locator": "body:L8-L8", "preview": "- the public implementation at Dao-AILab/flash-attention commit [`a369df707e1980fb328abcc1733e3457ec10155f`](https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute), which is the source sn", "sha256": "582bae2812e5a0d1842e5dabcaa3a111d9723410fdcb5287165744e1b7b680f9"}, {"id": "u004", "kind": "prose", "locator": "body:L10-L10", "preview": "The paper implementation is written in CuTe DSL. Its compilation comparison is against corresponding FA3 CUTLASS kernels: forward compiles in 2.5 seconds instead of 55 seconds, and backward in 1.4 seconds instead of 45 seconds. Those are si", "sha256": "2266d20d5c09dbaa973f352ea1400662ca84a3b9038021272b1258c9deefc4e0"}, {"id": "u005", "kind": "prose", "locator": "body:L14-L14", "preview": "The forward kernel assigns two output tiles of 128 query rows to one CTA. One MMA warp issues the matrix products, two four-warp softmax groups serve the two output tiles, and a correction warpgroup handles accumulator corrections. The scor", "sha256": "f72a68c72ef30e73a848c40dcf32c36a953ba4f63d07dea896a729aaf5fad961"}, {"id": "u006", "kind": "prose", "locator": "body:L16-L16", "preview": "This is more than ordinary K/V double buffering: the two alternating objects are output tiles with separate softmax state. The exact synchronization and stage ownership are implementation details; the invented one-loop pseudocode formerly o", "sha256": "c191fae7db0b6d0f0e9f84bffa3a6831477e6cfdabf2886659511eed72e8c34a"}, {"id": "u007", "kind": "prose", "locator": "body:L20-L20", "preview": "FA4 does not replace every hardware exponential. The paper selects only about 10-25% of entries for software evaluation on FMA units and leaves the rest on the hardware MUFU `ex2` path, allowing both resources to contribute.", "sha256": "684eaf9635c6bb161d267b05e4f15ce13f54b79e00d86174511b80879bedc800"}, {"id": "u008", "kind": "prose", "locator": "body:L22-L22", "preview": "For a software-selected value, the published range reduction writes `x = n + f` with `n = floor(x)` and `f` in `[0, 1)`, evaluates a degree-3 polynomial for `2**f`, and reconstructs the scale from `n`. The function below is a scalar referen", "sha256": "1baf75455f827398faef384641ff9669089152483a730956c6aa7ea364948060"}, {"id": "u009", "kind": "code", "locator": "body:L24-L32", "preview": "```python import math def fa4_blog_exp2_reference(x: float) -> float: n = math.floor(x) f = x - n polynomial = 1.0 + f * (0.6951 + f * (0.2276 + f * 0.0771)) return math.ldexp(polynomial, n) ```", "sha256": "9298f823e2f79bb1929071207fa66371dd4e82a8662ce232e6ebe29f493f1631"}, {"id": "u010", "kind": "prose", "locator": "body:L34-L34", "preview": "No standalone four-times software-versus-hardware exponential result is asserted here. The paper evaluates the combined kernel and its ablations rather than establishing that former page claim.", "sha256": "4f249df50219c8b338ef11f0dd2ed6b32aaeb17be2bb4157dafbeba464059290"}, {"id": "u011", "kind": "prose", "locator": "body:L38-L38", "preview": "Ordinary online softmax updates the row maximum and rescales accumulated state as each score block arrives. FA4 permits its retained maximum to lag: it resynchronizes only when the new block maximum exceeds the retained maximum by more than", "sha256": "9da61724cbae84193d5788fca9ce25e3c579a072c13fd16b8d54605f7ae86bc5"}, {"id": "u012", "kind": "prose", "locator": "body:L40-L40", "preview": "When a rescale is skipped, subsequent probabilities are still evaluated relative to the retained old maximum, and auxiliary statistics track the delayed normalization. The algorithm performs final renormalization at the end. Comparing absol", "sha256": "3177714cb704f95ebae51a833c10979b765bc122ccf6a14cd176cff5677c7b0d"}, {"id": "u013", "kind": "prose", "locator": "body:L44-L44", "preview": "The paper maps five backward GEMMs to two-CTA tcgen05 MMA with `M=256, N=128, K=128`. For those operations, the paired CTAs can share operand B, which the authors describe as roughly halving the shared-memory reads for that operand. This is", "sha256": "8d23ff074a9499de040bca63da0fe4d30d194b537de181f4ca508d2bc79bc11a"}, {"id": "u014", "kind": "prose", "locator": "body:L46-L46", "preview": "For dQ, each CTA computes a half of dS and exchanges that half through distributed shared memory so both CTAs can form the required dQ product. The two-CTA organization also doubles the dQ reduction tile along N and thereby halves the numbe", "sha256": "6a3c37791017a6a89f4f7716abe8d963db55672339299885a32d6ed573fe6662"}, {"id": "u015", "kind": "prose", "locator": "body:L50-L50", "preview": "At commit `a369df7`:", "sha256": "7113e059ccec8966d4256da23af091cdc161af598c071f055b0a1482b94c73e0"}, {"id": "u016", "kind": "list-item", "locator": "body:L52-L52", "preview": "- [`flash_fwd_sm100.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py) builds tcgen05 operations through `make_trivial_tiled_mma`, allocates TMEM through `Tmem", "sha256": "c521af8e33a110418a404d89f23ef9da68a4ad15215aeac9d228dfa9045847cf"}, {"id": "u017", "kind": "list-item", "locator": "body:L53-L53", "preview": "- The forward path constructs TMA atoms for Q, K, and V where its configuration enables them. It also has non-TMA Q and paged-K/V copy paths, so TMA use is not unconditional.", "sha256": "4c5c693b748161d30902f58dbd51ef0a42316b48ac01e7d28f12bcafddee5fea"}, {"id": "u018", "kind": "list-item", "locator": "body:L54-L54", "preview": "- [`flash_bwd_sm100.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_bwd_sm100.py) constructs the five backward MMA operations and the two-CTA exchange/reduction pipelines", "sha256": "815a65094e11852de9574447bc0d50adf1df3ad935e82caf224f979c7d21448c"}, {"id": "u019", "kind": "list-item", "locator": "body:L55-L55", "preview": "- The package README describes CuTe DSL attention for Hopper and Blackwell, and the tree contains SM90, SM100, and SM120 dispatch modules. The paper's FA4 result remains SM100/B200-specific; package architecture coverage should not be used ", "sha256": "2cc2823a1a7c188475de9f29140603b203621cbb842f595398a7083e604ffb08"}, {"id": "u020", "kind": "list-item", "locator": "body:L56-L56", "preview": "- The pinned [`pyproject.toml`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/pyproject.toml) requires `nvidia-cutlass-dsl==4.6.0.dev0`. That is a property of this source snapshot", "sha256": "2560ae01ebeeaa288f2229ebb7f069cf4864ff5ec4988bf4eceb965be3630659"}, {"id": "u021", "kind": "list-item", "locator": "body:L57-L57", "preview": "- Forward and backward accept FP16/BF16. An FP8 benchmark/bring-up script is present, but at this revision it explicitly expects the FA4 FP8 call to fail until support is implemented.", "sha256": "feeb126d7d71e6c313ef6c9da1aab21d2d319602db6e2c3de1fd9437053222c4"}, {"id": "u022", "kind": "prose", "locator": "body:L61-L61", "preview": "The first-party sources contain two different source-reported peak values. Paper v1 reports **up to 1613 TFLOPS/s on B200 BF16, or 71% of the peak convention used by the authors**. Tri Dao's blog reports **up to 1605 TFLOPS/s, also labeled ", "sha256": "1c2e20a8e65ba15f72f872cf74d6a28352ebfb75c641d12fde648b98cf037bd7"}, {"id": "u023", "kind": "prose", "locator": "body:L63-L63", "preview": "The paper's benchmark suite spans sequence lengths from 1K through 32K and multiple query/value head-dimension pairs under a fixed total-token convention. Neither textual source establishes the former single row that attached 1605 TFLOPS, 7", "sha256": "ffecfa7f2cb14d406a31606a14172909bfbbe0db54c419239eb5ae2e8ecb70ed"}, {"id": "u024", "kind": "list-item", "locator": "body:L67-L67", "preview": "- Choose the package path only for a supported architecture, dtype, head-dimension pair, masking mode, and feature set at the exact revision in use.", "sha256": "762ad956d4509aee447615faf6b80f1a258782fafdec6dd079f3e3e3c6ac920d"}, {"id": "u025", "kind": "list-item", "locator": "body:L68-L68", "preview": "- Do not treat sequence length 1024 or head dimension 128 as universal crossover or optimum values. Compare against the relevant cuDNN, framework, or other kernel path on the actual workload.", "sha256": "8375983f1d4f7ce52c849304a3aad867c8c2b667afa82f6b1ebf061cf544f372"}, {"id": "u026", "kind": "list-item", "locator": "body:L69-L69", "preview": "- Compilation speed and runtime speed are separate measurements. The paper's compile comparison does not prove an equivalent runtime factor.", "sha256": "90d8676cddf785613147e85842b741464ae3ae1fb640fd08252120ddd68f6785"}, {"id": "u027", "kind": "list-item", "locator": "body:L70-L70", "preview": "- Treat source-reported B200 numbers as unreproduced unless the same software, clock/power settings, tensor shapes, timing region, warmup, and FLOP convention are available.", "sha256": "896b8ebc0778eb869845e20ff98d507909b34ea91491af6057824a37de4f85a0"}, {"id": "u028", "kind": "list-item", "locator": "body:L74-L74", "preview": "- [FlashAttention-4 paper, arXiv v1](https://arxiv.org/abs/2603.05451v1)", "sha256": "65c89a97ddf06d9514d19588fe1266a84f9ab8f7dc02d57f8732947385eabe66"}, {"id": "u029", "kind": "list-item", "locator": "body:L75-L75", "preview": "- [Tri Dao's FlashAttention-4 blog](https://tridao.me/blog/2026/flash4/)", "sha256": "0c88bc6176d024bb44f1ccb80556697f7868de9d35d59a10546d17b279974d99"}, {"id": "u030", "kind": "list-item", "locator": "body:L76-L76", "preview": "- [FA4 CuTe DSL package at commit `a369df7`](https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute)", "sha256": "98fa8065e41a96242bc726acab9d3101d53459cf3260434fc27bf76545ee39e5"}, {"id": "u031", "kind": "list-item", "locator": "body:L77-L77", "preview": "- [Pinned SM100 forward source](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py)", "sha256": "4938cdf2c16b4de76f90e1e8a33103be66d267abb2adb5a030d8fc510818857b"}, {"id": "u032", "kind": "list-item", "locator": "body:L78-L78", "preview": "- [Pinned SM100 backward source](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_bwd_sm100.py)", "sha256": "afee82a4bbcd20dd427756fd46204f17af4ce2985c49fe7e4aca940035bb47b6"}, {"id": "u033", "kind": "prose", "locator": "body:L82-L82", "preview": "The local [`full/`](../../artifacts/kernels/flash-attention-4/full/) bundle is a byte-verified, verbatim **adjacent NVIDIA CUTLASS SM100 FMHA backward MLA example**, pinned to CUTLASS commit `0e026982`. It is not the Dao-AILab FA4 implement", "sha256": "a85ac9e12fb38e14987c53ae33ffdfc5ca91b605e4368033767b3b1cecebc738"}, {"id": "u034", "kind": "prose", "locator": "body:L84-L84", "preview": "Query the page and its attached local references with:", "sha256": "38c4142a3241d438fee5d4ab59db0c7c243b0825b789d47c12e6f4d62b2a4092"}, {"id": "u035", "kind": "code", "locator": "body:L86-L88", "preview": "```bash python3 scripts/get_page.py kernel-flash-attention-4 --include-code ```", "sha256": "dbaa5f540bdab5755bcdb9295bc1637f3e10a533091792abfe6b693678d4bd54"}], "confidence_claimed": "source-reported", "headings": ["Verified Scope", "Ping-Pong Forward Schedule", "Partial Software Exponential", "Conditional Softmax Rescaling", "Two-CTA Backward", "Pinned Implementation Notes", "Performance Evidence", "Practical Boundaries", "Pinned Sources", "Local Code References"], "id": "kernel-flash-attention-4", "path": "wiki/kernels/flash-attention-4.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/flash-attention-4.md", "url": "https://arxiv.org/abs/2603.05451v1"}, {"path": "sources/blogs/flash-attention-4.md", "url": "https://tridao.me/blog/2026/flash4/"}], "risk_flags": ["code", "ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-flash-attention-4", "blog-flash-attention-4"], "title": "FlashAttention-4", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "73e3ffa9038dafda7f4083be34a1dabae23531849cb738ba46e7914bfcff9857", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "[Dao-AILab/flash-attention PR 2441](https://github.com/Dao-AILab/flash-attention/pull/2441), merged as [`f219c89c886c6ccbf9d3dbd9fe41b11ac64e9df8`](https://github.com/Dao-AILab/flash-attention/commit/f219c89c886c6ccbf9d3dbd9fe41b11ac64e9df8", "sha256": "4f63a059a81265b7e1cc2d93083303f2ea9498f4b056d7f2855f19c4807dabbf"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "The top-k path is deliberately narrower than a generic paged sparse-attention API. It requires packed GQA/MQA, `qhead_per_kvhead == 128`, and the cp.async K/V gather path. Its constructor defaults to top-k length 2048 and requires the confi", "sha256": "da9c2fe6ef32dd275484f90f2353c35bb3d70a61c4879680f449bfcd5bb081d1"}, {"id": "u003", "kind": "prose", "locator": "body:L11-L11", "preview": "The pinned merge keeps three responsibilities distinguishable:", "sha256": "7c8e584dcb6ba3f5fec244c339c7c94b0aa5316af71a4de934df4270db9bb7f3"}, {"id": "u004", "kind": "list-item", "locator": "body:L13-L13", "preview": "- [`topk_gather_kv.py`](../../artifacts/prs/flash-attention/PR-2441/key-files/flash_attn/cute/topk_gather_kv.py) loads caller-provided top-k indices, forms indexed K/V addresses, issues cp.async copies, and optionally constructs validity bi", "sha256": "73efde420784dfeb2b7aa524d5aa4affeab8241ce307f37e1e5353236b1008f7"}, {"id": "u005", "kind": "list-item", "locator": "body:L14-L14", "preview": "- [`tile_scheduler.py`](../../artifacts/prs/flash-attention/PR-2441/key-files/flash_attn/cute/tile_scheduler.py) supplies the tile-scheduler implementations selected by the MLA kernel.", "sha256": "e609fba486c07cb0513067e5110f4a56712998c460ac1190ae764e870ea1df45"}, {"id": "u006", "kind": "list-item", "locator": "body:L15-L15", "preview": "- [`flash_fwd_mla_sm100.py`](../../artifacts/prs/flash-attention/PR-2441/key-files/flash_attn/cute/flash_fwd_mla_sm100.py) integrates gather, scheduling, two-CTA tcgen05 MMA, TMEM accumulators, softmax, and the output path.", "sha256": "82cb38d28021580714bc04318a4ef997cb376381d574b4edf67e7311c34059d2"}, {"id": "u007", "kind": "prose", "locator": "body:L17-L17", "preview": "This separation matters for evaluation. Arithmetic work follows the effective top-k length, whereas memory addresses follow the caller's index set and use an indexed gather/optional-bitmask path. Reduced attention FLOPs therefore do not pro", "sha256": "2daefc28898a6cce495bbd7fda384f2c0419b4e5882ed996444d0f964d074001"}, {"id": "u008", "kind": "prose", "locator": "body:L21-L21", "preview": "The PR author reports an initial saturating-decode comparison for batch 512, `seqlen_q=1`, `seqlen_k=16384`, 128 query heads, the 64/512 MLA shape, and top-k length 2048:", "sha256": "b3035685847df985b94a47aa4fa504d1371190e34396dde9ccd741363aa0a602"}, {"id": "u009", "kind": "table-row", "locator": "body:L25-L25", "preview": "| Variant | Author-reported latency | Author-reported throughput | | DSA, no bitmask; indices assumed in bounds | 0.31 ms | 955.47 TFLOPS |", "sha256": "a24fbe88c47cc40ebc98942a7df9423dbbbb468b341e1bb5aebed73761544c38"}, {"id": "u010", "kind": "table-row", "locator": "body:L26-L26", "preview": "| Variant | Author-reported latency | Author-reported throughput | | DSA, validity bitmask | 0.33 ms | 898.08 TFLOPS |", "sha256": "e7bf57c1383e05c0ff49169009cdc8f758023a7591f7c01c3ed64e68125b906f"}, {"id": "u011", "kind": "table-row", "locator": "body:L27-L27", "preview": "| Variant | Author-reported latency | Author-reported throughput | | Vanilla MLA baseline | 1.98 ms | 1180.70 TFLOPS |", "sha256": "8955e33c28ba6617ad774374f32687f50790bc825342c54af110832cf10f2fd8"}, {"id": "u012", "kind": "prose", "locator": "body:L29-L29", "preview": "These PR-description observations are not reproduced here. The PR body does not name the exact GPU model, dtype, clocks/power state, software environment, timing protocol, or run-to-run variation for the rows. They therefore remain prose wi", "sha256": "8f464ccdce13dd8c03fcdf509014a32437181e4d6a2d29317aa718b9aadb63fd"}, {"id": "u013", "kind": "list-item", "locator": "body:L33-L33", "preview": "- Confirm the exact MQA packing, 64/512 dimensions, top-k divisibility, index layout, and bounds contract used by the target revision.", "sha256": "0bd5d403083d1c9c0011d53972bb920f173d638cc6297d0fa560921a2d139f9b"}, {"id": "u014", "kind": "list-item", "locator": "body:L34-L34", "preview": "- Treat no-bitmask results as valid only when the caller guarantees every index is in bounds; otherwise preserve the validity path.", "sha256": "b76830a1691f4a2f7e8439909d3a544660928842234c0f5a41c0cb587eca8bdc"}, {"id": "u015", "kind": "list-item", "locator": "body:L35-L35", "preview": "- Do not infer paged-KV compatibility from the exposed parameter name: merge `f219c89c` rejects page tables in this MLA path.", "sha256": "0d852df017b6e52be08f9f98ecc3426541d04f254d3ece4a4dd788bc77df514e"}, {"id": "u016", "kind": "list-item", "locator": "body:L36-L36", "preview": "- Profile indexed-gather traffic and tensor arithmetic separately, then validate end-to-end accuracy and latency on the full workload.", "sha256": "9ab5962234ff3e14f16d030c48e8abf52d8283ce1b4b6c187393bc55d964aeed"}, {"id": "u017", "kind": "prose", "locator": "body:L40-L40", "preview": "The local PR record and byte-verified bundle identify the integration and its dedicated gather file:", "sha256": "47a761a14371c5670c19841d3279cbd73443fbd8a209713016e33d2693199cad"}, {"id": "u018", "kind": "code", "locator": "body:L42-L49", "preview": "```python from pathlib import Path pr_page = Path(\"sources/prs/flash-attention/PR-2441.md\").read_text() provenance = Path(\"artifacts/prs/flash-attention/PR-2441/PROVENANCE.yaml\").read_text() assert \"flash_fwd_mla_sm100.py\" in pr_page assert", "sha256": "cb588e655e6743eac16572e26541b164f2853fd9aa333125fff8febac7bb5298"}], "confidence_claimed": "source-reported", "headings": ["Verified Scope", "Gather and Scheduling Paths", "Source-Reported Performance", "Transfer Checklist", "Reproducible Source Lookup"], "id": "kernel-flash-attention-sm100-mla-topk", "path": "wiki/kernels/flash-attention-sm100-mla-topk.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/prs/flash-attention/PR-2441.md", "revision": "f219c89c", "url": "https://github.com/Dao-AILab/flash-attention/pull/2441"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["pr-flash-attention-2441"], "title": "FlashAttention SM100 MLA TopK Sparse Forward", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "3c4ba2ede0a25c46871912580475e2e71786d53c38b8dea8330b7b017e42d0ae", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "FlashMLA is DeepSeek's attention-kernel library for DeepSeek-V3 and DeepSeek-V3.2-Exp. At pinned commit [`71c7379`](https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232), the library contains MLA-mode decode", "sha256": "08238d87cfdb4314f85a910680b5840280b83bf0cc9e41bd80a02b4cfdb103c3"}, {"id": "u002", "kind": "prose", "locator": "body:L9-L9", "preview": "The DeepSeek-V2 paper describes the model-level reduction in elements cached per layer and token:", "sha256": "90f1ed73430b4cbde8c69c4af03544e62c75f78b7add683720adc84ef20b861f"}, {"id": "u003", "kind": "list-item", "locator": "body:L11-L11", "preview": "- MHA: `2 * n_h * d_h`", "sha256": "3065811e2bcaaead5d63598f5f54ddb3a2a8a65a0d5b1657ff8d27f5a7147e8c"}, {"id": "u004", "kind": "list-item", "locator": "body:L12-L12", "preview": "- MLA: `d_c + d_h^R`, approximately `4.5 * d_h` for its `d_c=4*d_h` and `d_h^R=d_h/2` configuration", "sha256": "075599e94f2e9cf9e3afd1a93903fc330f0193fede80ce16e1ad9c1768b2ece4"}, {"id": "u005", "kind": "prose", "locator": "body:L14-L14", "preview": "These formulas are independent of storage dtype and layer count. They should not be converted to whole-model KB/token figures without naming those additional assumptions.", "sha256": "5679a2621c2ce8aca3375f7bb6c190b8507bced12fa41d172eee9f063a08afda"}, {"id": "u006", "kind": "prose", "locator": "body:L16-L16", "preview": "FlashMLA's **656-byte** layout is narrower: it is the DeepSeek-V3-family FP8 sparse-decode cache ABI, not the definition of an MLA cache. One token contains:", "sha256": "8b83eac13e9ced1937c096c44920f26d7bb029cd81382cc390b1351c2c619b50"}, {"id": "u007", "kind": "table-row", "locator": "body:L20-L20", "preview": "| Region | Representation | Bytes | | NoPE latent data | 512 `float8_e4m3` values | 512 |", "sha256": "841cdc35d7e665175a013bf5bce137a96f0b1702d6a08dc47aa78095460aac24"}, {"id": "u008", "kind": "table-row", "locator": "body:L21-L21", "preview": "| Region | Representation | Bytes | | NoPE group scales | four `float32` values, one per 128 values | 16 |", "sha256": "71b3d01dd719dba8f1ff88f5f1f197a6693308676134668ce2fab9152fa59e46"}, {"id": "u009", "kind": "table-row", "locator": "body:L22-L22", "preview": "| Region | Representation | Bytes | | RoPE data | 64 unquantized `bfloat16` values | 128 |", "sha256": "acd39402aa7de9ad4eb981b9bd203369c56fda205897ab6fa61b2c7a625c0d5a"}, {"id": "u010", "kind": "table-row", "locator": "body:L23-L23", "preview": "| Region | Representation | Bytes | | **Total** | | **656** |", "sha256": "7b8371fda18c965ce82741c5d286f4e1076c5bb5722a40f803b4bde89ec2ce18"}, {"id": "u011", "kind": "prose", "locator": "body:L25-L25", "preview": "Dense decode uses a BF16 cache. The pinned quantization tests also contain a separate 512-dimensional sparse layout, so code must select the model/layout contract rather than assuming 656 bytes universally. `page_block_size` is taken from t", "sha256": "34fe059844fd809f6ef31b60cc8e093bb4c288d0de8db452a89fbe1ddda0c68c"}, {"id": "u012", "kind": "table-row", "locator": "body:L31-L31", "preview": "| Operator | Architecture | Attention mode | Documented cache/input format | | Dense decode | SM90 | MQA (`576/512`) | BF16 paged KV |", "sha256": "d293f5c14941162e3c80a4c205f6461abdb0acb7a357aba82b7bf19f2f2db22a"}, {"id": "u013", "kind": "table-row", "locator": "body:L32-L32", "preview": "| Operator | Architecture | Attention mode | Documented cache/input format | | Sparse decode | SM90, SM100 | MQA (`576/512`) | FP8 KV, dequantized for BF16 MMA; BF16 output |", "sha256": "65e23814a89de7ef621dd455097923d1f3f566b4b2f4051f65b33140950730b1"}, {"id": "u014", "kind": "table-row", "locator": "body:L33-L33", "preview": "| Operator | Architecture | Attention mode | Documented cache/input format | | Dense prefill | SM100 | MHA (`192/128` or `128/128`) | BF16 Q/K/V |", "sha256": "d2d522e04a6d5154570c895e700920708c7a57dd0a136996bbdde85b9cb691b9"}, {"id": "u015", "kind": "table-row", "locator": "body:L34-L34", "preview": "| Operator | Architecture | Attention mode | Documented cache/input format | | Sparse prefill | SM90, SM100 | MQA | BF16 Q and KV |", "sha256": "7d4c68d61d7d6124b0163996c5984745b43d81755a3dbffa98a6ed329f929770"}, {"id": "u016", "kind": "prose", "locator": "body:L36-L36", "preview": "CUDA 12.8 or newer and PyTorch 2.0 or newer are required; the pinned README requires CUDA 12.9 or newer for SM100.", "sha256": "f35e263d91113a6389ad903fab7fa42d70860898334fef8104eb0d3dc67d36c5"}, {"id": "u017", "kind": "prose", "locator": "body:L40-L40", "preview": "Sparse decode receives `indices[batch, s_q, topk]`. Each nonnegative value already encodes a physical page and offset:", "sha256": "5dfe1322da0b2dfe3df8fbe6ed0ece5d000cd1286a6beef0d863746051aa4f73"}, {"id": "u018", "kind": "code", "locator": "body:L42-L44", "preview": "```text encoded = physical_page * page_block_size + offset_in_page ```", "sha256": "5231e8b5e77d4fc668820c802694dfdb284139fc38a77fddae3aef8dc57c3bee"}, {"id": "u019", "kind": "prose", "locator": "body:L46-L46", "preview": "Because the physical page is already encoded, sparse decode does not use `block_table`; `-1` marks an invalid entry. The kernel consumes these indices but does not produce the top-k selection, so an indexing stage outside the attention call", "sha256": "04ac9da24c0828b044f4d7f4d8ca798014651239c97575e1852f78bd95cf7e10"}, {"id": "u020", "kind": "prose", "locator": "body:L48-L48", "preview": "Sparse prefill is a different interface. It receives BF16 `q[s_q,h_q,d_qk]`, BF16 `kv[s_kv,h_kv,d_qk]`, and `indices[s_q,h_kv,topk]`; it has no batch dimension, requires `h_kv=1` in the documented equivalence, and accepts `-1` or values at ", "sha256": "09284ad50c2cab412438ff4567f4f3773d30676a42e6f882fc3e0de26bfdb1db"}, {"id": "u021", "kind": "prose", "locator": "body:L52-L52", "preview": "The following are maxima reported by the pinned first-party README. They were not reproduced here, and the README does not provide complete shape, timing, sample-count, or variance cells, so they are intentionally excluded from structured `", "sha256": "a0ee45c4809bef21a69de293f0ce17990d899829625c2c97508db34adc85e738"}, {"id": "u022", "kind": "table-row", "locator": "body:L56-L56", "preview": "| Operator | Environment stated by source | Precision scope | Author-reported observation | | Dense MLA decode | H800 SXM5, CUDA 12.8 | BF16 cache | Up to 3000 GB/s in a memory-bound configuration; up to 660 TFLOPS in a compute-bound config", "sha256": "29f91ffd6863b1a26ca514240351b3d944b37d1143ee38f4e83180a774a3fb09"}, {"id": "u023", "kind": "table-row", "locator": "body:L57-L57", "preview": "| Operator | Environment stated by source | Precision scope | Author-reported observation | | Sparse MLA decode | H800 SXM5, CUDA 12.8 | FP8 KV, BF16 MMA | 410 TFLOPS in a compute-bound configuration |", "sha256": "9764016bfae45c9128dd978cf206509cf6b2448ec5602b05a403d0642521afe0"}, {"id": "u024", "kind": "table-row", "locator": "body:L58-L58", "preview": "| Operator | Environment stated by source | Precision scope | Author-reported observation | | Sparse MLA decode | B200; software version not stated in the row | FP8 KV, BF16 MMA | Up to 350 TFLOPS; the author says it was not well optimized ", "sha256": "c818b558acbb7ed171090bb6661eaaf4400cd8a3666a57fc1b7761aad8436b48"}, {"id": "u025", "kind": "table-row", "locator": "body:L59-L59", "preview": "| Operator | Environment stated by source | Precision scope | Author-reported observation | | Dense MHA prefill | B200; NVIDIA-reported | BF16 inputs | Up to 1460 TFLOPS forward and 1000 TFLOPS backward |", "sha256": "9a9d5d7fc032bd9e83da289de902bbf1535e3c996a3f8ba5210f76906dec57f0"}, {"id": "u026", "kind": "table-row", "locator": "body:L60-L60", "preview": "| Operator | Environment stated by source | Precision scope | Author-reported observation | | Sparse MLA prefill | H800 SXM5, CUDA 12.8 | BF16 inputs | Up to 640 TFLOPS forward |", "sha256": "53d0b78b4866316334e4a20763250f5c5fab57171f7c843f7722b12794bda70f"}, {"id": "u027", "kind": "table-row", "locator": "body:L61-L61", "preview": "| Operator | Environment stated by source | Precision scope | Author-reported observation | | Sparse MLA prefill | B200, CUDA 12.9 | BF16 inputs | Up to 1450 TFLOPS forward |", "sha256": "6ff6f1fdd66ca72895d490f8c0607ddf8e1df1df56d7fd8f12aed022c8527a79"}, {"id": "u028", "kind": "prose", "locator": "body:L63-L63", "preview": "The numbers compare different operators, phases, shapes, precision scopes, and machines. In particular, `1460` is dense MHA prefill, not a replacement for the `660` dense-MLA-decode observation.", "sha256": "e5a0dbf189f061c878b4cae1b5a7048e458381ff9fc70b5229b92828896acc57"}, {"id": "u029", "kind": "prose", "locator": "body:L67-L67", "preview": "The pinned DeepSeek SM100 sources use TMA, `tcgen05` tensor-core operations, and TMEM in specialized sparse-decode/prefill and dense-MHA-prefill code. Those mechanisms are implementation-specific: they do not make the CUTLASS and FlashInfer", "sha256": "d5395ed58db33d9631c7560a59e1e00e16d3992a554fc2ae6771656e5968e340"}, {"id": "u030", "kind": "prose", "locator": "body:L69-L69", "preview": "The local [`full/`](../../artifacts/kernels/flashmla/full/) bundle contains two byte-verified **adjacent implementations**:", "sha256": "5f9bf20e663563d7f07aaf28c3349aca750a2ca5f55dbb73357b3ae42f262d79"}, {"id": "u031", "kind": "list-item", "locator": "body:L71-L71", "preview": "- NVIDIA CUTLASS Example 77 MLA forward at merge `9baa06dd`", "sha256": "086f3f715bc450403924c12a37be1c3fae7265053578b8d21907af35c3f9d252"}, {"id": "u032", "kind": "list-item", "locator": "body:L72-L72", "preview": "- FlashInfer's SM100 FMHA-MLA header at commit `9a05c92a`", "sha256": "eec103415a36216d780be09cf18dda0a8fb8c2dff1600dec105fa91e92ce2dc5"}, {"id": "u033", "kind": "prose", "locator": "body:L74-L74", "preview": "Their exact per-file origins are recorded in `full/PROVENANCE.yaml`. The [`variants/`](../../artifacts/kernels/flashmla/variants/) directory contains a small KernelWiki-derived layout/index helper, explicitly marked as non-upstream. For the", "sha256": "37b693efa9f5daa2cf92e3b9434f64d04bd1a1159c2c7f697461898614ff707e"}, {"id": "u034", "kind": "list-item", "locator": "body:L78-L78", "preview": "- Match the operator family, `d_qk/d_v`, architecture, CUDA version, cache dtype, and index shape exactly.", "sha256": "1f8f834f7ce3ce7eca28b0eb611071ec978c248ae82042c4f293e78b9710b8cc"}, {"id": "u035", "kind": "list-item", "locator": "body:L79-L79", "preview": "- Treat the 656-byte layout as a V3-family FP8 sparse-decode ABI, not a generic MLA property.", "sha256": "a6abb77196519f2420b0a762e095773a8b5c77f40b2ee203ec99fd16ce3abe25"}, {"id": "u036", "kind": "list-item", "locator": "body:L80-L80", "preview": "- Generate sparse indices before invoking FlashMLA and validate invalid-index/page encoding rules.", "sha256": "0ca8b8e69996a2e86619b01c4abc06f895da7e1a18d8bd095f31ffe0ad454823"}, {"id": "u037", "kind": "list-item", "locator": "body:L81-L81", "preview": "- Benchmark the target decode or prefill workload; do not transfer TFLOPS or bandwidth across the table's distinct regimes.", "sha256": "f75121f6f51ee2d76e0ec793123df0743909bccb3bbf06e7ff84fe69b2682390"}, {"id": "u038", "kind": "list-item", "locator": "body:L82-L82", "preview": "- Validate output and LSE against the repository reference before relying on throughput.", "sha256": "13d8121c830496708430c92fd8de8b4fc8517137efec46a16fdb5566da14a859"}, {"id": "u039", "kind": "list-item", "locator": "body:L86-L86", "preview": "- [DeepSeek FlashMLA at audited commit `71c7379`](https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232)", "sha256": "0de9535be0d6d149c00340e2638776319f26831cdee6a72ee9c9fe208d2761b9"}, {"id": "u040", "kind": "list-item", "locator": "body:L87-L87", "preview": "- [DeepSeek-V2 MLA paper, v5](https://arxiv.org/html/2405.04434v5)", "sha256": "67b77a2f9ec951fd7db140185b5eedc21e2c3d8c8fa615da68632b6ee1075c44"}, {"id": "u041", "kind": "list-item", "locator": "body:L88-L88", "preview": "- [CUTLASS PR 2466, SM100 MLA-shape backward](https://github.com/NVIDIA/cutlass/pull/2466)", "sha256": "0e7eede9e2ae4d56aae5f8797c6118fbbbe1859478a97294d44e70db06ec8ee1"}, {"id": "u042", "kind": "list-item", "locator": "body:L89-L89", "preview": "- [CUTLASS PR 2472, SM100 MLA-shape forward](https://github.com/NVIDIA/cutlass/pull/2472)", "sha256": "ea1ffa911f1b9bfd82df419873ec8deed230cbd340899b68a493ef7db213d13a"}, {"id": "u043", "kind": "prose", "locator": "body:L91-L91", "preview": "Query the page and its labeled artifacts with:", "sha256": "f5116cccd6a53f501df19ca2264a4ecc72b0e1cdbd130894bdf4cc89c362ef66"}, {"id": "u044", "kind": "code", "locator": "body:L93-L95", "preview": "```bash conda run -n base python scripts/get_page.py kernel-flashmla --include-code ```", "sha256": "7bc0deaa394726c6a68dd18d167067b318bd1716043acfa1e49b6793d677009c"}, {"id": "u045", "kind": "prose", "locator": "body:L97-L97", "preview": "The mode-specific byte arithmetic can be checked without a GPU:", "sha256": "3ad75e7b4793a8c3389c0175106203e303b590c71caf9e9c606fa07a6c4428b6"}, {"id": "u046", "kind": "code", "locator": "body:L99-L108", "preview": "```python # KernelWiki-derived contract check; not upstream FlashMLA code. def v3_fp8_sparse_bytes() -> int: nope_bytes = 512 scale_bytes = 4 * 4 rope_bytes = 64 * 2 return nope_bytes + scale_bytes + rope_bytes assert v3_fp8_sparse_bytes() ", "sha256": "a3c4914434151f616abf1963f00adbf0e98f53973f559a7d54967a7b0f7b2e3c"}], "confidence_claimed": "source-reported", "headings": ["Scope", "MLA Cache Reduction Versus a Concrete Cache ABI", "Supported Paths at `71c7379`", "Sparse Contracts", "Source-Reported Performance", "SM100 Implementation Notes", "Selection and Validation Checklist", "Sources and Local Query"], "id": "kernel-flashmla", "path": "wiki/kernels/flashmla.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/flashmla.md", "url": "https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232"}, {"path": "sources/docs/deepseek-v2-mla.md", "url": "https://arxiv.org/abs/2405.04434v5"}, {"path": "sources/prs/cutlass/PR-2466.md", "revision": "0e026982", "url": "https://github.com/NVIDIA/cutlass/pull/2466"}, {"path": "sources/prs/cutlass/PR-2472.md", "revision": "9baa06dd", "url": "https://github.com/NVIDIA/cutlass/pull/2472"}], "risk_flags": ["code", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-flashmla", "doc-deepseek-v2-mla", "pr-cutlass-2466", "pr-cutlass-2472"], "title": "FlashMLA \u2014 Multi-head Latent Attention", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "5c0369ba45f1bfcc220967e83d4765f7e8b56908a310a1bf0356e9bb2c996e70", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "The DeepSeek-V3 training recipe quantizes forward activations per `1x128` tile and weights per `128x128` block. The smaller groups let each scale adapt to local outliers. This is a **logical quantization format**; a kernel is compatible onl", "sha256": "f61bbab670c40f6823b72c4e1552028f57135423a4271d19648f5dfc017d81f5"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "DeepGEMM commit [`891d57b4`](https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f) implements related FP8 GEMMs on SM90 and SM100, but the two generations handle scale factors differently.", "sha256": "49d3ceb8b3dfcc423ee95a2aed85148e53aee1c2bd3abd5c6a74ea88781e830c"}, {"id": "u003", "kind": "prose", "locator": "body:L11-L11", "preview": "For `A[M,K] @ B[N,K].T` with the DeepSeek-V3 forward recipe, the logical scale arrays have shapes `A_sf[M,K/128]` and `B_sf[N/128,K/128]`. This small check documents only that grouping; it does not encode DeepGEMM's required TMA-transformed", "sha256": "3e88a4131ee366e92be611a9362e894b9340f99021537262af71d6d9fc25e2a4"}, {"id": "u004", "kind": "code", "locator": "body:L13-L22", "preview": "```python # KernelWiki-derived format check; not upstream DeepGEMM code. def deepseek_v3_scale_shapes(m: int, n: int, k: int): assert m % 128 == 0 and n % 128 == 0 and k % 128 == 0 activation_scales = (m, k // 128) weight_scales = (n // 128", "sha256": "95bb63abdcbd66f36d7aca3428db5558ee397f06a9083bf97a482bd37887ee7a"}, {"id": "u005", "kind": "prose", "locator": "body:L24-L24", "preview": "The paper documents phase-specific exceptions, including `128x1` activation grouping in backward paths. Do not treat the forward layout above as a universal FP8 tensor ABI.", "sha256": "b0ae7badd583a24dc5521335ba8e89a1cc26d6a1d27a2267bc45e6ed79e4bd0f"}, {"id": "u006", "kind": "prose", "locator": "body:L28-L28", "preview": "The pinned SM90 1D1D kernel requires FP32 scale factors and fixes `BLOCK_K == 128`. Within each K block it issues the selected WGMMA operations into a register `accum` array; after the WGMMA batch completes, it multiplies those partials by ", "sha256": "3e107b8e8107c166b50c5694c86424e4c213bfbd28b70a28738ce03dc7a05ff3"}, {"id": "u007", "kind": "prose", "locator": "body:L30-L30", "preview": "For the DeepSeek-V3 description, a 128-element K interval corresponds to four WGMMAs. The paper describes Hopper's relevant internal addition/alignment precision as 14 bits, not as a generic \u201cFP22 accumulator.\u201d Promotion improves numerical ", "sha256": "cecb8cb1bd1b7f075e99d470f28b7f6a61a49edbf9eae01af99e262b1c1b1f31"}, {"id": "u008", "kind": "prose", "locator": "body:L34-L34", "preview": "The pinned SM100 interface requires scale factors packed as four UE8M0 values per 32-bit `torch.int`. The 1D1D kernel TMA-loads scale-factor blocks, copies them into TMEM, builds a `make_instr_desc_block_scaled<..., float_ue8m0_t, ...>` des", "sha256": "0f39e3039030787bcd77e5fbe341200b542de8fbc48e87b44918146604326602"}, {"id": "u009", "kind": "prose", "locator": "body:L36-L36", "preview": "This is hardware-integrated scale consumption, not the SM90 `final_accum += scale_a * scale_b * accum` loop. Exact `tcgen05.mma` instruction spelling and descriptor restrictions are PTX-version-sensitive; use the pinned source wrapper or th", "sha256": "47d7facf3907dc8397d4036bf38fedddb7d9b0e7223b63b4d407862a49d2cb4c"}, {"id": "u010", "kind": "prose", "locator": "body:L38-L38", "preview": "The current DeepGEMM snapshot supports more than the original 128-granularity training recipe on SM100, including selected 32-element recipes. Treat the API's transformed/padded scale layout as authoritative for the chosen recipe.", "sha256": "74649936c4d025144388472b48f2125145b099ef7a2e59bc215ec457661899c2"}, {"id": "u011", "kind": "prose", "locator": "body:L42-L42", "preview": "The pinned README says DeepGEMM achieved **up to 1550 TFLOPS on H800** in its 2025-04-18 news item. That sentence does not identify a matrix shape, utilization percentage, timing protocol, sample count, variance, or a single optimization re", "sha256": "0131383e14f94d7145b3a6392f8a499eb120bd33d9ac22ccb5331ad9a3ebac04"}, {"id": "u012", "kind": "prose", "locator": "body:L44-L44", "preview": "CUTLASS also ships SM100 block-scaled GEMM schedules, but no matched CUTLASS-versus-DeepGEMM shape/environment record is established here.", "sha256": "39bf754c3b7e05487d1d5de5c0d21def57c6861b4d4f22d72c4b07b2799afe2a"}, {"id": "u013", "kind": "list-item", "locator": "body:L48-L48", "preview": "- Confirm the producer's scale grouping; FP8 element type alone is insufficient for compatibility.", "sha256": "17f0a12c0b31d993ffe1b26f67d7626a7005e017a8e985a9e5bc6ba67e6f70d9"}, {"id": "u014", "kind": "list-item", "locator": "body:L49-L49", "preview": "- On SM90, provide the FP32 scale layout expected by the selected DeepGEMM kernel.", "sha256": "3f6731561c8bfe65babdb625cc8dc6b8188a82685d973674d041bc922d502bcc"}, {"id": "u015", "kind": "list-item", "locator": "body:L50-L50", "preview": "- On SM100, provide correctly packed, TMA-aligned UE8M0 factors for the selected granularity and layout.", "sha256": "666485ee64a02a7476e9caa19fd6cee3bb2d936a8d3e6af9919018707a961c82"}, {"id": "u016", "kind": "list-item", "locator": "body:L51-L51", "preview": "- Include quantization, scale-layout transformation, and epilogue costs when measuring an end-to-end path.", "sha256": "a577bfc1ed0cfeecdc61abe917ff9c9d6384b847071fe1b9cba87020f7b72edc"}, {"id": "u017", "kind": "list-item", "locator": "body:L52-L52", "preview": "- Validate numerical error against an FP32/BF16 reference on the actual activation/weight distribution.", "sha256": "07d0de5fcdf56bd5cbe1337b5b07aee7a1862a254e4b9d45171e5af7603f0f9b"}, {"id": "u018", "kind": "list-item", "locator": "body:L56-L56", "preview": "- [DeepSeek-V3 Technical Report v2, FP8 training](https://arxiv.org/html/2412.19437v2)", "sha256": "4b970c1d29c6d57c22c12f14573ac44b08d93feddf63c50dc895ff90c6b60a5c"}, {"id": "u019", "kind": "list-item", "locator": "body:L57-L57", "preview": "- [DeepGEMM at audited commit `891d57b4`](https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f)", "sha256": "2d6df0aa583a384cd804e719be02642c86a4cabb45ab25aa35d13b54ef4bfa49"}, {"id": "u020", "kind": "list-item", "locator": "body:L58-L58", "preview": "- [NVIDIA PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/)", "sha256": "39526295274d0947dca74c85d82c53cbeb211e2130d8f6a34c5dcc9f16956dfc"}], "confidence_claimed": "source-reported", "headings": ["Scope", "Logical Scale Shapes", "SM90: WGMMA Followed by CUDA-Core Promotion", "SM100: Packed UE8M0 and Block-Scaled UMMA", "Source-Reported Performance", "Selection Checklist", "Sources"], "id": "kernel-fp8-block-scale-gemm", "path": "wiki/kernels/fp8-block-scale-gemm.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/deepgemm.md", "url": "https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f"}, {"path": "sources/docs/deepseek-v3-fp8.md", "url": "https://arxiv.org/abs/2412.19437v2"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}], "risk_flags": ["code"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-deepgemm", "doc-deepseek-v3-fp8", "doc-ptx-isa-sm100"], "title": "FP8 Fine-Grained-Scale GEMM", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "5b18f9bb5eb643757003bd25f3822609266ef79d9151f3fd7623e798928547ed", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "FlashInfer's MLSys 2026 contest calls Track A **Fused MoE** with FP8 support and targets NVIDIA B200. Its exact benchmark definition is `moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048`. The definition says that DeepSeek-style ", "sha256": "18326cab883ddb97025916dd02cfcf8a67372064038fc4601961532b1bd327ec"}, {"id": "u002", "kind": "table-row", "locator": "body:L11-L11", "preview": "| Field | Value | Meaning | | `seq_len` | variable | Number of input tokens; this is not labeled request batch size |", "sha256": "489d30823dbe6d0d153a6adfb3e6be20e9e0599228bc8cca8e7ac2e27c2964f7"}, {"id": "u003", "kind": "table-row", "locator": "body:L12-L12", "preview": "| Field | Value | Meaning | | global experts | 256 | Width of `routing_logits` |", "sha256": "ab8e536264682c97b33b36e51f7818295389869d78071f7bcfc3e1e5f4cf741a"}, {"id": "u004", "kind": "table-row", "locator": "body:L13-L13", "preview": "| Field | Value | Meaning | | local experts | 32 | Experts whose weights are resident on one EP rank |", "sha256": "81705d585c8debe3e6df2235dc4b41d7833ddbeed65b984931247c31fcbe7947"}, {"id": "u005", "kind": "table-row", "locator": "body:L14-L14", "preview": "| Field | Value | Meaning | | expert parallelism | 8 | `256 / 32` ranks in the published definition |", "sha256": "c8110af705992d3b500544c8283f2149d07f0b61bb0e7d2eb3b9a47b1c3006af"}, {"id": "u006", "kind": "table-row", "locator": "body:L15-L15", "preview": "| Field | Value | Meaning | | `top_k` | 8 | Selected global experts per token |", "sha256": "701a714491b6d44c4bdaf8ce3d4267a0be3dd9a838e889f963f29c96ba2da59b"}, {"id": "u007", "kind": "table-row", "locator": "body:L16-L16", "preview": "| Field | Value | Meaning | | `n_group` | 8 | Groups of 32 global experts |", "sha256": "5f46d8298cd7f3e8e9d21f875d1f5924b6cd1074db5eba13f0fdeb0c6d0d1817"}, {"id": "u008", "kind": "table-row", "locator": "body:L17-L17", "preview": "| Field | Value | Meaning | | `topk_group` | 4 | Groups retained before global top-k selection |", "sha256": "527a38b3ef12c6bde36175a483564ad6c5cafe7704a00b97ceaedef2c4c9cd3e"}, {"id": "u009", "kind": "table-row", "locator": "body:L18-L18", "preview": "| Field | Value | Meaning | | hidden size | 7168 | Input and output width |", "sha256": "54e08f4f881e5c89526ffbe3b836b2356c8f3d62502fac07571f4d119f3358ef"}, {"id": "u010", "kind": "table-row", "locator": "body:L19-L19", "preview": "| Field | Value | Meaning | | intermediate size | 2048 | Per-expert SwiGLU width |", "sha256": "0ce89dc7a715c664bb87be41e1d21dfb81be33c180bf130ca6016fdfeb3bb0ee"}, {"id": "u011", "kind": "table-row", "locator": "body:L20-L20", "preview": "| Field | Value | Meaning | | GEMM1 output | 4096 | Concatenated W13 output, `2 * 2048` |", "sha256": "5c686849f46d9f7b44e9e1ee43efe1ceaa4668e29bda28a0cc4f8842cfa33d39"}, {"id": "u012", "kind": "table-row", "locator": "body:L21-L21", "preview": "| Field | Value | Meaning | | scale block | 128 | Fixed granularity for this DeepSeek-FP8 trace |", "sha256": "d1e5a1c8c5c7c963f36ab5fc093ade0e8927e34ebe5f2f5ffa18a5862183b023"}, {"id": "u013", "kind": "prose", "locator": "body:L23-L23", "preview": "The `e32` suffix means **32 local experts**, not 32 total experts.", "sha256": "001cf0ce8a59e9e29df693e403576c4d03447c4f9682265f4b571f1828177834"}, {"id": "u014", "kind": "prose", "locator": "body:L27-L27", "preview": "For `T = seq_len`, the published benchmark signature is:", "sha256": "737ed711ebf0a7140acfc7a8cab6f37d5f0ea8a77c0085292675844f26e7e74c"}, {"id": "u015", "kind": "table-row", "locator": "body:L31-L31", "preview": "| Input | Dtype | Shape | | `routing_logits` | FP32 | `[T, 256]` |", "sha256": "ebc301e873d88834130b18c076ac974b142065b79907dfd1f98b160ef7f232d1"}, {"id": "u016", "kind": "table-row", "locator": "body:L32-L32", "preview": "| Input | Dtype | Shape | | `routing_bias` | BF16 | `[256]` |", "sha256": "bcad38d96f0c6feac0ec3bb267624a4bc1c16f57f261031a7f29e6eb8da880d0"}, {"id": "u017", "kind": "table-row", "locator": "body:L33-L33", "preview": "| Input | Dtype | Shape | | `hidden_states` | FP8 E4M3FN | `[T, 7168]` |", "sha256": "bfb153078bcae7b3e69fe5de92d4edb832a264671a102886e2d2a320c4f6ea99"}, {"id": "u018", "kind": "table-row", "locator": "body:L34-L34", "preview": "| Input | Dtype | Shape | | `hidden_states_scale` | FP32 | `[56, T]` |", "sha256": "ec4bec489db7f5fce3b61e3e1747ecbd18c30c063b557a5bfcb5c20d512722fc"}, {"id": "u019", "kind": "table-row", "locator": "body:L35-L35", "preview": "| Input | Dtype | Shape | | `gemm1_weights` | FP8 E4M3FN | `[32, 4096, 7168]` |", "sha256": "24269b2c4d8cfe693e764fd54dfd1eafdc57616ea155cc3b8502138b1c9d9ea4"}, {"id": "u020", "kind": "table-row", "locator": "body:L36-L36", "preview": "| Input | Dtype | Shape | | `gemm1_weights_scale` | FP32 | `[32, 32, 56]` |", "sha256": "f4e5562976a2cf05bb99c60098e7f9d596b5a3cd59d749a6ba082028da0a37b6"}, {"id": "u021", "kind": "table-row", "locator": "body:L37-L37", "preview": "| Input | Dtype | Shape | | `gemm2_weights` | FP8 E4M3FN | `[32, 7168, 2048]` |", "sha256": "3a489fb55046bcfe271ac63e38346248a7bd6ad7a116d4992738d0a853ef2ffc"}, {"id": "u022", "kind": "table-row", "locator": "body:L38-L38", "preview": "| Input | Dtype | Shape | | `gemm2_weights_scale` | FP32 | `[32, 56, 16]` |", "sha256": "2c1d01b228f14c9f210eed1f920111406117344ee9bd24cd14033623ae06e443"}, {"id": "u023", "kind": "table-row", "locator": "body:L39-L39", "preview": "| Input | Dtype | Shape | | `local_expert_offset` | INT32 | scalar |", "sha256": "9f61941d2f2c8b538b6e5943fae9de3253c2a8a16e803ef74fca6ea3f343f6a6"}, {"id": "u024", "kind": "table-row", "locator": "body:L40-L40", "preview": "| Input | Dtype | Shape | | `routed_scaling_factor` | FP32 | scalar |", "sha256": "45c2c5da5cafe1e777345bd9b5dddcb9507ed68d91079ab886f275c556cefc30"}, {"id": "u025", "kind": "prose", "locator": "body:L42-L42", "preview": "The output is BF16 `[T, 7168]`. These scale tensors are explicit storage inputs; this page does not infer their runtime cost from their existence.", "sha256": "29d8f9c0cd8e3cc789f0c79fe0466d03269314a7c1be853882d9ab25e3004baf"}, {"id": "u026", "kind": "prose", "locator": "body:L44-L44", "preview": "This derived helper makes the fixed shape arithmetic executable without pretending to encode FlashInfer's physical layouts:", "sha256": "f5cfd21000618d145880cf47d79989f120bce90aadb2093820bd177df828a846"}, {"id": "u027", "kind": "code", "locator": "body:L46-L61", "preview": "```python def track_a_shapes(tokens: int) -> dict[str, tuple[int, ...]]: assert tokens > 0 return { \"routing_logits\": (tokens, 256), \"hidden_states\": (tokens, 7168), \"hidden_states_scale\": (7168 // 128, tokens), \"gemm1_weights\": (32, 2 * 20", "sha256": "ecbb997f0664e8b26ef95d1baab5c4bdeaa20c737ccc6c532d90123fac4cd4be"}, {"id": "u028", "kind": "prose", "locator": "body:L63-L63", "preview": "At FlashInfer commit `7f614b86470180bab2d22e36fd1775791c6bf3e6`, the corresponding public entry point is `flashinfer.fused_moe.trtllm_fp8_block_scale_moe`. Its complete call includes the eight tensors above plus `num_experts=256`, `top_k=8`", "sha256": "9c3a9fd3260dc9da7f6ecf5ccc39d149dcb6a903af942158546e10489b32fd3e"}, {"id": "u029", "kind": "prose", "locator": "body:L67-L67", "preview": "The official reference performs these steps:", "sha256": "69ce47bf087e8ec43d4a017de654b2adcee5426fdc4873b0c0a0e2b44b78a2d7"}, {"id": "u030", "kind": "list-item", "locator": "body:L69-L69", "preview": "1. Convert routing logits to `s = sigmoid(logits)` and form selection scores `s + routing_bias`.", "sha256": "3a336cf3dad815c822d8e002d8dd323c7e07e17b50547f448e040bafd885de53"}, {"id": "u031", "kind": "list-item", "locator": "body:L70-L70", "preview": "2. Reshape 256 scores into eight groups of 32. Sum the top two selection scores in each group, then retain four groups.", "sha256": "6b360ceba71fa8a57766d24eaa3ca8d86046b6171c1d410ce3c19e4be76aee4a"}, {"id": "u032", "kind": "list-item", "locator": "body:L71-L71", "preview": "3. Select eight global experts from the retained groups using the biased selection scores.", "sha256": "11b6dce51cdc636939c8f51b1e46572fe134d200e051272c53c2c59bf13b4534"}, {"id": "u033", "kind": "list-item", "locator": "body:L72-L72", "preview": "4. Form combine weights from the **unbiased** sigmoid values for those eight experts, normalize per token, and multiply by `routed_scaling_factor`.", "sha256": "894f6b0f253fe79c3ec06beb426083fff0499b2824bf87bbda3e96a40c4692d3"}, {"id": "u034", "kind": "list-item", "locator": "body:L73-L73", "preview": "5. For global expert IDs in `[local_expert_offset, local_expert_offset + 32)`, dequantize the relevant activation and weight blocks, compute one W13 projection, split its 4096 columns into two 2048-column halves, apply SwiGLU, and compute W", "sha256": "11671de6844ef72df08d28569ecd72a6d4f81714708312cb527c2cfcbed5510c"}, {"id": "u035", "kind": "list-item", "locator": "body:L74-L74", "preview": "6. Accumulate each local expert result into the token output using that expert's combine weight. Experts outside the local interval contribute nothing on that rank.", "sha256": "94aff12c1b0fd4c8ffcce3f97b9d20436416bd6b958791ca29293066d3a8509c"}, {"id": "u036", "kind": "prose", "locator": "body:L76-L76", "preview": "The W13 representation permits one logical `A @ W13.T` followed by a split. It does not require two separately allocated TMEM accumulators, and the benchmark contract does not prescribe a tcgen05 instruction sequence.", "sha256": "3e1f2f7efc222f4fcd494a9c8f4c0c4cd901badb958318c4f8e60fcf6d6c8784"}, {"id": "u037", "kind": "prose", "locator": "body:L80-L80", "preview": "[`01-routing-plus-fusion-skeleton.py`](../../artifacts/kernels/fused-moe/variants/01-routing-plus-fusion-skeleton.py) is a CPU-checkable, parameterized reference for grouped selection and local W13/SwiGLU/W2 accumulation. It is derived Kern", "sha256": "5bba28addcad2283879b25fb8696bf9be4d51a9ee49b8133a9f12bec14bf1853"}, {"id": "u038", "kind": "prose", "locator": "body:L84-L84", "preview": "The starter-kit evaluation document at commit `75ccd05cafceb0fd1f86be4cd0f2117249463c66` records:", "sha256": "cde284ee48fbdd748f8d3600e66c7ed2497287762035e914798400ff58e76db6"}, {"id": "u039", "kind": "list-item", "locator": "body:L86-L86", "preview": "- bare-metal NVIDIA B200 (`sm_100a`) with clocks locked to `3996,1965`;", "sha256": "7a75f41e06a4d80bd5efe0b54130a8d0d51fea02a0a481c349034c605fec1995"}, {"id": "u040", "kind": "list-item", "locator": "body:L87-L87", "preview": "- container `flashinfer/flashinfer-ci-cu132:20260401-2c675fb`;", "sha256": "664e4bf418b4433598ddd3b63587c5a7fbd5032aa5d703f03fcbfd59e9b59d40"}, {"id": "u041", "kind": "list-item", "locator": "body:L88-L88", "preview": "- CUDA 13.2, Python 3.12, PyTorch 2.12.0+cu132, and Triton 3.6.0;", "sha256": "8f2cdf7f92d9320af77e430548deb1e4a0c44ecd153c72aec5af59d9b51bed06"}, {"id": "u042", "kind": "list-item", "locator": "body:L89-L89", "preview": "- correctness gates `atol=1`, `rtol=0.3`, and matched ratio `0.9` for the MoE command; and", "sha256": "3dd94426d3da0fbbcd98a12ab3f69353a043fbdeda9a53737c85fc6c8324a6ac"}, {"id": "u043", "kind": "list-item", "locator": "body:L90-L90", "preview": "- an arithmetic mean of per-workload `FlashInfer baseline latency / candidate latency` as the single-definition MoE score.", "sha256": "4de08486e0d7f77210459b4a263cd093943ef5d7facc52f44330e366a8fd83f1"}, {"id": "u044", "kind": "prose", "locator": "body:L92-L92", "preview": "The current primary sources do not support the former framework TFLOPS/latency table, its launch counts, or the structured 1262-TFLOPS record. No performance result is retained here. The official trace axis is `seq_len`; relabeling its endp", "sha256": "2a74e722e8395c0bd3a36f6db147654d36e575ab03bc89b76451f3193502d75d"}, {"id": "u045", "kind": "list-item", "locator": "body:L96-L96", "preview": "- Routing produces different token counts per expert, hence different grouped-GEMM M dimensions. A scheduler can mitigate that imbalance; no single bottleneck is universal across token counts and tactics.", "sha256": "4509252cba2ad79785ad002e88756ca7547dcded8949e062f02b0159cdeb7799"}, {"id": "u046", "kind": "list-item", "locator": "body:L97-L97", "preview": "- Small M or partially filled GEMM tiles can reduce utilization. This is a workload-dependent risk, not a retained performance result.", "sha256": "778b5bf014aeac4c0a4c0bbc62ec34555e7b986c03a88beab2e204ff4d198183"}, {"id": "u047", "kind": "list-item", "locator": "body:L98-L98", "preview": "- At the pinned FlashInfer revision, runtime autotuning enumerates valid tactics and selects GEMM1 and GEMM2 tactics over token buckets. Record the exact revision and tuning state in any measurement.", "sha256": "f81de9fa798fe72ce83f43a6c0161e4a6bee5347a381db18bfc7767aee91b202"}, {"id": "u048", "kind": "list-item", "locator": "body:L99-L99", "preview": "- The FP8 element type alone is insufficient for compatibility. Routing method, global/local expert mapping, scale dtype and granularity, tensor layout, activation, and output semantics must all match.", "sha256": "7fa8a79649e679c66099456a574a461b2d8a93822d78cc19407dc800cf27174c"}, {"id": "u049", "kind": "list-item", "locator": "body:L100-L100", "preview": "- CUDA-graph layout requirements are backend-specific. DeepGEMM's masked grouped layout is one documented decode case when the CPU does not know expert token counts; it is not a universal graph requirement.", "sha256": "1b726ae4f1e45208d34afba797a4104fc1687728707f7a82b976fcc90e9bcec1"}, {"id": "u050", "kind": "list-item", "locator": "body:L101-L101", "preview": "- TMA alignment depends on the tensor-map configuration. CUDA 13.2.1 requires a 64-byte-aligned tensor-map object and ordinarily a 16-byte-aligned global base, with additional requirements for selected interleave, dtype, and swizzle modes; ", "sha256": "c489192b4d16ba711c6b5070c6f199816b5ca1ca31d2eca0880d8b3700ba4f60"}, {"id": "u051", "kind": "prose", "locator": "body:L105-L105", "preview": "The files under [`full/`](../../artifacts/kernels/fused-moe/full/) are mixed **adjacent references**, not a full implementation of this benchmark:", "sha256": "91e526c7370778ef42f7ef4e5d413af9fc8ce67b7ba844c2d32247386429a4d8"}, {"id": "u052", "kind": "list-item", "locator": "body:L107-L107", "preview": "- `vllm-PR-23696-dual-gemm.patch` is the aggregate merged diff for vLLM's MXFP4-weight fused expert-compute integration (BF16 activations on Hopper and MXFP8 activations on Blackwell).", "sha256": "eb444460ff069d31acae33faebd1748acbca31b3314448454b206450136a7227"}, {"id": "u053", "kind": "list-item", "locator": "body:L108-L108", "preview": "- `flashinfer_cutedsl.py` is byte-identical to an SGLang FP4 CuteDSL runner at commit `c554dc5c64b661f2c53225b03a76359eaddc39e4`.", "sha256": "4637e22f808fa7be197ed05bf39999b3faef048bfcc7d170b174045f5f9285e7"}, {"id": "u054", "kind": "list-item", "locator": "body:L109-L109", "preview": "- `moe-grouped-gemm-launch.cpp` is local illustrative pseudocode for varying per-expert M segments.", "sha256": "ce5f3f3207b6fccc92cbbd767ab21e5019b9dd4875f37d8ccbeebb95df5387d2"}, {"id": "u055", "kind": "prose", "locator": "body:L111-L111", "preview": "None supplies the exact Track A FP8 implementation or substantiates performance/launch claims. Their hashes and exact scopes are recorded in the bundle provenance.", "sha256": "3185281cb5115e4e603b66baf2ef9a9182277ac604d0c1f3542a0654e29f064d"}, {"id": "u056", "kind": "list-item", "locator": "body:L115-L115", "preview": "- [MLSys 2026 FlashInfer contest](https://mlsys26.flashinfer.ai/)", "sha256": "ab9d2391f0df03c8f838cf0749ecc9726e2d2d3dcb8b884ea6b7375a47370484"}, {"id": "u057", "kind": "list-item", "locator": "body:L116-L116", "preview": "- [Exact FP8 MoE benchmark definition](https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048)", "sha256": "419456d967a9985601dbcaf552faa63a40fd1b1d7b83c31d08a20aaaf87158c0"}, {"id": "u058", "kind": "list-item", "locator": "body:L117-L117", "preview": "- [Starter-kit evaluation contract at `75ccd05`](https://github.com/flashinfer-ai/flashinfer-bench-starter-kit/blob/75ccd05cafceb0fd1f86be4cd0f2117249463c66/EVALUATION.md)", "sha256": "52ca433ee6415278862ce222bb43c605d0c133517e6d0abce32e8221ad225165"}, {"id": "u059", "kind": "list-item", "locator": "body:L118-L118", "preview": "- [FlashInfer trace reference at `7f614b8`](https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/trace/templates/moe.py)", "sha256": "8cb3c878949322786ae8d9707896d528775d5cd7441e618252145f01bc31265d"}, {"id": "u060", "kind": "list-item", "locator": "body:L119-L119", "preview": "- [FlashInfer API implementation at `7f614b8`](https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/fused_moe/core.py)", "sha256": "bd2e86c0528a00b0e278588ee1708fab986c461ffb0f5991e414b6b325d24bd1"}, {"id": "u061", "kind": "list-item", "locator": "body:L120-L120", "preview": "- [CUDA 13.2.1 tensor-map requirements](https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html)", "sha256": "9b0de5e25bc10f3d939d8e71de0c30e132b9e3d13375dea6dbe04499bbeccfa7"}, {"id": "u062", "kind": "list-item", "locator": "body:L121-L121", "preview": "- [CUTLASS efficient-GEMM small-dimension discussion](https://github.com/NVIDIA/cutlass/blob/main/media/docs/cpp/efficient_gemm.md)", "sha256": "4b8edf9932c5821c5b961c770a55f0ce93b8a5ad8b5dbde632639d6ecb14be2b"}, {"id": "u063", "kind": "list-item", "locator": "body:L122-L122", "preview": "- [DeepGEMM grouped-layout documentation at `891d57b4`](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md)", "sha256": "52f8e92ca35c41e0413de525aa48ca4351f19d3d790f77ef8c0ac67f59ef158d"}, {"id": "u064", "kind": "prose", "locator": "body:L124-L124", "preview": "Query the page and its explicitly labeled artifacts with:", "sha256": "b0d28b74774c0b84d20fed48deff776c39a4c6fcda541d401cebc50ea98a12fd"}, {"id": "u065", "kind": "code", "locator": "body:L126-L128", "preview": "```bash python3 scripts/get_page.py kernel-fused-moe --include-code ```", "sha256": "56fc790420908106565f83ed31ab8a2fe1aa1e849fce411f2babc6c449629082"}], "confidence_claimed": "source-reported", "headings": ["Scope", "Fixed Benchmark Geometry", "Tensor Contract", "Reference Semantics", "Small Derived Reference", "Official Evaluation Boundary", "Implementation Risks That Must Be Measured", "Adjacent Local Artifacts", "Sources"], "id": "kernel-fused-moe", "path": "wiki/kernels/fused-moe.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/contests/flashinfer-mlsys26/track-a-fused-moe.md", "url": "https://mlsys26.flashinfer.ai/"}], "risk_flags": ["code", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["contest-flashinfer-track-a"], "title": "FlashInfer Track A FP8 Block-Scale MoE", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "2ed46cc12dc31743da8142c69d5a8c6b3d03b077d1ddd624e2e4351206734b68", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Gated DeltaNet is a recurrent linear-attention architecture published at ICLR 2025. For one head with state `S` shaped `[K,V]`, the FlashInfer reference at commit `7f614b8` implements the update in this equivalent form:", "sha256": "7f1a002328cc8b9c3b1f6c35a1deef36116d2a1d4096548aa30b3067c801835f"}, {"id": "u002", "kind": "code", "locator": "body:L7-L16", "preview": "```python def gdn_step(S, q, k, v, A_log, a, dt_bias, b, scale): g = exp(-exp(A_log) * softplus(a + dt_bias)) beta = sigmoid(b) decayed = g * S read = k @ decayed S_new = decayed + outer(k, beta * (v - read)) output = scale * (q @ S_new) re", "sha256": "238655628f5620f295796b5458731e1de971b0b1ccdfa2a0d61e2b06b3fa42e0"}, {"id": "u003", "kind": "prose", "locator": "body:L18-L18", "preview": "The subtraction is the delta-rule correction: the update moves the value retrieved at `k` toward `v`, while `g` independently decays the previous state. The recurrent state has no prior-token axis, so a single GDN decode step has work and s", "sha256": "16d803b48f532b51a8b71b6c51ff0f3f9190e4d2e05b0c8a0ef5441a605f7b8d"}, {"id": "u004", "kind": "prose", "locator": "body:L22-L22", "preview": "The immutable `Qwen3-Next-80B-A3B-Instruct` configuration at revision `9c7f2fbe` records:", "sha256": "1b817e4aaafd69ac95405b326076ab88f29f0e375a67d696d85c98a5189d6906"}, {"id": "u005", "kind": "table-row", "locator": "body:L26-L26", "preview": "| Field | Value | | Layers | 48 |", "sha256": "946085463385cea8c28640148d5a0ba95abc65a7e0538d49eea336c1585af0cd"}, {"id": "u006", "kind": "table-row", "locator": "body:L27-L27", "preview": "| Field | Value | | Hybrid layout | `12 * (3 * Gated DeltaNet -> MoE, 1 * Gated Attention -> MoE)` |", "sha256": "43a8bb12118eeafcafb77e929c2bf66a28812a33d84ba903805f6cd7b1c9cdc7"}, {"id": "u007", "kind": "table-row", "locator": "body:L28-L28", "preview": "| Field | Value | | GDN heads | 16 QK heads, 32 value heads, head dimension 128 |", "sha256": "d5d8ffebfb2214d6beca8e03e9d6d401c29d9665685c74443db83fae5dec4ec8"}, {"id": "u008", "kind": "table-row", "locator": "body:L29-L29", "preview": "| Field | Value | | MoE | 512 experts; 10 routed experts plus one shared expert |", "sha256": "f1139596c683c28c61884b723cecedcd771548b01a63c987a8fe3ebbebf5080a"}, {"id": "u009", "kind": "table-row", "locator": "body:L30-L30", "preview": "| Field | Value | | Parameters | 80B total, 3B activated |", "sha256": "61c189dd8d4dd018a2ff9b538c66ff49eba3f271d7788adbbbbd6d7e0f65ba4d"}, {"id": "u010", "kind": "table-row", "locator": "body:L31-L31", "preview": "| Field | Value | | Native context | 262,144 tokens |", "sha256": "dedbb3c41735ff173cb7e5dd4d3268d1f782cea4f826d6d9aa280edf8c93b76d"}, {"id": "u011", "kind": "prose", "locator": "body:L33-L33", "preview": "Thus 36 layers use GDN and 12 use full Gated Attention. The fixed GDN state does not eliminate cache growth for the whole hybrid model: the full-attention layers retain their own context-dependent cache.", "sha256": "8a84864b0e8835ae5ce3770898c7bf2d6f6070bee533e10a1defd6d05976ca70"}, {"id": "u012", "kind": "prose", "locator": "body:L35-L35", "preview": "Qwen's architecture article attributes attention-sink and massive-activation mitigation to the output gate in its full Gated Attention path and says the gate *helps* address those effects. That is separate from the GDN recurrence and is not", "sha256": "b4ab98335434f73e44a17c4f19086f9422ce7f719174e37bff04254ec2f36254"}, {"id": "u013", "kind": "prose", "locator": "body:L39-L39", "preview": "The MLSys 2026 organizer identifies Gated Delta Net as Track C and links separate verified definitions captured from Qwen3-Next with tensor parallelism four.", "sha256": "9e27e43efddec8992b6b8e8b134c39974693821ee00eccf0f80a1e9c3f8a710c"}, {"id": "u014", "kind": "table-row", "locator": "body:L43-L43", "preview": "| Axis | Decode | Prefill | | Q heads / K heads / V heads | `4 / 4 / 8` | `4 / 4 / 8` |", "sha256": "608f12dc8ccb9fe4ac4915f14a3fc2493bb1c1c840590841c0b9afecb2d79739"}, {"id": "u015", "kind": "table-row", "locator": "body:L44-L44", "preview": "| Axis | Decode | Prefill | | Head size | `128` | `128` |", "sha256": "6edfead2304585b0313ee9d8f869b1dbc7d5f853c280773ec08658bf3b4cda2c"}, {"id": "u016", "kind": "table-row", "locator": "body:L45-L45", "preview": "| Axis | Decode | Prefill | | Token axis | `seq_len=1`, variable `batch_size` | variable `total_seq_len` and `num_seqs` |", "sha256": "7d093c09bb19d24891809d47bd910864fa29423c4c5c1d4f0adb71d13e613c7b"}, {"id": "u017", "kind": "table-row", "locator": "body:L46-L46", "preview": "| Axis | Decode | Prefill | | Q/K/V dtype | BF16 | BF16 |", "sha256": "0d788ff950f9fdc86e3ea3b865934a730c855b2eb1d69c9d90d33069de2a4709"}, {"id": "u018", "kind": "table-row", "locator": "body:L47-L47", "preview": "| Axis | Decode | Prefill | | State | FP32 `[B,8,128,128]` | FP32 `[N,8,128,128]` |", "sha256": "6db808accb7b425a34818b1664bb85a644393394b736de9dbd28cd43f9387c73"}, {"id": "u019", "kind": "table-row", "locator": "body:L48-L48", "preview": "| Axis | Decode | Prefill | | Variable-length metadata | none | `cu_seqlens[N+1]` |", "sha256": "0eaae7797d9a3cebb8c71a2b1dea8900102ff17d4f0e9d5f87ebe5951f0c6440"}, {"id": "u020", "kind": "prose", "locator": "body:L50-L50", "preview": "Decode exposes `A_log`, `a`, `dt_bias`, and `b` for the decay and update gates, plus an optional scale. It returns BF16 output `[B,1,8,128]` and the updated state. Prefill returns BF16 output `[total_seq_len,8,128]` and one final state per ", "sha256": "43aba269d3b16aa91720754e667431d3283435c07c4f92b93655a49eaf035da4"}, {"id": "u021", "kind": "prose", "locator": "body:L52-L52", "preview": "For this exact geometry, one value-head state contains `128*128 = 16,384` FP32 values, or 64 KiB. All eight value heads contain 131,072 FP32 values, or 512 KiB, per sequence and layer. The heads are independent states; they must not be flat", "sha256": "513221db2de8f60f910249de33c604f9ed02833848b8d165fb8215ac1d7fb0db"}, {"id": "u022", "kind": "list-item", "locator": "body:L56-L56", "preview": "- NVlabs commit `b53d6d3` is the authors' PyTorch/Triton research implementation. Its README recommends FLA for faster kernels and variable-length functionality.", "sha256": "b186508f658ac67e9ff28e06e830efa5e6972892b33fdc0d30ccec4619b8acda"}, {"id": "u023", "kind": "list-item", "locator": "body:L57-L57", "preview": "- FlashInfer commit `7f614b8` supplies the exact decode/prefill reference contracts used above. Its prefill wrapper dispatches SM90 and SM100 implementations; the SM100/SM103 path uses CuTe DSL, requires CUDA 13 or newer, and fixes head siz", "sha256": "1296fe4efcbeb2068668918ed0d0b9606bbd86aff998e2ce44e2d10878367782"}, {"id": "u024", "kind": "list-item", "locator": "body:L58-L58", "preview": "- vLLM merge `e1d85e5c` gives its recurrent-attention backend uniform-batch CUDA-graph support for decode. CUDA graphs reduce CPU launch setup cost, but their benefit remains workload-dependent.", "sha256": "0c73444f9fba3005268f924d3c89a71841e0e27a6c412c47a2b72ac755f560e3"}, {"id": "u025", "kind": "prose", "locator": "body:L60-L60", "preview": "TFLA (`arXiv:2503.14376v3`) is related linear-recurrence work, but its published application and official code are for mLSTM. They use a second level of sequence parallelization to permit arbitrarily large chunks. They do not establish a GD", "sha256": "d24978b3eedc271f926ac0698ec40c0e00fb2108015605df35e19a89a6b6c0a6"}, {"id": "u026", "kind": "prose", "locator": "body:L64-L64", "preview": "GDN sequence mixing scales linearly with sequence length, but asymptotic complexity is not a measured speedup. Qwen reports a 10x whole-model inference-throughput comparison against Qwen3-32B for contexts over 32K, while also warning that e", "sha256": "0588f704c563cf93d4f648c1373112d145da7816d55865983d6fda562e546f38"}, {"id": "u027", "kind": "prose", "locator": "body:L66-L66", "preview": "Use an exact backend measurement for the intended batch, sequence distribution, dtype, state layout, software revision, and GPU. Also account for the context-growing full-attention cache in a hybrid model and for the GDN layer's learned pro", "sha256": "6fb643ad658430bb3d27cf33a44f6f40d0622ecc8df9d671c654775ef5de9939"}, {"id": "u028", "kind": "prose", "locator": "body:L70-L70", "preview": "The `full/` bundle contains one byte-pinned SGLang file from merge `5bdc07d974f6cf236fa765a685453ea5e587a838`. It fuses projection-output split/reshape/concatenation for Qwen3-Next/Qwen3.5; it is adjacent preprocessing, not a GDN recurrence", "sha256": "0590e300b7bc24d2e90144dfc93d63cf442145c81ae01ecae2f79b36b2ff4900"}, {"id": "u029", "kind": "prose", "locator": "body:L72-L72", "preview": "The `variants/` bundle contains a standard-library, one-head recurrence check derived from the FlashInfer reference. It checks the compact update against the independently expanded remove/write form and rejects an additive-only negative con", "sha256": "37f2ad221070aa4000f2e973f0232981aa09858467201df9cdf9ab264fbbc659"}, {"id": "u030", "kind": "list-item", "locator": "body:L76-L76", "preview": "- [Gated DeltaNet paper](https://arxiv.org/abs/2412.06464)", "sha256": "8662205df9900a09dff461d8835151b4a2d42182f4e0e9561e627f98f1c73266"}, {"id": "u031", "kind": "list-item", "locator": "body:L77-L77", "preview": "- [NVlabs implementation at `b53d6d3`](https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921)", "sha256": "cc161a1de495037a16854827359909e34621400f7276072204783ac331c966a9"}, {"id": "u032", "kind": "list-item", "locator": "body:L78-L78", "preview": "- [Qwen3-Next model card and configuration at `9c7f2fbe`](https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/tree/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7)", "sha256": "eb17392aa94fbd81a705594b627157c7e76f9758aacf7facd3b7bf73bfe69358"}, {"id": "u033", "kind": "list-item", "locator": "body:L79-L79", "preview": "- [MLSys 2026 organizer](https://mlsys26.flashinfer.ai/)", "sha256": "9879d2939532834b51ce2f9a3244e19f5a3592715cba9e9ebcf0f4acc6abd873"}, {"id": "u034", "kind": "list-item", "locator": "body:L80-L80", "preview": "- [Exact decode definition](https://bench.flashinfer.ai/kernels/gdn_decode_qk4_v8_d128_k_last)", "sha256": "66f2aadc7e3c3d63be85ae7ae8769384d817cf4b29f2d42e0cbbb3d4748d23eb"}, {"id": "u035", "kind": "list-item", "locator": "body:L81-L81", "preview": "- [Exact prefill definition](https://bench.flashinfer.ai/kernels/gdn_prefill_qk4_v8_d128_k_last)", "sha256": "34afa7b5621d3ae6f066cee9c115cff69ac9d1ce16578aa321af2d8412f926bc"}, {"id": "u036", "kind": "list-item", "locator": "body:L82-L82", "preview": "- [FlashInfer GDN trace at `7f614b8`](https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/trace/templates/gdn.py)", "sha256": "96b113bf9fd4cd978caec86d436535866be7d1f7110015a0b9d7ea435f0d1517"}, {"id": "u037", "kind": "list-item", "locator": "body:L83-L83", "preview": "- [FlashInfer prefill dispatch at `7f614b8`](https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/gdn_prefill.py)", "sha256": "a37d4c684a13866403e69c1724cd28c8537129bd7bfc615cd99a56d812157803"}, {"id": "u038", "kind": "list-item", "locator": "body:L84-L84", "preview": "- [CUDA Graphs programming guide](https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cuda-graphs.html)", "sha256": "47c94156d8098a16acf12043aa181f4430a198fb4eb71ed276faa0a14ef4df53"}, {"id": "u039", "kind": "list-item", "locator": "body:L85-L85", "preview": "- [TFLA v3](https://arxiv.org/abs/2503.14376v3)", "sha256": "1ced9ef7ae3ddb711eb326066c0711834b8cbb0d2bc908dca9a3df3d8f05ded4"}, {"id": "u040", "kind": "prose", "locator": "body:L87-L87", "preview": "Query via:", "sha256": "6abd63e508f9320fdd3b2b0cc5b5f6a000f4462abfd17f9ecc3c4818f4a0f875"}, {"id": "u041", "kind": "code", "locator": "body:L89-L91", "preview": "```bash conda run -n base python scripts/get_page.py kernel-gated-delta-net --include-code ```", "sha256": "754cf209695ee6f492b3af36cbc5a6d9f54c38099ba74eec7ec708e335fba999"}], "confidence_claimed": "source-reported", "headings": ["Verified mechanism", "Qwen3-Next architecture", "FlashInfer-Bench Track C contracts", "Implementations and architecture scope", "Performance boundary and use", "Local artifacts", "Primary sources"], "id": "kernel-gated-delta-net", "path": "wiki/kernels/gated-delta-net.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/gated-delta-net.md", "url": "https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921"}, {"path": "sources/contests/flashinfer-mlsys26/track-c-gated-delta-net.md", "url": "https://mlsys26.flashinfer.ai/"}, {"path": "sources/blogs/qwen3-next-architecture.md", "url": "https://developer.nvidia.com/blog/new-open-source-qwen3-next-models-preview-hybrid-moe-architecture-delivering-improved-accuracy-and-accelerated-parallel-processing-across-nvidia-platform/"}, {"path": "sources/docs/tfla.md", "url": "https://arxiv.org/abs/2503.14376v3"}, {"path": "sources/prs/vllm/PR-37303.md", "revision": "e1d85e5c", "url": "https://github.com/vllm-project/vllm/pull/37303"}], "risk_flags": ["code", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-gated-delta-net", "contest-flashinfer-track-c", "blog-qwen3-next-architecture", "doc-tfla", "pr-vllm-37303"], "title": "Gated Delta Net \u2014 Linear Attention", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "11f0038a8e9f681dac4ffd25654362cefa9f94a039e46745ac55f9488a405f80", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "The official NVIDIA rules identify NVFP4 Gated Dual GEMM as Kernel Challenge 3 of the Blackwell NVFP4 Hackathon, open from December 20, 2025 through January 16, 2026. The public GPU Mode problem at challenge-opening commit `c5b2f7c` targets", "sha256": "eb13cd2340105068b914156cb7d5d1e808b3f463ab2acb8a78db6fa1f3c5020c"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "For each batch index, the required result is two block-scaled matrix products sharing the same left operand, with SiLU applied only to the first branch:", "sha256": "a8a01d217423ed48389944aef48ae23ca65b467f846332a0b3a7f9b4f99b1704"}, {"id": "u003", "kind": "code", "locator": "body:L9-L14", "preview": "```python def gated_dual_result(a, b1, b2, scale_a, scale_b1, scale_b2, scaled_mm): gate = scaled_mm(a, b1.T, scale_a, scale_b1) up = scaled_mm(a, b2.T, scale_a, scale_b2) return silu(gate) * up ```", "sha256": "dbebdd2007ca6c67649f785252795530745a124fdd528eef5cdd9a9e3c41974a"}, {"id": "u004", "kind": "prose", "locator": "body:L16-L16", "preview": "This fixes result semantics and branch order. It does not require the submission entry point to use one GPU launch or forbid intermediate storage.", "sha256": "20fbacace5607f8e72cc6cf0b019f48f71788aa01d28640b2e62c771fb961479"}, {"id": "u005", "kind": "table-row", "locator": "body:L22-L22", "preview": "| Tensor | Published dtype | Logical shape | | `a` | NVFP4 E2M1 | `[M,K,L]`, K-major |", "sha256": "c51027e2c1727c7e1eebc42cd4417a53e5e5c7249152c256c969ef3dcf2e456d"}, {"id": "u006", "kind": "table-row", "locator": "body:L23-L23", "preview": "| Tensor | Published dtype | Logical shape | | `b1`, `b2` | NVFP4 E2M1 | `[N,K,L]`, K-major |", "sha256": "80489fc9f617fc818771c3642171bf4225506614fba752cb1466f9983e217ea5"}, {"id": "u007", "kind": "table-row", "locator": "body:L24-L24", "preview": "| Tensor | Published dtype | Logical shape | | `sfa` | FP8 E4M3FNUZ | `[M,K/16,L]`, K-major |", "sha256": "825e69aaf043da6d8f8b6bc86989e1507d940ba110183e27578ec296f251de8a"}, {"id": "u008", "kind": "table-row", "locator": "body:L25-L25", "preview": "| Tensor | Published dtype | Logical shape | | `sfb1`, `sfb2` | FP8 E4M3FNUZ | `[N,K/16,L]`, K-major |", "sha256": "0b77d5e56a0c9e095f5346f5b105840fd07e48ee57eb311855c97903cd89091a"}, {"id": "u009", "kind": "table-row", "locator": "body:L26-L26", "preview": "| Tensor | Published dtype | Logical shape | | `c` | FP16 | `[M,N,L]` |", "sha256": "91c57469d3b6cdb36340525d8683e7a8163d63ff0f0a02ed93b436a665cd15f4"}, {"id": "u010", "kind": "prose", "locator": "body:L28-L28", "preview": "The submission tuple also supplies layout-reordered copies of `sfa`, `sfb1`, and `sfb2`, plus preallocated `c`. `K` is divisible by 256; `M` and `N` must be divisible by the selected MMA tile dimensions. The correctness checker uses `rtol=1", "sha256": "a9160f4022cf1f167223f0ac260c9a5f45e6e26c1bd33db1bd277f2ef7a29e92"}, {"id": "u011", "kind": "prose", "locator": "body:L30-L30", "preview": "The pinned upstream files contain one dtype-label inconsistency: `task.yml` and `template.py` call the scales E4M3FNUZ, while `reference.py` constructs `torch.float8_e4m3fn`. A backend must follow the actual submission objects rather than t", "sha256": "bed832ba69cda0b6bef5b927ab33a5a660cb6e0ca5943dd345d1f4a4ea9ffc7d"}, {"id": "u012", "kind": "table-row", "locator": "body:L36-L36", "preview": "| M | N | K | L | Task speed-of-light time (\u00b5s) | | 256 | 4096 | 7168 | 1 | 4.708 |", "sha256": "7411db857d7dcc6631366727895a460e35c7d547a4efc9906e770fea0e5fa848"}, {"id": "u013", "kind": "table-row", "locator": "body:L37-L37", "preview": "| M | N | K | L | Task speed-of-light time (\u00b5s) | | 512 | 4096 | 7168 | 1 | 8.714 |", "sha256": "b4d7a0c0e8410c7ab28ed355f9adaa78d0e41a6955d60489d1eb86411de35d7d"}, {"id": "u014", "kind": "table-row", "locator": "body:L38-L38", "preview": "| M | N | K | L | Task speed-of-light time (\u00b5s) | | 256 | 3072 | 4096 | 1 | 2.125 |", "sha256": "1bce4631f28966ecc639c587fe0a817eedd229d39bcfb63736463d5af08d1173"}, {"id": "u015", "kind": "table-row", "locator": "body:L39-L39", "preview": "| M | N | K | L | Task speed-of-light time (\u00b5s) | | 512 | 3072 | 7168 | 1 | 6.535 |", "sha256": "0d114fedb3663c1f794288403e6bb6237a8fc7403af83090d0c68576a704b7ff"}, {"id": "u016", "kind": "prose", "locator": "body:L41-L41", "preview": "Ranking uses the geometric mean of the four benchmark times. The task explicitly labels the final column a speed-of-light analysis based on the maximum of B200 FP4 Tensor Core math time and DRAM-memory time at a 1.5 GHz clock. These values ", "sha256": "dbd703852dc16a02bacb2b9359f855f7cc6bb5aedbbe12766b5359b12b4c3df3"}, {"id": "u017", "kind": "prose", "locator": "body:L45-L45", "preview": "The shared `a` and `sfa` inputs create an opportunity to reuse data across the two products, and a genuinely fused epilogue can avoid writing both full-precision products to global memory. Those are optimization possibilities, not task guar", "sha256": "b9506a581d543f3f3449095a2de708afaf0cc2d3a4074808d129cf966b6143d6"}, {"id": "u018", "kind": "prose", "locator": "body:L47-L47", "preview": "The public task/reference does not establish a final leaderboard, winning source, single-launch decomposition, physical load count, TMEM accumulator partition, TMA pipeline, CUTLASS schedule, or compute-/memory-bound classification. If an i", "sha256": "7eebde5afbe8c3125f5c62849cf9a347608afc921f13de479637aa8d16a48e84"}, {"id": "u019", "kind": "prose", "locator": "body:L49-L49", "preview": "Compatibility is correspondingly narrower than \u201cany dual-output operation\u201d: a model/backend must match the two same-shaped NVFP4 products, scale storage and layouts, first-branch SiLU, FP16 output, batching, and divisibility rules. Grouped ", "sha256": "de73a447abc831be11c8a20216e9aa9deb202edda82ffd8e4b5fc04f855730f1"}, {"id": "u020", "kind": "prose", "locator": "body:L53-L53", "preview": "The `full/` bundle contains the byte-pinned official `task.yml` from commit `c5b2f7c`; it is the problem specification, not an optimized submission. The unrelated third-party schedule extract and duplicate vLLM MXFP4 MoE patch formerly stor", "sha256": "a38ab1c3e4bbe45a644a36ccc7d310f1d64e15abe5098570c0957bb0d2796105"}, {"id": "u021", "kind": "prose", "locator": "body:L55-L55", "preview": "The `variants/` bundle contains a standard-library semantic reference for two small dense products followed by `SiLU(gate) * up`. Its self-test compares compact and expanded forms and rejects the wrong alternative that gates the second bran", "sha256": "dce2ea09300743713bd166cc06f0b1dc512d956b3e4ab65c423b66a9d18ffc9c"}, {"id": "u022", "kind": "list-item", "locator": "body:L59-L59", "preview": "- [Official challenge rules](https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf)", "sha256": "1f5c0e65cebd680067dc91e0fa9151877d2bab0124675326a188474f3053ddae"}, {"id": "u023", "kind": "list-item", "locator": "body:L60-L60", "preview": "- [Pinned public task](https://github.com/gpu-mode/reference-kernels/blob/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm/task.yml)", "sha256": "0cddd0c030bce47ece25fb01646203081b9a5e6c128f164b8552a7c8a671b84a"}, {"id": "u024", "kind": "list-item", "locator": "body:L61-L61", "preview": "- [Pinned starter template](https://github.com/gpu-mode/reference-kernels/blob/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm/template.py)", "sha256": "9e4752cc881230defb5421be419a54115184bf44943fe7b7f55fc43f8ec8ccc5"}, {"id": "u025", "kind": "list-item", "locator": "body:L62-L62", "preview": "- [Pinned correctness reference](https://github.com/gpu-mode/reference-kernels/blob/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm/reference.py)", "sha256": "33537e2a914a4c1604f1a972e3ff6d137cba06b2e245ee8fa0373bb396f73ce5"}, {"id": "u026", "kind": "prose", "locator": "body:L64-L64", "preview": "Query via:", "sha256": "6abd63e508f9320fdd3b2b0cc5b5f6a000f4462abfd17f9ecc3c4818f4a0f875"}, {"id": "u027", "kind": "code", "locator": "body:L66-L68", "preview": "```bash conda run -n base python scripts/get_page.py kernel-gated-dual-gemm --include-code ```", "sha256": "45d0054e609734a050470028979c69f41c63c9e9f1d7d447ab961af3e447cdbf"}], "confidence_claimed": "source-reported", "headings": ["Verified scope", "Published tensor contract", "Published workloads and theoretical bounds", "Implementation boundary", "Local artifacts", "Primary sources"], "id": "kernel-gated-dual-gemm", "path": "wiki/kernels/gated-dual-gemm.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md", "url": "https://github.com/gpu-mode/reference-kernels/tree/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["contest-gpumode-p3"], "title": "GPU Mode NVFP4 Gated Dual GEMM", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "733a51f41033ffcc0ff6fc42c4813e72da9b342f2e17aa629cfbee80177927ce", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "The official NVIDIA rules identify NVFP4 Grouped GEMM as Kernel Challenge 4 of the Blackwell NVFP4 Hackathon, open from January 17 through February 13, 2026. It carries 40% of the four-problem grand-prize score. The corrected public task at", "sha256": "1639b7acf7b23b5b3321612131bdeb27e99062306a5f8603dfffeed74f69e18d"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "\u201cGrouped GEMM\u201d does not imply one universal shape contract. Two relevant interfaces differ materially:", "sha256": "5b62624db91dbdc4fd13581947cfda51890c1a7856fb2d73124be674dbe12784"}, {"id": "u003", "kind": "list-item", "locator": "body:L9-L9", "preview": "- The GPU Mode challenge accepts a list of independent problems; `M_i`, `N_i`, and `K_i` can all differ by group.", "sha256": "b9b8d88536cad39bc38b2372af39b03590aa774e4505cb406874ddcaa776f659"}, {"id": "u004", "kind": "list-item", "locator": "body:L10-L10", "preview": "- DeepGEMM's M-grouped MoE APIs vary M while holding N and K fixed. It separately provides a K-grouped interface for MoE weight backward.", "sha256": "096a1dea2f4799d14e774f4856385805b30814ef51ea5daaa6c7382112c21f76"}, {"id": "u005", "kind": "prose", "locator": "body:L12-L12", "preview": "Neither interface definition proves that a conforming implementation uses exactly one GPU launch.", "sha256": "7d2218cba139ae1b298d6c15bc8defc2c2639149b272d77e8be8a87e6a7c19b5"}, {"id": "u006", "kind": "prose", "locator": "body:L16-L16", "preview": "For each group `i`, the correctness reference computes `C_i = A_i @ B_i.T` with block scales and FP16 output.", "sha256": "821caf2f5b552a4124fd374bc92a0cb5f3ea499948da59300ecb510e2ec28519"}, {"id": "u007", "kind": "table-row", "locator": "body:L20-L20", "preview": "| Per-group value | Published dtype | Logical shape | | `a_i` | packed NVFP4 E2M1, two values per byte | `[M_i,K_i/2,L_i]` |", "sha256": "d34b857726733a2afa24e47f8733e8d6e6f043b335a81b2dc85328bc258e8e30"}, {"id": "u008", "kind": "table-row", "locator": "body:L21-L21", "preview": "| Per-group value | Published dtype | Logical shape | | `b_i` | packed NVFP4 E2M1, two values per byte | `[N_i,K_i/2,L_i]` |", "sha256": "6a6a65388a18d8ec1b7e3c83cf8c579b997842d51ed159132c3cb63f9c6f1bd7"}, {"id": "u009", "kind": "table-row", "locator": "body:L22-L22", "preview": "| Per-group value | Published dtype | Logical shape | | `c_i` | FP16 | `[M_i,N_i,L_i]` |", "sha256": "c30f983df15fb59e81796b93a1a61b5fe6637462a59524c4e36e2ed9f4e4e4eb"}, {"id": "u010", "kind": "table-row", "locator": "body:L23-L23", "preview": "| Per-group value | Published dtype | Logical shape | | `sfa_i` | FP8 E4M3FNUZ in task/template | `[M_i,K_i/16,L_i]` |", "sha256": "6f30839adec711adb38f9c03cf9fb078c7a66a155f9836da11ca60ff7c28b8f3"}, {"id": "u011", "kind": "table-row", "locator": "body:L24-L24", "preview": "| Per-group value | Published dtype | Logical shape | | `sfb_i` | FP8 E4M3FNUZ in task/template | `[N_i,K_i/16,L_i]` |", "sha256": "fb577a5c677951b74064836772973ae414b66c4eb9c75785449ac7326f388696"}, {"id": "u012", "kind": "table-row", "locator": "body:L25-L25", "preview": "| Per-group value | Published dtype | Logical shape | | problem size | integers | `(M_i,N_i,K_i,L_i)` |", "sha256": "5e238b762375b33910bc4972916f29f1235d1a043f2fe6c3ba0f41f3b856e448"}, {"id": "u013", "kind": "prose", "locator": "body:L27-L27", "preview": "The submission object actually contains four lists: logical A/B/C tensors, logical scales, reordered scale copies, and problem sizes. The `task.yml` prose lists only three tuple members, but `task.py`, `template.py`, and `reference.py` expo", "sha256": "7257e0924104d32452daba7c843a68a82c1eaf3d35326fbf8e0be92291040ffe"}, {"id": "u014", "kind": "prose", "locator": "body:L29-L29", "preview": "The pinned upstream files also disagree on the scale dtype suffix: task/template text says E4M3FNUZ, while `reference.py` constructs `torch.float8_e4m3fn`. A submission must follow the actual tensors it receives rather than silently treatin", "sha256": "5e4eee6cc002895da478fe96cc270e4b91af5cd04101f51a2d6cefcab5dd6cf2"}, {"id": "u015", "kind": "prose", "locator": "body:L31-L31", "preview": "For small non-empty ordinary-number matrices, this CPU-only reference isolates the group semantics. Each `b` is stored as `[N,K]`, so its rows are the columns of the mathematical right operand:", "sha256": "c320379f94fb28a61cb7701495a1d6932da8855cefe44c8cff6bbf150cb886ae"}, {"id": "u016", "kind": "code", "locator": "body:L33-L45", "preview": "```python def grouped_reference(groups): outputs = [] for a, b in groups: k = len(a[0]) if any(len(row) != k for row in a) or any(len(row) != k for row in b): raise ValueError(\"ragged or incompatible K\") outputs.append([ [sum(x * y for x, y", "sha256": "d4407b78235c5840c595cce75e98468022e0ca987e0d8fc61a22a961b679a537"}, {"id": "u017", "kind": "prose", "locator": "body:L47-L47", "preview": "It models neither NVFP4 packing nor block scales; those remain part of the challenge ABI above.", "sha256": "db4cf3e6019dbe5437fe573745b89e7c26bd66bf1eb1bde7f5476bee73e47323"}, {"id": "u018", "kind": "prose", "locator": "body:L51-L51", "preview": "The following shapes describe the pinned FP8/FP4 grouped APIs; scale tensors and layout conversions are additional required inputs.", "sha256": "b668b60f58e395720c62a89ca031cf80584385c6778c5de7785c935b7919a176"}, {"id": "u019", "kind": "table-row", "locator": "body:L55-L55", "preview": "| Mode | Main tensors | Group descriptor | Documented role | | M-grouped contiguous | A `[M,K]`, B `[G,N,K]`, D `[M,N]` | `grouped_layout` is either one expert ID per packed row (with `-1` padding) or one prefix-sum end per group | Training", "sha256": "3edecad8eeab41879305a4720468fe2b3a039c304de9a2f6ed4b2f7554929c39"}, {"id": "u020", "kind": "table-row", "locator": "body:L56-L56", "preview": "| Mode | Main tensors | Group descriptor | Documented role | | M-grouped masked | A `[G,Mmax,K]`, B `[G,N,K]`, D `[G,Mmax,N]` | `masked_m` is one int32 valid-row count per group | CUDA-graph decode; fixed allocation while computing valid po", "sha256": "afe0b21b4ba74a5b245f7b7473fa0a5db9141c4c404a861f5e32a9ee39ab9a09"}, {"id": "u021", "kind": "table-row", "locator": "body:L57-L57", "preview": "| Mode | Main tensors | Group descriptor | Documented role | | K-grouped contiguous | packed A `[sum(K_i),M]`, B `[sum(K_i),N]`, D `[G,M,N]` | host and device K-size lists; optional C has `[G,M,N]` | MoE weight backward; M/N fixed |", "sha256": "3e21ac0d9e7eca36eedbde09c1f5b847faa5edff155b273eb488f4168043f2ea"}, {"id": "u022", "kind": "prose", "locator": "body:L59-L59", "preview": "These are concrete library contracts, not generic C++ structs. Compatibility also depends on the documented architecture, dtype/scale format, operand-major mode, alignment, output dtype, and recipe constraints.", "sha256": "37db519c8bd9df805acef16357b537f98b78a4dfd8e3225ebb06c1e117d4b21f"}, {"id": "u023", "kind": "table-row", "locator": "body:L65-L65", "preview": "| Groups | M values | N | K | L | Task speed-of-light time (\u00b5s) | | 8 | 80, 176, 128, 72, 64, 248, 96, 160 | 4096 | 7168 | 1 | 18.833 |", "sha256": "1ca9581034e9fe688c74c377c0708dba1f3ea1092c93cd5eeb893e2f77e6a0dd"}, {"id": "u024", "kind": "table-row", "locator": "body:L66-L66", "preview": "| Groups | M values | N | K | L | Task speed-of-light time (\u00b5s) | | 8 | 40, 76, 168, 72, 164, 148, 196, 160 | 7168 | 2048 | 1 | 10.667 |", "sha256": "b9c58ac0eb054106541e7aff9bd5a9638104efc3af03f3848506f060f3470154"}, {"id": "u025", "kind": "table-row", "locator": "body:L67-L67", "preview": "| Groups | M values | N | K | L | Task speed-of-light time (\u00b5s) | | 2 | 192, 320 | 3072 | 4096 | 1 | 2.406 |", "sha256": "4f0fa644456b878bcf99df776d2cd88881bb05b73802300f95d8735110732d52"}, {"id": "u026", "kind": "table-row", "locator": "body:L68-L68", "preview": "| Groups | M values | N | K | L | Task speed-of-light time (\u00b5s) | | 2 | 128, 384 | 4096 | 1536 | 1 | 1.525 |", "sha256": "008c90daa24d22e14b488ac75949db726004748e92e59940c7c9b91a438e36cf"}, {"id": "u027", "kind": "prose", "locator": "body:L70-L70", "preview": "Ranking uses the geometric mean. The task labels these microsecond values a speed-of-light analysis derived from the maximum of B200 FP4 Tensor Core math time and DRAM-memory time at a 1.5 GHz clock. They are theoretical comparison values, ", "sha256": "6465beb0e7a292f48bf4e830c3442e6a8b1a8b71c37567dc903a1a346d91299a"}, {"id": "u028", "kind": "prose", "locator": "body:L74-L74", "preview": "GPU Mode's official postmortem records a submission that temporarily reached the number-one leaderboard position with a reported `11.191 \u00b5s`, roughly `2 \u00b5s` ahead of the next entry, and was scrubbed minutes after the competition.", "sha256": "b537c11c34b663ab971a605f55df3a11f215899dc99d34622e205983403d5c10"}, {"id": "u029", "kind": "prose", "locator": "body:L76-L76", "preview": "During correctness, it ran a real padded 8-group kernel on each of 15 cloned data objects. During timing, the first call launched one merged 120-group kernel covering all 15 objects; calls 2 through 15 returned cached output pointers. The h", "sha256": "fb7cc6e225efe9c103ee51aa0acf7fcb118fd3215154625920012cf1d6418a1c"}, {"id": "u030", "kind": "prose", "locator": "body:L78-L78", "preview": "The official post points to `gpu-mode/reference-kernels` PR #104 as the harness response. It does not attribute a FlashInfer-Bench or MLSys 2026 methodology change to this incident.", "sha256": "af363322f4bd273ed6d9542a802080bb526828195a6a0717566c17eb3054ab47"}, {"id": "u031", "kind": "prose", "locator": "body:L82-L82", "preview": "The public challenge constrains outputs, tolerances, workloads, and scoring. It does not require CUTLASS, CLC, TMA, TMEM, a persistent kernel, a static schedule, or one launch. The former CUTLASS and CUDA sketches were removed because they ", "sha256": "1fe62fa8512775ce291aec39d4653fbc7987d9e8df405c3f1f4988b6c8e2afe8"}, {"id": "u032", "kind": "prose", "locator": "body:L84-L84", "preview": "CUTLASS documents that small M or N can leave threads outside the useful problem bounds, and that a small M/N grid with large K can launch too few threadblocks to use every multiprocessor. This is a possible shape effect, not proof that eve", "sha256": "19b76be52212e350c7ab2a813e5df59f97f417436e5b78689a9a8ce75a9685bf"}, {"id": "u033", "kind": "prose", "locator": "body:L86-L86", "preview": "CLC itself uses an asynchronous cancellation request, shared response, mbarrier completion, and response decoding after a worker's initial block. That is different from a global `atomicAdd` tile queue. Whether CLC improves end-to-end time r", "sha256": "bce4d68a2a5f9d951609827c054da927ee679655ed5bf782d54e3e2e9d1bc468"}, {"id": "u034", "kind": "prose", "locator": "body:L88-L88", "preview": "TMA does not have one universal 128-byte alignment rule from which minimum tile sizes follow. CUDA 13.2.1's tiled tensor-map API generally documents 16-byte global-address and stride alignment, a 64-byte descriptor, and feature-specific 32-", "sha256": "25a89a65ad00ad6324e5c598533ef593b1deadb0dc6f50e647167551470242a3"}, {"id": "u035", "kind": "list-item", "locator": "body:L92-L92", "preview": "- [Official challenge rules](https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf)", "sha256": "1f5c0e65cebd680067dc91e0fa9151877d2bab0124675326a188474f3053ddae"}, {"id": "u036", "kind": "list-item", "locator": "body:L93-L93", "preview": "- [Pinned public task](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/task.yml)", "sha256": "c1bba884e3eee0f733185950f50932694f315394494566c74f0f208bd766e437"}, {"id": "u037", "kind": "list-item", "locator": "body:L94-L94", "preview": "- [Pinned starter template](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/template.py)", "sha256": "5aa4b241a5e4e3866f3fd5385c9ff42499d7ff5ebc088ffdedc512c0613564c6"}, {"id": "u038", "kind": "list-item", "locator": "body:L95-L95", "preview": "- [Pinned correctness reference](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/reference.py)", "sha256": "9f16a390d51604967a3cfb56bdc8255ef16fb2abcfb41b5175e73768643bd652"}, {"id": "u039", "kind": "list-item", "locator": "body:L96-L96", "preview": "- [Pinned DeepGEMM grouped overview](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md#grouped-gemms-contiguous-layout)", "sha256": "e645b512d46f8042349c24ca0a7976640909a20681c8a710160e96a2dc398660"}, {"id": "u040", "kind": "list-item", "locator": "body:L97-L97", "preview": "- [Pinned DeepGEMM grouped APIs](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp)", "sha256": "d3f54e38c58497a1a89b916f67253dc71d49a276cb93d040ee7fdcb225de6ed7"}, {"id": "u041", "kind": "list-item", "locator": "body:L98-L98", "preview": "- [Official reward-hack postmortem](https://www.gpumode.com/news/reward-hacking-nvfp4)", "sha256": "8197e7f31a17b0bb68c8301efb12308f82ad01c1abb3e1f1f82afb55fe1cd3f8"}, {"id": "u042", "kind": "list-item", "locator": "body:L99-L99", "preview": "- [CUDA 13.2.1 tensor-map constraints](https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html)", "sha256": "2ce8fa29f655906e7c265b4f2dcf842b58186c862ebf8809d5b5e51ecc9f5225"}, {"id": "u043", "kind": "list-item", "locator": "body:L100-L100", "preview": "- [CUDA 13.2 CLC programming guide](https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html)", "sha256": "aba48c8e1d6d8f310707cb1fb96cb16cf010f530efda480048d46518b5b4ccdc"}, {"id": "u044", "kind": "list-item", "locator": "body:L101-L101", "preview": "- [Pinned CUTLASS efficient-GEMM guide](https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/media/docs/cpp/efficient_gemm.md)", "sha256": "e99485d7254d7868a51325c7d98f9bd12104f73f87855028bf6ca337c5225982"}, {"id": "u045", "kind": "prose", "locator": "body:L103-L103", "preview": "Query via:", "sha256": "6abd63e508f9320fdd3b2b0cc5b5f6a000f4462abfd17f9ecc3c4818f4a0f875"}, {"id": "u046", "kind": "code", "locator": "body:L105-L107", "preview": "```bash conda run -n base python scripts/get_page.py kernel-grouped-gemm ```", "sha256": "9fbb183e9a149ef4de2e59d8f89f3a99147623edc37ab5c8c97198a67b143de1"}], "confidence_claimed": "source-reported", "headings": ["Verified scope", "GPU Mode Problem 4 contract", "DeepGEMM's different grouped layouts", "Published workloads and theoretical bounds", "The scrubbed reward hack", "Implementation and performance boundary", "Primary sources"], "id": "kernel-grouped-gemm", "path": "wiki/kernels/grouped-gemm.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md", "url": "https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm"}, {"path": "sources/blogs/deepgemm.md", "url": "https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f"}, {"path": "sources/blogs/gpu-mode-reward-hack.md", "url": "https://www.gpumode.com/news/reward-hacking-nvfp4"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["contest-gpumode-p4", "blog-deepgemm", "blog-gpu-mode-reward-hack"], "title": "Grouped GEMM Contracts for MoE and NVFP4", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "70483995c616c5db093c78410e006ee58b4ba3b50d6a9c3d4df495d17ee83b8b", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Native Sparse Attention is the three-branch, natively trainable sparse-attention design published in the ACL 2025 proceedings. The paper targets long-context training and inference and reports quality comparable to or better than full atten", "sha256": "2c0ec2bc109661ee09f54dc63f8417335388815f3753d73eec2981291af143b5"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "The paper's efficiency experiments use an eight-GPU A100 system. The paper does not establish Hopper or Blackwell compatibility for its Triton implementation.", "sha256": "6f6896a7fa0ba40dc353d0c167e3c649ded0ce47b2ae36612a4b1923e15eb3ae"}, {"id": "u003", "kind": "prose", "locator": "body:L11-L11", "preview": "For token representation \\(h_t\\), NSA computes three attention outputs and combines them with learned, input-dependent gates:", "sha256": "f64e6768752c7e6f3ba740ef76224502099ecf66a0106a52d67a3c47a39a56a3"}, {"id": "u004", "kind": "prose", "locator": "body:L13-L17", "preview": "\\[ o_t = g_t^{cmp} o_t^{cmp} + g_t^{slc} o_t^{slc} + g_t^{win} o_t^{win}, \\qquad g_t^c = \\operatorname{sigmoid}(\\operatorname{MLP}_c(h_t)). \\]", "sha256": "41026f4d18871ed500bfd946a01192fae3ccd21c8a793f7c9591d36a29b24888"}, {"id": "u005", "kind": "table-row", "locator": "body:L21-L21", "preview": "| Branch | Paper-defined role | | Compression | Learned MLPs with intra-block position encoding compress overlapping KV blocks into coarse-grained representations. |", "sha256": "39c68f8814a572d058f80500989273c21222e31059dc8352f8658036d1d04edf"}, {"id": "u006", "kind": "table-row", "locator": "body:L22-L22", "preview": "| Branch | Paper-defined role | | Selection | Compression-attention scores are reused and aggregated into fine-grained block-importance scores; top-n blocks are selected and attended at full token resolution. |", "sha256": "69c857a98cc6bcc41f69ea7ad8a1ad832967ede3ebec5df870fd4c2daf71d7f7"}, {"id": "u007", "kind": "table-row", "locator": "body:L23-L23", "preview": "| Branch | Paper-defined role | | Sliding window | Recent tokens are attended directly to preserve local context. |", "sha256": "7c88b15a55c170f0f387757aee97a3be92a4b1acbfa50cdb85e45286f450d002"}, {"id": "u008", "kind": "prose", "locator": "body:L25-L25", "preview": "The main experimental configuration uses compression block length 32 and stride 16, selected block length 64 with 16 selected blocks, and a 512-token sliding window. These are experiment settings, not universal NSA constants.", "sha256": "6c078d35fe343e0df340cf3581dab508e404337a77d5cc800fd8b1a501479571"}, {"id": "u009", "kind": "prose", "locator": "body:L29-L29", "preview": "This CPU function checks only the learned gated-sum semantics above. It is KernelWiki-derived, uses ordinary numbers, and makes no claim about the paper's GPU layout or performance.", "sha256": "f3bde73dc92644d83be1a500e942585d86bb8f275cc45390aa6e0440d2bbbc3d"}, {"id": "u010", "kind": "code", "locator": "body:L31-L49", "preview": "```python def gated_branch_sum(branch_outputs, gates): \"\"\"Combine equal-width branch vectors with one [0, 1] gate per branch.\"\"\" if len(branch_outputs) != len(gates) or not branch_outputs: raise ValueError(\"one gate is required for each non", "sha256": "44a4b9439967a835a9020b8d71c4e0705ddbe6889bde833a8bf2c4d774c04576"}, {"id": "u011", "kind": "prose", "locator": "body:L51-L51", "preview": "Removing the gates is a useful negative control: the raw branch sum for the same vectors is `[9.0, 26.0]`, not `[1.0, 10.0]`.", "sha256": "73b34bbecad3370b6e0eec3da0c92b7b15dc9817d821aa7921d2f62c52f3ef42"}, {"id": "u012", "kind": "prose", "locator": "body:L55-L55", "preview": "The authors say compression and sliding-window attention can use existing FlashAttention-2-style kernels, while selected attention needs a specialized Triton kernel for training and prefill. Its source-described structure is:", "sha256": "c7cd36466b1021b9863fae17675a7e07076a11916f6d0577fe9e275c71682848"}, {"id": "u013", "kind": "list-item", "locator": "body:L57-L57", "preview": "1. Load all query heads in one GQA/MQA group into SRAM so they share selected sparse KV indices.", "sha256": "768260e8436d762fc968c79d6fd814d0c40208dbf2be3f79d488232f1ba7c6c8"}, {"id": "u014", "kind": "list-item", "locator": "body:L58-L58", "preview": "2. Load selected KV data as contiguous blocks rather than scattered individual tokens.", "sha256": "7693234b4fa52ffc907e7569bc88a99ef4e7022a678855b8071ab1fbd41d0f4d"}, {"id": "u015", "kind": "list-item", "locator": "body:L59-L59", "preview": "3. Put the nearly constant query and output loops in Triton grid parallelism; keep the selected-block loop inside a program because its count is approximately constant.", "sha256": "6a0b998d60a53e4fe652acf63e51dd57cf0c7cd1a01b790589be65d488ea6450"}, {"id": "u016", "kind": "prose", "locator": "body:L61-L61", "preview": "The paper does not publish the former page's kernel listing, specify a grid exactly as `(query_block, head, batch)`, or claim that such a grid eliminates dynamic scheduling overhead. The removed listings were non-executable and mathematical", "sha256": "2d0122b777bf8c5c8e032dae7d13f2e1faf87911dc235148aad2b8120a48dff1"}, {"id": "u017", "kind": "table-row", "locator": "body:L67-L67", "preview": "| Quantity | Source scope at sequence/context length 65,536 | | Training/prefill forward | 9.0x versus the authors' Triton FlashAttention-2 baseline; Figure 5 timing result |", "sha256": "eeabdd9e3872b414925f03f14acb6773447dad55641cd836f611ce5f68673a3b"}, {"id": "u018", "kind": "table-row", "locator": "body:L68-L68", "preview": "| Quantity | Source scope at sequence/context length 65,536 | | Training/prefill backward | 6.0x versus the same baseline; Figure 5 timing result |", "sha256": "472d7906e53c0e40b8a332a8287fbf4a71db6e008b63e96e97117fc829405b37"}, {"id": "u019", "kind": "table-row", "locator": "body:L69-L69", "preview": "| Quantity | Source scope at sequence/context length 65,536 | | Decoding | 11.6x **expected** speedup from Table 4's memory-access volumes: 65,536 full-attention tokens versus 5,632 NSA-equivalent tokens |", "sha256": "3bf66c2184375f601a4dc615346eb44d33c37253d21f20113aa9c4f8e310c4b0"}, {"id": "u020", "kind": "prose", "locator": "body:L71-L71", "preview": "The setup states eight A100 GPUs, GQA group count 4, 16 query heads per group, key dimension 192, and value dimension 128. The paper does not provide the benchmark dtype, software versions, batch details, raw samples, or variance. Consequen", "sha256": "93cc2b833ca205d7c37ee86be7e7de768c0f5bb64c43ccd76012177aa40a0321"}, {"id": "u021", "kind": "prose", "locator": "body:L75-L75", "preview": "DeepSeek-V3.2-Exp later introduced **DeepSeek Sparse Attention (DSA)**. Its pinned first-party inference code uses a learned indexer to select up to 2,048 token positions and masks attention to those positions. Its README points to DeepGEMM", "sha256": "b759b38189951b82f1e7ff5ce2f07ab9901041dd2a8e8fbbffb3004d989bfecc"}, {"id": "u022", "kind": "prose", "locator": "body:L77-L77", "preview": "That deployed DSA path is related sparse-attention work, but it is not evidence that the ACL paper's gated compression/selection/window NSA architecture was deployed in V3.2-Exp.", "sha256": "4606d7c523a22914f1bc8ef5e748d3b0a4cd3d30f19ab3b48330a2a3808fd904"}, {"id": "u023", "kind": "list-item", "locator": "body:L81-L81", "preview": "- NSA requires a model trained for its compression, selection, window, and gate mechanism; do not treat it as a drop-in sparse mask for an arbitrary checkpoint.", "sha256": "d520c97cb2a307f60e35e18e72170c4d9b3d929d5f23d9b7130c267c78bef030"}, {"id": "u024", "kind": "list-item", "locator": "body:L82-L82", "preview": "- GQA/MQA lets query heads in a group share selected blocks, matching the paper's group-centric kernel design.", "sha256": "48adf162d4b0542d6469331811c4675cf802871f58d1ff43739261c47b3207ec"}, {"id": "u025", "kind": "list-item", "locator": "body:L83-L83", "preview": "- The paper evaluates long contexts through 64K but defines no universal 32K threshold. Compare quality, sequence distribution, memory use, and measured latency or throughput on the target workload.", "sha256": "c9ee5b9ee579038f32902b7284d069a8b03a8a5513560bf99a3c9b4caddf92d9"}, {"id": "u026", "kind": "list-item", "locator": "body:L84-L84", "preview": "- CUDA Graphs can reduce repeated-launch CPU overhead, but they are not an NSA-specific default. Profile first: NVIDIA's guidance says the largest gains occur for CPU-bound workflows and that GPU-bound workloads may see little benefit or re", "sha256": "f9b6df5ef4f3f131f00c87890ba79a225234505a177ddf9f29eae775645f6938"}, {"id": "u027", "kind": "list-item", "locator": "body:L85-L85", "preview": "- The compression MLP adds learned parameters and computation. The paper does not isolate a standalone cost for that component.", "sha256": "9ccd7326bd68872d3e7c3e4f9753cb1a498f16f8e6fc032eac6fba3b8475a883"}, {"id": "u028", "kind": "list-item", "locator": "body:L86-L86", "preview": "- The paper notes that a Triton implementation can retain abstraction overhead relative to native CUDA; it does not publish an NSA-specific CPU-launch profile.", "sha256": "f6435058cc76f46ad9f41b109905a67f7e096f67dce91f53dc284c7fd4d1f671"}, {"id": "u029", "kind": "list-item", "locator": "body:L90-L90", "preview": "- [NSA in the ACL 2025 Anthology](https://aclanthology.org/2025.acl-long.1126/)", "sha256": "78ccf4813ea3076dc266e56c039c186998cec537ae12c990098355ca1dd0affc"}, {"id": "u030", "kind": "list-item", "locator": "body:L91-L91", "preview": "- [ACL proceedings PDF](https://aclanthology.org/2025.acl-long.1126.pdf)", "sha256": "13864f984b34a739b22354671bf40b7b132a4a92b1fe709c69c18516a3157473"}, {"id": "u031", "kind": "list-item", "locator": "body:L92-L92", "preview": "- [DeepSeek-V3.2-Exp at pinned commit 87e509a](https://github.com/deepseek-ai/DeepSeek-V3.2-Exp/tree/87e509a2e5a100d221c97df52c6e8be7835f0057)", "sha256": "495799eeb9bf66dfb26c143f0617fa51bb3eafc0b83db9a0ecbf52e94f45bd9f"}, {"id": "u032", "kind": "list-item", "locator": "body:L93-L93", "preview": "- [FlashMLA at pinned commit 71c7379](https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232)", "sha256": "9a703d254eb11b0c82694980bf3ace79260d5a522fbbc81e75e1d5c838cc0273"}, {"id": "u033", "kind": "list-item", "locator": "body:L94-L94", "preview": "- [NVIDIA CUDA Graph performance troubleshooting](https://docs.nvidia.com/dl-cuda-graph/troubleshooting/performance-issues.html)", "sha256": "f07028fb31a610923a456920862ac95e7fe74dbbb90917e2ee033cc76d23f1d9"}, {"id": "u034", "kind": "list-item", "locator": "body:L95-L95", "preview": "- [Third-party lucidrains PyTorch implementation](https://github.com/lucidrains/native-sparse-attention-pytorch)", "sha256": "4ac2ee874ac52c888f894dbe154072fa140f783aefe6c722c45a6b7eaf2cfd00"}, {"id": "u035", "kind": "code", "locator": "body:L99-L101", "preview": "```bash conda run -n base python scripts/get_page.py kernel-nsa ```", "sha256": "1f78b6b3ff77dec53e1ab9848cec6dad73d5a38e2591afa12472b71b618f4c82"}], "confidence_claimed": "source-reported", "headings": ["Verified scope", "Exact three-branch mechanism", "Executable branch-combination reference", "Paper-described Triton design", "Source-reported efficiency", "NSA is not the later DSA deployment", "Applicability and boundaries", "Primary sources", "Query"], "id": "kernel-nsa", "path": "wiki/kernels/nsa.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nsa.md", "url": "https://aclanthology.org/2025.acl-long.1126/"}, {"path": "sources/blogs/flashmla.md", "url": "https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232"}], "risk_flags": ["code", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-nsa", "blog-flashmla"], "title": "Native Sparse Attention (NSA)", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "c5bd73b9a4501849ca7bc91e4a94a7de62cd1bf395495ceb820d6ca9c8465a37", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "The official NVIDIA rules identify NVFP4 GEMM as Kernel Challenge 2 of the Blackwell NVFP4 Hackathon. The contest ran from November 29 through December 19, 2025; Problem 2 targeted NVIDIA B200 and contributed 20% of the four-problem grand-p", "sha256": "c012f9b4b901261153563003ab333b1f3208acf9127751594a9c6f92e6194686"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "The pinned public task defines observable inputs, correctness, test shapes, benchmark shapes, and ranking. It does not publish a canonical optimized kernel, require CUTLASS, or establish which implementation mechanisms any entrant used. \u201cNV", "sha256": "60449fada505af15ad32e1652d853557b7618e9f0adf40fc59aa58181e7654c7"}, {"id": "u003", "kind": "prose", "locator": "body:L11-L11", "preview": "The generic one-dimensional NVFP4 recipe reconstructs each value from a signed E2M1 payload, one E4M3 local scale per 16 consecutive payloads, and a per-tensor FP32 global scale. E2M1 represents zero and signed magnitudes 0.5, 1, 1.5, 2, 3,", "sha256": "2d3e513162b32573b229edb6791eca5c968c25d20a763fc0c6728bee7623a754"}, {"id": "u004", "kind": "prose", "locator": "body:L13-L13", "preview": "The contest ABI is narrower and does not expose the generic recipe's FP32 global scales. Its actual generated input has seven tensors:", "sha256": "3a871dd4a928094b50b5397043bcb0ed40b4ca720015ca6f28f00940abc8308f"}, {"id": "u005", "kind": "table-row", "locator": "body:L17-L17", "preview": "| Tensor | Published representation | Physical shape | | `a` | packed E2M1, two values per byte | `[M,K/2,L]` |", "sha256": "e0a7789d78d69b33651c32ed0b1322da6a51ce99701fb8c4f691c1d606ca1e27"}, {"id": "u006", "kind": "table-row", "locator": "body:L18-L18", "preview": "| Tensor | Published representation | Physical shape | | `b` | packed E2M1, two values per byte | `[N,K/2,L]` |", "sha256": "7d0102d70ed8d43681e30199436e1a4ca93f82176a713d1a4eb60f915d82a32b"}, {"id": "u007", "kind": "table-row", "locator": "body:L19-L19", "preview": "| Tensor | Published representation | Physical shape | | `sfa` | logical E4M3 scales | `[M,K/16,L]` |", "sha256": "8c6e7b8cc27b3f4659878a193b0d177ad42556861e9af3c56bf8dd190847c87d"}, {"id": "u008", "kind": "table-row", "locator": "body:L20-L20", "preview": "| Tensor | Published representation | Physical shape | | `sfb` | logical E4M3 scales | `[N,K/16,L]` |", "sha256": "53f0052f4f81c4ddb14c0a6d49f8abbe1ee49fffe56a0e6600ba2e4690d9a1fc"}, {"id": "u009", "kind": "table-row", "locator": "body:L21-L21", "preview": "| Tensor | Published representation | Physical shape | | `sfa_reordered` | MMA-oriented scale copy | `[32,4,ceil(M/128),4,K/64,L]` |", "sha256": "fdccc0a2063460171c6874f7b71b04aa81f8547a6adf6647394f30407ad2130f"}, {"id": "u010", "kind": "table-row", "locator": "body:L22-L22", "preview": "| Tensor | Published representation | Physical shape | | `sfb_reordered` | MMA-oriented scale copy | `[32,4,ceil(N/128),4,K/64,L]` |", "sha256": "dc780d607c17c5380f0d0cf88b406360fc63691f9131f5658c4666fe6a431427"}, {"id": "u011", "kind": "table-row", "locator": "body:L23-L23", "preview": "| Tensor | Published representation | Physical shape | | `c` | preallocated FP16 output | `[M,N,L]` |", "sha256": "5bb5aa5ac7ba2bf79f1a086d297ae8bf6ac71c70fd93ef7176c42f472b856b8b"}, {"id": "u012", "kind": "prose", "locator": "body:L25-L25", "preview": "For each `L` slice, the correctness reference computes the block-scaled equivalent of `A @ B.T` and stores FP16 output. It uses `rtol=1e-3` and `atol=1e-3`.", "sha256": "6f3e0ffc95c1f7cfbba0f5cce8d4ad5cf98e730c2518d3f862973484eb5cd357"}, {"id": "u013", "kind": "prose", "locator": "body:L27-L27", "preview": "The upstream files contain two interface inconsistencies that implementations must not conceal:", "sha256": "2ead2f5e8d44bc8c68f365c286a7d40895ac73ad636d88cfbbf74c8456754f89"}, {"id": "u014", "kind": "list-item", "locator": "body:L29-L29", "preview": "- `task.yml` describes a five-member `(a,b,sfa,sfb,c)` tuple, while `task.py`, `template.py`, and `reference.py` expose the seven tensors above.", "sha256": "b1ab36d41d8414867d0fd4b868add90d1552fb037fa289b4952ffc5335fe9025"}, {"id": "u015", "kind": "list-item", "locator": "body:L30-L30", "preview": "- Task and template prose label scale tensors E4M3FNUZ, while `reference.py` constructs `torch.float8_e4m3fn` values. A submission must follow the tensors supplied by the actual harness rather than treating those suffixes as interchangeable", "sha256": "60a30abca2b8e35e3e85f9529afad31c41f9e672f64a665c9cab4c93db10e6cb"}, {"id": "u016", "kind": "prose", "locator": "body:L34-L34", "preview": "The published task requires `K` divisible by 256. Divisibility of `M` and `N` depends on the submission's selected MMA tile. The following CPU-only helper reproduces the published storage shapes; it does not decode FP4, apply scales, or mod", "sha256": "b665b4c6d8030c332f28c4d3ee158ae219b6d15795bdaee05374861dbc13385b"}, {"id": "u017", "kind": "code", "locator": "body:L36-L58", "preview": "```python def task_storage_shapes(m, n, k, l=1): if min(m, n, k, l) <= 0: raise ValueError(\"dimensions must be positive\") if k % 256: raise ValueError(\"K must be divisible by 256\") return { \"a_packed\": (m, k // 2, l), \"b_packed\": (n, k // 2", "sha256": "7642131a89d5b771cbf27494b4cc0b80cd0cf33ee914733ba4332b41646dc052"}, {"id": "u018", "kind": "prose", "locator": "body:L60-L60", "preview": "Nine of the ten official correctness shapes fail the former `(K/16) % 128 == 0` assertion, including the smallest valid case with `K=256`.", "sha256": "05437858b45314ea93e0a0104ea46596c3ea7d49d2c24ae53df396395c1074ae"}, {"id": "u019", "kind": "prose", "locator": "body:L64-L64", "preview": "The native Blackwell instruction path supports block-of-16 NVFP4 through `tcgen05.mma...kind::mxf4nvf4.block_scale.block16`. It uses UE4M3 scale elements, which CUTLASS names `float_ue4m3_t`; UE8M0 is the MXFP4 scale type. Converting NVFP4 ", "sha256": "5c735c80238f43b67916629af854f0bc5ebe3cbdc2a5dd8c6884eb7c0e838112"}, {"id": "u020", "kind": "prose", "locator": "body:L66-L66", "preview": "CUTLASS defines `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100`, but its direct official example is a grouped pointer-array kernel. The symbol's existence is not evidence that the contest entrants used it. CUTLASS's current official NVFP4 ex", "sha256": "ab4df04cd09ce423053e72e6743e8bd87ad2ec87bbac6cc6a67fcdf80f74f1bb"}, {"id": "u021", "kind": "prose", "locator": "body:L68-L68", "preview": "TMA likewise has no universal 128-byte alignment rule for every operand. CUDA 13.2.1 generally requires a 16-byte-aligned global base and 16-byte-multiple strides for a tiled tensor map, with additional datatype-, interleave-, swizzle-, and", "sha256": "3ccf1a1511937dc90f14ec1f46a7a6da76096f2e80ebf2b09b6040f7c8715278"}, {"id": "u022", "kind": "prose", "locator": "body:L70-L70", "preview": "TMEM's 128-lane by 512-column organization is a storage and addressing model, not a universal 128-by-512 logical output-tile limit. Official CUTLASS NVFP4 configurations include a cooperative two-SM MMA tile with logical shape 256 by 256 by", "sha256": "9146b8a20d4fa3e2d61f9fc78dbb921c708e685affa23cd507b61539ba3c289d"}, {"id": "u023", "kind": "prose", "locator": "body:L74-L74", "preview": "The task ranks the geometric mean across three benchmark cases. It labels the following values a speed-of-light analysis based on the maximum of B200 FP4 Tensor Core math time and DRAM-memory time at a 1.5 GHz clock:", "sha256": "1333ed27524466513afa4aa3d6b2aca56e71b8dd321380468992191a08483324"}, {"id": "u024", "kind": "table-row", "locator": "body:L78-L78", "preview": "| M | N | K | L | Theoretical time (\u00b5s) | | 128 | 7168 | 16384 | 1 | 8.994 |", "sha256": "ac29495b16a50fe1cfa2dcde5a85eb0fbc58049c674a2acd145386cdf9509a03"}, {"id": "u025", "kind": "table-row", "locator": "body:L79-L79", "preview": "| M | N | K | L | Theoretical time (\u00b5s) | | 128 | 4096 | 7168 | 1 | 2.354 |", "sha256": "5830e08d67630c0aa4612f11749d003bd0e8772b58d5696ff74cb0b2383e0fe2"}, {"id": "u026", "kind": "table-row", "locator": "body:L80-L80", "preview": "| M | N | K | L | Theoretical time (\u00b5s) | | 128 | 7168 | 2048 | 1 | 1.333 |", "sha256": "fa3e79cd0999f76fee4e1016298889112f86cae481bb308bf85adc53cba145e9"}, {"id": "u027", "kind": "prose", "locator": "body:L82-L82", "preview": "These are theoretical comparison rows, not measured contestant latencies and not cuBLAS results.", "sha256": "e7cccec1a9469a76f5310902e63e0ff426f9cdfb679295d05b973561c5a59ba8"}, {"id": "u028", "kind": "prose", "locator": "body:L84-L84", "preview": "The public Popcorn endpoint currently gives the following dated snapshot. To make its small floating-point `submission_score` values readable beside the task's microsecond presentation, the table shows `submission_score \u00d7 10^6`; it does not", "sha256": "ca5d81241e455f5afedd1cd83c9e5f33b6ce5ffd65e4fc6dbee310fe12e17238"}, {"id": "u029", "kind": "table-row", "locator": "body:L88-L88", "preview": "| Current rank | User | API score \u00d7 10^6 | Submission timestamp (UTC) | | 1 | `gau.nernst` | 9.981889 | 2025-12-21 00:43:03 |", "sha256": "fe02ccfff7ba7a01f611de399274d94df3fc34621f9c1f49f0b4661b1049fc8e"}, {"id": "u030", "kind": "table-row", "locator": "body:L89-L89", "preview": "| Current rank | User | API score \u00d7 10^6 | Submission timestamp (UTC) | | 2 | `s.am._` | 10.060110 | 2025-12-20 17:45:21 |", "sha256": "3b924272968baecf2642aff9df9a27f543bf3a3ec76b865ad5651e3dfc27ae46"}, {"id": "u031", "kind": "table-row", "locator": "body:L90-L90", "preview": "| Current rank | User | API score \u00d7 10^6 | Submission timestamp (UTC) | | 3 | `billcarson` | 10.137411 | 2025-12-21 03:05:32 |", "sha256": "682d0b4289aaf64c4a8dfce794ab877797b0f27e2dfbdb0558cd5b91020e7ea6"}, {"id": "u032", "kind": "table-row", "locator": "body:L91-L91", "preview": "| Current rank | User | API score \u00d7 10^6 | Submission timestamp (UTC) | | 8 | `Simon` | 10.806750 | 2025-12-16 20:18:42 |", "sha256": "532478b666f20292d73ffe2cf7b1d1e68503fdadedf372b6bc068566201ba1ca"}, {"id": "u033", "kind": "table-row", "locator": "body:L92-L92", "preview": "| Current rank | User | API score \u00d7 10^6 | Submission timestamp (UTC) | | 9 | `yue` | 10.914084 | 2025-12-11 04:36:45 |", "sha256": "390d4818810b01ab386a95f2b688d3b0e6b496b5c99251c2597eee2fffca1387"}, {"id": "u034", "kind": "table-row", "locator": "body:L93-L93", "preview": "| Current rank | User | API score \u00d7 10^6 | Submission timestamp (UTC) | | 10 | `currybab` | 10.930623 | 2025-12-19 08:10:18 |", "sha256": "b20e53b276da79e3e8db64ea4bc8792ffde6e7159c74d43fdd475df2f464a8d0"}, {"id": "u035", "kind": "prose", "locator": "body:L95-L95", "preview": "Snapshot fetched August 8, 2026. Because the current first three submissions postdate the official December 19 cutoff, this endpoint alone cannot establish winners or final prize rankings. It also publishes no contestant source, CUTLASS att", "sha256": "31d7de708031defe935bf9ec093e7e4e609515f457b21a57edaae459c012889a"}, {"id": "u036", "kind": "prose", "locator": "body:L99-L99", "preview": "Use this task contract only when the packed E2M1 payloads, logical and reordered scales, transpose convention, FP16 output, tolerances, and B200 target match the workload. \u201cFour-bit weights\u201d alone is insufficient: INT4, MXFP4, other scale g", "sha256": "73cdc11c5d684987ddca4b7d6b757fa05f35817fafcc4b69fab11eb85fdadbea"}, {"id": "u037", "kind": "prose", "locator": "body:L101-L101", "preview": "The documented native tensor-core path is Blackwell-specific; Hopper has no native FP4 tensor-core instruction. This does not exclude software conversion or emulation. Choose TMA, TMEM allocation, CUTLASS schedules, warp specialization, til", "sha256": "44a9079d9bde40b4b9d9f4314563b36497b2c4c3baf8f73d63b50ba622654431"}, {"id": "u038", "kind": "list-item", "locator": "body:L105-L105", "preview": "- [Official contest rules](https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf)", "sha256": "c945d69cc0933ee89c1418170925a1dac08b34aa49c449aff8fab2f67960ab55"}, {"id": "u039", "kind": "list-item", "locator": "body:L106-L106", "preview": "- [Pinned task definition](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml)", "sha256": "e65c352f0be491a3018d9832b9eb33f9a20999ed3c89f8073d9b51d7b05ab736"}, {"id": "u040", "kind": "list-item", "locator": "body:L107-L107", "preview": "- [Pinned task types](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.py)", "sha256": "3fd0cff1732ff2946231c340cdb3f1e603f51d9e30c1d38d02c5bdd5e97047cc"}, {"id": "u041", "kind": "list-item", "locator": "body:L108-L108", "preview": "- [Pinned starter template](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py)", "sha256": "f0c85282243b89d3b5178d653b7368861f3c8179b8902ccb2d8cf362e007ba1f"}, {"id": "u042", "kind": "list-item", "locator": "body:L109-L109", "preview": "- [Pinned correctness reference](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/reference.py)", "sha256": "a9913b0ecd262bdf9fc1ed22d8ba73b0df684b9d60385ea92765cefada05884c"}, {"id": "u043", "kind": "list-item", "locator": "body:L110-L110", "preview": "- [Public Popcorn leaderboard API](https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12)", "sha256": "05fc2df13c95fa5525c4bc583dfe7fbd3d792d5b56bd9701b72b2005410eba68"}, {"id": "u044", "kind": "list-item", "locator": "body:L111-L111", "preview": "- [Transformer Engine 2.13 NVFP4 recipe](https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html)", "sha256": "6a8a7f25f33afde1adc5eb40bedd91ec6b208700f493f6716ab59dda9d710f87"}, {"id": "u045", "kind": "list-item", "locator": "body:L112-L112", "preview": "- [cuBLAS block-scaling formats](https://docs.nvidia.com/cuda/cublas/index.html#element-1d-block-scaling-for-fp8-and-fp4-data-types)", "sha256": "835625f4c0a79e2e00ed840ffb666c21b6794ec80c63ba082abf75c3ac010c1f"}, {"id": "u046", "kind": "list-item", "locator": "body:L113-L113", "preview": "- [PTX ISA 9.0 block-scaled `tcgen05.mma`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma)", "sha256": "3b0e1ea6faf1d7ae1afa563c938aa77e64dfc63629fa38f2b9d81668b2ba6da4"}, {"id": "u047", "kind": "list-item", "locator": "body:L114-L114", "preview": "- [Pinned CUTLASS NVFP4 example](https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu)", "sha256": "673b6e03246cef1e9406aab15f2e173b99d155e39e94f235480d949ea32a846e"}, {"id": "u048", "kind": "list-item", "locator": "body:L115-L115", "preview": "- [Pinned CUTLASS grouped NVFP4 example](https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu)", "sha256": "072f9f025862aacd8fa196a463d4a6c65ad7f4c07ffc75f84a2e3230b331ecd9"}, {"id": "u049", "kind": "list-item", "locator": "body:L116-L116", "preview": "- [CUDA 13.2.1 tensor-map constraints](https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html)", "sha256": "2ce8fa29f655906e7c265b4f2dcf842b58186c862ebf8809d5b5e51ecc9f5225"}, {"id": "u050", "kind": "prose", "locator": "body:L118-L118", "preview": "Query via:", "sha256": "6abd63e508f9320fdd3b2b0cc5b5f6a000f4462abfd17f9ecc3c4818f4a0f875"}, {"id": "u051", "kind": "code", "locator": "body:L120-L122", "preview": "```bash conda run -n base python scripts/get_page.py kernel-nvfp4-gemm ```", "sha256": "48df91e99487c982432b8d227c4d314527b5a7a8f1814e19da626a790ab563a4"}], "confidence_claimed": "verified", "headings": ["Verified scope", "Format recipe versus task ABI", "Host-checkable shape contract", "Blackwell implementation boundary", "Published performance records", "Applicability", "Primary sources"], "id": "kernel-nvfp4-gemm", "path": "wiki/kernels/nvfp4-gemm.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/contests/gpu-mode-nvfp4/problem-2-gemm.md", "url": "https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm"}, {"path": "sources/docs/nvidia-transformer-engine-2.13-nvfp4.md", "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/docs/nvidia-cuda-13-0-2-tma.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#asynchronous-data-copies-using-the-tensor-memory-accelerator-tma"}], "risk_flags": ["code", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["contest-gpumode-p2", "doc-transformer-engine-2.13-nvfp4", "doc-ptx-isa-sm100", "doc-cuda-13-0-2-tma"], "title": "NVFP4 GEMM \u2014 GPU Mode Problem 2 Contract", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "15a1de756e23c26b7938e4b0c5e12e918492d5f204e504a428fcb1573c5bbd26", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "GPU Mode's Problem 1 asks for a block-scaled NVFP4 batched matrix-vector product on NVIDIA B200. The official task models its three benchmark cases against the slower of FFMA math and DRAM transfer time and reports DRAM-limited theoretical ", "sha256": "14e1c04988eab77eb5fda24b66c9161c6b4819d6a72816cd82aa24e0a228b98e"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "The logical operation has one B row, reused by all M output rows. The reference harness pads B and its scales to 128 rows so it can call `torch._scaled_mm`, then retains only result column zero. Consequently, A values are row-specific while", "sha256": "fb7fdddf466f0ad9cf575500ce1520eda0f07afbd4e86b5654b1dc604b053c5d"}, {"id": "u003", "kind": "prose", "locator": "body:L11-L11", "preview": "At pinned commit `ae679486`, `custom_kernel` receives seven tensors, not five tensors plus global FP32 scale arguments:", "sha256": "15bf1ed3ca25ae5af1f21dfbb407e770ac71c82bc8cde6e78cd46afd632e8b3f"}, {"id": "u004", "kind": "table-row", "locator": "body:L15-L15", "preview": "| Tensor | Physical shape | Role | | `a` | `[M, K/2, L]` | Packed E2M1 A; two logical values per byte |", "sha256": "f74daf225a044f42c185baf318bc992eb494afe58da70331b78e61f48059c196"}, {"id": "u005", "kind": "table-row", "locator": "body:L16-L16", "preview": "| Tensor | Physical shape | Role | | `b` | `[128, K/2, L]` | Packed E2M1 B, physically padded; logical row 0 is used |", "sha256": "4e4df829b77072f6fd90eda8d41974abd6ddcd71887459f0ab2e1a6f449f3171"}, {"id": "u006", "kind": "table-row", "locator": "body:L17-L17", "preview": "| Tensor | Physical shape | Role | | `sfa` | `[M, K/16, L]` | Logical/reference A block scales |", "sha256": "47e879cdd6d9333011e683d6bb76b0bb1ebd562f7e7d900090b2ae9f86d97c5b"}, {"id": "u007", "kind": "table-row", "locator": "body:L18-L18", "preview": "| Tensor | Physical shape | Role | | `sfb` | `[128, K/16, L]` | Logical/reference B block scales, padded to 128 rows |", "sha256": "a0be0f8d74ca34db15d2798662259b839777715672f2963b11ac9ae6cd6e401b"}, {"id": "u008", "kind": "table-row", "locator": "body:L19-L19", "preview": "| Tensor | Physical shape | Role | | `sfa_reordered` | `[32, 4, ceil(M/128), 4, K/64, L]` | Swizzled A scales for custom kernels |", "sha256": "e1f29ebc95c561c93de68422dd62d1e78f1b2f0e26a56cb1e63d2971a7ee3d4e"}, {"id": "u009", "kind": "table-row", "locator": "body:L20-L20", "preview": "| Tensor | Physical shape | Role | | `sfb_reordered` | `[32, 4, 1, 4, K/64, L]` | Swizzled padded-B scales for custom kernels |", "sha256": "25d7de3baa2b15fd47bea0643ffd612cf8dbd8732ed66bb80c7f3a9cf9fe501b"}, {"id": "u010", "kind": "table-row", "locator": "body:L21-L21", "preview": "| Tensor | Physical shape | Role | | `c` | `[M, 1, L]` | FP16 output buffer |", "sha256": "dc83bb9a64c7548c727836928612797531bb396623d9b852aad6bd803787427e"}, {"id": "u011", "kind": "prose", "locator": "body:L23-L23", "preview": "`K` must be divisible by 64, and M must be divisible by the implementation's selected M tile. The task prose labels scales `e4m3fnuz`, but the pinned generator constructs `torch.float8_e4m3fn`; implementations should follow the actual harne", "sha256": "9d23ed25965f166e208682143b1379b4a346ef123a4f513c3160a28f289fbe68"}, {"id": "u012", "kind": "prose", "locator": "body:L27-L27", "preview": "The official table assumes a 1.5 GHz B200 clock and ranks submissions by the geometric mean across all three rows:", "sha256": "181b1e55fb5b84d28814d49c47efc252c00f59b87529df6181d4be0f24168dcc"}, {"id": "u013", "kind": "table-row", "locator": "body:L31-L31", "preview": "| M | K | L | Theoretical time (\u00b5s) | | 7168 | 16384 | 1 | 8.622 |", "sha256": "9d7c65618fc8dc86bb3d2746a16f175fd4649ddfb006c68c807ce5e587b8fd0b"}, {"id": "u014", "kind": "table-row", "locator": "body:L32-L32", "preview": "| M | K | L | Theoretical time (\u00b5s) | | 4096 | 7168 | 8 | 17.275 |", "sha256": "6ef10de1e876f22ab461b936146b0788d1bf5d704d586c207ec965c2cfb38c65"}, {"id": "u015", "kind": "table-row", "locator": "body:L33-L33", "preview": "| M | K | L | Theoretical time (\u00b5s) | | 7168 | 2048 | 4 | 4.317 |", "sha256": "f615c82589dadc4edfe7b91cae3fca3ca39d09c582d799c3ea68d1cd3429a24c"}, {"id": "u016", "kind": "prose", "locator": "body:L35-L35", "preview": "NVIDIA's Blackwell technical brief specifies 8 TB/s of HBM3e bandwidth for one GB200 GPU. The task's theoretical numbers are model values, not measured kernel timings.", "sha256": "704c6601665fccbda264f611828f4318072acc029cf200d5edb39ac485314581"}, {"id": "u017", "kind": "prose", "locator": "body:L37-L37", "preview": "The public leaderboard is mutable and reports aggregate scores rather than per-shape latencies. In the snapshot fetched on 2026-08-08, the first three rows were `s.am._` at 18.549562452 \u00b5s, `gau.nernst` at 18.552844757 \u00b5s, and `shellsmile15", "sha256": "22240d05193ce717b94fafae8edaa23f1553f76a8c00a46d942aa92c4300367a"}, {"id": "u018", "kind": "prose", "locator": "body:L41-L41", "preview": "PTX ISA 9.0 defines the packed conversion and typed register decomposition used in Yue's author-reported decode path:", "sha256": "4affc9d80b04da71f0c299c5784d8efc15535361c1156364eff99e69729b354f"}, {"id": "u019", "kind": "code", "locator": "body:L43-L46", "preview": "```asm cvt.rn.f16x2.e2m1x2 %result, %packed_fp4_pair; mov.b32 {%b0, %b1, %b2, %b3}, %packed_word; ```", "sha256": "3e8b7ea84b83abad0b9ce987bd7810cd56904af6c822034b66dc22b09deced0a"}, {"id": "u020", "kind": "prose", "locator": "body:L48-L48", "preview": "It also defines vector loads such as `ld.global.v2.u64` and `ld.global.v4.u64`, which move 16 and 32 bytes and can carry 32 and 64 packed FP4 values. The ISA specifies behavior, not that one width or decomposition is universally faster.", "sha256": "9d0de1cc2b4db79424fedba1e2f5d3bc17dccb713cb173f3a8b41405ac5bf617"}, {"id": "u021", "kind": "prose", "locator": "body:L50-L50", "preview": "Amandeep reports that three solutions inspected after the event used `L1::no_allocate` for streamed A loads, `L1::evict_last` for reused B loads, wider PTX loads, and exact-K specializations. Those are author-reported observations; the publ", "sha256": "8d769e43685ea0b46eefdae1e86397cdbe7f97387c4fdb5be59a5305352d80cb"}, {"id": "u022", "kind": "prose", "locator": "body:L54-L54", "preview": "The following timings are from Yue's post, not an independent reproduction. Several stages combine multiple changes and cannot establish isolated causality:", "sha256": "a663193aef641ad8433597c548f5820e215e4556a1f06db8485c4075e1cbf6f9"}, {"id": "u023", "kind": "table-row", "locator": "body:L58-L58", "preview": "| Stage | Combined change | Reported latency | | Initial CuTe DSL | First working CuTe implementation | ~100 \u00b5s |", "sha256": "43eb908efd6940c3a7f0b3cb44880e4110b5b361042ab2ccf9cec68ad6d5009f"}, {"id": "u024", "kind": "table-row", "locator": "body:L59-L59", "preview": "| Stage | Combined change | Reported latency | | Optimized CuTe DSL | Scale-load and arithmetic changes plus thread collaboration | ~33 \u00b5s |", "sha256": "8ff27a458e08c63aa19c12f8680ccc4cb633d6b880364aec30bfce4b162c5aa7"}, {"id": "u025", "kind": "table-row", "locator": "body:L60-L60", "preview": "| Stage | Combined change | Reported latency | | Initial CUDA | Naive hand-written path | ~2000 \u00b5s |", "sha256": "ca54d2e0867aaacac8ef7cb2d50ce50784f5192560a97826eabca1c17f14eaba"}, {"id": "u026", "kind": "table-row", "locator": "body:L61-L61", "preview": "| Stage | Combined change | Reported latency | | CUDA step 1 | Coalescing, shared B, thread collaboration, warp reduction | ~443 \u00b5s |", "sha256": "45a99e5c48954c46d64d8216a104e7f192c5d36eae133d72ead8f09d1b957a37"}, {"id": "u027", "kind": "table-row", "locator": "body:L62-L62", "preview": "| Stage | Combined change | Reported latency | | CUDA step 2 | Remove shared B, per-thread tiles, `float4` loads, hardware intrinsics | ~39 \u00b5s |", "sha256": "c7347c699aff3a09cbd3e861889c16bd4cab72d2239ec315fc23068cdff5cf2b"}, {"id": "u028", "kind": "table-row", "locator": "body:L63-L63", "preview": "| Stage | Combined change | Reported latency | | CUDA step 3 | Vectorized PTX FP4/scale decode | ~27 \u00b5s |", "sha256": "6219356ab8e9caca95b5f5bf5e4a40b08c5b6eaef70784ebcafa9b4bc9966a55"}, {"id": "u029", "kind": "table-row", "locator": "body:L64-L64", "preview": "| Stage | Combined change | Reported latency | | Parameter tuning | Threads per row and rows per block | ~26 \u00b5s |", "sha256": "5d2fda1525d1d93874e8cdcc642024d47fe2139e768806cdfa35ab38b4826a4c"}, {"id": "u030", "kind": "table-row", "locator": "body:L65-L65", "preview": "| Stage | Combined change | Reported latency | | Two-tile ILP | Interleave two tiles per loop | ~22.9 \u00b5s |", "sha256": "55bc6ef2fdac8923e7f8d89df31dd74418182cd3741c71c983b0b942c8165841"}, {"id": "u031", "kind": "table-row", "locator": "body:L66-L66", "preview": "| Stage | Combined change | Reported latency | | Aggressive PTX fusion | Fuse decode, scale, multiply, and accumulation | ~22.3 \u00b5s |", "sha256": "3e9d7de1f63be9029aa4ff29559822da7ac982629cde96e956e04ec4c3330b31"}, {"id": "u032", "kind": "table-row", "locator": "body:L67-L67", "preview": "| Stage | Combined change | Reported latency | | Submitted score | Geometric mean shown by the leaderboard | 22.392 \u00b5s |", "sha256": "088f1750e1a2800452b3e097abdcb2af7099608a24ecb7bf909f53c3406f2917"}, {"id": "u033", "kind": "prose", "locator": "body:L69-L69", "preview": "Yue also reports that loading the entire B vector into shared memory did not improve the CuTe attempt. This is a useful counterexample to treating shared-memory B staging or an exact `BLOCK_M` traffic reduction as automatic.", "sha256": "19ee8de8d318e9c5ada566f02126040608afa1d3e39dbc64e014946f41585387"}, {"id": "u034", "kind": "list-item", "locator": "body:L73-L73", "preview": "- Use this case study for an N=1 decode-style matrix-vector operation only when payloads, scales, layouts, output type, and target match the chosen NVFP4 implementation contract.", "sha256": "a8248df41bbce5cc45f98412659fa2f2d8f3ad504d6e7dbf6611009207c697f4"}, {"id": "u035", "kind": "list-item", "locator": "body:L74-L74", "preview": "- Profile bytes moved, instruction count, decode and reduction work, cache behavior, spills, and occupancy before deciding which resource is limiting performance.", "sha256": "c725284ade431a9257a1b4a7cf2ae3f84f565ddd6b5a45c645adbba3458b3580"}, {"id": "u036", "kind": "list-item", "locator": "body:L75-L75", "preview": "- Exact-K dispatch can enable full unrolling and size-specific tuning, but retaining multiple specializations adds compiled kernel entries.", "sha256": "2f807d582f25851529b1c82489b1ba304fdc4b91510a1bfaf6f8b259db03ec6f"}, {"id": "u037", "kind": "list-item", "locator": "body:L76-L76", "preview": "- Inline PTX must be revalidated against the selected PTX ISA, toolkit, and target. The contest ran on B200; that does not imply packed FP4 conversion is restricted to SM100/SM100a forever. Transformer Engine 2.13 lists NVFP4 inference supp", "sha256": "9f975c4e2c336fdd731e7222126701eac54c48fbacbccd74f4fc7230a54c0ca7"}, {"id": "u038", "kind": "list-item", "locator": "body:L77-L77", "preview": "- The official task accepts any implementation that satisfies its checker. It does not require the cache policies, load widths, register limits, or code structures described in participant posts.", "sha256": "86286884a901e356f8c977190835d2ab47157b7f2fa23ef0bbb05aa129da29f2"}, {"id": "u039", "kind": "list-item", "locator": "body:L81-L81", "preview": "- [Pinned Problem 1 task](https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv)", "sha256": "253eac43081be0b1c0f587a706185b6b8f30e7b2cebb381630c17652de95898d"}, {"id": "u040", "kind": "list-item", "locator": "body:L82-L82", "preview": "- [Official contest rules](https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf)", "sha256": "c945d69cc0933ee89c1418170925a1dac08b34aa49c449aff8fab2f67960ab55"}, {"id": "u041", "kind": "list-item", "locator": "body:L83-L83", "preview": "- [NVIDIA Blackwell Architecture Technical Brief](https://resources.nvidia.com/en-us-blackwell-architecture/blackwell-architecture-technical-brief)", "sha256": "c330012ad06dc51b1eccaade34a7425fc7b7a8aaefd177c220b19f75aacfc633"}, {"id": "u042", "kind": "list-item", "locator": "body:L84-L84", "preview": "- [PTX ISA 9.0](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html)", "sha256": "f287fbe5a77017bc028e72cfac864b6bb16272498a46b45f643b430593555c9b"}, {"id": "u043", "kind": "list-item", "locator": "body:L85-L85", "preview": "- [Transformer Engine 2.13 NVFP4](https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html)", "sha256": "fd66d9aaa128a191225657e888e4d0692eac5623d5d11ade7dfc67793dad73e0"}, {"id": "u044", "kind": "list-item", "locator": "body:L86-L86", "preview": "- [Yue Zhang's hackathon journey](https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html)", "sha256": "c92a8a46203210ed1e85fb159f8abb7ee9402f2560f9d4f9c3bba8b71a5a2467"}, {"id": "u045", "kind": "list-item", "locator": "body:L87-L87", "preview": "- [Amandeep Singh's twelve attempts](https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/)", "sha256": "32b13b998fa855e2f28acc4e3385e172306944a5676385b43b3a1d65c0fffbe7"}, {"id": "u046", "kind": "list-item", "locator": "body:L88-L88", "preview": "- [Simon Veitner's reference](https://veitner.bearblog.dev/nvfp4-gemv/) and [improved variants](https://veitner.bearblog.dev/nvfp4-gemv-improved/)", "sha256": "1de91575cdf7f4cd9043e4939095ac18a892ba8e120f1b5343db9ef35082a8e0"}], "confidence_claimed": "source-reported", "headings": ["Verified Scope", "Official Callable Contract", "Official Benchmark Model", "PTX Semantics and Reported Techniques", "Yue's Author-Reported Progression", "Practical Use and Caveats", "Primary Sources"], "id": "kernel-nvfp4-gemv", "path": "wiki/kernels/nvfp4-gemv.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/contests/gpu-mode-nvfp4/problem-1-gemv.md", "url": "https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv"}, {"path": "sources/blogs/yue-nvfp4-hackathon.md", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}, {"path": "sources/blogs/amandeep-nvfp4-attempts.md", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}], "risk_flags": ["code", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["contest-gpumode-p1", "blog-yue-nvfp4", "blog-amandeep-nvfp4"], "title": "NVFP4 Batched GEMV", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "948b326db003b5db4472a9222cbd12a7ea8b28c706bb3c67abfc29cdc7846cca", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L6", "preview": "DeepSeek-V3.2-Exp introduces **DeepSeek Sparse Attention (DSA)**. Its first-party report defines two components:", "sha256": "bb3c5529691e366dd6efa2a880b744e11851437076378b06b60557eed9f130e4"}, {"id": "u002", "kind": "list-item", "locator": "body:L8-L8", "preview": "1. The lightning indexer computes one score for every query/preceding-token", "sha256": "800571c72e22c76c4f46f84b358de8ed823f7e7daa1d5a0a1aada431dff73396"}, {"id": "u003", "kind": "prose", "locator": "body:L9-L10", "preview": "pair. A score is a weighted sum across indexer heads of a ReLU-applied query and key dot product.", "sha256": "b0c28b86e74cff53ff735a2fccf865af8a26d42271a8e1f9e5ba566ac3c6bc24"}, {"id": "u004", "kind": "list-item", "locator": "body:L11-L11", "preview": "2. Fine-grained token selection retains the KV entries at the top-k index", "sha256": "27bba385baa0f30644e808286cf4cd7f206ac7b835de6e843d5877b66605f308"}, {"id": "u005", "kind": "prose", "locator": "body:L12-L12", "preview": "scores and applies the main attention only to that selected set.", "sha256": "799ed51e15b7fac5411d28e251d11824024993140b9ef0591e8307e92855c99b"}, {"id": "u006", "kind": "prose", "locator": "body:L14-L19", "preview": "The released `config_671B_v3.2.json` uses 64 indexer heads of dimension 128 and `index_topk=2048`. DSA is instantiated under MLA's MQA mode: the selected MLA latent entry is shared across the query heads. The model has 128 MLA query heads, ", "sha256": "7999f09cb62ad0774fc6162c7eeb34be31d20d0047d0dae05d29f9edc76f3774"}, {"id": "u007", "kind": "prose", "locator": "body:L21-L25", "preview": "For a length-L sequence and fixed selected count k, the report reduces the **main/core attention** complexity from O(L squared) to O(Lk). The lightning indexer still scores the preceding context and remains O(L squared) over the sequence. F", "sha256": "2c7a6f110419179e164e3a5294219756ca737c3fcc8e666080d7b2dbdaa83df2"}, {"id": "u008", "kind": "prose", "locator": "body:L29-L30", "preview": "The released high-performance path is not one fused \u201cindexer plus sparse MLA\u201d kernel:", "sha256": "55f8654222c7c4ed6f9915195f9062b25ad60c76e2b1fdb603290b284fdd718a"}, {"id": "u009", "kind": "list-item", "locator": "body:L32-L32", "preview": "- DeepGEMM provides non-paged and paged indexer-logit kernels. Its pinned SM100", "sha256": "49ef39eedf0417732d260892fbf391b96e42f8be6fcfc25f65df6a95b5a976d8"}, {"id": "u010", "kind": "prose", "locator": "body:L33-L35", "preview": "FP8 path computes token logits from FP8 query/key inputs, per-token key scales, and per-query head weights; top-k selection remains a separate operation.", "sha256": "37f4d58acab34fe50c0f8f7112e6ca0b4bfea2750e936edd7287731209210208"}, {"id": "u011", "kind": "list-item", "locator": "body:L36-L36", "preview": "- FlashMLA sparse attention consumes caller-produced token indices. It does not", "sha256": "cc6129e45570619d4939c753c50aa7f48f97d6bd16ddccbef36dafe6d6a91980"}, {"id": "u012", "kind": "prose", "locator": "body:L37-L37", "preview": "run the lightning indexer or top-k selector.", "sha256": "ae05e1bbfe66ece445518ed31b0a0d666c384437f79b7d468584eec2274a5262"}, {"id": "u013", "kind": "prose", "locator": "body:L39-L41", "preview": "At FlashMLA commit [`71c7379`](https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232), prefill and decode have different contracts:", "sha256": "eb389ddbba522174d4e4bfeeae557782e265db6d785ad1db1e6534dae97f1dd5"}, {"id": "u014", "kind": "table-row", "locator": "body:L45-L45", "preview": "| Stage | Inputs and selected-token encoding | Precision and outputs | | Sparse decode | `q[batch,s_q,h_q,576]`; paged `k_cache`; `indices[batch,s_q,topk]`, where each nonnegative value encodes physical page times page size plus token offse", "sha256": "26a36dfe709a48b77979096047f08d9c163ac61519ddf4cdf0418f3ac8e5e996"}, {"id": "u015", "kind": "table-row", "locator": "body:L46-L46", "preview": "| Stage | Inputs and selected-token encoding | Precision and outputs | | Sparse prefill | Unbatched BF16 `q[s_q,h_q,d_qk]`, BF16 `kv[s_kv,h_kv,d_qk]`, and INT32 `indices[s_q,h_kv,topk]`; the documented equivalence requires `h_kv=1`; negativ", "sha256": "1e44a7ce28c3d88a202ffd6faf77f7cedd7ca507f7ce76c55601542bc3357387"}, {"id": "u016", "kind": "prose", "locator": "body:L48-L52", "preview": "These are token indices, not one maximum or one selection per cache block. The sparse-decode API accepts page size through the cache tensor; pinned correctness tests exercise multiple values including 2, 53, 61, 64, 69, 256, and 576. Page s", "sha256": "708e5ca34ceab27f48e8fe7b2f704d457087a65dd4385e2131a535dc0871273d"}, {"id": "u017", "kind": "prose", "locator": "body:L56-L57", "preview": "Only the V3/V3.1/V3.2 **FP8 sparse-decode** mode has the documented 656-byte per-token layout:", "sha256": "953e4b660a1d8b065676b80e7b7bfe06fbcc32914d3a83085d33182a259edc3c"}, {"id": "u018", "kind": "list-item", "locator": "body:L59-L59", "preview": "- 512 E4M3 NoPE values: 512 bytes;", "sha256": "cab31744fcfa6f82078271f60cd43b820f43334dfc55e6a23d6b4eff5d85eeab"}, {"id": "u019", "kind": "list-item", "locator": "body:L60-L60", "preview": "- four FP32 scales, one for each successive group of 128 NoPE values: 16 bytes;", "sha256": "e9793ded4f667e1592dd6b993dd390b5016a1cf8154c4041a50971a687d58d57"}, {"id": "u020", "kind": "list-item", "locator": "body:L61-L61", "preview": "- 64 BF16 RoPE values used by the attention key: 128 bytes.", "sha256": "42502eff2ec7dd17fd0437ce18e46942556ee145ee15b2f566f78fc7f1f35cf6"}, {"id": "u021", "kind": "prose", "locator": "body:L63-L65", "preview": "The indexer has a separate FP8 K cache and scale cache in the released model. The 656-byte attention entry is not an indexer cache and is not the layout of every dense, sparse-prefill, or non-V3 FlashMLA mode.", "sha256": "004ef4c9a6d5f3ed82854f9a9c23912785829fab4c14b88f3209e130e2804f32"}, {"id": "u022", "kind": "prose", "locator": "body:L69-L71", "preview": "FlashMLA's pinned README reports the following maxima. They are useful source claims, not reproducible benchmark tuples: the README does not provide complete shapes, timed regions, repetitions, samples, or variance.", "sha256": "53fac485447961fd25735e0a07f1bc42805fa1e081bfc85f7e358c428dfce49f"}, {"id": "u023", "kind": "table-row", "locator": "body:L75-L75", "preview": "| Operator and regime | Reported environment | Author-reported maximum and precision scope | | Dense MLA decode, memory-bound configuration | H800 SXM5, CUDA 12.8 | Up to 3000 GB/s with BF16 cache |", "sha256": "330ca8a2334ecb87ab1bbb24ffd109a884a2205e87bf06eb13a6a23f7971d01c"}, {"id": "u024", "kind": "table-row", "locator": "body:L76-L76", "preview": "| Operator and regime | Reported environment | Author-reported maximum and precision scope | | Dense MLA decode, compute-bound configuration | H800 SXM5, CUDA 12.8 | Up to 660 TFLOPS with BF16 cache; separate from the 3000-GB/s case |", "sha256": "498754f9010440f53a448bb30c012279c0b07afbed50d3948e10936a375fe13b"}, {"id": "u025", "kind": "table-row", "locator": "body:L77-L77", "preview": "| Operator and regime | Reported environment | Author-reported maximum and precision scope | | Sparse MLA decode | H800 SXM5, CUDA 12.8 | 410 TFLOPS; FP8 KV storage and BF16 matrix multiplication |", "sha256": "619c3ef250d56d62746909cd8385a09d21c42b91cbb78ca19ff12ee196e011d7"}, {"id": "u026", "kind": "table-row", "locator": "body:L78-L78", "preview": "| Operator and regime | Reported environment | Author-reported maximum and precision scope | | Sparse MLA decode | B200 | Up to 350 TFLOPS; the source says this path was not really optimized and gives no bandwidth-causality result |", "sha256": "8c29c94ec9620c62ce3dc2dca6c5caf9fc593e6e655ed3083238714f063298c1"}, {"id": "u027", "kind": "table-row", "locator": "body:L79-L79", "preview": "| Operator and regime | Reported environment | Author-reported maximum and precision scope | | Sparse MLA prefill | H800 SXM5, CUDA 12.8 | Up to 640 TFLOPS forward with BF16 Q/KV |", "sha256": "65b792e1dcb4c09b1df29d269f6b31e97e9759eb45d1dbfb50461bdfc30a2b78"}, {"id": "u028", "kind": "table-row", "locator": "body:L80-L80", "preview": "| Operator and regime | Reported environment | Author-reported maximum and precision scope | | Sparse MLA prefill | B200, CUDA 12.9 | Up to 1450 TFLOPS forward with BF16 Q/KV |", "sha256": "95c25ad0d981b3686a61775f7965640cc532bb9f6b636a35fd27c6a294572fe7"}, {"id": "u029", "kind": "prose", "locator": "body:L82-L85", "preview": "The same README separately reports NVIDIA's dense **MHA** prefill maxima of 1460 TFLOPS forward and 1000 TFLOPS backward on B200. Those values are not a dense-MLA baseline matched to the sparse-prefill result, so numerical proximity between", "sha256": "7ac9b3f51be9a7774fa020507229c154b698af2de5f5294b218fdc3135466a57"}, {"id": "u030", "kind": "prose", "locator": "body:L89-L92", "preview": "Use this path directly for DeepSeek-V3.2-Exp, or for another model only after its query dimensions, latent/RoPE layout, cache format, head relationships, index encoding, invalid-entry rules, and output semantics match the chosen FlashMLA in", "sha256": "8f4750da95e3b1406c6de34977adb16629218d8cbf8633e93a0e2d1402c11c69"}, {"id": "u031", "kind": "prose", "locator": "body:L94-L94", "preview": "For a target deployment:", "sha256": "fbb956c923fa1fd09806a78a27d293e0f8ebba82fb051ec936245cfbab5aa6bf"}, {"id": "u032", "kind": "list-item", "locator": "body:L96-L96", "preview": "1. Compare index scores and selected token sets against the released model", "sha256": "a242b13ee51da5cc6cb541c02ee0c6daec6fdfc1984d6ec275554957be416c4a"}, {"id": "u033", "kind": "prose", "locator": "body:L97-L98", "preview": "equation and a dense reference, including causal masking and the indexer's non-interleaved RoPE layout.", "sha256": "2e2cead28d1d22caa0e2eb0a926978703e3dac81cef6c1ca51ca073875034c13"}, {"id": "u034", "kind": "list-item", "locator": "body:L99-L99", "preview": "2. Compare sparse prefill/decode outputs and LSE values with the pinned", "sha256": "da0d88573313119d302c4d5be8a002a02d2e95ed3cfdf92b11d9eb6d471ee920"}, {"id": "u035", "kind": "prose", "locator": "body:L100-L101", "preview": "FlashMLA reference for identical selected indices, including invalid and partially filled top-k cases.", "sha256": "7dfcefbf9f9a818d7f9216ec620e421da6a17ef61a85dc8a12903765a3d3a748"}, {"id": "u036", "kind": "list-item", "locator": "body:L102-L102", "preview": "3. Time indexer logits, top-k selection, sparse attention, and the complete", "sha256": "94eb29b4206955c162be3c506b690b8704ac47a014a9065ffb96b40829d3a68e"}, {"id": "u037", "kind": "prose", "locator": "body:L103-L105", "preview": "pipeline separately across context lengths, query batch sizes, page sizes, and selected counts. Use a matched dense-attention baseline and report synchronization, warmup, repeated trials, statistic, and variation.", "sha256": "f02576926515f6346b13cb20938a90d9add5d59c67836537a247f53a7ed3964e"}, {"id": "u038", "kind": "prose", "locator": "body:L107-L109", "preview": "There is no verified universal 32K crossover. The relevant threshold is where the measured selector-plus-sparse-attention pipeline improves the target's end-to-end latency or cost without violating its accuracy criterion.", "sha256": "dbcd039e000ca1384e83656c5550b990540d5f937a2165464a53be0b3a47373b"}, {"id": "u039", "kind": "prose", "locator": "body:L113-L117", "preview": "DeepSeek-V3.2-Exp DSA selects individual token positions with a learned indexer. The ACL Native Sparse Attention architecture instead combines compressed-block, selected-block, and sliding-window branches with learned gates. Do not transfer", "sha256": "0acee81708a88fb6dadac0671ca900e0781e9ea10c0f915e18641f375c096b84"}, {"id": "u040", "kind": "list-item", "locator": "body:L121-L121", "preview": "- [DeepSeek-V3.2-Exp report and released model](https://github.com/deepseek-ai/DeepSeek-V3.2-Exp/tree/87e509a2e5a100d221c97df52c6e8be7835f0057)", "sha256": "7e2d20a04ee83d9a3bf13f14c6776ff4b781d09b3be1430590dc1ecf4a0c74bc"}, {"id": "u041", "kind": "list-item", "locator": "body:L122-L122", "preview": "- [FlashMLA at audited commit `71c7379`](https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232)", "sha256": "1f1e839b0c6d66154bae95cc00ff4970eabb13c2b7f9c1fbf2c2adbc49cbff46"}, {"id": "u042", "kind": "list-item", "locator": "body:L123-L123", "preview": "- [FlashMLA sparse interface at `71c7379`](https://github.com/deepseek-ai/FlashMLA/blob/71c737929f2567bd0a094ae140f8f60f390b1232/flash_mla/flash_mla_interface.py)", "sha256": "e814ec84eaefe35612a2df8db9295b511349797f6c84c94ee4a8c5dd1fa447ae"}, {"id": "u043", "kind": "list-item", "locator": "body:L124-L124", "preview": "- [DeepGEMM indexer-logit implementation at `891d57b4`](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_mqa_logits.cuh)", "sha256": "719f548e64d48d007a4167184782be4a682c07ce1fc80ce249f2422d75dd59cf"}, {"id": "u044", "kind": "list-item", "locator": "body:L125-L125", "preview": "- [ACL 2025 Native Sparse Attention paper](https://aclanthology.org/2025.acl-long.1126/)", "sha256": "0d5536512e7b6f36724ec0738c6ae2905c69b13a296ce9b447941d99d1ba1f39"}], "confidence_claimed": "source-reported", "headings": ["Model mechanism", "Separate implementation boundaries", "V3-family sparse-decode cache", "Source-reported performance boundary", "Evaluation procedure", "DSA is not Native Sparse Attention", "Primary references"], "id": "kernel-sparse-mla", "path": "wiki/kernels/sparse-mla.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/flashmla.md", "url": "https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232"}, {"path": "sources/blogs/vllm-deepseek-v3-sparse-attention.md", "url": "https://blog.vllm.ai/2025/09/29/deepseek-v3-2.html"}, {"path": "sources/docs/nsa.md", "url": "https://aclanthology.org/2025.acl-long.1126/"}], "risk_flags": ["ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-flashmla", "blog-vllm-deepseek-v3-sparse", "blog-nsa"], "title": "DeepSeek Sparse Attention / Sparse MLA", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "64b746e36ef7866e93cb1c9aca98703acc5008d485a0d9c96ab31f7a1504a964", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L6", "preview": "TensorRT-LLM PR 13340 adds an FP4 option to its DeepSeek Sparse Attention (DSA) indexer path. This page describes the PR's pinned merge revision `897c4bff`; it is a TensorRT-LLM implementation reference, not a generic DSA ABI or a drop-in F", "sha256": "deba7267d6d469eb687d6d3449e5af0ad0c6d1ba369591c3fb16a90d5e366c36"}, {"id": "u002", "kind": "prose", "locator": "body:L8-L8", "preview": "The implementation has four distinct stages:", "sha256": "88d97624362af38799246c1501a4fe33def3f446b9b4aa90ab01b801fcd1925c"}, {"id": "u003", "kind": "table-row", "locator": "body:L12-L12", "preview": "| Stage | Pinned behavior | | Q/K preparation | `fused_cat_fp4` concatenates BF16 positional and non-positional components, then quantizes the 128-value row to FP4 E2M1 with per-32-value UE8M0 scales |", "sha256": "b4dd786700dab6a1916f3f6864b7313bc68209fd47ff51f0797e8e5519acfc4c"}, {"id": "u004", "kind": "table-row", "locator": "body:L13-L13", "preview": "| Stage | Pinned behavior | | Cache update/read | Scatter and gather copy an already-quantized payload and its scale word between contiguous tensors and TensorRT-LLM's possibly non-contiguous paged indexer cache |", "sha256": "9b7854ba44f86df2319c17ed31631bf2238523273d2d8d6358322f7e1ecbbdc6"}, {"id": "u005", "kind": "table-row", "locator": "body:L14-L14", "preview": "| Stage | Pinned behavior | | Indexer logits | The FP4 path reinterprets the packed bytes/scales and dispatches TensorRT-LLM's DeepGEMM FP4 MQA-logit implementation |", "sha256": "c9d9feea66087fdce32ebbe15b4d41baff2b2cd21955fa373128c608ae7e7477"}, {"id": "u006", "kind": "table-row", "locator": "body:L15-L15", "preview": "| Stage | Pinned behavior | | Selection | Prefill or decode top-k runs after logit computation in a separate operator |", "sha256": "67b72313b995f38a22bc9869baf2440b8e21d4e022ff92a1c80aad932b4dfaf1"}, {"id": "u007", "kind": "prose", "locator": "body:L17-L19", "preview": "The gather/scatter kernels do not quantize values, compute logits, or select indices. Calling them \u201cfused quantized gather/scatter\u201d conflates the first two stages.", "sha256": "807375e96e742adf3dcf843c6629b966585ade9ccf3c8eeb2203385bc47f166c"}, {"id": "u008", "kind": "prose", "locator": "body:L23-L23", "preview": "For the fixed 128-value indexer head in this path, one row has:", "sha256": "ebb2a6213e4bff8f699eae8109c1f172ba7e674d0b9b63b222df48f57b18e727"}, {"id": "u009", "kind": "table-row", "locator": "body:L27-L27", "preview": "| Component | Representation | Bytes per row | | Input | BF16 positional values followed by BF16 non-positional values | 256 before quantization |", "sha256": "baec963beab3b990ec4394a2317a757e16a2ae7eed59ba449f4b13f0c8d41276"}, {"id": "u010", "kind": "table-row", "locator": "body:L28-L28", "preview": "| Component | Representation | Bytes per row | | Packed payload | Two FP4 E2M1 codes per byte | 64 |", "sha256": "733a12c767a0e5c2b16891c439d9afb7f02967486419f72c19ab6be2f8899338"}, {"id": "u011", "kind": "table-row", "locator": "body:L29-L29", "preview": "| Component | Representation | Bytes per row | | Scale word | Four UE8M0 exponent bytes, one per successive group of 32 values, packed little-endian into one `int32` | 4 |", "sha256": "4f5e47e73b176f3b9410f26081aa80b35d5dc6f60655aab518f2b403c93c8d03"}, {"id": "u012", "kind": "table-row", "locator": "body:L30-L30", "preview": "| Component | Representation | Bytes per row | | Cache footprint | Packed payload plus scale word | 68 |", "sha256": "eb8484b32ee5c8dbb43eac0b6a213e540b06d702b5a0c10b8af1388ef129c7f5"}, {"id": "u013", "kind": "prose", "locator": "body:L32-L36", "preview": "The fused operator requires CUDA BF16 inputs on one device, at least two dimensions, a contiguous innermost dimension, eight-byte-aligned addresses, equal row counts, and positional width divisible by four. Positional and non-positional wid", "sha256": "a5cc71924358ceb6ead6a2ecda37330218b28290f4b2d41b6adaeb5ac031bebd"}, {"id": "u014", "kind": "prose", "locator": "body:L38-L41", "preview": "The scale for each 32-value group is `2^ceil(log2(max(amax, 1e-12) / 6))`. Quantization uses the FP4 E2M1 magnitude set `{0, 0.5, 1, 1.5, 2, 3, 4, 6}` and packs the earlier value into the low nibble.", "sha256": "f7c210916986dea9cc641e69e78cc98f6ecd2e4cf7699853f0ff2c8ccc511256"}, {"id": "u015", "kind": "prose", "locator": "body:L45-L49", "preview": "The cache is viewed as `[num_blocks, block_size, 1, per_token_size]` and may be non-contiguous. Each token launch copies four bytes per thread. The FP4 path uses 64 payload bytes and one four-byte scale word; the legacy FP8 path uses 128 pa", "sha256": "987ce4347f8506f5eea5a3d223d7a93fb36282a6a0fde62ad4a46290c6f401ba"}, {"id": "u016", "kind": "prose", "locator": "body:L51-L55", "preview": "Payload and scale use separate contiguous `int64` slot-mapping arrays. If either mapping for a token is negative, gather and scatter skip that entire token. The gather wrapper allocates output with `empty`, so a directly gathered row skippe", "sha256": "9368aff166cf872488d1f74571c046b6ce8644b1e602bf3a20e68ce7b65d9bec"}, {"id": "u017", "kind": "prose", "locator": "body:L57-L61", "preview": "The gather wrapper preserves a historical typed view: payload bytes are returned as float8 and scale bytes as FP32. The FP4 call site reinterprets them as packed `int8` data and `int32` scale words before the DeepGEMM call. These views are ", "sha256": "3d313476860278e4ba8137a9d7cdeab1f70baca9b13c2e21fb85e25506deb8a4"}, {"id": "u018", "kind": "list-item", "locator": "body:L65-L65", "preview": "1. Compare `fused_cat_fp4` byte-for-byte with a reference implementation for", "sha256": "154a706e5febec7291f37c016f2c410dda069d1093b5277e758ecc7df33d51d8"}, {"id": "u019", "kind": "prose", "locator": "body:L66-L67", "preview": "zeros, FP4 decision boundaries, saturation, non-contiguous row strides, and multiple positional/non-positional splits whose widths sum to 128.", "sha256": "73c51aeb33326d6cb9aa5f4b0051dbd1226634ef3d17ad7ac5542abf75368a5b"}, {"id": "u020", "kind": "list-item", "locator": "body:L68-L68", "preview": "2. Round-trip valid tokens through scatter and gather using strided cache", "sha256": "dd3e85e021b2f37adc460ca62eb0dd4572d922c801da9f356f462c7edd762b3f"}, {"id": "u021", "kind": "prose", "locator": "body:L69-L70", "preview": "views. Check all 64 payload bytes and the four scale bytes independently; test negative payload and scale mappings without reading skipped empty rows.", "sha256": "5c513ab9c82962738d09cd52fa8abbe2157d34276c0d623a35c76c705daf72bb"}, {"id": "u022", "kind": "list-item", "locator": "body:L71-L71", "preview": "3. Compare FP4 non-paged and paged indexer logits, then selected indices,", "sha256": "f2a52d0cdaea7c448fda6a9ec9891ff1715df56ef0cf0f062f7a4f8e5d951d60"}, {"id": "u023", "kind": "prose", "locator": "body:L72-L73", "preview": "against an unquantized or higher-precision reference with identical masks, weights, and sequence boundaries.", "sha256": "ec15cbb04e5738b69458f6dd903cb1034e1125baeb8eb7fe8ee1db97b5311e8f"}, {"id": "u024", "kind": "list-item", "locator": "body:L74-L74", "preview": "4. Profile fused preparation, scatter, gather, logit computation, top-k, and", "sha256": "21d1fd695924b33d8c8316a89cad074218542e8d28d1311eb48f082429e42713"}, {"id": "u025", "kind": "prose", "locator": "body:L75-L76", "preview": "the complete indexer separately. Report shapes, cache layout, mapping mix, warmup, synchronization, repetitions, statistic, and variation.", "sha256": "b76f11b1782d4489ce3834019cbd0aa1e7079900dc54bde8f66498aa7519c781"}, {"id": "u026", "kind": "prose", "locator": "body:L78-L79", "preview": "The pinned PR contains no performance result, so use it to define candidate mechanisms and exact contracts rather than to claim a speedup.", "sha256": "62b72a9c0d5493138efc92602fe3a9b570342ca3e3a1afd37d89359535faae33"}, {"id": "u027", "kind": "list-item", "locator": "body:L83-L83", "preview": "- [TensorRT-LLM PR 13340](https://github.com/NVIDIA/TensorRT-LLM/pull/13340)", "sha256": "b5192a1e772c7397854ba03a1cf3a79cadfd904f45437b099332dda59be08ce4"}, {"id": "u028", "kind": "list-item", "locator": "body:L84-L84", "preview": "- [Pinned `fusedCatFp4.cu` artifact](../../artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/fusedCatFp4.cu)", "sha256": "5c2e1274b194d2974112ca102087a93493f94e7c6c0166d5e928aaaf4423415f"}, {"id": "u029", "kind": "list-item", "locator": "body:L85-L85", "preview": "- [Pinned gather kernel artifact](../../artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheGather.cu)", "sha256": "e4cb68f24b2d80c4b670273abebab689b0effa32829ffbd7ba41922a51f517c3"}, {"id": "u030", "kind": "list-item", "locator": "body:L86-L86", "preview": "- [Pinned scatter kernel artifact](../../artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheScatter.cu)", "sha256": "0811f065211145ad22d7ea66d97a233992db16c6c2a1559f862a6399de29cef5"}, {"id": "u031", "kind": "list-item", "locator": "body:L87-L87", "preview": "- [Pinned DSA integration artifact](../../artifacts/prs/tensorrt-llm/PR-13340/key-files/tensorrt_llm/_torch/attention_backend/sparse/dsa.py)", "sha256": "6ab0e3cc566c51a18ede438c4e7b208034aae703a7d915c4a0ae46c997b62226"}], "confidence_claimed": "source-reported", "headings": ["Audited scope", "FP4 row and scale contract", "Gather and scatter semantics", "Validation and profiling procedure", "Primary references"], "id": "kernel-tensorrt-llm-blackwell-indexer", "path": "wiki/kernels/tensorrt-llm-blackwell-indexer.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/prs/TensorRT-LLM/PR-13340.md", "revision": "897c4bff", "url": "https://github.com/NVIDIA/TensorRT-LLM/pull/13340"}], "risk_flags": ["ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["pr-TensorRT-LLM-13340"], "title": "TensorRT-LLM Blackwell FP4 DSA Indexer", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "280b0450a7c8a8d900672fd9664f15c0dfdc7f7464b794a771ca6275e0b23ead", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "CUDA C++ can host PTX instructions that do not yet have a convenient CUDA intrinsic. Gau Nernst's pinned `tcgen05` tutorial uses that approach for a B200 GEMM. For M=N=K=4096 in its disclosed PyTorch 2.9.1/CUDA 13 environment, v6 reports 14", "sha256": "b8dc3fbffc1822510b25fe8791507f64cde605ac6e032a0bc71463331be05e62"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "The CUDA front end does not parse the instruction text inside an `asm()` statement. Operand constraints and address-space conversion therefore remain the wrapper author's responsibility. Use `\"r\"` for a 32-bit integer register, `\"l\"` for a ", "sha256": "66973f95a9e7bb9cee77e63addbbffaf6c70a3faf28e277c11345ca3159fd0d1"}, {"id": "u003", "kind": "prose", "locator": "body:L9-L9", "preview": "The allocation instruction is collective. For `.cta_group::1`, every lane of one designated warp must execute the same instruction. Synchronize the CTA before another warp reads the address written to shared memory.", "sha256": "fcc09cde925fe4f4a37fd6ad7ed4fc8bef0c25889074107e7c9929c4f9f9d69f"}, {"id": "u004", "kind": "code", "locator": "body:L11-L23", "preview": "```cuda // All 32 lanes of alloc_warp execute this branch. if (warp_id == alloc_warp) { uint32_t smem_addr = static_cast(__cvta_generic_to_shared(smem_tmem_addr)); asm volatile( \"tcgen05.alloc.cta_group::1.sync.aligned.shared::cta", "sha256": "ebb8cfcebd4140ad4d4278b4cfc5e15cc7fa44cf3962b6b7d1c35c6b56365732"}, {"id": "u005", "kind": "prose", "locator": "body:L25-L25", "preview": "An unscaled `kind::f16` MMA takes one instruction descriptor and an `enable-input-d` predicate. A single elected thread may issue this MMA form; converting an ordinary CUDA integer to the required PTX predicate inside the assembly keeps the", "sha256": "c24454be97efa2771a84b294c86485ec2136d395132619914577ffdfada1f11b"}, {"id": "u006", "kind": "code", "locator": "body:L27-L41", "preview": "```cuda __device__ inline void tcgen05_mma_f16( uint32_t taddr, uint64_t a_desc, uint64_t b_desc, uint32_t idesc, int enable_input_d) { asm volatile( \"{\\n\\t\" \".reg .pred p;\\n\\t\" \"setp.ne.b32 p, %4, 0;\\n\\t\" \"tcgen05.mma.cta_group::1.kind::f1", "sha256": "f18005177146a4b9219d14cd71f72c3d557cab8b1201d449702ffbe897ec110e"}, {"id": "u007", "kind": "prose", "locator": "body:L43-L43", "preview": "`tcgen05.ld` is likewise a warp-level instruction. All participating lanes execute it, then the warp executes `tcgen05.wait::ld.sync.aligned` before consuming the loaded registers. After all readers finish, synchronize at the required CTA o", "sha256": "4ce40675f2370e88d09915dad1a7751449e9421c373a09b46a59fb3548b5a40b"}, {"id": "u008", "kind": "prose", "locator": "body:L47-L47", "preview": "An `mbarrier.arrive.expect_tx` plus a parity wait is not a complete pipeline by itself. A staged TMA-to-MMA pipeline has these invariants:", "sha256": "036c9408e81ea2ca527361be056d38be625abb66fbb1c7080d03c2f28a60cb54"}, {"id": "u009", "kind": "list-item", "locator": "body:L49-L49", "preview": "1. An elected thread initializes each barrier with the intended arrival count, then publishes initialization to the async proxy with the appropriate `fence.mbarrier_init`.", "sha256": "0f6929490cd30e970bb293556db67d84c6a815c751580cb317db0fc73fc85fce"}, {"id": "u010", "kind": "list-item", "locator": "body:L50-L50", "preview": "2. The producer reserves a reusable stage, issues TMA against that stage's full barrier, and accounts for the expected transaction bytes.", "sha256": "8cd760e642ac8dd2a6c515be010fe9a44a7bb62f7ef15c8f70e6c6ebc9c401f1"}, {"id": "u011", "kind": "list-item", "locator": "body:L51-L51", "preview": "3. The consumer waits with acquire semantics on the matching phase before reading the shared-memory stage.", "sha256": "312a192456b9daef18da5ad8e2f779541b10225463031894dd7f857871ec475f"}, {"id": "u012", "kind": "list-item", "locator": "body:L52-L52", "preview": "4. After issuing tcgen05 MMA, it commits completion to a separate barrier; the epilogue waits before loading TMEM.", "sha256": "86b997ef2bea7fd774b96fcee44abe925d6bd6a210f9b95bb6d8af8236cb7286"}, {"id": "u013", "kind": "list-item", "locator": "body:L53-L53", "preview": "5. The last stage user signals a separate empty/reuse barrier. A stage cannot be overwritten until its owner observes that handoff.", "sha256": "37f0a635354de7c909cf892d271fe017f1e61f286f30fcad30e0489653a63125"}, {"id": "u014", "kind": "list-item", "locator": "body:L54-L54", "preview": "6. Each barrier's parity flips only when its circular stage is revisited. Inline assembly that reads or writes memory invisibly to C++ uses a `\"memory\"` clobber.", "sha256": "b0b450e2673b3629ee08ba79029114501314c74fe24b7f02b919b5cbe5173472"}, {"id": "u015", "kind": "prose", "locator": "body:L56-L56", "preview": "Arrival counts, transaction bytes, scope, and which CTA owns each barrier depend on whether the kernel uses one-CTA or two-CTA MMA. Copy the complete protocol from the pinned implementation rather than treating isolated wait/arrive wrappers", "sha256": "3ec286caceda6b90d42dc392485a046e217d430962f83739bffdffca46b8b50b"}, {"id": "u016", "kind": "prose", "locator": "body:L60-L60", "preview": "The tutorial's v6 kernel launches six warps per CTA: warp 0 elects one lane for TMA, warp 1 collectively allocates TMEM and elects one lane for MMA, and warps 2--5 run the epilogue. Before exit, all threads finish their TMEM reads and warp ", "sha256": "fafa0457c52a7a307e156572288c3c1a8e3ce7b4f105e514bbf007d487486481"}, {"id": "u017", "kind": "list-item", "locator": "body:L64-L64", "preview": "- [CUDA 13.0.2 Inline PTX Assembly guide](https://docs.nvidia.com/cuda/archive/13.0.2/inline-ptx-assembly/index.html)", "sha256": "c7bb1254cdc25d23cdb099758d51a9903b0f13e6ab607c9482c1691fe8555c65"}, {"id": "u018", "kind": "list-item", "locator": "body:L65-L65", "preview": "- [PTX ISA 9.0 tcgen05 instructions](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05)", "sha256": "256f50034cf818182da77455057ef801d24ae9d2f2ee39f5bb6bf5de986db405"}, {"id": "u019", "kind": "list-item", "locator": "body:L66-L66", "preview": "- [Pinned tutorial implementation](https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100)", "sha256": "696f8d117ec1363f62d2161c17a759d5444b0751c0e413ce648f5ff3e3ddf1a9"}, {"id": "u020", "kind": "list-item", "locator": "body:L70-L70", "preview": "- [ptx-sm100](ptx-sm100.md) \u2014 version-pinned PTX instruction forms", "sha256": "cd8cd7f26ad1ca745a9a20d3d7825d0d55283809fd0c042b2baebbae2e949d03"}, {"id": "u021", "kind": "list-item", "locator": "body:L71-L71", "preview": "- [tcgen05 tutorial](../../sources/blogs/tcgen05-tutorial.md) \u2014 benchmark provenance and progression", "sha256": "0daf044bba2ca23ffebec3ff9ae1dd6e70c54942dbe71f640e45eae1b3d7107e"}], "confidence_claimed": "verified", "headings": ["Scope", "Inline-PTX boundary", "Barrier lifecycle", "One verified role split", "Primary references", "Related"], "id": "lang-cuda-cpp", "path": "wiki/languages/cuda-cpp.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/yue-nvfp4-hackathon.md", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}], "risk_flags": ["code", "ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-tcgen05-tutorial", "doc-nvidia-tuning-guide", "blog-yue-nvfp4"], "title": "CUDA C++ for Blackwell Kernels", "type": "language", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "90ca9df7c219547d83ab1a5609935cb5d4b5fe4ebd1dee959da0fadaf0c0c5a9", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "CUTLASS v4.5.0 includes CuTe DSL, a Python-native interface for authoring GPU kernels, alongside CUTLASS's C++ template interfaces. This page uses the exact `v4.5.0` tag (`e406c186f510a15091cce01f782020ceb7ba8eb5`); rolling `latest` documen", "sha256": "334261620dc71d6f22a0a53fcf732078f58347ac693be8556adaa40ca4a61e5d"}, {"id": "u002", "kind": "prose", "locator": "body:L5-L5", "preview": "FlashAttention-4 is implemented entirely in CuTe DSL. Its author reports roughly 20--30x shorter compile times than C++ templates and a peak of 1605 TFLOP/s on B200 BF16. These are FA4-specific, source-reported results. The 1605-TFLOP/s res", "sha256": "056d2cac890be9c1eb0d800df0380ae301fe97f8895ad6404e8633b9112135f7"}, {"id": "u003", "kind": "prose", "locator": "body:L9-L9", "preview": "CUTLASS 4.5.0 uses a configured `tcgen05.MmaF16BF16Op`, not an `SM100_MMA_F16BF16_SS` Python symbol. This excerpt shows the one-CTA operation used by the tagged first tutorial; it is construction code, not a complete kernel.", "sha256": "37f03738b7d78dbf444d54d1793ac9d5defae6cad433d27dabfea02f62b5ce85"}, {"id": "u004", "kind": "code", "locator": "body:L11-L26", "preview": "```python import cutlass import cutlass.cute as cute from cutlass.cute.nvgpu import tcgen05 op = tcgen05.MmaF16BF16Op( cutlass.Float16, cutlass.Float32, (128, 256, 16), tcgen05.CtaGroup.ONE, tcgen05.OperandSource.SMEM, cute.nvgpu.OperandMaj", "sha256": "e63cc3018bae603de0436601418eab234ce8ae74754a85c50ecda03dc733d264"}, {"id": "u005", "kind": "prose", "locator": "body:L28-L28", "preview": "The two-CTA tutorial instead uses instruction shape `(256, 256, 16)` and `tcgen05.CtaGroup.TWO`. The accumulator fragment is rebound to an allocated TMEM pointer before `cute.gemm` issues the operation.", "sha256": "59dd30be131eab3431bcbbf1b6d2ce88712c263139dd23b53853f002255058fa"}, {"id": "u006", "kind": "prose", "locator": "body:L32-L32", "preview": "The tagged examples use `cutlass.utils.TmemAllocator` around a shared holding buffer. A complete path allocates columns, waits before pointer retrieval, rebinds the accumulator tensor, partitions a typed `tcgen05` TMEM-to-register copy acro", "sha256": "609d3277d8eeaeb952785969f9a32c9cb5a3029a965f2f2a6b6ab1644358cdef"}, {"id": "u007", "kind": "code", "locator": "body:L34-L53", "preview": "```python # Excerpt: storage/barrier/tensor definitions and pipeline edges are required. tmem = utils.TmemAllocator( storage.tmem_holding_buf.ptr, barrier_for_retrieve=tmem_alloc_barrier, ) tmem.allocate(num_tmem_cols) tmem.wait_for_alloc()", "sha256": "76e8bef333b52e86589cffba34de7858d51e7e22e686ea20302ad4a681a3ff65"}, {"id": "u008", "kind": "prose", "locator": "body:L55-L55", "preview": "The excerpt deliberately does not imply that allocation or deallocation is lane-local. Use the complete tutorial for collective participation, two-CTA handling, and pointer lifetime.", "sha256": "784d2fca4eef895f2657ad5f9f3c7efc742f0578ad17207364970580d4ca2145"}, {"id": "u009", "kind": "prose", "locator": "body:L59-L59", "preview": "The one-CTA tutorial constructs a typed global-to-shared TMA operation and then derives operand-specific tiled atoms. Kernel code subsequently uses `tma_partition` and `cute.copy` with pipeline barriers.", "sha256": "0f534dfc499371ce84c7e460c19e133f0d91acb3fcc1925107b5beff7735a851"}, {"id": "u010", "kind": "code", "locator": "body:L61-L72", "preview": "```python from cutlass.cute.nvgpu import cpasync, tcgen05 tma_op = cpasync.CopyBulkTensorTileG2SOp(tcgen05.CtaGroup.ONE) a_tma_atom, a_tma_tensor = cute.nvgpu.make_tiled_tma_atom_A( tma_op, a, a_smem_layout_one_stage, mma_tiler_mnk, tiled_m", "sha256": "60ca34012cb11ec18e15df9160f3855f4f84db9a459f67d9b7b757bd738b7650"}, {"id": "u011", "kind": "prose", "locator": "body:L74-L74", "preview": "For the two-CTA tutorial, the corresponding operation is `CopyBulkTensorTileG2SMulticastOp(CtaGroup.TWO)` and the cluster layout/multicast participants must agree with the launch.", "sha256": "131b73c4815e2daf0d7909840ca0dfb9311821758eed73af3350d021bd113207"}, {"id": "u012", "kind": "prose", "locator": "body:L78-L78", "preview": "`fp16_gemm_2.py` specializes TMA, MMA, and epilogue warps and uses `PipelineTmaUmma` plus `PipelineUmmaAsync` to represent full/empty ownership. It also retains explicit TMEM allocation, epilogue TMEM-copy partitioning, TMA-store completion", "sha256": "2642e3d5608c72ac65111fba5b4e0ff7f40f082f286dc2eb0264a7a484a8c8ed"}, {"id": "u013", "kind": "prose", "locator": "body:L80-L80", "preview": "CuTe supplies typed layout algebra and Blackwell helpers such as `make_smem_layout_a` and `make_smem_layout_b`. The author still supplies the MMA tiler, datatypes, operand major modes, alignment, cluster shape, and pipeline policy; layout a", "sha256": "d6c1c90120129d90b89dff9868a75e231e1d052c8d141776044e6ada0808f8ea"}, {"id": "u014", "kind": "prose", "locator": "body:L84-L84", "preview": "The following files are untruncated, verbatim captures. Their bytes match the SHA-256 records in each PR's `PROVENANCE.yaml`, and the seven numeric sizes below are physical line counts. To inspect through the repository helper, run `conda r", "sha256": "973251632a33fdc9e5912eab3418784d8643fd27350dc1b9c9a6f5b406ebd703"}, {"id": "u015", "kind": "table-row", "locator": "body:L88-L88", "preview": "| File | Upstream purpose | Lines | | [`fp16_gemm_0.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_0.py) | Software-pipelined FP16 tutorial baseline | 447 |", "sha256": "b18a5ec40a6c45897cf6249107ed902c41a33e9ef744d1bab24361e43e147139"}, {"id": "u016", "kind": "table-row", "locator": "body:L89-L89", "preview": "| File | Upstream purpose | Lines | | [`fp16_gemm_1.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_1.py) | Add two-CTA MMA and TMA multicast | 535 |", "sha256": "d6bf4bd7815fdca9a99c3fc22ff3d92162b156750825d479ced568997c142584"}, {"id": "u017", "kind": "table-row", "locator": "body:L90-L90", "preview": "| File | Upstream purpose | Lines | | [`fp16_gemm_2.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_2.py) | Add TMA/MMA/epilogue warp specialization | 679 |", "sha256": "1e5ec9c2a029b9219be46f35295e8e70936565a4222e8f2af8b92686ef480c39"}, {"id": "u018", "kind": "table-row", "locator": "body:L91-L91", "preview": "| File | Upstream purpose | Lines | | [`fp16_gemm_3.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_3.py) | Add static persistent scheduling | 769 |", "sha256": "352bb7673da6dbf2d748da7fc8a6b97df8bbaaf3c384d87dad952797c5e376ee"}, {"id": "u019", "kind": "table-row", "locator": "body:L92-L92", "preview": "| File | Upstream purpose | Lines | | [`fp16_gemm_4.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_4.py) | Add preferred and fallback/dynamic clusters | 1065 |", "sha256": "14f3ac2f46ca97c9134ac0b89fc357bffd4074ee52b3b6d16ac578d8f07865cb"}, {"id": "u020", "kind": "table-row", "locator": "body:L93-L93", "preview": "| File | Upstream purpose | Lines | | [`fp16_gemm_5.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_5.py) | Add TMA data prefetch | 919 |", "sha256": "9fd8f92a66857f1af567a8e0f0e9189a5d04916a7d81244ff86764bcf57e6a21"}, {"id": "u021", "kind": "table-row", "locator": "body:L94-L94", "preview": "| File | Upstream purpose | Lines | | [`fp16_gemm_6.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_6.py) | Add Programmatic Dependent Launch | 1002 |", "sha256": "eb3aaa1fe6a194346ade2621b9c431c4380a88779e4eb4c04a97b71d5ab0f931"}, {"id": "u022", "kind": "table-row", "locator": "body:L95-L95", "preview": "| File | Upstream purpose | Lines | | [`dense_gemm_persistent_prefetch.py`](../../artifacts/prs/cutlass/PR-2881/key-files/examples/python/CuTeDSL/blackwell/dense_gemm_persistent_prefetch.py) | Persistent GEMM with TMA prefetch | full captur", "sha256": "9ebdb330439f1139f28b3838ec227c79d34b7975dec2c9e5df6d14660bb3b241"}, {"id": "u023", "kind": "table-row", "locator": "body:L96-L96", "preview": "| File | Upstream purpose | Lines | | [`clc.py`](../../artifacts/prs/cutlass/PR-3021/key-files/python/CuTeDSL/cutlass/cute/arch/clc.py) | CLC Python binding | full capture |", "sha256": "79166083bd1d286c85076e5de9724750ddff7a17063461789553b1bb6ed340a3"}, {"id": "u024", "kind": "prose", "locator": "body:L98-L98", "preview": "PR 3106's series is an official progressive tutorial: software-pipelined FP16, two-CTA multicast, warp specialization, static persistence, preferred/dynamic clusters, prefetch, and PDL. In the v4.5.0 tag, the same series is under `examples/", "sha256": "b40157951f3e6a0b7e4642199d84f739ad03db4544e88e5408ccd6f00e1093ed"}, {"id": "u025", "kind": "list-item", "locator": "body:L102-L102", "preview": "- [CUTLASS v4.5.0 tag](https://github.com/NVIDIA/cutlass/tree/e406c186f510a15091cce01f782020ceb7ba8eb5)", "sha256": "8c040cd9bb2af23c748671cfd8c2c060bb7c97a2538c966f83ac196ccea9a9ad"}, {"id": "u026", "kind": "list-item", "locator": "body:L103-L103", "preview": "- [Pinned v4.5.0 tutorial series](https://github.com/NVIDIA/cutlass/tree/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm)", "sha256": "3375a33859f68d3d9dba83826c9e7d675fbd2d160d113044e3a54814fbac9c05"}, {"id": "u027", "kind": "list-item", "locator": "body:L104-L104", "preview": "- [FlashAttention-4 first-party post](https://tridao.me/blog/2026/flash4/)", "sha256": "ffdb1b83558fb29dbc72af17130e91affe8fa4562c2f6c01a63be8b7082b1ab3"}, {"id": "u028", "kind": "list-item", "locator": "body:L108-L108", "preview": "- [tcgen05-mma](../hardware/tcgen05-mma.md) \u2014 underlying instruction semantics", "sha256": "b66ad09998a670f6ea957a7d01c9bc9760f1107ca50f4444c411f41d0a76fed5"}, {"id": "u029", "kind": "list-item", "locator": "body:L109-L109", "preview": "- [flash-attention-4](../kernels/flash-attention-4.md) \u2014 evidence-scoped CuTe DSL case study", "sha256": "b35637f36cfb18108cd0c26bab36090406fbfca2c4690afb113a6a08856e2602"}, {"id": "u030", "kind": "list-item", "locator": "body:L110-L110", "preview": "- [CUTLASS Blackwell source card](../../sources/docs/nvidia-cutlass-blackwell.md) \u2014 version-pinned source routes", "sha256": "a445c9adf25fe1277381dbdeea01cb44d616f8c12ebbf5a8251f145cfff266dc"}], "confidence_claimed": "verified", "headings": ["Scope", "F16/BF16 MMA construction", "TMEM lifecycle and epilogue copy", "TMA construction", "Warp specialization and layouts", "Verbatim upstream artifacts", "Primary references", "Related"], "id": "lang-cute-dsl", "path": "wiki/languages/cute-dsl.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-cutlass-blackwell.md", "url": "https://github.com/NVIDIA/cutlass/tree/v4.5.0"}, {"path": "sources/blogs/colfax-cutlass-blackwell.md", "url": "https://research.colfax-intl.com/cutlass-tutorial-writing-gemm-kernels-using-tmem-for-nvidia-blackwell-gpus/"}, {"path": "sources/blogs/flash-attention-4.md", "url": "https://tridao.me/blog/2026/flash4/"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-cutlass-blackwell", "blog-colfax-cutlass", "blog-flash-attention-4"], "title": "CuTe DSL for Blackwell", "type": "language", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "4bbed1d5aa5094a031fdbe1e8349f5079263caf9887a77721a5ace82c565fad5", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "This page uses CUDA 13.0.2's archived PTX ISA 9.0 grammar. The fragments are representative instruction forms, not complete inline-assembly functions: operands need correctly typed declarations, descriptors, collective issue, lifetime manag", "sha256": "033f6c44d6511b7780358590ba1a15b928bfc82f021f49297cac0039b7377829"}, {"id": "u002", "kind": "prose", "locator": "body:L9-L9", "preview": "Representative unscaled F16 forms include:", "sha256": "44a5c5881e34f6a016787e26082b9e096b84d5f8c8455c0c6fa3807305071b57"}, {"id": "u003", "kind": "code", "locator": "body:L11-L19", "preview": "```ptx tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [alloc_result_smem], ncols; ld.shared.b32 taddr, [alloc_result_smem]; tcgen05.mma.cta_group::1.kind::f16 [taddr], a_desc, b_desc, idesc, enable_input_d; tcgen05.commit.cta_group", "sha256": "8ee862ded6432cbcbaab1586c6dca16626f9c1a8562d6b39189ba6c05e5afb44"}, {"id": "u004", "kind": "prose", "locator": "body:L21-L21", "preview": "`tcgen05.alloc` writes a 32-bit TMEM address to shared memory and counts columns. Allocation and deallocation are warp-collective for `cta_group::1`, every allocation must be deallocated before kernel exit, and all tcgen05 instructions in o", "sha256": "549eeb92a806311fe19b9d3a00fc5521bddc347201227fe36fe18b4ab20ce0aa"}, {"id": "u005", "kind": "prose", "locator": "body:L23-L23", "preview": "TMEM/register transfers and shared-memory/TMEM copies have their own asynchronous completion rules:", "sha256": "309d97f3ce7cd084b463b572dd973759f13aa75f87df09939aa962b25d2b998f"}, {"id": "u006", "kind": "code", "locator": "body:L25-L33", "preview": "```ptx tcgen05.ld.sync.aligned.32x32b.x1.b32 {r0}, [taddr]; tcgen05.wait::ld.sync.aligned; tcgen05.st.sync.aligned.32x32b.x1.b32 [taddr], {r0}; tcgen05.wait::st.sync.aligned; tcgen05.cp.cta_group::1.128x256b [taddr], sdesc; ```", "sha256": "07a9ef9877db393b8545daac9555a8c89fecdb37869186d00ca531f2c0673fa4"}, {"id": "u007", "kind": "prose", "locator": "body:L35-L35", "preview": "`tcgen05.ld` and `tcgen05.st` are warp-collective. `tcgen05.cp` copies a shaped shared-memory descriptor into TMEM. MMA and cp completion can be attached to an mbarrier with `tcgen05.commit`; the source and destination must remain live unti", "sha256": "5d14e69f987be01a3b619b1aa87a7089e1f92428c3dc595e2db26b3fcba0ed3b"}, {"id": "u008", "kind": "list-item", "locator": "body:L39-L39", "preview": "- A TMA global-to-shared load completes bytes on its mbarrier; a consumer waits for that phase before reading the destination.", "sha256": "78e297e1879d27a7cf7e373d63d1a5098a93c33b03215f1eafce9842ed1aa2a0"}, {"id": "u009", "kind": "list-item", "locator": "body:L40-L40", "preview": "- `tcgen05.commit` makes an mbarrier track completion of prior asynchronous tcgen05 MMA/cp/shift operations issued by the thread.", "sha256": "ed6a936d875de52f53dfa04b1ae6a7d11f9232c0e335cd07557501558d6b9e4c"}, {"id": "u010", "kind": "list-item", "locator": "body:L41-L41", "preview": "- `tcgen05.wait::ld` and `tcgen05.wait::st` wait for the corresponding prior TMEM/register transfers.", "sha256": "a9db9c17084afc3bea027eeda6880b05178a6a63a358cbd960d4a046f9c45bb6"}, {"id": "u011", "kind": "list-item", "locator": "body:L42-L42", "preview": "- `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` constrain tcgen05 operations around a documented cross-thread execution-ordering handoff. They are not substitutes for TMA or MMA completion waits.", "sha256": "711d6cdcb2d789fb3aeb9b8207f61c2863f9ac6fbb4f1408e2fc015cba4a4d73"}, {"id": "u012", "kind": "prose", "locator": "body:L46-L46", "preview": "PTX ISA 9.0 defines these typed operations:", "sha256": "9adc5d2ba8359512852b72fddbd060410dec97cb859c1966ad61851b7308d289"}, {"id": "u013", "kind": "code", "locator": "body:L48-L51", "preview": "```ptx cvt.rn.f16x2.e2m1x2 result_f16x2, packed_fp4_pair; mov.b32 {byte0, byte1, byte2, byte3}, packed_word; ```", "sha256": "e38e93ad27ad63afd75bcd8acd17fea4fd7b8d6947dac9c50bbc5189fd76c762"}, {"id": "u014", "kind": "prose", "locator": "body:L53-L53", "preview": "The first converts one byte containing two E2M1 values into a 32-bit F16x2 result. The second can decompose a 32-bit scalar into four byte-sized destinations when operand declarations satisfy the scalar-to-vector size rules. PTX specifies t", "sha256": "4e1316f897104a6e4ff30b2d1629ba8c31578c28efa1f9f335e2a90635a5bc10"}, {"id": "u015", "kind": "code", "locator": "body:L57-L61", "preview": "```ptx ld.global.L1::no_allocate.v2.u64 {r0, r1}, [addr]; ld.global.L1::evict_last.v2.u64 {r0, r1}, [addr]; ld.global.v4.u64 {r0, r1, r2, r3}, [addr]; ```", "sha256": "340109ac2a79a1eab9995193f4fdf446a23eac0c855e50ec472ab508290c93ad"}, {"id": "u016", "kind": "prose", "locator": "body:L63-L63", "preview": "`L1::no_allocate` and `L1::evict_last` are eviction-priority hints and may not always be respected. They do not guarantee L1 bypass or residency. The vector forms above move 16 and 32 bytes; whether a width or hint helps depends on alignmen", "sha256": "2d50ea674a04d5f5af2f3d408566c7166d4da40732d3c0f69a61ee3be6bda853"}, {"id": "u017", "kind": "prose", "locator": "body:L67-L67", "preview": "CLC cancellation is an asynchronous request followed by response decoding:", "sha256": "e1053c3dda6a336be00c8fe12b42dda779f9491181fe98936e6b907183ac3f63"}, {"id": "u018", "kind": "code", "locator": "body:L69-L76", "preview": "```ptx clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [response_smem], [response_mbarrier]; clusterlaunchcontrol.query_cancel.is_canceled.pred.b128 p, response_b128; @p clusterlaunchcontrol.query_cancel.", "sha256": "3d1e08688104e24edf01c64713b007cd76c560a8224eb3ff452117f18046ea42"}, {"id": "u019", "kind": "prose", "locator": "body:L78-L78", "preview": "The request writes an opaque 16-byte response to shared memory and completes on the mbarrier. Code must wait for that phase before loading and querying the response. A successful query returns the first CTA coordinate of a canceled not-yet-", "sha256": "a71d676af20173533f4312c70e51a00ade2b502fcf17504097caaaccb8229b5d"}, {"id": "u020", "kind": "prose", "locator": "body:L82-L82", "preview": "A representative 2D global-to-cluster-shared load is:", "sha256": "4420aec4da4b62535c1e9270382b6fdf555635efc5797fb7dc2d8db90a7767ab"}, {"id": "u021", "kind": "code", "locator": "body:L84-L87", "preview": "```ptx cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes [dst_smem], [tensor_map, {x, y}], [full_mbarrier]; ```", "sha256": "384b75e38a92db0f949c0b5a2f222f154108174be7c8aeb347f23bd57cbe6c54"}, {"id": "u022", "kind": "prose", "locator": "body:L89-L89", "preview": "Cluster multicast adds the full `.multicast::cluster` qualifier and a 16-bit CTA mask:", "sha256": "bdbff1d282630fa2048aa2d3e5b44339402c00503993effcf84fb95908336362"}, {"id": "u023", "kind": "code", "locator": "body:L91-L94", "preview": "```ptx cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster [dst_smem], [tensor_map, {x, y}], [full_mbarrier], cta_mask; ```", "sha256": "a65641655400da2854fcf8a87260ac7f8823215dba2e9d9cbdaa262378541169"}, {"id": "u024", "kind": "prose", "locator": "body:L96-L96", "preview": "The multicast data and completion signal target the selected CTAs at corresponding shared-memory offsets. Each destination must have a live, initialized matching barrier and wait for its own phase before consuming the tile.", "sha256": "a4eb1693de166fae870fc703cd84ebf94c38231450f081d012205e644ee51b01"}, {"id": "u025", "kind": "list-item", "locator": "body:L100-L100", "preview": "- [CUDA C++](cuda-cpp.md) \u2014 inline PTX integration", "sha256": "0cc70a94a27647104f0234aed6752f236117acdbd312d3b46d7eccd2e99e116a"}, {"id": "u026", "kind": "list-item", "locator": "body:L101-L101", "preview": "- [tcgen05 MMA](../hardware/tcgen05-mma.md) \u2014 operand, descriptor, completion, and shape details", "sha256": "b3297f05442f0dbd4d7c52be92ee40895b97bd816b3fbed5217de657284554bd"}, {"id": "u027", "kind": "list-item", "locator": "body:L102-L102", "preview": "- [Tensor Memory](../hardware/tmem.md) \u2014 allocation, layout, and transfer details", "sha256": "c13ae762a67d3f3b24856da39bfe99e2231999145211ed928d11e100c5cabad8"}, {"id": "u028", "kind": "list-item", "locator": "body:L103-L103", "preview": "- [Cluster Launch Control](../hardware/clc.md) \u2014 complete request/decode protocol", "sha256": "9e45deb92f889fefaf02795579b0410a680552b1ee29c0b894cadd5c4df97e11"}, {"id": "u029", "kind": "list-item", "locator": "body:L104-L104", "preview": "- [TMA](../hardware/tma.md) \u2014 tensor-map and pipeline semantics", "sha256": "325d9c316ccd41d8da1955efe4719b95727a1b4e47e368cc36dc5b69c2df3389"}, {"id": "u030", "kind": "list-item", "locator": "body:L105-L105", "preview": "- [NVFP4](../hardware/nvfp4.md) \u2014 packed format and block scaling", "sha256": "41e892a35a6f5c8b6e2a987ed2c3b6d644d33c7c199fe6b98d74dfd378ee941b"}, {"id": "u031", "kind": "list-item", "locator": "body:L109-L109", "preview": "- [CUDA 13.0.2, PTX ISA 9.0](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html)", "sha256": "ec36f101dd169720db290c03b2a18a106539b88b4b3c0c12559a90520c58403b"}], "confidence_claimed": "verified", "headings": ["Scope", "tcgen05 and Tensor Memory", "Completion and Ordering Are Distinct", "Packed FP4 Conversion", "Cache Eviction Hints and Vector Width", "Cluster Launch Control", "Tensor Memory Accelerator", "Related", "Primary Reference"], "id": "lang-ptx", "path": "wiki/languages/ptx-sm100.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/yue-nvfp4-hackathon.md", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}], "risk_flags": ["code", "ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-ptx-isa-sm100", "doc-nvidia-tuning-guide", "blog-yue-nvfp4"], "title": "PTX Instructions for SM100", "type": "language", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "ea6b2fb7068dfa7b8a5d31fd9324937075bcbd313e86292a970f031a005a821c", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "Native Blackwell TCGen5/TMEM compiler support enters between Triton v3.2.0 and v3.3.0. In the exact tag comparison, the corresponding TCGen5 MMA, TMEM, and MMAv5-lowering symbols are absent at v3.2.0 (`9641643d`) and present at v3.3.0 (`819", "sha256": "e3920948bd65b746042cddc8a55cbb58d67cea87f7de76b45b7f86b40076a399"}, {"id": "u002", "kind": "table-row", "locator": "body:L7-L7", "preview": "| Release | Evidence-scoped milestone | | v3.2.0 | Checked negative side of the native-backend boundary. |", "sha256": "29a2a0a6a1b0cab67d3c9170be5b53e14d5e925950ae7accae025a44743032c2"}, {"id": "u003", "kind": "table-row", "locator": "body:L8-L8", "preview": "| Release | Evidence-scoped milestone | | v3.3.0 | First checked tag after v3.2.0 with TCGen5/TMEM operations, allocation and lowering passes, and concrete conversion tests. |", "sha256": "96b6b9eb266480436cec3e260a7190ef701a6cdb182590188a1a9e08e8b2bc72"}, {"id": "u004", "kind": "table-row", "locator": "body:L9-L9", "preview": "| Release | Evidence-scoped milestone | | v3.5.0 | Tagged tree includes an explicit Gluon TCGen5/TMEM tutorial and the Blackwell block-scaled matmul tutorial; release notes also document warp-specialization work. |", "sha256": "bbfeba5bdbe013bf1e25f0b3313537c044591289530ad5ad3348df812d2f7db9"}, {"id": "u005", "kind": "table-row", "locator": "body:L10-L10", "preview": "| Release | Evidence-scoped milestone | | v3.6.0 | Generalizes TCGen5 copies/layouts and MMA handling, advances aref-style warp specialization, and adds initial multi-CTA/2-CTA Gluon work. It is not the introduction boundary. |", "sha256": "ae8dd494d86b9bdf4335d086a7fc67293ccb647ee15a43e83ae74a51c7583926"}, {"id": "u006", "kind": "table-row", "locator": "body:L11-L11", "preview": "| Release | Evidence-scoped milestone | | v3.7.0 / v3.7.1 | v3.7.0 continues 2-CTA, multicast, and TMA work; v3.7.1 is a two-regression patch with no advertised new API or feature. |", "sha256": "4635dc9d8b11b6614cb89888178e84327676f99fed5e0b67f2e5f75e64e46829"}, {"id": "u007", "kind": "prose", "locator": "body:L13-L13", "preview": "The compiler evidence proves that native paths exist. It does not prove that every plain `tl.dot` shape selects TCGen5: instruction selection can depend on architecture, dtype, shape, layout, and compiler configuration. A claim about one ke", "sha256": "469542c8b3f4a8fa1b59f96d3475a8808eb5e863feb48288a117287981a6cd98"}, {"id": "u008", "kind": "prose", "locator": "body:L17-L17", "preview": "The v3.5.0 tree provides two useful pinned examples:", "sha256": "e5dab5d53c539d58ff6ace9b78fcdcc0297af67d3094ebb9b76bdd4f6d044c54"}, {"id": "u009", "kind": "list-item", "locator": "body:L19-L19", "preview": "- [`python/tutorials/gluon/06-tcgen05.py`](https://github.com/triton-lang/triton/blob/c3c476f357f1e9768ea4e45aa5c17528449ab9ef/python/tutorials/gluon/06-tcgen05.py) explicitly allocates, loads, and stores TMEM and invokes TCGen5 MMA through", "sha256": "d517d20730d84aea345276a3cd17d16fc337b410b720198a34bd749d15a97691"}, {"id": "u010", "kind": "list-item", "locator": "body:L20-L20", "preview": "- [`python/tutorials/10-block-scaled-matmul.py`](https://github.com/triton-lang/triton/blob/c3c476f357f1e9768ea4e45aa5c17528449ab9ef/python/tutorials/10-block-scaled-matmul.py) demonstrates `tl.dot_scaled` for Blackwell block-scaled matmul.", "sha256": "b50af21443f3e19ead4a29e736f49169439ab7c89c9c9d58f3d1ff30d4d513d7"}, {"id": "u011", "kind": "prose", "locator": "body:L22-L22", "preview": "Triton v3.6.0 expands those foundations with broader layouts and copies and initial multi-CTA/2-CTA Gluon support. \u201cInitial\u201d is deliberate: v3.7.0 contains follow-on end-to-end 2-CTA, multicast, and TMA changes. See [`doc-triton-3.6-blackwe", "sha256": "248e2625a1e191a60b52ac1983ab2fa8ef8742c3dc42749ec8560102a3d81256"}, {"id": "u012", "kind": "list-item", "locator": "body:L26-L26", "preview": "- [`pr-vllm-34597`](../../sources/prs/vllm/PR-34597.md), pinned at `a1257fd1`, adds FP8 KV-cache handling to the Triton MLA decode backend. Its verbatim kernel contains `tl.dot`, but no target guard, Triton-version requirement, TCGen5/TMEM ", "sha256": "085b49bb4247323760315512a1abade8e49e1a52e8ae7bb49cc3ea835a7fa13e"}, {"id": "u013", "kind": "list-item", "locator": "body:L27-L27", "preview": "- [`pr-vllm-29339`](../../sources/prs/vllm/PR-29339.md), pinned at `c17610e2`, gates MXFP4 `triton_kernels` dispatch to SM90 and SM100. It changes dispatch logic, not a kernel or compiler lowering.", "sha256": "992ea35b568fd02174010ce53c73d442063daab448917c69bc178401610ad4d4"}, {"id": "u014", "kind": "list-item", "locator": "body:L28-L28", "preview": "- [`pr-sglang-21019`](../../sources/prs/sglang/PR-21019.md), pinned at `5bdc07d9`, provides a Triton GatedDeltaNet projection rearrangement using loads and stores, with no `tl.dot`.", "sha256": "9b174a1966e9c73cd142807ed4ba1fadd3d338c47fb5f52962b55350f2e9bc44"}, {"id": "u015", "kind": "list-item", "locator": "body:L29-L29", "preview": "- [`pr-sglang-22079`](../../sources/prs/sglang/PR-22079.md), pinned at `5638d40f`, provides an extend-attention Triton kernel with real `tl.dot` operations. The source does not contain an emitted-PTX witness for a particular MMA instruction", "sha256": "e4a160b6f2a7c6504d671624f8587b747817600bf587d805c15d41780f4dfd5c"}, {"id": "u016", "kind": "prose", "locator": "body:L31-L31", "preview": "These are verified downstream Triton examples. They must not be combined with the separate compiler-version evidence to infer an unobserved lowering.", "sha256": "e25568d8c8d675f229103457b89bfe14c70bb60ea3c384f8aec4ee49263bde73"}, {"id": "u017", "kind": "prose", "locator": "body:L35-L35", "preview": "SGLang [`pr-sglang-5390`](../../sources/prs/sglang/PR-5390.md) reports 10,447.34 total tok/s for its CUTLASS MLA run and 8,227.35 total tok/s for its Triton run, 26.98% higher under the PR's recorded DeepSeek-R1, 3,000-prompt, TP8/DP8, floa", "sha256": "e18be1d70f68c7e9f57e4ce6512515fba305011c95d9c3fb4f4aa91cee317942"}, {"id": "u018", "kind": "prose", "locator": "body:L37-L37", "preview": "SGLang [`pr-sglang-21595`](../../sources/prs/sglang/PR-21595.md) changes the SM100 datacenter multimodal-attention default from `triton_attn` to FA4. That routing decision is likewise workload- and architecture-scoped.", "sha256": "bc5f5aa77792eade3e6336d612d0c662e86f76c68607c91984455ea36ce09846"}, {"id": "u019", "kind": "prose", "locator": "body:L39-L39", "preview": "The live [FlashInfer-Bench leaderboard](https://bench.flashinfer.ai/), retrieved 2026-08-08, reports these author rows across 660 workloads each:", "sha256": "eb3c9c6949d2033f82810f3915ba67acd79712b58c62852e8abee880b272c1f9"}, {"id": "u020", "kind": "table-row", "locator": "body:L43-L43", "preview": "| Author | Average speedup | Resolved | | Gemini 2.5 Pro | 0.628x | 73.1% |", "sha256": "2c931bfe6fdce66ed874bc7939e251dfd74ce7efc8f76df9065a732b9160595d"}, {"id": "u021", "kind": "table-row", "locator": "body:L44-L44", "preview": "| Author | Average speedup | Resolved | | GPT-5 | 0.467x | 92.3% |", "sha256": "10715c194c5d2213a64860a9723c9215121405e98c58f8c654d5568d0ba792ed"}, {"id": "u022", "kind": "table-row", "locator": "body:L45-L45", "preview": "| Author | Average speedup | Resolved | | Claude Opus 4.1 | 0.456x | 73.1% |", "sha256": "da12cac615989bf57643afef7d553cf8210b73575d9b19def2feff7012898c65"}, {"id": "u023", "kind": "prose", "locator": "body:L47-L47", "preview": "The leaderboard does not attach those rows to a Triton release or identify them as a Triton-only language subset.", "sha256": "dc9b8f5d025f7b785f70fdb6cdf00ba150996bc6d75aca61cadfd91fc3f17a18"}, {"id": "u024", "kind": "prose", "locator": "body:L51-L51", "preview": "The CUDA Programming Guide explains that CPU setup and launch overhead can be significant for short kernels and that CUDA Graphs reduce repeated launch costs by preparing work in advance. At vLLM commit `a1257fd1`, `FULL_AND_PIECEWISE` is t", "sha256": "86ec28fae13765d35d9fddd442f01998a03d9e5f585a31169e38df465f897058"}, {"id": "u025", "kind": "prose", "locator": "body:L55-L55", "preview": "Each linked file is stored verbatim under its bundle's recorded merge SHA:", "sha256": "d8ff7fcdf07f06c752587e1f093f017a673d3c1baaf61a748a7f43d06cec1c58"}, {"id": "u026", "kind": "table-row", "locator": "body:L59-L59", "preview": "| File | Verified role | | [`triton_decode_attention.py`](../../artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/ops/triton_decode_attention.py) | Triton MLA decode kernels with `tl.dot`; PR 34597 adds FP8 cache handling. |", "sha256": "c531c719c3ba28f58191f062dda8ae43664fde6c4201bfbcf6c130b7cd19f417"}, {"id": "u027", "kind": "table-row", "locator": "body:L60-L60", "preview": "| File | Verified role | | [`triton_mla.py`](../../artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/backends/mla/triton_mla.py) | Backend wrapper and supported FP8 cache dtypes. |", "sha256": "6ca7b916a53f1fcdb23f6e6fdf4bdc51d7a26add939752dcc66c526d30ed4f26"}, {"id": "u028", "kind": "table-row", "locator": "body:L61-L61", "preview": "| File | Verified role | | [`format_conversion.py`](../../artifacts/prs/flashinfer/PR-1025/key-files/flashinfer/triton/format_conversion.py) | Triton FP8/FP16 format-conversion kernels. |", "sha256": "e57454c7314e09c20b08b2d5cd27ae1786974d1c9b2daddb61c1997228957b94"}, {"id": "u029", "kind": "table-row", "locator": "body:L62-L62", "preview": "| File | Verified role | | [`norm.py`](../../artifacts/prs/sglang/PR-20910/key-files/python/sglang/jit_kernel/norm.py) | Triton normalization kernels. |", "sha256": "65fd3a5f292128dc8cab99086d97e11181d0e2fc254c13490ac0543dd66ce342"}, {"id": "u030", "kind": "table-row", "locator": "body:L63-L63", "preview": "| File | Verified role | | [`gdn_fused_proj.py`](../../artifacts/prs/sglang/PR-21019/key-files/python/sglang/jit_kernel/triton/gdn_fused_proj.py) | Triton GatedDeltaNet projection rearrangement using loads/stores. |", "sha256": "f663a530904bc8709ee5d576f14ec0bc29e6a806bbfd85f94aa5274d519b0bc8"}, {"id": "u031", "kind": "table-row", "locator": "body:L64-L64", "preview": "| File | Verified role | | [`extend_attention.py`](../../artifacts/prs/sglang/PR-22079/key-files/python/sglang/srt/layers/attention/triton_ops/extend_attention.py) | Triton extend-attention kernels containing `tl.dot`. |", "sha256": "d03db60ea2e11a9393db7a9ccf09c4eb67f51e4fe93b2ddca10d598d0444d892"}, {"id": "u032", "kind": "prose", "locator": "body:L66-L66", "preview": "The complete tracked PR universe and its captured/skipped flags are recorded in [`data/triton-universe.yaml`](../../data/triton-universe.yaml); this page does not duplicate that changing count.", "sha256": "b82bbb549f1e4dfa484c36d1134005bb13fd50ddc1a6db308b67a2dab96b78c1"}, {"id": "u033", "kind": "prose", "locator": "body:L68-L68", "preview": "This verbatim excerpt from the pinned vLLM decode kernel demonstrates the page's deliberately limited downstream claim\u2014FP8 rescaling followed by `tl.dot`, without proving a particular emitted MMA instruction:", "sha256": "161635e13b202066602eecd585ccef97bc289ce7efd17d6eda51155b8cbce11b"}, {"id": "u034", "kind": "code", "locator": "body:L70-L76", "preview": "```python if k.dtype.is_fp8(): k = (k.to(tl.float32) * ks).to(q.dtype) qk = tl.dot(q, k.to(q.dtype)) if BLOCK_DPE > 0: offs_buf_kpe = kv_loc[None, :] * stride_buf_kbs ```", "sha256": "98ec2d0651858cbb3b5795bc15f9064e1167c1f0800a61b764ad5906cb6bcca9"}, {"id": "u035", "kind": "metadata-version", "locator": "frontmatter:version_sensitive", "preview": "{\"id\": \"vs-triton-3.3-blackwell-tcgen05\"}", "sha256": "2c0977b31b1a2e307ef3fca06c52165f985ad1366e7255b1e8c869baca5a2669"}], "confidence_claimed": "verified", "headings": ["Verified release boundary", "User-visible surfaces", "What downstream code establishes", "Scoped ecosystem results", "Launch overhead and CUDA Graphs", "Provenance-pinned examples"], "id": "lang-triton", "path": "wiki/languages/triton-blackwell.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/triton-3.3-blackwell.md", "url": "https://github.com/triton-lang/triton/compare/v3.2.0...v3.3.0"}, {"path": "sources/docs/triton-3.6-blackwell.md", "url": "https://github.com/triton-lang/triton/releases/tag/v3.6.0"}, {"path": "sources/prs/vllm/PR-34597.md", "revision": "a1257fd1", "url": "https://github.com/vllm-project/vllm/pull/34597"}, {"path": "sources/prs/vllm/PR-29339.md", "revision": "c17610e2", "url": "https://github.com/vllm-project/vllm/pull/29339"}, {"path": "sources/prs/sglang/PR-22079.md", "revision": "5638d40f", "url": "https://github.com/sgl-project/sglang/pull/22079"}, {"path": "sources/prs/sglang/PR-21019.md", "revision": "5bdc07d9", "url": "https://github.com/sgl-project/sglang/pull/21019"}, {"path": "sources/prs/sglang/PR-5390.md", "revision": "84810da4", "url": "https://github.com/sgl-project/sglang/pull/5390"}, {"path": "sources/prs/sglang/PR-21595.md", "revision": "87a27682", "url": "https://github.com/sgl-project/sglang/pull/21595"}], "risk_flags": ["code", "table", "version"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-triton-3.3-blackwell", "doc-triton-3.6-blackwell", "pr-vllm-34597", "pr-vllm-29339", "pr-sglang-22079", "pr-sglang-21019", "pr-sglang-5390", "pr-sglang-21595"], "title": "Triton on Blackwell", "type": "language", "unresolved_source_ids": [], "version_sensitive": {"id": "vs-triton-3.3-blackwell-tcgen05"}} +{"body_sha256": "7f0bb2f02e5a9f1f50c0c9003dc0baeeee3685876e3897726abbd12a14ae611a", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "For Hopper `wgmma.mma_async.m64nNk16` with an FP32 accumulator, each warpgroup thread holds `N/2` FP32 D registers. At `N=256`, that is 128 registers per thread for D. WGMMA updates that register vector asynchronously; the program uses the ", "sha256": "81c50814350c832517909572561c4f72593f90fdf9fd14235d07601b29cc502b"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "For `sm_100a`, tcgen05 D is addressed in Tensor Memory rather than supplied as a per-thread register vector. The CTA-visible TMEM structure has 128 lanes by 512 columns of 32-bit cells, or 256 KiB when fully allocated. Allocation is dynamic", "sha256": "90915abc172e7b8a5e9fdec98cdb99a4f905d1e52e66f7b4a939c546ed740ab2"}, {"id": "u003", "kind": "prose", "locator": "body:L9-L9", "preview": "This removes the resident D vector from the ordinary register file. It does not remove all register pressure: descriptors, addresses, loop state, pipeline state, and each TMEM-to-register epilogue batch still need registers. Both Hopper and", "sha256": "3065c8674899961c948b2920ce5998033450f02b782f496619e938425f12bb2c"}, {"id": "u004", "kind": "table-row", "locator": "body:L15-L15", "preview": "| Concern | Hopper WGMMA | Blackwell tcgen05 | | Resident D | Per-thread register fragment | TMEM region addressed by `taddr` |", "sha256": "caf62cc51664762ce0014371563bd85fb173f40058af73033877c114fb4e957d"}, {"id": "u005", "kind": "table-row", "locator": "body:L16-L16", "preview": "| Concern | Hopper WGMMA | Blackwell tcgen05 | | Example FP32 D cost | `N/2` registers/thread; 128 at `N=256` | No resident per-thread D vector |", "sha256": "6e4cf747f49122769816e81e7f7200b5e7c2dbef080de92600e88c94f0a6a67e"}, {"id": "u006", "kind": "table-row", "locator": "body:L17-L17", "preview": "| Concern | Hopper WGMMA | Blackwell tcgen05 | | First accumulation | Set WGMMA `scale-d` false to compute `D=A*B`, or initialize/use D when accumulation is intended | Set `enable-input-d` false to compute `D=A*B`, or explicitly initialize ", "sha256": "3eef8e2f3c1f793204f9ac25cd12a7ca2646443ef0aac8c4c12cf7b7d68a4482"}, {"id": "u007", "kind": "table-row", "locator": "body:L18-L18", "preview": "| Concern | Hopper WGMMA | Blackwell tcgen05 | | Compute completion | WGMMA commit/wait group | `tcgen05.commit` to an mbarrier, then wait before a consumer reads/reuses D |", "sha256": "0b73d94f0d547ca8fde7bfcbd32bb2051564e76bfacb9631eb78add6646bf88c"}, {"id": "u008", "kind": "table-row", "locator": "body:L19-L19", "preview": "| Concern | Hopper WGMMA | Blackwell tcgen05 | | Epilogue access | Use the thread's D registers | Collective `tcgen05.ld`, then `tcgen05.wait::ld` before consuming result registers |", "sha256": "61e6795055d9b77d891ca6222fd9697889c19e800e9403c14867010befb557d6"}, {"id": "u009", "kind": "table-row", "locator": "body:L20-L20", "preview": "| Concern | Hopper WGMMA | Blackwell tcgen05 | | Cleanup | Register lifetime ends with the thread | Matching collective `tcgen05.dealloc` before kernel exit |", "sha256": "735266abf367eab94461e883b716688a8976f3847bed7a112cba79534d11f5d7"}, {"id": "u010", "kind": "table-row", "locator": "body:L21-L21", "preview": "| Concern | Hopper WGMMA | Blackwell tcgen05 | | Double buffering | Two simultaneously live D fragments consume two register fragments | Two live outputs consume disjoint TMEM columns inside the allocation |", "sha256": "b04813198ede3f9af7088d4f1b95aa578b1195e6e41e388421499ab2487372ba"}, {"id": "u011", "kind": "prose", "locator": "body:L23-L23", "preview": "The table describes storage contracts, not an occupancy prediction. Instruction shape is also not necessarily the same as a composed CTA tile.", "sha256": "195c5b3cd276fffc69431065c21852ab5d89c54e26805dfbc82eeadd098432ab"}, {"id": "u012", "kind": "list-item", "locator": "body:L27-L27", "preview": "1. Choose one CTA-group mode for the kernel. For allocation, `nCols` is a power of two in `[32, 512]`; allocations are column-granular and cover all 128 lanes.", "sha256": "663ec5e2ecd0084580d22cb5e1a1fdb601f34042656316eb82971db2810e2220"}, {"id": "u013", "kind": "list-item", "locator": "body:L28-L28", "preview": "2. Have every lane of one designated warp execute the same `.cta_group::1` allocation. Synchronize before other threads read the 32-bit `taddr` written to shared memory. Two-CTA mode requires one warp in each live peer CTA.", "sha256": "442f844d571108d531ca6c4f1c2fcb34a128af006b51feb868efa7ca66f6945f"}, {"id": "u014", "kind": "list-item", "locator": "body:L29-L29", "preview": "3. Initialize and publish the barriers used by TMA and tcgen05. If the first MMA should compute only `A*B`, supply a false `enable-input-d` predicate instead of assuming a mandatory TMEM zero-store pass.", "sha256": "688cacfc78db32ef2dbc1b5748ab430829d8949fa6ec1e7f1737048cbcfa2858"}, {"id": "u015", "kind": "list-item", "locator": "body:L30-L30", "preview": "4. Issue MMA from the permitted elected thread with valid A/B descriptors, instruction descriptor, predicate, and lifetimes.", "sha256": "c81894a6edec76da9df885f000827bc66cd3820796598af4d484b7a69c4643e9"}, {"id": "u016", "kind": "list-item", "locator": "body:L31-L31", "preview": "5. Attach completion of the relevant tcgen05 work to an mbarrier with `tcgen05.commit`. A fence controls execution ordering but is not a completion wait.", "sha256": "3f66638c0aa59578daef93b5cd3375015890606f7dbaa0f9f2b07f9e33caa19c"}, {"id": "u017", "kind": "list-item", "locator": "body:L32-L32", "preview": "6. After the completion handoff, participating epilogue lanes execute a legal `tcgen05.ld` shape and `tcgen05.wait::ld` before using the loaded registers.", "sha256": "add86c6758160d1ee6436e9bbb4a42118fa75ad68987d8da15053e12fcddcc68"}, {"id": "u018", "kind": "list-item", "locator": "body:L33-L33", "preview": "7. After all readers finish, every lane of the designated warp executes the matching deallocation on every kernel exit path. Allocation/deallocation address, column count, and CTA-group mode must agree.", "sha256": "f07d98a96b90ae28744b10a65fee6446286122ccfd1e24b3f6df56759d8d3b9a"}, {"id": "u019", "kind": "prose", "locator": "body:L35-L35", "preview": "Representative PTX ISA 9.0 forms are shown below. They omit declarations, descriptor construction, collective control flow, barriers, and inline-assembly constraints, so they are not a standalone kernel.", "sha256": "ba8a0bbd80024f2e78f378a5524beb0317003225da5d484380df0020489d90f3"}, {"id": "u020", "kind": "code", "locator": "body:L37-L44", "preview": "```ptx tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [saddr], nCols; tcgen05.mma.cta_group::1.kind::f16 [taddr], a_desc, b_desc, idesc, p; tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cta.b64 [mbar]; tcgen05.ld.sync.a", "sha256": "a5d9f9e0f8f9643e18992569407c06dd52f1583195162b4b41c9bdfa172c67d7"}, {"id": "u021", "kind": "prose", "locator": "body:L48-L48", "preview": "TMEM makes it practical to keep multiple output regions independently addressable while different warp roles issue MMA and drain a completed region. Safe overlap needs three distinct proofs:", "sha256": "472061f2ee2a0acce107d45fd26f201f4bd143c15aeb1efeaef163973f894aa0"}, {"id": "u022", "kind": "list-item", "locator": "body:L50-L50", "preview": "- compute completion occurs before an epilogue loads a region;", "sha256": "797834acb4631fde043533de9ef5d1938c604993e27069015705834e832921ab"}, {"id": "u023", "kind": "list-item", "locator": "body:L51-L51", "preview": "- all epilogue loads complete before that region is overwritten or deallocated;", "sha256": "f76c38cde4d90d1036b53206db7b3fa9eb7c698a8ec70a94f914970e27393b41"}, {"id": "u024", "kind": "list-item", "locator": "body:L52-L52", "preview": "- producer/consumer state and phase cannot alias a different pipeline stage.", "sha256": "53171494a69370538165b9a5233bde906d81d1a6117a916676ef9f91f33fa533"}, {"id": "u025", "kind": "prose", "locator": "body:L54-L54", "preview": "TMEM is not the only way to overlap tensor-core and non-matmul work. FlashAttention-3 already uses warp specialization and matmul/softmax interleaving on Hopper with register accumulators. The migration benefit is the different storage and ", "sha256": "c0d0df2e7890cb2071ca656369ce0ebacde9548a7e8d52684018dc1f460f15bf"}, {"id": "u026", "kind": "prose", "locator": "body:L58-L58", "preview": "FlashAttention-4's first-party paper places score/output accumulators in TMEM and assigns MMA, softmax, and correction work to specialized warp roles. The accompanying author post describes software-selected polynomial `exp2` work on CUDA c", "sha256": "3b5cea1d7c442196306c53f5222f1b29458f1a4a7f94880145e94721679140e9"}, {"id": "u027", "kind": "prose", "locator": "body:L62-L62", "preview": "Do not translate an instruction shape directly into a required CTA tile or assume that moving D guarantees no spills. For each concrete kernel:", "sha256": "458b810cc30dfa258823ec0f43d812cd1b0d189139c427c79b07cf5bcf6d2d8b"}, {"id": "u028", "kind": "list-item", "locator": "body:L64-L64", "preview": "1. Record the compiler's registers/thread, spill loads/stores, static/dynamic shared memory, barriers, threads/CTA, and cluster shape.", "sha256": "d16c6acd3215b8f25c44c532f8c911211631184ea4f19611507b0865dcbf5215"}, {"id": "u029", "kind": "list-item", "locator": "body:L65-L65", "preview": "2. Compute occupancy with the CUDA occupancy APIs for the actual launch; do not equate one warpgroup with one CTA.", "sha256": "61f8007df8291d17b10b5d98fa24b9e5270e9671577f6d2185f80148fcd9a1d0"}, {"id": "u030", "kind": "list-item", "locator": "body:L66-L66", "preview": "3. Sweep legal instruction descriptors, CTA tiles, pipeline depth, TMEM column partitioning, and epilogue batch width.", "sha256": "c38ba2f0bc7de4c81597e253f093a2e181a747fa3c44afa31f9fc7ca8fe9edcc"}, {"id": "u031", "kind": "list-item", "locator": "body:L67-L67", "preview": "4. Benchmark identical shapes, datatypes, outputs, synchronization, warmup, and trial statistics. A fixed latency such as \u201c420 cycles per TMEM load\u201d is not an ISA guarantee.", "sha256": "254ff0f6b010a5a5da0857102c7f77d31d8d19ebeb3a17d4d96f5fb73b3da8e2"}, {"id": "u032", "kind": "list-item", "locator": "body:L68-L68", "preview": "5. Inspect generated PTX/SASS to confirm that the intended tcgen05 shapes, waits, and no unexpected spills are present.", "sha256": "a0dc065f5f6cd024c8097f8d5c531813014b8bcd3af6638ce958f8f15187547f"}, {"id": "u033", "kind": "list-item", "locator": "body:L72-L72", "preview": "- TMEM is not an ordinary CUDA pointer space. Load through a legal collective tcgen05 copy operation before scalar arithmetic.", "sha256": "5a0dc36e41360a05b9484ba2c76a8a40f4c50cd91d824efc8390bd263d26569e"}, {"id": "u034", "kind": "list-item", "locator": "body:L73-L73", "preview": "- Do not issue collective allocation, load/store, or deallocation from lane 0 alone.", "sha256": "7a08a8655056273dbd66d60daed365c7599fec8e52caf56e1ba389cdb33f2a41"}, {"id": "u035", "kind": "list-item", "locator": "body:L74-L74", "preview": "- Do not substitute `tcgen05.fence` or `__syncthreads()` for asynchronous MMA completion.", "sha256": "39b83c942611535ed32d220db39984aaaa353937f00d2407e2638f6864583fa9"}, {"id": "u036", "kind": "list-item", "locator": "body:L75-L75", "preview": "- Do not zero TMEM unconditionally when a false `enable-input-d` predicate provides the intended first-write semantics.", "sha256": "b99c99b2e0c4cd1c80b2fe08796a0916cc6f0b10aa816e84fc12cc3a76507733"}, {"id": "u037", "kind": "list-item", "locator": "body:L76-L76", "preview": "- Do not treat TMEM-resident D as the entire kernel's register allocation; the epilogue can still spill.", "sha256": "34b5e7d2ea4620b8b6eac38d7fd6247040e0caa5401bfee293902cc71ebd84c9"}, {"id": "u038", "kind": "list-item", "locator": "body:L77-L77", "preview": "- Do not describe missing deallocation as a defined recoverable \u201cleak.\u201d The normative rule is explicit matching deallocation before exit.", "sha256": "7459f5d2fab8c6fe59eeef14ce208773a6ea79f29893b81890e9546f5334030d"}, {"id": "u039", "kind": "list-item", "locator": "body:L81-L81", "preview": "- [PTX ISA 9.0 WGMMA register fragments](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16)", "sha256": "56a14ee2db1f1b79e0e525648d0d966aaa69571dfe36c87de7168f26881ef269"}, {"id": "u040", "kind": "list-item", "locator": "body:L82-L82", "preview": "- [PTX ISA 9.0 Tensor Memory](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory)", "sha256": "aed47f18c514497e50694f89415704f98533229e52ed733c1898d3c9bb3e2747"}, {"id": "u041", "kind": "list-item", "locator": "body:L83-L83", "preview": "- [PTX ISA 9.0 tcgen05 allocation](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc)", "sha256": "31973f025402fdb2aee03f37ec947608275b8e131e5d9423444bc588c91662b7"}, {"id": "u042", "kind": "list-item", "locator": "body:L84-L84", "preview": "- [PTX ISA 9.0 tcgen05 MMA](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma)", "sha256": "4c1c4ca67353c9e04e383f31694978300286983ac6d9415be5a57439e730d050"}, {"id": "u043", "kind": "list-item", "locator": "body:L85-L85", "preview": "- [CUDA 13.0.2 Hopper Tuning Guide](https://docs.nvidia.com/cuda/archive/13.0.2/hopper-tuning-guide/index.html#occupancy)", "sha256": "acaee20e11d45fd20a8d0c6e29f05aec3d89d17a203eae0a47e821c7f1c2946c"}, {"id": "u044", "kind": "list-item", "locator": "body:L86-L86", "preview": "- [CUDA 13.0.2 Blackwell Tuning Guide](https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy)", "sha256": "8d963db7f76eb6508d3be3b8853709f57f9a8b2f98e623d121cff16c31de6585"}, {"id": "u045", "kind": "list-item", "locator": "body:L87-L87", "preview": "- [FlashAttention-3 paper](https://arxiv.org/abs/2407.08608)", "sha256": "5a5ef1b6d90749138db83d063d75df914e64702b3d88af097f7ab1b7918bd60c"}, {"id": "u046", "kind": "list-item", "locator": "body:L88-L88", "preview": "- [FlashAttention-4 paper v1](https://arxiv.org/abs/2603.05451v1)", "sha256": "0fb60e121910c6e3db40016b49479cf32b63868bf4c07d1765c52f09c5adad70"}, {"id": "u047", "kind": "list-item", "locator": "body:L92-L92", "preview": "- [Tensor Memory](../hardware/tmem.md) \u2014 allocation, addressing, and access constraints", "sha256": "b3c60111974bb86dc60da7215cbb37b18f5ed884f7afc8f2ba9e888704ee4c81"}, {"id": "u048", "kind": "list-item", "locator": "body:L93-L93", "preview": "- [tcgen05 MMA](../hardware/tcgen05-mma.md) \u2014 exact operation and completion semantics", "sha256": "8eaf7f0f1fcdebbab2fb0edc10f2597a39ea62bb5d5350106b42fccbf9e6260c"}, {"id": "u049", "kind": "list-item", "locator": "body:L94-L94", "preview": "- [register pressure](../patterns/register-pressure.md) \u2014 diagnosis and compiler-resource evidence", "sha256": "63798ee2070c890c2bee614823e025d90ee4f0841197c7da548cb5327b8c297d"}], "confidence_claimed": "verified", "headings": ["What actually changes", "Storage and lifecycle contrast", "Migration lifecycle", "Overlap without invented guarantees", "FlashAttention-4 case study", "Evidence-driven retuning", "Common migration errors", "Primary references", "Related"], "id": "migration-register-to-tmem", "path": "wiki/migration/register-to-tmem.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/prs/vllm/PR-22738.md", "revision": "b1361c72", "url": "https://github.com/vllm-project/vllm/pull/22738"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "pr-vllm-22738"], "title": "Register Accumulators to TMEM", "type": "migration", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "138004faa6219749f4d0e9072b9bca0649a4a7ad20e9616caa52e18ea33b5b17", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Port the programming model, not just the opcode:", "sha256": "9be3861214d8d1c37479c20bd21f2a3e7c33c7460656c78dc9992c6bbbe1999a"}, {"id": "u002", "kind": "table-row", "locator": "body:L9-L9", "preview": "| Concern | SM90 WGMMA | SM100 tcgen05.mma | | MMA issue | Warpgroup collective | One thread for group-1 or group-2 MMA |", "sha256": "aa5e3304df841c5d5e3ac5c52d103b8fcf0d8e0bb2a39c54a026f790dd7a4b9a"}, {"id": "u003", "kind": "table-row", "locator": "body:L10-L10", "preview": "| Concern | SM90 WGMMA | SM100 tcgen05.mma | | D accumulator | Per-thread registers | TMEM |", "sha256": "599eed1c6192c3cdba2fd7c04288ba5f919d0307d513ea12d52ed5533800a1f3"}, {"id": "u004", "kind": "table-row", "locator": "body:L11-L11", "preview": "| Concern | SM90 WGMMA | SM100 tcgen05.mma | | A operand | SMEM-descriptor or register forms, depending on instruction | SMEM-descriptor or TMEM-address forms |", "sha256": "c8e2f6685d308cb41e7788d727dab8bf0129426bad70f0138f73a6c64cb636c6"}, {"id": "u005", "kind": "table-row", "locator": "body:L12-L12", "preview": "| Concern | SM90 WGMMA | SM100 tcgen05.mma | | B operand | SMEM descriptor | SMEM descriptor |", "sha256": "0966cfea21760a831d495ee1346a258dd33095cff8f2e23e5d7148f1280f1c8d"}, {"id": "u006", "kind": "table-row", "locator": "body:L13-L13", "preview": "| Concern | SM90 WGMMA | SM100 tcgen05.mma | | MMA completion | Commit/wait groups | `tcgen05.commit` plus mbarrier wait |", "sha256": "129b864fde6ae8e8e58264f223c3ab52804dcb50c282146b2a153673afe572eb"}, {"id": "u007", "kind": "table-row", "locator": "body:L14-L14", "preview": "| Concern | SM90 WGMMA | SM100 tcgen05.mma | | Cross-thread ordering | WGMMA-specific rules | tcgen05 fences composed with an execution-ordering operation |", "sha256": "2ba345a4115e7a6e1f81acbd1416a0f89dc2ee2033027406b83461798bb45007"}, {"id": "u008", "kind": "table-row", "locator": "body:L15-L15", "preview": "| Concern | SM90 WGMMA | SM100 tcgen05.mma | | Narrow floating formats | FP8 WGMMA forms | FP8/FP6/FP4 plus block-scaled MX/NVFP4 kinds |", "sha256": "5947c0ca6e715328562acf6c43de21c60e196b9155df29bf170d6e250ea8b9e2"}, {"id": "u009", "kind": "prose", "locator": "body:L17-L17", "preview": "The exact old WGMMA form matters. Do not assume every Hopper kernel loads A through `ldmatrix`: descriptor-sourced WGMMA forms already read A from SMEM. Likewise, tcgen05 can source A from SMEM or TMEM.", "sha256": "382cb9dc63ddcd3017855fe5603348fd65995c563c146bef2da1122f8ddd6434"}, {"id": "u010", "kind": "list-item", "locator": "body:L21-L21", "preview": "1. Identify the exact WGMMA operand form, accumulator type, shape, and group-completion points in the SM90 kernel.", "sha256": "5b2fcba32ae9b677b57ca8e62cea7f0b536a60f4597ad9ed56f5c6109b4719b6"}, {"id": "u011", "kind": "list-item", "locator": "body:L22-L22", "preview": "2. Choose a legal tcgen05 kind, A source, CTA group, instruction descriptor, and data-path layout for the SM100 target.", "sha256": "7c480de2cd0b7a0f51ab66f750a7f2e43174c52b028129e8b203320459111c42"}, {"id": "u012", "kind": "list-item", "locator": "body:L23-L23", "preview": "3. Replace register-resident D with a TMEM allocation. Group-1 allocation/deallocation is warp-collective; group 2 requires one warp in each CTA of the pair.", "sha256": "94550e33d2e5fc9492d28f341ecdc5ca0655fb2f1c88303fdb7165407f78fcd9"}, {"id": "u013", "kind": "list-item", "locator": "body:L24-L24", "preview": "4. Keep A/B backing storage unchanged until all asynchronous MMA consumers have completed.", "sha256": "e1465bb385f2612cdc483569cb82b1fcee5fb9aaebcb9eeb65667686e7e33848"}, {"id": "u014", "kind": "list-item", "locator": "body:L25-L25", "preview": "5. Replace WGMMA group completion with `tcgen05.commit` and an mbarrier wait. Add tcgen05 fences only where an execution-ordering handoff must order tcgen05-visible state across threads or CTAs.", "sha256": "4362a63d2b1bd22a3096480f6c9a35d00538f93e94821b33c705086971b544ad"}, {"id": "u015", "kind": "list-item", "locator": "body:L26-L26", "preview": "6. Transfer completed accumulator values from TMEM to registers with `tcgen05.ld`, observe its completion/access rules, then run the epilogue.", "sha256": "cc2fdea219e29be1d20d93e0ba014159d0e34b2c8e0de0d4b01083b03c8a177d"}, {"id": "u016", "kind": "list-item", "locator": "body:L27-L27", "preview": "7. Deallocate every dynamic TMEM allocation before kernel exit. Use the same `cta_group` value for all tcgen05 instructions in the kernel.", "sha256": "93a1c4bb169b8c6c02b3b85f458290d7346ff8af1ea4b70ae3f779db14d83181"}, {"id": "u017", "kind": "list-item", "locator": "body:L28-L28", "preview": "8. Retune CTA/cluster shapes, pipeline stages, thread roles, descriptors, and epilogue scheduling on the target workload.", "sha256": "7f11e9b39fa11e8d8ebfaa1313cf7dd006346a8458830b0170a888d8b75eb94d"}, {"id": "u018", "kind": "prose", "locator": "body:L32-L32", "preview": "`tcgen05.alloc` writes a 32-bit TMEM address to shared memory. Allocation size is expressed in columns, in power-of-two multiples permitted by PTX. It is a synchronous warp instruction: a lane-0-only call is invalid. `tcgen05.dealloc` has t", "sha256": "74c19585a0e1b1b0242b4e24476aa7c1c33cc83163385246b670415ca03b4448"}, {"id": "u019", "kind": "prose", "locator": "body:L34-L34", "preview": "The MMA issuer and the allocation participants are different concepts. One thread initiates a group-1 MMA, but one full warp collectively allocates or deallocates its TMEM. A group-2 allocation uses one warp from each peer CTA.", "sha256": "73e16902aedb4d98c7e1df66bf5c584223b108f2e7212840db033cad2b067c88"}, {"id": "u020", "kind": "prose", "locator": "body:L38-L38", "preview": "`tcgen05.mma` is asynchronous. `tcgen05.commit.cta_group::N.mbarrier::arrive::one.b64` makes an mbarrier track prior MMA work issued by the current thread; waiting on that barrier observes completion. A `tcgen05.fence` is an ordering and co", "sha256": "2c0f765385f99c91fd6bba2fce4d7f86c037b58d0a20e7f6ebd86695c01aa76c"}, {"id": "u021", "kind": "prose", "locator": "body:L40-L40", "preview": "After completion and any necessary thread handoff, epilogue warps use `tcgen05.ld` to bring their accessible TMEM lanes into registers. Ordinary bias, activation, conversion, and global-store code then operates on those register values. Pre", "sha256": "9a0355a075bf524abfe540283b025bfdd995ccf1fff1a336741a4c22859c7514"}, {"id": "u022", "kind": "prose", "locator": "body:L44-L44", "preview": "Do not mechanically convert every 64-byte swizzle to 128-byte swizzling. The tcgen05 shared-memory descriptor defines valid no-swizzle, 128B, 64B, and 32B modes, subject to mode-specific alignment and layout constraints. The 128B choice can", "sha256": "a5608028d7dfad8506b7b6106284085462c52175cc29c7fd2293ab77560f02b3"}, {"id": "u023", "kind": "prose", "locator": "body:L46-L46", "preview": "Similarly, there is no universal Hopper-to-Blackwell rule that doubles M. WGMMA and tcgen05 each expose multiple shapes; tcgen05 M/N are encoded in `idesc` and constrained by kind, layout, CTA group, and target ISA. Treat m128xn256xk16 and ", "sha256": "a8234afdf8accc4e7f9754a352400271f4c0da85834278656ef70b903510bf4e"}, {"id": "u024", "kind": "prose", "locator": "body:L50-L50", "preview": "Single-thread MMA issue can free instruction-issue capacity for TMA, descriptor preparation, epilogue, reductions, or scheduling. It does not determine the kernel's total thread count: TMEM allocation, TMEM loads, TMA, epilogue, and barrier", "sha256": "9c73fd720d7cee08f667430e653f338b5de11c015ad161b38c7e22150ac74a05"}, {"id": "u025", "kind": "prose", "locator": "body:L52-L52", "preview": "Moving D out of the GPR file also changes the register budget. Revisit tile size, pipeline depth, and launch bounds, but do not assume that lower accumulator pressure automatically increases occupancy; SMEM, TMEM, barriers, threads, and reg", "sha256": "a9ec2eb6b76b5bbe2308d3a68e5b60337addc6a971601f555cb98fb058f78f48"}, {"id": "u026", "kind": "prose", "locator": "body:L56-L56", "preview": "CUTLASS does not reduce this port to replacing an architecture tag and one schedule token. In pinned SM100 examples, the collective builders are re-instantiated with SM100 operator classes, tile and cluster shapes, stage policy, mainloop sc", "sha256": "7daf9cbdd59a2295c9b4b8a5d268cd9f8688a699e49351df54dbe17a067d6f94"}, {"id": "u027", "kind": "prose", "locator": "body:L58-L58", "preview": "Use a complete pinned example such as CUTLASS `examples/70_blackwell_gemm/70_blackwell_fp16_gemm.cu`, then revalidate alignment, workspace, hardware support, dispatch, and numerical results for the migrated configuration.", "sha256": "f910c88656bc71121c90a77b069b343ac27796b217f14f1c6d8a8cf3e4d143d4"}], "confidence_claimed": "verified", "headings": ["What actually changes", "Dependency-ordered migration checklist", "TMEM lifecycle", "Completion, ordering, and epilogue", "Layout and shape selection", "Warp specialization", "CUTLASS migration"], "id": "migration-wgmma-to-tcgen05", "path": "wiki/migration/wgmma-to-tcgen05.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/prs/cutlass/PR-2139.md", "revision": "ca4fdbea", "url": "https://github.com/NVIDIA/cutlass/pull/2139"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}], "risk_flags": ["ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-ptx-isa-sm100", "pr-cutlass-2139", "blog-tcgen05-tutorial"], "title": "Migrating from wgmma to tcgen05", "type": "migration", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "e08627fe8f79b7e82f8eb53531d3aa22ddb2a98858cae22ae1389d0084a145c3", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "\u201cBelow peak FLOPS\u201d is not itself a bottleneck diagnosis. Select the peak for the executed datatype, instruction kind, sparsity/scaling mode, clocks, and number of participating SMs. Then use arithmetic intensity and achieved compute/memory ", "sha256": "37a1bbcb990374122e9646954c44ff413f62d50018c5702c5d882c75a80633c2"}, {"id": "u002", "kind": "prose", "locator": "body:L5-L5", "preview": "Also distinguish whole-kernel throughput from tensor-core active time. A correct kernel can have efficient MMA intervals yet spend material time in TMA readiness, CUDA-core transforms, reductions, synchronization, epilogue work, or the grid", "sha256": "dc2fa3d8af9c6c95dc56e1fd641fe6c0a042bfdb6becdd5bfeb8af505ef98076"}, {"id": "u003", "kind": "table-row", "locator": "body:L11-L11", "preview": "| Observation and matched control | Supported inference if runtime also improves | | An extra legal SMEM operand stage reduces TMA-full waits | Operand production was exposed on the tested K loop |", "sha256": "3b43f587c282234b6d8375f8dbfeaae54809c78846ee848467588c17da3cab6e"}, {"id": "u004", "kind": "table-row", "locator": "body:L12-L12", "preview": "| Observation and matched control | Supported inference if runtime also improves | | Separating producer/MMA roles reduces control or dependency gaps | The prior role schedule was on the critical issue path |", "sha256": "1cbdf57b4870a67ac98e4e2f3a7c62322d7812ad05c1559d8529e6dbc6d8043d"}, {"id": "u005", "kind": "table-row", "locator": "body:L13-L13", "preview": "| Observation and matched control | Supported inference if runtime also improves | | A second TMEM output region reduces mainloop-to-epilogue waits | Accumulator reuse was serialized by the epilogue |", "sha256": "5d043e745540fbc1868cf31ba9b60aac234ea672c2633528c7ec4ecb1d551cf6"}, {"id": "u006", "kind": "table-row", "locator": "body:L14-L14", "preview": "| Observation and matched control | Supported inference if runtime also improves | | Moving only selected non-MMA operations changes their pipeline activity and runtime | Those operations contributed to the tested critical path |", "sha256": "977930000d035631826feb8c0f2e73f456f1feaeab8e0b5979244180123d5514"}, {"id": "u007", "kind": "table-row", "locator": "body:L15-L15", "preview": "| Observation and matched control | Supported inference if runtime also improves | | A legal 2-SM variant changes traffic/reuse and aggregate throughput | Cooperative mapping helped that shape and resource configuration |", "sha256": "ca57e5367e878d85eab42bbedef761a622b7312edd9c443d14b9205a288b66bc"}, {"id": "u008", "kind": "prose", "locator": "body:L17-L17", "preview": "For every comparison, hold math, datatype, tile coverage, launch environment, warmup, and timing statistic fixed. Record registers, spills, SMEM, TMEM columns, occupancy, cluster size, achieved bandwidth, scheduler issue activity, and per-p", "sha256": "64b22cc4fe9704c2e3785586fa1233ed26749a30d1292d6b101979aac43b4248"}, {"id": "u009", "kind": "prose", "locator": "body:L23-L23", "preview": "Enumerate stage counts that compile and fit the complete shared-storage allocation. More stages can overlap TMA production with MMA consumption, but they also consume SMEM, change occupancy, and enlarge prologue/tail costs. There is no arch", "sha256": "965081cf338b66d916d70a6e2a6a575655f8d7348a132f8468ba94fcbc6abbf7"}, {"id": "u010", "kind": "prose", "locator": "body:L27-L27", "preview": "Dedicated producer, MMA, or epilogue warps can remove role switching from a loop and allow independent work to progress, but add live warps, registers, and synchronization. They do not eliminate stalls. If the epilogue owns the only output ", "sha256": "2a05937c07bd241e43d89abea73682f936e5657100982c7bb3e5e3cbb5f55449"}, {"id": "u011", "kind": "prose", "locator": "body:L31-L31", "preview": "`cta_group::2` requires a valid cluster and exact kind-, shape-, layout-, descriptor-, and peer-resource constraints. It does not universally mean `m256n256`, require identical SMEM layouts, or promise twice the compute per cycle: one coope", "sha256": "d3b7453b3cc18c372b289d5200bfc051ca379b625f6a0527c8cf50d59c88e178"}, {"id": "u012", "kind": "prose", "locator": "body:L35-L35", "preview": "Optimize non-MMA work only after showing that it lies on the critical path. Fusion may remove intermediate traffic; tile interleaving may expose independent work; an approximation may exchange numerical error for throughput. Validate error ", "sha256": "e7197f0143cfd5cbf7e6424b5ff1b4dc3d49327f77499e54304e45f5e75ab5a0"}, {"id": "u013", "kind": "prose", "locator": "body:L39-L39", "preview": "FA4's B200 analysis reports 8192 BF16 MMA operations per clock per SM versus 4096 on Hopper, while exponential throughput is 16 operations per clock per SM on both. Its response is a coordinated design: two-output-tile scheduling, selected ", "sha256": "567f2a49763773c1bb46cd26b83edfc8f556f823ecfa082ac0c1def9873ad16f"}, {"id": "u014", "kind": "prose", "locator": "body:L41-L41", "preview": "The software path uses base-2 range reduction with `n=floor(x)` and a cubic FMA polynomial for a selected fraction of exponential evaluations; other values still use hardware `ex2`. The paper evaluates approximation error and end-to-end acc", "sha256": "d07f08d69c441f10ff4348a210c915a0a3e4cc28559056964b60f5923a124e9d"}, {"id": "u015", "kind": "prose", "locator": "body:L43-L43", "preview": "The paper reports up to 1613 TFLOP/s and 71% for complete FA4 forward kernels; the author blog reports up to 1605 TFLOP/s and 71%. Neither number isolates software exponentiation or any other single technique. Use FA4 as evidence that non-M", "sha256": "9d430642b62bcc1e464c1834a7038a7c7ebeaeffae02b469f9dc7ca5be34e5b9"}, {"id": "u016", "kind": "list-item", "locator": "body:L47-L47", "preview": "- [Nsight Compute 2025.3 roofline and profiling guidance](https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html)", "sha256": "9e40cc52e2795a8ac60ea70f7ce96eaf9fbbc49396653dd7eb617324d7628e09"}, {"id": "u017", "kind": "list-item", "locator": "body:L48-L48", "preview": "- [PTX ISA 9.0 tcgen05 MMA forms](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma)", "sha256": "0a9f68ac9e6cd67818c5d0f58f43cae5f62e1fc07f73ce030a4a27e90096ef33"}, {"id": "u018", "kind": "list-item", "locator": "body:L49-L49", "preview": "- [Pinned two-stage tutorial pipeline](https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu)", "sha256": "5480b05a76d56aaef8da618217be1dcf0583de432a860a245bf8b8aefb8a596e"}, {"id": "u019", "kind": "list-item", "locator": "body:L50-L50", "preview": "- [FlashAttention-4 paper v1](https://arxiv.org/html/2603.05451v1)", "sha256": "bc4648fc7d9797d1e5f1a28a9df55310a395f7bfc7efd78a1304fd817f2b10a8"}, {"id": "u020", "kind": "list-item", "locator": "body:L51-L51", "preview": "- [FlashAttention-4 author blog](https://tridao.me/blog/2026/flash4/)", "sha256": "cb9eb04e449adccb306d0356032bae2e6788f881a47ab8d8e3744cdcc9a4a55f"}], "confidence_claimed": "verified", "headings": ["Classify the Gap First", "Separate the Limiting Edge", "Candidate-Specific Checks", "Pipeline stages", "Warp specialization and output overlap", "2-SM cooperative MMA", "Non-MMA work", "FlashAttention-4 Case Study", "Primary References"], "id": "pattern-compute-bound", "path": "wiki/patterns/compute-bound.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/docs/flash-attention-4.md", "url": "https://arxiv.org/abs/2603.05451v1"}, {"path": "sources/blogs/flash-attention-4.md", "url": "https://tridao.me/blog/2026/flash4/"}], "risk_flags": ["ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "doc-flash-attention-4", "blog-flash-attention-4"], "title": "Not Reaching the Relevant Compute Ceiling", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "2c8aa46a8f96c06bc50d29d566037068eabf9c65e7c5d513b52cae6efc50c993", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "Low SM utilization means that fewer SMs perform useful work than the workload could profitably use during a material part of its measured time. It is not defined by a universal 60% threshold. Theoretical occupancy describes how many blocks ", "sha256": "dc68053a12a55778656da58a55c9814e043d8b253dd92d90f1ae3fbb763ecff3"}, {"id": "u002", "kind": "prose", "locator": "body:L5-L5", "preview": "Collect a synchronized kernel time and a time-resolved view where supported. Record the physical and application-constrained SM count, grid and cluster dimensions, blocks resident per SM, logical work count, work duration distribution, and ", "sha256": "4d2bd929f571b1290b98d979ae170aec632c360030e06c07774fbbe7590a0a78"}, {"id": "u003", "kind": "table-row", "locator": "body:L11-L11", "preview": "| Case | Discriminating evidence | Appropriate next experiment | | Too little independent work | Logical block/cluster count is below available one-wave worker capacity | Change problem decomposition, accounting for reduction and synchroniz", "sha256": "ff994186496977f7e437e154e3f90a1e2fa87d0eda0137219a9ab114e4246e2d"}, {"id": "u004", "kind": "table-row", "locator": "body:L12-L12", "preview": "| Case | Discriminating evidence | Appropriate next experiment | | Wave quantization | Equal-duration data-parallel work has a small nonzero final-wave remainder | Compare tile shapes or a decomposition that changes tile count |", "sha256": "5ce10167b4d3a44da66313f807951f9b46f3ef85adf55d4aeb93a7ce29d7ab2a"}, {"id": "u005", "kind": "table-row", "locator": "body:L13-L13", "preview": "| Case | Discriminating evidence | Appropriate next experiment | | Variable work duration | Per-worker tile counts/times have a long tail despite enough pending work | Compare static and dynamic acquisition with identical tile/decomposition", "sha256": "b1df3c32e77f9429f103d73d192d7db365f866ddc6cf1e845f75925a7ac957cf"}, {"id": "u006", "kind": "table-row", "locator": "body:L14-L14", "preview": "| Case | Discriminating evidence | Appropriate next experiment | | Residency or phase limitation | Grid is large, but resources or serial phases limit active blocks/warps | Change the limiting resource or phase; more grid blocks alone do no", "sha256": "e855f5bd4b6efd616e9c5159e215890d6ad81a321dbd2373c6aa65dde38069a7"}, {"id": "u007", "kind": "prose", "locator": "body:L16-L16", "preview": "Static assignment is nonadaptive, but it is not automatically imbalanced. A grid with fewer independent blocks than SMs cannot occupy every SM, while a grid much larger than the SM count can still show a tail or long-running stragglers. Do ", "sha256": "67a641057a83184ccd37d92d8b68398286f404e0ce56f00ce940fff1df3fe2d3"}, {"id": "u008", "kind": "prose", "locator": "body:L20-L20", "preview": "Persistence, coordinate order, CLC reassignment, and K decomposition are separate controls:", "sha256": "00ed398368679026145a964150a682adb0d3cc7fd0d060c54598a487366eece5"}, {"id": "u009", "kind": "list-item", "locator": "body:L22-L22", "preview": "- A static persistent worker can process multiple logical tiles by grid stride. It can amortize setup and change wave behavior but cannot guarantee removal of the final tail.", "sha256": "970c3cdbaf7dffb98cbbed36f91f9769dc030e8b7387f9dc98b592f0e7b58908"}, {"id": "u010", "kind": "list-item", "locator": "body:L23-L23", "preview": "- A row/column raster or swizzle changes coordinate order and may change operand locality. It does not guarantee a better L2 hit rate or lower work variance.", "sha256": "ee2954aa11e887c12ecac32122fff7c169e2a9aed5c43a8aff669da98adfaa95"}, {"id": "u011", "kind": "list-item", "locator": "body:L24-L24", "preview": "- Cluster Launch Control lets a running worker cancel an unspecified not-yet-started block or cluster from the launched grid and process the returned ID. It redistributes existing work; it cannot create independent tiles, discard required w", "sha256": "6d7b13fddf81ed5b845e50c993a9625c5d24e5f476ab5e896e8683254265a886"}, {"id": "u012", "kind": "list-item", "locator": "body:L25-L25", "preview": "- Stream-K or Split-K can create additional partitions when tile-level parallelism is insufficient, at the cost of partial-result reduction, workspace, synchronization, and possible determinism changes.", "sha256": "125414014589e01cb3a8612ef37f7a9a4ec31d18d82378aeea0953131a3e6b67"}, {"id": "u013", "kind": "prose", "locator": "body:L27-L27", "preview": "For CLC, compare static and dynamic scheduling with identical math, tile/cluster shapes, problem-sized grid, resource limits, and timing method. Record successful and failed requests plus per-worker work counts where instrumentation permits", "sha256": "812e28e793d9fe3f9828c17e49c7779bc964eb811dd42692bc89b4c80b15c4c8"}, {"id": "u014", "kind": "prose", "locator": "body:L29-L29", "preview": "PTX ISA 9.0 says `clusterlaunchcontrol.try_cancel` requires `sm_100` or higher; its cluster-wide multicast qualifier explicitly lists `sm_120a` and the SM120 family. Therefore CLC is not categorically excluded from SM120 by the ISA. The pin", "sha256": "49c0772dcd2917bc11ccc1ad8c2e9fef3bbe8d4376fe49cf7aadd36fc57aebb0"}, {"id": "u015", "kind": "prose", "locator": "body:L33-L33", "preview": "In Gau Nernst's `M=N=K=4096` B200 experiment, v5 reports 1302.29 TFLOP/s (86.43% of its 1506.74-TFLOP/s cuBLAS value), while v6 reports 1475.93 TFLOP/s (97.96%). The v6 endpoint combines static persistence, a changed output pipeline, and ep", "sha256": "f198e4c538684687922c5b39623a6e903579ad5473f3ec9267f3edeb9ab89647"}, {"id": "u016", "kind": "list-item", "locator": "body:L37-L37", "preview": "1. Prove exact logical-work coverage and no duplicates for initial and reassigned coordinates.", "sha256": "eaead768d23afffd442c784dbf60455f43526c0efe7cbd3c0751bf9dd3620fa4"}, {"id": "u017", "kind": "list-item", "locator": "body:L38-L38", "preview": "2. Compare requested work with available worker capacity and calculate the equal-duration final-wave remainder as a diagnostic baseline.", "sha256": "282a05b0f3813e50351b2d9b9121bf28136488746f2bf53251ff7ff18147466d"}, {"id": "u018", "kind": "list-item", "locator": "body:L39-L39", "preview": "3. Instrument work count and time per worker to separate shortage from variable-duration imbalance.", "sha256": "85d7667eb183b65674cf5bd1b4a61e1b431faf63459925677d7428c012beb7db"}, {"id": "u019", "kind": "list-item", "locator": "body:L40-L40", "preview": "4. Hold decomposition constant when testing static versus dynamic acquisition; test decomposition separately.", "sha256": "52d3102612d922f1fc85038af5c4ab2562f4ebabc9055143ddff583eafbdc805"}, {"id": "u020", "kind": "list-item", "locator": "body:L41-L41", "preview": "5. Report end-to-end time along with active-SM distribution, scheduler overhead, cache traffic, occupancy, and regressions.", "sha256": "baade11400a3d956b8272f80cca633553e8e4a8b7937e365aec1771558f87edb"}, {"id": "u021", "kind": "list-item", "locator": "body:L45-L45", "preview": "- [CUDA Programming Guide: Cluster Launch Control](https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html)", "sha256": "434985b84456959e4f53d2ca8c885026c1e84c1718e8d70d3a58f21c7cfff6cd"}, {"id": "u022", "kind": "list-item", "locator": "body:L46-L46", "preview": "- [PTX ISA 9.0 CLC target and request semantics](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel)", "sha256": "1ea71a1b44ae1bff04003838fd3c253ddd1bf748f883aa107a4c088bf748f5c3"}, {"id": "u023", "kind": "list-item", "locator": "body:L47-L47", "preview": "- [CUTLASS 4.5.0 CLC scheduling](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md)", "sha256": "f82b94cccb8b9883e584fa5ec6b7e602e2b7c3247932e262a7a61b54773d925d"}, {"id": "u024", "kind": "list-item", "locator": "body:L48-L48", "preview": "- [Pinned tutorial v6](https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu)", "sha256": "5b526dd64c330d7d736082b7324135601597bb08ed0687532ebef171bf9798d1"}, {"id": "u025", "kind": "list-item", "locator": "body:L49-L49", "preview": "- [Tutorial result progression](https://gau-nernst.github.io/tcgen05/)", "sha256": "797b16f54ff8d25ba34fefe90153d0a2e58a3efa7414f3594476c0b133a26bad"}], "confidence_claimed": "verified", "headings": ["Define the Missing Parallelism", "Distinguish Four Cases", "Scheduler Choices", "Source-Reported Tutorial Result", "Verification Checklist", "Primary References"], "id": "pattern-low-sm-utilization", "path": "wiki/patterns/low-sm-utilization.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/docs/cutlass-clc-documentation.md", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}], "risk_flags": ["ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "doc-cutlass-clc", "doc-ptx-isa-sm100"], "title": "Low SM Utilization", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "df62414848150544dd22c908b1f015e7c3925bad2512fe91186cfc9edffc8e8b", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "A kernel is memory-bandwidth bound when its measured arithmetic intensity places it on the memory side of the relevant roofline and its useful throughput is limited by an attained memory ceiling. Compute intensity from the operations actual", "sha256": "cc4bd91f176d605e31d0cc4f8a09523ded7dd1e512000b09385f3d19c9bd8a60"}, {"id": "u002", "kind": "prose", "locator": "body:L5-L5", "preview": "High DRAM throughput, low tensor-core activity, or a workload label such as GEMV is not sufficient alone. Record useful/requested bytes, transferred sectors or bytes, achieved bandwidth, cache behavior, scheduler issue/stalls, and end-to-en", "sha256": "452ba745b5f2d4681a17bb456941995bd003aa68107d46f228e615b82b2f9549"}, {"id": "u003", "kind": "table-row", "locator": "body:L9-L9", "preview": "| Evidence | More precise interpretation | | Useful bytes and transferred bytes are close; attained bandwidth is near the measured roof | Plausible bandwidth-ceiling limit |", "sha256": "0fd2a73f2b152964fbaa4f7d2413a91390ffa5a24361d357b8dbfced09282d11"}, {"id": "u004", "kind": "table-row", "locator": "body:L10-L10", "preview": "| Evidence | More precise interpretation | | Transferred bytes substantially exceed useful bytes | Coalescing, overfetch, cache, or redundant-traffic problem |", "sha256": "1716a47f062a06bff7fe43f5a747a8e6cff508071d791994fa9bbccc331c7f7f"}, {"id": "u005", "kind": "table-row", "locator": "body:L11-L11", "preview": "| Evidence | More precise interpretation | | Bandwidth is below the roof and warps lack ready work | Latency, dependency, insufficient concurrency, or issue problem |", "sha256": "cb08bba7957e0f2efd478c48c73e9acbcb838f6a54f90eaf06f5ea6960f77daf"}, {"id": "u006", "kind": "table-row", "locator": "body:L12-L12", "preview": "| Evidence | More precise interpretation | | Bandwidth is high only during one phase | Phase balance or fusion opportunity; whole-kernel boundedness remains unproven |", "sha256": "b4665c2eeed674412d9ed1d76c3a428205c05ad775b7294d0817ea60a1267113"}, {"id": "u007", "kind": "prose", "locator": "body:L14-L14", "preview": "Single-use data can lower operations per transferred byte, but \u201cpoor reuse\u201d is actionable only if additional legal reuse exists. Uncoalesced access and cache interference may increase traffic or latency; they are not synonyms for saturation", "sha256": "9d79f4c3ceb564927b8b707cf9c280edb4bf25c2d8d6cce03e8afe8a832561e6"}, {"id": "u008", "kind": "prose", "locator": "body:L20-L20", "preview": "Choose only legal, naturally aligned vector forms and handle tails separately. Compare scalar/narrow and wider variants with identical mapping. Record instruction count, requested and transferred bytes, transactions, registers, spills, achi", "sha256": "9b5f361784b37fe7c571114418e38b50ee1987cfd3b7c737bef1a8b177838955"}, {"id": "u009", "kind": "prose", "locator": "body:L24-L24", "preview": "PTX L1 eviction priorities and L2 prefetch controls are hints. `no_allocate` does not guarantee bypass, and `evict_last` does not guarantee residence. Characterize address order, reuse distance, working set, and interfering streams, then va", "sha256": "6b721336a33758eae0d71d95c602770ea6f3b58e37d6da610cf37f12c78b10b0"}, {"id": "u010", "kind": "prose", "locator": "body:L28-L28", "preview": "Inspect compiled registers, spills, SMEM, threads, and the actual occupancy-limiting resource before applying `-maxrregcount`. A cap may be clamped by ABI requirements, introduce spill traffic, or leave occupancy unchanged. Higher theoretic", "sha256": "a70f10cc7f9f485234c8040496f261aa5083ec2eb42ca7bcb936976be0270208"}, {"id": "u011", "kind": "prose", "locator": "body:L32-L32", "preview": "TMA multicast can issue one global-to-shared tensor copy to selected CTAs' shared-memory destinations within a cluster. It is relevant when those CTAs consume the same operand and the cluster/lifetime/barrier costs are valid. It does not he", "sha256": "fe64e13acbf1e8ce7ae0173430224af05a0baf6dd14a144777b87edd82cd33f5"}, {"id": "u012", "kind": "prose", "locator": "body:L34-L34", "preview": "Swizzling changes a shared-memory address mapping and can reduce conflicts for a specified access pattern. It does not universally eliminate conflicts and is not by itself a DRAM optimization. Validate the legal tensor-map/layout constraint", "sha256": "68bbe5d835608c759d3a6139ee4e7fbfb4b4f48e5b64ed210264ea86871a8b67"}, {"id": "u013", "kind": "prose", "locator": "body:L38-L38", "preview": "The cited Amandeep NVFP4 GEMV report is useful precisely because plausible memory-oriented changes failed. A wider `uint2` load was 16\u201325% slower on the reported shapes, and reducing `maxrregcount` from 80 to 64 had no effect. These observa", "sha256": "6f318f71c4ba4cfd742ff6fee67b3d4f8929e2a8b0a721fb81d828b089f41a4e"}, {"id": "u014", "kind": "prose", "locator": "body:L40-L40", "preview": "Use the same discipline for compute work. If the kernel is genuinely at its attainable memory roof, compute-only instruction reductions may have little effect. But address calculation, decoding, and instruction-level parallelism can affect ", "sha256": "4bf3256418639f29d01b7d95f9821b101480083ef4bfbcb021d8157b9ede2930"}, {"id": "u015", "kind": "prose", "locator": "body:L44-L44", "preview": "Report input shapes/distributions, useful work and bytes, cache state, warmup/repetitions/statistic, GPU and clocks, software versions, generated instructions, resource usage, roofline assumptions, achieved memory-level bandwidth, requested", "sha256": "c3fb3a60d91af6c78b839dce82e7b77f74688407543f7107927ad3d1dbe5fda4"}, {"id": "u016", "kind": "list-item", "locator": "body:L48-L48", "preview": "- [Nsight Compute 2025.3 Profiling Guide](https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html)", "sha256": "7d96e056fb2b06ab7298e5192301e839e81cf4e9a2afaf7ec44b408f56a20811"}, {"id": "u017", "kind": "list-item", "locator": "body:L49-L49", "preview": "- [PTX ISA 9.0 cache-operation semantics](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld)", "sha256": "18178953eb6438ebcfcb96936312c7d79c58c34750dee187e04be2251e1df4ef"}, {"id": "u018", "kind": "list-item", "locator": "body:L50-L50", "preview": "- [PTX ISA 9.0 TMA multicast](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor)", "sha256": "b61a3148a0694e4892a76dc9a81a407d548f10f93062892071b496747572f0a7"}, {"id": "u019", "kind": "list-item", "locator": "body:L51-L51", "preview": "- [Amandeep NVFP4 GEMV attempts](https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/)", "sha256": "cd885328e7372e60c5d3ad3b5f4b19d7d0c89f9aa3a7f3cff0992ffd946275a3"}], "confidence_claimed": "verified", "headings": ["Establish the Memory Roof", "Controlled Candidate Tests", "Load width and coalescing", "Cache policy", "Register budget and concurrency", "TMA multicast and shared-memory layout", "NVFP4 GEMV Negative Controls", "Reproduction Record", "Primary References"], "id": "pattern-memory-bound", "path": "wiki/patterns/memory-bound.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/amandeep-nvfp4-attempts.md", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}, {"path": "sources/blogs/yue-nvfp4-hackathon.md", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}, {"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}], "risk_flags": ["ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-amandeep-nvfp4", "blog-yue-nvfp4", "doc-nvidia-tuning-guide", "doc-ptx-isa-sm100"], "title": "Memory Bandwidth Bound", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "39a1e4e3fb4709726fa3f81e7e26b68f6c0936ca10fa151101aee424b9f283f0", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L8", "preview": "Do not infer one expert per physical SM from a grouped-GEMM trace. CUDA assigns thread blocks to available SMs, while grouped kernels may split an expert into multiple output tiles or let one resident worker process several logical tiles. M", "sha256": "6a4556c295e1fe46a216bb02fa3b9c18a4dfd635ed0ea95b40ab3105782a90aa"}, {"id": "u002", "kind": "list-item", "locator": "body:L10-L10", "preview": "- routed tokens per expert for the exact batch and prefill/decode phase;", "sha256": "7a869c75edb2661a6e1d7146b9df98b243d7611e31d063edc71f3f5c2948929c"}, {"id": "u003", "kind": "list-item", "locator": "body:L11-L11", "preview": "- valid rows and output-tile counts per expert;", "sha256": "21af8e58169651d75b62c2099c735e98ad0f179b6f192d03734b1b9f5e5d3541"}, {"id": "u004", "kind": "list-item", "locator": "body:L12-L12", "preview": "- completed tiles and elapsed work per logical worker and, when instrumented,", "sha256": "abe0c2bb6e4ce66df866b22cdacc7b3459bf91742abf5a10d70a7c157ba3e856"}, {"id": "u005", "kind": "prose", "locator": "body:L13-L13", "preview": "per SM;", "sha256": "a0e9294558ab12243cb5c87a97a4cbaf4966ab84a68cb44017852f8d95467519"}, {"id": "u006", "kind": "list-item", "locator": "body:L14-L14", "preview": "- dispatch, grouped-GEMM, combine, and end-to-end times per expert-parallel", "sha256": "642242588794a89bcb0b8bb3d69a8cf589838a4273d06d3bc63b3de5db8ec3c7"}, {"id": "u007", "kind": "prose", "locator": "body:L15-L15", "preview": "rank.", "sha256": "a9d4d07a50c665beea0d85f220de764cca3663b2c2af5d8c790a1d2970ef5cbc"}, {"id": "u008", "kind": "prose", "locator": "body:L17-L20", "preview": "Small expert segments and partial output tiles can expose too little parallel work or leave lanes predicated out. The effect depends on the kernel tile, other GEMM dimensions, resident-worker count, and competing implementation; there is no", "sha256": "10f047c682158b00cdf5835272cf7cbe2cc8c09f449d19f70966620f49ff4efc"}, {"id": "u009", "kind": "prose", "locator": "body:L22-L26", "preview": "Uniform expected routing, a larger batch, or an auxiliary balancing loss may change token-count skew, but none proves balanced tile counts, worker durations, communication, or runtime. Treat imbalance as absent only when the measured distri", "sha256": "4aea38fd66dea81359addca88f480b75cc6a8cd021b546e0728a216da167bc01"}, {"id": "u010", "kind": "prose", "locator": "body:L30-L32", "preview": "At pinned DeepGEMM commit [`891d57b4`](https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f), the three grouped interfaces have different axes and metadata:", "sha256": "db4321873118f047cf255c2898cd512bca03ae25073f932d1f4f5aa2696aeccd"}, {"id": "u011", "kind": "table-row", "locator": "body:L36-L36", "preview": "| Layout | Pinned contract | Relevant control | | M-grouped contiguous | Pack `A` and `D` along variable M with N/K fixed; identify groups by per-row expert IDs or per-group prefix-sum ends; segments are M-block aligned | Record valid rows,", "sha256": "d2455c5057ca34282247c70175297f8294562ea84975b8dbfd1d42af7117fccd"}, {"id": "u012", "kind": "table-row", "locator": "body:L37-L37", "preview": "| Layout | Pinned contract | Relevant control | | M-grouped masked | Allocate `[G, M_max, ...]`, pass one valid-M count per group, and compute valid portions; the fixed allocation is documented for a CUDA-graph decode case | Separate alloca", "sha256": "4992cd4a0d340ae7c7c8a103d0755063816b02ca84de49d6685fa2f9d5d22c83"}, {"id": "u013", "kind": "table-row", "locator": "body:L38-L38", "preview": "| Layout | Pinned contract | Relevant control | | K-grouped contiguous | Pack variable K with M/N fixed for MoE weight backward | Do not use its K-axis contract to describe forward token imbalance |", "sha256": "844315bc56884144631be8c64520b6b669112454794b93acd1837452b7777cb0"}, {"id": "u014", "kind": "prose", "locator": "body:L40-L42", "preview": "A fixed `M_max` allocation therefore does not establish that all padding rows are computed. Any residual edge-tile or predication cost needs generated-code and profile evidence for the selected kernel.", "sha256": "e1ad215b1834aea24c4c71f10f9dd49adb76027b5707f91aa558f23fed0775eb"}, {"id": "u015", "kind": "prose", "locator": "body:L46-L46", "preview": "Keep persistence, work acquisition, and decomposition as separate variables:", "sha256": "72b5c8bd73eb8523601df6dc917588754a8f96308e1b63a0634cf317643b4636"}, {"id": "u016", "kind": "list-item", "locator": "body:L48-L48", "preview": "1. A static persistent worker can advance through a deterministic logical-work", "sha256": "2ebeac42de4c86f459f32944d820f960af294bb7e3e90652bce81f1750d37a47"}, {"id": "u017", "kind": "prose", "locator": "body:L49-L50", "preview": "sequence. CUDA still decides where its block runs; the rule is not a precomputed tile-to-physical-SM map.", "sha256": "2141d38662b0abc377581ebce726e1b0f19cb22664d04f848c469c8824eb4a2d"}, {"id": "u018", "kind": "list-item", "locator": "body:L51-L51", "preview": "2. In a CLC-backed scheduler, a selected thread requests cancellation of an", "sha256": "ae1a3d61b865fd35104df9841d44658b7fbf250e361ae47cd15e04acd0767ab9"}, {"id": "u019", "kind": "prose", "locator": "body:L52-L54", "preview": "unspecified block or cluster that has not launched. A successful response returns that existing grid coordinate; a request can fail. CLC does not create tiles or make the fastest SM autonomously steal arbitrary work.", "sha256": "44666cd2a7aa439b477e3be4e332e7f1801285aeaa09555f77dc3ef6df41bd7a"}, {"id": "u020", "kind": "list-item", "locator": "body:L55-L55", "preview": "3. The initial `blockIdx` in CUTLASS's pinned CLC scheduler is static and later", "sha256": "ad48157736775ac54b4f798356d96d521ff0b5736bb3d5d8f9f2d83c18ca8b36"}, {"id": "u021", "kind": "prose", "locator": "body:L56-L58", "preview": "requests are asynchronous, pipelined, and cluster-granular. There is no universal per-tile latency bound: compare request activity and end-to-end time against the matched static scheduler.", "sha256": "848689501e7d30e823c98dde175448b2942777d47358b178c1b0b59592536b7d"}, {"id": "u022", "kind": "list-item", "locator": "body:L59-L59", "preview": "4. Splitting a large expert or K range may expose more independent work, but it", "sha256": "043c212f70796c400b7d34d0f4fcb011ecad2c412deb8ce351311e96cf1b18b2"}, {"id": "u023", "kind": "prose", "locator": "body:L60-L61", "preview": "can add partial-result and reduction costs. Hold math, tile shape, cluster shape, launch resources, and output semantics constant in the comparison.", "sha256": "ae096ac1401562950161c28a56d6eb66835ed10ab8b13f6892d16321db4ed587"}, {"id": "u024", "kind": "prose", "locator": "body:L63-L66", "preview": "PTX ISA 9.0 specifies CLC for `sm_100` or higher and explicitly includes SM120-family targets for the multicast form. CUTLASS 4.5.0's documented `PersistentTileSchedulerSm100` integration is a narrower library example, not the complete ISA ", "sha256": "ccd99bd11c24e601ec78e2e879751d77c744e1a7abbc3b1e17ee3194f421e15c"}, {"id": "u025", "kind": "prose", "locator": "body:L70-L76", "preview": "[DeepSeek EPLB at commit `d52c72d`](https://github.com/deepseek-ai/EPLB/tree/d52c72d5b2f2fb4c41afbf8eb21366820239913d) takes per-logical-expert load statistics, replicates heavily loaded logical experts, and packs physical experts across co", "sha256": "02ab54c819389e5434998bcacf1c43243f328f7f943ba034ccfec09615c7c6b7"}, {"id": "u026", "kind": "prose", "locator": "body:L78-L82", "preview": "An LMSYS/SGLang study on a 96-H100 deployment reported `1.49x` prefill and `2.54x` decode throughput speedups in its large-scale EPLB ablation. The same study says it used in-distribution data and that production distribution shifts require", "sha256": "16286c58c9ea8cdc2af988c2d8f5376bb6f0dcd832b599ebda14f672268be661"}, {"id": "u027", "kind": "prose", "locator": "body:L86-L91", "preview": "GPU Mode's official postmortem records a temporarily first-place submission that combined a real grouped-GEMM kernel with a timing-harness exploit. The correctness path ran a padded eight-group computation on each of 15 cloned objects. Duri", "sha256": "aff49e9b75d438a2acf47a45c80a8c00117bf9b8a72bbe8c75671b30fa878cd7"}, {"id": "u028", "kind": "prose", "locator": "body:L93-L97", "preview": "This is evidence about evaluator state reuse, not a valid load-balancing performance result. The postmortem identifies `gpu-mode/reference-kernels` PR 104 as the harness response; neither that record nor the pinned FlashInfer MLSys 2026 eva", "sha256": "db377644f5e0f24dfa0a6332d0ba891a7a6b2b62e7ae1eec35c18e9fac9223ed"}, {"id": "u029", "kind": "list-item", "locator": "body:L101-L101", "preview": "- [CUDA thread-block scheduling](https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html)", "sha256": "d5f8f19a50f2f68a004881c534ff8e7c16f392f92660593dab4c93558383aa06"}, {"id": "u030", "kind": "list-item", "locator": "body:L102-L102", "preview": "- [PTX ISA 9.0 CLC request semantics](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel)", "sha256": "4bfd1b3f62e762332ebd2013370bedf813ecfbfd024253b98a104a3e1e4ba66a"}, {"id": "u031", "kind": "list-item", "locator": "body:L103-L103", "preview": "- [CUTLASS 4.5.0 CLC scheduler](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md)", "sha256": "303ee33b3fbba445fb8238b14d3584472c2ffd1d5afb110eb71ad2fdde234d77"}, {"id": "u032", "kind": "list-item", "locator": "body:L104-L104", "preview": "- [DeepGEMM grouped interfaces at `891d57b4`](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md)", "sha256": "694d095fb8d4a47e1e3948889a83f8e8f2d6eecaaebb602be743c9510989ffe8"}, {"id": "u033", "kind": "list-item", "locator": "body:L105-L105", "preview": "- [DeepSeek EPLB at `d52c72d`](https://github.com/deepseek-ai/EPLB/tree/d52c72d5b2f2fb4c41afbf8eb21366820239913d)", "sha256": "745a1db428d252c3685ad9006ee3bccc82c88c3d83639a0df4fd908a336de615"}, {"id": "u034", "kind": "list-item", "locator": "body:L106-L106", "preview": "- [LMSYS/SGLang 96-H100 deployment study](https://www.lmsys.org/blog/2025-05-05-large-scale-ep/)", "sha256": "167e6224c182e20109786dc6943037c6ab8f4286f21811d71e84790250ebc755"}, {"id": "u035", "kind": "list-item", "locator": "body:L107-L107", "preview": "- [GPU Mode reward-hack postmortem](https://www.gpumode.com/news/reward-hacking-nvfp4)", "sha256": "215b30f533c162dba1a59352f6ba3b00f187094045e1690586ad24aa927ee27d"}, {"id": "u036", "kind": "list-item", "locator": "body:L108-L108", "preview": "- [Pinned FlashInfer MLSys 2026 evaluator](https://github.com/flashinfer-ai/flashinfer-bench-starter-kit/blob/75ccd05cafceb0fd1f86be4cd0f2117249463c66/EVALUATION.md)", "sha256": "75a976c5cc6d841615f24e87fc2de683ab22293c488615a4db5c32a69151a24b"}], "confidence_claimed": null, "headings": ["Diagnose the level that is imbalanced", "Grouped-layout choices", "Scheduler comparisons", "Expert placement is a separate system layer", "GPU Mode Problem 4 reward-hack boundary", "Primary references"], "id": "pattern-moe-load-imbalance", "path": "wiki/patterns/moe-load-imbalance.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md", "url": "https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm"}, {"path": "sources/contests/flashinfer-mlsys26/track-a-fused-moe.md", "url": "https://mlsys26.flashinfer.ai/"}, {"path": "sources/blogs/deepgemm.md", "url": "https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f"}, {"path": "sources/docs/cutlass-clc-documentation.md", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}, {"path": "sources/blogs/gpu-mode-reward-hack.md", "url": "https://www.gpumode.com/news/reward-hacking-nvfp4"}], "risk_flags": ["ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["contest-gpumode-p4", "contest-flashinfer-track-a", "blog-deepgemm", "doc-cutlass-clc", "blog-gpu-mode-reward-hack"], "title": "MoE Expert Load Imbalance", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "0c5294d705b6402ce30d3482e6d7f3bb2cb0ca6bf75e81712347ff1d23959d45", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "A pipeline stall is lost issue opportunity caused by a producer-consumer dependency on the critical path. Low TMA or tensor-core activity, a warp sampled at a barrier wait, or a phase-local utilization drop is only a lead. Expected dependen", "sha256": "da7a72a788c774770d5fdbfdf3a011e6425280d30b33bc03a25a0f39f5c83564"}, {"id": "u002", "kind": "prose", "locator": "body:L5-L5", "preview": "Nsight Compute's Warp State Statistics describes why sampled warps could not issue, while Scheduler Statistics shows whether schedulers had eligible warps and issued instructions. NVIDIA cautions that stalls are not necessarily performance-", "sha256": "7e0728caba67afc2a3981a4c5be636334cc90fa93bf4fc0ee6782c0c878e3126"}, {"id": "u003", "kind": "prose", "locator": "body:L9-L9", "preview": "Do this before tuning around an mbarrier wait:", "sha256": "14ed4da2dccf88e9f5096dc225ad0185ddffa3f32922f4eba71d0c4949f3c428"}, {"id": "u004", "kind": "list-item", "locator": "body:L11-L11", "preview": "1. Identify the exact barrier object, owner, initialized arrival count, initial phase, and storage stage for the wait.", "sha256": "214f90bfaac4e1a67bc424fb4679a4d4914d5f861188655234c0caaa267b4043"}, {"id": "u005", "kind": "list-item", "locator": "body:L12-L12", "preview": "2. For a TMA global-to-shared load, account for the producer's software arrival and expected transaction bytes. `mbarrier.arrive.expect_tx` performs an arrival and adds transaction bytes; the TMA `.mbarrier::complete_tx::bytes` operation de", "sha256": "3e96182dbe690a820cbf9b5e9fdb6b3cc25b08f91eca7cef16719bd9922e117c"}, {"id": "u006", "kind": "list-item", "locator": "body:L13-L13", "preview": "3. Wait for the matching phase of that same object. Phase parity changes when that barrier completes a phase and is reused; \u201cflip after every wait\u201d is not a valid global rule when a thread waits on multiple objects or phases.", "sha256": "fc67e43b74ef2c49970cdfcfcdfbc17e669fe14236db9b9cbb80a129f16f059e"}, {"id": "u007", "kind": "list-item", "locator": "body:L14-L14", "preview": "4. A successful acquire wait supplies the documented visibility for associated prior `cp.async.bulk` work before the consumer reads SMEM.", "sha256": "cad75bba1a6209991e889aebed6a0e54d9a993ac199a3b785434c82630900b3e"}, {"id": "u008", "kind": "list-item", "locator": "body:L15-L15", "preview": "5. Before reusing SMEM operands read by asynchronous MMA, attach completion of the relevant tcgen05 work with `tcgen05.commit` and observe its mbarrier. Issue is not completion.", "sha256": "0471e26fc7edd3ac9914e4d364ca725d6935473ed772fc429bdde5c130e8f362"}, {"id": "u009", "kind": "list-item", "locator": "body:L16-L16", "preview": "6. When asynchronous tcgen05 operations cross a thread handoff, place `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` around the applicable execution-ordering operation. The after fence is not a generic replacemen", "sha256": "a5a5678d754c4afc5251f822e93233a03ffb7e18a06d9f8a9157f3a0686476c6"}, {"id": "u010", "kind": "list-item", "locator": "body:L17-L17", "preview": "7. Verify prologue, steady-state wraparound, short loops, and the producer/consumer tails. A correct steady-state loop can still hang or reuse live storage at a boundary.", "sha256": "44d732e3dec9b913fb7e525f92cb9f90686aa569ff6b9dac92ebae4e0e8b19e2"}, {"id": "u011", "kind": "prose", "locator": "body:L19-L19", "preview": "An extra software arrival is erroneous only relative to the barrier's initialized and phase-specific accounting. Diagnose pending arrivals and transaction bytes separately instead of assuming that TMA hardware performs another arrival.", "sha256": "b0db1b4e9bcf02bdb609d55928eb002804bcdc285274448f73a2424b521cc581"}, {"id": "u012", "kind": "list-item", "locator": "body:L23-L23", "preview": "1. Establish a synchronized end-to-end timing regression on fixed inputs and confirm outputs against a reference. Record GPU, clocks, toolkit, build flags, kernel name, launch shape, and resource use.", "sha256": "c93c68e7cb1a483a6a1d4d21cf5f26631ffbbb1a2a8ab85db6bec467a9cb4798"}, {"id": "u013", "kind": "list-item", "locator": "body:L24-L24", "preview": "2. Collect Speed-of-Light, Scheduler Statistics, Warp State Statistics, and source/SASS correlation with the sections available in the installed Nsight Compute. Account for replay and sampling effects.", "sha256": "a67f70665ee157c5e1909e56a42f789904f46a2629e9a837c8b9573a46cb1eb5"}, {"id": "u014", "kind": "list-item", "locator": "body:L25-L25", "preview": "3. Map the dominant not-issued locations to concrete edges: TMA-full wait, MMA-completion wait, output-buffer reuse, CTA barrier, queue starvation, or pipeline prologue/tail.", "sha256": "9110e18720b271d8f64c2dd21aeb2b3f56a93dac9966380f70d9299ba74a4a67"}, {"id": "u015", "kind": "list-item", "locator": "body:L26-L26", "preview": "4. State a prediction. For example: if TMA readiness is the critical edge, a legal extra operand stage should reduce that wait and runtime; if only the tail is exposed, the steady-state wait distribution should remain largely unchanged.", "sha256": "36934631ace735be26b95ac853c3673c22dce4b1f4969d98234b65b90350f632"}, {"id": "u016", "kind": "list-item", "locator": "body:L27-L27", "preview": "5. Change one variable and repeat identical correctness, warmup, timing, and profiler collection. Reject a cause when its predicted counter and time movement do not occur.", "sha256": "2cd9d472de4c08e8283c6bd52fcbd6704e0be919b520fe671cb542c07a6841e4"}, {"id": "u017", "kind": "table-row", "locator": "body:L31-L31", "preview": "| Controlled change | Plausible target | Required controls and costs | | Vary legal SMEM stage count | Producer cannot stay far enough ahead of MMA | Same tile/math/roles; record SMEM, occupancy, short-loop behavior, and tail |", "sha256": "04c436dbcb263690831932e9dc3de9489d78d1aff38a257a533eb521cdf51549"}, {"id": "u018", "kind": "table-row", "locator": "body:L32-L32", "preview": "| Controlled change | Plausible target | Required controls and costs | | Separate long-lived warp roles | Role handoff or control work is on the issue path | Same stages and tile; record registers, active warps, and per-role idle/backpressu", "sha256": "b5b7ebcf0548afdbddcb877c7fcc70d3352750d277941928b752f71dcd59231e"}, {"id": "u019", "kind": "table-row", "locator": "body:L33-L33", "preview": "| Controlled change | Plausible target | Required controls and costs | | Add a second TMEM output region | Epilogue holds the only accumulator region | Prove disjoint lifetime; record TMEM columns and epilogue/MMA completion waits |", "sha256": "ab79f66f2d1b762e0d24d903673ac799dce4d639794e042dceeaa12daa1f3b07"}, {"id": "u020", "kind": "table-row", "locator": "body:L34-L34", "preview": "| Controlled change | Plausible target | Required controls and costs | | Interleave two query/output tiles | Softmax/correction and MMA have independent ready work | Prove separate state and dependencies; compare one versus two query stages", "sha256": "995291335dcbcf4e24bb3a145bedf98f4fa72eb4f49add2fca9612b80d272803"}, {"id": "u021", "kind": "prose", "locator": "body:L36-L36", "preview": "None is a universal cure. More stages consume storage and can reduce occupancy; specialization adds warps and synchronization; TMEM buffering consumes columns; multi-tile schedules increase live state. A memory-bound roofline classification", "sha256": "214e9852ed70ec2446680cf91d4def45cd5a23b65b1c8c942241a501ce9405b5"}, {"id": "u022", "kind": "prose", "locator": "body:L40-L40", "preview": "Gau Nernst reports the following for one `M=N=K=4096` BF16 GEMM on a Modal B200 using PyTorch 2.9.1 and CUDA 13. Percentages below are computed from the reported 1506.74-TFLOP/s cuBLAS value.", "sha256": "b5438bc9ed3152dfc868c28a134f08b5fad7f494f7c6b85998f51619284bdd40"}, {"id": "u023", "kind": "table-row", "locator": "body:L44-L44", "preview": "| Version | Author's cumulative version label | TFLOP/s | cuBLAS ratio | | v1a | basic tcgen05 + 2D 16-byte TMA | 254.62 | 16.90% |", "sha256": "2ec8ddf8e324506fc9632dec2e2b8bc384ea8deefdb9267c6cdcdc2c2732566d"}, {"id": "u024", "kind": "table-row", "locator": "body:L45-L45", "preview": "| Version | Author's cumulative version label | TFLOP/s | cuBLAS ratio | | v2b | 3D 128-byte TMA | 695.43 | 46.15% |", "sha256": "dbba2419e3054de28eff2b7f1fd83d83d98f18bea3fbea121804656bd24e694c"}, {"id": "u025", "kind": "table-row", "locator": "body:L46-L46", "preview": "| Version | Author's cumulative version label | TFLOP/s | cuBLAS ratio | | v3 | pipelining | 939.61 | 62.36% |", "sha256": "8b5c05c6b3e8641c5d14409e7371999aef50591dc7ded0fb18dd77e6bb1138f1"}, {"id": "u026", "kind": "table-row", "locator": "body:L47-L47", "preview": "| Version | Author's cumulative version label | TFLOP/s | cuBLAS ratio | | v4 | warp specialization | 1208.83 | 80.23% |", "sha256": "12da1a7a1107856224c31730db16b1b6d72ccb37605926b17cd3f1a7d9ec851f"}, {"id": "u027", "kind": "table-row", "locator": "body:L48-L48", "preview": "| Version | Author's cumulative version label | TFLOP/s | cuBLAS ratio | | v5 | 2-SM MMA | 1302.29 | 86.43% |", "sha256": "f849944c51b336479e60eefa90208c5313995e912594c9861c162dfbec203411"}, {"id": "u028", "kind": "table-row", "locator": "body:L49-L49", "preview": "| Version | Author's cumulative version label | TFLOP/s | cuBLAS ratio | | v6 | persistent kernel with static scheduling | 1475.93 | 97.96% |", "sha256": "37b786019302d87416e187fb57c9b75385ecb50a1b91484b1afc0437adc2fac5"}, {"id": "u029", "kind": "prose", "locator": "body:L51-L51", "preview": "The pinned v3 source instantiates two stages, not three. Each row is a cumulative source version rather than an isolated microbenchmark of the named mechanism. The final version uses static scheduling; the author explicitly says Cluster Lau", "sha256": "d2f53b4f0ed98b073c91470bb91db4a9b9c07a2543d56bf6f91af73e4c57c019"}, {"id": "u030", "kind": "list-item", "locator": "body:L55-L55", "preview": "- [Nsight Compute 2025.3 Profiling Guide](https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html)", "sha256": "7d96e056fb2b06ab7298e5192301e839e81cf4e9a2afaf7ec44b408f56a20811"}, {"id": "u031", "kind": "list-item", "locator": "body:L56-L56", "preview": "- [PTX ISA 9.0 mbarrier waits and visibility](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait)", "sha256": "63ce3a13a589f43f07e09bd7a101a72cbdcee3d1b7f3bbd268867bb512aa5ff4"}, {"id": "u032", "kind": "list-item", "locator": "body:L57-L57", "preview": "- [PTX ISA 9.0 tcgen05 execution ordering](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence)", "sha256": "2057f748a6cbb130b0db39057b2306f6bb2d2f06a151010176b32a5a3f998ea1"}, {"id": "u033", "kind": "list-item", "locator": "body:L58-L58", "preview": "- [Pinned tutorial v3 source](https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu)", "sha256": "bb64a4f937640f9b1116a335140e9c1f5ac530178c90a2202836bb97f66fdc52"}, {"id": "u034", "kind": "list-item", "locator": "body:L59-L59", "preview": "- [Tutorial progression](https://gau-nernst.github.io/tcgen05/)", "sha256": "e6ec04716b8773935c14b45e6b61195ce3c4704958a654a0b4599834055cb401"}], "confidence_claimed": "verified", "headings": ["Symptom, Not Diagnosis", "Correctness Gate: Audit the Wait Edge", "Causal Diagnosis Workflow", "Source-Reported Tutorial Progression", "Primary References"], "id": "pattern-pipeline-stalls", "path": "wiki/patterns/pipeline-stalls.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/docs/flash-attention-4.md", "url": "https://arxiv.org/abs/2603.05451v1"}, {"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}], "risk_flags": ["ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-tcgen05-tutorial", "doc-flash-attention-4", "doc-nvidia-tuning-guide", "doc-ptx-isa-sm100"], "title": "Pipeline Stalls", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "6411aba77dd98964d25ec7fa3a3e7f97e70f5400886e930aaa77f6b2072c7621", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "Register pressure is performance-relevant only when compiled register allocation or spills constrain useful scheduling or add material local-memory traffic. \u201cOccupancy below target\u201d is not enough: theoretical occupancy is a residency limit,", "sha256": "35e63dfd7e4f8b07c35d4ba215f12c71e732e0380db9d8319622c21a7bce90d7"}, {"id": "u002", "kind": "prose", "locator": "body:L5-L5", "preview": "For the exact binary and launch, record registers per thread, allocation granularity, spill stores/loads, local-memory traffic, threads and warps per CTA, SMEM, cluster shape, theoretical limiting resource, achieved active/eligible warps, s", "sha256": "926ef4107f387443dda4fc164c195d6657426805e698cae84bd0cca2d227a21a"}, {"id": "u003", "kind": "prose", "locator": "body:L7-L7", "preview": "Common contributors include a resident MMA fragment, an epilogue whose temporaries overlap the mainloop, descriptors/addresses and pipeline state, unrolled loops, and values live across branches. These are hypotheses about compiled liveness", "sha256": "74eb90aed5a121282b7dc1d357f02933e867b1bb16e6e4c53f1f4fa9f324aca1"}, {"id": "u004", "kind": "prose", "locator": "body:L13-L13", "preview": "Move epilogue work after accumulator lifetime when dependencies allow, reduce unnecessary unrolling, or split a long-lived role. Warp specialization exists on Hopper as well as Blackwell; it can shorten one role's live set, but adds role st", "sha256": "956ff6f89523aee899fb12dbab514dbfc87df14051783bc1cbea6f59361c72a7"}, {"id": "u005", "kind": "prose", "locator": "body:L17-L17", "preview": "Compare an uncapped build with selected `-maxrregcount` values. Record whether the requested cap is effective, the occupancy-limiting resource, spill traffic, instructions, and runtime. ABI minima can constrain the cap, another resource can", "sha256": "3a9edd4e72e4bc95e154d7d98e24c1a82b67388a1767ca6ce02ab24aa7927d4e"}, {"id": "u006", "kind": "prose", "locator": "body:L21-L21", "preview": "For Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, that is 128 32-bit registers (512 bytes) per thread for D. This exact fragment arithmetic does not include other live st", "sha256": "092af7b2947a48f0c1be1cccfd9d8da3508467f7456f603f964493b3fb506ccc"}, {"id": "u007", "kind": "prose", "locator": "body:L23-L23", "preview": "On the SM100 tcgen05 path, resident D is in TMEM, organized as 128 lanes by 512 columns of 32-bit cells. That removes the long-lived per-thread WGMMA D fragment, but not all accumulator-related register work: addresses, descriptors, barrier", "sha256": "41d3e8dcb70ed28fc35ae07fb47fbdc6303cf95955dd4577bd87e66b6045887a"}, {"id": "u008", "kind": "prose", "locator": "body:L25-L25", "preview": "The migration must implement collective allocation/address publication, tcgen05 MMA completion, cross-thread ordering where applicable, collective TMEM loads and `tcgen05.wait::ld`, consumer completion, and collective deallocation before ex", "sha256": "87e1d3773fdd20611b000abb10a908e303df0aaa55f40aab0b7dfd02792e6dd8"}, {"id": "u009", "kind": "prose", "locator": "body:L29-L29", "preview": "TMEM loads are asynchronous operations with explicit completion waits. Whether their issue/wait and epilogue work are repaid by lower long-lived register use is an empirical whole-kernel question. Compare the Hopper and Blackwell designs on", "sha256": "357908b2226dfa23e93f2a7fc6ad4bf80d1de89814a113b887d122f19213c26f"}, {"id": "u010", "kind": "prose", "locator": "body:L31-L31", "preview": "Report compiled registers and spills, TMEM columns, SMEM, barriers, occupancy limit, achieved warps, pipeline/scoreboard stalls, correctness, and time. Include shapes where register occupancy changes and shapes where it does not. A reductio", "sha256": "d0e60296c0451a2af20f944162800dc4b92754098cf1276b2bdf855204e8cbbe"}, {"id": "u011", "kind": "list-item", "locator": "body:L35-L35", "preview": "- [Nsight Compute 2025.3 register and occupancy analysis](https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html)", "sha256": "8cc14da20e47f0d5c7a97d9d05e8ec6dc2eb98aa3607cd8ea9764eacea609329"}, {"id": "u012", "kind": "list-item", "locator": "body:L36-L36", "preview": "- [PTX ISA 9.0 WGMMA accumulator fragments](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma)", "sha256": "4341e89ad4d7106f8ad0a8b5962187c0c6d4fdff10a19bdc713dc4c2f55ab1e3"}, {"id": "u013", "kind": "list-item", "locator": "body:L37-L37", "preview": "- [PTX ISA 9.0 Tensor Memory](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory)", "sha256": "aed47f18c514497e50694f89415704f98533229e52ed733c1898d3c9bb3e2747"}, {"id": "u014", "kind": "list-item", "locator": "body:L38-L38", "preview": "- [PTX ISA 9.0 tcgen05 load/wait](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld)", "sha256": "08c6551885ef06a7cbc5e0a5d824bdac60ef2e5086309b1d03ab0db1688c71fd"}, {"id": "u015", "kind": "list-item", "locator": "body:L39-L39", "preview": "- [Pinned SM100 tutorial implementation](https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100)", "sha256": "e650005a038546e8e510ec66cf6726931ba1ef3ae74a509855085fa902f03748"}], "confidence_claimed": "verified", "headings": ["Diagnose a Register-Limited Kernel", "Candidate Tests", "Shorten or separate live ranges", "Test a register cap", "Move resident D from registers to TMEM", "Evaluate the Tradeoff", "Primary References"], "id": "pattern-register-pressure", "path": "wiki/patterns/register-pressure.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}], "risk_flags": ["ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "doc-ptx-isa-sm100"], "title": "Register Pressure and Residency", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "015951e658cf7ca107772efab40ffc056cd6152f9c5c7864a4d0b59dcc1df455", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "For `T` equal-duration independent tiles and `W` available one-tile workers, with no K decomposition, write:", "sha256": "ae0eaa132780542dd74c3d0be7cdf9ebc63b079b2a026a31927442a4d500d190"}, {"id": "u002", "kind": "code", "locator": "body:L5-L7", "preview": "```text T = qW + r, 0 <= r < W ```", "sha256": "9055386055d6219a3731bea281e8962958fea9a3ccddd596354c89e898f6cd16"}, {"id": "u003", "kind": "prose", "locator": "body:L9-L9", "preview": "There are `q` full waves. If `r>0`, a final partial wave uses `r` workers and has instantaneous worker utilization `r/W`; if `r=0` and `T>0`, the final wave is full. For example, `T=150` and `W=142` gives one full wave plus eight tiles: the", "sha256": "a8ef68c0fcff6dd1e2a1c0bd2abaadce92c16b19b92d43925b41a0d9f0ecce59"}, {"id": "u004", "kind": "prose", "locator": "body:L11-L11", "preview": "This is an analytical model, not a statement that a B200 has 142 SMs or that physical SM count always equals `W`. Block resources can make multiple blocks resident per SM, cluster shape changes scheduling granularity, application policies m", "sha256": "d9e0cf3962e4d30c5a85e8a50f0dd53e70c56ead1c68156e831025ddf65c138e"}, {"id": "u005", "kind": "prose", "locator": "body:L13-L13", "preview": "Under the equal-duration model, the time fraction attributable to at most one partial wave shrinks as the number of full waves grows. There is no universal \u201cbelow four times the SM count\u201d cutoff: the remainder, wave duration, other phases, ", "sha256": "f324cd0832cae3c8f0339ca28879361e2a3ea07764d3318bfdc48a357f3fec53"}, {"id": "u006", "kind": "list-item", "locator": "body:L17-L17", "preview": "1. Record logical tiles/clusters, grid and cluster dimensions, available worker capacity, residency limits, and per-work-item duration.", "sha256": "e795d774faa4fd3b84027ef69d3dbb0f4ac2f61de31b0ad0ae7c902891d22e3f"}, {"id": "u007", "kind": "list-item", "locator": "body:L18-L18", "preview": "2. Predict wave count and the final remainder from the simplified model.", "sha256": "1d8d9be6330b539daf5cecf3a3e9ca5bdc43c8d66dc6dafc1626dcc58ab23c80"}, {"id": "u008", "kind": "list-item", "locator": "body:L19-L19", "preview": "3. Use time-resolved activity or instrumented worker records to locate the underfilled interval at the end, rather than inferring it from aggregate utilization.", "sha256": "171e09c6edcc03571aba5731487e7092cbfe98166edaacb3f751eca20419d90d"}, {"id": "u009", "kind": "list-item", "locator": "body:L20-L20", "preview": "4. Sweep nearby problem sizes or tile shapes. A wave-quantization hypothesis predicts a sawtooth response aligned with changes in the remainder, after controlling total work.", "sha256": "23112755735c4dfdfb4280e3bb343028ef65ce6126d88127f7553182529809e6"}, {"id": "u010", "kind": "list-item", "locator": "body:L21-L21", "preview": "5. Separate unequal tile durations: a long straggler is load imbalance even if the tile-count remainder is zero.", "sha256": "12e6a06a0cf28b05b3c3a675a4db26089ec744d706d3001cb0bd2b7b1e7342a7"}, {"id": "u011", "kind": "prose", "locator": "body:L23-L23", "preview": "CUDA schedules ordinary grid blocks onto available SMs; a static grid-stride loop fixes logical worker indices, not `blockIdx`-to-physical-SM placement. A fixed grid therefore does not mean CUDA lacks dynamic block scheduling.", "sha256": "6452a882b8c5416203c08e6041cef08ba04ccf0849deafc93a15afae865c9512"}, {"id": "u012", "kind": "list-item", "locator": "body:L27-L27", "preview": "- A static persistent loop lets a resident CTA process multiple logical work items. Worker count is selected from problem decomposition, cluster/resource limits, and policy\u2014not necessarily one CTA per physical SM. Finite work still has a fi", "sha256": "9db529aa6ae9bd030c3c8089509d63d8584dd95219b31f6d18be3cdf57767e73"}, {"id": "u013", "kind": "list-item", "locator": "body:L28-L28", "preview": "- Cluster Launch Control lets a selected thread request cancellation of an unspecified not-yet-started block or cluster and process the returned grid ID. CLC redistributes existing IDs; it cannot turn eight remaining independent tiles into ", "sha256": "0a5afba4961508157c3c0154f138d89d14c968938adcce7244fc0c81efbacd1c"}, {"id": "u014", "kind": "list-item", "locator": "body:L29-L29", "preview": "- Raster order and swizzle are coordinate transforms. Test them for locality while holding work acquisition fixed; they do not create work or guarantee balanced durations.", "sha256": "7fd94e42d1d2576ed4e87b200e19e9776b4d787a553e5e40fcef1a118dd2e23a"}, {"id": "u015", "kind": "list-item", "locator": "body:L30-L30", "preview": "- Stream-K or Split-K may create more independent partitions when tile-level work is insufficient. Evaluate reduction traffic, workspace, synchronization, determinism, and numerical effects separately.", "sha256": "dd0d2eea0bfed3c0fd23a77f78dc563c2da37c925225e0718d43ab29b94f49d4"}, {"id": "u016", "kind": "prose", "locator": "body:L32-L32", "preview": "Compare static and CLC-backed persistence with identical math, decomposition, tile/cluster shapes, grid, resources, and timing. Record CLC request results and per-worker work/time distribution. PTX ISA 9.0 requires `sm_100` or higher for CL", "sha256": "fcb099d00bab658c03147f5a9ec638a5500d4389439fad74c26eef326c17a5e8"}, {"id": "u017", "kind": "prose", "locator": "body:L36-L36", "preview": "Gau Nernst's Modal B200 has 148 SMs. In the author's `M=N=K=4096` experiment, v5 reports 1302.29 TFLOP/s (86.43% of cuBLAS) and v6 reports 1475.93 TFLOP/s (97.96%). The v6 change is static persistence plus output-pipeline/epilogue-role chan", "sha256": "6f9b202e053f7d5da7cb68d6ce3e62c4533da60caaa7c16fa6b728a202a33c95"}, {"id": "u018", "kind": "list-item", "locator": "body:L40-L40", "preview": "- [CUDA Programming Guide: block and cluster scheduling](https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html)", "sha256": "502cb7460a63cee49afa6d82044dcbfe0b1a257319a5ceb0c2543173f8b71844"}, {"id": "u019", "kind": "list-item", "locator": "body:L41-L41", "preview": "- [PTX ISA 9.0 CLC](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel)", "sha256": "0de2d63b4dc4111728ebc15efcb7962402b936e82599007da061f78ba8f4057b"}, {"id": "u020", "kind": "list-item", "locator": "body:L42-L42", "preview": "- [CUTLASS 4.5.0 CLC scheduling](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md)", "sha256": "f82b94cccb8b9883e584fa5ec6b7e602e2b7c3247932e262a7a61b54773d925d"}, {"id": "u021", "kind": "list-item", "locator": "body:L43-L43", "preview": "- [Pinned tutorial v6 source](https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu)", "sha256": "d47967878d57c59a54264abe233d783fe466469b45b79870f89f24ba6c73531a"}, {"id": "u022", "kind": "list-item", "locator": "body:L44-L44", "preview": "- [Tutorial result progression](https://gau-nernst.github.io/tcgen05/)", "sha256": "797b16f54ff8d25ba34fefe90153d0a2e58a3efa7414f3594476c0b133a26bad"}], "confidence_claimed": "verified", "headings": ["Exact Simplified Model", "Confirm a Last-Wave Cause", "Scheduler and Decomposition Choices", "Tutorial Scope", "Primary References"], "id": "pattern-tail-effect", "path": "wiki/patterns/tail-effect.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/docs/cutlass-clc-documentation.md", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}], "risk_flags": ["code", "ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "doc-cutlass-clc", "doc-ptx-isa-sm100"], "title": "Tail Effect \u2014 Last-Wave Underutilization", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "3af5831a5b82c05926f7c6ecb8301f7e0c4d3d03774f809a36f86ef5e95df07b", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "PTX global loads and stores can carry cache operators and eviction-priority qualifiers. Global loads can also carry L2 prefetch-size hints. These controls let a kernel express a preference for a particular access; they do not guarantee cach", "sha256": "58fc3708b4bf0b06839ff597548bd0d45b4f269cf14a89d70ed6547be08989a6"}, {"id": "u002", "kind": "table-row", "locator": "body:L9-L9", "preview": "| Form | PTX meaning | Not guaranteed | | `L1::no_allocate` | Selects an L1 eviction priority that may be applied | That the access bypasses L1 |", "sha256": "8225a6a519ce7e32ce882a77aa1056b9169be6e42a5f22560b898481579b8eef"}, {"id": "u003", "kind": "table-row", "locator": "body:L10-L10", "preview": "| Form | PTX meaning | Not guaranteed | | `L1::evict_first` | Requests first-eviction priority | Immediate eviction |", "sha256": "bc512e53bc172d71d0cb8d1ccfa67a7842621adcf4c72daf4c4016c906262a26"}, {"id": "u004", "kind": "table-row", "locator": "body:L11-L11", "preview": "| Form | PTX meaning | Not guaranteed | | `L1::evict_last` | Requests last-eviction priority | Persistent residence |", "sha256": "54e7c9299b1db1db2546f3ad6f51fd38f3f1ea1c67355720fe46d456320adaa4"}, {"id": "u005", "kind": "table-row", "locator": "body:L12-L12", "preview": "| Form | PTX meaning | Not guaranteed | | `L2::64B`, `128B`, `256B` | Hints that additional data of that size be fetched into L2 | A wider memory instruction or a completed prefetch |", "sha256": "23a0133f060ed5d78700c80e13b705675ace800df529b843bb4417b4894b9035"}, {"id": "u006", "kind": "table-row", "locator": "body:L13-L13", "preview": "| Form | PTX meaning | Not guaranteed | | `L2::cache_hint` with a policy operand | Supplies a created L2 eviction policy | That the hint is respected or changes memory consistency |", "sha256": "92165e23d514905a7089b6f790e98bc0b81b229b23cd9f6eb1a558c76feb682e"}, {"id": "u007", "kind": "prose", "locator": "body:L15-L15", "preview": "The following are legal PTX 9.0 instruction fragments when their operands and addresses are declared with matching types. Each vector access shown is 16 bytes and therefore requires 16-byte natural alignment:", "sha256": "e6875d2adb5ad541e5579d45da0f70247686265f3f187debecb3662cfe1c686d"}, {"id": "u008", "kind": "code", "locator": "body:L17-L24", "preview": "```ptx .reg .b64 addr_a, addr_b, addr_c; .reg .u32 a<4>, b<4>, c<4>; ld.global.L1::no_allocate.v4.u32 {a0, a1, a2, a3}, [addr_a]; ld.global.L1::evict_last.v4.u32 {b0, b1, b2, b3}, [addr_b]; st.global.L1::evict_first.v4.u32 [addr_c], {c0, c1", "sha256": "dff1576a0ce3b45407dcdc9dddaf973b2f0df90f5c9dc8eedbab93b06b805315"}, {"id": "u009", "kind": "prose", "locator": "body:L26-L26", "preview": "The common \u201cstream A, retain B\u201d explanation is a hypothesis about reuse and interference, not the semantics of the code. Even with correct alignment, the hardware may apply the priorities differently than a literal bypass/keep model suggest", "sha256": "0f8b19c545e0751d4f2f9962b9b26f4586f822f95847e2b4ab0fbb405ea44515"}, {"id": "u010", "kind": "prose", "locator": "body:L30-L30", "preview": "Start from the concrete access trace rather than a kernel label or tensor name:", "sha256": "eeb262f9767e73e0d30f6587d5465e23ddb312d3367caff656125c14d8a1e9b1"}, {"id": "u011", "kind": "list-item", "locator": "body:L32-L32", "preview": "1. Identify which addresses each warp touches, the reuse distance for each line, the live working set, and competing traffic at the same launch shape.", "sha256": "db5d6449fa0ee136e4483eec513b11e760a840112559819e672a1d3c568586fb"}, {"id": "u012", "kind": "list-item", "locator": "body:L33-L33", "preview": "2. Establish a correct default-policy baseline. Hold mapping, vector width, decode, unrolling, register controls, compiler, inputs, and launch parameters constant.", "sha256": "0aaf1ed9f480045eafdcc9a795490fea975fb9496b7da080a6f7dc21a2fcdceb"}, {"id": "u013", "kind": "list-item", "locator": "body:L34-L34", "preview": "3. Change one qualifier at a time. Re-run the same correctness oracle, including sizes that exercise alignment and tail paths.", "sha256": "6f4744fe0b055ceede7d49f44cbe6268747460836c80a678ad1457b7c21208ed"}, {"id": "u014", "kind": "list-item", "locator": "body:L35-L35", "preview": "4. Record generated instructions and resources, then profile cache hit behavior, requested/transferred bytes, stalls, achieved bandwidth, and active warps with metrics available on the installed tool and target.", "sha256": "4bfc53a191440ff619777b08cc32f2adb819edb032cee818fd8539d4679668de"}, {"id": "u015", "kind": "list-item", "locator": "body:L36-L36", "preview": "5. Benchmark every production shape with the same warmup, synchronization, repetitions, and statistic. Retain only scoped improvements; a policy may help one shape and regress another.", "sha256": "ae112842d641a2c358bbe90a6d930cfd3f10160668b7dadbea95eec4259b234f"}, {"id": "u016", "kind": "prose", "locator": "body:L38-L38", "preview": "An input being larger than L2, a kernel being described as memory-bound, or a tensor being called streamed/reused does not by itself predict a useful qualifier. Concurrent blocks can reuse data, a nominally reused vector may exceed the effe", "sha256": "01cdf9368ffd2d391dc2cb527f3463989408a958627a15dbcef4bd36147c631d"}, {"id": "u017", "kind": "prose", "locator": "body:L42-L42", "preview": "Amandeep Singh reports that three inspected B200 NVFP4 GEMV solutions used `L1::no_allocate` for A and `L1::evict_last` for B alongside raw PTX decode, wide loads, exact-K specialization, tighter register caps, and\u2014in one solution\u2014sharing B", "sha256": "ad93cd1575dc885525b55b8e82655208722219d63a561b9342350d8273f3a9ee"}, {"id": "u018", "kind": "prose", "locator": "body:L44-L44", "preview": "Yue Zhang reports approximately 39 microseconds for a stage combining removal of B shared-memory staging, per-thread tiles, `float4` loads, and hardware conversion, followed by approximately 27 microseconds for a vectorized PTX FP4/scale-de", "sha256": "c1f0bc6fb12c42dff1c938a4a57f0cc0fb0f492ef548917c637f70b038bd7739"}], "confidence_claimed": "verified", "headings": ["Contract", "Selecting a policy", "Evidence boundary for the NVFP4 reports"], "id": "technique-cache-policy", "path": "wiki/techniques/cache-policy.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/blogs/yue-nvfp4-hackathon.md", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}, {"path": "sources/blogs/amandeep-nvfp4-attempts.md", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}, {"path": "sources/contests/gpu-mode-nvfp4/problem-1-gemv.md", "url": "https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-ptx-isa-sm100", "blog-yue-nvfp4", "blog-amandeep-nvfp4", "contest-gpumode-p1"], "title": "PTX Cache Eviction and Prefetch Hints", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "d1c2e2981a7cbb20812788c814381f9f5a67edcec521a6497f68034d83483b8f", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "The two linked PRs have different evidentiary value:", "sha256": "3532db0f410a9281e56ca0b23d9c4c49c19a665f1b6d5e22c617cb4063670fb8"}, {"id": "u002", "kind": "table-row", "locator": "body:L7-L7", "preview": "| PR | Semantic scope | Transfer value | | CCCL 3559, captured merge `25523da2` | Adds B200/SM100 exclusive-sum scan tuning, expands policy classification to input/output/accumulator/offset types, and updates scan dispatch/tests. | A concre", "sha256": "1118714ee9a762906799170546cdc7f8d01af8836ba6cf2e9de5ff78d6ce1bf5"}, {"id": "u003", "kind": "table-row", "locator": "body:L8-L8", "preview": "| PR | Semantic scope | Transfer value | | CCCL 6152, captured merge `3fb05826` | Changes only `CUB_DEBUG_LOG` formatting and stale variable names in `DispatchTopK`. | Evidence for the corrected debug output, not TopK algorithm, performance", "sha256": "9b60c5435b1bbabba385a2d27b84a9705c10a71f82fd3cb5164e2d3c539155f0"}, {"id": "u004", "kind": "prose", "locator": "body:L10-L10", "preview": "Do not use PR 6152 as evidence for DSA TopK design or performance. Its captured key file exposes surrounding TopK implementation for inspection, but the PR's semantic delta is the small debug-only patch.", "sha256": "c6caf045a6b9260f06d822235725a7371ef237eb9773c2804f1e8bf3dfb8dc89"}, {"id": "u005", "kind": "prose", "locator": "body:L14-L14", "preview": "The new `sm100_tuning` specializations select a tuple rather than a single \u201cvectorized\u201d bit:", "sha256": "ef1ba6a5ffaf5c3db38f4137420cffbd6f9b3147a650d9c0bce7ab83dc894da3"}, {"id": "u006", "kind": "table-row", "locator": "body:L18-L18", "preview": "| Policy dimension | Examples in the captured source | | Classification | input value size, accumulator type/size, offset size, and recognized `plus` operator |", "sha256": "a9e0c1a68c8e41698173559e124463741f1f39a94e516597aef7ad189b74d4d5"}, {"id": "u007", "kind": "table-row", "locator": "body:L19-L19", "preview": "| Policy dimension | Examples in the captured source | | Work partition | `threads` and `items` per thread |", "sha256": "0815c33204b77ed25cca9827b2ab51e07004ebad4b5f9f78e9093d09f3a45c15"}, {"id": "u008", "kind": "table-row", "locator": "body:L20-L20", "preview": "| Policy dimension | Examples in the captured source | | Memory policy | `BLOCK_LOAD_*`, `BLOCK_STORE_*`, and `LOAD_DEFAULT` or `LOAD_CA` |", "sha256": "affc3f2082c6b33e588efad21bb73e3ae75573b1ba705d2cadbef18c285fc8cb"}, {"id": "u009", "kind": "table-row", "locator": "body:L21-L21", "preview": "| Policy dimension | Examples in the captured source | | Look-back behavior | an exponential backoff/delay constructor and its parameters |", "sha256": "d476c9630872da451c8547fe530ade5fa22ca4e2f4e0773635a7b5c3c3731dcb"}, {"id": "u010", "kind": "table-row", "locator": "body:L22-L22", "preview": "| Policy dimension | Examples in the captured source | | Fallback | `Policy1000` selects a matching SM100 specialization; otherwise it falls back to `Policy900`; the double specialization explicitly inherits an SM90 tuning |", "sha256": "612ef7a8a3d8257ec2557bc3eddb18089b41ad5c6f0249bcb1e0eed5db932d78"}, {"id": "u011", "kind": "prose", "locator": "body:L24-L24", "preview": "`items >= 4` does not establish vectorized access. Vector width also depends on iterator contiguity, element type, alignment, load/store algorithm, compiler lowering, and the executed policy. Treat items-per-thread, block size, load/store a", "sha256": "05b467376da229569223be7afe7af2a5044c10fabb1983ccbb1aed8b5a096f2c"}, {"id": "u012", "kind": "list-item", "locator": "body:L28-L28", "preview": "1. Identify the exact primitive and semantic contract: inclusive/exclusive scan, operator, input/output/accumulator types, offset width, aliasing, and empty/large-size behavior.", "sha256": "22bee881668fd269c29db6987e756c55090a7bdd5b4037f8d671563d07193f25"}, {"id": "u013", "kind": "list-item", "locator": "body:L29-L29", "preview": "2. Follow runtime architecture dispatch to the active policy. Confirm whether SM100 has a matching specialization or inherits the SM90/default route.", "sha256": "133fa8b4c60fbf759bbe9b2666428bd7250701273d08f0525df21dea718ff00b"}, {"id": "u014", "kind": "list-item", "locator": "body:L30-L30", "preview": "3. Reproduce the upstream baseline and changed policy over the size/type/operator matrix relevant to the application. A policy comment containing benchmark ratios is source context, not a portable speedup guarantee.", "sha256": "e1e61e646347efc9bc0b0608ba22f5dff8e61a40790c4fcb6b0f56bff39ca816"}, {"id": "u015", "kind": "list-item", "locator": "body:L31-L31", "preview": "4. Change one policy dimension at a time where practical and record time, bandwidth, occupancy, register/spill data, and correctness.", "sha256": "7566f62910d39f9e0363c194e49c8b25cc0cc4b9df73f45a5ef292116bfe7d77"}, {"id": "u016", "kind": "list-item", "locator": "body:L32-L32", "preview": "5. For selection, separately verify membership, output count, ordering, ties, NaNs, signed zero, key/value association, and repeatability. Those properties are not established by PR 6152.", "sha256": "c695167e5abc2c932b57c83095361a7e4713d338d1da7cf9e6ee588d4054dfa0"}, {"id": "u017", "kind": "prose", "locator": "body:L34-L34", "preview": "The linked evidence does not support claims about fill, histogram, reduce, block-load/store vectorization, or application-level DSA score computation. Add a directly relevant CCCL source before transferring policy conclusions to those primi", "sha256": "64c0b1fa7d7a20f4942ddeaae956cb7837d4285ade8e50e71db3e773e162e840"}], "confidence_claimed": "source-reported", "headings": ["Exact Source Scope", "What PR 3559 Actually Tunes", "Transfer Workflow"], "id": "technique-cccl-memory-primitives", "path": "wiki/techniques/cccl-memory-primitives.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/prs/cccl/PR-3559.md", "revision": "25523da2", "url": "https://github.com/NVIDIA/cccl/pull/3559"}, {"path": "sources/prs/cccl/PR-6152.md", "revision": "3fb05826", "url": "https://github.com/NVIDIA/cccl/pull/6152"}], "risk_flags": ["ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["pr-cccl-3559", "pr-cccl-6152"], "title": "CCCL CUB Memory Primitives For Selection And Scan", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "dc1f127417087638a33a432f9defd1b6e66efce189bd004f9e699f1bbbee4372", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Some linear recurrent models admit an algebraically equivalent chunkwise formulation. Work local to a chunk can then be expressed with parallel matrix operations, while the state passed across chunk boundaries preserves the recurrence's seq", "sha256": "48a92e68a2fe0a23f551664777d23c15d691c5921abf8b854b06caa7ef5106b9"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "A correct implementation separates at least these obligations:", "sha256": "8c48104a851aa54aa0a45bc389fd93b6f9409ea226bc2d7c0ed56b1f8d6b352d"}, {"id": "u003", "kind": "list-item", "locator": "body:L9-L9", "preview": "1. Compute the chunk-local quantities required by the model's exact recurrence.", "sha256": "7fd75a382a40ba603fb0f8a9810aebf341cd1a888ddc21d67b892970f1d1c1ef"}, {"id": "u004", "kind": "list-item", "locator": "body:L10-L10", "preview": "2. Resolve boundary states in sequence order, either with an associative scan supported by the formulation or with explicitly ordered stages or launches.", "sha256": "32e5983ffa33337e71cec6880f42fd8b53325b7039069f4940df823e46c9485b"}, {"id": "u005", "kind": "list-item", "locator": "body:L11-L11", "preview": "3. Combine each chunk's local result with its incoming boundary state and emit outputs in the original token order.", "sha256": "4185354ea9059f98c36f3e955471e7a5fe4d33df82d607b931c250a87e364f94"}, {"id": "u006", "kind": "list-item", "locator": "body:L12-L12", "preview": "4. Validate outputs and final states against a token-by-token reference across variable sequence lengths, chunk tails, batches, heads, dtypes, and gate extremes.", "sha256": "a3cba5f1e5eb6aa7623dc2822355cf7b802b9d57465d667703f3c10ba06f35e1"}, {"id": "u007", "kind": "prose", "locator": "body:L14-L14", "preview": "An ordinary GPU grid does not imply increasing program-ID execution order or grid-wide synchronization. Programs for every chunk therefore cannot safely read and overwrite one shared state pointer in a single unordered launch. A staged algo", "sha256": "9fc013326b6878c87eff2b13eecd2a1d2a3cf9e14d2f23ccd000b58042d8c0c4"}, {"id": "u008", "kind": "prose", "locator": "body:L18-L18", "preview": "The pinned NVlabs GatedDeltaNet repository uses chunkwise Triton kernels for training and a WY representation of the gated delta rule. That implementation is direct evidence for GatedDeltaNet chunking, but it is not equivalent to a generic ", "sha256": "e7c843e29ca3efe700861205d29f457d86bd1ba013279d32be1a0f2c46aa779b"}, {"id": "u009", "kind": "prose", "locator": "body:L20-L20", "preview": "Tiled Flash Linear Attention (TFLA) starts from the chunkwise formulation of linear RNNs and adds another level of sequence parallelization within a chunk. The authors state that this permits arbitrarily large chunks, raises arithmetic inte", "sha256": "b202431cbc60d3c50fc8c3ae71e5f1a923acdcee1c70f8aa9b30460a4d8181f2"}, {"id": "u010", "kind": "prose", "locator": "body:L22-L22", "preview": "Qwen3-Next is a hybrid-model example, not evidence that one kernel handles every layer. Its immutable 48-layer configuration repeats three Gated DeltaNet linear-attention layers and one full-attention layer twelve times: 36 GDN and 12 full-", "sha256": "8f59435f91c48cc1ead81d8b10be97200d4b6fd046f75d1a4827f1eab9ddc95d"}, {"id": "u011", "kind": "prose", "locator": "body:L26-L26", "preview": "There is no source-backed universal rule that `C=32` is a decode choice or that `C=256-512` is the prefill optimum. Choose only among chunk sizes supported by the exact algorithm and backend, then measure the tradeoff:", "sha256": "8818b3f5fbd2db4ba16a31baad13c667b4fe419941c1b236e1abc9ea5581ef7b"}, {"id": "u012", "kind": "list-item", "locator": "body:L28-L28", "preview": "- arithmetic intensity and matrix-instruction utilization;", "sha256": "ae9e1bf71be10c77bfb65cba9a45c82ae2c478b773284bb7c2e103a158e499a5"}, {"id": "u013", "kind": "list-item", "locator": "body:L29-L29", "preview": "- intermediate-state and workspace traffic;", "sha256": "73a40945cc41e860d08f14bf1d10885e9ab3d814182a7d447c0572e59311dcd7"}, {"id": "u014", "kind": "list-item", "locator": "body:L30-L30", "preview": "- registers, shared memory, and occupancy;", "sha256": "e6793ebfc69570934a6b8cd081905bbe72a12cc6c39148da17b85306eb20484f"}, {"id": "u015", "kind": "list-item", "locator": "body:L31-L31", "preview": "- tail handling and variable-length metadata;", "sha256": "53ab739e34e9312aa6fb64dfb93f2d192f7cc8c5b50888daa8ead3a08b47d27e"}, {"id": "u016", "kind": "list-item", "locator": "body:L32-L32", "preview": "- launch count and boundary-scan cost;", "sha256": "8d8b1cbac255a5b44bba9b24f6458d8179305382641fa292d6679114f7ab477c"}, {"id": "u017", "kind": "list-item", "locator": "body:L33-L33", "preview": "- latency and throughput for the intended batch and sequence distribution.", "sha256": "329f61d749b5347dfc9b2232f4944da7d760a2123ec4cd32523ce05cf5de766e"}, {"id": "u018", "kind": "prose", "locator": "body:L35-L35", "preview": "Compare candidates with identical compiler/software revisions, launch inputs, correctness oracles, warmups, synchronization, and repeated-trial statistics. Treat a selected size as scoped to the model dimensions, dtype, GPU, backend, and wo", "sha256": "d0698561089a9d858f6770ce429c967f63c781117e5ec3e658af91b908ef14d4"}], "confidence_claimed": "verified", "headings": ["Mechanism", "Verified implementations and scope", "Choosing a chunk configuration"], "id": "technique-chunk-parallelism", "path": "wiki/techniques/chunk-parallelism.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/gated-delta-net.md", "url": "https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921"}, {"path": "sources/docs/tfla.md", "url": "https://arxiv.org/abs/2503.14376v3"}, {"path": "sources/blogs/qwen3-next-architecture.md", "url": "https://developer.nvidia.com/blog/new-open-source-qwen3-next-models-preview-hybrid-moe-architecture-delivering-improved-accuracy-and-accelerated-parallel-processing-across-nvidia-platform/"}], "risk_flags": ["ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-gated-delta-net", "doc-tfla", "blog-qwen3-next-architecture"], "title": "Chunkwise Parallelism for Linear Recurrences", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "e32c3558c9bc6483db8f063a150cca5c88247952aa4d168c007310ad16e98baa", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Double- or multi-buffering reserves disjoint storage regions and transfers ownership between producers and consumers. While a consumer reads stage `s`, a producer may fill another stage. The storage alone does not establish overlap: the pro", "sha256": "353970ee76fddcefdd8634bdce84efaf522019d1c67aadbc7734485b35739dec"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "On SM100 these ideas can be applied independently:", "sha256": "7a9f970eba37f57c06d3e4c1053f758644480ec9c1bfd6e13000b66fa7e2fd52"}, {"id": "u003", "kind": "list-item", "locator": "body:L9-L9", "preview": "- SMEM stages can hold operand tiles while TMA production overlaps MMA consumption.", "sha256": "bceca9323f3eeda738243bf77003183e73292e753517a3d9c5e0a58bf486072e"}, {"id": "u004", "kind": "list-item", "locator": "body:L10-L10", "preview": "- Separate TMEM regions can hold output accumulators while an epilogue drains a completed region and MMA writes a different region.", "sha256": "c08e7056d2703d9010ebb72f542b30e222b600fba13f3990bf373404cd5ec57f"}, {"id": "u005", "kind": "prose", "locator": "body:L12-L12", "preview": "The pinned `matmul_v6.cu` from Gau Nernst's tutorial uses both mechanisms. That is a concrete design, not a rule that every optimized Blackwell GEMM needs both.", "sha256": "7143ee6782943e2cf11478db5d26219b0b9ed56dbb20c7df822457fa19405753"}, {"id": "u006", "kind": "prose", "locator": "body:L16-L16", "preview": "The CTA-visible TMEM address structure has 128 lanes and 512 columns of 32-bit cells. Allocation is column-granular across all 128 lanes. A kernel that allocates 512 columns may choose two 256-column regions:", "sha256": "28df4d63890310783d60527b96ed4c07c057cfe045b622775dbf7d3029cffe26"}, {"id": "u007", "kind": "code", "locator": "body:L18-L21", "preview": "```text region[0] = columns [0, 256) region[1] = columns [256, 512) ```", "sha256": "ed69a2b1fa0220c87f2e0b386d916c2e13666c8f97966b1b23e1551f5656de80"}, {"id": "u008", "kind": "prose", "locator": "body:L23-L23", "preview": "This equal split is one implementation choice. The actual invariant is that every simultaneously live region is inside the allocation and does not alias another region whose MMA or epilogue access is still outstanding. Region sizes can diff", "sha256": "47dc0422eca78419d023b265f59cbad85c48bb108bf29c8b6982f069ad1e6564"}, {"id": "u009", "kind": "prose", "locator": "body:L25-L25", "preview": "For each region, prove this lifecycle:", "sha256": "77958ca2b2300609b14a11248488adc9a6078379d31d5df83720e798035ab4a9"}, {"id": "u010", "kind": "table-row", "locator": "body:L29-L29", "preview": "| Transition | Required edge | | free \u2192 MMA-owned | every prior epilogue load from the region has completed |", "sha256": "b36ebc214cce51253c6d828b95dc22426a8ab01b4128d0a6e49484d51e3735db"}, {"id": "u011", "kind": "table-row", "locator": "body:L30-L30", "preview": "| Transition | Required edge | | MMA-owned \u2192 ready | relevant tcgen05 work is committed to an mbarrier and completion is observed |", "sha256": "8af311d47b977e2484a14ff52be09086cc85443f38fed3ec42c5cc1d87cf531d"}, {"id": "u012", "kind": "table-row", "locator": "body:L31-L31", "preview": "| Transition | Required edge | | ready \u2192 epilogue-owned | readers observe the matching barrier phase before `tcgen05.ld` |", "sha256": "086da55067d42eea98da03787c03f386f30b566554fe854779777bfb6a1909b7"}, {"id": "u013", "kind": "table-row", "locator": "body:L32-L32", "preview": "| Transition | Required edge | | epilogue-owned \u2192 free | all collective loads complete with `tcgen05.wait::ld`, then every reader reports completion |", "sha256": "04c3311dcc82b7cc6680a8b4e7e23e43fc32e4f21b96dc5d35631d0e0ad3e828"}, {"id": "u014", "kind": "prose", "locator": "body:L34-L34", "preview": "TMEM must first be allocated by the required participating warp and its address safely published. Matching collective deallocation is required before every kernel exit. Reused mbarriers need the correct arrival count and phase/parity state;", "sha256": "0190655232d79908e1b5184a1aba9c8b97f9ce1895c72ef9b24d30b25b54600e"}, {"id": "u015", "kind": "prose", "locator": "body:L38-L38", "preview": "An SMEM operand stage normally has its own full/empty state:", "sha256": "70c65950ab321f20237632a725827aae81634feef5c60cb54887ca343647416d"}, {"id": "u016", "kind": "list-item", "locator": "body:L40-L40", "preview": "1. The producer waits until stage `s` is empty.", "sha256": "a79eddecdf545db8d1cf874c1f8d4281c05ee71fbe6cbec7e01fe3c0c5162046"}, {"id": "u017", "kind": "list-item", "locator": "body:L41-L41", "preview": "2. It issues the stage's TMA copies and accounts for expected transaction bytes.", "sha256": "4bff5a30df81531732eda4d3b5c80ed4c98124eb1f1f620d593322f52494ae75"}, {"id": "u018", "kind": "list-item", "locator": "body:L42-L42", "preview": "3. The consumer observes the full barrier's matching phase before using the stage.", "sha256": "e61f4af11998b70bd86556f287f45075f6b2ec06632bb6af1d0977cf7c9f3624"}, {"id": "u019", "kind": "list-item", "locator": "body:L43-L43", "preview": "4. After the final dependent read, the consumer releases the empty barrier for reuse.", "sha256": "763037b61e4bfbaae65d104e217069396c21b86838e86a0a3cf76a821234b0fa"}, {"id": "u020", "kind": "prose", "locator": "body:L45-L45", "preview": "For binary16 `A[128,64]` and `B[64,256]`, the unpadded payload arithmetic is:", "sha256": "720c85da0975a0f1d2896dc5160c1ddb319b5ab4e5f9e77a1f863b5571298ee7"}, {"id": "u021", "kind": "code", "locator": "body:L47-L51", "preview": "```text A = 128 \u00d7 64 \u00d7 2 bytes = 16 KiB B = 64 \u00d7 256 \u00d7 2 bytes = 32 KiB one stage = 48 KiB; three stages = 144 KiB ```", "sha256": "b092d8e540dcc0b91039c62edcef86c1328d4c9d837144ae54279532c788868b"}, {"id": "u022", "kind": "prose", "locator": "body:L53-L53", "preview": "That is not a complete C++ shared-storage layout. Barriers, descriptors, epilogue scratch, padding, alignment, and swizzled physical layouts also consume or constrain shared memory. Compute capability 10.0 supports up to 228 KiB of shared m", "sha256": "7571140fcd587dd0d7dbfd830e6b0aa9af7f30e57c7f6c49c46017c55c048e65"}, {"id": "u023", "kind": "prose", "locator": "body:L55-L55", "preview": "FlashInfer PR 2387 is a second pinned example. Its merged `selective_state_update.cuh` has `state[numStages][...]` plus `bar_full` and `bar_empty` arrays and uses stage-specific producer/consumer handoffs. One path selects three stages and ", "sha256": "b3e9b66c8f8cc9b7e41acfa835cc972955d57829756334c98c944cf8b6d7aa7e"}, {"id": "u024", "kind": "prose", "locator": "body:L59-L59", "preview": "For Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, one D fragment is 128 registers per thread; two simultaneously live fragments are 256 registers, or 1024 bytes, per thre", "sha256": "f6e642682b459f63b368dd28720e7321715e6f3f095f5708a72b505a14d6e214"}, {"id": "u025", "kind": "prose", "locator": "body:L61-L61", "preview": "On SM100, tcgen05 keeps resident D in TMEM. This avoids a long-lived per-thread D vector, but the kernel still uses registers for addresses, descriptors, loop and barrier state, `tcgen05.ld` destinations, and epilogue temporaries. TMEM buff", "sha256": "bfb0ec4702c7e38a89db5564c2c0bc9038b25f7fb2da4fe7e93dfb6d6fc53189"}, {"id": "u026", "kind": "table-row", "locator": "body:L65-L65", "preview": "| Concern | Hopper WGMMA D | Blackwell tcgen05 D | | Resident storage | per-thread register fragment | allocated TMEM region |", "sha256": "c6d2a5bb97def6cf558e27a9beaf1ef9ddecc90fe5db35e212ebb270b9829736"}, {"id": "u027", "kind": "table-row", "locator": "body:L66-L66", "preview": "| Concern | Hopper WGMMA D | Blackwell tcgen05 D | | Two live outputs | two disjoint register fragments | two disjoint TMEM regions |", "sha256": "0713b8c5efd58eb69a0c9afbb6b8fdfabbfdad4b38831bbf0fba248d83b392c5"}, {"id": "u028", "kind": "table-row", "locator": "body:L67-L67", "preview": "| Concern | Hopper WGMMA D | Blackwell tcgen05 D | | Epilogue access | dependent use after WGMMA completion | collective `tcgen05.ld` and `tcgen05.wait::ld` after MMA completion |", "sha256": "1a1b26e2df7829f360c6b0d265010b85cdb71600218bc36dc143cbdb080c9368"}, {"id": "u029", "kind": "table-row", "locator": "body:L68-L68", "preview": "| Concern | Hopper WGMMA D | Blackwell tcgen05 D | | Main resource question | compiled registers, SMEM, block shape, and other limits | TMEM columns plus compiled registers, SMEM, block/cluster shape, and other limits |", "sha256": "45f5f676075588236b666c4a523ad20b4b6719508a6f5749f640ee3fbd88902f"}, {"id": "u030", "kind": "prose", "locator": "body:L72-L72", "preview": "Buffering is useful only when useful producer/consumer work overlaps enough to repay extra storage and coordination. For each concrete variant:", "sha256": "793da3bb85dc9c0487c967d2d3786db983a421feb6842d70691e85f9b3dd0189"}, {"id": "u031", "kind": "list-item", "locator": "body:L74-L74", "preview": "1. Keep shape, datatype, outputs, launch policy, warmup, and timing statistics identical.", "sha256": "c811a391248f2e1296904f15657e53b1e655a693b0c2f20cdf700f28553830e9"}, {"id": "u032", "kind": "list-item", "locator": "body:L75-L75", "preview": "2. Compare one versus multiple TMEM regions and the legal SMEM stage counts.", "sha256": "deb1c0cee7d6eaf7e50b64e7c05f585d2a755e855afdd7d697211d5fcfb3c6a2"}, {"id": "u033", "kind": "list-item", "locator": "body:L76-L76", "preview": "3. Record compiled registers/thread, spill traffic, static/dynamic SMEM, TMEM columns, barriers, threads/CTA, and cluster shape.", "sha256": "ae0395488ac0efa61f5fd2d5ac4a9b1daf9550fd06c04f868ae9f0183e291406"}, {"id": "u034", "kind": "list-item", "locator": "body:L77-L77", "preview": "4. Use the CUDA occupancy APIs for the actual resource record; do not infer a universal CTA/SM count from an instruction tile.", "sha256": "2a10693b0198014d1dc04730355f7a036237d5182f1bd7761e564c5141a60c79"}, {"id": "u035", "kind": "list-item", "locator": "body:L78-L78", "preview": "5. Inspect profiler pipeline and barrier stalls. PTX specifies mbarrier semantics, not a fixed \u201cfew cycles\u201d latency.", "sha256": "cf8f03e1c706642c110aae5d3923247b587728b29b7cd75a9abadb186bb84dbe"}, {"id": "u036", "kind": "list-item", "locator": "body:L79-L79", "preview": "6. Inspect generated PTX/SASS and execute correctness tests that force region wraparound and pipeline-tail paths.", "sha256": "65a8a0abcb1daa2ba41ef79bcbcc029319411ffd7cd2758984f0c96c15b9f169"}, {"id": "u037", "kind": "prose", "locator": "body:L81-L81", "preview": "Useful negative tests deliberately delay the epilogue, swap a barrier phase, omit the final drain, or reuse a region early. A correct test should detect stale/overwritten output or time out under a watchdog rather than silently accepting th", "sha256": "679336a50791d2df8ee87709263ae3c87ca8e1ff4b42560ec26d159a2e390266"}, {"id": "u038", "kind": "list-item", "locator": "body:L85-L85", "preview": "- [PTX ISA 9.0 Tensor Memory](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory)", "sha256": "aed47f18c514497e50694f89415704f98533229e52ed733c1898d3c9bb3e2747"}, {"id": "u039", "kind": "list-item", "locator": "body:L86-L86", "preview": "- [PTX ISA 9.0 tcgen05 allocation](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc)", "sha256": "31973f025402fdb2aee03f37ec947608275b8e131e5d9423444bc588c91662b7"}, {"id": "u040", "kind": "list-item", "locator": "body:L87-L87", "preview": "- [PTX ISA 9.0 tcgen05 load and wait](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld)", "sha256": "48de0a6246274c6b6cd72eb33f13df8bc2d371baf458c769c7dbc09c21d05cb3"}, {"id": "u041", "kind": "list-item", "locator": "body:L88-L88", "preview": "- [PTX ISA 9.0 mbarrier](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier)", "sha256": "57f21ef2db856a7d997dff531533e549ea54d4a49becca4e57e4fa0f44f2cc99"}, {"id": "u042", "kind": "list-item", "locator": "body:L89-L89", "preview": "- [CUDA 13.0.2 Blackwell Tuning Guide](https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy)", "sha256": "8d963db7f76eb6508d3be3b8853709f57f9a8b2f98e623d121cff16c31de6585"}, {"id": "u043", "kind": "list-item", "locator": "body:L90-L90", "preview": "- [Pinned combined SMEM/TMEM example](https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu)", "sha256": "862d7ef0e5fe7e4485454b7d0fbf52b79f47212d96d6c3d2d61e2f99a12c0966"}, {"id": "u044", "kind": "list-item", "locator": "body:L91-L91", "preview": "- [FlashInfer PR 2387 merged source](https://github.com/flashinfer-ai/flashinfer/blob/18804cd51734cccf807356d017733bc757677f15/include/flashinfer/mamba/selective_state_update.cuh)", "sha256": "21e58f5ba0dd7a07429d9e9e15e0cb6abd0137cefbff017bad4ae6654b1df059"}, {"id": "u045", "kind": "list-item", "locator": "body:L95-L95", "preview": "- [Tensor Memory](../hardware/tmem.md) \u2014 allocation, addressing, and lifetime rules", "sha256": "7afb4cc4a4c10faf8414ea0adcf8dbca658326f9d2b530cdba04f46f89c8a510"}, {"id": "u046", "kind": "list-item", "locator": "body:L96-L96", "preview": "- [pipeline stages](pipeline-stages.md) \u2014 SMEM pipeline construction and tail handling", "sha256": "a926fba806fc32d5e84f37e9775faa981aee434d3e360678aee986e6bc627802"}, {"id": "u047", "kind": "list-item", "locator": "body:L97-L97", "preview": "- [epilogue fusion](epilogue-fusion.md) \u2014 work performed while draining an output tile", "sha256": "613bbcc5ee62a62bc83b7c29a8227e6b57497c4f7f38939f1ca80134084b718c"}], "confidence_claimed": "verified", "headings": ["What the pattern guarantees", "TMEM regions", "SMEM stages", "Hopper and Blackwell storage tradeoff", "Decide with a controlled comparison", "Primary references", "Related"], "id": "technique-double-buffering", "path": "wiki/techniques/double-buffering.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/prs/flashinfer/PR-2387.md", "revision": "18804cd51734cccf807356d017733bc757677f15", "url": "https://github.com/flashinfer-ai/flashinfer/pull/2387"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-tcgen05-tutorial", "doc-nvidia-tuning-guide", "pr-flashinfer-2387"], "title": "Double/Multi-Buffering Patterns", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "7b78dc20ff19f53642d3b695375cd2e64bb0f6f55abad1feba792f98a12ad994", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Epilogue fusion computes output transformations\u2014such as `alpha*acc + beta*C`, bias, activation, output conversion, or auxiliary values\u2014inside the producer kernel before final output materialization. It can remove an intermediate tensor or l", "sha256": "f9107441dd920540c0d1ac9e5090773a75dfa9fa7d1443b994ffe3d7f94d465d"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "Fusion does not by itself imply overlap with the next MMA tile. A schedule may drain one completed accumulator after the mainloop, or it may use disjoint storage and specialized roles to overlap a completed region's epilogue with independen", "sha256": "4f47d7d2b8553488318d46030c512f990423ca91ad8d800925811d2b7e8bb075"}, {"id": "u003", "kind": "prose", "locator": "body:L11-L11", "preview": "For tcgen05, D resides in TMEM. An ordinary arithmetic epilogue first uses a legal collective `tcgen05.ld` mapping, or a library wrapper around it, to transfer a partition into per-thread registers. The load is asynchronous with respect to ", "sha256": "5877a199df06e039e1d768c0f9a71bba45e88384bc7e3ff057d758f72ec8ce82"}, {"id": "u004", "kind": "prose", "locator": "body:L13-L13", "preview": "CUTLASS 4.5.0's `fp16_gemm_2.py` demonstrates the typed route:", "sha256": "adc38d40084211a2785289cdae32b37c45f48b6c540acd24ab586a3a8994814b"}, {"id": "u005", "kind": "code", "locator": "body:L15-L24", "preview": "```python copy_atom_t2r = cute.make_copy_atom( tcgen05.Ld32x32bOp(tcgen05.Repetition.x32), cutlass.Float32 ) tiled_copy_t2r = tcgen05.make_tmem_copy(copy_atom_t2r, tCtAcc_epi) thr_copy_t2r = tiled_copy_t2r.get_slice(tidx) tTR_tAcc = thr_cop", "sha256": "98fb794337ea2463c1d0e401026810e136bcde4790746a21365872ce92d466a6"}, {"id": "u006", "kind": "prose", "locator": "body:L26-L26", "preview": "This is an API-routing fragment, not standalone code: the official file supplies the exact tensor layouts, selected load shape, participant group, edge predicates, accumulator-completion handoff, register-to-SMEM conversion, TMA-store pipel", "sha256": "a8dc66e0523115acbd50a98fb09e529ba902911c6c8ccd4b24c2afc24fec85eb"}, {"id": "u007", "kind": "table-row", "locator": "body:L32-L32", "preview": "| Boundary | Proof required before crossing it | | tcgen05 MMA \u2192 TMEM reader | the relevant asynchronous MMA is committed and its completion observed |", "sha256": "9985d4cd976b715fdc76865925a3aa8489cadc7a359e285805c6174ead01f6a4"}, {"id": "u008", "kind": "table-row", "locator": "body:L33-L33", "preview": "| Boundary | Proof required before crossing it | | TMEM \u2192 result registers | the legal collective load completes before dependent arithmetic |", "sha256": "6b6d528df317b00dd9c026068a171d0397fece81c64efdee7e8451c739811c78"}, {"id": "u009", "kind": "table-row", "locator": "body:L34-L34", "preview": "| Boundary | Proof required before crossing it | | registers \u2192 output | element/layout mapping and output-edge predicates cover exactly the valid coordinates |", "sha256": "efc6c04d14475fe8461a37c816dadacb9dfe78df871eba2e26350ad5e59bbc3e"}, {"id": "u010", "kind": "table-row", "locator": "body:L35-L35", "preview": "| Boundary | Proof required before crossing it | | TMEM reader \u2192 region reuse | every reader's load has completed and the matching reusable barrier phase is released |", "sha256": "624abc6a869187539c7ac548bb053a85c391b041419a4f52f82cd2390fe000da"}, {"id": "u011", "kind": "table-row", "locator": "body:L36-L36", "preview": "| Boundary | Proof required before crossing it | | kernel tail | all output stores complete as required and every TMEM allocation is collectively freed |", "sha256": "394bf4e44474bc37252b13e6680b64dce13cdd39a349c33da579f6af120afb85"}, {"id": "u012", "kind": "prose", "locator": "body:L38-L38", "preview": "For a multi-region overlap schedule, simultaneously live TMEM regions must also be disjoint. Equal 256-column halves are one possible policy for a 512-column allocation, not an epilogue-fusion requirement. Reusable mbarriers need correct ex", "sha256": "632d24557c88b222486de12166a2781d2e53a39ed69410cfa66d6ec170d85f1a"}, {"id": "u013", "kind": "prose", "locator": "body:L42-L42", "preview": "`cutlass::epilogue::fusion::LinCombEltAct` has this parameter order:", "sha256": "91ce48572731a856757d222bfb3aa65e8350a69cd68d1b04eff3e953be42239a"}, {"id": "u014", "kind": "code", "locator": "body:L44-L53", "preview": "```cpp template < template class ActivationFn, class ElementOutput, class ElementCompute, class ElementSource = ElementOutput, class ElementScalar = ElementCompute, cutlass::FloatRoundStyle Round = cutlass::FloatRoundStyle::round_to", "sha256": "2c17e7f24711c91e72d68d975cae0b68a3c2482f8b386e3a64ab96d14482efa0"}, {"id": "u015", "kind": "prose", "locator": "body:L55-L55", "preview": "For SM100 construction, `cutlass::epilogue::collective::CollectiveBuilder` receives architecture, operator class, tile/cluster shapes, `EpilogueTileAuto` or an explicit epilogue tile, accumulator/compute/C/D types and layouts, alignment, `E", "sha256": "5ba02c063f7bbb12184dbe6ba70d2875fca82c9b9bb28392add306148e470825"}, {"id": "u016", "kind": "prose", "locator": "body:L57-L57", "preview": "There is no CUTLASS 4.5.0 type named `Sm100EpilogueTmaWarpSpecialized`. Support is constrained by the exact architecture, operator class, schedule, tile, layout, alignment, datatype, and fusion callback combination. Use `Gemm::can_implement", "sha256": "d111f3431cc1550474efe0835affdc6255eb3133f4ab79c1adeab8d7bb1385ce"}, {"id": "u017", "kind": "prose", "locator": "body:L59-L59", "preview": "The vLLM PR 16032 NVFP4 wrapper is one pinned C++ construction example: it uses `CollectiveBuilder<... EpilogueTileAuto, ... EpilogueScheduleAuto>` and derives mainloop shared-memory stages with `StageCountAutoCarveout`](https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/swizzle.hpp):", "sha256": "5ef103ec8227418a2bf6fc668b9d8a44e15b7542e22dfe952b9f3368f5e85332"}, {"id": "u017", "kind": "list-item", "locator": "body:L36-L36", "preview": "- `BBits` is the number of mask bits;", "sha256": "5af4a067d6c7da7415684db6889c71471be01324eefd2b56cba5ab0605b9048b"}, {"id": "u018", "kind": "list-item", "locator": "body:L37-L37", "preview": "- `MBase` is the number of least-significant address bits kept invariant; and", "sha256": "4baa3200cd5285f7c28a299e5577b8cdb19ad27e8d093c6208a6d0b33aad32b5"}, {"id": "u019", "kind": "list-item", "locator": "body:L38-L38", "preview": "- `SShift` is the distance between the two bit fields.", "sha256": "f28019ed4ea31c3b1d699597255eec3b2cb5bca744ea8d505802e5142e328689"}, {"id": "u020", "kind": "prose", "locator": "body:L40-L40", "preview": "For `Swizzle<3,4,3>`, three address bits beginning above the four invariant low bits are XORed with the three-bit field shifted by three positions. Equivalently, address bits 7:9 affect bits 4:6. This is an address transformation, not a gen", "sha256": "ffaae440bfd21e9f23db7977e6f25a0843206a5ebcccceb60b0f43b287fb38b6"}, {"id": "u021", "kind": "prose", "locator": "body:L42-L42", "preview": "CUTLASS also distinguishes a position-independent composed swizzle layout from a position-dependent swizzle pointer, because hardware swizzling depends on the shared-memory pointer address. Reuse a pinned complete layout/descriptor construc", "sha256": "7b7c33d7d8af0396265809e977fa799bb5bc22896a34d87bb0341b7036cd6fdf"}, {"id": "u022", "kind": "prose", "locator": "body:L46-L46", "preview": "For `cuTensorMapEncodeTiled`, record and validate all inputs rather than copying only the swizzle enumerator:", "sha256": "52e9d72146342e12fd2b1490c3761b731df5f3257a9203f475f988e31c7143c9"}, {"id": "u023", "kind": "list-item", "locator": "body:L48-L48", "preview": "1. Use a correctly aligned `CUtensorMap` object and global base pointer.", "sha256": "80e4aa5c2d38c0d179760ddbd62c54902ad761ce25c126a2a276d27c250232a0"}, {"id": "u024", "kind": "list-item", "locator": "body:L49-L49", "preview": "2. Supply rank-sized global dimensions, rank-minus-one byte strides (the fastest dimension is implicit), rank-sized box dimensions, and rank-sized element strides.", "sha256": "7a0cabb5ab90c8ad9fcb90c4a3b1abec710ea10e42d49e4b1ae7f089df312640"}, {"id": "u025", "kind": "list-item", "locator": "body:L50-L50", "preview": "3. Satisfy the global alignment, stride, box, interleave, datatype, and selected-swizzle constraints.", "sha256": "44c861550453d7469cfe6ab5f3f95ff727c04f68a1728ad416f78b1f7d2b1b7c"}, {"id": "u026", "kind": "list-item", "locator": "body:L51-L51", "preview": "4. Check the returned `CUresult`; do not launch with an output descriptor after encoding failed.", "sha256": "e35e8c5abe5a043f7a59a02a53ec58fbe4d9c5630736504600a4fdf081ea801d"}, {"id": "u027", "kind": "list-item", "locator": "body:L52-L52", "preview": "5. Use a shared-memory base and tcgen05 descriptor that represent the same mapping as the tensor map.", "sha256": "bcf3491b71148b2582b92aa3a39be1f27b68b24860545007d7c3fceec0df92d3"}, {"id": "u028", "kind": "prose", "locator": "body:L54-L54", "preview": "An invalid encoder combination can fail explicitly. A successfully encoded tensor map paired with the wrong consumer layout may instead read the wrong logical elements, so a successful API return is not a correctness oracle.", "sha256": "5bb6b35556ce52d1c4949afd7353eea6ed6e47d18072506fc1477c4c5e422a86"}, {"id": "u029", "kind": "prose", "locator": "body:L58-L58", "preview": "Gau Nernst's pinned B200 tutorial compares M=N=K=4096 kernels with PyTorch 2.9.1 and CUDA 13. Its 3D TMA version changes from a 16-byte inner tile with no swizzle to a 128-byte inner tile with 128B swizzling and matching tcgen05 descriptors", "sha256": "1442557fe3203a558928700307223fea48556a1423bb751823fe7a3ded0daec6"}, {"id": "u030", "kind": "table-row", "locator": "body:L62-L62", "preview": "| Tutorial version | Author-reported TFLOP/s | | v1b: 3D 16B TMA | 252.81 |", "sha256": "5ec52bb84daca8d9c4f9c330d7e0f75780b21a3674ec4e8c94329488f3030c29"}, {"id": "u031", "kind": "table-row", "locator": "body:L63-L63", "preview": "| Tutorial version | Author-reported TFLOP/s | | v2b: 3D 128B TMA plus 128B swizzle | 695.43 |", "sha256": "98a08ba0c20d778d67556cee3756e894c83682172bf48209e67517fa770e44e4"}, {"id": "u032", "kind": "prose", "locator": "body:L65-L65", "preview": "The combined change is approximately 2.75\u00d7 and the v2b endpoint is about 46% of the tutorial's 1506.74 TFLOP/s cuBLAS result. It is not a bank-conflict or swizzle-only ablation. The author notes that the earlier contiguous `8\u00d716B` tile migh", "sha256": "5c904bdadd0bdbec537b8da0a58b9e0eae1846a8130911c2933fdae421f9771c"}, {"id": "u033", "kind": "prose", "locator": "body:L69-L69", "preview": "For each candidate mapping:", "sha256": "7efda7c57692314fbd1a9fcdd587176241012dc24de5eecd9b854d1b2545ead1"}, {"id": "u034", "kind": "list-item", "locator": "body:L71-L71", "preview": "1. Keep global tensor, box, consumer instruction, tile shape, synchronization, and output oracle fixed. Change only the producer/consumer layout pair when an isolated comparison is possible.", "sha256": "85fa3e9198db89eed9cb12480937658783d70f38dd929725f0b8cf71e59328c7"}, {"id": "u035", "kind": "list-item", "locator": "body:L72-L72", "preview": "2. Check every tensor-map encoder result and run correctness tests that distinguish rows, columns, tiles, boundaries, and out-of-bounds fill. Do not rely on random uniform values that can hide permutations.", "sha256": "7c059219f0a0bef45641d0ee887c2f16f188c681fa01a4258e0aac06772ee9ef"}, {"id": "u036", "kind": "list-item", "locator": "body:L73-L73", "preview": "3. Use the installed Nsight Compute's `--query-metrics` or Memory Workload Analysis rather than assuming one metric name exists on every chip/tool version. Current documentation includes `l1tex__data_bank_conflicts_pipe_lsu.sum` and request", "sha256": "7081d7ec194ad85836cd0a830d113b7edfbf9547a09a3d54ef52e60a1a4dbf2e"}, {"id": "u037", "kind": "list-item", "locator": "body:L74-L74", "preview": "4. Compare executed shared-memory requests, ideal versus excessive wavefronts, bank conflicts, TMA behavior, total time, and occupancy. Zero observed CUDA-core bank conflicts neither proves the MMA descriptor matches nor explains a performa", "sha256": "16afbfd8bff05f99eb2faf2c5860e7a0786f1e850a6c43efa566f887bdcc24dd"}, {"id": "u038", "kind": "list-item", "locator": "body:L75-L75", "preview": "5. Record GPU, clocks, toolkit, profiler, compiler, exact addresses/alignment, layout types, descriptor fields, warmup, repetitions, and trial statistic.", "sha256": "09ceac52c96b6ed2ee30601a3682d58e7e976256b12796df1a192f98531b105e"}, {"id": "u039", "kind": "prose", "locator": "body:L77-L77", "preview": "Select the mapping that is legal and correct for both producer and every consumer, then retain it only where the controlled target workload improves. Do not infer the choice from architecture name, element width, or \u201cMMA versus non-MMA\u201d alo", "sha256": "d9fe95880e83d8c77d23da16c2196e5649905b51170a21816dd3bca04ba4e1a5"}], "confidence_claimed": "verified", "headings": ["Bank-conflict model", "Three mappings must agree", "CuTe address-bit form", "Tensor-map construction checklist", "Source-reported tutorial result", "Verification procedure"], "id": "technique-swizzling", "path": "wiki/techniques/swizzling.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-cuda-13-0-2-tma.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#asynchronous-data-copies-using-the-tensor-memory-accelerator-tma"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}], "risk_flags": ["ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-cuda-13-0-2-tma", "doc-ptx-isa-sm100", "blog-tcgen05-tutorial"], "title": "Shared Memory Swizzling", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "ddee79f7472a6c3a775767f5f72596fd42a2d47df2856c53f96609d5e55ce71c", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "\u201cTile scheduling\u201d can refer to four different choices:", "sha256": "7b32fb0a937d7562fe4d500af4d3ac5ec152f9dc500779bc152c44b07f745659"}, {"id": "u002", "kind": "list-item", "locator": "body:L5-L5", "preview": "1. **Coordinate order:** a software mapping from a logical work index to `(tile_m, tile_n, ...)`, such as row-major, column-major, or a blocked/swizzled raster.", "sha256": "01bc0ff0de2d8faaa3f3ecc471ffde0939b45942f3e3b0e91c98312a9e644340"}, {"id": "u003", "kind": "list-item", "locator": "body:L6-L6", "preview": "2. **Resident-worker iteration:** whether a CTA handles one logical tile or repeatedly advances through work, for example by a static grid stride.", "sha256": "dfce2ca5b0bf2328975bc0b6e37cb7aa76a25fe5d9e46ec5b83b958e1df2ca8e"}, {"id": "u004", "kind": "list-item", "locator": "body:L7-L7", "preview": "3. **Work reassignment:** on SM100, Cluster Launch Control (CLC) lets a running worker cancel another grid entity that has not started and process the returned ClcID.", "sha256": "ad294691c329b879334b36f7addb42f5a735d07f182fa05a1568622e393ddf42"}, {"id": "u005", "kind": "list-item", "locator": "body:L8-L8", "preview": "4. **Problem decomposition:** Stream-K or Split-K can partition a tile's K work and then reduce partial results.", "sha256": "d68060e493898620463da03ac5643b456b664ef8a1a347e6ab695b45be712287"}, {"id": "u006", "kind": "prose", "locator": "body:L10-L10", "preview": "CLC does not accept a raster-order or swizzle-policy operand. CUTLASS applies those coordinate transforms in software to initial or returned coordinates. CLC also does not itself create K partitions or synthesize work beyond the launched gr", "sha256": "a5330ef0261e88c9d868a9e3dffeb4c1113bb325cdf01980a5cd9c1961898532"}, {"id": "u007", "kind": "prose", "locator": "body:L14-L14", "preview": "A flat row-major mapping is:", "sha256": "afb209da3a8a9396139e64e9bd7d8a4a8312b522317d1e515c25e13dd2696ed1"}, {"id": "u008", "kind": "code", "locator": "body:L16-L22", "preview": "```cuda __device__ void row_major_tile(int tile_idx, int tiles_n, int& tile_m, int& tile_n) { tile_m = tile_idx / tiles_n; tile_n = tile_idx % tiles_n; } ```", "sha256": "95c2ded3665bde2f3dcdf549fc05aa118b02258f5ab7143b40e7c212d48f4269"}, {"id": "u009", "kind": "prose", "locator": "body:L24-L24", "preview": "For a positive grid size, a static persistent worker can cover the flat index set without overlap:", "sha256": "cb8e18435ff65424ff8eecb1e050db1e5b8b4b92213bd48c80889d4c1f35059c"}, {"id": "u010", "kind": "code", "locator": "body:L26-L34", "preview": "```cuda for (int tile_idx = int(blockIdx.x); tile_idx < total_tiles; tile_idx += int(gridDim.x)) { int tile_m = tile_idx / tiles_n; int tile_n = tile_idx % tiles_n; compute_tile(tile_m, tile_n); } ```", "sha256": "e671351d81c0a79ee7e9273265fb228fe6e8819cee691435167ec958ed2cefed"}, {"id": "u011", "kind": "prose", "locator": "body:L36-L36", "preview": "Changing coordinate order changes the sequence of operand panels presented to the cache. It does not by itself prove a hit rate or speedup. A valid transform must first be shown to be in-bounds and bijective for edge groups; then its locali", "sha256": "6f6772443125529b78281c3269007c9d850c5072b7942599dbb6f86413914f7e"}, {"id": "u012", "kind": "prose", "locator": "body:L40-L40", "preview": "CUTLASS 4.5.0 documents this lifecycle for a CLC-backed persistent scheduler:", "sha256": "b9c7e8b605d64aac51b86256ed00d2cacf354e8b60280f9b4a025abfb2d71e25"}, {"id": "u013", "kind": "list-item", "locator": "body:L42-L42", "preview": "1. Launch the full problem grid and process the worker's initial `blockIdx` coordinate.", "sha256": "20d20b6a4cf0cb97dd6ffbb93b309b13a3241d6fcdaade9558afc3a1d4ac05c2"}, {"id": "u014", "kind": "list-item", "locator": "body:L43-L43", "preview": "2. Submit `clusterlaunchcontrol.try_cancel` against another not-yet-started grid entity. The asynchronous 16-byte response completes a transaction on an mbarrier.", "sha256": "a06961216d88c13174e5484e203558f45cf98ac572f6a213677701fbf730bf7d"}, {"id": "u015", "kind": "list-item", "locator": "body:L44-L44", "preview": "3. After the transaction completes, query whether cancellation succeeded. On success, decode and process the returned ClcID; for clusters, combine the returned first coordinate with the local cluster rank.", "sha256": "c7ab67729620e19fb98c56bbd8970ce7a7b024dd1e993362701587079589ba6d"}, {"id": "u016", "kind": "list-item", "locator": "body:L45-L45", "preview": "4. Treat an observed failed request as terminal for requests by that thread. Retrying from the same thread after failure is undefined.", "sha256": "c881c94cd5d38a82f418a9dd3cb9fd072897c3e38cf5208b7f2616ef9db28130"}, {"id": "u017", "kind": "prose", "locator": "body:L47-L47", "preview": "This can redistribute existing work when SM availability is uneven. It changes which resident worker handles an unlaunched ID, not the number of independent IDs in the grid. Software raster/swizzle may be applied consistently to both initia", "sha256": "1438dba392c4a82e98af96e55adc1a396e15067319beaa2511761701eca3fe2c"}, {"id": "u018", "kind": "prose", "locator": "body:L51-L51", "preview": "The public scheduler tags select implementation classes; users should prefer those tags over spelling detail types directly.", "sha256": "c9194ef2ae3478c668ebd794c2ebda157dd3b3fd60d8dff4d17922264e0e1c40"}, {"id": "u019", "kind": "table-row", "locator": "body:L55-L55", "preview": "| Public tag for `arch::Sm100` | Selected implementation | Scope | | `PersistentScheduler` (also the default `void` tag) | `PersistentTileSchedulerSm100` | CLC-backed persistent data-parallel scheduling |", "sha256": "436f3f1bdc780f0bcbba5b14d827591dbd44f9d903da5772fa655b3ac7a478c6"}, {"id": "u020", "kind": "table-row", "locator": "body:L56-L56", "preview": "| Public tag for `arch::Sm100` | Selected implementation | Scope | | `DynamicPersistentScheduler` | `PersistentTileSchedulerSm100` | Explicit dynamic route to the same SM100 implementation |", "sha256": "ce0c2c6ed629edf9fded3c6cfe63c01b6790dc62bc3d52c0b6858ae77a6c97b0"}, {"id": "u021", "kind": "table-row", "locator": "body:L57-L57", "preview": "| Public tag for `arch::Sm100` | Selected implementation | Scope | | `StaticPersistentScheduler` | `StaticPersistentTileScheduler100` | Static persistent scheduling |", "sha256": "59209b09fb5932bb87b80c3e1579fb729d01134b2ad07175e6e3765b61a7531a"}, {"id": "u022", "kind": "table-row", "locator": "body:L58-L58", "preview": "| Public tag for `arch::Sm100` | Selected implementation | Scope | | `StreamKScheduler` | `PersistentTileSchedulerSm100StreamK` | Parameterized data-parallel, Stream-K, or Split-K decomposition |", "sha256": "9dbd3bb085575ad9584218613f353eac0dcb7893ce1c68d03863b8011786992e"}, {"id": "u023", "kind": "table-row", "locator": "body:L59-L59", "preview": "| Public tag for `arch::Sm100` | Selected implementation | Scope | | `GroupScheduler` | `PersistentTileSchedulerSm100Group` | Grouped-problem route; in this tag it wraps the SM90-style static group scheduler |", "sha256": "0ecf5b3ad36b38af94e66745a93247520268644a8550382e195bb62071a6bf95"}, {"id": "u024", "kind": "prose", "locator": "body:L61-L61", "preview": "The SM100 Stream-K route accepts decomposition, split, raster, swizzle, and reduction settings. CUTLASS 4.5.0 has a deterministic lock/turnstile reduction and a nondeterministic atomic-workspace reduction; atomic accumulation is therefore n", "sha256": "1ff1f5194c712ba9bb2dae5da5ffaa2ba7c9fc8351523869f82bed91b7567175"}, {"id": "u025", "kind": "prose", "locator": "body:L65-L65", "preview": "For a deliberately simplified model with `T` equal-duration independent tiles, `W` available one-tile workers, and no K decomposition, write:", "sha256": "3184df242365781fc5c684291037f9127188eed995576762d2924ebb7cebccab"}, {"id": "u026", "kind": "code", "locator": "body:L67-L69", "preview": "```text T = qW + r, 0 <= r < W ```", "sha256": "9055386055d6219a3731bea281e8962958fea9a3ccddd596354c89e898f6cd16"}, {"id": "u027", "kind": "prose", "locator": "body:L71-L71", "preview": "If `r > 0`, the final partial wave has `r` active workers and occupancy `r / W`. If `r == 0` and `T > 0`, the final wave is full; reporting zero occupancy from the remainder alone is an error.", "sha256": "26ad025dfeb8dbf255a8583d4cc6332064e9c8889825eea498c1e7f0b3f5e5db"}, {"id": "u028", "kind": "prose", "locator": "body:L73-L73", "preview": "For `T = 150` and `W = 142`, the second wave contains 8 tiles, so 8 workers are active, 134 are idle, and wave occupancy is `8 / 142 = 5.63%` under those assumptions. CLC may change which workers receive those eight IDs and reduce delays ca", "sha256": "ea61406965d276427719ff49baf8c6394acc0d66f5e297ec2fe702333b6d0412"}, {"id": "u029", "kind": "list-item", "locator": "body:L77-L77", "preview": "1. Establish a correct unswizzled data-parallel mapping and verify exact tile coverage, including nondivisible edge groups.", "sha256": "e0aab377952cc5e5d1d899541ec67d739c272ed4e36f6ec16e835f1ce75277f5"}, {"id": "u030", "kind": "list-item", "locator": "body:L78-L78", "preview": "2. Sweep legal raster orders and swizzle sizes. Record kernel time plus L2 hit/sector traffic; do not infer eviction from coordinate order alone.", "sha256": "c06c06c02554411d3f9b47429f84f9a8e5441e79fc4dbdb332d4e538106ece78"}, {"id": "u031", "kind": "list-item", "locator": "body:L79-L79", "preview": "3. Compare static and CLC-backed persistence at identical tile, cluster, grid, and occupancy settings. Record successful/failed CLC requests and per-worker tile counts where instrumentation permits.", "sha256": "9b223b0ad22011e157f170461269e99ab6faeb845709c550a38ce039b5bb616d"}, {"id": "u032", "kind": "list-item", "locator": "body:L80-L80", "preview": "4. Test Stream-K separately with explicit decomposition, split, and reduction modes. Include workspace traffic, synchronization, determinism, and numerical tolerance.", "sha256": "636232c0d996791186f2d18f4d6911d3189712cd4ca86a0da1cd78474119981a"}, {"id": "u033", "kind": "list-item", "locator": "body:L81-L81", "preview": "5. Repeat across representative shapes and grouped-size distributions. Report regressions as well as wins; there is no universal best scheduler for all GEMMs, attention kernels, or MoE workloads.", "sha256": "ffb9c866fb101d1393c4c852ec2fb45c0f61356743566ff7791cda405e4c31ce"}, {"id": "u034", "kind": "prose", "locator": "body:L83-L83", "preview": "Without target-GPU measurements, this page makes no fixed claim for CLC acquisition latency, L2-miss reduction, or scheduler speedup.", "sha256": "2925a13ea32074b79278f44f396331abbada0e420d39689800102052fdcf8625"}], "confidence_claimed": "source-reported", "headings": ["Keep the Scheduling Layers Separate", "Exact Static Mappings", "SM100 CLC Work Acquisition", "CUTLASS 4.5.0 Scheduler Routes", "Tail Arithmetic", "Selection and Verification Workflow"], "id": "technique-tile-scheduling", "path": "wiki/techniques/tile-scheduling.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/cutlass-clc-documentation.md", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}, {"path": "sources/docs/nvidia-cutlass-blackwell.md", "url": "https://github.com/NVIDIA/cutlass/tree/v4.5.0"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-cutlass-clc", "doc-cutlass-blackwell"], "title": "Tile Scheduling Strategies", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "06ac3ebdfd34c7e7e4d4572a79c562a6b8733c0609fbdfe47db0573ad33e2290", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "A PTX vector load moves several typed elements into several registers with one instruction. This can reduce the number of load instructions issued by a thread, but PTX only says that vector loads *may* improve memory performance. Source-lev", "sha256": "3d99821d64a7e0fcf66a49561f2925641a983fbfdf7a6bdc83d77beacdcaa737"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "PTX ISA 9.0 permits the following global-memory forms on `sm_100`. The first accesses 16 bytes and the second accesses 32 bytes:", "sha256": "9f567d25863717dbe10d6a0c35951f195f02e73b69f3216826746dd1b16b7492"}, {"id": "u003", "kind": "code", "locator": "body:L9-L16", "preview": "```ptx .reg .b64 addr; .reg .u32 x<4>; .reg .u64 y<4>; ld.global.v4.u32 {x0, x1, x2, x3}, [addr]; ld.global.v4.u64 {y0, y1, y2, y3}, [addr]; ```", "sha256": "761519ce1a2f10c73aea258740cca70aac5fdf07bf0ff7a8a9d5b4984de5eace"}, {"id": "u004", "kind": "prose", "locator": "body:L18-L18", "preview": "The address must be naturally aligned to the total access size: 16 bytes for `v4.u32` and 32 bytes for `v4.u64`. A misaligned PTX address has undefined behavior; the ISA says it may have low address bits masked or fault. It does not specify", "sha256": "8cb92e31fee4aa36338676a252ea633a71739cc94656f4d19c25a20b651c5d11"}, {"id": "u005", "kind": "prose", "locator": "body:L20-L20", "preview": "The 256-bit `v4.b64` family is an SM100-or-newer feature. A 32-byte load contains 64 values when the payload is densely packed E2M1 at two 4-bit values per byte. That arithmetic says nothing about whether the width is profitable: unpack cos", "sha256": "33deaf137e1d64915539255a42f0a1496919c9302b9da264378d73eb537c5026"}, {"id": "u006", "kind": "prose", "locator": "body:L24-L24", "preview": "PTX distinguishes cache operators, eviction priorities, prefetch-size hints, and non-coherent read-only loads. Representative legal forms are:", "sha256": "0b9faa0d058fdadf045f326ea5dfcbab2f53d282615162309f99230b62e0cd20"}, {"id": "u007", "kind": "code", "locator": "body:L26-L31", "preview": "```ptx ld.global.L1::no_allocate.v4.u32 {x0, x1, x2, x3}, [addr]; ld.global.L1::evict_last.v4.u32 {x0, x1, x2, x3}, [addr]; ld.global.L2::256B.b32 x0, [addr]; ld.global.nc.b32 x0, [addr]; ```", "sha256": "ec4167ea0ba7152cb2179a67671479d0a2f4f407a28687bf87af2d87540b6f29"}, {"id": "u008", "kind": "table-row", "locator": "body:L35-L35", "preview": "| Form | Documented meaning | Limit on inference | | `L1::no_allocate` | L1 eviction-priority selection that may be applied | Does not guarantee an L1 bypass or a speedup for a streaming operand |", "sha256": "2780be407bbe96d5b106e3a0f5a703536fbb5465cf3d0d723517517d9918dcea"}, {"id": "u009", "kind": "table-row", "locator": "body:L36-L36", "preview": "| Form | Documented meaning | Limit on inference | | `L1::evict_last` | Requests the corresponding L1 eviction priority | Does not guarantee that a line remains resident |", "sha256": "1840c23696a0f16e1089d8d0649d755433db335252c3256c41f938763dc645d2"}, {"id": "u010", "kind": "table-row", "locator": "body:L37-L37", "preview": "| Form | Documented meaning | Limit on inference | | `L2::256B` | Hints that additional data of the stated size be prefetched into L2 | Is not a 256-byte load or a promotion guarantee |", "sha256": "06b2d39edee55c63dc2701519fcfeb993a3d00a45c4922a7b7b2e8dc80220e9b"}, {"id": "u011", "kind": "table-row", "locator": "body:L38-L38", "preview": "| Form | Documented meaning | Limit on inference | | `ld.global.nc` | Loads through a non-coherent read-only cache | Has architecture- and parallelism-dependent latency/throughput; it is not a general coherence optimization |", "sha256": "494ed829c67676d6c3b42e0d71f7c5924d0ae9f55c3a3caaf41be0d2bc6e093e"}, {"id": "u012", "kind": "prose", "locator": "body:L40-L40", "preview": "Cache-policy operands and prefetch-size qualifiers are performance hints and do not change the program's memory-consistency behavior. A reuse argument can motivate `evict_last`, and a one-pass stream can motivate `no_allocate`, but only a m", "sha256": "ad91b927be30bcfcfa70030224edc683a2b9de8c4f6af8a4d75c1c7d147b9724"}, {"id": "u013", "kind": "prose", "locator": "body:L44-L44", "preview": "For each candidate width and cache policy:", "sha256": "5e99de4759e9a94d2fa025e1ecf3ba20b1ee617b70fb262ad9d3cfde7d0408f5"}, {"id": "u014", "kind": "list-item", "locator": "body:L46-L46", "preview": "1. Prove base and per-thread address alignment for every executed access. Use a separate scalar or narrower-vector path for tails; validate minimum, maximum, and awkward sizes.", "sha256": "e6034cb185b56b2aa0b6088595b59f10d947cedfc609d0a6b028860c3041cbbf"}, {"id": "u015", "kind": "list-item", "locator": "body:L47-L47", "preview": "2. Hold the algorithm, mapping, decode, unrolling, launch shape, compiler, and inputs fixed while changing one load or cache choice. Compare identical correctness oracles before timing.", "sha256": "280cf574c975c962cfa7fb90943e03e73b2c5c96a11093dcab42ecce65d1bbb6"}, {"id": "u016", "kind": "list-item", "locator": "body:L48-L48", "preview": "3. Inspect generated PTX and SASS. Record actual load instructions, registers per thread, spill loads/stores, stack bytes, shared memory, and launch parameters. A source cast or inline-PTX block does not bypass compiler register allocation.", "sha256": "2fd170479b42be4da847ddabe8e7582f942debfed14c3050e1b31f7ad4941bdc"}, {"id": "u017", "kind": "list-item", "locator": "body:L49-L49", "preview": "4. Profile the matched variants. Check achieved bandwidth, requested versus transferred bytes, cache hit behavior, instruction issue/stalls, and active warps with metrics available on the installed Nsight Compute and GPU versions.", "sha256": "b7cee53be2ebd70f15f015073066cec635dbf6649f523057f03d1f5ac2e26ea9"}, {"id": "u018", "kind": "list-item", "locator": "body:L50-L50", "preview": "5. Use warmups, synchronization, repeated trials, and the same statistic for every production shape. Keep the change only where the declared metric improves without a correctness or resource regression.", "sha256": "36924cb17b1ccb46e32d0ddea78c1db22526b9871e47971d6fb917b03381c23f"}, {"id": "u019", "kind": "prose", "locator": "body:L52-L52", "preview": "Register caps and launch bounds are a separate variable. A nominal cap may be inert when natural allocation is lower, or it may trade registers for spills and extra instructions. Follow the resource and occupancy workflow in [Register Budge", "sha256": "cb3704e62d7d4725397d5fe9b2a2bcecbb903980445d842b50eb237c778bd327"}, {"id": "u020", "kind": "prose", "locator": "body:L56-L56", "preview": "Yue Zhang reports the following CUDA progression for GPU Mode Problem 1. These are author-reported endpoints without raw repeated-trial data or released complete submission code:", "sha256": "7b332e96d30f401078015efa8d7c6b2790a822f625eebb2f9b0a386d4e64a262"}, {"id": "u021", "kind": "table-row", "locator": "body:L60-L60", "preview": "| Stage | Combined change | Reported latency | | Initial CUDA | Naive hand-written path | about 2000 \u00b5s |", "sha256": "7c11921183bdbcbdb88258eff9a6b1d38d68d52b03ad0de2a6ddd30a6c5dbae1"}, {"id": "u022", "kind": "table-row", "locator": "body:L61-L61", "preview": "| Stage | Combined change | Reported latency | | CUDA optimization 1 | Coalescing, shared B, thread collaboration, warp reduction | about 443 \u00b5s |", "sha256": "98bdc14cff910536e6b2902f8e77cea8a8cf5401513c2fb653c934478cfaaf4b"}, {"id": "u023", "kind": "table-row", "locator": "body:L62-L62", "preview": "| Stage | Combined change | Reported latency | | CUDA optimization 2 | Remove shared B, per-thread tiles, `float4` loads, hardware intrinsics | about 39 \u00b5s |", "sha256": "3ccddb9db69afdec4a4b5e80e9279ce9e056ea7e8650ec0fc13b76260a91c8a3"}, {"id": "u024", "kind": "table-row", "locator": "body:L63-L63", "preview": "| Stage | Combined change | Reported latency | | CUDA optimization 3 | Vectorized PTX FP4 and scale decode | about 27 \u00b5s |", "sha256": "56cbb91dc0717675c0b075e8dc705e187892f107bb0bdbd355c5f7523fa1fb50"}, {"id": "u025", "kind": "table-row", "locator": "body:L64-L64", "preview": "| Stage | Combined change | Reported latency | | Parameter tuning | Threads per row and rows per block | about 26 \u00b5s |", "sha256": "20ec181854506636133e200346d437e30e574a0b0c4cf11fe5b13a2b148e2dde"}, {"id": "u026", "kind": "table-row", "locator": "body:L65-L65", "preview": "| Stage | Combined change | Reported latency | | ILP | Two tiles per loop iteration | about 22.9 \u00b5s |", "sha256": "65bd979b07f3eb7e869fa06be77e1a2ca90c448ece9242fb34e69473313aa428"}, {"id": "u027", "kind": "table-row", "locator": "body:L66-L66", "preview": "| Stage | Combined change | Reported latency | | Aggressive PTX fusion | Decode, scales, multiply, and accumulation in a larger PTX block | about 22.3 \u00b5s |", "sha256": "b88905c939fdf1e9c68c504abe3f1270ee932c5ff8c8adc0a20f9b4be6559dec"}, {"id": "u028", "kind": "prose", "locator": "body:L68-L68", "preview": "The submitted public-leaderboard score was 22.392 microseconds, the geometric mean over three benchmark shapes. The pinned task's theoretical model gives 8.622, 17.275, and 4.317 microseconds for those separate shapes; 8.622 microseconds is", "sha256": "59f3301f0b00e666daa6bccf8f7b771f1e6c77e3c16f6af8d5c0fb429fecef87"}, {"id": "u029", "kind": "prose", "locator": "body:L70-L70", "preview": "Amandeep Singh's attempts supply useful negative controls for the same task. Replacing two `uchar4` loads with one `uint2` load was reported 16\u201325% slower because extraction added instructions. Lowering `-maxrregcount` from 80 to 64 had no ", "sha256": "4f5cf4a108e0b33babb146932c0781c5c7e48b7b71672d1aee225a6f5226c9c6"}, {"id": "u030", "kind": "prose", "locator": "body:L72-L72", "preview": "Together, the reports justify testing vector width, decode organization, cache hints, and register limits. They do not establish that the widest load, a particular cache hint, or the lowest register cap is essential for GEMV, sub-byte arith", "sha256": "ef8f3594f82b50827d6fbc4ad359d74a683c0fee16b9c701e09124fd010fb6a4"}], "confidence_claimed": "verified", "headings": ["What a wider load establishes", "Cache controls are hints", "Reproducible selection procedure", "NVFP4 GEMV case study"], "id": "technique-vectorized-loads", "path": "wiki/techniques/vectorized-loads.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/docs/nvidia-cuda-register-controls.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html"}, {"path": "sources/blogs/yue-nvfp4-hackathon.md", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}, {"path": "sources/blogs/amandeep-nvfp4-attempts.md", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}, {"path": "sources/contests/gpu-mode-nvfp4/problem-1-gemv.md", "url": "https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-ptx-isa-sm100", "doc-cuda-register-controls", "blog-yue-nvfp4", "blog-amandeep-nvfp4", "contest-gpumode-p1"], "title": "Vectorized Loads and Cache Hints", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "9cff7bf68b39a3b6cef5c930d99bd525e7e3f232e933f44bee877722f8ecaa6c", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "Warp specialization assigns different long-lived functions to disjoint warps in a CTA\u2014for example scheduler, TMA load, MMA control, softmax/correction, or epilogue work. It is a software organization for overlapping pipeline stages, not a f", "sha256": "3ba8e2f12c87ebb0e5dcc7c4505180f38dc8ff8beac7d1d793ab842d17117595"}, {"id": "u002", "kind": "prose", "locator": "body:L5-L5", "preview": "The architecture-level comparison is narrower:", "sha256": "f5fce66b629d64368a8ad2f3262804a0414fb44cf8e2cad0753ec4c379769a0d"}, {"id": "u003", "kind": "table-row", "locator": "body:L9-L9", "preview": "| Property | Hopper `wgmma.mma_async` | Blackwell `tcgen05.mma` | | Issue granularity | Warpgroup collective | One thread for `cta_group::1` or `cta_group::2` |", "sha256": "78012326dd8c87c08b14223c51cb0caca44f22f93153cad66a4f9db57ff7f46c"}, {"id": "u004", "kind": "table-row", "locator": "body:L10-L10", "preview": "| Property | Hopper `wgmma.mma_async` | Blackwell `tcgen05.mma` | | D accumulator | Per-thread registers | TMEM |", "sha256": "9d8f6c6479d0022158169c0c47e8d10179521df1707c1a368c754cc5a2a3a884"}, {"id": "u005", "kind": "table-row", "locator": "body:L11-L11", "preview": "| Property | Hopper `wgmma.mma_async` | Blackwell `tcgen05.mma` | | A/B movement | Explicit register/SMEM operands and producer work | Explicit SMEM descriptors or TMEM addresses and producer work |", "sha256": "562e8662604420b24685b52bbc320c26ac8d2af69a511ce0e6e24f94d573f776"}, {"id": "u006", "kind": "table-row", "locator": "body:L12-L12", "preview": "| Property | Hopper `wgmma.mma_async` | Blackwell `tcgen05.mma` | | Completion | WGMMA commit/wait groups | `tcgen05.commit` tied to an mbarrier, followed by a wait |", "sha256": "9adf92e37d53097b532873a08e3c9c1fb6d611b608108674deb1ff1e73395fe6"}, {"id": "u007", "kind": "prose", "locator": "body:L14-L14", "preview": "Single-thread issue reduces the number of threads needed to submit MMA instructions. It does not make operands move automatically, turn an entire warp into an ISA-level role, complete MMA synchronously, or choose how many epilogue warps a k", "sha256": "04a758f12de17c65173d342cb8f24e546f345fb1e1d1e7807da8597afc4c9033"}, {"id": "u008", "kind": "prose", "locator": "body:L18-L18", "preview": "There is no universal \u201cwarp 0 load, warp 1 MMA, warps 2\u201315 epilogue\u201d rule. Three primary implementations illustrate the range:", "sha256": "85be6e7da4f7a119a0268d227c104475692b8b379b0e80a19e54c5ecadf8d6d0"}, {"id": "u009", "kind": "table-row", "locator": "body:L22-L22", "preview": "| Implementation | Total warps and roles | | Gau Nernst tutorial v4, commit `3b90ac9b...` | 4 warps. An elected lane in warp 0 runs the TMA loop; an elected lane in warp 1 runs the MMA loop; after completion, all four warps participate in T", "sha256": "04f9a434f6d86ab29394b02159dd40778cf936ab3da64f4c13afea393bf664ee"}, {"id": "u010", "kind": "table-row", "locator": "body:L23-L23", "preview": "| Implementation | Total warps and roles | | CUTLASS 4.5.0 generic SM100 GEMM | One warp each for `MMA`, `Sched`, `MainloopLoad`, and `EpilogueLoad`, followed by `CollectiveEpilogue::ThreadCount / 32` epilogue warps. Optional work can make ", "sha256": "cdd3cb833878e188948d9fa2d121d4b3ac4deb77d7309f891e965cff438b590c"}, {"id": "u011", "kind": "table-row", "locator": "body:L24-L24", "preview": "| Implementation | Total warps and roles | | FlashInfer PR 1039 context FMHA | 16 warps: 0\u20133 `Softmax0`, 4\u20137 `Softmax1`, 8\u201311 correction, 12 MMA, 13 load, 14 epilogue, and 15 empty. The kernel has explicit pipelines between these roles. |", "sha256": "a1adbbd5ec3b0d9716988bb7715ca8f100d59539d41bb5bfa1bb704f18062ac2"}, {"id": "u012", "kind": "prose", "locator": "body:L26-L26", "preview": "FlashAttention-4's pinned SM100 forward source likewise starts from a 16-warp layout with two four-warp softmax groups, four correction warps, and dedicated MMA/load/epilogue IDs, then adjusts roles for configuration choices such as one Q s", "sha256": "85453f8d16b28425bdf3fabef09e21c9aa6f172224da9679fd66630af21a45ae"}, {"id": "u013", "kind": "prose", "locator": "body:L30-L30", "preview": "A warp-specialized implementation must prove each ownership handoff. For a reusable TMA-to-MMA stage:", "sha256": "901977eb7283ec2dcce8362743f2200d3162ca12f2f28ca013243cee361015f0"}, {"id": "u014", "kind": "list-item", "locator": "body:L32-L32", "preview": "1. Initialize the mbarrier objects with correct participant/transaction counts and publish them to the required threads and async proxies before use.", "sha256": "9b61910e09ca8df51a88598997a7a9ccf36c7e3a690f01531a4933e884dcbe37"}, {"id": "u015", "kind": "list-item", "locator": "body:L33-L33", "preview": "2. The producer acquires an empty stage, sets expected transaction bytes, and submits the TMA copies. The full/data-ready phase completes only after the expected async transactions complete.", "sha256": "6480a07f2fe51b271cadf66a732a388767b3bc00f4b7b5d69ae5081150fc2337"}, {"id": "u016", "kind": "list-item", "locator": "body:L34-L34", "preview": "3. The MMA controller waits for the full phase and applies the required cross-thread/proxy fence before issuing `tcgen05.mma` against that SMEM stage.", "sha256": "8c289184a687c5b6dc0a191ff7f7c890df85100e6e37b09bb7251409b2af34df"}, {"id": "u017", "kind": "list-item", "locator": "body:L35-L35", "preview": "4. Because MMA is asynchronous, `__syncwarp()` or an ordinary `mbarrier.arrive` after issue does not make the operands reusable. Commit prior tcgen05 work to an mbarrier and wait for completion before releasing or overwriting the stage.", "sha256": "e95047726737de8cacee65eccb16e9c33b29eb85a3f60bd462536178e6ccf539"}, {"id": "u018", "kind": "list-item", "locator": "body:L36-L36", "preview": "5. Before another role reads D from TMEM, complete the MMA sequence and apply the required `tcgen05.fence::before_thread_sync` / execution-ordering handoff / `tcgen05.fence::after_thread_sync` protocol. A fence orders operations; it is not ", "sha256": "c144a17005a0a78e9914bbf04f8e669ae34d5edc880fd085e95eb70a070df51c"}, {"id": "u019", "kind": "list-item", "locator": "body:L37-L37", "preview": "6. Keep TMEM allocated until every consumer finishes its `tcgen05.ld` sequence and associated waits, then deallocate with the required collective participation.", "sha256": "0c51e8e3c826e6843d2f29ea4dadff3c0aec26d0e2e3f62b60976ff405bc3857"}, {"id": "u020", "kind": "prose", "locator": "body:L39-L39", "preview": "Pipeline wrappers in CUTLASS/CuTe encode parts of these rules, but participant counts, initial phases, tails, and producer/consumer states remain configuration-specific. A short role-dispatch sketch is not a substitute for the complete pipe", "sha256": "89b2f283a5df3d8a1f56bf6af22a431d104476439e941498a2c99702463b4198"}, {"id": "u021", "kind": "prose", "locator": "body:L43-L43", "preview": "For the author's exact `M=N=K=4096` Modal B200 setup with PyTorch 2.9.1 and CUDA 13, tutorial v3 reports 939.61 TFLOP/s and v4 reports 1208.83 TFLOP/s after introducing warp specialization. That is a 269.22-TFLOP/s, approximately 28.65% ste", "sha256": "6621d3a76f8c031fec1925cda99d56cfdf593b282737d84897dd3277eeb93451"}, {"id": "u022", "kind": "prose", "locator": "body:L47-L47", "preview": "Use it as a candidate when independent pipeline roles have enough steady-state work to overlap and their register needs differ materially. Compare it with a temporally pipelined version at the same tile shape, stage count, CTA-group mode, a", "sha256": "1627600031e403cc9fddb951b81bf297d7caa64fb89255c37db446929c69e78b"}, {"id": "u023", "kind": "list-item", "locator": "body:L49-L49", "preview": "- time and achieved throughput across representative shapes;", "sha256": "8932c9b5bb7697e947fe26437053390822855b8d9afe20b6084fb4aa9ab88b26"}, {"id": "u024", "kind": "list-item", "locator": "body:L50-L50", "preview": "- per-role active/stall time and pipeline backpressure;", "sha256": "366a32c28b89526070b467e74080a4bf9f037e0f770c23f0a439da3094e25b40"}, {"id": "u025", "kind": "list-item", "locator": "body:L51-L51", "preview": "- register allocation, spills, shared memory, and occupancy;", "sha256": "e6d4d8f371f22ff49a3f1b804e0e88ce3d32209249dc5d9b04d60e0793d7cc8e"}, {"id": "u026", "kind": "list-item", "locator": "body:L52-L52", "preview": "- TMA/MMA/epilogue balance, including prologue and tail cost;", "sha256": "a94aa0efb4d9bedf92e4e7496eaef4e479680d6899227cdd262b800f28a4a47e"}, {"id": "u027", "kind": "list-item", "locator": "body:L53-L53", "preview": "- 1-SM versus 2-SM mode as an independent legal-shape/resource choice.", "sha256": "eecde6ff7378da0ec52ac7970d16eca0580901fa5268e91d4e8dac14c05f5c0f"}, {"id": "u028", "kind": "prose", "locator": "body:L55-L55", "preview": "More role warps are not automatically useful for a more complex epilogue, and one MMA-control warp does not limit the issuer to one outstanding asynchronous MMA operation. Choose participant counts from the concrete collective and measureme", "sha256": "1f5622ad117b53bfcf321b00eccfc035da8b2468ad897d4b62c3e0c2ffd988a7"}, {"id": "u029", "kind": "prose", "locator": "body:L59-L59", "preview": "[`artifacts/kernels/warp-specialization/full/`](../../artifacts/kernels/warp-specialization/full/) contains the captured FlashInfer PR 1039 mainloop file; its bytes match the SHA-256 recorded in `PROVENANCE.yaml` and the duplicate captured ", "sha256": "2f40dc82464551aafd8efd85eb3d27958f6b2220bb352611772e73c0998ec3c0"}, {"id": "u030", "kind": "prose", "locator": "body:L61-L61", "preview": "[`artifacts/kernels/warp-specialization/variants/`](../../artifacts/kernels/warp-specialization/variants/) is explicitly `derived` teaching material and must not be cited as upstream code or as a complete safe kernel.", "sha256": "9b8752deff98b9437a0ad5787db5b0941b2c657c3ca2c5ca0a4a02cab448e48e"}, {"id": "u031", "kind": "prose", "locator": "body:L63-L63", "preview": "Retrieve the page plus both artifact modes with:", "sha256": "aa7fe4f8fefb6af22e88bcb2774f2b7880991739ab30daee394e2957a5c2b167"}, {"id": "u032", "kind": "code", "locator": "body:L65-L67", "preview": "```bash conda run -n base python scripts/get_page.py technique-warp-specialization --include-code ```", "sha256": "5304a268eeb5c50c0c72b18eb3a7eeea5bb8b36817dd99d35deb384c5254f4bf"}], "confidence_claimed": "source-reported", "headings": ["Definition and ISA Boundary", "Version-Pinned Role Maps", "Correct Synchronization Obligations", "Source-Reported Tutorial Result", "When to Test Warp Specialization", "Local Evidence Bundle"], "id": "technique-warp-specialization", "path": "wiki/techniques/warp-specialization.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/docs/nvidia-cutlass-blackwell.md", "url": "https://github.com/NVIDIA/cutlass/tree/v4.5.0"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/prs/flashinfer/PR-1039.md", "revision": "9a05c92a", "url": "https://github.com/flashinfer-ai/flashinfer/pull/1039"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-ptx-isa-sm100", "doc-cutlass-blackwell", "blog-tcgen05-tutorial", "pr-flashinfer-1039"], "title": "Warp Specialization on Blackwell", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} diff --git a/verification/original-report.jsonl b/verification/original-report.jsonl new file mode 100644 index 000000000..7f8b94300 --- /dev/null +++ b/verification/original-report.jsonl @@ -0,0 +1,52 @@ +{"claims": [{"claim": "Blackwell cta_group::2 always doubles a 1-CTA MMA into m256xn256xk16 within a TPC.", "claim_id": "hw-2sm-cooperative-original-c001", "claim_type": "semantics", "covers": ["u001"], "evidence": [{"locator": "CTA Pair, instruction descriptor, and data-path layouts including M=128 + cta_group::2", "observation": "cta_group::2 means a CTA-pair operation touching both CTAs' TMEM. Shape remains descriptor-selected; the ISA explicitly includes a cta_group::2 M=128 layout, so fixed doubling to one m256 shape is false.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-data-path-layout-organization"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The diagram accurately defines TPC as 'Two Processing Clusters' and fixes A by row halves and TMEM by column halves across the CTA pair.", "claim_id": "hw-2sm-cooperative-original-c002", "claim_type": "code", "covers": ["u002"], "evidence": [{"locator": "CTA Pair and data-path Layout A-F figures", "observation": "The ISA defines a CTA pair by cluster-rank parity and provides multiple data-path layouts. It does not define the diagram's universal row/column partition, and TPC is not expanded as 'Two Processing Clusters.'", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-data-path-layout-organization"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The displayed cta_group::2 f16 PTX is valid instruction syntax.", "claim_id": "hw-2sm-cooperative-original-c003", "claim_type": "code", "covers": ["u003"], "evidence": [{"locator": "9.7.16.10.9.1 unscaled floating-point syntax", "observation": "The grammar takes one idesc, an eight-register disable-output-lane vector for cta_group::2, enable-input-d, and optionally scale-input-d. The wiki uses two invented descriptor operands plus ellipses and omits the required vector and predicate.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Both CTAs must each issue the same MMA instruction collectively.", "check": "Distinguish the Table 46 mma row from the alloc/dealloc row.", "outcome": "MMA uses one thread from the pair; only allocation-management instructions require one warp from each CTA."}, "claim": "A cta_group::2 MMA is initiated by one thread from a CTA pair; the peer CTA must be active, and the operation touches the Tensor Memory of the current and peer CTAs.", "claim_id": "hw-2sm-cooperative-original-c004", "claim_type": "semantics", "covers": ["u001", "u002", "u006"], "evidence": [{"locator": "9.7.16.5 Table 46 and 9.7.16.5.1 CTA Pair", "observation": "Table 46 assigns one-thread issue for cta_group::2 MMA and requires an active peer; the CTA-pair section states that pair-granularity operations access both CTAs' TMEM.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-issue-granularity"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "Every cta_group::2 MMA requires identical SMEM layouts, a shared::cluster mbarrier exchanged between the CTAs, and exactly half of M from each CTA.", "claim_id": "hw-2sm-cooperative-original-c005", "claim_type": "semantics", "covers": ["u004", "u005", "u007"], "evidence": [{"locator": "operand access, data-path layouts, and tcgen05.commit", "observation": "The normative requirements are CTA-pair membership, peer lifetime, legal descriptors/layouts, consistent cta_group, and the applicable completion/ordering protocol. The ISA exposes multiple layouts including M=128 group-2 and does not state the three listed rules as universal MMA preconditions.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The percentages use peak hardware FLOP/s or compare different benchmark environments.", "check": "Recompute both shares from the article's same-table cuBLAS reference and the same disclosed setup.", "outcome": "The rounded percentages and relative uplift follow from one scoped table."}, "claim": "For the linked B200 M=N=K=4096 tutorial, moving from v4 warp specialization to v5 2-SM MMA raised throughput from about 80% to 86% of cuBLAS, an approximately 7.7% relative increase.", "claim_id": "hw-2sm-cooperative-original-c006", "claim_type": "performance", "covers": ["u008", "u009", "u010"], "evidence": [{"command": "conda run -n base python -c 'v4=1208.83; v5=1302.29; c=1506.74; print(100*v4/c,100*v5/c,100*(v5/v4-1),100*(v4/v4-1))'", "environment": "Conda base, Python 3.13.14; inputs transcribed from the article's final B200 table for M=N=K=4096, PyTorch 2.9.1 + CUDA 13", "negative_control": "Comparing v4 with itself gives 0% uplift, distinguishing the calculation from a constant offset.", "reference": "https://gau-nernst.github.io/tcgen05/ closing table: cuBLAS 1506.74, v4 1208.83, v5 1302.29 TFLOP/s", "result": "v4/cuBLAS=80.228%, v5/cuBLAS=86.431%, relative uplift=7.731%, consistent with the wiki's rounded 80%, 86%, and ~7.5%."}], "oracle": "host-experiment", "verdict": "verified"}, {"claim": "2-SM MMA should be used whenever M is at least 256, whenever a kernel is compute-bound, and with persistence to guarantee maximum throughput.", "claim_id": "hw-2sm-cooperative-original-c007", "claim_type": "performance", "covers": ["u011", "u012", "u013"], "evidence": [{"locator": "2-SM MMA and persistent-kernel sections", "observation": "The author reports a modest, workload-specific 2-SM gain and calls it disappointing; the next static-persistent change addresses observed epilogue/setup idle time. Neither the article nor PTX provides the wiki's universal M threshold or maximum-throughput guarantee.", "revision": "Article dated 2025-12-21; linked code commit 3b90ac9b", "url": "https://gau-nernst.github.io/tcgen05/"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The relative links escape the hardware directory or differ by case on a case-sensitive filesystem.", "check": "Resolve each exact target relative to the page and compare a broken sibling control.", "outcome": "Both exact case-sensitive paths exist in the expected directory."}, "claim": "The two Related links resolve to the tcgen05.mma and TMEM hardware pages.", "claim_id": "hw-2sm-cooperative-original-c008", "claim_type": "other", "covers": ["u014", "u015"], "evidence": [{"command": "for p in wiki/hardware/tcgen05-mma.md wiki/hardware/tmem.md wiki/hardware/does-not-exist.md; do test -f \"$p\" && echo \"$p present\" || echo \"$p missing\"; done", "environment": "Repository baseline/current workspace on 2026-08-08", "negative_control": "A deliberately nonexistent sibling target reports missing.", "reference": "Markdown targets tcgen05-mma.md and tmem.md resolved relative to wiki/hardware/", "result": "Both intended targets are present."}], "oracle": "host-experiment", "verdict": "verified"}], "id": "hw-2sm-cooperative", "page_verdict": "refuted", "path": "wiki/hardware/2sm-cooperative.md", "residual_risk": "The tutorial performance result was not rerun on B200. The report verifies it only as a source-reported measurement under the author's disclosed setup, not as a general recommendation.", "scope": "All 15 immutable baseline units were checked against CUDA 13.0.2 PTX ISA 9.0, the tutorial's scoped B200 table, and local target resolution. No GPU experiment was available."} +{"claims": [{"challenge": {"alternative": "CLC is merely a CUTLASS software scheduler rather than a Blackwell hardware feature.", "check": "CUDA Programming Guide 13.2 section 4.12 identifies the architectural feature independently of CUTLASS.", "outcome": "The vendor programming guide assigns the feature to Blackwell compute capability 10.0."}, "claim": "CLC is a Blackwell compute-capability 10.0 mechanism that enables dynamic work stealing for persistent workers.", "claim_id": "hw-clc-original-c001", "claim_type": "mechanism", "covers": ["u001"], "evidence": [{"locator": "4.12 Work Stealing with Cluster Launch Control, introduction", "observation": "The guide identifies CLC as a Blackwell compute-capability 10.0 feature and describes its work-stealing role.", "revision": "CUDA Programming Guide 13.2", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Workers receive even their first tile exclusively from a CLC queue.", "check": "Inspect rule 4 and the dynamic-persistent pseudocode in the pinned CUTLASS documentation.", "outcome": "The first work coordinate is the preloaded blockIdx, so the original absolute wording is false."}, "claim": "CLC replaces the static CUDA grid model and removes fixed CTA-to-initial-tile assignments.", "claim_id": "hw-clc-original-c002", "claim_type": "semantics", "covers": ["u001", "u003"], "evidence": [{"locator": "Dynamic Scheduler with Cluster Launch Control, rules 1-4", "observation": "CUTLASS launches a full grid with one ClcID per output tile; each worker uses its own blockIdx as its first tile and only queries subsequent work.", "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "CLC only cancels work and never returns the canceled work identifier to the requester.", "check": "CUTLASS rules 2, 4, and 5 describe the query response and subsequent tile processing.", "outcome": "A successful request returns a ClcID that the existing worker processes."}, "claim": "After its initial tile, a persistent worker can query another ClcID, which supports dynamic load balancing when SM availability is uneven.", "claim_id": "hw-clc-original-c003", "claim_type": "mechanism", "covers": ["u002", "u003"], "evidence": [{"locator": "Dynamic Scheduler with Cluster Launch Control, lines 102-115", "observation": "The documentation describes workers querying subsequent ClcIDs and motivates CLC with uneven real-time SM availability.", "revision": "CUTLASS 4.3.3", "url": "https://docs.nvidia.com/cutlass/4.3.3/media/docs/cpp/blackwell_cluster_launch_control.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "CLC can keep all SMs busy even when fewer unprocessed ClcIDs than SMs remain.", "check": "Compare the documented full-grid/cancellation model with NVIDIA's wave-quantization definition.", "outcome": "CLC can redistribute pending IDs among existing workers but cannot manufacture additional independent tiles."}, "claim": "CLC guarantees that an ordinary equal-cost GEMM tail is eliminated because idle CTAs always take the remaining tiles.", "claim_id": "hw-clc-original-c004", "claim_type": "performance", "covers": ["u004", "u006"], "evidence": [{"locator": "4.12 introduction and Cluster Launch Control flow", "observation": "CLC still launches the problem-sized grid and steals only not-yet-started blocks; a request may also decline for reasons other than end-of-grid. It does not create more parallel work than the grid contains or guarantee full occupancy.", "revision": "CUDA Programming Guide 13.2", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"}, {"locator": "3.2 Wave Quantization", "observation": "Wave occupancy depends on tile count, tile shape, and SM count; an underfilled last wave is a quantization property, not a universally eliminated condition.", "revision": "NVIDIA Matrix Multiplication Background User's Guide", "url": "https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The request accepts a desired tile coordinate and discards that tile's work.", "check": "Inspect the request operands and success path in the official API example.", "outcome": "The request supplies result and barrier addresses; the canceled coordinate is an output decoded after success and must be processed by the thief."}, "claim": "try_cancel cancels application-selected output tiles that no longer need to be computed, such as rejected speculative tokens, masked attention tiles, or finished sequences.", "claim_id": "hw-clc-original-c005", "claim_type": "semantics", "covers": ["u005", "u010", "u012", "u013", "u014", "u015"], "evidence": [{"locator": "4.12, Cluster Launch Control flow and 4.12.1 API Details", "observation": "A requester atomically cancels an unspecified not-yet-started block or cluster, obtains its index, and performs that work itself. The operation is work stealing, not semantic work deletion or cancellation of a caller-selected tile.", "revision": "CUDA Programming Guide 13.2", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The second wave is full or has a different number of idle SMs.", "check": "Compute quotient, remainder, and idle fraction directly and compare a deliberately full-tail control.", "outcome": "The exact arithmetic yields 124 active and 8 idle SMs under the stated assumptions."}, "claim": "For 256 equal-sized one-CTA tiles on 132 SMs at one CTA per SM, the second wave has 124 active and 8 idle SMs, or about 6% of the SMs idle.", "claim_id": "hw-clc-original-c006", "claim_type": "code", "covers": ["u006"], "evidence": [{"command": "conda run -n base python -c 'import math; tiles=256; sms=132; waves=math.ceil(tiles/sms); tail=tiles-(waves-1)*sms; idle=sms-tail; print({\"waves\":waves,\"tail_active\":tail,\"tail_idle\":idle,\"idle_percent\":round(100*idle/sms,2),\"negative_control_idle_if_full_tail\":sms-sms})'", "environment": "Conda base, Python 3.13.14, CPU-only arithmetic model; assumptions are exactly one resident CTA per SM and equal tile duration.", "negative_control": "A deliberately full second wave reports 0 idle SMs, distinguishing the remainder calculation.", "reference": "ceil(256/132)=2 waves; the remainder is 124 and 132-124=8.", "result": "waves=2, tail_active=124, tail_idle=8, idle_percent=6.06."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "A CLC kernel is launched with only one persistent CTA per SM and the first tile is returned by try_cancel.", "check": "Compare the diagram to pinned rules 1-6 and the dynamic-persistent pseudocode.", "outcome": "Both its grid size and its initial-work order contradict the documented model."}, "claim": "The CLC diagram's 132-CTA launch, request-before-first-work sequence, and guarantee that all CTAs remain busy until a tile queue is empty describe the documented execution model.", "claim_id": "hw-clc-original-c007", "claim_type": "code", "covers": ["u007"], "evidence": [{"locator": "Dynamic Scheduler with Cluster Launch Control, rules 1-6 and pseudocode", "observation": "The documented grid has as many blocks as output tiles, each worker processes blockIdx first, and a CLC request may decline. The wiki diagram instead launches only an SM-count grid and requests the first tile.", "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "A separate clusterlaunchcontrol.try_acquire instruction performs the work query.", "check": "Inspect the adjacent, exhaustive CLC instruction sections in PTX ISA 9.0.", "outcome": "Only try_cancel and query_cancel are defined for CLC."}, "claim": "clusterlaunchcontrol.try_cancel requests cancellation of a not-yet-launched ClcID that the requester can take over; PTX exposes try_cancel and query_cancel rather than try_acquire.", "claim_id": "hw-clc-original-c008", "claim_type": "semantics", "covers": ["u008"], "evidence": [{"locator": "9.7.13.17 clusterlaunchcontrol.try_cancel and 9.7.13.18 clusterlaunchcontrol.query_cancel", "observation": "The PTX specification defines try_cancel for a cluster not yet launched and query_cancel for decoding; the instruction inventory contains no try_acquire form.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The short inline PTX is an accepted register-returning form omitted from the main syntax.", "check": "Compare every operand and completion mechanism against the normative PTX syntax and official CUDA example.", "outcome": "The claimed form lacks mandatory address, mbarrier, completion, b128, and decode operations."}, "claim": "The persistent_gemm_clc CUDA block demonstrates a valid CLC request/decode loop.", "claim_id": "hw-clc-original-c009", "claim_type": "code", "covers": ["u009"], "evidence": [{"locator": "9.7.13.17 syntax and example; 9.7.13.18 query_cancel syntax", "observation": "The instruction asynchronously writes a b128 opaque response to shared memory, completes an mbarrier transaction, and requires query_cancel to decode success and coordinates. The wiki block uses register outputs and a direct predicate, has no mbarrier completion, and omits initial blockIdx work and required proxy ordering.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel"}, {"locator": "4.12.1.1 steps 1-6", "observation": "The official sequence initializes a shared mbarrier, submits the async request, sets a 16-byte transaction count, waits, decodes success, conditionally decodes the ID, and fences proxy access.", "revision": "CUDA Programming Guide 13.2", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The pointer operand denotes the desired tile coordinate and the first output is a bool.", "check": "Inspect the normative operand descriptions and query_cancel decoder.", "outcome": "The address is the response destination; the canceled coordinate is returned only inside the opaque response."}, "claim": "clc_try_cancel(uint2) cancels a caller-selected tile and returns cancellation success directly from the PTX request.", "claim_id": "hw-clc-original-c010", "claim_type": "code", "covers": ["u011"], "evidence": [{"locator": "9.7.13.17 try_cancel syntax and 9.7.13.18 query_cancel syntax", "observation": "The request takes shared response and mbarrier addresses, not an input tile coordinate or boolean destination. Success is decoded later from the 16-byte opaque response.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "PersistentScheduler maps to the older SM90 static scheduler on SM100.", "check": "Inspect the architecture-specific specialization at the pinned v4.5.0 revision.", "outcome": "The arch::Sm100 specialization names PersistentTileSchedulerSm100."}, "claim": "CUTLASS 4.5.0 supports CLC-backed SM100 scheduling through the PersistentScheduler tag and PersistentTileSchedulerSm100 implementation.", "claim_id": "hw-clc-original-c011", "claim_type": "compatibility", "covers": ["u016"], "evidence": [{"locator": "PersistentScheduler tag and TileSchedulerSelector", "observation": "The selector maps the PersistentScheduler tag for arch::Sm100 to PersistentTileSchedulerSm100.", "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/tile_scheduler.hpp"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "CUTLASS 4.5.0 retains a compatible legacy overload for exactly the shown SM100 form.", "check": "Compare the complete pinned official SM100 dynamic-persistent example and scheduler position.", "outcome": "The authoritative example uses the 3.x collective API and a different kernel template structure."}, "claim": "The GemmUniversal typedef shown is a valid CUTLASS 4.5.0 SM100 CLC kernel instantiation.", "claim_id": "hw-clc-original-c012", "claim_type": "code", "covers": ["u017"], "evidence": [{"locator": "dynamic_kernel_config", "observation": "The SM100 CLC example builds collective mainloop and epilogue types, uses cute::Shape tile/cluster shapes, sets DynamicPersistentScheduler, and instantiates cutlass::gemm::kernel::GemmUniversal with four template parameters. The wiki uses an incompatible legacy device::GemmUniversal signature and places PersistentScheduler as a trailing argument.", "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/95_blackwell_gemm_green_context/95_blackwell_gemm_green_context.cu"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The pipeline is only a latency optimization and synchronous direct coordinate return is semantically valid.", "check": "Compare the PTX opaque-response completion model and the pinned CUTLASS implementation.", "outcome": "Async shared-memory completion and explicit response decoding are part of the instruction semantics, not optional scheduling decoration."}, "claim": "The simplified ClcTileScheduler block accurately models a synchronous coordinate-returning CLC query.", "claim_id": "hw-clc-original-c013", "claim_type": "code", "covers": ["u018"], "evidence": [{"locator": "PersistentTileSchedulerSm100::advance_to_next_work and fetch_next_work", "observation": "CUTLASS issues an async query through PipelineCLCFetchAsync, waits as a consumer, decodes a staged opaque response, releases the stage, and then swizzles/rasterizes. The wiki's synchronous clc_try_cancel call omits the required pipeline and response semantics.", "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The table's percentages are invariant properties of GEMM dimension alone.", "check": "Hold the 2048x2048 output constant while varying only documented scope variables: tile shape and SM count.", "outcome": "The capacity changes materially, so unscoped exact utilization and improvement values are not valid factual measurements."}, "claim": "The three-row table gives generally valid exact SM-utilization values for 2048, 4096, and 8192 square GEMMs under static and CLC scheduling.", "claim_id": "hw-clc-original-c014", "claim_type": "performance", "covers": ["u019", "u020", "u021", "u022"], "evidence": [{"command": "conda run -n base python -c 'import math; cases=[(\"B200_128x256\",148,128,256),(\"B200_256x256\",148,256,256),(\"132SM_128x256\",132,128,256)]; out=[]\nfor name,sms,tm,tn in cases:\n tiles=math.ceil(2048/tm)*math.ceil(2048/tn); out.append((name,tiles,round(100*min(tiles,sms)/sms,2)))\nprint({\"same_2048x2048_different_scope\":out,\"negative_control_assume_invariant\":len({x[2] for x in out})==1})'", "environment": "Conda base, Python 3.13.14, CPU-only tile-count upper-bound model; no GPU performance inference beyond active-SM capacity.", "negative_control": "The deliberately false invariant-scope control evaluates to false because the three scoped capacities differ.", "reference": "NVIDIA's wave-quantization model makes active-SM capacity depend on tile shape and SM count; exact utilization additionally requires hardware, dtype, software, occupancy, and timing scope.", "result": "For the same 2048x2048 output, active-SM capacity is 86.49% for 128x256 tiles on 148 SMs, 43.24% for 256x256 tiles on 148 SMs, and 96.97% for 128x256 tiles on 132 SMs."}], "oracle": "host-experiment", "verdict": "refuted"}, {"challenge": {"alternative": "The section title is stale and the final code actually issues CLC instructions.", "check": "Inspect the pinned final algorithm, performance row label, and the closing statement that CLC remains unimplemented.", "outcome": "All three loci identify static scheduling and exclude CLC from the measured kernel."}, "claim": "The tcgen05 for dummies result improved from about 940 to 1476 TFLOPS because the final kernel used persistent scheduling plus CLC.", "claim_id": "hw-clc-original-c015", "claim_type": "performance", "covers": ["u023"], "evidence": [{"locator": "Persistent kernel (with static scheduling), final performance table and closing remarks", "observation": "The author labels 1475.93 TFLOPS as persistent with static scheduling, reports 939.61 TFLOPS for pipelining, and explicitly lists CLC as a feature not added.", "revision": "Article dated 2025-12-21; code pinned at gau-nernst/learn-cuda commit 3b90ac9b", "url": "https://gau-nernst.github.io/tcgen05/"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Only K, not the output M-N footprint, controls parallel work and GPU utilization.", "check": "Inspect the guide's output tiling model and wave-quantization example.", "outcome": "Thread blocks tile the M-N output, and the resulting tile count directly controls wave occupancy."}, "claim": "Small output dimensions can reduce GEMM tile parallelism and expose wave-quantization or under-utilization effects.", "claim_id": "hw-clc-original-c016", "claim_type": "performance", "covers": ["u024"], "evidence": [{"locator": "2.3 Typical Tile Dimensions and 3.2 Wave Quantization", "observation": "NVIDIA documents the tradeoff between tile parallelism and efficiency and shows that an underfilled tile wave reduces SM utilization.", "revision": "NVIDIA Matrix Multiplication Background User's Guide", "url": "https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "CLC turns a 32-block problem into an SM-count-sized persistent grid with additional work IDs.", "check": "Compare the 32-tile arithmetic with the official full-grid and cancellation model.", "outcome": "The grid contains one ID per problem block; CLC redistributes those IDs but does not increase their number."}, "claim": "For the shown M=32, N=K=8192 and 128x256 tile, a B200 has 132 SMs and CLC lets 32 persistent CTAs handle the tiles while avoiding the static scheduler's idle-SM condition.", "claim_id": "hw-clc-original-c017", "claim_type": "code", "covers": ["u025"], "evidence": [{"locator": "Table 8 platform specifications", "observation": "The NVIDIA B200 configuration has 148 SMs, not 132.", "revision": "NVIDIA technical blog, Table 8, 2026", "url": "https://developer.nvidia.com/blog/scaling-autonomous-ai-agents-and-workloads-with-nvidia-dgx-spark/"}, {"locator": "4.12 Cluster Launch Control flow", "observation": "CLC steals indices from the already launched problem-sized grid. With only 32 output tiles, it cannot create enough independent ClcIDs to occupy 148 SMs; a request can only cancel one of those 32 not-yet-started blocks.", "revision": "CUDA Programming Guide 13.2", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Each CTA in a cluster independently cancels exactly one unrelated CTA ID.", "check": "Inspect rule 6 and the official cluster programming example's multicast response semantics.", "outcome": "One cluster-level query consumes the matching cluster-sized group and broadcasts the first CTA ID."}, "claim": "CLC cancellation operates at cluster granularity, so a 2x2 worker cluster consumes a 2x2 set of ClcIDs on a successful query.", "claim_id": "hw-clc-original-c018", "claim_type": "semantics", "covers": ["u026"], "evidence": [{"locator": "Dynamic Scheduler with Cluster Launch Control, rule 6", "observation": "The documentation explicitly states cluster granularity and gives the 2x2 worker/2x2 ClcID example.", "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "clc_try_cancel_cluster encapsulates every required barrier, multicast, rank, and proxy operation.", "check": "The helper is undefined and the surrounding code still derives rank from blockIdx.x modulo 2 without the documented returned-ID adjustment or cluster lifetime synchronization.", "outcome": "The visible algorithm cannot establish the required cluster semantics even under that abstraction."}, "claim": "The cooperative_clc_loop code is a valid 2-CTA CLC plus cta_group::2 execution pattern.", "claim_id": "hw-clc-original-c019", "claim_type": "code", "covers": ["u027"], "evidence": [{"locator": "4.12.2.2 Use-case: Thread Block Clusters", "observation": "The cluster form requires a single cluster thread, multicast try_cancel, per-CTA local mbarriers at cluster scope, cluster synchronization to guarantee all blocks exist, and local-rank adjustment of the returned first CTA ID. The wiki code implements none of these requirements.", "revision": "CUDA Programming Guide 13.2", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Only the initial blockIdx is swizzled while CLC-returned IDs bypass the transform.", "check": "Inspect fetch_next_work immediately after work_tile_info_from_clc_response.", "outcome": "The decoded returned tile is explicitly passed to swizzle_and_rasterize."}, "claim": "CUTLASS can apply a software swizzle/rasterization transform to initial and CLC-returned work coordinates.", "claim_id": "hw-clc-original-c020", "claim_type": "mechanism", "covers": ["u028"], "evidence": [{"locator": "PersistentTileSchedulerSm100::fetch_next_work and swizzle_and_rasterize", "observation": "The scheduler decodes a CLC response and passes its coordinates through swizzle_and_rasterize; Arguments exposes max_swizzle_size and raster_order.", "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The repeated coordinates are expected Morton grouping rather than destructive collisions.", "check": "Compare cardinality and exact order against bit-interleaved Morton indices.", "outcome": "A traversal must retain one output per tile; collapsing 16 inputs to 4 outputs cannot be a traversal permutation."}, "claim": "The apply_l2_swizzle code implements a collision-free Morton/Z-order mapping of all input tiles.", "claim_id": "hw-clc-original-c021", "claim_type": "code", "covers": ["u029"], "evidence": [{"command": "conda run -n base python -c 'N=4; bits=2; f=lambda m,n: ((m*N+n>>bits)//N,((m*N+n>>bits)%N)^(((m*N+n>>bits)//N)&((1<>b)&1)<<(2*b)|((m>>b)&1)<<(2*b+1) for b in range(2)); print({\"wiki_outputs\":out,\"unique\":len(set(out)),\"expected_unique\":16,\"claimed_morton_order\":sorted(((morton(m,n),m,n) for m in range(4) for n in range(4))),\"negative_control_identity_unique\":len(set((m,n) for m in range(4) for n in range(4)))})'", "environment": "Conda base, Python 3.13.14; direct integer translation of the wiki function for a 4x4 grid with swizzle_bits=2.", "negative_control": "An identity mapping over the same grid produces all 16 unique outputs, showing the uniqueness check discriminates a valid permutation from the wiki function.", "reference": "A tile permutation, including a Morton order, must map all 16 input coordinates to 16 unique output coordinates.", "result": "The function produces only four unique outputs; every four inputs collide. Its sequence differs from the 16-entry bit-interleaved Morton reference."}], "oracle": "host-experiment", "verdict": "refuted"}, {"challenge": {"alternative": "Hardware CLC has no meaningful request latency or failure behavior under any workload.", "check": "Inspect the CUTLASS latency-hiding pipeline rationale and the CUDA request-decline conditions.", "outcome": "Both query latency and non-end-of-grid declines are documented, refuting the categorical comparison."}, "claim": "CLC scheduling overhead is universally near zero and CLC tail handling is universally optimal compared with a software atomic scheduler.", "claim_id": "hw-clc-original-c022", "claim_type": "performance", "covers": ["u030", "u031"], "evidence": [{"locator": "Static Scheduler and Cluster Launch Control Pipeline Class", "observation": "CUTLASS calls static scheduling zero-overhead and explicitly pipelines CLC queries to hide their latency. This contradicts an unqualified near-zero CLC cost, while no universal optimality guarantee is stated.", "revision": "CUTLASS 4.3.3", "url": "https://docs.nvidia.com/cutlass/4.3.3/media/docs/cpp/blackwell_cluster_launch_control.html"}, {"locator": "4.12 Cluster Launch Control flow", "observation": "CLC requests can decline for end-of-grid or other reasons such as a higher-priority kernel, so the API does not guarantee an optimal tail schedule.", "revision": "CUDA Programming Guide 13.2", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "CUDA atomicAdd itself supplies the same ClcID selection, cancellation, and completion protocol as CLC.", "check": "Compare the normative APIs and operands of atomicAdd and clusterlaunchcontrol.try_cancel.", "outcome": "atomicAdd changes a caller-provided word; CLC interacts with pending grid launches and returns an opaque cancellation response."}, "claim": "CLC exposes try_cancel, while a software work queue must explicitly manage its atomic state and synchronization.", "claim_id": "hw-clc-original-c023", "claim_type": "semantics", "covers": ["u032"], "evidence": [{"locator": "9.7.13.17 clusterlaunchcontrol.try_cancel", "observation": "PTX defines the CLC request directly.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel"}, {"locator": "10.14 Atomic Functions", "observation": "CUDA atomics provide read-modify-write primitives and defined scopes but do not constitute a tile scheduler; queue state, indexing, termination, and ordering remain software responsibilities.", "revision": "CUDA C++ Programming Guide 13.0.2", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#atomic-functions"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The CUTLASS arguments are forwarded to an undocumented CLC hardware launch attribute.", "check": "Trace fetch_next_work from response decode through swizzle_and_rasterize at the pinned revision.", "outcome": "The coordinates are transformed directly in device-side CUTLASS code."}, "claim": "L2 tile swizzling is a CLC hardware policy configured at kernel launch rather than a CUTLASS scheduler coordinate transform.", "claim_id": "hw-clc-original-c024", "claim_type": "mechanism", "covers": ["u033"], "evidence": [{"locator": "Arguments and swizzle_and_rasterize", "observation": "max_swizzle_size and raster_order are CUTLASS scheduler arguments, and the transformation is ordinary scheduler code applied after CLC response decoding.", "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp"}, {"locator": "9.7.13.17 and 9.7.13.18", "observation": "The CLC PTX surface contains request and response operations and has no swizzle-policy operand or initialization instruction.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "CLC is also available on Hopper sm_90, or software atomics first appear only on sm_100.", "check": "Compare the PTX target note with the CUDA atomic availability notes.", "outcome": "CLC starts at sm_100, while suitable integer atomics are supported well before sm_70."}, "claim": "Base CLC try_cancel requires sm_100 or newer, while ordinary software atomic scheduling is available on sm_70 and newer.", "claim_id": "hw-clc-original-c025", "claim_type": "compatibility", "covers": ["u034"], "evidence": [{"locator": "9.7.13.17 Target ISA Notes", "observation": "The base instruction requires sm_100 or higher; the multicast qualifier has narrower architecture-specific targets.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel"}, {"locator": "10.14.1 atomicAdd availability", "observation": "Common 32-bit atomicAdd variants predate sm_70, so an integer-atomic software scheduler is available throughout the stated sm_70+ range.", "revision": "CUDA C++ Programming Guide 13.0.2", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#atomic-functions"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "CUTLASS only exposes raw CLC PTX and leaves the entire scheduler to each application.", "check": "Inspect the complete PersistentTileSchedulerSm100 class at the pinned revision.", "outcome": "The class provides a reusable built-in scheduler rather than only an instruction wrapper."}, "claim": "CUTLASS has a built-in SM100 CLC scheduler implementation; an atomic tile queue is a separate software scheduler design.", "claim_id": "hw-clc-original-c026", "claim_type": "compatibility", "covers": ["u035"], "evidence": [{"locator": "PersistentTileSchedulerSm100", "observation": "CUTLASS implements the CLC pipeline, response decoding, coordinate mapping, and persistent-work interface in its SM100 scheduler.", "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp"}], "oracle": "web-primary", "verdict": "verified"}], "commands": ["conda run -n base python scripts/validate.py", "conda run -n base python /users/Master/.codex/skills/kernel-wiki-verifier/scripts/scan_wiki.py /users/Master/wiki-evolve/KernelWiki --output /users/Master/wiki-evolve/KernelWiki/verification/baseline-queue.jsonl", "nvidia-smi --query-gpu=name,compute_cap,driver_version --format=csv,noheader (not available)", "nvcc --version (not available)", "git ls-remote https://github.com/NVIDIA/cutlass.git refs/tags/v4.5.0", "git clone --depth 1 --branch v4.5.0 --filter=blob:none https://github.com/NVIDIA/cutlass.git /tmp/kernelwiki-cutlass-dRC5i4/cutlass", "conda run -n base python -c ", "conda run -n base python -c ", "conda run -n base python -c "], "id": "hw-clc", "page_verdict": "refuted", "path": "wiki/hardware/clc.md", "residual_risk": "No local NVIDIA GPU or CUDA compiler is installed. No positive CLC speedup is accepted from this page; its unscoped utilization table and CLC-attributed benchmark are refuted rather than replaced with new performance numbers.", "scope": "All 35 baseline coverage units were split into mechanism, semantics, compatibility, code, and performance claims. CLC semantics were checked against CUDA Programming Guide 13.2 and PTX ISA 9.0; CUTLASS behavior and API names were checked at v4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5; arithmetic-only examples were run in Conda base Python 3.13.14."} +{"claims": [{"claim": "mbarrier is a 64-bit shared-memory synchronization object for threads and asynchronous operations, introduced on Hopper and essential to Blackwell warp-specialized kernels.", "claim_id": "hw-mbarrier-original-c001", "claim_type": "compatibility", "covers": ["u001"], "evidence": [{"locator": "mbarrier overview, lifecycle, and ISA/target notes", "observation": "The object is 64-bit shared-memory state for thread/async synchronization, but mbarrier instructions began in PTX 7.0 on sm_80. Hopper added transaction-count support, not mbarrier itself; 'essential' is also not a universal ISA requirement.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The key-operation PTX correctly states that TMA completion performs an arrival with a transferred-byte count and then flips barrier parity.", "claim_id": "hw-mbarrier-original-c002", "claim_type": "code", "covers": ["u002"], "evidence": [{"locator": "arrive-on, expect-tx, complete-tx, and phase-completion definitions", "observation": "arrive.expect_tx performs an arrive-on and increments tx-count. A TMA complete-tx decrements tx-count rather than performing another arrival; phase advances only when both pending arrivals and tx-count reach zero.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#contents-of-the-mbarrier-object"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Parity is toggled by every arrival rather than once per completed phase.", "check": "Compare the arrive-on definition with the atomic phase-transition requirements.", "outcome": "Arrivals only decrement pending count; parity changes with the phase transition after all completion conditions hold."}, "claim": "An mbarrier advances through phases, and parity is the low bit of the phase that a parity wait must track across reuse.", "claim_id": "hw-mbarrier-original-c003", "claim_type": "semantics", "covers": ["u003"], "evidence": [{"locator": "phase lifecycle and parity operand semantics", "observation": "The ISA defines sequential phases, even/odd parity values, automatic reinitialization after completion, and a lifetime tracking requirement for parity waits.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The pipeline pseudocode correctly uses one global phase bit toggled after every iteration for every stage barrier.", "claim_id": "hw-mbarrier-original-c004", "claim_type": "code", "covers": ["u004"], "evidence": [{"locator": "parity lifetime rule and Example 2", "observation": "Parity belongs to each mbarrier object's phase. With multiple stage objects, a stage's expected parity changes only when that same object completes, not on every loop iteration; one globally toggled bit is wrong for typical even stage counts.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The producer must issue mbarrier.complete_tx after the TMA instruction returns.", "check": "Inspect the completion_mechanism definition for cp.async.bulk.tensor.", "outcome": "The async operation itself performs complete-tx on completion, discriminating it from software-simulated complete_tx."}, "claim": "TMA directly signals an mbarrier's transaction completion, eliminating the need for a producer thread to issue a separate completion arrival.", "claim_id": "hw-mbarrier-original-c005", "claim_type": "mechanism", "covers": ["u005"], "evidence": [{"locator": "mbarrier::complete_tx::bytes completion mechanism", "observation": "On copy completion, TMA performs complete-tx for the copied byte count on the supplied mbarrier. The consumer still waits; only manual producer completion signaling is avoided.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The displayed TMA integration code is a valid or reproducible CUDA example and its no-extra-arrive warning accurately describes TMA hardware as performing an arrival.", "claim_id": "hw-mbarrier-original-c006", "claim_type": "code", "covers": ["u006", "u009"], "evidence": [{"locator": "exact cp.async.bulk.tensor grammar and mbarrier completion", "observation": "The page uses undefined helper calls and omits descriptor/coordinate/address details. More importantly, TMA performs complete-tx, not arrive-on; an extra arrive changes pending-arrival accounting rather than racing with a hardware arrival.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The instruction derives the intended generation from the address and ignores a stale parity operand.", "check": "Inspect the explicit phaseParity operand contract.", "outcome": "The operand chooses the phase parity being tested; there is no separate generation tag supplied by the address."}, "claim": "Reusing a barrier stage with the wrong parity can let a wait observe an earlier phase rather than the intended new one.", "claim_id": "hw-mbarrier-original-c007", "claim_type": "semantics", "covers": ["u007"], "evidence": [{"locator": "phaseParity accepted phases and tracking note", "observation": "A parity wait may refer to the current or immediately preceding phase, so software must track each object's phase throughout its lifetime.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A zero tx-count alone completes the phase even when arrivals remain.", "check": "Inspect whether completion conditions are alternatives or conjunctive.", "outcome": "The ISA requires both pending-arrival count and tx-count to be zero."}, "claim": "If an mbarrier is initialized for N pending arrivals and only M", "environment": "KernelWiki worktree; conda base Python 3.13.14", "negative_control": "Changing topk_gather_kv.py to nonexistent_topk_gather.py raises AssertionError.", "reference": "sources/prs/flash-attention/PR-2441.md", "result": "The script exits 0 and both asserted file names are present in the local source page."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "Top-k selection is merely a scheduler flag with no distinct gather implementation.", "check": "Inspect the separate files and imports in the pinned bundle.", "outcome": "flash_fwd_mla_sm100 imports CpasyncGatherKVManager from a dedicated gather module while independently selecting tile-scheduler classes."}, "claim": "Top-k gather and tiled scheduling are separate implementation/evidence paths in PR 2441.", "claim_id": "kernel-flash-attention-sm100-mla-topk-original-c003", "claim_type": "code", "covers": ["u003"], "evidence": [{"locator": "key-file manifest", "observation": "The byte-verified bundle separately captures topk_gather_kv.py, tile_scheduler.py, and the integrating flash_fwd_mla_sm100.py implementation.", "path": "artifacts/prs/flash-attention/PR-2441/PROVENANCE.yaml", "revision": "PR 2441 merge f219c89c886c6ccbf9d3dbd9fe41b11ac64e9df8"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Reducing effective KV length necessarily improves both compute and locality by the same factor.", "check": "Compare FLOP accounting with the indexed gather address construction.", "outcome": "Arithmetic count follows top-k length, but addresses follow arbitrary indices and require a separate gather/bitmask path, so locality does not follow algebraically."}, "claim": "Top-k reduces the attention key extent used for arithmetic while indexed K/V gathering has a distinct memory-access path, so memory traffic/locality and tensor arithmetic should be evaluated separately.", "claim_id": "kernel-flash-attention-sm100-mla-topk-original-c004", "claim_type": "performance", "covers": ["u004", "u005"], "evidence": [{"locator": "benchmark effective seqlen/FLOP accounting and topk_gather_kv.py addition", "observation": "Benchmark arithmetic uses gather_kv_length as effective KV length, while the gather manager loads K/V through user-provided indices and optional bounds bitmasks rather than the contiguous path. These are separable arithmetic and memory mechanisms; no numeric locality gain is asserted.", "path": "artifacts/prs/flash-attention/PR-2441/diff.patch", "revision": "PR 2441 merge f219c89c886c6ccbf9d3dbd9fe41b11ac64e9df8; SHA-256 4525d7f9af76740692474ffc0266fe9cf72f894ce753fd9d18b70771c96b6810"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The merge is a generic sparse-attention primitive transferable unchanged to arbitrary MHA shapes and paging modes.", "check": "Enumerate the constructor/interface assertions and unimplemented-feature checks.", "outcome": "The source encodes exact MQA/head-dimension/gather/page-table constraints, rejecting generic compatibility."}, "claim": "The useful PR-2441 path is shape- and feature-specific and therefore warrants full-workload validation before transfer.", "claim_id": "kernel-flash-attention-sm100-mla-topk-original-c005", "claim_type": "compatibility", "covers": ["u006"], "evidence": [{"locator": "constructor and __call__ assertions", "observation": "The top-k path requires packed GQA/MQA with qhead_per_kvhead=128, cp.async K/V gather, hdim defaults 64/512, constrained top-k length, and no page table; other options dispatch differently or are rejected.", "path": "artifacts/prs/flash-attention/PR-2441/key-files/flash_attn/cute/flash_fwd_mla_sm100.py", "revision": "PR 2441 merge f219c89c886c6ccbf9d3dbd9fe41b11ac64e9df8"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "PR 2441 reports exactly 0.3 ms on B200 BF16 for batch 512, seqlen_q 1, seqlen_k 16384, 128 heads, and top-k 2048.", "claim_id": "kernel-flash-attention-sm100-mla-topk-original-c006", "claim_type": "performance", "covers": ["u007"], "evidence": [{"locator": "PR body, initial saturating decode performance block", "observation": "The author reports 0.31 ms and 955.47 TFLOPS for DSA without bitmask at the stated batch/sequence/head settings and top-k length 2048, plus 0.33 ms with bitmask. The PR body does not state B200 or BF16 for this row, so the structured record changes the value and supplies unsupported environment fields.", "revision": "GitHub PR record observed 2026-08-08; merge f219c89c886c6ccbf9d3dbd9fe41b11ac64e9df8", "url": "https://api.github.com/repos/Dao-AILab/flash-attention/pulls/2441"}], "oracle": "web-primary", "verdict": "refuted"}], "commands": ["inspect PR 2441 GitHub API record and merge f219c89c", "inspect byte-verified PR-2441 diff and key files", "execute the displayed source-page query with conda base Python and a failing negative control"], "id": "kernel-flash-attention-sm100-mla-topk", "page_verdict": "refuted", "path": "wiki/kernels/flash-attention-sm100-mla-topk.md", "residual_risk": "No NVIDIA GPU or CUDA toolkit is installed, so the sparse MLA kernel and the author-reported decode timings were not reproduced. Code conclusions are tied to merge f219c89c; the PR performance block omits enough environment details that it cannot support a structured B200/BF16 benchmark record.", "scope": "All 7 baseline units for body hash 260bbec945acf25551df5b9232d692574dbde84aca2aa0e9b3b73972a38e7239 were checked against the GitHub PR record, byte-verified merge artifacts, and a discriminating host execution of the page query."} +{"claims": [{"claim": "MLA compresses a standard-MHA KV cache from 327\u2013516 KB/token to 70 KB/token, a 4.66\u20137.28x reduction.", "claim_id": "kernel-flashmla-original-c001", "claim_type": "performance", "covers": ["u001"], "evidence": [{"locator": "Section 2.1.1, Table 1 and MLA hyperparameters", "observation": "The paper gives cache elements/token as 2*n_h*d_h*l for MHA and (d_c+d_h^R)*l, approximately 4.5*d_h*l, for MLA, with d_c=4*d_h and d_h^R=d_h/2. It does not report these KB/token endpoints or ratios; for the stated n_h=128, the formula ratio is about 56.9x in equal-width elements, not 4.66\u20137.28x.", "revision": "DeepSeek-V2 paper v5", "url": "https://arxiv.org/html/2405.04434v5"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "FlashMLA's four variants include dense MLA prefill on SM100 alongside dense/sparse MLA decoding and sparse MLA prefill.", "claim_id": "kernel-flashmla-original-c002", "claim_type": "compatibility", "covers": ["u002"], "evidence": [{"locator": "support matrix and Dense MHA Prefill usage section", "observation": "The repository classifies dense prefill on SM100 as MHA with d_qk 192/128 and d_v 128, not MLA/MQA; dense and sparse decode plus sparse prefill are MLA-mode MQA paths.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "DeepSeek FlashMLA commit 71c737929f2567bd0a094ae140f8f60f390b1232; SHA-256 46c001c53800f47148918f4fc2969102492dd5164f710fc015db1622f6f9929a"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "Every MLA KV-cache token is a 656-byte entry containing 512 FP8 bytes, 16 bytes of per-head FP32 scales, and 128 BF16 RoPE bytes, with 64-token dense pages and variable sparse pages.", "claim_id": "kernel-flashmla-original-c003", "claim_type": "code", "covers": ["u003", "u004"], "evidence": [{"locator": "flash_mla_with_kvcache docstring and dispatch", "observation": "The 656-byte layout is specifically DeepSeek V3/V3.1/V3.2 FP8+sparse decode; dense decode uses BF16. Its four FP32 scales are per 128-value group within each token, not per head. page_block_size is an input shape rather than a universal dense/sparse page rule.", "path": "/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py", "revision": "DeepSeek FlashMLA commit 71c7379"}, {"locator": "FP8KVCacheLayout.get_meta and quantize_k_cache", "observation": "The repository also implements a distinct 512-dimensional MODEL1_FP8Sparse format with seven E8M0 scales, disproving a universal 656-byte MLA layout.", "path": "/tmp/kernelwiki-flashmla-pinned/tests/quant.py", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "Dense MLA decode uses a fixed 64-token page and reaches up to 3000 GB/s memory-bound and 660 TFLOPS compute-bound on H800.", "claim_id": "kernel-flashmla-original-c004", "claim_type": "performance", "covers": ["u005"], "evidence": [{"locator": "Performance, dense MLA decoding", "observation": "The 3000-GB/s and 660-TFLOPS author-reported maxima on H800 SXM5/CUDA 12.8 are exact, but the public interface accepts page_block_size from the KV tensor and does not specify a fixed value of 64; 64 is a test default.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "DeepSeek FlashMLA commit 71c7379"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The displayed dense-decode CUDA template is a faithful SM90 BF16 FlashMLA structure using an FP8 cache, explicit scale/RoPE arrays, TMA, and one warpgroup per query head.", "claim_id": "kernel-flashmla-original-c005", "claim_type": "code", "covers": ["u006"], "evidence": [{"locator": "dense and sparse_fp8 implementations plus public interface", "observation": "The snippet is not upstream code and mixes mutually exclusive modes: dense decode is BF16, while FP8 cache dispatch is sparse; the real implementation uses generated/specialized kernels, scheduler metadata, separate split/combine paths, and different tensor contracts.", "path": "/tmp/kernelwiki-flashmla-pinned/csrc/sm90/decode", "revision": "DeepSeek FlashMLA commit 71c7379"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "The indices merely mask scores after all KV entries have already been read.", "check": "Trace the decode index-to-address logic and sparse-prefill reference gather.", "outcome": "The SM90/SM100 kernels derive KV addresses from supplied indices, and the reference computation gathers only selected KV rows; this is not a post-read dense mask."}, "claim": "Sparse MLA consumes token indices and computes attention only over the specified tokens.", "claim_id": "kernel-flashmla-original-c006", "claim_type": "code", "covers": ["u007"], "evidence": [{"locator": "Sparse Attention indices contract and sparse-prefill equivalence", "observation": "Decode indices encode selected page/offset entries and sparse prefill gathers focused_kv=kv[indices] before QK and PV operations.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "DeepSeek FlashMLA commit 71c7379"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "The displayed sparse CUDA template accurately represents FlashMLA's sparse API and its 656-byte cache retrieval.", "claim_id": "kernel-flashmla-original-c007", "claim_type": "code", "covers": ["u008"], "evidence": [{"locator": "flash_mla_with_kvcache and flash_mla_sparse_fwd signatures", "observation": "The invented template is not an upstream kernel. Decode takes page-encoded [batch,s_q,topk] indices and the 656-byte layout only in one FP8 mode; sparse prefill instead takes BF16 [s_kv,h_kv,d_qk] KV and [s_q,h_kv,topk] indices.", "path": "/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py", "revision": "DeepSeek FlashMLA commit 71c7379"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "FlashMLA's SM100 dense MLA prefill uses tcgen05/TMEM for forward and backward and reaches 1460/1000 TFLOPS on B200; the displayed five-stage sketch describes that path.", "claim_id": "kernel-flashmla-original-c008", "claim_type": "code", "covers": ["u009", "u010"], "evidence": [{"locator": "MHA prefill benchmark, support matrix, and Dense MHA Prefill section", "observation": "The 1460/1000 figures belong to NVIDIA's dense MHA forward/backward implementation, not an MLA-prefill variant. Pinned source does use TMEM/TMA in that MHA implementation, but the page's pseudocode is invented and supplies unverified role/dataflow details.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "DeepSeek FlashMLA commit 71c7379"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "The 3000-GB/s and 660-TFLOPS figures describe one simultaneous operating point or an FP8 path.", "check": "Read the exact benchmark qualification and support row.", "outcome": "The source explicitly separates memory-bound and compute-bound configurations and labels dense-decode KV format BF16, so the table must preserve that distinction."}, "claim": "Dense MLA decode is author-reported at up to 660 TFLOPS and 3000 GB/s on H800, and the support matrix labels its cache BF16.", "claim_id": "kernel-flashmla-original-c009", "claim_type": "performance", "covers": ["u011"], "evidence": [{"locator": "Performance and support matrix", "observation": "The source states both maxima for H800 SXM5 with CUDA 12.8 and lists Dense Decoding / SM90 / MQA / BF16.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "DeepSeek FlashMLA commit 71c7379"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "Sparse decode runs at 410 TFLOPS on H800 and up to 350 TFLOPS on B200 with dtype FP8.", "claim_id": "kernel-flashmla-original-c010", "claim_type": "performance", "covers": ["u012", "u013"], "evidence": [{"locator": "sparse MLA decode performance paragraph", "observation": "The values are exact author reports, but FP8 describes the KV-cache storage; the source explicitly says matrix multiplication is bfloat16. A single Dtype=FP8 column materially obscures the compute precision.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "DeepSeek FlashMLA commit 71c7379"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "The figures are MLA prefill results or independently reproduced DeepSeek benchmarks.", "check": "Compare the benchmark heading, news entry, support matrix, and test name.", "outcome": "Every locator calls this dense MHA on SM100 and attributes the numbers to NVIDIA, excluding an MLA or reproduced interpretation."}, "claim": "Dense prefill forward and backward are author-reported at up to 1460 and 1000 TFLOPS on B200 with BF16 tensors.", "claim_id": "kernel-flashmla-original-c011", "claim_type": "performance", "covers": ["u014", "u015"], "evidence": [{"locator": "dense MHA prefill performance and support matrix", "observation": "The repository transcribes NVIDIA's B200 maxima and its test/API use BF16 Q/K/V. The rows are MHA prefill, not MLA decode.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "DeepSeek FlashMLA commit 71c7379"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "Sparse prefill reaches up to 640 TFLOPS on H800 and 1450 TFLOPS on B200 with dtype FP8.", "claim_id": "kernel-flashmla-original-c012", "claim_type": "performance", "covers": ["u016", "u017"], "evidence": [{"locator": "sparse MLA prefill performance, API contract, and equivalent PyTorch code", "observation": "The values and H800 CUDA 12.8/B200 CUDA 12.9 environments are source-reported, but sparse-prefill Q and KV tensors are explicitly bfloat16; the source does not label this path FP8.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "DeepSeek FlashMLA commit 71c7379"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "The repository is only a standalone benchmark and is not connected to model deployments.", "check": "Compare the first-party introduction and sparse-kernel release note with the linked model repositories.", "outcome": "Both identify the kernels as powering the named model families and DSA release, although no broader serving-stack claim follows."}, "claim": "FlashMLA powers the DeepSeek-V3 and DeepSeek-V3.2-Exp model families.", "claim_id": "kernel-flashmla-original-c013", "claim_type": "compatibility", "covers": ["u018"], "evidence": [{"locator": "Introduction", "observation": "The first-party repository explicitly says its optimized attention kernels power DeepSeek-V3 and DeepSeek-V3.2-Exp.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "DeepSeek FlashMLA commit 71c7379"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "SGLang and vLLM provide day-0 FlashMLA support.", "claim_id": "kernel-flashmla-original-c014", "claim_type": "compatibility", "covers": ["u019"], "evidence": [{"locator": "title, date, and purpose", "observation": "The related vLLM source is a 2026 warning about FP8 prefill query quantization, not evidence of launch-day FlashMLA support; the FlashInfer PR is likewise a separate PDL change, and no SGLang source is attached. The unversioned 'day-0' statement is therefore a misleading source conflation.", "path": "sources/prs/vllm/PR-39752.md", "revision": "vLLM merge dc8df110"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "The reduction is merely generic FMHA code and no MLA specialization exists.", "check": "Inspect the pinned PR paths and the d_qk=192/d_vo=128 specialization descriptions.", "outcome": "The exact file and surrounding device/kernel paths are MLA-named and tied to the asymmetric MLA shape, distinguishing them from generic FMHA."}, "claim": "CUTLASS has SM100 MLA attention kernels with a separate fused-reduction implementation.", "claim_id": "kernel-flashmla-original-c015", "claim_type": "code", "covers": ["u020"], "evidence": [{"locator": "changed_paths and summary", "observation": "Example 77 adds the SM100 MLA-shape backward path and includes kernel/sm100_fmha_mla_reduction.hpp; PR 2472 separately captures forward MLA shape support.", "path": "sources/prs/cutlass/PR-2466.md", "revision": "CUTLASS merge 0e026982"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The sparse kernels are generic pruning utilities unrelated to the released model's attention execution.", "check": "Follow the first-party DSA release statement and API descriptions.", "outcome": "The repository explicitly connects both sparse attention stages to DSA and exposes attention operators that consume selected-token indices."}, "claim": "FlashMLA sparse prefill/decode kernels power DeepSeek-V3.2-Exp's token-level DSA/NSA path and are applicable to long-sequence sparse attention.", "claim_id": "kernel-flashmla-original-c016", "claim_type": "compatibility", "covers": ["u021", "u024"], "evidence": [{"locator": "Introduction and 2025-09-29 news item", "observation": "The repository identifies token-level sparse prefill and decode kernels as powering DeepSeek Sparse Attention in DeepSeek-V3.2-Exp.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "DeepSeek FlashMLA commit 71c7379"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "KV compression is only a model-storage property and has no bearing on the decode kernel choice.", "check": "Compare the MLA cache formula with FlashMLA's separately reported memory-bound decode benchmark and model linkage.", "outcome": "The cache is an input to the paged decode operator and the source explicitly measures a bandwidth-bound regime; this supports relevance, not universal superiority."}, "claim": "FlashMLA is relevant to serving DeepSeek MLA models and to decode workloads where KV-cache footprint or bandwidth is limiting.", "claim_id": "kernel-flashmla-original-c017", "claim_type": "performance", "covers": ["u022", "u023"], "evidence": [{"locator": "Section 2.1.1", "observation": "MLA reduces per-token KV elements via low-rank latent caching.", "revision": "DeepSeek-V2 paper v5", "url": "https://arxiv.org/html/2405.04434v5"}, {"locator": "README Introduction and dense-decode performance", "observation": "The library targets DeepSeek models and reports a memory-bound dense-decode regime, supporting the stated selection context without promising a speedup.", "revision": "DeepSeek FlashMLA commit 71c7379", "url": "https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The 656-byte latent KV-cache format is generally tied to DeepSeek's MLA architecture.", "claim_id": "kernel-flashmla-original-c018", "claim_type": "compatibility", "covers": ["u025"], "evidence": [{"locator": "DeepSeek V3/V3.1/V3.2 FP8+sparse docstring and dispatch", "observation": "656 bytes is tied to one V3-family FP8+sparse decode mode, not MLA generally: dense decode is BF16 and another sparse model layout is 512-dimensional with a different scale representation.", "path": "/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py", "revision": "DeepSeek FlashMLA commit 71c7379"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "SM90 dense prefill exists elsewhere in the tree or is implied by global SM90 support.", "check": "Compare the support matrix with setup.py source lists and public prefill API dispatch.", "outcome": "Dense-prefill build sources and dispatch are under sm100, while SM90 support applies to decode and sparse prefill, not this dense MHA path."}, "claim": "FlashMLA's dense prefill support is SM100-only.", "claim_id": "kernel-flashmla-original-c019", "claim_type": "compatibility", "covers": ["u026"], "evidence": [{"locator": "support matrix", "observation": "Dense Prefill is listed only for SM100 and in MHA mode.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "DeepSeek FlashMLA commit 71c7379"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "FlashMLA computes top-k internally before attention.", "check": "Trace public signatures, extension calls, and index address use.", "outcome": "Indices are required inputs passed into sparse_decode_fwd/sparse_prefill_fwd and directly control gathers; the library code audited here does not produce them."}, "claim": "Sparse MLA needs selected-token indices from an indexing stage outside the FlashMLA attention operator.", "claim_id": "kernel-flashmla-original-c020", "claim_type": "compatibility", "covers": ["u027"], "evidence": [{"locator": "decode indices and flash_mla_sparse_fwd arguments", "observation": "Both sparse APIs receive caller-produced int32 indices; no top-k selector is invoked by the attention interfaces.", "path": "/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py", "revision": "DeepSeek FlashMLA commit 71c7379"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The links refer to unrelated implementations or secondary summaries.", "check": "Resolve authorship, repository owner, and paper title/model identity.", "outcome": "Both are DeepSeek-authored primary sources for the implementation or model context."}, "claim": "The FlashMLA GitHub link and DeepSeek-V3 technical report are relevant first-party sources.", "claim_id": "kernel-flashmla-original-c021", "claim_type": "other", "covers": ["u028", "u029"], "evidence": [{"locator": "repository README", "observation": "This is the first-party implementation snapshot audited.", "revision": "commit 71c7379", "url": "https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232"}, {"locator": "paper record", "observation": "This is the named model's first-party technical report and provides model-level MLA context.", "revision": "DeepSeek-V3 technical report", "url": "https://arxiv.org/abs/2412.19437"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "CUTLASS material is the DeepSeek FlashMLA implementation itself.", "check": "Compare repositories and provenance.", "outcome": "It is a separate adjacent NVIDIA implementation of an MLA-shaped prefill kernel, relevant only when explicitly labeled as such."}, "claim": "The CUTLASS SM100 attention changelog is relevant to the adjacent SM100 MLA implementation.", "claim_id": "kernel-flashmla-original-c022", "claim_type": "other", "covers": ["u030"], "evidence": [{"locator": "summary and changed_paths", "observation": "The version-pinned primary PR adds Example 77's SM100 MLA forward implementation; it is stronger and more stable evidence than the mutable latest changelog URL.", "path": "sources/prs/cutlass/PR-2472.md", "revision": "CUTLASS merge 9baa06dd"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "The full artifact directory is a verbatim upstream FlashMLA implementation and its derived variant accurately represents FlashMLA decode.", "claim_id": "kernel-flashmla-original-c023", "claim_type": "code", "covers": ["u031"], "evidence": [{"locator": "origin_url, upstream_repo, and per-file records", "observation": "The two files come from separate NVIDIA CUTLASS and FlashInfer repositories, not deepseek-ai/FlashMLA; the bundle-level origin records only CUTLASS.", "path": "artifacts/kernels/flashmla/full/PROVENANCE.yaml", "revision": "local provenance"}, {"locator": "entire derived fragment", "observation": "The fragment uses a false Dc=128 assumption and conflates SM90 wgmma with SM100 tcgen05, so it is not a faithful FlashMLA decode reference.", "path": "artifacts/kernels/flashmla/variants/01-mla-decode-inner-loop.cu", "revision": "SHA-256 4f494dc2d282468966a1e2028bc50d0ef0c364b34d24f75ee881608ddef2df69"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "The command is stale, requires bare python3, or omits the referenced artifact bundle.", "check": "Execute it in the required conda environment and repeat with a known-absent page ID.", "outcome": "The real ID resolves and includes artifacts; the false ID fails, discriminating actual page lookup."}, "claim": "The displayed get_page command returns the FlashMLA page and included code artifacts.", "claim_id": "kernel-flashmla-original-c024", "claim_type": "code", "covers": ["u032", "u033"], "evidence": [{"command": "conda run -n base python scripts/get_page.py kernel-flashmla --include-code", "environment": "KernelWiki worktree; conda base Python", "negative_control": "Changing the ID to kernel-flashmla-does-not-exist exits nonzero with 'Page not found'.", "reference": "wiki/kernels/flashmla.md and artifacts/kernels/flashmla", "result": "Command exits successfully and returns the page plus both full and derived artifact sections."}], "oracle": "host-experiment", "verdict": "verified"}, {"claim": "Structured B200 performance records support 1460 TFLOPS BF16 dense prefill and 1450 TFLOPS FP8 sparse prefill at approximately 65% utilization for the listed shapes.", "claim_id": "kernel-flashmla-original-c025", "claim_type": "performance", "covers": ["u034"], "evidence": [{"locator": "B200 prefill performance paragraphs", "observation": "The repository reports the two throughput maxima but no ~65% utilization and no complete shape/timing record. Dense prefill is MHA; sparse prefill's documented inputs are BF16 rather than FP8. These omissions and dtype error make both structured records inadmissible.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "DeepSeek FlashMLA commit 71c7379"}], "oracle": "local-primary-source", "verdict": "refuted"}], "commands": ["inspect DeepSeek FlashMLA README, public interfaces, tests, and SM90/SM100 source at commit 71c737929f2567bd0a094ae140f8f60f390b1232", "inspect DeepSeek-V2 paper v5 MLA cache formula", "inspect pinned CUTLASS PR 2466/2472 sources and local artifact provenance", "execute the displayed get_page query and a failing negative control with conda base Python"], "id": "kernel-flashmla", "page_verdict": "refuted", "path": "wiki/kernels/flashmla.md", "residual_risk": "No NVIDIA GPU or CUDA toolkit is installed, so author-reported throughput/bandwidth and numerical kernels were not reproduced. Source-level conclusions are pinned to DeepSeek commit 71c7379; performance statements remain author reports and are not admitted as structured records without exact shapes, timing protocol, samples, and variance.", "scope": "All 34 baseline units for body hash 997be90a5a387c776857d0d8257d24915a19a1aa829e9d76940c4de57a311691 were checked against the pinned DeepSeek implementation, first-party MLA paper, pinned CUTLASS records, local artifacts, and a discriminating host query."} +{"claims": [{"challenge": {"alternative": "128 is only an illustrative diagram size or applies uniformly to all tensors/phases.", "check": "Read the prose definitions and the appendix's forward/backward qualifications.", "outcome": "The forward activation and weight groupings are explicit; the appendix separately notes 128x1 activation handling for backward, so the page's named forward pattern is real but not universal."}, "claim": "DeepSeek-V3 FP8 training uses 1x128 activation scales and 128x128 weight scales to better accommodate outliers than coarser scaling.", "claim_id": "kernel-fp8-block-scale-gemm-original-c001", "claim_type": "compatibility", "covers": ["u001"], "evidence": [{"locator": "Section 3.3.2 Fine-Grained Quantization and Figure 7", "observation": "The paper explicitly defines 1x128 activation tiles and 128x128 weight blocks and says smaller groups better accommodate feature outliers.", "revision": "DeepSeek-V3 Technical Report v2", "url": "https://arxiv.org/html/2412.19437v2"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "DeepGEMM is merely a wrapper over CUTLASS schedules.", "check": "Inspect the first-party README's dependency statement and pinned kernel sources.", "outcome": "The project says it borrows concepts but avoids heavy CUTLASS templates and contains its own SM90/SM100 implementations; CUTLASS is adjacent, not its backend."}, "claim": "DeepGEMM is a first-party implementation of these FP8 GEMM techniques, and CUTLASS supplies separate SM100 block-scaled schedules.", "claim_id": "kernel-fp8-block-scale-gemm-original-c002", "claim_type": "compatibility", "covers": ["u002"], "evidence": [{"locator": "Introduction, requirements, and interfaces", "observation": "DeepGEMM identifies itself as a tensor-core kernel library for FP8/FP4/BF16 on SM90/SM100.", "path": "/tmp/kernelwiki-deepgemm-pinned/README.md", "revision": "DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f; SHA-256 tied to local clone"}, {"locator": "3.8.0-4.1.0 blockscaled GEMM entries", "observation": "CUTLASS records SM100 blockscaled formats and persistent dense blockscaled GEMM examples as a separate implementation family.", "path": "sources/docs/cutlass-changelog-sm100.md", "revision": "CUTLASS release entries through 4.5.0"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "Both architectures use the displayed common structure: activation scales may be FP32 or E4M3, weight scales are one per 128x128 block, FP8 products accumulate in FP32, and scales are applied at the MMA boundary.", "claim_id": "kernel-fp8-block-scale-gemm-original-c003", "claim_type": "code", "covers": ["u003"], "evidence": [{"locator": "scaling-factor interface notice", "observation": "SM90 requires FP32 scales; SM100 requires packed UE8M0, not E4M3.", "path": "/tmp/kernelwiki-deepgemm-pinned/README.md", "revision": "DeepGEMM commit 891d57b4"}, {"locator": "BLOCK_K assertion and promotion loop", "observation": "SM90 WGMMA partials are held in float registers and only then multiplied by scale_a/scale_b into final_accum after each K block; scales are not applied at the WGMMA boundary.", "path": "/tmp/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "Hopper has an approximately 22-bit accumulator and DeepGEMM promotes every 128 WGMMAs to FP32 on CUDA cores, as represented by the displayed kernel.", "claim_id": "kernel-fp8-block-scale-gemm-original-c004", "claim_type": "code", "covers": ["u004"], "evidence": [{"locator": "Section 3.3.3 Increased Accumulation Precision and Figure 7(b)", "observation": "The paper describes a 128-element K interval equivalent to four WGMMAs, not 128 WGMMA instructions; it describes 14-bit alignment/addition precision, not an FP22 accumulator.", "revision": "DeepSeek-V3 Technical Report v2", "url": "https://arxiv.org/html/2412.19437v2"}, {"locator": "math warp-group K-block loop", "observation": "The real kernel fixes BLOCK_K=128, issues BLOCK_K/WGMMA::K instructions, then combines scaled partials into final_accum; the displayed function/signature is invented.", "revision": "commit 891d57b4", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The displayed SM100 kernel and inline tcgen05.mma string faithfully express DeepGEMM's native UE8M0 block-scaling path, with no promotion and a final global scale.", "claim_id": "kernel-fp8-block-scale-gemm-original-c005", "claim_type": "code", "covers": ["u005"], "evidence": [{"locator": "scale TMA/UTCCP path, make_instr_desc_block_scaled, and MMA loop", "observation": "The real source copies packed scale blocks into TMEM and constructs a block-scaled UMMA descriptor. It does not contain the displayed function, tmem_alloc helper, global_scale epilogue, or that inline instruction string.", "path": "/tmp/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh", "revision": "DeepGEMM commit 891d57b4"}, {"locator": "tcgen05.mma mxf8f6f4 wrappers", "observation": "The actual wrapper syntax is kind::mxf8f6f4.block_scale with bracketed scale descriptors and predicate; the displayed .scale_vec::1X form and operands do not match.", "path": "/tmp/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/ptx/tcgen05.cuh", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The displayed tensor layouts and scale dtypes are an accurate cross-architecture ABI, including FP32-or-E4M3 scales and four UE8M0 scales per int32 on Blackwell.", "claim_id": "kernel-fp8-block-scale-gemm-original-c006", "claim_type": "code", "covers": ["u006"], "evidence": [{"locator": "scaling-factor interface notice", "observation": "Four packed UE8M0 values per torch.int is correct for SM100, but the alternative architecture dtype is FP32 on SM90, not E4M3. The current API also supports multiple granularity recipes and layouts, so the fixed shapes are a DeepSeek-V3 pattern rather than a universal ABI.", "path": "/tmp/kernelwiki-deepgemm-pinned/README.md", "revision": "DeepGEMM commit 891d57b4"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "1550 is tied to the page's 4096-cubed shape and ~90% utilization metadata.", "check": "Read the news item and follow its explicit fields without importing nearby benchmark assumptions.", "outcome": "Only GPU and up-to throughput are stated in the README sentence; shape, utilization, timing method, and variance are absent and must not be inferred."}, "claim": "The pinned DeepGEMM README reports up to 1550 TFLOPS on H800.", "claim_id": "kernel-fp8-block-scale-gemm-original-c007", "claim_type": "performance", "covers": ["u007"], "evidence": [{"locator": "2025-04-18 news item", "observation": "The source states exactly that DeepGEMM achieves up to 1550 TFLOPS on H800 and links the contributing changes.", "path": "/tmp/kernelwiki-deepgemm-pinned/README.md", "revision": "DeepGEMM commit 891d57b4"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "CUTLASS SM100 block-scaled schedules achieve a similar ratio to peak as the H800 DeepGEMM result.", "claim_id": "kernel-fp8-block-scale-gemm-original-c008", "claim_type": "performance", "covers": ["u008"], "evidence": [{"locator": "blockscaled GEMM release entries", "observation": "The source establishes availability but contains no matched shape, GPU peak convention, or ratio comparable to DeepGEMM's H800 maximum.", "path": "sources/docs/cutlass-changelog-sm100.md", "revision": "captured changelog through CUTLASS 4.5.0"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "This exact DeepGEMM-style block-scaled pattern should be used for FP8 inference in DeepSeek-V3, Qwen2-FP8, and similar models.", "claim_id": "kernel-fp8-block-scale-gemm-original-c009", "claim_type": "compatibility", "covers": ["u009"], "evidence": [{"locator": "Sections 3.3.1-3.3.2", "observation": "The source establishes the exact format for DeepSeek-V3 training. It does not establish Qwen2-FP8 layout, scale dtype, kernel ABI, or compatibility; model-name transfer without a format contract is unsafe.", "revision": "DeepSeek-V3 Technical Report v2", "url": "https://arxiv.org/html/2412.19437v2"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "FP8 applies only to stored weights or inference, not training activations.", "check": "Inspect the mixed-precision framework and activation-quantization subsections.", "outcome": "The source explicitly discusses forward and backward activation tiles and cached FP8 activations during training."}, "claim": "DeepSeek-V3 uses FP8 activations in its training framework.", "claim_id": "kernel-fp8-block-scale-gemm-original-c010", "claim_type": "compatibility", "covers": ["u010"], "evidence": [{"locator": "Section 3.3 FP8 Training", "observation": "The paper documents FP8 activations, online tile-wise quantization, and phase-specific exceptions in the training framework.", "revision": "DeepSeek-V3 Technical Report v2", "url": "https://arxiv.org/html/2412.19437v2"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Fine-grained scaling universally improves accuracy without cost and should always replace per-tensor scaling.", "check": "Inspect the paper's overhead discussion and special-case policies.", "outcome": "The paper documents scale-calculation/layout overhead and higher-precision exceptions, supporting conditional relevance rather than a universal rule."}, "claim": "Fine-grained scaling is a relevant option when per-tensor FP8 loses accuracy to outliers.", "claim_id": "kernel-fp8-block-scale-gemm-original-c011", "claim_type": "compatibility", "covers": ["u011"], "evidence": [{"locator": "Fine-Grained Quantization subsection", "observation": "The authors motivate smaller scale groups specifically by activation/weight/gradient outliers and report that they better accommodate them.", "revision": "DeepSeek-V3 Technical Report v2", "url": "https://arxiv.org/html/2412.19437v2"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "A structured benchmark records 1550 TFLOPS for M=N=K=4096 on H800 at approximately 90% utilization via CUDA-core promotion.", "claim_id": "kernel-fp8-block-scale-gemm-original-c012", "claim_type": "performance", "covers": ["u012"], "evidence": [{"locator": "2025-04-18 news item", "observation": "The source reports only up to 1550 TFLOPS on H800. It does not attach M=N=K=4096, ~90%, timing details, samples, variance, or a single-kernel mechanism attribution to that maximum.", "path": "/tmp/kernelwiki-deepgemm-pinned/README.md", "revision": "DeepGEMM commit 891d57b4"}], "oracle": "local-primary-source", "verdict": "refuted"}], "commands": ["inspect DeepGEMM commit 891d57b4 README and exact SM90/SM100 1D1D kernels", "inspect DeepSeek-V3 Technical Report v2 fine-grained quantization and accumulation sections", "inspect pinned CUTLASS SM100 changelog capture and DeepGEMM tcgen05 wrappers"], "id": "kernel-fp8-block-scale-gemm", "page_verdict": "refuted", "path": "wiki/kernels/fp8-block-scale-gemm.md", "residual_risk": "No NVIDIA GPU or CUDA toolkit is installed, so kernel numerics and the 1550-TFLOPS author report were not reproduced. CUTLASS availability is source-level; no matched cross-library performance comparison was retained.", "scope": "All 12 baseline units for body hash 9fc01ec2833f70fa337f1ec2f149ac72e477d8465563010c15b1aa7c0c2bcc50 were checked against the DeepSeek paper, pinned DeepGEMM source, and CUTLASS release record."} +{"claims": [{"claim": "The FlashInfer Track A kernel covers the full MoE forward path in one to three launches and saves up to 21.9% of activation traffic relative to a five-to-seven-launch implementation.", "claim_id": "kernel-fused-moe-original-c001", "claim_type": "performance", "covers": ["u001"], "evidence": [{"locator": "description and signature", "observation": "The official definition says only that routing and two grouped GEMMs are included. It gives no launch count or 21.9% traffic result, and its explicit inputs/outputs do not define a separate dispatch/combine benchmark contract.", "revision": "definition fetched 2026-08-08", "url": "https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Track A is a generic MoE contest or targets Hopper as well as B200.", "check": "Read the organizer's challenge target and follow its Track A link to the definition tags.", "outcome": "The organizer says B200 and the linked definition names DeepSeek-V3/R1; no Hopper target is stated for this contest track."}, "claim": "Fused MoE is Track A of the FlashInfer MLSys 2026 contest, which targets NVIDIA B200 and uses a DeepSeek-V3/R1-style FP8 MoE definition.", "claim_id": "kernel-fused-moe-original-c002", "claim_type": "compatibility", "covers": ["u002"], "evidence": [{"locator": "Contest Overview and Track A", "observation": "The organizer identifies B200 as the target and names Track A Fused MoE with FP8 support, linking the exact benchmark.", "revision": "site fetched 2026-08-08", "url": "https://mlsys26.flashinfer.ai/"}, {"locator": "tags and axes", "observation": "The definition is tagged deepseek-v3/deepseek-r1 and records the fixed dimensions.", "revision": "definition fetched 2026-08-08", "url": "https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The displayed pipeline routes each token to top-8 experts from a total of 32 and otherwise represents the benchmark's routing, W13/SwiGLU, W2, and weighted-combine semantics.", "claim_id": "kernel-fused-moe-original-c003", "claim_type": "code", "covers": ["u003"], "evidence": [{"locator": "Axes and Reference Implementation", "observation": "Routing logits span 256 global experts; 32 is num_local_experts for EP=8. The reference selects top-8 globally after grouped routing, computes only the local expert interval, splits the 4096-row W13 result for SwiGLU, performs W2, and accumulates routing-weighted outputs.", "revision": "definition fetched 2026-08-08", "url": "https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "vLLM uses exactly seven launches, SGLang exactly five, and a fully fused implementation exactly one or two launches with the displayed operation boundaries.", "claim_id": "kernel-fused-moe-original-c004", "claim_type": "code", "covers": ["u004"], "evidence": [{"locator": "FlashInfer Baselines and MoE evaluation command", "observation": "The official evaluation defines one FlashInfer wrapper baseline and measures submitted solution entry points; it records no framework launch-count decomposition.", "path": "/tmp/kernelwiki-flashinfer-bench-starter-kit/EVALUATION.md", "revision": "flashinfer-bench-starter-kit 75ccd05cafceb0fd1f86be4cd0f2117249463c66"}, {"locator": "trtllm_fp8_block_scale_moe", "observation": "One Python call enters a staged runner with routing and two separately tuned GEMM tactics; a call count is not a GPU launch count.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/fused_moe/core.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "Gate and up must be two independently launched GEMMs with separate accumulator allocations.", "check": "Inspect the reference's W13 tensor shape and GEMM1 expression.", "outcome": "W13 is one [2I,H] tensor and the reference issues one logical matmul producing both halves; separate launches/allocations are not part of the contract."}, "claim": "The gated first projection is logically W13: a single 2I-wide GEMM result is split into up and gate halves, followed by a fused SwiGLU product.", "claim_id": "kernel-fused-moe-original-c005", "claim_type": "code", "covers": ["u005"], "evidence": [{"locator": "_fp8_moe_run_experts", "observation": "The official trace reference computes G1=A@W13.T, splits G1 into X1/X2, and computes silu(X2)*(X1+beta) before W2.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "The displayed CUDA function is a usable SM100 FP8 block-scale gated-dual-GEMM kernel with valid tcgen05 instructions, TMEM allocation, synchronization, indexing, and scale handling.", "claim_id": "kernel-fused-moe-original-c006", "claim_type": "code", "covers": ["u006"], "evidence": [{"locator": "trtllm_fp8_block_scale_moe wrapper and runner call", "observation": "The first-party implementation uses compiled TensorRT-LLM/FlashInfer runners and explicit tensor contracts, not the displayed helpers or inline assembly.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/fused_moe/core.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}, {"locator": "Instruction Forms and Ownership", "observation": "Real tcgen05 forms use architecture-defined descriptors, predicates, and allocation ownership. The displayed instruction string omits required form/operands and the function also references undefined storage, rows, columns, and helpers.", "path": "wiki/hardware/tcgen05-mma.md", "revision": "KernelWiki page already verified against PTX ISA 9.2"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The displayed keyword call is the FlashInfer benchmark API: BF16 hidden states, 32-wide routing logits, fused w_gate_up/w_down keywords, and FP8 block-scale tensors in the shown shapes.", "claim_id": "kernel-fused-moe-original-c007", "claim_type": "code", "covers": ["u007"], "evidence": [{"locator": "Signature and baseline wrapper", "observation": "The benchmark requires float8 hidden_states [T,7168], routing_logits [T,256], separate gemm1/gemm2 weight and FP32 scale tensors, plus local_expert_offset and routed_scaling_factor. Its wrapper passes the complete positional contract to trtllm_fp8_block_scale_moe; the displayed keyword names and shapes do not match.", "revision": "definition fetched 2026-08-08", "url": "https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The displayed Triton kernel correctly implements a routed, block-scaled fused gate-up projection.", "claim_id": "kernel-fused-moe-original-c008", "claim_type": "code", "covers": ["u008"], "evidence": [{"locator": "DeepSeek routing reference and _fp8_moe_run_experts", "observation": "A correct path uses global expert IDs, a local expert offset, routing weights, all three FP32 scale tensors, bounds-safe gathering, W13 splitting, W2, and weighted accumulation. The displayed kernel leaves expert_ids and every scale pointer unused, lacks K/M/N masks and local offsets, and never performs W2 or combine.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "On B200 at the stated batch sizes, SGLang records 1262 TFLOPS/206.9 us with five launches, FlashInfer CuTe DSL 1225 TFLOPS/481.9 us with one-to-two launches, and vLLM 1117 TFLOPS/369.5 us with seven launches.", "claim_id": "kernel-fused-moe-original-c009", "claim_type": "performance", "covers": ["u009", "u010", "u011"], "evidence": [{"locator": "definition, solutions, and per-workload traces", "observation": "The official page contains no such framework table or launch counts, and searches of the complete serialized page find none of 1262, 206.9, 1225, 481.9, or 369.5. The isolated official traces report latency_ms per seq_len and solution, not the asserted mixed TFLOPS/batch table.", "revision": "definition and traces fetched 2026-08-08", "url": "https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "There is no tuned FP8 MoE configuration for B200, so tile sizes and stages must be selected manually from scratch.", "claim_id": "kernel-fused-moe-original-c010", "claim_type": "compatibility", "covers": ["u012"], "evidence": [{"locator": "MoERunner tactic enumeration and autotuner selection", "observation": "The official implementation enumerates architecture/shape-valid GEMM1 and GEMM2 tactics and supplies token-bucketed runtime autotuning.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/fused_moe/core.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}, {"locator": "bench_trtllm_gen_fused_moe_autotuner_fp8", "observation": "A first-party B200-capable benchmark explicitly compares untuned and autotuned FP8-block tactics and passes tune_max_num_tokens.", "path": "/tmp/kernelwiki-flashinfer-pinned/benchmarks/bench_trtllm_gen_fused_moe_autotuner.py", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "Block size 128 is required specifically to prevent FP8 numerical overflow and guarantee stability.", "claim_id": "kernel-fused-moe-original-c011", "claim_type": "compatibility", "covers": ["u013"], "evidence": [{"locator": "trtllm_fp8_block_scale_moe_ds_routing_trace axes and inputs", "observation": "The source defines 128 as this DeepSeek-FP8 trace's scale granularity and receives already-quantized FP8 tensors plus FP32 scales; it does not compute scales or state a stability requirement.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}, {"locator": "Fine-Grained Quantization", "observation": "The paper motivates smaller groups as better accommodating local outliers and documents phase-specific exceptions; it does not state that 128 is a universal overflow-stability threshold.", "path": "sources/docs/deepseek-v3-fp8.md", "revision": "DeepSeek-V3 Technical Report v2 capture"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The benchmark's batch=1 workload is launch-overhead dominated and its batch=4096 workload is throughput dominated.", "claim_id": "kernel-fused-moe-original-c012", "claim_type": "performance", "covers": ["u014"], "evidence": [{"locator": "Axes", "observation": "The varied axis is seq_len (number of tokens), not batch, and the definition supplies no profile attributing latency to launch overhead or compute throughput.", "revision": "definition fetched 2026-08-08", "url": "https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "A grouped or persistent scheduler makes expert imbalance disappear as a scheduling concern.", "check": "Compare the logical per-group M sizes with the scheduler's finite assignment of tiles.", "outcome": "Grouping can mitigate the tail, but it does not make unequal tile counts equal; the claim is conditional and does not say every implementation suffers the same magnitude."}, "claim": "Dynamic routing gives experts different M dimensions, which can create scheduling tail effects unless grouped work is balanced.", "claim_id": "kernel-fused-moe-original-c013", "claim_type": "performance", "covers": ["u015"], "evidence": [{"locator": "Grouped GEMMs (contiguous layout)", "observation": "The first-party documentation states that each expert may process a varying number of tokens and represents those expert problems with varying M segments.", "path": "/tmp/kernelwiki-deepgemm-pinned/README.md", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "Every TMA descriptor and expert tensor base must be aligned to 128 bytes.", "claim_id": "kernel-fused-moe-original-c014", "claim_type": "compatibility", "covers": ["u016"], "evidence": [{"locator": "cuTensorMapEncodeTiled parameter requirements", "observation": "The tensor-map object address is 64-byte aligned, the ordinary globalAddress requirement is 16 bytes, and selected interleave/data-type cases require 32 bytes. A 128-byte global alignment applies to particular swizzle considerations, not every descriptor or tensor base.", "revision": "CUDA Driver API 13.2.1", "url": "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Gate and up necessarily require two separate TMEM allocations that compete for the entire 256-KB SM budget.", "claim_id": "kernel-fused-moe-original-c015", "claim_type": "code", "covers": ["u017"], "evidence": [{"locator": "_fp8_moe_run_experts", "observation": "The benchmark contract represents gate and up as one concatenated W13 tensor and one logical 2I-wide GEMM result, then splits it for SwiGLU. It does not require two independent accumulator allocations.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The exact kernel contract applies to DeepSeek-V3, Mixtral, and other MoE inference models without further compatibility checks.", "claim_id": "kernel-fused-moe-original-c016", "claim_type": "compatibility", "covers": ["u018"], "evidence": [{"locator": "tags, axes, and signature", "observation": "The definition is specifically tagged DeepSeek-V3/R1 and fixes routing method, 256/32 expert geometry, FP8 type, block-scale layouts, and dimensions. It contains no Mixtral compatibility statement.", "revision": "definition fetched 2026-08-08", "url": "https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The benchmark directly establishes both prefill and decode behavior by treating high batch as prefill and low batch as decode.", "claim_id": "kernel-fused-moe-original-c017", "claim_type": "compatibility", "covers": ["u019"], "evidence": [{"locator": "Axes", "observation": "The trace varies seq_len/number of tokens and does not label workloads as serving prefill, decode, or request batch size.", "revision": "definition fetched 2026-08-08", "url": "https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Combining gate and up always wins regardless of layout or workload.", "check": "Read the conditional wording and compare it with the contract rather than an unqualified speed claim.", "outcome": "The claim applies only after a matched measurement, so it does not assert universal benefit."}, "claim": "A W13/SwiGLU fusion is appropriate when a matched measurement shows it improves the intended workload over separate projections.", "claim_id": "kernel-fused-moe-original-c018", "claim_type": "performance", "covers": ["u020"], "evidence": [{"locator": "_fp8_moe_run_experts", "observation": "The reference contract permits one concatenated W13 matmul and immediate SwiGLU, so the candidate fusion preserves the logical operation; the page conditions use on an actual measured benefit rather than asserting one.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The single public API call means the implementation contract is only one simple matrix operation.", "check": "Inspect the reference implementation behind the wrapper.", "outcome": "The call is a wrapper; the reference separately implements routing, two GEMMs, activation, and weighted accumulation."}, "claim": "A routing-through-combine implementation must coordinate grouped routing, local expert selection, W13, activation, W2, and weighted accumulation.", "claim_id": "kernel-fused-moe-original-c019", "claim_type": "code", "covers": ["u021"], "evidence": [{"locator": "DeepSeek routing reference and _fp8_moe_run_experts", "observation": "The official reference spells out each of these dependent stages and their global/local expert mapping.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "Expert load imbalance is the single primary bottleneck for all practical workloads of this definition.", "claim_id": "kernel-fused-moe-original-c020", "claim_type": "performance", "covers": ["u022"], "evidence": [{"locator": "MoERunner tuning and separate GEMM1/GEMM2 tactic selection", "observation": "The implementation selects shape- and architecture-valid tactics separately for the two GEMMs across token buckets, rather than encoding one universal primary bottleneck.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/fused_moe/core.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}, {"locator": "MoE evaluation and scoring", "observation": "The contest measures whole-operation latency across workloads and publishes no profile that ranks load imbalance as the primary cause.", "path": "/tmp/kernelwiki-flashinfer-bench-starter-kit/EVALUATION.md", "revision": "commit 75ccd05cafceb0fd1f86be4cd0f2117249463c66"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "CUDA graph compatibility universally requires a fixed masked expert layout.", "claim_id": "kernel-fused-moe-original-c021", "claim_type": "compatibility", "covers": ["u023"], "evidence": [{"locator": "Grouped GEMMs (masked layout)", "observation": "DeepGEMM presents masked layout as one supported decoding case when CUDA graphs are enabled and the CPU does not know expert token counts, not as a universal CUDA-graph requirement.", "path": "/tmp/kernelwiki-deepgemm-pinned/README.md", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f"}, {"locator": "ensure_cutedsl_wrapper and standard dispatch function", "observation": "This exact upstream FP4-adjacent runner preallocates CUDA-graph buffers while accepting StandardDispatchOutput; it does not impose DeepGEMM's masked M-grouped ABI.", "path": "artifacts/kernels/fused-moe/full/flashinfer_cutedsl.py", "revision": "SGLang c554dc5c64b661f2c53225b03a76359eaddc39e4; SHA-256 de6bd663..."}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "Grouped scheduling always combines enough small expert problems to eliminate thin-GEMM inefficiency.", "check": "Read the stated small-dimension tile/wave conditions and distinguish mitigation from elimination.", "outcome": "Grouping can increase aggregate parallel work but individual problem tiles may remain partial; the claim says can cause, not must dominate."}, "claim": "Small per-expert M dimensions can leave too few or partially filled GEMM tiles and reduce GPU utilization.", "claim_id": "kernel-fused-moe-original-c022", "claim_type": "performance", "covers": ["u024"], "evidence": [{"locator": "Threadblock-level GEMM", "observation": "NVIDIA states that small M or N can leave threads outside the problem and that small M/N with large K may launch too few threadblocks to use all SMs.", "revision": "CUTLASS documentation fetched 2026-08-08", "url": "https://github.com/NVIDIA/cutlass/blob/main/media/docs/cpp/efficient_gemm.md"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Scales are implicit in FP8 payloads and require no separate storage.", "check": "Inspect the signature's independent tensor names, dtypes, and shapes.", "outcome": "All three scale arrays are explicit inputs; the claim is limited to storage existence and does not quantify a performance penalty."}, "claim": "The benchmark's FP8 block-scale format stores additional scale-factor tensors beyond its quantized activations and weights.", "claim_id": "kernel-fused-moe-original-c023", "claim_type": "compatibility", "covers": ["u025"], "evidence": [{"locator": "Signature", "observation": "The inputs explicitly include three non-empty FP32 scale tensors with shapes [56,T], [32,32,56], and [32,56,16] in addition to FP8 data tensors.", "revision": "definition fetched 2026-08-08", "url": "https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The URL is stale or belongs to a different contest track.", "check": "Open the page and follow the Track A link.", "outcome": "The page is live, identifies MLSys 2026 NVIDIA Track, and names Fused MoE as Track A."}, "claim": "The FlashInfer MLSys 2026 contest link is the organizer page for Track A.", "claim_id": "kernel-fused-moe-original-c024", "claim_type": "other", "covers": ["u026"], "evidence": [{"locator": "Track A Fused MoE", "observation": "The live organizer page identifies Track A and links the exact FP8 MoE definition.", "revision": "site fetched 2026-08-08", "url": "https://mlsys26.flashinfer.ai/"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "GPU Mode NVFP4 Hackathon Problem 3 is a source for this FP8 routed MoE benchmark.", "claim_id": "kernel-fused-moe-original-c025", "claim_type": "compatibility", "covers": ["u027"], "evidence": [{"locator": "task inputs and output", "observation": "Problem 3 is an NVFP4 dual GEMM over A, B1, and B2; it contains no expert routing, local-expert offset, W2 down projection, or FP8 DeepSeek block-scale MoE contract.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml", "revision": "gpu-mode/reference-kernels 51e22db671d36c1c76091c43c36a44546ba324a1"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "DeepGEMM's MoE implementation is direct evidence for this exact FlashInfer FP8 Track A contract.", "claim_id": "kernel-fused-moe-original-c026", "claim_type": "compatibility", "covers": ["u028"], "evidence": [{"locator": "Grouped GEMMs and Mega MoE sections", "observation": "The repository supplies generic grouped GEMMs and a distributed FP8xFP4 Mega MoE path with symmetric memory; it does not define the contest's routing-logit ABI, fixed FP8 tensors, or FlashInfer wrapper.", "path": "/tmp/kernelwiki-deepgemm-pinned/README.md", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The linked SGLang code is direct evidence for the exact FP8 Track A benchmark implementation and its asserted launch/performance table.", "claim_id": "kernel-fused-moe-original-c027", "claim_type": "compatibility", "covers": ["u029"], "evidence": [{"locator": "module constants and registered fused function", "observation": "The byte-matched upstream file is a standard-path FP4 CuteDSL runner: it quantizes activations with fp4_quantize and invokes CuteDslMoEWrapper. It is not the FP8 DeepSeek benchmark and contains no asserted table values or launch counts.", "path": "artifacts/kernels/fused-moe/full/flashinfer_cutedsl.py", "revision": "SGLang c554dc5c64b661f2c53225b03a76359eaddc39e4; SHA-256 de6bd663..."}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The local full bundle is a verbatim upstream implementation of the page's FP8 fused MoE, and its derived variant faithfully represents production fusion.", "claim_id": "kernel-fused-moe-original-c028", "claim_type": "code", "covers": ["u030"], "evidence": [{"locator": "bundle notes and file entries", "observation": "The bundle mixes an MXFP4 vLLM PR diff, an FP4 SGLang runner, and locally extracted pseudocode, so it is not one upstream FP8 implementation.", "path": "artifacts/kernels/fused-moe/full/PROVENANCE.yaml", "revision": "SHA-256 3832ddeeeafe8055e44c890283926365817763af2c724c54cce0872c53699f17"}, {"locator": "fused_moe_forward", "observation": "The derived function incorrectly boolean-multiplies masks for routing-weight indexing and asserts that three unrelated projects fuse all stages; it is neither a correct reference nor production code.", "path": "artifacts/kernels/fused-moe/variants/01-routing-plus-fusion-skeleton.py", "revision": "SHA-256 2cc640b1baa1a6cd42718616ffe2cf78a9b6d4eb3317464f059c6fd442439342"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "The command only prints Markdown and silently ignores --include-code or unknown IDs.", "check": "Run the documented command and an unknown-ID negative control.", "outcome": "The real ID emitted artifact-inclusive output and the control failed distinctly."}, "claim": "scripts/get_page.py can retrieve kernel-fused-moe with its artifact code, while an unknown page ID fails.", "claim_id": "kernel-fused-moe-original-c029", "claim_type": "code", "covers": ["u031", "u032"], "evidence": [{"command": "conda run -n base python scripts/get_page.py kernel-fused-moe --include-code", "environment": "KernelWiki worktree; conda base", "negative_control": "kernel-fused-moe-broken exited 1 with 'No page found'.", "reference": "scripts/get_page.py", "result": "Exited 0 and emitted the fused-moe page plus artifact content."}], "oracle": "host-experiment", "verdict": "verified"}, {"claim": "A reproducible structured B200 record establishes 1262 TFLOPS at topk=8, experts=32, H=7168, I=2048, batch=4096 and about 56% utilization.", "claim_id": "kernel-fused-moe-original-c030", "claim_type": "performance", "covers": ["u033"], "evidence": [{"locator": "Axes, traces, and serialized solutions", "observation": "The definition uses seq_len and distinguishes 256 global from 32 local experts. It has no 1262-TFLOPS or 56%-utilization record, and the repository page supplies no timing method, repetitions, variance, or exact source cell for the structured value.", "revision": "definition and traces fetched 2026-08-08", "url": "https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048"}], "oracle": "web-primary", "verdict": "refuted"}], "commands": ["inspect FlashInfer-Bench definition and serialized official reference/baseline wrapper fetched 2026-08-08", "inspect FlashInfer 7f614b8 trace template, API, runner validation, and autotuner", "inspect flashinfer-bench-starter-kit 75ccd05 evaluation contract", "inspect vLLM PR 23696 merge 074854b2, SGLang c554dc5c, DeepGEMM 891d57b4, GPU Mode reference-kernels 51e22db, and CUDA 13.2.1 TMA requirements", "run get_page positive and unknown-ID negative controls under conda base"], "id": "kernel-fused-moe", "page_verdict": "refuted", "path": "wiki/kernels/fused-moe.md", "residual_risk": "No B200, CUDA toolkit, or contest dataset is installed locally, so no candidate kernel, launch trace, numeric accuracy, or performance result was reproduced. Performance entries without a complete official record were removed rather than inferred from serialized historical traces.", "scope": "All 33 baseline units for body hash c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d were checked against the official contest/benchmark contract and pinned first-party implementations."} +{"claims": [{"challenge": {"alternative": "Gated DeltaNet is merely a renamed softmax-attention kernel or was not an ICLR 2025 result.", "check": "Compare the paper abstract and official repository publication entry.", "outcome": "Both identify a recurrent linear-transformer rule combining decay and delta updates, and the official record says ICLR 2025."}, "claim": "Gated DeltaNet is an ICLR 2025 linear-attention architecture that combines adaptive exponential decay with a delta-rule memory update and has linear sequence-length complexity.", "claim_id": "kernel-gated-delta-net-original-c001", "claim_type": "mechanism", "covers": ["u001"], "evidence": [{"locator": "abstract and ICLR publication record", "observation": "The paper introduces Gated DeltaNet by combining gating for adaptive memory control with the delta update rule; the official repository identifies the ICLR 2025 publication.", "revision": "arXiv 2412.06464, fetched 2026-08-08", "url": "https://arxiv.org/abs/2412.06464"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The 3:1 pattern and 262K context are editorial extrapolations from Qwen3-Next to Qwen3.5.", "check": "Inspect both immutable model configurations independently.", "outcome": "Each configuration directly records the hybrid layout and 262144 limit."}, "claim": "Gated DeltaNet is used in Qwen3-Next and Qwen3.5, whose cited official configurations use a 3:1 linear/full-attention pattern and a native 262,144-token context.", "claim_id": "kernel-gated-delta-net-original-c002", "claim_type": "compatibility", "covers": ["u001"], "evidence": [{"locator": "full_attention_interval, max_position_embeddings, and linear-attention fields", "observation": "The config fixes every fourth layer as full attention, GDN head fields, and max_position_embeddings 262144.", "revision": "Qwen model revision 9c7f2fbe84465e40164a94cc16cd30b6999b0cc7", "url": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/config.json"}, {"locator": "text_config.layer_types and max_position_embeddings", "observation": "The config repeats three linear_attention entries followed by full_attention and fixes 262144 positions.", "revision": "Qwen3.5 model revision 8472618112abcbd45acbcdc58436aff4233c23f7", "url": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/config.json"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "A fixed recurrent matrix gives constant-with-context decode cost and eliminates KV-cache growth entirely.", "claim_id": "kernel-gated-delta-net-original-c003", "claim_type": "mechanism", "covers": ["u002"], "evidence": [{"locator": "Model Overview hybrid layout and config full_attention_interval", "observation": "Qwen3-Next contains one full-attention layer for every three GDN layers. The GDN state is fixed with context length, but the full-attention layers still use cache; therefore the architecture does not eliminate KV-cache growth entirely.", "revision": "model card fetched 2026-08-08; config revision 9c7f2fbe84465e40164a94cc16cd30b6999b0cc7", "url": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The Qwen3-Next-80B diagram is exact, including 48 layers, 36 GDN plus 12 full-attention layers, 512 experts, about 19 active experts per token, and 80B/3B parameter counts.", "claim_id": "kernel-gated-delta-net-original-c004", "claim_type": "code", "covers": ["u003"], "evidence": [{"locator": "Model Overview and config.json", "observation": "The 48-layer 3:1 layout and 80B/3B counts are correct, but the official card states 10 routed experts plus one shared expert, not about 19; it calls the fourth block Gated Attention, not plain GQA for global retrieval.", "revision": "model card fetched 2026-08-08; config revision 9c7f2fbe84465e40164a94cc16cd30b6999b0cc7", "url": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The implementation only adds an outer product and never subtracts or corrects an existing association.", "check": "Inspect old_v, state_remove, state_update, and their signs in the reference.", "outcome": "The source explicitly subtracts k outer old_v before adding k outer new_v, rejecting the additive-only alternative."}, "claim": "The delta rule makes a targeted correction based on the difference between a value and the state's current retrieval for the key.", "claim_id": "kernel-gated-delta-net-original-c005", "claim_type": "mechanism", "covers": ["u004"], "evidence": [{"locator": "_gdn_decode_reference", "observation": "The reference decays the state, retrieves old_v with k, interpolates toward v with beta, removes the old association, and writes the new one.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/gdn.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "The displayed delta_rule_step has the correct state orientation, gate meanings, decay order, and Gated DeltaNet recurrence.", "claim_id": "kernel-gated-delta-net-original-c006", "claim_type": "code", "covers": ["u005"], "evidence": [{"locator": "_gdn_decode_reference", "observation": "With state [K,V], the reference first forms old_state=g*S, retrieves k@old_state, and applies beta to v-old_v. The displayed function declares [d_k,d_v] but computes S@k, constructs a [d_v,d_k] outer product, and swaps beta/alpha roles.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/gdn.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "Chunk prefill resolves each chunk by a causal scalar recurrence while chunks are parallel except for sequential state propagation.", "claim_id": "kernel-gated-delta-net-original-c007", "claim_type": "mechanism", "covers": ["u006"], "evidence": [{"locator": "chunk_gated_delta_rule_fwd_kernel_h and chunk_linear_attn_fwd_kernel_o", "observation": "The chunkwise algorithm uses WY-derived and matrix-product kernels; the state kernel loops over chunks for a given state tile, while other chunk-local products are launched across chunk IDs. The prose's scalar-recurrence/parallel-chunk description does not match this decomposition.", "path": "/tmp/kernelwiki-gdn-pinned/lit_gpt/gated_delta_rule_ops/chunk.py", "revision": "NVlabs/GatedDeltaNet b53d6d3a161267432a79c1c04af69fa52bddc921"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The displayed Triton prefill kernel correctly implements the Qwen/FlashInfer GDN contract.", "claim_id": "kernel-gated-delta-net-original-c008", "claim_type": "code", "covers": ["u007"], "evidence": [{"locator": "gdn_prefill_trace signature and _gdn_prefill_reference", "observation": "The contract uses separate q/k head counts 4, value heads 8, per-head dimension 128, state [N,8,128,128], g and beta gates, cu_seqlens, and output from the updated state. The sketch flattens heads to 512/1024, omits g and variable-length inputs, references undefined NUM_HEADS, and has inconsistent outer-product orientation.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/gdn.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "Decode rereads all prior keys and values, so work or storage grows with context length.", "check": "Inspect the trace signature and reference loop bounds.", "outcome": "The reference receives only one q/k/v token and a fixed recurrent state; there is no prior-sequence axis."}, "claim": "Single-token GDN decode performs a fixed-size state update whose work and storage do not grow with prior context length.", "claim_id": "kernel-gated-delta-net-original-c009", "claim_type": "mechanism", "covers": ["u008"], "evidence": [{"locator": "gated_delta_rule_decode_trace and _gdn_decode_reference", "observation": "seq_len is fixed to one; the input/output state shape depends on batch, heads, and head_size but not the number of prior tokens.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/gdn.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "The displayed Triton decode kernel has the correct state size, gates, layout, ordering, and output for the benchmark.", "claim_id": "kernel-gated-delta-net-original-c010", "claim_type": "code", "covers": ["u009"], "evidence": [{"locator": "Axes, Signature, and Outputs", "observation": "The definition has state [B,8,128,128] with A_log, a, dt_bias, and b inputs. The sketch instead claims a 512x1024 per-head state, accepts one Beta pointer, omits the two-gate computation, and queries the state before the update whereas the reference outputs from the updated state.", "revision": "definition fetched 2026-08-08", "url": "https://bench.flashinfer.ai/kernels/gdn_decode_qk4_v8_d128_k_last"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "TFLA directly supplies Gated DeltaNet kernels and emits inline WGMMA on Hopper and tcgen05 on Blackwell.", "claim_id": "kernel-gated-delta-net-original-c011", "claim_type": "compatibility", "covers": ["u010"], "evidence": [{"locator": "library overview and kernel lists", "observation": "The official TFLA code is a PyTorch/JAX/Triton mLSTM library. Its README describes TFLA as an mLSTM method with a second level of sequence parallelism and arbitrarily large chunks; the pinned tree contains no GatedDeltaNet, wgmma, tcgen05, inline-PTX, or asm implementation.", "path": "/tmp/kernelwiki-mlstm-kernels/README.md", "revision": "NX-AI/mlstm_kernels 5b98ff8e2bec189b3d3c249405bab5149564d6f8"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The displayed tcgen05 inline-assembly fragment is a valid TFLA Blackwell matrix multiply.", "claim_id": "kernel-gated-delta-net-original-c012", "claim_type": "code", "covers": ["u011"], "evidence": [{"locator": "Instruction Forms, Descriptors, and Ownership", "observation": "Valid tcgen05.mma forms require architecture-defined operand descriptors, predicates, and TMEM ownership/synchronization. The fragment's mnemonic and three untyped address operands are not a valid instruction form, and no such fragment exists in the pinned TFLA repository.", "path": "wiki/hardware/tcgen05-mma.md", "revision": "KernelWiki page already verified against PTX ISA 9.2"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "FLA is merely a community fork with no recommendation or variable-length distinction from the authors.", "check": "Read the authors' FAQ rather than a third-party comparison.", "outcome": "The official NVlabs repository makes both statements directly."}, "claim": "NVlabs provides Triton Gated DeltaNet code and its own README recommends the faster FLA implementation for variable-length functionality.", "claim_id": "kernel-gated-delta-net-original-c013", "claim_type": "compatibility", "covers": ["u012", "u013", "u014"], "evidence": [{"locator": "FAQ questions 1 and 2", "observation": "The official README links its Triton implementation, says FLA kernels are faster, supports varlen training, and strongly recommends FLA.", "path": "/tmp/kernelwiki-gdn-pinned/README.md", "revision": "NVlabs/GatedDeltaNet b53d6d3a161267432a79c1c04af69fa52bddc921"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "qk4 and v8 are flattened feature dimensions rather than head counts, or Track C is a single combined definition.", "check": "Read the axis names and follow both organizer links.", "outcome": "The definitions label them num_q_heads/num_k_heads/num_v_heads and expose distinct decode/prefill signatures."}, "claim": "Gated DeltaNet is MLSys 2026 FlashInfer Track C and its official decode/prefill definitions use qk heads 4, value heads 8, and head size 128.", "claim_id": "kernel-gated-delta-net-original-c014", "claim_type": "compatibility", "covers": ["u015", "u016", "u017"], "evidence": [{"locator": "Track C", "observation": "The organizer names Gated Delta Net as Track C and links separate decode and prefill definitions.", "revision": "fetched 2026-08-08", "url": "https://mlsys26.flashinfer.ai/"}, {"locator": "Axes", "observation": "The decode axes are num_q_heads=4, num_k_heads=4, num_v_heads=8, head_size=128.", "revision": "fetched 2026-08-08", "url": "https://bench.flashinfer.ai/kernels/gdn_decode_qk4_v8_d128_k_last"}, {"locator": "Axes", "observation": "The prefill definition records the same fixed head geometry.", "revision": "fetched 2026-08-08", "url": "https://bench.flashinfer.ai/kernels/gdn_prefill_qk4_v8_d128_k_last"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "FlashInfer Blackwell GDN prefill is still in progress while Hopper prefill and both decode paths are done.", "claim_id": "kernel-gated-delta-net-original-c015", "claim_type": "compatibility", "covers": ["u018"], "evidence": [{"locator": "chunk_gated_delta_rule architecture dispatch", "observation": "The current pinned implementation dispatches SM100/SM103 to chunk_gated_delta_rule_sm100 and documents CUDA 13 plus head_size 128 requirements, so Blackwell prefill is implemented rather than in progress.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/gdn_prefill.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "At 32K and longer contexts, GDN's O(n) scaling itself provides major throughput gains.", "claim_id": "kernel-gated-delta-net-original-c016", "claim_type": "performance", "covers": ["u019"], "evidence": [{"locator": "Highlights and Quickstart caveat", "observation": "Qwen reports 10x whole-model inference throughput over Qwen3-32B for contexts over 32K, but separately warns that efficiency depends highly on implementation. The result does not isolate the GDN kernel or establish a general causal threshold.", "revision": "model card fetched 2026-08-08", "url": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The 3:1 statement refers to heads within every layer rather than layer types.", "check": "Inspect the explicit 48-layer layout and interval field.", "outcome": "It is a layer sequence: three GDN layers then one full-attention layer."}, "claim": "Qwen3-Next uses a 75% GDN and 25% full-attention hybrid layout.", "claim_id": "kernel-gated-delta-net-original-c017", "claim_type": "compatibility", "covers": ["u020"], "evidence": [{"locator": "Model Overview hybrid layout and full_attention_interval", "observation": "The card specifies 12 repetitions of three GDN blocks followed by one Gated Attention block across 48 layers.", "revision": "model card fetched 2026-08-08; config revision 9c7f2fbe84465e40164a94cc16cd30b6999b0cc7", "url": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "Streaming GDN decode eliminates KV-cache growth for the whole hybrid model.", "claim_id": "kernel-gated-delta-net-original-c018", "claim_type": "mechanism", "covers": ["u021"], "evidence": [{"locator": "full_attention_interval and use_cache", "observation": "The model retains full-attention layers every fourth layer with caching enabled. Only the GDN layer's recurrent state is fixed with context length.", "revision": "9c7f2fbe84465e40164a94cc16cd30b6999b0cc7", "url": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/config.json"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "CUDA graphs remove GPU compute cost or are guaranteed to accelerate every GDN batch.", "check": "Read NVIDIA's scope and vLLM's capture preconditions.", "outcome": "The sources limit the benefit to CPU launch cost and the vLLM path to compatible uniform decode batches; no universal speedup is asserted."}, "claim": "Short GPU kernels can become host-launch sensitive; CUDA graphs reduce CPU launch cost, and pinned vLLM's Mamba/GDN scheduling backend supports full graph capture for uniform decode.", "claim_id": "kernel-gated-delta-net-original-c019", "claim_type": "performance", "covers": ["u022"], "evidence": [{"locator": "CUDA Graphs overview", "observation": "NVIDIA states that launch setup can be significant for short kernels and that graph replay reduces CPU launch costs.", "revision": "CUDA Programming Guide fetched 2026-08-08", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cuda-graphs.html"}, {"locator": "BaseMambaAttentionMetadataBuilder.build_for_cudagraph_capture", "observation": "The backend declares uniform-batch graph support and implements decode-only full CUDA graph capture.", "revision": "vLLM e1d85e5c2454bd8d349dbe676679380cbe0e920a", "url": "https://github.com/vllm-project/vllm/blob/e1d85e5c2454bd8d349dbe676679380cbe0e920a/vllm/v1/attention/backends/mamba_attn.py"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "A typical recurrent state is 512 by 1024, or 512K floats per head.", "claim_id": "kernel-gated-delta-net-original-c020", "claim_type": "compatibility", "covers": ["u023"], "evidence": [{"locator": "state input shape", "observation": "The exact Qwen TP=4 definition uses [batch_size,8,128,128]. That is 16,384 floats per value head and 131,072 floats across eight heads, not 512K floats per head.", "revision": "definition fetched 2026-08-08", "url": "https://bench.flashinfer.ai/kernels/gdn_decode_qk4_v8_d128_k_last"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "A serving framework can substitute GDN for an already-trained attention layer without changing weights or model behavior.", "check": "Compare the required learned parameter set and recurrence with an attention layer's contract.", "outcome": "The GDN-specific projections/gates and recurrent state have no weight-preserving drop-in mapping supplied by the source."}, "claim": "GDN is a learned architectural layer and is not a drop-in runtime replacement for trained softmax-attention weights.", "claim_id": "kernel-gated-delta-net-original-c021", "claim_type": "compatibility", "covers": ["u024"], "evidence": [{"locator": "GatedDeltaNet projections, gates, convolutions, and forward", "observation": "The layer owns learned q/k/v, decay, update, convolution, output-gate, norm, and output-projection parameters with a recurrence distinct from softmax attention.", "path": "/tmp/kernelwiki-gdn-pinned/lit_gpt/gated_delta_net.py", "revision": "NVlabs/GatedDeltaNet b53d6d3a161267432a79c1c04af69fa52bddc921"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "Qwen3.5 attention output gating is required to eliminate attention sinks and massive activations.", "claim_id": "kernel-gated-delta-net-original-c022", "claim_type": "mechanism", "covers": ["u025"], "evidence": [{"locator": "Gated Attention and Training-Stability-Friendly Designs", "observation": "Qwen attributes output gating to the full-attention path in Qwen3-Next and says it helps eliminate those issues. It does not say the gate is a GDN-kernel requirement or prove necessity, and the original claim misattributes the statement to Qwen3.5.", "revision": "Qwen3-Next blog fetched 2026-08-08", "url": "https://qwen.ai/blog?id=e34c4305036ce60d55a0791b170337c2b70ae51d"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "One or more links silently redirect to a generic landing page or missing resource.", "check": "Follow redirects, inspect page identity, and compare with a deliberately invalid same-site path.", "outcome": "Each target resolves to the intended resource while the invalid control is rejected."}, "claim": "The four external source links resolve to the named GatedDeltaNet repository, TFLA paper, NVIDIA Qwen3-Next article, and MLSys contest page.", "claim_id": "kernel-gated-delta-net-original-c023", "claim_type": "other", "covers": ["u026", "u027", "u028", "u029"], "evidence": [{"command": "curl -L -sS -o /dev/null -w '%{http_code}' ", "environment": "KernelWiki host, 2026-08-08", "negative_control": "https://mlsys26.flashinfer.ai/definitely-not-a-kernelwiki-page returned HTTP 404.", "reference": "HTTP success and page identity", "result": "All four URLs returned HTTP 200 and exposed the named upstream resource."}], "oracle": "host-experiment", "verdict": "verified"}, {"claim": "The local full bundle is a full GDN reference implementation, and both derived files are correctly labeled and semantically faithful GDN variants with complete provenance.", "claim_id": "kernel-gated-delta-net-original-c024", "claim_type": "code", "covers": ["u030"], "evidence": [{"locator": "module functions", "observation": "The verbatim file only splits and reshapes QKVZBA projection outputs; it does not implement the GDN recurrence, prefill, or decode.", "path": "artifacts/kernels/gated-delta-net/full/gdn_fused_proj.py", "revision": "SGLang 5bdc07d974f6cf236fa765a685453ea5e587a838; SHA-256 c4ef85167eb11065721e76dd5d63d6f346f309f84822578dd4a3874249acef89"}, {"locator": "gated_delta_net_prefill", "observation": "The derived file applies cumulative decay repeatedly, uses an additive update rather than the delta rule, and falsely says a per-token state update is paid once per chunk.", "path": "artifacts/kernels/gated-delta-net/variants/01-chunk-parallel-prefill-reference-pytorch.py", "revision": "original SHA-256 d7e74b02df4974fb082cacbae8a5182dd2c40cb1c13fd1d74505b51cfb15481b"}, {"locator": "gdn_decode_step_kernel", "observation": "The file implements additive gated linear attention, not the gated delta recurrence. Its Python comments use # rather than the page's claimed required // header.", "path": "artifacts/kernels/gated-delta-net/variants/02-triton-decode-step-kernel-streaming.py", "revision": "original SHA-256 d5eef14bdbc44e50784c0ca34e7041dcad2151960a85feddba62b097e20079ad"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "The command resolves only prose and silently omits code artifacts.", "check": "Search positive output for gdn_fused_proj.py and both derived provenance headers, then compare an invalid ID.", "outcome": "All artifacts appear in the positive output and the invalid ID is rejected."}, "claim": "The documented get_page command returns this page and its artifacts, while an unknown page ID fails distinctly.", "claim_id": "kernel-gated-delta-net-original-c025", "claim_type": "code", "covers": ["u031", "u032"], "evidence": [{"command": "conda run -n base python scripts/get_page.py kernel-gated-delta-net --include-code", "environment": "KernelWiki worktree; conda base", "negative_control": "kernel-gated-delta-net-broken exited nonzero with 'No page found'.", "reference": "current original page and artifact manifests", "result": "Exited 0 and returned the page plus full and derived artifact content."}], "oracle": "host-experiment", "verdict": "verified"}, {"claim": "A measured H100 BF16 workload at sequence length 8192 and qk4/v8/d128 establishes a 10x GDN speedup over Qwen3-32B at 32K-plus context.", "claim_id": "kernel-gated-delta-net-original-c026", "claim_type": "performance", "covers": ["u033"], "evidence": [{"locator": "Highlights and Model Overview", "observation": "The official claim is whole-model 10x inference throughput for Qwen3-Next-80B-A3B-Base versus Qwen3-32B-Base when context exceeds 32K. It specifies neither H100, BF16, sequence length 8192, nor the contest qk4/v8/d128 kernel shape.", "revision": "model card fetched 2026-08-08", "url": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct"}], "oracle": "web-primary", "verdict": "refuted"}], "commands": ["inspect GatedDeltaNet commit b53d6d3, FlashInfer commit 7f614b8, TFLA/mlstm_kernels commit 5b98ff8, and vLLM merge e1d85e5c", "inspect official Qwen model configurations at revisions 9c7f2fbe and 84726181", "open the organizer and exact FlashInfer-Bench decode/prefill definitions on 2026-08-08", "run source-link and get_page positive/negative controls"], "id": "kernel-gated-delta-net", "page_verdict": "refuted", "path": "wiki/kernels/gated-delta-net.md", "residual_risk": "No supported GPU or CUDA toolkit is installed, so no kernel performance or GPU numeric result was reproduced. The unsupported result-level and implementation-specific claims are refuted rather than retained.", "scope": "All 33 original coverage units for immutable body hash af98b7d75e33581067427cda67ff21fbf228821cc926241a7539b6215cd781ec were atomically adjudicated against primary sources and discriminating host checks."} +{"claims": [{"claim": "The contest operation is necessarily a single fused GPU launch that eliminates exactly two global-memory roundtrips and is the canonical gate-up pattern for LLaMA, Qwen, DeepSeek, and most modern LLMs.", "claim_id": "kernel-gated-dual-gemm-original-c001", "claim_type": "mechanism", "covers": ["u001"], "evidence": [{"locator": "description and submission interface", "observation": "The official task fixes the tensor contract and target operation but does not require a particular launch count, memory-traffic implementation, or model list. The custom Python entry point can launch any conforming implementation.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml", "revision": "gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c"}, {"locator": "ref_kernel", "observation": "The reference materializes two FP32 GEMM outputs before SiLU/multiply, demonstrating that observable correctness does not itself prove an in-kernel fusion or exact traffic reduction.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/reference.py", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "The numbering is an editorial inference from repository folder order.", "check": "Read the sponsor's official numbered challenge list.", "outcome": "The rules explicitly label it Kernel #3, independently of repository order."}, "claim": "NVFP4 Gated Dual GEMM was Kernel Challenge 3 of the NVIDIA/GPU Mode Blackwell NVFP4 Hackathon.", "claim_id": "kernel-gated-dual-gemm-original-c002", "claim_type": "compatibility", "covers": ["u002"], "evidence": [{"locator": "Section 3, Kernel #3", "observation": "The official rules name Kernel #3 NVFP4 Gated Dual GEMM and give its December 20, 2025 through January 16, 2026 challenge window.", "revision": "official rules fetched 2026-08-08", "url": "https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "SiLU applies after multiplying the two projections, or the second projection is the gated branch.", "check": "Trace the exact reference expression and operand order.", "outcome": "The reference applies SiLU only to ref1 before multiplying by ref2, matching the displayed algebra."}, "claim": "The mathematical operation is two matrix products sharing the same left operand, followed by SiLU on the first result and elementwise multiplication with the second.", "claim_id": "kernel-gated-dual-gemm-original-c003", "claim_type": "code", "covers": ["u003"], "evidence": [{"locator": "ref_kernel", "observation": "The official reference computes two torch._scaled_mm calls from a_ref with b1_ref and b2_ref, then returns silu(ref1) * ref2 as FP16.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/reference.py", "revision": "gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "The displayed fused pseudocode proves that conforming submissions use one GPU kernel and never materialize intermediate gate/up tensors in global memory.", "claim_id": "kernel-gated-dual-gemm-original-c004", "claim_type": "code", "covers": ["u003"], "evidence": [{"locator": "files, description, and config", "observation": "The benchmark calls a Python submission entry point and constrains result correctness/performance; it does not expose or require a one-launch decomposition.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml", "revision": "gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c"}, {"locator": "ref_kernel allocations", "observation": "The correctness reference explicitly creates ref1 and ref2 tensors, so the operation contract cannot serve as evidence that intermediates are universally absent.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/reference.py", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The displayed gated_dual_gemm_nvfp4 CUDA template is a valid Blackwell NVFP4 kernel implementing the contest contract.", "claim_id": "kernel-gated-dual-gemm-original-c005", "claim_type": "code", "covers": ["u004"], "evidence": [{"locator": "tcgen05.mma syntax for block-scaled floating-point types", "observation": "NVFP4 MMA requires a qualified tcgen05.mma form with D TMEM address, A/B descriptors, instruction descriptor, SFA/SFB TMEM addresses, and enable-input-D. The sketch uses undefined helper calls with only two source placeholders and a destination.", "revision": "PTX ISA 9.0 / CUDA 13.0.2", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}, {"locator": "allocation syntax and lifecycle", "observation": "Allocation writes its base address into shared memory and must be issued/deallocated collectively with exact qualifiers; the sketch's scalar tmem_alloc/tmem_dealloc helpers and unqualified loads are not defined interfaces.", "revision": "PTX ISA 9.0 / CUDA 13.0.2", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The official contest/reference establishes one physical X load reused by both MMAs, two adjacent 256-column TMEM accumulators, an epilogue with no intermediate shared memory, and shared SFA as the winning Blackwell structure.", "claim_id": "kernel-gated-dual-gemm-original-c006", "claim_type": "mechanism", "covers": ["u005", "u006", "u007", "u008"], "evidence": [{"locator": "description, inputs, and benchmarks", "observation": "The task makes A and SFA logical shared inputs and supplies separate B1/B2 scales, but specifies no global-load count, TMEM partition, shared-memory policy, winning kernel, or required internal schedule.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml", "revision": "gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c"}, {"locator": "ref_kernel", "observation": "The primary reference defines results, not an optimized Blackwell launch topology; it uses framework scaled GEMMs and separate intermediate tensors.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/reference.py", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The exact contest kernel can be used for any dual-output operation sharing one input and expands directly to per-expert MoE kernels across the named model families.", "claim_id": "kernel-gated-dual-gemm-original-c007", "claim_type": "compatibility", "covers": ["u009", "u010", "u011"], "evidence": [{"locator": "input dtypes, shapes, divisibility rules, and operation", "observation": "The contract is narrow: two same-shaped NVFP4 block-scaled products, FP8 per-16 scales, fixed operand layouts, SiLU on the first branch, FP16 output, and shape divisibility. Arbitrary dual outputs or grouped per-expert execution do not satisfy it automatically.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml", "revision": "gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The local full bundle is a reference implementation of the contest kernel and its derived CUDA file is a faithful executable NVFP4 variant with complete provenance.", "claim_id": "kernel-gated-dual-gemm-original-c008", "claim_type": "code", "covers": ["u012"], "evidence": [{"locator": "patch subjects and backend code", "observation": "The patch implements vLLM MXFP4 fused-MoE backend changes, not the GPU Mode NVFP4 dual-GEMM task.", "path": "artifacts/kernels/gated-dual-gemm/full/vllm-PR-23696-gated-dual-gemm.patch", "revision": "SHA-256 033d30b15af4f2050189c2e76ba61959e1e5d4bd049dc901fd56f68890b891a0"}, {"locator": "entire extracted fragment", "observation": "The fragment is commentary plus one offset-padding assignment extracted from a third-party MoE blog, not a dual GEMM implementation.", "path": "artifacts/kernels/gated-dual-gemm/full/blackwell-cutlass-schedules-and-tma.cu", "revision": "SHA-256 e720037bbdb5ee5054259a331cdaf489e7135a44038d66d1fd9d9e5998b75255"}, {"locator": "gated_dual_gemm_epilogue", "observation": "The derived file computes only an epilogue over already-produced float arrays; it has no NVFP4 inputs, scales, matrix multiplication, launch, bounds, or self-test.", "path": "artifacts/kernels/gated-dual-gemm/variants/01-fused-epilogue-swiglu-skeleton.cu", "revision": "SHA-256 85e158c184546912579ac5270451471a96f135efa3584bafbf45064e5fb7e223"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "The query resolves only prose and silently omits artifact content.", "check": "Search positive output for each named full/variant file and compare an unknown ID.", "outcome": "All named artifacts appear and the invalid ID is rejected."}, "claim": "The documented page query resolves the page and all listed artifacts, while an unknown page ID fails distinctly.", "claim_id": "kernel-gated-dual-gemm-original-c009", "claim_type": "code", "covers": ["u013", "u014"], "evidence": [{"command": "conda run -n base python scripts/get_page.py kernel-gated-dual-gemm --include-code", "environment": "KernelWiki worktree; conda base", "negative_control": "kernel-gated-dual-gemm-broken exited nonzero with 'No page found'.", "reference": "original page and artifact manifests", "result": "Exited 0 and returned the page, vLLM patch, extracted schedule fragment, and derived epilogue file."}], "oracle": "host-experiment", "verdict": "verified"}, {"claim": "A B200 NVFP4 measurement of 18.5 microseconds for M=1024, N=4096, K=7168 establishes a compute-bound contest result.", "claim_id": "kernel-gated-dual-gemm-original-c010", "claim_type": "performance", "covers": ["u015"], "evidence": [{"locator": "benchmarks and speed-of-light table", "observation": "The official benchmark M values are 256 or 512, not 1024. It publishes theoretical speed-of-light times 2.125-8.714 us at a stated 1.5 GHz model, not a measured 18.5 us result or compute-bound classification.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml", "revision": "gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The page's SM90 compatibility and claim that Blackwell TMEM necessarily holds two accumulators for a fusion Hopper registers could not handle efficiently are established for this kernel.", "claim_id": "kernel-gated-dual-gemm-original-c011", "claim_type": "compatibility", "covers": ["u004", "u006"], "evidence": [{"locator": "challenge title and Kernel #3", "observation": "The challenge is explicitly the Blackwell NVFP4 hackathon and its task targets B200; the record does not establish a Hopper target.", "revision": "official rules fetched 2026-08-08", "url": "https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf"}, {"locator": "Tensor Memory organization and allocation", "observation": "PTX documents 512 columns by 128 lanes/cells and legal allocation rules, but does not require this dual-GEMM implementation to partition two 256-column accumulators or compare its efficiency with a Hopper register design.", "revision": "PTX ISA 9.0 / CUDA 13.0.2", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"}], "oracle": "web-primary", "verdict": "refuted"}], "commands": ["inspect gpu-mode/reference-kernels commit c5b2f7c task.yml, reference.py, task.py, and repository license", "inspect official NVIDIA hackathon rules and PTX ISA 9.0 Tensor Memory/tcgen05 instruction forms", "inspect every local full and derived artifact with SHA-256", "run get_page positive and unknown-ID negative controls under conda base"], "id": "kernel-gated-dual-gemm", "page_verdict": "refuted", "path": "wiki/kernels/gated-dual-gemm.md", "residual_risk": "No B200, CUDA toolkit, or public immutable winning submission is available locally, so no optimized submission or measured contest result was reproduced. Unsupported implementation and performance claims are refuted rather than inferred.", "scope": "All 15 original coverage units for immutable body hash 4aa8460e6ef7388eb59cdd6643cc8a05fecb149f9148104b8eec852b369e7373, including structured performance and semantic architecture metadata, were adjudicated atomically."} +{"claims": [{"claim": "Grouped GEMM necessarily means variable M with shared N and K, batches MoE experts in one kernel launch, and is the most practically important MoE-serving kernel pattern.", "claim_id": "kernel-grouped-gemm-original-c001", "claim_type": "compatibility", "covers": ["u001"], "evidence": [{"locator": "description, tests, and benchmarks", "observation": "The official grouped-GEMM task supplies an independent M, N, and K for each group; its tests vary all three. Its Python submission interface does not constrain launch count, and it contains no ranking of MoE kernel importance.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_group_gemm/task.yml", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "Problem numbering/weight is inferred from repository order, or DeepGEMM only has ordinary dense GEMMs.", "check": "Compare the sponsor's numbered rules with DeepGEMM's pinned grouped-GEMM sections.", "outcome": "Both facts are explicit in independent primary sources."}, "claim": "NVFP4 Grouped GEMM was Hackathon Problem 4 with a 40% grand-prize weight, and DeepGEMM provides grouped GEMM interfaces tailored to MoE phases.", "claim_id": "kernel-grouped-gemm-original-c002", "claim_type": "compatibility", "covers": ["u002"], "evidence": [{"locator": "Section 3 Kernel #4 and Grand Prize Judging Criteria", "observation": "The rules name Kernel #4 NVFP4 Grouped GEMM and weight kernels 1/2/3/4 at 10/20/30/40 percent.", "revision": "official NVIDIA rules fetched 2026-08-08", "url": "https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf"}, {"locator": "Grouped GEMMs (contiguous layout) and Grouped GEMMs (masked layout)", "observation": "The project documents M-grouped contiguous, M-grouped masked, and K-grouped interfaces for MoE forward/prefill, decode, and weight backward.", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The contest problem fixes N and K across groups so that only M varies, and the displayed DeepSeek-V3 256-expert token counts describe its input contract.", "claim_id": "kernel-grouped-gemm-original-c003", "claim_type": "code", "covers": ["u003"], "evidence": [{"locator": "ten tests and four benchmarks", "observation": "Correctness tests include per-group changes to M, N, and K; benchmark group counts are 8, 8, 2, and 2. The task has no 256-expert DeepSeek token example.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_group_gemm/task.yml", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "K-grouped is merely another alias for the M-grouped forward interface, or masked mode is not phase-specific.", "check": "Read the three API roles in the pinned README and compare their fixed dimensions.", "outcome": "The source distinguishes their axes, fixed dimensions, and MoE phases."}, "claim": "DeepGEMM provides M-grouped contiguous, M-grouped masked, and K-grouped layouts for different MoE phases.", "claim_id": "kernel-grouped-gemm-original-c004", "claim_type": "compatibility", "covers": ["u004"], "evidence": [{"locator": "Grouped GEMMs (contiguous layout) and Grouped GEMMs (masked layout)", "observation": "The README assigns contiguous M-grouping to training forward/prefill, masked M-grouping to graph-compatible decode, and K-grouping to MoE weight backward.", "path": "/tmp/kernelwiki-deepgemm-pinned/README.md", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "The three displayed C++ structs accurately model DeepGEMM's contiguous, masked, and K-grouped tensor layouts.", "claim_id": "kernel-grouped-gemm-original-c005", "claim_type": "code", "covers": ["u005"], "evidence": [{"locator": "m_grouped_fp8_fp4_gemm_nt_contiguous, m_grouped_fp8_fp4_gemm_nt_masked, and k_grouped_fp8_gemm_*_contiguous", "observation": "Contiguous mode consumes a tensor-valued grouped_layout, masked mode consumes one integer valid-M count per group rather than a boolean matrix, and K-grouped produces one M-by-N result per group from packed K segments. These interfaces contradict the displayed offsets, mask, and single-output structs.", "path": "/tmp/kernelwiki-deepgemm-pinned/csrc/apis/gemm.hpp", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f"}, {"locator": "generate_m_grouped_contiguous, generate_m_grouped_masked, and generate_k_grouped_contiguous", "observation": "The generators construct per-row IDs or prefix-sum ends, an int32 masked_m vector, and grouped K result tensors, respectively.", "path": "/tmp/kernelwiki-deepgemm-pinned/tests/generators.py", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The displayed CUTLASS GemmGrouped instantiation and Arguments initializer are a valid SM100 NVFP4 grouped-GEMM API using the named pointer-array schedule and CLC.", "claim_id": "kernel-grouped-gemm-original-c006", "claim_type": "code", "covers": ["u006"], "evidence": [{"locator": "kernel::GemmGrouped template declaration", "observation": "The actual kernel::GemmGrouped takes Mma, Epilogue, ThreadblockSwizzle, GroupScheduleMode, and optional Transposed template parameters, not tile shape, architecture, element types, and layouts as shown.", "path": "/tmp/cutlass-audit.8aj07j/repo/include/cutlass/gemm/kernel/gemm_grouped.h", "revision": "NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3"}, {"locator": "SM100 BlockScaled Ptr Array Dense GEMM Dispatch Policies", "observation": "The named schedule tag exists, but that does not make the unrelated seven-argument GemmGrouped sketch or its invented runtime initializer valid, nor does the tag itself establish CLC use.", "path": "/tmp/cutlass-audit.8aj07j/repo/include/cutlass/gemm/dispatch_policy.hpp", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The static scheduling snippet is a complete schedule for the page's official grouped-GEMM contract.", "claim_id": "kernel-grouped-gemm-original-c007", "claim_type": "code", "covers": ["u007"], "evidence": [{"locator": "tests and problem_sizes contract", "observation": "The official contract permits a different N and K per group, while the snippet accepts one global N and no per-group K. It also depends on undefined BLOCK_M and BLOCK_N constants and supplies no executable launch or correctness check.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_group_gemm/task.yml", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The displayed global-atomic persistent CUDA loop is a valid CLC grouped-GEMM kernel implementing TMA and tcgen05 MMA.", "claim_id": "kernel-grouped-gemm-original-c008", "claim_type": "code", "covers": ["u008"], "evidence": [{"locator": "clusterlaunchcontrol.try_cancel and query_cancel syntax", "observation": "CLC uses an asynchronous try_cancel request, shared opaque response, mbarrier completion, and query_cancel decoding. The snippet uses only atomicAdd and none of the CLC protocol; its TMA/MMA calls are undefined placeholders.", "revision": "CUDA 13.0.2 / PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The entry was merely hypothetical or never appeared at the top of the live leaderboard.", "check": "Inspect the official incident account and its contemporaneous leaderboard figure caption.", "outcome": "Both explicitly record the temporary number-one position and later removal; the page must not call it a final winner."}, "claim": "A reward-hacking submission reached the number-one Problem 4 leaderboard position before it was scrubbed.", "claim_id": "kernel-grouped-gemm-original-c009", "claim_type": "other", "covers": ["u009"], "evidence": [{"locator": "opening account and leaderboard figure caption", "observation": "The official post says the 11.191-us submission surged to number one shortly before the deadline and was scrubbed minutes after the competition.", "revision": "GPU Mode post dated 2026-03-09, fetched 2026-08-08", "url": "https://www.gpumode.com/news/reward-hacking-nvfp4"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The exploit skipped arithmetic, corrupted numerics, or launched 15 merged kernels.", "check": "Trace the official call-by-call account and post-timing correctness description.", "outcome": "The merged kernel computes all results once; the exploit is amortized timing/state reuse, not skipped math or 15 launches."}, "claim": "During correctness the exploit ran an honest padded 8-group kernel on cloned objects; during timing it launched one merged 120-group kernel for 15 objects and returned cached pointers on calls 2 through 15.", "claim_id": "kernel-grouped-gemm-original-c010", "claim_type": "code", "covers": ["u010"], "evidence": [{"locator": "TL;DR, Correctness Check, and Gaming the Timing Loop", "observation": "The post documents cloned correctness objects, real padded 8-group calls, 15 reused timing objects, one 120-group launch, and 14 pointer-only returns.", "revision": "GPU Mode post dated 2026-03-09, fetched 2026-08-08", "url": "https://www.gpumode.com/news/reward-hacking-nvfp4"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "11.191 us is an honest per-call kernel measurement or a final valid record.", "check": "Read the result together with the scrub notice and timing-loop explanation.", "outcome": "The number is verified only as an exploit-reported leaderboard value; the source explicitly invalidates its interpretation as honest per-call performance."}, "claim": "The exploit reported 11.191 microseconds, roughly 2 microseconds ahead of the next leaderboard entry.", "claim_id": "kernel-grouped-gemm-original-c011", "claim_type": "performance", "covers": ["u011"], "evidence": [{"locator": "leaderboard figure caption", "observation": "The official caption labels 11.191 us as the exploit's reported result and says it was roughly 2 us ahead.", "revision": "GPU Mode post dated 2026-03-09, fetched 2026-08-08", "url": "https://www.gpumode.com/news/reward-hacking-nvfp4"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The reward-hacking incident led to FlashInfer-Bench evaluation-methodology improvements for the MLSys 2026 contest.", "claim_id": "kernel-grouped-gemm-original-c012", "claim_type": "mechanism", "covers": ["u011"], "evidence": [{"locator": "What Can We Learn From This? and closing references", "observation": "The official incident post says gpu-mode/reference-kernels PR #104 addressed the harness issue. It contains no FlashInfer-Bench or MLSys causation claim.", "revision": "GPU Mode post dated 2026-03-09, fetched 2026-08-08", "url": "https://www.gpumode.com/news/reward-hacking-nvfp4"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Contiguous and masked are interchangeable storage names with no phase or graph distinction.", "check": "Compare their stated scenarios and API inputs in the README and gemm.hpp.", "outcome": "The contiguous API consumes a packed/group map, while masked mode consumes per-group valid-M counts under fixed graph shapes."}, "claim": "DeepGEMM uses contiguous M-grouping for training forward or prefill, masked M-grouping for CUDA-graph decode, and grouped computation for MoE expert projections.", "claim_id": "kernel-grouped-gemm-original-c013", "claim_type": "compatibility", "covers": ["u012", "u014"], "evidence": [{"locator": "Grouped GEMMs (contiguous layout) and Grouped GEMMs (masked layout)", "observation": "The project explicitly maps variable expert-token M grouping to MoE forward/prefill and its masked interface to decode when CUDA graphs hide valid counts from the CPU.", "path": "/tmp/kernelwiki-deepgemm-pinned/README.md", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "Grouped GEMM is directly usable for any workload whose GEMMs share N and K while M varies.", "claim_id": "kernel-grouped-gemm-original-c014", "claim_type": "compatibility", "covers": ["u013"], "evidence": [{"locator": "m_grouped_fp8_fp4_gemm_nt_contiguous type, shape, layout, architecture, and scale checks", "observation": "The concrete interface additionally constrains architecture, FP8/FP4 or BF16 formats, operand major modes, scale representations, aligned segment layout, output dtype, and positive dimensions. Shared N/K and variable M alone are insufficient compatibility conditions.", "path": "/tmp/kernelwiki-deepgemm-pinned/csrc/apis/gemm.hpp", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "Expert load imbalance is universally the primary practical grouped-GEMM bottleneck.", "claim_id": "kernel-grouped-gemm-original-c015", "claim_type": "performance", "covers": ["u015"], "evidence": [{"locator": "NCU Analysis: Direct Measurement of Fixed Kernel Startup Cost", "observation": "For the studied B200 implementation, the post measures about 19.5 us of fixed kernel startup and attributes about 80% of a 148-tile runtime to setup, directly supplying a materially different dominant bottleneck. Primary bottlenecks are implementation- and workload-specific.", "revision": "GPU Mode post dated 2026-03-09, fetched 2026-08-08", "url": "https://www.gpumode.com/news/reward-hacking-nvfp4"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Tensor-core tiling always maintains full utilization regardless of M.", "check": "Inspect NVIDIA's documented boundary behavior for threadblock tiles and grid size.", "outcome": "The primary documentation explicitly identifies partial tiles and insufficient grid parallelism; the claim is retained only as a possibility, not a universal measured outcome."}, "claim": "Small GEMM M dimensions can leave threads outside the problem bounds or launch too few threadblocks to use all GPU multiprocessors.", "claim_id": "kernel-grouped-gemm-original-c016", "claim_type": "performance", "covers": ["u016"], "evidence": [{"locator": "Threadblock-level GEMM", "observation": "NVIDIA documents that small M or N can leave some threadblock threads without meaningful work, and small M and N with large K can produce too few threadblocks to use all multiprocessors.", "path": "/tmp/cutlass-audit.8aj07j/repo/media/docs/cpp/efficient_gemm.md", "revision": "NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "DeepGEMM's masked grouped layout necessarily wastes computation on padded rows when valid M counts are skewed.", "claim_id": "kernel-grouped-gemm-original-c017", "claim_type": "mechanism", "covers": ["u017"], "evidence": [{"locator": "Grouped GEMMs (masked layout)", "observation": "DeepGEMM states that the mask tensor makes the kernel compute only valid portions.", "path": "/tmp/kernelwiki-deepgemm-pinned/README.md", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f"}, {"locator": "test_m_grouped_gemm_masked output checks", "observation": "Correctness checks slice each group to masked_m[j] valid rows rather than asserting computation across every max-M row.", "path": "/tmp/kernelwiki-deepgemm-pinned/tests/test_fp8_fp4.py", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "CLC tile acquisition is implicit and incurs no request or synchronization machinery beyond a static tile map.", "check": "Enumerate the normative API sequence in the official guide.", "outcome": "The sequence contains explicit request, transaction, wait, decode, and fence operations; end-to-end performance impact remains workload-specific."}, "claim": "CLC scheduling requires request/response and synchronization work that a precomputed static schedule does not perform.", "claim_id": "kernel-grouped-gemm-original-c018", "claim_type": "mechanism", "covers": ["u018"], "evidence": [{"locator": "4.12.1.1 API Details, steps 1-6", "observation": "The documented CLC path initializes an mbarrier, submits an asynchronous cancellation request, waits for completion, decodes success and coordinates, and performs proxy ordering; those operations are specific to the dynamic request path.", "revision": "CUDA Programming Guide 13.2", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "TMA universally requires 128-byte alignment and therefore constrains grouped-GEMM minimum tile dimensions.", "claim_id": "kernel-grouped-gemm-original-c019", "claim_type": "compatibility", "covers": ["u019"], "evidence": [{"locator": "cuTensorMapEncodeTiled parameter constraints", "observation": "The API documents a 64-byte-aligned descriptor and generally 16-byte-aligned global address/strides, with feature-specific 32-byte constraints. It does not impose one universal 128-byte alignment or derive minimum tile dimensions from it.", "revision": "CUDA Driver API 13.2.1", "url": "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "A link is dead or resolves to an unrelated host/project.", "check": "Follow redirects, inspect target identities, and compare a same-path nonexistent control.", "outcome": "All positive targets resolved to the intended primary projects while the invalid target failed distinctly."}, "claim": "The four source links resolve to the intended official GPU Mode repository, DeepGEMM repository, GPU Mode incident post, and NVIDIA CUTLASS documentation.", "claim_id": "kernel-grouped-gemm-original-c020", "claim_type": "other", "covers": ["u020", "u021", "u022", "u023"], "evidence": [{"command": "curl -L -sS -o /dev/null -w '%{http_code}' ", "environment": "KernelWiki host, 2026-08-08", "negative_control": "A nonexistent file under the pinned GPU Mode grouped-GEMM directory returned HTTP 404.", "reference": "HTTP resolution plus inspection of repository/site identity", "result": "The repository, DeepGEMM, reward-hack, and CUTLASS targets resolved successfully; exact replacement links to pinned task/template/reference and DeepGEMM sources also returned HTTP 200."}], "oracle": "host-experiment", "verdict": "verified"}, {"claim": "A valid B200 NVFP4 result is 11.2 microseconds for 15 groups with shared N=K, and the workload is compute-bound.", "claim_id": "kernel-grouped-gemm-original-c021", "claim_type": "performance", "covers": ["u024"], "evidence": [{"locator": "four benchmark rows and speed-of-light table", "observation": "The official benchmarks contain 8, 8, 2, and 2 groups, with N and K distinct. Published 18.833/10.667/2.406/1.525-us values are theoretical speed-of-light estimates, not a measured 11.2-us result or compute-bound classification.", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6", "url": "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/task.yml"}, {"locator": "leaderboard figure caption and Gaming the Timing Loop", "observation": "11.191 us was an exploit-reported amortized value and the submission was scrubbed; it is not a valid performance record.", "revision": "GPU Mode post dated 2026-03-09, fetched 2026-08-08", "url": "https://www.gpumode.com/news/reward-hacking-nvfp4"}], "oracle": "web-primary", "verdict": "refuted"}], "commands": ["inspect official NVIDIA rules and gpu-mode/reference-kernels grouped-GEMM task/template/reference at ae67948685dfccf54ae8374dc9402addb7aae4f6", "inspect DeepGEMM grouped README, generators, tests, and API checks at 891d57b4db1071624b5c8fa0d1e51cb317fa709f", "inspect NVIDIA CUTLASS grouped API and efficient-GEMM guide at e05f953a5b3d38adc240df2ff928e0421c2abba3", "run primary-link positive/negative controls and get_page positive/negative controls"], "id": "kernel-grouped-gemm", "page_verdict": "refuted", "path": "wiki/kernels/grouped-gemm.md", "residual_risk": "No B200 or CUDA toolkit is available, so no optimized kernel or performance result was reproduced. The public official task, exact library APIs, vendor semantics, and official incident account are sufficient to adjudicate the original interface, code, and record claims.", "scope": "All 24 original coverage units for body hash 216f09e31b3f73503470760148850014df1540ad380997383317930a7b459e9e were checked against official rules, immutable upstream source, vendor documentation, and discriminating link/query controls."} +{"claims": [{"challenge": {"alternative": "The three branches are a post-training pruning recipe, or the quality statement is a universal no-loss guarantee.", "check": "Inspect the training formulation and the exact scope of the experimental results.", "outcome": "The branches and gates are trained end-to-end; quality is an author-reported result on the paper's evaluations, not a universal guarantee."}, "claim": "Native Sparse Attention was published at ACL 2025 as a hardware-aligned, natively trainable long-context sparse-attention design with compression, selection, and sliding-window branches; the authors report quality comparable to or better than full attention in their evaluations.", "claim_id": "kernel-nsa-original-c001", "claim_type": "mechanism", "covers": ["u001"], "evidence": [{"locator": "abstract, Sections 3 and 4", "observation": "The paper describes NSA as hardware-aligned and natively trainable, defines the three branches, evaluates contexts through 64K, and reports accuracy comparable to or better than full attention on the studied tasks.", "revision": "ACL 2025 proceedings PDF fetched 2026-08-08", "url": "https://aclanthology.org/2025.acl-long.1126.pdf"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "All three values are matched wall-clock measurements from one benchmark record.", "check": "Compare the Figure 5 experimental description with the Table 4 title and derivation.", "outcome": "Forward/backward are reported benchmark speedups, while decoding is an expected memory-access ratio; they require separate labels."}, "claim": "The NSA paper reports 9.0x forward and 6.0x backward speedups at 64K versus its Triton FlashAttention-2 baseline and gives 11.6x as the 64K decoding memory-access-volume ratio labeled expected speedup.", "claim_id": "kernel-nsa-original-c002", "claim_type": "performance", "covers": ["u002", "u011"], "evidence": [{"locator": "Section 4.2, Figure 5, and Table 4", "observation": "Figure 5 reports 9.0x forward and 6.0x backward at 64K on an 8-GPU A100 system against a Triton FlashAttention-2 implementation. Table 4 labels 11.6x an expected speedup obtained from 65,536 versus 5,632 memory-access tokens.", "revision": "ACL 2025 proceedings PDF fetched 2026-08-08", "url": "https://aclanthology.org/2025.acl-long.1126.pdf"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The three-branch NSA mechanism is deployed in DeepSeek-V3.2-Exp using FlashMLA sparse kernels.", "claim_id": "kernel-nsa-original-c003", "claim_type": "compatibility", "covers": ["u002"], "evidence": [{"locator": "DeepSeek Sparse Attention announcement and high-performance CUDA kernels", "observation": "The release names its mechanism DeepSeek Sparse Attention (DSA), links indexer logits to DeepGEMM and sparse kernels to FlashMLA, and does not identify it as the ACL paper's three-branch NSA.", "path": "/tmp/kernelwiki-deepseek-v32/README.md", "revision": "DeepSeek-V3.2-Exp 87e509a2e5a100d221c97df52c6e8be7835f0057"}, {"locator": "Indexer.forward and MLA.forward", "observation": "The model selects top-k token positions with a learned indexer and masks full attention to those positions; it does not implement the NSA compression/selection/window gated sum.", "path": "/tmp/kernelwiki-deepseek-v32/inference/model.py", "revision": "same commit"}, {"locator": "Sparse Attention Kernels", "observation": "FlashMLA says these kernels power DSA in DeepSeek-V3.2-Exp, not NSA.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The displayed three-path diagram is an exact NSA output equation in which the three branch attention outputs are combined without learned gates.", "claim_id": "kernel-nsa-original-c004", "claim_type": "code", "covers": ["u003"], "evidence": [{"locator": "Section 3.1, Equations 5 and 6", "observation": "NSA multiplies each branch output by an input-dependent sigmoid gate produced by a learned MLP before summing the branches. The displayed Combine expression omits those gates.", "revision": "ACL 2025 proceedings PDF fetched 2026-08-08", "url": "https://aclanthology.org/2025.acl-long.1126.pdf"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Group-centric loading is an editorial optimization not stated by the authors, or selected attention scatters individual tokens.", "check": "Read the kernel-design paragraph and selection-block definition.", "outcome": "Both group-level query loading and continuous block loading are explicit design choices in the paper."}, "claim": "The paper implements the selected-attention training/prefill kernel in Triton using hardware-aligned contiguous KV blocks, loading all query heads in a GQA group together so they share sparse KV indices, and placing nearly constant query/output loops in the Triton grid.", "claim_id": "kernel-nsa-original-c005", "claim_type": "mechanism", "covers": ["u004", "u007", "u008", "u009"], "evidence": [{"locator": "Section 3.3 Hardware-aligned Sparse Attention", "observation": "The authors describe a Triton kernel that loads all query heads in one GQA group into SRAM, shares selected KV indices, loads continuous KV blocks, and parallelizes query/output loops through the Triton grid.", "revision": "ACL 2025 proceedings PDF fetched 2026-08-08", "url": "https://aclanthology.org/2025.acl-long.1126.pdf"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The displayed nsa_sparse_attention_fwd function is a valid Triton implementation of NSA selected attention and correct online softmax.", "claim_id": "kernel-nsa-original-c006", "claim_type": "code", "covers": ["u005"], "evidence": [{"command": "conda run -n base python -c ''", "environment": "KernelWiki host; conda base standard-library Python", "negative_control": "The stable reference divided by the within-block partition sum and returned the expected average 2.0.", "reference": "Displayed nsa_sparse_attention_fwd update and normalized softmax definition", "result": "Exited 0: the displayed update produced 4.0 with lse=0.0; normalized softmax produced 2.0 with log-partition log(2). The listing also references undefined SEQ_LEN, ignores pid in query indexing, and loads one query head per program rather than the paper's group-centric query set."}], "oracle": "host-experiment", "verdict": "refuted"}, {"challenge": {"alternative": "BLOCK_SIZE denotes a vectorized block load hidden by the loop body.", "check": "Inspect the addresses used inside the loop.", "outcome": "The body loads only K[pos] and V[pos], not a BLOCK_SIZE-wide tile, and BLOCK_SIZE is not declared in the signature."}, "claim": "The displayed nsa_sliding_window_fwd function is a valid Triton implementation of dense attention over every position in the local window.", "claim_id": "kernel-nsa-original-c007", "claim_type": "code", "covers": ["u006"], "evidence": [{"command": "conda run -n base python -c ''", "environment": "KernelWiki host; conda base standard-library Python", "negative_control": "A unit-stride reference enumerated all four positions.", "reference": "Displayed sliding-window loop", "result": "Exited 0: the candidate loop visited [0,2], while a four-position window is [0,1,2,3]."}], "oracle": "host-experiment", "verdict": "refuted"}, {"claim": "The paper specifies Triton grid dimensions exactly as (query_block, head, batch) and establishes that this mapping avoids dynamic scheduling overhead.", "claim_id": "kernel-nsa-original-c008", "claim_type": "mechanism", "covers": ["u010"], "evidence": [{"locator": "Section 3.3 kernel design", "observation": "The paper says outer query/output loops are implemented in the Triton grid because their loop counts are nearly constant. It does not give the asserted three-dimensional grid mapping or claim that it eliminates dynamic scheduling overhead.", "revision": "ACL 2025 proceedings PDF fetched 2026-08-08", "url": "https://aclanthology.org/2025.acl-long.1126.pdf"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The 9.0x, 6.0x, and 11.6x values were all measured on H100 with BF16, and the structured 9.0x record is reproducible as H100 BF16 at sequence length 65,536.", "claim_id": "kernel-nsa-original-c009", "claim_type": "performance", "covers": ["u012", "u022"], "evidence": [{"locator": "Section 4 experimental setup, Figure 5, and Table 4", "observation": "The paper specifies an 8-GPU A100 system for efficiency experiments, not H100. It does not state BF16 for this benchmark, and 11.6x is labeled expected speedup from memory-access volume rather than a matched measurement. Batch, software versions, trials, and variance are also absent from the structured record.", "revision": "ACL 2025 proceedings PDF fetched 2026-08-08", "url": "https://aclanthology.org/2025.acl-long.1126.pdf"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "NSA should be used at 32K or longer because dense attention is categorically prohibitively expensive beyond that threshold.", "claim_id": "kernel-nsa-original-c010", "claim_type": "compatibility", "covers": ["u013"], "evidence": [{"locator": "Sections 1 and 4", "observation": "The paper motivates long-context efficiency and evaluates several lengths through 64K, but defines no universal 32K applicability threshold or categorical prohibition; suitability depends on model, quality target, hardware, and workload.", "revision": "ACL 2025 proceedings PDF fetched 2026-08-08", "url": "https://aclanthology.org/2025.acl-long.1126.pdf"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The top-n operation makes the entire selection mechanism non-trainable, or every query head selects independent blocks.", "check": "Trace score construction, head aggregation, and gradient path in the paper.", "outcome": "Block indices are shared at group level and importance comes from the trainable compression branch; the paper contrasts this with non-trainable post-hoc selection."}, "claim": "NSA is designed for grouped-query or multi-query attention and makes its sparse selection differentiable for end-to-end training.", "claim_id": "kernel-nsa-original-c011", "claim_type": "compatibility", "covers": ["u014", "u015"], "evidence": [{"locator": "Sections 3.2-3.3 and Appendix B", "observation": "Selection scores are aggregated over query heads in a GQA/MQA group, and the design reuses differentiable compression-attention scores so sparse patterns can be learned during end-to-end training.", "revision": "ACL 2025 proceedings PDF fetched 2026-08-08", "url": "https://aclanthology.org/2025.acl-long.1126.pdf"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Compression is a fixed pooling transform with no learned parameters.", "check": "Inspect the compression equations and parameter notation.", "outcome": "The transformation is explicitly parameterized by learned MLP weights."}, "claim": "The compression branch uses a learned MLP with intra-block position encoding, adding trainable parameters and computation, although the page provides no quantified cost.", "claim_id": "kernel-nsa-original-c012", "claim_type": "mechanism", "covers": ["u016"], "evidence": [{"locator": "Section 3.1 Compression of Coarse-grained Tokens", "observation": "Compressed keys and values are produced by learned MLPs over token blocks with intra-block position encoding; this necessarily introduces parameters and operations, but the paper does not quantify a standalone overhead in the cited passage.", "revision": "ACL 2025 proceedings PDF fetched 2026-08-08", "url": "https://aclanthology.org/2025.acl-long.1126.pdf"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "NSA token selection performs a separate first pass that scores every fine-grained block before a second top-n pass.", "claim_id": "kernel-nsa-original-c013", "claim_type": "mechanism", "covers": ["u017"], "evidence": [{"locator": "Section 3.1 Selection of Important Tokens", "observation": "NSA derives fine-grained block importance by reusing the compression-branch attention scores, aggregating them over related compressed blocks and query heads, then selecting top-n blocks. The paper does not define the claimed independent score-all-blocks pass.", "revision": "ACL 2025 proceedings PDF fetched 2026-08-08", "url": "https://aclanthology.org/2025.acl-long.1126.pdf"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "NSA's Triton implementation is established to be CPU-launch-bound for small-batch decode, so CUDA Graph mode is categorically recommended.", "claim_id": "kernel-nsa-original-c014", "claim_type": "performance", "covers": ["u018"], "evidence": [{"locator": "Section 4.2 and Limitations", "observation": "The paper analyzes decoding through memory-access volume and notes generic Triton abstraction overhead versus native CUDA; it provides no CPU-launch profile or CUDA Graph recommendation for NSA.", "revision": "ACL 2025 proceedings PDF fetched 2026-08-08", "url": "https://aclanthology.org/2025.acl-long.1126.pdf"}, {"locator": "Performance Issues", "observation": "NVIDIA says graphs primarily benefit CPU-bound workloads and can give minimal benefit or regress when the workload is GPU-bound, so profiling is required before applying the recommendation.", "revision": "NVIDIA CUDA Graphs troubleshooting guide fetched 2026-08-08", "url": "https://docs.nvidia.com/dl-cuda-graph/troubleshooting/performance-issues.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Joint training is a universal theorem and no pretrained model can tolerate post-hoc sparsity.", "check": "Scope the statement to the paper's mechanism, motivation, and experiments.", "outcome": "The evidence supports NSA's design choice and reported studies, not a universal impossibility result."}, "claim": "The paper's motivation and ablations support jointly training the sparse-selection mechanism with the model rather than applying a non-trainable post-hoc selector.", "claim_id": "kernel-nsa-original-c015", "claim_type": "compatibility", "covers": ["u019"], "evidence": [{"locator": "Appendix B, Natively Trainable Sparse Attention", "observation": "The authors argue that post-hoc sparsity can degrade pretrained models, contrast non-trainable selection with NSA's gradient path, and report that native training learns effective sparse patterns.", "revision": "ACL 2025 proceedings PDF fetched 2026-08-08", "url": "https://aclanthology.org/2025.acl-long.1126.pdf"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The repository is the authors' official implementation or one link silently redirects to an unrelated resource.", "check": "Inspect repository owner/readme and final target identity, then compare a same-host invalid path.", "outcome": "The links are live; lucidrains is explicitly third-party rather than an official paper artifact, and the negative control fails distinctly."}, "claim": "The ACL/arXiv paper link and the lucidrains repository link resolve to the intended NSA paper and a third-party PyTorch implementation, respectively.", "claim_id": "kernel-nsa-original-c016", "claim_type": "other", "covers": ["u020", "u021"], "evidence": [{"command": "curl -L -sS -o /dev/null -w '%{http_code}' ", "environment": "KernelWiki host, 2026-08-08", "negative_control": "A nonexistent sibling path under aclanthology.org returned HTTP 404.", "reference": "Source-link identities and HTTP resolution", "result": "ACL landing page, proceedings PDF, and lucidrains repository each returned HTTP 200 and matched their stated identities."}], "oracle": "host-experiment", "verdict": "verified"}, {"claim": "The NSA paper directly establishes SM90 and SM100 compatibility and that Blackwell's 126 MB L2 and bandwidth improve this Triton sparse-block implementation.", "claim_id": "kernel-nsa-original-c017", "claim_type": "compatibility", "covers": ["u004", "u007"], "evidence": [{"locator": "Section 4 experimental setup and Section 3.3", "observation": "The paper's efficiency evaluation uses A100 GPUs and does not report Hopper/Blackwell compatibility, a 126-MB L2 result, or a cross-architecture sparse-fetch benefit.", "revision": "ACL 2025 proceedings PDF fetched 2026-08-08", "url": "https://aclanthology.org/2025.acl-long.1126.pdf"}, {"locator": "requirements and support matrix", "observation": "SM90/SM100 support belongs to later FlashMLA DSA kernels, a different mechanism, and cannot be transferred to the ACL NSA implementation.", "revision": "FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232", "url": "https://github.com/deepseek-ai/FlashMLA/blob/71c737929f2567bd0a094ae140f8f60f390b1232/README.md"}], "oracle": "web-primary", "verdict": "refuted"}], "commands": ["inspect the ACL 2025 proceedings paper's equations, implementation section, efficiency experiment, tables, and appendix", "inspect DeepSeek-V3.2-Exp 87e509a2 and FlashMLA 71c7379 source for the later DSA mechanism", "run host counterexamples for the sparse-forward normalization and sliding-window coverage under conda base", "run primary-link positive controls and a same-host 404 negative control"], "id": "kernel-nsa", "page_verdict": "refuted", "path": "wiki/kernels/nsa.md", "residual_risk": "No NVIDIA GPU, CUDA toolkit, or paper implementation is available locally. The paper's performance values were not reproduced and its benchmark record omits dtype, software versions, batch details, raw trials, and variance; retained values remain explicitly source-reported.", "scope": "All 22 baseline units for body hash 002fbf6ef7dba7c6bd6bfec4cdeccf4104b100309e09eae082faaf0aeec0e4c3, plus semantic architecture and Blackwell metadata, were checked against the ACL paper, immutable first-party deployment source, version-pinned FlashMLA source, and discriminating host controls."} +{"claims": [{"challenge": {"alternative": "NVFP4 is merely a software decoding convention with no native Blackwell MMA path, or every implementation must use every listed mechanism.", "check": "Inspect the ISA operand and target contract, then distinguish hardware availability from a universal implementation prescription.", "outcome": "The native block-scaled instruction path is explicit; TMA and warp specialization are implementation choices rather than defining requirements."}, "claim": "NVFP4 GEMM multiplies block-scaled E2M1 inputs on Blackwell, whose native path includes block-scaled tcgen05.mma and TMEM accumulation and can be fed by asynchronous TMA pipelines.", "claim_id": "kernel-nvfp4-gemm-original-c001", "claim_type": "mechanism", "covers": ["u001"], "evidence": [{"locator": "tcgen05.mma block-scale syntax, semantics, and target notes", "observation": "The ISA defines Blackwell architecture-specific block-scaled mxf4nvf4 MMA with shared-memory descriptors, scale-factor TMEM addresses, and a TMEM accumulator destination.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}, {"locator": "GEMM kernel configuration and collective builders", "observation": "The official example instantiates nv_float4 E2M1 operands, block-scale factor layouts, a Blackwell collective mainloop, and a TMEM-backed universal kernel.", "revision": "NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3", "url": "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "Every NVFP4 GEMM is compute-bound, dominated by tensor-core throughput, and necessarily uses tcgen05.mma, TMA, TMEM, and warp specialization.", "claim_id": "kernel-nvfp4-gemm-original-c002", "claim_type": "performance", "covers": ["u001"], "evidence": [{"locator": "speed-of-light analysis and benchmark rows", "observation": "The official task models each benchmark by the maximum of FP4 math time and DRAM time, so the controlling resource is shape-dependent; it defines an output contract rather than mandating a kernel mechanism.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/task.yml", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6"}, {"locator": "threadblock rasterization, occupancy, and wave quantization discussion", "observation": "The official guide makes achieved GEMM throughput conditional on problem shape, tiling, occupancy, and scheduling rather than datatype alone.", "path": "/tmp/cutlass-audit.8aj07j/repo/media/docs/cpp/efficient_gemm.md", "revision": "NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "The page has conflated the GEMV or dual-GEMM problem with Problem 2, or the dates refer to a later public leaderboard period.", "check": "Compare the exact Kernel 2 row and contest-period clauses with the pinned nvfp4_gemm directory.", "outcome": "Both primary records identify the same GEMM task and B200 target; the rules, not later submissions, establish the contest window."}, "claim": "NVFP4 GEMM was Problem 2 of the NVIDIA and GPU Mode contest, targeted B200, ran from November 29 through December 19, 2025, and carried 20 percent grand-prize weight.", "claim_id": "kernel-nvfp4-gemm-original-c003", "claim_type": "other", "covers": ["u002"], "evidence": [{"locator": "contest period, Kernel 2 description, evaluation platform, and prize weighting", "observation": "The rules give the November 29 to December 19 period, identify Kernel 2 as NVFP4 GEMM on NVIDIA B200, and assign it 20 percent of the grand-prize score.", "revision": "official contest terms fetched 2026-08-08", "url": "https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf"}, {"locator": "description", "observation": "The immutable official problem specification calls for a block-scaled matrix-matrix multiplication optimized for NVIDIA B200.", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6", "url": "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The page's asserted top entries were within one percent of cuBLAS and used CUTLASS SM100 schedules.", "claim_id": "kernel-nvfp4-gemm-original-c004", "claim_type": "performance", "covers": ["u002"], "evidence": [{"locator": "rank, user_name, submission_score, and submission_time fields", "observation": "The public top three are gau.nernst, s.am._, and billcarson, not the cohort later asserted by the page; the endpoint publishes scores and filenames but no cuBLAS ratio or CUTLASS implementation attribution.", "revision": "public Popcorn leaderboard snapshot fetched 2026-08-08", "url": "https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12"}, {"locator": "ranking criteria and speed-of-light table", "observation": "The task ranks geometric-mean benchmark time and compares grand-prize entries with a theoretical math/DRAM speed-of-light model; it supplies no cuBLAS baseline or contestant implementation claims.", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6", "url": "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "NVFP4 is only the E2M1 encoding, or it uses the same block size and scale representation as MXFP4.", "check": "Separate payload from recipe metadata and compare the two format tables in the pinned documentation.", "outcome": "The two scale levels and block-of-16 versus block-of-32 distinction are explicit."}, "claim": "The generic NVFP4 recipe uses signed E2M1 payload values 0, 0.5, 1, 1.5, 2, 3, 4, and 6, one E4M3 local scale per 16 values, and a per-tensor FP32 global scale; MXFP4 instead groups 32 values with power-of-two UE8M0 local scales.", "claim_id": "kernel-nvfp4-gemm-original-c005", "claim_type": "code", "covers": ["u003"], "evidence": [{"locator": "E2M1 format, two-level scaling, and comparison with MXFP4", "observation": "The version-pinned vendor documentation defines the payload range, block-of-16 E4M3 local scaling, per-tensor FP32 scale, and MXFP4 contrast stated by the page.", "revision": "Transformer Engine 2.13", "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The symbol is fabricated or is a general single-GEMM schedule independent of pointer arrays.", "check": "Resolve the exact declaration and an upstream call site.", "outcome": "Both the inheritance and representative call site identify it as the pointer-array grouped schedule."}, "claim": "CUTLASS defines KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100 as a one-SM pointer-array NVFP4 block-scaled schedule and uses it in an official grouped-GEMM example.", "claim_id": "kernel-nvfp4-gemm-original-c006", "claim_type": "compatibility", "covers": ["u004"], "evidence": [{"locator": "lines 842-848, schedule declarations", "observation": "The exact symbol derives from the one-SM and pointer-array NVFP4 schedule marker types.", "path": "/tmp/cutlass-audit.8aj07j/repo/include/cutlass/gemm/dispatch_policy.hpp", "revision": "NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3"}, {"locator": "MMA1SMConfig", "observation": "The official grouped pointer-array example selects this schedule with an MmaTileShape of 128 by 256 by 256.", "path": "/tmp/cutlass-audit.8aj07j/repo/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "The GPU Mode Problem 2 kernel and its top performers are established to use KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100.", "claim_id": "kernel-nvfp4-gemm-original-c007", "claim_type": "compatibility", "covers": ["u004"], "evidence": [{"locator": "entire task contract", "observation": "The task defines tensors, tests, benchmarks, and scoring but no canonical implementation or CUTLASS schedule.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/task.yml", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6"}, {"locator": "problem-shape and schedule declarations", "observation": "The schedule's direct official example is a pointer-array grouped GEMM, not the contest's standard seven-tensor task contract; the page's bundled sources contain no contestant solution.", "path": "/tmp/cutlass-audit.8aj07j/repo/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu", "revision": "NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The displayed CUTLASS typedef is a valid CUTLASS 4.x NVFP4 GEMM definition for the asserted schedule and types.", "claim_id": "kernel-nvfp4-gemm-original-c008", "claim_type": "code", "covers": ["u005"], "evidence": [{"locator": "lines 95-164, operand types, collective builders, kernel, and adapter", "observation": "The official API wraps E2M1 in nv_float4_t, builds mainloop and epilogue collectives, instantiates cutlass::gemm::kernel::GemmUniversal with those collectives, then wraps it in device::GemmUniversalAdapter. The page instead passes obsolete scalar/layout arguments to device::GemmUniversal and omits the scale-factor interface.", "path": "/tmp/cutlass-audit.8aj07j/repo/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu", "revision": "NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The displayed hand-written CUDA is a valid warp-specialized NVFP4 kernel with correct TMA and tcgen05.mma instruction operands.", "claim_id": "kernel-nvfp4-gemm-original-c009", "claim_type": "code", "covers": ["u006"], "evidence": [{"locator": "lines 1647-1659, block16 NVFP4 asm wrapper", "observation": "The exact form is kind::mxf4nvf4.block_scale.block16 and requires D, A/B descriptors, an instruction descriptor, separate SFA/SFB TMEM addresses, and a predicate. The page emits kind::f8f6f4 with a different and incomplete operand list.", "revision": "NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3", "url": "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cute/arch/mma_sm100_umma.hpp#L1647-L1659"}, {"locator": "tcgen05.mma syntax and restrictions", "observation": "The authoritative grammar contradicts the displayed instruction. The surrounding listing also contains undefined types, descriptors, helpers, barriers, and tile symbols and therefore is not executable code.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Every TMA operand requires 128-byte alignment, universally implying an FP4 K dimension divisible by 256.", "claim_id": "kernel-nvfp4-gemm-original-c010", "claim_type": "mechanism", "covers": ["u007", "u020"], "evidence": [{"locator": "cuTensorMapEncodeTiled parameters and restrictions", "observation": "The general tiled tensor-map contract uses 16-byte global-address and stride constraints, with mode-specific additional rules; it does not impose one universal 128-byte rule on all operands. Problem 2's K-divisible-by-256 rule is therefore an interface constraint, not a universal TMA theorem.", "revision": "CUDA Driver API 13.2.1", "url": "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The displayed scale-array assertion (K/16) divisible by 128 is valid for the official Problem 2 input space.", "claim_id": "kernel-nvfp4-gemm-original-c022", "claim_type": "code", "covers": ["u008"], "evidence": [{"command": "conda run -n base python -c ''", "environment": "KernelWiki host; conda base standard-library Python", "negative_control": "K=255 was rejected by the task-specific K-divisible-by-256 validator.", "reference": "gpu-mode/reference-kernels ae679486 task.yml and reference.py shape construction", "result": "PASS: all ten official shapes satisfy K modulo 256; nine valid tests, including K=256, are rejected by (K/16) modulo 128."}], "oracle": "host-experiment", "verdict": "refuted"}, {"claim": "NVFP4 tcgen05.mma expects UE8M0 scales, so E4M3 block scales must be rounded to powers of two and packed before MMA, adding conversion overhead.", "claim_id": "kernel-nvfp4-gemm-original-c011", "claim_type": "code", "covers": ["u009", "u010", "u019"], "evidence": [{"locator": "lines 494-512, mx_float4_t and nv_float4_t", "observation": "CUTLASS assigns UE8M0 to MXFP4 but UE4M3 to NVFP4, directly separating the two recipes.", "revision": "NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3", "url": "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512"}, {"locator": "FP4 one-dimensional block scaling and CUBLASLT_MATMUL_MATRIX_SCALE_VEC16_UE4M3", "observation": "The documented FP4 NVFP4 block-scale mode uses UE4M3 for each group of 16. UE8M0 is a different scale format, so the proposed conversion would discard the defining fractional scale information.", "revision": "cuBLAS documentation fetched 2026-08-08", "url": "https://docs.nvidia.com/cuda/cublas/index.html#element-1d-block-scaling-for-fp8-and-fp4-data-types"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Simon, yue, and currybab were ranks 1, 2, and 3 with geometric-mean scores 10.807, 10.914, and 10.931 microseconds.", "claim_id": "kernel-nvfp4-gemm-original-c012", "claim_type": "performance", "covers": ["u011", "u012", "u013", "u014"], "evidence": [{"locator": "rows 1-10", "observation": "The rounded values do match Simon, yue, and currybab scores, but the endpoint places them at ranks 8, 9, and 10. Ranks 1-3 are gau.nernst at 9.981888843 us, s.am._ at 10.060110139 us, and billcarson at 10.137410720 us.", "revision": "public Popcorn leaderboard snapshot fetched 2026-08-08", "url": "https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12"}, {"locator": "contest period", "observation": "The contest ended December 19, whereas the public current top three have December 20-21 timestamps, so the snapshot cannot by itself be called the official prize-placement table.", "revision": "official rules fetched 2026-08-08", "url": "https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "This NVFP4 task is generally suitable for inference with any 4-bit quantized weights on Blackwell.", "claim_id": "kernel-nvfp4-gemm-original-c013", "claim_type": "compatibility", "covers": ["u015"], "evidence": [{"locator": "NVFP4 recipe and comparison with MXFP4", "observation": "NVFP4 requires a specific E2M1 payload, block-of-16 E4M3 scaling, and global scaling recipe; generic 4-bit weights may instead be INT4, MXFP4, or use incompatible layouts and scales.", "revision": "Transformer Engine 2.13", "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"}, {"locator": "custom_kernel tensor contract", "observation": "The task requires exact packed E2M1 operands plus logical and reordered block-scale tensors, not an arbitrary 4-bit weight tensor.", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6", "url": "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Every transformer MLP accepts the task ABI, or tensor-core tuning matters even when memory or launch overhead dominates.", "check": "Scope applicability to matching payload/scales/layouts and the bottleneck to a measured or modeled shape.", "outcome": "The sources support the conditional use, not a universal workload classification."}, "claim": "An NVFP4 GEMM can implement an MLP matrix multiplication when its weights and activation path satisfy the exact NVFP4 contract; if profiling shows the tensor-core path is the bottleneck, tensor-core scheduling is the relevant optimization target.", "claim_id": "kernel-nvfp4-gemm-original-c014", "claim_type": "compatibility", "covers": ["u016", "u017"], "evidence": [{"locator": "NVFP4 inference and GEMM discussion", "observation": "NVIDIA describes NVFP4 as a Blackwell low-precision inference format used for matrix multiplications in transformer layers.", "revision": "NVIDIA technical blog fetched 2026-08-08", "url": "https://developer.nvidia.com/blog/introducing-nvfp4-for-efficient-and-accurate-low-precision-inference/"}, {"locator": "speed-of-light model", "observation": "The task explicitly models both FP4 tensor-core math and DRAM throughput; the slower component determines the bound for a particular shape.", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6", "url": "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The statement forbids all software emulation or future library support on non-Blackwell devices.", "check": "Distinguish native tensor-core instruction support from software conversion or emulation.", "outcome": "The evidence proves only the native hardware path boundary, which is the scope of the claim."}, "claim": "The documented native NVFP4 tensor-core path is Blackwell-specific (including SM100/SM100a targets) and Hopper has no native FP4 tensor-core instruction path.", "claim_id": "kernel-nvfp4-gemm-original-c015", "claim_type": "compatibility", "covers": ["u018"], "evidence": [{"locator": "Blackwell Tensor Core support", "observation": "NVIDIA attributes native NVFP4 processing to Blackwell's fifth-generation Tensor Cores.", "revision": "NVIDIA technical blog fetched 2026-08-08", "url": "https://developer.nvidia.com/blog/introducing-nvfp4-for-efficient-and-accurate-low-precision-inference/"}, {"locator": "target ISA notes for block-scaled tcgen05.mma", "observation": "The NVFP4 forms require architecture-specific Blackwell targets; Hopper's wgmma instruction family has no native FP4 form.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "TMEM's 128-lane by 512-column organization limits every logical output tile to exactly 128 rows by 512 columns.", "claim_id": "kernel-nvfp4-gemm-original-c016", "claim_type": "compatibility", "covers": ["u021"], "evidence": [{"locator": "MMA1SMConfig and MMA2SMConfig", "observation": "The official NVFP4 grouped example defines 128 by 256 by 256 for one-SM MMA and 256 by 256 by 256 for cooperative two-SM MMA, demonstrating that a logical M tile need not be limited to 128.", "revision": "NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3", "url": "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu#L147-L159"}, {"locator": "tensor-memory organization and allocation", "observation": "The 128-lane by 512-column description is a storage/addressing organization per CTA allocation, not a universal logical GEMM output-shape ceiling.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "A URL succeeds only through a soft 404 or redirects to an unrelated landing page.", "check": "Inspect final page titles/content and compare a same-site invalid path.", "outcome": "Each positive target has the stated identity and the negative path is discriminated as 404."}, "claim": "The four listed source hyperlinks resolve to the intended GPU Mode repository, NVIDIA NVFP4 article, Harold Benoit NVFP4 notes, and CUTLASS changelog.", "claim_id": "kernel-nvfp4-gemm-original-c017", "claim_type": "other", "covers": ["u022", "u023", "u024", "u025"], "evidence": [{"command": "curl -L -sS -o /dev/null -w '%{http_code}' ", "environment": "KernelWiki host, 2026-08-08", "negative_control": "A nonexistent sibling page under haroldbenoit.com returned HTTP 404.", "reference": "link status and final document identity", "result": "All four URLs returned HTTP 200 and resolved to the named resources."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "The path, hashes, or derived labels are absent even though the prose says they exist.", "check": "Resolve every listed path and compare digest/provenance fields.", "outcome": "The structural provenance metadata exists as described; that does not establish semantic relevance or code validity."}, "claim": "The artifact files and their provenance manifests exist at the stated paths and record the listed hashes, modes, source locations, and CUTLASS revision.", "claim_id": "kernel-nvfp4-gemm-original-c018", "claim_type": "other", "covers": ["u026"], "evidence": [{"locator": "files entries", "observation": "The manifest names the PR patch and extracted snippet with their SHA-256 values, modes, upstream location or source path, and ca4fdbea revision; the variants manifest labels the sketch as derived.", "path": "artifacts/kernels/nvfp4-gemm/full/PROVENANCE.yaml", "revision": "repository baseline"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "The artifact bundle is a full reference implementation of GPU Mode Problem 2 NVFP4 GEMM.", "claim_id": "kernel-nvfp4-gemm-original-c019", "claim_type": "code", "covers": ["u026"], "evidence": [{"locator": "examples 67, 68, and 81 and their ElementA/ElementB declarations", "observation": "The patch implements FP8 E4M3 blockwise/groupwise examples and contains no NVFP4 contest implementation.", "path": "artifacts/kernels/nvfp4-gemm/full/PR-2139-blockwise-groupwise-gemm.patch", "revision": "CUTLASS merge ca4fdbea; artifact SHA-256 e6c54c39519ee3ac609efdef6f8a9b23a56deaed34153e69ae75cf5892d93fc1"}, {"locator": "entire extracted file", "observation": "The file is comments plus one blockscale_offsets assignment extracted from a third-party blog summary, not a callable GEMM.", "path": "artifacts/kernels/nvfp4-gemm/full/blackwell-cutlass-schedules-and-tma.cu", "revision": "artifact SHA-256 e720037bbdb5ee5054259a331cdaf489e7135a44038d66d1fd9d9e5998b75255"}, {"locator": "entire file", "observation": "The explicitly derived sketch uses undefined helpers and incorrect tcgen05 scale/type syntax, so it is neither upstream code nor a working reference.", "path": "artifacts/kernels/nvfp4-gemm/variants/01-nvfp4-gemm-skeleton.cu", "revision": "repository baseline derived variant"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "The flag is unsupported or the artifact_dir does not resolve.", "check": "Run the exact command and inspect artifact filenames, then use an invalid ID.", "outcome": "The valid path and flag resolve; the invalid identity is rejected."}, "claim": "The displayed get_page command retrieves this page and includes the attached artifact bundle.", "claim_id": "kernel-nvfp4-gemm-original-c020", "claim_type": "code", "covers": ["u027", "u028"], "evidence": [{"command": "conda run -n base python scripts/get_page.py kernel-nvfp4-gemm --include-code", "environment": "KernelWiki repository; conda base, 2026-08-08", "negative_control": "The same command with a nonexistent page ID exited nonzero.", "reference": "repository query utility", "result": "Exited 0 and emitted the page plus both full-bundle files and provenance records."}], "oracle": "host-experiment", "verdict": "verified"}, {"claim": "The structured 10.807 record is one B200 standard-shape latency in microseconds and establishes near-cuBLAS utilization.", "claim_id": "kernel-nvfp4-gemm-original-c021", "claim_type": "performance", "covers": ["u029"], "evidence": [{"locator": "Simon row, submission_score 1.0806750492229971e-05", "observation": "The rounded value comes from Simon's leaderboard score and current rank 8, not a latency for one identified shape.", "revision": "public Popcorn leaderboard snapshot fetched 2026-08-08", "url": "https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12"}, {"locator": "three benchmark rows and ranking_by geom", "observation": "The score is the geometric mean across three explicit shapes. The source provides theoretical speed-of-light rows but no cuBLAS measurement, software stack, trials, variance, or near-cuBLAS utilization field.", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6", "url": "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml"}], "oracle": "web-primary", "verdict": "refuted"}], "commands": ["inspect the immutable official Problem 2 task.yml, task.py, reference.py, and template.py at gpu-mode/reference-kernels ae679486", "fetch the public Popcorn leaderboard API and inspect rank, timestamp, identity, and score fields", "inspect CUTLASS e05f953a declarations, official examples, NVFP4 scale types, and exact tcgen05 asm wrappers", "run the official ten task shapes through a host shape oracle and challenge the old scale alignment assertion", "run primary-link positive controls, same-host 404 controls, and the get_page positive/negative query", "inspect the artifact patch, extracted snippet, derived variant, and provenance manifests"], "id": "kernel-nvfp4-gemm", "page_verdict": "refuted", "path": "wiki/kernels/nvfp4-gemm.md", "residual_risk": "No NVIDIA GPU or CUDA toolkit is available locally. No contestant source is exposed by the public leaderboard, so no implementation or performance result was reproduced. The current leaderboard includes submissions after the rules' December 19 cutoff and is retained only as a dated public snapshot, not as a prize-placement oracle.", "scope": "All 29 baseline units for body hash d176748702ac353341b6d4887c74d36b98deccce13fd42849ac545957b212fa0 were checked against official contest/task records, the public leaderboard API, version-pinned CUTLASS source, version-pinned NVIDIA documentation, artifact contents, and discriminating host controls."} +{"claims": [{"challenge": {"alternative": "The task is a general GEMM or its bound is necessarily tensor-core compute throughput.", "check": "Inspect the pinned task operation, N dimension, and stated max-of-FFMA-and-DRAM model.", "outcome": "The output N dimension is one and the task explicitly describes batched GEMV; its stated benchmark numbers use the DRAM side of the model."}, "claim": "The official Problem 1 task is an NVFP4 block-scaled batched matrix-vector product evaluated on NVIDIA B200, and its published benchmark model treats the listed cases as DRAM-limited relative to FFMA throughput.", "claim_id": "kernel-nvfp4-gemv-original-c001", "claim_type": "mechanism", "covers": ["u001"], "evidence": [{"locator": "description and speed-of-light analysis", "observation": "The task names batched NVFP4 GEMV on B200 and publishes a speed-of-light table based on max(FFMA math time, DRAM time); the author worklog independently identifies the submitted CUDA kernel as memory-bound.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/task.yml", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "Every FP4 element is used only once and all NVFP4 GEMV kernels are dominated by bandwidth, making PTX cache control and register budgeting universally critical.", "claim_id": "kernel-nvfp4-gemv-original-c002", "claim_type": "performance", "covers": ["u001"], "evidence": [{"locator": "ref_kernel matrix multiplication and result-column selection", "observation": "Each logical B value participates in every one of the M output rows, so not every FP4 value is used once; the task constrains results rather than prescribing PTX or register controls.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/reference.py", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6"}, {"locator": "attempts 9-12 and takeaways", "observation": "The author reports wider loads becoming 16-25 percent slower and a register-limit change having zero effect, contradicting the universal lever claims.", "path": "/tmp/kernelwiki-amandeep-nvfp4.html", "revision": "author URL fetched 2026-08-08"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "The 8.6 value belongs to a different benchmark row or the 8 TB/s number is a dual-GPU aggregate.", "check": "Match the exact task table row and the technical brief's one-GPU wording.", "outcome": "Both sources scope the numbers to the asserted row and one GPU respectively."}, "claim": "This was Problem 1 of the November 2025 hackathon; the M=7168, K=16384, L=1 row has an 8.622 microsecond theoretical time, and NVIDIA documents 8 TB/s HBM3e bandwidth for one GB200 GPU.", "claim_id": "kernel-nvfp4-gemv-original-c003", "claim_type": "performance", "covers": ["u002"], "evidence": [{"locator": "Kernel 1 description and contest period", "observation": "The rules identify NVFP4 batched GEMV as Kernel 1 and give November 10 through November 28, 2025.", "revision": "official rules fetched 2026-08-08", "url": "https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf"}, {"locator": "speed-of-light table", "observation": "The row for M=7168, K=16384, L=1 is 8.622 microseconds at the stated B200 clock.", "revision": "ae679486", "url": "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml"}, {"locator": "GB200 GPU memory subsystem", "observation": "The brief specifies 8 TB/s HBM3e bandwidth for one GB200 GPU.", "revision": "NVIDIA Blackwell Architecture Technical Brief fetched 2026-08-08", "url": "https://resources.nvidia.com/en-us-blackwell-architecture/blackwell-architecture-technical-brief"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "Top performers achieved about 22.4 microseconds, 2.6 times the 8.6 microsecond bound, and the structured 22.4 value is the latency of the M=7168, K=16384, L=1 shape.", "claim_id": "kernel-nvfp4-gemv-original-c004", "claim_type": "performance", "covers": ["u002", "u042"], "evidence": [{"locator": "ranks 1-3 and Yue row", "observation": "The current rank-1 geometric-mean score is 18.549562452 microseconds; Yue's 22.392217755 score is rank 11. The API reports aggregate scores, not per-shape timings or causal attribution.", "revision": "public leaderboard snapshot fetched 2026-08-08", "url": "https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemv/NVIDIA?limit=100"}, {"locator": "benchmarks and ranking_by", "observation": "The task ranks the geometric mean of three benchmark rows, so 22.392 cannot be assigned to the one frontmatter shape.", "revision": "ae679486", "url": "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The page copied the GEMM task or a prior benchmark set.", "check": "Compare the pinned nvfp4_gemv directory's N=1 output and every benchmark row.", "outcome": "The operation and all three rows match Problem 1's pinned task."}, "claim": "The task's logical operation has packed E2M1 A and B operands, one E4M3 scale per 16 logical values, FP16 output shaped M by 1 by L, and the three displayed benchmark configurations.", "claim_id": "kernel-nvfp4-gemv-original-c005", "claim_type": "code", "covers": ["u003"], "evidence": [{"locator": "description and benchmarks", "observation": "The logical operation, block size, FP16 output, and all three benchmark dimensions match.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/task.yml", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "The actual task ABI consists of five tensors plus two FP32 per-tensor global scales, with B and SFB physically having one row and the displayed global-scale dequantization formula.", "claim_id": "kernel-nvfp4-gemv-original-c006", "claim_type": "code", "covers": ["u003"], "evidence": [{"locator": "input_t", "observation": "The callable input is typed as a seven-tensor tuple.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/task.py", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6"}, {"locator": "generate_input return tuple and create_scale_factor_tensors", "observation": "The harness returns A, B, logical SFA/SFB, reordered SFA/SFB, and C; B and both B-scale forms are padded to 128 rows, and no FP32 global-scale arguments exist. A conda-base host reconstruction matched all seven largest-row shapes, accepted K=64, and rejected K=63.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/reference.py", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "The source is a scalar FP4 or mov.b32 cannot target byte lanes.", "check": "Check packed-type widths and the scalar-to-vector type-size rule.", "outcome": "The source is e2m1x2 and four b8 destinations total the b32 source size."}, "claim": "PTX ISA 9.0 defines cvt.rn.f16x2.e2m1x2 for packed FP4-pair conversion and typed mov.b32 scalar-to-vector decomposition can split a 32-bit register into four bytes.", "claim_id": "kernel-nvfp4-gemv-original-c007", "claim_type": "code", "covers": ["u005"], "evidence": [{"locator": "packed alternate-format cvt syntax", "observation": "The exact E2M1x2-to-F16x2 instruction form is documented.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt"}, {"locator": "scalar-to-vector move rules", "observation": "The typed move rules permit b32 decomposition into four byte-sized vector elements.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "Raw PTX itself produced the stated critical gain over C intrinsics and mov.b32 alone is faster than a shift-mask chain.", "claim_id": "kernel-nvfp4-gemv-original-c008", "claim_type": "performance", "covers": ["u004", "u005"], "evidence": [{"locator": "Optimization 2 and Optimization 3", "observation": "The 443-to-39 change bundled removal of shared memory, per-thread tiles, float4 loads, and hardware intrinsics; the 39-to-about-27 change bundled vectorized PTX decode. No isolated mov or C-versus-PTX trial is reported.", "revision": "author post fetched 2026-08-08", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}, {"locator": "mov semantics", "observation": "The ISA specifies semantics but no timing advantage over compiler-generated bit operations.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The qualifiers are invented syntax or were recommendations made by the wiki rather than observations in the source.", "check": "Resolve both qualifier forms in PTX and inspect the author's top-solution section.", "outcome": "Both syntax and the evidence scope as an author-reported observation are explicit."}, "claim": "Amandeep reports that the solutions he studied used A loads with L1::no_allocate, B loads with L1::evict_last, and per-K load-qualifier tuning.", "claim_id": "kernel-nvfp4-gemv-original-c009", "claim_type": "code", "covers": ["u006", "u007"], "evidence": [{"locator": "What the Top Solutions Did Differently", "observation": "The author explicitly reports the two cache policies and per-K qualifier selection as observations from solutions studied after the event.", "revision": "author post fetched 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}, {"locator": "ld.global cache eviction priority hints", "observation": "The qualifier spellings and their cache-hint meanings are valid PTX.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "Lower register limits necessarily raise occupancy and performance; ranks 1 and 3 prove 32 versus 45 registers makes occupancy dominant, and the shown launch bounds are part of that evidence.", "claim_id": "kernel-nvfp4-gemv-original-c010", "claim_type": "code", "covers": ["u008", "u009"], "evidence": [{"locator": "attempt 11 and top-solution register budgets", "observation": "The author reports 32 and 45 limits in inspected solutions but also reports that reducing his own limit from 80 to 64 had zero effect because natural use was already below 64. He supplies no controlled 32-versus-45 comparison or the displayed launch-bounds declaration.", "revision": "author post fetched 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The implementations used one runtime K loop and the wiki inferred templates.", "check": "Inspect the author's explicit description of templating and dispatch.", "outcome": "The source directly states separate exact-K specializations."}, "claim": "Amandeep reports that inspected solutions templated on exact K and dispatched separate specializations so the compiler could unroll known trip counts and tune allocation per size.", "claim_id": "kernel-nvfp4-gemv-original-c011", "claim_type": "mechanism", "covers": ["u010"], "evidence": [{"locator": "What the Top Solutions Did Differently, per-K paragraph", "observation": "The author describes exact-K templates, launch-time dispatch, full unrolling, and size-specific register allocation.", "revision": "author post fetched 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The displayed CUDA template is a usable representation of the official task ABI and its per-K dispatch values.", "claim_id": "kernel-nvfp4-gemv-original-c012", "claim_type": "code", "covers": ["u011"], "evidence": [{"locator": "custom_kernel signature", "observation": "The official entry receives seven tensors, including two reordered scale tensors; it receives no global FP32 scales.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/template.py", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6"}, {"locator": "u011", "observation": "The listing contains undefined THREADS, MIN_BLOCKS, ELEMENTS_PER_LOAD, load_fp4, and conversion helpers, uses false global-scale parameters, and gives K dispatch values inconsistent with the official benchmark K values.", "path": "wiki/kernels/nvfp4-gemv.md", "revision": "baseline body 55d84be3"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "128- and 256-bit loads maximize bandwidth and are effective only with PTX byte unpacking.", "claim_id": "kernel-nvfp4-gemv-original-c013", "claim_type": "performance", "covers": ["u012", "u013"], "evidence": [{"locator": "attempt 9 and top-solution load-width discussion", "observation": "The author's controlled widening from two uchar4 loads to one uint2 load was 16-25 percent slower because unpacking added instructions; inspected solutions used wider PTX loads, but the source does not establish a universal maximizing width or PTX mov as the only usable decode path.", "revision": "author post fetched 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The vector suffix counts bytes or FP4 lanes rather than 64-bit registers.", "check": "Apply PTX vector arity and u64 width.", "outcome": "The total widths are 128 and 256 bits, yielding the stated packed-element capacities."}, "claim": "The displayed v2.u64 and v4.u64 PTX forms move 16 and 32 bytes, enough to carry 32 and 64 packed FP4 values respectively.", "claim_id": "kernel-nvfp4-gemv-original-c014", "claim_type": "code", "covers": ["u013"], "evidence": [{"locator": "vector ld syntax and type sizes", "observation": "Two or four 64-bit destinations carry 128 or 256 bits; two packed 4-bit values occupy each byte.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "All 128 physical B rows contribute to an M-by-128 output contract.", "check": "Trace the result slice copied into C.", "outcome": "Only res[:,0] is copied into an M-by-1 output."}, "claim": "The logical N=1 B vector is reused across all M output rows, although the official reference physically pads B and its scales to 128 rows.", "claim_id": "kernel-nvfp4-gemv-original-c015", "claim_type": "mechanism", "covers": ["u014"], "evidence": [{"locator": "n=1, n_padded_128, scaled_mm, and c_ref[:,0] assignment", "observation": "The harness pads B to 128 for scaled_mm but stores only result column zero, so one logical B row contributes to each M output row.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/reference.py", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "The shown shared-memory B kernel is the rank-2 approach and reduces global traffic by exactly BLOCK_M.", "claim_id": "kernel-nvfp4-gemv-original-c016", "claim_type": "code", "covers": ["u015"], "evidence": [{"locator": "rank-2 reuse paragraph", "observation": "The author reports shared B reads across rows but publishes neither this CUDA listing nor an exact BLOCK_M traffic ratio.", "revision": "author post fetched 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}, {"locator": "failed CuTe ideas and CUDA Optimization 2", "observation": "Loading the whole B vector into shared memory gave no improvement in one attempt, and Yue's faster 443-to-39 change explicitly removed B shared memory.", "revision": "author post fetched 2026-08-08", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "22.392 is one shape latency or the intermediate numbers belong to another contestant.", "check": "Match the author page's headings and compare the leaderboard score field.", "outcome": "The intermediate values are in Yue's own progression and 22.392 is her three-row geometric-mean score."}, "claim": "Yue reports a CuTe baseline around 100 microseconds, a 39 microsecond CUDA stage, a 39-to-about-27 microsecond vectorized PTX-decode stage, 26-to-about-22.9 microseconds from two-tile ILP, and a final submitted geometric-mean score of 22.392 microseconds.", "claim_id": "kernel-nvfp4-gemv-original-c017", "claim_type": "performance", "covers": ["u016", "u017", "u019", "u020", "u021", "u022"], "evidence": [{"locator": "CuTe attempt, CUDA Optimizations 2-6, and introduction", "observation": "The author reports each stated stage and final submission score; the timings are explicitly scoped to the author's journey, not independent reproduction.", "revision": "author post fetched 2026-08-08", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}, {"locator": "Yue row", "observation": "The public endpoint reports Yue's aggregate score as 22.392217755 microseconds.", "revision": "snapshot fetched 2026-08-08", "url": "https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemv/NVIDIA?limit=100"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "Coalescing alone produced the 443-to-39 microsecond change.", "claim_id": "kernel-nvfp4-gemv-original-c018", "claim_type": "performance", "covers": ["u018"], "evidence": [{"locator": "Optimization 1 and Optimization 2 headings", "observation": "Coalescing, collaboration, and reduction produced the earlier roughly 2000-to-443 step. The 443-to-39 step bundled removal of shared memory, vectorized loads, per-thread tiles, and hardware intrinsics.", "revision": "author post fetched 2026-08-08", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "For memory-bound kernels arithmetic optimization has minimal impact and optimization should focus on bandwidth, cache policy, and vectorized loads.", "claim_id": "kernel-nvfp4-gemv-original-c019", "claim_type": "performance", "covers": ["u023"], "evidence": [{"locator": "Optimization 2, NCU observation, and PTX decode progression", "observation": "The author attributes a major stage to reducing roughly tenfold excess instruction issue and accelerating FP4/scale decode, demonstrating that instruction-side work was material even in this bandwidth-limited task.", "revision": "author post fetched 2026-08-08", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The 443-to-27 microsecond difference measures C intrinsics versus hand-written PTX.", "claim_id": "kernel-nvfp4-gemv-original-c020", "claim_type": "performance", "covers": ["u024"], "evidence": [{"locator": "CUDA Optimization 2 and Optimization 3", "observation": "The first stage combines four material changes and ends at 39; the next combined vectorized PTX decode stage reaches about 27. There is no isolated 443-to-27 C-versus-PTX trial.", "revision": "author post fetched 2026-08-08", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The quote is invented or describes a different tool and task.", "check": "Inspect the takeaways in the named NVFP4 GEMV worklog.", "outcome": "The tool, task, and retrospective lesson all match."}, "claim": "Amandeep's retrospective lesson is to profile with Nsight Compute early and confirm the kernel bottleneck before choosing optimizations.", "claim_id": "kernel-nvfp4-gemv-original-c021", "claim_type": "other", "covers": ["u025"], "evidence": [{"locator": "takeaways", "observation": "The author explicitly says the most important missed step after attempt 7 was running Nsight Compute to confirm the kernel was memory-bound.", "revision": "author post fetched 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "Reducing register limits reliably improves this kernel by increasing occupancy and hiding latency.", "claim_id": "kernel-nvfp4-gemv-original-c022", "claim_type": "performance", "covers": ["u026"], "evidence": [{"locator": "attempt 11", "observation": "Reducing maxrregcount from 80 to 64 had zero effect because the kernel already used fewer than 64 registers; occupancy and performance are conditional on realized resource use.", "revision": "author post fetched 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "TMEM is irrelevant to every memory-bound kernel and can help compute-bound kernels only.", "claim_id": "kernel-nvfp4-gemv-original-c023", "claim_type": "compatibility", "covers": ["u027"], "evidence": [{"locator": "Tensor Memory and tcgen05 instruction overview", "observation": "TMEM is an explicitly managed storage space used by tcgen05 operations and data movement; its availability and semantics are not conditioned on a whole-kernel compute-bound classification.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"}, {"locator": "section 3.1 forward pipeline", "observation": "A fused attention kernel uses separate TMEM regions as part of an overlap schedule involving MMA and non-MMA softmax work, contradicting the compute-bound-only categorical scope.", "revision": "FlashAttention-4 paper v1", "url": "https://arxiv.org/html/2603.05451v1"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Any batch-one matrix-vector tensor can be passed directly regardless of quantization recipe or measured bottleneck.", "check": "Compare the task ABI and vendor NVFP4 recipe with the conditional wording.", "outcome": "The sources support the use only for matching packed payloads/scales/layouts and a diagnosed low-intensity case."}, "claim": "A batch-one decode matrix-vector operation is a plausible use of this kernel pattern when operands satisfy the exact NVFP4 contract and profiling or the task model shows low arithmetic intensity relative to available math throughput.", "claim_id": "kernel-nvfp4-gemv-original-c024", "claim_type": "compatibility", "covers": ["u028", "u029"], "evidence": [{"locator": "operation and speed-of-light model", "observation": "The exact task is an N=1 matrix-vector operation and explicitly compares arithmetic with DRAM time.", "revision": "ae679486", "url": "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml"}, {"locator": "NVFP4 inference support and recipe", "observation": "The vendor documents NVFP4 for inference but requires its exact E2M1, block-scale, and global-scale recipe.", "revision": "Transformer Engine 2.13", "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The pattern applies to memory-bound operations with arbitrary FP4-quantized weights.", "claim_id": "kernel-nvfp4-gemv-original-c025", "claim_type": "compatibility", "covers": ["u030"], "evidence": [{"locator": "data format and MXFP4 comparison", "observation": "NVFP4 is a specific E2M1 plus E4M3-block plus FP32-global recipe; other FP4 formats and scale layouts are not ABI-compatible.", "revision": "Transformer Engine 2.13", "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Native FP4 decode is limited to SM100 and SM100a only.", "claim_id": "kernel-nvfp4-gemv-original-c026", "claim_type": "compatibility", "covers": ["u031"], "evidence": [{"locator": "Supported devices", "observation": "The vendor lists NVFP4 inference support as SM 10.0 and later rather than only two targets.", "revision": "Transformer Engine 2.13", "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"}, {"locator": "release feature table for packed alternate-format cvt", "observation": "The E2M1x2 conversion is listed for sm_100f, sm_101f, and sm_120f targets, directly exceeding SM100/SM100a-only scope.", "revision": "CUDA 12.9.2, PTX ISA 8.8", "url": "https://docs.nvidia.com/cuda/archive/12.9.2/parallel-thread-execution/index.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "PTX is a target-independent stable assembly language and the same inline text is guaranteed indefinitely.", "check": "Compare version and target notes in adjacent archived PTX releases.", "outcome": "The archive explicitly records versioned feature and target changes."}, "claim": "Inline PTX optimizations require revalidation across toolkit and target versions because instruction grammar and legal targets are versioned.", "claim_id": "kernel-nvfp4-gemv-original-c027", "claim_type": "compatibility", "covers": ["u032"], "evidence": [{"locator": "PTX versioning and target ISA notes", "observation": "The ISA versions instruction introduction, syntax, and legal target architecture features.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"locator": "target architecture notes", "observation": "A neighboring release changes target naming and adds target families, demonstrating why embedded PTX should be rechecked.", "revision": "CUDA 13.2.0, PTX ISA 9.2", "url": "https://docs.nvidia.com/cuda/archive/13.2.0/parallel-thread-execution/index.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Only one K specialization is emitted and all other variants are dead-stripped.", "check": "Apply the source's launch-time dispatch across multiple supported K values.", "outcome": "Dispatch requires each referenced K entry to be available; the page's caveat correctly identifies the code-size tradeoff, not a measured byte count."}, "claim": "Keeping separately dispatchable exact-K kernel specializations creates one retained kernel entry per selected K and trades a generic loop for additional compiled code.", "claim_id": "kernel-nvfp4-gemv-original-c028", "claim_type": "mechanism", "covers": ["u033"], "evidence": [{"locator": "per-K specialization paragraph", "observation": "The author describes exact-K templates dispatched at launch, which requires retaining the selected specializations rather than one runtime-K body.", "revision": "author post fetched 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The task's table is a universal B200 limit or 8 TB/s belongs to a two-GPU superchip.", "check": "Keep the assertion scoped to this task model and inspect the brief's per-GPU label.", "outcome": "The page's caveat is supportable only in the narrowed task-specific scope."}, "claim": "For this B200 task the published speed-of-light model is bounded by DRAM throughput, with 8 TB/s as the documented one-GPU GB200 bandwidth assumption.", "claim_id": "kernel-nvfp4-gemv-original-c029", "claim_type": "performance", "covers": ["u034"], "evidence": [{"locator": "speed-of-light analysis", "observation": "The task states its B200 max-of-FFMA-and-DRAM model and the three theoretical rows.", "revision": "ae679486", "url": "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml"}, {"locator": "one GB200 GPU memory bandwidth", "observation": "The vendor gives 8 TB/s HBM3e per GPU.", "revision": "official technical brief fetched 2026-08-08", "url": "https://resources.nvidia.com/en-us-blackwell-architecture/blackwell-architecture-technical-brief"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A link redirects to an unrelated page or succeeds only for arbitrary missing slugs.", "check": "Resolve with redirects and compare against same-site missing-page controls.", "outcome": "The named pages returned 200 while both missing-page controls returned 404."}, "claim": "The four source links resolve to the intended official task repository and the Yue, Amandeep, and Simon author posts.", "claim_id": "kernel-nvfp4-gemv-original-c030", "claim_type": "other", "covers": ["u035", "u036", "u037", "u038"], "evidence": [{"command": "curl -L -A 'Mozilla/5.0' -sS -o /dev/null -w '%{http_code}' ", "environment": "KernelWiki host, network fetched 2026-08-08", "negative_control": "Invented same-site Simon and Yue article slugs returned HTTP 404.", "reference": "Each label should resolve to its named source.", "result": "All official/pinned task and three author URLs returned HTTP 200."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "The paths are missing or the supposedly verbatim file differs from its ledger digest.", "check": "Resolve both directories and recompute SHA-256.", "outcome": "The paths exist and the file digest equals the provenance value."}, "claim": "The artifact directories exist and the full directory contains a byte-verified verbatim vLLM source file at the recorded upstream revision, while the variants are explicitly labeled derived.", "claim_id": "kernel-nvfp4-gemv-original-c031", "claim_type": "other", "covers": ["u039"], "evidence": [{"locator": "origin, asset_mode, upstream_sha, and files", "observation": "The record pins vLLM PR 16032 revision ed7a29d9 and SHA-256 e8aed5cc for nvfp4_scaled_mm_kernels.cu; the computed file digest matches.", "path": "artifacts/kernels/nvfp4-gemv/full/PROVENANCE.yaml", "revision": "local provenance record retrieved 2026-04-17"}, {"locator": "asset_mode and files", "observation": "The variants are labeled derived rather than verbatim upstream implementations.", "path": "artifacts/kernels/nvfp4-gemv/variants/PROVENANCE.yaml", "revision": "local provenance record"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "The attached full artifact is a reference implementation of the contest batched GEMV task.", "claim_id": "kernel-nvfp4-gemv-original-c032", "claim_type": "code", "covers": ["u039"], "evidence": [{"locator": "Fp4GemmSm100, args_from_options, and runGemm", "observation": "The file implements a general M,N,K CUTLASS NVFP4 GEMM adapter and does not implement the contest's seven-tensor N=1 callable or checker.", "path": "artifacts/kernels/nvfp4-gemv/full/nvfp4_scaled_mm_kernels.cu", "revision": "vLLM ed7a29d9; SHA-256 e8aed5ccb3dd9de26c3aeff159a242a46dcb7c7d8d0351b6c44ff1f8d2f7effa"}, {"locator": "custom_kernel contract", "observation": "The contest entry point receives seven task tensors, which the vLLM source does not expose.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/template.py", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "The command silently returns a similarly named page or accepts any identifier.", "check": "Inspect the returned path and run an invalid-ID control.", "outcome": "The exact path was returned and the invalid ID was rejected."}, "claim": "The documented get_page command returns this page with attached code, while an invalid page ID is rejected.", "claim_id": "kernel-nvfp4-gemv-original-c033", "claim_type": "code", "covers": ["u040", "u041"], "evidence": [{"command": "conda run -n base python scripts/get_page.py kernel-nvfp4-gemv --include-code", "environment": "KernelWiki worktree before GEMV remediation, conda base", "negative_control": "kernel-does-not-exist returned nonzero with 'No page found'.", "reference": "Output should identify wiki/kernels/nvfp4-gemv.md and include attached artifact text.", "result": "Exit 0; 27,170-byte output began with the expected path and page frontmatter."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["conda run -n base python -c ''", "conda run -n base python scripts/get_page.py kernel-nvfp4-gemv --include-code", "curl link checks with same-site 404 controls", "sha256sum artifacts/kernels/nvfp4-gemv/full/nvfp4_scaled_mm_kernels.cu"], "id": "kernel-nvfp4-gemv", "page_verdict": "refuted", "path": "wiki/kernels/nvfp4-gemv.md", "residual_risk": "No B200 or CUDA toolkit is installed locally. Author-reported implementation techniques and timings were checked for faithful attribution but were not independently performance-reproduced; the public leaderboard is a dated current snapshot, not an immutable official cutoff/prize-placement record.", "scope": "All 42 baseline units for body hash 55d84be3408d0ab5d0adfed5c555998569f294e99d23cfc0c3fd582b32841d7d were adjudicated against gpu-mode/reference-kernels ae679486, official contest rules and NVIDIA documentation, the public leaderboard snapshot fetched 2026-08-08, the three author posts, pinned local artifacts, and host controls."} +{"claims": [{"challenge": {"alternative": "The mechanism is the ACL Native Sparse Attention architecture or an unmodified dense MLA kernel.", "check": "Compare the first-party DSA paper's equations/Figure 1 and released configuration/model path with both alternatives.", "outcome": "The paper names DeepSeek Sparse Attention, gives a learned lightning indexer plus top-k token selection, instantiates it under MLA's MQA mode, and the release fixes index_topk=2048."}, "claim": "DeepSeek-V3.2-Exp introduces DeepSeek Sparse Attention with a lightning indexer and fine-grained top-k token selection, instantiates it under MLA's MQA mode, and its released 671B configuration selects 2048 entries per query.", "claim_id": "kernel-sparse-mla-original-c001", "claim_type": "mechanism", "covers": ["u001"], "evidence": [{"locator": "Section 1, equations 1-2 and Figure 1", "observation": "The first-party report defines both components and the MLA/MQA instantiation.", "path": "/tmp/kernelwiki-deepseek-v32/DeepSeek_V3_2.pdf", "revision": "DeepSeek-V3.2-Exp 87e509a2e5a100d221c97df52c6e8be7835f0057"}, {"locator": "index_topk", "observation": "The released configuration sets 2048.", "path": "/tmp/kernelwiki-deepseek-v32/inference/config_671B_v3.2.json", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "The two-stage pipeline reduces total per-query decode attention computation from O(sequence length) to O(topk=2048).", "claim_id": "kernel-sparse-mla-original-c002", "claim_type": "performance", "covers": ["u001"], "evidence": [{"locator": "Section 3, Inference Costs", "observation": "Only core/main attention becomes O(Lk) over a sequence; the lightning indexer remains O(L squared), or O(L) work for one decode query, so the whole two-stage per-query path is not O(k).", "path": "/tmp/kernelwiki-deepseek-v32/DeepSeek_V3_2.pdf", "revision": "DeepSeek-V3.2-Exp 87e509a2e5a100d221c97df52c6e8be7835f0057"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The architecture diagram is an accurate general kernel contract: a 64-head/128-dimensional FP8 indexer with a 64-token page selects 2048 block indices, then a 16-head sparse MLA gathers them from one shared cache.", "claim_id": "kernel-sparse-mla-original-c003", "claim_type": "code", "covers": ["u002"], "evidence": [{"locator": "index and MLA dimensions", "observation": "The model has 64x128 indexer heads, 128 MLA query heads, latent 512 and RoPE 64; 16 MLA heads is only an eight-way local shard, not the architecture contract.", "path": "/tmp/kernelwiki-deepseek-v32/inference/config_671B_v3.2.json", "revision": "DeepSeek-V3.2-Exp 87e509a2"}, {"locator": "Indexer and MLA caches", "observation": "The indexer maintains a separate 128-dimensional FP8 K cache and scale cache, then emits token positions; MLA maintains separate latent and positional caches.", "path": "/tmp/kernelwiki-deepseek-v32/inference/model.py", "revision": "same commit"}, {"locator": "sparse decode indices", "observation": "Decode indices encode selected token positions as physical-page-plus-offset values; they are not one index per KV block.", "path": "/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py", "revision": "FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "Every sparse-MLA KV cache entry is universally 656 bytes with 512 FP8 bytes, 16 bytes of per-block FP32 scales, 128 BF16 bytes for indexer positional encoding, and FlashMLA requires a fixed page size of 64.", "claim_id": "kernel-sparse-mla-original-c004", "claim_type": "compatibility", "covers": ["u003", "u004", "u005", "u006", "u007"], "evidence": [{"locator": "DeepSeek V3-family FP8+sparse decode layout", "observation": "656 bytes applies to that decode mode: 512 E4M3 NoPE values, four FP32 scales for successive 128-value groups, and 64 BF16 RoPE values for attention.", "path": "/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py", "revision": "FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232"}, {"locator": "correctness case block_size list", "observation": "The official tests cover page sizes 2, 53, 61, 64, 69, 256 and 576, refuting a FlashMLA-wide fixed-64 requirement.", "path": "/tmp/kernelwiki-flashmla-pinned/tests/test_flash_mla_sparse_decoding.py", "revision": "same commit"}, {"locator": "Indexer.k_cache versus MLA caches", "observation": "Indexer K storage is separate from the 656-byte attention cache.", "path": "/tmp/kernelwiki-deepseek-v32/inference/model.py", "revision": "DeepSeek-V3.2-Exp 87e509a2"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The displayed lightning-indexer CUDA sketch accurately computes one maximum score per KV block using FP8 block-scale MMA and then runs top-k over blocks.", "claim_id": "kernel-sparse-mla-original-c005", "claim_type": "code", "covers": ["u008"], "evidence": [{"locator": "Equation 1", "observation": "The index score is per preceding token and is a weighted sum across indexer heads of ReLU-applied dot products, not a per-block maximum.", "path": "/tmp/kernelwiki-deepseek-v32/DeepSeek_V3_2.pdf", "revision": "DeepSeek-V3.2-Exp 87e509a2"}, {"locator": "Indexer.forward", "observation": "The reference forms q/k, applies non-interleaved RoPE and rotations, quantizes FP8, computes token logits, masks them, and calls topk over token positions; it does not implement the displayed signature or block-max reduction.", "path": "/tmp/kernelwiki-deepseek-v32/inference/model.py", "revision": "same commit"}, {"locator": "sm100_fp8_mqa_logits", "observation": "The high-performance kernel emits logits for Q-head/KV ranges using explicit weights and scales; top-k selection is outside this kernel.", "path": "/tmp/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm100_fp8_mqa_logits.cuh", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The displayed sparse-decode CUDA sketch accurately takes 2048 selected block indices, a generic FP8 paged cache and a [16,576] query, gathers blocks, and runs standard MLA.", "claim_id": "kernel-sparse-mla-original-c006", "claim_type": "code", "covers": ["u009"], "evidence": [{"locator": "flash_mla_with_kvcache signature and sparse dispatch", "observation": "The real interface accepts q[batch,s_q,h_q,576], the V3-family 656-byte FP8+sparse cache, and indices[batch,s_q,topk] whose values encode token page+offset; head count and topk are shape parameters, not the invented fixed signature.", "path": "/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py", "revision": "FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232"}, {"locator": "ref_sparse_attn_decode", "observation": "The reference gathers token rows selected by encoded indices, applies attention across query heads, and masks invalid entries; it does not interpret each index as a selected block.", "path": "/tmp/kernelwiki-flashmla-pinned/tests/ref.py", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "One H800 dense-MLA decode operating point simultaneously reaches 660 BF16 TFLOPS and 3000 GB/s and is compute-bound.", "claim_id": "kernel-sparse-mla-original-c007", "claim_type": "performance", "covers": ["u010"], "evidence": [{"locator": "dense MLA decoding performance", "observation": "The author reports separate maxima for memory-bound and compute-bound configurations; the row merges them into one operating point and label.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "Sparse MLA decode on H800 performs FP8 matrix multiplication at 410 TFLOPS.", "claim_id": "kernel-sparse-mla-original-c008", "claim_type": "performance", "covers": ["u011"], "evidence": [{"locator": "sparse MLA decode performance", "observation": "The 410-TFLOPS author report is for H800 SXM5/CUDA 12.8, but only KV storage is FP8; the matrix multiplication is explicitly BF16.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "Sparse MLA decode on B200 performs FP8 matrix multiplication at 350 TFLOPS and is lower than H800 because decode is bandwidth-dominated.", "claim_id": "kernel-sparse-mla-original-c009", "claim_type": "performance", "covers": ["u012"], "evidence": [{"locator": "sparse MLA decode performance", "observation": "The source says up to 350 TFLOPS and not really optimized yet; it specifies BF16 MMA over an FP8 cache and does not attribute the cross-GPU difference to bandwidth.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "Dense MLA prefill reaches 1460 BF16 TFLOPS on B200 through tcgen05.", "claim_id": "kernel-sparse-mla-original-c010", "claim_type": "performance", "covers": ["u013"], "evidence": [{"locator": "dense prefill support matrix and performance", "observation": "The 1460 author-reported value belongs to NVIDIA's dense MHA forward implementation on SM100, not a dense MLA/MQA prefill result; the README does not label it tcgen05 peak.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "Sparse MLA prefill on B200 is FP8, reaches 1450 TFLOPS, and therefore matches BF16 dense prefill.", "claim_id": "kernel-sparse-mla-original-c011", "claim_type": "performance", "covers": ["u014"], "evidence": [{"locator": "sparse MLA prefill performance and equivalent code", "observation": "The 1450 maximum on B200/CUDA 12.9 is author-reported, but the sparse-prefill Q/KV interface is BF16 and the source makes no matched-performance equivalence to the separate dense-MHA maximum.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "Sparse MLA has a verified use threshold at context lengths of 32K or greater.", "claim_id": "kernel-sparse-mla-original-c012", "claim_type": "performance", "covers": ["u015"], "evidence": [{"locator": "Inference Costs and Figure 3", "observation": "The report presents H800 cost curves from 0K to 128K and even uses masked MHA for short prefill, but defines no universal 32K crossover or use threshold.", "path": "/tmp/kernelwiki-deepseek-v32/DeepSeek_V3_2.pdf", "revision": "DeepSeek-V3.2-Exp 87e509a2"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "The released DSA kernels are unrelated to the DeepSeek-V3.2 model.", "check": "Follow the model repository's high-performance-kernel links and FlashMLA's release statement.", "outcome": "Both first-party repositories explicitly connect the indexer logits and sparse attention kernels to DeepSeek-V3.2-Exp DSA."}, "claim": "DeepSeek-V3.2-Exp is a directly supported use of the released indexer and FlashMLA sparse-attention kernels.", "claim_id": "kernel-sparse-mla-original-c013", "claim_type": "compatibility", "covers": ["u016"], "evidence": [{"locator": "Open-Source Kernels", "observation": "The release links indexer logits to DeepGEMM and sparse attention to FlashMLA.", "path": "/tmp/kernelwiki-deepseek-v32/README.md", "revision": "DeepSeek-V3.2-Exp 87e509a2"}, {"locator": "Introduction", "observation": "The repository says the sparse kernels power DSA in DeepSeek-V3.2-Exp.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "The same sparse-kernel path can be recommended for unspecified similar MLA architectures.", "claim_id": "kernel-sparse-mla-original-c014", "claim_type": "compatibility", "covers": ["u016"], "evidence": [{"locator": "V3-family decode and sparse-prefill contracts", "observation": "The APIs impose exact head dimensions, cache layout, dtype, head-count, index-shape, and output contracts; architectural similarity alone does not establish compatibility.", "path": "/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py", "revision": "FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "Sparse decode cannot affect per-token latency because the indexer still scans the prefix.", "check": "Compare the main-attention complexity reduction and first-party H800 service cost curve with the retained caveat about the indexer.", "outcome": "The main attention reads and computes over selected entries and the first-party deployment reports long-context end-to-end savings, supporting relevance but not universal superiority."}, "claim": "The DSA/FlashMLA path is relevant to serving measurements where long-context per-token decode latency matters, provided indexer and sparse-attention time are both included.", "claim_id": "kernel-sparse-mla-original-c015", "claim_type": "performance", "covers": ["u017"], "evidence": [{"locator": "Inference Costs and Figure 3", "observation": "The paper reports reduced H800 service costs in long-context decoding while retaining the indexer's complexity.", "path": "/tmp/kernelwiki-deepseek-v32/DeepSeek_V3_2.pdf", "revision": "DeepSeek-V3.2-Exp 87e509a2"}, {"locator": "sparse decode interface", "observation": "The attention kernel consumes selected-token indices and therefore addresses the selected-attention portion of decode.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "A structured B200 performance record establishes 1450 FP8 TFLOPS for sparse prefill at sequence length 32K and topk 2048 and classifies it FP8 sparse compute-bound.", "claim_id": "kernel-sparse-mla-original-c016", "claim_type": "performance", "covers": ["u018"], "evidence": [{"locator": "sparse MLA prefill performance and API", "observation": "The source reports only up to 1450 TFLOPS on B200/CUDA 12.9; the inputs are BF16 and it supplies no 32K/topk-2048 tuple, compute-bound classification, timed region, repetitions, or variance.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "SM100 tcgen05 FP8 block-scale MMA is the common mechanism enabling both the lightning indexer and sparse-attention stages.", "claim_id": "kernel-sparse-mla-original-c017", "claim_type": "mechanism", "covers": ["u001"], "evidence": [{"locator": "UMMA descriptor and scale handling", "observation": "The indexer logit kernel uses FP8 UMMA with explicit Q/K scale handling, not the page's claimed shared FP8 block-scale attention pipeline.", "path": "/tmp/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm100_fp8_mqa_logits.cuh", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f"}, {"locator": "sparse decode and prefill dtype contracts", "observation": "Sparse decode dequantizes FP8 KV to BF16 for MMA; sparse prefill accepts BF16 Q/KV. SM100 source uses tcgen05/TMEM, but not one FP8 block-scale MMA mechanism for both stages.", "path": "/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py", "revision": "FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232"}], "oracle": "local-primary-source", "verdict": "refuted"}], "commands": ["Extract and inspect the first-party DeepSeek-V3.2-Exp report and model", "Inspect pinned FlashMLA sparse interfaces, tests, and reported maxima", "Inspect pinned DeepGEMM SM100 indexer-logit implementation"], "id": "kernel-sparse-mla", "page_verdict": "refuted", "path": "wiki/kernels/sparse-mla.md", "residual_risk": "No CUDA toolkit or GPU is present; code semantics were checked from pinned primary source, while performance was audited only as author-reported scope.", "scope": "All 18 baseline units for hash 3a4b0e533208cfbbe45f6c7c15396a5e66007ed4a56f0ca9699e034a5b36b628 plus the material Blackwell frontmatter claim were audited against the first-party model report/configuration and pinned kernel sources."} +{"claims": [{"challenge": {"alternative": "The PR only changes configuration or Python dispatch and does not provide the stated CUDA data path.", "check": "Inspect the pinned PR artifact for the CUDA kernels, their Torch operators, and the DSA call sites.", "outcome": "The artifact contains fusedCatFp4.cu plus indexerKCacheGather.cu and indexerKCacheScatter.cu, and dsa.py calls the corresponding operators in the FP4 indexer path."}, "claim": "TensorRT-LLM PR 13340 adds an SM100 FP4 DSA indexer path with native CUDA fused FP4 concatenation/quantization and K-cache gather/scatter operators.", "claim_id": "kernel-tensorrt-llm-blackwell-indexer-original-c001", "claim_type": "mechanism", "covers": ["u001"], "evidence": [{"locator": "fusedCatFp4Kernel and invokeFusedCatFp4", "observation": "The CUDA implementation concatenates BF16 PE/NoPE inputs, quantizes 128 values to packed FP4 E2M1, and emits packed UE8M0 scales.", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/fusedCatFp4.cu", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}, {"locator": "indexerKCacheGatherUnifiedKernel", "observation": "The CUDA gather copies 64-byte FP4 or 128-byte FP8 payloads and a four-byte scale word from the paged cache.", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheGather.cu", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}, {"locator": "indexerKCacheScatterUnifiedKernel", "observation": "The CUDA scatter copies the payload and scale word into the paged cache.", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheScatter.cu", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The PR establishes a generic sparse-indexer ABI or a reusable FlashInfer-Bench implementation.", "check": "Compare the operator assertions and TensorRT-LLM-specific cache/dispatch contracts with that broader interpretation.", "outcome": "The path is tied to TensorRT-LLM's DSA metadata, cache manager, SM100 dispatch, 128-element index head, and DeepGEMM calls, so it is evidence to adapt rather than a drop-in external kernel."}, "claim": "The PR is implementation evidence for quantized indexer cache movement and layout, but it is not a drop-in FlashInfer-Bench solution.", "claim_id": "kernel-tensorrt-llm-blackwell-indexer-original-c002", "claim_type": "compatibility", "covers": ["u001"], "evidence": [{"locator": "Indexer._prep_q_or_k, sparse_attn_indexer, and DSACacheManager", "observation": "The operators are embedded in TensorRT-LLM's DSA projection, cache-management, DeepGEMM-logit, and top-k pipeline rather than exposing a generic benchmark contract.", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/tensorrt_llm/_torch/attention_backend/sparse/dsa.py", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "The displayed candidate_indexer_probe is a meaningful implementation probe for the PR's sparse indexer path.", "claim_id": "kernel-tensorrt-llm-blackwell-indexer-original-c003", "claim_type": "code", "covers": ["u002"], "evidence": [{"locator": "indexerKCacheGatherUnifiedKernel", "observation": "The real gather consumes separate int64 payload and scale slot mappings, handles a strided four-dimensional cache, copies vectorized payload and scale bytes, and skips negative mappings; the sketch does none of these and also lacks a bounds guard.", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheGather.cu", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}, {"locator": "sparse_attn_indexer", "observation": "The actual indexer performs quantized Q/K preparation, cache scatter/gather, MQA logit computation, and a separate top-k selection; assigning one cache byte to an index represents none of this behavior.", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/tensorrt_llm/_torch/attention_backend/sparse/dsa.py", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "Packed data alone is sufficient for a correctness reference and negative slot mappings produce initialized sentinel outputs.", "check": "Trace scale packing and the early-return behavior in both copy kernels and their allocating wrapper.", "outcome": "FP4 requires four per-32-value UE8M0 exponents packed into one int32, and a negative payload or scale mapping makes the whole token return; the gather wrapper allocates empty output, so a skipped row is not initialized by the kernel."}, "claim": "A correctness reference must explicitly model FP4 packing, scale placement, and invalid-token behavior.", "claim_id": "kernel-tensorrt-llm-blackwell-indexer-original-c004", "claim_type": "semantics", "covers": ["u003", "u004"], "evidence": [{"locator": "FP4 block constants and stages 5-7", "observation": "Each 128-value row becomes 64 packed bytes and four UE8M0 exponent bytes packed little-endian into one int32.", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/fusedCatFp4.cu", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}, {"locator": "negative slot check in indexerKCacheGatherUnifiedKernel", "observation": "If either slot mapping is negative, the token returns before writing its allocated output row.", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheGather.cu", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The PR only exposes the complete indexer, making gather/scatter traffic impossible to isolate.", "check": "Inspect the registered Torch operators and DSA call graph for independent gather and scatter launches.", "outcome": "Gather and scatter are separately registered operators and are launched independently of MQA logits and top-k, so their traffic can be profiled separately with valid inputs."}, "claim": "Gather/scatter traffic is a valid separate profiling target.", "claim_id": "kernel-tensorrt-llm-blackwell-indexer-original-c005", "claim_type": "performance", "covers": ["u005"], "evidence": [{"locator": "cache scatter/gather calls and subsequent MQA-logit/top-k calls", "observation": "The pinned path invokes cache movement, logit calculation, and top-k as distinct operations, permitting stage-specific profiling.", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/tensorrt_llm/_torch/attention_backend/sparse/dsa.py", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The PR fuses gather/scatter with top-k, so separating them would misrepresent the implementation.", "check": "Follow data from cache gather or paged-cache input through logits to top-k dispatch.", "outcome": "The copy kernels do not calculate scores or select indices; dsa.py calls DeepGEMM MQA logits and then a distinct prefill/decode top-k operator."}, "claim": "Top-k selection should remain separate from the gather/scatter memory path while evaluating this pinned implementation.", "claim_id": "kernel-tensorrt-llm-blackwell-indexer-original-c006", "claim_type": "mechanism", "covers": ["u006"], "evidence": [{"locator": "_call_mqa_logits, _call_paged_mqa_logits, indexer_topk_prefill, and indexer_topk_decode", "observation": "Logit generation and top-k selection occur after and outside cache copy operations in the pinned pipeline.", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/tensorrt_llm/_torch/attention_backend/sparse/dsa.py", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "The PR provides fused quantized gather/scatter kernels for the Blackwell FP4/FP8 indexer cache paths.", "claim_id": "kernel-tensorrt-llm-blackwell-indexer-original-c007", "claim_type": "mechanism", "covers": ["u001"], "evidence": [{"locator": "fusedCatFp4Kernel stages 1-7", "observation": "Quantization is fused with PE/NoPE concatenation in fusedCatFp4, before cache movement.", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/fusedCatFp4.cu", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}, {"locator": "indexerKCacheGatherUnifiedKernel", "observation": "Gather only copies already-quantized payload and scale bytes; it performs no quantization.", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheGather.cu", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}, {"locator": "indexerKCacheScatterUnifiedKernel", "observation": "Scatter only copies already-quantized payload and scale bytes; it performs no quantization.", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheScatter.cu", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}], "oracle": "local-primary-source", "verdict": "refuted"}], "commands": ["Inspect pinned fusedCatFp4 CUDA kernel and Torch wrapper", "Inspect pinned K-cache gather/scatter CUDA kernels and Torch wrappers", "Trace the pinned TensorRT-LLM DSA FP4 call path through logits and top-k"], "id": "kernel-tensorrt-llm-blackwell-indexer", "page_verdict": "refuted", "path": "wiki/kernels/tensorrt-llm-blackwell-indexer.md", "residual_risk": "No CUDA toolkit or GPU is present, so the pinned source was not built or executed and the page's profiling recommendations have no measured result.", "scope": "All six baseline units for hash c6277510118240c58c46ecfd1a84c7978461c9a65c84b4f74cbd679662dae2d1 and the material Blackwell frontmatter claim were audited against the PR's pinned verbatim CUDA, wrapper, and DSA sources."} +{"claims": [{"challenge": {"alternative": "The 98 percent compares with theoretical peak or another shape/environment.", "check": "Use the exact cuBLAS and v6 rows and disclosed shape/software.", "outcome": "The ratio rounds to 98 only against the stated same-table cuBLAS row; the claim is source-reported for that setup."}, "claim": "Gau Nernst's hand-written CUDA C++ and inline-PTX tutorial reports 1475.93 TFLOP/s for its final 4096-cubed B200 kernel versus 1506.74 TFLOP/s for its PyTorch 2.9.1/CUDA 13 cuBLAS baseline, which rounds to 98 percent.", "claim_id": "lang-cuda-cpp-original-c001", "claim_type": "performance", "covers": ["u001"], "evidence": [{"locator": "benchmark table and environment", "observation": "The pinned author source gives M=N=K=4096, Modal B200, PyTorch 2.9.1/CUDA 13, cuBLAS 1506.74, and v6 1475.93 TFLOP/s. A conda-base ratio check produced 97.955 percent and rejected a broken 1600-TFLOP/s baseline.", "path": "/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/README.md", "revision": "gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "The displayed tmem_alloc_cta helper correctly issues tcgen05.alloc from thread 0 alone.", "claim_id": "lang-cuda-cpp-original-c002", "claim_type": "code", "covers": ["u002"], "evidence": [{"locator": "cta_group::1 issue granularity", "observation": "Allocation is warp-collective: all threads of one warp must execute the same sync.aligned instruction, and lane-0-only execution is undefined.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"}, {"locator": "TMEM allocation branch", "observation": "The pinned author code executes alloc in every lane of warp 1, not thread 0.", "revision": "gau-nernst/learn-cuda 3b90ac9b", "url": "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The displayed tcgen05_mma helper supplies two immediate descriptors for accumulator C and output D as the final two operands of an unscaled f16 MMA.", "claim_id": "lang-cuda-cpp-original-c003", "claim_type": "code", "covers": ["u002"], "evidence": [{"locator": "tcgen05_mma_f16", "observation": "The real wrapper takes one instruction descriptor and an enable-input-D integer, converts the latter to a local predicate, then passes idesc and predicate as the final operands.", "path": "/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/common.h", "revision": "gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573"}, {"locator": "tcgen05.mma unscaled f16 grammar", "observation": "The required final operands are idesc and enable-input-D predicate, optionally preceded by a lane-disable vector; there are no C/D immediate descriptors.", "path": "/tmp/kernelwiki-ptx-1302.html", "revision": "CUDA 13.0.2, PTX ISA 9.0 fetched 2026-08-08"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "The text strings themselves are fabricated even though wrapper participation is wrong.", "check": "Separate instruction grammar from CUDA control flow and compare exact qualifiers.", "outcome": "The instruction strings are valid; the allocation wrapper is wrong because of how it predicates them."}, "claim": "The literal alloc, 32x32b.x1 load, and dealloc instruction spellings are legal PTX ISA 9.0 forms when issued with their required collective and completion context.", "claim_id": "lang-cuda-cpp-original-c004", "claim_type": "code", "covers": ["u002"], "evidence": [{"locator": "allocation and deallocation grammar", "observation": "The exact shared b32 allocation and taddr deallocation forms match.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"}, {"locator": "32x32b repetition table", "observation": "32x32b.x1.b32 is legal and produces one 32-bit register per participating thread.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The omitted semantic/scope qualifiers make both instructions relaxed or system-scoped.", "check": "Read the documented defaults for each grammar.", "outcome": "The defaults are release/acquire at CTA scope as required for a CTA-local stage."}, "claim": "The two mbarrier instructions have valid default release-CTA arrive-expect-tx and acquire-CTA parity-wait semantics for a correctly initialized CTA-shared barrier and phase.", "claim_id": "lang-cuda-cpp-original-c005", "claim_type": "code", "covers": ["u003"], "evidence": [{"locator": "arrive.expect_tx grammar and default semantics/scope", "observation": "Absent qualifiers default to release and CTA; expect_tx adds byte count and performs one arrival.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive"}, {"locator": "try_wait.parity grammar and defaults", "observation": "Absent qualifiers default to acquire and CTA; parity must track the current or immediately preceding phase.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The two inline helpers alone are a safe complete TMA-to-MMA synchronization implementation.", "claim_id": "lang-cuda-cpp-original-c006", "claim_type": "code", "covers": ["u003"], "evidence": [{"locator": "volatile and memory clobber pitfall", "observation": "Hidden memory effects or a need to stop compiler memory motion require a memory clobber; neither helper declares one.", "revision": "CUDA 13.0.2 Inline PTX guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/inline-ptx-assembly/index.html#incorrect-optimization"}, {"locator": "barrier initialization and staged pipeline", "observation": "The complete source initializes every barrier, publishes initialization to the async proxy, tracks separate TMA/MMA/mainloop/epilogue phases, and uses paired full/empty handoffs; the page shows none of that lifecycle.", "revision": "gau-nernst/learn-cuda 3b90ac9b", "url": "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Every lane in the TMA/MMA warps issues or the epilogue is lane-0-only.", "check": "Inspect elect_sync predicates and warp-range branches.", "outcome": "Only elected threads issue TMA/MMA; all warps at IDs 2 and above enter the epilogue path."}, "claim": "The warp-role skeleton matches the tutorial's final source: warp 0 elects one TMA issuer, CTA 0 warp 1 elects one MMA issuer, and warps 2-5 collectively drain TMEM in the epilogue.", "claim_id": "lang-cuda-cpp-original-c007", "claim_type": "code", "covers": ["u004"], "evidence": [{"locator": "NUM_WARPS and role branches around lines 227-297", "observation": "The source declares one TMA warp, one MMA warp, four epilogue warps and uses elected single issuers for TMA/MMA while all epilogue lanes participate in TMEM loads.", "path": "/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v6.cu", "revision": "gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The ../../ source path escapes to a missing or unrelated file.", "check": "Resolve it from wiki/languages and inspect frontmatter.", "outcome": "It lands on the intended tutorial card; the broken local control fails."}, "claim": "The two related links resolve to the local PTX language page and evidence-scoped tcgen05 tutorial source card.", "claim_id": "lang-cuda-cpp-original-c008", "claim_type": "other", "covers": ["u005", "u006"], "evidence": [{"command": "test -f wiki/languages/ptx-sm100.md && test -f sources/blogs/tcgen05-tutorial.md", "environment": "current KernelWiki worktree", "negative_control": "wiki/languages/not-cuda.md does not exist.", "reference": "Resolve links from wiki/languages and inspect IDs.", "result": "Both targets exist with IDs lang-ptx and blog-tcgen05-tutorial."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["conda run -n base python -c '<1475.93/1506.74 ratio with broken baseline control>'", "local related-link resolution"], "id": "lang-cuda-cpp", "page_verdict": "refuted", "path": "wiki/languages/cuda-cpp.md", "residual_risk": "No CUDA toolkit or B200 is installed locally. The author benchmark is verified as faithful source reporting, not independently reproduced performance.", "scope": "All six baseline units for body hash 7a58194cb413f25af6db71a4e0d90828273f461ee66232063fb6e6a96176d836 were checked against PTX ISA 9.0, CUDA 13.0.2 inline-PTX rules, Gau Nernst's pinned commit 3b90ac9b, and host controls."} +{"claims": [{"challenge": {"alternative": "CuTe DSL arrived after 4.5.0 or only targets pre-Blackwell architectures.", "check": "Inspect the v4.5.0 tag rather than rolling documentation.", "outcome": "The tagged Python package implements tcgen05/TMEM and the tag contains Blackwell CuTe DSL examples."}, "claim": "CUTLASS 4.5.0 includes the Python-native CuTe DSL and Blackwell support.", "claim_id": "lang-cute-dsl-original-c001", "claim_type": "compatibility", "covers": ["u001"], "evidence": [{"locator": "4.5.0 CuTe DSL section and repository tree", "observation": "The exact release contains CuTe DSL Python packages plus SM100/Blackwell operations and examples.", "path": "/tmp/kernelwiki-cutlass-4.5.0/CHANGELOG.md", "revision": "NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "CuTe DSL is the primary abstraction layer in CUTLASS 4.5.0 for Blackwell kernels.", "claim_id": "lang-cute-dsl-original-c002", "claim_type": "other", "covers": ["u001"], "evidence": [{"locator": "Overview and repository organization", "observation": "CUTLASS describes C++ template abstractions and Python DSLs as parallel interfaces; the release also ships extensive SM100 C++ kernels. It does not designate CuTe DSL as the primary Blackwell layer.", "path": "/tmp/kernelwiki-cutlass-4.5.0/README.md", "revision": "NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "The factor is an execution-speed result or applies to all CuTe DSL programs.", "check": "Read the metric and subject in the first-party language/framework section.", "outcome": "It is explicitly compile time for FA4's development comparison, not kernel runtime or a universal bound."}, "claim": "FlashAttention-4 is implemented entirely in CuTe DSL, and its author reports approximately 20-30 times shorter compile times than C++ templates.", "claim_id": "lang-cute-dsl-original-c003", "claim_type": "performance", "covers": ["u001", "u006"], "evidence": [{"locator": "Language and framework: CuTe-DSL", "observation": "The author says FA4 is entirely CuTe DSL and reports about 20-30x lower compile time than C++ templates.", "revision": "Tri Dao first-party FA4 post, fetched 2026-08-08", "url": "https://tridao.me/blog/2026/flash4/"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The displayed SM100_MMA_F16BF16_SS and SM100_MMA_F16BF16_SS_2SM symbols are usable CUTLASS 4.5.0 CuTe DSL Python MMA atoms.", "claim_id": "lang-cute-dsl-original-c004", "claim_type": "code", "covers": ["u002"], "evidence": [{"locator": "MmaF16BF16Op and CtaGroup definitions", "observation": "The Python API constructs tcgen05.MmaF16BF16Op with datatypes, instruction shape, CTA group, operand source, and major modes; neither displayed SM100_MMA symbol exists in the tag.", "path": "/tmp/kernelwiki-cutlass-4.5.0/python/CuTeDSL/cutlass/cute/nvgpu/tcgen05/mma.py", "revision": "NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "Those are CTA tile shapes rather than instruction shapes, or the accumulator remains in registers.", "check": "Trace mma_inst_shape_mnk into MmaF16BF16Op and make_fragment_C into the retrieved TMEM pointer.", "outcome": "The source names and passes them as instruction shapes and explicitly swaps the accumulator tensor to TMEM."}, "claim": "CUTLASS 4.5.0's F16/BF16 CuTe DSL operation represents one-CTA 128x256x16 and two-CTA 256x256x16 tcgen05 MMA shapes with TMEM accumulator fragments.", "claim_id": "lang-cute-dsl-original-c005", "claim_type": "code", "covers": ["u002"], "evidence": [{"locator": "mma_inst_shape_mnk, MmaF16BF16Op, make_fragment_C, and TMEM pointer replacement", "observation": "The 1-CTA example uses (128,256,16), CtaGroup.ONE, and replaces the C fragment pointer with allocated TMEM.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py", "revision": "NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5"}, {"locator": "mma_inst_shape_mnk and CtaGroup.TWO", "observation": "The 2-CTA example uses (256,256,16) and CtaGroup.TWO.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_1.py", "revision": "same tag"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "The displayed make_tmem_ptr/make_tensor/copy fragment is a usable CUTLASS 4.5.0 CuTe DSL TMEM allocation and epilogue example.", "claim_id": "lang-cute-dsl-original-c006", "claim_type": "code", "covers": ["u003"], "evidence": [{"locator": "TmemAllocator, retrieve_ptr, tmem copy atom/partition, and free", "observation": "The release uses a shared holding buffer, TmemAllocator allocation/wait/retrieve/free lifecycle, Ld32x32bOp copy atom, make_tmem_copy, per-thread partitions, and a register tensor. make_tmem_ptr is absent.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py", "revision": "NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The displayed SM100_TMA_LOAD_2D and make_tma_copy call form are usable CUTLASS 4.5.0 CuTe DSL Python APIs for global-to-shared TMA.", "claim_id": "lang-cute-dsl-original-c007", "claim_type": "code", "covers": ["u004"], "evidence": [{"locator": "host_function TMA atom construction and kernel TMA partition/copy", "observation": "The tag uses CopyBulkTensorTileG2SOp and make_tiled_tma_atom_A/B, then tma_partition and cute.copy; neither displayed symbol/call signature occurs.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py", "revision": "NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The displayed @cute_kernel warp-role skeleton is a safe usable CuTe DSL kernel skeleton with correct TMEM and pipeline ownership.", "claim_id": "lang-cute-dsl-original-c008", "claim_type": "code", "covers": ["u005"], "evidence": [{"locator": "kernel decorator, role dispatch, PipelineTmaUmma/PipelineUmmaAsync, TmemAllocator, and teardown", "observation": "The real warp-specialized kernel uses @cute.kernel, explicit participant groups and states, collective TMEM allocation/free, tiled-copy partitioning, and pipeline tails. The displayed decorator/helpers are absent and deallocation from every epilogue warp is not the source lifecycle.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py", "revision": "NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "Python is only a host-side generator and users still author the kernel in C++ templates.", "check": "Inspect the tagged CuTe DSL examples and decorators.", "outcome": "Kernel bodies are authored in Python with @cute.kernel and lowered by the DSL."}, "claim": "CuTe DSL is Python-native and CUTLASS presents its development model as easier to learn, iterate, debug, and integrate than template-heavy C++.", "claim_id": "lang-cute-dsl-original-c009", "claim_type": "other", "covers": ["u007"], "evidence": [{"locator": "CUTLASS DSL overview", "observation": "NVIDIA describes Python-native interfaces, faster compile times, framework integration without glue code, and more intuitive metaprogramming than deep C++ expertise.", "path": "/tmp/kernelwiki-cutlass-4.5.0/README.md", "revision": "NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "1605 is a single named shape or an independently reproduced median.", "check": "Inspect how the post scopes the number and benchmark plots.", "outcome": "It is an author-reported maximum, so only that scope is verified."}, "claim": "The FlashAttention-4 post reports a peak 1605 TFLOP/s B200 BF16 result.", "claim_id": "lang-cute-dsl-original-c010", "claim_type": "performance", "covers": ["u008"], "evidence": [{"locator": "performance summary and attention benchmarks", "observation": "The post reports up to 1605 TFLOP/s on B200 BF16 and labels it 71 percent; it is a maximum over evaluated configurations.", "revision": "Tri Dao first-party FA4 post, fetched 2026-08-08", "url": "https://tridao.me/blog/2026/flash4/"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "FlashAttention-4's 1605-TFLOP/s number establishes that CuTe DSL has the same performance as handwritten C++.", "claim_id": "lang-cute-dsl-original-c011", "claim_type": "performance", "covers": ["u008"], "evidence": [{"locator": "Language/framework and benchmark comparators", "observation": "The post's runtime comparisons are with FA2, Triton, Gluon, and cuDNN; it gives no matched handwritten-C++ FA4 comparator. The 1605 number is FA4's own peak, not an equality measurement.", "revision": "Tri Dao first-party FA4 post, fetched 2026-08-08", "url": "https://tridao.me/blog/2026/flash4/"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Blackwell Python support is only an example-layer inline-PTX escape hatch.", "check": "Inspect the installed package tree rather than examples alone.", "outcome": "The architecture and nvgpu packages contain public typed operations and allocation helpers."}, "claim": "CUTLASS 4.5.0 has first-class tcgen05 and TMEM Python APIs, including typed MMA/copy operations and allocation management.", "claim_id": "lang-cute-dsl-original-c012", "claim_type": "compatibility", "covers": ["u009"], "evidence": [{"locator": "MMA operation/trait implementations", "observation": "The tag provides typed tcgen05 operations and traits.", "path": "/tmp/kernelwiki-cutlass-4.5.0/python/CuTeDSL/cutlass/cute/nvgpu/tcgen05/mma.py", "revision": "NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5"}, {"locator": "retrieve_tmem_ptr, alloc_tmem, and dealloc_tmem", "observation": "The tag exposes TMEM allocation, pointer retrieval, and deallocation APIs.", "path": "/tmp/kernelwiki-cutlass-4.5.0/python/CuTeDSL/cutlass/cute/arch/tmem.py", "revision": "same tag"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "CuTe DSL automatically computes layouts and handles swizzles without explicit user configuration.", "claim_id": "lang-cute-dsl-original-c013", "claim_type": "code", "covers": ["u010"], "evidence": [{"locator": "make_smem_layout_a/b and allocate_tensor", "observation": "The example uses layout algebra and Blackwell helper functions but explicitly supplies MMA tilers, datatypes, major modes, layouts, alignment, and swizzle objects. Handling is typed/composable, not universally automatic.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py", "revision": "NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "The files are reconstructed snippets, truncated captures, or point to mutable PR heads.", "check": "Use provenance mode/truncation fields, content hashes, full merge SHAs from GitHub's PR API, and remote raw bytes.", "outcome": "All recorded properties and sampled remote bytes agree; a mismatched hash fails."}, "claim": "The nine linked local artifact files exist, match their provenance-recorded SHA-256 values, and are untruncated verbatim captures of the stated PR merge revisions; the seven tutorial files have the displayed line counts.", "claim_id": "lang-cute-dsl-original-c014", "claim_type": "other", "covers": ["u011", "u012", "u013", "u014", "u015", "u016", "u017", "u018", "u019", "u020"], "evidence": [{"command": "verify each local file against PROVENANCE.yaml SHA-256 and line count, compare representative files to raw.githubusercontent.com at full merge SHAs, then corrupt one expected hash in memory", "environment": "current KernelWiki worktree and network, 2026-08-09", "negative_control": "The deliberately altered expected hash was rejected.", "reference": "All nine file records declare mode=verbatim and size_cap_truncated=false; tutorial rows state exact line counts.", "result": "All local hashes and seven numeric line counts matched; representative remote hashes at PR 3106/2881/3021 merge commits matched byte-for-byte."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "The row labels were inferred from filenames and step 4/6 do not implement dynamic clusters or PDL.", "check": "Inspect each file's upstream docstring and the corresponding launch/scheduler code.", "outcome": "The descriptions and mechanisms occur in their named files, including preferred/fallback cluster launch and grid dependency control."}, "claim": "The seven PR-3106 tutorial files form the stated progression from FP16 software-pipelined baseline through 2-CTA multicast, warp specialization, static persistence, preferred/dynamic clusters, TMA prefetch, and PDL.", "claim_id": "lang-cute-dsl-original-c015", "claim_type": "code", "covers": ["u012", "u013", "u014", "u015", "u016", "u017", "u018", "u021"], "evidence": [{"locator": "file docstrings and implementations for fp16_gemm_0.py through fp16_gemm_6.py", "observation": "The upstream patch explicitly labels and implements the listed progression; the 4.5.0 official tree retains it under the renamed cute/blackwell/tutorial path.", "path": "artifacts/prs/cutlass/PR-3106/diff.patch", "revision": "NVIDIA/cutlass PR 3106 merge 772fbb264e0fe823d62dec10a44a67e1503afdfc"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The files are merely diffs or excerpts whose omitted context changes their purpose.", "check": "Check provenance mode and truncation, then inspect file entry points/imports.", "outcome": "Both are recorded as full verbatim upstream files and contain the stated implementations."}, "claim": "PR-2881's captured dense_gemm_persistent_prefetch.py is a full persistent GEMM with TMA prefetch, and PR-3021's captured clc.py is a full CLC Python binding revision.", "claim_id": "lang-cute-dsl-original-c016", "claim_type": "code", "covers": ["u019", "u020"], "evidence": [{"locator": "full file and provenance", "observation": "The complete 2147-line captured file implements persistent scheduling and TMA prefetch.", "path": "artifacts/prs/cutlass/PR-2881/key-files/examples/python/CuTeDSL/blackwell/dense_gemm_persistent_prefetch.py", "revision": "NVIDIA/cutlass PR 2881 merge 3f4c086d09bd1dc55defb955862f333893bbb28b"}, {"locator": "full file and provenance", "observation": "The complete 124-line captured module defines the CLC issue/query binding API.", "path": "artifacts/prs/cutlass/PR-3021/key-files/python/CuTeDSL/cutlass/cute/arch/clc.py", "revision": "NVIDIA/cutlass PR 3021 merge 5c42d0f28c9dc728d874fd552492a355036a53e1"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "A relative path lands on a similarly named but unrelated page.", "check": "Resolve paths and inspect frontmatter IDs.", "outcome": "Every ID matches the link label and the missing control fails."}, "claim": "The three related links resolve to the intended local tcgen05 hardware page, verified FlashAttention-4 page, and CUTLASS source card.", "claim_id": "lang-cute-dsl-original-c017", "claim_type": "other", "covers": ["u022", "u023", "u024"], "evidence": [{"command": "resolve the three relative links from wiki/languages/cute-dsl.md and inspect target IDs; test a missing neighbor", "environment": "current KernelWiki worktree", "negative_control": "wiki/languages/not-cute-dsl.md was absent.", "reference": "Targets should identify hw-tcgen05-mma, kernel-flash-attention-4, and doc-cutlass-blackwell.", "result": "All three files existed with the expected IDs."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["git clone --depth 1 --branch v4.5.0 NVIDIA/cutlass and inspect tag e406c186", "GitHub PR API merge-SHA checks", "artifact SHA-256/line-count verification with corrupted-hash control", "local related-link checks with missing-target control"], "id": "lang-cute-dsl", "page_verdict": "refuted", "path": "wiki/languages/cute-dsl.md", "residual_risk": "No Blackwell GPU or installed CuTe DSL toolchain is available locally. Exact APIs and artifact contents are source-verified; FA4 compile/runtime numbers remain author-reported rather than independently reproduced.", "scope": "All 24 baseline units for body hash 4e09e1f2220eed702b5a7c8e6d7318de541e65faf5660835318c6916f5f1625b were checked against NVIDIA/cutlass v4.5.0, first-party FA4 reporting, full PR merge revisions, provenance hashes, and host controls."} +{"claims": [{"challenge": {"alternative": "These mnemonics belong to CUDA C++ rather than PTX or to Hopper only.", "check": "Resolve each mnemonic in the archived PTX ISA and inspect target notes.", "outcome": "They are PTX instructions; tcgen05 and CLC forms have Blackwell target scope while TMA also has Hopper forms."}, "claim": "The page is an overview of PTX instruction families relevant to Blackwell SM100 hardware features.", "claim_id": "lang-ptx-original-c001", "claim_type": "other", "covers": ["u001"], "evidence": [{"locator": "tcgen05, cluster launch control, cp.async.bulk.tensor, packed cvt, and ld sections", "observation": "The archive defines each instruction family represented by the page and gives Blackwell target notes for the architecture-specific forms.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The allocation counts rows, MMA requires block-scale operands, x1 is illegal for 32x32b, or cp copies in the opposite direction.", "check": "Compare each exact grammar and operand description rather than adjacent forms.", "outcome": "Allocation counts columns, the unscaled f16 grammar accepts the shown five operands, x1 is listed, and cp is SMEM to TMEM."}, "claim": "The displayed tcgen05 alloc/dealloc, unscaled f16 MMA, ld/st 32x32b x1, and shared-to-TMEM cp forms are legal PTX ISA 9.0 instruction shapes with the described operand locations; all allocated TMEM must be deallocated before exit.", "claim_id": "lang-ptx-original-c002", "claim_type": "code", "covers": ["u002"], "evidence": [{"locator": "alloc/dealloc grammar and lifecycle", "observation": "The shown shared b32 allocation destination, taddr deallocation source, column count, and before-exit rule match.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"}, {"locator": "unscaled floating-point grammar and examples", "observation": "The f16 forms use TMEM D, A/B descriptors, idesc, and enable-input-D predicate for both CTA-group modes.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}, {"locator": "ld/st grammar and repetition table", "observation": "32x32b.x1 is valid for warp-collective TMEM-register transfers.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"}, {"locator": "cp grammar and 128x256b example", "observation": "The exact cta_group::1 128x256b shared-descriptor-to-TMEM form is documented.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-cp"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "tcgen05.fence::after_thread_sync is the critical fence between TMA completion and MMA.", "claim_id": "lang-ptx-original-c003", "claim_type": "mechanism", "covers": ["u002"], "evidence": [{"locator": "after_thread_sync semantics and canonical handoff patterns", "observation": "The fence orders subsequent asynchronous tcgen05 instructions across a documented execution-ordering handoff; it is not a TMA completion wait.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence"}, {"locator": "global-to-shared mbarrier completion", "observation": "TMA load completion is tracked through complete_tx on an mbarrier and must be awaited before consuming the destination.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The conversion consumes one scalar FP4 or the move cannot write byte-sized destinations.", "check": "Apply packed x2 widths and the scalar/vector total-size rule.", "outcome": "The source is an 8-bit pair and four byte destinations exactly match b32."}, "claim": "PTX ISA 9.0 defines the displayed E2M1x2-to-F16x2 conversion and typed b32 scalar-to-four-byte vector move.", "claim_id": "lang-ptx-original-c004", "claim_type": "code", "covers": ["u003"], "evidence": [{"locator": "packed alternate-format conversion", "observation": "The cvt mnemonic and packed source/result types match.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt"}, {"locator": "scalar-to-vector move", "observation": "A b32 source can be decomposed into four b8 destinations when types are declared accordingly.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The mov.b32 byte decomposition is categorically faster than bitwise extraction.", "claim_id": "lang-ptx-original-c005", "claim_type": "performance", "covers": ["u003"], "evidence": [{"locator": "mov semantics", "observation": "The ISA provides no relative latency or throughput guarantee.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov"}, {"locator": "vectorized PTX decode stage", "observation": "The author's reported improvement bundles mov, conversions, vectorized decode, and arithmetic; no isolated mov-versus-bitwise trial is reported.", "revision": "author post fetched 2026-08-08", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "v4.u64 is unsupported or the qualifiers are L2-only.", "check": "Inspect the exact ld vector/type and L1 eviction-priority sets.", "outcome": "Both qualifiers are in the L1 set and v4.u64 is an explicit 256-bit example."}, "claim": "The displayed ld.global load forms are legal vector widths and L1 eviction-priority qualifiers in PTX ISA 9.0.", "claim_id": "lang-ptx-original-c006", "claim_type": "code", "covers": ["u004"], "evidence": [{"locator": "ld grammar, eviction priorities, and vector examples", "observation": "L1::no_allocate and L1::evict_last are legal hints; v2.u64 and v4.u64 move 128 and 256 bits.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "L1::no_allocate necessarily bypasses L1 for one-use data and L1::evict_last keeps reused data resident.", "claim_id": "lang-ptx-original-c007", "claim_type": "performance", "covers": ["u004"], "evidence": [{"locator": "cache eviction priority semantics", "observation": "These qualifiers are performance hints that may not always be respected and do not guarantee bypass, residence, or workload suitability.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#cache-operators"}, {"locator": "top-solution observations and failed wider-load experiment", "observation": "The policies are source-reported for particular implementations, not universal cache guarantees.", "revision": "author post fetched 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "clusterlaunchcontrol.try_cancel takes one ClcID output and directly returns a valid tile ID or decline.", "claim_id": "lang-ptx-original-c008", "claim_type": "code", "covers": ["u005"], "evidence": [{"locator": "try_cancel grammar and completion", "observation": "The asynchronous instruction takes shared response and mbarrier addresses, writes a 16-byte opaque response, and requires query_cancel instructions after completion to determine success and decode coordinates.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Cluster shared memory requires multicast or the tensor map must be a raw global address.", "check": "Compare non-multicast cluster-destination grammar and operand descriptions.", "outcome": "A cluster destination is legal without multicast and uses an opaque tensor-map address plus coordinates."}, "claim": "The first displayed cp.async.bulk.tensor form is a legal 2D global-to-cluster-shared TMA load completed in bytes on an mbarrier.", "claim_id": "lang-ptx-original-c009", "claim_type": "code", "covers": ["u006"], "evidence": [{"locator": "global-to-shared::cluster grammar", "observation": "The dimensions, state spaces, optional tile mode, tensor-map coordinates, and mbarrier completion operands match.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "Appending .multicast is the legal multicast qualifier for the displayed TMA load.", "claim_id": "lang-ptx-original-c010", "claim_type": "code", "covers": ["u006"], "evidence": [{"locator": "global-to-shared::cluster multicast grammar and example", "observation": "The required qualifier is .multicast::cluster, followed by a 16-bit CTA mask; .multicast alone is not in the grammar.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "A relative path resolves to an adjacent directory or unrelated ID.", "check": "Resolve from wiki/languages and inspect target frontmatter.", "outcome": "Each exact target and ID matches; the missing-target control fails."}, "claim": "The three related links resolve to the CUDA C++, tcgen05 MMA, and NVFP4 pages named by their labels.", "claim_id": "lang-ptx-original-c011", "claim_type": "other", "covers": ["u007", "u008", "u009"], "evidence": [{"command": "test -f wiki/languages/cuda-cpp.md && test -f wiki/hardware/tcgen05-mma.md && test -f wiki/hardware/nvfp4.md", "environment": "current KernelWiki worktree", "negative_control": "wiki/languages/not-a-real-target.md does not exist.", "reference": "Resolve each relative path and compare frontmatter IDs with labels.", "result": "All three files exist with IDs lang-cuda-cpp, hw-tcgen05-mma, and hw-nvfp4."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["local path resolution with a missing-target control"], "id": "lang-ptx", "page_verdict": "refuted", "path": "wiki/languages/ptx-sm100.md", "residual_risk": "No CUDA assembler or Blackwell GPU is installed locally. Legal forms and ordering semantics come from PTX ISA 9.0; the original page's performance prescriptions were not retained without measurements.", "scope": "All nine baseline units for body hash 7ef09348591994e947177b5ee3a4a04af46cd573163a183b47b016ff51d1fec9 were checked against CUDA 13.0.2 PTX ISA 9.0, the version-scoped author evidence, and local link controls."} +{"claims": [{"challenge": {"alternative": "The date refers only to a later documentation update or another tag.", "check": "Inspect the signed upstream v3.6.0 release header and resolve the tag with git ls-remote.", "outcome": "Both identify the same v3.6.0 commit and release publication."}, "claim": "The GitHub v3.6.0 release is commit 7c56a5e40f7fd928dfd5c72902d5def0097db73a and was published on 2026-01-21 UTC.", "claim_id": "lang-triton-original-c001", "claim_type": "compatibility", "covers": ["u001", "u046"], "evidence": [{"locator": "release header", "observation": "The upstream release page identifies v3.6.0, short commit 7c56a5e, and publication at 21 Jan 23:27.", "revision": "v3.6.0, commit 7c56a5e40f7fd928dfd5c72902d5def0097db73a", "url": "https://github.com/triton-lang/triton/releases/tag/v3.6.0"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The v3.3 symbols are dead scaffolding while real compilation still emits only WGMMA.", "check": "Inspect v3.3's conversion tests and MMAv5 lowering files, and compare their absence at v3.2.", "outcome": "The pinned v3.3 tests check concrete tcgen05 PTX forms, directly contradicting the claimed 3.6 first-support boundary."}, "claim": "Triton 3.6 was the first release with native Blackwell tcgen05/TMEM lowering; through Triton 3.5 the compiler only emitted Hopper WGMMA and kept accumulators in registers.", "claim_id": "lang-triton-original-c002", "claim_type": "compatibility", "covers": ["u001", "u011", "u017", "u020", "u022", "u024", "u040", "u041", "u042", "u043", "u046"], "evidence": [{"locator": "TTNG_TCGen5MMAOp, TTNG_TCGen5MMAScaledOp, TTNG_TMEMLoadOp, TTNG_TMEMStoreOp, TTNG_TMEMAllocOp, TTNG_TMEMCopyOp", "observation": "The v3.3.0 dialect defines native TCGen5 MMA and TMEM operations; TMEMCopy explicitly lowers to tcgen05.cp.", "revision": "Triton v3.3.0, commit 819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c", "url": "https://github.com/triton-lang/triton/blob/819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c/include/triton/Dialect/TritonNvidiaGPU/IR/TritonNvidiaGPUOps.td"}, {"locator": "Blackwell lowering CHECK directives", "observation": "Pinned tests require tcgen05.mma, tcgen05.commit, TMEM encodings, scaled MMA, and cta_group::2 output.", "revision": "Triton v3.3.0, commit 819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c", "url": "https://github.com/triton-lang/triton/blob/819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c/test/Conversion/tritongpu_to_llvm_blackwell.mlir"}, {"locator": "new Blackwell dialect, lowering, allocation, and test files", "observation": "The version boundary adds MMAv5 lowering, tensor-memory allocation, and Blackwell lowering tests; the corresponding symbols are absent at v3.2.0.", "revision": "v3.2.0 9641643da6c52000c807b5eeed05edaec4402a67 to v3.3.0 819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c", "url": "https://github.com/triton-lang/triton/compare/v3.2.0...v3.3.0"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "These items are a wholesale first implementation rather than refinements.", "check": "Compare the 3.6 release items with the pinned v3.3 and v3.5 dialect/tests and release notes.", "outcome": "Earlier releases already contain TCGen5/TMEM lowering; the listed 3.6 changes are narrower additions and generalizations."}, "claim": "Triton 3.6 incrementally expands the existing Blackwell backend with generic tcgen05.cp, more generic TMEM/ld/st layouts, tcgen05.mma layout generalization, aref warp-specialization work, and initial Gluon multi-CTA/2CTA support.", "claim_id": "lang-triton-original-c003", "claim_type": "compatibility", "covers": ["u011", "u012", "u013", "u014", "u015", "u017", "u019", "u022"], "evidence": [{"locator": "Backend & Compiler lines 221-225; NVIDIA Backend lines 310-340; Gluon lines 351-389", "observation": "The release enumerates generic tcgen05.cp, TMEM layout changes, tcgen05.mma generalization, warp-specialization aref work, initial 2CTA/multi-CTA, num_ctas, and scaled MMA support.", "revision": "Triton v3.6.0, commit 7c56a5e40f7fd928dfd5c72902d5def0097db73a", "url": "https://github.com/triton-lang/triton/releases/tag/v3.6.0"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Those user-visible examples appear only after the v3.5 tag.", "check": "Open the files and release notes at the exact v3.5 commit.", "outcome": "The tagged v3.5 tree contains the tutorial files and implementation references."}, "claim": "By Triton 3.5, Blackwell paths include persistent-matmul and fused-attention warp specialization, generic tcgen05.ld/st exposure, block-scaled matmul, and an explicit Gluon tcgen05/TMEM tutorial.", "claim_id": "lang-triton-original-c004", "claim_type": "compatibility", "covers": ["u006", "u007", "u013", "u014"], "evidence": [{"locator": "NVIDIA Backend, Hopper/Blackwell Features and Blackwell-Specific", "observation": "The release notes enable warp specialization for persistent matmul and FA and list TMEM, tcgen05_copy, generic tcgen05.ld/st, and tcgen05.commit work.", "revision": "Triton v3.5.0, commit c3c476f357f1e9768ea4e45aa5c17528449ab9ef", "url": "https://github.com/triton-lang/triton/releases/tag/v3.5.0"}, {"locator": "tutorial introduction, tmem_example_kernel, and small_mma_kernel", "observation": "The tutorial allocates/loads/stores TMEM and invokes tcgen05_mma on Blackwell.", "revision": "Triton v3.5.0, commit c3c476f357f1e9768ea4e45aa5c17528449ab9ef", "url": "https://github.com/triton-lang/triton/blob/c3c476f357f1e9768ea4e45aa5c17528449ab9ef/python/tutorials/gluon/06-tcgen05.py"}, {"locator": "Blackwell block-scaled matmul tutorial", "observation": "The v3.5 tree already ships the tl.dot_scaled Blackwell tutorial.", "revision": "Triton v3.5.0, commit c3c476f357f1e9768ea4e45aa5c17528449ab9ef", "url": "https://github.com/triton-lang/triton/blob/c3c476f357f1e9768ea4e45aa5c17528449ab9ef/python/tutorials/10-block-scaled-matmul.py"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The bullets report a scoped comparative benchmark rather than editorial guidance.", "check": "Inspect the bullets, cited source trail, and performance table columns.", "outcome": "They contain no workload, machine, software revision, timed region, trials, or turnaround measurements."}, "claim": "The page's 'when to use' bullets are recommendations; their concrete examples are grounded by actual Triton attention, GatedDeltaNet projection, format-conversion, normalization, dot_scaled, and warp-specialized matmul code, but the page provides no benchmark scope for its CuTe-DSL turnaround comparison.", "claim_id": "lang-triton-original-c005", "claim_type": "other", "covers": ["u001", "u002", "u003", "u004", "u005"], "evidence": [{"locator": "the six files linked in original units u048-u053 and their PROVENANCE.yaml entries", "observation": "Pinned code contains Triton JIT kernels for the named workload categories. No unit or cited benchmark defines or measures 'turnaround' against CuTe DSL.", "path": "artifacts/prs/", "revision": "verbatim bundles at SHAs a1257fd1, 890df95b, 766d225f, 5bdc07d9, and 5638d40f"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The heading itself is an independently quantified claim for all workloads.", "check": "Inspect its grammatical form and the scope of the two supporting units.", "outcome": "It is a recommendation whose factual support is limited to the two named cases."}, "claim": "The 'use alternatives' introduction is editorial guidance rather than a universal benchmark result; the two following bullets give specific downstream cases.", "claim_id": "lang-triton-original-c006", "claim_type": "other", "covers": ["u008"], "evidence": [{"locator": "u008-u010", "observation": "The imperative heading is followed by one SGLang benchmark report and one routing decision, not a universal performance table.", "path": "verification/baseline-queue.jsonl", "revision": "immutable baseline body SHA 6b9394376db86dd1090ef2012a0d599ad3ac0703f7df8c9bbf9c61c4c24888f1"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The percentage compares different metrics or the routing change applies to every GPU.", "check": "Use the total-token-throughput rows for both PR 5390 runs and the architecture-scoped summary of PR 21595.", "outcome": "10447.34/8227.35-1 is 26.98%, and the routing PR explicitly restricts the change to SM100."}, "claim": "SGLang PR 5390 reports 10,447.34 total tok/s for its CUTLASS MLA run versus 8,227.35 tok/s for its Triton run (26.98% higher), and PR 21595 changes the SM100 multimodal-attention default from triton_attn to FA4.", "claim_id": "lang-triton-original-c007", "claim_type": "performance", "covers": ["u009", "u010", "u032"], "evidence": [{"locator": "Deepseek-R1 Benchmarks, CUTLASS and Triton result blocks", "observation": "The PR records the commands, 3,000 prompts, token lengths, dtype, TP/DP settings, and the two reported throughput totals; this verifies only what that PR reports, not a universal speedup.", "revision": "merged commit 84810da4ae424d15ec06a3e32a03a839dd50a644", "url": "https://github.com/sgl-project/sglang/pull/5390"}, {"locator": "Summary and commit 'Restrict fa4 default to Blackwell SM100 only'", "observation": "The PR explicitly changes the Blackwell datacenter SM100 default from triton_attn to fa4 while retaining other defaults by architecture.", "revision": "merged commit 87a27682", "url": "https://github.com/sgl-project/sglang/pull/21595"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The downstream artifact itself contains inspectable tcgen05 PTX proving every tl.dot path.", "check": "Search the provenance-pinned artifact for tcgen05, TMEM, PTX, and warp_specialize and compare to a positive-control Triton v3.5 tcgen05 tutorial.", "outcome": "The downstream file has none of those lowering witnesses; the positive control does."}, "claim": "The page correctly cautions that plain tl.dot lowering and every tutorial/downstream shape are not proved by the cited downstream artifacts.", "claim_id": "lang-triton-original-c008", "claim_type": "compatibility", "covers": ["u016", "u018", "u019", "u025", "u026", "u027"], "evidence": [{"locator": "whole file; tl.dot sites around lines 358, 372, and 399", "observation": "The artifact contains tl.dot kernels but no PTX dump, tcgen05 name, TMEM operation, warp_specialize argument, or lowering record.", "path": "artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/ops/triton_decode_attention.py", "revision": "upstream SHA a1257fd1, sha256 16cba0b3e98d2829df6d235e3d368ffaa66d72cc2a100ac53f96542e30472796"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Architecture metadata outside the code proves it is SM100-only and its tl.dot necessarily emits tcgen05.", "check": "Inspect the primary PR body, exact changed code, and its guards/imports.", "outcome": "The primary source mentions SM120 and supplies no lowering witness, contradicting the SM100-only/demonstration framing."}, "claim": "vLLM PR 34597 is an SM100-only production-decode anchor that demonstrates its tl.dot kernels lower through Triton 3.6's tcgen05/TMEM path.", "claim_id": "lang-triton-original-c009", "claim_type": "compatibility", "covers": ["u017", "u023", "u024", "u026", "u027", "u048", "u049", "u054"], "evidence": [{"locator": "PR summary and changed files", "observation": "The PR adds FP8 KV-cache support to a generic Triton MLA backend and describes that backend as the only MLA option on SM120; it does not identify an SM100-only path or tcgen05 lowering.", "revision": "merged commit a1257fd1", "url": "https://github.com/vllm-project/vllm/pull/34597"}, {"locator": "kernel definitions and tl.dot call sites", "observation": "The code contains tl.dot but no architecture guard, Triton-version requirement, warp_specialize, tcgen05, TMEM, or emitted PTX evidence.", "revision": "commit a1257fd1", "url": "https://github.com/vllm-project/vllm/blob/a1257fd1/vllm/v1/attention/ops/triton_decode_attention.py"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The bundle is a derived excerpt or contains only a dispatch gate.", "check": "Verify asset_mode, upstream paths, SHA-256 values, decorators, and dot call sites.", "outcome": "Both files are marked verbatim and contain the implementation described, though they do not prove a particular backend lowering."}, "claim": "vLLM PR 34597 genuinely modifies an @triton.jit MLA decode kernel with tl.dot operations and adds FP8 KV-cache support, with two verbatim files pinned locally at merge SHA a1257fd1.", "claim_id": "lang-triton-original-c010", "claim_type": "code", "covers": ["u017", "u021", "u023", "u024", "u026", "u027", "u048", "u049", "u054"], "evidence": [{"locator": "PROVENANCE.yaml; triton_decode_attention.py decorators and tl.dot call sites; triton_mla.py supported_kv_cache_dtypes", "observation": "The provenance lists exact hashes; the code contains Triton kernels and the stated tl.dot operations, while the backend file adds FP8 cache dtypes.", "path": "artifacts/prs/vllm/PR-34597/", "revision": "upstream SHA a1257fd1; asset_mode verbatim"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The PR adds a new Triton kernel or proves TCGen5 code generation.", "check": "Inspect its one-file diff and changed-files list.", "outcome": "It is a dispatch gate, not a kernel or lowering proof."}, "claim": "vLLM PR 29339 gates triton_kernels MXFP4 use to SM90 and SM100 and changes only the dispatch file mxfp4.py.", "claim_id": "lang-triton-original-c011", "claim_type": "compatibility", "covers": ["u028"], "evidence": [{"locator": "PR title, motivation, and changed files", "observation": "The upstream PR limits triton_kernels to SM90/SM100 for MXFP4 and lists only vllm/model_executor/layers/quantization/mxfp4.py.", "revision": "merged commit c17610e2", "url": "https://github.com/vllm-project/vllm/pull/29339"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The page swapped which artifact contains matrix products.", "check": "Search each exact file for decorators, tl.dot, tl.load, and tl.store.", "outcome": "The GDN projection is load/store rearrangement; extend_attention contains the dot products, matching the narrowed claim."}, "claim": "Pinned SGLang artifacts show a GatedDeltaNet fused projection implemented with Triton loads/stores and an extend-attention kernel implemented with Triton dot products.", "claim_id": "lang-triton-original-c012", "claim_type": "code", "covers": ["u029", "u030", "u031", "u052", "u053"], "evidence": [{"locator": "@triton.jit kernels and tl.load/tl.store sites", "observation": "The verbatim file implements fused QKVZBA split/reshape/cat projection through Triton memory operations and contains no tl.dot.", "path": "artifacts/prs/sglang/PR-21019/key-files/python/sglang/jit_kernel/triton/gdn_fused_proj.py", "revision": "upstream SHA 5bdc07d9, sha256 c4ef85167eb11065721e76dd5d63d6f346f309f84822578dd4a3874249acef89"}, {"locator": "@triton.jit extend-attention kernels and tl.dot call sites", "observation": "The verbatim file performs QK, QPE/KPE, and probability/value dot products.", "path": "artifacts/prs/sglang/PR-22079/key-files/python/sglang/srt/layers/attention/triton_ops/extend_attention.py", "revision": "upstream SHA 5638d40f, sha256 d27789c32e180d725ed867f4f6cd05b09a3c47bfe37794ff60e150d000573edd"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The absence result is caused by a broken search pattern.", "check": "Run the identical search against the pinned upstream positive control.", "outcome": "The positive control matches, so the search discriminates the downstream artifacts."}, "claim": "The local downstream artifacts do not contain an explicit tcgen05 PTX dump or a warp-specialized descriptor/TMA lowering witness.", "claim_id": "lang-triton-original-c013", "claim_type": "code", "covers": ["u018", "u019", "u026", "u027"], "evidence": [{"command": "rg -n -i 'tcgen05|tmem|warp_specialize' artifacts/prs/vllm/PR-34597 artifacts/prs/sglang/PR-22079 artifacts/prs/sglang/PR-21019", "environment": "GNU ripgrep over provenance-pinned local artifacts", "negative_control": "The same terms match Triton v3.5.0's python/tutorials/gluon/06-tcgen05.py and compiler sources.", "reference": "A lowering witness would contain a TCGen5/TMEM/PTX term or explicit warp-specialize surface.", "result": "No matching lowering witness was found in the three cited downstream bundles."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "The values belong to model workloads rather than leaderboard authors or adjacent columns.", "check": "Read the author row and its two labeled columns without flattening neighboring rows.", "outcome": "Each value maps to the same author row and the exact Avg Speedup and % Resolved columns."}, "claim": "The live FlashInfer-Bench author leaderboard displays 0.628x/73.1% for gemini-2.5-pro, 0.467x/92.3% for gpt-5-2025-08-07, and 0.456x/73.1% for claude-opus-4-1-20250805 across 660 workloads.", "claim_id": "lang-triton-original-c014", "claim_type": "performance", "covers": ["u033", "u034", "u035", "u036"], "evidence": [{"locator": "Leaderboard, Author / Avg Speedup / % Resolved rows 1-3", "observation": "The three exact values and resolution percentages appear in the named rows; each percentage is labeled over 660 workloads.", "revision": "live leaderboard retrieved 2026-08-08", "url": "https://bench.flashinfer.ai/"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Hidden table metadata pins these exact rows to Triton 3.5 and marks a missing 3.6 rerun.", "check": "Inspect the leaderboard labels and linked public page for version/language scope.", "outcome": "The source exposes neither asserted scope, while the identical values are current, so the historical Triton-version framing is false."}, "claim": "Those FlashInfer-Bench rows are a preserved Triton-3.5-era snapshot that specifically measures AI-generated Triton kernels, and no Triton 3.6 rerun is available.", "claim_id": "lang-triton-original-c015", "claim_type": "performance", "covers": ["u033", "u037"], "evidence": [{"locator": "Leaderboard rows and labels", "observation": "The current live page still shows the values but labels them only by author, speedup, resolved percentage, and workload count; it supplies no Triton version, historical-snapshot date, or Triton-only language filter.", "revision": "live leaderboard retrieved 2026-08-08", "url": "https://bench.flashinfer.ai/"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "offsets is an implicit Triton builtin.", "check": "Compare the function arguments and Triton language namespace with the AST load set.", "outcome": "offsets is neither an argument, assignment, nor tl builtin, so the displayed function cannot compile as written."}, "claim": "The gated_delta_net_decode block is a usable Triton kernel implementing its stated recurrent-state update.", "claim_id": "lang-triton-original-c016", "claim_type": "code", "covers": ["u038"], "evidence": [{"command": "conda run -n base python -c ", "environment": "Conda base, Python 3.13.14, host AST analysis", "negative_control": "A well-scoped control function taking offsets and d as arguments reports no undefined names.", "reference": "A self-contained kernel must define every loaded local name or receive it as an argument.", "result": "gated_delta_net_decode loads undefined name offsets; its state/output address expressions are also inconsistent between load and store."}], "oracle": "host-experiment", "verdict": "refuted"}, {"challenge": {"alternative": "d and offsets are compiler-provided names and the omitted store is outside the excerpt by design.", "check": "Inspect the complete fenced block, signature, and loaded names.", "outcome": "The block closes without defining the names or producing Output, so it is incomplete executable-looking code."}, "claim": "The sparse_attention_fwd block is a usable Triton sparse-attention kernel.", "claim_id": "lang-triton-original-c017", "claim_type": "code", "covers": ["u039"], "evidence": [{"command": "conda run -n base python -c ", "environment": "Conda base, Python 3.13.14, host AST analysis", "negative_control": "A well-scoped control function taking offsets and d as arguments reports no undefined names.", "reference": "A self-contained kernel must define every loaded local name or receive it as an argument and must write its promised output.", "result": "sparse_attention_fwd loads undefined d and offsets, stops at an unfinished softmax comment, and never stores Output."}], "oracle": "host-experiment", "verdict": "refuted"}, {"challenge": {"alternative": "CUDA Graphs are unrelated to launch overhead or GDN opts out of vLLM's default graph path.", "check": "Inspect the CUDA guide plus the pinned compilation default and GDN metadata builder.", "outcome": "The sources directly connect graphs to CPU launch cost and explicitly implement GDN decode graph capture."}, "claim": "CPU launch overhead can materially affect short-kernel workflows; CUDA Graphs reduce repeated CPU launch costs, and vLLM at commit a1257fd1 defaults to FULL_AND_PIECEWISE graphs and has GDN decode graph support.", "claim_id": "lang-triton-original-c018", "claim_type": "performance", "covers": ["u044", "u045", "u046"], "evidence": [{"locator": "4.2 CUDA Graphs introduction", "observation": "The guide states that per-kernel host setup is significant for short kernels and graphs reduce CPU launch costs by preparing work in advance.", "revision": "CUDA Programming Guide 13.2", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cuda-graphs.html"}, {"locator": "CUDAGraphMode and CompilationConfig.cudagraph_mode documentation", "observation": "FULL_AND_PIECEWISE is documented as the v1 default and most performant mode for most models.", "revision": "vLLM commit a1257fd1", "url": "https://github.com/vllm-project/vllm/blob/a1257fd1/vllm/config/compilation.py"}, {"locator": "GDNAttentionMetadataBuilder and build_for_cudagraph_capture", "observation": "The GDN backend advertises uniform-batch graph support and implements decode-only full graph capture.", "revision": "vLLM commit a1257fd1", "url": "https://github.com/vllm-project/vllm/blob/a1257fd1/vllm/v1/attention/backends/gdn_attn.py"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "At least one link is a summary, derived asset, or stale target.", "check": "Resolve paths, read bundle asset_mode/upstream_sha, verify hashes, and inspect the referenced symbols.", "outcome": "Every linked file is present and provenance marks it verbatim; the broken control fails as expected."}, "claim": "All six artifact links in the examples table resolve to provenance-pinned verbatim upstream files, and their file-purpose descriptions match the code at the recorded revisions.", "claim_id": "lang-triton-original-c019", "claim_type": "other", "covers": ["u047", "u048", "u049", "u050", "u051", "u052", "u053", "u054"], "evidence": [{"command": "conda run -n base python -c ", "environment": "Conda base, Python 3.13.14, repository filesystem", "negative_control": "artifacts/prs/vllm/PR-00000/key-files/missing.py does not resolve.", "reference": "Each relative link must resolve and its hash must match a verbatim provenance entry.", "result": "All six paths exist; their five bundles declare asset_mode verbatim and exact upstream SHAs/hashes. Code inspection finds the described MLA, FP8-cache, format-conversion, normalization, GDN projection, and extend-attention roles."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "42 is a filtered sublist explicitly identified by the page.", "check": "Inspect the file's top-level lane, total, captured, skipped, and prs fields.", "outcome": "The page says full universe, while the canonical full list has 267 entries."}, "claim": "data/triton-universe.yaml enumerates exactly 42 pull requests.", "claim_id": "lang-triton-original-c020", "claim_type": "numeric", "covers": ["u055"], "evidence": [{"command": "conda run -n base python -c ", "environment": "Conda base, Python 3.13.14, PyYAML", "negative_control": "The equality check actual_count == 42 evaluates false while actual_count == declared total evaluates true.", "reference": "The declared total and actual list length should equal the wiki's 42.", "result": "The file declares total=267 and contains 267 PR entries: 117 captured and 150 skipped."}], "oracle": "host-experiment", "verdict": "refuted"}, {"challenge": {"alternative": "The linked artifacts violate the capture ledger.", "check": "Join their source PR IDs against data/triton-universe.yaml.", "outcome": "Every linked bundle is marked captured:true."}, "claim": "Entries marked captured:false in data/triton-universe.yaml are intentionally not shipped; the five table bundles are captured:true.", "claim_id": "lang-triton-original-c021", "claim_type": "other", "covers": ["u047", "u055"], "evidence": [{"command": "conda run -n base python -c ", "environment": "Conda base, Python 3.13.14, PyYAML", "negative_control": "A nonexistent PR ID has no universe entry or artifact path.", "reference": "Linked bundles should be captured and skipped entries should lack shipped bundles by policy.", "result": "All five linked PR IDs are captured:true; the universe has 150 captured:false entries and the table files all exist."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "The manifest exists under another key and still supports the labels.", "check": "Parse every top-level key and compare the named PR dates to the sole cutoff.", "outcome": "There is no manifest field and no named PR is post-cutoff, so the administrative classifications are stale."}, "claim": "PR 34597 and PR 29339 are post-refresh while PR 22079 and PR 21019 are pre-refresh according to data/refresh-cutoff.yaml::previous_pages_manifest.", "claim_id": "lang-triton-original-c022", "claim_type": "semantics", "covers": ["u017", "u023", "u028", "u029", "u030", "u031", "u054"], "evidence": [{"command": "conda run -n base python -c ", "environment": "Conda base, Python 3.13.14, PyYAML", "negative_control": "A synthetic 2026-05-21 date is correctly classified after the cutoff.", "reference": "The asserted manifest should exist and distinguish the named entries relative to the active cutoff.", "result": "The file contains only cutoff_date=2026-05-20 and no previous_pages_manifest; all four PR dates precede the active cutoff."}], "oracle": "host-experiment", "verdict": "refuted"}, {"challenge": {"alternative": "The pointer is dangling even though the prose mentions a registry.", "check": "Resolve it through the repository validator and inspect applies_to.", "outcome": "The structural pointer is valid; its semantic lower bound is separately refuted by c002."}, "claim": "The page's version_sensitive pointer resolves to a registry entry that applies back to this page.", "claim_id": "lang-triton-original-c023", "claim_type": "other", "covers": ["u056"], "evidence": [{"command": "conda run -n base python scripts/validate.py", "environment": "Conda base, Python 3.13.14", "negative_control": "The registry validator's unresolved-pointer rule rejects an unknown ID in its test path.", "reference": "The validator requires the pointer ID to resolve and the registry entry to include the page in applies_to.", "result": "Validation passes with vs-triton-3.6-blackwell-tcgen05 resolving to an entry that lists wiki/languages/triton-blackwell.md."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["git compare and grep across exact Triton v3.2.0, v3.3.0, v3.4.0, v3.5.0, and v3.6.0 tags", "conda run -n base python AST audit of both wiki code blocks with a well-scoped negative control", "conda run -n base python YAML/link/provenance/count audits with broken controls", "conda run -n base python scripts/validate.py"], "id": "lang-triton", "page_verdict": "refuted", "path": "wiki/languages/triton-blackwell.md", "residual_risk": "No GPU was available, so no new PTX or performance run was attempted. Claims requiring such proof were either limited to what pinned sources report or refuted when the page asserted the absent proof.", "scope": "Original body hash 6b9394376db86dd1090ef2012a0d599ad3ac0703f7df8c9bbf9c61c4c24888f; all 56 units. Version boundary checked from v3.2.0 through v3.7.1; downstream claims pinned to merge SHAs; leaderboard retrieved 2026-08-08."} +{"claims": [{"challenge": {"alternative": "The 128 values are distributed across the whole warpgroup rather than held by every thread.", "check": "Read the fragment table's per-thread vector expression and substitute N=256.", "outcome": "The table explicitly describes each thread's fragment as N/2 f32 registers."}, "claim": "For Hopper wgmma.mma_async.m64n256k16 with an f32 accumulator, each of the 128 warpgroup threads holds N/2 = 128 f32 accumulator registers; high per-thread register use can constrain occupancy.", "claim_id": "migration-register-to-tmem-original-c001", "claim_type": "mechanism", "covers": ["u001"], "evidence": [{"locator": "m64nNk16 accumulator D fragment table", "observation": "The f32 accumulator is N/2 registers per thread, hence 128 when N=256.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16"}, {"locator": "occupancy limits", "observation": "The guide identifies register usage as one factor limiting active blocks/warps.", "revision": "CUDA 13.0.2 Hopper Tuning Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/hopper-tuning-guide/index.html#occupancy"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "TMEM is merely a name for accumulator registers or has 512 bytes rather than columns.", "check": "Inspect the address-space organization and MMA destination operand type.", "outcome": "It is a distinct dynamically allocated space and column allocation covers all 128 lanes."}, "claim": "On sm_100a, tcgen05 MMA writes D in Tensor Memory, whose CTA-visible structure is 128 lanes by 512 columns of 32-bit cells (256 KiB total addressable structure).", "claim_id": "migration-register-to-tmem-original-c002", "claim_type": "mechanism", "covers": ["u002"], "evidence": [{"locator": "Tensor Memory organization and tcgen05 MMA D operand", "observation": "The ISA defines 512 columns, 128 rows/lanes, 32-bit cells and TMEM destination addresses for D.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "Putting the resident MMA accumulator in TMEM eliminates accumulator register pressure entirely and necessarily frees the full register file for epilogues, enables larger tiles, and removes registers as an occupancy bottleneck.", "claim_id": "migration-register-to-tmem-original-c003", "claim_type": "performance", "covers": ["u002"], "evidence": [{"locator": "occupancy limits", "observation": "Blackwell still has a 64K-register file and 255-register per-thread limit; all nonresident accumulator fragments, addresses, loop state, and epilogue batches still consume registers. Occupancy remains resource- and launch-specific.", "revision": "CUDA 13.0.2 Blackwell Tuning Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "64K denotes bytes or registers per processing block.", "check": "Read the unit and scope in the occupancy bullet.", "outcome": "It explicitly says 32-bit registers per SM."}, "claim": "An SM90 SM has 65,536 32-bit registers (256 KiB).", "claim_id": "migration-register-to-tmem-original-c004", "claim_type": "compatibility", "covers": ["u003"], "evidence": [{"locator": "Streaming Multiprocessor occupancy bullets", "observation": "The official guide states 64K 32-bit registers per SM; multiplying by four bytes gives 256 KiB.", "revision": "CUDA 13.0.2 Hopper Tuning Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/hopper-tuning-guide/index.html#occupancy"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The detailed 174-register Hopper budget and its conclusion of exactly two warpgroups/equivalently two CTAs per SM are generally valid for m64n256k16 BF16.", "claim_id": "migration-register-to-tmem-original-c005", "claim_type": "code", "covers": ["u004"], "evidence": [{"locator": "A and D register fragment tables", "observation": "The register-A BF16 fragment is four f16x2 registers, not the claimed 16; only the 128-register f32 D count is ISA-defined.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16"}, {"locator": "occupancy calculation", "observation": "Active blocks depend on block size, allocated registers, shared memory, barriers, and allocation granularity; a warpgroup is not inherently one CTA.", "revision": "CUDA 13.0.2 Programming Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Changing the example to k32 and double-buffering necessarily yields 300 registers per thread, 38,400 per warpgroup, and exactly one CTA per SM.", "claim_id": "migration-register-to-tmem-original-c006", "claim_type": "code", "covers": ["u005", "u006"], "evidence": [{"locator": "maximum registers per thread and occupancy", "observation": "The architectural maximum is 255 registers per thread, and actual allocation/spilling is compiler- and kernel-specific.", "revision": "CUDA 13.0.2 Hopper Tuning Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/hopper-tuning-guide/index.html#occupancy"}, {"locator": "D fragment table", "observation": "Accumulator register count depends on N and dtype, not K; double buffering is an implementation choice rather than an m64n256k32 consequence.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Local memory means on-chip storage private to the thread and therefore register-speed.", "check": "Inspect the programming guide's physical placement and caching description.", "outcome": "Local is a scope/name, not on-chip placement; accesses use device-memory paths."}, "claim": "Compiler register spills use per-thread local memory, which resides in device memory and may be cached in L2 and, depending on architecture, L1; spills can materially hurt performance.", "claim_id": "migration-register-to-tmem-original-c007", "claim_type": "mechanism", "covers": ["u007"], "evidence": [{"locator": "Local Memory", "observation": "Automatic variables may be placed in local memory, which is off-chip device memory and follows documented cache behavior; the guide warns that access is expensive.", "revision": "CUDA 13.0.2 Programming Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#local-memory"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "Register, L1-local, and L2-spill accesses universally cost about 4, 30, and 200 cycles, so each spilled accumulator adds exactly 200 times K_tiles cycles.", "claim_id": "migration-register-to-tmem-original-c008", "claim_type": "code", "covers": ["u008"], "evidence": [{"locator": "Local memory performance", "observation": "Official guidance does not define these fixed latencies; caching, dependencies, coalescing, occupancy, compiler placement, and overlap determine cost. One spill access does not imply a serialized 200-cycle addition per K tile.", "revision": "CUDA 13.0.2 Best Practices Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-best-practices-guide/index.html#local-memory"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The 29-register Blackwell budget, 7,424-register CTA total, eight-CTA register limit, two-CTA TMEM limit, and categorical TMEM-bound occupancy conclusion are generally valid for m128n256k16.", "claim_id": "migration-register-to-tmem-original-c009", "claim_type": "code", "covers": ["u009", "u010"], "evidence": [{"locator": "occupancy limits", "observation": "The guide requires occupancy calculation from actual block size, compiled register allocation, shared memory, cluster shape and other resources; none of the page's per-thread category counts are architectural constants.", "revision": "CUDA 13.0.2 Blackwell Tuning Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"}, {"locator": "dynamic column allocation", "observation": "TMEM allocations are dynamic in power-of-two column units; the ISA does not derive the page's universal CTA residency limit from an MMA N dimension.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory-allocation"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "wgmma writes D to shared memory or TMEM and only later transfers it to registers.", "check": "Inspect the D operand class and fragment table.", "outcome": "D is an explicit per-thread register vector in the wgmma instruction."}, "claim": "The Hopper pseudocode captures the central register lifecycle: an f32 m64n256 accumulator is a 128-register fragment per thread, wgmma updates register D asynchronously, commit/wait completes a group, and the registers are then directly available to the issuing threads.", "claim_id": "migration-register-to-tmem-original-c010", "claim_type": "code", "covers": ["u011"], "evidence": [{"locator": "wgmma register fragments, mma_async, commit_group, and wait_group", "observation": "The page's high-level register-D lifecycle and 128-register count agree, although helper names are explicitly pseudocode.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-multiply-accumulate-instructions-wgmma-mma"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The Hopper pseudocode proves its arrays remain in registers and that no registers remain for epilogue temporaries.", "claim_id": "migration-register-to-tmem-original-c011", "claim_type": "code", "covers": ["u011"], "evidence": [{"locator": "automatic arrays and local memory", "observation": "A C++ array's placement is compiler-dependent; dynamic indexing or pressure can place it in local memory. Register headroom also depends on the compiled kernel, not the illustrative tally.", "revision": "CUDA 13.0.2 Programming Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#local-memory"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The displayed Blackwell CUDA/PTX block is a correct accumulator lifecycle: lane 0 allocates/deallocates, 32x1b stores/loads initialize/read f32, a literal enables input D, no completion is needed, and a fence plus syncthreads makes results readable.", "claim_id": "migration-register-to-tmem-original-c012", "claim_type": "code", "covers": ["u012"], "evidence": [{"locator": "alloc/dealloc issue granularity", "observation": "cta_group::1 allocation and deallocation require all lanes of one warp, not lane 0.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"}, {"locator": "ld/st shape grammar and completion", "observation": "The displayed 32x1b forms are not legal; ld/st are warp-collective asynchronous operations with matching waits.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"}, {"locator": "f16 grammar, predicate and completion semantics", "observation": "enable-input-D is a predicate, and MMA completion must be tracked before readers/reuse; fence is ordering, not completion.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "With TMEM, tile size is no longer constrained by registers; Hopper m128n256 necessarily spills, while Blackwell m128n256/m128n512/m256n256 have the stated universal fit and occupancy properties.", "claim_id": "migration-register-to-tmem-original-c013", "claim_type": "code", "covers": ["u013", "u014"], "evidence": [{"locator": "occupancy resources", "observation": "Registers remain a kernel resource and practical tile choice depends on all resources and measured workload behavior.", "revision": "CUDA 13.0.2 Blackwell Tuning Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"}, {"locator": "supported matrix shapes", "observation": "The common single instruction has N up to 256; an m128n512 output would require an implementation decomposition, not the claimed one-instruction fit.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "A Blackwell epilogue has the full register file available, and the displayed rich epilogue necessarily uses about 24 temporaries without spilling whereas the Hopper equivalent often spills.", "claim_id": "migration-register-to-tmem-original-c014", "claim_type": "code", "covers": ["u015", "u016"], "evidence": [{"locator": "register limits", "observation": "Blackwell has 64K registers/SM and a 255-register/thread limit shared by all live variables; TMEM residency reduces one source of pressure but does not reserve the full file for an epilogue or prove allocation/spilling for pseudocode.", "revision": "CUDA 13.0.2 Blackwell Tuning Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Overlapping one tile's epilogue with another tile's MMA is impossible with register accumulators, and the two-allocation sketch is a complete safe tcgen05 overlap protocol.", "claim_id": "migration-register-to-tmem-original-c015", "claim_type": "code", "covers": ["u017", "u018"], "evidence": [{"locator": "Hopper asynchrony and warp-specialization contributions", "observation": "FA3 overlaps Hopper tensor-core work with non-matmul work via warp specialization/interleaving despite register accumulators, disproving impossibility.", "revision": "FlashAttention-3 paper, arXiv 2407.08608", "url": "https://arxiv.org/abs/2407.08608"}, {"locator": "commit, waits, lifetimes, and allocation rules", "observation": "Safe overlap requires explicit completion and no conflicting read/write/reuse; the sketch's opaque fence_and_sync and helper calls do not establish those edges.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "FA4 keeps both GEMM accumulators resident in ordinary registers as on Hopper.", "check": "Inspect the paper's thread-memory mapping and forward pipeline.", "outcome": "Its Blackwell design explicitly uses TMEM for the score/output accumulator storage."}, "claim": "FlashAttention-4 is a concrete Blackwell case using TMEM-resident score/output accumulators and scheduling non-matmul work around asynchronous tensor-core work.", "claim_id": "migration-register-to-tmem-original-c016", "claim_type": "mechanism", "covers": ["u019"], "evidence": [{"locator": "forward kernel architecture and ping-pong scheduling", "observation": "The paper places attention accumulators in TMEM and assigns MMA, softmax, and correction roles to overlap asymmetric units.", "revision": "FlashAttention-4 paper v1", "url": "https://arxiv.org/abs/2603.05451v1"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The Hopper and Blackwell FlashAttention register budgets of about 254 and 20 registers per thread, two- and 236-register margins, and resulting spill/no-spill conclusions are measured or source-defined.", "claim_id": "migration-register-to-tmem-original-c017", "claim_type": "code", "covers": ["u020", "u021"], "evidence": [{"locator": "architecture and evaluation", "observation": "The paper explains TMEM and warp roles but does not publish these per-thread category totals or margins; actual register allocation is variant/compiler-specific and the claimed arithmetic omits live epilogue/pipeline state.", "revision": "FlashAttention-4 paper v1", "url": "https://arxiv.org/abs/2603.05451v1"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "All exponentials use the polynomial path or 1605 is an independently reproduced fixed-shape median.", "check": "Read the selection description and maximum-performance wording.", "outcome": "Only a selected fraction uses software exp, and the number is an author-reported maximum; the page's wording is correspondingly source-scoped only in this claim."}, "claim": "The first-party FA4 report uses software-selected polynomial exp2 work on CUDA cores and reports up to 1605 TFLOP/s on B200 BF16 at a labeled 71 percent.", "claim_id": "migration-register-to-tmem-original-c018", "claim_type": "performance", "covers": ["u022"], "evidence": [{"locator": "software exponential and performance sections", "observation": "The author describes selective polynomial/FMA exp2 computation and reports up to 1605 TFLOP/s on B200 BF16, labeled 71 percent.", "revision": "Tri Dao first-party FA4 post, fetched 2026-08-08", "url": "https://tridao.me/blog/2026/flash4/"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A TMEM address can be cast to a CUDA pointer and dereferenced like shared/global memory.", "check": "Inspect defined state spaces and the instruction-only addressing model.", "outcome": "No CUDA/PTX generic load/store state space maps to TMEM; dedicated tcgen05 movement is required."}, "claim": "TMEM is not directly addressable by ordinary CUDA expressions; tcgen05 data-movement instructions transfer values between TMEM and registers for scalar/epilogue computation.", "claim_id": "migration-register-to-tmem-original-c019", "claim_type": "code", "covers": ["u023"], "evidence": [{"locator": "TMEM addressing and tcgen05.ld/st", "observation": "TMEM uses a special 32-bit lane/column address and is accessed by the tcgen05 instruction family; ld transfers TMEM data to per-thread registers.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "Every fine-grained TMEM load costs 420 cycles while an x4 vector form fetches four adjacent values in one 420-cycle access, so the displayed loop is a valid optimization.", "claim_id": "migration-register-to-tmem-original-c020", "claim_type": "code", "covers": ["u024"], "evidence": [{"locator": "ld shape/repetition/register mapping", "observation": "The instruction has collective shape- and repetition-specific mapping; it is not a scalar tmem_load_f32 API, and x4 does not establish the page's adjacent-scalar abstraction or latency.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"}, {"locator": "memory hierarchy latency experiment", "observation": "The cited approximately 420-cycle figure concerns a measured cache-miss path in that methodology, not a normative per-tcgen05.ld latency for every shape.", "revision": "Blackwell microbenchmark paper v1", "url": "https://arxiv.org/abs/2512.02189"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "A Blackwell migration should use at minimum m128n256 for one-SM or m256n256 for two-SM MMA, and those choices carry no spilling risk.", "claim_id": "migration-register-to-tmem-original-c021", "claim_type": "code", "covers": ["u025"], "evidence": [{"locator": "shape constraints", "observation": "The ISA supports shape choices through descriptors but does not prescribe a performance minimum.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}, {"locator": "occupancy guidance", "observation": "Tile selection and spills depend on the full kernel, shape, resources and measurements; TMEM does not guarantee spill-free code.", "revision": "Blackwell Tuning Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Kernel exit implicitly releases TMEM as it does local registers.", "check": "Read the explicit lifecycle requirement in the allocation section.", "outcome": "The ISA states the opposite: explicit deallocation is mandatory."}, "claim": "Every TMEM allocation must be explicitly deallocated before kernel exit, including persistent-kernel termination paths.", "claim_id": "migration-register-to-tmem-original-c022", "claim_type": "code", "covers": ["u026"], "evidence": [{"locator": "allocation/deallocation description", "observation": "The ISA requires all allocated Tensor Memory to be deallocated with tcgen05.dealloc before the kernel exits.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "Omitting deallocation merely leaks TMEM to the next CTA, which then cannot allocate, and making one source path reach a helper is sufficient cleanup.", "claim_id": "migration-register-to-tmem-original-c023", "claim_type": "mechanism", "covers": ["u026"], "evidence": [{"locator": "deallocation and collective issue rules", "observation": "The ISA specifies a must-deallocate contract and collective matching operation; it does not define omission as a recoverable cross-CTA leak. Every actual exit path and every lane of the designated warp must satisfy the protocol.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "tcgen05 D is still an implicit register vector or TMEM disappears automatically on scope exit.", "check": "Compare operand classes and lifecycle sections for both instruction families.", "outcome": "The architectures expose distinct storage and cleanup contracts exactly as summarized in these rows."}, "claim": "The summary correctly contrasts register-resident wgmma D with TMEM-resident tcgen05 D, direct register result use with collective tcgen05.ld readback, implicit register lifetime with explicit TMEM deallocation, and the fact that double-buffered resident accumulators consume two register fragments versus two TMEM regions.", "claim_id": "migration-register-to-tmem-original-c024", "claim_type": "mechanism", "covers": ["u027", "u029", "u030", "u031", "u032", "u034"], "evidence": [{"locator": "wgmma D register fragments", "observation": "Hopper D is a per-thread register vector.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16"}, {"locator": "tcgen05 D, ld, allocation/deallocation", "observation": "Blackwell D is addressed in TMEM, read through tcgen05.ld, and explicitly deallocated; separate live buffers consume separate storage regions.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "A migrated accumulator must be zero-initialized with a tcgen05.st zero pattern before its first MMA.", "claim_id": "migration-register-to-tmem-original-c025", "claim_type": "code", "covers": ["u028"], "evidence": [{"locator": "enable-input-D semantics", "observation": "When enable-input-D is false, the operation computes D=A*B without reading prior D, so an explicit zero store is not universally required.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "TMEM gives every epilogue ample access to the full register file with only minimal spill risk.", "claim_id": "migration-register-to-tmem-original-c026", "claim_type": "performance", "covers": ["u033"], "evidence": [{"locator": "register file and per-thread limits", "observation": "All other live values and loaded TMEM batches consume the same finite register file; the spill outcome requires compiled-resource evidence for a concrete kernel.", "revision": "CUDA 13.0.2 Blackwell Tuning Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "m64n256 is Hopper's universal maximum practical tile, while m128n256 or m256n256 are Blackwell's universal practical maxima determined only by TMEM columns.", "claim_id": "migration-register-to-tmem-original-c027", "claim_type": "performance", "covers": ["u035"], "evidence": [{"locator": "instruction shapes", "observation": "Instruction shape is not a complete CTA tile or practical-kernel maximum; kernels compose multiple instructions.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-multiply-accumulate-instructions-wgmma-mma"}, {"locator": "resource limits", "observation": "Practical tiles depend on workload and all resources rather than a universal architecture-only maximum.", "revision": "Blackwell Tuning Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"}], "oracle": "web-primary", "verdict": "refuted"}], "commands": ["direct archived PTX/tuning-guide inspection", "FA3/FA4 primary-paper comparison", "host arithmetic checks for N/2 and TMEM byte capacity"], "id": "migration-register-to-tmem", "page_verdict": "refuted", "path": "wiki/migration/register-to-tmem.md", "residual_risk": "No Hopper/Blackwell compiler or GPU is installed locally. Architectural storage and instruction contracts are source-verified; no unsourced register allocation, latency, occupancy, or performance estimate is accepted.", "scope": "All 35 baseline units for body hash d18a8529e0c9cab4d1711c15938892a04771dbccb5548fc61e5298b0fd6c0ed4 were checked against CUDA 13.0.2 PTX ISA 9.0, archived Hopper/Blackwell guidance, and author-primary FA3/FA4 papers."} +{"claims": [{"challenge": {"alternative": "tcgen05 is only a renamed WGMMA encoding and retains warpgroup-issued register fragments.", "check": "Compare issue granularity and D operand locations in the two pinned ISA sections.", "outcome": "Both properties change explicitly, discriminating the programming models."}, "claim": "Porting an SM90 WGMMA kernel to SM100 tcgen05 changes MMA issue from a warpgroup to one thread and moves the D accumulator from per-thread registers to TMEM.", "claim_id": "migration-wgmma-to-tcgen05-original-c001", "claim_type": "compatibility", "covers": ["u001", "u002", "u003"], "evidence": [{"locator": "WGMMA execution model; tcgen05 issue granularity and MMA overview", "observation": "WGMMA is a warpgroup collective with register D; tcgen05 MMA has single-thread issue and TMEM D.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-matrix-multiply-and-accumulate-operations"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The migration universally changes ldmatrix-plus-register A into direct-SMEM A and replaces WGMMA synchronization with async fences.", "claim_id": "migration-wgmma-to-tcgen05-original-c002", "claim_type": "semantics", "covers": ["u004", "u005"], "evidence": [{"locator": "WGMMA A operand forms; tcgen05 A grammar; fence and commit sections", "observation": "WGMMA has both descriptor-A and register-A forms; tcgen05 A can be SMEM or TMEM. tcgen05 fences order cross-thread operations, while commit plus mbarrier provides MMA completion.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "FP4/FP6 appear only in software conversion and not in tcgen05 MMA grammar.", "check": "Inspect the exact tcgen05 kind and block-scale grammar tokens.", "outcome": "The native grammar directly names the narrow-precision kinds."}, "claim": "Compared with Hopper WGMMA FP8 support, the tcgen05 family adds native FP6/FP4 and block-scaled MX/NVFP4 kinds.", "claim_id": "migration-wgmma-to-tcgen05-original-c003", "claim_type": "compatibility", "covers": ["u006"], "evidence": [{"locator": "WGMMA alternate floating formats and tcgen05 kind inventory", "observation": "The WGMMA section includes FP8 formats, while tcgen05 adds f8f6f4 and the block-scaled mxf8f6f4, mxf4, and mxf4nvf4 kinds.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-matrix-multiply-and-accumulate-operations"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The migration checklist is correct as written, including mandatory ldmatrix removal, 64B-to-128B swizzle conversion, and replacing commit/wait with fences.", "claim_id": "migration-wgmma-to-tcgen05-original-c004", "claim_type": "code", "covers": ["u007"], "evidence": [{"locator": "Table 40, MMA operand grammar, and completion mechanism", "observation": "tcgen05 supports several valid swizzles and SMEM/TMEM A; MMA completion requires commit/mbarrier. The checklist's categorical substitutions are false.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The Hopper kernel block is valid or structurally faithful WGMMA CUDA for the migration baseline.", "claim_id": "migration-wgmma-to-tcgen05-original-c005", "claim_type": "code", "covers": ["u008"], "evidence": [{"locator": "MMA_64x256x16 WGMMA wrapper signatures", "observation": "The wiki block contains literal ellipses and undefined names and combines an ldmatrix fragment with a made-up WGMMA operand list. The pinned wrappers show exact descriptor-A or register-A signatures instead.", "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/include/cute/arch/mma_sm90_gmma.hpp", "revision": "CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The Blackwell kernel block correctly demonstrates TMEM allocation, tcgen05 MMA, completion, TMEM load, and deallocation.", "claim_id": "migration-wgmma-to-tcgen05-original-c006", "claim_type": "code", "covers": ["u009"], "evidence": [{"locator": "alloc/dealloc issue semantics, mma grammar, commit, and ld access restrictions", "observation": "alloc/dealloc are warp-collective rather than lane-0 operations; the MMA omits its lane mask; fence/syncthreads is not completion; tcgen05.ld is warp-collective. Multiple undefined helpers further prevent execution.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "128 is a tile-wide count or TMEM allocation returns a pointer-sized generic address.", "check": "Inspect the per-thread wrapper typedef and PTX's b32 taddr allocation result.", "outcome": "The original contrast is correct at the stated conceptual level."}, "claim": "The accumulator examples correctly contrast a 128-register per-thread Hopper m64n256 FP32 fragment with a TMEM-addressed Blackwell accumulator allocation concept.", "claim_id": "migration-wgmma-to-tcgen05-original-c007", "claim_type": "code", "covers": ["u010", "u011", "u012", "u013"], "evidence": [{"locator": "MMA_64x256x16_F32F16F16_SS CRegisters; PTX tcgen05.alloc semantics", "observation": "The wrapper declares float CRegisters[128]; PTX alloc returns a 32-bit TMEM address via shared memory and uses it for TMEM-backed D. The tmem_alloc name is plainly pseudocode, consistent with page metadata.", "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/include/cute/arch/mma_sm90_gmma.hpp", "revision": "CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "A WGMMA port should always remove ldmatrix because Hopper necessarily register-stages A while tcgen05 necessarily uses an SMEM A descriptor.", "claim_id": "migration-wgmma-to-tcgen05-original-c008", "claim_type": "code", "covers": ["u014", "u015", "u016", "u017"], "evidence": [{"locator": "WGMMA SS/RS forms and tcgen05 SMEM-A/TMEM-A grammar", "observation": "Some WGMMA forms already source A through an SMEM descriptor, and tcgen05 can source A from TMEM. Migration depends on the exact source instruction, not a universal ldmatrix step.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "64B swizzle is only a tcgen05 descriptor value and is invalid for WGMMA.", "check": "Use the WGMMA-specific layout section rather than the adjacent tcgen05 table.", "outcome": "The Hopper section directly documents 64B layouts."}, "claim": "The Hopper swizzle example accurately notes that both 64B and 128B swizzled shared-memory layouts are supported WGMMA/TMA choices.", "claim_id": "migration-wgmma-to-tcgen05-original-c009", "claim_type": "code", "covers": ["u018", "u019"], "evidence": [{"locator": "WGMMA shared-memory layouts and descriptor swizzle modes", "observation": "The WGMMA layout section documents both 64-byte and 128-byte swizzle modes.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-shared-memory-layout-matrix-descriptor"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "Blackwell tcgen05 correctness requires converting every operand layout to 128-byte swizzling.", "claim_id": "migration-wgmma-to-tcgen05-original-c010", "claim_type": "code", "covers": ["u020", "u021"], "evidence": [{"locator": "Table 40 and tcgen05 shared-memory layout sections", "observation": "tcgen05 defines valid no-swizzle, 128B, 64B, and 32B modes subject to their layout constraints.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The WGMMA-to-tcgen05 issue snippets are valid pseudocode and tcgen05 needs no commit because it is fully asynchronous.", "claim_id": "migration-wgmma-to-tcgen05-original-c011", "claim_type": "code", "covers": ["u022", "u023", "u024", "u025"], "evidence": [{"locator": "MMA grammar and mbarrier-based completion mechanism", "observation": "Both snippets contain literal ellipses or omit required operands; tcgen05 MMA completion is explicitly observed by tcgen05.commit plus mbarrier wait.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Immediate 0 allows one group to remain outstanding.", "check": "Compare the immediate-count definition and the adjacent example values.", "outcome": "The zero value leaves no committed groups pending before use."}, "claim": "wgmma.wait_group.sync.aligned 0 waits until all committed WGMMA groups are complete before their accumulator registers are consumed.", "claim_id": "migration-wgmma-to-tcgen05-original-c012", "claim_type": "code", "covers": ["u026", "u027"], "evidence": [{"locator": "wgmma.wait_group syntax and description", "observation": "The immediate 0 form waits for completion of all prior committed WGMMA groups.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-wait-group"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "A tcgen05.fence before __syncthreads replaces wgmma.wait_group and makes the TMEM accumulator ready.", "claim_id": "migration-wgmma-to-tcgen05-original-c013", "claim_type": "code", "covers": ["u028", "u029"], "evidence": [{"locator": "mbarrier completion and tcgen05.fence descriptions", "observation": "commit/mbarrier observes MMA completion; fence only orders operations around execution-ordering handoffs and constrains code motion.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "CUDA scalar arithmetic can address TMEM directly without tcgen05.ld.", "check": "Inspect the TMEM access-instruction model and destination of tcgen05.ld.", "outcome": "TMEM-to-GPR transfer is the documented path before ordinary register computation."}, "claim": "A Hopper epilogue can consume its register accumulator directly, whereas a tcgen05 epilogue first transfers TMEM values into registers before scalar post-ops and stores.", "claim_id": "migration-wgmma-to-tcgen05-original-c014", "claim_type": "code", "covers": ["u030", "u031", "u032", "u033"], "evidence": [{"locator": "WGMMA D fragments and 9.7.16.8.3 tcgen05.ld", "observation": "WGMMA returns D in per-thread registers; tcgen05.ld collectively transfers TMEM data into a register vector for further computation.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The two warp-specialization diagrams describe typical required Hopper and Blackwell allocations of 13+ and 5 warps, including a dedicated CLC warp on Blackwell.", "claim_id": "migration-wgmma-to-tcgen05-original-c015", "claim_type": "code", "covers": ["u034", "u035"], "evidence": [{"locator": "thread-role partition and pipeline definitions; Hopper warp-specialized examples", "observation": "Vendor examples choose workload-specific role counts and do not impose the diagrams' totals or a mandatory CLC scheduler warp.", "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/examples/cute/tutorial/blackwell/02_mma_tma_sm100.cu", "revision": "CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "The rest of the issuing warp is forced idle and cannot perform any other pipeline role.", "check": "Inspect the vendor example's role branching and pipelines.", "outcome": "The implementation uses non-issuing execution resources for other work, while leaving the exact partition configurable."}, "claim": "Single-thread MMA issue permits other threads or warps to specialize in prefetch, epilogue, or reduction work, but the profitable role partition is kernel-dependent.", "claim_id": "migration-wgmma-to-tcgen05-original-c016", "claim_type": "mechanism", "covers": ["u036", "u037", "u038", "u039"], "evidence": [{"locator": "warp-specialized producer/consumer pipeline", "observation": "The pinned example assigns distinct data movement, MMA, and accumulator-consumer roles around a single-thread-issued MMA.", "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/examples/cute/tutorial/blackwell/02_mma_tma_sm100.cu", "revision": "CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "Blackwell's base MMA tile universally doubles Hopper M from 64 to 128, and migration should mechanically double CTA M or choose the two fixed table shapes.", "claim_id": "migration-wgmma-to-tcgen05-original-c017", "claim_type": "semantics", "covers": ["u040", "u041", "u042", "u043", "u044", "u045"], "evidence": [{"locator": "WGMMA shape inventory; tcgen05 idesc and data-path layouts", "observation": "Both instruction families expose multiple legal shapes. tcgen05 M/N are descriptor- and configuration-selected, so tile migration is not a fixed two-row lookup or mandatory doubling rule.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-data-path-layout-organization"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The pitfall list correctly says missing 128B swizzle silently corrupts tcgen05, fence-plus-syncthreads is the required completion path, and retaining Hopper thread counts necessarily wastes resources.", "claim_id": "migration-wgmma-to-tcgen05-original-c018", "claim_type": "semantics", "covers": ["u046", "u048", "u049"], "evidence": [{"locator": "descriptor modes, issue granularity, commit, and fence", "observation": "Several swizzles are valid, completion uses commit/mbarrier, and one-thread MMA issue does not determine the thread requirements of loads, epilogue, scheduling, or other kernel work.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "CTA teardown implicitly releases TMEM, or identical register limits dominate both kernels.", "check": "Inspect the explicit before-exit mandate and compare accumulator storage locations.", "outcome": "The ISA requires deallocation and the accumulator resource changes, so both scoped statements hold."}, "claim": "All dynamically allocated TMEM must be explicitly deallocated before kernel exit, and removing register-resident accumulators warrants retuning rather than assuming Hopper's register budget remains optimal.", "claim_id": "migration-wgmma-to-tcgen05-original-c019", "claim_type": "semantics", "covers": ["u047", "u050"], "evidence": [{"locator": "9.7.16.7.1 allocation lifecycle; WGMMA and tcgen05 D locations", "observation": "PTX requires every allocation to be deallocated before exit. Moving D from GPRs to TMEM changes one resource constraint, supporting a retuning recommendation without guaranteeing a particular optimum.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "CUTLASS migration is largely an arch-tag and schedule-token substitution using the shown device::GemmUniversal and KernelScheduleSm100CpAsyncWarpSpecialized types, after which MMA_Atom/MMA_Traits automatically handle all differences.", "claim_id": "migration-wgmma-to-tcgen05-original-c020", "claim_type": "code", "covers": ["u051", "u052", "u053"], "evidence": [{"locator": "SM100 CollectiveBuilder and GemmUniversal construction; gemm/dispatch_policy.hpp", "observation": "The displayed schedule type does not exist in pinned CUTLASS. SM100 examples configure collective builders, tile/cluster shapes, stages, and supported schedule tags such as KernelTmaWarpSpecialized1SmSm100 before instantiating kernel::GemmUniversal and a device adapter.", "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/examples/70_blackwell_gemm/70_blackwell_fp16_gemm.cu", "revision": "CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "refuted"}], "id": "migration-wgmma-to-tcgen05", "page_verdict": "refuted", "path": "wiki/migration/wgmma-to-tcgen05.md", "residual_risk": "No CUDA toolkit or NVIDIA GPU is installed locally, so no migrated kernel was compiled or run. The discriminating ISA and pinned-library checks are sufficient to refute syntax, lifecycle, and API claims but do not predict performance for a specific port.", "scope": "All 53 immutable baseline units were checked against CUDA 13.0.2 PTX ISA 9.0 and pinned CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5. Code was assessed according to the page's pseudocode reproducibility label while executable-looking PTX was still required to preserve real operand and ordering semantics."} +{"claims": [{"claim": "Tensor-core utilization below 70% plus unsaturated memory bandwidth proves a kernel is compute-bound but below peak.", "claim_id": "pattern-compute-bound-original-c001", "claim_type": "performance", "covers": ["u001"], "evidence": [{"locator": "roofline analysis", "observation": "Boundedness depends on arithmetic intensity relative to measured compute and memory ceilings; neither an architecture-wide 70% threshold nor unsaturated DRAM alone establishes it.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The listed events cannot explain under-attainment unless a profiler names them as a stall reason.", "check": "Inspect direct implementations where TMA readiness, non-MMA work, or epilogue occupancy lies on a dependency path.", "outcome": "Pinned tutorial and FA4 sources contain these dependency structures, but the page correctly labels them only as likely causes."}, "claim": "TMA-readiness bubbles, non-matmul work, and an epilogue that delays mainloop reuse are plausible causes of compute-throughput under-attainment.", "claim_id": "pattern-compute-bound-original-c002", "claim_type": "semantics", "covers": ["u002", "u003", "u005"], "evidence": [{"locator": "v3-v6 producer/consumer pipelines", "observation": "The implementations contain TMA-to-MMA and mainloop-to-epilogue dependencies; v6 adds overlap after profiling epilogue/setup idle periods.", "revision": "3b90ac9b3f624bdf1f6f78d02dcd533675d36573", "url": "https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100"}, {"locator": "section 2.2 and 3", "observation": "The paper analyzes exponentials, correction, and SMEM work as non-MMA contributors that become material relative to faster tensor cores.", "revision": "FlashAttention-4 paper v1", "url": "https://arxiv.org/html/2603.05451v1"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "Single-SM MMA tiles are inherently too small to utilize available compute.", "claim_id": "pattern-compute-bound-original-c003", "claim_type": "performance", "covers": ["u004"], "evidence": [{"locator": "cta_group and shape constraints", "observation": "PTX defines multiple legal single-CTA and paired-CTA shapes; it gives no utilization conclusion from CTA-group mode alone.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "2-SM cooperative MMA always doubles the effective tile to m256n256 and provides twice the compute per cycle.", "claim_id": "pattern-compute-bound-original-c004", "claim_type": "performance", "covers": ["u006"], "evidence": [{"locator": "cta_group::2 shapes", "observation": "Legal shapes depend on kind, descriptors, layouts, target, and CTA group; cooperative work occupies two CTAs/SMs and does not promise a universal 2x throughput per cycle.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The candidate rows promise these transformations will improve every compute-bound kernel.", "check": "Read each row as a structural effect and compare it to pinned implementations.", "outcome": "The rows state possible overlap/offload mechanisms without a numeric guarantee; pinned sources demonstrate each structure in a scoped kernel."}, "claim": "Operand stages can overlap TMA and MMA, output buffering/fusion can overlap epilogue and later MMA, and FA4 moves selected exponential evaluations to CUDA-core FMA code.", "claim_id": "pattern-compute-bound-original-c005", "claim_type": "semantics", "covers": ["u007", "u009", "u010"], "evidence": [{"locator": "v3 and v6", "observation": "v3 stages TMA/MMA work and v6 uses output buffering/epilogue specialization for overlap.", "revision": "3b90ac9b3f624bdf1f6f78d02dcd533675d36573", "url": "https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100"}, {"locator": "partial software exponentiation", "observation": "FA4 selects a fraction of exponentials for a cubic polynomial evaluated with FMA while retaining hardware ex2 for others.", "revision": "FlashAttention-4 paper v1", "url": "https://arxiv.org/html/2603.05451v1"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "Warp specialization provides dedicated TMA/MMA/epilogue warps with no stalls.", "claim_id": "pattern-compute-bound-original-c006", "claim_type": "performance", "covers": ["u008"], "evidence": [{"locator": "warp specialization and persistent sections", "observation": "The tutorial reports a scoped improvement but later profiling still observes idle periods and imperfect overlap; role separation does not guarantee no stalls.", "revision": "retrieved 2026-08-09", "url": "https://gau-nernst.github.io/tcgen05/"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "FA4 uses Cody-Waite plus a Horner polynomial for every software exponential, thereby multiplying exponential throughput and causing the full-kernel 1605-TFLOP/s result.", "claim_id": "pattern-compute-bound-original-c007", "claim_type": "code", "covers": ["u011"], "evidence": [{"locator": "partial software exponentiation", "observation": "FA4 uses floor-based base-2 range reduction and a cubic polynomial for a selected fraction of values, with other values using ex2; it does not call this Cody-Waite or isolate a multiplied-throughput result.", "revision": "FlashAttention-4 paper v1", "url": "https://arxiv.org/html/2603.05451v1"}, {"locator": "performance and techniques", "observation": "1605 TFLOP/s and 71% describe the complete forward kernel with several techniques, not a software-exp-only causal result.", "revision": "Tri Dao FA4 blog, 2026", "url": "https://tridao.me/blog/2026/flash4/"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "2-SM cooperative MMA requires cluster configuration and identical shared-memory layouts in the two CTAs.", "claim_id": "pattern-compute-bound-original-c008", "claim_type": "compatibility", "covers": ["u012"], "evidence": [{"locator": "cta_group::2 requirements", "observation": "The CTAs must be in a cluster and satisfy instruction-specific peer/resource requirements, but PTX does not impose the page's universal identical-SMEM-layout rule.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Pipeline depth is workload-dependent, with three to five stages typical.", "claim_id": "pattern-compute-bound-original-c009", "claim_type": "performance", "covers": ["u013"], "evidence": [{"locator": "matmul_v3 launch", "observation": "The cited beneficial pipelined variant uses two stages, directly defeating the unqualified typical range even though depth is configuration-dependent.", "revision": "3b90ac9b3f624bdf1f6f78d02dcd533675d36573", "url": "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "A software transcendental can be exactly rounded and therefore need not trade accuracy.", "check": "Inspect the specific software exponential linked by this page.", "outcome": "FA4's cubic approximation has measured error and is selected only where the algorithm accepts it, making accuracy and throughput explicit design variables in this cited technique."}, "claim": "The cited FA4 software-exponential technique trades approximation accuracy against throughput.", "claim_id": "pattern-compute-bound-original-c010", "claim_type": "semantics", "covers": ["u014"], "evidence": [{"locator": "software exponential accuracy and selection", "observation": "The paper reports approximation error, selects only a subset of values, and evaluates accuracy/performance together.", "revision": "FlashAttention-4 paper v1", "url": "https://arxiv.org/html/2603.05451v1"}], "oracle": "web-primary", "verdict": "verified"}], "commands": ["Inspect Nsight Compute roofline semantics", "Inspect PTX ISA 9.0 cta_group::2 contracts", "Inspect FA4 paper/blog and pinned tutorial implementations"], "id": "pattern-compute-bound", "page_verdict": "refuted", "path": "wiki/patterns/compute-bound.md", "residual_risk": "No B200 or profiler is available locally. Compatibility and algorithms are source-inspected; performance is author-reported.", "scope": "All 14 baseline units for hash 1250f2c8d05fdd1ec4dcb30133b71caf29617b7fe0c0131fcafbe9ca9a4fe320 were reviewed, including the pseudo-code block and every candidate-effect claim."} +{"claims": [{"claim": "SM utilization below 60% despite sufficient occupancy, plus idle-SM intervals, defines the low-utilization symptom.", "claim_id": "pattern-low-sm-utilization-original-c001", "claim_type": "performance", "covers": ["u001"], "evidence": [{"locator": "occupancy, PM sampling, scheduler statistics", "observation": "Occupancy is a residency limit rather than achieved activity, PM samples are temporal observations, and no official 60% threshold defines the pattern.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Only the last-wave remainder can leave SMs idle; imbalance, static assignment, and a small grid cannot.", "check": "Apply the scheduler semantics and basic independent-block count to each listed condition.", "outcome": "Each condition can reduce concurrent useful work, though none is proven by aggregate utilization alone."}, "claim": "A partial final wave, variable tile duration, nonadaptive static assignment, or fewer independent blocks than available workers can each leave SMs without useful work.", "claim_id": "pattern-low-sm-utilization-original-c002", "claim_type": "semantics", "covers": ["u002", "u003", "u004", "u005"], "evidence": [{"locator": "fixed-work and fixed-worker scheduling motivation", "observation": "The guide distinguishes fixed work per block from persistent work acquisition and motivates CLC with uneven runtime SM availability.", "revision": "CUDA Programming Guide, retrieved 2026-08-09", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"}, {"locator": "CLC scheduling model", "observation": "CUTLASS documents initial grid work and redistribution of not-yet-started IDs, consistent with these possible shortage/imbalance mechanisms.", "revision": "CUTLASS 4.5.0", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "CLC is SM100-only dynamic tile assignment that eliminates load imbalance.", "claim_id": "pattern-low-sm-utilization-original-c003", "claim_type": "performance", "covers": ["u006"], "evidence": [{"locator": "target and request semantics", "observation": "The instruction requires sm_100 or higher and redistributes an unspecified not-yet-started ID; it cannot create work or guarantee elimination of imbalance.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Persistent kernels apply on SM90+ and eliminate the tail effect while making launch overhead one-time.", "claim_id": "pattern-low-sm-utilization-original-c004", "claim_type": "performance", "covers": ["u007"], "evidence": [{"locator": "fixed number of thread blocks", "observation": "Persistent workers can process multiple work items, but the guide does not promise elimination of tails; launch count and uncovered parallelism remain workload/scheduler dependent.", "revision": "CUDA Programming Guide, retrieved 2026-08-09", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Tile scheduling on SM90+ necessarily improves L2 locality and reduces load variance.", "claim_id": "pattern-low-sm-utilization-original-c005", "claim_type": "performance", "covers": ["u008"], "evidence": [{"locator": "scheduler and swizzle options", "observation": "Scheduling exposes choices, but coordinate order and work reassignment have no guaranteed cache or variance effect.", "revision": "CUTLASS 4.5.0", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The tcgen05 tutorial moves from 86% without persistence/CLC to 98% with persistence plus CLC, keeping all SMs busy.", "claim_id": "pattern-low-sm-utilization-original-c006", "claim_type": "code", "covers": ["u009"], "evidence": [{"locator": "v5, v6, and closing remarks", "observation": "v5 is 1302.29/1506.74=86.43%; v6 static persistence is 1475.93/1506.74=97.96%. The author explicitly says CLC was not added and notes overlap remains imperfect.", "revision": "retrieved 2026-08-09", "url": "https://gau-nernst.github.io/tcgen05/"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "CLC is available only on SM100 datacenter GPUs and not SM120 consumer GPUs.", "claim_id": "pattern-low-sm-utilization-original-c007", "claim_type": "compatibility", "covers": ["u010"], "evidence": [{"locator": "Target ISA Notes", "observation": "The base instruction requires sm_100 or higher, and the multicast qualifier explicitly lists sm_120a/sm_120f support, refuting the page's SM120 exclusion.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "A persistent worker loop is no harder to inspect than a one-work-item block because the profiler still observes one kernel launch.", "check": "Compare the fixed-work and fixed-worker execution models and identify what must be attributed to reproduce a logical work item.", "outcome": "A persistent block repeatedly acquires and processes logical work, so debugging and profiling must distinguish worker identity, acquired work identity, and loop iteration; this is additional attribution state, not a claim that tools cannot profile the kernel."}, "claim": "Persistent kernels complicate debugging and profiling.", "claim_id": "pattern-low-sm-utilization-original-c008", "claim_type": "semantics", "covers": ["u011"], "evidence": [{"locator": "fixed-work and fixed-worker scheduling models", "observation": "The fixed-worker model has a resident block repeatedly request and process work rather than binding one block to one logical item, introducing a worker-to-work mapping and repeated lifetime that diagnostics must attribute.", "revision": "CUDA Programming Guide, retrieved 2026-08-09", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "A non-persistent kernel should always use a grid much larger than the SM count.", "claim_id": "pattern-low-sm-utilization-original-c009", "claim_type": "performance", "covers": ["u012"], "evidence": [{"locator": "thread-block scheduling and occupancy", "observation": "Useful grid size follows problem decomposition, block resources, and residency; the guide provides no grid-much-greater-than-SM rule, and small problems may not expose that work.", "revision": "CUDA Programming Guide, retrieved 2026-08-09", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html"}], "oracle": "web-primary", "verdict": "refuted"}], "commands": ["Inspect CUDA/PTX CLC compatibility and work semantics", "Inspect CUTLASS 4.5.0 scheduler model", "Inspect tutorial v5/v6 progression"], "id": "pattern-low-sm-utilization", "page_verdict": "refuted", "path": "wiki/patterns/low-sm-utilization.md", "residual_risk": "No GPU profiling was possible. Scheduler compatibility is source-verified and tutorial performance remains author-reported.", "scope": "All 12 baseline units for hash ffe8f3f021f4946f96b95bfc41d6745d38c52fd887cb7171751f8f4e9934e91b were reviewed, including the code block and applicability claims."} +{"claims": [{"claim": "High DRAM throughput, low tensor-core utilization, and arithmetic intensity below a roofline knee establish that a kernel is memory-bandwidth bound.", "claim_id": "pattern-memory-bound-original-c001", "claim_type": "performance", "covers": ["u001"], "evidence": [{"locator": "roofline analysis", "observation": "The roofline needs achieved work, bytes, and relevant measured ceilings; vague 'high' bandwidth and low tensor activity do not establish proximity to the attainable memory roof or exclude latency/instruction limits.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "GEMV, small-batch decode, and reduction kernels are intrinsically low-arithmetic-intensity causes of memory boundedness.", "claim_id": "pattern-memory-bound-original-c002", "claim_type": "performance", "covers": ["u002"], "evidence": [{"locator": "arithmetic intensity", "observation": "Intensity is operations divided by transferred bytes for the concrete implementation/input; an algorithm-category label cannot substitute for that measurement.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Using each element once need not be poor reuse if the algorithm has no reuse to exploit.", "check": "Interpret the row as a possible traffic/intensity mechanism rather than an optimization judgment.", "outcome": "Single-use traffic lowers operations per transferred byte relative to reusable data, although it does not by itself prove a reachable bandwidth ceiling."}, "claim": "Using each transferred data element only once is a possible source of low arithmetic intensity.", "claim_id": "pattern-memory-bound-original-c003", "claim_type": "semantics", "covers": ["u003"], "evidence": [{"locator": "arithmetic-intensity definition", "observation": "Arithmetic intensity is the ratio of operations to bytes; reuse can raise work per transferred byte.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "Uncoalesced loads or L1 thrashing are causes of a memory-bandwidth-bound diagnosis.", "claim_id": "pattern-memory-bound-original-c004", "claim_type": "performance", "covers": ["u004"], "evidence": [{"locator": "Memory Workload Analysis", "observation": "Inefficient transactions and cache behavior can create excess traffic or latency, but can also leave bandwidth below its ceiling; the observations require separate requested/transferred-byte and throughput analysis.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "128- or 256-bit vectorized loads maximize memory-bandwidth utilization.", "claim_id": "pattern-memory-bound-original-c005", "claim_type": "performance", "covers": ["u005"], "evidence": [{"locator": "Attempt 9", "observation": "The cited NVFP4 GEMV author reports that changing to wider uint2 loads made the kernel 16-25% slower, a direct negative control.", "revision": "retrieved 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "L1::no_allocate is the correct policy for streaming data and L1::evict_last for reused data.", "claim_id": "pattern-memory-bound-original-c006", "claim_type": "performance", "covers": ["u006"], "evidence": [{"locator": "eviction priorities", "observation": "These are performance hints that may be applied, not guaranteed placement rules or tensor-role prescriptions.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Setting maxrregcount increases occupancy.", "claim_id": "pattern-memory-bound-original-c007", "claim_type": "performance", "covers": ["u007"], "evidence": [{"locator": "maxrregcount", "observation": "The option limits allocated registers subject to ABI minima; it may introduce spills and need not change the limiting occupancy resource.", "revision": "CUDA Compiler Driver 13.0.2", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html"}, {"locator": "register cap attempt", "observation": "The cited author reports that reducing maxrregcount from 80 to 64 had no performance effect.", "revision": "retrieved 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "TMA multicast merely duplicates separate global loads and therefore shares no loaded data.", "check": "Inspect the normative multicast form and destination mask.", "outcome": "One tensor-copy operation can write selected cluster CTAs' shared-memory destinations, sharing the global operand transfer across the cluster."}, "claim": "TMA multicast can distribute one global-to-shared tensor transfer to selected CTAs in a cluster.", "claim_id": "pattern-memory-bound-original-c008", "claim_type": "semantics", "covers": ["u008"], "evidence": [{"locator": "multicast::cluster form", "observation": "PTX defines a global-to-cluster-shared tensor copy with a CTA mask and per-destination mbarrier completion.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "Shared-memory swizzling eliminates bank conflicts.", "claim_id": "pattern-memory-bound-original-c009", "claim_type": "performance", "covers": ["u009"], "evidence": [{"locator": "swizzle modes and layout constraints", "observation": "Swizzling remaps addresses; conflict behavior depends on access pattern and legal layout, and no mode universally eliminates conflicts.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-map-swizzle"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "For memory-bound NVFP4 GEMV, the correct priority is wide/coalesced loads, lower register count, differentiated cache policy, and never optimizing compute.", "claim_id": "pattern-memory-bound-original-c010", "claim_type": "code", "covers": ["u010"], "evidence": [{"locator": "Attempts and Takeaways", "observation": "The author's wider-load and lower-register-cap attempts failed or regressed, and the retrospective recommends profiling the actual bottleneck rather than this fixed priority list.", "revision": "retrieved 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "One should always profile before optimizing because a wrong assumption wastes effort.", "claim_id": "pattern-memory-bound-original-c011", "claim_type": "semantics", "covers": ["u011"], "evidence": [{"locator": "roofline, guided analysis, and profiling overhead", "observation": "Profiling supplies concrete bottleneck evidence, but collection has replay/overhead costs and the guide does not make profiling a prerequisite for every optimization; correctness, static legality, and already-established representative evidence are counterexamples to the absolute sequence.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "B200 has 8 TB/s of bandwidth, and a speed-of-light calculation determines achievable performance.", "claim_id": "pattern-memory-bound-original-c012", "claim_type": "performance", "covers": ["u012"], "evidence": [{"locator": "B200 specifications", "observation": "A platform specification can give a nominal memory-bandwidth value, but roofline supplies an upper-bound model; achieved performance also depends on access efficiency, clocks, operation mix, and measured ceilings.", "revision": "retrieved 2026-08-09", "url": "https://www.nvidia.com/en-us/data-center/blackwell/"}, {"locator": "roofline limitations", "observation": "Roofline classifies observed work/traffic against ceilings; it does not determine one exact achievable result.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Instruction-level parallelism and compute optimizations necessarily have diminishing returns for memory-bound kernels.", "claim_id": "pattern-memory-bound-original-c013", "claim_type": "performance", "covers": ["u013"], "evidence": [{"locator": "scheduler and memory analysis", "observation": "A memory-roof classification does not show whether instruction issue, address/decode work, latency hiding, or transaction inefficiency keeps achieved bandwidth below the attainable ceiling.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}], "oracle": "web-primary", "verdict": "refuted"}], "commands": ["Inspect Nsight Compute roofline/memory semantics", "Inspect PTX load hints, multicast, and swizzle contracts", "Inspect cited NVFP4 GEMV negative controls"], "id": "pattern-memory-bound", "page_verdict": "refuted", "path": "wiki/patterns/memory-bound.md", "residual_risk": "No target memory counters were collected locally. Hardware behavior is source-scoped and cited performance is author-reported.", "scope": "All 13 baseline units for hash 55576e61fe10aa796637b7056b452f0d94f5b0e727643e78a422e2b721eb30ff were audited, including the code-comment prescription and numerical roof claim."} +{"claims": [{"claim": "MoE grouped GEMM imbalance is necessarily per-expert compute-time imbalance in which each SM owns one expert and latency is set by the slowest expert.", "claim_id": "pattern-moe-load-imbalance-original-c001", "claim_type": "mechanism", "covers": ["u001"], "evidence": [{"locator": "grouped GEMM interfaces", "observation": "Grouped work is represented as multiple variable-size GEMMs; the interface does not assign one expert to one physical SM or define latency as the maximum expert time.", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"}, {"locator": "thread-block scheduling", "observation": "CUDA schedules thread blocks to available SMs and exposes no fixed expert-to-SM ownership.", "revision": "retrieved 2026-08-09", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Trained MoE routers commonly send exactly 80 percent of tokens to 20 percent of experts.", "claim_id": "pattern-moe-load-imbalance-original-c002", "claim_type": "performance", "covers": ["u002"], "evidence": [{"locator": "README policies and example", "observation": "The official project consumes measured expert loads and supplies no universal 80/20 distribution or frequency claim.", "revision": "EPLB d52c72d5b2f2fb4c41afbf8eb21366820239913d", "url": "https://github.com/deepseek-ai/EPLB/blob/d52c72d5b2f2fb4c41afbf8eb21366820239913d/README.md"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Static tile assignment means a precomputed tile-to-physical-SM mapping that cannot rebalance at runtime.", "claim_id": "pattern-moe-load-imbalance-original-c003", "claim_type": "semantics", "covers": ["u003"], "evidence": [{"locator": "thread-block scheduling", "observation": "CUDA dynamically places blocks on available SMs; software can precompute logical work ownership but not a portable tile-to-physical-SM map.", "revision": "retrieved 2026-08-09", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "A fixed M-max masked grouped layout necessarily executes computation on every padding row.", "claim_id": "pattern-moe-load-imbalance-original-c004", "claim_type": "mechanism", "covers": ["u004"], "evidence": [{"locator": "Masked grouped GEMM", "observation": "The documented masked interface takes valid M counts and says it computes only valid portions while retaining fixed allocation for CUDA graphs.", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Every M smaller than the output tile has full tensor-core utilization because the instruction shape is unchanged.", "check": "Inspect NVIDIA's small-dimension GEMM discussion and the grouped valid-M representation.", "outcome": "Partial tiles can leave threads predicated out and small problems can expose too little parallel work, although the effect remains kernel- and shape-specific."}, "claim": "Small per-expert M or partial output tiles can reduce useful parallel work and utilization, but this is a configuration-specific risk rather than an M less than BLOCK_M rule.", "claim_id": "pattern-moe-load-imbalance-original-c005", "claim_type": "performance", "covers": ["u005"], "evidence": [{"locator": "small GEMM dimensions and tile quantization", "observation": "CUTLASS explains that small dimensions can leave threads out of bounds and insufficient threadblocks, without defining a universal threshold.", "revision": "CUTLASS 4.5.0", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/efficient_gemm.md"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "CLC is hardware dynamic tile assignment in which the fastest SMs autonomously grab more tiles.", "claim_id": "pattern-moe-load-imbalance-original-c006", "claim_type": "semantics", "covers": ["u006"], "evidence": [{"locator": "request and response semantics", "observation": "A selected thread asynchronously requests cancellation of an unspecified unlaunched block or cluster and handles success/failure; the ISA does not define autonomous fastest-SM work stealing.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Persistent kernels inherently amortize launch overhead by looping over a dynamic work queue.", "claim_id": "pattern-moe-load-imbalance-original-c007", "claim_type": "performance", "covers": ["u007"], "evidence": [{"locator": "static and CLC-backed persistent schedulers", "observation": "CUTLASS documents both a zero-overhead static next-work rule and a CLC-backed dynamic rule; persistence does not imply a dynamic queue, and launch benefit is workload-dependent.", "revision": "CUTLASS 4.5.0", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "DeepGEMM contiguous grouped layout always uses an offsets array to delimit packed variable-M experts.", "claim_id": "pattern-moe-load-imbalance-original-c008", "claim_type": "semantics", "covers": ["u008"], "evidence": [{"locator": "Contiguous grouped GEMM", "observation": "The packed M layout accepts either a per-row expert index or a per-group prefix-sum end representation and requires aligned segments; it is not a single offsets-array ABI.", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "DeepGEMM masked grouped layout supports a CUDA-graph decode case but necessarily wastes compute on padding.", "claim_id": "pattern-moe-load-imbalance-original-c009", "claim_type": "mechanism", "covers": ["u009"], "evidence": [{"locator": "Masked grouped GEMM", "observation": "The README documents CUDA-graph-friendly decode and valid-M counts, explicitly limiting computation to valid portions rather than all allocated rows.", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "K-grouped is another forward/prefill representation or also varies M and N.", "check": "Inspect the pinned README and API shape assertions.", "outcome": "They assign K-grouped to MoE weight backward, pack the K axis, and retain fixed M/N with grouped outputs."}, "claim": "DeepGEMM K-grouped layout is documented for MoE weight-gradient computation with variable K and fixed M and N.", "claim_id": "pattern-moe-load-imbalance-original-c010", "claim_type": "semantics", "covers": ["u010"], "evidence": [{"locator": "K-grouped GEMM", "observation": "The README gives the weight-backward use and fixed-axis contract.", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"}, {"locator": "k_grouped_fp8_gemm functions", "observation": "API checks establish packed variable K operands and fixed M/N grouped outputs.", "revision": "same commit", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "EPLB replication across GPUs generically provides 1.49x prefill and 2.54x decode speedups.", "claim_id": "pattern-moe-load-imbalance-original-c011", "claim_type": "performance", "covers": ["u011"], "evidence": [{"locator": "Ablation Study: EPLB, Overall Results and limitations", "observation": "The figures are from a large-scale 96-H100 SGLang deployment using in-distribution data; the authors explicitly limit generalization to production distribution shifts.", "revision": "LMSYS post published 2025-05-05, retrieved 2026-08-09", "url": "https://www.lmsys.org/blog/2025-05-05-large-scale-ep/"}, {"locator": "replicate_experts and rebalance_experts", "observation": "The library replicates loaded logical experts and packs physical experts across configured GPUs/nodes but supplies no universal speedup contract.", "revision": "EPLB d52c72d5b2f2fb4c41afbf8eb21366820239913d", "url": "https://github.com/deepseek-ai/EPLB/blob/d52c72d5b2f2fb4c41afbf8eb21366820239913d/eplb.py"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The submission only skipped work and never produced correct outputs.", "check": "Inspect the official postmortem's correctness and timing paths separately.", "outcome": "It ran a real padded eight-group kernel on cloned objects for correctness, then merged and cached work only under timing."}, "claim": "GPU Mode's official postmortem says the temporarily first-place submission combined a real correct kernel path with a timing-harness exploit.", "claim_id": "pattern-moe-load-imbalance-original-c012", "claim_type": "mechanism", "covers": ["u012", "u013"], "evidence": [{"locator": "correctness and timing analysis", "observation": "The postmortem distinguishes real correctness computation from the timing-specific exploit.", "revision": "retrieved 2026-08-09", "url": "https://www.gpumode.com/news/reward-hacking-nvfp4"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The exploit merely reused one result without performing all timed inputs.", "check": "Inspect object counts, merged group count, return behavior, and timing division in the official postmortem.", "outcome": "Fifteen cloned eight-group inputs became one 120-group call, calls 2-15 returned cached pointers, and the harness divided by fifteen."}, "claim": "During timing the exploit merged fifteen eight-group inputs into one 120-group call, returned cached pointers for calls 2 through 15, and benefited from division by fifteen.", "claim_id": "pattern-moe-load-imbalance-original-c013", "claim_type": "mechanism", "covers": ["u014"], "evidence": [{"locator": "timing exploit sequence", "observation": "The official account gives the same counts and amortization behavior.", "revision": "retrieved 2026-08-09", "url": "https://www.gpumode.com/news/reward-hacking-nvfp4"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The GPU Mode reward-hack incident prompted the MLSys 2026 FlashInfer contest to add runtime isolation and subprocess evaluation.", "claim_id": "pattern-moe-load-imbalance-original-c014", "claim_type": "mechanism", "covers": ["u015"], "evidence": [{"locator": "evaluation response", "observation": "The postmortem points to gpu-mode/reference-kernels PR 104 and does not claim a FlashInfer/MLSys response.", "revision": "retrieved 2026-08-09", "url": "https://www.gpumode.com/news/reward-hacking-nvfp4"}, {"locator": "evaluation contract", "observation": "The contest contract documents its evaluator but attributes no design change to the GPU Mode incident.", "revision": "starter kit 75ccd05cafceb0fd1f86be4cd0f2117249463c66", "url": "https://github.com/flashinfer-ai/flashinfer-bench-starter-kit/blob/75ccd05cafceb0fd1f86be4cd0f2117249463c66/EVALUATION.md"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "CLC is available only on SM100 data-center GPUs and not on SM120 consumer Blackwell.", "claim_id": "pattern-moe-load-imbalance-original-c015", "claim_type": "compatibility", "covers": ["u016"], "evidence": [{"locator": "Target ISA Notes", "observation": "The instruction requires sm_100 or higher and the multicast form explicitly lists sm_120a and the SM120 family.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Dynamic scheduling adds a small overhead to every tile relative to a static precomputed schedule.", "claim_id": "pattern-moe-load-imbalance-original-c016", "claim_type": "performance", "covers": ["u017"], "evidence": [{"locator": "CLC rules and pipeline", "observation": "The initial blockIdx is fully static and incurs no CLC request; later asynchronous requests are pipelined and cluster-granular. The source gives no universal small latency bound.", "revision": "CUTLASS 4.5.0", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Small experts have a universal minimum viable tile-size floor below which a scheduling technique cannot benefit.", "claim_id": "pattern-moe-load-imbalance-original-c017", "claim_type": "performance", "covers": ["u018"], "evidence": [{"locator": "small GEMM dimensions", "observation": "The document describes gradual, shape- and tiling-dependent underutilization from predication and insufficient blocks, not a universal viability threshold.", "revision": "CUTLASS 4.5.0", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/efficient_gemm.md"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "EPLB works only at cluster scale and is categorically not a single-device mechanism.", "claim_id": "pattern-moe-load-imbalance-original-c018", "claim_type": "compatibility", "covers": ["u019"], "evidence": [{"locator": "rebalance_experts parameters and policies", "observation": "The planner accepts explicit node and GPU counts and operates on load tensors; its intended use is expert-parallel placement across GPUs, which can be intra-node and is not synonymous with cluster scale. It is distinct from a device-kernel tile scheduler.", "revision": "EPLB d52c72d5b2f2fb4c41afbf8eb21366820239913d", "url": "https://github.com/deepseek-ai/EPLB/blob/d52c72d5b2f2fb4c41afbf8eb21366820239913d/eplb.py"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Uniform routing, a very large batch, or an auxiliary load-balancing loss by itself proves that runtime MoE load imbalance is not an issue.", "claim_id": "pattern-moe-load-imbalance-original-c019", "claim_type": "performance", "covers": ["u020", "u021", "u022"], "evidence": [{"locator": "EPLB case studies and limitations", "observation": "The deployment diagnoses imbalance from measured per-expert/rank loads and throughput, notes phase/distribution differences, and treats benefit as batch- and distribution-dependent; none of the three proxies is a sufficient runtime negative test.", "revision": "published 2025-05-05, retrieved 2026-08-09", "url": "https://www.lmsys.org/blog/2025-05-05-large-scale-ep/"}], "oracle": "web-primary", "verdict": "refuted"}], "commands": ["Inspect CUDA/PTX block scheduling and CLC semantics", "Inspect pinned DeepGEMM grouped-layout contracts", "Inspect pinned EPLB placement code and scoped 96-H100 ablation", "Inspect official reward-hack and FlashInfer evaluation records"], "id": "pattern-moe-load-imbalance", "page_verdict": "refuted", "path": "wiki/patterns/moe-load-imbalance.md", "residual_risk": "No GPU timeline or expert trace was available; utilization and scheduler-benefit claims are retained only as measurement-dependent risks.", "scope": "All 22 baseline units for hash dd70d3638677328e71242ec9fa14891b9ed9465328f129378f5b835f52741275 were audited across routing, layouts, schedulers, distributed placement, performance, compatibility, and reward-hack history."} +{"claims": [{"claim": "Low TMA/tcgen05 activity, phase-local tensor-core drops, and warps observed at mbarrier.try_wait establish pipeline stalls in an otherwise compute-bound kernel.", "claim_id": "pattern-pipeline-stalls-original-c001", "claim_type": "semantics", "covers": ["u001"], "evidence": [{"locator": "Warp State Statistics and sampling", "observation": "Warp states are sampled symptoms, stalls are not necessarily performance-impacting, and NVIDIA says to focus on them when schedulers fail to issue; source correlation does not by itself prove the causal pipeline or a compute-bound classification.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Two stages cannot hide a three-cycle latency chain.", "claim_id": "pattern-pipeline-stalls-original-c002", "claim_type": "performance", "covers": ["u002"], "evidence": [{"locator": "matmul_v3 launch and pipeline", "observation": "The cited tutorial's pipelined v3 instantiates NUM_STAGES=2 and reports a benefit; neither it nor the tuning guide defines a generic three-cycle chain or a universal depth rule.", "revision": "3b90ac9b3f624bdf1f6f78d02dcd533675d36573", "url": "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "A stale phase can be harmless because parity repeats every other reuse.", "check": "Inspect the PTX validity window for state/parity waits and the per-iteration example.", "outcome": "A wait may identify only the current or immediately preceding phase, so incorrect per-object phase tracking can wait for the wrong reuse or invoke an invalid state."}, "claim": "Incorrect mbarrier phase tracking can make a consumer wait for or observe the wrong reuse of a barrier.", "claim_id": "pattern-pipeline-stalls-original-c003", "claim_type": "semantics", "covers": ["u003"], "evidence": [{"locator": "state and phaseParity validity", "observation": "test_wait/try_wait accepts state or parity for the current or immediately preceding phase, requiring phase state to be maintained for each reused object.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "Missing tcgen05.fence::after_thread_sync between a TMA wait and MMA issue lets MMA read SMEM before the TMA transfer is visible.", "claim_id": "pattern-pipeline-stalls-original-c004", "claim_type": "semantics", "covers": ["u004"], "evidence": [{"locator": "acquire wait ordering", "observation": "A successful acquire wait provides visibility for prior cp.async.bulk operations associated with the barrier.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait"}, {"locator": "tcgen05 fence description", "observation": "after_thread_sync orders subsequent asynchronous tcgen05 operations after prior tcgen05/execution-ordering operations across threads; it is not the general TMA-to-SMEM visibility primitive asserted by the page.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Using one tile serializes every warp on that tile's softmax or epilogue.", "claim_id": "pattern-pipeline-stalls-original-c005", "claim_type": "performance", "covers": ["u005"], "evidence": [{"locator": "sections 3.1.1-3.1.2", "observation": "FA4 motivates two-tile overlap for its specific attention schedule, but does not establish that every single-tile kernel serializes all warps or that this is a general stall cause.", "revision": "FlashAttention-4 paper v1", "url": "https://arxiv.org/html/2603.05451v1"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "After a TMA operation, hardware performs an mbarrier arrival in addition to the producer's manual arrival, so the producer over-arrives and releases the next stage stale.", "claim_id": "pattern-pipeline-stalls-original-c006", "claim_type": "semantics", "covers": ["u006"], "evidence": [{"locator": "arrive.expect_tx and complete_tx semantics", "observation": "arrive.expect_tx performs the software arrival and increases tx-count; a TMA .complete_tx operation decrements transaction bytes rather than performing a second arrival. An unmatched extra arrive can still break a phase, but the stated mechanism is false.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Three to five pipeline stages are typical on Blackwell, and warp specialization eliminates role-switching stalls.", "claim_id": "pattern-pipeline-stalls-original-c007", "claim_type": "performance", "covers": ["u007", "u008"], "evidence": [{"locator": "NUM_STAGES instantiation", "observation": "The cited v3 uses two stages, which is a direct counterexample to the proposed typical range.", "revision": "3b90ac9b3f624bdf1f6f78d02dcd533675d36573", "url": "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu"}, {"locator": "warp specialization", "observation": "The tutorial reports a source-specific speedup and distinct role loops, not elimination of all role-switching or pipeline stalls.", "revision": "retrieved 2026-08-09", "url": "https://gau-nernst.github.io/tcgen05/"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The candidate descriptions imply guaranteed speedups.", "check": "Compare them with the exact TMEM ownership and FA4 scheduling contracts.", "outcome": "The rows only identify the structural overlap each technique can test; the underlying sources require disjoint state and synchronization and do not guarantee speedup."}, "claim": "Double-buffering can use separate TMEM output regions while MMA and output consumption overlap, and FA4's ping-pong schedule alternates two query/output tiles between MMA and softmax work.", "claim_id": "pattern-pipeline-stalls-original-c008", "claim_type": "semantics", "covers": ["u009", "u010"], "evidence": [{"locator": "TMEM output stages and epilogue handoff", "observation": "The pinned tutorial source uses distinct output buffers so epilogue and MMA work can overlap.", "revision": "3b90ac9b3f624bdf1f6f78d02dcd533675d36573", "url": "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu"}, {"locator": "section 3.1.2", "observation": "FA4 assigns two output tiles per block and overlaps one tile's MMA with the other's softmax.", "revision": "FlashAttention-4 paper v1", "url": "https://arxiv.org/html/2603.05451v1"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The six-step diagnosis checklist gives generally correct profiler metrics, phase progression, TMA accounting, tcgen05 fencing, and stage-count advice.", "claim_id": "pattern-pipeline-stalls-original-c009", "claim_type": "code", "covers": ["u011"], "evidence": [{"locator": "metric collection and Warp State Statistics", "observation": "Available metrics and sampled stall states are tool/chip dependent and must be interpreted with scheduler issue evidence.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}, {"locator": "after_thread_sync semantics", "observation": "The checklist's mandatory after_thread_sync placement for TMA-to-MMA is a semantic conflation, and parity changes per barrier phase/reuse rather than after every wait globally.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The tcgen05 tutorial progresses from a one-stage 62%-of-cuBLAS kernel to three-stage 70%, warp-specialized 80%, 2-SM 86%, and persistent-plus-CLC 98%, with the stated isolated causes.", "claim_id": "pattern-pipeline-stalls-original-c010", "claim_type": "performance", "covers": ["u012", "u013", "u014", "u015", "u016"], "evidence": [{"locator": "final progression and version sections", "observation": "The exact results are v1a 254.62, v2b 695.43, v3 pipelining 939.61, v4 warp specialization 1208.83, v5 2-SM 1302.29, and v6 static persistence 1475.93 TFLOP/s versus 1506.74 cuBLAS. The pinned v3 source uses two stages, and the author explicitly did not add CLC.", "revision": "retrieved 2026-08-09", "url": "https://gau-nernst.github.io/tcgen05/"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Too many stages exceed a 228-KiB shared-memory budget.", "claim_id": "pattern-pipeline-stalls-original-c011", "claim_type": "compatibility", "covers": ["u017"], "evidence": [{"locator": "occupancy and shared memory", "observation": "Compute capability 10.0 has a 228-KiB per-SM limit, but whether a stage count exceeds it depends on bytes per stage, fixed storage, per-block limits, and configuration; the unqualified statement has no tile payload.", "revision": "CUDA 13.0.2 Blackwell Tuning Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Output comparison alone is enough, so assertions on phase invariants add no debugging value.", "check": "Inspect the phase validity window and the failure modes addressed by NVIDIA's synchronization diagnostics.", "outcome": "A stale or wrong per-object phase can wait on the wrong reuse or deadlock; checking the application's expected phase/ownership state during development localizes that protocol error before a final-value mismatch or hang."}, "claim": "Phase-tracking bugs are notoriously hard to debug, so development assertions should be added.", "claim_id": "pattern-pipeline-stalls-original-c012", "claim_type": "semantics", "covers": ["u018"], "evidence": [{"locator": "state and phaseParity validity", "observation": "A phase-aware wait is valid only for the current or immediately preceding phase, so the program must maintain the correct per-object phase across reuse.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait"}, {"locator": "Racecheck and Synccheck synchronization tracing and deadlock detection", "observation": "NVIDIA provides specialized development diagnostics for asynchronous-copy synchronization, barrier misuse, synchronization tracing, and deadlocks, confirming that these state protocols need explicit development-time checks beyond output timing.", "revision": "Compute Sanitizer documentation, retrieved 2026-08-09", "url": "https://docs.nvidia.com/compute-sanitizer/ComputeSanitizer/index.html"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "Pipelining is a waste of effort on memory-bound kernels.", "claim_id": "pattern-pipeline-stalls-original-c013", "claim_type": "performance", "covers": ["u019"], "evidence": [{"locator": "roofline analysis", "observation": "A memory-bound classification identifies a limiting ceiling; it does not imply that latency-hiding, issue overlap, or a staged transfer change cannot improve the measured kernel.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline"}], "oracle": "web-primary", "verdict": "refuted"}], "commands": ["Inspect PTX ISA 9.0 mbarrier and tcgen05 ordering contracts", "Inspect Nsight Compute 2025.3 profiling semantics", "Inspect tcgen05 tutorial article and pinned v3/v6 source"], "id": "pattern-pipeline-stalls", "page_verdict": "refuted", "path": "wiki/patterns/pipeline-stalls.md", "residual_risk": "No Blackwell GPU or Nsight Compute installation is available locally, so no target-side profiler experiment was performed. The tutorial measurements remain author-reported.", "scope": "All 19 baseline body units for hash ec21f3208711a7056069ab687fce39dd6bc96f6ee8fd093b42542af45799b592 were reviewed, including diagnostic semantics, synchronization claims, candidates, the checklist, and every progression number."} +{"claims": [{"claim": "Below-target occupancy plus local-memory spills establishes that high registers per thread are the limiting cause.", "claim_id": "pattern-register-pressure-original-c001", "claim_type": "performance", "covers": ["u001"], "evidence": [{"locator": "Launch Statistics and Occupancy", "observation": "The profiler reports registers, spills, theoretical/achieved occupancy, and limiting resources separately; spilling does not prove registers are the occupancy limiter or that a target occupancy is useful.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Accumulator, epilogue, or control state cannot contribute to register pressure because the compiler always shortens their lifetimes.", "check": "Inspect WGMMA fragment ownership and compiled-source liveness/resource reporting.", "outcome": "WGMMA D is a per-thread register fragment and extra live epilogue/control values can overlap its lifetime; exact allocation remains compiler/configuration dependent."}, "claim": "Hopper WGMMA accumulators and simultaneously live epilogue or control-flow state can contribute to per-thread register use.", "claim_id": "pattern-register-pressure-original-c002", "claim_type": "semantics", "covers": ["u002", "u003", "u004"], "evidence": [{"locator": "WGMMA D operands", "observation": "WGMMA accumulator fragments are distributed across warpgroup threads in registers.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma"}, {"locator": "Launch Statistics and source analysis", "observation": "Compiled registers and spills reflect live program state and can be correlated with source/generated code.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "SM100 TMEM is a dedicated 256-KB memory that simply moves accumulators out of registers.", "claim_id": "pattern-register-pressure-original-c003", "claim_type": "compatibility", "covers": ["u005"], "evidence": [{"locator": "organization and access", "observation": "TMEM is organized as 128 lanes by 512 32-bit columns and tcgen05 D resides there, but access/lifecycle and logical capacity depend on instruction kind/packing; 'SM100 only' and a simple byte-memory migration omit target and access constraints.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Warp specialization applies only to SM100+ and reduces each warp's register requirement.", "claim_id": "pattern-register-pressure-original-c004", "claim_type": "compatibility", "covers": ["u006"], "evidence": [{"locator": "Hopper warp-specialized kernels", "observation": "CUTLASS documents warp-specialized GEMM on Hopper, refuting SM100+ applicability; register effects depend on role code, compiler allocation, and optional register reallocation.", "revision": "CUTLASS documentation, retrieved 2026-08-09", "url": "https://docs.nvidia.com/cutlass/media/docs/cpp/hopper_gemm.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Register-to-TMEM migration means replacing every accumulator register reference mechanically.", "check": "Compare the migration page and PTX completion/load lifecycle.", "outcome": "The cited approach explicitly replaces resident D storage while adding allocation, completion, collective load, wait, and epilogue handoffs."}, "claim": "Register-to-TMEM migration is a systematic SM90-WGMMA to SM100-tcgen05 change for resident D accumulators, with new lifecycle and synchronization requirements.", "claim_id": "pattern-register-pressure-original-c005", "claim_type": "semantics", "covers": ["u007"], "evidence": [{"locator": "tcgen05 MMA, allocation, completion, and load", "observation": "tcgen05 uses TMEM D and specifies allocation, asynchronous completion, collective load, wait, and deallocation mechanisms.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "A Hopper m64n256 FP32 WGMMA accumulator costs about 128 registers/thread, while Blackwell frees exactly all 128, uses zero accumulator registers, has 256 KB TMEM, and the largest 1-SM MMA always occupies half so double buffering is available.", "claim_id": "pattern-register-pressure-original-c006", "claim_type": "code", "covers": ["u008"], "evidence": [{"locator": "m64nNk16 D fragment", "observation": "At N=256 the FP32 D fragment is N/2=128 registers per thread, validating only the Hopper arithmetic.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma"}, {"locator": "organization and tcgen05 access", "observation": "Resident D is in 128x512 TMEM cells, but epilogue tcgen05.ld destinations and other state still use registers; capacity/shape/packing constraints do not imply a universal half-capacity largest MMA or exactly 128 freed registers.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Consumer Blackwell SM120 also exposes TMEM/tcgen05.", "check": "Inspect PTX target notes for tensor-memory instructions.", "outcome": "The tcgen05 TMEM instruction family is restricted to architecture-specific data-center targets such as sm_100a and is not a generic sm_120 path."}, "claim": "The TMEM/tcgen05 path described is available on SM100-class data-center targets rather than SM120 consumer Blackwell.", "claim_id": "pattern-register-pressure-original-c007", "claim_type": "compatibility", "covers": ["u009"], "evidence": [{"locator": "Target ISA Notes", "observation": "Tensor-memory instructions list architecture-specific data-center targets and do not provide the described SM120 consumer path.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "TMEM is automatically managed at kernel entry/exit.", "check": "Inspect alloc/dealloc semantics and exit rule.", "outcome": "PTX requires collective allocation and deallocation, and exiting without deallocation is invalid."}, "claim": "TMEM requires an explicit allocation and deallocation lifecycle.", "claim_id": "pattern-register-pressure-original-c008", "claim_type": "semantics", "covers": ["u010"], "evidence": [{"locator": "allocation/deallocation lifecycle", "observation": "PTX defines collective alloc/dealloc and requires deallocation before CTA exit.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "TMEM-to-register transfer latency is offset by the registers freed from accumulators.", "claim_id": "pattern-register-pressure-original-c009", "claim_type": "performance", "covers": ["u011"], "evidence": [{"locator": "load and wait semantics", "observation": "tcgen05.ld is asynchronous and requires its load wait before use, but the ISA gives no fixed latency or guarantee that freed resident state offsets its cost.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"}], "oracle": "web-primary", "verdict": "refuted"}], "commands": ["Inspect Nsight Compute register/occupancy semantics", "Inspect WGMMA fragment and TMEM lifecycle contracts", "Inspect Hopper warp-specialization documentation"], "id": "pattern-register-pressure", "page_verdict": "refuted", "path": "wiki/patterns/register-pressure.md", "residual_risk": "No target compilation or occupancy experiment is possible without a CUDA toolkit/GPU. Register effects beyond ISA fragment arithmetic remain compiler-specific.", "scope": "All 11 baseline units for hash 6fd15f8aae2d1ca78d038b8af9ec545760eac38cdbb46e46985f19c35a84a491 were reviewed, including the arithmetic code block and target claims."} +{"claims": [{"claim": "Performance necessarily drops whenever total_tiles modulo num_SMs is nonzero because the last wave idles many SMs.", "claim_id": "pattern-tail-effect-original-c001", "claim_type": "performance", "covers": ["u001"], "evidence": [{"locator": "occupancy and workload distribution", "observation": "Physical SM count is not automatically the one-wave worker count; residency, concurrent work, tile durations, scheduling, and the remainder magnitude determine observed impact.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "When N is divisible by M, N%M=0 means the final wave has zero active workers.", "check": "Apply Euclidean division N=qM+r and account for the r=0 case.", "outcome": "For r>0 the partial wave has r workers; for r=0 and N>0 the final wave is full, not empty."}, "claim": "In an equal-duration, one-tile-per-worker model, N tiles on M workers require ceil(N/M) waves and a nonzero remainder r=N%M makes the final partial wave use r workers.", "claim_id": "pattern-tail-effect-original-c002", "claim_type": "semantics", "covers": ["u002"], "evidence": [{"locator": "problem grid and persistent scheduling model", "observation": "The documented model treats launched grid IDs as work acquired by workers, consistent with the page's simplified wave arithmetic when its assumptions hold.", "revision": "CUTLASS 4.5.0", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "A static grid-stride schedule leaves remainder tiles on a few fixed SMs, while a non-persistent fixed grid has no dynamic block scheduling.", "claim_id": "pattern-tail-effect-original-c003", "claim_type": "semantics", "covers": ["u003", "u004"], "evidence": [{"locator": "thread-block scheduling", "observation": "CUDA schedules blocks onto available SMs; blockIdx-to-SM assignment is not fixed by a grid stride. Persistence changes repeated work acquisition, not whether the ordinary grid is dynamically scheduled by hardware.", "revision": "retrieved 2026-08-09", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "CLC is hardware dynamic scheduling in which idle SMs grab tiles on demand.", "claim_id": "pattern-tail-effect-original-c004", "claim_type": "semantics", "covers": ["u005"], "evidence": [{"locator": "request semantics", "observation": "A designated thread asynchronously requests cancellation of an unspecified not-yet-started block/cluster and processes a successful response; SMs do not autonomously grab arbitrary tiles.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Persistent kernels use an SM-count grid, iterate over tiles, and have no wave boundary.", "claim_id": "pattern-tail-effect-original-c005", "claim_type": "performance", "covers": ["u006"], "evidence": [{"locator": "fixed number of thread blocks", "observation": "Persistent launch size is a software choice and repeated work still terminates with a final subset; persistence does not guarantee one CTA per physical SM or removal of all tail behavior.", "revision": "retrieved 2026-08-09", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Raster order and swizzle patterns improve worker distribution.", "claim_id": "pattern-tail-effect-original-c006", "claim_type": "performance", "covers": ["u007"], "evidence": [{"locator": "rasterization and swizzle", "observation": "These are software coordinate transforms applied to work IDs; they can affect locality but do not create IDs or guarantee a more even worker-duration distribution.", "revision": "CUTLASS 4.5.0", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "A B200 has 142 SMs; for 150 tiles CLC turns a 142+8 two-wave launch into one persistent wave with all SMs busy and causes the tutorial's 86%-to-98% result.", "claim_id": "pattern-tail-effect-original-c007", "claim_type": "code", "covers": ["u008"], "evidence": [{"locator": "persistent section and results", "observation": "The author's Modal B200 has 148 SMs, v6 uses static persistence without CLC, overlap remains imperfect, and the 86.43%-to-97.96% cumulative step includes other pipeline changes.", "revision": "retrieved 2026-08-09", "url": "https://gau-nernst.github.io/tcgen05/"}, {"locator": "work semantics", "observation": "CLC cannot turn eight independent IDs into 142 or 148 independent pieces of work.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Tail effects are significant only for tile counts below four times the SM count.", "claim_id": "pattern-tail-effect-original-c008", "claim_type": "performance", "covers": ["u009"], "evidence": [{"locator": "workload distribution", "observation": "No four-wave cutoff is defined; impact depends on remainder, wave duration, residency, imbalance, and whole-kernel fraction.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The absolute last-wave cost grows with problem size, so it is not amortized.", "check": "For equal-duration work, divide at most one partial-wave duration by q full waves plus the final wave.", "outcome": "The fractional contribution is bounded by roughly one over the number of waves and shrinks as q grows, under the simplified assumptions."}, "claim": "For equal-duration independent tiles and fixed worker capacity, a single final partial wave is fractionally amortized as the number of full waves becomes large.", "claim_id": "pattern-tail-effect-original-c009", "claim_type": "performance", "covers": ["u010"], "evidence": [{"locator": "data-parallel tile scheduling", "observation": "The scheduler model supports the independent-work assumptions; the amortization follows directly from the wave-count arithmetic and is explicitly scoped.", "revision": "CUTLASS 4.5.0", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "CLC is available only on SM100 datacenter GPUs.", "claim_id": "pattern-tail-effect-original-c010", "claim_type": "compatibility", "covers": ["u011"], "evidence": [{"locator": "Target ISA Notes", "observation": "The instruction requires sm_100 or higher and its multicast qualifier explicitly supports sm_120a and the SM120 family.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"}], "oracle": "web-primary", "verdict": "refuted"}], "commands": ["Audit wave arithmetic and CUDA block scheduling", "Inspect PTX/CUDA CLC semantics and targets", "Inspect tutorial hardware/result scope"], "id": "pattern-tail-effect", "page_verdict": "refuted", "path": "wiki/patterns/tail-effect.md", "residual_risk": "No GPU trace was available; the equal-duration model is analytical and real duration/residency must be measured.", "scope": "All 11 baseline units for hash 796c1dd09a9e76eff0996bf41283998fce447e492dd300571596ec89ad147ed9 were audited, including arithmetic, scheduler, compatibility, and example claims."} +{"claims": [{"claim": "L1 eviction qualifiers control cache admission and are critical whenever a memory-bound kernel's L1 working set matters more than compute.", "claim_id": "technique-cache-policy-original-c001", "claim_type": "semantics", "covers": ["u001"], "evidence": [{"locator": "9.7.9.8 ld, eviction priorities", "observation": "The ISA says eviction policies may be applied during access; it does not define these qualifiers as mandatory admission controls or prescribe them by bottleneck category.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The PTX pattern makes A bypass L1, keeps B resident, and evicts C immediately, thereby preventing cache pollution.", "claim_id": "technique-cache-policy-original-c002", "claim_type": "code", "covers": ["u002"], "evidence": [{"locator": "9.7.9.8 ld and 9.7.9.12 st", "observation": "The displayed vector load/store forms are legal on SM100 with natural alignment, but L1 qualifiers are eviction priorities that may apply. Their names do not guarantee bypass, residence, immediate eviction, or performance.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-st"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The rank-1 NVFP4 GEMV submission used an aggressive A no_allocate policy for K=16384 and a relaxed policy for K=2048.", "claim_id": "technique-cache-policy-original-c003", "claim_type": "performance", "covers": ["u003", "u004", "u005"], "evidence": [{"locator": "Post-event Observations", "observation": "The cited retrospective reports A no_allocate, B evict_last, exact-K specializations, and caps among inspected solutions, but publishes no contestant code and does not assign the page's per-K qualifier policies to rank 1.", "revision": "retrieved 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}, {"locator": "Public leaderboard and task directory", "observation": "The public contest materials expose scores and a task contract, not contestant source or technique metadata.", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6", "url": "https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Cache policy plus PTX byte unpacking partly caused a 16x 443-to-27 microsecond gain and cache policy can dominate memory-bound kernels.", "claim_id": "technique-cache-policy-original-c004", "claim_type": "performance", "covers": ["u006", "u007"], "evidence": [{"locator": "CUDA optimization 1 through 3", "observation": "The 443-to-27 endpoints include coalescing, shared-memory changes, collaboration, reductions, per-thread tiles, vector width, hardware conversion, and PTX decode. No cache-policy-only run or dominant-lever result is reported.", "revision": "retrieved 2026-08-08", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Cache differentiation should be used for memory-bound kernels, clear streaming/reuse patterns, inputs larger than B200's 126 MB L2, and separate GEMM M/N loading patterns.", "claim_id": "technique-cache-policy-original-c005", "claim_type": "performance", "covers": ["u008", "u009", "u010", "u011"], "evidence": [{"locator": "9.7.9.8 ld", "observation": "The ISA defines legal hints but no input-size, tensor-role, GEMM-axis, or bottleneck rule that predicts benefit. Cache behavior depends on the exact address stream, reuse distance, concurrency, and compiled access path.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld"}, {"locator": "Attempts 8-12 and Takeaways", "observation": "The author's task-specific conclusion is to profile the actual bottleneck and run controlled variants; workload categories alone did not predict beneficial transformations.", "revision": "retrieved 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}], "oracle": "web-primary", "verdict": "refuted"}], "commands": ["Inspect archived PTX 9.0 ld/st eviction-priority grammar and semantics", "Compare the cited author reports with the page's contestant and causal attributions", "Check the pinned public contest boundary for source and technique visibility"], "id": "technique-cache-policy", "page_verdict": "refuted", "path": "wiki/techniques/cache-policy.md", "residual_risk": "No B200 or released rank-1 source was available; unsupported contestant attribution was therefore removed rather than guessed.", "scope": "All 11 baseline body units and the frontmatter performance claim were checked against archived PTX, the cited reports, and pinned public contest materials."} +{"claims": [{"claim": "The two linked CCCL PRs are broadly reusable SM100/SM90 policy and dispatch sources for scan, top-k, fill, histogram, reduce and block load/store, including DSA TopK tuning.", "claim_id": "technique-cccl-memory-primitives-original-c001", "claim_type": "compatibility", "covers": ["u001"], "evidence": [{"locator": "changed files and sm100_tuning additions", "observation": "PR 3559 specifically adds B200/SM100 exclusive-sum scan policies and dispatch/type plumbing; it does not cover top-k, fill, histogram or reduce.", "path": "artifacts/prs/cccl/PR-3559/diff.patch", "revision": "CCCL PR 3559 captured merge 25523da2"}, {"locator": "complete 13-line debug-log fix", "observation": "PR 6152 only fixes stale CUB_DEBUG_LOG formatting/variable names in DispatchTopK; it is not an SM100 tuning or DSA integration PR.", "path": "artifacts/prs/cccl/PR-6152/diff.patch", "revision": "CCCL PR 6152 captured merge 3fb05826"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The displayed PrimitivePolicy is a meaningful vectorization probe because ITEMS_PER_THREAD >= 4 implies vectorized memory access.", "claim_id": "technique-cccl-memory-primitives-original-c002", "claim_type": "code", "covers": ["u002"], "evidence": [{"locator": "sm100_tuning specializations and ScanPolicyWrapper", "observation": "SM100 scan policies independently choose threads, items, delay constructor, load/store algorithms and cache modifier by input/accumulator/offset/operator classification. Item count alone neither specifies nor proves vectorized access.", "path": "artifacts/prs/cccl/PR-3559/key-files/cub/cub/device/dispatch/tuning/tuning_scan.cuh", "revision": "CCCL PR 3559 captured merge 25523da2"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "The PR's B200 title means every path automatically uses a new SM100 policy.", "check": "Follow type/operator classification through select_agent_policy100 and inspect explicit fallbacks.", "outcome": "Coverage is selective, so the page's architecture-specific check is necessary."}, "claim": "A transfer review should determine whether an upstream change has a distinct SM100 policy or merely inherits an earlier architecture's policy.", "claim_id": "technique-cccl-memory-primitives-original-c003", "claim_type": "compatibility", "covers": ["u003"], "evidence": [{"locator": "sm90_tuning, sm100_tuning and policy_hub Policy900/Policy1000", "observation": "The source has explicit SM90 and SM100 policy layers; some SM100 types use new specializations while double falls back to an SM90 tuning.", "path": "artifacts/prs/cccl/PR-3559/key-files/cub/cub/device/dispatch/tuning/tuning_scan.cuh", "revision": "CCCL PR 3559 captured merge 25523da2"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "TopK ties and relaxed atomic ordering are harmless because any equal-valued item is semantically interchangeable.", "check": "Compare set membership, output order, and application identity requirements under equal keys.", "outcome": "CCCL treats these as distinct determinism contracts; an application that consumes indices or stable order must validate the required contract before transferring a relaxed or atomic variant."}, "claim": "Atomic or relaxed variants require determinism and tie-breaking validation before transfer.", "claim_id": "technique-cccl-memory-primitives-original-c004", "claim_type": "semantics", "covers": ["u004"], "evidence": [{"locator": "determinism definitions and TopK example", "observation": "CCCL distinguishes selected-set determinism from output-order determinism for parallel algorithms, so transfer validation must state which result and tie behavior are required.", "revision": "CCCL documentation, retrieved 2026-08-09", "url": "https://nvidia.github.io/cccl/unstable/cccl/determinism.html"}, {"locator": "DeviceTopK execution environments", "observation": "DeviceTopK exposes algorithm-specific determinism and ordering controls rather than promising one result order for every execution environment.", "revision": "CCCL documentation, retrieved 2026-08-09", "url": "https://nvidia.github.io/cccl/unstable/cub/environment.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Only combined indexer latency matters because logits and TopK are fused into one inseparable operation.", "check": "Trace the pinned DSA call path from score generation to selected indices.", "outcome": "The implementation has separate logits and TopK dispatch boundaries, so isolating their costs is both possible and necessary to attribute a TopK-policy change."}, "claim": "For DSA TopK, radix/select cost should be isolated from score-computation cost.", "claim_id": "technique-cccl-memory-primitives-original-c005", "claim_type": "performance", "covers": ["u005"], "evidence": [{"locator": "sparse_attn_indexer logits dispatch and prefill/decode TopK selection", "observation": "The pinned DSA path first computes indexer logits through separate MQA-logits kernels, then dispatches distinct prefill/decode TopK selection, making score computation and selection separable measured stages.", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/tensorrt_llm/_torch/attention_backend/sparse/dsa.py", "revision": "TensorRT-LLM PR 13340 captured merge 897c4bff"}], "oracle": "local-primary-source", "verdict": "verified"}], "commands": ["inspect complete captured diffs and key files for CCCL PRs 3559 and 6152", "inspect official CCCL determinism and DeviceTopK execution-environment contracts", "trace pinned TensorRT-LLM DSA logits and TopK dispatch boundaries"], "id": "technique-cccl-memory-primitives", "page_verdict": "refuted", "path": "wiki/techniques/cccl-memory-primitives.md", "residual_risk": "No GPU benchmark was run. Determinism requirements are source-verified at the CCCL contract level, and DSA cost isolation is source-verified from a pinned implementation boundary rather than benchmarked on target hardware.", "scope": "All five baseline units and linked source scopes were checked against the complete captured PR diffs/key files."} +{"claims": [{"claim": "GatedDeltaNet, RetNet, and Mamba are all O(n) linear-attention variants whose naive implementations are sequential, and Blackwell TMEM permits larger chunks with better tensor-core utilization.", "claim_id": "technique-chunk-parallelism-original-c001", "claim_type": "semantics", "covers": ["u001"], "evidence": [{"locator": "Abstract", "observation": "TFLA concerns linear RNNs and mLSTM, says practical benefits require optimized kernels, and introduces within-chunk sequence parallelization. It does not define Mamba as linear attention or connect chunk size to Blackwell TMEM.", "revision": "arXiv:2503.14376v3", "url": "https://arxiv.org/abs/2503.14376v3"}, {"locator": "training implementation", "observation": "The pinned GatedDeltaNet training path uses chunkwise Triton kernels with a WY representation, not the page's generic additive state update; its architecture parameters do not imply a TMEM capacity rule.", "revision": "NVlabs/GatedDeltaNet b53d6d3a161267432a79c1c04af69fa52bddc921", "url": "https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The displayed Triton kernel correctly implements chunk-parallel linear attention with sequential inter-chunk state propagation.", "claim_id": "technique-chunk-parallelism-original-c002", "claim_type": "code", "covers": ["u002"], "evidence": [{"locator": "Triton programming model, program instances", "observation": "Separate Triton program instances in one launch have no grid-wide execution order or synchronization. Every chunk here reads and writes the same State address, so the claimed previous-chunk dependency is a data race rather than a sequential propagation.", "revision": "Triton language documentation, retrieved 2026-08-09", "url": "https://triton-lang.org/main/programming-guide/chapter-01/introduction.html"}, {"locator": "chunkwise training implementation", "observation": "The pinned GDN implementation uses a multi-stage WY/chunkwise algorithm; the snippet has undefined offsets, shapes, masking, update_state, head/batch addressing, and state ordering and is not an executable equivalent.", "revision": "NVlabs/GatedDeltaNet b53d6d3a161267432a79c1c04af69fa52bddc921", "url": "https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Chunks of 32 are a low-latency decode choice, chunks of 256-512 improve prefill tensor-core throughput, and TFLA uses recursive two-level tiling to allow arbitrary chunk sizes.", "claim_id": "technique-chunk-parallelism-original-c003", "claim_type": "performance", "covers": ["u003", "u004", "u005"], "evidence": [{"locator": "Abstract and algorithm scope", "observation": "TFLA introduces an additional level of sequence parallelization within each chunk and claims arbitrarily large chunks for its mLSTM algorithm. It does not prescribe 32 for decode, 256-512 for prefill, call the construction recursive, or establish Blackwell results.", "revision": "arXiv:2503.14376v3", "url": "https://arxiv.org/abs/2503.14376v3"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Chunk parallelism should be used for all linear attention and recurrent-state models including Mamba, GatedDeltaNet, and the Delta Rule.", "claim_id": "technique-chunk-parallelism-original-c004", "claim_type": "compatibility", "covers": ["u006", "u007"], "evidence": [{"locator": "Abstract and algorithm scope", "observation": "TFLA establishes its two-level sequence-parallel construction for a specific linear-RNN/mLSTM formulation, not a universal rule for every recurrent model.", "revision": "arXiv:2503.14376v3", "url": "https://arxiv.org/abs/2503.14376v3"}, {"locator": "chunkwise training implementation", "observation": "GatedDeltaNet uses a model-specific WY-form chunkwise algorithm and layouts rather than the page's universal generic update.", "revision": "b53d6d3a161267432a79c1c04af69fa52bddc921", "url": "https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921"}, {"locator": "selective_scan interface and implementation", "observation": "Mamba exposes a selective-scan recurrence and implementation, so transfer requires a model-specific equivalent transformation and profitability test.", "revision": "retrieved 2026-08-09", "url": "https://github.com/state-spaces/mamba"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The 3:1 ratio is an informal description rather than an exact model configuration.", "check": "Expand the pinned layer_types pattern over num_hidden_layers.", "outcome": "The 48-layer configuration repeats three linear-attention entries and one full-attention entry twelve times, yielding 36 and 12 layers."}, "claim": "Qwen3-Next uses a 3:1 hybrid ratio of Gated DeltaNet linear-attention layers to full-attention layers.", "claim_id": "technique-chunk-parallelism-original-c005", "claim_type": "compatibility", "covers": ["u008"], "evidence": [{"locator": "config.json layer_types and num_hidden_layers", "observation": "The immutable 48-layer configuration repeats three linear_attention entries followed by one full_attention entry.", "revision": "Qwen3-Next-80B-A3B-Instruct 9c7f2fbe84465e40164a94cc16cd30b6999b0cc7", "url": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/config.json"}], "oracle": "web-primary", "verdict": "verified"}], "commands": ["Inspect TFLA v3 algorithm and scope", "Inspect pinned GatedDeltaNet chunkwise implementation boundary", "Discriminate the Triton snippet's inter-program state race", "Expand the immutable Qwen3-Next layer pattern"], "id": "technique-chunk-parallelism", "page_verdict": "refuted", "path": "wiki/techniques/chunk-parallelism.md", "residual_risk": "No GPU execution was available. The universal prescription is refuted by the model-specific scope and distinct direct implementations; profitability for any concrete transformed model remains hardware- and workload-dependent.", "scope": "All eight baseline body units and the frontmatter TMEM/chunk-size claim were checked against the cited papers, pinned repository, Triton execution model, and immutable model configuration."} +{"claims": [{"challenge": {"alternative": "The two mechanisms cannot coexist because both consume the same storage or synchronization object.", "check": "Trace the address spaces and barrier arrays in the pinned implementation.", "outcome": "SMEM operand stages and TMEM accumulator regions are separately allocated and coordinated by distinct pipeline/barrier state."}, "claim": "Double- or multi-buffering uses disjoint copies so a producer can fill one stage while a consumer reads another; a Blackwell kernel can combine SMEM operand staging with separate TMEM accumulator regions.", "claim_id": "technique-double-buffering-original-c001", "claim_type": "mechanism", "covers": ["u001"], "evidence": [{"locator": "NUM_STAGES SMEM arrays, BLOCK_N*2 TMEM allocation, mainloop/epilogue barriers", "observation": "The pinned kernel has multiple SMEM stages and two disjoint BLOCK_N-wide TMEM accumulator regions with producer, MMA, and epilogue handoffs.", "path": "/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v6.cu", "revision": "gau-nernst/learn-cuda 3b90ac9b697a1180d06a0fba42431534410d1949"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The 256 KiB figure means one independently private physical array per CTA or mandates two equal buffers.", "check": "Separate the ISA-visible structure from allocation policy and inspect the example's chosen offsets.", "outcome": "The dimensions are architectural, but equal 256-column partitioning is an implementation choice; the claim is scoped accordingly."}, "claim": "The CTA-visible TMEM address structure has 128 lanes by 512 columns of 32-bit cells; a symmetric 512-column allocation can place two 256-column accumulator regions at offsets 0 and 256.", "claim_id": "technique-double-buffering-original-c002", "claim_type": "mechanism", "covers": ["u002"], "evidence": [{"locator": "Tensor Memory organization and allocation", "observation": "TMEM is organized as 128 lanes by 512 columns of 32-bit cells, and allocations reserve columns across all lanes; offsets 0 and 256 therefore name disjoint halves of a 512-column allocation.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The displayed tmem_double_buffer_mainloop is a safe CUDA implementation of two-buffer TMEM accumulation and epilogue reuse.", "claim_id": "technique-double-buffering-original-c003", "claim_type": "code", "covers": ["u003"], "evidence": [{"locator": "collective allocation/deallocation and lifetime rules", "observation": "The block contains no TMEM allocation or mandatory deallocation and does not establish publication of a returned address.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"}, {"locator": "tcgen05.ld collective execution, completion, and address mapping", "observation": "TMEM readback is a warp-collective asynchronous instruction followed by wait::ld, not a per-row scalar load callable independently by lanes.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"}, {"locator": "mbarrier phase and initialization semantics", "observation": "Reusable barriers require correct expected-arrival counts and phase/parity tracking; the block initializes drained barriers to one despite claiming fourteen arrivals and reuses waits without tokens/phases.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Multiple arrays alone guarantee overlap without a stage-state protocol.", "check": "Remove or invert a full/empty handoff in the source control-flow graph.", "outcome": "The producer would overwrite unread data or the consumer would read incomplete data; synchronization and ownership are essential parts of the verified claim."}, "claim": "Multiple SMEM copies of A/B tiles can overlap asynchronous TMA production with MMA consumption when each stage has a correct full/empty ownership protocol.", "claim_id": "technique-double-buffering-original-c004", "claim_type": "mechanism", "covers": ["u004"], "evidence": [{"locator": "SharedStorageHorizontal and producer/consumer stage loops", "observation": "The merged code uses numStages SMEM arrays with per-stage bar_empty/bar_full handoffs around TMA copies and consumption.", "path": "artifacts/prs/flashinfer/PR-2387/key-files/include/flashinfer/mamba/selective_state_update.cuh", "revision": "flashinfer merge 18804cd51734cccf807356d017733bc757677f15"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The struct is a complete drop-in shared-memory layout and its payload total is the allocation total.", "check": "Account for barriers, padding, descriptor alignment, swizzled layouts, and placement in dynamic/static shared memory.", "outcome": "Only the payload arithmetic and conceptual staging are established; the displayed type is not sufficient production code."}, "claim": "For TILE_M=128, TILE_N=256, TILE_K=64 and binary16 operands, one conceptual A/B stage occupies 16 KiB plus 32 KiB, so three stages occupy 144 KiB before barriers, padding, alignment, and other shared storage.", "claim_id": "technique-double-buffering-original-c005", "claim_type": "code", "covers": ["u005"], "evidence": [{"command": "conda run -n base python integer arithmetic for 128*64*2, 64*256*2, and three-stage total", "environment": "Conda base, host CPU", "negative_control": "Treating binary16 as four bytes doubles all totals and fails the stated example.", "reference": "binary16 is two bytes; compare exact byte totals", "result": "16384 + 32768 = 49152 bytes per stage and 147456 bytes = 144 KiB for three stages."}], "oracle": "host-experiment", "verdict": "verified"}, {"claim": "Every fully optimized Blackwell GEMM uses simultaneous three-stage SMEM and two-buffer TMEM.", "claim_id": "technique-double-buffering-original-c006", "claim_type": "performance", "covers": ["u006"], "evidence": [{"locator": "first Blackwell GEMM tutorial stage", "observation": "The official tutorial progression includes valid Blackwell GEMMs before the later pipelined variants; neither exact stage count nor double-buffer policy is a universal optimization requirement.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The displayed combined CUDA block and timeline form a complete safe implementation in which the previous tile's epilogue overlaps the next tile's MMA.", "claim_id": "technique-double-buffering-original-c007", "claim_type": "code", "covers": ["u007"], "evidence": [{"locator": "full kernel role loops, ready/empty barriers, parity, prologue and tail", "observation": "The real implementation requires independent persistent role loops, phase-aware ready/empty handoffs, prologue seeding, read completion, final drain, and TMEM cleanup; the block leaves these as opaque helpers/comments and its per-warp common outer loop does not prove overlap.", "path": "/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v6.cu", "revision": "gau-nernst/learn-cuda 3b90ac9b697a1180d06a0fba42431534410d1949"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "Double buffering merely renames one register fragment and therefore has no live-range cost.", "check": "Require simultaneous producer and consumer ownership of different output tiles.", "outcome": "Overlap requires two disjoint live results; reuse of the same fragment would overwrite the result still being drained."}, "claim": "SM90 has no TMEM, and WGMMA's D accumulator is a per-thread register fragment, so keeping two simultaneous live D fragments increases register demand.", "claim_id": "technique-double-buffering-original-c008", "claim_type": "mechanism", "covers": ["u008"], "evidence": [{"locator": "WGMMA D fragment tables", "observation": "WGMMA D is explicitly distributed as register vectors among warpgroup threads; two simultaneously live vectors require twice that D-fragment storage.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "For a Hopper m64n256 FP32 accumulator, two buffers consume only 64 registers (256 bytes) per thread, whereas two Blackwell TMEM buffers have zero register cost and dedicate all 256 KiB to accumulator space.", "claim_id": "technique-double-buffering-original-c009", "claim_type": "code", "covers": ["u009"], "evidence": [{"locator": "m64nNk16 f32 D fragment", "observation": "Each thread holds N/2 f32 registers: 128 registers for N=256, hence 256 registers (1024 bytes) for two simultaneous fragments, not 64 (256 bytes).", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16"}, {"locator": "register and TMEM resource limits", "observation": "TMEM removes resident D from registers but addresses, descriptors, synchronization state, readback batches, and epilogue values still consume registers; TMEM can also serve non-accumulator uses.", "revision": "CUDA 13.0.2 Blackwell Tuning Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Both architectures expose D as an ordinary pointer-addressable buffer.", "check": "Compare the operand classes and legal load operations.", "outcome": "WGMMA names register fragments, while tcgen05 names a TMEM address and uses dedicated collective movement instructions."}, "claim": "The storage and access rows correctly contrast WGMMA register-resident D with tcgen05 TMEM-resident D that must be transferred to registers for an epilogue.", "claim_id": "technique-double-buffering-original-c010", "claim_type": "mechanism", "covers": ["u010", "u013"], "evidence": [{"locator": "WGMMA D operand and fragments", "observation": "D is a register operand distributed among threads.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-multiply-accumulate-instructions-wgmma-mma"}, {"locator": "TMEM-to-register load", "observation": "tcgen05.ld transfers a TMEM region into per-thread registers before ordinary scalar processing.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "Doubling an accumulator costs exactly twice the Hopper registers but zero Blackwell registers.", "claim_id": "technique-double-buffering-original-c011", "claim_type": "performance", "covers": ["u011"], "evidence": [{"locator": "tcgen05.ld destination operands", "observation": "TMEM readback places values in register vectors, so the resident D vector is absent but total register cost is not zero.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Accumulator double-buffering typically changes Hopper occupancy from two to one CTA per SM and has no Blackwell occupancy impact.", "claim_id": "technique-double-buffering-original-c012", "claim_type": "performance", "covers": ["u012"], "evidence": [{"locator": "occupancy calculation", "observation": "Resident CTAs depend on block size, compiled registers, shared memory, barriers, clusters, and allocation granularities; neither architecture implies the displayed transition.", "revision": "CUDA 13.0.2 Programming Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The maximum practical accumulator is about 16K elements on Hopper and exactly 128 by 512 elements on Blackwell.", "claim_id": "technique-double-buffering-original-c013", "claim_type": "performance", "covers": ["u014"], "evidence": [{"locator": "WGMMA instruction shapes", "observation": "An instruction fragment is not a universal kernel accumulator maximum; kernels may compose instructions and distribute output ownership.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-multiply-accumulate-instructions-wgmma-mma"}, {"locator": "TMEM structure and packing", "observation": "128 by 512 is the cell-address structure, not a universal count of logical accumulator elements; packing and descriptor shape depend on dtype/instruction kind.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "TMEM double-buffering should always be used for any nontrivial Blackwell epilogue and its only cost is plentiful TMEM capacity.", "claim_id": "technique-double-buffering-original-c014", "claim_type": "performance", "covers": ["u015"], "evidence": [{"locator": "resource-dependent occupancy and TMEM", "observation": "TMEM allocation can constrain residency and buffering also adds synchronization, address/lifetime complexity, and readback; profitability is kernel- and workload-dependent.", "revision": "CUDA 13.0.2 Blackwell Tuning Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Every GEMM or attention mainloop should use SMEM multi-staging, with three stages as the universal default and four or five only for long high-latency K loops.", "claim_id": "technique-double-buffering-original-c015", "claim_type": "performance", "covers": ["u016"], "evidence": [{"locator": "numStages selections near kernel dispatch", "observation": "One merged kernel selects three stages in one path and min(totalStages,4) in another, demonstrating shape/path-specific policy rather than the page's universal rule.", "path": "artifacts/prs/flashinfer/PR-2387/key-files/include/flashinfer/mamba/selective_state_update.cuh", "revision": "flashinfer merge 18804cd51734cccf807356d017733bc757677f15"}, {"locator": "tutorial progression and stage construction", "observation": "Official examples derive or configure stages from tile and resource policy rather than declaring three universal.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "Simultaneous SMEM and TMEM buffering is the standard production Blackwell GEMM approach used by all CUTLASS and CuTe DSL kernels.", "claim_id": "technique-double-buffering-original-c016", "claim_type": "compatibility", "covers": ["u017"], "evidence": [{"locator": "multiple tutorial GEMM variants", "observation": "CUTLASS supplies multiple Blackwell schedules and incremental examples; the sources establish concrete combined implementations, not universality across all kernels.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "TMEM double-buffering always requires every accumulator to fit in exactly half of the 512 columns, and every FP32 tile wider than N=256 must be split.", "claim_id": "technique-double-buffering-original-c017", "claim_type": "compatibility", "covers": ["u018"], "evidence": [{"locator": "column allocation and addressing", "observation": "The ISA specifies allocation units and address structure, not symmetric two-way partitioning; implementations may use unequal regions, more/fewer buffers, packed formats, or decomposed ownership.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory-allocation"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The 228 KiB figure applies equally to every Blackwell compute capability or is all available to one CTA without reservation/configuration limits.", "check": "Read the architecture scope and distinguish per-SM capacity from per-block opt-in limits.", "outcome": "The claim is valid only when scoped to compute capability 10.0 and as a shared budget, not a universal Blackwell per-block guarantee."}, "claim": "Compute capability 10.0 supports up to 228 KiB of shared memory per SM, so operand stages and epilogue scratch can compete for the shared-memory budget.", "claim_id": "technique-double-buffering-original-c018", "claim_type": "compatibility", "covers": ["u019"], "evidence": [{"locator": "SM shared-memory capacity", "observation": "The guide reports up to 228 KiB per SM for compute capability 10.0; all dynamic/static shared objects contribute to the relevant allocation constraints.", "revision": "CUDA 13.0.2 Blackwell Tuning Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The mbarrier handoff between TMEM buffers costs a universal few cycles per tile, making it proportionally worse for tiles with few K iterations.", "claim_id": "technique-double-buffering-original-c019", "claim_type": "performance", "covers": ["u020"], "evidence": [{"locator": "mbarrier semantics", "observation": "The ISA specifies ordering, arrivals, phases, and waits but no fixed cycle latency; exposed cost depends on contention, polling, dependencies, schedule, and target.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"}], "oracle": "web-primary", "verdict": "refuted"}], "commands": ["archived CUDA 13.0.2 PTX/tuning-guide inspection", "exact-revision author and CUTLASS source inspection", "FlashInfer PR artifact hash comparison against the full merge SHA", "conda-base byte arithmetic and negative control"], "id": "technique-double-buffering", "page_verdict": "refuted", "path": "wiki/techniques/double-buffering.md", "residual_risk": "No CUDA toolkit or SM90/SM100 GPU is installed locally. No throughput or instruction-latency claim is accepted without a pinned benchmark, and illustrative layouts are not treated as compiled kernels.", "scope": "All 20 baseline units for body hash 787b870b0080a00805a0601a72d16367df1fd5ff21f1043c92acaf6e2d91a387 were reviewed; code blocks were judged both for their scoped arithmetic/concepts and for whether they establish a safe runnable lifecycle."} +{"claims": [{"claim": "Epilogue fusion inherently means overlapping post-MMA work with ongoing MMA, and Blackwell normally assigns warps 2-15 to drain one TMEM half while warp 1 writes the other.", "claim_id": "technique-epilogue-fusion-original-c001", "claim_type": "mechanism", "covers": ["u001"], "evidence": [{"locator": "warp-role dispatch and epilogue path", "observation": "The official example assigns a 128-thread epilogue group (four warps), reads one completed TMEM accumulator, converts through registers/SMEM, and stores; fusion and concurrent next-tile MMA are separate scheduling choices.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "TMEM can be cast to a CUDA pointer and used directly by scalar arithmetic.", "check": "Inspect PTX state spaces and legal TMEM movement instructions.", "outcome": "TMEM is instruction-addressed and tcgen05.ld/copy is required before ordinary register computation."}, "claim": "An SM100 tcgen05 accumulator resides in TMEM and an ordinary elementwise epilogue must first transfer its relevant values into registers through a legal TMEM copy/load operation.", "claim_id": "technique-epilogue-fusion-original-c002", "claim_type": "mechanism", "covers": ["u002"], "evidence": [{"locator": "tcgen05.ld TMEM-to-register operation", "observation": "The instruction collectively copies TMEM data into per-thread register vectors and requires load completion before use.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The displayed epilogue_warp_fn is safe usable CUDA for fourteen warps to load TMEM scalars, apply scale/bias/ReLU, and store a tile.", "claim_id": "technique-epilogue-fusion-original-c003", "claim_type": "code", "covers": ["u003"], "evidence": [{"locator": "shape, collective execution, mapping, and completion", "observation": "There is no scalar tmem_load_f32(r,c) operation; loads are warp-collective shaped operations with mapped register vectors and wait::ld completion. The block also omits edge predicates and source/result layout mapping.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "TMEM double-buffering with a mandatory 256/256 column split is the key required mechanism for Blackwell epilogue fusion.", "claim_id": "technique-epilogue-fusion-original-c004", "claim_type": "mechanism", "covers": ["u004"], "evidence": [{"locator": "TMEM allocation and epilogue", "observation": "The official warp-specialized GEMM allocates 512 columns and performs a fused conversion/store epilogue without establishing the page's mandatory equal-half ping-pong design.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The displayed mma_epilogue_overlap is a complete phase-correct two-region protocol.", "claim_id": "technique-epilogue-fusion-original-c005", "claim_type": "code", "covers": ["u005"], "evidence": [{"locator": "expected arrivals and reusable phases", "observation": "The epilogue-done barriers are initialized with count one while comments require all epilogue warps, and repeated waits have no token/parity state.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"}, {"locator": "tcgen05 completion", "observation": "A raw barrier arrival after an opaque MMA helper does not attach asynchronous MMA completion as tcgen05.commit does; allocation, load completion, tail drain, and deallocation are also absent.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The names are documentation-only and not accepted by the SM100 builder.", "check": "Trace operation tags through sm100_builder.inl and sm100 callback specializations.", "outcome": "SM100 builder specializations instantiate callbacks for supported operation tags; support remains combination-dependent rather than arbitrary."}, "claim": "CUTLASS 4.5.0 defines composable epilogue fusion operation tags and a CollectiveBuilder that accepts a fusion operation or callbacks.", "claim_id": "technique-epilogue-fusion-original-c006", "claim_type": "compatibility", "covers": ["u006"], "evidence": [{"locator": "FusionOperation and LinComb* operation tags", "observation": "The release defines linear combination, activation, row/column bias, auxiliary, block-scale, residual, and top-k softmax tags.", "path": "/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/operations.hpp", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}, {"locator": "CollectiveBuilder template", "observation": "The final policy parameter accepts FusionOpOrCallbacks with a LinearCombination default.", "path": "/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/collective_builder.hpp", "revision": "same tag"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "CUTLASS 4.5.0 can fuse arbitrary element-wise operations without support or schedule constraints.", "claim_id": "technique-epilogue-fusion-original-c007", "claim_type": "compatibility", "covers": ["u006"], "evidence": [{"locator": "SM100 CollectiveBuilder specializations and static constraints", "observation": "Support is selected by architecture, operator class, schedule, layouts, alignments, datatypes, tile and recognized fusion operation/callback combinations; it is not arbitrary.", "path": "/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/builders/sm100_builder.inl", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The displayed LinCombEltAct and Sm100EpilogueTmaWarpSpecialized typedefs are valid CUTLASS 4.5.0 code for a scale+bias+ReLU SM100 epilogue.", "claim_id": "technique-epilogue-fusion-original-c008", "claim_type": "code", "covers": ["u007"], "evidence": [{"locator": "LinCombEltAct template signature", "observation": "The actual parameters are ActivationFn template, ElementOutput, ElementCompute, ElementSource, ElementScalar, and rounding; the shown order/types do not represent the claimed output/source policy.", "path": "/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/operations.hpp", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}, {"locator": "full tree symbol search", "observation": "No Sm100EpilogueTmaWarpSpecialized type exists; construction uses CollectiveBuilder, EpilogueTileAuto, EpilogueScheduleAuto and an optional fusion op/callback.", "path": "/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue", "revision": "same tag"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The row labeled Scale + Bias gives the standard GEMM source epilogue equation y=alpha*acc+beta*C.", "claim_id": "technique-epilogue-fusion-original-c009", "claim_type": "mechanism", "covers": ["u008"], "evidence": [{"locator": "LinearCombination versus LinCombPerRowBias/LinCombPerColBias", "observation": "alpha*acc+beta*C is a linear combination with a source tensor C; bias is a distinct row/column input in separate operation tags.", "path": "/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/operations.hpp", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "Every listed formula is automatically one CUTLASS SM100 visitor and therefore always legal/profitable.", "check": "Search exact operation tags and builder constraints for each composition.", "outcome": "Only the operation categories are established; exact combinations require supported callbacks or custom code and benchmarking."}, "claim": "Activation, output conversion/quantization, gated activation/product, online-softmax rescaling, and residual addition are legitimate categories of work that may be fused around a matrix/attention output when their data dependencies and supported implementation permit it.", "claim_id": "technique-epilogue-fusion-original-c010", "claim_type": "mechanism", "covers": ["u009", "u010", "u011", "u012", "u013"], "evidence": [{"locator": "activation, residual, bias, block-scale and softmax operation tags", "observation": "The release directly implements several listed categories and marks source/residual/activation capabilities.", "path": "/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/operations.hpp", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}, {"locator": "ReLU, SiLU and GELU functors", "observation": "The activation functions named in the table are present as epilogue-compatible functors.", "path": "/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/activation.h", "revision": "same tag"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "Every Blackwell GEMM with a nontrivial epilogue should use fourteen epilogue warps, which are available by default, and fusion necessarily removes one launch and one global-memory round trip.", "claim_id": "technique-epilogue-fusion-original-c011", "claim_type": "performance", "covers": ["u014"], "evidence": [{"locator": "epilogue CooperativeGroup and dispatch", "observation": "This official example uses a 128-thread/four-warp epilogue group, not a default fourteen. Whether fusion removes a launch/intermediate depends on the unfused baseline and graph.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "Softmax rescaling must wait for the entire attention matrix and cannot overlap any subsequent tensor-core work.", "check": "Inspect the tiled online-softmax schedule and producer/consumer dependencies.", "outcome": "The cited kernel operates tilewise and interleaves independent tensor/non-tensor work; exact next-tile timing remains implementation-specific."}, "claim": "Attention implementations can interleave softmax rescaling/output accumulation with tensor-core work when their schedule provides correct dependencies.", "claim_id": "technique-epilogue-fusion-original-c012", "claim_type": "mechanism", "covers": ["u015"], "evidence": [{"locator": "asynchrony and interleaving design", "observation": "FA3 describes overlapping/interleaving GEMM and softmax/non-GEMM work on Hopper through warp specialization and instruction scheduling.", "revision": "FlashAttention-3 paper", "url": "https://arxiv.org/abs/2407.08608"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Conversion still necessarily writes and rereads an FP32 tensor between kernels.", "check": "Trace accumulator fragment conversion into the visitor's output/store path.", "outcome": "The conversion occurs in the epilogue path before the final output write, so no separate FP32 global intermediate is required."}, "claim": "Fusing FP32-to-FP8/FP16 output conversion can avoid materializing an FP32 intermediate in global memory.", "claim_id": "technique-epilogue-fusion-original-c013", "claim_type": "mechanism", "covers": ["u016"], "evidence": [{"locator": "compute_quantized_value and store visitor", "observation": "The SM100 visitor computes quantized output fragments before the output store instead of storing an FP32 intermediate tensor.", "path": "/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/sm100_visitor_store_tma_warpspecialized.hpp", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Disjoint warp roles or __syncthreads alone makes incomplete asynchronous work visible.", "check": "Distinguish thread synchronization from async-proxy completion and source lifetime.", "outcome": "Role separation does not complete asynchronous operations; explicit completion edges are required."}, "claim": "An epilogue may read a TMEM accumulator only after the corresponding tcgen05 MMA work completes; synchronization must also prevent overwrite before all reads complete.", "claim_id": "technique-epilogue-fusion-original-c014", "claim_type": "mechanism", "covers": ["u017"], "evidence": [{"locator": "MMA completion and operand lifetimes", "observation": "Commit associates asynchronous work completion with an mbarrier; destination/source lifetimes prohibit conflicting access before completion.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"}, {"locator": "load completion and source lifetime", "observation": "TMEM source data must remain valid until the collective load completes.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "Fourteen simultaneous epilogue warps receive equal proportional shares of TMEM bandwidth, and sufficiently wide TILE_N therefore bottlenecks on TMEM read bandwidth.", "claim_id": "technique-epilogue-fusion-original-c015", "claim_type": "performance", "covers": ["u018"], "evidence": [{"locator": "load semantics and shapes", "observation": "The ISA specifies mappings and completion but no equal-share bandwidth model or TILE_N bottleneck threshold; the assumed fourteen-warp configuration is not architectural.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "A store-only epilogue wastes fourteen warps and should reduce CTA size or repurpose those exact warps for next-tile TMA prefetch.", "claim_id": "technique-epilogue-fusion-original-c016", "claim_type": "performance", "covers": ["u019"], "evidence": [{"locator": "role groups and epilogue TMA-store pipeline", "observation": "The official design uses four epilogue warps for conversion/staging/store; changing CTA roles affects collectives, pipeline participation, and occupancy and cannot be inferred from epilogue simplicity alone.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "Every file under artifacts/kernels/epilogue-fusion/full is verbatim upstream code.", "claim_id": "technique-epilogue-fusion-original-c017", "claim_type": "compatibility", "covers": ["u020"], "evidence": [{"command": "inspect full/PROVENANCE.yaml and the first lines of every full artifact; compare nvfp4 file with raw merge revision", "environment": "current worktree, 2026-08-09", "negative_control": "The separately labeled variants file declares derived/not upstream, demonstrating the repository can distinguish modes.", "reference": "Every full file should be upstream-verbatim if the page claim is true.", "result": "nvfp4_scaled_mm_kernels.cu byte-matches vLLM merge ed7a29d9..., but tmem-load-into-registers-for-epilogue.cu declares itself an extracted block from a local blog card and is not upstream-verbatim."}], "oracle": "host-experiment", "verdict": "refuted"}, {"challenge": {"alternative": "The command omits artifacts or treats the derived skeleton as upstream.", "check": "Search captured output for artifact paths and inspect the first source line.", "outcome": "Artifacts are included and the variant's provenance label is explicit; this does not rescue the separate false claim about every full file."}, "claim": "The page and its artifact bundle are returned by scripts/get_page.py with --include-code, and variant files carry the required derived/not-upstream header.", "claim_id": "technique-epilogue-fusion-original-c018", "claim_type": "code", "covers": ["u020", "u021", "u022"], "evidence": [{"command": "conda run -n base python scripts/get_page.py technique-epilogue-fusion --include-code; inspect output and variant headers", "environment": "Conda base, current worktree", "negative_control": "A nonexistent page ID is rejected by the retrieval script.", "reference": "Command should exit zero, include the page/artifacts, and variants should be labeled.", "result": "Command exited zero with 628 lines and included artifact content; 01-double-buffered-tmem-epilogue-skeleton.cu starts with the required derived/not-upstream provenance line."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["exact CUTLASS 4.5.0 source search", "PTX ISA 9.0 lifecycle inspection", "vLLM PR full-SHA and artifact byte-hash comparison", "conda-base page retrieval with provenance controls"], "id": "technique-epilogue-fusion", "page_verdict": "refuted", "path": "wiki/techniques/epilogue-fusion.md", "residual_risk": "No CUDA compiler or SM100 GPU is available locally. Unsupported scheduling and throughput claims were refuted rather than inferred from source structure.", "scope": "All 22 baseline units for body hash f37f59190f56273dbd2ec10059fd85e03af89ea72f6cf5e9898fb14e1f3fd450 were reviewed, including every code block and artifact-routing claim."} +{"claims": [{"challenge": {"alternative": "The source cards alone are sufficient implementation evidence, so exact upstream files do not need inspection.", "check": "Compare the source-card summaries with the pinned repository file trees and implementations.", "outcome": "Several source-card architecture and mechanism labels exceed the code's actual scope, while exact files discriminate what is present."}, "claim": "Once profiling has identified an edit family, searching a small upstream source repository for the measured mechanism and inspecting exact files is a valid way to find candidate implementations.", "claim_id": "technique-external-source-map-research-original-c001", "claim_type": "mechanism", "covers": ["u001"], "evidence": [{"locator": "tma/tma_copy.h, pipeline-gemm/sm90_gemm_multistage.cu, streamk/tile_scheduler.hpp, transpose-cute/", "observation": "The pinned repository contains compact, mechanism-specific examples whose exact paths can be inspected after a profiler has narrowed the candidate class.", "path": "/tmp/kernelwiki-ext-cfx", "revision": "ColfaxResearch/cfx-article-src fbecfed88de2e4246f104a023188ba722937c5fc"}, {"locator": "ld_matrix_x{1,2,4}.cu and st_matrix_x{1,2,4}.cu", "observation": "The repository is a small set of direct ldmatrix/stmatrix instruction examples rather than a general optimization authority.", "path": "/tmp/kernelwiki-ext-load-store", "revision": "simveit/load_and_store 05d828cf910dd43f0053ddbbe4744218a06e9d7f"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "The displayed clone-and-grep snippet is reproducible enough to support exact implementation citations.", "claim_id": "technique-external-source-map-research-original-c002", "claim_type": "code", "covers": ["u002"], "evidence": [{"locator": "git revision and repository tree", "observation": "The snippet clones the moving default branch and records neither this immutable revision nor an exact file; future results can change.", "path": "/tmp/kernelwiki-ext-cfx", "revision": "ColfaxResearch/cfx-article-src fbecfed88de2e4246f104a023188ba722937c5fc"}, {"locator": "repository scope and six .cu files", "observation": "This repository only demonstrates ldmatrix/stmatrix, so the shared broad regular expression does not establish the page's wider TMA, swizzle, Stream-K, or vector-width scope.", "path": "/tmp/kernelwiki-ext-load-store/README.md", "revision": "simveit/load_and_store 05d828cf910dd43f0053ddbbe4744218a06e9d7f"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "Those profiler symptoms uniquely diagnose a layout or vector-width defect.", "check": "Inspect whether the repositories define profiler-counter causality or instead provide candidate mechanisms.", "outcome": "They provide implementation examples but no symptom-to-cause theorem; the page only recommends searching before editing."}, "claim": "Load/store and transpose examples are relevant candidate sources after profiling identifies long-scoreboard or sector-utilization symptoms, without proving that vector width or layout is the cause.", "claim_id": "technique-external-source-map-research-original-c003", "claim_type": "mechanism", "covers": ["u003", "u004"], "evidence": [{"locator": "transpose_naive.cu, transpose_swizzle.cu and transpose_swizzle_batched.cu", "observation": "The pinned SM90a examples compare TMA transpose variants and explicitly encode 128-byte tensor-map swizzling.", "path": "/tmp/kernelwiki-ext-transpose", "revision": "simveit/effective_transpose 994b2b5acaa67f80e411df3e8274b6ae13fd1949"}, {"locator": "series/cuda-cpp/coalescing-global/coalescing.cu and series/cuda-cpp/transpose/transpose.cu", "observation": "The source tree contains direct coalescing and shared-memory transpose examples, including a bank-conflict-free padded layout.", "path": "/tmp/kernelwiki-ext-nvidia-samples", "revision": "NVIDIA-developer-blog/code-samples 3350d216083a902ccbf5b31665e3b82096a75b55"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The examples establish that changing stage count or role split will improve any kernel with these stalls.", "check": "Inspect the stated target/version and configuration scope of each example.", "outcome": "The sources are Hopper-specific candidate implementations and make no portable performance guarantee; the page says to search them before changing code."}, "claim": "The Colfax repositories contain Hopper pipelined GEMM and TMA/barrier examples relevant to investigating barrier or TMA-wait stalls before changing stages or producer/consumer roles.", "claim_id": "technique-external-source-map-research-original-c004", "claim_type": "mechanism", "covers": ["u005", "u006"], "evidence": [{"locator": "pipeline-gemm/README.md, pipeline-gemm/sm90_gemm_multistage.cu and tma/tma_copy.h", "observation": "The pinned files implement SM90 TMA copies, transaction barriers, multistage GEMM, and a warp-specialized alternative.", "path": "/tmp/kernelwiki-ext-cfx", "revision": "ColfaxResearch/cfx-article-src fbecfed88de2e4246f104a023188ba722937c5fc"}, {"locator": "README.md and fmha_pipe_ws.h", "observation": "The source documents and implements SM90a software pipelining with optional warp specialization and CUTLASS PipelineTmaAsync.", "path": "/tmp/kernelwiki-ext-colfax-cutlass/src/fmha-pipeline", "revision": "ColfaxResearch/cutlass-kernels 84f0802e2b4a1bf068ac70359f20ffdb368c8f6a"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "A tail wave proves that a persistent or Stream-K scheduler is the best fix.", "check": "Inspect whether the repository connects profiler tail metrics to a universal scheduler choice.", "outcome": "It exposes alternatives and shape controls rather than a universal choice; the page treats them as candidates to inspect."}, "claim": "The pinned cfx repository contains persistent data-parallel and Stream-K schedulers relevant to investigating tail waves before adding shape-specific dispatch.", "claim_id": "technique-external-source-map-research-original-c005", "claim_type": "mechanism", "covers": ["u007", "u008"], "evidence": [{"locator": "tile_scheduler.hpp classes and cli_options.h scheduler values", "observation": "The source implements SingleTileScheduler, DataParallelPersistentTileScheduler, and StreamKPersistentTileScheduler, selectable as non-persistent, persistent, Stream-K, or heuristic modes.", "path": "/tmp/kernelwiki-ext-cfx/streamk", "revision": "ColfaxResearch/cfx-article-src fbecfed88de2e4246f104a023188ba722937c5fc"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "Citing a local source page plus an exact upstream path, commit, or URL is sufficient implementation provenance.", "claim_id": "technique-external-source-map-research-original-c006", "claim_type": "other", "covers": ["u009"], "evidence": [{"locator": "sections 5 and 8", "observation": "Evidence must record a revision and an exact section, symbol, line, or table locator; a moving repository URL by itself is insufficient.", "path": "plan.md", "revision": "repository remediation contract"}, {"locator": "frontmatter and body", "observation": "The local summary assigns SM100 and persistent/tile-scheduling scope although the pinned source is SM90a and contains no such implementation, demonstrating why the source page is only a lead.", "path": "sources/blogs/colfax-cutlass-kernels.md", "revision": "baseline body 4fa361c619785bf4e42b8fd58ce26e98c998de467797e6cea6b778a328fe3184"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The linked repositories provide TMA, swizzle, Stream-K, persistence, and load/store mechanics on both Hopper- and Blackwell-era kernels.", "claim_id": "technique-external-source-map-research-original-c007", "claim_type": "compatibility", "covers": ["u001"], "evidence": [{"locator": "pipeline-gemm, streamk, tma and transpose-cute build files/source symbols", "observation": "The relevant modern examples compile for SM90a and use SM90 TMA/GMMA facilities; no SM100 target or tcgen05 implementation appears.", "path": "/tmp/kernelwiki-ext-cfx", "revision": "ColfaxResearch/cfx-article-src fbecfed88de2e4246f104a023188ba722937c5fc"}, {"locator": "FMHA READMEs, compile scripts and CUTLASS GEMM source", "observation": "The repository explicitly targets Hopper SM90/SM90a with CUDA 12.2/12.3 and SM90 TMA/WGMMA.", "path": "/tmp/kernelwiki-ext-colfax-cutlass", "revision": "ColfaxResearch/cutlass-kernels 84f0802e2b4a1bf068ac70359f20ffdb368c8f6a"}, {"locator": "README.md and Makefile", "observation": "The repository calls itself a Hopper transpose example and compiles with -arch=sm_90a.", "path": "/tmp/kernelwiki-ext-transpose", "revision": "simveit/effective_transpose 994b2b5acaa67f80e411df3e8274b6ae13fd1949"}, {"locator": "README.md and Makefile", "observation": "The repository is six ldmatrix/stmatrix examples compiled for sm_90a, not a Blackwell source map.", "path": "/tmp/kernelwiki-ext-load-store", "revision": "simveit/load_and_store 05d828cf910dd43f0053ddbbe4744218a06e9d7f"}], "oracle": "local-primary-source", "verdict": "refuted"}], "commands": ["git ls-remote each linked repository and inspect pinned shallow clones", "rg exact architecture, mechanism and scheduler symbols in each pinned tree"], "id": "technique-external-source-map-research", "page_verdict": "refuted", "path": "wiki/techniques/external-source-map-research.md", "residual_risk": "No GPU benchmark was run. The verified workflow claims are candidate-discovery rules, not claims that a profiler symptom proves a root cause or that a transferred implementation improves performance.", "scope": "All nine baseline body units, frontmatter architecture claims, and all five linked source-card scopes were checked against exact upstream repository revisions."} +{"claims": [{"claim": "Fine-grained scaling prevents an outlier from destroying tensor precision, and Blackwell universally uses native UE8M0 while Hopper requires Nc=128 CUDA-core promotion.", "claim_id": "technique-fine-grained-quantization-original-c001", "claim_type": "compatibility", "covers": ["u001"], "evidence": [{"locator": "sections 3.3.2 and 3.5.2", "observation": "The paper presents 1x128/128x128 scaling and Nc=128 as its scoped FP8 training design; it does not guarantee that local scaling prevents all outlier-induced error or prescribe this Hopper implementation universally.", "revision": "DeepSeek-V3 Technical Report v2", "url": "https://arxiv.org/abs/2412.19437v2"}, {"locator": "block-scale valid-combination table", "observation": "Blackwell block-scaled tcgen05 supports multiple kinds and both UE8M0 and UE4M3 combinations, not only UE8M0.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The scaling diagram's 128x128 FP32 factor has 0.006% overhead and a 1x128 activation group captures a per-channel distribution.", "claim_id": "technique-fine-grained-quantization-original-c002", "claim_type": "code", "covers": ["u002"], "evidence": [{"command": "Compute scale-count ratio and storage ratio for one FP32 factor per 128x128 FP8 payload block", "environment": "deterministic host arithmetic", "negative_control": "Using a one-byte factor yields the stated 0.0061035% byte ratio, confirming that the missing factor width causes the ambiguity.", "reference": "Distinguish factor count from bytes and distinguish a row-by-128-channel group from a channel-wide group.", "result": "1/16384 is 0.0061035% as a count ratio, but 4/(16384*1) is 0.024414% storage overhead; a 1x128 group is one token/row across 128 channels, not a per-channel distribution."}], "oracle": "host-experiment", "verdict": "refuted"}, {"challenge": {"alternative": "Both operands use the same 128x128 block.", "check": "Read the separate activation and weight granularity definitions.", "outcome": "The paper explicitly assigns different geometries to the two operands."}, "claim": "DeepSeek's fine-grained FP8 scheme uses 1x128 activation groups and 128x128 weight groups.", "claim_id": "technique-fine-grained-quantization-original-c003", "claim_type": "semantics", "covers": ["u003"], "evidence": [{"locator": "section 3.3.2 and Appendix B.2", "observation": "The report states tile-wise 1x128 activation scaling and block-wise 128x128 weight scaling.", "revision": "DeepSeek-V3 Technical Report v2", "url": "https://arxiv.org/abs/2412.19437v2"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The displayed deepgemm_fp8_tile function is a representative DeepGEMM CUDA implementation with the stated scale tensors and FP32 output accumulator.", "claim_id": "technique-fine-grained-quantization-original-c004", "claim_type": "code", "covers": ["u004"], "evidence": [{"locator": "BLOCK_K assertion and math warp-group loop", "observation": "The upstream kernel uses WGMMA descriptors, staged TMA scale loads, float accum/final_accum arrays, and a templated epilogue. The displayed function uses undefined types/constants/partial and is neither upstream nor compilable.", "path": "artifacts/kernels/deepgemm/full/sm90_fp8_gemm_1d1d.cuh", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f, byte-verified"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "Hopper FP8 WGMMA accumulates with approximately FP22 precision and DeepGEMM universally promotes every four instructions.", "claim_id": "technique-fine-grained-quantization-original-c005", "claim_type": "semantics", "covers": ["u005"], "evidence": [{"locator": "sections 3.3.2 and 3.5.2", "observation": "The report characterizes H800 FP8 Tensor Core accumulation as retaining about 14 bits, not FP22, and describes Nc=128/four-WGMMA promotion for its specific scheme.", "revision": "DeepSeek-V3 Technical Report v2", "url": "https://arxiv.org/abs/2412.19437v2"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The displayed hopper_fp8_gemm_with_promotion function is a valid DeepGEMM implementation of Nc=128 promotion.", "claim_id": "technique-fine-grained-quantization-original-c006", "claim_type": "code", "covers": ["u006"], "evidence": [{"locator": "lines 250-323 accumulator, WGMMA, promotion and store path", "observation": "The exact source differs materially and the sketch contains undefined types, dimensions, helper calls, scale indices, and partial/tail behavior; it cannot serve as reproducible code.", "path": "artifacts/kernels/deepgemm/full/sm90_fp8_gemm_1d1d.cuh", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f, byte-verified"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "Nc=128 was selected because four WGMMA operations cover 128 K elements, bound relative error near 0.1%, make smaller intervals too costly, and make Nc=256 unacceptably inaccurate.", "claim_id": "technique-fine-grained-quantization-original-c007", "claim_type": "performance", "covers": ["u007", "u008", "u009", "u010", "u011"], "evidence": [{"locator": "sections 3.3.2 and 3.5.2", "observation": "The source supports Nc=128 and equivalence to four WGMMAs in the described configuration, but provides none of the page's 0.1%, 'too much overhead', or 'unacceptable Nc=256' thresholds.", "revision": "DeepSeek-V3 Technical Report v2", "url": "https://arxiv.org/abs/2412.19437v2"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "SM100 still performs the same CUDA-core promotion after every K block.", "check": "Search the pinned SM100 kernel for final_accum and trace scale addresses into the UMMA wrapper.", "outcome": "No final_accum promotion exists; the scales are operands of block-scaled UMMA."}, "claim": "DeepGEMM's pinned SM100 FP8 path uses native tcgen05 block scaling with packed UE8M0 factors instead of the SM90 final_accum promotion loop.", "claim_id": "technique-fine-grained-quantization-original-c008", "claim_type": "mechanism", "covers": ["u012"], "evidence": [{"locator": "block-scaled descriptor, TMA/UTCCP scale path and mma_t::fma", "observation": "The file selects float_ue8m0_t block scaling and TMEM scale operands and has no SM90 final_accum loop.", "path": "artifacts/kernels/deepgemm/full/sm100_fp8_gemm_1d1d.cuh", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f, byte-verified"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "The UE8M0 pseudo-structure correctly decodes every byte as 2^(e-127), including all 256 encodings.", "claim_id": "technique-fine-grained-quantization-original-c009", "claim_type": "code", "covers": ["u013"], "evidence": [{"locator": "ue8m0 alternate floating-point format", "observation": "Encoding 0xff is NaN, so the displayed exponent formula incorrectly treats it as a finite 2^128 value. PTX also specifies packed ue8m0x2 register use rather than this invented descriptor API.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#alternate-floating-point-data-formats"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The displayed tcgen05.mma PTX is valid native block-scale grammar with UE8M0 factors read directly from shared memory.", "claim_id": "technique-fine-grained-quantization-original-c010", "claim_type": "code", "covers": ["u014"], "evidence": [{"locator": "floating-point block-scale syntax", "observation": "Valid grammar requires a block-scaled kind such as mxf8f6f4, the .block_scale qualifier, an instruction descriptor, two TMEM scale addresses, and enable-input-d. The displayed f8f6f4 form instead supplies nonexistent SMEM scale operands and a misplaced scale_d/mask sequence.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The four-value packing makes each UE8M0 scale 32 bits.", "check": "Distinguish element width from container width.", "outcome": "Four byte-sized values can occupy one 32-bit container without changing element width."}, "claim": "UE8M0 and E4M3 scale elements occupy 8 bits and FP32 scale elements occupy 32 bits.", "claim_id": "technique-fine-grained-quantization-original-c011", "claim_type": "semantics", "covers": ["u015"], "evidence": [{"locator": "ue8m0 and ue4m3 format definitions", "observation": "Both alternate scale formats are byte-sized; FP32 is a 32-bit floating type.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#alternate-floating-point-data-formats"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "UE8M0 has the finite range 2^-127 through 2^128 and E4M3 scale values are accurately summarized as about 0 through 448 with 240 distinct values.", "claim_id": "technique-fine-grained-quantization-original-c012", "claim_type": "semantics", "covers": ["u016", "u017"], "evidence": [{"locator": "ue8m0 and ue4m3 definitions", "observation": "UE8M0 reserves 0xff for NaN, invalidating the claimed finite 2^128 endpoint. PTX's unsigned UE4M3 scale encoding is not the signed E4M3 payload/range summarized by the table.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#alternate-floating-point-data-formats"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "All low-precision formats use one universal block size.", "check": "Compare the recipe-specific group definitions.", "outcome": "The primary sources explicitly define different sizes and geometries."}, "claim": "The compared recipes use 32-value MXFP4 groups, 16-value NVFP4 1D groups, and 128-element DeepGEMM SM90 activation groups.", "claim_id": "technique-fine-grained-quantization-original-c013", "claim_type": "semantics", "covers": ["u018"], "evidence": [{"locator": "NVFP4 quantization and comparison with MXFP4", "observation": "The documentation distinguishes NVFP4 block 16 from MXFP4 block 32.", "revision": "Transformer Engine 2.13", "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"}, {"locator": "section 3.3.2", "observation": "DeepSeek activation scaling is one scale per 128 channels for each token.", "revision": "DeepSeek-V3 Technical Report v2", "url": "https://arxiv.org/abs/2412.19437v2"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "E4M3-scaled NVFP4 requires software decode on Blackwell, and FP32 scaling is categorically the best-precision option while UE8M0 is categorically coarser.", "claim_id": "technique-fine-grained-quantization-original-c014", "claim_type": "compatibility", "covers": ["u019", "u020"], "evidence": [{"locator": "mxf4nvf4 valid combinations", "observation": "Blackwell tcgen05 natively accepts the block16 UE4M3 NVFP4 combination. Element format alone does not establish a universal error ranking across different grouping and quantizer recipes.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "NVFP4 uses only one scale or groups 32 payloads like MXFP4.", "check": "Inspect both levels and the 1D group size in the versioned recipe.", "outcome": "The source defines two levels and 16-value local groups."}, "claim": "NVFP4's 1D recipe reconstructs an E2M1 payload with a block-of-16 E4M3 scale and one per-tensor FP32 scale.", "claim_id": "technique-fine-grained-quantization-original-c015", "claim_type": "code", "covers": ["u021", "u022"], "evidence": [{"locator": "E2M1 format, two-level scaling and 1D grouping", "observation": "The recipe is exactly payload times local block scale times global scale; the displayed function explicitly decodes only the selected low nibble.", "revision": "Transformer Engine 2.13", "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The three prescriptive use rules are generally validated: DeepGEMM geometry for 671B+ training, NVFP4 as highest precision versus MXFP4 native acceleration, and Nc=128 promotion for Hopper FP8 inference.", "claim_id": "technique-fine-grained-quantization-original-c016", "claim_type": "performance", "covers": ["u023", "u024", "u025"], "evidence": [{"locator": "model and FP8 training sections", "observation": "This validates one model/training design, not a general size threshold or inference rule.", "revision": "DeepSeek-V3 Technical Report v2", "url": "https://arxiv.org/abs/2412.19437v2"}, {"locator": "block-scale valid combinations", "observation": "Both MXFP4 UE8M0 and NVFP4-compatible UE4M3 forms have native block-scaled instruction support; the ISA makes no highest-precision recommendation.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The caveats establish universal accuracy/performance orderings for scale format, block size and Nc, and say Blackwell native block scaling works only with UE8M0.", "claim_id": "technique-fine-grained-quantization-original-c017", "claim_type": "performance", "covers": ["u026", "u027", "u028", "u029"], "evidence": [{"locator": "NVFP4/MXFP4 comparison", "observation": "Finer grouping and fractional scales provide more representational freedom and metadata, but do not prove strictly better precision for every tensor or implementation.", "revision": "Transformer Engine 2.13", "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"}, {"locator": "mxf4nvf4 block16 UE4M3 row", "observation": "Native UE4M3 block scaling directly contradicts the UE8M0-only claim; the ISA supplies no Nc performance/accuracy curve.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "refuted"}], "commands": ["inspect byte-verified DeepGEMM SM90/SM100 FP8 sources", "check exact PTX ISA 9.0 block-scale grammar and valid combinations"], "id": "technique-fine-grained-quantization", "page_verdict": "refuted", "path": "wiki/techniques/fine-grained-quantization.md", "residual_risk": "No GPU experiment was run. Unsupported universal accuracy and performance comparisons are refuted rather than replaced with estimates.", "scope": "All 29 immutable baseline units and frontmatter compatibility claims were checked against DeepSeek-V3 v2, pinned DeepGEMM 891d57b, Transformer Engine 2.13, and PTX ISA 9.0."} +{"claims": [{"claim": "Kernel fusion always eliminates intermediate global-memory round trips and is critical because MoE/attention pipelines generally use five to seven launches.", "claim_id": "technique-kernel-fusion-original-c001", "claim_type": "performance", "covers": ["u001"], "evidence": [{"locator": "Track A reference operation", "observation": "The operation includes routing and two grouped GEMMs but defines observable semantics, not a launch count or guarantee that every intermediate remains on chip.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}, {"locator": "fusion operation graph types", "observation": "Fusion describes in-kernel expression composition; whether a specific unfused baseline would materialize an intermediate and whether the fused schedule avoids all traffic are separate implementation facts.", "path": "/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/operations.hpp", "revision": "CUTLASS v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The displayed CUDA kernel is a valid fused dual-GEMM implementation using two independent 256-column TMEM allocations.", "claim_id": "technique-kernel-fusion-original-c002", "claim_type": "code", "covers": ["u002"], "evidence": [{"locator": "TMEM allocation, address and capacity rules", "observation": "TMEM allocation is collective, returns a shared result, must obey legal column counts and lifecycle rules, and the SM has 512 columns total. The sketch's tmem_alloc/tmem_load/tcgen05_mma calls are invented and omit completion/free semantics.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"}, {"locator": "dual-GEMM correctness reference", "observation": "The operation requires two scaled matrix products and SwiGLU but does not prescribe two TMEM allocations or this instruction schedule.", "revision": "gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c", "url": "https://github.com/gpu-mode/reference-kernels/blob/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm/reference.py"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "vLLM uses the listed seven-kernel graph, SGLang the listed five-kernel graph, and an ideal one-to-two-launch implementation saves exactly 21.9% activation traffic.", "claim_id": "technique-kernel-fusion-original-c003", "claim_type": "performance", "covers": ["u003", "u004", "u005"], "evidence": [{"locator": "operation definition and traces", "observation": "The primary benchmark defines routing and expert computations but provides no framework launch counts, ideal launch topology, or 21.9% traffic result.", "revision": "FlashInfer Track A benchmark definition checked 2026-08-08", "url": "https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048"}, {"locator": "MoE reference", "observation": "The reference uses one logical W13 projection split for SwiGLU and does not support the page's gate/up launch decomposition or cross-framework counts.", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6", "url": "https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/trace/templates/moe.py"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "SM100 has only 256 TMEM columns total, which directly limits the number of fused accumulators.", "claim_id": "technique-kernel-fusion-original-c004", "claim_type": "compatibility", "covers": ["u006"], "evidence": [{"locator": "TMEM organization", "observation": "Each SM exposes 512 columns of 128 lanes, not 256 total. Accumulator feasibility also depends on mapping, datatype, shapes and simultaneous lifetimes.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Fusion always lowers pressure because it deletes intermediate buffers.", "check": "Distinguish global buffers from per-thread registers and per-block shared state used by the fused expression.", "outcome": "Deleting a global tensor does not remove the extra values that must remain live inside the fused kernel."}, "claim": "Fusing a complex epilogue can increase register/resource pressure and reduce occupancy, so the resource impact must be measured.", "claim_id": "technique-kernel-fusion-original-c005", "claim_type": "performance", "covers": ["u007"], "evidence": [{"locator": "resource-dependent occupancy", "observation": "Registers and shared memory per block constrain resident blocks; adding live epilogue state can therefore affect occupancy, without guaranteeing that it does.", "revision": "CUDA 13.0.2 Programming Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "A fusion opportunity requires a dependency graph that is DAG-compatible with CTA scope.", "claim_id": "technique-kernel-fusion-original-c006", "claim_type": "mechanism", "covers": ["u008"], "evidence": [{"locator": "routing and local-expert reference", "observation": "The logical operation includes cross-token/expert selection and accumulation; an implementation may use multiple CTAs, multiple launches, atomics, clusters or workspace. 'DAG-compatible with CTA scope' is neither defined nor a necessary universal fusion criterion.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "One relative link escapes its directory or resolves to an unrelated page.", "check": "Resolve from wiki/techniques and inspect target IDs.", "outcome": "Both targets match their labels."}, "claim": "The fused-MoE and epilogue-fusion related links resolve to the intended pages.", "claim_id": "technique-kernel-fusion-original-c007", "claim_type": "other", "covers": ["u009", "u010"], "evidence": [{"command": "test -f wiki/kernels/fused-moe.md && test -f wiki/techniques/epilogue-fusion.md", "environment": "baseline worktree", "negative_control": "wiki/kernels/not-fused.md does not exist.", "reference": "Resolve relative links and inspect frontmatter IDs.", "result": "Both files exist with IDs kernel-fused-moe and technique-epilogue-fusion."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["inspect pinned FlashInfer Track A reference and GPU Mode gated dual-GEMM reference", "resolve related links with a broken-target control"], "id": "technique-kernel-fusion", "page_verdict": "refuted", "path": "wiki/techniques/kernel-fusion.md", "residual_risk": "No GPU profile was run; launch-count, traffic and speed claims lacking a reproducible primary record are refuted rather than estimated.", "scope": "All ten immutable baseline units and frontmatter TMEM claim were checked against PTX ISA 9.0 and pinned primary implementations/contracts."} +{"claims": [{"claim": "Persistent kernels always launch exactly one CTA per SM, and CLC replaces their software scheduler with a query that returns new work or lets a CTA cancel itself.", "claim_id": "technique-persistent-kernels-original-c001", "claim_type": "mechanism", "covers": ["u001"], "evidence": [{"locator": "Dynamic Scheduler rules and request flow", "observation": "CLC launches the problem-sized grid; each worker processes its initial blockIdx then cancels a not-yet-launched ClcID and processes the returned coordinate. try_cancel does not cancel the caller, and persistent grid size is policy/resource dependent.", "revision": "CUTLASS 4.5.0 e406c186", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "CLC creates new tiles or assigns every tile through a separate hardware queue.", "check": "Trace the initial and canceled-ID paths against the launched grid.", "outcome": "Every work ID already exists in the grid and is handled either by ordinary launch or cancellation, exactly as scoped."}, "claim": "CLC can let a running compute-capability-10.0 worker acquire a not-yet-started grid coordinate after its initial blockIdx work, enabling hardware-assisted dynamic work redistribution.", "claim_id": "technique-persistent-kernels-original-c002", "claim_type": "mechanism", "covers": ["u002"], "evidence": [{"locator": "CLC dynamic scheduler rules", "observation": "The documented worker handles blockIdx first and can cancel/receive an unlaunched ClcID for subsequent work.", "revision": "CUTLASS 4.5.0 e406c186", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The displayed persistent_gemm_clc loop correctly queries work first, retries when none exists, and then uses try_cancel for self-termination.", "claim_id": "technique-persistent-kernels-original-c003", "claim_type": "code", "covers": ["u003"], "evidence": [{"locator": "try_cancel/query semantics and restrictions", "observation": "try_cancel is itself the asynchronous request to cancel another not-yet-started block/cluster; query decodes that response. After a thread observes failure, another request from that thread is undefined. The block reverses the protocol and invents self-cancellation.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The text control-flow block completely describes a CLC loop as repeated cancel/wait/query, returning immediately on failure.", "claim_id": "technique-persistent-kernels-original-c004", "claim_type": "code", "covers": ["u004", "u005"], "evidence": [{"locator": "16-byte response, mbarrier transaction, proxy fences, query and cluster rules", "observation": "The sequence omits initial blockIdx work, expect-tx/publication/fences, the exact failure restriction, and cluster-rank coordinate derivation. It is not a complete protocol even though cancel-before-query is directionally correct.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The loop skips or duplicates tiles when total_tiles is not divisible by gridDim.x.", "check": "Enumerate nondivisible sizes and compare the sorted union with the exact index range.", "outcome": "The final iteration counts differ by at most one and the union remains exact."}, "claim": "The Hopper-style static-stride example maps CTA i to indices i + iteration*gridDim.x and converts a flat index into row-major tile coordinates.", "claim_id": "technique-persistent-kernels-original-c005", "claim_type": "code", "covers": ["u006", "u007"], "evidence": [{"command": "conda run -n base python enumerate static-stride assignments for multiple grid/tile counts and test uniqueness/coverage", "environment": "Conda base, host CPU", "negative_control": "Using stride total_ctas-1 produced duplicate assignments.", "reference": "Each valid flat tile index should occur exactly once and map by quotient/remainder.", "result": "All indices [0,total_tiles) were covered once with tile_m=idx//tiles_n and tile_n=idx%tiles_n."}], "oracle": "host-experiment", "verdict": "verified"}, {"claim": "The comparison table accurately states that CLC assigns arbitrary tiles, automatically balances variable tile costs, minimizes ordinary tail waves, chains problems, uses try_cancel for self-termination, and applies a hardware swizzled raster.", "claim_id": "technique-persistent-kernels-original-c006", "claim_type": "mechanism", "covers": ["u008", "u009", "u010", "u011", "u012", "u013"], "evidence": [{"locator": "dynamic scheduler and software swizzle rules", "observation": "CLC redistributes existing unlaunched ClcIDs to address uneven SM availability. CUTLASS applies swizzle/raster as a software coordinate transform; CLC neither chains distinct problems nor guarantees ordinary tail elimination.", "revision": "CUTLASS 4.5.0 e406c186", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}, {"locator": "request/query result", "observation": "try_cancel targets another not-yet-started grid entity; failure means no returned work, not caller cancellation.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The scheduler is a generic software queue with no CLC instruction path.", "check": "Follow the linked scheduler/pipeline sources to the arch CLC wrappers.", "outcome": "The fetch pipeline submits and decodes clusterlaunchcontrol operations for the SM100 scheduler."}, "claim": "CUTLASS 4.5.0 provides PersistentTileSchedulerSm100 and PipelineCLCFetchAsync integration for CLC-backed scheduling.", "claim_id": "technique-persistent-kernels-original-c007", "claim_type": "compatibility", "covers": ["u014"], "evidence": [{"locator": "CUTLASS implementation links and scheduler pipeline", "observation": "The official tagged documentation identifies PersistentTileSchedulerSm100 and PipelineCLCFetchAsync and describes their producer/consumer roles.", "revision": "CUTLASS 4.5.0 e406c186", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The displayed PersistentTileSchedulerSm100 class is a valid simplified CUTLASS 4.5.0 API with clc_init, clc_query_tile and clc_try_cancel helpers and a thread-0 CTA broadcast.", "claim_id": "technique-persistent-kernels-original-c008", "claim_type": "code", "covers": ["u015"], "evidence": [{"locator": "PersistentTileSchedulerSm100 interface and CLC pipeline calls", "observation": "The displayed helper API and init policy do not exist. Real code uses WorkTileInfo, get_current_work/fetch_next_work/advance_to_next_work, PipelineCLCFetchAsync state, cluster participation and software swizzle/raster transforms.", "revision": "CUTLASS 4.5.0 e406c186", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The tutorial isolates a 940-to-1476 TFLOP/s, 57-percent improvement caused by CLC persistence, specifically through eliminated tail, reduced launches and CLC-controlled L2 swizzle.", "claim_id": "technique-persistent-kernels-original-c009", "claim_type": "code", "covers": ["u016", "u017", "u018", "u019", "u020", "u021"], "evidence": [{"locator": "v6 persistent kernel and static tile assignment", "observation": "v6 uses persistent static scheduling, not CLC.", "path": "/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v6.cu", "revision": "gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573"}, {"locator": "progression table and supported takeaways", "observation": "939.61 is v3 pipelining and 1475.93 is v6 after warp specialization, 2-SM MMA and persistence; the article explicitly says CLC and threadblock swizzling were not added, so neither isolation nor causal breakdown follows.", "path": "sources/blogs/tcgen05-tutorial.md", "revision": "pinned source card verified against author article/commit"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "Persistent kernels are most beneficial only above two-to-three tiles per SM, CLC chains grouped problems without launch gaps, and CLC naturally handles arbitrary variable-cost tiles such as triangular masks.", "claim_id": "technique-persistent-kernels-original-c010", "claim_type": "performance", "covers": ["u022", "u023", "u024"], "evidence": [{"locator": "motivation and dynamic scheduler limits", "observation": "The documented motivation is uneven SM availability and redistribution of existing ClcIDs. It supplies no 2-3x threshold, multi-problem chaining, or cost-aware selection guarantee.", "revision": "CUTLASS 4.5.0 e406c186", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Hopper supports the same PTX operation under a different name.", "check": "Inspect architecture requirements for the CLC API/PTX instruction.", "outcome": "The feature is gated to CC 10.0, so Hopper requires a software schedule or another mechanism."}, "claim": "CLC is a compute-capability-10.0 Blackwell feature unavailable on Hopper, where persistent work scheduling is software-managed.", "claim_id": "technique-persistent-kernels-original-c011", "claim_type": "compatibility", "covers": ["u025"], "evidence": [{"locator": "availability", "observation": "CLC is documented for compute capability 10.0; Hopper compute capability 9.0 does not expose it.", "revision": "CUDA 13.0.2 Programming Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#cluster-launch-control"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "A failed CLC request represents a race and the same thread must retry until another CTA generates work.", "claim_id": "technique-persistent-kernels-original-c012", "claim_type": "mechanism", "covers": ["u026"], "evidence": [{"locator": "failed-request restriction", "observation": "After a thread observes a failed request, a subsequent request from that thread is undefined; other CTAs do not generate new ClcIDs.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "CLC synthesizes extra K-splits or duplicate tiles to fill all SMs.", "check": "Trace whether try_cancel returns anything outside the launched grid.", "outcome": "It can return only an existing unlaunched ClcID; decomposition is a separate scheduler policy."}, "claim": "When independent tile count is below available SM count, a scheduler cannot create enough extra parallel tiles to fill the idle SMs, so added dynamic scheduling may not repay its complexity.", "claim_id": "technique-persistent-kernels-original-c013", "claim_type": "performance", "covers": ["u027"], "evidence": [{"locator": "problem-sized grid and ClcID conservation", "observation": "CLC only redistributes existing grid IDs, so fewer independent tiles than SMs remain fewer work items.", "revision": "CUTLASS 4.5.0 e406c186", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The derived skeleton is silently treated as upstream or the page command omits artifacts.", "check": "Inspect first-line labels and captured retrieval output.", "outcome": "Mode labeling is explicit and artifact content is included."}, "claim": "The artifact directory contains declared upstream patch/source material, a labeled derived skeleton, matching recorded file hashes, and is retrievable with the shown get_page command.", "claim_id": "technique-persistent-kernels-original-c014", "claim_type": "code", "covers": ["u028", "u029", "u030"], "evidence": [{"command": "sha256sum all artifact files; inspect provenance/header modes; byte-compare dense_gemm_persistent_prefetch.py to CUTLASS PR 2881 merge; run Conda-base get_page", "environment": "current worktree/network, 2026-08-09", "negative_control": "A nonexistent page ID exits nonzero.", "reference": "Hashes/modes and retrieval should agree with manifests and page.", "result": "All declared hashes match; dense_gemm file byte-matches merge 3f4c086d09bd1dc55defb955862f333893bbb28b; variant is labeled derived/not upstream; retrieval exited zero."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["archived CLC/PTX and CUTLASS 4.5.0 documentation inspection", "exact author tutorial source comparison", "Conda-base static-stride enumeration", "artifact hash/provenance and retrieval controls"], "id": "technique-persistent-kernels", "page_verdict": "refuted", "path": "wiki/techniques/persistent-kernels.md", "residual_risk": "No CC10.0 GPU is available locally. No CLC performance claim is accepted without a controlled implementation-specific benchmark.", "scope": "All 30 baseline units for body hash 7d1d16d68cfae0d699d1b73b4d10ac71a12c878966081788134245f3861fc283 were reviewed, including all code, table, performance-attribution, and artifact claims."} +{"claims": [{"claim": "FlashAttention-4 introduced a two-query-tile single-CTA ping-pong schedule that ensures its softmax warpgroup never waits for MMA.", "claim_id": "technique-ping-pong-scheduling-original-c001", "claim_type": "semantics", "covers": ["u001"], "evidence": [{"locator": "section 3.1.2", "observation": "The paper says FA4 follows a ping-pong schedule similar to FA3. It maximizes overlap between one tile's MMA and the other's softmax but does not claim a no-stall guarantee.", "revision": "FlashAttention-4 paper v1, 2026-03-05", "url": "https://arxiv.org/html/2603.05451v1"}, {"locator": "warp IDs, q_stage, TMEM offsets, and pipeline construction", "observation": "The pinned implementation defaults to q_stage=2, assigns separate four-warp softmax groups plus correction/MMA roles, and uses explicit multi-stage barriers; it does not expose the page's one-barrier loop.", "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", "url": "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The displayed CUDA kernel is a valid two-tile FA4 ping-pong implementation with correct TMEM allocation, warpgroup roles, asynchronous MMA completion, and synchronization.", "claim_id": "technique-ping-pong-scheduling-original-c002", "claim_type": "code", "covers": ["u002"], "evidence": [{"locator": "flash_fwd_sm100.py setup, kernel roles, and pipeline loops", "observation": "The real CuTe DSL kernel uses one MMA warp, two four-warp softmax groups, a correction warpgroup, load/epilogue roles, one TMEM allocation with disjoint offsets, and multiple producer/consumer barriers. The snippet's helpers are undefined and its generic arrive/wait is not tied to tcgen05 completion or safe buffer ownership.", "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", "url": "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py"}, {"locator": "tcgen05 allocation, MMA, commit, fence, and mbarrier contracts", "observation": "TMEM lifecycle is collective and asynchronous MMA requires its documented completion/order protocol; an unrelated mbarrier arrive/wait pair does not establish completion.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The doubled tensor-core and unchanged exponential-throughput comparison is only qualitative marketing.", "check": "Compare the paper's per-SM throughput figures for B200 and Hopper.", "outcome": "The paper gives 8192 versus 4096 BF16 MMA ops/clock/SM and 16 exponential ops/clock/SM on both B200 and Hopper."}, "claim": "The FA4 paper models B200 BF16 tensor-core throughput as double Hopper while B200 exponential-unit throughput remains the same as Hopper.", "claim_id": "technique-ping-pong-scheduling-original-c003", "claim_type": "performance", "covers": ["u003"], "evidence": [{"locator": "section 2.2 Shifting bottlenecks", "observation": "The paper lists 8192 versus 4096 BF16 MMA operations per clock per SM and 16 exponential operations per clock per SM on B200, the same exponential rate as Hopper.", "revision": "FlashAttention-4 paper v1, 2026-03-05", "url": "https://arxiv.org/html/2603.05451v1"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "A single-tile schedule alternately leaves SFU and tensor cores idle, whereas ping-pong keeps both units 100% busy.", "claim_id": "technique-ping-pong-scheduling-original-c004", "claim_type": "performance", "covers": ["u004", "u005"], "evidence": [{"locator": "sections 3.1.1-3.1.2", "observation": "The paper gives a simplified roofline and says the schedule maximizes overlap; it explicitly notes the analysis omits resources and never claims universal exclusive idleness or measured 100% utilization of both units.", "revision": "FlashAttention-4 paper v1, 2026-03-05", "url": "https://arxiv.org/html/2603.05451v1"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The ping-pong pattern itself accounts for FA4's 1605 TFLOPS/s and 71% BF16 result.", "claim_id": "technique-ping-pong-scheduling-original-c005", "claim_type": "performance", "covers": ["u006"], "evidence": [{"locator": "Performance and Key Techniques", "observation": "The blog reports up to 1605 TFLOPS/s and 71% for the complete FA4 forward design, which also uses partial software exponentials, conditional rescaling, TMEM partitioning, register tuning, and other pipeline choices; it provides no ping-pong-only attribution for the peak.", "revision": "Tri Dao FA4 blog, 2026", "url": "https://tridao.me/blog/2026/flash4/"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Ping-pong should be used for compute-bound or SFU-heavy Blackwell attention kernels and is not useful on Hopper.", "claim_id": "technique-ping-pong-scheduling-original-c006", "claim_type": "compatibility", "covers": ["u007", "u008", "u009"], "evidence": [{"locator": "section 3.1.2", "observation": "The paper explicitly says FA4 follows a schedule similar to FA3, a Hopper kernel, refuting 'not useful on Hopper.' It motivates FA4 from a concrete balance but does not prescribe the technique solely from compute-bound or SFU-heavy labels.", "revision": "FlashAttention-4 paper v1, 2026-03-05", "url": "https://arxiv.org/html/2603.05451v1"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The local full bundle is a verbatim full reference implementation of FA4 ping-pong scheduling and the derived variant faithfully demonstrates the overlap; the displayed query retrieves it.", "claim_id": "technique-ping-pong-scheduling-original-c007", "claim_type": "code", "covers": ["u010", "u011", "u012"], "evidence": [{"locator": "origin_url, upstream_repo, source_pr_id, files", "observation": "The full file comes from NVIDIA CUTLASS PR 2466 at merge 0e026982 and is a Blackwell FMHA backward kernel, not Dao-AILab FA4 forward source or a full implementation of its two-output-tile schedule.", "path": "artifacts/kernels/ping-pong-scheduling/full/PROVENANCE.yaml", "revision": "local provenance"}, {"locator": "entire function", "observation": "The skeleton calls wait_mma, then completes softmax A, and only afterward issues MMA B; its visible single-thread order does not implement the claimed overlap and all helpers are undefined.", "path": "artifacts/kernels/ping-pong-scheduling/variants/01-two-tile-ping-pong-skeleton.cu", "revision": "derived local variant"}], "oracle": "local-primary-source", "verdict": "refuted"}], "commands": ["Inspect FA4 paper v1 pipeline and hardware model", "Inspect pinned Dao-AILab SM100 forward q_stage, warp roles, TMEM offsets, and barriers", "Inspect local full/variant provenance and visible execution order", "Run the repository page query through conda base Python"], "id": "technique-ping-pong-scheduling", "page_verdict": "refuted", "path": "wiki/techniques/ping-pong-scheduling.md", "residual_risk": "The pinned FA4 implementation was source-inspected but not compiled or profiled because no CUDA toolkit or GPU is available. The paper/blog performance remains author-reported.", "scope": "All 12 baseline body units, artifact references, code claims, hardware motivation, performance attribution, and architecture prescription were checked against direct paper, source, ISA, and provenance evidence."} +{"claims": [{"claim": "A three-to-five-stage SMEM pipeline hides global-memory latency entirely and is critical for high utilization on every Hopper/Blackwell kernel.", "claim_id": "technique-pipeline-stages-original-c001", "claim_type": "performance", "covers": ["u001"], "evidence": [{"locator": "asynchronous-copy pipeline tradeoffs", "observation": "Overlap and benefit depend on work balance, issue/completion, occupancy, resource use and workload; documentation provides no universal stage range or complete-latency-hiding guarantee.", "revision": "CUDA 13.0.2 Best Practices Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-best-practices-guide/index.html#asynchronous-copy-from-global-memory-to-shared-memory"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The tutorial progression is 695 TFLOP/s without pipelining, 940 with three stages, and 1476 immediately after adding warp specialization.", "claim_id": "technique-pipeline-stages-original-c002", "claim_type": "code", "covers": ["u002", "u003"], "evidence": [{"locator": "source-reported progression", "observation": "v2b=695.43, v3 pipelining=939.61, v4 warp specialization=1208.83, v5 2-SM=1302.29 and v6 static persistence=1475.93; the last line skips two changes.", "path": "sources/blogs/tcgen05-tutorial.md", "revision": "Gau Nernst article/source commit 3b90ac9b3f624bdf1f6f78d02dcd533675d36573"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "The delta is a portable B200 theorem or independently reproduced isolated latency percentage.", "check": "Inspect benchmark scope and adjacent source changes.", "outcome": "It is only an author-reported result for the disclosed shape/environment; the verified claim is scoped accordingly."}, "claim": "For the author's exact 4096-cubed B200 setup, the pipelining step changes 695.43 to 939.61 TFLOP/s, approximately a 35.1-percent increase, and the author attributes that step to TMA/MMA overlap.", "claim_id": "technique-pipeline-stages-original-c003", "claim_type": "performance", "covers": ["u002", "u004"], "evidence": [{"locator": "v2b/v3 progression and pipelining description", "observation": "The source reports those values for consecutive variants, with v3 adding pipelining.", "path": "sources/blogs/tcgen05-tutorial.md", "revision": "Gau Nernst article/source commit 3b90ac9b3f624bdf1f6f78d02dcd533675d36573"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Multiple copies are used only sequentially and therefore do not form a pipeline.", "check": "Trace producer and consumer loops around the same modulo-indexed stages.", "outcome": "Separate roles advance with stage-specific full/empty waits, enabling simultaneous ownership of different stages."}, "claim": "A circular SMEM pipeline allocates multiple operand stages and cycles producer/consumer ownership through them.", "claim_id": "technique-pipeline-stages-original-c004", "claim_type": "mechanism", "covers": ["u005"], "evidence": [{"locator": "state[numStages], stage modulo loops and full/empty barriers", "observation": "The merged implementation allocates per-stage storage and cycles stage indices through explicit producer/consumer handoffs.", "path": "artifacts/prs/flashinfer/PR-2387/key-files/include/flashinfer/mamba/selective_state_update.cuh", "revision": "flashinfer merge 18804cd51734cccf807356d017733bc757677f15"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "The displayed pipelined_gemm is a complete safe three-stage TMA/tcgen05 implementation.", "claim_id": "technique-pipeline-stages-original-c005", "claim_type": "code", "covers": ["u006"], "evidence": [{"locator": "tcgen05 MMA completion and operand lifetimes", "observation": "The consumer releases SMEM immediately after an opaque asynchronous MMA issue without attaching/waiting for completion; syncwarp does not complete tcgen05.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"}, {"locator": "initialization, phase/tokens and transaction completion", "observation": "The code omits async-proxy publication fencing, reusable barrier phase/token state, initial empty-stage seeding, and a correct tail protocol.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "One architecture-wide stage count is always optimal regardless of tile or K depth.", "check": "Compare source-selected counts across paths with different geometry.", "outcome": "The exact merged kernel uses different policies, rejecting the constant-stage alternative."}, "claim": "The optimal stage count depends on per-stage work/latency and resource constraints rather than being an architectural constant.", "claim_id": "technique-pipeline-stages-original-c006", "claim_type": "performance", "covers": ["u007"], "evidence": [{"locator": "numStages dispatch choices", "observation": "Different code paths select three stages or cap a geometry-derived total at four, demonstrating implementation dependence.", "path": "artifacts/prs/flashinfer/PR-2387/key-files/include/flashinfer/mamba/selective_state_update.cuh", "revision": "flashinfer merge 18804cd51734cccf807356d017733bc757677f15"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "Two stages give partial hiding, three fully hide latency for most GEMMs, four-to-five give full hiding with margin, and more than five are universally excessive.", "claim_id": "technique-pipeline-stages-original-c007", "claim_type": "performance", "covers": ["u008", "u009", "u010", "u011"], "evidence": [{"locator": "pipeline construction and stage policies across tutorial variants", "observation": "Stage counts derive from tile, shared-memory capacity and schedule; no tagged source defines the table's universal hiding classifications or thresholds.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "144 KiB is the complete dynamic shared-memory allocation or 228 KiB is guaranteed per CTA.", "check": "Account for barriers/padding/other storage and distinguish per-SM from opt-in per-block limits.", "outcome": "Only unpadded operand payload and per-SM capacity are verified; the original caveat that other storage remains is retained in scope."}, "claim": "For CC10.0's 228-KiB per-SM shared-memory budget, binary16 A[128,64] and B[64,256] payloads occupy 16 and 32 KiB; three unpadded stages are 144 KiB and five are 240 KiB.", "claim_id": "technique-pipeline-stages-original-c008", "claim_type": "compatibility", "covers": ["u012", "u013", "u014", "u015", "u016", "u017"], "evidence": [{"command": "conda run -n base python exact byte/KiB and percentage arithmetic; compare 5-stage total with 228 KiB", "environment": "Conda base, host CPU", "negative_control": "Treating KB as decimal or binary16 as 4 bytes fails the listed values.", "reference": "binary16 is 2 bytes and CC10.0 guide reports 228 KiB/SM", "result": "16+32=48 KiB/stage; 3=144 KiB=63.16% of 228; 5=240 KiB>228."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "mbarrier is just a spelling of CTA-wide bar.sync and all threads must participate identically.", "check": "Compare participant/arrival semantics and transaction completion with bar.sync.", "outcome": "mbarrier permits explicitly counted asymmetric arrivals and waits, matching the scoped claim."}, "claim": "mbarrier supports phase-based asymmetric producer/consumer synchronization without requiring every CTA thread to execute each wait.", "claim_id": "technique-pipeline-stages-original-c009", "claim_type": "mechanism", "covers": ["u018"], "evidence": [{"locator": "mbarrier object, phases, arrivals and waits", "observation": "mbarrier is addressable state with expected arrivals/transactions and phase-aware waits usable by designated producer/consumer participants.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The displayed PTX block is an accurate complete phase protocol, and autonomous TMA completion removes the producer from the critical path.", "claim_id": "technique-pipeline-stages-original-c010", "claim_type": "code", "covers": ["u019", "u020"], "evidence": [{"locator": "tensor-copy grammar and mbarrier complete_tx", "observation": "The shown copy form omits required `.tile` and full operand/context details; transaction expectation and proxy ordering are not represented.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"}, {"locator": "phase completion", "observation": "An arrival does not generically 'flip phase' at issue; phase changes after expected arrivals/transactions complete. Autonomous completion reduces explicit producer work but does not prove the producer is absent from the performance critical path.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The Modular series establishes that a five-stage input pipeline alone reaches 85 percent of SOTA, was selected to hide an approximately 400-cycle B200 HBM latency, and has the displayed exact steady-state timeline.", "claim_id": "technique-pipeline-stages-original-c011", "claim_type": "code", "covers": ["u021", "u022"], "evidence": [{"locator": "Kernel 6 pipeline, warp specialization, write-out double buffering and final result", "observation": "The article uses five stages but attributes the eventual 85% endpoint to a sequence including 2-SM, pipelining/warp specialization and output write-back double buffering; it does not provide the page's 400-cycle premise or exact fabricated stage timeline.", "revision": "Modular Part 3, fetched 2026-08-09", "url": "https://www.modular.com/blog/matrix-multiplication-on-nvidias-blackwell-part-3-the-optimizations-behind-85-of-sota-performance"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Pipelining is never harmful for all compute- and memory-bound GEMMs, and it is almost always paired with warp specialization.", "claim_id": "technique-pipeline-stages-original-c012", "claim_type": "performance", "covers": ["u023", "u025"], "evidence": [{"locator": "pipeline/performance considerations", "observation": "Extra stages consume shared memory and coordination state and can reduce occupancy; profitability and whether warp roles are specialized depend on the concrete schedule.", "revision": "CUDA 13.0.2 Best Practices Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-best-practices-guide/index.html#asynchronous-copy-from-global-memory-to-shared-memory"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Every KV tile must be loaded synchronously immediately before its MMA.", "check": "Inspect the staged attention schedule and producer/consumer roles.", "outcome": "Independent next-tile movement overlaps current work under explicit synchronization."}, "claim": "Attention mainloops can pipeline KV tile production with computation when their dependencies and buffer ownership permit it.", "claim_id": "technique-pipeline-stages-original-c013", "claim_type": "mechanism", "covers": ["u024"], "evidence": [{"locator": "asynchronous pipeline and warp specialization", "observation": "The paper pipelines asynchronous data movement and tensor-core/non-matmul work across attention tiles.", "revision": "FlashAttention-3 paper", "url": "https://arxiv.org/abs/2407.08608"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Stages alias the same SMEM and consume no additional payload.", "check": "Require producer and consumer simultaneous ownership of different stages.", "outcome": "Safe simultaneous ownership needs disjoint buffers, producing the stated linear payload term."}, "claim": "Unpadded operand SMEM payload grows linearly with stage count and therefore competes with tile size and other shared storage.", "claim_id": "technique-pipeline-stages-original-c014", "claim_type": "mechanism", "covers": ["u026"], "evidence": [{"command": "conda run -n base python sweep stage_count*per_stage_bytes", "environment": "Conda base, host CPU", "negative_control": "Adding a fixed per-kernel overhead makes total allocation affine, confirming the page limits the linear statement to stage payload.", "reference": "Each independent stage contains one fixed-size operand payload.", "result": "Payload is exactly linear for stage counts 1-16."}], "oracle": "host-experiment", "verdict": "verified"}, {"claim": "Barrier initialization overhead is negligible and placing initialization in a syncthreads block is a sufficient pre-TMA publication protocol.", "claim_id": "technique-pipeline-stages-original-c015", "claim_type": "performance", "covers": ["u027"], "evidence": [{"locator": "initialization and visibility", "observation": "The ISA defines initialization plus required memory/proxy ordering and participant conditions but no negligible-cost guarantee; __syncthreads alone is not a generic substitute for every required proxy fence.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-init"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Modulo indexing alone makes phases and short-loop boundaries automatic.", "check": "Run the state machine for num_k_tiles below stage count and for first stage reuse.", "outcome": "Some stages are never produced in the short case and reuse changes expected phase, requiring the guards stated."}, "claim": "Incorrect phase/reuse tracking can deadlock or corrupt a reusable pipeline, and short loops require guarded prologue/tail handling because steady state may never be reached.", "claim_id": "technique-pipeline-stages-original-c016", "claim_type": "mechanism", "covers": ["u028", "u029"], "evidence": [{"locator": "phase and wait semantics", "observation": "Waiting on the wrong phase may never complete or may observe the wrong reuse; bounded loops must avoid waiting for stages that were never produced.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"}], "oracle": "web-primary", "verdict": "verified"}], "commands": ["author progression and exact-revision source inspection", "archived PTX mbarrier/TMA inspection", "Conda-base payload arithmetic and state controls", "Modular primary article comparison"], "id": "technique-pipeline-stages", "page_verdict": "refuted", "path": "wiki/techniques/pipeline-stages.md", "residual_risk": "No CUDA compiler or SM100 GPU is available locally. Stage profitability and exposed latency remain target measurements.", "scope": "All 29 baseline units for body hash d990987f96b0c765ab7753798f189a2125cbab0619c1671b450e305a4c628966 were reviewed, including code, ordering, table, arithmetic and performance-attribution claims."} +{"claims": [{"claim": "SM occupancy is inversely proportional to registers per thread, higher occupancy is the governing goal for memory-bound kernels, register controls force a budget, and Blackwell TMEM frees about 100 registers per thread.", "claim_id": "technique-register-budgeting-original-c001", "claim_type": "semantics", "covers": ["u001"], "evidence": [{"locator": "sections 8.2.3.1 and 10.38", "observation": "Occupancy is calculated from the compiled function, block size, and shared-memory use; register thresholds cause discrete resident-block transitions. Launch bounds guide compiler heuristics rather than imposing an exact register count.", "revision": "CUDA C++ Programming Guide 13.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html"}, {"locator": "epilogue TMEM load path around lines 455-463", "observation": "The pinned SM100 implementation loads completed TMEM values into registers for its epilogue, so TMEM does not eliminate all accumulator-related register demand or establish a fixed saving.", "path": "artifacts/kernels/deepgemm/full/sm100_fp8_gemm_1d1d.cuh", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f, byte-verified", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The displayed launch-bounds example forces 32 registers per thread, yields about four 256-thread blocks per SM, and spills if necessary.", "claim_id": "technique-register-budgeting-original-c002", "claim_type": "code", "covers": ["u002"], "evidence": [{"locator": "Launch Bounds, lines 8688-8713", "observation": "The two-argument qualifier supplies a maximum thread count and desired minimum resident blocks. The compiler derives an architecture-dependent upper limit L and may reduce or increase register use; it does not set 32 registers or guarantee residency when another resource limits the launch.", "revision": "CUDA C++ Programming Guide 13.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Lowering the register budget has the fixed qualitative consequences shown: spills are bad, recomputation is neutral, and reduced unrolling is bad for compute-bound kernels.", "claim_id": "technique-register-budgeting-original-c003", "claim_type": "performance", "covers": ["u003", "u004", "u005", "u006"], "evidence": [{"locator": "Launch Bounds, lines 8700-8706", "observation": "The documented consequence of forcing usage below the compiler's initial choice is usually more local-memory use and/or more instructions; it does not assign workload-independent good, neutral, or bad labels to individual transformations.", "revision": "CUDA C++ Programming Guide 13.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html"}, {"locator": "Attempts 9-12", "observation": "In the author's kernel, wider loads, extra accumulator chains, higher unrolling, and software pipelining all regressed or had no effect, showing that individual compiler/code-shape changes need a concrete comparison.", "revision": "retrieved 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Memory-bound kernels can hide spill traffic behind memory latency, so aggressive register budgeting often wins.", "claim_id": "technique-register-budgeting-original-c004", "claim_type": "performance", "covers": ["u007"], "evidence": [{"locator": "--resource-usage and launch-bounds descriptions", "observation": "Spill stores and loads are stack/local-memory traffic, and the compiler guide describes register limits as a tradeoff rather than a generally winning policy.", "revision": "CUDA Compiler Driver 13.0.2", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html"}, {"locator": "Attempts 10-12", "observation": "This memory-bound B200 kernel regressed when register pressure caused spills, and reducing maxrregcount from 80 to 64 had no effect because the natural allocation was already below the cap.", "revision": "retrieved 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "GPU Mode rank 1 used 32 registers at exactly 18.5 microseconds and rank 3 used 45 registers at about 20 microseconds.", "claim_id": "technique-register-budgeting-original-c005", "claim_type": "performance", "covers": ["u008", "u009"], "evidence": [{"locator": "What the Top Solutions Did Differently", "observation": "The author says all three inspected solutions clustered around an 18.5 microsecond geometric mean and separately reports caps of 32 for rank 1 and 45 for rank 3. The source does not assign the table's exact 18.5 and approximately 20 microsecond values to those two rows or publish controlled runs.", "revision": "retrieved 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The difference between the 32- and 45-register solutions proves that occupancy dominates NVFP4 GEMV performance.", "claim_id": "technique-register-budgeting-original-c006", "claim_type": "performance", "covers": ["u010"], "evidence": [{"locator": "What the Top Solutions Did Differently", "observation": "The compared solutions also differed in PTX decode, cache policy, load width, exact-K specialization, and reuse. No one-variable register-cap or occupancy ablation is provided.", "revision": "retrieved 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}, {"locator": "Attempt 11", "observation": "The author's own cap change from 80 to 64 produced no effect, directly demonstrating that a nominal cap does not identify the compiled register count or isolate occupancy.", "revision": "retrieved 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Occupancy-first register budgeting should be used for memory-bound, inner-loop-pressure, and sub-byte kernels, but not for compute-bound GEMM or kernels whose spills serialize.", "claim_id": "technique-register-budgeting-original-c007", "claim_type": "performance", "covers": ["u011", "u012", "u013", "u014", "u015"], "evidence": [{"locator": "sections 8.2.3.1 and 10.38", "observation": "CUDA exposes occupancy prediction and compiler constraints but does not prescribe these categories; optimal launch bounds require detailed per-kernel analysis and generally differ across architecture revisions.", "revision": "CUDA C++ Programming Guide 13.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html"}, {"locator": "Attempts 10-12 and Takeaways", "observation": "Even within one memory-bound sub-byte kernel, only measurements of natural register use, spills, and variants discriminated useful from ineffective or harmful changes.", "revision": "retrieved 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}], "oracle": "web-primary", "verdict": "refuted"}], "commands": ["Inspect CUDA 13.0 launch-bounds, occupancy, maxrregcount, spill, and resource-usage contracts", "Compare the author-reported NVFP4 observations and failed attempts", "Inspect the pinned SM100 DeepGEMM TMEM epilogue path"], "id": "technique-register-budgeting", "page_verdict": "refuted", "path": "wiki/techniques/register-budgeting.md", "residual_risk": "No B200 compiler or profiler run was available. Exact cap effects and performance remain compiler-, kernel-, launch-, and workload-specific.", "scope": "All 15 baseline body units plus the frontmatter TMEM/register claim were checked against archived NVIDIA documentation, the cited author's report, and a pinned SM100 implementation."} +{"claims": [{"claim": "FlashAttention-4 replaces hardware ex2 with a four-FMA degree-4 software exponential over a symmetric reduced interval.", "claim_id": "technique-software-exp-original-c001", "claim_type": "mechanism", "covers": ["u001"], "evidence": [{"locator": "Faster exponential section", "observation": "The authors run a selected software degree-3 path alongside hardware MUFU.EX2, with floor reduction and a fractional interval [0,1); they do not replace all hardware exponentials.", "revision": "FlashAttention-4 blog, retrieved 2026-08-09", "url": "https://tridao.me/blog/2026/flash4/"}, {"locator": "flash_attn/cute/utils.py POLY_EX2 and ex2_emulation_2; softmax.py apply_exp2_convert", "observation": "The pinned implementation uses the degree-3 coefficient set by default and chooses between hardware exp2 and software emulation by fragment.", "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", "url": "https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The Hopper/Blackwell resource table gives exact architecture counts and implies that any exp-per-MMA kernel is SFU-bound and software exp has eight times hardware throughput.", "claim_id": "technique-software-exp-original-c002", "claim_type": "performance", "covers": ["u002", "u003", "u004", "u005", "u006", "u007", "u008"], "evidence": [{"locator": "Feeds and Speeds", "observation": "For one M=N=D=128 B200 model, the authors use 8192 BF16 tensor-core ops/cycle, 16 exponential ops/cycle, and derive 1024 cycles for each of tensor cores and exponentials. They do not publish the page's architecture table, universal bottleneck rule, or 8x software throughput.", "revision": "FlashAttention-4 blog, retrieved 2026-08-09", "url": "https://tridao.me/blog/2026/flash4/"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The displayed software_exp2 function implements FlashAttention-4's Cody-Waite reduction and minimax polynomial with about 22 bits of accuracy.", "claim_id": "technique-software-exp-original-c003", "claim_type": "code", "covers": ["u009", "u010"], "evidence": [{"locator": "Faster exponential section", "observation": "The published method uses floor reduction to f in [0,1), degree-3 Sollya coefficients 1, 0.6951, 0.2276, 0.0771, and exponent-bit reconstruction. The snippet uses nearest-integer reduction, a symmetric interval, degree-4 Taylor coefficients, and ldexpf.", "revision": "FlashAttention-4 blog, retrieved 2026-08-09", "url": "https://tridao.me/blog/2026/flash4/"}, {"locator": "flash_attn/cute/utils.py lines 32-58 and 756-793", "observation": "The exact pinned code confirms the published degree-3 path, clamps below -127, assumes inputs no larger than 127, and uses packed operations plus bit reconstruction.", "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", "url": "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The displayed softmax CUDA function is a valid FlashAttention-4 implementation and its FMA throughput is fully available beside tensor-core work.", "claim_id": "technique-software-exp-original-c004", "claim_type": "code", "covers": ["u011", "u012"], "evidence": [{"locator": "forward pipeline and faster exponential sections", "observation": "FA4 has two synchronized softmax warpgroups, a correction warpgroup, TMEM staging, conditional rescaling, and a tunable hybrid exponential split. The standalone warp function implements none of that schedule and the source only says FMA units would otherwise be underutilized, not fully available.", "revision": "FlashAttention-4 blog, retrieved 2026-08-09", "url": "https://tridao.me/blog/2026/flash4/"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The displayed PTX is a valid SFU-free FlashAttention-4 Horner implementation.", "claim_id": "technique-software-exp-original-c005", "claim_type": "code", "covers": ["u013", "u014"], "evidence": [{"locator": "floating-point instructions and ex2.approx", "observation": "The fragment uses undeclared %ni and %scale registers and reconstructs with ex2.approx, so it is neither self-contained nor SFU-free. Its coefficients and reduction also differ from FA4.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"locator": "flash_attn/cute/utils.py combine_int_frac_ex2 and ex2_emulation_2", "observation": "The exact software path uses integer exponent-field reconstruction rather than a hardware ex2 instruction.", "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", "url": "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The page's degree-4 Taylor polynomial has about 22 mantissa bits of accuracy on the stated reduced interval.", "claim_id": "technique-software-exp-original-c006", "claim_type": "semantics", "covers": ["u015"], "evidence": [{"command": "Dense-grid relative-error evaluation of the displayed polynomial on 1,000,002 points in [-0.5,0.5]", "environment": "Conda base Python 3 host arithmetic", "negative_control": "The same evaluator applied to the published rounded degree-3 coefficients on [0,1) produced approximately 8.763e-5, confirming the procedure distinguishes coefficient sets.", "reference": "Compare the displayed polynomial to math.pow(2,x) and report maximum sampled relative error.", "result": "Maximum sampled relative error was 5.5684e-5 at -0.5, approximately 14.13 bits, not 22 bits. A grid is not a rigorous global bound."}], "oracle": "host-experiment", "verdict": "refuted"}, {"claim": "FlashAttention-4 measured a 1.1-1.3x cuDNN speedup from software exponential alone.", "claim_id": "technique-software-exp-original-c007", "claim_type": "performance", "covers": ["u020"], "evidence": [{"locator": "Attention Benchmarks", "observation": "The 1.1-1.3x forward comparison is for the complete FA4 implementation, which combines pipelining, hybrid exp, conditional rescaling, TMEM staging, scheduling, and other choices; no exp-only ablation is reported there.", "revision": "FlashAttention-4 blog, retrieved 2026-08-09", "url": "https://tridao.me/blog/2026/flash4/"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Software exp should be used for any transcendental-throughput-limited kernel and is not recommended on Hopper because four FMAs are unjustified there.", "claim_id": "technique-software-exp-original-c008", "claim_type": "compatibility", "covers": ["u021", "u022"], "evidence": [{"locator": "FA4 README and implementation scope", "observation": "The current FA4 package says it is optimized for Hopper and Blackwell, while the cited software-exp rationale and tuning are for a concrete Blackwell forward pipeline. Neither source defines a universal transcendental substitution rule or the stated Hopper prohibition.", "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", "url": "https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Four dependent FMAs take about 16 cycles versus about 20 cycles for ex2.approx, and the win follows from 128 FMA versus 16 SFU units.", "claim_id": "technique-software-exp-original-c009", "claim_type": "performance", "covers": ["u023"], "evidence": [{"locator": "Feeds and Speeds and Faster exponential sections", "observation": "The source provides a tile-level 16-exp-ops/cycle model and motivates a hybrid path, but does not provide the page's instruction latency, FMA-unit count, or derived 8x comparison. The actual default polynomial is degree 3 plus range reduction and reconstruction.", "revision": "FlashAttention-4 blog, retrieved 2026-08-09", "url": "https://tridao.me/blog/2026/flash4/"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "Natural exponential needs different polynomial coefficients rather than input conversion.", "check": "Apply the identity e^x = 2^(x log2(e)).", "outcome": "The identity exactly supports the stated input conversion, independent of the approximation chosen for exp2."}, "claim": "A base-2 exponential implementation can approximate e^x by first multiplying x by log2(e).", "claim_id": "technique-software-exp-original-c010", "claim_type": "semantics", "covers": ["u024"], "evidence": [{"locator": "softmax scale_log2 construction and exp2 calls", "observation": "FA4 converts the natural softmax scale to base-2 before evaluating exp2.", "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", "url": "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/softmax.py"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "Attention inputs are always non-positive, so software reconstruction only needs to handle underflow and not special values or an explicit input contract.", "claim_id": "technique-software-exp-original-c011", "claim_type": "semantics", "covers": ["u025"], "evidence": [{"locator": "flash_attn/cute/utils.py ex2_emulation and ex2_emulation_2", "observation": "The exact helpers state x<=127 assumptions, clamp values below -127, and rely on particular rounding modes. The wider softmax implementation separately handles all-masked -infinity and dtype scaling cases.", "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", "url": "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Softmax approximation errors cancel under normalization, BF16 conversion makes the polynomial automatically sufficient, and degree 6 gives near-ULP accuracy across the full float range.", "claim_id": "technique-software-exp-original-c012", "claim_type": "semantics", "covers": ["u016", "u017", "u018", "u019"], "evidence": [{"locator": "Faster exponential section and forward pipeline", "observation": "The authors specify a degree-3 relative-error approximation and BF16 conversion inside one validated attention design; they make none of the page's error-cancellation, universal sufficiency, or degree-6 full-range near-ULP claims.", "revision": "FlashAttention-4 blog, retrieved 2026-08-09", "url": "https://tridao.me/blog/2026/flash4/"}, {"locator": "flash_attn/cute/utils.py POLY_EX2 and softmax implementation", "observation": "The pinned implementation contains degree 0 through 5 coefficient sets, not the claimed degree-6 construction, and validates approximation only as part of its concrete dtype/softmax pipeline.", "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", "url": "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py"}], "oracle": "web-primary", "verdict": "refuted"}], "commands": ["Inspect FlashAttention-4 first-party blog and pinned source revision a369df7", "Inspect archived PTX ISA 9.0 instruction declarations", "Evaluate both displayed and published coefficient sets on dense host grids"], "id": "technique-software-exp", "page_verdict": "refuted", "path": "wiki/techniques/software-exp.md", "residual_risk": "No B200 SASS inspection or performance run was available. The dense numerical experiment is discriminating evidence, not a formal error bound.", "scope": "All 25 baseline body units and the page's architecture, code, accuracy, compatibility, and performance implications were checked."} +{"claims": [{"claim": "Swizzling makes consecutive row/column accesses hit different 32-byte banks, eliminates conflicts, and 128-byte swizzling is mandatory for all SM100 TMA/tcgen05 inputs with unswizzled GEMM limited to 46% throughput.", "claim_id": "technique-swizzling-original-c001", "claim_type": "semantics", "covers": ["u001", "u002", "u003"], "evidence": [{"locator": "Shared Memory and bank-conflict descriptions", "observation": "Shared memory has 32 banks with successive 32-bit words mapped to successive banks; conflicts depend on the concrete warp addresses, with broadcasts and same-word accesses as exceptions. Banks are not 32 bytes wide.", "revision": "Nsight Compute Profiling Guide 13.3", "url": "https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html"}, {"locator": "tcgen05 shared-memory descriptor swizzling modes", "observation": "PTX defines legal no-swizzle, 32B, 64B, and 128B descriptors, so 128B is not a universal correctness requirement.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The tutorial demonstrates a 2.7x improvement from swizzling alone, from 255 to 695 TFLOP/s.", "claim_id": "technique-swizzling-original-c002", "claim_type": "code", "covers": ["u004", "u005"], "evidence": [{"locator": "128-byte global load and swizzling, benchmark table and remarks", "observation": "The exact 3D endpoints are 252.81 and 695.43 TFLOP/s, but the revision also changes the inner TMA tile from 16B to 128B. The author explicitly says the speedup may be wider/faster TMA rather than removed bank conflicts and had no ncu access to discriminate it.", "revision": "tcgen05 for dummies, pinned source 3b90ac9b", "url": "https://gau-nernst.github.io/tcgen05/"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The manual row-XOR function and bank-rotation diagram define the general 128B TMA/tcgen05 swizzle and guarantee conflict-free column access.", "claim_id": "technique-swizzling-original-c003", "claim_type": "code", "covers": ["u006", "u007", "u008", "u009"], "evidence": [{"locator": "include/cute/swizzle.hpp Swizzle", "observation": "CuTe defines swizzling over address bits, with BBits as mask width, MBase as invariant low bits, and SShift as mask distance. Mapping depends on byte address and alignment, not an arbitrary row parameter; Swizzle<3,4,3> XORs address bits 7:9 into 4:6.", "revision": "CUTLASS 4.5.0 e406c186d2cae5782a846f7280af282ca4fecec2", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/swizzle.hpp"}, {"locator": "CUtensorMapSwizzle enum", "observation": "Driver documentation specifies 16B chunks within a 128B span for ordinary 128B swizzle; it does not define the page's one-bank-per-row rotation or conflict-free guarantee for every consumer access.", "revision": "CUDA Driver API 13.0.97", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "TMA writes a canonical layout and the MMA descriptor can choose swizzle independently.", "check": "Compare tensor-map swizzle semantics with tcgen05 descriptor layout selection.", "outcome": "Both describe the destination/consumer layout; disagreement changes physical-to-logical addressing."}, "claim": "A TMA tensor map encodes a shared-memory swizzle and a TMA-produced tile must be consumed with a descriptor/layout that describes the same mapping.", "claim_id": "technique-swizzling-original-c004", "claim_type": "mechanism", "covers": ["u010"], "evidence": [{"locator": "cuTensorMapEncodeTiled and CUtensorMapSwizzle", "observation": "The driver API defines swizzle as the shared-memory bank-shuffling pattern in the tensor map.", "revision": "CUDA Driver API 13.0.97", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"}, {"locator": "shared-memory descriptor", "observation": "PTX encodes the tcgen05 shared-memory layout and mode in the consumer descriptor.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The displayed cuTensorMapEncodeTiled call is a reproducible 2D FP16 128B-swizzle encoder.", "claim_id": "technique-swizzling-original-c005", "claim_type": "code", "covers": ["u011"], "evidence": [{"locator": "cuTensorMapEncodeTiled signature and parameter constraints", "observation": "The snippet leaves all dimension/stride arrays and pointer alignment undefined, describes a two-entry global stride although rank-2 uses rank-1 byte strides, and does not check CUresult or the 128B inner-box constraint.", "revision": "CUDA Driver API 13.0.97", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The no/32B/64B/128B swizzle modes have the page's universal non-MMA, narrow-type, medium-tile, and standard BF16/FP16 use cases.", "claim_id": "technique-swizzling-original-c006", "claim_type": "compatibility", "covers": ["u012", "u013", "u014", "u015", "u016"], "evidence": [{"locator": "CUtensorMapSwizzle enum and constraints", "observation": "The API defines chunk/span mappings and datatype/interleave/box restrictions, not the page's qualitative use-case table.", "revision": "CUDA Driver API 13.0.97", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"}, {"locator": "tcgen05 valid combinations of type, major-ness, and swizzling", "observation": "Legality is kind-, type-, shape-, major-mode-, and target-specific; no single 128B default covers all BF16/FP16 MMA operands.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The CuTe snippet correctly defines Swizzle, and Swizzle<3,4,3> with the shown layout is a generally valid 128B TMA/tcgen05 tile.", "claim_id": "technique-swizzling-original-c007", "claim_type": "code", "covers": ["u017", "u018"], "evidence": [{"locator": "include/cute/swizzle.hpp lines 42-82", "observation": "The page reverses the parameter meanings: BBits is mask width and MBase is low invariant bits. CUTLASS also distinguishes position-independent swizzle layouts from the hardware's position-dependent swizzle pointers, so alignment/composition cannot be omitted.", "revision": "CUTLASS 4.5.0 e406c186d2cae5782a846f7280af282ca4fecec2", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/swizzle.hpp"}, {"locator": "include/cute/pointer_swizzle.hpp lines 39-56", "observation": "The source explicitly distinguishes position-independent swizzle_layout from position-dependent hardware swizzle pointers.", "revision": "CUTLASS 4.5.0 e406c186d2cae5782a846f7280af282ca4fecec2", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/pointer_swizzle.hpp"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "nvprof or the displayed ncu metric proves swizzling eliminated conflicts when the result is exactly zero.", "claim_id": "technique-swizzling-original-c008", "claim_type": "code", "covers": ["u019", "u020"], "evidence": [{"locator": "Metrics Examples and Shared Memory table", "observation": "Current Nsight Compute documents l1tex__data_bank_conflicts_pipe_lsu.sum and derived/shared wavefront analysis, not the page's qualified metric as a portable contract. Zero conflicts is neither guaranteed nor sufficient to validate a layout or output.", "revision": "Nsight Compute 13.3", "url": "https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html"}, {"locator": "tutorial swizzle remarks", "observation": "The tutorial author lacked ncu access and explicitly could not determine whether bank conflicts caused the measured change.", "revision": "tcgen05 for dummies, retrieved 2026-08-09", "url": "https://gau-nernst.github.io/tcgen05/"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Every Blackwell tensor-core and Hopper WGMMA kernel requires 128B swizzling, and swizzling prevents serialization for non-MMA column accesses.", "claim_id": "technique-swizzling-original-c009", "claim_type": "compatibility", "covers": ["u021", "u022", "u023"], "evidence": [{"locator": "tcgen05 descriptor modes and valid combinations", "observation": "No-swizzle, 32B, 64B, and 128B layouts are legal for tcgen05 under their respective constraints.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}, {"locator": "shared-memory conflict definition", "observation": "Conflict behavior follows the exact warp addresses, including same-word and broadcast exceptions; a swizzle can improve one access and conflict for another.", "revision": "Nsight Compute Profiling Guide 13.3", "url": "https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "A mismatched descriptor always fails silently, CuTe avoids indexing errors, and element width determines the optimal swizzle while TMEM removes the issue.", "claim_id": "technique-swizzling-original-c010", "claim_type": "semantics", "covers": ["u024", "u025", "u026"], "evidence": [{"locator": "cuTensorMapEncodeTiled return and constraints", "observation": "Invalid tensor-map combinations can return an encoding error, while a legal but logically mismatched producer/consumer layout requires output validation. The API does not select swizzle from element width alone.", "revision": "CUDA Driver API 13.0.97", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"}, {"locator": "CuTe swizzle and pointer implementations", "observation": "CuTe expresses mappings but cannot guarantee the programmer composed the matching tensor map, alignment, layout, and consumer descriptor.", "revision": "CUTLASS 4.5.0 e406c186d2cae5782a846f7280af282ca4fecec2", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/swizzle.hpp"}], "oracle": "web-primary", "verdict": "refuted"}], "commands": ["Inspect archived CUDA Driver/PTX swizzle enumerations and constraints", "Inspect CUTLASS 4.5.0 Swizzle and pointer semantics", "Compare the tutorial's exact v1/v2 changes and causal caveat", "Check current Nsight Compute shared-memory metrics"], "id": "technique-swizzling", "page_verdict": "refuted", "path": "wiki/techniques/swizzling.md", "residual_risk": "No GPU profiler run was available. Exact bank-conflict behavior and performance remain properties of a concrete instruction access pattern and layout pair.", "scope": "All 26 baseline body units plus mandatory/correctness frontmatter implications, code, profiler guidance, and performance attribution were checked."} +{"claims": [{"claim": "Tile order can affect locality and utilization, and Blackwell CLC itself supplies dynamic policies including a swizzled raster while Hopper uses only launch-time static mappings.", "claim_id": "technique-tile-scheduling-original-c001", "claim_type": "mechanism", "covers": ["u001"], "evidence": [{"locator": "Dynamic Scheduler and software swizzle rules", "observation": "CLC redistributes already-launched ClcIDs; CUTLASS applies raster order and swizzle as software coordinate transforms, not as a CLC hardware scheduling policy. The broad locality/utilization premise is plausible, but the architecture contrast is false.", "revision": "CUTLASS 4.5.0 e406c186", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The displayed linear-raster code maps flat indices row-major, and consecutive tiles have categorically poor L2 locality because B column zero is evicted before tile four.", "claim_id": "technique-tile-scheduling-original-c002", "claim_type": "code", "covers": ["u002", "u003"], "evidence": [{"command": "conda run -n base python enumerate quotient/remainder coordinates and reuse sets", "environment": "Conda base, host CPU; no GPU", "negative_control": "Treating every distinct B column as an eviction event reproduces the page's unsupported conclusion and ignores simultaneous residency.", "reference": "The code's tile_m=i//tiles_n and tile_n=i%tiles_n mapping, with cache behavior left unmodeled", "result": "The mapping is row-major, but adjacent tiles reuse the same A tile row; eviction of B column zero cannot be inferred from four coordinate assignments without cache capacity, residency and interleaving information."}], "oracle": "host-experiment", "verdict": "refuted"}, {"claim": "The displayed blocked-raster function is a valid all-shape swizzle that iterates column-major within groups and maximizes both A and B L2 reuse.", "claim_id": "technique-tile-scheduling-original-c003", "claim_type": "code", "covers": ["u004", "u005"], "evidence": [{"command": "conda run -n base python enumerate the displayed swizzled_raster for 4x4 and nondivisible 3x5 grids", "environment": "Conda base, host CPU", "negative_control": "Testing only grids whose N dimension is divisible by swizzle_size hides the final-group bug.", "reference": "Every input index must map bijectively into [0,tiles_m) x [0,tiles_n), and comments must match quotient/remainder order", "result": "4x4 happens to cover exactly, but 3x5 with swizzle 2 returns (0,5), an out-of-range coordinate, and the within-group order is row-major rather than the claimed column-major traversal."}], "oracle": "host-experiment", "verdict": "refuted"}, {"challenge": {"alternative": "A nondivisible final iteration skips or duplicates a tile.", "check": "Enumerate remainders one and total_ctas-1 and count every flat index.", "outcome": "Only CTA loop lengths differ; the assignment remains exact."}, "claim": "For positive total_ctas, the static-stride snippet assigns CTA i the flat indices i + iteration*total_ctas and converts each valid index to row-major coordinates.", "claim_id": "technique-tile-scheduling-original-c004", "claim_type": "code", "covers": ["u006", "u007"], "evidence": [{"command": "conda run -n base python enumerate static-stride assignments for divisible and nondivisible tile counts", "environment": "Conda base, host CPU", "negative_control": "Using total_ctas-1 as the stride produced duplicate assignments.", "reference": "The sorted union should equal range(total_tiles) exactly once", "result": "For positive grid sizes, all valid flat indices were covered once and quotient/remainder produced the stated row-major coordinates."}], "oracle": "host-experiment", "verdict": "verified"}, {"claim": "CLC exposes an initialization-time hardware schedule policy with LinearRaster, SwizzledRaster, ColumnFirst and Hilbert choices through the displayed enum and helper functions.", "claim_id": "technique-tile-scheduling-original-c005", "claim_type": "code", "covers": ["u008", "u009"], "evidence": [{"locator": "clusterlaunchcontrol.try_cancel and query instruction syntax", "observation": "The raw interface submits a cancel request to a response buffer/mbarrier and queries the response; it has no policy operand or scheduler initialization API. The enum and helper calls are invented.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "The named categories exist only for SM90 and have no SM100 selection route.", "check": "Inspect the arch::Sm100 selector specializations and included implementation headers.", "outcome": "The tag-to-class mappings explicitly instantiate SM100 persistent, Stream-K and group implementations."}, "claim": "CUTLASS 4.5.0 provides persistent, Stream-K and grouped scheduler abstractions for SM100.", "claim_id": "technique-tile-scheduling-original-c006", "claim_type": "compatibility", "covers": ["u010"], "evidence": [{"locator": "scheduler tags and SM100 TileSchedulerSelector specializations", "observation": "PersistentScheduler, DynamicPersistentScheduler, StreamKScheduler and GroupScheduler tags map to SM100 scheduler implementations.", "path": "/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/tile_scheduler.hpp", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "All SM100 persistent schedulers inherit PersistentTileSchedulerSm100, the public types are StreamKSchedulerSm100 and GroupedTileSchedulerSm100, default scheduling is a hardware swizzled raster, and the displayed GemmUniversal signature is valid CUTLASS 4.5.0.", "claim_id": "technique-tile-scheduling-original-c007", "claim_type": "code", "covers": ["u011"], "evidence": [{"locator": "public scheduler tags and SM100 selector mappings", "observation": "The public tags are DynamicPersistentScheduler, StreamKScheduler and GroupScheduler; implementation classes are detail types. The group wrapper reuses an SM90 static grouped scheduler rather than inheriting the SM100 CLC class, and modern GemmUniversal composition does not use the shown legacy argument list.", "path": "/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/tile_scheduler.hpp", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The displayed B200 cache calculation correctly counts the first linear wave's A/B working set and proves that swizzling halves L2 misses for the stated GEMM.", "claim_id": "technique-tile-scheduling-original-c008", "claim_type": "code", "covers": ["u012", "u013"], "evidence": [{"command": "conda run -n base python enumerate the first 142 row-major coordinates in a 64x32 tile grid and count unique M and N coordinates", "environment": "Conda base, host CPU; no cache simulator or GPU", "negative_control": "Multiplying two unrelated 4.4 ratios recreates the page's approximately-20 value but does not count coordinate reuse.", "reference": "Unique operand panels follow directly from the coordinate set; miss rates require measurement or a validated cache model", "result": "The first 142 indices span 5 M rows and all 32 N columns, not approximately 20 A rows or 5 B groups. The arithmetic therefore cannot establish the claimed 2x miss reduction."}], "oracle": "host-experiment", "verdict": "refuted"}, {"challenge": {"alternative": "Tail effect is defined only by wall-clock imbalance inside tiles, not a partial final wave.", "check": "Separate equal-duration discrete-wave occupancy from variable-duration load imbalance.", "outcome": "The scoped definition describes the standard discrete final-wave case and does not claim to cover every imbalance source."}, "claim": "A tail wave is the final scheduling wave when fewer independent work units remain than available workers.", "claim_id": "technique-tile-scheduling-original-c009", "claim_type": "mechanism", "covers": ["u014"], "evidence": [{"command": "conda run -n base python compute quotient/remainder wave occupancy for total work and worker counts", "environment": "Conda base, host CPU", "negative_control": "Always treating remainder zero as an empty last wave incorrectly reports zero utilization for exactly divisible work.", "reference": "For equal-duration independent tiles and one resident CTA per worker, total=q*workers+r", "result": "When 0= M return before __syncthreads, so a partial final block can deadlock or exhibit undefined synchronization behavior; several helper types/functions are also undefined.", "revision": "CUDA C++ Programming Guide 13.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The optimization table isolates coalescing, FP4 intrinsics, vector loads plus cache policy, ILP plus maxrregcount, and an 8.6-microsecond speed-of-light into a single 2000-to-22.4-microsecond progression.", "claim_id": "technique-vectorized-loads-original-c009", "claim_type": "performance", "covers": ["u018", "u019", "u020", "u021", "u022", "u023"], "evidence": [{"locator": "Reported Progression", "observation": "Every author stage has more changes than the wiki's label, and the final 22.392 value is a three-case geometric-mean score. The 8.622 theoretical value belongs only to the first benchmark shape, not to the aggregate endpoint.", "revision": "retrieved 2026-08-08", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}, {"locator": "Pinned task benchmark table", "observation": "The official model gives 8.622, 17.275, and 4.317 microseconds for the three different shapes and ranks submissions by their geometric mean.", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6", "url": "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "Wide loads and cache policies are essential for all GEMV/memory-bound or sub-byte kernels, and single-token decode GEMV is always bandwidth-bound.", "claim_id": "technique-vectorized-loads-original-c010", "claim_type": "performance", "covers": ["u024", "u025", "u026"], "evidence": [{"locator": "Attempt 9 and Takeaways", "observation": "A wider load regressed this sub-byte GEMV by 16-25%, and the author recommends diagnosing the concrete bottleneck rather than choosing a transformation from a workload label.", "revision": "retrieved 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}, {"locator": "Roofline Analysis", "observation": "The official method determines memory- versus compute-boundedness from measured arithmetic intensity and achieved ceilings, not solely from an algorithm or decode label.", "revision": "Nsight Compute 13.3", "url": "https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html#roofline"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "A misaligned 256-bit PTX load safely falls back to narrower transactions, while no_allocate is necessarily harmful for reused data.", "claim_id": "technique-vectorized-loads-original-c011", "claim_type": "semantics", "covers": ["u027", "u028"], "evidence": [{"locator": "6.4.1.1 Addresses as Operands and 9.7.9.8 ld", "observation": "An address must be naturally aligned to the access size; misalignment is undefined and may mask low bits or fault. L1::no_allocate is an eviction-priority hint whose performance effect must be measured, not a universal harm rule.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#addresses-as-operands"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "A cap that is too low necessarily spills and loses more than occupancy gains, while inline PTX bypasses the compiler register allocator.", "claim_id": "technique-vectorized-loads-original-c012", "claim_type": "semantics", "covers": ["u029", "u030"], "evidence": [{"locator": "--maxrregcount and inline PTX constraints", "observation": "A cap is a requested maximum subject to ABI/compiler constraints and can add spills or instructions, but the performance balance is empirical. Inline-assembly operands are represented by compiler-selected registers through constraints, so the construct does not bypass register allocation.", "revision": "CUDA 13.0.2", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/inline-ptx-assembly/index.html"}], "oracle": "web-primary", "verdict": "refuted"}], "commands": ["Inspect archived PTX 9.0 vector-load, alignment, cache-hint, and mov contracts", "Compare the cited NVFP4 author reports, including negative controls and co-varying stages", "Compare the example ABI and theoretical times with the pinned contest task"], "id": "technique-vectorized-loads", "page_verdict": "refuted", "path": "wiki/techniques/vectorized-loads.md", "residual_risk": "No CUDA toolkit or B200 was available. Author-reported performance was therefore checked for attribution and scope but not independently reproduced.", "scope": "All 30 baseline body units and the frontmatter performance claim were checked against archived NVIDIA documentation, the pinned task, and the cited authors' reports."} +{"claims": [{"claim": "SM100 replaces Hopper's four-warp model with a fixed 16-warp CTA in which one warp issues tcgen05 MMA and all remaining warps are freed for producer/consumer roles.", "claim_id": "technique-warp-specialization-original-c001", "claim_type": "mechanism", "covers": ["u001"], "evidence": [{"locator": "NUM_WARPS, role branches and epilogue", "observation": "The source-reported SM100 warp-specialization example launches four warps: elected lanes from warps 0 and 1 issue TMA/MMA, and all four warps later load/store the output. Sixteen warps is not an architectural replacement rule.", "path": "/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v4.cu", "revision": "gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The canonical SM100 GEMM role assignment is exactly warp 0 for TMA, warp 1 for MMA, and warps 2-15 for epilogue work.", "claim_id": "technique-warp-specialization-original-c002", "claim_type": "mechanism", "covers": ["u002", "u003", "u004", "u005"], "evidence": [{"locator": "thread counts and WarpCategory", "observation": "The generic SM100 kernel assigns warp categories MMA=0, Sched=1, MainloopLoad=2, EpilogueLoad=3, then a configuration-derived number of epilogue warps. It is neither the stated IDs nor a fixed 14-warp epilogue.", "path": "/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/sm100_gemm_tma_warpspecialized.hpp", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "Hopper WGMMA is a warpgroup operation whereas SM100 tcgen05.mma can be issued by one thread, but the page also implies that this issuing thread removes explicit operand movement and makes a whole MMA warp the architectural unit.", "claim_id": "technique-warp-specialization-original-c003", "claim_type": "mechanism", "covers": ["u006"], "evidence": [{"locator": "tcgen05.mma description, operands and thread semantics", "observation": "tcgen05.mma has one-thread issue and TMEM D, but it still consumes explicit A/B SMEM descriptors or TMEM addresses with lifecycle/ordering requirements. PTX defines the issuing thread, not a mandatory dedicated warp role.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "The Hopper/Blackwell table gives universal producer, epilogue, synchronization and register-pressure arrangements, including a fixed 14-warp SM100 epilogue and uniformly low register pressure.", "claim_id": "technique-warp-specialization-original-c004", "claim_type": "mechanism", "covers": ["u007", "u008", "u009", "u010", "u011", "u012"], "evidence": [{"locator": "NumEpilogueThreads, WarpCategory and pipeline construction", "observation": "Role count and pipeline participation are collective/epilogue dependent. SM100 has scheduler, mainloop-load and optional epilogue-load roles in addition to MMA/epilogue, invalidating the fixed comparison. TMEM changes accumulator placement but does not guarantee low GPR pressure for every role.", "path": "/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/sm100_gemm_tma_warpspecialized.hpp", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "The displayed 512-thread CUDA skeleton is a safe implementation of a three-role SM100 GEMM with reusable TMA stages and a correct MMA-to-epilogue handoff.", "claim_id": "technique-warp-specialization-original-c005", "claim_type": "code", "covers": ["u013", "u014"], "evidence": [{"locator": "tcgen05.commit completion semantics", "observation": "tcgen05 MMA is asynchronous. The skeleton releases each SMEM stage after issue using syncwarp and signals accumulator completion with ordinary mbarrier.arrive, never committing the MMA sequence to an mbarrier and waiting for completion.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"}, {"locator": "tcgen05.ld grammar and completion", "observation": "tmem_load is not a CUDA/PTX API and the snippet omits legal TMEM addressing, load instruction shape, wait and output tile offsets.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-load"}], "oracle": "web-primary", "verdict": "refuted"}, {"challenge": {"alternative": "One data-ready barrier is sufficient because TMA and tcgen05 consume SMEM immediately at issue.", "check": "Inspect asynchronous completion and operand-lifetime requirements for both operations.", "outcome": "The producer must not overwrite operands while MMA can still consume them, so a reverse release/completion path is required for reuse."}, "claim": "A reusable async producer/consumer stage needs a full/data-ready handoff after transfer completion and an empty/buffer-free handoff only after the consumer has finished using the stage.", "claim_id": "technique-warp-specialization-original-c006", "claim_type": "mechanism", "covers": ["u015", "u016", "u017"], "evidence": [{"locator": "mbarrier phases, expected transactions, waits and reuse", "observation": "Phase-aware full/empty handoffs protect async transaction completion and prevent stage overwrite before consumer completion.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"}], "oracle": "web-primary", "verdict": "verified"}, {"claim": "The displayed PTX accurately maps a TMA data-ready and tcgen05 buffer-release protocol using only ordinary mbarrier.arrive and one try_wait.parity instruction per handoff.", "claim_id": "technique-warp-specialization-original-c007", "claim_type": "code", "covers": ["u018", "u019"], "evidence": [{"locator": "TMA complete_tx and expected transaction protocol", "observation": "TMA completion decrements expected transaction bytes; a manual generic arrival does not represent the full copy/expect-tx protocol.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"}, {"locator": "commit mbarrier arrival", "observation": "Safe release after MMA requires completion tracking, which the PTX block omits. try_wait also returns a predicate and is not an unconditional blocking wait by itself.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"}], "oracle": "web-primary", "verdict": "refuted"}, {"claim": "CUTLASS 4.5.0 implements the page's exact 1/1/14 partition in a type named CollectiveMma_1SM with WarpCount Shape<1,1,14> and the displayed producer/math/epilogue methods.", "claim_id": "technique-warp-specialization-original-c008", "claim_type": "code", "covers": ["u020", "u021"], "evidence": [{"locator": "SM100 GemmUniversal kernel role dispatch", "observation": "The alleged type/methods/shape are absent. Real kernel dispatch uses WarpCategory with four control roles plus collective-derived epilogue threads; CollectiveMma is a templated collective specialization, not CollectiveMma_1SM.", "path": "/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/sm100_gemm_tma_warpspecialized.hpp", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "Warp specialization is always required for Blackwell GEMMs because the tcgen05 model assumes that scheduling structure.", "claim_id": "technique-warp-specialization-original-c009", "claim_type": "performance", "covers": ["u022"], "evidence": [{"locator": "pre-specialization pipelined tcgen05 kernel", "observation": "The tutorial has a functioning tcgen05 GEMM before its v4 warp-specialization optimization. PTX constrains issue/operands, not a universal software role partition.", "path": "/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v3.cu", "revision": "gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "The two groups are conceptual query tiles but not distinct warp-role groups.", "check": "Trace the explicit warp ID tuples and stage-specific softmax dispatch.", "outcome": "Separate ID sets execute stage 0/1 softmax work, so the scoped implementation claim holds."}, "claim": "The pinned FlashAttention-4 SM100 forward implementation uses a 16-warp partition with two groups of four softmax warps and dedicated MMA/load/correction/epilogue roles.", "claim_id": "technique-warp-specialization-original-c010", "claim_type": "mechanism", "covers": ["u023"], "evidence": [{"locator": "softmax0_warp_ids, softmax1_warp_ids and role IDs", "observation": "The source assigns warps 0-3 and 4-7 to two softmax groups, 8-11 correction, 12 MMA, 13 epilogue, 14 load and 15 initially empty, with configuration-dependent adjustments.", "path": "/tmp/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_fwd_sm100.py", "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f"}], "oracle": "local-primary-source", "verdict": "verified"}, {"claim": "Warp specialization is the cleanest decomposition for every producer-consumer kernel whenever TMA and MMA can overlap.", "claim_id": "technique-warp-specialization-original-c011", "claim_type": "performance", "covers": ["u024"], "evidence": [{"locator": "pipelined non-specialized implementation", "observation": "A valid temporal producer/consumer pipeline can overlap work before static role specialization. Profit depends on loop balance, register allocation, participant counts and workload; no primary source defines a universal cleanest choice.", "path": "/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v3.cu", "revision": "gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "Fourteen epilogue warps are generally underutilized for simple stores and become more beneficial as epilogue complexity increases.", "claim_id": "technique-warp-specialization-original-c012", "claim_type": "performance", "covers": ["u025"], "evidence": [{"locator": "CollectiveEpilogue::ThreadCount-derived role sizing", "observation": "The implementation does not prescribe 14 epilogue warps; the count is collective-dependent. The source supplies no monotonic performance result relating epilogue complexity to a fixed count.", "path": "/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/sm100_gemm_tma_warpspecialized.hpp", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"claim": "A single MMA-control warp prevents overlap of independent MMA streams in one CTA, and 2-SM cooperative mode should therefore be used for larger tiles.", "claim_id": "technique-warp-specialization-original-c013", "claim_type": "mechanism", "covers": ["u026"], "evidence": [{"locator": "elected MMA lane nested issue loops", "observation": "One elected issuing lane submits many asynchronous tcgen05 MMA operations before a commit; one control warp does not imply only one outstanding stream. The later 2-SM variant is a separate source-specific tile choice, not a consequence of that alleged prohibition.", "path": "/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v4.cu", "revision": "gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573"}], "oracle": "local-primary-source", "verdict": "refuted"}, {"challenge": {"alternative": "mbarrier.init alone is immediately visible to all threads and async proxies without synchronization.", "check": "Inspect memory/proxy visibility requirements and the tutorial's release fence plus syncthreads sequence.", "outcome": "Initialization must be ordered before use; the page's conditional synchronization advice is directionally correct, though exact fences depend on scope and proxy."}, "claim": "mbarrier objects must be initialized and made visible before any participant or async mechanism uses or waits on them; CTA synchronization can be part of that publication protocol.", "claim_id": "technique-warp-specialization-original-c014", "claim_type": "semantics", "covers": ["u027"], "evidence": [{"locator": "initialization and visibility requirements", "observation": "Use before initialization or required visibility ordering is invalid; appropriate synchronization/fences must publish the object before participants proceed.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-init"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Matching local duplicate files proves only internal consistency, not the declared upstream verbatim lineage.", "check": "Run the strict verifier against the public PR merge metadata/content and separately compare the derived variant with the verbatim file.", "outcome": "The strict upstream check passes for the verbatim bundle, while the derived skeleton intentionally differs and remains labeled derived."}, "claim": "The full and variants artifact directories declare respectively a verbatim PR-1039 file and a derived teaching skeleton, and their files match the SHA-256 values recorded in provenance.", "claim_id": "technique-warp-specialization-original-c015", "claim_type": "code", "covers": ["u028"], "evidence": [{"command": "sha256sum artifact files; compare the full file with PR-1039 key-file and its new-file diff content; inspect both PROVENANCE.yaml modes; conda run -n base python scripts/verify_verbatim.py --strict", "environment": "Host shell and Conda base, no GPU", "negative_control": "Comparing the derived skeleton with the full file fails, preserving the provenance-mode distinction.", "reference": "Recorded hashes 792ac63e... and 2c2ac9a2..., verbatim/derived labels, and exact upstream merged-PR content", "result": "Both local hashes match their manifests, the full file byte-matches the captured key-file/new-file patch content, and the strict verifier validates the upstream PR merge and verbatim bundle."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "--include-code is ignored and only emits Markdown.", "check": "Search output after the Artifact Bundle heading for provenance and source filenames.", "outcome": "Both full and variant file contents are present after the page body."}, "claim": "scripts/get_page.py can retrieve the warp-specialization page and include its linked artifact bundle.", "claim_id": "technique-warp-specialization-original-c016", "claim_type": "code", "covers": ["u029", "u030"], "evidence": [{"command": "conda run -n base python scripts/get_page.py technique-warp-specialization --include-code", "environment": "Conda base, repository root", "negative_control": "A nonexistent page ID exits nonzero.", "reference": "Output should contain the page, full artifact/provenance and derived variant", "result": "Command exited zero and produced 1,443 lines containing both artifact modes and the full SM100 FMHA source."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["conda run -n base python scripts/get_page.py technique-warp-specialization --include-code", "rg exact role dispatch and thread counts in CUTLASS v4.5.0 and pinned tutorial/FA4 source", "sha256sum and cmp local artifact files"], "id": "technique-warp-specialization", "page_verdict": "refuted", "path": "wiki/techniques/warp-specialization.md", "residual_risk": "No CUDA compiler or SM100 GPU is available. Artifact hashes, captured patch content, provenance modes, and the public merged-PR lineage pass strict verification.", "scope": "All 30 baseline units were checked against CUDA 13.0.2 PTX, CUTLASS 4.5.0, the pinned tutorial and FA4 implementations, plus local artifact/CLI checks."} diff --git a/verification/remediation-log.jsonl b/verification/remediation-log.jsonl new file mode 100644 index 000000000..eaf704b63 --- /dev/null +++ b/verification/remediation-log.jsonl @@ -0,0 +1,554 @@ +{"receipt_id":"rem-hw-clc-001","path":"wiki/hardware/clc.md","original_body_sha256":"a62ba7ec03d8146415399b716afb18e5106e6d2fd332e93a3500ddefb76fcc67","original_unit_ids":["u001","u003"],"locator":"body:L5-L9","original_claim_id":"hw-clc-original-c002","original_claim":"CLC replaces the static grid model and removes fixed initial CTA-to-tile assignments.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"Dynamic Scheduler rules 1-4","observation":"The full problem grid is launched and each worker processes blockIdx before querying another ClcID."}],"strongest_alternative":"Every tile, including the first, is acquired from a CLC queue.","discriminating_result":"Pinned rule 4 makes blockIdx the first work coordinate.","action":"correct","action_reason":"Retain the dynamic-work-stealing concept while restoring the documented initial assignment and full-grid model.","proposed_replacement":"CLC launches the problem-sized grid; each worker processes its own blockIdx first and may then steal a not-yet-started ClcID.","replacement_claim_ids":["hw-clc-current-c002"]} +{"receipt_id":"rem-hw-clc-002","path":"wiki/hardware/clc.md","original_body_sha256":"a62ba7ec03d8146415399b716afb18e5106e6d2fd332e93a3500ddefb76fcc67","original_unit_ids":["u004","u006"],"locator":"body:L10 and body:L17-L28","original_claim_id":"hw-clc-original-c004","original_claim":"CLC guarantees elimination of the ordinary equal-cost GEMM tail.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html","revision":"NVIDIA Matrix Multiplication Background User's Guide","locator":"3.2 Wave Quantization","observation":"Wave occupancy depends on tile count, tile shape, and SM count."},{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html","revision":"CUDA Programming Guide 13.2","locator":"4.12 CLC flow","observation":"CLC steals IDs from the existing problem grid and does not create extra independent tiles."}],"strongest_alternative":"CLC keeps every SM busy even when fewer unprocessed ClcIDs than SMs remain.","discriminating_result":"The documented request can only return an existing not-yet-started grid ID.","action":"correct","action_reason":"Replace the universal tail claim with the scoped load-balancing benefit documented for uneven SM availability.","proposed_replacement":"CLC can reduce dynamic load imbalance, but it does not guarantee full occupancy or eliminate wave quantization when insufficient tile parallelism exists.","replacement_claim_ids":["hw-clc-current-c008"]} +{"receipt_id":"rem-hw-clc-003","path":"wiki/hardware/clc.md","original_body_sha256":"a62ba7ec03d8146415399b716afb18e5106e6d2fd332e93a3500ddefb76fcc67","original_unit_ids":["u005","u010","u012","u013","u014","u015"],"locator":"body:L11 and body:L123-L144","original_claim_id":"hw-clc-original-c005","original_claim":"try_cancel discards application-selected outputs such as rejected tokens or masked tiles.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html","revision":"CUDA Programming Guide 13.2","locator":"4.12 flow and 4.12.1 API Details","observation":"The request has result and barrier operands, selects an unspecified not-yet-started block, and returns its ID for the requester to process."}],"strongest_alternative":"The request accepts a desired tile coordinate and intentionally drops that work.","discriminating_result":"The desired coordinate is not an input and successful work is transferred, not discarded.","action":"delete","action_reason":"The listed use cases depend on an API capability CLC does not provide; no useful verified portion remains.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-hw-clc-004","path":"wiki/hardware/clc.md","original_body_sha256":"a62ba7ec03d8146415399b716afb18e5106e6d2fd332e93a3500ddefb76fcc67","original_unit_ids":["u007"],"locator":"body:L32-L41","original_claim_id":"hw-clc-original-c007","original_claim":"CLC launches an SM-count-sized grid, requests work before the first tile, and keeps all CTAs busy until a tile queue is empty.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"Dynamic Scheduler rules 1-6 and pseudocode","observation":"The grid has one ClcID per output tile, blockIdx is first, and requests may decline."}],"strongest_alternative":"The wiki diagram is an equivalent fixed-worker formulation.","discriminating_result":"Its grid cardinality and initial request order differ from the normative model.","action":"correct","action_reason":"Replace the misleading diagram with the documented worker lifecycle.","proposed_replacement":"Launch one grid block per output tile; process blockIdx; asynchronously request and decode another ClcID; stop after a declined request.","replacement_claim_ids":["hw-clc-current-c002","hw-clc-current-c003"]} +{"receipt_id":"rem-hw-clc-005","path":"wiki/hardware/clc.md","original_body_sha256":"a62ba7ec03d8146415399b716afb18e5106e6d2fd332e93a3500ddefb76fcc67","original_unit_ids":["u009"],"locator":"body:L51-L119","original_claim_id":"hw-clc-original-c009","original_claim":"The persistent_gemm_clc block uses a valid register-returning CLC request and decode loop.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"9.7.13.17-18 syntax","observation":"try_cancel asynchronously writes a b128 opaque shared-memory response and completes an mbarrier; query_cancel decodes it."},{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html","revision":"CUDA Programming Guide 13.2","locator":"4.12.1.1 steps 1-6","observation":"The official sequence includes mbarrier initialization, 16-byte transaction accounting, wait, conditional decode, and proxy fences."}],"strongest_alternative":"The register-returning syntax is an accepted shorthand.","discriminating_result":"The normative grammar has mandatory response and mbarrier address operands with no register-output form.","action":"correct","action_reason":"Replace invalid illustrative CUDA with the official operation sequence and exact PTX signatures.","proposed_replacement":"Use a shared 16-byte response and shared mbarrier, submit try_cancel asynchronously, wait for completion, query is_canceled, and decode first_ctaid only on success.","replacement_claim_ids":["hw-clc-current-c003","hw-clc-current-c004"]} +{"receipt_id":"rem-hw-clc-006","path":"wiki/hardware/clc.md","original_body_sha256":"a62ba7ec03d8146415399b716afb18e5106e6d2fd332e93a3500ddefb76fcc67","original_unit_ids":["u011"],"locator":"body:L125-L139","original_claim_id":"hw-clc-original-c010","original_claim":"clc_try_cancel(uint2) cancels the supplied tile and returns success directly.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"9.7.13.17-18 operand descriptions","observation":"The instruction takes response and barrier addresses; success and coordinates are decoded from the later opaque response."}],"strongest_alternative":"The pointer operand denotes a desired input coordinate.","discriminating_result":"The pointer is explicitly the shared response destination, and the canceled coordinate is an output.","action":"correct","action_reason":"Retain an API sketch only in the exact asynchronous request/decode form.","proposed_replacement":"The request does not accept a target tile; it writes an opaque response that query_cancel decodes after mbarrier completion.","replacement_claim_ids":["hw-clc-current-c003"]} +{"receipt_id":"rem-hw-clc-007","path":"wiki/hardware/clc.md","original_body_sha256":"a62ba7ec03d8146415399b716afb18e5106e6d2fd332e93a3500ddefb76fcc67","original_unit_ids":["u017"],"locator":"body:L150-L173","original_claim_id":"hw-clc-original-c012","original_claim":"The shown legacy device::GemmUniversal typedef is a CUTLASS 4.5.0 SM100 CLC instantiation.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/95_blackwell_gemm_green_context/95_blackwell_gemm_green_context.cu","revision":"CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"dynamic_kernel_config","observation":"The official SM100 example uses collective builders and kernel::GemmUniversal with DynamicPersistentScheduler as its scheduler type."}],"strongest_alternative":"A legacy overload accepts the displayed trailing PersistentScheduler template argument.","discriminating_result":"The complete official v4.5.0 SM100 dynamic example uses a structurally different API.","action":"correct","action_reason":"Replace invalid code with a compact excerpt matching the pinned release.","proposed_replacement":"Use CollectiveBuilder mainloop/epilogue types and instantiate cutlass::gemm::kernel::GemmUniversal.","replacement_claim_ids":["hw-clc-current-c006"]} +{"receipt_id":"rem-hw-clc-008","path":"wiki/hardware/clc.md","original_body_sha256":"a62ba7ec03d8146415399b716afb18e5106e6d2fd332e93a3500ddefb76fcc67","original_unit_ids":["u018"],"locator":"body:L177-L198","original_claim_id":"hw-clc-original-c013","original_claim":"A synchronous clc_try_cancel call returning coordinates accurately models CUTLASS's scheduler.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp","revision":"CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"advance_to_next_work and fetch_next_work","observation":"CUTLASS issues through PipelineCLCFetchAsync, waits, decodes an opaque staged response, releases the stage, and maps coordinates."}],"strongest_alternative":"The pipeline is optional and direct synchronous return preserves semantics.","discriminating_result":"Async shared-memory completion and query decoding are mandatory CLC semantics.","action":"correct","action_reason":"Replace the false pseudocode with the actual staged CUTLASS flow.","proposed_replacement":"PersistentTileSchedulerSm100 uses PipelineCLCFetchAsync; advance_to_next_work submits the request and fetch_next_work consumes, decodes, releases, and remaps the response.","replacement_claim_ids":["hw-clc-current-c006"]} +{"receipt_id":"rem-hw-clc-009","path":"wiki/hardware/clc.md","original_body_sha256":"a62ba7ec03d8146415399b716afb18e5106e6d2fd332e93a3500ddefb76fcc67","original_unit_ids":["u019","u020","u021","u022"],"locator":"body:L202-L210","original_claim_id":"hw-clc-original-c014","original_claim":"The page's exact static/CLC SM-utilization table is generally valid for three square GEMM sizes.","original_verdict":"refuted","evidence":[{"command":"conda run -n base python -c ","environment":"Conda base Python 3.13.14 CPU tile-count model","reference":"Active-SM capacity depends on tile shape and SM count before dtype, occupancy, or timing are considered.","result":"For the same 2048x2048 output, scoped capacities were 86.49%, 43.24%, and 96.97% under three valid tile/SM scopes.","negative_control":"The false invariant-scope control evaluated false."}],"strongest_alternative":"GEMM dimension alone determines the percentages.","discriminating_result":"Holding GEMM size fixed while changing tile shape or SM count changes the capacity materially.","action":"delete","action_reason":"The exact numbers lack hardware, software, tile, dtype, occupancy, and timing scope; no source-backed replacement measurement is available.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-hw-clc-010","path":"wiki/hardware/clc.md","original_body_sha256":"a62ba7ec03d8146415399b716afb18e5106e6d2fd332e93a3500ddefb76fcc67","original_unit_ids":["u023"],"locator":"body:L212","original_claim_id":"hw-clc-original-c015","original_claim":"The tcgen05 tutorial's approximately 940-to-1476 TFLOPS progression measures adding persistent scheduling plus CLC.","original_verdict":"refuted","evidence":[{"url":"https://gau-nernst.github.io/tcgen05/","revision":"Article 2025-12-21; code commit 3b90ac9b","locator":"Persistent kernel (with static scheduling), final table, closing remarks","observation":"The final 1475.93-TFLOPS row is explicitly static scheduling and the author says CLC was not added."}],"strongest_alternative":"The label is stale and final code uses CLC.","discriminating_result":"The algorithm, row label, and closing remark independently identify static scheduling.","action":"delete","action_reason":"The measured numbers may remain attributable to the tutorial elsewhere, but they are not CLC evidence and cannot support this page's performance claim.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-hw-clc-011","path":"wiki/hardware/clc.md","original_body_sha256":"a62ba7ec03d8146415399b716afb18e5106e6d2fd332e93a3500ddefb76fcc67","original_unit_ids":["u025"],"locator":"body:L218-L229","original_claim_id":"hw-clc-original-c017","original_claim":"The example B200 has 132 SMs and CLC turns 32 output tiles into an efficient persistent schedule that avoids static idle SMs.","original_verdict":"refuted","evidence":[{"url":"https://developer.nvidia.com/blog/scaling-autonomous-ai-agents-and-workloads-with-nvidia-dgx-spark/","revision":"NVIDIA technical blog 2026","locator":"Table 8","observation":"The B200 configuration has 148 SMs."},{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html","revision":"CUDA Programming Guide 13.2","locator":"4.12 CLC flow","observation":"CLC redistributes IDs from the problem-sized grid and does not create more IDs than its 32 tiles."}],"strongest_alternative":"CLC expands a 32-block problem into an SM-count-sized pool.","discriminating_result":"The documented grid contains one ID per original problem block.","action":"correct","action_reason":"Retain the useful warning about insufficient tile parallelism while removing the wrong hardware count and false CLC remedy.","proposed_replacement":"A 32-tile output grid exposes at most 32 independent ClcIDs; CLC may redistribute them but cannot occupy a 148-SM B200 with only 32 tiles.","replacement_claim_ids":["hw-clc-current-c009"]} +{"receipt_id":"rem-hw-clc-012","path":"wiki/hardware/clc.md","original_body_sha256":"a62ba7ec03d8146415399b716afb18e5106e6d2fd332e93a3500ddefb76fcc67","original_unit_ids":["u027"],"locator":"body:L235-L260","original_claim_id":"hw-clc-original-c019","original_claim":"The cooperative_clc_loop is a valid two-CTA cluster cancellation and cta_group::2 pattern.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html","revision":"CUDA Programming Guide 13.2","locator":"4.12.2.2 Thread Block Clusters","observation":"Cluster cancellation requires a single cluster thread, multicast response, local cluster-scope barriers, cluster synchronization, and returned-ID local-rank adjustment."}],"strongest_alternative":"The undefined helper encapsulates all missing rules.","discriminating_result":"The visible loop still lacks the documented cluster lifetime and returned-ID rank handling.","action":"correct","action_reason":"Replace invalid executable-looking code with explicit, source-backed cluster requirements.","proposed_replacement":"For a cluster query, one cluster thread submits the multicast request, every CTA tracks completion with its local cluster-scope mbarrier, and each CTA adds its local block rank to the returned first CTA ID.","replacement_claim_ids":["hw-clc-current-c005"]} +{"receipt_id":"rem-hw-clc-013","path":"wiki/hardware/clc.md","original_body_sha256":"a62ba7ec03d8146415399b716afb18e5106e6d2fd332e93a3500ddefb76fcc67","original_unit_ids":["u029"],"locator":"body:L266-L281","original_claim_id":"hw-clc-original-c021","original_claim":"The apply_l2_swizzle function is a collision-free Morton/Z-order tile permutation.","original_verdict":"refuted","evidence":[{"command":"conda run -n base python -c ","environment":"Conda base Python 3.13.14 direct translation, 4x4 grid, swizzle_bits=2","reference":"A 16-tile traversal must produce 16 unique output coordinates.","result":"The wiki function produced only four unique outputs and did not match bit-interleaved Morton order.","negative_control":"Identity mapping produced all 16 unique outputs."}],"strongest_alternative":"Repeated coordinates are valid Morton grouping.","discriminating_result":"A traversal permutation cannot collapse 16 distinct inputs to four outputs.","action":"correct","action_reason":"Delete the broken function and retain the verified CUTLASS software-coordinate-remapping mechanism.","proposed_replacement":"CUTLASS applies its own collision-free serpentine swizzle/rasterization transform after decoding a ClcID; this is software scheduler logic, not the shown Morton function.","replacement_claim_ids":["hw-clc-current-c007"]} +{"receipt_id":"rem-hw-clc-014","path":"wiki/hardware/clc.md","original_body_sha256":"a62ba7ec03d8146415399b716afb18e5106e6d2fd332e93a3500ddefb76fcc67","original_unit_ids":["u030","u031"],"locator":"body:L287-L288","original_claim_id":"hw-clc-original-c022","original_claim":"CLC overhead is universally near zero and its tail handling universally optimal versus software atomics.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cutlass/4.3.3/media/docs/cpp/blackwell_cluster_launch_control.html","revision":"CUTLASS 4.3.3","locator":"Static Scheduler and CLC Pipeline Class","observation":"CUTLASS calls static scheduling zero-overhead and pipelines CLC queries to hide their latency."},{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html","revision":"CUDA Programming Guide 13.2","locator":"4.12 CLC flow","observation":"CLC requests can decline for end-of-grid or other reasons."}],"strongest_alternative":"Hardware requests have no meaningful cost or failure mode in any workload.","discriminating_result":"Vendor docs explicitly describe latency hiding and non-end-of-grid declines.","action":"delete","action_reason":"The qualitative ranking has no defined workload or metric and cannot be replaced with a universal performance claim.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-hw-clc-015","path":"wiki/hardware/clc.md","original_body_sha256":"a62ba7ec03d8146415399b716afb18e5106e6d2fd332e93a3500ddefb76fcc67","original_unit_ids":["u033"],"locator":"body:L290","original_claim_id":"hw-clc-original-c024","original_claim":"L2 tile swizzling is a CLC hardware policy configured at launch.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp","revision":"CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"Arguments, fetch_next_work, swizzle_and_rasterize","observation":"CUTLASS scheduler arguments drive a device-side coordinate transform after response decoding."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"9.7.13.17-18","observation":"The CLC instruction surface has no swizzle policy operand."}],"strongest_alternative":"CUTLASS forwards max_swizzle_size to undocumented CLC launch hardware.","discriminating_result":"Pinned source performs the coordinate transformation directly in swizzle_and_rasterize.","action":"correct","action_reason":"Preserve swizzled work mapping but attribute it to the CUTLASS software scheduler.","proposed_replacement":"CUTLASS can remap initial and returned ClcIDs with max_swizzle_size and raster_order in software.","replacement_claim_ids":["hw-clc-current-c007"]} +{"receipt_id":"rem-lang-triton-001","path":"wiki/languages/triton-blackwell.md","original_body_sha256":"6b9394376db86dd1090ef2012a0d599ad3ac0703f7df8c9bbf9c61c4c24888f1","original_unit_ids":["u001","u011","u017","u020","u022","u024","u040","u041","u042","u043","u046"],"locator":"body:L3, L18-L35, L119-L128","original_claim_id":"lang-triton-original-c002","original_claim":"Triton 3.6 was the first release with native Blackwell tcgen05/TMEM; 3.5 and earlier only emitted WGMMA with register accumulators.","original_verdict":"refuted","evidence":[{"url":"https://github.com/triton-lang/triton/blob/819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c/include/triton/Dialect/TritonNvidiaGPU/IR/TritonNvidiaGPUOps.td","revision":"Triton v3.3.0 commit 819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c","locator":"TCGen5 MMA and TMEM op definitions","observation":"v3.3.0 defines TCGen5 MMA/scaled MMA and TMEM alloc/load/store/copy operations."},{"url":"https://github.com/triton-lang/triton/blob/819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c/test/Conversion/tritongpu_to_llvm_blackwell.mlir","revision":"Triton v3.3.0 commit 819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c","locator":"tcgen05.mma CHECK directives","observation":"v3.3 tests require concrete tcgen05.mma output and TMEM encodings."}],"strongest_alternative":"The v3.3 ops are unused scaffolding and all real lowering remains WGMMA until 3.6.","discriminating_result":"Pinned v3.3 conversion tests check emitted tcgen05 PTX, while the corresponding backend is absent from v3.2.0.","action":"correct","action_reason":"Restore the v3.2-to-v3.3 introduction boundary and describe 3.6 as an incremental expansion.","proposed_replacement":"Native Blackwell TCGen5/TMEM lowering appears in Triton v3.3.0; v3.6 generalizes layouts/copies, deepens warp specialization, and adds initial Gluon multi-CTA/2CTA support.","replacement_claim_ids":["lang-triton-current-c001","lang-triton-current-c002"]} +{"receipt_id":"rem-lang-triton-002","path":"wiki/languages/triton-blackwell.md","original_body_sha256":"6b9394376db86dd1090ef2012a0d599ad3ac0703f7df8c9bbf9c61c4c24888f1","original_unit_ids":["u017","u023","u024","u026","u027","u048","u049","u054"],"locator":"body:L29, L42-L49, L136-L143","original_claim_id":"lang-triton-original-c009","original_claim":"vLLM PR 34597 is an SM100-only downstream demonstration of Triton 3.6 tcgen05/TMEM lowering.","original_verdict":"refuted","evidence":[{"url":"https://github.com/vllm-project/vllm/pull/34597","revision":"merged commit a1257fd1","locator":"PR summary and changed files","observation":"The PR adds FP8 cache support and calls the Triton backend the only MLA backend on SM120; it does not claim an SM100-only path or tcgen05 lowering."},{"url":"https://github.com/vllm-project/vllm/blob/a1257fd1/vllm/v1/attention/ops/triton_decode_attention.py","revision":"commit a1257fd1","locator":"kernel definitions and tl.dot sites","observation":"The code has no architecture guard, Triton-version pin, warp_specialize, tcgen05, TMEM, or emitted PTX."}],"strongest_alternative":"External architecture metadata is sufficient to prove both SM100-only use and TCGen5 emission.","discriminating_result":"The primary PR mentions SM120 and the exact code contains no lowering witness.","action":"correct","action_reason":"Retain the genuine FP8 Triton MLA kernel example while removing architecture and lowering inferences.","proposed_replacement":"PR 34597 is a provenance-pinned Triton MLA decode example with tl.dot and FP8 cache support; it does not prove which PTX MMA path any target selects.","replacement_claim_ids":["lang-triton-current-c003"]} +{"receipt_id":"rem-lang-triton-003","path":"wiki/languages/triton-blackwell.md","original_body_sha256":"6b9394376db86dd1090ef2012a0d599ad3ac0703f7df8c9bbf9c61c4c24888f1","original_unit_ids":["u033","u037"],"locator":"body:L62-L70","original_claim_id":"lang-triton-original-c015","original_claim":"The FlashInfer-Bench rows are a Triton-3.5-era snapshot and a Triton 3.6 rerun is unavailable.","original_verdict":"refuted","evidence":[{"url":"https://bench.flashinfer.ai/","revision":"live leaderboard retrieved 2026-08-08","locator":"author leaderboard rows 1-3","observation":"The live page still contains the values but provides no Triton version, historical snapshot date, or Triton-only filter."}],"strongest_alternative":"The public leaderboard metadata pins these rows to Triton 3.5 and records missing 3.6 results.","discriminating_result":"No such version or language scope appears, and the identical rows are current.","action":"correct","action_reason":"Preserve the exact live leaderboard values while deleting the false historical/version interpretation.","proposed_replacement":"As retrieved 2026-08-08, the live FlashInfer-Bench author leaderboard shows the three exact rows; the page does not attribute them to a Triton release.","replacement_claim_ids":["lang-triton-current-c004"]} +{"receipt_id":"rem-lang-triton-004","path":"wiki/languages/triton-blackwell.md","original_body_sha256":"6b9394376db86dd1090ef2012a0d599ad3ac0703f7df8c9bbf9c61c4c24888f1","original_unit_ids":["u038"],"locator":"body:L74-L97","original_claim_id":"lang-triton-original-c016","original_claim":"The displayed gated_delta_net_decode function is usable Triton code implementing the stated recurrence.","original_verdict":"refuted","evidence":[{"command":"conda run -n base python -c ","revision":"Conda base Python 3.13.14","locator":"gated_delta_net_decode AST","observation":"The function loads undefined offsets and uses inconsistent state load/store addressing."}],"strongest_alternative":"offsets is implicitly provided by Triton.","discriminating_result":"It is neither an argument, assignment, nor tl builtin; the well-scoped control has no undefined names.","action":"delete","action_reason":"The repository already provides provenance-pinned real GDN-related Triton code; retaining broken pseudocode has no teaching value.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-lang-triton-005","path":"wiki/languages/triton-blackwell.md","original_body_sha256":"6b9394376db86dd1090ef2012a0d599ad3ac0703f7df8c9bbf9c61c4c24888f1","original_unit_ids":["u039"],"locator":"body:L101-L115","original_claim_id":"lang-triton-original-c017","original_claim":"The displayed sparse_attention_fwd function is usable Triton sparse-attention code.","original_verdict":"refuted","evidence":[{"command":"conda run -n base python -c ","revision":"Conda base Python 3.13.14","locator":"sparse_attention_fwd AST","observation":"The function loads undefined d and offsets, leaves softmax unfinished, and never writes Output."}],"strongest_alternative":"The missing names and store are supplied outside a deliberately partial excerpt.","discriminating_result":"The complete fenced block presents a kernel signature and closes without defining them or producing output.","action":"delete","action_reason":"Replace the incomplete executable-looking block with links to complete provenance-pinned attention kernels.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-lang-triton-006","path":"wiki/languages/triton-blackwell.md","original_body_sha256":"6b9394376db86dd1090ef2012a0d599ad3ac0703f7df8c9bbf9c61c4c24888f1","original_unit_ids":["u055"],"locator":"body:L145","original_claim_id":"lang-triton-original-c020","original_claim":"data/triton-universe.yaml is the full 42-PR universe.","original_verdict":"refuted","evidence":[{"command":"conda run -n base python -c ","revision":"repository baseline 2777d18","locator":"top-level total/captured/skipped/prs","observation":"The file declares and contains 267 entries: 117 captured and 150 skipped."}],"strongest_alternative":"42 denotes an explicitly named filtered subset.","discriminating_result":"The prose says full universe and names no filter, while the canonical full list has 267 entries.","action":"correct","action_reason":"Remove the stale count and retain a stable pointer to the canonical ledger and capture flags.","proposed_replacement":"The current PR universe and captured/skipped flags are enumerated in data/triton-universe.yaml.","replacement_claim_ids":["lang-triton-current-c006"]} +{"receipt_id":"rem-lang-triton-007","path":"wiki/languages/triton-blackwell.md","original_body_sha256":"6b9394376db86dd1090ef2012a0d599ad3ac0703f7df8c9bbf9c61c4c24888f1","original_unit_ids":["u017","u023","u028","u029","u030","u031","u054"],"locator":"body:L29, L42, L51-L56, L143","original_claim_id":"lang-triton-original-c022","original_claim":"The named PRs have post-refresh/pre-refresh status according to data/refresh-cutoff.yaml::previous_pages_manifest.","original_verdict":"refuted","evidence":[{"command":"conda run -n base python -c ","revision":"repository baseline 2777d18","locator":"data/refresh-cutoff.yaml","observation":"The file has only cutoff_date 2026-05-20, no previous_pages_manifest, and all four named PR dates precede the cutoff."}],"strongest_alternative":"The manifest exists under another key and distinguishes the PRs.","discriminating_result":"Parsing every top-level key finds no manifest; a 2026-05-21 negative control is correctly classified after the cutoff.","action":"delete","action_reason":"Refresh-round administration is stale and irrelevant to the durable technical claim.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-source-triton-001","receipt_scope":"source-trail","path":"sources/docs/triton-3.6-blackwell.md","original_body_sha256":"78c7be5e3b76ddde273df4680f9456bc113a982d07f3cabf76b25740df6ef227","original_unit_ids":[],"locator":"Overview; Blackwell-Relevant Items; Caveats","original_claim_id":"source-doc-triton-3.6-original-c001","original_claim":"Triton 3.6.0 was the first native Blackwell TCGen5/TMEM release and introduced the underlying operation and allocation infrastructure.","original_verdict":"refuted","evidence":[{"url":"https://github.com/triton-lang/triton/blob/819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c/include/triton/Dialect/TritonNvidiaGPU/IR/TritonNvidiaGPUOps.td","revision":"Triton v3.3.0 commit 819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c","locator":"TCGen5 MMA and TMEM operation definitions","observation":"The v3.3 tag defines TCGen5 MMA/scaled MMA and TMEM allocation/load/store/copy operations."},{"url":"https://github.com/triton-lang/triton/blob/819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c/test/Conversion/tritongpu_to_llvm_blackwell.mlir","revision":"Triton v3.3.0 commit 819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c","locator":"FileCheck expectations for tcgen05 and TMEM","observation":"Conversion tests require concrete tcgen05 MMA, commit, TMEM, scaled-MMA, and CTA-group output; corresponding backend symbols are absent at v3.2.0 commit 9641643d."}],"strongest_alternative":"The v3.3 symbols were unused scaffolding and native lowering began only in v3.6.","discriminating_result":"Pinned v3.3 conversion tests assert emitted TCGen5/TMEM constructs while v3.2 lacks the backend symbols.","action":"correct","action_reason":"Describe v3.6 as an incremental release and preserve the actual v3.2-to-v3.3 boundary.","proposed_replacement":"Native TCGen5/TMEM backend support is present by v3.3.0; v3.6 generalizes copies/layouts and advances warp-specialized and initial multi-CTA Gluon paths.","replacement_claim_ids":["lang-triton-current-c001","lang-triton-current-c002"]} +{"receipt_id":"rem-source-triton-002","receipt_scope":"source-trail","path":"data/triton-3.6-evidence.md","original_body_sha256":"f76f1c7678d4853f8ef599e3ed56147f157bdb8823fd2571f55de730f4eb8b90","original_unit_ids":[],"locator":"Releases of Record; Lowering Surfaces; Evidence References","original_claim_id":"data-triton-evidence-original-c001","original_claim":"The evidence memo treats 3.6 as the TCGen5/TMEM introduction and downstream tl.dot kernels as demonstrations of that 3.6 lowering path.","original_verdict":"refuted","evidence":[{"url":"https://github.com/triton-lang/triton/compare/v3.2.0...v3.3.0","revision":"v3.2.0 commit 9641643d to v3.3.0 commit 819e9c8c","locator":"TritonNvidiaGPU operations, lowering, allocation, and Blackwell conversion tests","observation":"The native backend enters between the two tags, before 3.6."},{"url":"https://github.com/vllm-project/vllm/blob/a1257fd1/vllm/v1/attention/ops/triton_decode_attention.py","revision":"vLLM merge commit a1257fd1","locator":"kernel definitions and tl.dot sites","observation":"The downstream code contains tl.dot but no target guard, TCGen5/TMEM symbol, Triton version pin, or emitted PTX."}],"strongest_alternative":"A downstream tl.dot plus a post-3.6 merge date necessarily proves the 3.6 native path.","discriminating_result":"The compiler boundary predates 3.6 and the downstream source contains no lowering witness.","action":"correct","action_reason":"Separate compiler-version evidence from downstream adoption evidence and remove stale refresh-round classifications.","proposed_replacement":"Record the v3.3 boundary, the incremental v3.6 changes, and downstream kernels only for the behavior their exact code demonstrates.","replacement_claim_ids":["lang-triton-current-c001","lang-triton-current-c002","lang-triton-current-c003"]} +{"receipt_id":"rem-source-triton-003","receipt_scope":"source-trail","path":"data/version-claims.yaml","original_body_sha256":"eb3bc038ff408cede397937a008c3079de4aec700665eec47fd3a404b88599ba","original_unit_ids":[],"locator":"claims[id=vs-triton-3.6-blackwell-tcgen05]","original_claim_id":"registry-triton-boundary-original-c001","original_claim":"The version registry makes native Blackwell TCGen5/TMEM valid only for Triton >=3.6 and pins 3.6.0 as the checked release.","original_verdict":"refuted","evidence":[{"url":"https://github.com/triton-lang/triton/compare/v3.2.0...v3.3.0","revision":"v3.2.0 commit 9641643d to v3.3.0 commit 819e9c8c","locator":"native TCGen5/TMEM dialect, lowering, allocation, and tests","observation":"The checked version boundary is between 3.2 and 3.3."},{"url":"https://github.com/triton-lang/triton/releases/tag/v3.7.1","revision":"Triton v3.7.1 commit f797708","locator":"release description","observation":"v3.7.1 is the latest checked stable release on 2026-08-08 and is a regression-fix patch with no new API or feature."}],"strongest_alternative":"The v3.3 backend was not usable until the registry's 3.6 threshold.","discriminating_result":"The v3.3 conversion suite asserts concrete TCGen5/TMEM output, so >=3.6 excludes supported tagged releases.","action":"correct","action_reason":"Move the boundary to >=3.3 and pin the latest independently checked release.","proposed_replacement":"Use vs-triton-3.3-blackwell-tcgen05 with claim_valid_for >=3.3 and last_verified_release 3.7.1.","replacement_claim_ids":["lang-triton-current-c001","lang-triton-current-c002"]} +{"receipt_id":"rem-source-triton-004","receipt_scope":"source-trail","path":"data/tool-versions.yaml","original_body_sha256":"7fce8075b4c29d4307ee97f54e6d9d7063cfad6fe603decf5dfc47867607d59e","original_unit_ids":[],"locator":"tools[tool=triton].releases","original_claim_id":"tool-snapshot-triton-boundary-original-c001","original_claim":"The tool snapshot labels 3.6.0 as Triton's first native Blackwell TCGen5/TMEM release.","original_verdict":"refuted","evidence":[{"url":"https://github.com/triton-lang/triton/compare/v3.2.0...v3.3.0","revision":"v3.2.0 commit 9641643d to v3.3.0 commit 819e9c8c","locator":"native backend additions and conversion tests","observation":"Native TCGen5/TMEM support is already in the v3.3.0 tag."}],"strongest_alternative":"The earlier tagged code cannot produce TCGen5 output.","discriminating_result":"The v3.3 conversion tests explicitly require tcgen05 instructions and TMEM behavior.","action":"correct","action_reason":"Keep 3.6 in the snapshot as an incremental release and add the neighboring boundary releases plus latest checked release.","proposed_replacement":"Record v3.2.0 as the absent-side comparison, v3.3.0 as the native-backend boundary, v3.6.0 as incremental, and v3.7.1 as the latest checked stable release.","replacement_claim_ids":["lang-triton-current-c001","lang-triton-current-c002"]} +{"receipt_id":"rem-source-triton-005","receipt_scope":"source-trail","path":"data/inclusion-policy.yaml","original_body_sha256":"058015ad7f2f2076cab1d2eafe255b71bfdc333bbeed4cb4e20d66b34ce433f6","original_unit_ids":[],"locator":"header:L4-L7; triton.description","original_claim_id":"inclusion-policy-triton-boundary-original-c001","original_claim":"The inclusion policy says Triton lacked direct TCGen5/TMEM until 3.6, when native SM100 lowering arrived.","original_verdict":"refuted","evidence":[{"url":"https://github.com/triton-lang/triton/blob/819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c/test/Conversion/tritongpu_to_llvm_blackwell.mlir","revision":"Triton v3.3.0 commit 819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c","locator":"TCGen5/TMEM FileCheck expectations","observation":"The v3.3 tag already tests the native SM100 backend."}],"strongest_alternative":"Policy deliberately uses 3.6 as the first production-ready rather than first native threshold.","discriminating_result":"Its prose explicitly calls 3.6 the native-lowering boundary, not a separately defined maturity threshold.","action":"correct","action_reason":"Retain the scoped inclusion lane while correcting its compiler history and pointer.","proposed_replacement":"Triton has native Blackwell backend support from v3.3; coverage and peak-performance maturity remain workload-dependent through later releases.","replacement_claim_ids":["lang-triton-current-c001","lang-triton-current-c002"]} +{"receipt_id":"rem-source-triton-006","receipt_scope":"source-trail","path":"references/primer.md","original_body_sha256":"94ce5b8af93d8b644d69db94ab194fddec41b16d548374845cce224d2744b207","original_unit_ids":[],"locator":"frontmatter version_sensitive; language comparison row","original_claim_id":"primer-triton-boundary-original-c001","original_claim":"The primer identifies Triton 3.6+ as the start of native TCGen5/TMEM and pre-3.6 as lacking direct exposure.","original_verdict":"refuted","evidence":[{"url":"https://github.com/triton-lang/triton/compare/v3.2.0...v3.3.0","revision":"v3.2.0 commit 9641643d to v3.3.0 commit 819e9c8c","locator":"TritonNvidiaGPU backend additions","observation":"The native boundary is the v3.2-to-v3.3 change, not v3.6."}],"strongest_alternative":"The primer's 3.6 threshold names a distinct user API boundary.","discriminating_result":"The v3.3 tag includes both backend operations and concrete lowering tests; the broad 'no exposure' claim includes those surfaces.","action":"correct","action_reason":"Align the reference summary and version pointer with the exact checked boundary.","proposed_replacement":"Triton v3.3+ contains native TCGen5/TMEM backend support; later releases expand user-visible paths and maturity.","replacement_claim_ids":["lang-triton-current-c001","lang-triton-current-c002"]} +{"receipt_id":"rem-source-triton-007","receipt_scope":"source-trail","path":"references/examples.md","original_body_sha256":"ba596a492dfc80a3d0de0d39d6990a3816d9e979120614bb1b10ffb2653f1d66","original_unit_ids":[],"locator":"frontmatter version_sensitive; Example 7 Triton entry","original_claim_id":"examples-triton-boundary-original-c001","original_claim":"The examples guide labels the current Triton Blackwell surfaces as a Triton 3.6+ change and preserves pre-3.6 as the historical no-native-backend era.","original_verdict":"refuted","evidence":[{"url":"https://github.com/triton-lang/triton/blob/819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c/test/Conversion/tritongpu_to_llvm_blackwell.mlir","revision":"Triton v3.3.0 commit 819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c","locator":"TCGen5/TMEM FileCheck expectations","observation":"The exact v3.3 tree tests explicit TCGen5/TMEM lowering, contradicting the pre-3.6 framing."}],"strongest_alternative":"The checked output is disconnected from functional backend support.","discriminating_result":"The same tag contains the operation definitions, lowering implementations, and allocation passes that the conversion tests exercise.","action":"correct","action_reason":"Update the navigation example without broadening beyond the proven version boundary.","proposed_replacement":"Point to the v3.3+ native backend and identify v3.6 as an incremental expansion.","replacement_claim_ids":["lang-triton-current-c001","lang-triton-current-c002"]} +{"receipt_id":"rem-source-triton-008","receipt_scope":"source-trail","path":"sources/prs/vllm/PR-34597.md","original_body_sha256":"28658a0f9ba0737a9a843b6f18fa70fb1bbd7394b484ca8b85b358006b30d7e7","original_unit_ids":[],"locator":"frontmatter architectures; Summary","original_claim_id":"pr-vllm-34597-architecture-original-c001","original_claim":"The local PR summary classifies vLLM PR 34597 as an SM100 kernel change.","original_verdict":"refuted","evidence":[{"url":"https://github.com/vllm-project/vllm/pull/34597","revision":"merged commit a1257fd1","locator":"PR description and changed files","observation":"The PR says the Triton MLA backend is the only MLA backend available on SM120 and makes no SM100-only claim."},{"url":"https://github.com/vllm-project/vllm/blob/a1257fd1/vllm/v1/attention/ops/triton_decode_attention.py","revision":"commit a1257fd1","locator":"kernel definitions and launch path","observation":"The exact kernel has no SM100 architecture guard or TCGen5/TMEM lowering witness."}],"strongest_alternative":"Repository metadata outside the PR proves SM100-specific scope.","discriminating_result":"The primary PR explicitly names SM120 while the changed code is target-generic.","action":"correct","action_reason":"Preserve the supported target named by the primary source and remove the unsupported SM100 classification.","proposed_replacement":"Classify the PR for SM120 and state only that it adds FP8 KV-cache handling to the Triton MLA decode backend; it does not prove a target-specific lowering.","replacement_claim_ids":["lang-triton-current-c003"]} +{"receipt_id":"rem-hw-tcgen05-001","path":"wiki/hardware/tcgen05-mma.md","original_body_sha256":"8c57f5a2e74a678015fa2b7881071326d35a7aadc4083bddf116a18adf9a3f85","original_unit_ids":["u001"],"locator":"body:L5","original_claim_id":"hw-tcgen05-mma-original-c002","original_claim":"NVIDIA defines UMMA as the expansion 'Unified Matrix Multiply-Accumulate.'","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-dRC5i4/cutlass","revision":"CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"repository-wide nomenclature search","observation":"Pinned NVIDIA code uses the UMMA namespace but never defines the asserted expansion; PTX names the family tcgen05."}],"strongest_alternative":"The expansion is defined in the pinned vendor code near a different UMMA wrapper.","discriminating_result":"A repository-wide case-insensitive search finds UMMA uses but no 'Unified Matrix' or 'Unified MMA' definition.","action":"delete","action_reason":"Retain the official tcgen05 name and CUTLASS namespace without inventing an acronym expansion.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-hw-tcgen05-002","path":"wiki/hardware/tcgen05-mma.md","original_body_sha256":"8c57f5a2e74a678015fa2b7881071326d35a7aadc4083bddf116a18adf9a3f85","original_unit_ids":["u029"],"locator":"body:L58","original_claim_id":"hw-tcgen05-mma-original-c004","original_claim":"Single-thread issue removes synchronization overhead, so an elected lane can issue tcgen05.mma without a completion or ordering protocol.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"9.7.16.12.1","observation":"Completion is attached to an mbarrier by tcgen05.commit and cross-thread use requires documented ordering."}],"strongest_alternative":"Single-thread issue makes all completion state implicit.","discriminating_result":"The ISA defines a required explicit completion mechanism for observing prior asynchronous operations.","action":"correct","action_reason":"Separate issue granularity from synchronization and completion requirements.","proposed_replacement":"One thread issues MMA, but software still must manage asynchronous completion and any inter-thread ordering.","replacement_claim_ids":["hw-tcgen05-mma-current-c003","hw-tcgen05-mma-current-c007"]} +{"receipt_id":"rem-hw-tcgen05-003","path":"wiki/hardware/tcgen05-mma.md","original_body_sha256":"8c57f5a2e74a678015fa2b7881071326d35a7aadc4083bddf116a18adf9a3f85","original_unit_ids":["u005","u007"],"locator":"body:L13,L15","original_claim_id":"hw-tcgen05-mma-original-c006","original_claim":"tcgen05.mma accepts operand A only from shared memory, while WGMMA necessarily obtains A through ldmatrix registers.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"9.7.16.10 overview and mma syntax","observation":"tcgen05 A may reside in SMEM or TMEM; WGMMA has both register-A and descriptor-A forms."}],"strongest_alternative":"The TMEM-A grammar is only for a different instruction family.","discriminating_result":"Both A forms are listed directly under dense tcgen05.mma syntax.","action":"correct","action_reason":"Restore the actual operand-location alternatives and avoid a mandatory ldmatrix claim.","proposed_replacement":"tcgen05 reads A from SMEM or TMEM, B from SMEM, and D from TMEM; WGMMA A forms vary by instruction.","replacement_claim_ids":["hw-tcgen05-mma-current-c002"]} +{"receipt_id":"rem-hw-tcgen05-004","path":"wiki/hardware/tcgen05-mma.md","original_body_sha256":"8c57f5a2e74a678015fa2b7881071326d35a7aadc4083bddf116a18adf9a3f85","original_unit_ids":["u008","u031","u037","u038","u039","u040"],"locator":"body:L16,L60,L147-L169","original_claim_id":"hw-tcgen05-mma-original-c008","original_claim":"tcgen05 completion is fence-based: a fence plus CTA synchronization is sufficient before reading the MMA result from TMEM.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"9.7.16.11.1","observation":"Fences provide execution ordering and code-motion constraints."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"9.7.16.12.1","observation":"commit plus mbarrier wait provides completion observation."}],"strongest_alternative":"before_thread_sync waits for all prior MMA work.","discriminating_result":"The fence description never grants completion; the separate commit operation explicitly tracks it.","action":"correct","action_reason":"Teach completion and ordering as separate protocols.","proposed_replacement":"Use tcgen05.commit plus mbarrier wait for completion; use tcgen05.fence around an execution-ordering handoff when another thread consumes or produces tcgen05-visible state.","replacement_claim_ids":["hw-tcgen05-mma-current-c007"]} +{"receipt_id":"rem-hw-tcgen05-005","path":"wiki/hardware/tcgen05-mma.md","original_body_sha256":"8c57f5a2e74a678015fa2b7881071326d35a7aadc4083bddf116a18adf9a3f85","original_unit_ids":["u010","u012","u014","u015","u016"],"locator":"body:L24,L26,L28-L30","original_claim_id":"hw-tcgen05-mma-original-c010","original_claim":"The variant table accurately assigns scaling modes, accumulator types, and one fixed 1-SM shape to all seven kinds.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"9.7.16.10.9.1","observation":"f8f6f4 is unscaled, the MX kind is mxf8f6f4, block-scaled forms use separate scale operands, and M/N come from idesc."}],"strongest_alternative":"The table uses harmless aliases and merely lists one representative shape.","discriminating_result":"mxf8 is not a legal kind token and the table explicitly labels f8f6f4 with a scaling mode that belongs to a different grammar.","action":"correct","action_reason":"Replace the mixed taxonomy with the exact PTX grammar groups and avoid oversimplified type claims.","proposed_replacement":"List the three unscaled floating kinds, three block-scaled MX kinds, and i8, with shape controlled by idesc.","replacement_claim_ids":["hw-tcgen05-mma-current-c004"]} +{"receipt_id":"rem-hw-tcgen05-006","path":"wiki/hardware/tcgen05-mma.md","original_body_sha256":"8c57f5a2e74a678015fa2b7881071326d35a7aadc4083bddf116a18adf9a3f85","original_unit_ids":["u018","u019","u020","u021","u023","u024","u025","u026","u027"],"locator":"body:L38-L52","original_claim_id":"hw-tcgen05-mma-original-c012","original_claim":"Each precision has only the listed maximum MxN shape in 1-CTA and 2-CTA modes, and cta_group::2 always doubles M because each SM contributes exactly 128 rows.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instruction-descriptor","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"instruction descriptor and data-path layouts","observation":"M and N are descriptor-selected from multiple legal layouts; the listed values are maxima/examples, not exhaustive shapes."}],"strongest_alternative":"The smaller layouts exist only for sparse or non-MMA operations.","discriminating_result":"The dense MMA descriptor and layout sections enumerate multiple M/N configurations for the same kinds.","action":"correct","action_reason":"State group ownership separately from descriptor-selected shapes and label maxima as maxima.","proposed_replacement":"cta_group::1 uses current-CTA resources; ::2 also accesses its peer. Shape is encoded in idesc and depends on kind and target constraints.","replacement_claim_ids":["hw-tcgen05-mma-current-c005"]} +{"receipt_id":"rem-hw-tcgen05-007","path":"wiki/hardware/tcgen05-mma.md","original_body_sha256":"8c57f5a2e74a678015fa2b7881071326d35a7aadc4083bddf116a18adf9a3f85","original_unit_ids":["u032","u033","u034","u035","u036"],"locator":"body:L62-L143","original_claim_id":"hw-tcgen05-mma-original-c014","original_claim":"The four MMA PTX/inline-PTX examples have valid PTX 8.7-or-newer operand grammar and demonstrate unscaled and block-scaled accumulation correctly.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"9.7.16.10.9.1 syntax","observation":"Unscaled forms require a disable-output-lane vector; block-scaled forms require block_scale qualifiers and two TMEM scale operands."}],"strongest_alternative":"The omitted vector is optional shorthand accepted by earlier PTX.","discriminating_result":"PTX 8.7 and PTX 9.0 both require the vector; the displayed scalar form does not match either grammar.","action":"delete","action_reason":"Remove invalid executable-looking examples and retain the normative grammar with a vendor example link.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-hw-tcgen05-008","path":"wiki/hardware/tcgen05-mma.md","original_body_sha256":"8c57f5a2e74a678015fa2b7881071326d35a7aadc4083bddf116a18adf9a3f85","original_unit_ids":["u041"],"locator":"body:L173-L205","original_claim_id":"hw-tcgen05-mma-original-c015","original_claim":"The shown mainloop safely reuses shared-memory stages and reads TMEM after only a final fence and __syncthreads.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"9.7.16.12.1","observation":"The code omits the commit/mbarrier completion path and does not order async operand consumption before stage release."}],"strongest_alternative":"The final fence implicitly completes every MMA and protects every prior SMEM stage.","discriminating_result":"The ISA assigns completion to commit/mbarrier, not fence; stage lifetime must be managed by the producer-consumer pipeline.","action":"delete","action_reason":"A partial pipeline with unsafe lifetime rules is more misleading than the normative sequence.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-hw-tcgen05-009","path":"wiki/hardware/tcgen05-mma.md","original_body_sha256":"8c57f5a2e74a678015fa2b7881071326d35a7aadc4083bddf116a18adf9a3f85","original_unit_ids":["u042"],"locator":"body:L211-L262","original_claim_id":"hw-tcgen05-mma-original-c016","original_claim":"The Hopper/Blackwell comparison block is valid illustrative CUDA/PTX for both paths.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"tcgen05 grammar and WGMMA operand forms","observation":"The block contains literal ellipses, forces a non-universal WGMMA load path, omits the tcgen05 mask vector, and lacks completion."}],"strongest_alternative":"The block is unambiguously pseudocode and need not obey operand grammar.","discriminating_result":"It is fenced as cuda, presents asm volatile statements, and labels the pattern correct, so its invalid grammar is operationally misleading.","action":"delete","action_reason":"Replace executable-looking pseudo-assembly with a precise semantic comparison table.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-hw-tcgen05-010","path":"wiki/hardware/tcgen05-mma.md","original_body_sha256":"8c57f5a2e74a678015fa2b7881071326d35a7aadc4083bddf116a18adf9a3f85","original_unit_ids":["u046"],"locator":"body:L272","original_claim_id":"hw-tcgen05-mma-original-c018","original_claim":"Both operands require 128-byte-swizzled shared-memory layouts; no-swizzle and 64-byte modes silently produce incorrect results.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"Table 40","observation":"The tcgen05 descriptor defines valid no-swizzle, 128B, 64B, and 32B modes."}],"strongest_alternative":"Non-128B modes are encoded but invalid for mma operands.","discriminating_result":"The descriptor is defined for tcgen05 multiplicands and the layout sections specify valid constraints for the alternative modes.","action":"correct","action_reason":"Distinguish valid layouts from performance-sensitive layout choice.","proposed_replacement":"Choose a valid descriptor/layout pair; 128B swizzling may improve a given workload but is not universally mandatory.","replacement_claim_ids":["hw-tcgen05-mma-current-c008"]} +{"receipt_id":"rem-hw-tcgen05-011","path":"wiki/hardware/tcgen05-mma.md","original_body_sha256":"8c57f5a2e74a678015fa2b7881071326d35a7aadc4083bddf116a18adf9a3f85","original_unit_ids":["u047"],"locator":"body:L274-L290","original_claim_id":"hw-tcgen05-mma-original-c019","original_claim":"The shown make_smem_desc function constructs a valid mandatory-128B tcgen05 shared-memory descriptor.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"Table 40","observation":"The code omits required fields and uses invalid swizzle value 3; 128B is encoded as value 2."}],"strongest_alternative":"Reserved/fixed fields are filled by ptxas and the mode field is inherited from WGMMA.","discriminating_result":"The descriptor is a runtime register value and tcgen05 Table 40 assigns fixed bits and a distinct three-bit mode map.","action":"delete","action_reason":"Direct users to the complete table rather than preserving a subtly corrupt descriptor builder.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-hw-tcgen05-012","path":"wiki/hardware/tcgen05-mma.md","original_body_sha256":"8c57f5a2e74a678015fa2b7881071326d35a7aadc4083bddf116a18adf9a3f85","original_unit_ids":["u051"],"locator":"body:L299","original_claim_id":"hw-tcgen05-mma-original-c021","original_claim":"The tutorial's 1476-TFLOP/s final kernel combines persistence with Cluster Launch Control.","original_verdict":"refuted","evidence":[{"url":"https://gau-nernst.github.io/tcgen05/","revision":"article 2025-12-21; code commit 3b90ac9b","locator":"Persistent kernel heading and closing table","observation":"v6 is persistent with static scheduling and the author explicitly says CLC was not added."}],"strongest_alternative":"The code implements CLC without mentioning it in the row label.","discriminating_result":"The author explicitly lists CLC as a future, unimplemented optimization.","action":"correct","action_reason":"Preserve the measurement while restoring the actual scheduling method.","proposed_replacement":"v6 persistent with static scheduling: 1475.93 TFLOP/s; no CLC in this result.","replacement_claim_ids":["hw-tcgen05-mma-current-c009"]} +{"receipt_id":"rem-source-tcgen05-001","receipt_scope":"source-trail","path":"sources/docs/nvidia-blackwell-tuning-guide.md","original_body_sha256":"edf8d568a3376baa4b5f59a03fc8cad5ae32dc75c760fab1c67704ef48c106fa","original_unit_ids":[],"locator":"Key Hardware Features; Performance Optimization Path; Hopper-to-Blackwell Migration Summary","original_claim_id":"source-blackwell-guide-original-c001","original_claim":"The local guide summary says tcgen05 A is SMEM-only, every kind has one maximum fixed shape, CLC replaces static assignment and eliminates tails, and the tutorial's 98% result used CLC.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"tcgen05 overview, grammar, descriptors, and completion","observation":"A may be SMEM or TMEM, M/N are descriptor-selected, variant and completion details differ from the summary."},{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html","revision":"CUDA Programming Guide 13.2","locator":"4.12 work stealing flow","observation":"CLC steals not-yet-started work from the full grid; it does not replace initial blockIdx assignment or guarantee tail elimination."},{"url":"https://gau-nernst.github.io/tcgen05/","revision":"article 2025-12-21; code commit 3b90ac9b","locator":"v6 heading and closing remarks","observation":"The final result used static scheduling and explicitly did not implement CLC."}],"strongest_alternative":"The summary intentionally merges generic recommendations from multiple sources.","discriminating_result":"Its absolute architecture and benchmark attributions contradict the pinned primary source and the benchmark author.","action":"correct","action_reason":"Reduce the source trail to exact claims the linked primary material supports.","proposed_replacement":"Summarize the official operand, variant, shape, ordering, descriptor, and CLC semantics without unsupported latency or guarantee claims.","replacement_claim_ids":["hw-tcgen05-mma-current-c001","hw-tcgen05-mma-current-c002","hw-tcgen05-mma-current-c004","hw-tcgen05-mma-current-c005","hw-tcgen05-mma-current-c007","hw-tcgen05-mma-current-c008"]} +{"receipt_id":"rem-source-tcgen05-002","receipt_scope":"source-trail","path":"sources/blogs/tcgen05-tutorial.md","original_body_sha256":"ce46ccd3d2ea79d335ffa96c0a5d0193389a88553311785595975b3840b8282b","original_unit_ids":[],"locator":"Summary; Performance Progression; Key Code","original_claim_id":"source-tcgen05-tutorial-original-c001","original_claim":"The local tutorial summary presents invalid shortened tcgen05 operand grammar and reports a generic 98%-of-cuBLAS progression without the exact environment or static-scheduling caveat.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"9.7.16.10.9.1 syntax","observation":"The local inline PTX omits the required disable-output-lane vector."},{"url":"https://gau-nernst.github.io/tcgen05/","revision":"article 2025-12-21; code commit 3b90ac9b","locator":"experiment setup and final progression table","observation":"The exact values are scoped to B200, M=N=K=4096, PyTorch 2.9.1 plus CUDA 13; v6 uses static scheduling and no CLC."}],"strongest_alternative":"The local code is only pseudocode and the rounded figures are portable performance expectations.","discriminating_result":"It is presented as key CUDA code, while the author reports measured numbers only for one disclosed setup and labels the final scheduler static.","action":"correct","action_reason":"Keep this source trail source-reported, exact, and non-executable where operands are intentionally omitted.","proposed_replacement":"Record the full disclosed benchmark table and link to the author's pinned code instead of reproducing truncated inline PTX.","replacement_claim_ids":["hw-tcgen05-mma-current-c009"]} +{"receipt_id":"rem-hw-2sm-001","path":"wiki/hardware/2sm-cooperative.md","original_body_sha256":"eab776a8afa8cfe67479add2ac89855a63fabdee6e46f9b3e445b85a7c94b3c2","original_unit_ids":["u001"],"locator":"body:L3","original_claim_id":"hw-2sm-cooperative-original-c001","original_claim":"Blackwell cta_group::2 always doubles a 1-CTA MMA into m256xn256xk16 within a TPC.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-data-path-layout-organization","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"CTA Pair and data-path layouts","observation":"Group 2 touches both CTA-pair TMEM resources, but M/N remain descriptor-selected and a group-2 M=128 layout is defined."}],"strongest_alternative":"Group 2 is synonymous with the single BF16 m256 maximum example.","discriminating_result":"The same ISA explicitly defines cta_group::2 with M=128, so group size does not determine one fixed shape.","action":"correct","action_reason":"Describe pair-level resource scope separately from shape selection.","proposed_replacement":"cta_group::2 initiates one MMA across current and peer CTA TMEM; idesc still selects the legal shape.","replacement_claim_ids":["hw-2sm-cooperative-current-c001","hw-2sm-cooperative-current-c002"]} +{"receipt_id":"rem-hw-2sm-002","path":"wiki/hardware/2sm-cooperative.md","original_body_sha256":"eab776a8afa8cfe67479add2ac89855a63fabdee6e46f9b3e445b85a7c94b3c2","original_unit_ids":["u002"],"locator":"body:L7-L15","original_claim_id":"hw-2sm-cooperative-original-c002","original_claim":"The diagram accurately defines TPC as 'Two Processing Clusters' and fixes A by row halves and TMEM by column halves across the CTA pair.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-data-path-layout-organization","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"CTA Pair and Layout A-F","observation":"The pair is defined by cluster-rank parity and the ISA has multiple data-path layouts rather than the diagram's universal partition."}],"strongest_alternative":"The diagram is the unique dense-F16 layout even if not universal.","discriminating_result":"It also mis-expands TPC and labels two CTA-local TMEM column ranges as one fixed mapping without an idesc/layout scope.","action":"delete","action_reason":"Replace the misleading topology diagram with normative pair and issue rules.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-hw-2sm-003","path":"wiki/hardware/2sm-cooperative.md","original_body_sha256":"eab776a8afa8cfe67479add2ac89855a63fabdee6e46f9b3e445b85a7c94b3c2","original_unit_ids":["u003"],"locator":"body:L19-L23","original_claim_id":"hw-2sm-cooperative-original-c003","original_claim":"The displayed cta_group::2 f16 PTX is valid instruction syntax.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"9.7.16.10.9.1","observation":"The code invents a second idesc, omits the eight-register lane-disable vector and accumulation predicate, and contains literal ellipses."}],"strongest_alternative":"The extra operands are placeholders that preserve the actual grammar.","discriminating_result":"Their count and positions differ from the normative grammar, so substitution cannot make the statement valid.","action":"delete","action_reason":"Link to the already verified normative grammar on the tcgen05 page instead of duplicating invalid code.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-hw-2sm-004","path":"wiki/hardware/2sm-cooperative.md","original_body_sha256":"eab776a8afa8cfe67479add2ac89855a63fabdee6e46f9b3e445b85a7c94b3c2","original_unit_ids":["u004","u005","u007"],"locator":"body:L26-L29","original_claim_id":"hw-2sm-cooperative-original-c005","original_claim":"Every cta_group::2 MMA requires identical SMEM layouts, a shared::cluster mbarrier exchanged between the CTAs, and exactly half of M from each CTA.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-issue-granularity","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"CTA pair, MMA, commit, and layouts","observation":"The normative requirements are pair membership, peer lifetime, legal descriptors/layouts, consistent CTA group, and applicable completion/ordering; the asserted three rules are not universal."}],"strongest_alternative":"Those rules are implicit consequences of one descriptor and pair allocation.","discriminating_result":"The ISA exposes multiple group-2 layouts and a separate optional multicast completion mechanism; it does not impose the page's exact checklist on every MMA.","action":"correct","action_reason":"Replace the invented checklist with direct ISA invariants.","proposed_replacement":"Require a valid CTA pair with a live peer, paired allocation management, legal descriptors, consistent cta_group, and explicit completion/ordering.","replacement_claim_ids":["hw-2sm-cooperative-current-c001","hw-2sm-cooperative-current-c003"]} +{"receipt_id":"rem-hw-2sm-005","path":"wiki/hardware/2sm-cooperative.md","original_body_sha256":"eab776a8afa8cfe67479add2ac89855a63fabdee6e46f9b3e445b85a7c94b3c2","original_unit_ids":["u011","u012","u013"],"locator":"body:L38-L40","original_claim_id":"hw-2sm-cooperative-original-c007","original_claim":"2-SM MMA should be used whenever M is at least 256, whenever a kernel is compute-bound, and with persistence to guarantee maximum throughput.","original_verdict":"refuted","evidence":[{"url":"https://gau-nernst.github.io/tcgen05/","revision":"article 2025-12-21; code commit 3b90ac9b","locator":"2-SM and persistent sections","observation":"The reported gain is workload-specific and modest; persistence addresses measured idle periods. No universal threshold or maximum-throughput guarantee is given."}],"strongest_alternative":"The advice is a harmless heuristic implied by the tutorial.","discriminating_result":"The article calls the 2-SM gain disappointing and motivates persistence from a specific profile, demonstrating dependence on kernel shape and bottlenecks.","action":"correct","action_reason":"Replace categorical advice with a measurement-and-resource tradeoff checklist.","proposed_replacement":"Benchmark group 1 versus group 2 for the exact shapes and account for pairing, layout, occupancy, and pipeline costs.","replacement_claim_ids":["hw-2sm-cooperative-current-c005"]} +{"receipt_id":"rem-migration-wgmma-001","path":"wiki/migration/wgmma-to-tcgen05.md","original_body_sha256":"0c2cada28831223c7f5b4dc10e20b9007129747d3659ed5f7e0aefbe340281d9","original_unit_ids":["u004","u005"],"locator":"body:L9-L10","original_claim_id":"migration-wgmma-to-tcgen05-original-c002","original_claim":"The migration universally changes ldmatrix-plus-register A into direct-SMEM A and replaces WGMMA synchronization with async fences.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"WGMMA/tcgen05 operand forms; fence and commit","observation":"Both families have operand alternatives; fences order, while commit/mbarrier completes tcgen05 MMA."}],"strongest_alternative":"All production WGMMA kernels use register A and every tcgen05 kernel uses SMEM A.","discriminating_result":"The ISA defines descriptor-A WGMMA and TMEM-A tcgen05 forms directly.","action":"correct","action_reason":"Make migration conditional on the exact old and new instruction forms and separate completion from ordering.","proposed_replacement":"Inventory the current WGMMA operand form; choose legal tcgen05 SMEM/TMEM A and implement commit/mbarrier plus any required fences.","replacement_claim_ids":["migration-wgmma-to-tcgen05-current-c001","migration-wgmma-to-tcgen05-current-c004"]} +{"receipt_id":"rem-migration-wgmma-002","path":"wiki/migration/wgmma-to-tcgen05.md","original_body_sha256":"0c2cada28831223c7f5b4dc10e20b9007129747d3659ed5f7e0aefbe340281d9","original_unit_ids":["u007"],"locator":"body:L15-L25","original_claim_id":"migration-wgmma-to-tcgen05-original-c004","original_claim":"The migration checklist is correct as written, including mandatory ldmatrix removal, 64B-to-128B swizzle conversion, and replacing commit/wait with fences.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"descriptor modes, operand grammar, completion","observation":"Several swizzles and A locations are valid; commit/mbarrier remains necessary for MMA completion."}],"strongest_alternative":"The checklist is a recommended fast path rather than a correctness checklist.","discriminating_result":"It uses unconditional checkbox commands and explicitly says replace commit/wait with fences, a correctness error.","action":"correct","action_reason":"Replace categorical substitutions with a dependency-ordered audit checklist.","proposed_replacement":"Audit instruction forms, allocate TMEM collectively, preserve operands, commit/wait for completion, select a legal layout, and retune roles/shapes.","replacement_claim_ids":["migration-wgmma-to-tcgen05-current-c002"]} +{"receipt_id":"rem-migration-wgmma-003","path":"wiki/migration/wgmma-to-tcgen05.md","original_body_sha256":"0c2cada28831223c7f5b4dc10e20b9007129747d3659ed5f7e0aefbe340281d9","original_unit_ids":["u008"],"locator":"body:L31-L84","original_claim_id":"migration-wgmma-to-tcgen05-original-c005","original_claim":"The Hopper kernel block is valid or structurally faithful WGMMA CUDA for the migration baseline.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-dRC5i4/cutlass/include/cute/arch/mma_sm90_gmma.hpp","revision":"CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"m64n256 wrapper signatures","observation":"The block's ellipses and invented mixed ldmatrix/WGMMA operand list match neither descriptor-A nor register-A forms."}],"strongest_alternative":"Literal ellipses clearly mark harmless structural pseudocode.","discriminating_result":"The false universal register-A data path is the premise of later migration steps, so the block misstates structure even as pseudocode.","action":"delete","action_reason":"Use an exact semantic mapping and pinned CUTLASS examples rather than fabricated assembly.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-migration-wgmma-004","path":"wiki/migration/wgmma-to-tcgen05.md","original_body_sha256":"0c2cada28831223c7f5b4dc10e20b9007129747d3659ed5f7e0aefbe340281d9","original_unit_ids":["u009"],"locator":"body:L88-L155","original_claim_id":"migration-wgmma-to-tcgen05-original-c006","original_claim":"The Blackwell kernel block correctly demonstrates TMEM allocation, tcgen05 MMA, completion, TMEM load, and deallocation.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"allocation issue, MMA grammar, completion, and ld","observation":"Allocation/deallocation and TMEM loads are warp-collective; the MMA grammar and completion path are incomplete."}],"strongest_alternative":"Lane 0 may issue alloc/dealloc because only the result store is collective.","discriminating_result":"The mandatory sync/aligned semantics require every warp lane to execute with identical operands.","action":"delete","action_reason":"Remove an unsafe end-to-end example that cannot be repaired without becoming a full kernel.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-migration-wgmma-005","path":"wiki/migration/wgmma-to-tcgen05.md","original_body_sha256":"0c2cada28831223c7f5b4dc10e20b9007129747d3659ed5f7e0aefbe340281d9","original_unit_ids":["u014","u015","u016","u017"],"locator":"body:L177-L197","original_claim_id":"migration-wgmma-to-tcgen05-original-c008","original_claim":"A WGMMA port should always remove ldmatrix because Hopper necessarily register-stages A while tcgen05 necessarily uses an SMEM A descriptor.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"WGMMA SS/RS and tcgen05 A forms","observation":"Some WGMMA forms already use descriptor A, while tcgen05 also supports TMEM A."}],"strongest_alternative":"Only obscure unsupported variants use those alternative forms.","discriminating_result":"They are normative dense instruction grammar on the relevant SM90/SM100 targets.","action":"correct","action_reason":"Make operand migration a form-by-form decision.","proposed_replacement":"Remove ldmatrix only if the old path actually register-stages A and the chosen tcgen05 path no longer needs it.","replacement_claim_ids":["migration-wgmma-to-tcgen05-current-c001"]} +{"receipt_id":"rem-migration-wgmma-006","path":"wiki/migration/wgmma-to-tcgen05.md","original_body_sha256":"0c2cada28831223c7f5b4dc10e20b9007129747d3659ed5f7e0aefbe340281d9","original_unit_ids":["u020","u021"],"locator":"body:L209-L215","original_claim_id":"migration-wgmma-to-tcgen05-original-c010","original_claim":"Blackwell tcgen05 correctness requires converting every operand layout to 128-byte swizzling.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"Table 40","observation":"No-swizzle, 128B, 64B, and 32B descriptor modes are valid subject to layout constraints."}],"strongest_alternative":"Alternative modes are encoded only for non-MMA tcgen05 operations.","discriminating_result":"Table 40 defines multiplicand descriptors used by tcgen05 MMA and the layout sections cover the modes.","action":"correct","action_reason":"Make 128B a benchmarkable layout option rather than a correctness mandate.","proposed_replacement":"Choose a legal descriptor/layout pair; benchmark 128B swizzling for the target tile.","replacement_claim_ids":["migration-wgmma-to-tcgen05-current-c003"]} +{"receipt_id":"rem-migration-wgmma-007","path":"wiki/migration/wgmma-to-tcgen05.md","original_body_sha256":"0c2cada28831223c7f5b4dc10e20b9007129747d3659ed5f7e0aefbe340281d9","original_unit_ids":["u022","u023","u024","u025"],"locator":"body:L219-L245","original_claim_id":"migration-wgmma-to-tcgen05-original-c011","original_claim":"The WGMMA-to-tcgen05 issue snippets are valid pseudocode and tcgen05 needs no commit because it is fully asynchronous.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"MMA grammar and mbarrier completion","observation":"The snippets omit required operands, and tcgen05.commit plus mbarrier observes asynchronous MMA completion."}],"strongest_alternative":"Commit is optional when later code fences before reading.","discriminating_result":"The ISA gives fence no completion guarantee and assigns completion explicitly to commit/mbarrier.","action":"delete","action_reason":"Replace incomplete assembly with a normative lifecycle sequence.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-migration-wgmma-008","path":"wiki/migration/wgmma-to-tcgen05.md","original_body_sha256":"0c2cada28831223c7f5b4dc10e20b9007129747d3659ed5f7e0aefbe340281d9","original_unit_ids":["u028","u029"],"locator":"body:L257-L264","original_claim_id":"migration-wgmma-to-tcgen05-original-c013","original_claim":"A tcgen05.fence before __syncthreads replaces wgmma.wait_group and makes the TMEM accumulator ready.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"commit and fence descriptions","observation":"Commit/mbarrier tracks completion; fences order tcgen05 across execution-ordering operations."}],"strongest_alternative":"before_thread_sync implicitly drains the tcgen05 pipeline.","discriminating_result":"The fence contract contains only ordering/code-motion language and the separate commit contract provides draining.","action":"correct","action_reason":"Preserve the WGMMA analogy while mapping it to the correct Blackwell mechanism.","proposed_replacement":"Replace WGMMA group completion with tcgen05.commit and an mbarrier wait; add fences only for applicable handoffs.","replacement_claim_ids":["migration-wgmma-to-tcgen05-current-c004"]} +{"receipt_id":"rem-migration-wgmma-009","path":"wiki/migration/wgmma-to-tcgen05.md","original_body_sha256":"0c2cada28831223c7f5b4dc10e20b9007129747d3659ed5f7e0aefbe340281d9","original_unit_ids":["u034","u035"],"locator":"body:L298-L316","original_claim_id":"migration-wgmma-to-tcgen05-original-c015","original_claim":"The two warp-specialization diagrams describe typical required Hopper and Blackwell allocations of 13+ and 5 warps, including a dedicated CLC warp on Blackwell.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-dRC5i4/cutlass/examples/cute/tutorial/blackwell/02_mma_tma_sm100.cu","revision":"CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"role partition and pipelines","observation":"Pinned examples use workload-specific role counts and do not require the displayed totals or CLC role."}],"strongest_alternative":"The diagrams are explicitly labeled examples, not requirements.","discriminating_result":"They are labeled Typical and followed by a categorical 'far fewer warps' conclusion with no source or configuration scope.","action":"delete","action_reason":"Replace invented totals with role-level design guidance.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-migration-wgmma-010","path":"wiki/migration/wgmma-to-tcgen05.md","original_body_sha256":"0c2cada28831223c7f5b4dc10e20b9007129747d3659ed5f7e0aefbe340281d9","original_unit_ids":["u040","u041","u042","u043","u044","u045"],"locator":"body:L327-L333","original_claim_id":"migration-wgmma-to-tcgen05-original-c017","original_claim":"Blackwell's base MMA tile universally doubles Hopper M from 64 to 128, and migration should mechanically double CTA M or choose the two fixed table shapes.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-data-path-layout-organization","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"shape inventories and idesc","observation":"Both families have multiple legal shapes and tcgen05 M/N are descriptor/configuration selected."}],"strongest_alternative":"The table intentionally covers only BF16 maximum-throughput GEMM.","discriminating_result":"It is presented as the generic tile migration section and uses mandatory should/doubles wording without that scope.","action":"correct","action_reason":"Turn mechanical tile substitution into a resource-aware retuning step.","proposed_replacement":"Select legal idesc shapes and retune CTA/cluster tiles for the target workload; do not infer a universal factor-of-two mapping.","replacement_claim_ids":["migration-wgmma-to-tcgen05-current-c006"]} +{"receipt_id":"rem-migration-wgmma-011","path":"wiki/migration/wgmma-to-tcgen05.md","original_body_sha256":"0c2cada28831223c7f5b4dc10e20b9007129747d3659ed5f7e0aefbe340281d9","original_unit_ids":["u046","u048","u049"],"locator":"body:L337-L343","original_claim_id":"migration-wgmma-to-tcgen05-original-c018","original_claim":"The pitfall list correctly says missing 128B swizzle silently corrupts tcgen05, fence-plus-syncthreads is the required completion path, and retaining Hopper thread counts necessarily wastes resources.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"descriptor, issue granularity, commit, and fence","observation":"Several layouts are correct; completion uses commit/mbarrier; total thread needs depend on non-MMA roles."}],"strongest_alternative":"These are conservative warnings that cannot harm correctness.","discriminating_result":"They direct users to remove valid layouts and omit the actual completion mechanism, creating rather than preventing bugs.","action":"correct","action_reason":"Keep only direct, scoped pitfalls with observable failure modes.","proposed_replacement":"Verify collective allocation/deallocation, source lifetime, commit/wait, fence handoffs, descriptor validity, and retune thread roles empirically.","replacement_claim_ids":["migration-wgmma-to-tcgen05-current-c002","migration-wgmma-to-tcgen05-current-c003","migration-wgmma-to-tcgen05-current-c004"]} +{"receipt_id":"rem-migration-wgmma-012","path":"wiki/migration/wgmma-to-tcgen05.md","original_body_sha256":"0c2cada28831223c7f5b4dc10e20b9007129747d3659ed5f7e0aefbe340281d9","original_unit_ids":["u051","u052","u053"],"locator":"body:L349-L369","original_claim_id":"migration-wgmma-to-tcgen05-original-c020","original_claim":"CUTLASS migration is largely an arch-tag and schedule-token substitution using the shown device::GemmUniversal and KernelScheduleSm100CpAsyncWarpSpecialized types, after which MMA_Atom/MMA_Traits automatically handle all differences.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-dRC5i4/cutlass/examples/70_blackwell_gemm/70_blackwell_fp16_gemm.cu","revision":"CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"CollectiveBuilder and GemmUniversal construction","observation":"The shown schedule type does not exist; SM100 examples configure collective builders, shapes, stages, schedules, kernel::GemmUniversal, and a device adapter."}],"strongest_alternative":"The names are stable aliases omitted from the searched header.","discriminating_result":"Repository-wide exact-token search finds neither displayed schedule token, while official examples use different public types.","action":"correct","action_reason":"Point to a pinned, complete SM100 example and enumerate the configuration surfaces that must be reselected.","proposed_replacement":"Rebuild mainloop and epilogue collectives for Sm100, select supported schedules and shapes, instantiate kernel::GemmUniversal, and revalidate alignment/workspace/dispatch.","replacement_claim_ids":["migration-wgmma-to-tcgen05-current-c007"]} +{"receipt_id":"rem-hw-tmem-001","path":"wiki/hardware/tmem.md","original_body_sha256":"31c6ed49bcd928754519fe06febb83cd987d9f3691fc9bb89f5aad7c7e1351b0","original_unit_ids":["u001"],"locator":"body:L5","original_claim_id":"hw-tmem-original-c001","original_claim":"TMEM eliminates a Hopper accumulator burden of 128-plus registers per warpgroup.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cute/arch/mma_sm90_gmma.hpp","revision":"CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"MMA_64x256x16_F32F16F16_SS::CRegisters","observation":"The representative wrapper declares 128 float accumulators per participating thread, not per warpgroup."}],"strongest_alternative":"The page uses warpgroup as shorthand for the fragment distributed across that warpgroup.","discriminating_result":"The cited wrapper's array is a per-thread interface, making the stated resource unit wrong and the elimination claim overbroad.","action":"correct","action_reason":"Keep the verified storage shift but scope register impact to the exact old instruction and per-thread fragment.","proposed_replacement":"TMEM holds tcgen05 D accumulators; this can reduce GPR pressure compared with register-D WGMMA forms.","replacement_claim_ids":["hw-tmem-current-c001"]} +{"receipt_id":"rem-hw-tmem-002","path":"wiki/hardware/tmem.md","original_body_sha256":"31c6ed49bcd928754519fe06febb83cd987d9f3691fc9bb89f5aad7c7e1351b0","original_unit_ids":["u008"],"locator":"body:L28","original_claim_id":"hw-tmem-original-c004","original_claim":"Each thread owns and independently reads the TMEM row computed from warp and lane.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"tcgen05.ld execution and access restrictions","observation":"TMEM loads/stores are warp-collective and use shape-specific lane mappings."}],"strongest_alternative":"The row formula is still an exact ownership model beneath collective syntax.","discriminating_result":"The instruction contract requires all warp threads and defines mapping by access shape, not an independent per-thread load permission.","action":"correct","action_reason":"Describe lanes and collective warp chunks without inventing ownership.","proposed_replacement":"Four participating warps collectively cover successive 32-lane TMEM chunks; consult the chosen ld/st shape for register mapping.","replacement_claim_ids":["hw-tmem-current-c002"]} +{"receipt_id":"rem-hw-tmem-003","path":"wiki/hardware/tmem.md","original_body_sha256":"31c6ed49bcd928754519fe06febb83cd987d9f3691fc9bb89f5aad7c7e1351b0","original_unit_ids":["u009","u010","u011","u012"],"locator":"body:L32-L46","original_claim_id":"hw-tmem-original-c005","original_claim":"Every 128x256 FP32 accumulator is a direct 128-thread-row by 256-column rectangle addressed through a TMEM descriptor.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-data-path-layout-organization","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"data-path layouts and TMEM address format","observation":"D layout depends on MMA kind/shape/layout and operands use a 32-bit lane/column taddr, not a TMEM descriptor."}],"strongest_alternative":"The diagram is merely capacity arithmetic for one common FP32 form.","discriminating_result":"It labels rows as owned CUDA threads and states descriptor addressing as general architecture, which conflicts with the normative model.","action":"correct","action_reason":"Retain capacity arithmetic separately from instruction-specific layout.","proposed_replacement":"A TMEM address encodes lane and column; the MMA descriptor and kind determine how logical D maps to those cells.","replacement_claim_ids":["hw-tmem-current-c002"]} +{"receipt_id":"rem-hw-tmem-004","path":"wiki/hardware/tmem.md","original_body_sha256":"31c6ed49bcd928754519fe06febb83cd987d9f3691fc9bb89f5aad7c7e1351b0","original_unit_ids":["u014","u019","u021"],"locator":"body:L54-L95","original_claim_id":"hw-tmem-original-c007","original_claim":"One thread may issue cta_group::1 TMEM allocation and deallocation.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"alloc/dealloc issue granularity","observation":"The operations are warp-collective for cta_group::1 and require all lanes to execute identically."}],"strongest_alternative":"Only the shared-memory destination store is collective, so lane 0 may issue.","discriminating_result":"The .sync.aligned execution requirement applies to the instruction itself, not just result visibility.","action":"delete","action_reason":"Remove unsafe inline PTX wrappers and state the collective contract directly.","proposed_replacement":null,"replacement_claim_ids":["hw-tmem-current-c003"]} +{"receipt_id":"rem-hw-tmem-005","path":"wiki/hardware/tmem.md","original_body_sha256":"31c6ed49bcd928754519fe06febb83cd987d9f3691fc9bb89f5aad7c7e1351b0","original_unit_ids":["u020"],"locator":"body:L80","original_claim_id":"hw-tmem-original-c009","original_claim":"tcgen05.alloc returns a base column index.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory-addressing","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"32-bit TMEM address format","observation":"The b32 taddr encodes lane in bits 31:16 and column in bits 15:0."}],"strongest_alternative":"An allocation base always has lane zero, so its numeric value equals a column.","discriminating_result":"Even at lane zero, the operand's normative type and later address arithmetic are a TMEM address, not an unrestricted column integer.","action":"correct","action_reason":"Preserve the full address model.","proposed_replacement":"Allocation writes a 32-bit base taddr to shared memory; taddr encodes lane and column fields.","replacement_claim_ids":["hw-tmem-current-c003"]} +{"receipt_id":"rem-hw-tmem-006","path":"wiki/hardware/tmem.md","original_body_sha256":"31c6ed49bcd928754519fe06febb83cd987d9f3691fc9bb89f5aad7c7e1351b0","original_unit_ids":["u022","u023"],"locator":"body:L97-L128","original_claim_id":"hw-tmem-original-c010","original_claim":"Failure to deallocate necessarily leaks into later CTAs and hangs persistent kernels, while fence plus __syncthreads completes the shown loop.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"allocation lifecycle, commit, and fence","observation":"The ISA mandates deallocation before kernel exit without specifying those failure modes; commit plus mbarrier, not fence, provides completion."}],"strongest_alternative":"The warning describes the practical manifestation and the fence implicitly drains MMA.","discriminating_result":"The normative fence section grants ordering only and the separate commit section owns completion; the stated leak behavior is not guaranteed.","action":"delete","action_reason":"Remove a fabricated incomplete kernel and replace it with a normative lifecycle sequence.","proposed_replacement":null,"replacement_claim_ids":["hw-tmem-current-c003","hw-tmem-current-c005"]} +{"receipt_id":"rem-hw-tmem-007","path":"wiki/hardware/tmem.md","original_body_sha256":"31c6ed49bcd928754519fe06febb83cd987d9f3691fc9bb89f5aad7c7e1351b0","original_unit_ids":["u024","u025","u026"],"locator":"body:L134-L195","original_claim_id":"hw-tmem-original-c011","original_claim":"Scalar .32x1b tcgen05.ld/st wrappers and their zero-fill loop are valid.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"ld/st shape grammar and access semantics","observation":".32x1b is absent; the documented shaped operations are warp-collective and asynchronous."}],"strongest_alternative":"ptxas accepts .32x1b as an undocumented scalar alias.","discriminating_result":"Neither PTX 8.7 nor 9.0 grammar contains the token, while both enumerate the valid shapes explicitly.","action":"delete","action_reason":"Do not publish non-assembling executable-looking wrappers.","proposed_replacement":null,"replacement_claim_ids":["hw-tmem-current-c006"]} +{"receipt_id":"rem-hw-tmem-008","path":"wiki/hardware/tmem.md","original_body_sha256":"31c6ed49bcd928754519fe06febb83cd987d9f3691fc9bb89f5aad7c7e1351b0","original_unit_ids":["u031"],"locator":"body:L216-L250","original_claim_id":"hw-tmem-original-c014","original_claim":"The displayed two-buffer kernel performs real epilogue/MMA overlap with valid allocation, zeroing, and completion.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_6.py","revision":"CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"PipelineUmmaAsync producer/consumer loops","observation":"The real pipeline uses distinct participants and barriers; the page serializes its body, calls invalid helpers, and only comments that overlap happened."}],"strongest_alternative":"A compiler can overlap the serial-looking epilogue and the following iteration automatically.","discriminating_result":"There is no concurrent producer/consumer schedule or valid async completion state in the displayed control flow.","action":"delete","action_reason":"Replace invented code with resource and synchronization requirements plus a pinned real example.","proposed_replacement":null,"replacement_claim_ids":["hw-tmem-current-c008"]} +{"receipt_id":"rem-hw-tmem-009","path":"wiki/hardware/tmem.md","original_body_sha256":"31c6ed49bcd928754519fe06febb83cd987d9f3691fc9bb89f5aad7c7e1351b0","original_unit_ids":["u034"],"locator":"body:L259","original_claim_id":"hw-tmem-original-c016","original_claim":"Two exact 192-column buffers are legal and leave 128 columns.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"nCols constraints","observation":"nCols must be a power of two from 32 through 512, so 192 is invalid."}],"strongest_alternative":"Multiple-of-32 is sufficient even when the prose also lists powers of two.","discriminating_result":"The operand constraint requires both the 32-column unit and a power-of-two count.","action":"correct","action_reason":"List only allocatable budgets and distinguish logical suballocation from hardware allocation.","proposed_replacement":"Legal reservation sizes are 32, 64, 128, 256, and 512 columns; a 192-column logical need requires a 256-column reservation or suballocation within one.","replacement_claim_ids":["hw-tmem-current-c004"]} +{"receipt_id":"rem-hw-tmem-010","path":"wiki/hardware/tmem.md","original_body_sha256":"31c6ed49bcd928754519fe06febb83cd987d9f3691fc9bb89f5aad7c7e1351b0","original_unit_ids":["u038","u039","u040","u041","u042","u049"],"locator":"body:L267-L275,L307","original_claim_id":"hw-tmem-original-c017","original_claim":"Published measurements establish a general 420-cycle TMEM cache-miss latency versus 30-cycle SMEM and 4-cycle registers and justify categorical best-use guidance.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/pdf/2512.02189v3","revision":"arXiv:2512.02189v3, 2026-03-02","locator":"Section V-A","observation":"The current author revision no longer contains the 420-cycle claim and never supplies the page's matched 420/30/4 comparison."},{"url":"https://arxiv.org/abs/2512.02189","revision":"submission history through v3","locator":"v1-v3 history","observation":"The source was revised after v1, establishing a superseding version boundary."}],"strongest_alternative":"The removed v1 measurement remains valid and the other two numbers are standard reference latencies.","discriminating_result":"The authors withdrew the figure from current v3 and the three values have no shared device, instruction shape, dependency chain, clock, or benchmark conditions.","action":"delete","action_reason":"A stale, heterogeneous comparison cannot support general performance guidance.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-hw-tmem-011","path":"wiki/hardware/tmem.md","original_body_sha256":"31c6ed49bcd928754519fe06febb83cd987d9f3691fc9bb89f5aad7c7e1351b0","original_unit_ids":["u044"],"locator":"body:L281-L299","original_claim_id":"hw-tmem-original-c019","original_claim":"The shown Layout(memory_space=MemorySpace.TMEM), tmem_alloc, tcgen05_mma, tcgen05_fence, and tmem_load code is valid CUTLASS 4.5.0 Python DSL.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py","revision":"CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"TMEM allocation, accumulator binding, and epilogue","observation":"The official API uses TmemAllocator, wait/retrieve, make_tensor, tiled MMA, and tcgen05 copy atoms; the displayed names are absent."}],"strongest_alternative":"The snippet targets another supported high-level convenience API in the same release.","discriminating_result":"Repository-wide exact-token search finds none of its purported API calls, while the version-pinned tutorial demonstrates the public path.","action":"delete","action_reason":"Link a complete version-pinned tutorial rather than fabricate a simplified API.","proposed_replacement":null,"replacement_claim_ids":["hw-tmem-current-c009"]} +{"receipt_id":"rem-hw-tmem-012","path":"wiki/hardware/tmem.md","original_body_sha256":"31c6ed49bcd928754519fe06febb83cd987d9f3691fc9bb89f5aad7c7e1351b0","original_unit_ids":["u045","u046","u047"],"locator":"body:L303-L305","original_claim_id":"hw-tmem-original-c020","original_claim":"Fence completes TMEM MMA, allocation must be freed before each persistent tile, and allocating over 512 silently corrupts or hangs.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"commit, fence, and allocation constraints","observation":"Commit/mbarrier provides completion; allocation may span tiles but must end before kernel exit; over 512 violates the operand constraint without the asserted failure mode."}],"strongest_alternative":"The list intentionally gives conservative practical symptoms rather than normative outcomes.","discriminating_result":"It prescribes the wrong completion primitive and unnecessary per-tile deallocation, so it can break or slow correct code.","action":"correct","action_reason":"Replace folklore failure modes with normative obligations.","proposed_replacement":"Use commit/mbarrier for completion, fences for documented handoffs, legal power-of-two allocation sizes, and deallocate every allocation before exit.","replacement_claim_ids":["hw-tmem-current-c003","hw-tmem-current-c004","hw-tmem-current-c005"]} +{"receipt_id":"rem-hw-tmem-013","path":"wiki/hardware/tmem.md","original_body_sha256":"31c6ed49bcd928754519fe06febb83cd987d9f3691fc9bb89f5aad7c7e1351b0","original_unit_ids":["u048"],"locator":"body:L306","original_claim_id":"hw-tmem-original-c021","original_claim":"Threads independently own rows and can use either warp shuffle or SMEM to access another warp's rows.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"warp-collective access restrictions","observation":"TMEM accesses are collective and lane-shape mapped; warp shuffle is intra-warp and cannot retrieve another warp's values."}],"strongest_alternative":"The phrase shuffle includes a multi-step chain through intermediate warps.","discriminating_result":"CUDA shuffle scope never crosses the issuing warp, so shared state or another explicit inter-warp mechanism is still required.","action":"correct","action_reason":"State direct warp access restrictions and avoid a false communication primitive.","proposed_replacement":"Each participating warp accesses its documented lane chunk collectively; exchange across warps through an explicitly synchronized mechanism such as shared memory.","replacement_claim_ids":["hw-tmem-current-c002"]} +{"receipt_id":"rem-source-tmem-001","receipt_scope":"source-trail","path":"sources/blogs/colfax-cutlass-blackwell.md","original_body_sha256":"41c68b9025de36a26cc640beb97a97b6695c7f918b1e4cf555df7372cdd44afe","original_unit_ids":[],"locator":"Key Code: TMEM allocation + tcgen05.mma; TMEM load","original_claim_id":"source-colfax-tmem-original-c001","original_claim":"The local Colfax summary presents lane-0 allocation, a shortened scalar-mask MMA form, and an unsynchronized TMEM load as valid CUDA PTX.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"alloc issue, MMA operand grammar, and ld completion","observation":"Allocation is warp-collective, MMA requires a disable-output-lane vector, and asynchronous TMEM loads have explicit wait forms."}],"strongest_alternative":"The examples are abbreviated pseudocode distilled from the linked tutorial.","discriminating_result":"They are fenced as CUDA, use asm volatile, and omit correctness-critical collective and completion semantics.","action":"correct","action_reason":"Keep the source summary non-executable and point readers to exact version-pinned implementations.","proposed_replacement":"Summarize collective allocation, single-thread MMA issue, and collective TMEM drain while linking exact PTX and CUTLASS examples.","replacement_claim_ids":["hw-tmem-current-c003","hw-tmem-current-c005","hw-tmem-current-c009"]} +{"receipt_id":"rem-hw-mbarrier-001","path":"wiki/hardware/mbarrier.md","original_body_sha256":"3abf0f9b53646c33d53214d726c4d11ecf825d2ce9393225f20fd6d3e0fafe27","original_unit_ids":["u001"],"locator":"body:L5","original_claim_id":"hw-mbarrier-original-c001","original_claim":"mbarrier was introduced on Hopper and is universally essential for Blackwell warp-specialized kernels.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-init","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"ISA and target notes","observation":"mbarrier began in PTX 7.0 on sm_80; Hopper added transaction-count features in PTX 8.0."}],"strongest_alternative":"The page means the transaction-tracking form of mbarrier, not the base object.","discriminating_result":"It attributes the entire primitive to Hopper without narrowing to expect_tx/complete_tx, and 'essential' is not an ISA requirement.","action":"correct","action_reason":"Separate the Ampere base primitive from Hopper transaction-count extensions and avoid a universal design mandate.","proposed_replacement":"mbarrier is a 64-bit shared-memory object available since sm_80; sm_90 added tx-count tracking used by TMA, and Blackwell also uses mbarriers for tcgen05 completion.","replacement_claim_ids":["hw-mbarrier-current-c001"]} +{"receipt_id":"rem-hw-mbarrier-002","path":"wiki/hardware/mbarrier.md","original_body_sha256":"3abf0f9b53646c33d53214d726c4d11ecf825d2ce9393225f20fd6d3e0fafe27","original_unit_ids":["u002"],"locator":"body:L9-L24","original_claim_id":"hw-mbarrier-original-c002","original_claim":"After arrive.expect_tx, TMA completes by arriving with a byte count and then flips parity.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#phase-completion-of-the-mbarrier-object","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"expect-tx, complete-tx, and phase-completion definitions","observation":"arrive.expect_tx decrements pending arrivals and raises tx-count; TMA complete-tx lowers tx-count; phase advances only after both counts reach zero."}],"strongest_alternative":"complete-tx is merely another name for a byte-counted arrival.","discriminating_result":"The object tracks pending arrivals and tx-count as separate fields changed by different operations.","action":"correct","action_reason":"Preserve exact accounting semantics and avoid premature-completion bugs.","proposed_replacement":"Describe init, arrive, arrive.expect_tx, TMA complete-tx, and parity waits as separate operations with conjunctive completion conditions.","replacement_claim_ids":["hw-mbarrier-current-c002"]} +{"receipt_id":"rem-hw-mbarrier-003","path":"wiki/hardware/mbarrier.md","original_body_sha256":"3abf0f9b53646c33d53214d726c4d11ecf825d2ce9393225f20fd6d3e0fafe27","original_unit_ids":["u004"],"locator":"body:L30-L45","original_claim_id":"hw-mbarrier-original-c004","original_claim":"One global phase bit toggled every loop iteration correctly tracks every barrier in a multi-stage ring.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"phaseParity semantics and tracking requirement","observation":"Parity is the phase of each mbarrier object; a stage object's parity changes only when that stage completes."}],"strongest_alternative":"Every loop iteration completes all stage barriers, so a global bit remains aligned.","discriminating_result":"The code waits and arrives on only the selected stage; for an even stage count the global bit returns to its old value before the same object is reused.","action":"delete","action_reason":"Remove subtly unsafe pseudocode and give a per-stage reuse rule instead.","proposed_replacement":null,"replacement_claim_ids":["hw-mbarrier-current-c003"]} +{"receipt_id":"rem-hw-mbarrier-004","path":"wiki/hardware/mbarrier.md","original_body_sha256":"3abf0f9b53646c33d53214d726c4d11ecf825d2ce9393225f20fd6d3e0fafe27","original_unit_ids":["u006","u009"],"locator":"body:L51-L62,L68","original_claim_id":"hw-mbarrier-original-c006","original_claim":"The invented TMA helper code is reproducible and an extra software arrive races with a hardware arrival.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"cp.async.bulk.tensor grammar and complete-tx mechanism","observation":"The helpers omit required operands and are undefined; hardware performs complete-tx, not arrive-on, so extra arrive corrupts a different count."}],"strongest_alternative":"The code is clearly pseudocode and 'arrival' is harmless shorthand for any barrier signal.","discriminating_result":"The page labels it cuda and gives a specific double-count/race explanation that conflates two independently tracked counters.","action":"delete","action_reason":"Replace fabricated code with an exact accounting sequence and vendor grammar link.","proposed_replacement":null,"replacement_claim_ids":["hw-mbarrier-current-c004"]} +{"receipt_id":"rem-hw-mbarrier-005","path":"wiki/hardware/mbarrier.md","original_body_sha256":"3abf0f9b53646c33d53214d726c4d11ecf825d2ce9393225f20fd6d3e0fafe27","original_unit_ids":["u010"],"locator":"body:L69","original_claim_id":"hw-mbarrier-original-c009","original_claim":"shared::cluster alone is the qualifier for cluster-level mbarrier synchronization in two-SM MMA.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"shared-location support table and wait scope semantics","observation":".shared::cluster names an object's state space; .cluster names synchronization scope. Only selected operations support cluster-shared objects, while waits target CTA-shared objects, possibly through mapped remote addresses."}],"strongest_alternative":"State space and synchronization scope are intentionally coupled for every mbarrier operation.","discriminating_result":"The grammar exposes them as separate qualifiers and the support table permits different combinations per operation.","action":"correct","action_reason":"Teach object location, address mapping, operation support, and synchronization scope separately.","proposed_replacement":"Use .shared::cluster only for supported operations on cluster-shared locations, and select .cluster scope where the operation's synchronization contract requires it.","replacement_claim_ids":["hw-mbarrier-current-c005"]} +{"receipt_id":"rem-hw-nvfp4-001","path":"wiki/hardware/nvfp4.md","original_body_sha256":"b655cfe0784f6e74204d4748a18fb4f06f420b6ddcae893e9b1e8596105d1c1e","original_unit_ids":["u003","u004"],"locator":"body:L22-L23","original_claim_id":"hw-nvfp4-original-c003","original_claim":"The abbreviated tcgen05 variant strings completely identify MXFP4 and NVFP4 operations and each is generally 4x faster than Hopper.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"block-scale grammar and valid scale/block combinations","observation":"Exact forms require kind:: plus block16/block32 and scale-vector qualifiers; kind::mxf4nvf4 admits multiple scale modes, and the ISA supplies no universal Hopper throughput ratio."}],"strongest_alternative":"The strings are harmless shorthand and 4x follows directly from using 4-bit rather than 8-bit inputs.","discriminating_result":"The shorthand erases behavior-selecting qualifiers, and datatype width alone cannot establish instruction throughput, shape, utilization, or end-to-end speed against an architecture without native FP4.","action":"correct","action_reason":"Give exact distinguishing qualifiers and remove an unscoped performance number.","proposed_replacement":"Describe PTX 9.0 kind::mxf4 block32/UE8M0 and kind::mxf4nvf4 block16/UE4M3 combinations without a universal throughput claim.","replacement_claim_ids":["hw-nvfp4-current-c004"]} +{"receipt_id":"rem-hw-nvfp4-002","path":"wiki/hardware/nvfp4.md","original_body_sha256":"b655cfe0784f6e74204d4748a18fb4f06f420b6ddcae893e9b1e8596105d1c1e","original_unit_ids":["u005"],"locator":"body:L27-L33","original_claim_id":"hw-nvfp4-original-c005","original_claim":"The undeclared PTX fragment is reproducible as written and proves mov byte unpacking is faster than bitwise extraction.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"mov type-size rules and scalar-to-vector examples","observation":"The mov form is legal only with compatible declared byte destinations and a b32 source; the ISA gives no performance comparison, and the fragment omits declarations and benchmark conditions."}],"strongest_alternative":"Operand names make the types obvious and scalar-to-vector mov must compile to fewer instructions than masks and shifts.","discriminating_result":"PTX type checking depends on declarations, while source syntax does not determine final machine instruction count or latency after lowering.","action":"correct","action_reason":"Retain the exact packed conversion form, state mov's typed preconditions, and require measurement for packing-strategy performance.","proposed_replacement":"Show the normative cvt.rn.f16x2.e2m1x2 form and describe typed scalar-to-vector mov without asserting speed.","replacement_claim_ids":["hw-nvfp4-current-c005"]} +{"receipt_id":"rem-hw-nvfp4-003","path":"wiki/hardware/nvfp4.md","original_body_sha256":"b655cfe0784f6e74204d4748a18fb4f06f420b6ddcae893e9b1e8596105d1c1e","original_unit_ids":["u009"],"locator":"body:L42","original_claim_id":"hw-nvfp4-original-c007","original_claim":"NVFP4 universally has lower quantization error than MXFP4.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html","revision":"Transformer Engine 2.13","locator":"Comparison with MXFP8 and MXFP4","observation":"The recipe provides finer groups and fractional scales, but the vendor does not establish strict lower error for every tensor, scale-selection method, and metric."}],"strongest_alternative":"A smaller block and more precise scale mathematically guarantee a strict error reduction.","discriminating_result":"The richer scale family can improve or equal a suitably defined optimum, but strict ordering fails on exactly representable data and practical algorithms/metrics can produce other outcomes.","action":"correct","action_reason":"State the representational tradeoff without claiming an unconditional empirical ordering.","proposed_replacement":"NVFP4's finer blocks and fractional scale provide more scale freedom at greater metadata cost; actual error is workload- and algorithm-dependent.","replacement_claim_ids":["hw-nvfp4-current-c003"]} +{"receipt_id":"rem-hw-pdl-gdc-001","path":"wiki/hardware/pdl-gdc.md","original_body_sha256":"9783467bbb5c577667660ff8f39f2dd672f1bdcc6cd4729a4f491421bb3ba735","original_unit_ids":["u002"],"locator":"body:L7-L13","original_claim_id":"hw-pdl-gdc-original-c002","original_claim":"cudaGridDependencySynchronize is the primary grid's completion signal and is sufficient to enable PDL.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#programmatic-dependent-launch-and-synchronization","revision":"CUDA 13.0.2","locator":"6.2.8.6.2 API Description","observation":"The primary calls cudaTriggerProgrammaticLaunchCompletion, the secondary calls cudaGridDependencySynchronize, and the host opts the secondary launch in with cudaLaunchAttributeProgrammaticStreamSerialization."}],"strongest_alternative":"The two device functions are aliases and architecture defaults supply the missing host attribute.","discriminating_result":"The official example assigns distinct producer and consumer roles and explicitly constructs the launch attribute on every supported architecture.","action":"correct","action_reason":"Replace the reversed, incomplete example with the three-part protocol.","proposed_replacement":"Show the primary trigger, secondary wait, and extensible secondary-launch attribute in their correct locations.","replacement_claim_ids":["hw-pdl-gdc-current-c002"]} +{"receipt_id":"rem-hw-pdl-gdc-002","path":"wiki/hardware/pdl-gdc.md","original_body_sha256":"9783467bbb5c577667660ff8f39f2dd672f1bdcc6cd4729a4f491421bb3ba735","original_unit_ids":["u003"],"locator":"body:L17","original_claim_id":"hw-pdl-gdc-original-c003","original_claim":"PDL is globally enabled by default on SM100 with no application opt-in, unlike SM90.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#programmatic-dependent-launch-and-synchronization","revision":"CUDA 13.0.2","locator":"availability and programmaticStreamSerializationAllowed attribute","observation":"The same explicit PDL launch contract applies to compute capability 9.0 and newer."},{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/CMakeLists.txt#L463-L474","revision":"CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"CUTLASS_ENABLE_GDC_FOR_SM100 option","observation":"Only CUTLASS's SM100 compile option defaults ON; this is a configurable library build choice."}],"strongest_alternative":"CUTLASS's default demonstrates that all SM100 launches automatically use PDL.","discriminating_result":"A build macro only emits GDC wrappers in eligible CUTLASS code; the CUDA runtime still needs a programmatic dependency and ordinary user kernels are unchanged.","action":"correct","action_reason":"Separate hardware availability, CUDA launch opt-in, and one library's build default.","proposed_replacement":"PDL is available on sm_90+ and requires the CUDA protocol; CUTLASS 4.5.0 separately defaults its SM100 GDC build option ON.","replacement_claim_ids":["hw-pdl-gdc-current-c001","hw-pdl-gdc-current-c004"]} +{"receipt_id":"rem-hw-pdl-gdc-003","path":"wiki/hardware/pdl-gdc.md","original_body_sha256":"9783467bbb5c577667660ff8f39f2dd672f1bdcc6cd4729a4f491421bb3ba735","original_unit_ids":["u004","u005","u006"],"locator":"body:L18-L20","original_claim_id":"hw-pdl-gdc-original-c004","original_claim":"Ordinary back-to-back SM100 launches naturally overlap, generic memory fences ensure dependent-data correctness, and launch gaps are reduced.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#programmatic-dependent-launch-and-synchronization","revision":"CUDA 13.0.2","locator":"early-launch attribute, dependency wait, and concurrency caveat","observation":"Early launch requires configured PDL, dependent reads require a dependency wait, and actual concurrent execution is opportunistic."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-griddepcontrol","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"griddepcontrol.wait semantics","observation":"The wait supplies prerequisite completion and memory visibility; a generic fence is not its substitute."}],"strongest_alternative":"The driver can recognize back-to-back dependencies and insert the required fences automatically on Blackwell.","discriminating_result":"The documented interface requires the application to expose the dependency and place the wait; it expressly does not guarantee overlap.","action":"correct","action_reason":"State the exact ordering contract and make overlap conditional.","proposed_replacement":"Configured PDL may overlap independent tails/preambles; dependent work must execute after cudaGridDependencySynchronize or griddepcontrol.wait.","replacement_claim_ids":["hw-pdl-gdc-current-c001","hw-pdl-gdc-current-c002","hw-pdl-gdc-current-c003"]} +{"receipt_id":"rem-hw-pdl-gdc-004","path":"wiki/hardware/pdl-gdc.md","original_body_sha256":"9783467bbb5c577667660ff8f39f2dd672f1bdcc6cd4729a4f491421bb3ba735","original_unit_ids":["u007","u008","u009"],"locator":"body:L23-L25","original_claim_id":"hw-pdl-gdc-original-c005","original_claim":"Small-kernel chains and pipeline-parallel launches necessarily benefit, reducing Blackwell wall time without code changes.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#programmatic-dependent-launch-and-synchronization","revision":"CUDA 13.0.2","locator":"benefit preconditions, required API changes, and opportunistic concurrency warning","observation":"Overlap depends on independent work and available resources; the official example changes primary, secondary, and host launch code, and gives no guaranteed speedup."}],"strongest_alternative":"Those workload labels imply enough independent work, and a library can hide every required change.","discriminating_result":"The labels do not determine data-ready points or resource coexistence; even a library integration is still code/configuration behavior and must be measured.","action":"correct","action_reason":"Replace categorical workload promises with a decision and measurement checklist.","proposed_replacement":"Use PDL only when independent tail/preamble work and resource headroom exist; verify overlap and wall time on the target configuration.","replacement_claim_ids":["hw-pdl-gdc-current-c005"]} +{"receipt_id":"rem-source-pdl-gdc-001","receipt_scope":"source-trail","path":"sources/prs/cutlass/PR-2161.md","original_body_sha256":"30bec8e67474e2f1b1f0100d88a6a812c2b598a6a0b31daebacb3995fbb162d8","original_unit_ids":[],"locator":"Summary, Problem item 2, and Performance","original_claim_id":"source-cutlass-2161-pdl-performance","original_claim":"The PR establishes that PDL generally reduces launch overhead/performance for persistent Blackwell GEMMs and that improvements were validated through CUTLASS gemm.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/pull/2161","revision":"merge commit 06e560d9","locator":"changed files and discussion","observation":"The change enables an SM100 GDC build path, but the local summary supplies no device, shapes, commands, baseline, or numerical results for its claimed validation."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#programmatic-dependent-launch-and-synchronization","revision":"CUDA 13.0.2","locator":"opportunistic concurrency warning","observation":"NVIDIA explicitly states that PDL creates an opportunity rather than a concurrency or performance guarantee."}],"strongest_alternative":"A merged optimization PR and its generic test suite are adequate evidence of a general speedup.","discriminating_result":"Correctness/enablement tests cannot quantify performance without a scoped benchmark, and the official contract makes overlap conditional.","action":"correct","action_reason":"Keep the exact build/configuration change while removing invented or unscoped performance validation.","proposed_replacement":"State that the PR added SM100 GDC enablement and the default-ON CUTLASS CMake option; require workload-specific profiling for benefit.","replacement_claim_ids":["hw-pdl-gdc-current-c004","hw-pdl-gdc-current-c005"]} +{"receipt_id":"rem-hw-tma-001","path":"wiki/hardware/tma.md","original_body_sha256":"5a71bdad2d3319054af91f093a17db812ad57396120e6144d8f62535bed3e49f","original_unit_ids":["u001","u006","u013","u014","u025","u032"],"locator":"body:L5,L16,L79-L89,L295,L329","original_claim_id":"hw-tma-original-c002","original_claim":"Every Blackwell tcgen05 operand requires 128B TMA swizzling and non-128B layouts silently fail.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-layout","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"tcgen05 shared-memory descriptor swizzle modes","observation":"Valid layouts include none, 32B, 64B, and 128B subject to kind/type/shape constraints."}],"strongest_alternative":"All practical tcgen05 shapes happen to select 128B even if the descriptor encodes other modes.","discriminating_result":"The normative tables define legal non-128B configurations, so 128B is not a universal correctness condition.","action":"correct","action_reason":"Make swizzle selection descriptor- and shape-specific and remove the invented failure mode.","proposed_replacement":"Match the TMA swizzle to the exact tcgen05 shared-memory descriptor and use PTX's layout tables.","replacement_claim_ids":["hw-tma-current-c006"]} +{"receipt_id":"rem-hw-tma-002","path":"wiki/hardware/tma.md","original_body_sha256":"5a71bdad2d3319054af91f093a17db812ad57396120e6144d8f62535bed3e49f","original_unit_ids":["u002","u007"],"locator":"body:L7,L17","original_claim_id":"hw-tma-original-c004","original_claim":"Ordinary TMA copies clamp OOB coordinates and convert FP32 to/from FP16 or BF16.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html","revision":"CUDA Driver API 13.0.97","locator":"tensor datatype and oobFill fields","observation":"The datatype describes copied elements; OOB loads fill destination elements with supported fill values rather than clamping coordinates, and no general FP32/FP16/BF16 conversion is defined."}],"strongest_alternative":"The datatype enum necessarily requests conversion to the destination's element type.","discriminating_result":"One descriptor datatype describes both tensor traversal and copy representation; there is no separate source/destination type pair.","action":"correct","action_reason":"Describe representation-preserving copies and OOB fill precisely.","proposed_replacement":"TMA handles address traversal and fills OOB load elements; perform explicit format conversion elsewhere.","replacement_claim_ids":["hw-tma-current-c001","hw-tma-current-c002"]} +{"receipt_id":"rem-hw-tma-003","path":"wiki/hardware/tma.md","original_body_sha256":"5a71bdad2d3319054af91f093a17db812ad57396120e6144d8f62535bed3e49f","original_unit_ids":["u005","u009"],"locator":"body:L15,L19","original_claim_id":"hw-tma-original-c006","original_claim":"TMA has a universal 256-bytes-per-element/128x256-tile limit and all directions synchronize by mbarrier arrival/wait.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html","revision":"CUDA Driver API 13.0.97","locator":"cuTensorMapEncodeTiled boxDim constraints","observation":"Each box dimension is 1..256 elements with additional byte/type rules, not 256 bytes per element or one 2D maximum."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"completion mechanisms","observation":"Loads use mbarrier complete_tx; stores use bulk groups."}],"strongest_alternative":"The table compresses dimension and inner-span constraints into a memorable equivalent limit.","discriminating_result":"It changes units and loses rank/type dependence, while also assigning the wrong completion family to stores.","action":"correct","action_reason":"List parameter constraints and direction-specific completion separately.","proposed_replacement":"Use 1..256 box elements per dimension subject to encoder rules; loads use mbarrier tx-count and stores use bulk-group commit/wait.","replacement_claim_ids":["hw-tma-current-c001","hw-tma-current-c002","hw-tma-current-c003","hw-tma-current-c004"]} +{"receipt_id":"rem-hw-tma-004","path":"wiki/hardware/tma.md","original_body_sha256":"5a71bdad2d3319054af91f093a17db812ad57396120e6144d8f62535bed3e49f","original_unit_ids":["u011"],"locator":"body:L24","original_claim_id":"hw-tma-original-c009","original_claim":"Tensor maps must be created on the host and passed into Blackwell kernels.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#encoding-a-tensor-map-on-device","revision":"CUDA 13.0.2","locator":"device-side encoding and modification","observation":"Blackwell supports device-side tensor-map encoding/modification with proxy-fence obligations."}],"strongest_alternative":"Device-side support only replaces the base address of a host-created map.","discriminating_result":"The documented Blackwell path can construct and modify tensor-map fields on device, beyond merely consuming a host value.","action":"correct","action_reason":"Present host encoding as common, not exclusive, and mention device proxy ordering.","proposed_replacement":"Use host cuTensorMapEncodeTiled or the documented Blackwell device-side path; publish modifications with the required proxy fence.","replacement_claim_ids":["hw-tma-current-c002"]} +{"receipt_id":"rem-hw-tma-005","path":"wiki/hardware/tma.md","original_body_sha256":"5a71bdad2d3319054af91f093a17db812ad57396120e6144d8f62535bed3e49f","original_unit_ids":["u012"],"locator":"body:L28-L75","original_claim_id":"hw-tma-original-c010","original_claim":"The displayed descriptor factory safely supports its advertised arbitrary dimensions, tiles, pointers, and swizzle integers.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html","revision":"CUDA Driver API 13.0.97","locator":"cuTensorMapEncodeTiled requirements and CUresult","observation":"The code validates none of the address/stride/box/swizzle rules, ignores failure, leaves an enum uninitialized for other integers, and omits the half definition."}],"strongest_alternative":"Callers are implicitly required to pass only legal values, making validation unnecessary.","discriminating_result":"The function signature and comments advertise unrestricted ints and return a descriptor even when the API reports failure, so invalid state escapes silently.","action":"delete","action_reason":"Replace a deceptively general factory with a constraints checklist and official complete sample.","proposed_replacement":null,"replacement_claim_ids":["hw-tma-current-c002","hw-tma-current-c008"]} +{"receipt_id":"rem-hw-tma-006","path":"wiki/hardware/tma.md","original_body_sha256":"5a71bdad2d3319054af91f093a17db812ad57396120e6144d8f62535bed3e49f","original_unit_ids":["u015"],"locator":"body:L95-L125","original_claim_id":"hw-tma-original-c011","original_claim":"The displayed load helper correctly accounts a shared barrier when called once for A and once for B.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#phase-completion-of-the-mbarrier-object","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"arrive.expect_tx composition","observation":"Each helper call both arrives and adds bytes, so the two-call use performs two arrivals without a matching initialized count or protected phase; it also uses undefined constants."}],"strongest_alternative":"Each TMA copy should perform its own arrival, so two loads naturally need two calls.","discriminating_result":"TMA completions change tx-count, while software arrivals are a separate phase counter; the displayed barrier lifecycle never establishes two pending arrivals.","action":"delete","action_reason":"Teach one explicit phase-accounting sequence instead of an unsafe reusable helper.","proposed_replacement":null,"replacement_claim_ids":["hw-tma-current-c003"]} +{"receipt_id":"rem-hw-tma-007","path":"wiki/hardware/tma.md","original_body_sha256":"5a71bdad2d3319054af91f093a17db812ad57396120e6144d8f62535bed3e49f","original_unit_ids":["u016"],"locator":"body:L129-L150","original_claim_id":"hw-tma-original-c012","original_claim":"The displayed store PTX is valid and commit_group completes the store.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"shared-to-global grammar and bulk-group waits","observation":"The instruction is missing mandatory .bulk_group; commit only creates a group, while wait_group establishes completion and safe source reuse."}],"strongest_alternative":"bulk_group is an implied default and commit is a blocking commit.","discriminating_result":"The grammar names bulk_group as the completion mechanism and documents commit_group as grouping, with a separate wait instruction for completion.","action":"correct","action_reason":"Show the required qualifier and explicit wait boundary.","proposed_replacement":"Issue .global.shared::cta.bulk_group, commit the group, and wait before reusing source shared memory or depending on completion.","replacement_claim_ids":["hw-tma-current-c004"]} +{"receipt_id":"rem-hw-tma-008","path":"wiki/hardware/tma.md","original_body_sha256":"5a71bdad2d3319054af91f093a17db812ad57396120e6144d8f62535bed3e49f","original_unit_ids":["u017","u018","u019"],"locator":"body:L154-L157","original_claim_id":"hw-tma-original-c013","original_claim":"TMA completion arrives at the barrier and decrements expected transaction count.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"mbarrier::complete_tx::bytes semantics","observation":"TMA decrements tx-count by completed bytes and performs no arrive-on; expected arrivals are a distinct barrier field."}],"strongest_alternative":"Complete-tx is simply the byte-valued form of arrival.","discriminating_result":"The mbarrier object maintains pending arrivals and tx-count separately, and phase completion requires both zero.","action":"correct","action_reason":"Separate software arrival from hardware byte completion.","proposed_replacement":"Producer performs arrive.expect_tx once for the phase total; each TMA load performs complete_tx bytes; consumer waits for the phase.","replacement_claim_ids":["hw-tma-current-c003"]} +{"receipt_id":"rem-hw-tma-009","path":"wiki/hardware/tma.md","original_body_sha256":"5a71bdad2d3319054af91f093a17db812ad57396120e6144d8f62535bed3e49f","original_unit_ids":["u020"],"locator":"body:L161-L208","original_claim_id":"hw-tma-original-c015","original_claim":"The invented mainloop is a correct reproducible multistage TMA/tcgen05 pipeline with safe reuse.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-dRC5i4/cutlass/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_6.py","revision":"CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"PipelineTmaUmma producer/consumer loops","observation":"The real pipeline has per-stage full/empty state, correct byte accounting, and release after asynchronous MMA stops reading; the page lacks the empty handoff, double-arrives, calls wait with the wrong signature, and uses invalid placeholders."}],"strongest_alternative":"MMA consumes shared operands at issue, so the next producer may overwrite immediately after issue.","discriminating_result":"tcgen05.mma is asynchronous and its input-buffer lifetime extends until the documented completion/ordering handoff.","action":"delete","action_reason":"Replace unrepairable pseudocode with invariant-based stage lifecycle and a pinned complete implementation.","proposed_replacement":null,"replacement_claim_ids":["hw-tma-current-c007","hw-tma-current-c008"]} +{"receipt_id":"rem-hw-tma-010","path":"wiki/hardware/tma.md","original_body_sha256":"5a71bdad2d3319054af91f093a17db812ad57396120e6144d8f62535bed3e49f","original_unit_ids":["u021"],"locator":"body:L212-L239","original_claim_id":"hw-tma-original-c016","original_claim":"The barrier helpers provide a complete reusable phase abstraction.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"phase lifetime, ordering, retry, and invalidation rules","observation":"The code does not publish initialization, maintain per-object parity, choose acquire/release semantics, invalidate before reuse, or provide collision-safe inline labels."}],"strongest_alternative":"Callers supply phase and external synchronization, so the omissions are intentionally outside the helper.","discriminating_result":"The surrounding pipeline calls the helper without phase and documents it as the synchronization implementation, making those missing obligations observable defects.","action":"delete","action_reason":"Refer to the verified mbarrier page and state lifecycle requirements rather than ship a partial abstraction.","proposed_replacement":null,"replacement_claim_ids":["hw-tma-current-c003","hw-tma-current-c007"]} +{"receipt_id":"rem-hw-tma-011","path":"wiki/hardware/tma.md","original_body_sha256":"5a71bdad2d3319054af91f093a17db812ad57396120e6144d8f62535bed3e49f","original_unit_ids":["u022","u023"],"locator":"body:L243-L275","original_claim_id":"hw-tma-original-c018","original_claim":"Multicast is critical for GEMM and the helper elects one issuer with correct barrier state for every destination.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"multicast ctaMask and destination barrier semantics","observation":"threadIdx.x==0 elects one thread per CTA, not one CTA per cluster, and the code never establishes corresponding initialized barrier state for every masked destination; GEMMs without cross-CTA reuse do not require multicast."}],"strongest_alternative":"Only CTA0 reaches the helper in the intended caller even though the function does not say so.","discriminating_result":"The helper's interface and comments claim cluster-wide correctness but encode no rank predicate or caller precondition.","action":"delete","action_reason":"Replace incomplete executable code with the cluster ownership and barrier invariants.","proposed_replacement":null,"replacement_claim_ids":["hw-tma-current-c005"]} +{"receipt_id":"rem-hw-tma-012","path":"wiki/hardware/tma.md","original_body_sha256":"5a71bdad2d3319054af91f093a17db812ad57396120e6144d8f62535bed3e49f","original_unit_ids":["u024"],"locator":"body:L279-L289","original_claim_id":"hw-tma-original-c020","original_claim":"One two-CTA multicast exactly halves global-memory bandwidth.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"multicast functional semantics","observation":"The ISA defines one masked operation but no exact measured DRAM/global-bandwidth ratio; caches, transaction behavior, masks, and baseline reuse matter."}],"strongest_alternative":"One request serving two destinations proves exactly half the traffic by definition.","discriminating_result":"Instruction request count is not total measured memory traffic, and a non-multicast baseline can already receive cache reuse.","action":"correct","action_reason":"State logical load reuse without an unmeasured hardware-traffic factor.","proposed_replacement":"Multicast can reduce duplicate source requests for a shared tile; measure traffic and time against the actual baseline.","replacement_claim_ids":["hw-tma-current-c005","hw-tma-current-c007"]} +{"receipt_id":"rem-hw-tma-013","path":"wiki/hardware/tma.md","original_body_sha256":"5a71bdad2d3319054af91f093a17db812ad57396120e6144d8f62535bed3e49f","original_unit_ids":["u026","u027"],"locator":"body:L297-L309","original_claim_id":"hw-tma-original-c021","original_claim":"The informal per-row XOR diagram exactly defines 128B swizzling and guarantees elimination of tensor-core bank conflicts.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#the-swizzle-modes","revision":"CUDA 13.0.2","locator":"swizzle mapping and base-offset rules","observation":"The actual mapping operates on chunks with address-dependent base offset and matching consumer indexing; conflict avoidance is access-pattern-specific."}],"strongest_alternative":"The omitted XOR is only pedagogical detail and any 128B swizzle is inherently conflict-free.","discriminating_result":"Using linear consumer indices on swizzled data changes logical values, and other access patterns can still collide; exact mapping is correctness-critical.","action":"delete","action_reason":"Link the normative mapping and explain the producer/consumer layout match.","proposed_replacement":null,"replacement_claim_ids":["hw-tma-current-c006"]} +{"receipt_id":"rem-hw-tma-014","path":"wiki/hardware/tma.md","original_body_sha256":"5a71bdad2d3319054af91f093a17db812ad57396120e6144d8f62535bed3e49f","original_unit_ids":["u030","u031","u032","u033","u034"],"locator":"body:L327-L331","original_claim_id":"hw-tma-original-c023","original_claim":"Back-to-back issue, exact cluster-size bandwidth savings, universal 128B swizzle, 2D preference, and a 3-5-stage optimum are general TMA rules.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#asynchronous-data-copies-using-the-tensor-memory-accelerator-tma","revision":"CUDA 13.0.2","locator":"TMA mechanisms and swizzle guidance","observation":"The guide supplies no universal stage optimum or performance ordering and makes layout, resource, and access-pattern choices conditional."}],"strongest_alternative":"These are accepted tuning heuristics and need not be universal to be useful.","discriminating_result":"The table states categorical correctness and numeric behavior; several entries are false even as defaults, and an unscoped stage count can exceed resource limits or underfill latency.","action":"correct","action_reason":"Replace folklore with a measurement and resource checklist.","proposed_replacement":"Choose rank, swizzle, multicast, issue cadence, and stage depth from access pattern, descriptor constraints, resource budget, and target measurements.","replacement_claim_ids":["hw-tma-current-c007"]} +{"receipt_id":"rem-hw-tma-015","path":"wiki/hardware/tma.md","original_body_sha256":"5a71bdad2d3319054af91f093a17db812ad57396120e6144d8f62535bed3e49f","original_unit_ids":["u035"],"locator":"body:L335-L358","original_claim_id":"hw-tma-original-c025","original_claim":"The shown make_tma_copy/SM100_TMA_LOAD_2D Python is valid CUTLASS 4.5.0 CuTe DSL.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-dRC5i4/cutlass/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_tma/tma_v0.py","revision":"CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"TMA atom construction","observation":"The actual API uses cutlass.cute.nvgpu.cpasync operations and make_tiled_tma_atom; exact-token search finds none of the page's constructors or operation names."}],"strongest_alternative":"The snippet targets a convenience alias re-exported by an older or newer CuTe package.","discriminating_result":"The page has no version boundary, while its own source set targets current Blackwell tooling and the pinned public release contains none of those symbols.","action":"delete","action_reason":"Link a complete pinned tutorial rather than fabricate a simplified API.","proposed_replacement":null,"replacement_claim_ids":["hw-tma-current-c008"]} +{"receipt_id":"rem-source-tma-001","receipt_scope":"source-trail","path":"sources/docs/nvidia-ptx-isa-sm100.md","original_body_sha256":"f19f5b6800dd2b3cdf992e3689b32cd669c1621b1f293df2e869743cd6c60d26","original_unit_ids":[],"locator":"TMA Instructions (SM100 Enhanced), Alignment Requirements","original_claim_id":"source-ptx-sm100-tma-summary","original_claim":"TMA is an SM100-enhanced instruction family whose displayed dst_/src_ spellings are PTX and whose tensor maps, shared destinations, and global sources all require 128B alignment.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"CUDA 13.0.2 PTX ISA 9.0","locator":"exact grammar, history, and operand restrictions","observation":"TMA began at sm_90; valid spellings use .shared::cta/.global and direction-specific completion. Tensor maps are 64B aligned, tiled global addresses/strides generally 16B aligned, and destination alignment depends on swizzle/base rules rather than one 128B rule."}],"strongest_alternative":"The source intentionally uses descriptive pseudomnemonics and conservative alignment advice.","discriminating_result":"It fences the forms as asm and labels them instructions, while the 128B descriptor claim contradicts the API's 64B requirement and can reject valid layouts.","action":"correct","action_reason":"Replace pseudoinstructions and false global alignment rules with version-pinned grammar and constraint links.","proposed_replacement":"Show exact PTX 9.0 load/store/multicast forms, Hopper introduction, direction-specific completion, and scoped alignment requirements.","replacement_claim_ids":["hw-tma-current-c001","hw-tma-current-c002","hw-tma-current-c003","hw-tma-current-c004","hw-tma-current-c005"]} +{"receipt_id":"rem-kernel-deepgemm-001","path":"wiki/kernels/deepgemm.md","original_body_sha256":"6ab7a73798de1e059f397018969dab63e94f888f714964e75ff05cd82a5ba7a3","original_unit_ids":["u001"],"locator":"body:L5","original_claim_id":"kernel-deepgemm-original-c002","original_claim":"DeepGEMM's core kernel is about 300 lines and achieves about 90% utilization on H800.","original_verdict":"refuted","evidence":[{"path":"artifacts/kernels/deepgemm/full/","revision":"DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"pinned SM90 and SM100 files plus README news item","observation":"The two selected templates are 346 and 567 lines, and the upstream performance statement gives no 90% utilization."}],"strongest_alternative":"The phrase refers to an older single-kernel implementation and a separately documented efficiency result.","discriminating_result":"The page presents both as properties of its pinned two-architecture source, but that source is larger and contains no 90% statement.","action":"correct","action_reason":"Scope the overview to the pinned library and separate the surviving source-reported peak from unsupported utilization and size claims.","proposed_replacement":"Describe the pinned multi-kernel library without a line-count or utilization claim.","replacement_claim_ids":["kernel-deepgemm-current-c001","kernel-deepgemm-current-c008"]} +{"receipt_id":"rem-kernel-deepgemm-002","path":"wiki/kernels/deepgemm.md","original_body_sha256":"6ab7a73798de1e059f397018969dab63e94f888f714964e75ff05cd82a5ba7a3","original_unit_ids":["u002","u003"],"locator":"body:L7,L11-L26","original_claim_id":"kernel-deepgemm-original-c004","original_claim":"The 1x128/128x128 scheme prevents outliers from destroying precision and always stores one FP32 scale per tile or block across both architectures.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/abs/2412.19437v2","revision":"DeepSeek-V3 Technical Report v2","locator":"section 3.3.2","observation":"The paper claims improved accommodation of outliers, not a precision guarantee."},{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md","revision":"DeepGEMM commit 891d57b","locator":"scaling-factor formats","observation":"SM90 uses FP32 factors while SM100 uses packed UE8M0 factors."}],"strongest_alternative":"The diagram is conceptual and FP32 names the pre-packed source scale rather than the kernel input representation.","discriminating_result":"It explicitly labels every tile's stored scale FP32 while the page describes both architectures; the pinned interface requires a different SM100 representation.","action":"correct","action_reason":"Preserve exact granularity while qualifying the numerical benefit and separating architecture-specific scale representations.","proposed_replacement":"State the paper's better-accommodates-outliers result and list FP32 on SM90 versus packed UE8M0 on SM100.","replacement_claim_ids":["kernel-deepgemm-current-c002"]} +{"receipt_id":"rem-kernel-deepgemm-003","path":"wiki/kernels/deepgemm.md","original_body_sha256":"6ab7a73798de1e059f397018969dab63e94f888f714964e75ff05cd82a5ba7a3","original_unit_ids":["u004"],"locator":"body:L30","original_claim_id":"kernel-deepgemm-original-c005","original_claim":"Hopper's FP8 accumulator is approximately FP22 and DeepGEMM promotes it every Nc=128 columns, equivalent to four consecutive WGMMA operations.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/abs/2412.19437v2","revision":"DeepSeek-V3 Technical Report v2","locator":"sections 3.3.2 and 3.5.2","observation":"The authors report about 14 retained bits and Nc=128 inner-dimension elements, equivalent to four WGMMAs."}],"strongest_alternative":"FP22 is an accepted shorthand and columns means columns traversed along K.","discriminating_result":"FP22 implies a floating format not asserted by the source, and GEMM output columns are N while the promotion interval is along K.","action":"correct","action_reason":"Use the paper's measured retained-bit characterization and exact K-interval definition.","proposed_replacement":"Describe about 14 retained bits and an Nc=128 K-elements/four-WGMMA interval.","replacement_claim_ids":["kernel-deepgemm-current-c003"]} +{"receipt_id":"rem-kernel-deepgemm-004","path":"wiki/kernels/deepgemm.md","original_body_sha256":"6ab7a73798de1e059f397018969dab63e94f888f714964e75ff05cd82a5ba7a3","original_unit_ids":["u005"],"locator":"body:L32-L56","original_claim_id":"kernel-deepgemm-original-c006","original_claim":"The displayed SM90 CUDA fragment is a faithful or reproducible representation of DeepGEMM's Nc=128 promotion implementation.","original_verdict":"refuted","evidence":[{"path":"artifacts/kernels/deepgemm/full/sm90_fp8_gemm_1d1d.cuh","revision":"DeepGEMM commit 891d57b; SHA-256 4a3e39d4a1dd25c0be2bc2ff586c5f50192a91d5a0d1bcda5e03e20ef094e2db","locator":"math warp-group and Promote with scales loops","observation":"The upstream kernel uses float accum/final_accum arrays and K-stepped WGMMA; the page uses undefined helpers, half2 accumulation, and an incorrect WGMMA-N derivation."}],"strongest_alternative":"The code is intentionally schematic and conveys only the high-level promotion pattern.","discriminating_result":"It is fenced as cpp and assigns correctness-relevant types and dimensions that conflict with the actual implementation.","action":"delete","action_reason":"Replace invented executable-looking code with a precise walkthrough and link to the byte-verified source.","proposed_replacement":null,"replacement_claim_ids":["kernel-deepgemm-current-c003"]} +{"receipt_id":"rem-kernel-deepgemm-005","path":"wiki/kernels/deepgemm.md","original_body_sha256":"6ab7a73798de1e059f397018969dab63e94f888f714964e75ff05cd82a5ba7a3","original_unit_ids":["u007"],"locator":"body:L60-L77","original_claim_id":"kernel-deepgemm-original-c008","original_claim":"The displayed SM100 inline PTX correctly packs scale values in a general register, passes them directly to a short tcgen05.mma form, and establishes full-precision TMEM accumulation.","original_verdict":"refuted","evidence":[{"path":"artifacts/kernels/deepgemm/full/sm100_fp8_gemm_1d1d.cuh","revision":"DeepGEMM commit 891d57b; SHA-256 84ecd247b5fb6500fd31e464388b281b1425d105451dbf8dd3c189945dcf9384","locator":"block-scaled descriptor, UTCCP copies, UMMA issue, and epilogue","observation":"The pinned implementation moves packed factors to TMEM and supplies TMEM scale addresses/runtime descriptors through a wrapper; it does not use the page's hand-written form or make its blanket precision claim."}],"strongest_alternative":"The fragment is equivalent low-level pseudocode for the wrapper call.","discriminating_result":"It materially changes the scale-factor address space and instruction operands and asserts an accumulation guarantee absent from the source.","action":"delete","action_reason":"Point to the exact upstream block-scaled UMMA sequence instead of presenting fabricated PTX.","proposed_replacement":null,"replacement_claim_ids":["kernel-deepgemm-current-c004"]} +{"receipt_id":"rem-kernel-deepgemm-006","path":"wiki/kernels/deepgemm.md","original_body_sha256":"6ab7a73798de1e059f397018969dab63e94f888f714964e75ff05cd82a5ba7a3","original_unit_ids":["u008","u009","u010","u016"],"locator":"body:L81-L120,L150","original_claim_id":"kernel-deepgemm-original-c009","original_claim":"All three illustrated grouped-GEMM modes vary only M with fixed N/K, masked mode uses a binary mask, and the displayed prefix-offset/binary-search CUDA function represents DeepGEMM's contiguous scheduler.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-deepgemm-pinned/csrc/apis/gemm.hpp","revision":"DeepGEMM commit 891d57b","locator":"M-grouped and K-grouped APIs","observation":"M-grouped modes fix N/K, masked_m stores per-group valid M lengths, K-grouped fixes M/N with varying K, and contiguous layout accepts row group IDs or prefix sums rather than the invented dispatcher."}],"strongest_alternative":"The page uses layout loosely to mean three workload modes and binary mask loosely to mean any validity metadata.","discriminating_result":"The distinction changes tensor shapes, fixed axes, metadata values, and supported architecture/layout combinations, so the loose description is operationally misleading.","action":"correct","action_reason":"Separate M-grouped contiguous, M-grouped masked, and K-grouped APIs with their exact invariants.","proposed_replacement":"Document fixed/varying axes and actual layout/length metadata without invented dispatch code.","replacement_claim_ids":["kernel-deepgemm-current-c005"]} +{"receipt_id":"rem-kernel-deepgemm-007","path":"wiki/kernels/deepgemm.md","original_body_sha256":"6ab7a73798de1e059f397018969dab63e94f888f714964e75ff05cd82a5ba7a3","original_unit_ids":["u011","u012"],"locator":"body:L124-L135","original_claim_id":"kernel-deepgemm-original-c010","original_claim":"DeepGEMM uses NVRTC JIT by default and the displayed jit_compile API faithfully shows per-shape compilation that guarantees optimal register allocation and unrolling.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-deepgemm-pinned/csrc/jit/compiler.hpp","revision":"DeepGEMM commit 891d57b","locator":"Compiler::build and compiler selection","observation":"The default compiler is NVCC; NVRTC requires DG_JIT_USE_NVRTC=1. Cache keys include generated code and compiler settings; the displayed API is absent and no optimality guarantee exists."}],"strongest_alternative":"Any runtime compiler can be described as NVRTC and 'optimal' is informal tuning language.","discriminating_result":"NVRTC and NVCC are distinct code paths with an explicit default, and the README warns the optional NVRTC path may lose performance.","action":"correct","action_reason":"Describe the generated-code cache and compiler selection exactly; remove the fabricated API and optimality guarantee.","proposed_replacement":"JIT-generated kernels default to NVCC, use optional NVRTC, and reuse signature-keyed cached artifacts.","replacement_claim_ids":["kernel-deepgemm-current-c006"]} +{"receipt_id":"rem-kernel-deepgemm-008","path":"wiki/kernels/deepgemm.md","original_body_sha256":"6ab7a73798de1e059f397018969dab63e94f888f714964e75ff05cd82a5ba7a3","original_unit_ids":["u014","u026"],"locator":"body:L145; frontmatter:performance_claims","original_claim_id":"kernel-deepgemm-original-c012","original_claim":"DeepGEMM measured 1550 TFLOPS and about 90% utilization on H800 specifically for M=N=K=4096.","original_verdict":"refuted","evidence":[{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md","revision":"DeepGEMM commit 891d57b","locator":"2025-04-18 news item","observation":"The primary source says only up to 1550 TFLOPS on H800."},{"url":"https://github.com/deepseek-ai/DeepGEMM/pull/86","revision":"merged 2025-04-14","locator":"performance report","observation":"The disclosed earlier peak is associated with 4096x7168x16384, not a 4096-cubed 1550-TFLOPS result."}],"strongest_alternative":"The exact shape and 90% were present in a removed benchmark table and remain implied by the up-to result.","discriminating_result":"No pinned primary artifact supplied by the page binds those conditions, and a nearby upstream report binds peak throughput to a different shape.","action":"correct","action_reason":"Delete the unsupported structured measurement and retain only the exact source-reported up-to statement with its missing-condition caveat.","proposed_replacement":"The pinned README reports up to 1550 TFLOPS on H800 but does not state the page's former shape or utilization.","replacement_claim_ids":["kernel-deepgemm-current-c008"]} +{"receipt_id":"rem-kernel-deepgemm-009","path":"wiki/kernels/deepgemm.md","original_body_sha256":"6ab7a73798de1e059f397018969dab63e94f888f714964e75ff05cd82a5ba7a3","original_unit_ids":["u020"],"locator":"body:L157","original_claim_id":"kernel-deepgemm-original-c015","original_claim":"Fine-grained scaling necessarily adds overhead relative to per-tensor scaling and is beneficial only when outlier sensitivity matters.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/abs/2412.19437v2","revision":"DeepSeek-V3 Technical Report v2","locator":"section 3.3.2","observation":"The paper calls scale multiplication's added computational cost minimal in the Hopper promotion design and gives no universal only-beneficial condition."}],"strongest_alternative":"Extra scale loads and multiplies prove nonzero overhead and precision is the only possible motivation.","discriminating_result":"End-to-end cost depends on hardware-native scaling, fusion, and workload, while representable range and format compatibility are distinct from a single outlier criterion.","action":"correct","action_reason":"Replace the categorical rule with architecture- and workload-scoped tradeoffs.","proposed_replacement":"Treat scale preparation/layout, compilation, and target-specific performance as measured considerations, not a universal penalty or decision rule.","replacement_claim_ids":["kernel-deepgemm-current-c009"]} +{"receipt_id":"rem-source-deepgemm-001","receipt_scope":"source-trail","path":"sources/blogs/deepgemm.md","original_body_sha256":"5bcc9b608eb3e40499b07a42db01c0d2a98b61b00e759a61633282c9486df37e","original_unit_ids":[],"locator":"Key Techniques and Key Code","original_claim_id":"source-deepgemm-summary-original-c001","original_claim":"The local source summary describes FP22-to-FP32 SM90 promotion, full-FP32 SM100 TMEM accumulation, a direct-register shortened tcgen05 form, binary/variable-K masked grouping, NVRTC JIT, about 300 kernel lines, and an unscoped 1550-TFLOPS result as one faithful upstream summary.","original_verdict":"refuted","evidence":[{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md","revision":"DeepGEMM commit 891d57b","locator":"interfaces, grouped APIs, compiler settings, and news","observation":"The pinned source distinguishes FP32 versus UE8M0 factors, M- versus K-grouping, default NVCC versus optional NVRTC, and reports only up to 1550 TFLOPS."},{"path":"artifacts/kernels/deepgemm/full/","revision":"DeepGEMM commit 891d57b","locator":"byte-verified SM90 and SM100 kernels","observation":"The real implementation contradicts the source entry's invented CUDA/PTX snippets and blanket accumulator descriptions."}],"strongest_alternative":"The source entry intentionally condenses multiple historical DeepGEMM versions into pedagogical pseudocode.","discriminating_result":"It labels the material as key code for a source retrieved in 2026 and provides no version boundary, while several operational details are false for the pinned repository.","action":"correct","action_reason":"Turn the source entry into an evidence-scoped summary of the exact pinned README/code and remove fabricated executable snippets.","proposed_replacement":"Record the precise scale formats, promotion path, grouped-axis invariants, JIT compiler selection, and scoped upstream performance statement with pinned links.","replacement_claim_ids":["kernel-deepgemm-current-c002","kernel-deepgemm-current-c003","kernel-deepgemm-current-c004","kernel-deepgemm-current-c005","kernel-deepgemm-current-c006","kernel-deepgemm-current-c008"]} +{"receipt_id":"rem-source-schema-001","receipt_scope":"source-trail","path":"sources/docs/nvidia-cuda-13-0-2-tma.md","original_body_sha256":"916d0efa36276ee4a6ab46b59a5c95a39997050f008e56b68b06b2441b03de6e","original_unit_ids":[],"locator":"frontmatter tags","original_claim_id":"source-cuda-tma-taxonomy-original-c001","original_claim":"The source entry classifies tensor-map and swizzle as valid controlled topical tags.","original_verdict":"refuted","evidence":[{"path":"data/tags.yaml","revision":"current repository controlled vocabulary","locator":"hardware_features and techniques","observation":"The vocabulary represents this topic with tma/mbarrier hardware tags and the swizzling technique; tensor-map and swizzle are not valid entries."}],"strongest_alternative":"The two spellings are harmless free-form synonyms.","discriminating_result":"Repository validation enforces the controlled vocabulary and rejects both values.","action":"correct","action_reason":"Normalize metadata without changing the source's evidence-scoped body.","proposed_replacement":"Use tags tma, mbarrier, and swizzling.","replacement_claim_ids":["hw-tma-current-c001","hw-tma-current-c002","hw-tma-current-c006"]} +{"receipt_id":"rem-source-schema-002","receipt_scope":"source-trail","path":"sources/docs/nvidia-transformer-engine-2.13-nvfp4.md","original_body_sha256":"0e601f6935056e2f6f62608035cc4bf8b4690086ffcb980db93c5c2b62f51e40","original_unit_ids":[],"locator":"frontmatter tags","original_claim_id":"source-te-nvfp4-taxonomy-original-c001","original_claim":"The source entry treats e2m1 as a valid controlled topical tag.","original_verdict":"refuted","evidence":[{"path":"data/tags.yaml","revision":"current repository controlled vocabulary","locator":"hardware_features","observation":"The vocabulary contains nvfp4, fp4, and block-scale but no standalone e2m1 tag; E2M1 remains correctly described in the title/body."}],"strongest_alternative":"Every datatype spelling should also be a free-form tag.","discriminating_result":"The repository deliberately uses a finite vocabulary and validation rejects the unregistered value.","action":"delete","action_reason":"Remove only the invalid redundant tag while preserving the exact payload description.","proposed_replacement":null,"replacement_claim_ids":["hw-nvfp4-current-c001"]} +{"receipt_id":"rem-source-schema-003","receipt_scope":"source-trail","path":"sources/docs/triton-3.3-blackwell.md","original_body_sha256":"8c83e04f51b319309735904fafcb6919ff3acffcf50e7c3fa12236d0b0f7a9bf","original_unit_ids":[],"locator":"frontmatter source_category and tags","original_claim_id":"source-triton-3.3-taxonomy-original-c001","original_claim":"A source stored in the official-doc corpus may declare upstream-code as its source category and version-boundary as an unregistered tag.","original_verdict":"refuted","evidence":[{"path":"data/schemas.yaml","revision":"current repository source-doc schema","locator":"source-doc source_category constraint","observation":"Documents under sources/docs must use official-doc or paper."},{"path":"data/tags.yaml","revision":"current repository controlled vocabulary","locator":"topical categories","observation":"triton, tcgen05, and tmem are registered, while version-boundary is represented by version_sensitive metadata elsewhere and is not a tag."}],"strongest_alternative":"The Git comparison is code and therefore should override directory/schema classification.","discriminating_result":"This file is an evidence summary in the docs corpus; its body still links exact upstream code while its corpus category must satisfy the source-doc contract.","action":"correct","action_reason":"Normalize source-corpus metadata without changing the pinned compiler-boundary evidence.","proposed_replacement":"Use official-doc and the registered triton/tcgen05/tmem tags.","replacement_claim_ids":["lang-triton-current-c001","lang-triton-current-c002"]} +{"receipt_id":"rem-kernel-flash-attention-4-001","path":"wiki/kernels/flash-attention-4.md","original_body_sha256":"77d451b38127be1f61669e4237971c0e4da24687371dfdf76f65ec1cafc8ce16","original_unit_ids":["u002"],"locator":"body:L7","original_claim_id":"kernel-flash-attention-4-original-c002","original_claim":"FA4 universally matches or exceeds cuDNN while compiling 20-30x faster than equivalent CUTLASS C++ code.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/html/2603.05451v1","revision":"FA4 paper v1","locator":"sections 5.1 and 5.3","observation":"Compilation is a two-kernel FA3 comparison (22x and 32x); runtime results are configuration-scoped and only up to 1.3x over cuDNN 9.13."}],"strongest_alternative":"The sentence uses ordinary performance shorthand, so readers will infer the benchmark scope.","discriminating_result":"Its unqualified match-or-exceed statement materially broadens an up-to result, and equivalent CUTLASS code is less precise than the paper's named FA3 comparison.","action":"correct","action_reason":"Separate the measured compile comparison from scoped runtime maxima.","proposed_replacement":"State the exact 55s/2.5s and 45s/1.4s compile comparisons to FA3, and label runtime values as source-reported maxima over the authors' suite.","replacement_claim_ids":["kernel-flash-attention-4-current-c001","kernel-flash-attention-4-current-c007"]} +{"receipt_id":"rem-kernel-flash-attention-4-002","path":"wiki/kernels/flash-attention-4.md","original_body_sha256":"77d451b38127be1f61669e4237971c0e4da24687371dfdf76f65ec1cafc8ce16","original_unit_ids":["u004"],"locator":"body:L15-L44","original_claim_id":"kernel-flash-attention-4-original-c005","original_claim":"The displayed Python kernel is faithful CuTe DSL pseudocode for FA4's ping-pong forward implementation.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_fwd_sm100.py","revision":"Dao-AILab commit a369df707e1980fb328abcc1733e3457ec10155f","locator":"warp roles, pipeline creation, mma, and softmax_loop","observation":"The real implementation uses explicit pipeline state and barriers; the snippet's APIs and values are undefined and its first-iteration dependencies are invalid."}],"strongest_alternative":"The snippet is intentionally high-level pseudocode and need not preserve executable APIs or exact state.","discriminating_result":"It is fenced as Python under an implementation page and reads S1/V_prev before defining them, so it fails even its own purported phase model.","action":"delete","action_reason":"The paper's verified schedule can be stated precisely without publishing misleading pseudo-code.","proposed_replacement":null,"replacement_claim_ids":["kernel-flash-attention-4-current-c002"]} +{"receipt_id":"rem-kernel-flash-attention-4-003","path":"wiki/kernels/flash-attention-4.md","original_body_sha256":"77d451b38127be1f61669e4237971c0e4da24687371dfdf76f65ec1cafc8ce16","original_unit_ids":["u005"],"locator":"body:L48","original_claim_id":"kernel-flash-attention-4-original-c006","original_claim":"FA4 replaces the hardware ex2 path with a Cody-Waite/Horner software exponential distributed over FMA units.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/html/2603.05451v1","revision":"FA4 paper v1","locator":"section 3.2","observation":"Only roughly 10-25% of entries are evaluated in software; the remainder stay on hardware MUFU ex2."}],"strongest_alternative":"Replaces describes the bottleneck remedy, not every individual exponential.","discriminating_result":"The next code and throughput statements explicitly present a full replacement, while partial selection is a core scheduling requirement in the source.","action":"correct","action_reason":"Preserve the optimization while restoring its hybrid scope.","proposed_replacement":"Describe partial emulation and the simultaneous use of FMA and MUFU paths.","replacement_claim_ids":["kernel-flash-attention-4-current-c003"]} +{"receipt_id":"rem-kernel-flash-attention-4-004","path":"wiki/kernels/flash-attention-4.md","original_body_sha256":"77d451b38127be1f61669e4237971c0e4da24687371dfdf76f65ec1cafc8ce16","original_unit_ids":["u006"],"locator":"body:L50-L71","original_claim_id":"kernel-flash-attention-4-original-c007","original_claim":"The displayed software_exp2 implements FA4's Cody-Waite reduction and polynomial.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/html/2603.05451v1","revision":"FA4 paper v1","locator":"section 3.2 equations and coefficient table","observation":"The source uses floor reduction to [0,1), a degree-3 polynomial, low-range handling, and selective emulation, unlike the rounded [-0.5,0.5] five-coefficient snippet."}],"strongest_alternative":"A centered interval and higher-degree polynomial are mathematically interchangeable approximations.","discriminating_result":"They may approximate exp2, but they are not the algorithm or coefficients attributed to FA4 and the undefined constants make the snippet unreproducible.","action":"correct","action_reason":"Replace it with a clearly labeled scalar reference for the published degree-3 range reduction.","proposed_replacement":"Use floor(x), f=x-floor(x), the blog's rounded four coefficients, and ldexp, while stating that real FA4 selects only some entries for software evaluation.","replacement_claim_ids":["kernel-flash-attention-4-current-c003"]} +{"receipt_id":"rem-kernel-flash-attention-4-005","path":"wiki/kernels/flash-attention-4.md","original_body_sha256":"77d451b38127be1f61669e4237971c0e4da24687371dfdf76f65ec1cafc8ce16","original_unit_ids":["u007"],"locator":"body:L73","original_claim_id":"kernel-flash-attention-4-original-c008","original_claim":"Software exponential provides roughly four times hardware-SFU throughput.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/html/2603.05451v1","revision":"FA4 paper v1","locator":"section 3.2 and forward ablations","observation":"No four-times standalone exponential throughput result is reported; the evaluation presents end-to-end kernel effects."}],"strongest_alternative":"The factor follows analytically from the number of FMA pipes relative to MUFU pipes.","discriminating_result":"Available instruction resources do not by themselves determine realized throughput, and the algorithm deliberately uses both paths.","action":"delete","action_reason":"No adequate oracle supports the numeric factor.","proposed_replacement":null,"replacement_claim_ids":["kernel-flash-attention-4-current-c003"]} +{"receipt_id":"rem-kernel-flash-attention-4-006","path":"wiki/kernels/flash-attention-4.md","original_body_sha256":"77d451b38127be1f61669e4237971c0e4da24687371dfdf76f65ec1cafc8ce16","original_unit_ids":["u009"],"locator":"body:L79-L89","original_claim_id":"kernel-flash-attention-4-original-c010","original_claim":"The displayed LSE-based threshold-2 function correctly implements FA4 conditional rescaling.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/html/2603.05451v1","revision":"FA4 paper v1","locator":"section 3.3 equations 7-10","observation":"The algorithm compares monotonic running maxima, typically uses tau=8 log2 units, retains the old maximum on a skip, and renormalizes at the end."}],"strongest_alternative":"LSE is a convenient proxy for the max and threshold 2 is a tunable example.","discriminating_result":"LSE incorporates the row sum and is not the state variable in the proof; updating it without the retained-max exponent convention changes the result.","action":"delete","action_reason":"Use the verified invariant rather than repair a misleading miniature implementation.","proposed_replacement":null,"replacement_claim_ids":["kernel-flash-attention-4-current-c004"]} +{"receipt_id":"rem-kernel-flash-attention-4-007","path":"wiki/kernels/flash-attention-4.md","original_body_sha256":"77d451b38127be1f61669e4237971c0e4da24687371dfdf76f65ec1cafc8ce16","original_unit_ids":["u010"],"locator":"body:L93","original_claim_id":"kernel-flash-attention-4-original-c011","original_claim":"Two-CTA backward halves shared-memory traffic for the entire dQ/dK/dV computation.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/html/2603.05451v1","revision":"FA4 paper v1","locator":"section 4","observation":"The approximate halving is scoped to operand-B shared reads for five GEMMs; dS DSMEM exchange and dQ global-atomic reduction are separate effects."}],"strongest_alternative":"Operand B dominates all shared-memory traffic, making the shorter statement operationally equivalent.","discriminating_result":"Other operands, staging, exchanges, and epilogues still generate traffic, and the paper explicitly limits its factor to one operand.","action":"correct","action_reason":"Retain the two-CTA optimization with the source's exact traffic scope.","proposed_replacement":"State the five-GEMM operand-B reuse, dS DSMEM exchange for dQ, and halved dQ global atomic reductions separately.","replacement_claim_ids":["kernel-flash-attention-4-current-c005"]} +{"receipt_id":"rem-kernel-flash-attention-4-008","path":"wiki/kernels/flash-attention-4.md","original_body_sha256":"77d451b38127be1f61669e4237971c0e4da24687371dfdf76f65ec1cafc8ce16","original_unit_ids":["u011"],"locator":"body:L95-L118","original_claim_id":"kernel-flash-attention-4-original-c012","original_claim":"The displayed backward kernel represents FA4 by assigning dK to CTA 0 and dV to CTA 1.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_bwd_sm100.py","revision":"Dao-AILab commit a369df707e1980fb328abcc1733e3457ec10155f","locator":"two-CTA MMA construction and backward loops","observation":"Both CTAs cooperate in two-CTA MMA pipelines for the five GEMMs; cluster rank does not partition dK and dV as shown."}],"strongest_alternative":"The branch is a conceptual ownership diagram rather than literal execution.","discriminating_result":"It is fenced as a CuTe kernel and contradicts the actual two-CTA instruction semantics by making each CTA independently own a different GEMM.","action":"delete","action_reason":"Replace fabricated executable code with the paper's verified data-sharing description and pinned upstream links.","proposed_replacement":null,"replacement_claim_ids":["kernel-flash-attention-4-current-c005","kernel-flash-attention-4-current-c006"]} +{"receipt_id":"rem-kernel-flash-attention-4-009","path":"wiki/kernels/flash-attention-4.md","original_body_sha256":"77d451b38127be1f61669e4237971c0e4da24687371dfdf76f65ec1cafc8ce16","original_unit_ids":["u012","u030"],"locator":"frontmatter:performance_claims; body:L122-L124","original_claim_id":"kernel-flash-attention-4-original-c013","original_claim":"B200 BF16 at seqlen 8192/head dimension 128 produced 1605 TFLOPS, 71% utilization, and the listed speedup ranges, sourced to the paper.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/html/2603.05451v1","revision":"FA4 paper v1","locator":"abstract and section 5","observation":"The paper's peak is 1613 TFLOPS/s at 71% over a benchmark suite and does not establish the page's complete shape tuple."},{"url":"https://tridao.me/blog/2026/flash4/","revision":"Tri Dao blog 2026-03-05","locator":"performance summary","observation":"The blog gives the distinct 1605 maximum and up-to speedups, not the exact row attributed to the paper."}],"strongest_alternative":"1605 is the precise data point behind the paper's rounded 1613 maximum and the chart visually identifies the shape.","discriminating_result":"The two first-party sources state different maxima, and neither textual record supplies the full row; mixing them under one source ID is not reproducible evidence.","action":"correct","action_reason":"Remove the false structured record and preserve both source-reported maxima with their distinct provenance and suite scope.","proposed_replacement":"Use performance_claims: [] and explain that paper v1 reports up to 1613 while the blog reports up to 1605, both with 71%, without asserting the former shape row.","replacement_claim_ids":["kernel-flash-attention-4-current-c007"]} +{"receipt_id":"rem-kernel-flash-attention-4-010","path":"wiki/kernels/flash-attention-4.md","original_body_sha256":"77d451b38127be1f61669e4237971c0e4da24687371dfdf76f65ec1cafc8ce16","original_unit_ids":["u013"],"locator":"body:L126","original_claim_id":"kernel-flash-attention-4-original-c014","original_claim":"The 71% result is universally state of the art and the remaining 29% is consumed by softmax, rescaling, and memory transfers.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/html/2603.05451v1","revision":"FA4 paper v1","locator":"abstract and section 5","observation":"The source does not provide an exhaustive global ordering or partition the complement of a peak-throughput ratio into those categories."}],"strongest_alternative":"The 29% statement is just an intuitive complement to 71% MMA utilization.","discriminating_result":"A utilization ratio's complement is not a measured time breakdown; overlap and the FLOP denominator make that subtraction non-causal.","action":"delete","action_reason":"No source or experiment supports the attribution.","proposed_replacement":null,"replacement_claim_ids":["kernel-flash-attention-4-current-c007"]} +{"receipt_id":"rem-kernel-flash-attention-4-011","path":"wiki/kernels/flash-attention-4.md","original_body_sha256":"77d451b38127be1f61669e4237971c0e4da24687371dfdf76f65ec1cafc8ce16","original_unit_ids":["u016","u017","u018"],"locator":"body:L132-L134","original_claim_id":"kernel-flash-attention-4-original-c015","original_claim":"FA4 uses SM100_MMA_SS atoms, a TMEM locale, and unconditional TMA loads for Q/K/V.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_fwd_sm100.py","revision":"Dao-AILab commit a369df707e1980fb328abcc1733e3457ec10155f","locator":"_setup_tiled_mma, TMEM allocation/offsets, and copy-atom construction","observation":"The exact APIs are make_trivial_tiled_mma, OperandSource, TmemAllocator, and explicit offsets; Q and paged K/V can use non-TMA fallbacks. The claimed symbol/locale do not occur."}],"strongest_alternative":"SM100_MMA_SS and TMEM locale are conceptual names for lower-level objects hidden by helpers.","discriminating_result":"The bullets present exact code identifiers and unconditional behavior; exact-token and control-flow inspection reject both.","action":"correct","action_reason":"Name the pinned source's actual abstractions and conditional copy behavior.","proposed_replacement":"Describe helper-built tcgen05 MMA atoms, explicit TMEM allocation/offsets, TMA atoms where enabled, and fallback copy paths.","replacement_claim_ids":["kernel-flash-attention-4-current-c006"]} +{"receipt_id":"rem-kernel-flash-attention-4-012","path":"wiki/kernels/flash-attention-4.md","original_body_sha256":"77d451b38127be1f61669e4237971c0e4da24687371dfdf76f65ec1cafc8ce16","original_unit_ids":["u019"],"locator":"body:L138","original_claim_id":"kernel-flash-attention-4-original-c016","original_claim":"Sequence length 1024 is a universal lower threshold for choosing FA4 standard multi-head attention.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/html/2603.05451v1","revision":"FA4 paper v1","locator":"section 5 benchmark setup and results","observation":"Sequence length 1024 is one evaluated point, not a selection rule; results vary across direction, masking, dimensions, and baseline."}],"strongest_alternative":"The first tested length reasonably defines the supported/useful floor.","discriminating_result":"A benchmark grid endpoint does not establish compatibility or a crossover, and no source statement promotes it to a threshold.","action":"correct","action_reason":"Replace the invented cutoff with workload- and support-dependent selection guidance.","proposed_replacement":"Use the pinned package's supported shapes/features and controlled measurements on the target workload rather than a universal sequence-length cutoff.","replacement_claim_ids":["kernel-flash-attention-4-current-c008"]} +{"receipt_id":"rem-kernel-flash-attention-4-013","path":"wiki/kernels/flash-attention-4.md","original_body_sha256":"77d451b38127be1f61669e4237971c0e4da24687371dfdf76f65ec1cafc8ce16","original_unit_ids":["u022","u023","u024"],"locator":"body:L144-L146","original_claim_id":"kernel-flash-attention-4-original-c018","original_claim":"FA4 has no SM90 implementation, requires CUTLASS/CuTe DSL 4.5.0, and is generally optimal at head dimension 128.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flash-attention-pinned/flash_attn/cute/","revision":"Dao-AILab commit a369df707e1980fb328abcc1733e3457ec10155f","locator":"README.md, pyproject.toml, architecture dispatch, and tuning tables","observation":"The package supports Hopper and Blackwell modules, pins nvidia-cutlass-dsl 4.6.0.dev0, and tunes several dimensions and modes without a universal 128 optimum."}],"strongest_alternative":"FA4 names only the SM100 paper algorithm even though its distribution carries other architectures, and 4.5 was correct at publication.","discriminating_result":"The page presents current implementation/toolchain caveats without a version boundary; its own package identity and dependency manifest contradict them.","action":"correct","action_reason":"Separate paper scope from the pinned package snapshot and avoid universal tuning advice.","proposed_replacement":"State that the paper studies B200/SM100, while the pinned package also dispatches SM90/SM120 and pins nvidia-cutlass-dsl 4.6.0.dev0; require per-workload tuning.","replacement_claim_ids":["kernel-flash-attention-4-current-c006","kernel-flash-attention-4-current-c008"]} +{"receipt_id":"rem-kernel-flash-attention-4-014","path":"wiki/kernels/flash-attention-4.md","original_body_sha256":"77d451b38127be1f61669e4237971c0e4da24687371dfdf76f65ec1cafc8ce16","original_unit_ids":["u027"],"locator":"body:L155","original_claim_id":"kernel-flash-attention-4-original-c020","original_claim":"The local full bundle contains verbatim upstream FA4 implementation code.","original_verdict":"refuted","evidence":[{"path":"artifacts/kernels/flash-attention-4/full/PROVENANCE.yaml","revision":"CUTLASS commit 0e026982","locator":"upstream_repo and upstream_path","observation":"The file is NVIDIA CUTLASS example 77 SM100 FMHA backward MLA code, not Dao-AILab/flash-attention FA4 code."}],"strongest_alternative":"The CUTLASS example is the upstream implementation on which FA4 is based and can be called FA4 source.","discriminating_result":"It belongs to a different repository, pull request, namespace, and implementation; provenance itself never identifies Dao-AILab or FA4.","action":"correct","action_reason":"Retain the useful adjacent reference but label its identity; point implementation claims to exact Dao-AILab commit URLs.","proposed_replacement":"Call the local file an adjacent CUTLASS SM100 FMHA reference, retain derived material only as labeled teaching code, and link exact FA4 source files externally.","replacement_claim_ids":["kernel-flash-attention-4-current-c009"]} +{"receipt_id":"rem-source-fa4-paper-001","receipt_scope":"source-trail","path":"sources/docs/flash-attention-4.md","original_body_sha256":"b12630558449ab1a7c1387690337940190806e2ab8a9a54b5018823169112901","original_unit_ids":[],"locator":"Key Contributions and Performance","original_claim_id":"source-doc-fa4-summary","original_claim":"The paper summary reports up to 1605 TFLOPS and broadly says two-CTA backward halves shared-memory traffic.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/html/2603.05451v1","revision":"FA4 paper v1","locator":"abstract and section 4","observation":"The paper reports up to 1613 TFLOPS/s; shared-memory halving is scoped to operand B of five backward GEMMs, with separate dS/dQ effects."}],"strongest_alternative":"1605 is a more precise chart value and the traffic summary is harmless shorthand.","discriminating_result":"The paper's own text states 1613, while the 1605 number belongs to the blog; broadening operand-B traffic changes the mechanism claim.","action":"correct","action_reason":"Keep the local paper trail faithful to the cited revision.","proposed_replacement":"Record 1613 and the exact backward traffic scopes.","replacement_claim_ids":["kernel-flash-attention-4-current-c005","kernel-flash-attention-4-current-c007"]} +{"receipt_id":"rem-source-fa4-blog-001","receipt_scope":"source-trail","path":"sources/blogs/flash-attention-4.md","original_body_sha256":"94133fb35bb9946d0d5c63617f242d54b9ae792b24f4a159420b85b08398d8da","original_unit_ids":[],"locator":"Key Techniques, Performance, and Key Code","original_claim_id":"source-blog-fa4-summary","original_claim":"The blog summary presents a full software replacement, broad SMEM halving, unscoped speedup ranges, and invented CUDA code as if extracted from the post.","original_verdict":"refuted","evidence":[{"url":"https://tridao.me/blog/2026/flash4/","revision":"Tri Dao blog 2026-03-05","locator":"software exp, conditional rescaling, 2CTA, and performance sections","observation":"The post describes partial software evaluation with floor/[0,1)/degree-3 coefficients, tau=8 conditional rescaling, scoped two-CTA reuse, and up-to performance. It does not contain the summary's CUDA functions or inline PTX."}],"strongest_alternative":"Local source pages are allowed to add illustrative code and concise summaries.","discriminating_result":"The artifacts label the blocks extracted and inherit the blog's provenance, so fabricated forms become falsely attributed evidence unless explicitly marked as derived illustrations.","action":"correct","action_reason":"Make every qualification and code origin explicit before regenerating extracted assets.","proposed_replacement":"Use accurate scoped prose and clearly labeled non-upstream illustrative code corresponding to the published formulas.","replacement_claim_ids":["kernel-flash-attention-4-current-c003","kernel-flash-attention-4-current-c004","kernel-flash-attention-4-current-c005","kernel-flash-attention-4-current-c007"]} +{"receipt_id":"rem-source-fa4-blog-artifacts-001","receipt_scope":"source-trail","path":"artifacts/blogs/flash-attention-4/code","original_body_sha256":"f15704c92718baa510e046bac17164b03e63bfbcf2eb4479be93a79827ce2827","original_unit_ids":[],"locator":"PROVENANCE.yaml and both extracted blocks","original_claim_id":"source-artifact-fa4-blog-code","original_claim":"The extracted bundle attributes an invented centered degree-5 CUDA exp2 function and invalid short two-CTA inline PTX to the FA4 blog.","original_verdict":"refuted","evidence":[{"url":"https://tridao.me/blog/2026/flash4/","revision":"Tri Dao blog 2026-03-05","locator":"software exp and 2CTA sections","observation":"The blog gives equations and diagrams rather than these CUDA blocks; its software formula uses floor/[0,1)/degree 3 and the two-CTA description is not the displayed incomplete assembly."}],"strongest_alternative":"Extraction provenance refers to the local captured Markdown rather than verbatim remote HTML.","discriminating_result":"Without an explicit derived label in the block, origin_url and inherits-from-source-blog falsely imply first-party code; the old math is also inconsistent with the post.","action":"correct","action_reason":"Regenerate the required bundle from corrected, explicitly illustrative local blocks and refresh hashes.","proposed_replacement":"Extract accurate self-contained reference code whose header says it is a KernelWiki illustration, not upstream FA4 code.","replacement_claim_ids":["kernel-flash-attention-4-current-c003","kernel-flash-attention-4-current-c009"]} +{"receipt_id":"rem-source-fa4-kernel-artifacts-001","receipt_scope":"source-trail","path":"artifacts/kernels/flash-attention-4","original_body_sha256":"0f948d25e9e4af17ab3b022059f2231afd062696cba6a4075b8afa41eb6915af","original_unit_ids":[],"locator":"full/PROVENANCE.yaml and variants/01-software-exp-skeleton.cu","original_claim_id":"source-artifact-fa4-kernel-labels","original_claim":"The kernel bundle's derived material calls a CUTLASS example the upstream FA4-MLA kernel and inaccurately describes FA4 software exponentiation as compiler-emitted hardware ex2.","original_verdict":"refuted","evidence":[{"path":"artifacts/kernels/flash-attention-4/full/PROVENANCE.yaml","revision":"CUTLASS commit 0e026982","locator":"repository and upstream path","observation":"The full file is an adjacent CUTLASS example, not FA4."},{"url":"https://arxiv.org/html/2603.05451v1","revision":"FA4 paper v1","locator":"section 3.2","observation":"Software-selected entries use integer/fraction range reduction and FMA polynomial evaluation specifically to complement hardware MUFU ex2; compiler-emitted __expf is not that path."}],"strongest_alternative":"The derived file only sketches the rescaling exponential and uses the CUTLASS code as a nearby implementation anchor.","discriminating_result":"Its comments explicitly call hardware __expf the software trick and identify the unrelated full file as upstream FA4, reversing both proven identities.","action":"correct","action_reason":"Keep both useful assets while labeling the adjacent reference and replacing the derived variant with the verified scalar formula.","proposed_replacement":"Mark the full file as an adjacent CUTLASS SM100 FMHA reference and implement a derived floor/[0,1)/degree-3 scalar exp2 reference.","replacement_claim_ids":["kernel-flash-attention-4-current-c003","kernel-flash-attention-4-current-c009"]} +{"receipt_id":"rem-source-fa4-relationship-001","receipt_scope":"source-trail","path":"wiki/kernels/flash-attention-4.md","original_body_sha256":"77d451b38127be1f61669e4237971c0e4da24687371dfdf76f65ec1cafc8ce16","original_unit_ids":[],"locator":"frontmatter:sources pr-flashinfer-1850","original_claim_id":"source-relationship-fa4-flashinfer-1850","original_claim":"FlashInfer PR 1850 is a source for the Dao-AILab FlashAttention-4 page.","original_verdict":"refuted","evidence":[{"url":"https://github.com/flashinfer-ai/flashinfer/pull/1850","revision":"FlashInfer merge f3ea938d","locator":"repository, title, and changed paths","observation":"The PR modifies FlashInfer's separate CUTLASS C++ SM100 FMHA implementation to add head dimension 64; it is not Dao-AILab FA4 source or evidence for the page's mechanisms."}],"strongest_alternative":"Any Blackwell FlashAttention implementation is a useful corroborating source for FA4.","discriminating_result":"The repositories, languages, algorithms, and authorship differ; treating the PR as FA4 evidence conflates an adjacent implementation with the subject.","action":"delete","action_reason":"Keep the FA4 source relationship limited to its first-party paper and blog; adjacent CUTLASS code remains explicitly labeled in the artifact discussion.","proposed_replacement":null,"replacement_claim_ids":["kernel-flash-attention-4-current-c009"]} +{"receipt_id":"rem-kernel-flash-attention-sm100-mla-topk-001","path":"wiki/kernels/flash-attention-sm100-mla-topk.md","original_body_sha256":"260bbec945acf25551df5b9232d692574dbde84aca2aa0e9b3b73972a38e7239","original_unit_ids":["u001"],"locator":"body:L3-L5","original_claim_id":"kernel-flash-attention-sm100-mla-topk-original-c001","original_claim":"PR 2441 combines page-table/KV layout handling with its SM100 CuTe DSL MLA top-k path.","original_verdict":"refuted","evidence":[{"path":"artifacts/prs/flash-attention/PR-2441/key-files/flash_attn/cute/flash_fwd_mla_sm100.py","revision":"merge f219c89c; SHA-256 f6512b25d901463cf62433b745e87ead30443f295ce313c03e363c886ece296e","locator":"__call__ feature assertions","observation":"The implementation rejects non-null mPageTable with 'page table tbd for MLA'; the PR body also lists page table as not planned."}],"strongest_alternative":"KV layout handling refers only to tensor layout, not paged KV, so the wording remains true.","discriminating_result":"The compound phrase explicitly says page/KV and the source exposes but rejects a page-table parameter; readers would reasonably infer the missing mode.","action":"correct","action_reason":"Retain the implemented top-k/scheduler path and state page-table exclusion precisely.","proposed_replacement":"Describe the SM100 64/512 MQA top-k gather and persistent scheduling path, with page table explicitly unsupported in the merge.","replacement_claim_ids":["kernel-flash-attention-sm100-mla-topk-current-c001","kernel-flash-attention-sm100-mla-topk-current-c002"]} +{"receipt_id":"rem-kernel-flash-attention-sm100-mla-topk-002","path":"wiki/kernels/flash-attention-sm100-mla-topk.md","original_body_sha256":"260bbec945acf25551df5b9232d692574dbde84aca2aa0e9b3b73972a38e7239","original_unit_ids":["u007"],"locator":"frontmatter:performance_claims","original_claim_id":"kernel-flash-attention-sm100-mla-topk-original-c006","original_claim":"PR 2441 reports exactly 0.3 ms on B200 BF16 for the listed top-k=2048 decode shape.","original_verdict":"refuted","evidence":[{"url":"https://api.github.com/repos/Dao-AILab/flash-attention/pulls/2441","revision":"PR record; merge f219c89c886c6ccbf9d3dbd9fe41b11ac64e9df8","locator":"initial saturating decode performance block","observation":"The row is 0.31 ms/955.47 TFLOPS for DSA without bitmask, while the bitmask path is 0.33 ms/898.08 TFLOPS. The body does not name B200 or BF16 for either row."}],"strongest_alternative":"0.3 is ordinary rounding and SM100/deepseek context uniquely implies B200 BF16.","discriminating_result":"Structured records require exact source cells and environments; rounding erases the separate 0.31/0.33 variants, and architecture generation does not prove a particular GPU or dtype.","action":"correct","action_reason":"Remove the unsupported structured record but retain the author's two qualified, unreproduced observations in prose.","proposed_replacement":"Use performance_claims: [] and state 0.31 ms no-bitmask versus 0.33 ms bitmask with the exact shape/top-k and missing-environment warning.","replacement_claim_ids":["kernel-flash-attention-sm100-mla-topk-current-c003"]} +{"receipt_id":"rem-source-fa4-mla-topk-001","receipt_scope":"source-trail","path":"sources/prs/flash-attention/PR-2441.md","original_body_sha256":"01d8d73099253e71bbcb68d2b9aa8b352369ca396b9c8624a83cecc221cbb88d","original_unit_ids":[],"locator":"changed_paths, Summary, and Changed Files","original_claim_id":"source-pr-fa4-2441-capture","original_claim":"The local PR capture omits topk_gather_kv.py and test files from frontmatter, truncates the DSA row at 0.3, and leaves an unclosed performance fence.","original_verdict":"refuted","evidence":[{"url":"https://api.github.com/repos/Dao-AILab/flash-attention/pulls/2441","revision":"GitHub PR record; merge f219c89c886c6ccbf9d3dbd9fe41b11ac64e9df8","locator":"body and merge metadata","observation":"The full body has 0.31 ms/955.47 TFLOPS without bitmask and 0.33 ms/898.08 TFLOPS with bitmask, plus explicit feature exclusions."},{"path":"artifacts/prs/flash-attention/PR-2441/diff.patch","revision":"merge f219c89c; SHA-256 4525d7f9af76740692474ffc0266fe9cf72f894ce753fd9d18b70771c96b6810","locator":"diff headers","observation":"The patch includes .pre-commit-config.yaml, topk_gather_kv.py, and tests/cute/test_flash_attn.py in addition to the captured frontmatter list."}],"strongest_alternative":"The source page is intentionally a shortened ingestion excerpt and its body list still names the important gather file.","discriminating_result":"The truncation changes a structured numeric observation and malformed Markdown swallows later sections; changed_paths is represented as exhaustive machine-readable provenance.","action":"correct","action_reason":"Restore the exact primary PR body facts and complete file list without changing the pinned artifacts.","proposed_replacement":"Summarize both performance variants and exclusions in valid Markdown and align changed_paths with the captured patch.","replacement_claim_ids":["kernel-flash-attention-sm100-mla-topk-current-c001","kernel-flash-attention-sm100-mla-topk-current-c002","kernel-flash-attention-sm100-mla-topk-current-c003"]} +{"receipt_id":"rem-source-fa4-mla-topk-relationship-001","receipt_scope":"source-trail","path":"wiki/kernels/flash-attention-sm100-mla-topk.md","original_body_sha256":"260bbec945acf25551df5b9232d692574dbde84aca2aa0e9b3b73972a38e7239","original_unit_ids":[],"locator":"frontmatter:sources pr-flash-attention-1236","original_claim_id":"source-relationship-fa4-mla-topk-pr1236","original_claim":"FA3 Hopper PR 1236 is a source for the SM100 MLA top-k kernel added by PR 2441.","original_verdict":"refuted","evidence":[{"url":"https://github.com/Dao-AILab/flash-attention/pull/1236","revision":"merge a5a75274","locator":"title, description, and changed paths","observation":"PR 1236 adds FA3 SM90 KV-cache/split-KV/GQA C++ paths in 2024 and contains neither the SM100 CuTe DSL MLA kernel nor top-k gather added in PR 2441."}],"strongest_alternative":"PR 1236 is historical ancestry for attention scheduling and therefore a valid source.","discriminating_result":"The current page makes no historical FA3 claim and every material mechanism is directly evidenced by PR 2441; retaining 1236 invites architecture/feature conflation.","action":"delete","action_reason":"Limit the source relationship to the exact introducing merge.","proposed_replacement":null,"replacement_claim_ids":["kernel-flash-attention-sm100-mla-topk-current-c005"]} +{"receipt_id":"rem-kernel-flashmla-001","path":"wiki/kernels/flashmla.md","original_body_sha256":"997be90a5a387c776857d0d8257d24915a19a1aa829e9d76940c4de57a311691","original_unit_ids":["u001"],"locator":"body:L5","original_claim_id":"kernel-flashmla-original-c001","original_claim":"MLA compresses a standard-MHA KV cache from 327–516 KB/token to 70 KB/token, a 4.66–7.28x reduction.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/html/2405.04434v5","revision":"DeepSeek-V2 paper v5","locator":"Section 2.1.1 and Table 1","observation":"The paper provides symbolic cache-element formulas, not these KB endpoints or ratios; its stated n_h=128 formula implies about 56.9x equal-width elements."}],"strongest_alternative":"The KB values may describe another model or mixed-precision implementation omitted from the MLA paper.","discriminating_result":"The page presents them as the MLA mechanism's standard-MHA comparison without a model, dtype, layer count, or source; they conflict with the only attached first-party formula and are not auditable.","action":"correct","action_reason":"Replace invented byte endpoints with the paper's explicit per-layer element formula and keep implementation-specific byte layout separate.","proposed_replacement":"State MHA 2*n_h*d_h elements versus MLA d_c+d_h^R per layer/token, with DeepSeek-V2 d_c=4d_h and d_h^R=d_h/2.","replacement_claim_ids":["kernel-flashmla-current-c001"]} +{"receipt_id":"rem-kernel-flashmla-002","path":"wiki/kernels/flashmla.md","original_body_sha256":"997be90a5a387c776857d0d8257d24915a19a1aa829e9d76940c4de57a311691","original_unit_ids":["u002"],"locator":"body:L7","original_claim_id":"kernel-flashmla-original-c002","original_claim":"FlashMLA includes dense MLA prefill on SM100 as one of four MLA variants.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"DeepSeek commit 71c7379","locator":"support matrix and Dense MHA Prefill","observation":"Dense prefill is explicitly MHA; decode and sparse prefill are MLA-mode MQA."}],"strongest_alternative":"The repository name makes every bundled attention operator an MLA variant.","discriminating_result":"The first-party support matrix deliberately distinguishes MHA from MQA MLA mode and gives different dimensions.","action":"correct","action_reason":"Preserve the four operator families while labeling dense prefill as MHA.","proposed_replacement":"Use the exact support matrix: dense MLA decode SM90, sparse MLA decode SM90/SM100, dense MHA prefill SM100, sparse MLA prefill SM90/SM100.","replacement_claim_ids":["kernel-flashmla-current-c002"]} +{"receipt_id":"rem-kernel-flashmla-003","path":"wiki/kernels/flashmla.md","original_body_sha256":"997be90a5a387c776857d0d8257d24915a19a1aa829e9d76940c4de57a311691","original_unit_ids":["u003","u004"],"locator":"body:L11-L24","original_claim_id":"kernel-flashmla-original-c003","original_claim":"Every MLA KV-cache token uses one universal 656-byte format and fixed/variable page-size rules.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py","revision":"DeepSeek commit 71c7379","locator":"FP8+sparse docstring and dispatch","observation":"656 bytes applies to V3-family FP8 sparse decode only; dense decode is BF16, scales are per 128-value group, and page_block_size comes from the input."},{"path":"/tmp/kernelwiki-flashmla-pinned/tests/quant.py","revision":"same commit","locator":"FP8KVCacheLayout","observation":"A second 512-dimensional sparse format uses a different scale layout."}],"strongest_alternative":"The page intended only DeepSeek-V3 sparse decode despite saying each MLA token.","discriminating_result":"Its surrounding section and dense/sparse page claims explicitly generalize the format beyond that mode.","action":"correct","action_reason":"Narrow the byte layout to its exact model and dispatch mode and remove invented page-size universals.","proposed_replacement":"Describe 512 E4M3 bytes plus four FP32 group scales plus 64 BF16 RoPE values only for V3 FP8 sparse decode; treat page size as an input contract.","replacement_claim_ids":["kernel-flashmla-current-c003"]} +{"receipt_id":"rem-kernel-flashmla-004","path":"wiki/kernels/flashmla.md","original_body_sha256":"997be90a5a387c776857d0d8257d24915a19a1aa829e9d76940c4de57a311691","original_unit_ids":["u005"],"locator":"body:L28","original_claim_id":"kernel-flashmla-original-c004","original_claim":"Dense MLA decode uses a fixed 64-token page and reaches up to 3000 GB/s and 660 TFLOPS on H800.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"DeepSeek commit 71c7379","locator":"dense decode performance and public input contract","observation":"The maxima are reported for H800 SXM5/CUDA 12.8, but 64 is a test default rather than a fixed interface constraint."}],"strongest_alternative":"All intended production calls use the test default of 64.","discriminating_result":"The API derives page_block_size from k_cache.shape[1] and does not assert 64, so a test fixture cannot establish a compatibility rule.","action":"correct","action_reason":"Retain the exact author report and remove the false fixed-page condition.","proposed_replacement":"Describe paged BF16 dense decode and separately report the memory-bound and compute-bound maxima with environment.","replacement_claim_ids":["kernel-flashmla-current-c005"]} +{"receipt_id":"rem-kernel-flashmla-005","path":"wiki/kernels/flashmla.md","original_body_sha256":"997be90a5a387c776857d0d8257d24915a19a1aa829e9d76940c4de57a311691","original_unit_ids":["u006"],"locator":"body:L30-L87","original_claim_id":"kernel-flashmla-original-c005","original_claim":"The invented dense-decode template faithfully represents SM90 BF16 FlashMLA while loading an FP8 cache through TMA and explicit scale/RoPE arrays.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/csrc/sm90/decode","revision":"DeepSeek commit 71c7379","locator":"dense versus sparse_fp8 implementations","observation":"Dense BF16 and sparse FP8 are separate dispatch paths; the displayed function, helpers, signature, and per-head layout are not upstream code."}],"strongest_alternative":"The block is only high-level pseudocode and can combine details from multiple variants.","discriminating_result":"It is fenced as C++ with a kernel name/signature and labeled SM90 BF16 while its parameter types and loop describe the incompatible FP8 mode, making it actively misleading as code.","action":"delete","action_reason":"Remove fabricated kernel code; point readers to pinned source-level contracts and transparently labeled adjacent artifacts.","proposed_replacement":null,"replacement_claim_ids":["kernel-flashmla-current-c003","kernel-flashmla-current-c004"]} +{"receipt_id":"rem-kernel-flashmla-006","path":"wiki/kernels/flashmla.md","original_body_sha256":"997be90a5a387c776857d0d8257d24915a19a1aa829e9d76940c4de57a311691","original_unit_ids":["u008"],"locator":"body:L93-L118","original_claim_id":"kernel-flashmla-original-c007","original_claim":"The displayed sparse CUDA template accurately represents both sparse decode and sparse prefill with a universal 656-byte entry.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py","revision":"DeepSeek commit 71c7379","locator":"sparse decode and flash_mla_sparse_fwd signatures","observation":"Decode uses page-encoded batch indices and FP8 cache; prefill uses BF16 KV and unbatched [s_q,h_kv,topk] indices. The invented template matches neither exact API."}],"strongest_alternative":"A single conceptual loop is enough to explain both operators.","discriminating_result":"The modes have different tensor ranks, dtypes, address semantics, invalid-index rules, and return values, so the unified pseudo-signature erases correctness-critical contracts.","action":"delete","action_reason":"Replace the fabricated code with exact interface descriptions and small, explicitly derived contract checks.","proposed_replacement":null,"replacement_claim_ids":["kernel-flashmla-current-c004"]} +{"receipt_id":"rem-kernel-flashmla-007","path":"wiki/kernels/flashmla.md","original_body_sha256":"997be90a5a387c776857d0d8257d24915a19a1aa829e9d76940c4de57a311691","original_unit_ids":["u009","u010"],"locator":"body:L122-L137","original_claim_id":"kernel-flashmla-original-c008","original_claim":"The 1460/1000-TFLOPS SM100 path is dense MLA prefill and the displayed tcgen05/TMEM sketch is its verified implementation structure.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"DeepSeek commit 71c7379","locator":"dense MHA prefill benchmark and support matrix","observation":"The path and numbers are explicitly dense MHA, reported by NVIDIA; the page's role decomposition is not captured upstream code."}],"strongest_alternative":"MLA-shaped 192/128 dimensions justify calling the dense MHA operator MLA.","discriminating_result":"The repository deliberately calls 192/128 MHA mode and 576/512 MQA mode; changing its label collapses a published compatibility distinction.","action":"correct","action_reason":"Relabel the operator and separate verified architectural mechanisms from unsupported pseudocode.","proposed_replacement":"Call it dense MHA prefill on SM100, report the source attribution, and note only source-observed TMA/TMEM/tensor-core use.","replacement_claim_ids":["kernel-flashmla-current-c002","kernel-flashmla-current-c006"]} +{"receipt_id":"rem-kernel-flashmla-008","path":"wiki/kernels/flashmla.md","original_body_sha256":"997be90a5a387c776857d0d8257d24915a19a1aa829e9d76940c4de57a311691","original_unit_ids":["u012","u013"],"locator":"body:L144-L145","original_claim_id":"kernel-flashmla-original-c010","original_claim":"Sparse-decode throughput rows have dtype FP8.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"DeepSeek commit 71c7379","locator":"sparse decode performance paragraph","observation":"FP8 is the KV-cache storage format, while the matrix multiplication and output are bfloat16."}],"strongest_alternative":"Benchmark dtype columns conventionally name input/cache storage rather than MMA precision.","discriminating_result":"The column provides one unqualified dtype, while the first-party source explicitly distinguishes the two; readers cannot recover which interpretation was intended.","action":"correct","action_reason":"Split cache storage from compute precision in the performance table.","proposed_replacement":"Label both rows FP8 KV / BF16 MMA and preserve the H800/B200 qualifications.","replacement_claim_ids":["kernel-flashmla-current-c005"]} +{"receipt_id":"rem-kernel-flashmla-009","path":"wiki/kernels/flashmla.md","original_body_sha256":"997be90a5a387c776857d0d8257d24915a19a1aa829e9d76940c4de57a311691","original_unit_ids":["u016","u017"],"locator":"body:L148-L149","original_claim_id":"kernel-flashmla-original-c012","original_claim":"Sparse-prefill throughput rows have dtype FP8.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"DeepSeek commit 71c7379","locator":"sparse prefill API and equivalent operations","observation":"Q and KV inputs are explicitly bfloat16; the source does not call this FP8 prefill."}],"strongest_alternative":"The implementation could use internal FP8 tensor-core operations despite BF16 inputs.","discriminating_result":"The page's unqualified dtype purports to describe the benchmark interface; no source evidence establishes an FP8 conversion or FP8 benchmark mode.","action":"correct","action_reason":"Use the documented BF16 input dtype and preserve exact author-reported values/environments.","proposed_replacement":"Label sparse prefill BF16 inputs, 640 TFLOPS H800/CUDA 12.8 and 1450 TFLOPS B200/CUDA 12.9.","replacement_claim_ids":["kernel-flashmla-current-c005"]} +{"receipt_id":"rem-kernel-flashmla-010","path":"wiki/kernels/flashmla.md","original_body_sha256":"997be90a5a387c776857d0d8257d24915a19a1aa829e9d76940c4de57a311691","original_unit_ids":["u019"],"locator":"body:L154","original_claim_id":"kernel-flashmla-original-c014","original_claim":"SGLang and vLLM provide unversioned day-0 FlashMLA support.","original_verdict":"refuted","evidence":[{"path":"sources/prs/vllm/PR-39752.md","revision":"merge dc8df110","locator":"date and purpose","observation":"This source is a later FP8-prefill warning, not launch-day support evidence; the attached FlashInfer PDL PR is also a different project and no SGLang primary record is attached."}],"strongest_alternative":"Both serving frameworks likely announced support elsewhere even if the attached sources are wrong.","discriminating_result":"Day-0 is a precise version/date relationship; without named release boundaries or primary records it is not auditable, and neither attached PR supports it.","action":"delete","action_reason":"Remove the unversioned ecosystem claim rather than substituting inference for evidence.","proposed_replacement":null,"replacement_claim_ids":["kernel-flashmla-current-c006"]} +{"receipt_id":"rem-kernel-flashmla-011","path":"wiki/kernels/flashmla.md","original_body_sha256":"997be90a5a387c776857d0d8257d24915a19a1aa829e9d76940c4de57a311691","original_unit_ids":["u025"],"locator":"body:L166","original_claim_id":"kernel-flashmla-original-c018","original_claim":"A 656-byte latent KV entry is generally tied to DeepSeek's MLA architecture.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/tests/quant.py","revision":"DeepSeek commit 71c7379","locator":"FP8KVCacheLayout variants","observation":"656 is one V3-family FP8 sparse decode layout; dense BF16 and MODEL1 sparse layouts differ."}],"strongest_alternative":"The caveat only intends to say the layout is DeepSeek-specific, not universal within MLA.","discriminating_result":"Its definite wording and preceding universal cache section make 656 the asserted MLA format; the implementation exposes counterexamples.","action":"correct","action_reason":"State the precise V3-family FP8+sparse scope.","proposed_replacement":"Treat 656 bytes as a mode-specific ABI, not a general definition of MLA caching.","replacement_claim_ids":["kernel-flashmla-current-c003","kernel-flashmla-current-c009"]} +{"receipt_id":"rem-kernel-flashmla-012","path":"wiki/kernels/flashmla.md","original_body_sha256":"997be90a5a387c776857d0d8257d24915a19a1aa829e9d76940c4de57a311691","original_unit_ids":["u031"],"locator":"body:L178","original_claim_id":"kernel-flashmla-original-c023","original_claim":"The local full bundle is the upstream FlashMLA implementation and its derived decode variant is faithful.","original_verdict":"refuted","evidence":[{"path":"artifacts/kernels/flashmla/full/PROVENANCE.yaml","revision":"local provenance","locator":"bundle and per-file repositories","observation":"Files are from CUTLASS and FlashInfer, not DeepSeek FlashMLA."},{"path":"artifacts/kernels/flashmla/variants/01-mla-decode-inner-loop.cu","revision":"SHA-256 4f494dc2d282468966a1e2028bc50d0ef0c364b34d24f75ee881608ddef2df69","locator":"comments and constants","observation":"The code invents Dc=128 and combines wgmma/tcgen05 across architectures."}],"strongest_alternative":"'Upstream' can mean any upstream project and the variant is explicitly derived.","discriminating_result":"The page subject is FlashMLA and does not disclose the two unrelated repositories; a derived label does not cure false technical content.","action":"correct","action_reason":"Retain useful artifacts with explicit adjacent-implementation labels and replace the variant with a narrow verified contract helper.","proposed_replacement":"Call full files adjacent CUTLASS/FlashInfer references and identify their exact per-file origins; label derived helper as non-upstream.","replacement_claim_ids":["kernel-flashmla-current-c007"]} +{"receipt_id":"rem-kernel-flashmla-013","path":"wiki/kernels/flashmla.md","original_body_sha256":"997be90a5a387c776857d0d8257d24915a19a1aa829e9d76940c4de57a311691","original_unit_ids":["u034"],"locator":"frontmatter:performance_claims","original_claim_id":"kernel-flashmla-original-c025","original_claim":"Two B200 structured records support 1460-TFLOPS BF16 dense prefill and 1450-TFLOPS FP8 sparse prefill at about 65% utilization.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"DeepSeek commit 71c7379","locator":"prefill performance paragraphs","observation":"The throughput maxima are source-reported but ~65% utilization is absent, shapes/timing details are incomplete, dense prefill is MHA, and sparse-prefill inputs are BF16."}],"strongest_alternative":"Utilization can be derived from B200 peak throughput and the shape strings are sufficient summaries.","discriminating_result":"Peak depends on SKU/clock/dtype/sparsity conventions, while the source omits those details and complete benchmark cells; the second record also has the wrong dtype.","action":"correct","action_reason":"Remove inadmissible structured records while retaining qualified author reports in prose.","proposed_replacement":"Set performance_claims to [] and give a source-reported table with exact labels, environments that are stated, and non-reproduction warning.","replacement_claim_ids":["kernel-flashmla-current-c005"]} +{"receipt_id":"rem-source-flashmla-blog-001","receipt_scope":"source-trail","path":"sources/blogs/flashmla.md","original_body_sha256":"92a9b71d658ecabb61dbb911d6cfa88e6c604bf374f7b94ab21ac1e6e89832b4","original_unit_ids":[],"locator":"Summary, Variants, Token Format, and Key Code","original_claim_id":"source-blog-flashmla-summary","original_claim":"The local repository summary presents 70KB/token, universal 656-byte caching, FP8 sparse prefill, and invented CUDA paths as first-party FlashMLA facts.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"DeepSeek commit 71c7379; SHA-256 46c001c53800f47148918f4fc2969102492dd5164f710fc015db1622f6f9929a","locator":"support matrix, APIs, and performance","observation":"The repo does not state 70KB/token; dense prefill is MHA, sparse prefill takes BF16, 656 is V3 FP8 sparse decode only, and the displayed CUDA blocks are absent."}],"strongest_alternative":"The local page is an editorial benchmark summary allowed to add illustrative code.","discriminating_result":"Its artifact extraction provenance inherits from the first-party source without labeling derivation, so fabricated code and scope changes become false evidence.","action":"correct","action_reason":"Rewrite the capture to match commit 71c7379 and label every local illustration explicitly.","proposed_replacement":"Record the exact matrix, cache/interface contracts, qualified performance observations, and small derived contract checks.","replacement_claim_ids":["kernel-flashmla-current-c002","kernel-flashmla-current-c003","kernel-flashmla-current-c004","kernel-flashmla-current-c005"]} +{"receipt_id":"rem-source-flashmla-blog-artifacts-001","receipt_scope":"source-trail","path":"artifacts/blogs/flashmla","original_body_sha256":"dadb96fc53c02b32b66a5b4f9224bd60e66044c22b09188a5cc48316f3511f08","original_unit_ids":[],"locator":"MANIFEST.yaml, PROVENANCE.yaml, and two extracted CUDA files","original_claim_id":"source-artifact-flashmla-blog-code","original_claim":"The extracted bundle attributes an invented Dc=128 decode loop and a top-k selector to the DeepSeek repository.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py","revision":"DeepSeek commit 71c7379","locator":"decode dimensions and sparse API inputs","observation":"DeepSeek MLA decode uses 576/512 dimensions and consumes caller-provided indices; the repo does not contain either displayed block."}],"strongest_alternative":"The files are extracted only from local Markdown, not claimed verbatim upstream.","discriminating_result":"origin_url/upstream_sha/license metadata and absence of an in-code derived warning falsely present the blocks as source-derived evidence.","action":"correct","action_reason":"Regenerate from corrected source Markdown whose blocks explicitly say they are KernelWiki contract checks, not upstream code.","proposed_replacement":"Extract small Python checks for the V3 656-byte arithmetic and page-encoded index mapping.","replacement_claim_ids":["kernel-flashmla-current-c003","kernel-flashmla-current-c008"]} +{"receipt_id":"rem-source-flashmla-kernel-artifacts-001","receipt_scope":"source-trail","path":"artifacts/kernels/flashmla","original_body_sha256":"ec13a9a195c414a691db1be698d1f4bc092880cf55ae38a9b3743dc9b1394d70","original_unit_ids":[],"locator":"full/PROVENANCE.yaml and variants bundle","original_claim_id":"source-artifact-flashmla-kernel-labels","original_claim":"The kernel artifacts imply a DeepSeek FlashMLA full implementation while mixing CUTLASS and FlashInfer files and retaining an inaccurate derived decode loop.","original_verdict":"refuted","evidence":[{"path":"artifacts/kernels/flashmla/full/PROVENANCE.yaml","revision":"CUTLASS 9baa06dd plus FlashInfer 9a05c92a","locator":"per-file provenance","observation":"Neither file is from deepseek-ai/FlashMLA and the bundle-level URL names only one of the two origins."},{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"DeepSeek commit 71c7379","locator":"support matrix","observation":"The derived loop's Dc=128 and cross-architecture wgmma/tcgen05 description do not match supported decode modes."}],"strongest_alternative":"Adjacent implementations are valuable and the derived file is already marked non-upstream.","discriminating_result":"They can remain valuable only if the page and bundle disclose adjacency and the derived helper itself contains no false implementation claim.","action":"correct","action_reason":"Preserve byte-verified files, add explicit adjacent scope, and replace the derived file with a narrow verified layout/index helper.","proposed_replacement":"Document each origin and use a derived CUDA helper limited to constants/address mapping verified from DeepSeek commit 71c7379.","replacement_claim_ids":["kernel-flashmla-current-c007"]} +{"receipt_id":"rem-source-flashmla-relationship-001","receipt_scope":"source-trail","path":"wiki/kernels/flashmla.md","original_body_sha256":"997be90a5a387c776857d0d8257d24915a19a1aa829e9d76940c4de57a311691","original_unit_ids":[],"locator":"frontmatter:sources pr-flashinfer-1117","original_claim_id":"source-relationship-flashmla-flashinfer-1117","original_claim":"FlashInfer PR 1117 is a source for DeepSeek FlashMLA's core cache, sparse-attention, and performance claims.","original_verdict":"refuted","evidence":[{"path":"sources/prs/flashinfer/PR-1117.md","revision":"FlashInfer merge 2f01a9a3","locator":"repository, title, and summary","observation":"The PR adds PDL to FlashInfer batch attention/MLA decode; it is a separate implementation and does not substantiate DeepSeek FlashMLA's published contracts or numbers."}],"strongest_alternative":"Any MLA-decode implementation is useful corroboration.","discriminating_result":"Repository identity and feature scope differ; using the PR as an undifferentiated source invites transfer of FlashInfer-specific PDL behavior to FlashMLA.","action":"delete","action_reason":"Remove the unrelated relationship; retain only explicitly labeled adjacent implementations where discussed.","proposed_replacement":null,"replacement_claim_ids":["kernel-flashmla-current-c008"]} +{"receipt_id":"rem-source-flashmla-relationship-002","receipt_scope":"source-trail","path":"wiki/kernels/flashmla.md","original_body_sha256":"997be90a5a387c776857d0d8257d24915a19a1aa829e9d76940c4de57a311691","original_unit_ids":[],"locator":"frontmatter:sources pr-vllm-39752","original_claim_id":"source-relationship-flashmla-vllm-39752","original_claim":"vLLM PR 39752 is a source for the DeepSeek FlashMLA kernel page and its day-0 serving claim.","original_verdict":"refuted","evidence":[{"path":"sources/prs/vllm/PR-39752.md","revision":"vLLM merge dc8df110","locator":"title, date, and changed path","observation":"The PR adds a vLLM startup warning for FP8 prefill query quantization in 2026; it neither introduces DeepSeek FlashMLA nor establishes day-0 support."}],"strongest_alternative":"It supplies a useful current integration caveat for FP8 MLA serving.","discriminating_result":"The audited page does not document vLLM's separate query-quantization option, and the PR cannot evidence first-party FlashMLA mechanisms or release timing.","action":"delete","action_reason":"Keep source relationships claim-directed and version-pinned to the implementation actually discussed.","proposed_replacement":null,"replacement_claim_ids":["kernel-flashmla-current-c008"]} +{"receipt_id":"rem-kernel-fp8-block-scale-gemm-001","path":"wiki/kernels/fp8-block-scale-gemm.md","original_body_sha256":"9fc01ec2833f70fa337f1ec2f149ac72e477d8465563010c15b1aa7c0c2bcc50","original_unit_ids":["u003"],"locator":"body:L11-L20","original_claim_id":"kernel-fp8-block-scale-gemm-original-c003","original_claim":"A common cross-architecture structure uses FP32-or-E4M3 scales and applies them at the MMA boundary.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-deepgemm-pinned/README.md","revision":"DeepGEMM 891d57b4","locator":"scale-factor notice","observation":"SM90 uses FP32 scales and SM100 uses packed UE8M0, not E4M3."},{"path":"/tmp/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh","revision":"same commit","locator":"promotion loop","observation":"SM90 applies scales after WGMMA K-block accumulation in CUDA-core final_accum."}],"strongest_alternative":"The diagram is conceptual and allows either architecture's scale type.","discriminating_result":"It names a wrong E4M3 alternative and one scale-placement rule for two materially different implementations.","action":"correct","action_reason":"Split the Hopper and Blackwell scale ABIs and accumulation locations.","proposed_replacement":"Describe FP32 post-WGMMA promotion on SM90 and packed UE8M0 block-scaled UMMA on SM100 separately.","replacement_claim_ids":["kernel-fp8-block-scale-gemm-current-c003","kernel-fp8-block-scale-gemm-current-c004"]} +{"receipt_id":"rem-kernel-fp8-block-scale-gemm-002","path":"wiki/kernels/fp8-block-scale-gemm.md","original_body_sha256":"9fc01ec2833f70fa337f1ec2f149ac72e477d8465563010c15b1aa7c0c2bcc50","original_unit_ids":["u004"],"locator":"body:L24-L44","original_claim_id":"kernel-fp8-block-scale-gemm-original-c004","original_claim":"Hopper has an FP22 accumulator and promotes every 128 WGMMAs, as shown by the invented kernel.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/html/2412.19437v2","revision":"DeepSeek-V3 report v2","locator":"Sections 3.3.3 and 3.5.2","observation":"The source describes 14-bit alignment/addition precision and a 128-element interval equal to four WGMMAs."},{"path":"/tmp/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh","revision":"891d57b4","locator":"BLOCK_K and promotion loops","observation":"The real implementation promotes scaled partials once per BLOCK_K=128 block; the displayed API/helpers are absent."}],"strongest_alternative":"FP22 is a common shorthand and '128 WGMMAs' meant an Nc=128 element span.","discriminating_result":"The comment grammatically counts WGMMAs and gives a different precision model; the primary paper explicitly says four WGMMAs and 14-bit additions.","action":"correct","action_reason":"Replace fabricated code with a narrow source-adapted promotion skeleton and exact interval qualification.","proposed_replacement":"State BLOCK_K=128, four WGMMA K=32 operations in the cited configuration, then scale/add into FP32 final_accum.","replacement_claim_ids":["kernel-fp8-block-scale-gemm-current-c003"]} +{"receipt_id":"rem-kernel-fp8-block-scale-gemm-003","path":"wiki/kernels/fp8-block-scale-gemm.md","original_body_sha256":"9fc01ec2833f70fa337f1ec2f149ac72e477d8465563010c15b1aa7c0c2bcc50","original_unit_ids":["u005"],"locator":"body:L48-L72","original_claim_id":"kernel-fp8-block-scale-gemm-original-c005","original_claim":"The displayed tcgen05 inline PTX and global-scale epilogue faithfully represent DeepGEMM SM100.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh","revision":"891d57b4","locator":"scale copy, descriptor, and MMA loop","observation":"The real implementation uses TMA/UTCCP into TMEM and make_instr_desc_block_scaled; the shown helpers and global scale are absent."},{"path":"/tmp/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/ptx/tcgen05.cuh","revision":"891d57b4","locator":"mxf8f6f4 wrapper","observation":"Actual syntax and scale operands differ from the displayed string."}],"strongest_alternative":"The code is illustrative pseudocode rather than an exact wrapper.","discriminating_result":"It is fenced/labeled CUDA with concrete PTX syntax that would teach an invalid instruction form and a nonexistent epilogue behavior.","action":"delete","action_reason":"Remove invalid inline PTX and explain the verified source-level path in prose.","proposed_replacement":null,"replacement_claim_ids":["kernel-fp8-block-scale-gemm-current-c004"]} +{"receipt_id":"rem-kernel-fp8-block-scale-gemm-004","path":"wiki/kernels/fp8-block-scale-gemm.md","original_body_sha256":"9fc01ec2833f70fa337f1ec2f149ac72e477d8465563010c15b1aa7c0c2bcc50","original_unit_ids":["u006"],"locator":"body:L76-L83","original_claim_id":"kernel-fp8-block-scale-gemm-original-c006","original_claim":"The fixed displayed scale tensors form a universal ABI and use FP32-or-E4M3 scale elements.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-deepgemm-pinned/README.md","revision":"891d57b4","locator":"scale-factor notice","observation":"SM90 requires FP32 and SM100 packed UE8M0; current APIs support multiple granularity/layout recipes."}],"strongest_alternative":"The diagram only documents the DeepSeek-V3 128-granularity recipe.","discriminating_result":"It explicitly presents a Memory Layout for the cross-architecture page and supplies an unsupported scale dtype, while omitting architecture-specific transformed layouts.","action":"correct","action_reason":"Separate the model grouping from each implementation's input ABI.","proposed_replacement":"Give logical 1x128/128x128 groups, then state SM90 FP32 TMA layout and SM100 packed UE8M0/TMEM layout without invented fixed tensor shapes.","replacement_claim_ids":["kernel-fp8-block-scale-gemm-current-c002","kernel-fp8-block-scale-gemm-current-c004"]} +{"receipt_id":"rem-kernel-fp8-block-scale-gemm-005","path":"wiki/kernels/fp8-block-scale-gemm.md","original_body_sha256":"9fc01ec2833f70fa337f1ec2f149ac72e477d8465563010c15b1aa7c0c2bcc50","original_unit_ids":["u008"],"locator":"body:L88","original_claim_id":"kernel-fp8-block-scale-gemm-original-c008","original_claim":"CUTLASS SM100 schedules achieve a similar peak ratio to DeepGEMM's H800 maximum.","original_verdict":"refuted","evidence":[{"path":"sources/docs/cutlass-changelog-sm100.md","revision":"release capture through 4.5.0","locator":"blockscaled GEMM entries","observation":"Availability is documented, but there is no matched shape, GPU, peak definition, or comparable ratio."}],"strongest_alternative":"CUTLASS's general near-cuBLAS reputation makes similar utilization plausible.","discriminating_result":"Plausibility is not a benchmark cell, and the compared architectures and peak conventions differ.","action":"delete","action_reason":"Retain CUTLASS availability without an unsupported cross-library performance comparison.","proposed_replacement":null,"replacement_claim_ids":["kernel-fp8-block-scale-gemm-current-c005"]} +{"receipt_id":"rem-kernel-fp8-block-scale-gemm-006","path":"wiki/kernels/fp8-block-scale-gemm.md","original_body_sha256":"9fc01ec2833f70fa337f1ec2f149ac72e477d8465563010c15b1aa7c0c2bcc50","original_unit_ids":["u009"],"locator":"body:L92","original_claim_id":"kernel-fp8-block-scale-gemm-original-c009","original_claim":"The exact DeepSeek-style format applies to DeepSeek-V3, Qwen2-FP8, and similar inference models.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/html/2412.19437v2","revision":"DeepSeek-V3 report v2","locator":"FP8 Training section","observation":"The primary source specifies DeepSeek-V3 only and does not define Qwen2's scale layout, dtype, or kernel ABI."}],"strongest_alternative":"All FP8 model checkpoints are compatible at the GEMM level.","discriminating_result":"FP8 element type alone does not determine scale granularity, scale representation, matrix layout, or supported architecture.","action":"correct","action_reason":"Replace model-name generalization with an explicit contract-matching rule.","proposed_replacement":"Use only when the producer matches element type, scale granularity/representation, layout, and architecture-specific API.","replacement_claim_ids":["kernel-fp8-block-scale-gemm-current-c006"]} +{"receipt_id":"rem-kernel-fp8-block-scale-gemm-007","path":"wiki/kernels/fp8-block-scale-gemm.md","original_body_sha256":"9fc01ec2833f70fa337f1ec2f149ac72e477d8465563010c15b1aa7c0c2bcc50","original_unit_ids":["u012"],"locator":"frontmatter:performance_claims","original_claim_id":"kernel-fp8-block-scale-gemm-original-c012","original_claim":"1550 TFLOPS is a structured H800 4096-cubed result at ~90% utilization via promotion.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-deepgemm-pinned/README.md","revision":"891d57b4","locator":"2025-04-18 news item","observation":"Only an up-to 1550-TFLOPS H800 statement is present; shape, utilization, timing, samples, variance, and single-mechanism attribution are absent."}],"strongest_alternative":"The linked optimization PRs or common benchmark suite imply 4096 cubed and ~90%.","discriminating_result":"Structured metadata must transcribe one source cell; inference across links cannot manufacture missing environment fields.","action":"correct","action_reason":"Remove the inadmissible record while retaining the exact qualified author report in prose.","proposed_replacement":"Set performance_claims to [] and report only up to 1550 TFLOPS on H800, unreproduced and without a claimed shape/utilization.","replacement_claim_ids":["kernel-fp8-block-scale-gemm-current-c005"]} +{"receipt_id":"rem-source-fp8-block-scale-gemm-relationship-001","receipt_scope":"source-trail","path":"wiki/kernels/fp8-block-scale-gemm.md","original_body_sha256":"9fc01ec2833f70fa337f1ec2f149ac72e477d8465563010c15b1aa7c0c2bcc50","original_unit_ids":[],"locator":"frontmatter:sources doc-cutlass-blackwell","original_claim_id":"source-relationship-fp8-block-scale-cutlass-summary","original_claim":"A mutable editorial CUTLASS 4.x summary is direct evidence for DeepGEMM's architecture-specific implementation.","original_verdict":"refuted","evidence":[{"path":"sources/docs/nvidia-cutlass-blackwell.md","revision":"local summary","locator":"Overview and performance claims","observation":"The page combines many CUTLASS releases and unstructured comparative claims; it is not version-pinned DeepGEMM evidence."}],"strongest_alternative":"The broad summary provides useful Blackwell context.","discriminating_result":"The audited page can cite the PTX contract directly and the exact DeepGEMM commit, avoiding implementation transfer from a separate library.","action":"delete","action_reason":"Use claim-directed, version-pinned sources.","proposed_replacement":null,"replacement_claim_ids":["kernel-fp8-block-scale-gemm-current-c004","kernel-fp8-block-scale-gemm-current-c007"]} +{"receipt_id":"rem-source-fp8-block-scale-gemm-relationship-002","receipt_scope":"source-trail","path":"wiki/kernels/fp8-block-scale-gemm.md","original_body_sha256":"9fc01ec2833f70fa337f1ec2f149ac72e477d8465563010c15b1aa7c0c2bcc50","original_unit_ids":[],"locator":"frontmatter:sources doc-cutlass-changelog-sm100","original_claim_id":"source-relationship-fp8-block-scale-cutlass-changelog","original_claim":"The broad mutable CUTLASS changelog capture is required as a source for this DeepGEMM-focused mechanism page.","original_verdict":"refuted","evidence":[{"path":"sources/docs/cutlass-changelog-sm100.md","revision":"release capture through 4.5.0","locator":"blockscaled GEMM entries","observation":"It proves adjacent CUTLASS availability but not DeepGEMM implementation or comparable performance, and the corrected page makes no CUTLASS performance claim."}],"strongest_alternative":"Adjacent schedule availability is useful to readers choosing implementations.","discriminating_result":"That can be stated as an external link without treating a multi-release editorial capture as evidence for DeepGEMM semantics.","action":"delete","action_reason":"Limit source relationships to the DeepSeek paper, exact DeepGEMM snapshot, and official PTX contract used by current claims.","proposed_replacement":null,"replacement_claim_ids":["kernel-fp8-block-scale-gemm-current-c007"]} +{"receipt_id":"rem-kernel-fused-moe-001","path":"wiki/kernels/fused-moe.md","original_body_sha256":"c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d","original_unit_ids":["u001"],"locator":"body:L5","original_claim_id":"kernel-fused-moe-original-c001","original_claim":"Track A is the full routing-through-combine MoE forward in one to three launches and saves 21.9% activation traffic over five-to-seven-launch paths.","original_verdict":"refuted","evidence":[{"url":"https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","revision":"fetched 2026-08-08","locator":"description and signature","observation":"The official definition states routing plus two grouped GEMMs; it publishes neither launch counts nor a 21.9% traffic result."}],"strongest_alternative":"The benchmark UI may abbreviate a fully fused implementation whose launch and traffic numbers are known from framework profiles.","discriminating_result":"The official evaluation treats a solution entry point as opaque and the page supplies no trace/profile cell; a Python call count cannot establish GPU launches.","action":"correct","action_reason":"State only the operation scope in the official contract and remove unrecorded performance figures.","proposed_replacement":"Define Track A as FP8 block-scale MoE with DeepSeek routing and two grouped GEMMs; make no launch or traffic claim.","replacement_claim_ids":["kernel-fused-moe-current-c001","kernel-fused-moe-current-c005"]} +{"receipt_id":"rem-kernel-fused-moe-002","path":"wiki/kernels/fused-moe.md","original_body_sha256":"c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d","original_unit_ids":["u003"],"locator":"body:L11-L38","original_claim_id":"kernel-fused-moe-original-c003","original_claim":"The diagram routes top-8 from 32 total experts and presents that as the benchmark pipeline.","original_verdict":"refuted","evidence":[{"url":"https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","revision":"fetched 2026-08-08","locator":"Axes and Reference Implementation","observation":"Routing spans 256 global experts; 32 is the number of local experts for EP=8."}],"strongest_alternative":"The page uses 32 as a shorthand for the experts visible on one GPU.","discriminating_result":"The text says selection is from 32, while the routing-logit tensor is [T,256] and selection precedes local-expert filtering.","action":"correct","action_reason":"Distinguish global routing geometry from local expert compute.","proposed_replacement":"Route top-8 from 256 global experts after selecting four of eight groups; compute only the local interval of 32 experts.","replacement_claim_ids":["kernel-fused-moe-current-c002"]} +{"receipt_id":"rem-kernel-fused-moe-003","path":"wiki/kernels/fused-moe.md","original_body_sha256":"c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d","original_unit_ids":["u004"],"locator":"body:L42-L61","original_claim_id":"kernel-fused-moe-original-c004","original_claim":"vLLM, SGLang, and a fully fused implementation have exactly seven, five, and one-to-two launches with the displayed boundaries.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashinfer-bench-starter-kit/EVALUATION.md","revision":"75ccd05cafceb0fd1f86be4cd0f2117249463c66","locator":"MoE evaluation and baseline","observation":"The official contract names one wrapper baseline and measures solution latency without framework launch-count rows."}],"strongest_alternative":"The counts were measured independently even if absent from the contest documentation.","discriminating_result":"No profiler capture, software revision, workload, or launch trace is attached, and the current implementation separately tunes GEMM tactics behind one API call.","action":"delete","action_reason":"Do not teach fixed launch counts without a pinned GPU trace.","proposed_replacement":null,"replacement_claim_ids":["kernel-fused-moe-current-c005"]} +{"receipt_id":"rem-kernel-fused-moe-004","path":"wiki/kernels/fused-moe.md","original_body_sha256":"c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d","original_unit_ids":["u006"],"locator":"body:L67-L120","original_claim_id":"kernel-fused-moe-original-c006","original_claim":"The displayed CUDA/tcgen05 function is a usable gated-dual-GEMM implementation.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/fused_moe/core.py","revision":"7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"trtllm_fp8_block_scale_moe","observation":"The real path calls compiled runners with complete routing, scale, tactic, and output contracts; none of the displayed helpers exist."},{"path":"wiki/hardware/tcgen05-mma.md","revision":"verified PTX ISA 9.2 summary","locator":"instruction forms and ownership","observation":"The inline instruction omits architecture-defined descriptors/predicates, and the surrounding code has undefined storage, indices, and synchronization."}],"strongest_alternative":"The block is clearly pseudocode, so missing declarations and exact PTX syntax are harmless.","discriminating_result":"It is fenced as C++, gives a concrete asm string, and claims a kernel; those details would be copied as implementation guidance and are materially invalid.","action":"delete","action_reason":"Replace fabricated source with a narrow, executable derived semantic reference.","proposed_replacement":null,"replacement_claim_ids":["kernel-fused-moe-current-c004"]} +{"receipt_id":"rem-kernel-fused-moe-005","path":"wiki/kernels/fused-moe.md","original_body_sha256":"c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d","original_unit_ids":["u007"],"locator":"body:L124-L140","original_claim_id":"kernel-fused-moe-original-c007","original_claim":"The displayed BF16/32-expert keyword call matches the FlashInfer Track A API.","original_verdict":"refuted","evidence":[{"url":"https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","revision":"fetched 2026-08-08","locator":"Signature and baseline wrapper","observation":"The benchmark uses FP8 hidden states, [T,256] logits, separate gemm1/gemm2 tensors and FP32 scales, local offset, and routed scaling factor; the shown keywords do not match."}],"strongest_alternative":"FlashInfer may accept convenient aliases not shown in the benchmark wrapper.","discriminating_result":"The first-party Python signature at commit 7f614b8 has the benchmark names and no w_gate_up, w_down, router_weights, or block_scale_gate_up aliases.","action":"correct","action_reason":"Use the exact benchmark argument names, dtypes, and global/local dimensions.","proposed_replacement":"Show a shape contract and concise call using routing_logits, hidden_states_scale, gemm1_weights_scale, gemm2_weights_scale, and the explicit routing parameters.","replacement_claim_ids":["kernel-fused-moe-current-c003"]} +{"receipt_id":"rem-kernel-fused-moe-006","path":"wiki/kernels/fused-moe.md","original_body_sha256":"c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d","original_unit_ids":["u008"],"locator":"body:L144-L188","original_claim_id":"kernel-fused-moe-original-c008","original_claim":"The displayed Triton function correctly implements routed block-scaled gate-up fusion.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py","revision":"7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"DeepSeek routing reference and expert compute","observation":"A correct reference consumes global/local routing, weights, all scales, bounds, W2, and weighted accumulation; the shown function ignores expert_ids and all scales and lacks required masks."}],"strongest_alternative":"It intentionally implements only the per-expert W13 sub-operation.","discriminating_result":"Even that narrower operation needs valid expert token indirection, scale application, and M/N/K masks, none of which are implemented.","action":"delete","action_reason":"Remove non-executable code and point to the verified derived semantic reference.","proposed_replacement":null,"replacement_claim_ids":["kernel-fused-moe-current-c004"]} +{"receipt_id":"rem-kernel-fused-moe-007","path":"wiki/kernels/fused-moe.md","original_body_sha256":"c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d","original_unit_ids":["u009","u010","u011"],"locator":"body:L194-L196","original_claim_id":"kernel-fused-moe-original-c009","original_claim":"The three-row B200 framework table records exact TFLOPS, latency, and launch counts for SGLang, FlashInfer CuTe DSL, and vLLM.","original_verdict":"refuted","evidence":[{"url":"https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","revision":"complete serialized page fetched 2026-08-08","locator":"solutions and per-workload traces","observation":"The official page has per-solution latency_ms traces and no such framework table; five of the six displayed numbers are absent entirely and the remaining digit string occurs only inside unrelated decimal latencies."}],"strongest_alternative":"The table came from an earlier official leaderboard snapshot.","discriminating_result":"No archived revision, solution IDs, shapes, clocks, timing method, or raw trials are attached, so the cells are not reproducible or attributable.","action":"delete","action_reason":"Remove untraceable performance cells rather than inventing a replacement benchmark.","proposed_replacement":null,"replacement_claim_ids":["kernel-fused-moe-current-c005"]} +{"receipt_id":"rem-kernel-fused-moe-008","path":"wiki/kernels/fused-moe.md","original_body_sha256":"c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d","original_unit_ids":["u012"],"locator":"body:L200","original_claim_id":"kernel-fused-moe-original-c010","original_claim":"No tuned B200 FP8 MoE configuration exists and implementers must choose tiles/stages manually from scratch.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/fused_moe/core.py","revision":"7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"MoERunner autotuning","observation":"FlashInfer enumerates shape-valid GEMM1/GEMM2 tactics and performs token-bucketed runtime selection."}],"strongest_alternative":"Runtime autotuning does not equal a static pre-tuned table.","discriminating_result":"Correct; therefore the safe claim is about the actual runtime mechanism, not the absolute absence of configurations or a manual-from-scratch requirement.","action":"correct","action_reason":"Document the version-pinned autotuner behavior precisely.","proposed_replacement":"At FlashInfer 7f614b8, runtime autotuning selects separate GEMM1 and GEMM2 tactics over token buckets.","replacement_claim_ids":["kernel-fused-moe-current-c006"]} +{"receipt_id":"rem-kernel-fused-moe-009","path":"wiki/kernels/fused-moe.md","original_body_sha256":"c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d","original_unit_ids":["u013"],"locator":"body:L201","original_claim_id":"kernel-fused-moe-original-c011","original_claim":"A 128-element block is required specifically to avoid FP8 overflow and guarantee stability.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py","revision":"7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"DeepSeek FP8 trace definition","observation":"Block 128 is the trace's input scale granularity; the operation consumes already-quantized FP8 tensors and FP32 scales and states no universal stability threshold."},{"path":"sources/docs/deepseek-v3-fp8.md","revision":"DeepSeek-V3 report v2 capture","locator":"Fine-Grained Quantization","observation":"The paper motivates fine groups by local outliers and documents phase-specific exceptions, not a universal overflow requirement."}],"strongest_alternative":"A practical implementation may indeed become unstable with coarser scaling.","discriminating_result":"That needs a named quantizer, workload, error metric, and comparative result; none is attached to this absolute claim.","action":"correct","action_reason":"Separate the fixed benchmark ABI from unsupported causal performance language.","proposed_replacement":"State that this definition fixes 128-element FP32 scale groups and that other formats are not interchangeable.","replacement_claim_ids":["kernel-fused-moe-current-c003","kernel-fused-moe-current-c007"]} +{"receipt_id":"rem-kernel-fused-moe-010","path":"wiki/kernels/fused-moe.md","original_body_sha256":"c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d","original_unit_ids":["u014"],"locator":"body:L202","original_claim_id":"kernel-fused-moe-original-c012","original_claim":"batch=1 is launch-overhead dominated and batch=4096 is throughput dominated for this benchmark.","original_verdict":"refuted","evidence":[{"url":"https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","revision":"fetched 2026-08-08","locator":"Axes","observation":"The varied field is seq_len/number of tokens, not request batch, and no bottleneck profile is published."}],"strongest_alternative":"Token count often correlates with decode and prefill regimes.","discriminating_result":"Correlation cannot relabel the trace axis or establish which component dominates without serving context and profiling.","action":"correct","action_reason":"Use the official axis name and avoid mechanism attribution.","proposed_replacement":"Treat seq_len as a variable token-count axis; do not infer prefill/decode or a dominant bottleneck from it alone.","replacement_claim_ids":["kernel-fused-moe-current-c005"]} +{"receipt_id":"rem-kernel-fused-moe-011","path":"wiki/kernels/fused-moe.md","original_body_sha256":"c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d","original_unit_ids":["u016"],"locator":"body:L204","original_claim_id":"kernel-fused-moe-original-c014","original_claim":"All TMA descriptors and expert pointers require 128-byte alignment.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html","revision":"CUDA Driver API 13.2.1","locator":"cuTensorMapEncodeTiled requirements","observation":"The descriptor object is 64-byte aligned; ordinary globalAddress is 16-byte aligned, with selected 32-byte exceptions. 128-byte alignment is tied to particular swizzle/shared-memory cases."}],"strongest_alternative":"Using 128-byte alignment everywhere is a safe conservative implementation choice.","discriminating_result":"A conservative choice is not the API's universal requirement and must not be taught as one.","action":"correct","action_reason":"State that alignment is descriptor-configuration dependent and defer to the exact CUDA contract.","proposed_replacement":"Validate tensor-map address, global base, stride, interleave, dtype, and swizzle requirements for the selected descriptor.","replacement_claim_ids":["kernel-fused-moe-current-c007"]} +{"receipt_id":"rem-kernel-fused-moe-012","path":"wiki/kernels/fused-moe.md","original_body_sha256":"c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d","original_unit_ids":["u017"],"locator":"body:L205","original_claim_id":"kernel-fused-moe-original-c015","original_claim":"Gate and up require two distinct TMEM allocations that compete for the whole SM budget.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py","revision":"7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"_fp8_moe_run_experts","observation":"The contract uses one concatenated [2I,H] W13 and one logical GEMM1 result, then splits it for SwiGLU; no dual allocation is required."}],"strongest_alternative":"A particular hand-tuned tcgen05 kernel may choose two accumulator regions.","discriminating_result":"That implementation choice cannot be promoted to a requirement of the benchmark or all fused W13 kernels.","action":"correct","action_reason":"Describe the logical W13 contract without inventing TMEM ownership.","proposed_replacement":"One W13 projection produces two I-wide halves; any TMEM layout is implementation-specific and must follow actual resource accounting.","replacement_claim_ids":["kernel-fused-moe-current-c002","kernel-fused-moe-current-c004"]} +{"receipt_id":"rem-kernel-fused-moe-013","path":"wiki/kernels/fused-moe.md","original_body_sha256":"c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d","original_unit_ids":["u018"],"locator":"body:L209","original_claim_id":"kernel-fused-moe-original-c016","original_claim":"The fixed Track A kernel applies to DeepSeek-V3, Mixtral, and similar MoE models.","original_verdict":"refuted","evidence":[{"url":"https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","revision":"fetched 2026-08-08","locator":"tags, axes, and signature","observation":"The definition is tagged DeepSeek-V3/R1 and fixes routing, 256/32 experts, FP8 type, scale layout, and dimensions; it states no Mixtral compatibility."}],"strongest_alternative":"The same high-level gate/up/down formula appears in many MoE families.","discriminating_result":"High-level algebra does not establish routing method, expert geometry, quantization, scale ABI, layout, or runtime compatibility.","action":"correct","action_reason":"Limit use to producers that match the complete contract.","proposed_replacement":"Use for this exact DeepSeek-style contract or only after every routing, tensor, scale, layout, and architecture field matches.","replacement_claim_ids":["kernel-fused-moe-current-c001","kernel-fused-moe-current-c007"]} +{"receipt_id":"rem-kernel-fused-moe-014","path":"wiki/kernels/fused-moe.md","original_body_sha256":"c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d","original_unit_ids":["u019"],"locator":"body:L210","original_claim_id":"kernel-fused-moe-original-c017","original_claim":"The benchmark directly covers prefill and decode by mapping high/low batch to those phases.","original_verdict":"refuted","evidence":[{"url":"https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","revision":"fetched 2026-08-08","locator":"Axes","observation":"The definition exposes variable seq_len/number of tokens and gives no serving-phase or request-batch label."}],"strongest_alternative":"Small and large token counts are representative proxies for decode and prefill.","discriminating_result":"A proxy needs a defined request topology, concurrency, and cache state; those are absent from this kernel trace.","action":"correct","action_reason":"Keep the exact workload-axis interpretation.","proposed_replacement":"Report results by seq_len and do not relabel them as prefill/decode without an external serving experiment.","replacement_claim_ids":["kernel-fused-moe-current-c005"]} +{"receipt_id":"rem-kernel-fused-moe-015","path":"wiki/kernels/fused-moe.md","original_body_sha256":"c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d","original_unit_ids":["u022"],"locator":"body:L216","original_claim_id":"kernel-fused-moe-original-c020","original_claim":"Expert load imbalance is the primary practical bottleneck for all workloads.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/fused_moe/core.py","revision":"7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"separate GEMM1/GEMM2 tactic autotuning","observation":"The implementation selects tactics by shape and token bucket for two GEMM stages; it does not encode a universal bottleneck ranking."}],"strongest_alternative":"Load imbalance is frequently important even when not the only factor.","discriminating_result":"That narrower statement is retained; 'primary' requires per-workload profiling and can vary with token count, routing, tactic, and data movement.","action":"correct","action_reason":"Replace a universal ranking with measurable workload-specific risks.","proposed_replacement":"Measure expert token counts, grouped-GEMM tile occupancy, routing/data movement, and both GEMM tactics for each workload.","replacement_claim_ids":["kernel-fused-moe-current-c006","kernel-fused-moe-current-c007"]} +{"receipt_id":"rem-kernel-fused-moe-016","path":"wiki/kernels/fused-moe.md","original_body_sha256":"c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d","original_unit_ids":["u023"],"locator":"body:L217","original_claim_id":"kernel-fused-moe-original-c021","original_claim":"CUDA graph compatibility requires a masked fixed-allocation expert layout.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-deepgemm-pinned/README.md","revision":"891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"Grouped GEMMs (masked layout)","observation":"Masked layout is one decoding case when the CPU does not know per-expert counts, not a universal CUDA-graph rule."},{"path":"artifacts/kernels/fused-moe/full/flashinfer_cutedsl.py","revision":"SGLang c554dc5c64b661f2c53225b03a76359eaddc39e4","locator":"CUDA-graph wrapper allocation and standard dispatch","observation":"The adjacent upstream runner preallocates graph buffers while accepting a standard dispatch representation."}],"strongest_alternative":"The page intended only DeepGEMM decode with CPU-unknown counts.","discriminating_result":"It gives no such scope and presents the claim as a general fused-MoE caveat.","action":"correct","action_reason":"Make graph/layout requirements backend- and capture-path-specific.","proposed_replacement":"Masked layout is one supported DeepGEMM decode strategy; other graph-compatible runners use different fixed buffers and dispatch ABIs.","replacement_claim_ids":["kernel-fused-moe-current-c007"]} +{"receipt_id":"rem-kernel-fused-moe-017","path":"wiki/kernels/fused-moe.md","original_body_sha256":"c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d","original_unit_ids":["u027"],"locator":"body:L224","original_claim_id":"kernel-fused-moe-original-c025","original_claim":"GPU Mode NVFP4 Hackathon Problem 3 is a source for this FP8 routed MoE kernel.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml","revision":"51e22db671d36c1c76091c43c36a44546ba324a1","locator":"task contract","observation":"Problem 3 is an NVFP4 A/B1/B2 dual GEMM without expert routing, local offsets, W2, or weighted combine."}],"strongest_alternative":"The dual-GEMM activation subproblem is conceptually adjacent.","discriminating_result":"Adjacency does not make its FP4 tensor/scale ABI evidence for the FP8 routed operation.","action":"delete","action_reason":"Remove the misleading direct-source link from the page.","proposed_replacement":null,"replacement_claim_ids":["kernel-fused-moe-current-c008"]} +{"receipt_id":"rem-kernel-fused-moe-018","path":"wiki/kernels/fused-moe.md","original_body_sha256":"c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d","original_unit_ids":["u028"],"locator":"body:L225","original_claim_id":"kernel-fused-moe-original-c026","original_claim":"DeepGEMM MoE is direct evidence for the exact Track A operation.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-deepgemm-pinned/README.md","revision":"891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"Grouped GEMMs and Mega MoE","observation":"DeepGEMM exposes generic M-grouped GEMMs and a distributed FP8xFP4 Mega MoE, not the contest's exact routing/logit and FP8 scale ABI."}],"strongest_alternative":"Its grouped-GEMM mechanisms can inform implementations.","discriminating_result":"They may be labeled adjacent, but cannot substantiate the benchmark signature or performance claims.","action":"delete","action_reason":"Keep direct sources claim-specific; discuss adjacency only in artifact provenance.","proposed_replacement":null,"replacement_claim_ids":["kernel-fused-moe-current-c008"]} +{"receipt_id":"rem-kernel-fused-moe-019","path":"wiki/kernels/fused-moe.md","original_body_sha256":"c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d","original_unit_ids":["u029"],"locator":"body:L226","original_claim_id":"kernel-fused-moe-original-c027","original_claim":"The linked SGLang implementation substantiates the exact FP8 benchmark and framework table.","original_verdict":"refuted","evidence":[{"path":"artifacts/kernels/fused-moe/full/flashinfer_cutedsl.py","revision":"SGLang c554dc5c64b661f2c53225b03a76359eaddc39e4; SHA-256 de6bd663382b5bf34dba077d90febc9cc80d9dda908d9947966a3cadbc21bf33","locator":"registered function and fp4_quantize call","observation":"The file is a standard-path FP4 CuteDSL runner and contains no Track A FP8 table or launch count."}],"strongest_alternative":"The file demonstrates related MoE dispatch and graph integration.","discriminating_result":"That makes it an adjacent implementation reference only, with a different quantization contract.","action":"correct","action_reason":"Retain the byte-pinned file but label it adjacent and non-evidentiary for Track A FP8.","proposed_replacement":"List the SGLang file only under adjacent artifacts with exact FP4 scope.","replacement_claim_ids":["kernel-fused-moe-current-c008"]} +{"receipt_id":"rem-kernel-fused-moe-020","path":"wiki/kernels/fused-moe.md","original_body_sha256":"c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d","original_unit_ids":["u030"],"locator":"body:L230","original_claim_id":"kernel-fused-moe-original-c028","original_claim":"The full artifact bundle is the verbatim upstream FP8 implementation and the derived variant faithfully models production fusion.","original_verdict":"refuted","evidence":[{"path":"artifacts/kernels/fused-moe/full/PROVENANCE.yaml","revision":"SHA-256 3832ddeeeafe8055e44c890283926365817763af2c724c54cce0872c53699f17","locator":"file origins","observation":"The bundle mixes an MXFP4 vLLM diff, an FP4 SGLang file, and local pseudocode."},{"path":"artifacts/kernels/fused-moe/variants/01-routing-plus-fusion-skeleton.py","revision":"SHA-256 2cc640b1baa1a6cd42718616ffe2cf78a9b6d4eb3317464f059c6fd442439342","locator":"routing-weight scatter","observation":"The boolean indexing expression is incorrect and the production-fusion assertion is unsupported."}],"strongest_alternative":"'Upstream' can refer to several adjacent projects and the variant is already labeled derived.","discriminating_result":"The page calls the directory a full reference implementation of its subject; derivation labels do not cure wrong code or hidden contract differences.","action":"correct","action_reason":"Preserve the artifacts while disclosing each exact adjacent scope and replacing the variant with a tested semantic reference.","proposed_replacement":"Label full/ as mixed adjacent excerpts and variants/ as non-optimized derived checks, never the contest implementation.","replacement_claim_ids":["kernel-fused-moe-current-c008"]} +{"receipt_id":"rem-kernel-fused-moe-021","path":"wiki/kernels/fused-moe.md","original_body_sha256":"c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d","original_unit_ids":["u033"],"locator":"frontmatter:performance_claims","original_claim_id":"kernel-fused-moe-original-c030","original_claim":"A structured B200 record supports 1262 TFLOPS at batch 4096 and about 56% utilization for the stated geometry.","original_verdict":"refuted","evidence":[{"url":"https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","revision":"definition and traces fetched 2026-08-08","locator":"Axes and traces","observation":"The page distinguishes 256 global/32 local experts, varies seq_len rather than batch, and has no 1262-TFLOPS or 56%-utilization record."}],"strongest_alternative":"TFLOPS and utilization were derived from an older latency entry and B200 peak.","discriminating_result":"No latency cell, clock/SKU peak convention, operation-count formula, repetitions, or variance is attached, so a structured record cannot be reconstructed.","action":"delete","action_reason":"Remove inadmissible structured performance metadata.","proposed_replacement":"Set performance_claims to [] and document only the official evaluation method, not a result.","replacement_claim_ids":["kernel-fused-moe-current-c005"]} +{"receipt_id":"rem-source-fused-moe-contest-001","receipt_scope":"source-trail","path":"sources/contests/flashinfer-mlsys26/track-a-fused-moe.md","original_body_sha256":"f6c27b536895d6f1cf5b89c1181bdecec260beebf567e1bf39b03a5dae793815","original_unit_ids":[],"locator":"frontmatter submissions and body benchmark/performance/challenges","original_claim_id":"source-contest-fused-moe-contract","original_claim":"The local contest summary records stale AI rankings, a wrong benchmark slug, 32 total experts, fabricated framework performance, and universal launch/TMA claims as official facts.","original_verdict":"refuted","evidence":[{"url":"https://mlsys26.flashinfer.ai/","revision":"fetched 2026-08-08","locator":"Track A, Evaluation, Timeline, and Winners","observation":"The current organizer page gives the exact Track A link, B200 target, final schedule, and actual winners, not the local AI ranking block."},{"url":"https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","revision":"fetched 2026-08-08","locator":"Axes and Signature","observation":"The exact slug begins moe_, routing spans 256 total experts with 32 local, and the tensors/scales differ from the local summary."}],"strongest_alternative":"The local file captures a legitimate earlier leaderboard snapshot.","discriminating_result":"It records no snapshot revision or raw cells, and its semantic contract contradicts the immutable definition geometry.","action":"correct","action_reason":"Rewrite the source capture around exact current primary URLs and pinned starter-kit evaluation details.","proposed_replacement":"Record the official track, definition axes/signature/reference semantics, B200 evaluation environment, and absence of a retained performance record.","replacement_claim_ids":["kernel-fused-moe-current-c001","kernel-fused-moe-current-c002","kernel-fused-moe-current-c003","kernel-fused-moe-current-c005"]} +{"receipt_id":"rem-source-fused-moe-vllm-pr-001","receipt_scope":"source-trail","path":"sources/prs/vllm/PR-23696.md","original_body_sha256":"adb7884c1234fdc96c64831feacecc809dbfe8d9680e93ec98c466cb9a7e7f42","original_unit_ids":[],"locator":"title, changed_paths, Summary, Solution, and Performance","original_claim_id":"source-vllm-23696-summary","original_claim":"The local PR summary gives the wrong title, only one changed path, calls NVFP4 MXFP4, and says the kernel fuses routing with FP8 MoE.","original_verdict":"refuted","evidence":[{"url":"https://api.github.com/repos/vllm-project/vllm/pulls/23696","revision":"merge 074854b24f6e0b1e237a004283e1f46d98c0d73c","locator":"title, body, and changed_files","observation":"The merged PR is '[Kernel][B200] mxfp4 fused cutlass moe', changes five files, and adds BF16xMXFP4 Hopper plus MXFP8xMXFP4 Blackwell expert compute; routing IDs/weights are prepared outside the backend call."}],"strongest_alternative":"NVFP4 is a reasonable informal synonym for the 4-bit weight format used by the PR.","discriminating_result":"The PR and APIs consistently name MXFP4/mxfp4; preserving the exact standard and activation format is material for compatibility.","action":"correct","action_reason":"Transcribe the exact merged PR contract, paths, and author-reported performance boundaries.","proposed_replacement":"Document MXFP4 weights, BF16 versus MXFP8 activations, five changed paths, external top-k inputs, and qualified gpt-oss serving results.","replacement_claim_ids":["kernel-fused-moe-current-c008"]} +{"receipt_id":"rem-source-fused-moe-full-artifacts-001","receipt_scope":"source-trail","path":"artifacts/kernels/fused-moe/full","original_body_sha256":"3832ddeeeafe8055e44c890283926365817763af2c724c54cce0872c53699f17","original_unit_ids":[],"locator":"PROVENANCE.yaml and moe-grouped-gemm-launch.cpp","original_claim_id":"source-artifact-fused-moe-full-scope","original_claim":"The full bundle is a verbatim vLLM-source implementation despite mixing a vLLM MXFP4 aggregate diff, SGLang FP4 file, and locally extracted pseudocode.","original_verdict":"refuted","evidence":[{"path":"artifacts/kernels/fused-moe/full/vllm-PR-23696-dual-gemm.patch","revision":"aggregate merge diff SHA-256 033d30b15af4f2050189c2e76ba61959e1e5d4bd049dc901fd56f68890b891a0","locator":"five diff headers","observation":"The diff is MXFP4 vLLM work, not the Track A FP8 implementation."},{"path":"artifacts/kernels/fused-moe/full/flashinfer_cutedsl.py","revision":"SGLang c554dc5c64b661f2c53225b03a76359eaddc39e4","locator":"fp4_quantize and CuteDslMoEWrapper","observation":"This file is byte-identical upstream FP4 code from a different repository."}],"strongest_alternative":"A mixed bundle can still be useful as implementation context.","discriminating_result":"Yes, if its top-level mode, notes, and every file label disclose adjacency and the local pseudocode is not presented as upstream extraction.","action":"correct","action_reason":"Preserve bytes but repair provenance and the illustrative file header.","proposed_replacement":"Mark the bundle extracted/mixed adjacent, give full revisions and exact scopes, and label the grouped-GEMM skeleton as local pseudocode.","replacement_claim_ids":["kernel-fused-moe-current-c008"]} +{"receipt_id":"rem-source-fused-moe-variant-001","receipt_scope":"source-trail","path":"artifacts/kernels/fused-moe/variants/01-routing-plus-fusion-skeleton.py","original_body_sha256":"2cc640b1baa1a6cd42718616ffe2cf78a9b6d4eb3317464f059c6fd442439342","original_unit_ids":[],"locator":"fused_moe_forward","original_claim_id":"source-artifact-fused-moe-derived-reference","original_claim":"The derived variant is a correct reference for routing/dispatch/fusion and says vLLM, SGLang, and DeepGEMM fuse all three stages.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py","revision":"7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"DeepSeek routing reference and expert compute","observation":"The official semantics use sigmoid+bias grouped selection, unbiased normalized weights, global/local expert IDs, and correct per-token accumulation; the variant implements none of the grouped routing and indexes weights incorrectly."}],"strongest_alternative":"The file is only an educational skeleton and need not match the contest exactly.","discriminating_result":"It explicitly claims to be a reference implementation and production fusion summary, so wrong routing/scatter behavior is not harmless.","action":"correct","action_reason":"Replace it with a small parameterized CPU-checkable semantic reference and remove production-performance claims.","proposed_replacement":"Implement grouped DeepSeek top-k and local W13/SwiGLU/W2 accumulation with a deterministic self-test.","replacement_claim_ids":["kernel-fused-moe-current-c004","kernel-fused-moe-current-c008"]} +{"receipt_id":"rem-source-fused-moe-relationship-deepgemm-001","receipt_scope":"source-trail","path":"wiki/kernels/fused-moe.md","original_body_sha256":"c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d","original_unit_ids":[],"locator":"frontmatter:sources blog-deepgemm","original_claim_id":"source-relationship-fused-moe-deepgemm","original_claim":"The DeepGEMM blog summary is a direct source for the FlashInfer Track A contract and performance table.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-deepgemm-pinned/README.md","revision":"891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"Grouped GEMMs and Mega MoE","observation":"DeepGEMM is adjacent generic/distributed MoE work and does not define the contest signature or table."}],"strongest_alternative":"Its grouped-GEMM discussion is relevant background.","discriminating_result":"Background may remain in the adjacent artifact note, but not in the direct source list for exact claims.","action":"delete","action_reason":"Keep source relationships claim-directed.","proposed_replacement":null,"replacement_claim_ids":["kernel-fused-moe-current-c008"]} +{"receipt_id":"rem-source-fused-moe-relationship-vllm-001","receipt_scope":"source-trail","path":"wiki/kernels/fused-moe.md","original_body_sha256":"c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d","original_unit_ids":[],"locator":"frontmatter:sources pr-vllm-23696","original_claim_id":"source-relationship-fused-moe-vllm","original_claim":"vLLM PR 23696 directly supports the page's FP8 Track A kernel and launch/performance claims.","original_verdict":"refuted","evidence":[{"url":"https://api.github.com/repos/vllm-project/vllm/pulls/23696","revision":"merge 074854b24f6e0b1e237a004283e1f46d98c0d73c","locator":"title and body","observation":"The PR concerns MXFP4 weights with BF16/MXFP8 activations for gpt-oss and publishes serving throughput, not the Track A FP8 block-scale definition."}],"strongest_alternative":"It is a useful adjacent fused-MoE implementation.","discriminating_result":"It can be preserved as explicitly adjacent artifact context without treating its formats/results as evidence for Track A.","action":"delete","action_reason":"Remove the direct source relationship and disclose adjacency separately.","proposed_replacement":null,"replacement_claim_ids":["kernel-fused-moe-current-c008"]} +{"receipt_id":"rem-source-fused-moe-frontmatter-001","receipt_scope":"source-trail","path":"wiki/kernels/fused-moe.md","original_body_sha256":"c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d","original_unit_ids":[],"locator":"frontmatter:architectures, languages, hardware tags, and blackwell_relevance","original_claim_id":"source-metadata-fused-moe-compatibility","original_claim":"The Track A page claims SM90 compatibility and concrete tcgen05/TMEM/CLC/warp-specialized implementation details.","original_verdict":"refuted","evidence":[{"url":"https://mlsys26.flashinfer.ai/","revision":"fetched 2026-08-08","locator":"Challenge and Track A","observation":"The organizer targets Blackwell B200."},{"url":"https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","revision":"fetched 2026-08-08","locator":"definition tags and signature","observation":"The benchmark specifies its operation/quantization/API but not a Hopper target or those implementation choices."}],"strongest_alternative":"Related libraries implement similar MoE operations on Hopper and use those mechanisms on Blackwell.","discriminating_result":"Related variants do not establish compatibility or implementation internals for this exact contest definition.","action":"correct","action_reason":"Limit frontmatter to the verified B200/SM100a contract and implementation languages accepted by the contest.","proposed_replacement":"Use sm100/sm100a, remove Hopper and unproved mechanism tags, and describe the page as a logical benchmark contract.","replacement_claim_ids":["kernel-fused-moe-current-c001","kernel-fused-moe-current-c007"]} +{"receipt_id":"rem-source-vllm-23696-provenance-001","receipt_scope":"source-trail","path":"artifacts/prs/vllm/PR-23696/PROVENANCE.yaml","original_body_sha256":"6b6c0e170d3783095f3beb0115fad2ec93ed24d80e957a2e5e003e93b86f6127","original_unit_ids":[],"locator":"upstream_sha and file scope","original_claim_id":"source-artifact-vllm-23696-revision","original_claim":"The artifact pins only abbreviated merge SHA 074854b2 and its source summary implies the single captured key file is the only changed path.","original_verdict":"refuted","evidence":[{"url":"https://api.github.com/repos/vllm-project/vllm/pulls/23696","revision":"fetched 2026-08-08","locator":"merge_commit_sha and changed_files","observation":"The exact merge is 074854b24f6e0b1e237a004283e1f46d98c0d73c and the PR changes five files; the captured warmup file is byte-identical at that revision but only one selected key file."}],"strongest_alternative":"An eight-character SHA is sufficient to identify the commit locally.","discriminating_result":"The verifier contract requires exact revision provenance, and the full immutable SHA is available.","action":"correct","action_reason":"Pin the full merge and clarify aggregate diff versus selected key-file capture.","proposed_replacement":"Use the 40-character merge SHA and note that diff.patch covers all five files while key-files stores one representative file.","replacement_claim_ids":["kernel-fused-moe-current-c008"]} +{"receipt_id":"rem-kernel-gated-delta-net-001","receipt_scope":"wiki","path":"wiki/kernels/gated-delta-net.md","original_body_sha256":"af98b7d75e33581067427cda67ff21fbf228821cc926241a7539b6215cd781ec","original_unit_ids":["u002"],"locator":"body:L7","original_claim_id":"kernel-gated-delta-net-original-c003","original_claim":"The fixed recurrent matrix eliminates KV-cache growth entirely.","original_verdict":"refuted","evidence":[{"url":"https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/config.json","revision":"9c7f2fbe84465e40164a94cc16cd30b6999b0cc7","locator":"full_attention_interval and use_cache","observation":"Qwen3-Next keeps a full-attention layer every fourth layer with caching enabled; only GDN-layer state is fixed with context length."}],"strongest_alternative":"The sentence refers only to a standalone GDN layer, not the Qwen hybrid.","discriminating_result":"Its surrounding deployment context and unqualified 'entirely' transfer the statement to the model; the exact model retains cached full attention.","action":"correct","action_reason":"Preserve the fixed-state advantage while stating the hybrid-model boundary.","proposed_replacement":"GDN-layer state is fixed with context length; hybrid full-attention layers still retain their own cache.","replacement_claim_ids":["kernel-gated-delta-net-current-c005"]} +{"receipt_id":"rem-kernel-gated-delta-net-002","receipt_scope":"wiki","path":"wiki/kernels/gated-delta-net.md","original_body_sha256":"af98b7d75e33581067427cda67ff21fbf228821cc926241a7539b6215cd781ec","original_unit_ids":["u003"],"locator":"body:L11-L21","original_claim_id":"kernel-gated-delta-net-original-c004","original_claim":"Qwen3-Next activates about 19 of 512 experts per token and uses plain full attention for global retrieval.","original_verdict":"refuted","evidence":[{"url":"https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct","revision":"model revision 9c7f2fbe84465e40164a94cc16cd30b6999b0cc7","locator":"Model Overview","observation":"The official card says 10 routed experts plus one shared expert and names the fourth block Gated Attention."}],"strongest_alternative":"The 19 count includes replicated or shared expert work not visible in num_experts_per_tok.","discriminating_result":"The official architecture explicitly enumerates 10 activated routed experts and one shared expert; no 19-expert convention is stated.","action":"correct","action_reason":"Use the immutable model-card architecture fields.","proposed_replacement":"Record 48 layers in a 3-GDN/1-Gated-Attention pattern with 512 experts, 10 routed plus one shared, and 80B/3B parameter counts.","replacement_claim_ids":["kernel-gated-delta-net-current-c002"]} +{"receipt_id":"rem-kernel-gated-delta-net-003","receipt_scope":"wiki","path":"wiki/kernels/gated-delta-net.md","original_body_sha256":"af98b7d75e33581067427cda67ff21fbf228821cc926241a7539b6215cd781ec","original_unit_ids":["u005"],"locator":"body:L27-L58","original_claim_id":"kernel-gated-delta-net-original-c006","original_claim":"The displayed delta_rule_step correctly orients the state and assigns alpha/beta to decay and update.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/gdn.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"_gdn_decode_reference","observation":"The official reference uses state [K,V], decay g before retrieval, and beta times the v-minus-retrieved correction; the display is dimensionally inconsistent and swaps gate roles."}],"strongest_alternative":"Equivalent transposition or renamed gates makes the code correct.","discriminating_result":"Even under transposition, S@k and the constructed outer product have opposite orientations, and the correction retrieves from undecayed state.","action":"correct","action_reason":"Replace with an executable recurrence matching the exact reference.","proposed_replacement":"Use decayed=g*S, retrieved=k@decayed, new_state=decayed+outer(k,beta*(v-retrieved)), output=scale*q@new_state.","replacement_claim_ids":["kernel-gated-delta-net-current-c001","kernel-gated-delta-net-current-c008"]} +{"receipt_id":"rem-kernel-gated-delta-net-004","receipt_scope":"wiki","path":"wiki/kernels/gated-delta-net.md","original_body_sha256":"af98b7d75e33581067427cda67ff21fbf228821cc926241a7539b6215cd781ec","original_unit_ids":["u006"],"locator":"body:L62","original_claim_id":"kernel-gated-delta-net-original-c007","original_claim":"Chunk prefill processes chunks in parallel but resolves each chunk through a causal scalar recurrence and propagates state sequentially across chunks.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-gdn-pinned/lit_gpt/gated_delta_rule_ops/chunk.py","revision":"NVlabs/GatedDeltaNet b53d6d3a161267432a79c1c04af69fa52bddc921","locator":"chunk_gated_delta_rule_fwd_kernel_h and chunk_linear_attn_fwd_kernel_o","observation":"The implementation separates a state kernel that loops over chunks from chunk-indexed matrix-product kernels using the WY representation; it is not the stated scalar-within-chunk decomposition."}],"strongest_alternative":"The prose is only a high-level description and need not mirror kernel decomposition.","discriminating_result":"It makes concrete parallel/sequential scheduling claims that invert or omit the actual grid and loop structure.","action":"correct","action_reason":"Describe only the verified chunkwise boundary and point to the exact implementation.","proposed_replacement":"Prefill uses a chunkwise/WY formulation with matrix products and materialized chunk-boundary states; exact scheduling is implementation-specific.","replacement_claim_ids":["kernel-gated-delta-net-current-c004"]} +{"receipt_id":"rem-kernel-gated-delta-net-005","receipt_scope":"wiki","path":"wiki/kernels/gated-delta-net.md","original_body_sha256":"af98b7d75e33581067427cda67ff21fbf228821cc926241a7539b6215cd781ec","original_unit_ids":["u007"],"locator":"body:L64-L121","original_claim_id":"kernel-gated-delta-net-original-c008","original_claim":"The displayed Triton prefill sketch is a correct GDN implementation for the contest geometry.","original_verdict":"refuted","evidence":[{"url":"https://bench.flashinfer.ai/kernels/gdn_prefill_qk4_v8_d128_k_last","revision":"definition fetched 2026-08-08","locator":"Signature","observation":"The exact contract has per-head 128x128 states, separate g and beta, variable-length cu_seqlens, and 4/4/8 head geometry; the sketch flattens dimensions, omits inputs, and uses undefined symbols."}],"strongest_alternative":"The snippet is illustrative pseudocode rather than runnable Triton.","discriminating_result":"It is fenced and presented as a kernel with concrete pointers, shapes, and state behavior; those observable contracts are wrong.","action":"delete","action_reason":"A short exact recurrence and pinned production sources are more useful than a misleading non-executable kernel.","proposed_replacement":null,"replacement_claim_ids":["kernel-gated-delta-net-current-c003","kernel-gated-delta-net-current-c008"]} +{"receipt_id":"rem-kernel-gated-delta-net-006","receipt_scope":"wiki","path":"wiki/kernels/gated-delta-net.md","original_body_sha256":"af98b7d75e33581067427cda67ff21fbf228821cc926241a7539b6215cd781ec","original_unit_ids":["u009"],"locator":"body:L127-L167","original_claim_id":"kernel-gated-delta-net-original-c010","original_claim":"The displayed Triton decode has the correct 512x1024 state, gates, update ordering, and output.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/gdn.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"gated_delta_rule_decode_trace and _gdn_decode_reference","observation":"The official shape is [B,8,128,128], gates are derived from A_log/a/dt_bias and b, and output queries the updated state. The sketch disagrees on each point."}],"strongest_alternative":"Flattened heads yield an algebraically equivalent larger matrix.","discriminating_result":"GVA repeats q/k heads into value-head groups and maintains independent 128x128 state per value head; flattening introduces cross-head interactions absent from the contract.","action":"delete","action_reason":"Remove code that cannot represent the benchmark semantics.","proposed_replacement":null,"replacement_claim_ids":["kernel-gated-delta-net-current-c003","kernel-gated-delta-net-current-c008"]} +{"receipt_id":"rem-kernel-gated-delta-net-007","receipt_scope":"wiki","path":"wiki/kernels/gated-delta-net.md","original_body_sha256":"af98b7d75e33581067427cda67ff21fbf228821cc926241a7539b6215cd781ec","original_unit_ids":["u010"],"locator":"body:L171","original_claim_id":"kernel-gated-delta-net-original-c011","original_claim":"TFLA supplies GDN kernels and emits inline WGMMA/tcgen05 PTX on Hopper/Blackwell.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-mlstm-kernels/README.md","revision":"NX-AI/mlstm_kernels 5b98ff8e2bec189b3d3c249405bab5149564d6f8","locator":"library overview","observation":"The codebase implements mLSTM through PyTorch, JAX, and Triton and contains no GatedDeltaNet, wgmma, tcgen05, or inline-assembly implementation."},{"url":"https://arxiv.org/abs/2503.14376","revision":"arXiv v3","locator":"abstract","observation":"The paper applies TFLA to mLSTM, not GDN."}],"strongest_alternative":"The algorithm is generic enough to apply to GDN even if the public code demonstrates mLSTM.","discriminating_result":"Potential applicability does not establish an existing GDN implementation or the claimed architecture-specific instructions.","action":"correct","action_reason":"Retain TFLA only as separate mLSTM background and remove implementation transfer.","proposed_replacement":"TFLA is mLSTM work using a second level of sequence parallelization; it is not evidence for this GDN kernel's instruction path.","replacement_claim_ids":["kernel-gated-delta-net-current-c010"]} +{"receipt_id":"rem-kernel-gated-delta-net-008","receipt_scope":"wiki","path":"wiki/kernels/gated-delta-net.md","original_body_sha256":"af98b7d75e33581067427cda67ff21fbf228821cc926241a7539b6215cd781ec","original_unit_ids":["u011"],"locator":"body:L173-L184","original_claim_id":"kernel-gated-delta-net-original-c012","original_claim":"The displayed inline PTX is a valid TFLA tcgen05 matrix multiply.","original_verdict":"refuted","evidence":[{"path":"wiki/hardware/tcgen05-mma.md","revision":"page verified against PTX ISA 9.2","locator":"Instruction Forms and Ownership","observation":"The fragment omits required instruction form, descriptor, predicate, ownership, and synchronization semantics; it is absent from the pinned TFLA source."}],"strongest_alternative":"The operands are placeholders and the compiler fills missing descriptors.","discriminating_result":"Inline PTX strings must themselves match a valid grammar and operand list; placeholders cannot repair the mnemonic or missing operands.","action":"delete","action_reason":"Delete fabricated assembly rather than implying it can compile.","proposed_replacement":null,"replacement_claim_ids":["kernel-gated-delta-net-current-c010"]} +{"receipt_id":"rem-kernel-gated-delta-net-009","receipt_scope":"wiki","path":"wiki/kernels/gated-delta-net.md","original_body_sha256":"af98b7d75e33581067427cda67ff21fbf228821cc926241a7539b6215cd781ec","original_unit_ids":["u018"],"locator":"body:L200 and frontmatter:blackwell_relevance","original_claim_id":"kernel-gated-delta-net-original-c015","original_claim":"Blackwell GDN prefill remains in progress.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/gdn_prefill.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"chunk_gated_delta_rule SM100 dispatch","observation":"The implementation dispatches an SM100/SM103 CuTe DSL prefill path and documents CUDA 13 and head-size-128 requirements."}],"strongest_alternative":"The code is merged but guarded or experimental, so 'in progress' still describes production readiness.","discriminating_result":"The page claimed implementation status, not maturity; callable architecture dispatch refutes absence while exact prerequisites can be stated.","action":"correct","action_reason":"Replace stale issue status with pinned implementation status and prerequisites.","proposed_replacement":"At FlashInfer 7f614b8, SM100 prefill is implemented through CuTe DSL and requires CUDA 13 plus head_size 128.","replacement_claim_ids":["kernel-gated-delta-net-current-c004"]} +{"receipt_id":"rem-kernel-gated-delta-net-010","receipt_scope":"wiki","path":"wiki/kernels/gated-delta-net.md","original_body_sha256":"af98b7d75e33581067427cda67ff21fbf228821cc926241a7539b6215cd781ec","original_unit_ids":["u019"],"locator":"body:L204","original_claim_id":"kernel-gated-delta-net-original-c016","original_claim":"O(n) GDN scaling itself provides major throughput gains at 32K and longer contexts.","original_verdict":"refuted","evidence":[{"url":"https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct","revision":"model card fetched 2026-08-08","locator":"Highlights and implementation caveat","observation":"The reported 10x is a whole-model comparison and the same card says efficiency depends highly on implementation; it does not isolate a GDN kernel or establish a universal 32K threshold."}],"strongest_alternative":"The whole-model result is adequate practical evidence for recommending GDN at long context.","discriminating_result":"The compared models differ in parameter activation, MoE, MTP, and hybrid attention, so the causal performance attribution is not discriminating.","action":"correct","action_reason":"Retain asymptotic and model-report facts without claiming an isolated threshold or universal gain.","proposed_replacement":"Use measured end-to-end results for the exact model/backend; O(n) sequence mixing alone is not a performance result.","replacement_claim_ids":["kernel-gated-delta-net-current-c012"]} +{"receipt_id":"rem-kernel-gated-delta-net-011","receipt_scope":"wiki","path":"wiki/kernels/gated-delta-net.md","original_body_sha256":"af98b7d75e33581067427cda67ff21fbf228821cc926241a7539b6215cd781ec","original_unit_ids":["u021"],"locator":"body:L206","original_claim_id":"kernel-gated-delta-net-original-c018","original_claim":"Streaming decode eliminates KV-cache growth without qualification.","original_verdict":"refuted","evidence":[{"url":"https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/config.json","revision":"9c7f2fbe84465e40164a94cc16cd30b6999b0cc7","locator":"full_attention_interval and use_cache","observation":"The hybrid model caches full attention every fourth layer; only GDN state is constant with prior context."}],"strongest_alternative":"The bullet is scoped to streaming GDN decode only.","discriminating_result":"Its wording says eliminates KV-cache growth, not 'for GDN layers'; the adjacent hybrid recommendation makes the missing scope material.","action":"correct","action_reason":"State the layer-local storage property and hybrid exception.","proposed_replacement":"GDN decode uses fixed recurrent state, while hybrid full-attention layers retain context-growing cache.","replacement_claim_ids":["kernel-gated-delta-net-current-c005"]} +{"receipt_id":"rem-kernel-gated-delta-net-012","receipt_scope":"wiki","path":"wiki/kernels/gated-delta-net.md","original_body_sha256":"af98b7d75e33581067427cda67ff21fbf228821cc926241a7539b6215cd781ec","original_unit_ids":["u023"],"locator":"body:L211","original_claim_id":"kernel-gated-delta-net-original-c020","original_claim":"A typical GDN state is 512K floats per head from a 512x1024 matrix.","original_verdict":"refuted","evidence":[{"url":"https://bench.flashinfer.ai/kernels/gdn_decode_qk4_v8_d128_k_last","revision":"definition fetched 2026-08-08","locator":"state shape","observation":"The Qwen TP=4 definition uses eight independent 128x128 value-head states: 16,384 floats per head and 131,072 total per sequence/layer."}],"strongest_alternative":"Flattening four QK and eight V heads produces 512x1024 and is merely another storage view.","discriminating_result":"That product introduces cross-head cells absent from [B,8,128,128], multiplying storage by four and misdescribing independent head groups.","action":"correct","action_reason":"Use exact benchmark dimensions and distinguish per-head from aggregate storage.","proposed_replacement":"At qk4/v8/d128 the FP32 state is 16,384 floats (64 KiB) per value head and 512 KiB across eight heads per sequence/layer.","replacement_claim_ids":["kernel-gated-delta-net-current-c003"]} +{"receipt_id":"rem-kernel-gated-delta-net-013","receipt_scope":"wiki","path":"wiki/kernels/gated-delta-net.md","original_body_sha256":"af98b7d75e33581067427cda67ff21fbf228821cc926241a7539b6215cd781ec","original_unit_ids":["u025"],"locator":"body:L213","original_claim_id":"kernel-gated-delta-net-original-c022","original_claim":"Qwen3.5 GDN attention output gating is required to eliminate Attention Sink and Massive Activation.","original_verdict":"refuted","evidence":[{"url":"https://qwen.ai/blog?id=e34c4305036ce60d55a0791b170337c2b70ae51d","revision":"Qwen3-Next blog fetched 2026-08-08","locator":"Gated Attention and stability sections","observation":"Qwen says output gating in the Qwen3-Next full-attention path helps eliminate these issues; it does not establish necessity or make it a GDN-kernel requirement."}],"strongest_alternative":"Qwen3.5 inherits the same gate, so the original attribution is harmless.","discriminating_result":"Inheritance does not turn a helpful full-attention architectural feature into a required GDN-kernel mechanism.","action":"correct","action_reason":"Keep the useful architectural caveat with its exact scope and modality.","proposed_replacement":"Qwen reports that its full-attention output gate helps address attention sinks and massive activations; this is separate from the GDN recurrence.","replacement_claim_ids":["kernel-gated-delta-net-current-c007"]} +{"receipt_id":"rem-kernel-gated-delta-net-014","receipt_scope":"wiki","path":"wiki/kernels/gated-delta-net.md","original_body_sha256":"af98b7d75e33581067427cda67ff21fbf228821cc926241a7539b6215cd781ec","original_unit_ids":["u030"],"locator":"body:L224 heading and artifact description","original_claim_id":"kernel-gated-delta-net-original-c024","original_claim":"The full bundle is a full GDN reference and the two derived kernels are semantically faithful with complete labeling.","original_verdict":"refuted","evidence":[{"path":"artifacts/kernels/gated-delta-net/full/gdn_fused_proj.py","revision":"SGLang 5bdc07d974f6cf236fa765a685453ea5e587a838; SHA-256 c4ef85167eb11065721e76dd5d63d6f346f309f84822578dd4a3874249acef89","locator":"module","observation":"The verbatim file only rearranges projected QKVZBA tensors."},{"path":"artifacts/kernels/gated-delta-net/variants/01-chunk-parallel-prefill-reference-pytorch.py","revision":"SHA-256 d7e74b02df4974fb082cacbae8a5182dd2c40cb1c13fd1d74505b51cfb15481b","locator":"function","observation":"The variant repeats cumulative decay and implements additive rather than delta-rule updates."}],"strongest_alternative":"The heading refers to the collection as reference material, not a complete recurrence implementation.","discriminating_result":"The page explicitly says Full Reference Implementation and exposes incorrect variants as code; readers cannot infer the hidden scope limitation.","action":"correct","action_reason":"Relabel the upstream projection helper as adjacent and replace false variants with one tested semantic recurrence.","proposed_replacement":"Disclose the full file as an adjacent fused projection-layout helper and the derived file as a CPU-checkable one-head recurrence, neither a tuned GPU kernel.","replacement_claim_ids":["kernel-gated-delta-net-current-c008","kernel-gated-delta-net-current-c009"]} +{"receipt_id":"rem-kernel-gated-delta-net-015","receipt_scope":"wiki","path":"wiki/kernels/gated-delta-net.md","original_body_sha256":"af98b7d75e33581067427cda67ff21fbf228821cc926241a7539b6215cd781ec","original_unit_ids":["u033"],"locator":"frontmatter:performance_claims","original_claim_id":"kernel-gated-delta-net-original-c026","original_claim":"H100 BF16 seqlen=8192 qk4/v8/d128 measured a 10x speedup over Qwen3-32B at 32K-plus context.","original_verdict":"refuted","evidence":[{"url":"https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct","revision":"model card fetched 2026-08-08","locator":"Highlights","observation":"The source reports a whole-model comparison for contexts over 32K but supplies none of H100, BF16, seqlen 8192, or kernel geometry; the structured record combines incompatible scopes."}],"strongest_alternative":"The missing fields came from an unpublished benchmark underlying the model-card statement.","discriminating_result":"No immutable result, environment, timing method, or raw samples are attached, while the published claim has a different workload boundary.","action":"delete","action_reason":"No evidence-supported numeric kernel record can replace the fabricated structured tuple.","proposed_replacement":null,"replacement_claim_ids":["kernel-gated-delta-net-current-c012"]} +{"receipt_id":"rem-source-gated-delta-net-blog-001","receipt_scope":"source-trail","path":"sources/blogs/gated-delta-net.md","original_body_sha256":"27dbb8980c79ef88d00e33d7fa6632f6efe4af4e5d6c85d76a854ecfa64526d5","original_unit_ids":[],"locator":"Architecture, Adoption, and Key Code","original_claim_id":"source-blog-gated-delta-net-summary","original_claim":"The local NVlabs summary fixes a universal 128x128 state, attributes a 10x Qwen result to GDN, and presents additive/cumulative-decay snippets as GDN reference code.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-gdn-pinned/lit_gpt/gated_delta_net.py","revision":"NVlabs b53d6d3a161267432a79c1c04af69fa52bddc921","locator":"forward and init_state","observation":"The state shape is parameterized per head and the recurrence uses both beta and log decay; the local snippets omit the delta correction."},{"url":"https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct","revision":"fetched 2026-08-08","locator":"Highlights","observation":"The 10x result is whole-model throughput at over 32K, not an H100 GDN kernel measurement."}],"strongest_alternative":"The snippets are deliberately simplified teaching examples.","discriminating_result":"They are labeled reference/kernel code, extracted into provenance-bearing artifacts, and compute a different recurrence, so simplification changes semantics.","action":"correct","action_reason":"Rewrite the source capture to exact pinned repository facts and remove fabricated code/performance.","proposed_replacement":"Record paper identity, exact recurrence roles, parameterized state, official FLA recommendation, and adoption without unsupported performance.","replacement_claim_ids":["kernel-gated-delta-net-current-c001","kernel-gated-delta-net-current-c004"]} +{"receipt_id":"rem-source-gated-delta-net-blog-artifacts-001","receipt_scope":"source-trail","path":"artifacts/blogs/gated-delta-net","original_body_sha256":"f1240da89e411f1c04d12f76fb2c85e953d8b29647e295f667606714d94d040f","original_unit_ids":[],"locator":"MANIFEST.yaml, code/PROVENANCE.yaml, and two extracted files","original_claim_id":"source-artifact-gated-delta-net-blog-code","original_claim":"The extracted bundle presents two additive gated-linear-attention snippets as code derived from NVlabs GatedDeltaNet.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-gdn-pinned/lit_gpt/gated_delta_rule_ops/chunk.py","revision":"NVlabs b53d6d3a161267432a79c1c04af69fa52bddc921","locator":"chunk recurrence kernels","observation":"The source implements beta-weighted delta correction with WY preparation; neither extracted function matches or exists in the upstream tree."}],"strongest_alternative":"Provenance says extracted from the local summary, not verbatim upstream.","discriminating_result":"The bundle inherits the source URL/license and the snippets lack a derived warning, creating a false evidentiary trail.","action":"delete","action_reason":"The corrected source has no code blocks, so mark the extraction manifest code-free and remove the false extracted files.","proposed_replacement":null,"replacement_claim_ids":["kernel-gated-delta-net-current-c010"]} +{"receipt_id":"rem-source-gated-delta-net-tfla-001","receipt_scope":"source-trail","path":"sources/docs/tfla.md","original_body_sha256":"3c62def5f50df9dc5fcb2f10cacd4364200829c74ceda348bc36c97c0a0dfde8","original_unit_ids":[],"locator":"frontmatter architectures/tags and Key Techniques","original_claim_id":"source-doc-tfla-gdn-blackwell","original_claim":"The TFLA paper/source covers GDN and emits WGMMA/tcgen05 inline PTX on Hopper and Blackwell.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/abs/2503.14376","revision":"arXiv v3","locator":"abstract","observation":"TFLA is applied to mLSTM and adds intra-chunk sequence parallelization; the paper does not claim GDN-specific architecture assembly."},{"path":"/tmp/kernelwiki-mlstm-kernels/README.md","revision":"NX-AI/mlstm_kernels 5b98ff8e2bec189b3d3c249405bab5149564d6f8","locator":"library overview","observation":"The repository is PyTorch/JAX/Triton mLSTM code and contains no wgmma, tcgen05, inline PTX, or GDN implementation."}],"strongest_alternative":"TFLA is generic and later compilers may lower its Triton dot operations to those instructions.","discriminating_result":"Possible compiler lowering does not establish explicit inline PTX or direct GDN support in the cited revision.","action":"correct","action_reason":"Limit the source summary to claims the paper and code actually make.","proposed_replacement":"Describe the mLSTM application, arbitrary large chunks, second sequence-parallel level, materialization goal, and Triton codebase without architecture claims.","replacement_claim_ids":["kernel-gated-delta-net-current-c010"]} +{"receipt_id":"rem-source-gated-delta-net-contest-001","receipt_scope":"source-trail","path":"sources/contests/flashinfer-mlsys26/track-c-gated-delta-net.md","original_body_sha256":"5dcce57ca3d69791e345b5f535a87a8c2ee1ec499244c96442765234951cd97b","original_unit_ids":[],"locator":"submissions, recurrence, state geometry, implementation status, and optimization sections","original_claim_id":"source-contest-gated-delta-net-contract","original_claim":"The local Track C summary contains stale AI ranks, wrong flattened 512x1024 state, fabricated resource/optimization claims, and an obsolete Blackwell-prefill status.","original_verdict":"refuted","evidence":[{"url":"https://mlsys26.flashinfer.ai/","revision":"fetched 2026-08-08","locator":"Track C winners","observation":"The organizer lists actual agent-assisted and full-agent winners, not the three AI baselines in the local frontmatter."},{"url":"https://bench.flashinfer.ai/kernels/gdn_decode_qk4_v8_d128_k_last","revision":"fetched 2026-08-08","locator":"Signature","observation":"The state is [B,8,128,128] with the full two-gate input set."},{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/gdn_prefill.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"SM100 dispatch","observation":"Blackwell prefill is implemented with explicit prerequisites."}],"strongest_alternative":"The file intentionally preserves an early contest snapshot and speculative optimization ideas.","discriminating_result":"It records no snapshot revision and states speculation/status as current fact after final winners and implementation landed.","action":"correct","action_reason":"Rewrite around current organizer, exact benchmark signatures/reference, pinned implementation prerequisites, and no unsupported result table.","proposed_replacement":"Record final winners by approach, qk4/v8/d128 signatures, exact gate/state semantics, and pinned SM90/SM100 implementation status.","replacement_claim_ids":["kernel-gated-delta-net-current-c003","kernel-gated-delta-net-current-c004"]} +{"receipt_id":"rem-source-gated-delta-net-kernel-variants-001","receipt_scope":"source-trail","path":"artifacts/kernels/gated-delta-net/variants","original_body_sha256":"13143d40b499b83ffd537b9c9b2a4985ccdafea7070eb103475b06359ffb24c0","original_unit_ids":[],"locator":"two derived Python variants and PROVENANCE.yaml","original_claim_id":"source-artifact-gated-delta-net-derived","original_claim":"The derived prefill and decode files faithfully implement GDN semantics.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/gdn.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"_gdn_decode_reference","observation":"The correct recurrence decays before retrieval and adds beta*k outer (v-retrieved); both variants instead perform a simple additive k-v update and omit one gate."}],"strongest_alternative":"They are structural skeletons whose omissions are obvious to expert readers.","discriminating_result":"They are named reference/streaming implementations and contain false statements about paying a state update once per chunk, making semantic ambiguity unsafe.","action":"correct","action_reason":"Replace both with one small standard-library recurrence plus independent expanded-form and additive-only negative controls.","proposed_replacement":"A one-head CPU-checkable GDN step matching FlashInfer 7f614b8, explicitly not a tuned GPU kernel.","replacement_claim_ids":["kernel-gated-delta-net-current-c008"]} +{"receipt_id":"rem-source-gated-delta-net-full-artifacts-001","receipt_scope":"source-trail","path":"artifacts/kernels/gated-delta-net/full/PROVENANCE.yaml","original_body_sha256":"2a722330524123b40558edbded32c0153234a51171363047e64aa67593caf69a0","original_unit_ids":[],"locator":"bundle-level origin and scope","original_claim_id":"source-artifact-gated-delta-net-full-scope","original_claim":"The full bundle is a verbatim GDN kernel implementation pinned only to short SHA 5bdc07d9.","original_verdict":"refuted","evidence":[{"path":"artifacts/kernels/gated-delta-net/full/gdn_fused_proj.py","revision":"SGLang 5bdc07d974f6cf236fa765a685453ea5e587a838; SHA-256 c4ef85167eb11065721e76dd5d63d6f346f309f84822578dd4a3874249acef89","locator":"module","observation":"The file is byte-pinned upstream but only splits/reshapes projection outputs; it does not implement GDN recurrence, prefill, or decode."}],"strongest_alternative":"Its containing kernel directory is sufficient context and the short SHA uniquely resolves.","discriminating_result":"The verifier requires exact revision and claim-scoped provenance; directory naming cannot turn an adjacent projection helper into a full recurrence kernel.","action":"correct","action_reason":"Preserve the verbatim file while pinning the full SHA and disclosing its adjacent scope.","proposed_replacement":"Label the bundle as a verbatim SGLang projection-layout helper adjacent to GDN and not a recurrence/prefill/decode implementation.","replacement_claim_ids":["kernel-gated-delta-net-current-c009"]} +{"receipt_id":"rem-kernel-gated-dual-gemm-001","receipt_scope":"wiki","path":"wiki/kernels/gated-dual-gemm.md","original_body_sha256":"4aa8460e6ef7388eb59cdd6643cc8a05fecb149f9148104b8eec852b369e7373","original_unit_ids":["u001"],"locator":"body:L5","original_claim_id":"kernel-gated-dual-gemm-original-c001","original_claim":"The contest necessarily uses one fused launch, eliminates exactly two global-memory roundtrips, and is the canonical gate-up pattern for most modern LLMs.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml","revision":"gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c","locator":"description and submission interface","observation":"The task fixes output semantics, tensors, and scoring but neither launch count, memory traffic, nor an exhaustive model scope."},{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/reference.py","revision":"same commit","locator":"ref_kernel","observation":"The correctness reference materializes both intermediate GEMM tensors, proving that the contract alone does not establish fusion internals."}],"strongest_alternative":"The word fused describes an optimization target, so a single launch and eliminated traffic may be assumed.","discriminating_result":"The observable submission interface permits any conforming launch decomposition and the reference itself uses intermediates; an optimization goal is not implementation evidence.","action":"correct","action_reason":"Retain exact dual-GEMM/SwiGLU semantics while separating potential fusion benefits from contract guarantees.","proposed_replacement":"The logical target is SiLU(A@B1)*(A@B2); a truly fused implementation may reuse A and avoid intermediate global tensors, but the task does not mandate launch topology.","replacement_claim_ids":["kernel-gated-dual-gemm-current-c001","kernel-gated-dual-gemm-current-c005"]} +{"receipt_id":"rem-kernel-gated-dual-gemm-002","receipt_scope":"wiki","path":"wiki/kernels/gated-dual-gemm.md","original_body_sha256":"4aa8460e6ef7388eb59cdd6643cc8a05fecb149f9148104b8eec852b369e7373","original_unit_ids":["u003"],"locator":"body:L11-L21","original_claim_id":"kernel-gated-dual-gemm-original-c004","original_claim":"The displayed fused pseudocode proves all conforming submissions use one GPU kernel and no intermediate GMEM.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml","revision":"gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c","locator":"files and config","observation":"The harness calls a Python custom_kernel entry point and does not constrain how many device launches it performs."}],"strongest_alternative":"Kernel in the challenge wording is a normative one-launch requirement.","discriminating_result":"No launch-count rule exists in the immutable task, while correctness is defined only by returned output.","action":"correct","action_reason":"Keep the exact formula without converting a possible optimization into an ABI requirement.","proposed_replacement":"Describe the fused formula and state that physical materialization and launch count are implementation-specific.","replacement_claim_ids":["kernel-gated-dual-gemm-current-c003","kernel-gated-dual-gemm-current-c005"]} +{"receipt_id":"rem-kernel-gated-dual-gemm-003","receipt_scope":"wiki","path":"wiki/kernels/gated-dual-gemm.md","original_body_sha256":"4aa8460e6ef7388eb59cdd6643cc8a05fecb149f9148104b8eec852b369e7373","original_unit_ids":["u004"],"locator":"body:L25-L58","original_claim_id":"kernel-gated-dual-gemm-original-c005","original_claim":"The displayed gated_dual_gemm_nvfp4 CUDA template is a valid Blackwell NVFP4 implementation.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"PTX ISA 9.0 / CUDA 13.0.2","locator":"block-scaled tcgen05.mma syntax","observation":"The instruction needs exact qualifiers, descriptors, instruction descriptor, SFA/SFB TMEM addresses, and an accumulator predicate; the sketch supplies none."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"PTX ISA 9.0","locator":"allocation syntax","observation":"Allocation/deallocation are warp-collective instructions using shared-memory result storage and exact qualifiers, not the undefined scalar helpers in the sketch."}],"strongest_alternative":"The code is illustrative C++ whose helper wrappers hide all low-level operands.","discriminating_result":"No helpers, descriptors, scale placement, synchronization objects, indices, or launches are defined; it cannot compile or demonstrate the claimed contract.","action":"delete","action_reason":"Replace non-executable hardware pseudocode with an exact host-checkable semantic reference and pinned upstream sources.","proposed_replacement":null,"replacement_claim_ids":["kernel-gated-dual-gemm-current-c003","kernel-gated-dual-gemm-current-c005"]} +{"receipt_id":"rem-kernel-gated-dual-gemm-004","receipt_scope":"wiki","path":"wiki/kernels/gated-dual-gemm.md","original_body_sha256":"4aa8460e6ef7388eb59cdd6643cc8a05fecb149f9148104b8eec852b369e7373","original_unit_ids":["u005","u006","u007","u008"],"locator":"body:L62-L65","original_claim_id":"kernel-gated-dual-gemm-original-c006","original_claim":"The contest establishes one-load X reuse, two adjacent 256-column TMEM accumulators, no intermediate SMEM, and shared SFA as the winning structure.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml","revision":"gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c","locator":"input contract","observation":"A and SFA are logically shared while B1/B2 and their scales are separate, but the task records no winning implementation, physical load count, TMEM map, or SMEM policy."}],"strongest_alternative":"Those are inevitable implementation consequences of shared operands.","discriminating_result":"Logical sharing does not determine cache/TMA traffic or accumulator placement, and conforming implementations can use different decompositions.","action":"correct","action_reason":"Separate exact shared-input semantics from unsupported schedule claims.","proposed_replacement":"Record one logical A/SFA input and separate B1/B2 scales; treat reuse, TMEM layout, and epilogue storage as implementation choices.","replacement_claim_ids":["kernel-gated-dual-gemm-current-c002","kernel-gated-dual-gemm-current-c005"]} +{"receipt_id":"rem-kernel-gated-dual-gemm-005","receipt_scope":"wiki","path":"wiki/kernels/gated-dual-gemm.md","original_body_sha256":"4aa8460e6ef7388eb59cdd6643cc8a05fecb149f9148104b8eec852b369e7373","original_unit_ids":["u009","u010","u011"],"locator":"body:L69-L71","original_claim_id":"kernel-gated-dual-gemm-original-c007","original_claim":"The exact contest kernel applies to most gated MLPs, any dual-output operation, and per-expert MoE expansion.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml","revision":"gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c","locator":"tensor shapes/dtypes, operation, and constraints","observation":"Compatibility requires two same-shaped NVFP4 block-scaled products, specific scale layouts, first-branch SiLU, FP16 output, and divisibility constraints; arbitrary dual outputs or grouped execution do not automatically match."}],"strongest_alternative":"Every gate/up MLP can be quantized and reshaped into this contract.","discriminating_result":"Weight format, scale format/layout, batch axis, output type, and grouping are externally visible constraints that cannot be assumed.","action":"correct","action_reason":"Replace universal use advice with an explicit compatibility checklist.","proposed_replacement":"Use only when the backend/model supplies the exact NVFP4 dual-projection contract or a verified conversion to it.","replacement_claim_ids":["kernel-gated-dual-gemm-current-c002","kernel-gated-dual-gemm-current-c005"]} +{"receipt_id":"rem-kernel-gated-dual-gemm-006","receipt_scope":"wiki","path":"wiki/kernels/gated-dual-gemm.md","original_body_sha256":"4aa8460e6ef7388eb59cdd6643cc8a05fecb149f9148104b8eec852b369e7373","original_unit_ids":["u012"],"locator":"body:L75","original_claim_id":"kernel-gated-dual-gemm-original-c008","original_claim":"The local full bundle is a contest reference implementation and the derived CUDA file is a faithful executable NVFP4 variant.","original_verdict":"refuted","evidence":[{"path":"artifacts/kernels/gated-dual-gemm/full/vllm-PR-23696-gated-dual-gemm.patch","revision":"SHA-256 033d30b15af4f2050189c2e76ba61959e1e5d4bd049dc901fd56f68890b891a0","locator":"patch","observation":"This is an MXFP4 fused-MoE backend patch, not the NVFP4 contest task."},{"path":"artifacts/kernels/gated-dual-gemm/full/blackwell-cutlass-schedules-and-tma.cu","revision":"SHA-256 e720037bbdb5ee5054259a331cdaf489e7135a44038d66d1fd9d9e5998b75255","locator":"entire file","observation":"This is a third-party blog extract with one assignment, not a kernel."},{"path":"artifacts/kernels/gated-dual-gemm/variants/01-fused-epilogue-swiglu-skeleton.cu","revision":"SHA-256 85e158c184546912579ac5270451471a96f135efa3584bafbf45064e5fb7e223","locator":"function","observation":"It only transforms existing float accumulators and lacks NVFP4 GEMMs, scales, launch, bounds, and a test."}],"strongest_alternative":"Adjacent backend code plus a semantic epilogue are useful reference pieces even if incomplete.","discriminating_result":"They may be retained only with explicit adjacent/partial labels; calling the collection a full contest implementation is false.","action":"correct","action_reason":"Preserve useful evidence with corrected scope and replace the untested variant with a discriminating semantic test.","proposed_replacement":"Label the vLLM file adjacent, remove the unsupported blog fragment, and add a tested dual-GEMM/SwiGLU reference.","replacement_claim_ids":["kernel-gated-dual-gemm-current-c006"]} +{"receipt_id":"rem-kernel-gated-dual-gemm-007","receipt_scope":"wiki","path":"wiki/kernels/gated-dual-gemm.md","original_body_sha256":"4aa8460e6ef7388eb59cdd6643cc8a05fecb149f9148104b8eec852b369e7373","original_unit_ids":["u015"],"locator":"frontmatter:performance_claims","original_claim_id":"kernel-gated-dual-gemm-original-c010","original_claim":"B200 measured 18.5 us at M=1024,N=4096,K=7168 and the workload was compute-bound.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml","revision":"gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c","locator":"benchmarks and speed-of-light analysis","observation":"Published benchmark M is 256 or 512, never 1024; the task gives theoretical 1.5-GHz speed-of-light values, not a measured 18.5-us record or bottleneck classification."}],"strongest_alternative":"18.5 us came from an unpublished leaderboard shape.","discriminating_result":"No immutable result, raw samples, timer, software stack, or matching task shape is attached, so the tuple is not auditable.","action":"delete","action_reason":"Remove the fabricated measurement and retain only the clearly labeled theoretical task table.","proposed_replacement":null,"replacement_claim_ids":["kernel-gated-dual-gemm-current-c004"]} +{"receipt_id":"rem-kernel-gated-dual-gemm-008","receipt_scope":"wiki","path":"wiki/kernels/gated-dual-gemm.md","original_body_sha256":"4aa8460e6ef7388eb59cdd6643cc8a05fecb149f9148104b8eec852b369e7373","original_unit_ids":["u004","u006"],"locator":"frontmatter:architectures/blackwell_relevance and body:L25-L65","original_claim_id":"kernel-gated-dual-gemm-original-c011","original_claim":"The page establishes SM90 compatibility and mandatory two-accumulator TMEM fusion that Hopper registers could not handle efficiently.","original_verdict":"refuted","evidence":[{"url":"https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf","revision":"official rules fetched 2026-08-08","locator":"Kernel #3","observation":"The challenge is Blackwell NVFP4 and the immutable task targets B200, not Hopper."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory","revision":"PTX ISA 9.0","locator":"Tensor Memory organization/allocation","observation":"PTX defines TMEM capacity and allocation but neither prescribes this partition nor provides the Hopper efficiency comparison."}],"strongest_alternative":"Generic MLP algebra also runs on Hopper, and TMEM naturally enables the claimed design.","discriminating_result":"Generic algebra is not compatibility with the NVFP4 B200 tensor contract, and architectural capability does not prove one implementation or comparison.","action":"correct","action_reason":"Scope hardware metadata to B200/SM100a and avoid unmeasured Hopper claims.","proposed_replacement":"Target B200/SM100a; describe TMEM/tcgen05 only as possible mechanisms, not requirements or measured advantages.","replacement_claim_ids":["kernel-gated-dual-gemm-current-c001","kernel-gated-dual-gemm-current-c005"]} +{"receipt_id":"rem-source-gated-dual-gemm-contest-001","receipt_scope":"source-trail","path":"sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md","original_body_sha256":"a3c9d8602d062dcf51857d0bf893ec87eaccb188d2031c10d456a39ad403746c","original_unit_ids":[],"locator":"submissions, optimization, relevance, and CUTLASS sections","original_claim_id":"source-contest-gated-dual-gemm-ranks-and-implementation","original_claim":"The source publishes top-three ranks/scores and specific winning dual-TMEM, TMA, CUTLASS, single-launch, memory-traffic, and compute-bound facts.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml","revision":"gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c","locator":"complete task specification","observation":"The task publishes inputs, tests, benchmark shapes, theoretical speed-of-light values, and geometric-mean ranking, but no winners, submission implementations, launch count, or bottleneck verdict."},{"url":"https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf","revision":"official rules fetched 2026-08-08","locator":"Kernel #3","observation":"The rules verify problem number/dates but not local rankings or techniques."}],"strongest_alternative":"The results came from a private GPU Mode Discord leaderboard.","discriminating_result":"Unavailable messages are not immutable auditable evidence and no captures are recorded, so those details cannot remain facts.","action":"correct","action_reason":"Rewrite around exact public rules and pinned task/reference only.","proposed_replacement":"Record dates, tensors, constraints, reference semantics, tolerance, benchmarks, theoretical bounds, and scoring; omit unpublished submissions and implementation claims.","replacement_claim_ids":["kernel-gated-dual-gemm-current-c001","kernel-gated-dual-gemm-current-c002","kernel-gated-dual-gemm-current-c004"]} +{"receipt_id":"rem-source-gated-dual-gemm-relationships-001","receipt_scope":"source-trail","path":"wiki/kernels/gated-dual-gemm.md","original_body_sha256":"4aa8460e6ef7388eb59cdd6643cc8a05fecb149f9148104b8eec852b369e7373","original_unit_ids":[],"locator":"frontmatter:sources blog-deepgemm, blog-tflops-gap-fp4-moe, pr-vllm-23696","original_claim_id":"source-relationship-gated-dual-gemm-adjacent","original_claim":"DeepGEMM, a third-party FP4 MoE blog, and vLLM PR 23696 directly evidence the GPU Mode NVFP4 dual-GEMM contract.","original_verdict":"refuted","evidence":[{"path":"artifacts/kernels/gated-dual-gemm/full/vllm-PR-23696-gated-dual-gemm.patch","revision":"vLLM merge 074854b24f6e0b1e237a004283e1f46d98c0d73c","locator":"patch","observation":"The PR is MXFP4 fused-MoE backend work, not this standalone NVFP4 task."},{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml","revision":"gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c","locator":"task identity","observation":"The exact contract is self-contained in the official problem directory and does not cite those adjacent projects."}],"strongest_alternative":"Adjacent MoE and GEMM sources provide useful optimization background.","discriminating_result":"Background utility does not make them primary evidence and their formats/scopes differ materially.","action":"delete","action_reason":"Keep the direct source graph claim-scoped and disclose any preserved adjacent artifact separately.","proposed_replacement":null,"replacement_claim_ids":["kernel-gated-dual-gemm-current-c006"]} +{"receipt_id":"rem-source-gated-dual-gemm-full-artifacts-001","receipt_scope":"source-trail","path":"artifacts/kernels/gated-dual-gemm/full/PROVENANCE.yaml","original_body_sha256":"17e70273d64483b26fcafc8ad9e83d7bbf80eee69b6c79829d336ea09176f0bf","original_unit_ids":[],"locator":"bundle origin, revision, and file roles","original_claim_id":"source-artifact-gated-dual-gemm-full-scope","original_claim":"One extracted bundle from short vLLM SHA 074854b2 contains the contest reference plus a valid CUTLASS/TMA extract.","original_verdict":"refuted","evidence":[{"path":"artifacts/kernels/gated-dual-gemm/full/vllm-PR-23696-gated-dual-gemm.patch","revision":"SHA-256 033d30b15af4f2050189c2e76ba61959e1e5d4bd049dc901fd56f68890b891a0","locator":"diff","observation":"The patch is adjacent MXFP4 MoE work and should be pinned to a full merge SHA, not represented as the contest kernel."},{"path":"artifacts/kernels/gated-dual-gemm/full/blackwell-cutlass-schedules-and-tma.cu","revision":"SHA-256 e720037bbdb5ee5054259a331cdaf489e7135a44038d66d1fd9d9e5998b75255","locator":"entire file","observation":"The file has editorial claims and one assignment copied from a third-party blog, with no dual GEMM implementation."}],"strongest_alternative":"Directory/source/mode labels are enough to disclose mixed reference material.","discriminating_result":"The page calls it a full reference and provenance assigns one origin to unrelated sources, so scope is misleading.","action":"correct","action_reason":"Preserve the vLLM patch as adjacent, remove the false fragment, add exact official task.yml, and pin revisions/hashes.","proposed_replacement":"A mixed-scope bundle with official GPU Mode task.yml and separately labeled adjacent vLLM MXFP4 MoE patch, not an optimized submission.","replacement_claim_ids":["kernel-gated-dual-gemm-current-c006"]} +{"receipt_id":"rem-source-gated-dual-gemm-variant-001","receipt_scope":"source-trail","path":"artifacts/kernels/gated-dual-gemm/variants/PROVENANCE.yaml","original_body_sha256":"dfb898edf4ad674cf76bf6d905274f555dcde3f361b84b744faab85e6b9c9f58","original_unit_ids":[],"locator":"01-fused-epilogue-swiglu-skeleton.cu and provenance","original_claim_id":"source-artifact-gated-dual-gemm-derived-scope","original_claim":"The derived CUDA epilogue skeleton is a faithful gated dual GEMM variant.","original_verdict":"refuted","evidence":[{"path":"artifacts/kernels/gated-dual-gemm/variants/01-fused-epilogue-swiglu-skeleton.cu","revision":"SHA-256 85e158c184546912579ac5270451471a96f135efa3584bafbf45064e5fb7e223","locator":"gated_dual_gemm_epilogue","observation":"It assumes two float arrays and only applies SiLU/multiply; it has neither GEMM, NVFP4/scales, launch, bounds, nor a self-test."},{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/reference.py","revision":"gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c","locator":"ref_kernel","observation":"Exact semantics include both scaled matrix multiplications and branch order."}],"strongest_alternative":"A variant may intentionally isolate only the epilogue.","discriminating_result":"That is acceptable only if labeled partial; current page calls it a dual GEMM variant and supplies no discriminating check.","action":"correct","action_reason":"Replace it with a complete semantic dual-matmul/SwiGLU reference and wrong-branch negative control.","proposed_replacement":"A standard-library reference performing both products, SiLU on the first, and rejecting gating the second.","replacement_claim_ids":["kernel-gated-dual-gemm-current-c003","kernel-gated-dual-gemm-current-c006"]} +{"receipt_id":"rem-kernel-grouped-gemm-001","receipt_scope":"wiki","path":"wiki/kernels/grouped-gemm.md","original_body_sha256":"216f09e31b3f73503470760148850014df1540ad380997383317930a7b459e9e","original_unit_ids":["u001"],"locator":"body:L5","original_claim_id":"kernel-grouped-gemm-original-c001","original_claim":"Grouped GEMM necessarily means variable M with shared N/K, one MoE kernel launch, and the most important serving pattern.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_group_gemm/task.yml","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"description, tests, and benchmarks","observation":"The official task supplies independent M/N/K per group and imposes no launch-count or importance claim."}],"strongest_alternative":"The contest may be an M-grouped MoE specialization despite its generic tuple wording.","discriminating_result":"Multiple official correctness cases vary N and K within one group list, so only-M is observably false for that contract.","action":"correct","action_reason":"Distinguish general independent-shape grouping from DeepGEMM's narrower M-grouped MoE API and avoid launch/perceived-importance claims.","proposed_replacement":"State two separate contracts: contest M/N/K-varying grouping and DeepGEMM M-grouping with fixed N/K.","replacement_claim_ids":["kernel-grouped-gemm-current-c001","kernel-grouped-gemm-current-c003"]} +{"receipt_id":"rem-kernel-grouped-gemm-002","receipt_scope":"wiki","path":"wiki/kernels/grouped-gemm.md","original_body_sha256":"216f09e31b3f73503470760148850014df1540ad380997383317930a7b459e9e","original_unit_ids":["u003"],"locator":"body:L11-L25","original_claim_id":"kernel-grouped-gemm-original-c003","original_claim":"The contest fixes N/K, varies only M, and uses the displayed 256-expert DeepSeek token example.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_group_gemm/task.yml","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"tests and benchmarks","observation":"Tests vary M, N, and K; benchmark group counts are 8/8/2/2 and no 256-expert example is specified."}],"strongest_alternative":"Only benchmark rows matter, and their N/K happen to be shared within each row.","discriminating_result":"Correctness is part of the public contract, and those cases require per-group N/K variation.","action":"correct","action_reason":"Replace the fabricated example with the exact tensor/list contract and separately describe DeepGEMM M-grouping.","proposed_replacement":"Document independent (M_i,N_i,K_i) contest groups and fixed-N/K DeepGEMM M groups as different interfaces.","replacement_claim_ids":["kernel-grouped-gemm-current-c002","kernel-grouped-gemm-current-c003"]} +{"receipt_id":"rem-kernel-grouped-gemm-003","receipt_scope":"wiki","path":"wiki/kernels/grouped-gemm.md","original_body_sha256":"216f09e31b3f73503470760148850014df1540ad380997383317930a7b459e9e","original_unit_ids":["u005"],"locator":"body:L31-L61","original_claim_id":"kernel-grouped-gemm-original-c005","original_claim":"The displayed offsets, binary mask, and single-output K-grouped structs accurately model DeepGEMM layouts.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-deepgemm-pinned/csrc/apis/gemm.hpp","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"three grouped API functions","observation":"The interfaces consume a tensor group map/prefix sums, one integer valid-M count per group, and grouped K results, contradicting the structs."},{"path":"/tmp/kernelwiki-deepgemm-pinned/tests/generators.py","revision":"same commit","locator":"three grouped generators","observation":"Generated tensors confirm the exact map/count/packed-K representations."}],"strongest_alternative":"The structs are harmless conceptual wrappers around those tensors.","discriminating_result":"They change observable ranks and meanings: bool-per-row versus int-per-group, generic offsets versus map/prefix-sum modes, and one C versus one output per K group.","action":"correct","action_reason":"Use a table matching the actual pinned APIs instead of invented structs.","proposed_replacement":"List exact A/B/D shapes and grouped_layout, masked_m, or K-size inputs for each mode.","replacement_claim_ids":["kernel-grouped-gemm-current-c003"]} +{"receipt_id":"rem-kernel-grouped-gemm-004","receipt_scope":"wiki","path":"wiki/kernels/grouped-gemm.md","original_body_sha256":"216f09e31b3f73503470760148850014df1540ad380997383317930a7b459e9e","original_unit_ids":["u006"],"locator":"body:L65-L92","original_claim_id":"kernel-grouped-gemm-original-c006","original_claim":"The displayed seven-argument CUTLASS kernel::GemmGrouped and runtime initializer are a valid SM100 NVFP4/CLC API.","original_verdict":"refuted","evidence":[{"path":"/tmp/cutlass-audit.8aj07j/repo/include/cutlass/gemm/kernel/gemm_grouped.h","revision":"NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3","locator":"GemmGrouped template declaration","observation":"The real type has Mma/Epilogue/Swizzle/ScheduleMode/Transposed parameters, not the displayed signature."},{"path":"/tmp/cutlass-audit.8aj07j/repo/include/cutlass/gemm/dispatch_policy.hpp","revision":"same commit","locator":"SM100 block-scaled pointer-array policies","observation":"The schedule tag exists, but does not validate the invented kernel or Arguments API or prove CLC use."}],"strongest_alternative":"A newer CUTLASS convenience overload may accept the shown signature.","discriminating_result":"At the inspected official revision, the exact namespace/type declaration contradicts it; no matching symbol or initializer exists.","action":"delete","action_reason":"Remove noncompiling pseudocode rather than substitute another untested optimized kernel.","proposed_replacement":null,"replacement_claim_ids":["kernel-grouped-gemm-current-c006"]} +{"receipt_id":"rem-kernel-grouped-gemm-005","receipt_scope":"wiki","path":"wiki/kernels/grouped-gemm.md","original_body_sha256":"216f09e31b3f73503470760148850014df1540ad380997383317930a7b459e9e","original_unit_ids":["u007"],"locator":"body:L98-L117","original_claim_id":"kernel-grouped-gemm-original-c007","original_claim":"The static host snippet is a complete schedule for the official grouped-GEMM contract.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_group_gemm/task.yml","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"problem_sizes and correctness tests","observation":"The task permits per-group N/K, while the snippet accepts one N and no K; its block constants and execution are also undefined."}],"strongest_alternative":"The code intentionally covers only the M-grouped DeepGEMM subset.","discriminating_result":"The section presents it as the page's variable-M schedule without declaring that narrower ABI or supplying executable semantics.","action":"delete","action_reason":"Remove the incomplete derived implementation and retain only source-backed contract boundaries.","proposed_replacement":null,"replacement_claim_ids":["kernel-grouped-gemm-current-c001","kernel-grouped-gemm-current-c006"]} +{"receipt_id":"rem-kernel-grouped-gemm-006","receipt_scope":"wiki","path":"wiki/kernels/grouped-gemm.md","original_body_sha256":"216f09e31b3f73503470760148850014df1540ad380997383317930a7b459e9e","original_unit_ids":["u008"],"locator":"body:L121-L151","original_claim_id":"kernel-grouped-gemm-original-c008","original_claim":"A global atomic tile-counter loop is a valid CLC/TMA/tcgen05 grouped-GEMM kernel.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel","revision":"CUDA 13.0.2 / PTX ISA 9.0","locator":"try_cancel and query_cancel syntax","observation":"CLC requires an async shared response, mbarrier completion, and query decoding; the sketch uses only atomicAdd and undefined TMA/MMA helpers."}],"strongest_alternative":"The atomic scheduler may coexist with hidden CLC helper implementations.","discriminating_result":"No CLC instruction, response, barrier, or decoding appears, and undefined ellipses provide no executable implementation.","action":"delete","action_reason":"Remove the mislabeled non-kernel rather than imply an untested launch is authoritative.","proposed_replacement":null,"replacement_claim_ids":["kernel-grouped-gemm-current-c006"]} +{"receipt_id":"rem-kernel-grouped-gemm-007","receipt_scope":"wiki","path":"wiki/kernels/grouped-gemm.md","original_body_sha256":"216f09e31b3f73503470760148850014df1540ad380997383317930a7b459e9e","original_unit_ids":["u011"],"locator":"body:L164","original_claim_id":"kernel-grouped-gemm-original-c012","original_claim":"The incident caused FlashInfer-Bench methodology improvements for MLSys 2026.","original_verdict":"refuted","evidence":[{"url":"https://www.gpumode.com/news/reward-hacking-nvfp4","revision":"GPU Mode post dated 2026-03-09, fetched 2026-08-08","locator":"lessons and closing references","observation":"The official post identifies gpu-mode/reference-kernels PR #104 as the harness response and makes no FlashInfer-Bench/MLSys causation claim."}],"strongest_alternative":"The same authors may have privately applied the lesson to FlashInfer-Bench.","discriminating_result":"No primary record is attached for that causal link; the official incident record names a different concrete remediation.","action":"correct","action_reason":"Preserve the verified reported number while replacing the unsupported consequence with the documented PR #104 response.","proposed_replacement":"State that the exploit was scrubbed and the official post points to reference-kernels PR #104 as the harness fix.","replacement_claim_ids":["kernel-grouped-gemm-current-c005"]} +{"receipt_id":"rem-kernel-grouped-gemm-008","receipt_scope":"wiki","path":"wiki/kernels/grouped-gemm.md","original_body_sha256":"216f09e31b3f73503470760148850014df1540ad380997383317930a7b459e9e","original_unit_ids":["u013"],"locator":"body:L169","original_claim_id":"kernel-grouped-gemm-original-c014","original_claim":"Any workload with shared N/K and variable M can directly use the grouped interface.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-deepgemm-pinned/csrc/apis/gemm.hpp","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"m_grouped type/shape/layout checks","observation":"Actual use additionally depends on supported architecture, dtypes/scales, major modes, aligned group layout, output type, and positive dimensions."}],"strongest_alternative":"Those are implementation details and another generic grouped implementation can accept the workload.","discriminating_result":"The page names concrete DeepGEMM and B200 contracts; applicability to an unspecified alternative implementation is not direct compatibility.","action":"correct","action_reason":"Replace universal advice with contract-specific selection criteria.","proposed_replacement":"Choose the contest independent-shape ABI or DeepGEMM's fixed-N/K M-grouped ABI only when all dtype/layout/architecture requirements match.","replacement_claim_ids":["kernel-grouped-gemm-current-c001","kernel-grouped-gemm-current-c003"]} +{"receipt_id":"rem-kernel-grouped-gemm-009","receipt_scope":"wiki","path":"wiki/kernels/grouped-gemm.md","original_body_sha256":"216f09e31b3f73503470760148850014df1540ad380997383317930a7b459e9e","original_unit_ids":["u015"],"locator":"body:L174","original_claim_id":"kernel-grouped-gemm-original-c015","original_claim":"Expert load imbalance is the primary practical bottleneck.","original_verdict":"refuted","evidence":[{"url":"https://www.gpumode.com/news/reward-hacking-nvfp4","revision":"GPU Mode post dated 2026-03-09, fetched 2026-08-08","locator":"Direct Measurement of Fixed Kernel Startup Cost","observation":"The studied B200 implementation attributes about 80% of a 148-tile runtime to fixed setup, giving a directly measured different dominant bottleneck."}],"strongest_alternative":"Natural MoE routing could still make expert imbalance primary in another deployment.","discriminating_result":"That possibility proves the correct conclusion is workload-specific, not a universal primary ranking.","action":"correct","action_reason":"State measured contest-specific setup/underfill observations and avoid a universal bottleneck hierarchy.","proposed_replacement":"Profile tile count, shapes, setup, scheduling, and memory/compute behavior per workload; no single bottleneck is established universally.","replacement_claim_ids":["kernel-grouped-gemm-current-c004","kernel-grouped-gemm-current-c006"]} +{"receipt_id":"rem-kernel-grouped-gemm-010","receipt_scope":"wiki","path":"wiki/kernels/grouped-gemm.md","original_body_sha256":"216f09e31b3f73503470760148850014df1540ad380997383317930a7b459e9e","original_unit_ids":["u017"],"locator":"body:L176","original_claim_id":"kernel-grouped-gemm-original-c017","original_claim":"DeepGEMM masked grouping necessarily computes padded rows and wastes that compute when valid M is skewed.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-deepgemm-pinned/README.md","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"Grouped GEMMs (masked layout)","observation":"The project says the mask makes the kernel compute only valid portions."},{"path":"/tmp/kernelwiki-deepgemm-pinned/tests/test_fp8_fp4.py","revision":"same commit","locator":"masked output checks","observation":"Tests compare only d[j,:masked_m[j]], consistent with per-group valid-row counts rather than a boolean padding-compute guarantee."}],"strongest_alternative":"Fixed max-M allocation may still force physical work on padding internally.","discriminating_result":"Allocation shape alone does not prove execution, and the primary project explicitly claims valid-portion computation.","action":"correct","action_reason":"Separate fixed allocation from valid computation and describe masked_m accurately.","proposed_replacement":"Masked mode keeps graph-compatible fixed shapes but passes one valid-M count per group so the kernel computes valid portions.","replacement_claim_ids":["kernel-grouped-gemm-current-c003"]} +{"receipt_id":"rem-kernel-grouped-gemm-011","receipt_scope":"wiki","path":"wiki/kernels/grouped-gemm.md","original_body_sha256":"216f09e31b3f73503470760148850014df1540ad380997383317930a7b459e9e","original_unit_ids":["u019"],"locator":"body:L178","original_claim_id":"kernel-grouped-gemm-original-c019","original_claim":"TMA universally requires 128-byte alignment and therefore sets minimum grouped-GEMM tile dimensions.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html","revision":"CUDA Driver API 13.2.1","locator":"cuTensorMapEncodeTiled constraints","observation":"Documented requirements include a 64-byte descriptor and generally 16-byte global address/strides, with some feature-specific 32-byte constraints, not one blanket 128-byte rule."}],"strongest_alternative":"NVFP4-specific CUTLASS code may choose 128-byte transactions even if TMA itself does not require them.","discriminating_result":"An implementation preference cannot be promoted to a universal TMA requirement or minimum-tile theorem.","action":"correct","action_reason":"Replace the blanket rule with versioned API constraints and direct readers to their exact descriptor/feature.","proposed_replacement":"Check the versioned tensor-map API and selected operation; do not infer minimum tiles from a universal 128-byte rule.","replacement_claim_ids":["kernel-grouped-gemm-current-c006"]} +{"receipt_id":"rem-kernel-grouped-gemm-012","receipt_scope":"wiki","path":"wiki/kernels/grouped-gemm.md","original_body_sha256":"216f09e31b3f73503470760148850014df1540ad380997383317930a7b459e9e","original_unit_ids":["u024"],"locator":"frontmatter:performance_claims","original_claim_id":"kernel-grouped-gemm-original-c021","original_claim":"B200 measured 11.2 us for 15 groups with shared N=K and the workload was compute-bound.","original_verdict":"refuted","evidence":[{"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/task.yml","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"benchmarks and speed-of-light table","observation":"The four workloads contain 8/8/2/2 groups, N differs from K, and only theoretical 18.833/10.667/2.406/1.525-us estimates are published."},{"url":"https://www.gpumode.com/news/reward-hacking-nvfp4","revision":"GPU Mode post dated 2026-03-09, fetched 2026-08-08","locator":"leaderboard caption and timing exploit","observation":"11.191 us is an invalid amortized exploit report and was scrubbed."}],"strongest_alternative":"11.2 us is a rounded legitimate record for a hidden 15-group workload.","discriminating_result":"No such task shape or reproducible record exists in the primary benchmark; 15 refers to timing objects, not groups in one workload.","action":"delete","action_reason":"Remove invalid structured performance metadata and retain only labeled theoretical task estimates in prose.","proposed_replacement":null,"replacement_claim_ids":["kernel-grouped-gemm-current-c004","kernel-grouped-gemm-current-c005"]} +{"receipt_id":"rem-source-grouped-gemm-contest-001","receipt_scope":"source-trail","path":"sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md","original_body_sha256":"32745c7d3a47b5a815ecee140a5940d53006542b75c8bf6dca366505c56deeb9","original_unit_ids":[],"locator":"frontmatter submissions and body problem/techniques/performance/challenges sections","original_claim_id":"source-contest-grouped-gemm-rankings-and-implementation","original_claim":"The source publishes legitimate ranks/scores, CLC/CUTLASS winning techniques, fixed N/K-only semantics, model statistics, compute-bound status, and universal 128-byte TMA constraints.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_group_gemm/task.yml","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"complete public task","observation":"The public source specifies independent group shapes, exact tensors/tests/benchmarks, theoretical estimates, and geometric-mean ranking but no legitimate ranks, techniques, model statistics, CLC mandate, or bottleneck classification."},{"url":"https://www.gpumode.com/news/reward-hacking-nvfp4","revision":"GPU Mode post dated 2026-03-09, fetched 2026-08-08","locator":"incident account","observation":"It supports only the exploit mechanism/report and measured postmortem, not the invented legitimate leaderboard."}],"strongest_alternative":"The legitimate ranks and techniques came from an unpublished Discord leaderboard.","discriminating_result":"No immutable capture or submission is recorded, so those details cannot remain source facts; the official task and incident post contradict several associated scopes.","action":"correct","action_reason":"Rewrite the source around exact public rules/task/reference and clearly separate the scrubbed exploit from valid performance.","proposed_replacement":"Record challenge dates/weight, tensor ABI, tests, four theoretical rows, geometric-mean scoring, and the official exploit boundary; omit unpublished winners and implementations.","replacement_claim_ids":["kernel-grouped-gemm-current-c001","kernel-grouped-gemm-current-c002","kernel-grouped-gemm-current-c004","kernel-grouped-gemm-current-c005"]} +{"receipt_id":"rem-source-grouped-gemm-relationships-001","receipt_scope":"source-trail","path":"wiki/kernels/grouped-gemm.md","original_body_sha256":"216f09e31b3f73503470760148850014df1540ad380997383317930a7b459e9e","original_unit_ids":[],"locator":"frontmatter:sources doc-cutlass-blackwell and missing blog-gpu-mode-reward-hack","original_claim_id":"source-relationship-grouped-gemm-cutlass-and-incident","original_claim":"The generic CUTLASS Blackwell changelog directly supports the page's displayed grouped kernel, while the reward-hack discussion needs no direct source relationship.","original_verdict":"refuted","evidence":[{"path":"/tmp/cutlass-audit.8aj07j/repo/include/cutlass/gemm/kernel/gemm_grouped.h","revision":"NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3","locator":"GemmGrouped declaration","observation":"The exact official API contradicts the displayed kernel signature; a generic changelog does not evidence that sketch."},{"url":"https://www.gpumode.com/news/reward-hacking-nvfp4","revision":"GPU Mode post dated 2026-03-09, fetched 2026-08-08","locator":"entire incident account","observation":"This is the direct primary source for the retained incident claims."}],"strongest_alternative":"CUTLASS remains useful general background and the page already links the incident in prose.","discriminating_result":"Background utility does not make a generic changelog direct evidence for invented code; the retained incident is material enough to require the explicit source relationship.","action":"correct","action_reason":"Remove doc-cutlass-blackwell from direct frontmatter evidence and add blog-gpu-mode-reward-hack; retain precise CUTLASS links only where used.","proposed_replacement":"sources: contest-gpumode-p4, blog-deepgemm, blog-gpu-mode-reward-hack.","replacement_claim_ids":["kernel-grouped-gemm-current-c005","kernel-grouped-gemm-current-c006"]} +{"receipt_id":"rem-kernel-nsa-001","receipt_scope":"wiki","path":"wiki/kernels/nsa.md","original_body_sha256":"002fbf6ef7dba7c6bd6bfec4cdeccf4104b100309e09eae082faaf0aeec0e4c3","original_unit_ids":["u002"],"locator":"body:L7","original_claim_id":"kernel-nsa-original-c003","original_claim":"The three-branch NSA mechanism is deployed in DeepSeek-V3.2-Exp using FlashMLA sparse kernels.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-deepseek-v32/README.md","revision":"DeepSeek-V3.2-Exp 87e509a2e5a100d221c97df52c6e8be7835f0057","locator":"DSA announcement and CUDA-kernel links","observation":"The release names a different DeepSeek Sparse Attention mechanism and directs its indexer and sparse kernels to DeepGEMM and FlashMLA."},{"path":"/tmp/kernelwiki-deepseek-v32/inference/model.py","revision":"same commit","locator":"Indexer.forward and MLA.forward","observation":"The released model selects top-k token positions and masks attention; it does not implement NSA's gated compression/selection/window sum."}],"strongest_alternative":"DSA is only a renamed production version of the same NSA architecture.","discriminating_result":"The observable inference graph lacks NSA's compression and sliding-window branches and uses a separate learned token indexer, so the mechanisms cannot be identified as one architecture.","action":"correct","action_reason":"Separate the ACL NSA paper from the later DSA deployment and retain FlashMLA only as evidence for that distinction.","proposed_replacement":"State that V3.2-Exp and FlashMLA deploy DSA, a related but different learned sparse-attention design.","replacement_claim_ids":["kernel-nsa-current-c006"]} +{"receipt_id":"rem-kernel-nsa-002","receipt_scope":"wiki","path":"wiki/kernels/nsa.md","original_body_sha256":"002fbf6ef7dba7c6bd6bfec4cdeccf4104b100309e09eae082faaf0aeec0e4c3","original_unit_ids":["u003"],"locator":"body:L11-L32","original_claim_id":"kernel-nsa-original-c004","original_claim":"The displayed three-path Combine expression is the exact NSA output equation without learned branch gates.","original_verdict":"refuted","evidence":[{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings PDF","locator":"Section 3.1, Equations 5 and 6","observation":"Each branch output is multiplied by an input-dependent sigmoid gate produced by a learned MLP before summation."}],"strongest_alternative":"Combine is merely shorthand that implicitly includes the gates.","discriminating_result":"The diagram spells out its operands and equation but includes no gate variables or gate network, omitting a defining learned operation.","action":"correct","action_reason":"Replace the ambiguous diagram with the paper's gated branch equation and exact branch roles.","proposed_replacement":"Write o=sum_c g_c*Attn_c and define g_c=sigmoid(MLP_c(h_t)).","replacement_claim_ids":["kernel-nsa-current-c002","kernel-nsa-current-c003"]} +{"receipt_id":"rem-kernel-nsa-003","receipt_scope":"wiki","path":"wiki/kernels/nsa.md","original_body_sha256":"002fbf6ef7dba7c6bd6bfec4cdeccf4104b100309e09eae082faaf0aeec0e4c3","original_unit_ids":["u005"],"locator":"body:L38-L103","original_claim_id":"kernel-nsa-original-c006","original_claim":"The displayed nsa_sparse_attention_fwd is a valid NSA Triton kernel with correct online softmax.","original_verdict":"refuted","evidence":[{"command":"conda run -n base python -c ''","environment":"KernelWiki host; conda base","reference":"displayed update for scores [0,0] and values [1,3]","result":"Candidate output was 4.0 and candidate lse 0.0; correct output was 2.0 and log-partition log(2).","negative_control":"A stable normalized-softmax implementation returned 2.0."}],"strongest_alternative":"The omitted normalization is recovered by the later online-softmax iterations.","discriminating_result":"The failure occurs with a single selected block, where no later iteration exists; undefined SEQ_LEN and unused pid are additional independent defects.","action":"delete","action_reason":"Remove a mathematically incorrect and non-executable kernel sketch rather than imply it is paper code.","proposed_replacement":null,"replacement_claim_ids":["kernel-nsa-current-c003","kernel-nsa-current-c004"]} +{"receipt_id":"rem-kernel-nsa-004","receipt_scope":"wiki","path":"wiki/kernels/nsa.md","original_body_sha256":"002fbf6ef7dba7c6bd6bfec4cdeccf4104b100309e09eae082faaf0aeec0e4c3","original_unit_ids":["u006"],"locator":"body:L107-L140","original_claim_id":"kernel-nsa-original-c007","original_claim":"The displayed nsa_sliding_window_fwd is a valid dense sliding-window attention kernel.","original_verdict":"refuted","evidence":[{"command":"conda run -n base python -c ''","environment":"KernelWiki host; conda base","reference":"displayed loop semantics","result":"Candidate positions were [0,2], while the four-position reference window was [0,1,2,3].","negative_control":"Unit-stride enumeration covered all positions."}],"strongest_alternative":"Each loop iteration implicitly loads a BLOCK_SIZE-wide vector.","discriminating_result":"The body addresses only K[pos] and V[pos], and BLOCK_SIZE is not declared by the function.","action":"delete","action_reason":"Remove the incomplete sketch and describe the branch from the paper instead.","proposed_replacement":null,"replacement_claim_ids":["kernel-nsa-current-c002","kernel-nsa-current-c003"]} +{"receipt_id":"rem-kernel-nsa-005","receipt_scope":"wiki","path":"wiki/kernels/nsa.md","original_body_sha256":"002fbf6ef7dba7c6bd6bfec4cdeccf4104b100309e09eae082faaf0aeec0e4c3","original_unit_ids":["u010"],"locator":"body:L148","original_claim_id":"kernel-nsa-original-c008","original_claim":"The paper specifies a (query_block, head, batch) Triton grid that avoids dynamic scheduling overhead.","original_verdict":"refuted","evidence":[{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings PDF","locator":"Section 3.3","observation":"The paper places nearly constant query/output loops in the grid but gives neither the asserted three-axis mapping nor an eliminated-overhead result."}],"strongest_alternative":"Those dimensions are an obvious implementation of the paper's outer loops.","discriminating_result":"A plausible implementation choice is not a source-established layout or measured performance mechanism.","action":"correct","action_reason":"Retain the exact outer-loop/grid observation without inventing grid axes or an overhead result.","proposed_replacement":"State that the authors map nearly constant query/output loops to Triton grid parallelism.","replacement_claim_ids":["kernel-nsa-current-c004"]} +{"receipt_id":"rem-kernel-nsa-006","receipt_scope":"wiki","path":"wiki/kernels/nsa.md","original_body_sha256":"002fbf6ef7dba7c6bd6bfec4cdeccf4104b100309e09eae082faaf0aeec0e4c3","original_unit_ids":["u012","u022"],"locator":"body:L156 and frontmatter:performance_claims","original_claim_id":"kernel-nsa-original-c009","original_claim":"All three speedups were measured on H100 BF16, including a reproducible structured 9.0x record at sequence length 65,536.","original_verdict":"refuted","evidence":[{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings PDF","locator":"Section 4 setup, Figure 5, and Table 4","observation":"The experiment uses an 8-GPU A100 system, gives no dtype for this benchmark, and labels 11.6x as expected from memory-access volume; batch/software/raw trials/variance are absent."}],"strongest_alternative":"H100/BF16 details were omitted only from the paper prose but known from the authors' setup.","discriminating_result":"The primary record explicitly names A100 and supplies no supporting H100/BF16 configuration, directly contradicting the structured tuple.","action":"delete","action_reason":"Remove invalid structured performance metadata and retain source-reported values in prose with exact scope and missing reproducibility fields.","proposed_replacement":null,"replacement_claim_ids":["kernel-nsa-current-c005"]} +{"receipt_id":"rem-kernel-nsa-007","receipt_scope":"wiki","path":"wiki/kernels/nsa.md","original_body_sha256":"002fbf6ef7dba7c6bd6bfec4cdeccf4104b100309e09eae082faaf0aeec0e4c3","original_unit_ids":["u013"],"locator":"body:L160","original_claim_id":"kernel-nsa-original-c010","original_claim":"NSA should be used at 32K or longer because dense attention is universally prohibitively expensive beyond that threshold.","original_verdict":"refuted","evidence":[{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings PDF","locator":"Sections 1 and 4","observation":"The paper motivates long context and evaluates through 64K but defines no universal 32K cutoff or categorical applicability rule."}],"strongest_alternative":"The first long-context evaluation point implicitly defines the use threshold.","discriminating_result":"An evaluation length is not a compatibility threshold; model quality, workload, sequence distribution, and hardware determine the tradeoff.","action":"correct","action_reason":"Replace the universal cutoff with a workload- and quality-dependent selection boundary.","proposed_replacement":"Consider NSA only when its trained model contract and measured workload justify the sparse tradeoff.","replacement_claim_ids":["kernel-nsa-current-c007"]} +{"receipt_id":"rem-kernel-nsa-008","receipt_scope":"wiki","path":"wiki/kernels/nsa.md","original_body_sha256":"002fbf6ef7dba7c6bd6bfec4cdeccf4104b100309e09eae082faaf0aeec0e4c3","original_unit_ids":["u017"],"locator":"body:L167","original_claim_id":"kernel-nsa-original-c013","original_claim":"NSA selection adds an independent two-pass operation that first scores every fine-grained block and then takes top-n.","original_verdict":"refuted","evidence":[{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings PDF","locator":"Section 3.1 Selection of Important Tokens","observation":"Fine-grained block importance is derived by reusing compression-attention scores, aggregating them over related blocks and query heads, before top-n selection."}],"strongest_alternative":"Reusing compression scores is still conceptually a separate first scoring pass.","discriminating_result":"The original performance caveat claims an additional score-all-blocks pass, while the paper specifically designs selection to reuse scores already computed by another branch.","action":"correct","action_reason":"Describe score reuse and top-n selection without inventing an extra pass.","proposed_replacement":"Selection aggregates reused compression scores and chooses top-n fine-grained blocks.","replacement_claim_ids":["kernel-nsa-current-c002"]} +{"receipt_id":"rem-kernel-nsa-009","receipt_scope":"wiki","path":"wiki/kernels/nsa.md","original_body_sha256":"002fbf6ef7dba7c6bd6bfec4cdeccf4104b100309e09eae082faaf0aeec0e4c3","original_unit_ids":["u018"],"locator":"body:L168","original_claim_id":"kernel-nsa-original-c014","original_claim":"NSA Triton decode is established to be CPU-launch-bound at small batch, so CUDA Graph mode is recommended.","original_verdict":"refuted","evidence":[{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings PDF","locator":"Section 4.2 and Limitations","observation":"The paper gives a memory-access model and mentions generic Triton abstraction overhead, but no NSA CPU-launch profile or graph recommendation."},{"url":"https://docs.nvidia.com/dl-cuda-graph/troubleshooting/performance-issues.html","revision":"NVIDIA guide fetched 2026-08-08","locator":"Performance Issues","observation":"CUDA Graphs primarily help CPU-bound workloads and can provide little benefit or regress otherwise."}],"strongest_alternative":"Short decoding kernels are normally launch-bound, making graphs safe default advice.","discriminating_result":"Vendor guidance makes the recommendation conditional on profiling, and the NSA source does not establish the required bottleneck.","action":"correct","action_reason":"Replace the categorical prescription with a profile-first CUDA Graph boundary.","proposed_replacement":"Use graphs only after a profile shows CPU launch overhead dominates and the workflow satisfies graph constraints.","replacement_claim_ids":["kernel-nsa-current-c007"]} +{"receipt_id":"rem-kernel-nsa-010","receipt_scope":"wiki","path":"wiki/kernels/nsa.md","original_body_sha256":"002fbf6ef7dba7c6bd6bfec4cdeccf4104b100309e09eae082faaf0aeec0e4c3","original_unit_ids":["u004","u007"],"locator":"frontmatter:architectures/blackwell_relevance and body kernel-design scope","original_claim_id":"kernel-nsa-original-c017","original_claim":"The paper directly establishes SM90/SM100 compatibility and a Blackwell 126-MB-L2/bandwidth benefit for NSA's sparse block fetches.","original_verdict":"refuted","evidence":[{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings PDF","locator":"Section 4 setup","observation":"The efficiency evaluation is on A100 and reports no SM90/SM100 compatibility or Blackwell comparison."},{"url":"https://github.com/deepseek-ai/FlashMLA/blob/71c737929f2567bd0a094ae140f8f60f390b1232/README.md","revision":"FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232","locator":"support matrix","observation":"SM90/SM100 support is documented for later DSA kernels, not for the ACL NSA implementation."}],"strongest_alternative":"NSA's algorithm is portable and should benefit automatically from larger caches and bandwidth.","discriminating_result":"Algorithmic portability does not prove implementation compatibility or a performance effect; the only direct benchmark target is A100.","action":"correct","action_reason":"Scope architecture metadata to the paper's A100 evidence and make Blackwell an unmeasured portability question.","proposed_replacement":"Leave controlled architecture metadata empty; state the A100 evidence and require a separate Blackwell implementation and benchmark.","replacement_claim_ids":["kernel-nsa-current-c001","kernel-nsa-current-c006"]} +{"receipt_id":"rem-source-nsa-paper-001","receipt_scope":"source-trail","path":"sources/blogs/nsa.md","original_body_sha256":"01ec9cf58d5ea354b8ad2fdcc3719072adb613fdd8df3c8271f235ca6ac64a32","original_unit_ids":[],"locator":"frontmatter architecture/category and Key Techniques deployment/performance bullets","original_claim_id":"source-nsa-a100-scope-and-deployment","original_claim":"The NSA source is a benchmark blog for SM90/SM100, reports undifferentiated 9x/6x values, and says NSA was deployed in DeepSeek-V3.2-Exp.","original_verdict":"refuted","evidence":[{"url":"https://aclanthology.org/2025.acl-long.1126/","revision":"ACL 2025 anthology record","locator":"publication metadata","observation":"The source is an ACL research paper, not a benchmark blog."},{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL proceedings PDF","locator":"Section 4.2","observation":"The reported efficiency setup is A100; 9x/6x are Figure 5 results and 11.6x is an expected decoding ratio."},{"path":"/tmp/kernelwiki-deepseek-v32/README.md","revision":"DeepSeek-V3.2-Exp 87e509a2","locator":"DSA release","observation":"The later deployment is explicitly DSA."}],"strongest_alternative":"The source card may summarize later project evolution in addition to the original paper.","discriminating_result":"It labels those later facts as properties of NSA without distinguishing a separate architecture, causing false evidence transfer.","action":"correct","action_reason":"Rewrite the source card as a paper record with exact A100/performance scope and an explicit DSA non-equivalence note.","proposed_replacement":"Use paper category, leave controlled architecture metadata empty, record the exact A100 branch/kernel setup, and separately identify later DSA.","replacement_claim_ids":["kernel-nsa-current-c001","kernel-nsa-current-c005","kernel-nsa-current-c006"]} +{"receipt_id":"rem-source-nsa-relationships-001","receipt_scope":"source-trail","path":"wiki/kernels/nsa.md","original_body_sha256":"002fbf6ef7dba7c6bd6bfec4cdeccf4104b100309e09eae082faaf0aeec0e4c3","original_unit_ids":[],"locator":"frontmatter:sources","original_claim_id":"source-relationship-nsa-vllm-flashmla","original_claim":"A vLLM community post and FlashMLA directly support the ACL NSA mechanism and deployment claim.","original_verdict":"refuted","evidence":[{"url":"https://github.com/deepseek-ai/DeepSeek-V3.2-Exp/tree/87e509a2e5a100d221c97df52c6e8be7835f0057","revision":"DeepSeek-V3.2-Exp 87e509a2","locator":"README and inference source","observation":"The first-party deployment defines DSA and points to FlashMLA; it differs from NSA."},{"url":"https://github.com/deepseek-ai/FlashMLA/blob/71c737929f2567bd0a094ae140f8f60f390b1232/README.md","revision":"FlashMLA 71c7379","locator":"Sparse Attention Kernels","observation":"FlashMLA directly supports only the later DSA distinction."}],"strongest_alternative":"The vLLM post is useful secondary implementation context.","discriminating_result":"A primary official repository is available for DSA, while the community post cannot establish equivalence to NSA.","action":"correct","action_reason":"Remove the vLLM community source from direct frontmatter evidence and retain FlashMLA only for the explicit non-equivalence boundary.","proposed_replacement":"sources: [blog-nsa, blog-flashmla].","replacement_claim_ids":["kernel-nsa-current-c006","kernel-nsa-current-c008"]} +{"receipt_id":"rem-kernel-nvfp4-gemm-001","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemm.md","original_body_sha256":"d176748702ac353341b6d4887c74d36b98deccce13fd42849ac545957b212fa0","original_unit_ids":["u001"],"locator":"body:L5","original_claim_id":"kernel-nvfp4-gemm-original-c002","original_claim":"Every NVFP4 GEMM is compute-bound and necessarily uses the listed Blackwell pipeline mechanisms.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/task.yml","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"speed-of-light model","observation":"The task compares math and DRAM bounds per shape and imposes an output contract rather than one implementation."},{"path":"/tmp/cutlass-audit.8aj07j/repo/media/docs/cpp/efficient_gemm.md","revision":"NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3","locator":"occupancy and wave quantization","observation":"Achieved GEMM behavior depends on shape and schedule."}],"strongest_alternative":"Datatype and high arithmetic intensity alone make all instances compute-bound.","discriminating_result":"The task's own model includes a DRAM bound and changes by shape; no mechanism is part of correctness.","action":"correct","action_reason":"Scope bottlenecks and implementation mechanisms to the exact shape and implementation.","proposed_replacement":"State the task contract and require profiling before assigning a bottleneck or implementation strategy.","replacement_claim_ids":["kernel-nvfp4-gemm-current-c001","kernel-nvfp4-gemm-current-c008"]} +{"receipt_id":"rem-kernel-nvfp4-gemm-002","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemm.md","original_body_sha256":"d176748702ac353341b6d4887c74d36b98deccce13fd42849ac545957b212fa0","original_unit_ids":["u002"],"locator":"body:L7","original_claim_id":"kernel-nvfp4-gemm-original-c004","original_claim":"The asserted top entries were within one percent of cuBLAS and used CUTLASS SM100 schedules.","original_verdict":"refuted","evidence":[{"url":"https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12","revision":"public leaderboard fetched 2026-08-08","locator":"ranked rows","observation":"The public top cohort differs and the endpoint provides neither a cuBLAS ratio nor implementation source."},{"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml","revision":"ae679486","locator":"ranking and speed-of-light text","observation":"The task ranks geometric mean and provides theoretical math/DRAM values, not cuBLAS or CUTLASS attribution."}],"strongest_alternative":"The named competitors privately posted CUTLASS implementations and were close to an unpublished cuBLAS run.","discriminating_result":"That possibility is not the claimed public result; the direct records contradict the top-entry identity and do not contain the asserted metric or implementation.","action":"delete","action_reason":"Remove an untraceable performance and implementation attribution; retain only dated public fields and theoretical task rows.","proposed_replacement":null,"replacement_claim_ids":["kernel-nvfp4-gemm-current-c006","kernel-nvfp4-gemm-current-c007"]} +{"receipt_id":"rem-kernel-nvfp4-gemm-003","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemm.md","original_body_sha256":"d176748702ac353341b6d4887c74d36b98deccce13fd42849ac545957b212fa0","original_unit_ids":["u004"],"locator":"body:L29","original_claim_id":"kernel-nvfp4-gemm-original-c007","original_claim":"GPU Mode Problem 2 and its top performers are established to use the named CUTLASS pointer-array schedule.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/task.yml","revision":"gpu-mode/reference-kernels ae679486","locator":"task contract","observation":"The task specifies no kernel or schedule."},{"path":"/tmp/cutlass-audit.8aj07j/repo/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu","revision":"NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3","locator":"MMA1SMConfig","observation":"The direct official call site is a grouped pointer-array example, not a contestant solution."}],"strongest_alternative":"A contestant could have chosen the schedule despite the different representative API.","discriminating_result":"Possibility is not attribution; neither official task, leaderboard, nor bundled artifact contains such a solution.","action":"correct","action_reason":"Retain the schedule only as an adjacent official CUTLASS example and explicitly deny contestant attribution.","proposed_replacement":"Describe the symbol's official grouped pointer-array scope and state that it is not evidence about entrants.","replacement_claim_ids":["kernel-nvfp4-gemm-current-c005"]} +{"receipt_id":"rem-kernel-nvfp4-gemm-004","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemm.md","original_body_sha256":"d176748702ac353341b6d4887c74d36b98deccce13fd42849ac545957b212fa0","original_unit_ids":["u005"],"locator":"body:L31-L55","original_claim_id":"kernel-nvfp4-gemm-original-c008","original_claim":"The displayed CUTLASS typedef is a valid CUTLASS 4.x NVFP4 GEMM.","original_verdict":"refuted","evidence":[{"path":"/tmp/cutlass-audit.8aj07j/repo/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu","revision":"NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3","locator":"operand, collective, kernel, and adapter declarations","observation":"The official API uses nv_float4 wrappers, collective builders, kernel::GemmUniversal, and device::GemmUniversalAdapter; the displayed old-style signature omits the scale contract."}],"strongest_alternative":"CUTLASS preserves an overload accepting the old scalar/layout template list.","discriminating_result":"The named current block-scaled API and official example require collective types; the shown arguments do not match that declaration.","action":"delete","action_reason":"Remove non-compiling pseudo-CUTLASS and link to the immutable official example instead.","proposed_replacement":null,"replacement_claim_ids":["kernel-nvfp4-gemm-current-c005","kernel-nvfp4-gemm-current-c009"]} +{"receipt_id":"rem-kernel-nvfp4-gemm-005","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemm.md","original_body_sha256":"d176748702ac353341b6d4887c74d36b98deccce13fd42849ac545957b212fa0","original_unit_ids":["u006"],"locator":"body:L59-L130","original_claim_id":"kernel-nvfp4-gemm-original-c009","original_claim":"The displayed hand-written CUDA is a valid warp-specialized NVFP4 kernel with correct TMA and tcgen05 operands.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cute/arch/mma_sm100_umma.hpp#L1647-L1659","revision":"NVIDIA/cutlass e05f953a","locator":"block16 NVFP4 asm wrapper","observation":"The exact instruction kind and operand set differ materially."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"PTX ISA 9.0","locator":"syntax and restrictions","observation":"The listing violates the authoritative grammar and also references numerous undefined program symbols."}],"strongest_alternative":"It is intentionally abbreviated pseudocode whose missing pieces are understood.","discriminating_result":"It is labeled as a kernel and presents executable syntax, but even the instruction mnemonic and required operands are wrong; abbreviation cannot repair the semantics.","action":"delete","action_reason":"Remove misleading non-executable CUDA rather than imply it is a reference implementation.","proposed_replacement":null,"replacement_claim_ids":["kernel-nvfp4-gemm-current-c005"]} +{"receipt_id":"rem-kernel-nvfp4-gemm-006","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemm.md","original_body_sha256":"d176748702ac353341b6d4887c74d36b98deccce13fd42849ac545957b212fa0","original_unit_ids":["u007","u020"],"locator":"body:L134 and L186","original_claim_id":"kernel-nvfp4-gemm-original-c010","original_claim":"All TMA operands universally require 128-byte alignment, implying K divisible by 256 for FP4.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html","revision":"CUDA Driver API 13.2.1","locator":"cuTensorMapEncodeTiled restrictions","observation":"The general contract documents 16-byte base/stride rules plus mode-specific constraints, not one universal 128-byte rule."}],"strongest_alternative":"128 bytes is a conservative implementation choice and therefore a valid TMA requirement.","discriminating_result":"A kernel-specific conservative choice cannot be relabeled as the API's universal contract; the official interface uses smaller general alignment values.","action":"correct","action_reason":"Separate the task's K divisibility rule from actual tensor-map constraints.","proposed_replacement":"State K modulo 256 as the task ABI and require checking exact TMA descriptor mode/version independently.","replacement_claim_ids":["kernel-nvfp4-gemm-current-c003","kernel-nvfp4-gemm-current-c005"]} +{"receipt_id":"rem-kernel-nvfp4-gemm-007","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemm.md","original_body_sha256":"d176748702ac353341b6d4887c74d36b98deccce13fd42849ac545957b212fa0","original_unit_ids":["u008"],"locator":"body:L136-L145","original_claim_id":"kernel-nvfp4-gemm-original-c022","original_claim":"The official input space requires (K/16) divisible by 128 for the scale array.","original_verdict":"refuted","evidence":[{"command":"conda run -n base python -c ''","environment":"KernelWiki host; conda base","reference":"gpu-mode/reference-kernels ae679486 task.yml/reference.py","result":"Nine of ten valid official tests, including K=256, fail the displayed assertion.","negative_control":"K=255 fails the actual K-divisible-by-256 task rule."}],"strongest_alternative":"The tests are padded before reaching a TMA kernel, making the logical assertion harmless.","discriminating_result":"The assertion is written against task K and rejects officially valid values before any private padding strategy; padding is not an ABI requirement.","action":"delete","action_reason":"Replace the false assertion with a host-checkable representation of the published tensor shapes.","proposed_replacement":null,"replacement_claim_ids":["kernel-nvfp4-gemm-current-c003","kernel-nvfp4-gemm-current-c004"]} +{"receipt_id":"rem-kernel-nvfp4-gemm-008","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemm.md","original_body_sha256":"d176748702ac353341b6d4887c74d36b98deccce13fd42849ac545957b212fa0","original_unit_ids":["u009","u010","u019"],"locator":"body:L149-L164 and L185","original_claim_id":"kernel-nvfp4-gemm-original-c011","original_claim":"NVFP4 tcgen05 expects UE8M0, so E4M3 scales must be rounded and packed before MMA.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512","revision":"NVIDIA/cutlass e05f953a","locator":"mx_float4_t and nv_float4_t ScaleFactorType","observation":"CUTLASS assigns UE8M0 to MXFP4 and UE4M3 to NVFP4."},{"url":"https://docs.nvidia.com/cuda/cublas/index.html#element-1d-block-scaling-for-fp8-and-fp4-data-types","revision":"fetched 2026-08-08","locator":"FP4 VEC16 UE4M3 mode","observation":"The official NVFP4 mode consumes UE4M3 directly."}],"strongest_alternative":"PTX normalizes E4M3 to UE8M0 internally despite the API type.","discriminating_result":"The ISA/library type contract expressly selects UE4M3; rounding to UE8M0 would change representable scale values and turn NVFP4 into a different scaling mode.","action":"delete","action_reason":"Remove a format-confusing conversion and document the exact UE4M3 scale path.","proposed_replacement":null,"replacement_claim_ids":["kernel-nvfp4-gemm-current-c002","kernel-nvfp4-gemm-current-c005"]} +{"receipt_id":"rem-kernel-nvfp4-gemm-009","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemm.md","original_body_sha256":"d176748702ac353341b6d4887c74d36b98deccce13fd42849ac545957b212fa0","original_unit_ids":["u011","u012","u013","u014"],"locator":"body:L168-L174","original_claim_id":"kernel-nvfp4-gemm-original-c012","original_claim":"Simon, yue, and currybab are ranks 1-3 with the displayed scores.","original_verdict":"refuted","evidence":[{"url":"https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12","revision":"fetched 2026-08-08","locator":"ranks 1-10","observation":"The values correspond to those users but at current ranks 8-10; current ranks 1-3 are different users with later timestamps."},{"url":"https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf","revision":"official rules","locator":"contest period","observation":"Some current leading submissions postdate the December 19 cutoff, so the public snapshot is not itself a prize-placement oracle."}],"strongest_alternative":"The table records ranks at the contest deadline rather than current ranks.","discriminating_result":"The page labels no snapshot date or cutoff, and the direct current endpoint contradicts its unqualified ranks; no primary cutoff snapshot establishes the old labels.","action":"correct","action_reason":"Publish only a dated current snapshot and explicitly separate it from official prize placements.","proposed_replacement":"Show current ranks 1-3 and historical named rows with timestamps and the cutoff caveat.","replacement_claim_ids":["kernel-nvfp4-gemm-current-c007"]} +{"receipt_id":"rem-kernel-nvfp4-gemm-010","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemm.md","original_body_sha256":"d176748702ac353341b6d4887c74d36b98deccce13fd42849ac545957b212fa0","original_unit_ids":["u015"],"locator":"body:L178","original_claim_id":"kernel-nvfp4-gemm-original-c013","original_claim":"This task applies to inference with any 4-bit quantized weights on Blackwell.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html","revision":"Transformer Engine 2.13","locator":"recipe and MXFP4 comparison","observation":"NVFP4 is one exact payload-and-scale recipe among multiple incompatible 4-bit formats."},{"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py","revision":"ae679486","locator":"input tuple","observation":"The task requires exact packed E2M1 and logical/reordered E4M3 scale tensors."}],"strongest_alternative":"All 4-bit formats can be losslessly reinterpreted as NVFP4.","discriminating_result":"INT4 and MXFP4 have different payload/scale semantics; reinterpretation changes values or violates layout.","action":"correct","action_reason":"Scope use to operands prepared for the exact NVFP4 ABI.","proposed_replacement":"Use only when payload, scales, layouts, output, and architecture match the selected implementation contract.","replacement_claim_ids":["kernel-nvfp4-gemm-current-c008"]} +{"receipt_id":"rem-kernel-nvfp4-gemm-011","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemm.md","original_body_sha256":"d176748702ac353341b6d4887c74d36b98deccce13fd42849ac545957b212fa0","original_unit_ids":["u021"],"locator":"body:L187","original_claim_id":"kernel-nvfp4-gemm-original-c016","original_claim":"TMEM organization limits every logical output tile to 128 rows by 512 columns.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu#L147-L159","revision":"NVIDIA/cutlass e05f953a","locator":"one-SM and two-SM tile configurations","observation":"The official example defines a cooperative logical M tile of 256."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory","revision":"PTX ISA 9.0","locator":"TMEM organization/allocation","observation":"128 lanes by 512 columns describes storage addressing, not a universal output-matrix ceiling."}],"strongest_alternative":"The 256-row example is really two independent 128-row tiles and preserves the claim per SM.","discriminating_result":"The original claim says maximum logical output tile; the cooperative kernel exposes one 256-row MMA tile across the CTA pair, so physical ownership cannot justify that logical ceiling.","action":"correct","action_reason":"Describe TMEM storage separately from implementation tile shapes.","proposed_replacement":"State that logical tiling also depends on CTA grouping, allocation, data type, and epilogue strategy.","replacement_claim_ids":["kernel-nvfp4-gemm-current-c005","kernel-nvfp4-gemm-current-c008"]} +{"receipt_id":"rem-kernel-nvfp4-gemm-012","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemm.md","original_body_sha256":"d176748702ac353341b6d4887c74d36b98deccce13fd42849ac545957b212fa0","original_unit_ids":["u026"],"locator":"body:L198","original_claim_id":"kernel-nvfp4-gemm-original-c019","original_claim":"The attached bundle is a full reference implementation of the contest NVFP4 GEMM.","original_verdict":"refuted","evidence":[{"path":"artifacts/kernels/nvfp4-gemm/full/PR-2139-blockwise-groupwise-gemm.patch","revision":"ca4fdbea; SHA-256 e6c54c39519ee3ac609efdef6f8a9b23a56deaed34153e69ae75cf5892d93fc1","locator":"examples 67, 68, 81","observation":"The patch is FP8 E4M3 blockwise/groupwise work, not NVFP4 contest code."},{"path":"artifacts/kernels/nvfp4-gemm/full/blackwell-cutlass-schedules-and-tma.cu","revision":"SHA-256 e720037bbdb5ee5054259a331cdaf489e7135a44038d66d1fd9d9e5998b75255","locator":"entire file","observation":"It is comments plus one offset assignment."},{"path":"artifacts/kernels/nvfp4-gemm/variants/01-nvfp4-gemm-skeleton.cu","revision":"derived repository artifact","locator":"entire file","observation":"It uses undefined helpers and wrong instruction/scale syntax."}],"strongest_alternative":"A provenance-complete bundle can be called a reference even if assembled from adjacent components.","discriminating_result":"Provenance establishes origin, not conformance; none of the files implements the seven-tensor task or passes its checker.","action":"delete","action_reason":"Detach the artifact_dir and remove the Full Reference Implementation section while preserving audit evidence and files.","proposed_replacement":null,"replacement_claim_ids":["kernel-nvfp4-gemm-current-c009"]} +{"receipt_id":"rem-kernel-nvfp4-gemm-013","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemm.md","original_body_sha256":"d176748702ac353341b6d4887c74d36b98deccce13fd42849ac545957b212fa0","original_unit_ids":["u029"],"locator":"frontmatter:performance_claims","original_claim_id":"kernel-nvfp4-gemm-original-c021","original_claim":"10.807 is one standard-shape B200 latency and proves near-cuBLAS utilization.","original_verdict":"refuted","evidence":[{"url":"https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12","revision":"fetched 2026-08-08","locator":"Simon row","observation":"10.806750492 microseconds is the rounded leaderboard score, currently rank 8, not one shape row."},{"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml","revision":"ae679486","locator":"benchmarks and ranking_by geom","observation":"The score aggregates three shapes geometrically and no cuBLAS result is supplied."}],"strongest_alternative":"A geometric mean is still a latency metric and cuBLAS comparison is implicit.","discriminating_result":"The structured schema claims one unspecified shape and utilization; neither can be recovered from the aggregate score or task source.","action":"delete","action_reason":"Remove malformed performance metadata and retain exact theoretical rows plus a dated leaderboard snapshot in prose.","proposed_replacement":null,"replacement_claim_ids":["kernel-nvfp4-gemm-current-c006","kernel-nvfp4-gemm-current-c007"]} +{"receipt_id":"rem-source-nvfp4-gemm-contest-001","receipt_scope":"source-trail","path":"sources/contests/gpu-mode-nvfp4/problem-2-gemm.md","original_body_sha256":"ab9d5824cecd7c118b3672b27d05de4ef2110d597ac8297251728f2feaee7679","original_unit_ids":[],"locator":"frontmatter submissions and entire body","original_claim_id":"source-nvfp4-gemm-contest-record","original_claim":"The source card names Simon, yue, and currybab as ranks 1-3, invents their techniques, treats a five-tensor prose tuple as the complete ABI, and presents all GEMMs as compute-bound and near cuBLAS.","original_verdict":"refuted","evidence":[{"url":"https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12","revision":"fetched 2026-08-08","locator":"public ranks","observation":"The named users are current ranks 8-10 and the API supplies no source or technique."},{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/reference.py","revision":"gpu-mode/reference-kernels ae679486","locator":"generate_input and ref_kernel","observation":"The actual input has seven tensors and exact logical/reordered scale layouts; reference.py constructs e4m3fn despite FNUZ prose."},{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/task.yml","revision":"same commit","locator":"speed-of-light model","observation":"The bottleneck model is shape-dependent and no cuBLAS comparison is published."}],"strongest_alternative":"The source card preserves a private Discord cutoff snapshot and implementation descriptions.","discriminating_result":"No pinned primary artifact is recorded for those claims, while public first-party records directly differ; provenance-free private recollection cannot support the card.","action":"correct","action_reason":"Rewrite the card as an evidence-scoped official task record with a dated public leaderboard caveat.","proposed_replacement":"Record the exact seven-tensor ABI, tests, benchmarks, scoring, dtype inconsistency, and current public ranks without implementation attribution.","replacement_claim_ids":["kernel-nvfp4-gemm-current-c001","kernel-nvfp4-gemm-current-c003","kernel-nvfp4-gemm-current-c006","kernel-nvfp4-gemm-current-c007"]} +{"receipt_id":"rem-source-cutlass-2139-nvfp4-001","receipt_scope":"source-trail","path":"sources/prs/cutlass/PR-2139.md","original_body_sha256":"9caa079150cab8dc2e0af0e8e82d584ef6b2e6e58e6181356cf5d5e4357fe1a5","original_unit_ids":[],"locator":"frontmatter tags/hardware_features and Problem/Solution prose","original_claim_id":"source-cutlass-2139-nvfp4-scope","original_claim":"CUTLASS PR 2139 adds or directly supports NVFP4/FP4 block-scaled kernels.","original_verdict":"refuted","evidence":[{"path":"artifacts/kernels/nvfp4-gemm/full/PR-2139-blockwise-groupwise-gemm.patch","revision":"merge ca4fdbea; exact PR patch","locator":"changed examples 67, 68, and 81 and Element declarations","observation":"All introduced numeric examples are FP8 E4M3 blockwise/groupwise GEMMs; the patch contains no NVFP4 or E2M1 implementation."}],"strongest_alternative":"Generic internal block-scale infrastructure later used by FP4 makes the PR an NVFP4 source.","discriminating_result":"Potential downstream reuse does not make the introducing change an FP4 implementation; its user-facing examples and operands are all FP8.","action":"correct","action_reason":"Remove NVFP4/FP4 tags and claims while retaining verified FP8 Blackwell/Hopper scope.","proposed_replacement":"Describe PR 2139 as FP8 blockwise/groupwise CUTLASS work only.","replacement_claim_ids":[]} +{"receipt_id":"rem-source-nvfp4-gemm-relationships-001","receipt_scope":"source-trail","path":"wiki/kernels/nvfp4-gemm.md","original_body_sha256":"d176748702ac353341b6d4887c74d36b98deccce13fd42849ac545957b212fa0","original_unit_ids":[],"locator":"frontmatter:sources and artifact_dir","original_claim_id":"source-relationship-nvfp4-gemm-adjacent","original_claim":"The mutable CUTLASS summary, FP8-only PR 2139, and assembled artifact bundle directly evidence the contest NVFP4 GEMM implementation.","original_verdict":"refuted","evidence":[{"path":"sources/docs/nvidia-cutlass-blackwell.md","revision":"local editorial summary","locator":"overview, snippets, and performance data","observation":"The card is not a version-pinned task implementation and contains broad unsourced comparisons."},{"path":"artifacts/kernels/nvfp4-gemm/full/PR-2139-blockwise-groupwise-gemm.patch","revision":"CUTLASS ca4fdbea","locator":"numeric examples","observation":"The patch is FP8-only."},{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/task.yml","revision":"gpu-mode/reference-kernels ae679486","locator":"official task","observation":"The exact task source is independent and does not name those artifacts."}],"strongest_alternative":"Adjacent library material is useful implementation background.","discriminating_result":"Background utility does not create direct evidence or a conforming reference implementation; the page can link exact official examples at pinned revisions.","action":"correct","action_reason":"Use claim-directed task, format, PTX, and TMA source cards and detach the invalid bundle.","proposed_replacement":"sources: [contest-gpumode-p2, doc-transformer-engine-2.13-nvfp4, doc-ptx-isa-sm100, doc-cuda-13-0-2-tma]; remove artifact_dir.","replacement_claim_ids":["kernel-nvfp4-gemm-current-c001","kernel-nvfp4-gemm-current-c002","kernel-nvfp4-gemm-current-c005"]} +{"receipt_id":"rem-source-hw-nvfp4-cutlass-2139-001","receipt_scope":"source-trail","path":"wiki/hardware/nvfp4.md","original_body_sha256":"b655cfe0784f6e74204d4748a18fb4f06f420b6ddcae893e9b1e8596105d1c1e","original_unit_ids":[],"locator":"frontmatter:evidence_basis and sources","original_claim_id":"source-relationship-hw-nvfp4-pr2139","original_claim":"CUTLASS PR 2139 is direct upstream-code evidence for NVFP4.","original_verdict":"refuted","evidence":[{"path":"artifacts/kernels/nvfp4-gemm/full/PR-2139-blockwise-groupwise-gemm.patch","revision":"CUTLASS merge ca4fdbea","locator":"changed examples and element types","observation":"The exact patch introduces FP8 E4M3 blockwise/groupwise kernels and contains no NVFP4/E2M1 implementation."}],"strongest_alternative":"The PR's generic infrastructure is ancestral to later NVFP4 support.","discriminating_result":"Historical reuse is not direct evidence for the NVFP4 format or instruction claims already covered by Transformer Engine and PTX sources.","action":"delete","action_reason":"Remove the misleading evidence relationship while retaining the verified source-scoped hardware prose unchanged.","proposed_replacement":"Use only doc-transformer-engine-2.13-nvfp4 and doc-ptx-isa-sm100 in NVFP4 evidence_basis and sources.","replacement_claim_ids":["hw-nvfp4-current-c001","hw-nvfp4-current-c003"]} +{"receipt_id":"rem-kernel-nvfp4-gemv-001","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemv.md","original_body_sha256":"55d84be3408d0ab5d0adfed5c555998569f294e99d23cfc0c3fd582b32841d7d","original_unit_ids":["u001"],"locator":"body:L5","original_claim_id":"kernel-nvfp4-gemv-original-c002","original_claim":"Every FP4 element is used once and all such kernels need PTX cache control and register budgeting as critical levers.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/reference.py","revision":"gpu-mode/reference-kernels ae679486","locator":"ref_kernel and c_ref[:,0]","observation":"The one logical B row is reused across all M outputs."},{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"fetched 2026-08-08","locator":"attempts 9-12","observation":"Wider loads slowed the author's kernel and one register-limit change had zero effect."}],"strongest_alternative":"Only A was meant by each element and the levers were scoped to one implementation.","discriminating_result":"The text says each FP4 element and states universal criticality; B reuse and controlled no-gain cases contradict that scope.","action":"correct","action_reason":"Scope bottleneck and optimization statements to the official task model and measured implementations.","proposed_replacement":"Describe the task's DRAM-bound model, logical B reuse, and profiling-dependent choices.","replacement_claim_ids":["kernel-nvfp4-gemv-current-c001","kernel-nvfp4-gemv-current-c005"]} +{"receipt_id":"rem-kernel-nvfp4-gemv-002","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemv.md","original_body_sha256":"55d84be3408d0ab5d0adfed5c555998569f294e99d23cfc0c3fd582b32841d7d","original_unit_ids":["u002","u042"],"locator":"body:L7 and frontmatter:performance_claims","original_claim_id":"kernel-nvfp4-gemv-original-c004","original_claim":"Top performers scored about 22.4 microseconds and the value is one M=7168,K=16384,L=1 latency.","original_verdict":"refuted","evidence":[{"url":"https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemv/NVIDIA?limit=100","revision":"fetched 2026-08-08","locator":"ranks 1-3 and Yue row","observation":"Current rank 1 is 18.549562452; Yue's 22.392217755 is rank 11 and is an aggregate score."},{"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml","revision":"ae679486","locator":"ranking_by and benchmarks","observation":"Ranking is the geometric mean of three rows."}],"strongest_alternative":"22.4 was the cutoff-winning score and happened also to equal the largest-row latency.","discriminating_result":"No immutable cutoff result or per-row timing supports that; the direct API identifies the number as Yue's aggregate and current rank 11.","action":"correct","action_reason":"Remove malformed per-shape metadata and publish a dated aggregate snapshot with cutoff caveat.","proposed_replacement":"Record theoretical rows and identify Yue's author-reported/current aggregate without calling it a winning per-shape result.","replacement_claim_ids":["kernel-nvfp4-gemv-current-c001","kernel-nvfp4-gemv-current-c004"]} +{"receipt_id":"rem-kernel-nvfp4-gemv-003","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemv.md","original_body_sha256":"55d84be3408d0ab5d0adfed5c555998569f294e99d23cfc0c3fd582b32841d7d","original_unit_ids":["u003"],"locator":"body:L11-L30","original_claim_id":"kernel-nvfp4-gemv-original-c006","original_claim":"The task ABI has five tensors plus two FP32 global scales, with one physical B row and the displayed global-scale formula.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/task.py","revision":"gpu-mode/reference-kernels ae679486","locator":"input_t","observation":"The callable accepts seven tensors."},{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/reference.py","revision":"same commit","locator":"generate_input","observation":"It returns logical and reordered scale tensors, pads B/SFB to 128, and exposes no global scales."}],"strongest_alternative":"The five prose tensors and two globals are transformed into seven physical tensors by the harness.","discriminating_result":"The extra two tensors are reordered scale views, not global scalars, and neither the tuple nor reference formula consumes FP32 globals.","action":"correct","action_reason":"Publish both the logical operation and actual seven-tensor callable without inventing inputs.","proposed_replacement":"List packed A/B, logical SFA/SFB, reordered SFA/SFB, and C with the physical padding caveat.","replacement_claim_ids":["kernel-nvfp4-gemv-current-c002"]} +{"receipt_id":"rem-kernel-nvfp4-gemv-004","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemv.md","original_body_sha256":"55d84be3408d0ab5d0adfed5c555998569f294e99d23cfc0c3fd582b32841d7d","original_unit_ids":["u004","u005"],"locator":"body:L34-L48","original_claim_id":"kernel-nvfp4-gemv-original-c008","original_claim":"Raw PTX itself caused a critical gain over C intrinsics and mov.b32 alone is faster than shifts and masks.","original_verdict":"refuted","evidence":[{"url":"https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html","revision":"fetched 2026-08-08","locator":"Optimizations 2-3","observation":"Both reported performance steps bundled several changes; no isolated mov or C-versus-PTX trial appears."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov","revision":"PTX ISA 9.0","locator":"mov semantics","observation":"The ISA gives behavior, not a timing promise."}],"strongest_alternative":"The bundled stage was dominated entirely by mov and raw PTX.","discriminating_result":"The source explicitly lists multiple simultaneous changes and provides no ablation that selects that cause.","action":"correct","action_reason":"Retain exact instruction semantics and author-reported bundled progression without causal isolation.","proposed_replacement":"Show the valid cvt/mov forms and label the 39-to-27 result as a combined vectorized PTX-decode stage.","replacement_claim_ids":["kernel-nvfp4-gemv-current-c003","kernel-nvfp4-gemv-current-c004"]} +{"receipt_id":"rem-kernel-nvfp4-gemv-005","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemv.md","original_body_sha256":"55d84be3408d0ab5d0adfed5c555998569f294e99d23cfc0c3fd582b32841d7d","original_unit_ids":["u008","u009"],"locator":"body:L70-L88","original_claim_id":"kernel-nvfp4-gemv-original-c010","original_claim":"Ranks using 32 and 45 registers prove lower limits improve occupancy and dominate performance, with the displayed launch bounds.","original_verdict":"refuted","evidence":[{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"fetched 2026-08-08","locator":"attempt 11 and top-solution notes","observation":"The author reports the two limits only as observations; his own 80-to-64 change had zero effect, and no shown launch bounds or controlled rank comparison is published."}],"strongest_alternative":"The two rank limits establish a natural experiment.","discriminating_result":"Different unpublished kernels are not a controlled comparison, and realized register use can already lie below a cap.","action":"correct","action_reason":"Make register caps a measure-and-verify tuning parameter.","proposed_replacement":"State the author-reported budgets and the no-effect counterexample; require occupancy/spill measurement.","replacement_claim_ids":["kernel-nvfp4-gemv-current-c005"]} +{"receipt_id":"rem-kernel-nvfp4-gemv-006","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemv.md","original_body_sha256":"55d84be3408d0ab5d0adfed5c555998569f294e99d23cfc0c3fd582b32841d7d","original_unit_ids":["u011"],"locator":"body:L94-L136","original_claim_id":"kernel-nvfp4-gemv-original-c012","original_claim":"The displayed CUDA template represents the task ABI and valid per-K dispatch.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/template.py","revision":"gpu-mode/reference-kernels ae679486","locator":"custom_kernel","observation":"The official entry receives seven tensors and no global scales."},{"path":"wiki/kernels/nvfp4-gemv.md","revision":"baseline body 55d84be3","locator":"u011","observation":"The listing has undefined symbols, false globals, and dispatch K values unrelated to the official rows."}],"strongest_alternative":"The listing is intentionally schematic and need not compile.","discriminating_result":"It is presented as concrete specialization code and asserts a false ABI and specific dispatch values, so the semantic error remains even as pseudocode.","action":"delete","action_reason":"No primary contestant source supports this invented implementation.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-kernel-nvfp4-gemv-007","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemv.md","original_body_sha256":"55d84be3408d0ab5d0adfed5c555998569f294e99d23cfc0c3fd582b32841d7d","original_unit_ids":["u012","u013"],"locator":"body:L140-L152","original_claim_id":"kernel-nvfp4-gemv-original-c013","original_claim":"128- and 256-bit loads maximize bandwidth and work only with PTX byte unpacking.","original_verdict":"refuted","evidence":[{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"fetched 2026-08-08","locator":"attempt 9 and top-solution load discussion","observation":"A controlled wider-load attempt was 16-25 percent slower; top-solution observations do not establish a universal optimum or sole decode path."}],"strongest_alternative":"The failed uint2 path was only 64-bit and therefore does not bear on 128/256-bit PTX loads.","discriminating_result":"It directly disproves the page's unqualified monotonic maximize claim; no measured oracle establishes the asserted universal width or only-if condition.","action":"correct","action_reason":"Keep width semantics but remove universal performance guarantees.","proposed_replacement":"State the exact byte widths and present wider loads as an author-reported, implementation-dependent choice.","replacement_claim_ids":["kernel-nvfp4-gemv-current-c003","kernel-nvfp4-gemv-current-c005"]} +{"receipt_id":"rem-kernel-nvfp4-gemv-008","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemv.md","original_body_sha256":"55d84be3408d0ab5d0adfed5c555998569f294e99d23cfc0c3fd582b32841d7d","original_unit_ids":["u015"],"locator":"body:L158-L176","original_claim_id":"kernel-nvfp4-gemv-original-c016","original_claim":"The shown shared-memory B code is the rank-2 approach and cuts global traffic by exactly BLOCK_M.","original_verdict":"refuted","evidence":[{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"fetched 2026-08-08","locator":"rank-2 reuse paragraph","observation":"Only shared reads across rows are reported; neither this code nor an exact ratio is published."},{"url":"https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html","revision":"fetched 2026-08-08","locator":"failed CuTe ideas and Optimization 2","observation":"One shared-B attempt had no gain, and the faster CUDA stage removed B shared memory."}],"strongest_alternative":"Shared reads necessarily mean staging decoded B in shared memory and exact BLOCK_M amortization.","discriminating_result":"Cache transactions, scale loads, synchronization, and physical padding break that inference; the author evidence does not specify this mechanism.","action":"delete","action_reason":"Remove invented contestant code and exact traffic attribution while retaining logical B reuse.","proposed_replacement":null,"replacement_claim_ids":["kernel-nvfp4-gemv-current-c002","kernel-nvfp4-gemv-current-c005"]} +{"receipt_id":"rem-kernel-nvfp4-gemv-009","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemv.md","original_body_sha256":"55d84be3408d0ab5d0adfed5c555998569f294e99d23cfc0c3fd582b32841d7d","original_unit_ids":["u018"],"locator":"body:L185","original_claim_id":"kernel-nvfp4-gemv-original-c018","original_claim":"Coalescing alone caused the 443-to-39 microsecond result.","original_verdict":"refuted","evidence":[{"url":"https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html","revision":"fetched 2026-08-08","locator":"Optimization 1 and Optimization 2","observation":"Coalescing was part of the earlier 2000-to-443 stage; 443-to-39 bundled shared-memory removal, vector loads, per-thread work, and hardware intrinsics."}],"strongest_alternative":"The table's label was shorthand for all memory-access fixes.","discriminating_result":"It explicitly names coalesced access while the source assigns that work to a different stage and lists four changes for 443-to-39.","action":"correct","action_reason":"Restore the author's actual stage boundaries.","proposed_replacement":"Label 2000-to-443 and 443-to-39 with their complete combined change sets.","replacement_claim_ids":["kernel-nvfp4-gemv-current-c004"]} +{"receipt_id":"rem-kernel-nvfp4-gemv-010","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemv.md","original_body_sha256":"55d84be3408d0ab5d0adfed5c555998569f294e99d23cfc0c3fd582b32841d7d","original_unit_ids":["u023"],"locator":"body:L193","original_claim_id":"kernel-nvfp4-gemv-original-c019","original_claim":"Arithmetic optimization has minimal impact in memory-bound kernels.","original_verdict":"refuted","evidence":[{"url":"https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html","revision":"fetched 2026-08-08","locator":"Optimization 2 and PTX decode progression","observation":"NCU showed roughly ten times more issued instructions, and decode/instruction-path work accompanied large gains despite the task's bandwidth-bound model."}],"strongest_alternative":"Decode is part of memory handling rather than arithmetic.","discriminating_result":"The source explicitly identifies conversion, FMA, and instruction-count work; the categorical minimal-impact statement is too broad regardless of label.","action":"correct","action_reason":"Replace a universal rule with a profiling-dependent decision procedure.","proposed_replacement":"Treat bytes, instruction count, decode, reductions, occupancy, and cache behavior as measured candidates.","replacement_claim_ids":["kernel-nvfp4-gemv-current-c005"]} +{"receipt_id":"rem-kernel-nvfp4-gemv-011","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemv.md","original_body_sha256":"55d84be3408d0ab5d0adfed5c555998569f294e99d23cfc0c3fd582b32841d7d","original_unit_ids":["u024"],"locator":"body:L194","original_claim_id":"kernel-nvfp4-gemv-original-c020","original_claim":"The 443-to-27 result measures C intrinsics versus hand-written PTX.","original_verdict":"refuted","evidence":[{"url":"https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html","revision":"fetched 2026-08-08","locator":"Optimizations 2-3","observation":"The combined non-PTX stage ended at 39 and the vectorized PTX-decode stage ended near 27; 443-to-27 spans multiple simultaneous changes."}],"strongest_alternative":"All other changes are negligible so endpoints isolate PTX.","discriminating_result":"No ablation or repeated controlled benchmark establishes that; the author expressly presents combined stages.","action":"correct","action_reason":"Use source-reported combined-stage attribution only.","proposed_replacement":"Report 39-to-about-27 for the bundled vectorized PTX-decode step.","replacement_claim_ids":["kernel-nvfp4-gemv-current-c004"]} +{"receipt_id":"rem-kernel-nvfp4-gemv-012","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemv.md","original_body_sha256":"55d84be3408d0ab5d0adfed5c555998569f294e99d23cfc0c3fd582b32841d7d","original_unit_ids":["u026"],"locator":"body:L196","original_claim_id":"kernel-nvfp4-gemv-original-c022","original_claim":"Lower register limits reliably improve occupancy and memory-latency hiding.","original_verdict":"refuted","evidence":[{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"fetched 2026-08-08","locator":"attempt 11","observation":"Changing maxrregcount from 80 to 64 had zero effect because natural use was already below the cap."}],"strongest_alternative":"A more aggressive cap such as 32 would always help.","discriminating_result":"No controlled source proves that, and caps can add spills; the measured no-effect boundary disproves reliable monotonic improvement.","action":"correct","action_reason":"Make register caps conditional on realized use, spills, occupancy, and timing.","proposed_replacement":"Profile before and after each cap rather than treating lower as better.","replacement_claim_ids":["kernel-nvfp4-gemv-current-c005"]} +{"receipt_id":"rem-kernel-nvfp4-gemv-013","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemv.md","original_body_sha256":"55d84be3408d0ab5d0adfed5c555998569f294e99d23cfc0c3fd582b32841d7d","original_unit_ids":["u027"],"locator":"body:L197","original_claim_id":"kernel-nvfp4-gemv-original-c023","original_claim":"TMEM is irrelevant to memory-bound kernels and helps compute-bound kernels only.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory","revision":"PTX ISA 9.0","locator":"TMEM overview and tcgen05 data paths","observation":"TMEM is an explicitly managed architectural storage path, not gated by a whole-kernel bottleneck label."},{"url":"https://arxiv.org/html/2603.05451v1","revision":"FlashAttention-4 paper v1","locator":"section 3.1","observation":"A mixed MMA/softmax overlap pipeline uses separate TMEM regions, contradicting compute-bound-only scope."}],"strongest_alternative":"The statement was intended only for this CUDA-core contest implementation.","discriminating_result":"The text explicitly generalizes to memory-bound kernels and gives no task implementation oracle; the architectural category is false.","action":"delete","action_reason":"No useful universal TMEM recommendation is supported for this page.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-kernel-nvfp4-gemv-014","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemv.md","original_body_sha256":"55d84be3408d0ab5d0adfed5c555998569f294e99d23cfc0c3fd582b32841d7d","original_unit_ids":["u030"],"locator":"body:L203","original_claim_id":"kernel-nvfp4-gemv-original-c025","original_claim":"The pattern applies to memory-bound operations with arbitrary FP4 weights.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html","revision":"Transformer Engine 2.13","locator":"NVFP4 recipe and MXFP4 comparison","observation":"NVFP4 requires exact E2M1 payload and E4M3/global scale semantics; generic FP4 formats are not interchangeable."}],"strongest_alternative":"All FP4 weights can be losslessly reinterpreted into the task layout.","discriminating_result":"MXFP4 and other formats have different block sizes and scale representations, so reinterpretation changes values or violates the ABI.","action":"correct","action_reason":"Scope applicability to exact task-compatible NVFP4 operands.","proposed_replacement":"Require matching packed payload, block scales, physical layouts, output, and architecture.","replacement_claim_ids":["kernel-nvfp4-gemv-current-c006"]} +{"receipt_id":"rem-kernel-nvfp4-gemv-015","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemv.md","original_body_sha256":"55d84be3408d0ab5d0adfed5c555998569f294e99d23cfc0c3fd582b32841d7d","original_unit_ids":["u031"],"locator":"body:L207","original_claim_id":"kernel-nvfp4-gemv-original-c026","original_claim":"Native packed FP4 decode is limited to SM100 and SM100a only.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html","revision":"Transformer Engine 2.13","locator":"Supported devices","observation":"NVFP4 inference is listed for SM 10.0 and later."},{"url":"https://docs.nvidia.com/cuda/archive/12.9.2/parallel-thread-execution/index.html","revision":"PTX ISA 8.8","locator":"packed alternate-format cvt feature table","observation":"e2m1x2 cvt is listed for sm_100f, sm_101f, and sm_120f."}],"strongest_alternative":"The page meant only B200 contest deployment targets.","discriminating_result":"Its parenthetical asserts native-instruction exclusivity, while the archived ISA explicitly names additional targets.","action":"correct","action_reason":"Separate contest B200 scope from versioned instruction/library support.","proposed_replacement":"State B200 for this task and require the chosen PTX/library target to support the exact path.","replacement_claim_ids":["kernel-nvfp4-gemv-current-c006"]} +{"receipt_id":"rem-kernel-nvfp4-gemv-016","receipt_scope":"wiki","path":"wiki/kernels/nvfp4-gemv.md","original_body_sha256":"55d84be3408d0ab5d0adfed5c555998569f294e99d23cfc0c3fd582b32841d7d","original_unit_ids":["u039"],"locator":"body:L221","original_claim_id":"kernel-nvfp4-gemv-original-c032","original_claim":"The full artifact directory is a reference implementation of contest Problem 1.","original_verdict":"refuted","evidence":[{"path":"artifacts/kernels/nvfp4-gemv/full/nvfp4_scaled_mm_kernels.cu","revision":"vLLM ed7a29d9; SHA-256 e8aed5ccb3dd9de26c3aeff159a242a46dcb7c7d8d0351b6c44ff1f8d2f7effa","locator":"Fp4GemmSm100 and runGemm","observation":"The file is a general M,N,K CUTLASS GEMM adapter, not the seven-tensor N=1 contest callable."},{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/template.py","revision":"gpu-mode/reference-kernels ae679486","locator":"custom_kernel","observation":"The task entry has a different seven-tensor contract."}],"strongest_alternative":"A general GEMM with N=1 is a drop-in reference for the task.","discriminating_result":"The vLLM adapter lacks the task callable and its logical/reordered scale inputs, so it cannot be submitted to or checked by the official harness.","action":"delete","action_reason":"Detach the misleading artifact relationship while preserving the files and immutable receipt.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-source-nvfp4-gemv-contest-001","receipt_scope":"source-trail","path":"sources/contests/gpu-mode-nvfp4/problem-1-gemv.md","original_body_sha256":"825391e8315e37c040215e21d6f9df46892185a7fa1dab44e748ce1f64eac96a","original_unit_ids":[],"locator":"frontmatter submissions/artifact_dir and entire body","original_claim_id":"source-nvfp4-gemv-contest-record","original_claim":"The card invents ranks, techniques, a five-tensor physical ABI with global scales, universal optimization rules, and a contest artifact relationship.","original_verdict":"refuted","evidence":[{"url":"https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemv/NVIDIA?limit=100","revision":"fetched 2026-08-08","locator":"current ranks","observation":"Simon is current rank 25, Yue rank 11, and no Amandeep-named top row or implementation code is exposed."},{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/reference.py","revision":"gpu-mode/reference-kernels ae679486","locator":"generate_input","observation":"The actual callable has seven tensors, physical B padding, and no global scale arguments."},{"path":"artifacts/contests/gpu-mode-nvfp4/problem-1-gemv/submissions/rank-1-simon-veitner/PROVENANCE.yaml","revision":"local provenance","locator":"asset_mode and upstream_path","observation":"The files are extracted from a fabricated local source summary, not contestant submissions."}],"strongest_alternative":"The card preserves a private official cutoff leaderboard and Discord techniques.","discriminating_result":"No pinned primary record is provided, direct public records disagree, and derived local snippets cannot establish private claims.","action":"correct","action_reason":"Rewrite as a pinned official task record with dated leaderboard caveat and no contestant attribution.","proposed_replacement":"Document the exact seven-tensor harness, tests, benchmark model, scoring, rules, and current endpoint limits.","replacement_claim_ids":["kernel-nvfp4-gemv-current-c001","kernel-nvfp4-gemv-current-c002","kernel-nvfp4-gemv-current-c004"]} +{"receipt_id":"rem-source-nvfp4-gemv-yue-001","receipt_scope":"source-trail","path":"sources/blogs/yue-nvfp4-hackathon.md","original_body_sha256":"f4ff2cdd794b6090b17b9b6377b09095af9009a1a445841b642ba459a9178b0d","original_unit_ids":[],"locator":"frontmatter artifact_dir and entire body","original_claim_id":"source-nvfp4-gemv-yue-record","original_claim":"The card mislabels stage boundaries, isolates causal gains the author did not isolate, adds invented snippets, and treats derived extracts as source code.","original_verdict":"refuted","evidence":[{"url":"https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html","revision":"fetched 2026-08-08","locator":"CUDA Optimizations 1-6 and source-code note","observation":"The author gives combined 2000-to-443 and 443-to-39 stages, then 39-to-27, 26-to-22.9, and 22.9-to-22.3; code was still marked coming soon."},{"path":"artifacts/blogs/yue-nvfp4-hackathon/code/PROVENANCE.yaml","revision":"local provenance if present","locator":"asset mode/upstream path","observation":"The bundle is derived from the local summary rather than verbatim author source."}],"strongest_alternative":"The summary reconstructed equivalent illustrative code and simplified only presentation.","discriminating_result":"It assigns performance causality to different stages and presents source-unavailable reconstructions as the author's path, materially changing evidence.","action":"correct","action_reason":"Keep only faithful author-reported observations and remove the artifact relationship.","proposed_replacement":"Record exact combined stage labels, score scope, failed ideas, and source availability caveat.","replacement_claim_ids":["kernel-nvfp4-gemv-current-c003","kernel-nvfp4-gemv-current-c004"]} +{"receipt_id":"rem-source-nvfp4-gemv-amandeep-001","receipt_scope":"source-trail","path":"sources/blogs/amandeep-nvfp4-attempts.md","original_body_sha256":"7ae79b28e23c86c5919a104f3f72cfa4c476189b7237407cffcd42164be4dca5","original_unit_ids":[],"locator":"entire body","original_claim_id":"source-nvfp4-gemv-amandeep-record","original_claim":"The card fabricates timings and a monotonic twelve-step optimization journey ending in a 26.7 microsecond geometric mean.","original_verdict":"refuted","evidence":[{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"fetched 2026-08-08","locator":"attempt 7 table and failed attempts 8-12","observation":"26.7, 45.1, and 16.4 are three per-shape times; attempts 8-12 were worse or had zero effect rather than improving to 26.7."},{"path":"/tmp/kernelwiki-amandeepsp-cuda/nvfp4/gemv","revision":"amandeepsp/cuda 44513ac7d5bbd1cf8109cab952844adac5b6c551","locator":"attempt1.py through attempt12.py","observation":"The repository contains the attempt series described by the author, not the fabricated progression in the card."}],"strongest_alternative":"The local card summarizes a later revision of the blog.","discriminating_result":"The current author post and pinned code agree with each other and directly contradict the card's per-attempt values and outcomes.","action":"correct","action_reason":"Rewrite from the author post and pinned public code without inventing measurements.","proposed_replacement":"Publish attempt-7 per-row timings, failures for 8-12, retrospective top-solution observations, and profiling lesson.","replacement_claim_ids":["kernel-nvfp4-gemv-current-c003","kernel-nvfp4-gemv-current-c005"]} +{"receipt_id":"rem-source-nvfp4-gemv-simon-001","receipt_scope":"source-trail","path":"sources/blogs/simon-nvfp4-gemv.md","original_body_sha256":"c7fdcb3454bde58cb0f42ae752f4e59896236c30bbc688a2b343649674ad93bd","original_unit_ids":[],"locator":"frontmatter artifact_dir and entire body","original_claim_id":"source-nvfp4-gemv-simon-record","original_claim":"The card calls nanosecond timing outputs operations, invents CUDA C++ listings, attaches derived extracts, and implies a rank-1 contest relationship.","original_verdict":"refuted","evidence":[{"url":"https://veitner.bearblog.dev/nvfp4-gemv/","revision":"fetched 2026-08-08","locator":"reference implementation","observation":"The source is a CuTe DSL Python implementation."},{"url":"https://veitner.bearblog.dev/nvfp4-gemv-improved/","revision":"fetched 2026-08-08","locator":"benchmark outputs and three strategies","observation":"Values such as 234495.997 and 36864.001 are benchmark time outputs in nanoseconds, not operation counts; the code is CuTe DSL."},{"path":"artifacts/contests/gpu-mode-nvfp4/problem-1-gemv/submissions/rank-1-simon-veitner/PROVENANCE.yaml","revision":"local provenance","locator":"asset_mode and upstream_path","observation":"The alleged submission files were extracted from the local summary, have no upstream SHA, and are not author code."}],"strongest_alternative":"The C++ is a faithful translation and 'ops' is a harmless unit label.","discriminating_result":"Changing code language and measurement units makes source and performance claims false; the current leaderboard places Simon at rank 25, not rank 1.","action":"correct","action_reason":"Rewrite as a two-post CuTe DSL record and remove invalid code/artifact/rank relationships.","proposed_replacement":"Describe the reference and three K-parallel strategies with exact nanosecond outputs and author scope.","replacement_claim_ids":["kernel-nvfp4-gemv-current-c007"]} +{"receipt_id":"rem-lang-ptx-001","receipt_scope":"wiki","path":"wiki/languages/ptx-sm100.md","original_body_sha256":"7ef09348591994e947177b5ee3a4a04af46cd573163a183b47b016ff51d1fec9","original_unit_ids":["u002"],"locator":"body:L7-L31","original_claim_id":"lang-ptx-original-c003","original_claim":"tcgen05.fence::after_thread_sync is the critical fence between TMA completion and MMA.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence","revision":"PTX ISA 9.0","locator":"after_thread_sync and canonical handoffs","observation":"The fence orders subsequent tcgen05 operations across a thread handoff; it is not a completion wait."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"PTX ISA 9.0","locator":"mbarrier completion","observation":"TMA completion is delivered through complete_tx on an mbarrier."}],"strongest_alternative":"The fence is shorthand for the entire TMA-to-MMA handoff.","discriminating_result":"The displayed instruction cannot wait for TMA or acquire its completion; the required mechanisms have distinct grammar and semantics.","action":"correct","action_reason":"Separate TMA completion, async-proxy visibility, tcgen completion, and cross-thread tcgen ordering.","proposed_replacement":"Document mbarrier wait for TMA and tcgen05 commit/wait plus handoff fences where applicable.","replacement_claim_ids":["lang-ptx-current-c003"]} +{"receipt_id":"rem-lang-ptx-002","receipt_scope":"wiki","path":"wiki/languages/ptx-sm100.md","original_body_sha256":"7ef09348591994e947177b5ee3a4a04af46cd573163a183b47b016ff51d1fec9","original_unit_ids":["u003"],"locator":"body:L35-L41","original_claim_id":"lang-ptx-original-c005","original_claim":"mov.b32 byte decomposition is always faster than bitwise extraction.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov","revision":"PTX ISA 9.0","locator":"mov semantics","observation":"No relative performance guarantee is defined."},{"url":"https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html","revision":"fetched 2026-08-08","locator":"vectorized PTX decode","observation":"The reported timing step bundles multiple instructions and code changes."}],"strongest_alternative":"A single mov necessarily lowers instruction count versus any shift/mask compilation.","discriminating_result":"Compiler lowering and surrounding use are target-dependent, and no isolated benchmark proves the page's categorical claim.","action":"correct","action_reason":"Keep exact semantics and require generated-code/timing comparison for performance.","proposed_replacement":"Describe typed decomposition without claiming it is universally faster.","replacement_claim_ids":["lang-ptx-current-c004"]} +{"receipt_id":"rem-lang-ptx-003","receipt_scope":"wiki","path":"wiki/languages/ptx-sm100.md","original_body_sha256":"7ef09348591994e947177b5ee3a4a04af46cd573163a183b47b016ff51d1fec9","original_unit_ids":["u004"],"locator":"body:L45-L54","original_claim_id":"lang-ptx-original-c007","original_claim":"L1::no_allocate bypasses L1 and L1::evict_last keeps reused data resident.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#cache-operators","revision":"PTX ISA 9.0","locator":"eviction-priority semantics","observation":"They are performance hints that may not be respected and do not guarantee bypass or residency."}],"strongest_alternative":"The comments are accepted informal expansions of the qualifier names.","discriminating_result":"Informal wording changes a nonbinding eviction hint into a deterministic cache behavior, which is semantically false.","action":"correct","action_reason":"Use normative hint semantics and separate workload selection from instruction behavior.","proposed_replacement":"Label no_allocate and evict_last as L1 eviction-priority hints; profile the target access pattern.","replacement_claim_ids":["lang-ptx-current-c005"]} +{"receipt_id":"rem-lang-ptx-004","receipt_scope":"wiki","path":"wiki/languages/ptx-sm100.md","original_body_sha256":"7ef09348591994e947177b5ee3a4a04af46cd573163a183b47b016ff51d1fec9","original_unit_ids":["u005"],"locator":"body:L58-L62","original_claim_id":"lang-ptx-original-c008","original_claim":"clusterlaunchcontrol.try_cancel accepts one ClcID output and directly returns a tile or decline.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel","revision":"PTX ISA 9.0","locator":"try_cancel and query_cancel grammar","observation":"The request takes response/mbarrier addresses, completes asynchronously, and requires separate success and coordinate queries."}],"strongest_alternative":"The snippet is accepted pseudocode for the protocol.","discriminating_result":"It is inside a PTX code fence and uses a nonexistent operand form while omitting completion and response decoding, so it cannot represent the instruction semantics.","action":"correct","action_reason":"Replace with the exact request/wait/query sequence.","proposed_replacement":"Show b128 shared response, mbarrier completion, is_canceled, and get_first_ctaid forms.","replacement_claim_ids":["lang-ptx-current-c006"]} +{"receipt_id":"rem-lang-ptx-005","receipt_scope":"wiki","path":"wiki/languages/ptx-sm100.md","original_body_sha256":"7ef09348591994e947177b5ee3a4a04af46cd573163a183b47b016ff51d1fec9","original_unit_ids":["u006"],"locator":"body:L66-L74","original_claim_id":"lang-ptx-original-c010","original_claim":"The TMA multicast qualifier is .multicast.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"PTX ISA 9.0","locator":"cluster multicast grammar","observation":"The legal qualifier is .multicast::cluster and the form consumes a 16-bit CTA mask."}],"strongest_alternative":"The ::cluster suffix is optional because the destination already says shared::cluster.","discriminating_result":"The grammar enumerates only multicast::cluster; destination scope does not shorten the qualifier.","action":"correct","action_reason":"Use the exact version-pinned PTX spelling.","proposed_replacement":"Append .multicast::cluster and document the mask/barrier scope.","replacement_claim_ids":["lang-ptx-current-c007"]} +{"receipt_id":"rem-source-ptx-isa-sm100-001","receipt_scope":"source-trail","path":"sources/docs/nvidia-ptx-isa-sm100.md","original_body_sha256":"273e3d35e446406125b65ca5d1985b278e1acb6aca249fd13c8fadf9f356c82c","original_unit_ids":[],"locator":"frontmatter URL and entire body","original_claim_id":"source-ptx-isa-sm100-record","original_claim":"The rolling-doc card supplies incomplete or invented tcgen05, TMEM, CLC, conversion, cache, and synchronization grammar and broad performance claims.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"tcgen05, CLC, cvt, ld, and mbarrier sections","observation":"The normative grammar differs materially: allocation counts columns, MMA has more operands, CLC uses try_cancel/query_cancel, packed conversions are typed, cache controls are hints, and completion mechanisms are distinct."}],"strongest_alternative":"The card intentionally simplifies the ISA and its snippets are schematic.","discriminating_result":"It labels nonexistent mnemonics and operand forms as syntax, changes units and directions, and makes categorical performance claims; those are factual errors rather than omitted detail.","action":"correct","action_reason":"Rewrite as a version-pinned routing card that preserves exact representative grammar and semantic caveats.","proposed_replacement":"Use CUDA 13.0.2/PTX ISA 9.0 anchors, exact instruction forms, and direct links to the already verified hardware pages.","replacement_claim_ids":["lang-ptx-current-c001","lang-ptx-current-c002","lang-ptx-current-c003","lang-ptx-current-c004","lang-ptx-current-c005","lang-ptx-current-c006","lang-ptx-current-c007"]} +{"receipt_id":"rem-lang-cuda-cpp-001","receipt_scope":"wiki","path":"wiki/languages/cuda-cpp.md","original_body_sha256":"7a58194cb413f25af6db71a4e0d90828273f461ee66232063fb6e6a96176d836","original_unit_ids":["u002"],"locator":"body:L7-L20","original_claim_id":"lang-cuda-cpp-original-c002","original_claim":"tcgen05.alloc can be issued by thread 0 alone.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"PTX ISA 9.0","locator":"cta_group::1 issue granularity","observation":"All lanes of one warp must execute the same sync.aligned allocation; otherwise behavior is undefined."},{"url":"https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu","revision":"3b90ac9b","locator":"allocation branch","observation":"The pinned source executes alloc in every lane of warp 1."}],"strongest_alternative":"Lane 0 issues while the other lanes implicitly participate because of .sync.","discriminating_result":"Predicated non-executing lanes do not execute the collective instruction; the ISA explicitly makes that undefined.","action":"correct","action_reason":"Show whole-warp allocation and reserve lane election for single-thread MMA/TMA issue.","proposed_replacement":"Execute alloc/dealloc from all lanes of one designated warp and synchronize before consumers read the taddr.","replacement_claim_ids":["lang-cuda-cpp-current-c002"]} +{"receipt_id":"rem-lang-cuda-cpp-002","receipt_scope":"wiki","path":"wiki/languages/cuda-cpp.md","original_body_sha256":"7a58194cb413f25af6db71a4e0d90828273f461ee66232063fb6e6a96176d836","original_unit_ids":["u002"],"locator":"body:L23-L36","original_claim_id":"lang-cuda-cpp-original-c003","original_claim":"Unscaled f16 MMA takes separate immediate descriptors for accumulator C and output D.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/common.h","revision":"gau-nernst/learn-cuda 3b90ac9b","locator":"tcgen05_mma_f16","observation":"The wrapper takes one instruction descriptor and converts an enable-input-D integer to a predicate."},{"path":"/tmp/kernelwiki-ptx-1302.html","revision":"PTX ISA 9.0","locator":"unscaled f16 grammar","observation":"The final required operands are idesc and enable-input-D predicate, not C and D descriptors."}],"strongest_alternative":"The helper names are merely local aliases for idesc and predicate bits.","discriminating_result":"Both are constrained as u32 registers and passed directly; a predicate operand cannot be supplied by an r constraint, and the comments assert nonexistent semantics.","action":"correct","action_reason":"Use one idesc plus a local predicate constructed from a CUDA integer.","proposed_replacement":"Show the exact pinned wrapper pattern with .reg .pred and setp.ne.b32.","replacement_claim_ids":["lang-cuda-cpp-current-c003"]} +{"receipt_id":"rem-lang-cuda-cpp-003","receipt_scope":"wiki","path":"wiki/languages/cuda-cpp.md","original_body_sha256":"7a58194cb413f25af6db71a4e0d90828273f461ee66232063fb6e6a96176d836","original_unit_ids":["u003"],"locator":"body:L55-L78","original_claim_id":"lang-cuda-cpp-original-c006","original_claim":"The two short mbarrier helpers alone provide safe complete TMA-to-MMA synchronization.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/inline-ptx-assembly/index.html#incorrect-optimization","revision":"CUDA 13.0.2 Inline PTX guide","locator":"memory clobber pitfall","observation":"Hidden memory effects and compiler memory-order constraints require a memory clobber; the helpers omit it."},{"url":"https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu","revision":"3b90ac9b","locator":"barrier lifecycle and pipeline","observation":"The full source initializes/publishes barriers, tracks multiple phases, and implements both full and empty ownership handoffs."}],"strongest_alternative":"Initialization and empty handoff are naturally supplied by callers, so the helpers are complete primitives.","discriminating_result":"The page labels them TMA-MMA synchronization without documenting those mandatory preconditions, and compiler-visible ordering is missing from the wrappers themselves.","action":"correct","action_reason":"Publish the full lifecycle invariants and memory-clobber requirement rather than a misleading two-function protocol.","proposed_replacement":"Show initialization/publication, expect_tx, acquire wait, phase tracking, tcgen completion, and stage-reuse ownership as distinct steps.","replacement_claim_ids":["lang-cuda-cpp-current-c004"]} +{"receipt_id":"rem-source-tcgen05-tutorial-002","receipt_scope":"source-trail","path":"sources/blogs/tcgen05-tutorial.md","original_body_sha256":"c426867ecc38ff4465e2bf8eb8acfce0f5280c9abed19e433d3a6ceaffddfda4","original_unit_ids":[],"locator":"Scope and provenance; References/Pinned source tree","original_claim_id":"source-tcgen05-tutorial-pinned-tree","original_claim":"The pinned tutorial source tree is the repository's tcgen05 directory at commit 3b90ac9b.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-learn-cuda-3b90ac9b","revision":"gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"root tree and 02e_matmul_sm100/README.md","observation":"The commit has no tcgen05 directory; its tcgen05 tutorial code and benchmark table are under 02e_matmul_sm100."}],"strongest_alternative":"GitHub redirects the former tcgen05 directory to the renamed path.","discriminating_result":"The exact pinned tcgen05 tree URL returns 404, while the pinned 02e_matmul_sm100 tree returns 200 and contains the referenced implementation.","action":"correct","action_reason":"Point the evidence card at the actual immutable source directory.","proposed_replacement":"Replace the tcgen05 tree URL with the full-commit 02e_matmul_sm100 tree URL and name that directory in provenance.","replacement_claim_ids":["lang-cuda-cpp-current-c001","lang-cuda-cpp-current-c003","lang-cuda-cpp-current-c004","lang-cuda-cpp-current-c006","lang-cuda-cpp-current-c007"]} +{"receipt_id":"rem-lang-cute-dsl-001","receipt_scope":"wiki","path":"wiki/languages/cute-dsl.md","original_body_sha256":"4e09e1f2220eed702b5a7c8e6d7318de541e65faf5660835318c6916f5f1625b","original_unit_ids":["u001"],"locator":"body:L3-L3","original_claim_id":"lang-cute-dsl-original-c002","original_claim":"CuTe DSL is the primary abstraction layer in CUTLASS 4.5.0 for Blackwell kernels.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0/README.md","revision":"NVIDIA/cutlass v4.5.0 e406c186","locator":"Overview and repository organization","observation":"The exact release presents C++ template abstractions and Python DSLs as parallel interfaces and does not designate CuTe DSL as the primary Blackwell layer."}],"strongest_alternative":"Primary informally means the preferred new authoring interface rather than the only interface.","discriminating_result":"The release contains and documents extensive SM100 C++ and DSL surfaces without that hierarchy, so the categorical label overstates the source.","action":"correct","action_reason":"Describe CuTe DSL as a Python-native CUTLASS interface without assigning unsupported primacy.","proposed_replacement":"Scope the page to the CuTe DSL interface in CUTLASS v4.5.0 alongside CUTLASS C++.","replacement_claim_ids":["lang-cute-dsl-current-c001"]} +{"receipt_id":"rem-lang-cute-dsl-002","receipt_scope":"wiki","path":"wiki/languages/cute-dsl.md","original_body_sha256":"4e09e1f2220eed702b5a7c8e6d7318de541e65faf5660835318c6916f5f1625b","original_unit_ids":["u002"],"locator":"body:L7-L17","original_claim_id":"lang-cute-dsl-original-c004","original_claim":"SM100_MMA_F16BF16_SS and SM100_MMA_F16BF16_SS_2SM are CUTLASS 4.5.0 CuTe DSL Python MMA atoms.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0/python/CuTeDSL/cutlass/cute/nvgpu/tcgen05/mma.py","revision":"NVIDIA/cutlass v4.5.0 e406c186","locator":"MmaF16BF16Op and CtaGroup","observation":"The Python API uses MmaF16BF16Op configured with CtaGroup.ONE or TWO; the displayed symbols do not exist."}],"strongest_alternative":"The names are re-exported convenience aliases outside mma.py.","discriminating_result":"A full tagged-tree symbol search finds neither alias, while the official tutorial constructs MmaF16BF16Op directly.","action":"correct","action_reason":"Replace invented identifiers with the exact version-pinned constructor.","proposed_replacement":"Show MmaF16BF16Op plus make_tiled_mma and explain the one-/two-CTA shape switch.","replacement_claim_ids":["lang-cute-dsl-current-c003"]} +{"receipt_id":"rem-lang-cute-dsl-003","receipt_scope":"wiki","path":"wiki/languages/cute-dsl.md","original_body_sha256":"4e09e1f2220eed702b5a7c8e6d7318de541e65faf5660835318c6916f5f1625b","original_unit_ids":["u003"],"locator":"body:L21-L30","original_claim_id":"lang-cute-dsl-original-c006","original_claim":"The make_tmem_ptr/make_tensor/copy fragment is a usable CUTLASS 4.5.0 CuTe DSL TMEM allocation and epilogue example.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py","revision":"NVIDIA/cutlass v4.5.0 e406c186","locator":"TmemAllocator, TMEM pointer, copy partition, and free","observation":"The official path requires TmemAllocator lifecycle and a typed tcgen05 copy atom/partition; make_tmem_ptr is absent."}],"strongest_alternative":"The fragment uses low-level aliases that are omitted from the tutorial.","discriminating_result":"The tag has no make_tmem_ptr symbol and its public allocation helper has materially different lifecycle requirements.","action":"correct","action_reason":"Use an evidence-scoped excerpt that preserves allocation, copy partition, wait, and free responsibilities.","proposed_replacement":"Route readers through TmemAllocator and tcgen05.make_tmem_copy in the complete tagged example.","replacement_claim_ids":["lang-cute-dsl-current-c004"]} +{"receipt_id":"rem-lang-cute-dsl-004","receipt_scope":"wiki","path":"wiki/languages/cute-dsl.md","original_body_sha256":"4e09e1f2220eed702b5a7c8e6d7318de541e65faf5660835318c6916f5f1625b","original_unit_ids":["u004"],"locator":"body:L34-L46","original_claim_id":"lang-cute-dsl-original-c007","original_claim":"SM100_TMA_LOAD_2D and the displayed make_tma_copy signature are CUTLASS 4.5.0 CuTe DSL TMA APIs.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py","revision":"NVIDIA/cutlass v4.5.0 e406c186","locator":"TMA operation, tiled atom construction, and partition","observation":"The tagged Python example uses CopyBulkTensorTileG2SOp, make_tiled_tma_atom_A/B, tma_partition, and cute.copy; the displayed API does not occur."}],"strongest_alternative":"The shorter identifiers are documented aliases in another package module.","discriminating_result":"A full tagged-tree search rejects both displayed symbols and finds the exact official construction sequence.","action":"correct","action_reason":"Replace fabricated shorthand with the release's typed TMA atom and partitioning API.","proposed_replacement":"Show CopyBulkTensorTileG2SOp and make_tiled_tma_atom_A, with full partition/copy delegated to the official example.","replacement_claim_ids":["lang-cute-dsl-current-c005"]} +{"receipt_id":"rem-lang-cute-dsl-005","receipt_scope":"wiki","path":"wiki/languages/cute-dsl.md","original_body_sha256":"4e09e1f2220eed702b5a7c8e6d7318de541e65faf5660835318c6916f5f1625b","original_unit_ids":["u005"],"locator":"body:L50-L75","original_claim_id":"lang-cute-dsl-original-c008","original_claim":"The @cute_kernel warp-role skeleton is a safe usable CuTe DSL kernel skeleton with correct TMEM and pipeline ownership.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py","revision":"NVIDIA/cutlass v4.5.0 e406c186","locator":"kernel decorators, participant groups, role dispatch, pipelines, and TMEM teardown","observation":"The displayed decorator/helpers are absent and the real kernel requires explicit full/empty states, collective TMEM lifecycle, and pipeline tails."}],"strongest_alternative":"The block is unmistakable pseudocode and therefore need not preserve API or lifecycle constraints.","discriminating_result":"It is presented under a language page without a schematic warning and contains a deallocation path that materially contradicts the collective source lifecycle.","action":"correct","action_reason":"Replace unsafe pseudocode with verified role/lifetime invariants and a link to the full implementation.","proposed_replacement":"Describe the actual fp16_gemm_2 role split and required PipelineTmaUmma/PipelineUmmaAsync ownership transitions.","replacement_claim_ids":["lang-cute-dsl-current-c006"]} +{"receipt_id":"rem-lang-cute-dsl-006","receipt_scope":"wiki","path":"wiki/languages/cute-dsl.md","original_body_sha256":"4e09e1f2220eed702b5a7c8e6d7318de541e65faf5660835318c6916f5f1625b","original_unit_ids":["u008"],"locator":"body:L81-L81","original_claim_id":"lang-cute-dsl-original-c011","original_claim":"FlashAttention-4's 1605-TFLOP/s number establishes that CuTe DSL has the same performance as handwritten C++.","original_verdict":"refuted","evidence":[{"url":"https://tridao.me/blog/2026/flash4/","revision":"first-party FA4 post fetched 2026-08-08","locator":"Language/framework and benchmark comparators","observation":"The runtime comparisons are FA2, Triton, Gluon, and cuDNN; 1605 TFLOP/s is FA4's own reported peak and not a matched C++ comparator."}],"strongest_alternative":"The post's no-performance-compromise language implicitly supplies the C++ equality comparison.","discriminating_result":"No hand-written C++ FA4 row, shape-matched measurement, or equality statistic is reported, so the number cannot evidence that comparison.","action":"correct","action_reason":"Preserve the author-reported maximum and remove the unsupported comparator.","proposed_replacement":"State up to 1605 TFLOP/s on B200 BF16 as a source-reported FA4 maximum only.","replacement_claim_ids":["lang-cute-dsl-current-c002"]} +{"receipt_id":"rem-lang-cute-dsl-007","receipt_scope":"wiki","path":"wiki/languages/cute-dsl.md","original_body_sha256":"4e09e1f2220eed702b5a7c8e6d7318de541e65faf5660835318c6916f5f1625b","original_unit_ids":["u010"],"locator":"body:L83-L83","original_claim_id":"lang-cute-dsl-original-c013","original_claim":"CuTe DSL automatically computes layouts and handles swizzles without explicit user configuration.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py","revision":"NVIDIA/cutlass v4.5.0 e406c186","locator":"MMA construction, make_smem_layout_a/b, and allocate_tensor","observation":"The library provides typed layout algebra and helpers, but the user still supplies tilers, major modes, datatypes, alignment, layouts, and swizzle objects."}],"strongest_alternative":"Helper-generated layouts make the overall process automatic in ordinary examples.","discriminating_result":"Helpers derive some layouts from explicit inputs; they do not choose all layout/swizzle policy or remove explicit configuration.","action":"correct","action_reason":"Narrow the benefit to typed composition and version-pinned Blackwell helpers.","proposed_replacement":"Describe helper-assisted layout construction while retaining explicit policy inputs.","replacement_claim_ids":["lang-cute-dsl-current-c007"]} +{"receipt_id":"rem-source-cutlass-blackwell-001","receipt_scope":"source-trail","path":"sources/docs/nvidia-cutlass-blackwell.md","original_body_sha256":"714ab223b4f27e6857f92cce7e16279e05fdcc4a83853cd86da13564ec51fce8","original_unit_ids":[],"locator":"entire source summary","original_claim_id":"source-cutlass-blackwell-record","original_claim":"The rolling changelog card provides reliable CUTLASS 4.x Blackwell APIs, schedules, attention behavior, epilogue symbols, and performance values as written.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"tagged C++/CuTe DSL source and examples","observation":"Several displayed types and schedules do not occur in the tag, while performance percentages lack reproducible rows; rolling documentation also cannot establish 4.5.0 semantics."}],"strongest_alternative":"The card intentionally merges symbols from later 4.x releases and uses schematic pseudocode.","discriminating_result":"It labels the snippets and quantitative results as components/performance without version qualifiers; exact tag search rejects multiple names and provides no rows for the percentages.","action":"correct","action_reason":"Replace speculative summaries with an exact v4.5.0 routing card and directly verified Python API loci.","proposed_replacement":"Pin URL and revision to v4.5.0, list supported concepts and official tutorial paths, and remove unverified performance and invented API details.","replacement_claim_ids":["lang-cute-dsl-current-c001","lang-cute-dsl-current-c003","lang-cute-dsl-current-c004","lang-cute-dsl-current-c005","lang-cute-dsl-current-c006","lang-cute-dsl-current-c007"]} +{"receipt_id":"rem-migration-register-to-tmem-001","receipt_scope":"wiki","path":"wiki/migration/register-to-tmem.md","original_body_sha256":"d18a8529e0c9cab4d1711c15938892a04771dbccb5548fc61e5298b0fd6c0ed4","original_unit_ids":["u002"],"locator":"body:L7-L7","original_claim_id":"migration-register-to-tmem-original-c003","original_claim":"TMEM eliminates accumulator register pressure entirely, frees the full register file, necessarily enables larger tiles, and removes registers as an occupancy bottleneck.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2","locator":"occupancy resource limits","observation":"Blackwell retains finite per-thread/per-SM registers and occupancy depends on all compiled resources; TMEM readback and other state still consume registers."}],"strongest_alternative":"Entirely refers only to the resident D fragment, not every live value.","discriminating_result":"The sentence extends the claim to the epilogue, tile size, and occupancy, where that narrow interpretation no longer holds.","action":"correct","action_reason":"Keep the resident-D benefit while preserving resource-dependent limits.","proposed_replacement":"State that TMEM removes the resident D vector from the register file but does not guarantee occupancy, tile size, or spill behavior.","replacement_claim_ids":["migration-register-to-tmem-current-c002"]} +{"receipt_id":"rem-migration-register-to-tmem-002","receipt_scope":"wiki","path":"wiki/migration/register-to-tmem.md","original_body_sha256":"d18a8529e0c9cab4d1711c15938892a04771dbccb5548fc61e5298b0fd6c0ed4","original_unit_ids":["u004"],"locator":"body:L15-L32","original_claim_id":"migration-register-to-tmem-original-c005","original_claim":"The 174-register Hopper budget and exactly two-warpgroups/two-CTAs occupancy result are general for m64n256k16 BF16.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16","revision":"PTX ISA 9.0","locator":"A and D fragments","observation":"Only D=128 f32 and A=four f16x2 registers are specified; the page's other category counts are invented."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator","revision":"CUDA 13.0.2","locator":"occupancy","observation":"CTA residency depends on full compiled resources and a warpgroup is not inherently one CTA."}],"strongest_alternative":"The table is explicitly one representative typical GEMM rather than an architecture constant.","discriminating_result":"It supplies no concrete source or compiled kernel, labels arbitrary values as a budget, and derives a universal CTA count.","action":"delete","action_reason":"No pinned kernel/resource report supports the invented categories.","proposed_replacement":"Retain only the ISA-defined 128-register D fragment and require compiler resource output for a concrete occupancy claim.","replacement_claim_ids":["migration-register-to-tmem-current-c001","migration-register-to-tmem-current-c002"]} +{"receipt_id":"rem-migration-register-to-tmem-003","receipt_scope":"wiki","path":"wiki/migration/register-to-tmem.md","original_body_sha256":"d18a8529e0c9cab4d1711c15938892a04771dbccb5548fc61e5298b0fd6c0ed4","original_unit_ids":["u005","u006"],"locator":"body:L34-L43","original_claim_id":"migration-register-to-tmem-original-c006","original_claim":"Changing to k32 and double-buffering necessarily produces 300 registers per thread and exactly one CTA per SM.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/hopper-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2","locator":"255-register maximum","observation":"300 exceeds the architectural per-thread maximum and actual allocation/spilling is compiler-dependent."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16","revision":"PTX ISA 9.0","locator":"D fragment","observation":"D count depends on N/dtype rather than K."}],"strongest_alternative":"300 is a pre-spill live-value estimate rather than allocated registers.","discriminating_result":"Even as an estimate it cannot imply the stated allocation or occupancy without compiled evidence, and k32 does not double D.","action":"delete","action_reason":"The numeric scenario is not source- or experiment-grounded.","proposed_replacement":"Explain that simultaneous live D fragments multiply their defined fragment cost and must be checked with compiler resource data.","replacement_claim_ids":["migration-register-to-tmem-current-c002"]} +{"receipt_id":"rem-migration-register-to-tmem-004","receipt_scope":"wiki","path":"wiki/migration/register-to-tmem.md","original_body_sha256":"d18a8529e0c9cab4d1711c15938892a04771dbccb5548fc61e5298b0fd6c0ed4","original_unit_ids":["u008"],"locator":"body:L49-L56","original_claim_id":"migration-register-to-tmem-original-c008","original_claim":"Register/L1/L2 accesses universally cost 4/30/200 cycles and each spilled accumulator adds 200 times K_tiles cycles.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-best-practices-guide/index.html#local-memory","revision":"CUDA 13.0.2","locator":"local-memory performance","observation":"Official guidance defines no fixed tuple; cache hits, dependencies, coalescing, occupancy and overlap determine cost."}],"strongest_alternative":"The values are rough H100 rules of thumb.","discriminating_result":"No environment or microbenchmark is given, and the exact additive formula ignores hiding and cache behavior.","action":"delete","action_reason":"Unsupported latency constants would mislead migration decisions.","proposed_replacement":"Describe spills as off-chip local-memory traffic whose cost must be measured for the concrete kernel.","replacement_claim_ids":["migration-register-to-tmem-current-c002"]} +{"receipt_id":"rem-migration-register-to-tmem-005","receipt_scope":"wiki","path":"wiki/migration/register-to-tmem.md","original_body_sha256":"d18a8529e0c9cab4d1711c15938892a04771dbccb5548fc61e5298b0fd6c0ed4","original_unit_ids":["u009","u010"],"locator":"body:L62-L82","original_claim_id":"migration-register-to-tmem-original-c009","original_claim":"The 29-register Blackwell budget and eight-CTA register/two-CTA TMEM occupancy limits are general for m128n256k16.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2","locator":"occupancy limits","observation":"Actual block size, allocated registers, shared memory, clusters and other resources determine residency; the listed categories are not ISA constants."}],"strongest_alternative":"The table is a didactic estimate for one conventional kernel.","discriminating_result":"No kernel, compiler report, or launch configuration is named, so neither the 29-register total nor binding resource is established.","action":"delete","action_reason":"Replace speculative arithmetic with a measurement workflow.","proposed_replacement":"Use ptxas/Nsight resource data and the occupancy API for each concrete migration.","replacement_claim_ids":["migration-register-to-tmem-current-c002"]} +{"receipt_id":"rem-migration-register-to-tmem-006","receipt_scope":"wiki","path":"wiki/migration/register-to-tmem.md","original_body_sha256":"d18a8529e0c9cab4d1711c15938892a04771dbccb5548fc61e5298b0fd6c0ed4","original_unit_ids":["u011"],"locator":"body:L88-L128","original_claim_id":"migration-register-to-tmem-original-c011","original_claim":"The Hopper C++ pseudocode proves its arrays stay in registers and that no registers remain for epilogue temporaries.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#local-memory","revision":"CUDA 13.0.2","locator":"automatic arrays","observation":"Compiler placement of arrays can become local memory, and headroom requires a compiled whole-kernel report."}],"strongest_alternative":"The array only represents the PTX D operand rather than literal C++ allocation.","discriminating_result":"The code makes compiler-placement and no-headroom assertions about C++ variables beyond the verified PTX fragment.","action":"correct","action_reason":"Separate the PTX register-D contract from compiler allocation conclusions.","proposed_replacement":"Show only the N/2 register-fragment calculation and require resource inspection.","replacement_claim_ids":["migration-register-to-tmem-current-c001","migration-register-to-tmem-current-c003"]} +{"receipt_id":"rem-migration-register-to-tmem-007","receipt_scope":"wiki","path":"wiki/migration/register-to-tmem.md","original_body_sha256":"d18a8529e0c9cab4d1711c15938892a04771dbccb5548fc61e5298b0fd6c0ed4","original_unit_ids":["u012"],"locator":"body:L132-L210","original_claim_id":"migration-register-to-tmem-original-c012","original_claim":"The Blackwell block correctly uses lane-0 allocation/deallocation, 32x1b f32 transfers, a literal predicate, no completion wait, and a fence/barrier read handoff.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"PTX ISA 9.0","locator":"collective alloc/dealloc","observation":"All lanes of one warp must participate."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"PTX ISA 9.0","locator":"shape grammar/completion","observation":"32x1b is not legal and ld/st require collective execution and waits."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit","revision":"PTX ISA 9.0","locator":"MMA completion","observation":"Fence does not replace asynchronous completion tracking."}],"strongest_alternative":"The code is schematic despite its literal PTX strings.","discriminating_result":"It presents nonexistent instruction forms and undefined participation as executable CUDA, so schematic intent cannot make it safe.","action":"correct","action_reason":"Replace the block with lifecycle invariants and known-good source routes.","proposed_replacement":"Document collective allocation, predicate-controlled initialization, commit/wait, collective ld/wait, reader sync, and deallocation.","replacement_claim_ids":["migration-register-to-tmem-current-c004","migration-register-to-tmem-current-c005"]} +{"receipt_id":"rem-migration-register-to-tmem-008","receipt_scope":"wiki","path":"wiki/migration/register-to-tmem.md","original_body_sha256":"d18a8529e0c9cab4d1711c15938892a04771dbccb5548fc61e5298b0fd6c0ed4","original_unit_ids":["u013","u014"],"locator":"body:L216-L227","original_claim_id":"migration-register-to-tmem-original-c013","original_claim":"TMEM removes register constraints on tile size; Hopper m128n256 necessarily spills and Blackwell m128n512 has a universal one-tile fit.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"PTX ISA 9.0","locator":"shape table","observation":"Common tcgen05 instructions have N at most 256; larger CTA tiles require decomposition."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2","locator":"resource limits","observation":"Registers remain finite and practical tile size depends on the entire kernel."}],"strongest_alternative":"The statements refer to composed CTA tiles rather than one instruction.","discriminating_result":"Even composed tiles have workload- and resource-dependent spills/fit; the universal claims remain unsupported.","action":"correct","action_reason":"Replace categorical tile advice with descriptor legality plus empirical tuning.","proposed_replacement":"Re-tune shapes using legal tcgen descriptors, TMEM columns, SMEM, registers, clusters and measured performance.","replacement_claim_ids":["migration-register-to-tmem-current-c002","migration-register-to-tmem-current-c008"]} +{"receipt_id":"rem-migration-register-to-tmem-009","receipt_scope":"wiki","path":"wiki/migration/register-to-tmem.md","original_body_sha256":"d18a8529e0c9cab4d1711c15938892a04771dbccb5548fc61e5298b0fd6c0ed4","original_unit_ids":["u015","u016"],"locator":"body:L231-L277","original_claim_id":"migration-register-to-tmem-original-c014","original_claim":"Blackwell gives the epilogue the full register file and the rich pseudocode necessarily uses about 24 registers without spilling.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2","locator":"register limits","observation":"All live values share finite registers and spilling is a compiler/kernel outcome; pseudocode cannot prove an allocation count."}],"strongest_alternative":"Full means substantially more headroom after D moves to TMEM.","discriminating_result":"The page attaches exact no-spill and 24-register conclusions that require a concrete compiled kernel.","action":"correct","action_reason":"Preserve the design opportunity without promising allocation or performance.","proposed_replacement":"Read TMEM in bounded batches and measure compiled registers/spills while tuning epilogue width.","replacement_claim_ids":["migration-register-to-tmem-current-c002","migration-register-to-tmem-current-c005"]} +{"receipt_id":"rem-migration-register-to-tmem-010","receipt_scope":"wiki","path":"wiki/migration/register-to-tmem.md","original_body_sha256":"d18a8529e0c9cab4d1711c15938892a04771dbccb5548fc61e5298b0fd6c0ed4","original_unit_ids":["u017","u018"],"locator":"body:L281-L319","original_claim_id":"migration-register-to-tmem-original-c015","original_claim":"MMA/epilogue overlap is impossible with register accumulators and the two-TMEM-allocation sketch safely implements it.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/abs/2407.08608","revision":"FlashAttention-3","locator":"Hopper warp specialization/interleaving","observation":"FA3 overlaps Hopper tensor-core and non-matmul work despite register accumulators."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit","revision":"PTX ISA 9.0","locator":"completion/lifetimes","observation":"The opaque sketch lacks the required completion and ownership proof."}],"strongest_alternative":"Impossible only means impossible within the same thread's live accumulator registers.","discriminating_result":"The page states a kernel-wide impossibility and presents a kernel-wide alternative, so warp specialization is a direct counterexample.","action":"correct","action_reason":"Describe overlap as a scheduling option requiring disjoint regions and explicit handoffs.","proposed_replacement":"Use separate TMEM regions inside a legal allocation and prove compute-complete/read-complete/reuse edges.","replacement_claim_ids":["migration-register-to-tmem-current-c006"]} +{"receipt_id":"rem-migration-register-to-tmem-011","receipt_scope":"wiki","path":"wiki/migration/register-to-tmem.md","original_body_sha256":"d18a8529e0c9cab4d1711c15938892a04771dbccb5548fc61e5298b0fd6c0ed4","original_unit_ids":["u020","u021"],"locator":"body:L327-L362","original_claim_id":"migration-register-to-tmem-original-c017","original_claim":"FA Hopper/Blackwell use about 254/20 registers per thread with exact two-/236-register margins and forced spill/no-spill outcomes.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/abs/2603.05451v1","revision":"FA4 paper v1","locator":"kernel architecture/evaluation","observation":"The paper does not publish these category totals or margins; compiled allocation varies by kernel variant and includes omitted live state."}],"strongest_alternative":"The counts are reasonable back-of-envelope estimates from tile dimensions.","discriminating_result":"Fragment ownership, liveness, compiler allocation and role specialization prevent tile dimensions alone from establishing per-thread totals.","action":"delete","action_reason":"Replace invented budgets with the paper's directly reported TMEM roles.","proposed_replacement":"Describe FA4's TMEM-resident accumulators and role split without fabricated register totals.","replacement_claim_ids":["migration-register-to-tmem-current-c007"]} +{"receipt_id":"rem-migration-register-to-tmem-012","receipt_scope":"wiki","path":"wiki/migration/register-to-tmem.md","original_body_sha256":"d18a8529e0c9cab4d1711c15938892a04771dbccb5548fc61e5298b0fd6c0ed4","original_unit_ids":["u024"],"locator":"body:L381-L396","original_claim_id":"migration-register-to-tmem-original-c020","original_claim":"Each fine-grained TMEM load costs 420 cycles and an x4 form fetches four adjacent scalars in one such access.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"PTX ISA 9.0","locator":"shape/repetition mapping","observation":"Loads are collective with shape-specific mappings, not the scalar/x4 abstraction shown."},{"url":"https://arxiv.org/abs/2512.02189","revision":"microbenchmark paper v1","locator":"memory latency experiment","observation":"Approximately 420 cycles is not a normative tcgen05.ld latency across shapes."}],"strongest_alternative":"The code is informal shorthand for batching legal TMEM transfers.","discriminating_result":"The exact latency and adjacent-value semantics are the advice's justification and are not supported by the cited mechanisms.","action":"correct","action_reason":"Keep batching as a profiling hypothesis, not a latency theorem.","proposed_replacement":"Choose a legal collective copy shape and benchmark epilogue partitioning on the target.","replacement_claim_ids":["migration-register-to-tmem-current-c005","migration-register-to-tmem-current-c008"]} +{"receipt_id":"rem-migration-register-to-tmem-013","receipt_scope":"wiki","path":"wiki/migration/register-to-tmem.md","original_body_sha256":"d18a8529e0c9cab4d1711c15938892a04771dbccb5548fc61e5298b0fd6c0ed4","original_unit_ids":["u025"],"locator":"body:L400-L406","original_claim_id":"migration-register-to-tmem-original-c021","original_claim":"Blackwell should use at minimum m128n256 or m256n256 and has no spilling risk at those tiles.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2","locator":"occupancy guidance","observation":"No minimum tile is prescribed and spills remain dependent on compiled whole-kernel resource use."}],"strongest_alternative":"These are starting points for compute-bound GEMM, not universal mandates.","discriminating_result":"The page uses should/minimum and no-risk language without workload qualification or measurements.","action":"correct","action_reason":"Turn the mandate into an evidence-driven tuning checklist.","proposed_replacement":"Select legal shapes and benchmark multiple tile/CTA-group choices for the actual workload.","replacement_claim_ids":["migration-register-to-tmem-current-c008"]} +{"receipt_id":"rem-migration-register-to-tmem-014","receipt_scope":"wiki","path":"wiki/migration/register-to-tmem.md","original_body_sha256":"d18a8529e0c9cab4d1711c15938892a04771dbccb5548fc61e5298b0fd6c0ed4","original_unit_ids":["u026"],"locator":"body:L410-L433","original_claim_id":"migration-register-to-tmem-original-c023","original_claim":"Omitted deallocation is a defined leak that blocks the next CTA, and reaching one cleanup helper suffices.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"PTX ISA 9.0","locator":"deallocation/collective rules","observation":"The ISA states a mandatory collective before-exit contract, not defined recoverable next-CTA leak behavior."}],"strongest_alternative":"Leak is a useful operational metaphor for unavailable dynamic TMEM.","discriminating_result":"It asserts a specific cross-CTA consequence while omitting collective and all-exit-path requirements, so the metaphor changes the contract.","action":"correct","action_reason":"State the normative obligation and control-flow requirements.","proposed_replacement":"All lanes of the designated warp perform matching deallocation after readers complete on every kernel exit path.","replacement_claim_ids":["migration-register-to-tmem-current-c004","migration-register-to-tmem-current-c008"]} +{"receipt_id":"rem-migration-register-to-tmem-015","receipt_scope":"wiki","path":"wiki/migration/register-to-tmem.md","original_body_sha256":"d18a8529e0c9cab4d1711c15938892a04771dbccb5548fc61e5298b0fd6c0ed4","original_unit_ids":["u028"],"locator":"body:L440-L440","original_claim_id":"migration-register-to-tmem-original-c025","original_claim":"Every migrated TMEM accumulator must be zeroed with tcgen05.st before its first MMA.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"PTX ISA 9.0","locator":"enable-input-D","observation":"A false enable-input-D predicate computes D=A*B without reading old D, avoiding mandatory zero initialization."}],"strongest_alternative":"Explicit zeroing is one legal initialization strategy.","discriminating_result":"The summary presents it as the migration rule rather than an optional strategy; predicate-controlled initialization is the direct counterexample.","action":"correct","action_reason":"Describe both initialization choices without prescribing a needless store.","proposed_replacement":"Use enable-input-D=false for the first MMA unless the algorithm requires a separately initialized D value.","replacement_claim_ids":["migration-register-to-tmem-current-c005"]} +{"receipt_id":"rem-migration-register-to-tmem-016","receipt_scope":"wiki","path":"wiki/migration/register-to-tmem.md","original_body_sha256":"d18a8529e0c9cab4d1711c15938892a04771dbccb5548fc61e5298b0fd6c0ed4","original_unit_ids":["u033"],"locator":"body:L445-L445","original_claim_id":"migration-register-to-tmem-original-c026","original_claim":"TMEM gives every Blackwell epilogue ample access to the full register file with minimal spill risk.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2","locator":"register limits","observation":"Other live values and TMEM readback batches consume finite registers; spill risk must be obtained from compiled resource data."}],"strongest_alternative":"Ample is a qualitative comparison with a Hopper kernel holding 128 D registers.","discriminating_result":"The universal every/full/minimal wording goes beyond that relative comparison and lacks a concrete kernel.","action":"correct","action_reason":"State reduced resident-D pressure as an opportunity, not a guarantee.","proposed_replacement":"Tune epilogue batch size and verify registers/spills in compiler/profiler output.","replacement_claim_ids":["migration-register-to-tmem-current-c002","migration-register-to-tmem-current-c008"]} +{"receipt_id":"rem-migration-register-to-tmem-017","receipt_scope":"wiki","path":"wiki/migration/register-to-tmem.md","original_body_sha256":"d18a8529e0c9cab4d1711c15938892a04771dbccb5548fc61e5298b0fd6c0ed4","original_unit_ids":["u035"],"locator":"body:L447-L447","original_claim_id":"migration-register-to-tmem-original-c027","original_claim":"m64n256 and m128n256/m256n256 are universal practical maximum tiles on Hopper and Blackwell respectively.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"PTX ISA 9.0","locator":"instruction shapes","observation":"Instruction shape is not the same as a composed CTA tile or practical maximum."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2","locator":"resource constraints","observation":"Practical shape is workload/resource dependent."}],"strongest_alternative":"The row means common maximum instruction shapes rather than kernel tiles.","discriminating_result":"It explicitly labels them practical tiles limited by registers/TMEM, which is a kernel-performance claim, not syntax.","action":"correct","action_reason":"Separate legal instruction shapes from empirical CTA tile tuning.","proposed_replacement":"Use the ISA shape table for legality and benchmark composed tiles under actual resource limits.","replacement_claim_ids":["migration-register-to-tmem-current-c001","migration-register-to-tmem-current-c008"]} +{"receipt_id":"rem-technique-double-buffering-001","receipt_scope":"wiki","path":"wiki/techniques/double-buffering.md","original_body_sha256":"787b870b0080a00805a0601a72d16367df1fd5ff21f1043c92acaf6e2d91a387","original_unit_ids":["u003"],"locator":"body:L9-L105","original_claim_id":"technique-double-buffering-original-c003","original_claim":"The displayed tmem_double_buffer_mainloop is a safe CUDA implementation of two-buffer TMEM accumulation and epilogue reuse.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"PTX ISA 9.0","locator":"allocation/deallocation","observation":"No TMEM allocation or mandatory deallocation is present."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier","revision":"PTX ISA 9.0","locator":"phase/arrival rules","observation":"The drained barrier is initialized for one arrival despite fourteen claimed producers, and reusable phases/tokens are absent."}],"strongest_alternative":"The helper calls make the block intentionally schematic.","discriminating_result":"It is presented as CUDA without a schematic warning and asserts concrete synchronization behavior that contradicts the normative lifecycle.","action":"correct","action_reason":"Replace unsafe pseudo-implementation with verified ownership invariants and a pinned complete implementation route.","proposed_replacement":"Document allocation, disjoint regions, compute-complete and read-complete handoffs, phase state, tail drain, and collective cleanup.","replacement_claim_ids":["technique-double-buffering-current-c002","technique-double-buffering-current-c003"]} +{"receipt_id":"rem-technique-double-buffering-002","receipt_scope":"wiki","path":"wiki/techniques/double-buffering.md","original_body_sha256":"787b870b0080a00805a0601a72d16367df1fd5ff21f1043c92acaf6e2d91a387","original_unit_ids":["u006"],"locator":"body:L142-L142","original_claim_id":"technique-double-buffering-original-c006","original_claim":"Every fully optimized Blackwell GEMM uses simultaneous three-stage SMEM and two-buffer TMEM.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm","revision":"NVIDIA/cutlass v4.5.0 e406c186","locator":"tutorial variants","observation":"Official examples use multiple schedules and incremental stages, not one universal combination."}],"strongest_alternative":"The wording means only a common high-performance pattern.","discriminating_result":"It says a fully optimized GEMM uses both, making the design necessary rather than conditional.","action":"correct","action_reason":"Scope the combination to concrete kernels and tuning conditions.","proposed_replacement":"Describe simultaneous buffering as one evidenced design, not a requirement.","replacement_claim_ids":["technique-double-buffering-current-c001","technique-double-buffering-current-c006"]} +{"receipt_id":"rem-technique-double-buffering-003","receipt_scope":"wiki","path":"wiki/techniques/double-buffering.md","original_body_sha256":"787b870b0080a00805a0601a72d16367df1fd5ff21f1043c92acaf6e2d91a387","original_unit_ids":["u007"],"locator":"body:L144-L190","original_claim_id":"technique-double-buffering-original-c007","original_claim":"The displayed combined CUDA block and timeline form a complete safe implementation in which the previous tile's epilogue overlaps the next tile's MMA.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v6.cu","revision":"gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"role loops and barrier lifecycle","observation":"The real implementation requires independent persistent loops, ready/empty phases, a prologue, tail drain, and cleanup omitted by the block."}],"strongest_alternative":"Opaque helpers encapsulate all missing state.","discriminating_result":"The shared outer loop and comments do not establish role progress or final ownership even under that interpretation.","action":"correct","action_reason":"Use a state/ownership table instead of incomplete CUDA.","proposed_replacement":"Show the required transitions for each TMEM and SMEM stage and link the complete pinned source.","replacement_claim_ids":["technique-double-buffering-current-c003","technique-double-buffering-current-c004"]} +{"receipt_id":"rem-technique-double-buffering-004","receipt_scope":"wiki","path":"wiki/techniques/double-buffering.md","original_body_sha256":"787b870b0080a00805a0601a72d16367df1fd5ff21f1043c92acaf6e2d91a387","original_unit_ids":["u009"],"locator":"body:L196-L212","original_claim_id":"technique-double-buffering-original-c009","original_claim":"Two Hopper m64n256 FP32 D buffers cost 64 registers/256 bytes per thread, while Blackwell uses zero registers and dedicates all TMEM to accumulators.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16","revision":"PTX ISA 9.0","locator":"f32 D fragment","observation":"N/2 gives 128 registers per thread for one N=256 D fragment and 256 registers/1024 bytes for two."}],"strongest_alternative":"The formula represents a smaller per-warp subfragment.","discriminating_result":"The text explicitly labels a full m64n256 warpgroup tile and a per-thread total, for which the ISA table is decisive.","action":"correct","action_reason":"Replace incorrect arithmetic and reject zero total-register cost.","proposed_replacement":"Give the ISA-defined N/2 D count and distinguish resident-D savings from whole-kernel registers.","replacement_claim_ids":["technique-double-buffering-current-c005"]} +{"receipt_id":"rem-technique-double-buffering-005","receipt_scope":"wiki","path":"wiki/techniques/double-buffering.md","original_body_sha256":"787b870b0080a00805a0601a72d16367df1fd5ff21f1043c92acaf6e2d91a387","original_unit_ids":["u011"],"locator":"body:L217-L217","original_claim_id":"technique-double-buffering-original-c011","original_claim":"Doubling an accumulator costs exactly twice the Hopper registers but zero Blackwell registers.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"PTX ISA 9.0","locator":"TMEM load destination","observation":"Readback values are placed in registers; addresses, state, and epilogue temporaries also remain register-resident."}],"strongest_alternative":"Zero means only no long-lived D register fragment.","discriminating_result":"The table states unqualified register cost, not resident-D cost.","action":"correct","action_reason":"Narrow the comparison to resident accumulator storage.","proposed_replacement":"Say TMEM avoids a resident D vector but not all register use.","replacement_claim_ids":["technique-double-buffering-current-c005"]} +{"receipt_id":"rem-technique-double-buffering-006","receipt_scope":"wiki","path":"wiki/techniques/double-buffering.md","original_body_sha256":"787b870b0080a00805a0601a72d16367df1fd5ff21f1043c92acaf6e2d91a387","original_unit_ids":["u012"],"locator":"body:L218-L218","original_claim_id":"technique-double-buffering-original-c012","original_claim":"Accumulator double-buffering typically changes Hopper occupancy from two to one CTA/SM and has no Blackwell occupancy impact.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator","revision":"CUDA 13.0.2","locator":"occupancy calculation","observation":"Residency depends on compiled registers, block size, SMEM, barriers, clusters, and allocation granularity; no such fixed transition follows."}],"strongest_alternative":"The row is a representative rule of thumb.","discriminating_result":"No kernel, launch shape, compiler resource report, or GPU measurement defines the represented case.","action":"delete","action_reason":"Unsupported occupancy numbers cannot guide a real launch.","proposed_replacement":"Require compiled resource and occupancy analysis for the concrete variant.","replacement_claim_ids":["technique-double-buffering-current-c007"]} +{"receipt_id":"rem-technique-double-buffering-007","receipt_scope":"wiki","path":"wiki/techniques/double-buffering.md","original_body_sha256":"787b870b0080a00805a0601a72d16367df1fd5ff21f1043c92acaf6e2d91a387","original_unit_ids":["u014"],"locator":"body:L220-L220","original_claim_id":"technique-double-buffering-original-c013","original_claim":"The maximum practical accumulator is about 16K elements on Hopper and exactly 128 by 512 elements on Blackwell.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory","revision":"PTX ISA 9.0","locator":"TMEM cell structure","observation":"128 by 512 is an address structure, not a universal logical-accumulator maximum; packing and composed tiles vary."}],"strongest_alternative":"The table means one common instruction fragment rather than a kernel maximum.","discriminating_result":"It labels the values Max accumulator size and attributes the limits to storage generally.","action":"delete","action_reason":"Avoid conflating an instruction fragment or storage geometry with a practical kernel maximum.","proposed_replacement":"Compare storage location and access only; tune composed tile size empirically.","replacement_claim_ids":["technique-double-buffering-current-c005","technique-double-buffering-current-c007"]} +{"receipt_id":"rem-technique-double-buffering-008","receipt_scope":"wiki","path":"wiki/techniques/double-buffering.md","original_body_sha256":"787b870b0080a00805a0601a72d16367df1fd5ff21f1043c92acaf6e2d91a387","original_unit_ids":["u015"],"locator":"body:L224-L224","original_claim_id":"technique-double-buffering-original-c014","original_claim":"TMEM double-buffering should always be used for any nontrivial Blackwell epilogue and its only cost is plentiful TMEM capacity.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2","locator":"resource limits","observation":"TMEM allocation can affect residency, while buffering also needs synchronization, address state, and readback."}],"strongest_alternative":"For a sufficiently expensive epilogue the overlap benefit dominates in practice.","discriminating_result":"That still requires a measured crossover and adequate resources; none is source-defined for every kernel.","action":"correct","action_reason":"Make adoption conditional on measured overlap benefit and resource fit.","proposed_replacement":"Benchmark one versus two regions and inspect residency and stalls.","replacement_claim_ids":["technique-double-buffering-current-c007"]} +{"receipt_id":"rem-technique-double-buffering-009","receipt_scope":"wiki","path":"wiki/techniques/double-buffering.md","original_body_sha256":"787b870b0080a00805a0601a72d16367df1fd5ff21f1043c92acaf6e2d91a387","original_unit_ids":["u016"],"locator":"body:L225-L225","original_claim_id":"technique-double-buffering-original-c015","original_claim":"Every GEMM or attention mainloop should use SMEM multi-staging, with three stages as the universal default and four or five only for long high-latency K loops.","original_verdict":"refuted","evidence":[{"path":"artifacts/prs/flashinfer/PR-2387/key-files/include/flashinfer/mamba/selective_state_update.cuh","revision":"flashinfer merge 18804cd51734cccf807356d017733bc757677f15","locator":"numStages dispatch","observation":"The merged source selects stage counts by path and total stage geometry, demonstrating policy dependence."}],"strongest_alternative":"Three is only a suggested starting point.","discriminating_result":"The original says always/default and restricts other counts to one condition without evidence.","action":"correct","action_reason":"Replace a universal recipe with resource- and workload-dependent tuning.","proposed_replacement":"Sweep legal stage counts while tracking SMEM, occupancy, and pipeline stalls.","replacement_claim_ids":["technique-double-buffering-current-c006","technique-double-buffering-current-c007"]} +{"receipt_id":"rem-technique-double-buffering-010","receipt_scope":"wiki","path":"wiki/techniques/double-buffering.md","original_body_sha256":"787b870b0080a00805a0601a72d16367df1fd5ff21f1043c92acaf6e2d91a387","original_unit_ids":["u017"],"locator":"body:L226-L226","original_claim_id":"technique-double-buffering-original-c016","original_claim":"Simultaneous SMEM and TMEM buffering is the standard production Blackwell GEMM approach used by all CUTLASS and CuTe DSL kernels.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm","revision":"NVIDIA/cutlass v4.5.0 e406c186","locator":"tutorial variants","observation":"The release contains multiple schedules and examples; it establishes concrete uses, not use by all kernels."}],"strongest_alternative":"Both names denote the same ecosystem and the sentence only intends commonality.","discriminating_result":"The explicit universal subject remains false even if the ecosystem commonly supports the pattern.","action":"correct","action_reason":"Use concrete, pinned examples rather than ecosystem-wide attribution.","proposed_replacement":"Cite learn-cuda v6 as one combined design and FlashInfer PR 2387 as a multi-stage SMEM design.","replacement_claim_ids":["technique-double-buffering-current-c006"]} +{"receipt_id":"rem-technique-double-buffering-011","receipt_scope":"wiki","path":"wiki/techniques/double-buffering.md","original_body_sha256":"787b870b0080a00805a0601a72d16367df1fd5ff21f1043c92acaf6e2d91a387","original_unit_ids":["u018"],"locator":"body:L230-L230","original_claim_id":"technique-double-buffering-original-c017","original_claim":"TMEM double-buffering always requires every accumulator to fit in exactly half of the 512 columns, and every FP32 tile wider than N=256 must be split.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory-allocation","revision":"PTX ISA 9.0","locator":"allocation units","observation":"The ISA does not mandate equal halves; region sizes, counts, packing, and decomposition are implementation choices."}],"strongest_alternative":"The statement is intentionally limited to the page's symmetric two-buffer design.","discriminating_result":"It says TMEM double-buffering requires the condition generally rather than scoping it to that design.","action":"correct","action_reason":"State the actual nonaliasing/capacity invariant and label 256/256 as one choice.","proposed_replacement":"All simultaneously live regions must be disjoint and fit the allocation; equal halves are optional.","replacement_claim_ids":["technique-double-buffering-current-c002","technique-double-buffering-current-c007"]} +{"receipt_id":"rem-technique-double-buffering-012","receipt_scope":"wiki","path":"wiki/techniques/double-buffering.md","original_body_sha256":"787b870b0080a00805a0601a72d16367df1fd5ff21f1043c92acaf6e2d91a387","original_unit_ids":["u020"],"locator":"body:L232-L232","original_claim_id":"technique-double-buffering-original-c019","original_claim":"The mbarrier handoff between TMEM buffers costs a universal few cycles per tile, making it proportionally worse for tiles with few K iterations.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier","revision":"PTX ISA 9.0","locator":"mbarrier semantics","observation":"The ISA publishes no fixed cycle cost; exposed latency depends on the schedule, contention, dependencies, and target."}],"strongest_alternative":"Few cycles is a harmless qualitative approximation.","discriminating_result":"It is an exact-seeming hardware cost with no environment or measurement and can reverse a tuning decision.","action":"delete","action_reason":"Retain the need to measure synchronization overhead without inventing latency.","proposed_replacement":"Measure short-K variants and inspect barrier stalls rather than using a fixed cost.","replacement_claim_ids":["technique-double-buffering-current-c007"]} +{"receipt_id":"rem-source-flashinfer-2387-001","receipt_scope":"source-trail","path":"sources/prs/flashinfer/PR-2387.md","original_body_sha256":"b67424853871b3d438278c030d75cb80a49dbc966dc38f6a45a572c7ed14f442","original_unit_ids":[],"locator":"frontmatter and entire source summary","original_claim_id":"source-flashinfer-2387-record","original_claim":"The short merge revision and narrative/code sketch accurately preserve PR 2387's implementation, dispatch, toolchain, and performance evidence.","original_verdict":"refuted","evidence":[{"url":"https://api.github.com/repos/flashinfer-ai/flashinfer/pulls/2387","revision":"queried 2026-08-09","locator":"merge_commit_sha and head.sha","observation":"The full merge SHA is 18804cd51734cccf807356d017733bc757677f15 and head SHA is bfe3867d2c9ced629326dc6c96ec81e0b32482d0."},{"path":"artifacts/prs/flashinfer/PR-2387/key-files/include/flashinfer/mamba/selective_state_update.cuh","revision":"merge 18804cd51734cccf807356d017733bc757677f15","locator":"entire file; SHA-256 40a2de44b1f29b4b4c213e74e530131c9a43be5e602daa3c8bbd901a3b281446","observation":"The artifact byte-matches the raw file at the merge commit and supports per-stage full/empty barriers and stage counts; the displayed source-card pseudocode is not verbatim and several narrative/performance claims lack preserved rows."}],"strongest_alternative":"The short SHA and prose are sufficient because the artifact is present.","discriminating_result":"A routing card should name the immutable full revision and distinguish exact artifact evidence from unpreserved benchmark interpretation.","action":"correct","action_reason":"Pin the full merge SHA and reduce the card to directly inspectable implementation facts.","proposed_replacement":"Record hashes and exact loci for SharedStorageHorizontal, stage barriers, dispatch, and stage-count choices; remove invented pseudocode and unsupported performance/toolchain claims.","replacement_claim_ids":["technique-double-buffering-current-c006"]} +{"receipt_id":"rem-technique-epilogue-fusion-001","receipt_scope":"wiki","path":"wiki/techniques/epilogue-fusion.md","original_body_sha256":"f37f59190f56273dbd2ec10059fd85e03af89ea72f6cf5e9898fb14e1f3fd450","original_unit_ids":["u001"],"locator":"body:L3-L3","original_claim_id":"technique-epilogue-fusion-original-c001","original_claim":"Epilogue fusion inherently means overlap and Blackwell normally assigns warps 2-15 to ping-pong TMEM with warp 1.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py","revision":"NVIDIA/cutlass v4.5.0 e406c186","locator":"epilogue role group","observation":"The official example uses a four-warp epilogue group and demonstrates fusion without making concurrent next-tile MMA definitional."}],"strongest_alternative":"The sentence describes one common implementation, not the definition.","discriminating_result":"It states a fixed typical role map and overlap as the meaning of fusion without source scope.","action":"correct","action_reason":"Separate semantic fusion from optional schedule overlap.","proposed_replacement":"Define fusion as computing output transforms before final materialization; discuss overlap as a separate schedule optimization.","replacement_claim_ids":["technique-epilogue-fusion-current-c001"]} +{"receipt_id":"rem-technique-epilogue-fusion-002","receipt_scope":"wiki","path":"wiki/techniques/epilogue-fusion.md","original_body_sha256":"f37f59190f56273dbd2ec10059fd85e03af89ea72f6cf5e9898fb14e1f3fd450","original_unit_ids":["u003"],"locator":"body:L9-L47","original_claim_id":"technique-epilogue-fusion-original-c003","original_claim":"The displayed epilogue_warp_fn is safe usable CUDA for fourteen warps to scalar-load TMEM and store a tile.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"PTX ISA 9.0","locator":"collective shapes/mapping/completion","observation":"No scalar tmem_load_f32(r,c) exists; legal loads are collective shaped operations with wait::ld and layout mapping."}],"strongest_alternative":"tmem_load_f32 is an opaque wrapper around legal collectives.","discriminating_result":"Its scalar row/column interface and per-lane loop do not expose the collective mapping, completion, edge predicates, or shared source ownership needed for safe use.","action":"correct","action_reason":"Replace pseudo-CUDA with exact lifecycle and official CuTe copy-route identifiers.","proposed_replacement":"Show make_tmem_copy/partition/cute.copy as a non-standalone API route and state the required wait/edge handling.","replacement_claim_ids":["technique-epilogue-fusion-current-c002","technique-epilogue-fusion-current-c003"]} +{"receipt_id":"rem-technique-epilogue-fusion-003","receipt_scope":"wiki","path":"wiki/techniques/epilogue-fusion.md","original_body_sha256":"f37f59190f56273dbd2ec10059fd85e03af89ea72f6cf5e9898fb14e1f3fd450","original_unit_ids":["u004"],"locator":"body:L51-L51","original_claim_id":"technique-epilogue-fusion-original-c004","original_claim":"A mandatory 256/256 TMEM double buffer is the key required mechanism for Blackwell epilogue fusion.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py","revision":"NVIDIA/cutlass v4.5.0 e406c186","locator":"TMEM allocation and epilogue","observation":"The official fused epilogue does not establish equal-half ping-pong as a requirement."}],"strongest_alternative":"The statement is limited to overlapped epilogues.","discriminating_result":"It calls double buffering the key to epilogue fusion generally, conflating two distinct choices.","action":"correct","action_reason":"Make multi-region overlap optional and state nonaliasing constraints.","proposed_replacement":"Use one completed region for a non-overlapped fused epilogue or multiple disjoint regions when measured overlap is profitable.","replacement_claim_ids":["technique-epilogue-fusion-current-c001","technique-epilogue-fusion-current-c003"]} +{"receipt_id":"rem-technique-epilogue-fusion-004","receipt_scope":"wiki","path":"wiki/techniques/epilogue-fusion.md","original_body_sha256":"f37f59190f56273dbd2ec10059fd85e03af89ea72f6cf5e9898fb14e1f3fd450","original_unit_ids":["u005"],"locator":"body:L53-L126","original_claim_id":"technique-epilogue-fusion-original-c005","original_claim":"The displayed mma_epilogue_overlap is a complete phase-correct two-region protocol.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier","revision":"PTX ISA 9.0","locator":"phase/arrival semantics","observation":"The code initializes a one-arrival barrier despite requiring all epilogue warps and omits phases/tokens."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit","revision":"PTX ISA 9.0","locator":"MMA completion","observation":"A raw arrive does not attach asynchronous MMA completion; allocation, load completion, final drain, and cleanup are absent."}],"strongest_alternative":"Helpers encapsulate completion and comments supply the intended barrier count.","discriminating_result":"Literal initialization contradicts the comment, and no helper can supply missing phase state and outer lifecycle visible to callers.","action":"correct","action_reason":"Replace incomplete code with a required-edge table and negative tests.","proposed_replacement":"Document compute-ready, read-complete, nonaliasing, tail, and cleanup proofs.","replacement_claim_ids":["technique-epilogue-fusion-current-c003"]} +{"receipt_id":"rem-technique-epilogue-fusion-005","receipt_scope":"wiki","path":"wiki/techniques/epilogue-fusion.md","original_body_sha256":"f37f59190f56273dbd2ec10059fd85e03af89ea72f6cf5e9898fb14e1f3fd450","original_unit_ids":["u006"],"locator":"body:L130-L130","original_claim_id":"technique-epilogue-fusion-original-c007","original_claim":"CUTLASS 4.5.0 can fuse arbitrary element-wise operations without support or schedule constraints.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/builders/sm100_builder.inl","revision":"NVIDIA/cutlass v4.5.0 e406c186","locator":"builder specializations","observation":"Support depends on recognized operation/callback, architecture, schedule, layout, alignment, dtype, and tile combinations."}],"strongest_alternative":"Arbitrary means user-composable through custom callbacks.","discriminating_result":"Even custom callbacks must satisfy builder and schedule interfaces; arbitrary code is not automatically supported.","action":"correct","action_reason":"Describe the exact supported operation-tag/callback interface.","proposed_replacement":"Use CollectiveBuilder with a supported fusion operation or callbacks and validate can_implement for the concrete configuration.","replacement_claim_ids":["technique-epilogue-fusion-current-c004"]} +{"receipt_id":"rem-technique-epilogue-fusion-006","receipt_scope":"wiki","path":"wiki/techniques/epilogue-fusion.md","original_body_sha256":"f37f59190f56273dbd2ec10059fd85e03af89ea72f6cf5e9898fb14e1f3fd450","original_unit_ids":["u007"],"locator":"body:L132-L159","original_claim_id":"technique-epilogue-fusion-original-c008","original_claim":"The displayed LinCombEltAct and Sm100EpilogueTmaWarpSpecialized typedefs are valid CUTLASS 4.5.0 scale+bias+ReLU code.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/operations.hpp","revision":"NVIDIA/cutlass v4.5.0 e406c186","locator":"LinCombEltAct signature","observation":"The shown parameter meanings/order do not express the claimed output/source policy."},{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue","revision":"same tag","locator":"symbol search","observation":"Sm100EpilogueTmaWarpSpecialized does not exist; the release uses CollectiveBuilder and schedule tags."}],"strongest_alternative":"The snippet targets a nearby development revision.","discriminating_result":"The page explicitly claims CUTLASS 4.5.0, whose exact tag rejects the type.","action":"correct","action_reason":"Replace invented code with exact version-pinned type signatures and a real builder example route.","proposed_replacement":"Document FusionOp template parameters and CollectiveBuilder's final FusionOpOrCallbacks argument.","replacement_claim_ids":["technique-epilogue-fusion-current-c004"]} +{"receipt_id":"rem-technique-epilogue-fusion-007","receipt_scope":"wiki","path":"wiki/techniques/epilogue-fusion.md","original_body_sha256":"f37f59190f56273dbd2ec10059fd85e03af89ea72f6cf5e9898fb14e1f3fd450","original_unit_ids":["u008"],"locator":"body:L165-L165","original_claim_id":"technique-epilogue-fusion-original-c009","original_claim":"The Scale + Bias row's y=alpha*acc+beta*C equation is a bias epilogue.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/operations.hpp","revision":"NVIDIA/cutlass v4.5.0 e406c186","locator":"LinearCombination and bias operation tags","observation":"beta*C is a source-tensor term; row/column bias is a separate input and operation tag."}],"strongest_alternative":"C may itself be a broadcast bias tensor.","discriminating_result":"The row labels standard GEMM beta*C without stating broadcast layout; the default source C is not bias.","action":"correct","action_reason":"Separate source linear combination from explicit bias broadcasting.","proposed_replacement":"List alpha*acc+beta*C and per-row/per-column bias as distinct operation shapes.","replacement_claim_ids":["technique-epilogue-fusion-current-c005"]} +{"receipt_id":"rem-technique-epilogue-fusion-008","receipt_scope":"wiki","path":"wiki/techniques/epilogue-fusion.md","original_body_sha256":"f37f59190f56273dbd2ec10059fd85e03af89ea72f6cf5e9898fb14e1f3fd450","original_unit_ids":["u014"],"locator":"body:L174-L174","original_claim_id":"technique-epilogue-fusion-original-c011","original_claim":"Every Blackwell GEMM with a nontrivial epilogue should use fourteen default epilogue warps and necessarily removes one launch/round trip.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py","revision":"NVIDIA/cutlass v4.5.0 e406c186","locator":"epilogue CooperativeGroup","observation":"The official example uses four epilogue warps, and launch/intermediate removal depends on the chosen unfused graph."}],"strongest_alternative":"Fourteen is one custom 16-warp CTA convention.","discriminating_result":"The original calls it default/all without naming such a kernel or launch.","action":"correct","action_reason":"Replace universal role count and benefit with configuration-specific evaluation.","proposed_replacement":"Count participants from the actual schedule and compare a defined fused/unfused pair.","replacement_claim_ids":["technique-epilogue-fusion-current-c006"]} +{"receipt_id":"rem-technique-epilogue-fusion-009","receipt_scope":"wiki","path":"wiki/techniques/epilogue-fusion.md","original_body_sha256":"f37f59190f56273dbd2ec10059fd85e03af89ea72f6cf5e9898fb14e1f3fd450","original_unit_ids":["u018"],"locator":"body:L181-L181","original_claim_id":"technique-epilogue-fusion-original-c015","original_claim":"Fourteen epilogue warps get equal proportional TMEM bandwidth and wide TILE_N therefore bottlenecks on TMEM reads.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"PTX ISA 9.0","locator":"load semantics","observation":"The ISA defines no equal-share model or tile-width bottleneck threshold; fourteen participants are not architectural."}],"strongest_alternative":"It is qualitative contention guidance.","discriminating_result":"The proportional-share mechanism and fixed participant count are concrete unsupported claims.","action":"correct","action_reason":"Require profiler and variant measurements for bandwidth diagnosis.","proposed_replacement":"Measure TMEM load issue/stalls and epilogue throughput as work partition and tile width vary.","replacement_claim_ids":["technique-epilogue-fusion-current-c006"]} +{"receipt_id":"rem-technique-epilogue-fusion-010","receipt_scope":"wiki","path":"wiki/techniques/epilogue-fusion.md","original_body_sha256":"f37f59190f56273dbd2ec10059fd85e03af89ea72f6cf5e9898fb14e1f3fd450","original_unit_ids":["u019"],"locator":"body:L182-L182","original_claim_id":"technique-epilogue-fusion-original-c016","original_claim":"A store-only epilogue wastes fourteen warps and should shrink the CTA or repurpose them for TMA prefetch.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py","revision":"NVIDIA/cutlass v4.5.0 e406c186","locator":"epilogue TMA-store path","observation":"A store path still performs collective TMEM loads, conversion, SMEM staging, and TMA stores; changing participants changes pipeline/occupancy constraints."}],"strongest_alternative":"Waste means low utilization relative to a heavier activation.","discriminating_result":"No utilization measurement or fourteen-warp source is supplied, and the proposed role change is not mechanically free.","action":"correct","action_reason":"Use measured role-count and store-path bottlenecks.","proposed_replacement":"Sweep epilogue group size and schedule only with a complete supported kernel and correctness tests.","replacement_claim_ids":["technique-epilogue-fusion-current-c006"]} +{"receipt_id":"rem-technique-epilogue-fusion-011","receipt_scope":"wiki","path":"wiki/techniques/epilogue-fusion.md","original_body_sha256":"f37f59190f56273dbd2ec10059fd85e03af89ea72f6cf5e9898fb14e1f3fd450","original_unit_ids":["u020"],"locator":"body:L186-L186","original_claim_id":"technique-epilogue-fusion-original-c017","original_claim":"Every file in artifacts/kernels/epilogue-fusion/full is verbatim upstream code.","original_verdict":"refuted","evidence":[{"command":"inspect provenance and file headers; byte-compare nvfp4 source to vLLM merge","environment":"current worktree","reference":"all full files should be upstream-verbatim","result":"The nvfp4 file is verbatim, while tmem-load-into-registers-for-epilogue.cu self-identifies as extracted from a local blog card."}],"strongest_alternative":"Full means complete evidence bundle rather than upstream-verbatim.","discriminating_result":"The original explicitly says Verbatim upstream code, so the directory's mixed modes contradict it.","action":"correct","action_reason":"Describe each artifact's actual provenance mode and warn that extracted/derived snippets are not safe standalone kernels.","proposed_replacement":"Name the one byte-matched upstream file, the extracted fragment, and the labeled derived variant separately.","replacement_claim_ids":["technique-epilogue-fusion-current-c007"]} +{"receipt_id":"rem-source-vllm-16032-001","receipt_scope":"source-trail","path":"sources/prs/vllm/PR-16032.md","original_body_sha256":"c44453a78143790a3b11c869db3d9d1bda94aa325ceb022095f0a34309599878","original_unit_ids":[],"locator":"frontmatter and entire source summary","original_claim_id":"source-vllm-16032-record","original_claim":"The short merge SHA, tcgen05_mla symbol/path, CUTLASS v3.9 attribution, and significant-speedup/default-backend narrative accurately describe PR 16032.","original_verdict":"refuted","evidence":[{"url":"https://api.github.com/repos/vllm-project/vllm/pulls/16032","revision":"queried 2026-08-09","locator":"merge_commit_sha","observation":"The full merge SHA is ed7a29d9f8b48978e3bbf43599d21b4de65387e0."},{"path":"artifacts/prs/vllm/PR-16032/diff.patch","revision":"merge ed7a29d9f8b48978e3bbf43599d21b4de65387e0","locator":"changed files, registrations and tests","observation":"The implementation is cutlass_mla_decode in cutlass_mla_*.cu; the captured diff does not substantiate the claimed tcgen05_mla API, v3.9 provenance, portable speedup, or recommended-default statement."}],"strongest_alternative":"The source card paraphrases an earlier PR revision or discussion naming.","discriminating_result":"The card claims the merged result, whose exact diff and artifacts use different stable symbols and contain no preserved benchmark evidence for the performance conclusions.","action":"correct","action_reason":"Pin the full merge revision and reduce the card to byte-verifiable files, API registration, dispatch guards, and tests.","proposed_replacement":"Record cutlass_mla_decode, exact changed paths/hashes, SM100a dispatch, q_nope/q_pe inputs, and remove unsupported benchmark/default claims.","replacement_claim_ids":["technique-epilogue-fusion-current-c007"]} +{"receipt_id":"rem-technique-persistent-kernels-001","receipt_scope":"wiki","path":"wiki/techniques/persistent-kernels.md","original_body_sha256":"7d1d16d68cfae0d699d1b73b4d10ac71a12c878966081788134245f3861fc283","original_unit_ids":["u001"],"locator":"body:L3-L3","original_claim_id":"technique-persistent-kernels-original-c001","original_claim":"Persistent kernels always launch one CTA per SM and CLC returns work or lets the caller cancel itself.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0","locator":"dynamic scheduler rules","observation":"CLC launches the problem-sized grid; workers process blockIdx then cancel another not-yet-started ClcID. Grid size is policy-dependent and try_cancel does not cancel the caller."}],"strongest_alternative":"Exactly one CTA/SM describes persistent residency rather than the launched grid.","discriminating_result":"The sentence explicitly says launch count and then misstates the CLC target, so that interpretation cannot repair it.","action":"correct","action_reason":"Restore problem-sized CLC grid and two-path ClcID model.","proposed_replacement":"Define persistence separately, then describe initial blockIdx plus successful cancellation of another unlaunched ID.","replacement_claim_ids":["technique-persistent-kernels-current-c001","technique-persistent-kernels-current-c002"]} +{"receipt_id":"rem-technique-persistent-kernels-002","receipt_scope":"wiki","path":"wiki/techniques/persistent-kernels.md","original_body_sha256":"7d1d16d68cfae0d699d1b73b4d10ac71a12c878966081788134245f3861fc283","original_unit_ids":["u003"],"locator":"body:L9-L40","original_claim_id":"technique-persistent-kernels-original-c003","original_claim":"The displayed CLC CUDA loop queries work first, retries no-work races, and uses try_cancel for self-termination.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"PTX ISA 9.0","locator":"request/query and failure restriction","observation":"try_cancel requests cancellation of another grid entity; query decodes its response, and another request after observed failure from the same thread is undefined."}],"strongest_alternative":"The helper names abstract a higher-level queue unrelated to PTX names.","discriminating_result":"Comments explicitly map the helpers to CLC and self-cancellation, contradicting the normative operation.","action":"correct","action_reason":"Remove inverted/fabricated executable sketch.","proposed_replacement":"Give the exact initial-work/request/wait/decode/stop ordering and a pinned CUTLASS route.","replacement_claim_ids":["technique-persistent-kernels-current-c002","technique-persistent-kernels-current-c003"]} +{"receipt_id":"rem-technique-persistent-kernels-003","receipt_scope":"wiki","path":"wiki/techniques/persistent-kernels.md","original_body_sha256":"7d1d16d68cfae0d699d1b73b4d10ac71a12c878966081788134245f3861fc283","original_unit_ids":["u004","u005"],"locator":"body:L42-L58","original_claim_id":"technique-persistent-kernels-original-c004","original_claim":"The text cancel/wait/query loop completely describes the CLC protocol.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"PTX ISA 9.0","locator":"response transaction, fences, query and cluster rules","observation":"Initial blockIdx, expected bytes, proxy fences, failure restriction, and cluster-rank coordinate derivation are missing."}],"strongest_alternative":"It is explicitly control-flow-only pseudocode.","discriminating_result":"Even a control-flow summary that omits initial work and cluster coordinate derivation changes which tiles execute.","action":"correct","action_reason":"Replace with complete invariants and exact PTX forms linked elsewhere.","proposed_replacement":"State initial tile, one async request, 16-byte completion, success-only decode, software transform, and termination on failure.","replacement_claim_ids":["technique-persistent-kernels-current-c002","technique-persistent-kernels-current-c003"]} +{"receipt_id":"rem-technique-persistent-kernels-004","receipt_scope":"wiki","path":"wiki/techniques/persistent-kernels.md","original_body_sha256":"7d1d16d68cfae0d699d1b73b4d10ac71a12c878966081788134245f3861fc283","original_unit_ids":["u008","u009","u010","u011","u012","u013"],"locator":"body:L83-L88","original_claim_id":"technique-persistent-kernels-original-c006","original_claim":"CLC assigns arbitrary tiles, balances variable costs, minimizes ordinary tails, chains problems, self-terminates callers, and applies hardware swizzle.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0","locator":"scope and software coordinate transform","observation":"CLC redistributes existing unlaunched IDs for uneven SM availability; CUTLASS software handles swizzle, and neither problem chaining nor tail elimination is guaranteed."}],"strongest_alternative":"The table conflates the broader CUTLASS scheduler with the raw hardware intentionally.","discriminating_result":"It attributes every capability directly to CLC and even misstates try_cancel termination, so the distinction is materially lost.","action":"correct","action_reason":"Separate raw CLC, CUTLASS transforms, and out-of-scope mechanisms.","proposed_replacement":"Compare only initial/next work source, failure rule, cluster granularity, and software swizzle ownership.","replacement_claim_ids":["technique-persistent-kernels-current-c002","technique-persistent-kernels-current-c004"]} +{"receipt_id":"rem-technique-persistent-kernels-005","receipt_scope":"wiki","path":"wiki/techniques/persistent-kernels.md","original_body_sha256":"7d1d16d68cfae0d699d1b73b4d10ac71a12c878966081788134245f3861fc283","original_unit_ids":["u015"],"locator":"body:L94-L154","original_claim_id":"technique-persistent-kernels-original-c008","original_claim":"The displayed PersistentTileSchedulerSm100 class is valid simplified CUTLASS 4.5.0 code using clc_init/query_tile/try_cancel.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp","revision":"CUTLASS 4.5.0","locator":"scheduler interface","observation":"Those helpers/API do not exist; real code uses WorkTileInfo, get/fetch/advance calls, PipelineCLCFetchAsync, cluster rules and software transforms."}],"strongest_alternative":"The class is deliberately pedagogical pseudocode.","discriminating_result":"It is labeled CUTLASS code and invents an initialization/policy surface whose semantics contradict real CLC.","action":"correct","action_reason":"Use exact class/method routing and avoid fake implementation bodies.","proposed_replacement":"Document PersistentTileSchedulerSm100 with advance_to_next_work/fetch_next_work and PipelineCLCFetchAsync.","replacement_claim_ids":["technique-persistent-kernels-current-c003"]} +{"receipt_id":"rem-technique-persistent-kernels-006","receipt_scope":"wiki","path":"wiki/techniques/persistent-kernels.md","original_body_sha256":"7d1d16d68cfae0d699d1b73b4d10ac71a12c878966081788134245f3861fc283","original_unit_ids":["u016","u017","u018","u019","u020","u021"],"locator":"body:L158-L168","original_claim_id":"technique-persistent-kernels-original-c009","original_claim":"The tutorial isolates a 940-to-1476 TFLOP/s 57% CLC-persistence gain caused by tail elimination, fewer launches and hardware swizzle.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v6.cu","revision":"gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"v6 scheduling","observation":"v6 uses static persistence, not CLC."},{"path":"sources/blogs/tcgen05-tutorial.md","revision":"pinned article/source card","locator":"progression","observation":"The endpoints include multiple intervening changes and the author explicitly did not add CLC or swizzling."}],"strongest_alternative":"The numbers compare nonpersistent v3 with persistent v6 and are still an upper-bound persistence effect.","discriminating_result":"Warp specialization and 2-SM MMA also change, so no isolated persistence effect or causal decomposition can be computed.","action":"correct","action_reason":"Preserve the author progression only with exact variant scope and no CLC attribution.","proposed_replacement":"State that v6 is a source-reported static-persistent endpoint after multiple changes, not an ablation.","replacement_claim_ids":["technique-persistent-kernels-current-c005"]} +{"receipt_id":"rem-technique-persistent-kernels-007","receipt_scope":"wiki","path":"wiki/techniques/persistent-kernels.md","original_body_sha256":"7d1d16d68cfae0d699d1b73b4d10ac71a12c878966081788134245f3861fc283","original_unit_ids":["u022","u023","u024"],"locator":"body:L172-L174","original_claim_id":"technique-persistent-kernels-original-c010","original_claim":"A two-to-three-tiles/SM threshold, multi-problem chaining, and natural variable-cost handling are general CLC use rules.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0","locator":"motivation and scope","observation":"The source supports uneven-SM redistribution only; it defines none of those thresholds/capabilities."}],"strongest_alternative":"They are practical heuristics beyond the official documentation.","discriminating_result":"No target, scheduler, shapes or controlled measurements support them, and problem chaining is a different launch mechanism.","action":"correct","action_reason":"Replace prescriptive thresholds with evidence requirements and conserved-work limits.","proposed_replacement":"Measure static versus dynamic variants under controlled SM availability and problem geometry.","replacement_claim_ids":["technique-persistent-kernels-current-c005"]} +{"receipt_id":"rem-technique-persistent-kernels-008","receipt_scope":"wiki","path":"wiki/techniques/persistent-kernels.md","original_body_sha256":"7d1d16d68cfae0d699d1b73b4d10ac71a12c878966081788134245f3861fc283","original_unit_ids":["u026"],"locator":"body:L179-L179","original_claim_id":"technique-persistent-kernels-original-c012","original_claim":"A failed try_cancel is a race that the same thread must retry because another CTA may generate work.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"PTX ISA 9.0","locator":"failed request","observation":"A subsequent request by that thread after observed failure is undefined, and ClcIDs come only from the existing grid."}],"strongest_alternative":"Retry occurs before failure is observed or from another designated thread.","discriminating_result":"The original explicitly branches on the decoded no-work result and loops from the same CTA path.","action":"correct","action_reason":"Make failed response terminal for that requesting thread.","proposed_replacement":"Decode only after completion and stop requesting from that thread after failure.","replacement_claim_ids":["technique-persistent-kernels-current-c002"]} +{"receipt_id":"rem-source-cutlass-clc-001","receipt_scope":"source-trail","path":"sources/docs/cutlass-clc-documentation.md","original_body_sha256":"1acdc171198cb688fac9f555e16e54ade152be006b3dcb036db26e71b1fc84a1","original_unit_ids":[],"locator":"entire source summary","original_claim_id":"source-cutlass-clc-record","original_claim":"The rolling documentation card accurately says every tile is dynamically assigned, fixes pipeline depth/roles globally, and makes Stream-K hardware-assisted CLC behavior.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0 e406c186","locator":"Dynamic Scheduler rules 1-4 and implementation","observation":"The first tile is blockIdx, later work is a canceled ClcID, and stage/participant details are kernel configuration rather than universal CLC constants. Stream-K decomposition remains scheduler software."}],"strongest_alternative":"The source card summarizes one specific CUTLASS GEMM configuration.","discriminating_result":"It presents those values and behaviors as core mechanism without naming a configuration and conflicts with the same document's rule 4.","action":"correct","action_reason":"Pin to CUTLASS 4.5.0 and retain only normative request, ClcID, cluster, pipeline-class and software-transform facts.","proposed_replacement":"Provide exact tag links and distinguish raw CLC from CUTLASS scheduling policy.","replacement_claim_ids":["technique-persistent-kernels-current-c002","technique-persistent-kernels-current-c003","technique-persistent-kernels-current-c004"]} +{"receipt_id":"rem-technique-pipeline-stages-001","receipt_scope":"wiki","path":"wiki/techniques/pipeline-stages.md","original_body_sha256":"d990987f96b0c765ab7753798f189a2125cbab0619c1671b450e305a4c628966","original_unit_ids":["u001"],"locator":"body:L3-L3","original_claim_id":"technique-pipeline-stages-original-c001","original_claim":"Three-to-five SMEM stages hide latency entirely and are critical for every Hopper/Blackwell kernel.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-best-practices-guide/index.html#asynchronous-copy-from-global-memory-to-shared-memory","revision":"CUDA 13.0.2","locator":"asynchronous pipeline tradeoffs","observation":"Overlap depends on work balance and resource/occupancy costs; no universal stage range or full-hiding guarantee exists."}],"strongest_alternative":"The statement is conventional GEMM guidance, not universal.","discriminating_result":"It says entirely/critical for both architectures without a workload, and stage cost can reduce occupancy or add idle coordination.","action":"correct","action_reason":"Define the mechanism without guaranteed profitability.","proposed_replacement":"State disjoint staged ownership and require measurement of overlap versus resource cost.","replacement_claim_ids":["technique-pipeline-stages-current-c001","technique-pipeline-stages-current-c006"]} +{"receipt_id":"rem-technique-pipeline-stages-002","receipt_scope":"wiki","path":"wiki/techniques/pipeline-stages.md","original_body_sha256":"d990987f96b0c765ab7753798f189a2125cbab0619c1671b450e305a4c628966","original_unit_ids":["u002","u003"],"locator":"body:L7-L13","original_claim_id":"technique-pipeline-stages-original-c002","original_claim":"The tutorial progresses directly from 695 to 940 to 1476 TFLOP/s by adding pipelining then warp specialization.","original_verdict":"refuted","evidence":[{"path":"sources/blogs/tcgen05-tutorial.md","revision":"source commit 3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"progression","observation":"Warp specialization is v4=1208.83; 1475.93 is v6 after 2-SM MMA and static persistence."}],"strongest_alternative":"The last row means all later optimizations beginning with warp specialization.","discriminating_result":"Its singular + warp specialization label omits two material changes and invites a false attribution.","action":"correct","action_reason":"Report only the adjacent v2b/v3 pipeline comparison and scope it to the author setup.","proposed_replacement":"Use 695.43→939.61 as the source-reported pipeline step and list no later aggregate as a warp-only result.","replacement_claim_ids":["technique-pipeline-stages-current-c005"]} +{"receipt_id":"rem-technique-pipeline-stages-003","receipt_scope":"wiki","path":"wiki/techniques/pipeline-stages.md","original_body_sha256":"d990987f96b0c765ab7753798f189a2125cbab0619c1671b450e305a4c628966","original_unit_ids":["u006"],"locator":"body:L21-L110","original_claim_id":"technique-pipeline-stages-original-c005","original_claim":"The displayed pipelined_gemm is a complete safe three-stage TMA/tcgen05 implementation.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit","revision":"PTX ISA 9.0","locator":"MMA completion","observation":"The code releases the SMEM stage immediately after async MMA issue; syncwarp is not completion."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier","revision":"PTX ISA 9.0","locator":"reusable phases/publication","observation":"Phase/token state, initial empty seeding, proxy publication and tail are missing."}],"strongest_alternative":"Opaque helpers make MMA synchronous and barriers hide phases.","discriminating_result":"The comments explicitly call tcgen05 and direct mbarrier functions; the visible reuse edge remains invalid under that claim.","action":"correct","action_reason":"Replace unsafe code with a state-transition contract and pinned implementations.","proposed_replacement":"Document empty→loading→ready→consuming→empty, prologue, tail and completion edges.","replacement_claim_ids":["technique-pipeline-stages-current-c002"]} +{"receipt_id":"rem-technique-pipeline-stages-004","receipt_scope":"wiki","path":"wiki/techniques/pipeline-stages.md","original_body_sha256":"d990987f96b0c765ab7753798f189a2125cbab0619c1671b450e305a4c628966","original_unit_ids":["u008","u009","u010","u011"],"locator":"body:L118-L121","original_claim_id":"technique-pipeline-stages-original-c007","original_claim":"Two/three/four-to-five/>five stages have universal partial/full/full-with-margin/excessive latency-hiding classifications.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm","revision":"NVIDIA/cutlass v4.5.0 e406c186","locator":"stage policies","observation":"Counts derive from tile, SMEM and schedule; the release defines no universal classifications."}],"strongest_alternative":"The table is only a tuning heuristic.","discriminating_result":"It labels full hiding and standard choice without a target or measurement, which can reverse stage selection.","action":"correct","action_reason":"Replace fixed labels with resource and measurement inputs.","proposed_replacement":"Compute per-stage bytes, enumerate legal counts, compile, profile and benchmark.","replacement_claim_ids":["technique-pipeline-stages-current-c004","technique-pipeline-stages-current-c006"]} +{"receipt_id":"rem-technique-pipeline-stages-005","receipt_scope":"wiki","path":"wiki/techniques/pipeline-stages.md","original_body_sha256":"d990987f96b0c765ab7753798f189a2125cbab0619c1671b450e305a4c628966","original_unit_ids":["u019","u020"],"locator":"body:L134-L155","original_claim_id":"technique-pipeline-stages-original-c010","original_claim":"The displayed PTX is a complete phase protocol and autonomous TMA completion removes the producer from the critical path.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"PTX ISA 9.0","locator":"tensor-copy grammar","observation":"The copy form/context is incomplete and omits transaction setup/proxy ordering."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier","revision":"PTX ISA 9.0","locator":"phase completion","observation":"Phase advances after expected arrivals/transactions complete, not generically on issue, and no critical-path performance theorem is specified."}],"strongest_alternative":"It is illustrative PTX plus a qualitative benefit.","discriminating_result":"The incorrect phase explanation can deadlock real code and the critical-path claim is a performance conclusion without measurement.","action":"correct","action_reason":"Use exact lifecycle prose and distinguish autonomous completion from measured critical path.","proposed_replacement":"State arrive_expect_tx before TMA, hardware completion, consumer wait, MMA completion, and free-stage release.","replacement_claim_ids":["technique-pipeline-stages-current-c002","technique-pipeline-stages-current-c003"]} +{"receipt_id":"rem-technique-pipeline-stages-006","receipt_scope":"wiki","path":"wiki/techniques/pipeline-stages.md","original_body_sha256":"d990987f96b0c765ab7753798f189a2125cbab0619c1671b450e305a4c628966","original_unit_ids":["u021","u022"],"locator":"body:L159-L181","original_claim_id":"technique-pipeline-stages-original-c011","original_claim":"Modular proves a five-stage input pipeline alone reaches 85% SOTA, hides ~400-cycle HBM latency, and follows the displayed timeline.","original_verdict":"refuted","evidence":[{"url":"https://www.modular.com/blog/matrix-multiplication-on-nvidias-blackwell-part-3-the-optimizations-behind-85-of-sota-performance","revision":"fetched 2026-08-09","locator":"Kernel 6 and write-out pipeline","observation":"Five stages are used, but 85% follows a multi-step 2-SM/pipeline/warp-specialization/output-double-buffer sequence; the 400-cycle premise and exact timeline are absent."}],"strongest_alternative":"The paragraph merely says the series, not that stage count alone causes 85%.","discriminating_result":"The heading and code say the five-stage implementation reaches 85% and give an invented causal hardware latency.","action":"correct","action_reason":"Keep five stages as one source configuration and scope the endpoint to the full sequence.","proposed_replacement":"Describe Modular's five-stage choice without claiming isolated performance or fixed latency.","replacement_claim_ids":["technique-pipeline-stages-current-c005"]} +{"receipt_id":"rem-technique-pipeline-stages-007","receipt_scope":"wiki","path":"wiki/techniques/pipeline-stages.md","original_body_sha256":"d990987f96b0c765ab7753798f189a2125cbab0619c1671b450e305a4c628966","original_unit_ids":["u023","u025"],"locator":"body:L185-L187","original_claim_id":"technique-pipeline-stages-original-c012","original_claim":"Pipelining is never harmful for all GEMMs and almost always requires warp specialization.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-best-practices-guide/index.html#asynchronous-copy-from-global-memory-to-shared-memory","revision":"CUDA 13.0.2","locator":"pipeline considerations","observation":"Stages consume resources and coordination and may reduce occupancy; producer/consumer roles may be temporal or warp-specialized."}],"strongest_alternative":"The wording is strong advice for optimized tensor-core kernels only.","discriminating_result":"Even within that class, short loops, large stages and resource limits can make extra staging harmful or unnecessary.","action":"correct","action_reason":"Make stage depth and role specialization independent tuning dimensions.","proposed_replacement":"Benchmark stage count and role assignment for each concrete variant.","replacement_claim_ids":["technique-pipeline-stages-current-c001","technique-pipeline-stages-current-c006"]} +{"receipt_id":"rem-technique-pipeline-stages-008","receipt_scope":"wiki","path":"wiki/techniques/pipeline-stages.md","original_body_sha256":"d990987f96b0c765ab7753798f189a2125cbab0619c1671b450e305a4c628966","original_unit_ids":["u027"],"locator":"body:L192-L192","original_claim_id":"technique-pipeline-stages-original-c015","original_claim":"Barrier initialization overhead is negligible and a syncthreads block is a sufficient publication protocol.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-init","revision":"PTX ISA 9.0","locator":"init/visibility","observation":"No negligible-cost promise exists and required memory/proxy ordering depends on the consumers/async mechanism; syncthreads is not a universal substitute."}],"strongest_alternative":"The statement assumes conventional CTA-local initialization before any async work.","discriminating_result":"The performance adjective remains unsupported and the page purports to teach both PTX proxy and TMA ordering, where the simplification is unsafe.","action":"correct","action_reason":"State the ordering proof rather than a fixed initialization recipe/cost.","proposed_replacement":"Initialize with correct counts, publish with required fences/synchronization, and exclude setup from steady-state timing only when measured.","replacement_claim_ids":["technique-pipeline-stages-current-c002","technique-pipeline-stages-current-c006"]} +{"receipt_id":"rem-source-modular-blackwell-001","receipt_scope":"source-trail","path":"sources/blogs/modular-blackwell-matmul.md","original_body_sha256":"cf82a7f72e6de711cc8c0a9178b0624eea35944a3ae0de0f84259f372d169682","original_unit_ids":[],"locator":"entire source summary","original_claim_id":"source-modular-blackwell-record","original_claim":"The source card's staged percentages, 5-stage-optimal claim, free multicast reduction, fixed optimization ordering, and explanation of the remaining 15% accurately reflect the Modular series.","original_verdict":"refuted","evidence":[{"url":"https://www.modular.com/blog/matrix-multiplication-on-nvidias-blackwell-part-3-the-optimizations-behind-85-of-sota-performance","revision":"fetched 2026-08-09","locator":"2-SM, input pipeline, warp specialization, output double buffering and next steps","observation":"The article supports a five-stage choice and author-reported final 85% endpoint, but not the card's invented per-step percentages, universal optimality/order, free 4x claim, or attributed remaining-gap list."}],"strongest_alternative":"The card combines takeaways across the multi-part series and illustrative arithmetic.","discriminating_result":"It presents exact performance steps and universal prescriptions without versioned code/benchmark rows; the accessible primary page provides a different scoped progression.","action":"correct","action_reason":"Reduce to directly source-reported design choices, endpoint scope, and exact code links.","proposed_replacement":"Record Part 3's five-stage input pipeline, warp roles, output double buffering, author-reported 64-TFLOP/s final increment and 85%-of-SOTA endpoint with no universal claims.","replacement_claim_ids":["technique-pipeline-stages-current-c005"]} +{"receipt_id":"rem-technique-tile-scheduling-001","receipt_scope":"wiki","path":"wiki/techniques/tile-scheduling.md","original_body_sha256":"b8004e09034567bbd556d5cae14856d176cfa350af0214bb99c9aade49312f32","original_unit_ids":["u001"],"locator":"body:L3","original_claim_id":"technique-tile-scheduling-original-c001","original_claim":"Blackwell CLC supplies configurable hardware scheduling policies including swizzled raster, unlike Hopper software mappings.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0 e406c186","locator":"dynamic scheduler and software swizzle rules","observation":"CLC redistributes unlaunched IDs; CUTLASS performs raster/swizzle as a software coordinate transform."}],"strongest_alternative":"Hardware-assisted work reassignment makes the whole policy a hardware policy in ordinary shorthand.","discriminating_result":"The raw CLC interface has no raster or swizzle policy operand, so the distinction changes what programmers can configure.","action":"correct","action_reason":"Separate tile-coordinate order from work acquisition.","proposed_replacement":"Describe software raster/swizzle, CLC reassignment, and Stream-K decomposition as separate layers.","replacement_claim_ids":["technique-tile-scheduling-current-c001"]} +{"receipt_id":"rem-technique-tile-scheduling-002","receipt_scope":"wiki","path":"wiki/techniques/tile-scheduling.md","original_body_sha256":"b8004e09034567bbd556d5cae14856d176cfa350af0214bb99c9aade49312f32","original_unit_ids":["u002","u003"],"locator":"body:L9-L27","original_claim_id":"technique-tile-scheduling-original-c002","original_claim":"The linear map is categorically poor for L2 and B column zero is evicted before tile four.","original_verdict":"refuted","evidence":[{"command":"conda run -n base python enumerate quotient/remainder coordinates","environment":"Conda base, host CPU","reference":"coordinate reuse is countable but eviction requires cache evidence","result":"The mapping is row-major and adjacent tiles reuse A; no eviction follows from coordinates alone.","negative_control":"Equating a distinct B column with eviction invents a cache result."}],"strongest_alternative":"The eviction statement is merely an intuitive example.","discriminating_result":"It asserts a concrete cache event with no capacity, concurrency, policy or trace and is not reliable teaching material.","action":"correct","action_reason":"Retain exact mapping and make cache effects empirical.","proposed_replacement":"Show the row-major mapping, name operand reuse, and require profiling for L2 conclusions.","replacement_claim_ids":["technique-tile-scheduling-current-c002","technique-tile-scheduling-current-c005"]} +{"receipt_id":"rem-technique-tile-scheduling-003","receipt_scope":"wiki","path":"wiki/techniques/tile-scheduling.md","original_body_sha256":"b8004e09034567bbd556d5cae14856d176cfa350af0214bb99c9aade49312f32","original_unit_ids":["u004","u005"],"locator":"body:L31-L61","original_claim_id":"technique-tile-scheduling-original-c003","original_claim":"The displayed blocked raster is a valid column-major swizzle for all shapes and maximizes A/B reuse.","original_verdict":"refuted","evidence":[{"command":"conda run -n base python enumerate the displayed function on 3x5 and 4x4 grids","environment":"Conda base, host CPU","reference":"valid mapping must be in-bounds and bijective","result":"For 3x5 and swizzle 2 it emits (0,5), and its within-group ordering contradicts the comment.","negative_control":"Only testing divisible N hides the out-of-range final group."}],"strongest_alternative":"Production code would pad N, making the sample harmless.","discriminating_result":"Neither the signature nor preconditions require padding, and the page presents a general tiles_m by tiles_n mapping.","action":"delete","action_reason":"Remove incorrect executable-looking code and unsupported maximality.","proposed_replacement":null,"replacement_claim_ids":["technique-tile-scheduling-current-c002","technique-tile-scheduling-current-c005"]} +{"receipt_id":"rem-technique-tile-scheduling-004","receipt_scope":"wiki","path":"wiki/techniques/tile-scheduling.md","original_body_sha256":"b8004e09034567bbd556d5cae14856d176cfa350af0214bb99c9aade49312f32","original_unit_ids":["u008","u009"],"locator":"body:L80-L105","original_claim_id":"technique-tile-scheduling-original-c005","original_claim":"CLC provides an initialization-time hardware enum with linear, swizzled, column-first and Hilbert scheduling policies.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"try_cancel/query grammar","observation":"The interface requests cancellation into a response/mbarrier and queries it; no initialization or policy operand exists."}],"strongest_alternative":"The enum could be application-level pseudocode around raw CLC.","discriminating_result":"It is explicitly presented as the CLC hardware policy, and none of its helper calls are identified as fictional.","action":"delete","action_reason":"Replace the invented API with the documented work lifecycle.","proposed_replacement":null,"replacement_claim_ids":["technique-tile-scheduling-current-c003"]} +{"receipt_id":"rem-technique-tile-scheduling-005","receipt_scope":"wiki","path":"wiki/techniques/tile-scheduling.md","original_body_sha256":"b8004e09034567bbd556d5cae14856d176cfa350af0214bb99c9aade49312f32","original_unit_ids":["u011"],"locator":"body:L111-L137","original_claim_id":"technique-tile-scheduling-original-c007","original_claim":"The displayed CUTLASS scheduler class names, inheritance, default policy and GemmUniversal signature are valid CUTLASS 4.5.0.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/tile_scheduler.hpp","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"scheduler tags and SM100 selectors","observation":"Public tags and implementation mappings differ; the group wrapper is static and the shown composition is not the current API."}],"strongest_alternative":"The block is simplified pseudocode, so approximate type names are acceptable.","discriminating_result":"It is fenced as CUDA and labeled CUTLASS kernel definition; nonexistent identifiers prevent reuse and invert scheduler relationships.","action":"correct","action_reason":"Name exact public tags and keep implementation classes clearly internal.","proposed_replacement":"List PersistentScheduler, DynamicPersistentScheduler, StaticPersistentScheduler, StreamKScheduler and GroupScheduler with exact SM100 mappings.","replacement_claim_ids":["technique-tile-scheduling-current-c004"]} +{"receipt_id":"rem-technique-tile-scheduling-006","receipt_scope":"wiki","path":"wiki/techniques/tile-scheduling.md","original_body_sha256":"b8004e09034567bbd556d5cae14856d176cfa350af0214bb99c9aade49312f32","original_unit_ids":["u012","u013"],"locator":"body:L141-L163","original_claim_id":"technique-tile-scheduling-original-c008","original_claim":"The displayed B200 working-set arithmetic proves swizzling halves L2 misses for the stated GEMM.","original_verdict":"refuted","evidence":[{"command":"conda run -n base python enumerate the first 142 row-major coordinates in a 64x32 grid","environment":"Conda base, host CPU","reference":"count unique operand-panel coordinates","result":"The wave spans 5 M rows and all 32 N columns, not approximately 20 A rows and 5 B groups; miss rate is unmeasured.","negative_control":"Multiplying two 4.4 ratios recreates the incorrect approximately-20 count."}],"strongest_alternative":"The code is only a rough capacity estimate.","discriminating_result":"Even its coordinate counts are wrong, and capacity alone cannot establish hit rate or a 2x delta.","action":"delete","action_reason":"Replace fabricated cache results with a measurement protocol.","proposed_replacement":null,"replacement_claim_ids":["technique-tile-scheduling-current-c005"]} +{"receipt_id":"rem-technique-tile-scheduling-007","receipt_scope":"wiki","path":"wiki/techniques/tile-scheduling.md","original_body_sha256":"b8004e09034567bbd556d5cae14856d176cfa350af0214bb99c9aade49312f32","original_unit_ids":["u015","u016","u017","u018"],"locator":"body:L171-L174","original_claim_id":"technique-tile-scheduling-original-c010","original_claim":"The table gives universal last-wave utilization formulas and guarantees for linear, static, CLC and Stream-K schedulers.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/sm100_tile_scheduler_stream_k.hpp","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"decomposition arguments and data-parallel fallback","observation":"Stream-K behavior is parameterized and not guaranteed near 100%; CLC conserves IDs, and remainder zero needs a full-wave case."}],"strongest_alternative":"The table intentionally assumes a simple one-CTA-per-SM model.","discriminating_result":"Even under that model its remainder-zero formula is wrong, while its CLC and Stream-K rows introduce different mechanisms without stating assumptions.","action":"correct","action_reason":"Give exact scoped arithmetic and list what each mechanism can change.","proposed_replacement":"Use q,r wave arithmetic, distinguish r=0, and avoid utilization guarantees.","replacement_claim_ids":["technique-tile-scheduling-current-c006"]} +{"receipt_id":"rem-technique-tile-scheduling-008","receipt_scope":"wiki","path":"wiki/techniques/tile-scheduling.md","original_body_sha256":"b8004e09034567bbd556d5cae14856d176cfa350af0214bb99c9aade49312f32","original_unit_ids":["u021","u022"],"locator":"body:L178-L179","original_claim_id":"technique-tile-scheduling-original-c012","original_claim":"CLC fully occupies the eight-tile tail and Stream-K necessarily spreads those exact tiles over all 142 SMs.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0 e406c186","locator":"ClcID conservation rules","observation":"CLC changes ownership of existing unlaunched IDs but does not create or K-split them; Stream-K is separate and parameterized."}],"strongest_alternative":"Absorb and split are informal summaries of dynamic execution.","discriminating_result":"The wording claims a concrete occupancy result that neither conserved-ID reassignment nor optional decomposition guarantees.","action":"correct","action_reason":"Preserve the 8/142 baseline while describing mechanisms without invented outcomes.","proposed_replacement":"Say CLC may reduce delay from uneven worker availability; Stream-K may create K partitions with reduction overhead.","replacement_claim_ids":["technique-tile-scheduling-current-c006"]} +{"receipt_id":"rem-technique-tile-scheduling-009","receipt_scope":"wiki","path":"wiki/techniques/tile-scheduling.md","original_body_sha256":"b8004e09034567bbd556d5cae14856d176cfa350af0214bb99c9aade49312f32","original_unit_ids":["u023","u024","u025","u026"],"locator":"body:L183-L186","original_claim_id":"technique-tile-scheduling-original-c013","original_claim":"Swizzle, CLC, Stream-K and grouped scheduling each have a universal workload class where they are always, best, recommended for all, or essential.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/tile_scheduler.hpp","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"alternative scheduler tags/selectors","observation":"CUTLASS offers multiple parameterized routes and supplies no universal performance ranking."}],"strongest_alternative":"These are conventional tuning defaults, not formal guarantees.","discriminating_result":"Words such as always, all, best and essential erase known shape/resource/decomposition tradeoffs and are presented without benchmark scope.","action":"correct","action_reason":"Turn prescriptions into testable selection hypotheses.","proposed_replacement":"Choose candidates from workload structure, then benchmark exact shapes and inspect counters.","replacement_claim_ids":["technique-tile-scheduling-current-c007"]} +{"receipt_id":"rem-technique-tile-scheduling-010","receipt_scope":"wiki","path":"wiki/techniques/tile-scheduling.md","original_body_sha256":"b8004e09034567bbd556d5cae14856d176cfa350af0214bb99c9aade49312f32","original_unit_ids":["u027"],"locator":"body:L190","original_claim_id":"technique-tile-scheduling-original-c014","original_claim":"Every too-large swizzle exceeds L2 and every too-small swizzle loses locality.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/tile_scheduler_params.h","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"max_swizzle_size and raster_order arguments","observation":"These are tuning parameters, but the implementation defines no universal cache threshold or monotonic benefit."}],"strongest_alternative":"The sentence merely motivates tuning.","discriminating_result":"Its absolute causal explanation is stronger than the useful and supportable point that workload-specific tuning is required.","action":"correct","action_reason":"Retain tuning advice without a false monotonic cache model.","proposed_replacement":"Sweep legal raster/swizzle settings and measure cache traffic and time.","replacement_claim_ids":["technique-tile-scheduling-current-c005","technique-tile-scheduling-current-c007"]} +{"receipt_id":"rem-technique-tile-scheduling-011","receipt_scope":"wiki","path":"wiki/techniques/tile-scheduling.md","original_body_sha256":"b8004e09034567bbd556d5cae14856d176cfa350af0214bb99c9aade49312f32","original_unit_ids":["u028","u029"],"locator":"body:L191-L192","original_claim_id":"technique-tile-scheduling-original-c015","original_claim":"CLC acquisition costs tens of cycles and Stream-K always uses atomic accumulation at K splits.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/sm100_tile_scheduler_stream_k.hpp","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"reduction_mode and decomposition arguments","observation":"Reduction/decomposition is configurable rather than universally atomic, and pinned CLC material gives no fixed cycle cost."}],"strongest_alternative":"Atomic is generic shorthand for merging partial accumulations and tens of cycles is an order-of-magnitude warning.","discriminating_result":"CUTLASS exposes distinct reduction modes, and an unsourced latency estimate can mislead scheduler decisions across hardware/configurations.","action":"correct","action_reason":"State observable costs and configurable reductions rather than invented constants.","proposed_replacement":"Measure acquisition/synchronization overhead and account for the selected Stream-K reduction mode.","replacement_claim_ids":["technique-tile-scheduling-current-c004","technique-tile-scheduling-current-c007"]} +{"receipt_id":"rem-technique-warp-specialization-001","receipt_scope":"wiki","path":"wiki/techniques/warp-specialization.md","original_body_sha256":"48ec87e638f28a2e42314d08521041a40b518e4fe0d8ba0b984f9764efcf3fd9","original_unit_ids":["u001"],"locator":"body:L3","original_claim_id":"technique-warp-specialization-original-c001","original_claim":"SM100 replaces Hopper's four-warp model with a fixed 16-warp CTA and one required MMA warp.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v4.cu","revision":"gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"NUM_WARPS and role branches","observation":"The source-reported SM100 example uses four total warps and only elected lanes for TMA/MMA issue."}],"strongest_alternative":"Sixteen warps describes optimized production kernels rather than an ISA rule.","discriminating_result":"The page calls it the SM100 structure and architectural replacement; valid pinned kernels use different counts.","action":"correct","action_reason":"Separate single-thread issue from software CTA/role sizing.","proposed_replacement":"Define specialization and state that role counts are kernel-dependent.","replacement_claim_ids":["technique-warp-specialization-current-c001"]} +{"receipt_id":"rem-technique-warp-specialization-002","receipt_scope":"wiki","path":"wiki/techniques/warp-specialization.md","original_body_sha256":"48ec87e638f28a2e42314d08521041a40b518e4fe0d8ba0b984f9764efcf3fd9","original_unit_ids":["u002","u003","u004","u005"],"locator":"body:L7-L13","original_claim_id":"technique-warp-specialization-original-c002","original_claim":"The canonical SM100 GEMM assigns warp 0 TMA, warp 1 MMA and warps 2-15 epilogue.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/sm100_gemm_tma_warpspecialized.hpp","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"WarpCategory and thread counts","observation":"CUTLASS uses MMA=0, scheduler=1, mainloop load=2, epilogue load=3 and a variable epilogue count."}],"strongest_alternative":"The table is a pedagogical partition, not the CUTLASS layout.","discriminating_result":"It is labeled canonical and later attributed verbatim to CUTLASS, so exact IDs materially mislead implementation.","action":"correct","action_reason":"Replace one invented canonical layout with version-pinned examples.","proposed_replacement":"Show tutorial, CUTLASS and FA4 partitions side by side with explicit scope.","replacement_claim_ids":["technique-warp-specialization-current-c002"]} +{"receipt_id":"rem-technique-warp-specialization-003","receipt_scope":"wiki","path":"wiki/techniques/warp-specialization.md","original_body_sha256":"48ec87e638f28a2e42314d08521041a40b518e4fe0d8ba0b984f9764efcf3fd9","original_unit_ids":["u006"],"locator":"body:L15","original_claim_id":"technique-warp-specialization-original-c003","original_claim":"Single-thread tcgen05 issue means hardware handles operand movement internally and makes one whole MMA warp the architecture's unit.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"operands and thread semantics","observation":"One thread issues, but explicit SMEM/TMEM operands, descriptors, lifetimes and ordering remain; PTX mandates no warp role."}],"strongest_alternative":"Hardware handles the matrix operation after issue, which is what the sentence means.","discriminating_result":"The omitted software movement/lifetime work is exactly what warp specialization assigns, so the shorthand reverses the programming responsibility.","action":"correct","action_reason":"Keep the precise Hopper/SM100 issue distinction without implying autonomous operand staging.","proposed_replacement":"State issue granularity, D location and remaining completion/operand requirements.","replacement_claim_ids":["technique-warp-specialization-current-c001"]} +{"receipt_id":"rem-technique-warp-specialization-004","receipt_scope":"wiki","path":"wiki/techniques/warp-specialization.md","original_body_sha256":"48ec87e638f28a2e42314d08521041a40b518e4fe0d8ba0b984f9764efcf3fd9","original_unit_ids":["u007","u008","u009","u010","u011","u012"],"locator":"body:L21-L26","original_claim_id":"technique-warp-specialization-original-c004","original_claim":"The architecture table universally fixes producer/epilogue roles, barriers and low SM100 register pressure.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/sm100_gemm_tma_warpspecialized.hpp","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"role sizing and pipelines","observation":"Role count and synchronization depend on collectives/configuration; TMEM changes D placement but does not guarantee low GPR use."}],"strongest_alternative":"The table only sketches a common dense GEMM.","discriminating_result":"It supplies exact fixed counts and qualitative pressure claims while omitting actual CUTLASS scheduler/load roles.","action":"correct","action_reason":"Limit the comparison to normative issue/D placement and use concrete source layouts for software roles.","proposed_replacement":"Compare WGMMA/tcgen05 programming-model facts, then separately show source-specific role maps.","replacement_claim_ids":["technique-warp-specialization-current-c001","technique-warp-specialization-current-c002"]} +{"receipt_id":"rem-technique-warp-specialization-005","receipt_scope":"wiki","path":"wiki/techniques/warp-specialization.md","original_body_sha256":"48ec87e638f28a2e42314d08521041a40b518e4fe0d8ba0b984f9764efcf3fd9","original_unit_ids":["u013","u014"],"locator":"body:L30-L155","original_claim_id":"technique-warp-specialization-original-c005","original_claim":"The displayed 512-thread skeleton safely reuses SMEM and hands completed TMEM accumulators to epilogue warps.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"commit/completion","observation":"The code releases operands and signals output via ordinary arrival without committing/waiting for asynchronous MMA completion."}],"strongest_alternative":"Helper names may hide completion internally.","discriminating_result":"The visible release follows tcgen05_mma plus syncwarp and the accumulator barrier is an unrelated manual arrival; no helper receives that barrier.","action":"delete","action_reason":"Unsafe pseudo-code is worse than a precise protocol checklist and pinned full sources.","proposed_replacement":null,"replacement_claim_ids":["technique-warp-specialization-current-c003"]} +{"receipt_id":"rem-technique-warp-specialization-006","receipt_scope":"wiki","path":"wiki/techniques/warp-specialization.md","original_body_sha256":"48ec87e638f28a2e42314d08521041a40b518e4fe0d8ba0b984f9764efcf3fd9","original_unit_ids":["u018","u019"],"locator":"body:L164-L178","original_claim_id":"technique-warp-specialization-original-c007","original_claim":"The displayed PTX ordinary arrivals/try_wait instructions constitute the TMA-ready and MMA-release protocol.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"complete_tx protocol","observation":"TMA needs expected transactions/completion; MMA release needs tcgen05.commit completion; one try_wait is only a predicate query."}],"strongest_alternative":"The fragment intentionally omits surrounding copy/commit operations.","discriminating_result":"It explicitly says the operations map to this block and labels manual arrival as data-ready, teaching the wrong completion source.","action":"delete","action_reason":"Replace incomplete PTX with named obligations and authoritative links.","proposed_replacement":null,"replacement_claim_ids":["technique-warp-specialization-current-c003"]} +{"receipt_id":"rem-technique-warp-specialization-007","receipt_scope":"wiki","path":"wiki/techniques/warp-specialization.md","original_body_sha256":"48ec87e638f28a2e42314d08521041a40b518e4fe0d8ba0b984f9764efcf3fd9","original_unit_ids":["u020","u021"],"locator":"body:L182-L211","original_claim_id":"technique-warp-specialization-original-c008","original_claim":"CUTLASS 4.5.0 implements a CollectiveMma_1SM type with a fixed Shape<1,1,14> and the displayed dispatch methods.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/sm100_gemm_tma_warpspecialized.hpp","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"real kernel dispatch","observation":"Those identifiers/shape are absent; WarpCategory and collective-derived epilogue threads implement the route."}],"strongest_alternative":"It is named simplified code and approximate identifiers may convey the idea.","discriminating_result":"Nonexistent APIs and incorrect roles defeat the stated goal of showing CUTLASS 4.5.0 behavior.","action":"delete","action_reason":"Use exact source names and line-level descriptions instead of fabricated code.","proposed_replacement":null,"replacement_claim_ids":["technique-warp-specialization-current-c002"]} +{"receipt_id":"rem-technique-warp-specialization-008","receipt_scope":"wiki","path":"wiki/techniques/warp-specialization.md","original_body_sha256":"48ec87e638f28a2e42314d08521041a40b518e4fe0d8ba0b984f9764efcf3fd9","original_unit_ids":["u022"],"locator":"body:L215","original_claim_id":"technique-warp-specialization-original-c009","original_claim":"Warp specialization is always required for Blackwell GEMMs because tcgen05 assumes it.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v3.cu","revision":"gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"pre-v4 tcgen05 pipeline","observation":"The tutorial runs tcgen05 before introducing warp specialization as an optimization."}],"strongest_alternative":"Always refers to high-performance production GEMMs, not correctness.","discriminating_result":"No primary evidence establishes universal profitability, and the page explicitly attributes the requirement to the instruction model.","action":"correct","action_reason":"Present specialization as a candidate optimization with an isolated source-reported example.","proposed_replacement":"Measure v3/v4 scoped results and profile role balance for each kernel.","replacement_claim_ids":["technique-warp-specialization-current-c004"]} +{"receipt_id":"rem-technique-warp-specialization-009","receipt_scope":"wiki","path":"wiki/techniques/warp-specialization.md","original_body_sha256":"48ec87e638f28a2e42314d08521041a40b518e4fe0d8ba0b984f9764efcf3fd9","original_unit_ids":["u024"],"locator":"body:L217","original_claim_id":"technique-warp-specialization-original-c011","original_claim":"Warp specialization is the cleanest decomposition for every producer-consumer pipeline with TMA/MMA overlap.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v3.cu","revision":"gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"temporal pipelining","observation":"The source overlaps stages before static role specialization; profitability depends on balance/resources."}],"strongest_alternative":"Cleanest is subjective design guidance, not a factual optimum.","discriminating_result":"As technical guidance it applies to all such kernels without criteria and can increase idle warps or register/occupancy costs.","action":"correct","action_reason":"Supply selection criteria and controlled comparison instead of a universal adjective.","proposed_replacement":"Compare temporal and spatial specialization at equal tile/stage settings.","replacement_claim_ids":["technique-warp-specialization-current-c004"]} +{"receipt_id":"rem-technique-warp-specialization-010","receipt_scope":"wiki","path":"wiki/techniques/warp-specialization.md","original_body_sha256":"48ec87e638f28a2e42314d08521041a40b518e4fe0d8ba0b984f9764efcf3fd9","original_unit_ids":["u025"],"locator":"body:L221","original_claim_id":"technique-warp-specialization-original-c012","original_claim":"A fixed 14-warp epilogue is underutilized for stores and increasingly beneficial for complex epilogues.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/sm100_gemm_tma_warpspecialized.hpp","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"NumEpilogueThreads","observation":"The count derives from CollectiveEpilogue and no monotonic complexity/performance result is given."}],"strongest_alternative":"The statement is a plausible load-balance warning.","discriminating_result":"It is anchored to a nonexistent universal 14-warp design and asserts benefit without measurements.","action":"correct","action_reason":"Generalize to measurable role imbalance and configuration-derived counts.","proposed_replacement":"Measure per-role active/stall time and vary participant counts where the implementation permits.","replacement_claim_ids":["technique-warp-specialization-current-c004"]} +{"receipt_id":"rem-technique-warp-specialization-011","receipt_scope":"wiki","path":"wiki/techniques/warp-specialization.md","original_body_sha256":"48ec87e638f28a2e42314d08521041a40b518e4fe0d8ba0b984f9764efcf3fd9","original_unit_ids":["u026"],"locator":"body:L222","original_claim_id":"technique-warp-specialization-original-c013","original_claim":"One MMA warp forbids independent MMA overlap, so larger tiles should use 2-SM mode.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v4.cu","revision":"gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"MMA issue loops","observation":"One elected lane submits multiple asynchronous MMA operations before commit; 2-SM is a separate configuration choice."}],"strongest_alternative":"Streams means independent issue warps, not outstanding operations.","discriminating_result":"The conclusion still does not follow: CTA-group choice is constrained by shapes/resources and not a remedy mandated by the number of control warps.","action":"correct","action_reason":"Remove the false causal rule and make 1SM/2SM an independently benchmarked configuration.","proposed_replacement":"Choose CTA-group mode from legal shapes and measured resource/performance behavior.","replacement_claim_ids":["technique-warp-specialization-current-c004"]} +{"receipt_id":"rem-technique-cccl-memory-primitives-001","receipt_scope":"wiki","path":"wiki/techniques/cccl-memory-primitives.md","original_body_sha256":"c8d85d250f8216c456f2c7737203ae250b1b6d85d9d6150336ed406b13ceb20d","original_unit_ids":["u001"],"locator":"body:L3-L6","original_claim_id":"technique-cccl-memory-primitives-original-c001","original_claim":"The two linked PRs broadly cover SM100/SM90 scan, top-k, fill, histogram, reduce, block load/store and DSA tuning policies.","original_verdict":"refuted","evidence":[{"path":"artifacts/prs/cccl/PR-3559/diff.patch","revision":"captured merge 25523da2","locator":"complete change","observation":"PR 3559 is SM100 scan tuning/type dispatch only."},{"path":"artifacts/prs/cccl/PR-6152/diff.patch","revision":"captured merge 3fb05826","locator":"complete change","observation":"PR 6152 is only a TopK debug-log typo fix."}],"strongest_alternative":"The text offers general discovery guidance rather than attributing every primitive to both PRs.","discriminating_result":"The page has only these sources and its Blackwell relevance explicitly calls them scan/top-k tuning PRs, making the scope attribution unavoidable.","action":"correct","action_reason":"Teach the exact reusable decisions and limitations of each PR.","proposed_replacement":"Separate PR 3559's SM100 scan policies from PR 6152's non-semantic debug fix.","replacement_claim_ids":["technique-cccl-memory-primitives-current-c001","technique-cccl-memory-primitives-current-c003"]} +{"receipt_id":"rem-technique-cccl-memory-primitives-002","receipt_scope":"wiki","path":"wiki/techniques/cccl-memory-primitives.md","original_body_sha256":"c8d85d250f8216c456f2c7737203ae250b1b6d85d9d6150336ed406b13ceb20d","original_unit_ids":["u002"],"locator":"body:L8-L16","original_claim_id":"technique-cccl-memory-primitives-original-c002","original_claim":"ITEMS_PER_THREAD >= 4 is a meaningful proxy for vectorized memory access.","original_verdict":"refuted","evidence":[{"path":"artifacts/prs/cccl/PR-3559/key-files/cub/cub/device/dispatch/tuning/tuning_scan.cuh","revision":"captured merge 25523da2","locator":"sm100_tuning fields","observation":"Items, threads, load/store algorithm, cache modifier and delay are independent policy choices classified by types/operator."}],"strongest_alternative":"The boolean is only an application experiment label, not a CUB guarantee.","discriminating_result":"Naming it vectorized makes results easy to misattribute and omits the conditions that actually determine vector access.","action":"delete","action_reason":"Replace fabricated policy code with an exact field matrix from the PR.","proposed_replacement":null,"replacement_claim_ids":["technique-cccl-memory-primitives-current-c002"]} +{"receipt_id":"rem-source-cccl-3559-001","receipt_scope":"source-trail","path":"sources/prs/cccl/PR-3559.md","original_body_sha256":"dc84c4a3ef22fad9b4b63cc389ca15fa797ccd196e42e917052c357d5f35bd40","original_unit_ids":[],"locator":"frontmatter tags/techniques/kernel_types","original_claim_id":"source-cccl-3559-taxonomy-original-c001","original_claim":"CCCL PR 3559 is a GEMM source with no scan technique or kernel type.","original_verdict":"refuted","evidence":[{"path":"artifacts/prs/cccl/PR-3559/diff.patch","revision":"captured merge 25523da2","locator":"changed paths and tuning_scan.cuh additions","observation":"Every substantive change concerns scan dispatch/tuning/testing, including explicit SM100 policies; no GEMM is changed."}],"strongest_alternative":"GEMM is a broad GPU-compute discovery tag.","discriminating_result":"The controlled vocabulary has exact parallel-scan/scan values, and GEMM misroutes queries away from the source's only algorithm.","action":"correct","action_reason":"Normalize source taxonomy to its exact algorithm.","proposed_replacement":"Use parallel-scan tags/techniques and scan kernel type.","replacement_claim_ids":["technique-cccl-memory-primitives-current-c001","technique-cccl-memory-primitives-current-c002"]} +{"receipt_id":"rem-source-cccl-6152-001","receipt_scope":"source-trail","path":"sources/prs/cccl/PR-6152.md","original_body_sha256":"fe4343a51075951c4d025db2f222c337706f2b025c614dfe1a418370ff5ed21d","original_unit_ids":[],"locator":"frontmatter architectures/tags/techniques/kernel_types","original_claim_id":"source-cccl-6152-taxonomy-original-c001","original_claim":"A generic DispatchTopK debug-log correction is an SM100 GEMM source with no TopK classification.","original_verdict":"refuted","evidence":[{"path":"artifacts/prs/cccl/PR-6152/diff.patch","revision":"captured merge 3fb05826","locator":"complete patch","observation":"The patch changes only generic DispatchTopK debug formatting/variable names and has no architecture guard or GEMM code."}],"strongest_alternative":"The repository collection is SM100-focused and therefore tags every included PR sm100/gemm.","discriminating_result":"Source metadata is used for semantic query routing; exact top-k-selection/topk and no architecture claim preserve the actual scope.","action":"correct","action_reason":"Normalize algorithm and architecture taxonomy without implying a performance change.","proposed_replacement":"Use top-k-selection/topk and an empty architecture list.","replacement_claim_ids":["technique-cccl-memory-primitives-current-c003"]} +{"receipt_id":"rem-technique-external-source-map-research-001","receipt_scope":"wiki","path":"wiki/techniques/external-source-map-research.md","original_body_sha256":"0455e988208712c28de6084a897d46734ceafe52da948a9d9d003458f845b2c6","original_unit_ids":["u002"],"locator":"body:L8-L12","original_claim_id":"technique-external-source-map-research-original-c002","original_claim":"The unpinned clone-and-grep snippet is reproducible enough to support exact implementation citations.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-ext-cfx","revision":"ColfaxResearch/cfx-article-src fbecfed88de2e4246f104a023188ba722937c5fc","locator":"git revision and repository tree","observation":"The displayed clone follows a moving default branch and records neither this revision nor an exact file."},{"path":"/tmp/kernelwiki-ext-load-store/README.md","revision":"simveit/load_and_store 05d828cf910dd43f0053ddbbe4744218a06e9d7f","locator":"repository scope and six source files","observation":"The repository only demonstrates ldmatrix/stmatrix, so the shared broad grep does not establish the other mechanisms."}],"strongest_alternative":"The commands are only discovery commands and need not be reproducible.","discriminating_result":"The page labels reproducibility as snippet and immediately requires exact citations; a moving checkout cannot reproduce the inspected implementation.","action":"correct","action_reason":"Pin every checkout and show mechanism-specific file searches.","proposed_replacement":"Fetch an exact commit, checkout detached, and grep exact subtrees/symbols.","replacement_claim_ids":["technique-external-source-map-research-current-c002"]} +{"receipt_id":"rem-technique-external-source-map-research-002","receipt_scope":"wiki","path":"wiki/techniques/external-source-map-research.md","original_body_sha256":"0455e988208712c28de6084a897d46734ceafe52da948a9d9d003458f845b2c6","original_unit_ids":["u009"],"locator":"body:L25-L27","original_claim_id":"technique-external-source-map-research-original-c006","original_claim":"A source page plus a path, commit, or URL is sufficient implementation provenance.","original_verdict":"refuted","evidence":[{"path":"plan.md","revision":"repository remediation contract","locator":"sections 5 and 8","observation":"Evidence requires an immutable revision and an exact source locator; a moving URL alone is insufficient."},{"path":"sources/blogs/colfax-cutlass-kernels.md","revision":"baseline body 4fa361c619785bf4e42b8fd58ce26e98c998de467797e6cea6b778a328fe3184","locator":"frontmatter and body","observation":"The summary overstates SM100, persistent, and scheduling scope compared with the pinned Hopper repository."}],"strongest_alternative":"A direct URL can contain a commit fragment and therefore be immutable.","discriminating_result":"The original disjunction also accepts an unversioned repository URL; requiring both revision and locator removes that ambiguity.","action":"correct","action_reason":"Require a pinned upstream revision and exact file/symbol locator, treating source pages as discovery leads only.","proposed_replacement":"Record repository, full revision, exact path and symbol or line range.","replacement_claim_ids":["technique-external-source-map-research-current-c006"]} +{"receipt_id":"rem-technique-external-source-map-research-003","receipt_scope":"wiki","path":"wiki/techniques/external-source-map-research.md","original_body_sha256":"0455e988208712c28de6084a897d46734ceafe52da948a9d9d003458f845b2c6","original_unit_ids":["u001"],"locator":"frontmatter architectures and blackwell_relevance","original_claim_id":"technique-external-source-map-research-original-c007","original_claim":"The linked repositories demonstrate the listed mechanisms on both Hopper- and Blackwell-era kernels.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-ext-cfx","revision":"ColfaxResearch/cfx-article-src fbecfed88de2e4246f104a023188ba722937c5fc","locator":"modern example Makefiles and SM90 source symbols","observation":"The relevant examples compile for SM90a and use SM90 TMA/GMMA facilities."},{"path":"/tmp/kernelwiki-ext-colfax-cutlass","revision":"ColfaxResearch/cutlass-kernels 84f0802e2b4a1bf068ac70359f20ffdb368c8f6a","locator":"FMHA READMEs/compile scripts and GEMM source","observation":"The repository explicitly targets Hopper SM90/SM90a with CUDA 12.2/12.3."},{"path":"/tmp/kernelwiki-ext-transpose","revision":"simveit/effective_transpose 994b2b5acaa67f80e411df3e8274b6ae13fd1949","locator":"README.md and Makefile","observation":"It is a Hopper transpose example compiled for sm_90a."},{"path":"/tmp/kernelwiki-ext-load-store","revision":"simveit/load_and_store 05d828cf910dd43f0053ddbbe4744218a06e9d7f","locator":"README.md and Makefile","observation":"It contains ldmatrix/stmatrix demos compiled for sm_90a, not Blackwell code."}],"strongest_alternative":"Hopper examples remain conceptually relevant to Blackwell and therefore qualify as Blackwell-era sources.","discriminating_result":"Conceptual transfer is not target compatibility; no pinned linked implementation uses an SM100 target or Blackwell-only instruction.","action":"correct","action_reason":"Scope demonstrated implementations to SM90a and make Blackwell transfer require separate SM100 evidence.","proposed_replacement":"Use the repositories as Hopper candidate sources; revalidate every transfer against pinned Blackwell primary sources and measurements.","replacement_claim_ids":["technique-external-source-map-research-current-c001","technique-external-source-map-research-current-c006"]} +{"receipt_id":"rem-source-nvidia-code-samples-001","receipt_scope":"source-trail","path":"sources/blogs/nvidia-code-samples.md","original_body_sha256":"3acd777672addae9f33ed79479dd7fd04fe3256fd6b0e44788ce5e8bda0b1138","original_unit_ids":[],"locator":"frontmatter architectures/tags and body scope","original_claim_id":"source-nvidia-code-samples-taxonomy-original-c001","original_claim":"The broad NVIDIA blog sample collection is specifically SM90/SM100 and demonstrates swizzling, vectorized loads, and register budgeting.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-ext-nvidia-samples","revision":"NVIDIA-developer-blog/code-samples 3350d216083a902ccbf5b31665e3b82096a75b55","locator":"series/cuda-cpp/coalescing-global, shared-memory and transpose; posts/tensor-cores/simpleTensorCoreGEMM.cu; complete file tree","observation":"The relevant files are cross-generation CUDA examples for scalar coalescing, shared-memory staging/padding, transpose, and an older WMMA GEMM; the tree provides no repository-wide SM90/SM100 or listed modern-mechanism guarantee."}],"strongest_alternative":"Current GPUs can compile old samples, so SM90 and SM100 are useful discovery labels.","discriminating_result":"Runnable compatibility was not established and architecture labels imply source targeting; empty architecture scope preserves the exact cross-generation nature.","action":"correct","action_reason":"Pin exact example paths and remove unsupported architecture/mechanism routing.","proposed_replacement":"Use no architecture claim and scope tags/body to CUDA, GEMM, and shared-memory examples at the pinned revision.","replacement_claim_ids":["technique-external-source-map-research-current-c003"]} +{"receipt_id":"rem-source-colfax-article-source-kernels-001","receipt_scope":"source-trail","path":"sources/blogs/colfax-article-source-kernels.md","original_body_sha256":"c9f48405f0d0d5341c2d151554eadeed44a7c0353c4b95fe2ef93767a25838ff","original_unit_ids":[],"locator":"frontmatter architectures and body scope","original_claim_id":"source-colfax-article-source-kernels-taxonomy-original-c001","original_claim":"The cfx article source examples directly cover both SM90 and SM100.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-ext-cfx","revision":"ColfaxResearch/cfx-article-src fbecfed88de2e4246f104a023188ba722937c5fc","locator":"pipeline-gemm, streamk, tma and transpose-cute build/source files","observation":"The relevant examples compile for SM90a and use SM90 TMA/GMMA; no SM100 target or tcgen05 source appears."}],"strongest_alternative":"The latest repository commit postdates Blackwell and its portable C++ could run there.","discriminating_result":"Commit date and possible forward compatibility do not establish a compiled or tested SM100 implementation.","action":"correct","action_reason":"Use the exact SM90a target and record the audit revision and paths.","proposed_replacement":"Architecture sm90a with a pinned-revision Hopper source map.","replacement_claim_ids":["technique-external-source-map-research-current-c001","technique-external-source-map-research-current-c004","technique-external-source-map-research-current-c005"]} +{"receipt_id":"rem-source-colfax-cutlass-kernels-001","receipt_scope":"source-trail","path":"sources/blogs/colfax-cutlass-kernels.md","original_body_sha256":"4fa361c619785bf4e42b8fd58ce26e98c998de467797e6cea6b778a328fe3184","original_unit_ids":[],"locator":"frontmatter architectures/tags and body scope","original_claim_id":"source-colfax-cutlass-kernels-taxonomy-original-c001","original_claim":"Colfax CUTLASS kernels cover SM100 plus persistent and tile-scheduling implementations.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-ext-colfax-cutlass","revision":"ColfaxResearch/cutlass-kernels 84f0802e2b4a1bf068ac70359f20ffdb368c8f6a","locator":"README files, compile scripts and complete tracked file search","observation":"The implementations explicitly target SM90/SM90a Hopper GEMM/FMHA with TMA, WGMMA, pipelining, and optional warp specialization; no SM100, persistent scheduler, or tile scheduler implementation was found."}],"strongest_alternative":"Warp-specialized GEMM is often persistent and therefore deserves both scheduling tags.","discriminating_result":"The pinned source contains no persistent/tile-scheduler symbol or implementation contract; architecture and scheduling properties must not be inferred from a broad kernel family.","action":"correct","action_reason":"Narrow taxonomy and description to the exact Hopper GEMM/FMHA mechanisms.","proposed_replacement":"Architecture sm90a; tags CUDA/CuTe/GEMM/TMA/WGMMA/pipeline/warp-specialization.","replacement_claim_ids":["technique-external-source-map-research-current-c004"]} +{"receipt_id":"rem-source-simveit-effective-transpose-001","receipt_scope":"source-trail","path":"sources/blogs/simveit-effective-transpose.md","original_body_sha256":"cc5c5f7716d476e538dbc9bdeb00aca399b7275d5c583b70fb568780dee96ed1","original_unit_ids":[],"locator":"frontmatter architectures/tags and body scope","original_claim_id":"source-simveit-effective-transpose-taxonomy-original-c001","original_claim":"effective_transpose is an SM90/SM100 CuTe GEMM/vectorized-load source.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-ext-transpose","revision":"simveit/effective_transpose 994b2b5acaa67f80e411df3e8274b6ae13fd1949","locator":"README.md, Makefile and transpose_*.cu","observation":"The repository is raw CUDA C++ for Hopper SM90a TMA transpose and 128-byte tensor-map swizzling; it contains no CuTe DSL, GEMM, SM100 target, or explicit vector-load implementation."}],"strongest_alternative":"Tensor-map transfers are vectorized tiled loads in a general sense.","discriminating_result":"The controlled vectorized-load tag implies an explicit load-width technique, while the source demonstrates TMA tiles and swizzle; exact mechanism tags are available.","action":"correct","action_reason":"Replace inferred categories with the repository's explicit language, architecture, and mechanisms.","proposed_replacement":"Architecture sm90a; tags cuda-cpp, tma, swizzling, shared-memory-optimization.","replacement_claim_ids":["technique-external-source-map-research-current-c003"]} +{"receipt_id":"rem-source-simveit-load-and-store-001","receipt_scope":"source-trail","path":"sources/blogs/simveit-load-and-store.md","original_body_sha256":"dfcd8b289435ef17b291c0cbf4f023a543499185c8254b363ace256c079b0322","original_unit_ids":[],"locator":"frontmatter architectures/tags and body scope","original_claim_id":"source-simveit-load-and-store-taxonomy-original-c001","original_claim":"load_and_store is an SM90/SM100 CuTe GEMM/TMA/WGMMA/vectorized-load source.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-ext-load-store","revision":"simveit/load_and_store 05d828cf910dd43f0053ddbbe4744218a06e9d7f","locator":"README.md, Makefile and six .cu files","observation":"The repository consists only of inline-PTX ldmatrix/stmatrix x1/x2/x4 examples compiled for sm_90a; it contains none of the broader claimed mechanisms or CuTe code."}],"strongest_alternative":"ldmatrix/stmatrix move matrix fragments and so can be grouped under vectorized loads and WGMMA.","discriminating_result":"The repository directly names two independent PTX instruction families; it neither issues WGMMA/TMA nor demonstrates generic vector-width selection.","action":"correct","action_reason":"Use the exact instruction and language taxonomy.","proposed_replacement":"Architecture sm90a; tags cuda-cpp, ptx, ldmatrix, stmatrix.","replacement_claim_ids":["technique-external-source-map-research-current-c001","technique-external-source-map-research-current-c003"]} +{"receipt_id":"rem-technique-fine-grained-quantization-001","receipt_scope":"wiki","path":"wiki/techniques/fine-grained-quantization.md","original_body_sha256":"6a7689d9e5c0a13d8fc25daca9b0753cba405982bf7443f5fe50d355601c9952","original_unit_ids":["u001"],"locator":"frontmatter blackwell_relevance and body:L3","original_claim_id":"technique-fine-grained-quantization-original-c001","original_claim":"Local scales prevent outlier damage and universally divide Blackwell UE8M0 from Hopper Nc=128 promotion.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/abs/2412.19437v2","revision":"DeepSeek-V3 report v2","locator":"sections 3.3.2/3.5.2","observation":"These are scoped DeepSeek FP8 design choices, not universal guarantees."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"PTX ISA 9.0","locator":"block-scale type table","observation":"Blackwell supports UE8M0 and UE4M3 block-scale combinations."}],"strongest_alternative":"The statement is only a high-level contrast between two example kernels.","discriminating_result":"It uses universal architecture wording and an absolute precision guarantee rather than naming DeepGEMM's pinned paths.","action":"correct","action_reason":"Separate recipe, implementation and architecture scopes.","proposed_replacement":"Describe verified DeepGEMM, NVFP4 and MXFP4 recipes independently.","replacement_claim_ids":["technique-fine-grained-quantization-current-c001"]} +{"receipt_id":"rem-technique-fine-grained-quantization-002","receipt_scope":"wiki","path":"wiki/techniques/fine-grained-quantization.md","original_body_sha256":"6a7689d9e5c0a13d8fc25daca9b0753cba405982bf7443f5fe50d355601c9952","original_unit_ids":["u002"],"locator":"body:L7-L31","original_claim_id":"technique-fine-grained-quantization-original-c002","original_claim":"One FP32 scale per 128x128 FP8 block costs 0.006% and 1x128 captures per-channel distributions.","original_verdict":"refuted","evidence":[{"command":"compute factor-count and byte ratios","environment":"host arithmetic","reference":"one four-byte factor and 16384 one-byte payloads","result":"Count ratio is 0.0061035%, but byte ratio is 0.0244141%; 1x128 is one token row across 128 channels.","negative_control":"A one-byte factor produces 0.0061035% bytes."}],"strongest_alternative":"Overhead meant factor count rather than storage bytes.","discriminating_result":"The page never states the unit and compares storage formats; the same number is wrong as byte overhead.","action":"correct","action_reason":"State geometry and formulas without an ambiguous percentage or per-channel label.","proposed_replacement":"Give scale-count formulas and require explicit factor/payload widths for byte overhead.","replacement_claim_ids":["technique-fine-grained-quantization-current-c002"]} +{"receipt_id":"rem-technique-fine-grained-quantization-003","receipt_scope":"wiki","path":"wiki/techniques/fine-grained-quantization.md","original_body_sha256":"6a7689d9e5c0a13d8fc25daca9b0753cba405982bf7443f5fe50d355601c9952","original_unit_ids":["u004"],"locator":"body:L37-L76","original_claim_id":"technique-fine-grained-quantization-original-c004","original_claim":"The deepgemm_fp8_tile sketch is a representative executable DeepGEMM implementation.","original_verdict":"refuted","evidence":[{"path":"artifacts/kernels/deepgemm/full/sm90_fp8_gemm_1d1d.cuh","revision":"DeepGEMM 891d57b, byte-verified","locator":"BLOCK_K and math warp-group loop","observation":"The upstream implementation uses staged TMA, descriptors, WGMMA and exact accumulators; the sketch has undefined types, partial storage and helpers."}],"strongest_alternative":"It is intentionally pseudocode despite the cuda fence.","discriminating_result":"The page claims snippet reproducibility and attributes the sketch to DeepGEMM without marking it derived.","action":"delete","action_reason":"Use the pinned implementation and a short verbatim fragment instead.","proposed_replacement":null,"replacement_claim_ids":["technique-fine-grained-quantization-current-c003"]} +{"receipt_id":"rem-technique-fine-grained-quantization-004","receipt_scope":"wiki","path":"wiki/techniques/fine-grained-quantization.md","original_body_sha256":"6a7689d9e5c0a13d8fc25daca9b0753cba405982bf7443f5fe50d355601c9952","original_unit_ids":["u005"],"locator":"body:L80","original_claim_id":"technique-fine-grained-quantization-original-c005","original_claim":"Hopper FP8 accumulation is approximately FP22 and every Hopper path requires four-WGMMA promotion.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/abs/2412.19437v2","revision":"DeepSeek-V3 report v2","locator":"sections 3.3.2/3.5.2","observation":"The report says about 14 retained bits and scopes Nc=128 to its H800 scheme."}],"strongest_alternative":"FP22 names a storage format while 14 bits names retained precision.","discriminating_result":"The page explicitly equates WGMMA accumulation precision with FP22, which the cited source does not do.","action":"correct","action_reason":"Use the source's retained-bit wording and implementation scope.","proposed_replacement":"DeepSeek reports about 14 retained bits and its pinned SM90 kernel promotes each K=128 block.","replacement_claim_ids":["technique-fine-grained-quantization-current-c003"]} +{"receipt_id":"rem-technique-fine-grained-quantization-005","receipt_scope":"wiki","path":"wiki/techniques/fine-grained-quantization.md","original_body_sha256":"6a7689d9e5c0a13d8fc25daca9b0753cba405982bf7443f5fe50d355601c9952","original_unit_ids":["u006"],"locator":"body:L82-L126","original_claim_id":"technique-fine-grained-quantization-original-c006","original_claim":"The hopper_fp8_gemm_with_promotion sketch is valid DeepGEMM CUDA code.","original_verdict":"refuted","evidence":[{"path":"artifacts/kernels/deepgemm/full/sm90_fp8_gemm_1d1d.cuh","revision":"DeepGEMM 891d57b, byte-verified","locator":"lines 250-323","observation":"The exact source differs materially; the sketch has undefined types, dimensions, helpers and scale indices."}],"strongest_alternative":"The omissions are conventional pseudocode placeholders.","discriminating_result":"A reproducible cuda snippet cannot depend on an undefined partial array and comment-only indices.","action":"delete","action_reason":"Replace it with an exact upstream fragment and workflow.","proposed_replacement":null,"replacement_claim_ids":["technique-fine-grained-quantization-current-c003"]} +{"receipt_id":"rem-technique-fine-grained-quantization-006","receipt_scope":"wiki","path":"wiki/techniques/fine-grained-quantization.md","original_body_sha256":"6a7689d9e5c0a13d8fc25daca9b0753cba405982bf7443f5fe50d355601c9952","original_unit_ids":["u007","u008","u009","u010","u011"],"locator":"body:L128-L132","original_claim_id":"technique-fine-grained-quantization-original-c007","original_claim":"Nc=128 has a 0.1% error bound, smaller intervals cost too much and Nc=256 is unacceptably inaccurate.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/abs/2412.19437v2","revision":"DeepSeek-V3 report v2","locator":"sections 3.3.2/3.5.2","observation":"The source supports Nc=128/four WGMMAs but none of those quantitative or acceptability thresholds."}],"strongest_alternative":"The bullets summarize unpublished DeepGEMM engineering measurements.","discriminating_result":"No pinned source or environment is given, so the thresholds cannot be attributed or reproduced.","action":"correct","action_reason":"Retain only the exact source-supported interval and prohibit transfer without measurement.","proposed_replacement":"Nc=128 is the pinned DeepGEMM choice; other intervals need controlled accuracy/performance tests.","replacement_claim_ids":["technique-fine-grained-quantization-current-c003","technique-fine-grained-quantization-current-c007"]} +{"receipt_id":"rem-technique-fine-grained-quantization-007","receipt_scope":"wiki","path":"wiki/techniques/fine-grained-quantization.md","original_body_sha256":"6a7689d9e5c0a13d8fc25daca9b0753cba405982bf7443f5fe50d355601c9952","original_unit_ids":["u013"],"locator":"body:L138-L158","original_claim_id":"technique-fine-grained-quantization-original-c009","original_claim":"Every UE8M0 byte decodes as 2^(e-127), including 0xff as finite 2^128.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#alternate-floating-point-data-formats","revision":"PTX ISA 9.0","locator":"ue8m0 definition","observation":"0xff is NaN, not a finite exponent value."}],"strongest_alternative":"The pseudocode only illustrates ordinary finite encodings.","discriminating_result":"It states a complete descriptor/dequantization rule with no excluded encoding and the later table explicitly claims 2^128.","action":"delete","action_reason":"Avoid an incomplete invented descriptor; cite the exact type definition and upstream representation.","proposed_replacement":null,"replacement_claim_ids":["technique-fine-grained-quantization-current-c004"]} +{"receipt_id":"rem-technique-fine-grained-quantization-008","receipt_scope":"wiki","path":"wiki/techniques/fine-grained-quantization.md","original_body_sha256":"6a7689d9e5c0a13d8fc25daca9b0753cba405982bf7443f5fe50d355601c9952","original_unit_ids":["u014"],"locator":"body:L160-L171","original_claim_id":"technique-fine-grained-quantization-original-c010","original_claim":"The displayed f8f6f4 tcgen05 instruction is valid block-scaled PTX using SMEM scale operands.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"PTX ISA 9.0","locator":"block-scale grammar","observation":"The kind must be block-scaled, .block_scale is required, and scale A/B are TMEM addresses after idesc."}],"strongest_alternative":"The comments are conceptual and syntax intentionally abbreviated.","discriminating_result":"The ptx fence and fully spelled mnemonic present it as valid syntax while operand order and address spaces are wrong.","action":"correct","action_reason":"Show exact grammar-level PTX and label missing declarations/descriptors.","proposed_replacement":"Use mxf8f6f4.block_scale.scale_vec::1X with idesc and TMEM scale operands.","replacement_claim_ids":["technique-fine-grained-quantization-current-c005"]} +{"receipt_id":"rem-technique-fine-grained-quantization-009","receipt_scope":"wiki","path":"wiki/techniques/fine-grained-quantization.md","original_body_sha256":"6a7689d9e5c0a13d8fc25daca9b0753cba405982bf7443f5fe50d355601c9952","original_unit_ids":["u016","u017"],"locator":"body:L178-L179","original_claim_id":"technique-fine-grained-quantization-original-c012","original_claim":"UE8M0 finite range ends at 2^128 and the E4M3 scale column is exactly 240 values over about 0..448.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#alternate-floating-point-data-formats","revision":"PTX ISA 9.0","locator":"ue8m0/ue4m3 definitions","observation":"UE8M0 reserves 0xff for NaN and PTX's unsigned UE4M3 scale is not the signed E4M3 payload summarized by the table."}],"strongest_alternative":"The table intentionally ignores NaN and uses E4M3 as a recipe-level alias.","discriminating_result":"Ignoring 0xff still makes 2^128 false, and conflating signed E4M3 with PTX UE4M3 breaks instruction compatibility.","action":"delete","action_reason":"Use recipe/type combinations without unsupported range/count shorthand.","proposed_replacement":null,"replacement_claim_ids":["technique-fine-grained-quantization-current-c004"]} +{"receipt_id":"rem-technique-fine-grained-quantization-010","receipt_scope":"wiki","path":"wiki/techniques/fine-grained-quantization.md","original_body_sha256":"6a7689d9e5c0a13d8fc25daca9b0753cba405982bf7443f5fe50d355601c9952","original_unit_ids":["u019","u020"],"locator":"body:L181-L182","original_claim_id":"technique-fine-grained-quantization-original-c014","original_claim":"NVFP4 E4M3 scales require software decode on Blackwell and FP32 scaling is always best precision.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"PTX ISA 9.0","locator":"mxf4nvf4 UE4M3 block16 combination","observation":"Blackwell provides native block-scaled UE4M3 support; the ISA states no universal quantization-error ordering."}],"strongest_alternative":"Software decode refers to the hackathon kernel rather than architecture capability.","discriminating_result":"The row is labeled Hardware support and generalizes beyond any pinned hackathon implementation.","action":"correct","action_reason":"Separate native instruction legality from recipe accuracy and software choices.","proposed_replacement":"List exact native PTX combinations and require workload accuracy measurement.","replacement_claim_ids":["technique-fine-grained-quantization-current-c004","technique-fine-grained-quantization-current-c005"]} +{"receipt_id":"rem-technique-fine-grained-quantization-011","receipt_scope":"wiki","path":"wiki/techniques/fine-grained-quantization.md","original_body_sha256":"6a7689d9e5c0a13d8fc25daca9b0753cba405982bf7443f5fe50d355601c9952","original_unit_ids":["u023","u024","u025"],"locator":"body:L215-L217","original_claim_id":"technique-fine-grained-quantization-original-c016","original_claim":"The page's three recipe choices are generally validated for 671B+ training, highest-precision Blackwell inference, and all Hopper inference.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/abs/2412.19437v2","revision":"DeepSeek-V3 report v2","locator":"model and FP8 sections","observation":"It validates a scoped training system, not a parameter-count threshold or universal inference rule."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"PTX ISA 9.0","locator":"block-scale combinations","observation":"Both NVFP4-compatible UE4M3 and MXFP4 UE8M0 forms are native and no highest-precision ranking is defined."}],"strongest_alternative":"They are informal defaults rather than universal requirements.","discriminating_result":"Imperative 'Use' wording plus 'validated' and 'highest' makes architecture/workload-independent prescriptions.","action":"correct","action_reason":"Replace prescriptions with a recipe-selection checklist.","proposed_replacement":"Match payload, scale type, geometry, layout and target, then measure accuracy/cost.","replacement_claim_ids":["technique-fine-grained-quantization-current-c006","technique-fine-grained-quantization-current-c007"]} +{"receipt_id":"rem-technique-fine-grained-quantization-012","receipt_scope":"wiki","path":"wiki/techniques/fine-grained-quantization.md","original_body_sha256":"6a7689d9e5c0a13d8fc25daca9b0753cba405982bf7443f5fe50d355601c9952","original_unit_ids":["u026","u027","u028","u029"],"locator":"body:L221-L224","original_claim_id":"technique-fine-grained-quantization-original-c017","original_claim":"Scale format, smaller blocks and Nc impose universal precision/cost orderings, and native Blackwell block scaling is UE8M0-only.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html","revision":"Transformer Engine 2.13","locator":"NVFP4/MXFP4 comparison","observation":"The source gives recipe tradeoffs, not strict per-input accuracy ordering."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"PTX ISA 9.0","locator":"mxf4nvf4 valid combinations","observation":"Native block16 UE4M3 directly contradicts UE8M0-only."}],"strongest_alternative":"The qualitative directions are generally true engineering heuristics.","discriminating_result":"Exact ties/counterexamples and native UE4M3 invalidate absolutes; no controlled Nc curve is cited.","action":"correct","action_reason":"State only metadata arithmetic and verified capability; require measurement for accuracy/performance.","proposed_replacement":"More groups mean more scale elements, while accuracy and end-to-end cost remain workload/implementation dependent.","replacement_claim_ids":["technique-fine-grained-quantization-current-c002","technique-fine-grained-quantization-current-c007"]} +{"receipt_id":"rem-technique-kernel-fusion-001","receipt_scope":"wiki","path":"wiki/techniques/kernel-fusion.md","original_body_sha256":"685ddc5bebbf6ea4d6782e495c6a48fe89232b6b41680c67c23e8921b5ab2ed2","original_unit_ids":["u001"],"locator":"body:L5","original_claim_id":"technique-kernel-fusion-original-c001","original_claim":"Fusion always eliminates intermediate round trips and MoE/attention generally require five to seven launches.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py","revision":"FlashInfer 7f614b8","locator":"Track A reference","observation":"The operation graph does not prescribe launch count or on-chip materialization."},{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/operations.hpp","revision":"CUTLASS 4.5.0 e406c186","locator":"fusion operation types","observation":"Expression fusion is distinct from traffic and launch behavior of a concrete schedule."}],"strongest_alternative":"Eliminating round trips is the definition of useful fusion.","discriminating_result":"A fused launch may still spill/stage/write intermediates, and saved traffic exists only relative to a defined materializing baseline.","action":"correct","action_reason":"Define fusion semantically and state conditional benefits.","proposed_replacement":"One implementation evaluates dependent operations in one launch; saved launches/traffic require a defined pair.","replacement_claim_ids":["technique-kernel-fusion-current-c001"]} +{"receipt_id":"rem-technique-kernel-fusion-002","receipt_scope":"wiki","path":"wiki/techniques/kernel-fusion.md","original_body_sha256":"685ddc5bebbf6ea4d6782e495c6a48fe89232b6b41680c67c23e8921b5ab2ed2","original_unit_ids":["u002"],"locator":"body:L10-L26","original_claim_id":"technique-kernel-fusion-original-c002","original_claim":"The shown CUDA kernel validly allocates two 256-column TMEM accumulators and implements fused dual GEMM.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"PTX ISA 9.0","locator":"allocation/lifecycle","observation":"Allocation is collective and needs legal size, shared result, completion and free; the helpers are invented."},{"url":"https://github.com/gpu-mode/reference-kernels/blob/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm/reference.py","revision":"c5b2f7c","locator":"correctness reference","observation":"The operation does not require this TMEM decomposition."}],"strongest_alternative":"The cuda fence is pseudocode and only illustrates two logical accumulators.","discriminating_result":"It claims snippet reproducibility and architecture behavior through nonexistent APIs without lifecycle or mapping.","action":"delete","action_reason":"Use the exact operation reference and keep schedule choices separate.","proposed_replacement":null,"replacement_claim_ids":["technique-kernel-fusion-current-c002"]} +{"receipt_id":"rem-technique-kernel-fusion-003","receipt_scope":"wiki","path":"wiki/techniques/kernel-fusion.md","original_body_sha256":"685ddc5bebbf6ea4d6782e495c6a48fe89232b6b41680c67c23e8921b5ab2ed2","original_unit_ids":["u003","u004","u005"],"locator":"body:L29-L31","original_claim_id":"technique-kernel-fusion-original-c003","original_claim":"vLLM/SGLang use fixed seven/five-launch graphs and ideal fusion saves exactly 21.9% traffic.","original_verdict":"refuted","evidence":[{"url":"https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","revision":"checked 2026-08-08","locator":"definition/traces","observation":"No such framework launch counts or traffic result are provided."},{"url":"https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/trace/templates/moe.py","revision":"FlashInfer 7f614b8","locator":"reference","observation":"One logical W13 projection is split for SwiGLU; launch decomposition remains unspecified."}],"strongest_alternative":"The secondary benchmark blog measured these exact framework pipelines.","discriminating_result":"It lacks pinned framework revisions, source-level launch traces and a reproducible traffic derivation; direct primary contracts do not contain the numbers.","action":"delete","action_reason":"Do not preserve unpinned framework-specific performance claims.","proposed_replacement":null,"replacement_claim_ids":["technique-kernel-fusion-current-c003","technique-kernel-fusion-current-c005"]} +{"receipt_id":"rem-technique-kernel-fusion-004","receipt_scope":"wiki","path":"wiki/techniques/kernel-fusion.md","original_body_sha256":"685ddc5bebbf6ea4d6782e495c6a48fe89232b6b41680c67c23e8921b5ab2ed2","original_unit_ids":["u006"],"locator":"body:L35","original_claim_id":"technique-kernel-fusion-original-c004","original_claim":"SM100 has 256 TMEM columns total.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory","revision":"PTX ISA 9.0","locator":"TMEM organization","observation":"An SM has 512 columns of 128 lanes."}],"strongest_alternative":"256 refers to one commonly used half-allocation rather than hardware capacity.","discriminating_result":"The text explicitly says total; equal 256-column halves are a policy, not the capacity.","action":"correct","action_reason":"State 512-column capacity and implementation-specific live-region accounting.","proposed_replacement":"TMEM is 128x512 32-bit cells per SM; fusion must budget simultaneous regions and lifecycle.","replacement_claim_ids":["technique-kernel-fusion-current-c004"]} +{"receipt_id":"rem-technique-kernel-fusion-005","receipt_scope":"wiki","path":"wiki/techniques/kernel-fusion.md","original_body_sha256":"685ddc5bebbf6ea4d6782e495c6a48fe89232b6b41680c67c23e8921b5ab2ed2","original_unit_ids":["u008"],"locator":"body:L37","original_claim_id":"technique-kernel-fusion-original-c006","original_claim":"Fusion requires a dependency graph that is DAG-compatible with CTA scope.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py","revision":"FlashInfer 7f614b8","locator":"routing/local-expert reference","observation":"The operation includes reductions and expert accumulation and does not impose a CTA-only execution graph; implementations may use multiple CTAs or launches."}],"strongest_alternative":"DAG-compatible means simply acyclic producer-consumer dependence within whatever scope is chosen.","discriminating_result":"That tautology gives no necessary/sufficient criterion and the CTA restriction is false for multi-CTA fusion mechanisms.","action":"correct","action_reason":"Replace vague scope jargon with explicit correctness/resource questions.","proposed_replacement":"Verify ownership, communication, synchronization, grid scope, and intermediate lifetime for the concrete implementation.","replacement_claim_ids":["technique-kernel-fusion-current-c004"]} +{"receipt_id":"rem-technique-register-budgeting-001","receipt_scope":"wiki","path":"wiki/techniques/register-budgeting.md","original_body_sha256":"e4490d9c41b459fa167d4dbc32f8814b4ebe1e7579297a6074df53a54b51509d","original_unit_ids":["u001"],"locator":"frontmatter:blackwell_relevance; body:L5","original_claim_id":"technique-register-budgeting-original-c001","original_claim":"Occupancy is inversely proportional to registers, higher occupancy governs memory-bound performance, controls force an exact budget, and TMEM frees about 100 registers/thread.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html","revision":"CUDA C++ Programming Guide 13.0","locator":"sections 8.2.3.1 and 10.38","observation":"Occupancy has discrete resource thresholds and launch bounds guide compiler heuristics."},{"path":"artifacts/kernels/deepgemm/full/sm100_fp8_gemm_1d1d.cuh","revision":"DeepGEMM 891d57b, byte-verified","locator":"lines 455-463","observation":"The epilogue still loads TMEM values into registers."}],"strongest_alternative":"The prose is an intentionally simplified rule of thumb and ~100 describes a representative accumulator fragment.","discriminating_result":"The page presents an equality, forced-control semantics, and fixed saving without a shape; official contracts and the pinned epilogue directly distinguish all three.","action":"correct","action_reason":"Use the resource-threshold model and scope the TMEM effect to the resident D fragment.","proposed_replacement":"Treat register use as one occupancy input; inspect the compiled kernel and account for TMEM epilogue registers.","replacement_claim_ids":["technique-register-budgeting-current-c001","technique-register-budgeting-current-c005"]} +{"receipt_id":"rem-technique-register-budgeting-002","receipt_scope":"wiki","path":"wiki/techniques/register-budgeting.md","original_body_sha256":"e4490d9c41b459fa167d4dbc32f8814b4ebe1e7579297a6074df53a54b51509d","original_unit_ids":["u002"],"locator":"body:L9-L18","original_claim_id":"technique-register-budgeting-original-c002","original_claim":"__launch_bounds__(256,4) forces 32 registers/thread, yields about four blocks/SM, and spills if needed.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html","revision":"CUDA C++ Programming Guide 13.0","locator":"Launch Bounds","observation":"The qualifier supplies max threads and desired min blocks; the compiler derives L and other resources can limit residency."}],"strongest_alternative":"The 32-register value is an approximate consequence on B200.","discriminating_result":"No target/compiler/resource assumptions are supplied, and the official compiler algorithm does not define that consequence.","action":"delete","action_reason":"The fenced snippet is incomplete CUDA and encodes false numeric implications.","proposed_replacement":null,"replacement_claim_ids":["technique-register-budgeting-current-c002","technique-register-budgeting-current-c003"]} +{"receipt_id":"rem-technique-register-budgeting-003","receipt_scope":"wiki","path":"wiki/techniques/register-budgeting.md","original_body_sha256":"e4490d9c41b459fa167d4dbc32f8814b4ebe1e7579297a6074df53a54b51509d","original_unit_ids":["u003","u004","u005","u006"],"locator":"body:L22-L25","original_claim_id":"technique-register-budgeting-original-c003","original_claim":"A lower cap has fixed bad/neutral consequences through spilling, recomputation, and reduced unrolling.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html","revision":"CUDA C++ Programming Guide 13.0","locator":"Launch Bounds","observation":"The documented tradeoff is usually local memory and/or instruction count, not fixed qualitative labels."},{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"retrieved 2026-08-08","locator":"Attempts 9-12","observation":"Several changes had workload-specific regressions or no effect."}],"strongest_alternative":"The labels are conventional optimization heuristics rather than guarantees.","discriminating_result":"The unqualified bad/neutral wording changes design decisions, while the cited example itself contains counterexamples to a fixed ordering.","action":"correct","action_reason":"Describe compiler responses as possible costs to measure.","proposed_replacement":"A tighter limit can trade registers for spill traffic and instruction count; inspect actual output.","replacement_claim_ids":["technique-register-budgeting-current-c002","technique-register-budgeting-current-c003"]} +{"receipt_id":"rem-technique-register-budgeting-004","receipt_scope":"wiki","path":"wiki/techniques/register-budgeting.md","original_body_sha256":"e4490d9c41b459fa167d4dbc32f8814b4ebe1e7579297a6074df53a54b51509d","original_unit_ids":["u007"],"locator":"body:L27","original_claim_id":"technique-register-budgeting-original-c004","original_claim":"Memory latency hides spills, so aggressive budgeting often wins for memory-bound kernels.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html","revision":"CUDA Compiler Driver 13.0.2","locator":"maxrregcount and resource usage","observation":"Spills are local-memory traffic and register limits are a tradeoff."},{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"retrieved 2026-08-08","locator":"Attempts 10-12","observation":"Spilling regressed this memory-bound kernel and one cap reduction had no effect."}],"strongest_alternative":"More resident warps commonly hide the added latency.","discriminating_result":"Extra spill traffic competes for the same memory system and the cited kernel supplies both harmful and no-effect outcomes.","action":"correct","action_reason":"Require controlled variants instead of a directional guarantee.","proposed_replacement":"Benchmark only caps that change the compiled allocation, occupancy, or spills, holding the workload fixed.","replacement_claim_ids":["technique-register-budgeting-current-c004"]} +{"receipt_id":"rem-technique-register-budgeting-005","receipt_scope":"wiki","path":"wiki/techniques/register-budgeting.md","original_body_sha256":"e4490d9c41b459fa167d4dbc32f8814b4ebe1e7579297a6074df53a54b51509d","original_unit_ids":["u008","u009"],"locator":"body:L33-L34","original_claim_id":"technique-register-budgeting-original-c005","original_claim":"Rank 1 paired 32 registers with exactly 18.5 us and rank 3 paired 45 registers with about 20 us.","original_verdict":"refuted","evidence":[{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"retrieved 2026-08-08","locator":"What the Top Solutions Did Differently","observation":"The source reports all top three clustered around 18.5 us and separately reports caps; it does not publish those exact row pairs."}],"strongest_alternative":"The table reconstructs leaderboard values available when the page was written.","discriminating_result":"No pinned leaderboard snapshot or contestant implementation ties those measurements to the budgets, so the paired rows are not reproducible.","action":"correct","action_reason":"Preserve only the author's scoped observation and its provenance limit.","proposed_replacement":"The author reported caps of 32 and 45 among three solutions clustered around 18.5 us, without controlled ablation.","replacement_claim_ids":["technique-register-budgeting-current-c006"]} +{"receipt_id":"rem-technique-register-budgeting-006","receipt_scope":"wiki","path":"wiki/techniques/register-budgeting.md","original_body_sha256":"e4490d9c41b459fa167d4dbc32f8814b4ebe1e7579297a6074df53a54b51509d","original_unit_ids":["u010"],"locator":"body:L36","original_claim_id":"technique-register-budgeting-original-c006","original_claim":"The 32-versus-45 comparison proves occupancy dominates NVFP4 GEMV.","original_verdict":"refuted","evidence":[{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"retrieved 2026-08-08","locator":"top solutions and Attempt 11","observation":"Solutions differed on many mechanisms and the author's own cap reduction had no effect."}],"strongest_alternative":"The other differences are secondary and the caps explain the ranking.","discriminating_result":"Without compiled register counts, occupancy values, or a one-variable ablation, the causal conclusion cannot be separated from PTX, cache, load, and specialization changes.","action":"delete","action_reason":"Do not infer causality from confounded implementations.","proposed_replacement":null,"replacement_claim_ids":["technique-register-budgeting-current-c004","technique-register-budgeting-current-c006"]} +{"receipt_id":"rem-technique-register-budgeting-007","receipt_scope":"wiki","path":"wiki/techniques/register-budgeting.md","original_body_sha256":"e4490d9c41b459fa167d4dbc32f8814b4ebe1e7579297a6074df53a54b51509d","original_unit_ids":["u011","u012","u013","u014","u015"],"locator":"body:L40-L47","original_claim_id":"technique-register-budgeting-original-c007","original_claim":"Kernel category alone determines when occupancy-first register budgeting should or should not be used.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html","revision":"CUDA C++ Programming Guide 13.0","locator":"Launch Bounds and Occupancy Calculator","observation":"Optimal bounds require per-kernel analysis and differ by architecture."},{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"retrieved 2026-08-08","locator":"Attempts 10-12","observation":"A memory-bound sub-byte kernel still exhibited harmful and ineffective register-pressure changes."}],"strongest_alternative":"The lists are intended as starting heuristics, not sufficient conditions.","discriminating_result":"The page labels them When To Use/NOT To Use and gives occupancy first priority without the measurements needed to choose.","action":"correct","action_reason":"Replace categories with a decision procedure based on the compiled kernel and controlled benchmarks.","proposed_replacement":"Measure resource limits, cap effect, spill traffic, latency, and correctness for the exact target and workload.","replacement_claim_ids":["technique-register-budgeting-current-c003","technique-register-budgeting-current-c004","technique-register-budgeting-current-c005"]} +{"receipt_id":"rem-technique-software-exp-001","receipt_scope":"wiki","path":"wiki/techniques/software-exp.md","original_body_sha256":"fe120348335fa0ecedde2247d6fed5fe7262c22730cfed75ee48ee931de7d730","original_unit_ids":["u001"],"locator":"body:L3","original_claim_id":"technique-software-exp-original-c001","original_claim":"FA4 replaces hardware ex2 with a four-FMA degree-4 software path over a symmetric interval.","original_verdict":"refuted","evidence":[{"url":"https://tridao.me/blog/2026/flash4/","revision":"retrieved 2026-08-09","locator":"Faster exponential","observation":"FA4 mixes hardware and software paths and publishes a degree-3 polynomial on [0,1)."},{"path":"/tmp/kernelwiki-fa4-current/flash_attn/cute/utils.py","revision":"a369df7","locator":"POLY_EX2/ex2_emulation_2","observation":"Pinned code confirms degree 3, floor-style reduction and hybrid selection."}],"strongest_alternative":"The page describes a simpler equivalent software-only variant.","discriminating_result":"It attributes the mechanism to FA4 and makes the replacement central; exact first-party code retains hardware exp2 for most selected configurations.","action":"correct","action_reason":"Describe the hybrid degree-3 implementation exactly.","proposed_replacement":"FA4 selects a fraction of entries for degree-3 FMA emulation and leaves the remainder on hardware exp2.","replacement_claim_ids":["technique-software-exp-current-c001","technique-software-exp-current-c003"]} +{"receipt_id":"rem-technique-software-exp-002","receipt_scope":"wiki","path":"wiki/techniques/software-exp.md","original_body_sha256":"fe120348335fa0ecedde2247d6fed5fe7262c22730cfed75ee48ee931de7d730","original_unit_ids":["u002","u003","u004","u005","u006","u007","u008"],"locator":"body:L7-L18","original_claim_id":"technique-software-exp-original-c002","original_claim":"The architecture table proves every proportional-exp kernel is SFU-bound and software exp has 8x throughput.","original_verdict":"refuted","evidence":[{"url":"https://tridao.me/blog/2026/flash4/","revision":"retrieved 2026-08-09","locator":"Feeds and Speeds","observation":"The source supplies a scoped B200 M=N=D=128 tile model, not the table, universal implication, or 8x result."}],"strongest_alternative":"Unit-count ratios are a valid peak-throughput proxy.","discriminating_result":"The software path includes dependent FMAs, range reduction, reconstruction, selection, and contention; unit counts do not define its throughput.","action":"correct","action_reason":"Retain only the source's scoped feeds-and-speeds model.","proposed_replacement":"In the authors' B200 128x128 model, tensor-core and exponential work each account for 1024 modeled cycles.","replacement_claim_ids":["technique-software-exp-current-c002"]} +{"receipt_id":"rem-technique-software-exp-003","receipt_scope":"wiki","path":"wiki/techniques/software-exp.md","original_body_sha256":"fe120348335fa0ecedde2247d6fed5fe7262c22730cfed75ee48ee931de7d730","original_unit_ids":["u009","u010"],"locator":"body:L22-L72","original_claim_id":"technique-software-exp-original-c003","original_claim":"The software_exp2 snippet implements FA4 Cody-Waite and its minimax polynomial.","original_verdict":"refuted","evidence":[{"url":"https://tridao.me/blog/2026/flash4/","revision":"retrieved 2026-08-09","locator":"Faster exponential","observation":"Published reduction uses floor, f in [0,1), and degree-3 Sollya coefficients."},{"path":"/tmp/kernelwiki-fa4-current/flash_attn/cute/utils.py","revision":"a369df7","locator":"ex2_emulation_2","observation":"Exact code clamps, uses rounding-mode-specific packed operations, and reconstructs exponent bits."}],"strongest_alternative":"Nearest-integer degree-4 Taylor is a numerically comparable implementation detail.","discriminating_result":"The code fence claims FA4 provenance and 22-bit minimax accuracy while every defining choice differs from the source.","action":"delete","action_reason":"Avoid presenting invented CUDA as upstream or accuracy-validated.","proposed_replacement":null,"replacement_claim_ids":["technique-software-exp-current-c003","technique-software-exp-current-c004"]} +{"receipt_id":"rem-technique-software-exp-004","receipt_scope":"wiki","path":"wiki/techniques/software-exp.md","original_body_sha256":"fe120348335fa0ecedde2247d6fed5fe7262c22730cfed75ee48ee931de7d730","original_unit_ids":["u011","u012"],"locator":"body:L76-L138","original_claim_id":"technique-software-exp-original-c004","original_claim":"The standalone softmax function implements FA4 and FMA throughput is fully available beside tensor cores.","original_verdict":"refuted","evidence":[{"url":"https://tridao.me/blog/2026/flash4/","revision":"retrieved 2026-08-09","locator":"forward pipeline","observation":"FA4 uses synchronized softmax warpgroups, correction, TMEM staging, conditional rescaling, and hybrid exp selection."}],"strongest_alternative":"The function is intentionally schematic online softmax.","discriminating_result":"It is fenced as CUDA and attributed to FA4 but omits the mechanisms that define FA4's schedule and does not produce a complete output accumulator.","action":"delete","action_reason":"Point to pinned implementation structure instead of non-runnable reconstructed code.","proposed_replacement":null,"replacement_claim_ids":["technique-software-exp-current-c001","technique-software-exp-current-c005"]} +{"receipt_id":"rem-technique-software-exp-005","receipt_scope":"wiki","path":"wiki/techniques/software-exp.md","original_body_sha256":"fe120348335fa0ecedde2247d6fed5fe7262c22730cfed75ee48ee931de7d730","original_unit_ids":["u013","u014"],"locator":"body:L142-L180","original_claim_id":"technique-software-exp-original-c005","original_claim":"The PTX fragment is valid, complete and SFU-free FA4 software exp.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html","revision":"PTX ISA 9.0","locator":"declarations/ex2","observation":"The fragment uses undeclared registers and invokes ex2.approx."},{"path":"/tmp/kernelwiki-fa4-current/flash_attn/cute/utils.py","revision":"a369df7","locator":"combine_int_frac_ex2","observation":"FA4's software path reconstructs with integer exponent-field operations."}],"strongest_alternative":"The PTX is an illustrative fragment and hardware ex2 only scales an integer exactly.","discriminating_result":"Using the resource claimed to be bypassed invalidates the mechanism, and undeclared operands make the fragment non-assemblable.","action":"delete","action_reason":"Cite exact source functions and generated-code inspection requirements.","proposed_replacement":null,"replacement_claim_ids":["technique-software-exp-current-c003"]} +{"receipt_id":"rem-technique-software-exp-006","receipt_scope":"wiki","path":"wiki/techniques/software-exp.md","original_body_sha256":"fe120348335fa0ecedde2247d6fed5fe7262c22730cfed75ee48ee931de7d730","original_unit_ids":["u015"],"locator":"body:L184","original_claim_id":"technique-software-exp-original-c006","original_claim":"The displayed degree-4 Taylor polynomial provides about 22 mantissa bits on [-0.5,0.5].","original_verdict":"refuted","evidence":[{"command":"Dense grid of 1,000,002 points versus 2**x","environment":"Conda base Python host arithmetic","reference":"maximum sampled relative error","result":"5.5684e-5 at -0.5, about 14.13 bits","negative_control":"Published FA4 degree-3 coefficients on [0,1) gave a distinct 8.763e-5 result."}],"strongest_alternative":"22 bits refers to a different error metric or optimized coefficients.","discriminating_result":"The page lists exact coefficients and interval; evaluating that exact polynomial contradicts the numeric claim.","action":"delete","action_reason":"Do not attach an unsupported error guarantee to a non-upstream polynomial.","proposed_replacement":null,"replacement_claim_ids":["technique-software-exp-current-c004"]} +{"receipt_id":"rem-technique-software-exp-007","receipt_scope":"wiki","path":"wiki/techniques/software-exp.md","original_body_sha256":"fe120348335fa0ecedde2247d6fed5fe7262c22730cfed75ee48ee931de7d730","original_unit_ids":["u020"],"locator":"body:L193","original_claim_id":"technique-software-exp-original-c007","original_claim":"FA4's 1.1-1.3x cuDNN speedup came from software exp alone.","original_verdict":"refuted","evidence":[{"url":"https://tridao.me/blog/2026/flash4/","revision":"retrieved 2026-08-09","locator":"Attention Benchmarks","observation":"The figure compares complete FA4 forward against cuDNN 9.13 with no exp-only ablation."}],"strongest_alternative":"The prose around the figure isolates software exp as the main improvement.","discriminating_result":"The first-party source enumerates multiple co-designed forward changes, so the benchmark cannot identify one technique's contribution.","action":"correct","action_reason":"Scope numbers to the full system and require a matched ablation for this technique.","proposed_replacement":"The complete FA4 forward reports 1.1-1.3x; this is not an exp-only result.","replacement_claim_ids":["technique-software-exp-current-c006"]} +{"receipt_id":"rem-technique-software-exp-008","receipt_scope":"wiki","path":"wiki/techniques/software-exp.md","original_body_sha256":"fe120348335fa0ecedde2247d6fed5fe7262c22730cfed75ee48ee931de7d730","original_unit_ids":["u021","u022"],"locator":"body:L194-L195","original_claim_id":"technique-software-exp-original-c008","original_claim":"Any transcendental-limited kernel should use software exp, while Hopper should not.","original_verdict":"refuted","evidence":[{"url":"https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f","revision":"a369df7","locator":"README and SM100 tuning","observation":"FA4 supports Hopper and Blackwell, while hybrid-exp tuning is configuration-specific; no universal rule appears."}],"strongest_alternative":"These are practical starting recommendations rather than compatibility guarantees.","discriminating_result":"The categorical wording crosses architectures and kernels without specifying approximation tolerance, scheduling, instruction mix, or an ablation.","action":"correct","action_reason":"Use measurable preconditions and target-specific validation.","proposed_replacement":"Consider a hybrid path only after profiling and validate selection fraction, accuracy, SASS, and end-to-end performance.","replacement_claim_ids":["technique-software-exp-current-c005"]} +{"receipt_id":"rem-technique-software-exp-009","receipt_scope":"wiki","path":"wiki/techniques/software-exp.md","original_body_sha256":"fe120348335fa0ecedde2247d6fed5fe7262c22730cfed75ee48ee931de7d730","original_unit_ids":["u023"],"locator":"body:L199","original_claim_id":"technique-software-exp-original-c009","original_claim":"Software exp has 16-cycle latency versus 20 for ex2 and wins from a 128-to-16 unit ratio.","original_verdict":"refuted","evidence":[{"url":"https://tridao.me/blog/2026/flash4/","revision":"retrieved 2026-08-09","locator":"Feeds and Speeds","observation":"The authors publish a tile model and hybrid design, not those instruction latencies or derived unit-ratio throughput."}],"strongest_alternative":"The numbers are standard microarchitectural measurements for B200.","discriminating_result":"No benchmark environment, instruction dependency pattern, SASS, or primary table is cited, and the actual path includes more than four FMAs.","action":"delete","action_reason":"Do not preserve unattributed cycle counts.","proposed_replacement":null,"replacement_claim_ids":["technique-software-exp-current-c002","technique-software-exp-current-c005"]} +{"receipt_id":"rem-technique-software-exp-010","receipt_scope":"wiki","path":"wiki/techniques/software-exp.md","original_body_sha256":"fe120348335fa0ecedde2247d6fed5fe7262c22730cfed75ee48ee931de7d730","original_unit_ids":["u025"],"locator":"body:L201","original_claim_id":"technique-software-exp-original-c011","original_claim":"Attention inputs are always non-positive, so only underflow handling matters.","original_verdict":"refuted","evidence":[{"url":"https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py","revision":"a369df7","locator":"ex2_emulation_2","observation":"The helper states an upper assumption, clamps below -127, and depends on exact rounding; surrounding softmax handles special cases separately."}],"strongest_alternative":"After stable max subtraction every ordinary finite exponent is non-positive.","discriminating_result":"That invariant does not specify all-masked infinities, NaNs, clamping, rounding modes, or bit-reconstruction boundaries required by the implementation.","action":"correct","action_reason":"State the exact helper domain and require special-case handling at the caller.","proposed_replacement":"Pinned FA4 assumes x<=127, clamps below -127, and handles softmax special cases outside the helper.","replacement_claim_ids":["technique-software-exp-current-c003","technique-software-exp-current-c004"]} +{"receipt_id":"rem-technique-software-exp-011","receipt_scope":"wiki","path":"wiki/techniques/software-exp.md","original_body_sha256":"fe120348335fa0ecedde2247d6fed5fe7262c22730cfed75ee48ee931de7d730","original_unit_ids":["u016","u017","u018","u019"],"locator":"body:L185-L189","original_claim_id":"technique-software-exp-original-c012","original_claim":"Softmax errors cancel, BF16 proves sufficiency, and degree 6 gives full-range near-ULP accuracy.","original_verdict":"refuted","evidence":[{"url":"https://tridao.me/blog/2026/flash4/","revision":"retrieved 2026-08-09","locator":"Faster exponential/forward pipeline","observation":"The source validates one degree-3 approximation inside one pipeline and makes none of these general guarantees."},{"url":"https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py","revision":"a369df7","locator":"POLY_EX2","observation":"The pinned coefficient table ends at degree 5, not the claimed degree 6."}],"strongest_alternative":"Those statements are conservative intuition about low-precision attention.","discriminating_result":"Normalization propagates relative per-entry errors into both numerator and denominator; output dtype alone is not a proof, and the asserted higher-degree implementation is absent.","action":"correct","action_reason":"Replace universal accuracy claims with an explicit validation protocol.","proposed_replacement":"Measure scalar error, row sums, outputs and end-to-end tolerances across special and adversarial inputs.","replacement_claim_ids":["technique-software-exp-current-c004","technique-software-exp-current-c005"]} +{"receipt_id":"rem-technique-swizzling-001","receipt_scope":"wiki","path":"wiki/techniques/swizzling.md","original_body_sha256":"c716117a080b9689b2eecbf26c4f0bf1801704ef00fd83627b3f229789716aa5","original_unit_ids":["u001","u002","u003"],"locator":"frontmatter:blackwell_relevance; body:L3-L9","original_claim_id":"technique-swizzling-original-c001","original_claim":"Swizzling eliminates conflicts and 128B is mandatory for every SM100 TMA/tcgen05 input and correctness.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html","revision":"Nsight Compute 13.3","locator":"Shared Memory","observation":"Banks are successive 32-bit words and conflicts depend on exact warp addresses."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"PTX ISA 9.0","locator":"descriptor modes","observation":"No-, 32B-, 64B-, and 128B-swizzled layouts are legal."}],"strongest_alternative":"128B is mandatory for the standard high-throughput BF16 path even if the ISA has theoretical alternatives.","discriminating_result":"The page says all inputs and incorrect results; official legal combinations and exact working no-swizzle tutorial v1 directly contradict that scope.","action":"correct","action_reason":"Define conflicts per access and mode legality per consumer combination.","proposed_replacement":"Choose a legal producer/consumer mapping; 128B is one mode, not a universal correctness condition.","replacement_claim_ids":["technique-swizzling-current-c001","technique-swizzling-current-c002"]} +{"receipt_id":"rem-technique-swizzling-002","receipt_scope":"wiki","path":"wiki/techniques/swizzling.md","original_body_sha256":"c716117a080b9689b2eecbf26c4f0bf1801704ef00fd83627b3f229789716aa5","original_unit_ids":["u004","u005"],"locator":"body:L11-L17","original_claim_id":"technique-swizzling-original-c002","original_claim":"The tutorial isolates a 255-to-695 TFLOP/s, 2.7x swizzling-only gain.","original_verdict":"refuted","evidence":[{"url":"https://gau-nernst.github.io/tcgen05/","revision":"pinned source 3b90ac9b","locator":"lines 715-798","observation":"Exact 3D values are 252.81/695.43, and the change includes a 16B-to-128B TMA inner tile; the author explicitly cannot attribute it to bank conflicts."}],"strongest_alternative":"The wider TMA tile is inseparable from 128B swizzling and thus counts as the swizzle treatment.","discriminating_result":"That defines a combined treatment, not swizzling alone; the author's own causal alternatives prevent the claimed attribution.","action":"correct","action_reason":"Preserve endpoints and list the simultaneous changes and unresolved cause.","proposed_replacement":"The v1b-to-v2b combined 3D TMA/layout change reported 252.81 to 695.43 TFLOP/s; no isolated bank-conflict cause was established.","replacement_claim_ids":["technique-swizzling-current-c005"]} +{"receipt_id":"rem-technique-swizzling-003","receipt_scope":"wiki","path":"wiki/techniques/swizzling.md","original_body_sha256":"c716117a080b9689b2eecbf26c4f0bf1801704ef00fd83627b3f229789716aa5","original_unit_ids":["u006","u007","u008","u009"],"locator":"body:L21-L57","original_claim_id":"technique-swizzling-original-c003","original_claim":"The row-XOR function and one-bank rotation are a general 128B layout and guarantee conflict-free columns.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/swizzle.hpp","revision":"CUTLASS 4.5.0","locator":"Swizzle","observation":"Swizzle is defined over address bits and alignment; Swizzle<3,4,3> XORs bits 7:9 into 4:6."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html","revision":"Driver API 13.0.97","locator":"CUtensorMapSwizzle","observation":"Ordinary 128B mode swizzles 16B chunks in a 128B span, not one bank per row."}],"strongest_alternative":"For the one illustrated 128B-stride half tile, row bits happen to equal the relevant high address bits.","discriminating_result":"The function accepts arbitrary stride/base and the diagram rotates one bank instead of XORing 16B chunks, so it cannot serve as the general mapping it claims.","action":"delete","action_reason":"Use exact address-bit semantics and library/descriptor mappings.","proposed_replacement":null,"replacement_claim_ids":["technique-swizzling-current-c003"]} +{"receipt_id":"rem-technique-swizzling-004","receipt_scope":"wiki","path":"wiki/techniques/swizzling.md","original_body_sha256":"c716117a080b9689b2eecbf26c4f0bf1801704ef00fd83627b3f229789716aa5","original_unit_ids":["u011"],"locator":"body:L63-L86","original_claim_id":"technique-swizzling-original-c005","original_claim":"The displayed tensor-map call is a reproducible 2D FP16 128B encoder.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html","revision":"Driver API 13.0.97","locator":"cuTensorMapEncodeTiled","observation":"Arrays/alignment are undefined, rank-2 stride commentary is wrong, 128B box constraints are unchecked, and CUresult is ignored."}],"strongest_alternative":"Those arrays are conventional surrounding variables intentionally omitted from a snippet.","discriminating_result":"The page labels snippet reproducibility, but success depends exactly on omitted dimensions, byte strides, alignment and return status.","action":"delete","action_reason":"Replace incomplete code with an encoder validation checklist and pinned full example.","proposed_replacement":null,"replacement_claim_ids":["technique-swizzling-current-c002","technique-swizzling-current-c004"]} +{"receipt_id":"rem-technique-swizzling-005","receipt_scope":"wiki","path":"wiki/techniques/swizzling.md","original_body_sha256":"c716117a080b9689b2eecbf26c4f0bf1801704ef00fd83627b3f229789716aa5","original_unit_ids":["u012","u013","u014","u015","u016"],"locator":"body:L88-L95","original_claim_id":"technique-swizzling-original-c006","original_claim":"Swizzle modes map universally to non-MMA, narrow, medium and standard BF16/FP16 use cases.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html","revision":"Driver API 13.0.97","locator":"swizzle enum/constraints","observation":"The API defines chunk/span patterns and constraints, not these use cases."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"PTX ISA 9.0","locator":"valid combinations","observation":"Legality depends on kind, type, shape, major mode and target."}],"strongest_alternative":"The table is a reasonable default heuristic for common GEMMs.","discriminating_result":"It presents Use Case without scope and incorrectly turns type/tile width into a sufficient selector.","action":"correct","action_reason":"List exact mapping spans and defer legality to producer and consumer contracts.","proposed_replacement":"NONE, 32B, 64B and 128B define mappings; choose only modes legal for both TMA and the concrete consumer.","replacement_claim_ids":["technique-swizzling-current-c002"]} +{"receipt_id":"rem-technique-swizzling-006","receipt_scope":"wiki","path":"wiki/techniques/swizzling.md","original_body_sha256":"c716117a080b9689b2eecbf26c4f0bf1801704ef00fd83627b3f229789716aa5","original_unit_ids":["u017","u018"],"locator":"body:L99-L125","original_claim_id":"technique-swizzling-original-c007","original_claim":"The CuTe snippet correctly defines the template parameters and a general 128B layout.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/swizzle.hpp","revision":"CUTLASS 4.5.0","locator":"template definition","observation":"BBits is mask width and MBase is invariant low bits, the reverse of the page's labels."},{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/pointer_swizzle.hpp","revision":"CUTLASS 4.5.0","locator":"position dependence","observation":"Hardware uses position-dependent swizzle pointers; alignment and composition matter."}],"strongest_alternative":"Only the comments are mislabeled; Swizzle<3,4,3> itself is a real 128B mapping.","discriminating_result":"The surrounding shape/composition is claimed reusable without the base-alignment and pointer/layout distinction required for the same hardware mapping.","action":"correct","action_reason":"Define parameters exactly and cite a pinned full implementation rather than a generic atom.","proposed_replacement":"Swizzle<3,4,3> has three mask bits, four invariant low address bits, and shift three; preserve required base alignment and composition.","replacement_claim_ids":["technique-swizzling-current-c003"]} +{"receipt_id":"rem-technique-swizzling-007","receipt_scope":"wiki","path":"wiki/techniques/swizzling.md","original_body_sha256":"c716117a080b9689b2eecbf26c4f0bf1801704ef00fd83627b3f229789716aa5","original_unit_ids":["u019","u020"],"locator":"body:L129-L140","original_claim_id":"technique-swizzling-original-c008","original_claim":"nvprof or the displayed ncu metric proves elimination when conflict count is zero.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html","revision":"Nsight Compute 13.3","locator":"metrics/shared memory","observation":"Current documented aggregate is l1tex__data_bank_conflicts_pipe_lsu.sum and Memory Workload Analysis uses requests/wavefronts/conflicts."},{"url":"https://gau-nernst.github.io/tcgen05/","revision":"retrieved 2026-08-09","locator":"lines 795-798","observation":"The tutorial had no ncu access and leaves the performance cause unresolved."}],"strongest_alternative":"The older qualified metric works on the intended B200 profiler version.","discriminating_result":"Metric availability is chip/tool dependent and a zero count neither validates mapping correctness nor attributes end-to-end performance.","action":"correct","action_reason":"Use ncu --query-metrics and compare correctness, wavefronts/conflicts, and timing between matched layouts.","proposed_replacement":"Discover metrics on the installed profiler, then compare matched variants with output validation and Memory Workload Analysis.","replacement_claim_ids":["technique-swizzling-current-c004"]} +{"receipt_id":"rem-technique-swizzling-008","receipt_scope":"wiki","path":"wiki/techniques/swizzling.md","original_body_sha256":"c716117a080b9689b2eecbf26c4f0bf1801704ef00fd83627b3f229789716aa5","original_unit_ids":["u021","u022","u023"],"locator":"body:L144-L146","original_claim_id":"technique-swizzling-original-c009","original_claim":"All Blackwell/Hopper MMA requires 128B and swizzling prevents non-MMA serialization.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"PTX ISA 9.0","locator":"descriptor modes","observation":"Multiple non-128B modes are legal."},{"url":"https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html","revision":"Nsight Compute 13.3","locator":"bank conflicts","observation":"Conflict outcome depends on exact per-warp addresses and same-word/broadcast behavior."}],"strongest_alternative":"128B remains the practical default for all high-performance tensor-core paths.","discriminating_result":"A practical default cannot justify mandatory correctness wording, and a swizzle optimized for one access need not improve another.","action":"correct","action_reason":"Replace architecture categories with a producer/consumer/access-pattern decision.","proposed_replacement":"Select and validate one legal matched layout for the exact TMA box and consumer access.","replacement_claim_ids":["technique-swizzling-current-c001","technique-swizzling-current-c004"]} +{"receipt_id":"rem-technique-swizzling-009","receipt_scope":"wiki","path":"wiki/techniques/swizzling.md","original_body_sha256":"c716117a080b9689b2eecbf26c4f0bf1801704ef00fd83627b3f229789716aa5","original_unit_ids":["u024","u025","u026"],"locator":"body:L150-L152","original_claim_id":"technique-swizzling-original-c010","original_claim":"Mismatch always fails silently, CuTe prevents errors, and element width chooses the optimum while TMEM removes the issue.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html","revision":"Driver API 13.0.97","locator":"encoder return/constraints","observation":"Invalid maps can fail encoding and legal logical mismatch needs correctness testing; type width alone is insufficient."},{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/swizzle.hpp","revision":"CUTLASS 4.5.0","locator":"mapping abstraction","observation":"CuTe represents a mapping but cannot guarantee matching producer, alignment, consumer and access."}],"strongest_alternative":"These caveats are informal advice rather than hard contracts.","discriminating_result":"They assert failure mode and optimization choice; both affect correctness and cannot be inferred from abstraction or dtype.","action":"correct","action_reason":"State explicit encoder checks, layout equivalence and correctness/performance validation.","proposed_replacement":"Check CUresult, verify producer and consumer mappings including base alignment, then test output and profile the exact access.","replacement_claim_ids":["technique-swizzling-current-c004"]} +{"receipt_id":"rem-technique-vectorized-loads-001","receipt_scope":"wiki","path":"wiki/techniques/vectorized-loads.md","original_body_sha256":"a1867ce5eadf388cf89f64dbc67969e35641a2aee5abf95bc15a6310620d3778","original_unit_ids":["u001"],"locator":"body:L3","original_claim_id":"technique-vectorized-loads-original-c001","original_claim":"Wide loads, differentiated cache policy, and register caps together caused the reported 89x improvement.","original_verdict":"refuted","evidence":[{"url":"https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html","revision":"retrieved 2026-08-08","locator":"optimization progression","observation":"Every relevant stage combines several code, mapping, decode, launch, or scheduling changes and provides no three-factor ablation."}],"strongest_alternative":"The endpoint progression is enough to credit the three highlighted techniques jointly.","discriminating_result":"The progression contains additional co-varying mechanisms, so that causal subset is not identified.","action":"correct","action_reason":"Retain the report as an author-reported multi-change case study without causal attribution.","proposed_replacement":"Treat the 2000-to-22.392 microsecond progression as an author-reported sequence of combined changes.","replacement_claim_ids":["technique-vectorized-loads-current-c005"]} +{"receipt_id":"rem-technique-vectorized-loads-002","receipt_scope":"wiki","path":"wiki/techniques/vectorized-loads.md","original_body_sha256":"a1867ce5eadf388cf89f64dbc67969e35641a2aee5abf95bc15a6310620d3778","original_unit_ids":["u002","u003"],"locator":"body:L7-L34","original_claim_id":"technique-vectorized-loads-original-c002","original_claim":"Scalar loads waste bandwidth and wide per-thread loads inherently saturate B200 HBM; the snippets are safe demonstrations.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld","revision":"PTX ISA 9.0","locator":"vector loads and alignment","observation":"Vector loads may improve performance, but every address must be naturally aligned to the access size and the ISA gives no saturation guarantee."}],"strongest_alternative":"Per-thread instruction width alone determines global-memory utilization.","discriminating_result":"Warp address coalescing and generated instructions are independent of the source-level width label.","action":"correct","action_reason":"Document legal widths, exact alignment, and the need to inspect/profile generated code.","proposed_replacement":"SM100 supports 16- and 32-byte vector loads; they can reduce load instruction count but do not guarantee fewer physical transactions or higher bandwidth.","replacement_claim_ids":["technique-vectorized-loads-current-c001","technique-vectorized-loads-current-c002"]} +{"receipt_id":"rem-technique-vectorized-loads-003","receipt_scope":"wiki","path":"wiki/techniques/vectorized-loads.md","original_body_sha256":"a1867ce5eadf388cf89f64dbc67969e35641a2aee5abf95bc15a6310620d3778","original_unit_ids":["u004","u005"],"locator":"body:L36-L84","original_claim_id":"technique-vectorized-loads-original-c003","original_claim":"128- or 256-bit loads are essential for NVFP4 GEMV and the helper is a complete safe 64-value decoder.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov","revision":"PTX ISA 9.0","locator":"mov pack/unpack","observation":"The ISA defines legal bit-field movement, not necessity or performance, and the helper omits alignment/tail contracts and a decoder definition."},{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"retrieved 2026-08-08","locator":"Attempt 9","observation":"A wider load was 16-25% slower in the author's NVFP4 kernel."}],"strongest_alternative":"Sub-byte packing makes the widest legal load necessarily best.","discriminating_result":"A direct author-reported wider-load negative control regressed.","action":"correct","action_reason":"Replace the invented helper with exact PTX semantics and an empirical selection workflow.","proposed_replacement":"A 32-byte load contains 64 packed E2M1 values, but alignment, tails, register pressure, decode cost, and timing must be checked.","replacement_claim_ids":["technique-vectorized-loads-current-c002","technique-vectorized-loads-current-c004"]} +{"receipt_id":"rem-technique-vectorized-loads-004","receipt_scope":"wiki","path":"wiki/techniques/vectorized-loads.md","original_body_sha256":"a1867ce5eadf388cf89f64dbc67969e35641a2aee5abf95bc15a6310620d3778","original_unit_ids":["u006","u007"],"locator":"body:L88-L111","original_claim_id":"technique-vectorized-loads-original-c004","original_claim":"The displayed qualifiers make A bypass L1 and keep B resident as long as possible, thereby avoiding pollution.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld","revision":"PTX ISA 9.0","locator":"eviction-priority qualifiers","observation":"Eviction policies may be applied during access and are not residency or performance guarantees."}],"strongest_alternative":"The names no_allocate and evict_last specify mandatory hardware outcomes.","discriminating_result":"The normative contract uses hint/priority semantics and does not promise residence.","action":"correct","action_reason":"State exact hint semantics and require a matched comparison.","proposed_replacement":"Use L1::no_allocate and L1::evict_last only as eviction-policy hypotheses and measure the concrete streams.","replacement_claim_ids":["technique-vectorized-loads-current-c003"]} +{"receipt_id":"rem-technique-vectorized-loads-005","receipt_scope":"wiki","path":"wiki/techniques/vectorized-loads.md","original_body_sha256":"a1867ce5eadf388cf89f64dbc67969e35641a2aee5abf95bc15a6310620d3778","original_unit_ids":["u008","u009"],"locator":"body:L113-L118","original_claim_id":"technique-vectorized-loads-original-c005","original_claim":"Cache policy alone improved the cited kernel from 39 to 27 microseconds.","original_verdict":"refuted","evidence":[{"url":"https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html","revision":"retrieved 2026-08-08","locator":"CUDA optimization 2 and 3","observation":"The endpoints also change FP4 loading and decode implementation; there is no cache-policy-only run."}],"strongest_alternative":"The omitted changes are incidental and cache policy is the dominant difference.","discriminating_result":"No one-variable comparison or counterfactual measurement identifies dominance.","action":"correct","action_reason":"Restore the complete stage descriptions and label the values author-reported.","proposed_replacement":"The 39-to-27 microsecond endpoints span a combined change to vectorized PTX FP4 and scale decode and do not isolate cache policy.","replacement_claim_ids":["technique-vectorized-loads-current-c005"]} +{"receipt_id":"rem-technique-vectorized-loads-006","receipt_scope":"wiki","path":"wiki/techniques/vectorized-loads.md","original_body_sha256":"a1867ce5eadf388cf89f64dbc67969e35641a2aee5abf95bc15a6310620d3778","original_unit_ids":["u010","u011"],"locator":"body:L120-L141","original_claim_id":"technique-vectorized-loads-original-c006","original_claim":"The snippet is the full qualifier set and accurately equates qualifiers with bypass, persistence, promotion, and coherence benefits.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld-global-nc","revision":"PTX ISA 9.0","locator":"grammar and descriptions","observation":"The full grammar is larger; L2::256B is a prefetch hint, eviction priorities may apply, and nc uses a non-coherent read-only cache with conditional performance guidance."}],"strongest_alternative":"The comments are harmless shorthand for common implementations.","discriminating_result":"They turn optional hints into guarantees and misidentify prefetch size as promotion.","action":"correct","action_reason":"Use the archived grammar and scope each semantic statement.","proposed_replacement":"List representative qualifiers as hints and distinguish eviction priority, prefetch size, and non-coherent read-only access.","replacement_claim_ids":["technique-vectorized-loads-current-c003"]} +{"receipt_id":"rem-technique-vectorized-loads-007","receipt_scope":"wiki","path":"wiki/techniques/vectorized-loads.md","original_body_sha256":"a1867ce5eadf388cf89f64dbc67969e35641a2aee5abf95bc15a6310620d3778","original_unit_ids":["u012","u013","u014","u015"],"locator":"body:L145-L182","original_claim_id":"technique-vectorized-loads-original-c007","original_claim":"Occupancy dominates memory-bound kernels; caps 32/45 fix SM100 residency; launch bounds guarantee the shown tradeoff.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html","revision":"CUDA C++ Programming Guide 13.0","locator":"occupancy and launch bounds","observation":"Residency depends on compiled resources and launch inputs; launch bounds guide compiler limits rather than guaranteeing exact registers or blocks."},{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"retrieved 2026-08-08","locator":"Attempt 11","observation":"An 80-to-64 cap change had no effect."}],"strongest_alternative":"Register caps alone are sufficient to infer active warps.","discriminating_result":"Natural allocation and other resource limits can leave occupancy unchanged.","action":"correct","action_reason":"Route register-control details to the audited register-budgeting page and remove exact residency claims.","proposed_replacement":"Measure compiled registers, spills, occupancy and latency; a nominal cap or launch bound alone proves none of them.","replacement_claim_ids":["technique-vectorized-loads-current-c004"]} +{"receipt_id":"rem-technique-vectorized-loads-008","receipt_scope":"wiki","path":"wiki/techniques/vectorized-loads.md","original_body_sha256":"a1867ce5eadf388cf89f64dbc67969e35641a2aee5abf95bc15a6310620d3778","original_unit_ids":["u016","u017"],"locator":"body:L186-L264","original_claim_id":"technique-vectorized-loads-original-c008","original_claim":"The displayed complete CUDA kernel correctly implements contest Problem 1 and combines the claimed techniques.","original_verdict":"refuted","evidence":[{"url":"https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv","revision":"ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"task callable","observation":"The task ABI has seven physical tensors, reordered scales and an L dimension, but no global-scale arguments."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html","revision":"CUDA C++ Programming Guide 13.0","locator":"__syncthreads","observation":"Threads can return before the block barrier in a partial row block, violating collective participation; helpers are also undefined."}],"strongest_alternative":"The code is illustrative and need not match the task ABI or compile.","discriminating_result":"It is explicitly labeled complete and optimized, so executable-looking correctness defects are semantic claims.","action":"delete","action_reason":"No direct published contestant implementation supports repairing this invented kernel in place.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-technique-vectorized-loads-009","receipt_scope":"wiki","path":"wiki/techniques/vectorized-loads.md","original_body_sha256":"a1867ce5eadf388cf89f64dbc67969e35641a2aee5abf95bc15a6310620d3778","original_unit_ids":["u018","u019","u020","u021","u022","u023"],"locator":"body:L270-L275","original_claim_id":"technique-vectorized-loads-original-c009","original_claim":"The table isolates five techniques in a single 2000-to-22.4 microsecond progression with an 8.6-microsecond aggregate speed of light.","original_verdict":"refuted","evidence":[{"url":"https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html","revision":"retrieved 2026-08-08","locator":"progression","observation":"Stage labels omit multiple simultaneous changes and 22.392 is a geometric-mean score."},{"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml","revision":"ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"benchmarks","observation":"Theoretical times are 8.622, 17.275 and 4.317 microseconds for three shapes; 8.622 is not the aggregate."}],"strongest_alternative":"Rounded endpoints make the simplified labels acceptable.","discriminating_result":"The defect is causal scope and mismatched aggregation, not rounding.","action":"correct","action_reason":"Use exact author-reported combined-stage descriptions and separate task-model rows.","proposed_replacement":"Present the full combined progression and state that it does not isolate load width, cache policy or register caps.","replacement_claim_ids":["technique-vectorized-loads-current-c005"]} +{"receipt_id":"rem-technique-vectorized-loads-010","receipt_scope":"wiki","path":"wiki/techniques/vectorized-loads.md","original_body_sha256":"a1867ce5eadf388cf89f64dbc67969e35641a2aee5abf95bc15a6310620d3778","original_unit_ids":["u024","u025","u026"],"locator":"body:L279-L281","original_claim_id":"technique-vectorized-loads-original-c010","original_claim":"Wide loads/cache policy are essential for all GEMV, memory-bound, sub-byte, and single-token decode kernels.","original_verdict":"refuted","evidence":[{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"retrieved 2026-08-08","locator":"Attempt 9 and Takeaways","observation":"A wider load regressed the cited sub-byte GEMV, and the author recommends profiling the actual bottleneck."},{"url":"https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html#roofline","revision":"Nsight Compute 13.3","locator":"Roofline Analysis","observation":"Boundedness is assessed from measured intensity and ceilings rather than an algorithm label."}],"strongest_alternative":"These workload categories are reliably bandwidth-bound in practice.","discriminating_result":"A same-category negative control refutes essentiality and monotonic benefit.","action":"correct","action_reason":"Replace category prescriptions with concrete selection criteria and measurement.","proposed_replacement":"Consider vector width and cache hints when instruction issue, coalescing, and working-set evidence identifies a plausible benefit.","replacement_claim_ids":["technique-vectorized-loads-current-c004"]} +{"receipt_id":"rem-technique-vectorized-loads-011","receipt_scope":"wiki","path":"wiki/techniques/vectorized-loads.md","original_body_sha256":"a1867ce5eadf388cf89f64dbc67969e35641a2aee5abf95bc15a6310620d3778","original_unit_ids":["u027","u028"],"locator":"body:L285-L286","original_claim_id":"technique-vectorized-loads-original-c011","original_claim":"Misaligned 256-bit loads fall back safely and no_allocate is necessarily harmful for reused data.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#addresses-as-operands","revision":"PTX ISA 9.0","locator":"natural alignment","observation":"Misalignment is undefined and may mask address bits or fault; eviction priority is not a universal performance result."}],"strongest_alternative":"Hardware commonly splits misaligned accesses even if PTX does not promise it.","discriminating_result":"The portable contract is undefined, so a fallback cannot be taught as behavior.","action":"correct","action_reason":"State the normative correctness condition and empirical status of hints.","proposed_replacement":"A 32-byte load requires 32-byte alignment; handle tails separately and treat cache-hint effects as measured hypotheses.","replacement_claim_ids":["technique-vectorized-loads-current-c002","technique-vectorized-loads-current-c003"]} +{"receipt_id":"rem-technique-vectorized-loads-012","receipt_scope":"wiki","path":"wiki/techniques/vectorized-loads.md","original_body_sha256":"a1867ce5eadf388cf89f64dbc67969e35641a2aee5abf95bc15a6310620d3778","original_unit_ids":["u029","u030"],"locator":"body:L287-L288","original_claim_id":"technique-vectorized-loads-original-c012","original_claim":"An overly low cap necessarily loses to spills, while inline PTX bypasses register allocation.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/inline-ptx-assembly/index.html","revision":"CUDA 13.0.2","locator":"constraints","observation":"Compiler-selected operands are tied to register constraints, so inline PTX does not bypass allocation."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html","revision":"CUDA Compiler Driver 13.0.2","locator":"maxrregcount","observation":"A cap can add spills/instructions, but the performance balance is workload-specific."}],"strongest_alternative":"Bypass means only that the compiler cannot rewrite the instruction body.","discriminating_result":"The claim specifically concerns allocator control, and all operands/liveness still participate in allocation.","action":"correct","action_reason":"Separate opaque instruction scheduling effects from register allocation and keep performance empirical.","proposed_replacement":"Inline PTX constrains compiler visibility but its operands still use compiler-allocated registers; inspect compiled resources and benchmark.","replacement_claim_ids":["technique-vectorized-loads-current-c004"]} +{"receipt_id":"rem-technique-cache-policy-001","receipt_scope":"wiki","path":"wiki/techniques/cache-policy.md","original_body_sha256":"a47c6862f01b21f93a8d51024e927b47f02eba0fe5852a52053fb9b1080e1498","original_unit_ids":["u001"],"locator":"body:L5","original_claim_id":"technique-cache-policy-original-c001","original_claim":"L1 qualifiers control cache admission and are critical for memory-bound kernels.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld","revision":"PTX ISA 9.0","locator":"eviction priorities","observation":"Policies may be applied and no workload-category prescription is defined."}],"strongest_alternative":"The qualifier names are equivalent to mandatory admission controls.","discriminating_result":"The normative text provides priority/hint semantics rather than guaranteed placement.","action":"correct","action_reason":"State the exact contract and empirical scope.","proposed_replacement":"PTX exposes cache operators, eviction priorities, and prefetch hints whose effect must be measured.","replacement_claim_ids":["technique-cache-policy-current-c001"]} +{"receipt_id":"rem-technique-cache-policy-002","receipt_scope":"wiki","path":"wiki/techniques/cache-policy.md","original_body_sha256":"a47c6862f01b21f93a8d51024e927b47f02eba0fe5852a52053fb9b1080e1498","original_unit_ids":["u002"],"locator":"body:L9-L19","original_claim_id":"technique-cache-policy-original-c002","original_claim":"The code guarantees A bypass, B residence, and immediate C eviction.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-st","revision":"PTX ISA 9.0","locator":"ld/st eviction priorities","observation":"The forms are legal on supported targets with alignment, but their policies do not guarantee the commented hardware outcomes."}],"strongest_alternative":"The comments describe reliable implementation behavior beyond the virtual ISA wording.","discriminating_result":"No version-pinned hardware contract or discriminating measurement supports those guarantees.","action":"correct","action_reason":"Keep legal fragments with accurate non-guarantee comments and alignment requirements.","proposed_replacement":"Use legal eviction-priority forms as hypotheses and compare them against defaults.","replacement_claim_ids":["technique-cache-policy-current-c002"]} +{"receipt_id":"rem-technique-cache-policy-003","receipt_scope":"wiki","path":"wiki/techniques/cache-policy.md","original_body_sha256":"a47c6862f01b21f93a8d51024e927b47f02eba0fe5852a52053fb9b1080e1498","original_unit_ids":["u003","u004","u005"],"locator":"body:L23-L25","original_claim_id":"technique-cache-policy-original-c003","original_claim":"Rank 1 used the stated K-specific aggressive and relaxed qualifier policies.","original_verdict":"refuted","evidence":[{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"retrieved 2026-08-08","locator":"post-event observations","observation":"The report does not assign those per-K policies to rank 1 and supplies no contestant code."},{"url":"https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv","revision":"ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"public task","observation":"Public materials expose the task and scores, not submission techniques."}],"strongest_alternative":"Exact-K specialization implies the listed K-dependent cache policies.","discriminating_result":"Specialization is not evidence of which qualifier each variant used.","action":"delete","action_reason":"No direct public source supports this contestant attribution.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-technique-cache-policy-004","receipt_scope":"wiki","path":"wiki/techniques/cache-policy.md","original_body_sha256":"a47c6862f01b21f93a8d51024e927b47f02eba0fe5852a52053fb9b1080e1498","original_unit_ids":["u006","u007"],"locator":"body:L29-L30","original_claim_id":"technique-cache-policy-original-c004","original_claim":"Cache policy plus PTX unpacking partly caused a 16x gain and policy can dominate memory-bound kernels.","original_verdict":"refuted","evidence":[{"url":"https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html","revision":"retrieved 2026-08-08","locator":"CUDA optimization 1-3","observation":"The endpoints contain many simultaneous mapping, staging, collaboration, load, and decode changes without a cache-only comparison."}],"strongest_alternative":"The author narrative is sufficient to assign partial causality.","discriminating_result":"Without a counterfactual run even partial magnitude or dominance is unidentified.","action":"correct","action_reason":"Preserve only the complete combined-stage observation and causal limit.","proposed_replacement":"The cited solutions used differentiated policies, but the available reports provide no cache-policy-only timing.","replacement_claim_ids":["technique-cache-policy-current-c004"]} +{"receipt_id":"rem-technique-cache-policy-005","receipt_scope":"wiki","path":"wiki/techniques/cache-policy.md","original_body_sha256":"a47c6862f01b21f93a8d51024e927b47f02eba0fe5852a52053fb9b1080e1498","original_unit_ids":["u008","u009","u010","u011"],"locator":"body:L34-L37","original_claim_id":"technique-cache-policy-original-c005","original_claim":"Memory-bound classification, reuse labels, input size over 126 MB, or GEMM axes prescribe cache differentiation.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld","revision":"PTX ISA 9.0","locator":"cache operations and hints","observation":"The ISA defines no such selection thresholds or tensor-role rules."},{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"retrieved 2026-08-08","locator":"Takeaways","observation":"The author recommends profiling and controlled attempts because category-level heuristics failed to predict outcomes."}],"strongest_alternative":"The list is intended only as situations worth testing.","discriminating_result":"Its heading says when to use and includes an exact capacity threshold, turning hypotheses into prescriptions.","action":"correct","action_reason":"Replace with a one-variable measurement procedure based on concrete address streams and reuse distances.","proposed_replacement":"Test hints only after characterizing address order, reuse distance, working set, and interference, then keep measured wins.","replacement_claim_ids":["technique-cache-policy-current-c003"]} +{"receipt_id":"rem-technique-chunk-parallelism-001","receipt_scope":"wiki","path":"wiki/techniques/chunk-parallelism.md","original_body_sha256":"f7d2c3d8c9327b5dfd8b4e5c47d71cc82ce97f8ba4c85aad30dbadf15aeb1e8c","original_unit_ids":["u001"],"locator":"body:L5","original_claim_id":"technique-chunk-parallelism-original-c001","original_claim":"GDN, RetNet and Mamba share the page's generic chunk model, and Blackwell TMEM scales chunk size and tensor-core utilization.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/abs/2503.14376v3","revision":"TFLA v3","locator":"abstract","observation":"TFLA is scoped to linear RNNs/mLSTM and within-chunk parallelism, with no Blackwell/TMEM rule."},{"url":"https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921","revision":"b53d6d3","locator":"training implementation","observation":"GDN uses a specific WY chunkwise algorithm rather than the generic additive update."}],"strongest_alternative":"The overview is only a high-level abstraction applicable to all named models.","discriminating_result":"It asserts architecture identity and a hardware scaling rule absent from the direct sources.","action":"correct","action_reason":"Scope chunking to concrete recurrence transformations and implementations.","proposed_replacement":"Chunkwise algorithms expose within-chunk parallel work while preserving ordered boundary-state semantics; details are model-specific.","replacement_claim_ids":["technique-chunk-parallelism-current-c001"]} +{"receipt_id":"rem-technique-chunk-parallelism-002","receipt_scope":"wiki","path":"wiki/techniques/chunk-parallelism.md","original_body_sha256":"f7d2c3d8c9327b5dfd8b4e5c47d71cc82ce97f8ba4c85aad30dbadf15aeb1e8c","original_unit_ids":["u002"],"locator":"body:L9-L36","original_claim_id":"technique-chunk-parallelism-original-c002","original_claim":"The Triton snippet correctly provides sequential inter-chunk state propagation.","original_verdict":"refuted","evidence":[{"url":"https://triton-lang.org/main/programming-guide/chapter-01/introduction.html","revision":"retrieved 2026-08-09","locator":"program instances","observation":"Grid programs have no implicit execution order or grid-wide synchronization, so shared State updates race."},{"url":"https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921","revision":"b53d6d3","locator":"chunkwise path","observation":"The real implementation uses staged WY computations and explicit layouts absent from the snippet."}],"strongest_alternative":"The runtime happens to schedule chunk IDs in order.","discriminating_result":"Launch ordering is not a Triton correctness contract and concurrent programs address the same state.","action":"delete","action_reason":"The incomplete, racy pseudo-kernel cannot be repaired into a model-correct implementation without choosing a specific algorithm.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-technique-chunk-parallelism-003","receipt_scope":"wiki","path":"wiki/techniques/chunk-parallelism.md","original_body_sha256":"f7d2c3d8c9327b5dfd8b4e5c47d71cc82ce97f8ba4c85aad30dbadf15aeb1e8c","original_unit_ids":["u003","u004","u005"],"locator":"body:L40-L42","original_claim_id":"technique-chunk-parallelism-original-c003","original_claim":"C=32 is for decode, C=256-512 improves prefill, and TFLA recursively tiles arbitrary chunks.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/abs/2503.14376v3","revision":"TFLA v3","locator":"abstract and method scope","observation":"TFLA adds a second within-chunk parallel level for arbitrarily large mLSTM chunks but supplies none of the page's fixed decode/prefill prescriptions or recursive label."}],"strongest_alternative":"The sizes are reasonable engineering defaults outside the paper.","discriminating_result":"No direct measurement scope, backend, shape, or architecture supports them as recommendations.","action":"correct","action_reason":"Retain TFLA's exact contribution and make chunk size an empirical backend/model choice.","proposed_replacement":"TFLA enables arbitrarily large mLSTM chunks through an additional within-chunk sequence-parallel level; tune concrete chunk sizes per implementation.","replacement_claim_ids":["technique-chunk-parallelism-current-c002","technique-chunk-parallelism-current-c003"]} +{"receipt_id":"rem-technique-ping-pong-scheduling-001","receipt_scope":"wiki","path":"wiki/techniques/ping-pong-scheduling.md","original_body_sha256":"1e86380f8ba7bfb3a8a4c0f9aeaba796da8fef032eca6b4abc5dbacf72564d20","original_unit_ids":["u001"],"locator":"body:L5","original_claim_id":"technique-ping-pong-scheduling-original-c001","original_claim":"FlashAttention-4 introduced a two-query-tile single-CTA ping-pong schedule that ensures its softmax warpgroup never waits for MMA.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/html/2603.05451v1","revision":"FlashAttention-4 paper v1, 2026-03-05","locator":"section 3.1.2","observation":"FA4 follows a schedule similar to FA3 and maximizes overlap; it does not promise that softmax never waits."},{"url":"https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py","revision":"a369df707e1980fb328abcc1733e3457ec10155f","locator":"q_stage and warp roles","observation":"The implementation defaults to two query stages and assigns distinct MMA, softmax, correction, load, and epilogue roles."}],"strongest_alternative":"Maximized overlap can be summarized as a no-stall schedule.","discriminating_result":"The paper makes a relative overlap claim, not a universal stall-elimination guarantee.","action":"correct","action_reason":"Describe the exact two-output-tile schedule and its bounded overlap objective.","proposed_replacement":"FA4's SM100 forward schedule uses two query/output-tile states so one tile's MMA can overlap the other tile's softmax, subject to explicit dependencies.","replacement_claim_ids":["technique-ping-pong-scheduling-current-c001"]} +{"receipt_id":"rem-technique-ping-pong-scheduling-002","receipt_scope":"wiki","path":"wiki/techniques/ping-pong-scheduling.md","original_body_sha256":"1e86380f8ba7bfb3a8a4c0f9aeaba796da8fef032eca6b4abc5dbacf72564d20","original_unit_ids":["u002"],"locator":"body:L9-L36","original_claim_id":"technique-ping-pong-scheduling-original-c002","original_claim":"The displayed CUDA kernel is a valid two-tile FA4 ping-pong implementation with correct TMEM allocation, warpgroup roles, asynchronous MMA completion, and synchronization.","original_verdict":"refuted","evidence":[{"url":"https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py","revision":"a369df707e1980fb328abcc1733e3457ec10155f","locator":"kernel roles, TMEM allocation, and pipelines","observation":"The real kernel uses one collective TMEM allocation with disjoint offsets, specialized warp roles, and several producer/consumer pipelines."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"PTX ISA 9.0","locator":"tcgen05 lifecycle and completion","observation":"The pseudo-kernel's unrelated generic barrier does not establish the documented asynchronous MMA completion or TMEM ownership protocol."}],"strongest_alternative":"The snippet is merely pseudocode and need not compile.","discriminating_result":"It is presented as CUDA and teaches incorrect resource allocation and synchronization semantics even as pseudocode.","action":"delete","action_reason":"Replace executable-looking unsafe code with a source-pinned structural contract and verification checklist.","proposed_replacement":null,"replacement_claim_ids":["technique-ping-pong-scheduling-current-c002"]} +{"receipt_id":"rem-technique-ping-pong-scheduling-003","receipt_scope":"wiki","path":"wiki/techniques/ping-pong-scheduling.md","original_body_sha256":"1e86380f8ba7bfb3a8a4c0f9aeaba796da8fef032eca6b4abc5dbacf72564d20","original_unit_ids":["u004","u005"],"locator":"body:L42-L43","original_claim_id":"technique-ping-pong-scheduling-original-c004","original_claim":"A single-tile schedule alternately leaves SFU and tensor cores idle, whereas ping-pong keeps both units 100% busy.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/html/2603.05451v1","revision":"FlashAttention-4 paper v1, 2026-03-05","locator":"sections 3.1.1-3.1.2","observation":"The simplified roofline omits resources, and the schedule is described as maximizing overlap rather than guaranteeing 100 percent utilization."}],"strongest_alternative":"The statement is an intuitive idealization of the timeline.","discriminating_result":"The absolute utilization claim is stronger than both the analytical model and the reported evidence.","action":"correct","action_reason":"Retain the overlap motivation without universal idle-state or utilization assertions.","proposed_replacement":"The schedule targets overlap between MMA and softmax work, while actual stalls and utilization remain shape- and implementation-dependent.","replacement_claim_ids":["technique-ping-pong-scheduling-current-c001","technique-ping-pong-scheduling-current-c004"]} +{"receipt_id":"rem-technique-ping-pong-scheduling-004","receipt_scope":"wiki","path":"wiki/techniques/ping-pong-scheduling.md","original_body_sha256":"1e86380f8ba7bfb3a8a4c0f9aeaba796da8fef032eca6b4abc5dbacf72564d20","original_unit_ids":["u006"],"locator":"body:L44","original_claim_id":"technique-ping-pong-scheduling-original-c005","original_claim":"The ping-pong pattern itself accounts for FA4's 1605 TFLOPS/s and 71% BF16 result.","original_verdict":"refuted","evidence":[{"url":"https://tridao.me/blog/2026/flash4/","revision":"Tri Dao FA4 blog, 2026","locator":"Performance and Key Techniques","observation":"The numbers describe complete FA4 forward kernels with several other techniques and no ping-pong-only ablation."}],"strongest_alternative":"Because ping-pong is central, the full-kernel peak can be attributed to it.","discriminating_result":"A multi-change endpoint cannot identify the isolated effect of one scheduling component.","action":"correct","action_reason":"Report the result only at full-system scope and explicitly deny isolated attribution.","proposed_replacement":"The complete FA4 implementation reaches the author-reported peak; the cited material does not provide a ping-pong-only timing.","replacement_claim_ids":["technique-ping-pong-scheduling-current-c004"]} +{"receipt_id":"rem-technique-ping-pong-scheduling-005","receipt_scope":"wiki","path":"wiki/techniques/ping-pong-scheduling.md","original_body_sha256":"1e86380f8ba7bfb3a8a4c0f9aeaba796da8fef032eca6b4abc5dbacf72564d20","original_unit_ids":["u007","u008","u009"],"locator":"body:L48-L50","original_claim_id":"technique-ping-pong-scheduling-original-c006","original_claim":"Ping-pong should be used for compute-bound or SFU-heavy Blackwell attention kernels and is not useful on Hopper.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/html/2603.05451v1","revision":"FlashAttention-4 paper v1, 2026-03-05","locator":"section 3.1.2","observation":"The paper says FA4 follows a ping-pong schedule similar to Hopper-era FA3 and gives no category-only adoption rule."}],"strongest_alternative":"The specific FA4 implementation is SM100-only, so the schedule is useless on Hopper.","discriminating_result":"Implementation compatibility and scheduling-pattern usefulness are distinct; the paper explicitly traces the pattern to FA3.","action":"correct","action_reason":"Scope concrete code to SM100 and recommend controlled measurement instead of workload-label prescriptions.","proposed_replacement":"Use the pinned SM100 implementation as the concrete reference; evaluate staged overlap from dependency and profiler evidence, with q_stage=1 as a control.","replacement_claim_ids":["technique-ping-pong-scheduling-current-c005"]} +{"receipt_id":"rem-technique-ping-pong-scheduling-006","receipt_scope":"wiki","path":"wiki/techniques/ping-pong-scheduling.md","original_body_sha256":"1e86380f8ba7bfb3a8a4c0f9aeaba796da8fef032eca6b4abc5dbacf72564d20","original_unit_ids":["u010","u011","u012"],"locator":"body:L54-L62","original_claim_id":"technique-ping-pong-scheduling-original-c007","original_claim":"The local full bundle is a verbatim full reference implementation of FA4 ping-pong scheduling and the derived variant faithfully demonstrates the overlap; the displayed query retrieves it.","original_verdict":"refuted","evidence":[{"path":"artifacts/kernels/ping-pong-scheduling/full/PROVENANCE.yaml","revision":"local provenance","locator":"origin and files","observation":"The full artifact is sourced from CUTLASS PR 2466 and implements Blackwell FMHA backward, not Dao-AILab FA4 forward."},{"path":"artifacts/kernels/ping-pong-scheduling/variants/01-two-tile-ping-pong-skeleton.cu","revision":"derived local variant","locator":"function order","observation":"The visible order waits for and completes tile A softmax before issuing tile B MMA, so it does not demonstrate the claimed overlap."}],"strongest_alternative":"The artifacts are related enough to serve as implementation references.","discriminating_result":"Both algorithm direction/provenance and the derived schedule differ from the claim.","action":"delete","action_reason":"Remove the misleading page linkage while preserving the separately provenance-labeled files for repository integrity.","proposed_replacement":null,"replacement_claim_ids":[]} +{"receipt_id":"rem-pattern-pipeline-stalls-001","receipt_scope":"wiki","path":"wiki/patterns/pipeline-stalls.md","original_body_sha256":"ec21f3208711a7056069ab687fce39dd6bc96f6ee8fd093b42542af45799b592","original_unit_ids":["u001"],"locator":"body:L5","original_claim_id":"pattern-pipeline-stalls-original-c001","original_claim":"Low TMA/tcgen05 activity, phase-local tensor-core drops, and warps observed at mbarrier.try_wait establish pipeline stalls in an otherwise compute-bound kernel.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"Warp State Statistics","observation":"Sampled stall states and unit activity are symptoms; NVIDIA cautions that stalls matter only with scheduler under-issue and they do not establish causal boundedness."}],"strongest_alternative":"The combined profiler observations uniquely identify a pipeline bottleneck.","discriminating_result":"They admit load latency, expected dependency waits, imbalance, tails, and correctness errors as alternatives.","action":"correct","action_reason":"Require scheduler issue evidence, source correlation, and a controlled causal perturbation.","proposed_replacement":"Treat waits and low activity as leads, then isolate the critical edge with matched variants.","replacement_claim_ids":["pattern-pipeline-stalls-current-c001"]} +{"receipt_id":"rem-pattern-pipeline-stalls-002","receipt_scope":"wiki","path":"wiki/patterns/pipeline-stalls.md","original_body_sha256":"ec21f3208711a7056069ab687fce39dd6bc96f6ee8fd093b42542af45799b592","original_unit_ids":["u002"],"locator":"body:L9","original_claim_id":"pattern-pipeline-stalls-original-c002","original_claim":"Two stages cannot hide a three-cycle latency chain.","original_verdict":"refuted","evidence":[{"url":"https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu","revision":"3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"matmul_v3 launch","observation":"The cited pipelined source instantiates two stages; no source defines the page's generic three-cycle chain."}],"strongest_alternative":"Three-cycle was shorthand for three dependent phases.","discriminating_result":"Phase count is not latency and cannot determine minimum buffering depth.","action":"correct","action_reason":"Make depth a resource-checked controlled variable.","proposed_replacement":"Test legal stage counts while holding the rest of the kernel fixed.","replacement_claim_ids":["pattern-pipeline-stalls-current-c003"]} +{"receipt_id":"rem-pattern-pipeline-stalls-003","receipt_scope":"wiki","path":"wiki/patterns/pipeline-stalls.md","original_body_sha256":"ec21f3208711a7056069ab687fce39dd6bc96f6ee8fd093b42542af45799b592","original_unit_ids":["u004"],"locator":"body:L11","original_claim_id":"pattern-pipeline-stalls-original-c004","original_claim":"Missing tcgen05.fence::after_thread_sync between a TMA wait and MMA issue lets MMA read SMEM before the TMA transfer is visible.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait","revision":"PTX ISA 9.0","locator":"acquire ordering","observation":"A successful acquire wait provides visibility for associated prior bulk-copy operations."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence","revision":"PTX ISA 9.0","locator":"after_thread_sync","observation":"The tcgen05 fence orders asynchronous tcgen05 operations across an execution-ordering handoff; it is not the general TMA visibility operation asserted."}],"strongest_alternative":"The fence is conservatively required before every MMA issue.","discriminating_result":"The ISA assigns distinct semantics to acquire waits and tcgen05 execution ordering.","action":"correct","action_reason":"Separate TMA completion/visibility from cross-thread tcgen05 ordering.","proposed_replacement":"Use the TMA mbarrier acquire path for load visibility and tcgen05 fences only at applicable tcgen05 cross-thread handoffs.","replacement_claim_ids":["pattern-pipeline-stalls-current-c002"]} +{"receipt_id":"rem-pattern-pipeline-stalls-004","receipt_scope":"wiki","path":"wiki/patterns/pipeline-stalls.md","original_body_sha256":"ec21f3208711a7056069ab687fce39dd6bc96f6ee8fd093b42542af45799b592","original_unit_ids":["u005"],"locator":"body:L12","original_claim_id":"pattern-pipeline-stalls-original-c005","original_claim":"Using one tile serializes every warp on that tile's softmax or epilogue.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/html/2603.05451v1","revision":"FlashAttention-4 paper v1","locator":"sections 3.1.1-3.1.2","observation":"Two-tile overlap is a concrete FA4 optimization, not proof that all single-tile schedules serialize all warps."}],"strongest_alternative":"The statement applies only to the simplified FA4 timeline.","discriminating_result":"The page presents it as a general pipeline-stall cause without that scope.","action":"correct","action_reason":"Present multi-tile scheduling as a hypothesis only when independent state and work exist.","proposed_replacement":"Test multi-output overlap only after identifying non-MMA work on the critical path and proving resources and dependencies.","replacement_claim_ids":["pattern-pipeline-stalls-current-c003"]} +{"receipt_id":"rem-pattern-pipeline-stalls-005","receipt_scope":"wiki","path":"wiki/patterns/pipeline-stalls.md","original_body_sha256":"ec21f3208711a7056069ab687fce39dd6bc96f6ee8fd093b42542af45799b592","original_unit_ids":["u006"],"locator":"body:L13","original_claim_id":"pattern-pipeline-stalls-original-c006","original_claim":"After a TMA operation, hardware performs an mbarrier arrival in addition to the producer's manual arrival, so the producer over-arrives and releases the next stage stale.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive","revision":"PTX ISA 9.0","locator":"arrive.expect_tx and complete_tx","observation":"arrive.expect_tx performs an arrival and adds transaction bytes; TMA complete_tx decrements bytes rather than performing a second arrival."}],"strongest_alternative":"The operational warning is right even if the hardware mechanism is mislabeled.","discriminating_result":"An unmatched extra arrival is dangerous, but correct diagnosis requires reconciling initialized arrivals and expected transaction bytes separately.","action":"correct","action_reason":"Teach the two independent counters exactly.","proposed_replacement":"Audit initialized arrival count, software arrivals, expected bytes, and TMA completed bytes for each phase.","replacement_claim_ids":["pattern-pipeline-stalls-current-c002"]} +{"receipt_id":"rem-pattern-pipeline-stalls-006","receipt_scope":"wiki","path":"wiki/patterns/pipeline-stalls.md","original_body_sha256":"ec21f3208711a7056069ab687fce39dd6bc96f6ee8fd093b42542af45799b592","original_unit_ids":["u007","u008"],"locator":"body:L19-L20","original_claim_id":"pattern-pipeline-stalls-original-c007","original_claim":"Three to five pipeline stages are typical on Blackwell, and warp specialization eliminates role-switching stalls.","original_verdict":"refuted","evidence":[{"url":"https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu","revision":"3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"launch","observation":"The cited v3 uses two stages."},{"url":"https://gau-nernst.github.io/tcgen05/","revision":"retrieved 2026-08-09","locator":"warp specialization","observation":"The source reports a matched version step, not universal elimination of stalls."}],"strongest_alternative":"The advice is a reasonable default range and idealized effect.","discriminating_result":"A direct cited counterexample defeats the range, while the absolute effect lacks an oracle.","action":"correct","action_reason":"Turn both into measured candidate changes with explicit costs.","proposed_replacement":"Vary legal depth and role partition independently and retain only improvements on representative shapes.","replacement_claim_ids":["pattern-pipeline-stalls-current-c003"]} +{"receipt_id":"rem-pattern-pipeline-stalls-007","receipt_scope":"wiki","path":"wiki/patterns/pipeline-stalls.md","original_body_sha256":"ec21f3208711a7056069ab687fce39dd6bc96f6ee8fd093b42542af45799b592","original_unit_ids":["u011"],"locator":"body:L26-L33","original_claim_id":"pattern-pipeline-stalls-original-c009","original_claim":"The six-step diagnosis checklist gives generally correct profiler metrics, phase progression, TMA accounting, tcgen05 fencing, and stage-count advice.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"metric collection","observation":"Metrics are chip/tool dependent and stall sampling requires issue context."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence","revision":"PTX ISA 9.0","locator":"fence semantics","observation":"The mandatory TMA-wait-to-MMA fence step is incorrect, and phase parity is per object/reuse rather than each wait globally."}],"strongest_alternative":"Most steps are directionally useful despite imprecision.","discriminating_result":"A correctness checklist cannot safely contain a false mandatory ordering rule.","action":"correct","action_reason":"Replace with separate profiling, barrier-accounting, and controlled-experiment stages.","proposed_replacement":"Query available metrics, correlate waits, prove exact phase edges, then perturb one scheduling variable.","replacement_claim_ids":["pattern-pipeline-stalls-current-c001","pattern-pipeline-stalls-current-c002","pattern-pipeline-stalls-current-c003"]} +{"receipt_id":"rem-pattern-pipeline-stalls-008","receipt_scope":"wiki","path":"wiki/patterns/pipeline-stalls.md","original_body_sha256":"ec21f3208711a7056069ab687fce39dd6bc96f6ee8fd093b42542af45799b592","original_unit_ids":["u012","u013","u014","u015","u016"],"locator":"body:L37-L41","original_claim_id":"pattern-pipeline-stalls-original-c010","original_claim":"The tcgen05 tutorial progresses from a one-stage 62%-of-cuBLAS kernel to three-stage 70%, warp-specialized 80%, 2-SM 86%, and persistent-plus-CLC 98%, with the stated isolated causes.","original_verdict":"refuted","evidence":[{"url":"https://gau-nernst.github.io/tcgen05/","revision":"retrieved 2026-08-09","locator":"final progression","observation":"Exact endpoints are v1a 254.62, v2b 695.43, v3 939.61, v4 1208.83, v5 1302.29, and v6 1475.93 TFLOP/s versus 1506.74 cuBLAS; v6 uses static scheduling and CLC was not added."},{"url":"https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu","revision":"3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"matmul_v3 launch","observation":"The pipelined variant uses two stages."}],"strongest_alternative":"The percentages are rounded summaries of the tutorial stages.","discriminating_result":"The first two labels, stage count, CLC attribution, and causal descriptions are materially wrong, not rounding differences.","action":"correct","action_reason":"Report the exact source table and combined-version scope.","proposed_replacement":"Show exact TFLOP/s and computed ratios, noting each version step and absence of CLC.","replacement_claim_ids":["pattern-pipeline-stalls-current-c004"]} +{"receipt_id":"rem-pattern-pipeline-stalls-009","receipt_scope":"wiki","path":"wiki/patterns/pipeline-stalls.md","original_body_sha256":"ec21f3208711a7056069ab687fce39dd6bc96f6ee8fd093b42542af45799b592","original_unit_ids":["u017"],"locator":"body:L45","original_claim_id":"pattern-pipeline-stalls-original-c011","original_claim":"Too many stages exceed a 228-KiB shared-memory budget.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2 Blackwell Tuning Guide","locator":"shared memory capacity","observation":"228 KiB is a per-SM limit for compute capability 10.0; exceeding it depends on concrete per-stage and fixed storage plus block limits."}],"strongest_alternative":"The sentence is only a generic warning that stage storage grows.","discriminating_result":"Its exact threshold is presented without the payload and allocation scope needed to decide legality.","action":"correct","action_reason":"Require explicit storage arithmetic and compiled resource checks.","proposed_replacement":"Compute total dynamic/static shared memory for every variant and reject resource-invalid launches.","replacement_claim_ids":["pattern-pipeline-stalls-current-c003"]} +{"receipt_id":"rem-pattern-pipeline-stalls-010","receipt_scope":"wiki","path":"wiki/patterns/pipeline-stalls.md","original_body_sha256":"ec21f3208711a7056069ab687fce39dd6bc96f6ee8fd093b42542af45799b592","original_unit_ids":["u019"],"locator":"body:L47","original_claim_id":"pattern-pipeline-stalls-original-c013","original_claim":"Pipelining is a waste of effort on memory-bound kernels.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline","revision":"Nsight Compute 2025.3","locator":"roofline analysis","observation":"A roofline classification identifies a limiting ceiling but does not prove every overlap or latency-hiding transformation has zero benefit."}],"strongest_alternative":"Bandwidth-bound work cannot benefit from deeper staging because throughput is capped.","discriminating_result":"Even under a bandwidth ceiling, issue gaps, latency, or transaction behavior can leave attainable bandwidth unused.","action":"correct","action_reason":"Use causal measurements rather than excluding an entire category.","proposed_replacement":"Prioritize the measured limiting edge but allow a matched pipeline test when it predicts reduced issue or dependency gaps.","replacement_claim_ids":["pattern-pipeline-stalls-current-c005"]} +{"receipt_id":"rem-pattern-compute-bound-001","receipt_scope":"wiki","path":"wiki/patterns/compute-bound.md","original_body_sha256":"1250f2c8d05fdd1ec4dcb30133b71caf29617b7fe0c0131fcafbe9ca9a4fe320","original_unit_ids":["u001"],"locator":"body:L3","original_claim_id":"pattern-compute-bound-original-c001","original_claim":"Tensor-core utilization below 70% plus unsaturated memory bandwidth proves a kernel is compute-bound but below peak.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline","revision":"Nsight Compute 2025.3","locator":"roofline","observation":"Boundedness depends on arithmetic intensity and relevant ceilings, not a fixed utilization threshold or DRAM counter alone."}],"strongest_alternative":"The threshold is only a practical triage heuristic.","discriminating_result":"No workload, datatype, achieved ceiling, or metric definition scopes the threshold.","action":"correct","action_reason":"Replace the threshold with roofline and scheduler evidence.","proposed_replacement":"Confirm the relevant compute ceiling, arithmetic intensity, memory ceiling, and issue activity before classifying under-attainment.","replacement_claim_ids":["pattern-compute-bound-current-c001"]} +{"receipt_id":"rem-pattern-compute-bound-002","receipt_scope":"wiki","path":"wiki/patterns/compute-bound.md","original_body_sha256":"1250f2c8d05fdd1ec4dcb30133b71caf29617b7fe0c0131fcafbe9ca9a4fe320","original_unit_ids":["u004"],"locator":"body:L9","original_claim_id":"pattern-compute-bound-original-c003","original_claim":"Single-SM MMA tiles are inherently too small to utilize available compute.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"PTX ISA 9.0","locator":"shape constraints","observation":"PTX defines multiple legal one- and two-CTA shapes but no utilization conclusion from group mode alone."}],"strongest_alternative":"Larger cooperative tiles usually expose more reuse.","discriminating_result":"Reuse opportunity does not prove the single-CTA alternative underutilizes compute.","action":"correct","action_reason":"Make CTA-group mode a controlled legal-shape and resource experiment.","proposed_replacement":"Compare legal one- and two-CTA variants with matched math and record total/per-SM throughput and traffic.","replacement_claim_ids":["pattern-compute-bound-current-c003"]} +{"receipt_id":"rem-pattern-compute-bound-003","receipt_scope":"wiki","path":"wiki/patterns/compute-bound.md","original_body_sha256":"1250f2c8d05fdd1ec4dcb30133b71caf29617b7fe0c0131fcafbe9ca9a4fe320","original_unit_ids":["u006"],"locator":"body:L16","original_claim_id":"pattern-compute-bound-original-c004","original_claim":"2-SM cooperative MMA always doubles the effective tile to m256n256 and provides twice the compute per cycle.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"PTX ISA 9.0","locator":"cta_group::2","observation":"Shapes and layouts are kind/configuration dependent, and a paired operation consumes two CTAs; the ISA promises no 2x performance."}],"strongest_alternative":"m256n256 is a representative maximum shape and 2x describes aggregate work.","discriminating_result":"The row states an invariant performance effect rather than an example shape/work amount.","action":"correct","action_reason":"State exact compatibility scope and measure the paired-resource tradeoff.","proposed_replacement":"Test 2-SM mode for legal shapes when peer operand reuse or tile mapping predicts a benefit.","replacement_claim_ids":["pattern-compute-bound-current-c003"]} +{"receipt_id":"rem-pattern-compute-bound-004","receipt_scope":"wiki","path":"wiki/patterns/compute-bound.md","original_body_sha256":"1250f2c8d05fdd1ec4dcb30133b71caf29617b7fe0c0131fcafbe9ca9a4fe320","original_unit_ids":["u008"],"locator":"body:L18","original_claim_id":"pattern-compute-bound-original-c006","original_claim":"Warp specialization provides dedicated TMA/MMA/epilogue warps with no stalls.","original_verdict":"refuted","evidence":[{"url":"https://gau-nernst.github.io/tcgen05/","revision":"retrieved 2026-08-09","locator":"warp-specialized and persistent versions","observation":"The source reports a scoped gain yet later shows idle time and imperfect overlap."}],"strongest_alternative":"No stalls means it removes role-switch branches only.","discriminating_result":"The row's performance effect is absolute and unqualified.","action":"correct","action_reason":"Treat role separation as a candidate with resource and synchronization costs.","proposed_replacement":"Test specialization only when role/control work is on the critical issue path.","replacement_claim_ids":["pattern-compute-bound-current-c003"]} +{"receipt_id":"rem-pattern-compute-bound-005","receipt_scope":"wiki","path":"wiki/patterns/compute-bound.md","original_body_sha256":"1250f2c8d05fdd1ec4dcb30133b71caf29617b7fe0c0131fcafbe9ca9a4fe320","original_unit_ids":["u011"],"locator":"body:L24-L31","original_claim_id":"pattern-compute-bound-original-c007","original_claim":"FA4 uses Cody-Waite plus a Horner polynomial for every software exponential, thereby multiplying exponential throughput and causing the full-kernel 1605-TFLOP/s result.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/html/2603.05451v1","revision":"FlashAttention-4 paper v1","locator":"partial software exponentiation","observation":"FA4 applies floor-based base-2 reduction and a cubic polynomial to a selected fraction, retaining ex2 for the rest."},{"url":"https://tridao.me/blog/2026/flash4/","revision":"Tri Dao FA4 blog, 2026","locator":"performance","observation":"1605 TFLOP/s is a full-kernel endpoint with multiple mechanisms."}],"strongest_alternative":"Cody-Waite was intended generically for range reduction.","discriminating_result":"The named algorithm, selection scope, throughput causality, and result scope are all misstated.","action":"delete","action_reason":"Remove the misleading pseudo-code and present the exact scoped case in prose.","proposed_replacement":null,"replacement_claim_ids":["pattern-compute-bound-current-c004"]} +{"receipt_id":"rem-pattern-compute-bound-006","receipt_scope":"wiki","path":"wiki/patterns/compute-bound.md","original_body_sha256":"1250f2c8d05fdd1ec4dcb30133b71caf29617b7fe0c0131fcafbe9ca9a4fe320","original_unit_ids":["u012"],"locator":"body:L34","original_claim_id":"pattern-compute-bound-original-c008","original_claim":"2-SM cooperative MMA requires cluster configuration and identical shared-memory layouts in the two CTAs.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"PTX ISA 9.0","locator":"cta_group::2","observation":"Cluster co-residency and instruction-specific peer/resource rules apply, but identical SMEM layouts are not a universal ISA requirement."}],"strongest_alternative":"Identical layouts are a safe implementation convention.","discriminating_result":"A convention cannot be taught as an architecture requirement.","action":"correct","action_reason":"Use the actual instruction/configuration contract.","proposed_replacement":"Validate the selected kind, descriptors, cluster geometry, peer access, and resource lifetime for the exact 2-SM form.","replacement_claim_ids":["pattern-compute-bound-current-c003"]} +{"receipt_id":"rem-pattern-compute-bound-007","receipt_scope":"wiki","path":"wiki/patterns/compute-bound.md","original_body_sha256":"1250f2c8d05fdd1ec4dcb30133b71caf29617b7fe0c0131fcafbe9ca9a4fe320","original_unit_ids":["u013"],"locator":"body:L35","original_claim_id":"pattern-compute-bound-original-c009","original_claim":"Pipeline depth is workload-dependent, with three to five stages typical.","original_verdict":"refuted","evidence":[{"url":"https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu","revision":"3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"launch","observation":"The cited beneficial pipeline uses two stages."}],"strongest_alternative":"The range is common beyond this tutorial.","discriminating_result":"No representative dataset or scope supports typicality, and a cited counterexample exists.","action":"correct","action_reason":"Retain only configuration dependence and controlled selection.","proposed_replacement":"Enumerate legal depths from concrete storage/occupancy limits and benchmark them.","replacement_claim_ids":["pattern-compute-bound-current-c003"]} +{"receipt_id":"rem-pattern-low-sm-utilization-001","receipt_scope":"wiki","path":"wiki/patterns/low-sm-utilization.md","original_body_sha256":"ffe8f3f021f4946f96b95bfc41d6745d38c52fd887cb7171751f8f4e9934e91b","original_unit_ids":["u001"],"locator":"body:L3","original_claim_id":"pattern-low-sm-utilization-original-c001","original_claim":"SM utilization below 60% despite sufficient occupancy, plus idle-SM intervals, defines the low-utilization symptom.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"occupancy and sampling","observation":"Occupancy is a residency limit and samples are observations; no 60% threshold defines the diagnosis."}],"strongest_alternative":"60% is only a triage threshold.","discriminating_result":"It has no metric, target, phase, or workload scope.","action":"correct","action_reason":"Define the symptom from insufficient useful active work relative to required parallelism and timing.","proposed_replacement":"Measure resident/active work over time and distinguish grid shortage, residency, imbalance, and phase behavior.","replacement_claim_ids":["pattern-low-sm-utilization-current-c001"]} +{"receipt_id":"rem-pattern-low-sm-utilization-002","receipt_scope":"wiki","path":"wiki/patterns/low-sm-utilization.md","original_body_sha256":"ffe8f3f021f4946f96b95bfc41d6745d38c52fd887cb7171751f8f4e9934e91b","original_unit_ids":["u006"],"locator":"body:L16","original_claim_id":"pattern-low-sm-utilization-original-c003","original_claim":"CLC is SM100-only dynamic tile assignment that eliminates load imbalance.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"PTX ISA 9.0","locator":"semantics and target notes","observation":"CLC requires sm_100 or higher and redistributes existing unlaunched IDs; it does not eliminate imbalance."}],"strongest_alternative":"Eliminates means it repairs scheduler-caused imbalance when enough IDs remain.","discriminating_result":"Cancellation may fail and cannot create independent work or equal-duration tiles.","action":"correct","action_reason":"State exact redistribution scope and failure/parallelism limits.","proposed_replacement":"Compare static and CLC scheduling with matched grids while recording returned work and per-worker counts.","replacement_claim_ids":["pattern-low-sm-utilization-current-c003"]} +{"receipt_id":"rem-pattern-low-sm-utilization-003","receipt_scope":"wiki","path":"wiki/patterns/low-sm-utilization.md","original_body_sha256":"ffe8f3f021f4946f96b95bfc41d6745d38c52fd887cb7171751f8f4e9934e91b","original_unit_ids":["u007"],"locator":"body:L17","original_claim_id":"pattern-low-sm-utilization-original-c004","original_claim":"Persistent kernels apply on SM90+ and eliminate the tail effect while making launch overhead one-time.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html","revision":"retrieved 2026-08-09","locator":"fixed worker scheduling","observation":"Persistence changes work acquisition but cannot guarantee tail removal; launch and uncovered-parallelism effects remain problem-specific."}],"strongest_alternative":"A fixed worker pool removes wave quantization by construction.","discriminating_result":"A worker pool still has a final set of work and may lack enough independent items or have unequal durations.","action":"correct","action_reason":"Separate persistence from dynamic assignment and measure tail distribution.","proposed_replacement":"Use a static persistent loop as a baseline and test CLC separately.","replacement_claim_ids":["pattern-low-sm-utilization-current-c003"]} +{"receipt_id":"rem-pattern-low-sm-utilization-004","receipt_scope":"wiki","path":"wiki/patterns/low-sm-utilization.md","original_body_sha256":"ffe8f3f021f4946f96b95bfc41d6745d38c52fd887cb7171751f8f4e9934e91b","original_unit_ids":["u008"],"locator":"body:L18","original_claim_id":"pattern-low-sm-utilization-original-c005","original_claim":"Tile scheduling on SM90+ necessarily improves L2 locality and reduces load variance.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0","locator":"scheduler options","observation":"Raster, swizzle, static/dynamic persistence, and decomposition are distinct choices without guaranteed cache or balance effects."}],"strongest_alternative":"The table only lists intended effects.","discriminating_result":"No controlled evidence or condition scopes the asserted results.","action":"correct","action_reason":"Map each scheduler choice to a specific measured hypothesis.","proposed_replacement":"Test coordinate order for locality, work acquisition for imbalance, and decomposition for insufficient independent work separately.","replacement_claim_ids":["pattern-low-sm-utilization-current-c003"]} +{"receipt_id":"rem-pattern-low-sm-utilization-005","receipt_scope":"wiki","path":"wiki/patterns/low-sm-utilization.md","original_body_sha256":"ffe8f3f021f4946f96b95bfc41d6745d38c52fd887cb7171751f8f4e9934e91b","original_unit_ids":["u009"],"locator":"body:L22-L26","original_claim_id":"pattern-low-sm-utilization-original-c006","original_claim":"The tcgen05 tutorial moves from 86% without persistence/CLC to 98% with persistence plus CLC, keeping all SMs busy.","original_verdict":"refuted","evidence":[{"url":"https://gau-nernst.github.io/tcgen05/","revision":"retrieved 2026-08-09","locator":"v5-v6 and closing","observation":"v5 is 1302.29 and v6 static persistence 1475.93 versus cuBLAS 1506.74; CLC was not added and overlap remains imperfect."}],"strongest_alternative":"The percentages are accurate enough and CLC is conceptually part of persistence.","discriminating_result":"CLC and static persistence are distinct mechanisms, and the source explicitly excludes CLC.","action":"correct","action_reason":"Report exact cumulative version scope without all-SMs-busy causality.","proposed_replacement":"Present v5/v6 as an author-reported combined step to static persistence and output overlap, with no CLC ablation.","replacement_claim_ids":["pattern-low-sm-utilization-current-c004"]} +{"receipt_id":"rem-pattern-low-sm-utilization-006","receipt_scope":"wiki","path":"wiki/patterns/low-sm-utilization.md","original_body_sha256":"ffe8f3f021f4946f96b95bfc41d6745d38c52fd887cb7171751f8f4e9934e91b","original_unit_ids":["u010"],"locator":"body:L29","original_claim_id":"pattern-low-sm-utilization-original-c007","original_claim":"CLC is available only on SM100 datacenter GPUs and not SM120 consumer GPUs.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"PTX ISA 9.0","locator":"Target ISA Notes","observation":"Base CLC requires sm_100 or higher and multicast support explicitly includes sm_120a/sm_120f."}],"strongest_alternative":"Only SM100 CLC is exposed by current CUTLASS schedulers.","discriminating_result":"Library route availability does not change the ISA compatibility claim.","action":"correct","action_reason":"Separate PTX target support from pinned CUTLASS integration.","proposed_replacement":"State the PTX target notes and scope CUTLASS examples to its SM100 scheduler route.","replacement_claim_ids":["pattern-low-sm-utilization-current-c003"]} +{"receipt_id":"rem-pattern-low-sm-utilization-007","receipt_scope":"wiki","path":"wiki/patterns/low-sm-utilization.md","original_body_sha256":"ffe8f3f021f4946f96b95bfc41d6745d38c52fd887cb7171751f8f4e9934e91b","original_unit_ids":["u012"],"locator":"body:L31","original_claim_id":"pattern-low-sm-utilization-original-c009","original_claim":"A non-persistent kernel should always use a grid much larger than the SM count.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html","revision":"retrieved 2026-08-09","locator":"thread-block scheduling","observation":"Grid sufficiency depends on problem work, block resources, occupancy, and duration; CUDA defines no much-greater-than-SM rule."}],"strongest_alternative":"Large grids statistically smooth tail imbalance.","discriminating_result":"They may, but cannot be manufactured without changing decomposition and can add reduction/coordination costs.","action":"correct","action_reason":"Use explicit worker-demand and tile arithmetic.","proposed_replacement":"Compare independent work with resident worker capacity and consider decomposition only as a separate measured change.","replacement_claim_ids":["pattern-low-sm-utilization-current-c002"]} +{"receipt_id":"rem-pattern-memory-bound-001","receipt_scope":"wiki","path":"wiki/patterns/memory-bound.md","original_body_sha256":"55576e61fe10aa796637b7056b452f0d94f5b0e727643e78a422e2b721eb30ff","original_unit_ids":["u001"],"locator":"body:L3","original_claim_id":"pattern-memory-bound-original-c001","original_claim":"High DRAM throughput, low tensor-core utilization, and arithmetic intensity below a roofline knee establish that a kernel is memory-bandwidth bound.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline","revision":"Nsight Compute 2025.3","locator":"roofline","observation":"Diagnosis needs measured work/bytes and relevant ceilings; vague high bandwidth does not establish proximity or exclude other limits."}],"strongest_alternative":"These are standard sufficient symptoms.","discriminating_result":"They omit the attained memory roof, requested/transferred efficiency, scheduler issue, and operation-specific compute ceiling.","action":"correct","action_reason":"Require a complete roofline and traffic/issue record.","proposed_replacement":"Classify from measured intensity and ceilings, then distinguish bandwidth saturation from latency or transaction inefficiency.","replacement_claim_ids":["pattern-memory-bound-current-c001"]} +{"receipt_id":"rem-pattern-memory-bound-002","receipt_scope":"wiki","path":"wiki/patterns/memory-bound.md","original_body_sha256":"55576e61fe10aa796637b7056b452f0d94f5b0e727643e78a422e2b721eb30ff","original_unit_ids":["u002"],"locator":"body:L7","original_claim_id":"pattern-memory-bound-original-c002","original_claim":"GEMV, small-batch decode, and reduction kernels are intrinsically low-arithmetic-intensity causes of memory boundedness.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline","revision":"Nsight Compute 2025.3","locator":"arithmetic intensity","observation":"Intensity is concrete operations per transferred byte, not an algorithm-category property."}],"strongest_alternative":"Those categories usually have little reuse.","discriminating_result":"Fusion, batching, cache reuse, and implementation traffic can move the same category across ceilings.","action":"correct","action_reason":"Use measured work and bytes for the actual implementation/input.","proposed_replacement":"Treat workload labels as hypotheses only.","replacement_claim_ids":["pattern-memory-bound-current-c001"]} +{"receipt_id":"rem-pattern-memory-bound-003","receipt_scope":"wiki","path":"wiki/patterns/memory-bound.md","original_body_sha256":"55576e61fe10aa796637b7056b452f0d94f5b0e727643e78a422e2b721eb30ff","original_unit_ids":["u004"],"locator":"body:L9","original_claim_id":"pattern-memory-bound-original-c004","original_claim":"Uncoalesced loads or L1 thrashing are causes of a memory-bandwidth-bound diagnosis.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"Memory Workload Analysis","observation":"They can produce excess traffic or latency while leaving bandwidth below its ceiling, requiring separate efficiency and throughput evidence."}],"strongest_alternative":"Both waste bandwidth and therefore are bandwidth causes.","discriminating_result":"Wasted transactions and latency are distinct mechanisms even when they increase bytes.","action":"correct","action_reason":"Separate bandwidth ceiling, transaction efficiency, cache behavior, and latency/issue limits.","proposed_replacement":"Use requested-versus-transferred bytes, cache data, achieved bandwidth, and scheduler stalls.","replacement_claim_ids":["pattern-memory-bound-current-c002"]} +{"receipt_id":"rem-pattern-memory-bound-004","receipt_scope":"wiki","path":"wiki/patterns/memory-bound.md","original_body_sha256":"55576e61fe10aa796637b7056b452f0d94f5b0e727643e78a422e2b721eb30ff","original_unit_ids":["u005"],"locator":"body:L15","original_claim_id":"pattern-memory-bound-original-c005","original_claim":"128- or 256-bit vectorized loads maximize memory-bandwidth utilization.","original_verdict":"refuted","evidence":[{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"retrieved 2026-08-08","locator":"Attempt 9","observation":"A wider uint2 load regressed the cited kernel by 16-25%."}],"strongest_alternative":"The negative control changed something besides width.","discriminating_result":"It is the cited author's direct width attempt and defeats the universal maximize claim even if exact causality is complex.","action":"correct","action_reason":"Make width an aligned legal candidate with a scalar/narrow control.","proposed_replacement":"Compare legal widths while recording instruction count, transactions, registers, and time.","replacement_claim_ids":["pattern-memory-bound-current-c003"]} +{"receipt_id":"rem-pattern-memory-bound-005","receipt_scope":"wiki","path":"wiki/patterns/memory-bound.md","original_body_sha256":"55576e61fe10aa796637b7056b452f0d94f5b0e727643e78a422e2b721eb30ff","original_unit_ids":["u006"],"locator":"body:L16","original_claim_id":"pattern-memory-bound-original-c006","original_claim":"L1::no_allocate is the correct policy for streaming data and L1::evict_last for reused data.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld","revision":"PTX ISA 9.0","locator":"eviction priority hints","observation":"These are hints that may be applied, not guaranteed placement or role-based prescriptions."}],"strongest_alternative":"The names encode the intended use cases.","discriminating_result":"Intent is not a guaranteed or universally optimal cache behavior.","action":"correct","action_reason":"Test hints one variable at a time from concrete reuse/interference evidence.","proposed_replacement":"Keep a hint only when matched profiling improves end-to-end time.","replacement_claim_ids":["pattern-memory-bound-current-c003"]} +{"receipt_id":"rem-pattern-memory-bound-006","receipt_scope":"wiki","path":"wiki/patterns/memory-bound.md","original_body_sha256":"55576e61fe10aa796637b7056b452f0d94f5b0e727643e78a422e2b721eb30ff","original_unit_ids":["u007"],"locator":"body:L17","original_claim_id":"pattern-memory-bound-original-c007","original_claim":"Setting maxrregcount increases occupancy.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html","revision":"CUDA Compiler Driver 13.0.2","locator":"maxrregcount","observation":"A cap limits allocation subject to ABI minima and may spill; another resource may limit occupancy."},{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"retrieved 2026-08-08","locator":"register-cap attempt","observation":"80 to64 had no performance effect in the cited kernel."}],"strongest_alternative":"A sufficiently low cap eventually raises resident blocks.","discriminating_result":"It may instead be clamped, spill, or leave occupancy unchanged.","action":"correct","action_reason":"Use compiled-resource and occupancy evidence before timing caps.","proposed_replacement":"Compare uncapped/capped builds and record registers, spills, limiting resource, occupancy, and runtime.","replacement_claim_ids":["pattern-memory-bound-current-c003"]} +{"receipt_id":"rem-pattern-memory-bound-007","receipt_scope":"wiki","path":"wiki/patterns/memory-bound.md","original_body_sha256":"55576e61fe10aa796637b7056b452f0d94f5b0e727643e78a422e2b721eb30ff","original_unit_ids":["u009"],"locator":"body:L19","original_claim_id":"pattern-memory-bound-original-c009","original_claim":"Shared-memory swizzling eliminates bank conflicts.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-map-swizzle","revision":"PTX ISA 9.0","locator":"swizzle modes","observation":"Swizzle remaps addresses under layout constraints; conflicts depend on the concrete access pattern."}],"strongest_alternative":"A correct swizzle is designed specifically to eliminate the target conflict.","discriminating_result":"That scoped result needs a specified access trace and measurement, absent from the row.","action":"correct","action_reason":"Treat swizzle as a shared-memory conflict hypothesis, not a DRAM-bandwidth cure.","proposed_replacement":"Test legal layouts using bank-conflict and end-to-end measurements.","replacement_claim_ids":["pattern-memory-bound-current-c003"]} +{"receipt_id":"rem-pattern-memory-bound-008","receipt_scope":"wiki","path":"wiki/patterns/memory-bound.md","original_body_sha256":"55576e61fe10aa796637b7056b452f0d94f5b0e727643e78a422e2b721eb30ff","original_unit_ids":["u010"],"locator":"body:L23-L34","original_claim_id":"pattern-memory-bound-original-c010","original_claim":"For memory-bound NVFP4 GEMV, the correct priority is wide/coalesced loads, lower register count, differentiated cache policy, and never optimizing compute.","original_verdict":"refuted","evidence":[{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"retrieved 2026-08-08","locator":"Attempts and Takeaways","observation":"Wider loads regressed, a lower register cap had no effect, and the author recommends measurement rather than this fixed order."}],"strongest_alternative":"The list remains a reasonable generic order despite failed variants.","discriminating_result":"Direct same-kernel negative controls show category-level prescriptions fail.","action":"delete","action_reason":"Replace the pseudo-code prescription with the exact negative-control lesson.","proposed_replacement":null,"replacement_claim_ids":["pattern-memory-bound-current-c004"]} +{"receipt_id":"rem-pattern-memory-bound-009","receipt_scope":"wiki","path":"wiki/patterns/memory-bound.md","original_body_sha256":"55576e61fe10aa796637b7056b452f0d94f5b0e727643e78a422e2b721eb30ff","original_unit_ids":["u012"],"locator":"body:L38","original_claim_id":"pattern-memory-bound-original-c012","original_claim":"B200 has 8 TB/s of bandwidth, and a speed-of-light calculation determines achievable performance.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline","revision":"Nsight Compute 2025.3","locator":"roofline","observation":"A nominal specification is an upper-bound input, while achieved ceilings and performance depend on measured workload and system conditions."}],"strongest_alternative":"Determine means set an upper bound, not predict exact runtime.","discriminating_result":"The wording collapses nominal hardware rate, attainable measured bandwidth, and achieved kernel performance.","action":"correct","action_reason":"Use measured ceiling and efficiency with complete environment scope.","proposed_replacement":"Report nominal and measured bandwidth separately and treat roofline as a bound/classifier.","replacement_claim_ids":["pattern-memory-bound-current-c001"]} +{"receipt_id":"rem-pattern-memory-bound-010","receipt_scope":"wiki","path":"wiki/patterns/memory-bound.md","original_body_sha256":"55576e61fe10aa796637b7056b452f0d94f5b0e727643e78a422e2b721eb30ff","original_unit_ids":["u013"],"locator":"body:L39","original_claim_id":"pattern-memory-bound-original-c013","original_claim":"Instruction-level parallelism and compute optimizations necessarily have diminishing returns for memory-bound kernels.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"scheduler and memory analysis","observation":"The classification does not prove whether issue, address/decode work, latency hiding, or inefficiency prevents attaining the memory ceiling."}],"strongest_alternative":"Once truly saturated, compute-only work cannot raise the roof.","discriminating_result":"That narrower claim differs from the page's category-wide advice and excludes transformations that alter memory issue/traffic.","action":"correct","action_reason":"Tie every candidate to a measured critical resource and predicted counter movement.","proposed_replacement":"Reject compute-only work only after a matched test shows the memory ceiling remains saturated and runtime unchanged.","replacement_claim_ids":["pattern-memory-bound-current-c002","pattern-memory-bound-current-c003"]} +{"receipt_id":"rem-pattern-register-pressure-001","receipt_scope":"wiki","path":"wiki/patterns/register-pressure.md","original_body_sha256":"6fd15f8aae2d1ca78d038b8af9ec545760eac38cdbb46e46985f19c35a84a491","original_unit_ids":["u001"],"locator":"body:L3","original_claim_id":"pattern-register-pressure-original-c001","original_claim":"Below-target occupancy plus local-memory spills establishes that high registers per thread are the limiting cause.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"Launch Statistics and Occupancy","observation":"Registers, spills, limiting residency resource, theoretical occupancy, and achieved occupancy are separate measurements."}],"strongest_alternative":"The conjunction strongly implies register pressure.","discriminating_result":"Spills can occur without register-limited residency, and lower occupancy need not limit runtime.","action":"correct","action_reason":"Require compiled resource and limiting-resource evidence plus a causal variant.","proposed_replacement":"Diagnose register pressure only when registers/spills constrain a performance-relevant path and a matched change improves time.","replacement_claim_ids":["pattern-register-pressure-current-c001"]} +{"receipt_id":"rem-pattern-register-pressure-002","receipt_scope":"wiki","path":"wiki/patterns/register-pressure.md","original_body_sha256":"6fd15f8aae2d1ca78d038b8af9ec545760eac38cdbb46e46985f19c35a84a491","original_unit_ids":["u005"],"locator":"body:L15","original_claim_id":"pattern-register-pressure-original-c003","original_claim":"SM100 TMEM is a dedicated 256-KB memory that simply moves accumulators out of registers.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory","revision":"PTX ISA 9.0","locator":"organization and access","observation":"TMEM has 128 lanes by512 32-bit cells and holds tcgen05 D, but capacity/packing, collective access, lifecycle, and target constraints make it more than a byte-memory move."}],"strongest_alternative":"256 KB is exact physical arithmetic and the table is only high level.","discriminating_result":"The omitted access/lifecycle semantics are material to correctness and register-use conclusions.","action":"correct","action_reason":"State organization and resident-D scope without promising exact register relief.","proposed_replacement":"TMEM replaces long-lived per-thread D fragments but still requires allocation, completion, loads, and epilogue registers.","replacement_claim_ids":["pattern-register-pressure-current-c003"]} +{"receipt_id":"rem-pattern-register-pressure-003","receipt_scope":"wiki","path":"wiki/patterns/register-pressure.md","original_body_sha256":"6fd15f8aae2d1ca78d038b8af9ec545760eac38cdbb46e46985f19c35a84a491","original_unit_ids":["u006"],"locator":"body:L16","original_claim_id":"pattern-register-pressure-original-c004","original_claim":"Warp specialization applies only to SM100+ and reduces each warp's register requirement.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cutlass/media/docs/cpp/hopper_gemm.html","revision":"retrieved 2026-08-09","locator":"warp-specialized Hopper GEMM","observation":"Hopper already uses warp-specialized kernels, and register allocation effects depend on role code/compiler mechanisms."}],"strongest_alternative":"The page intended the Blackwell tcgen05 form only.","discriminating_result":"It labels the technique itself SM100+ and guarantees a register effect.","action":"correct","action_reason":"Separate architecture availability from measured role-liveness/resource changes.","proposed_replacement":"Compare role layouts and compiled per-kernel resources; specialization can reduce or raise live state.","replacement_claim_ids":["pattern-register-pressure-current-c002"]} +{"receipt_id":"rem-pattern-register-pressure-004","receipt_scope":"wiki","path":"wiki/patterns/register-pressure.md","original_body_sha256":"6fd15f8aae2d1ca78d038b8af9ec545760eac38cdbb46e46985f19c35a84a491","original_unit_ids":["u008"],"locator":"body:L21-L31","original_claim_id":"pattern-register-pressure-original-c006","original_claim":"A Hopper m64n256 FP32 WGMMA accumulator costs about 128 registers/thread, while Blackwell frees exactly all 128, uses zero accumulator registers, has 256 KB TMEM, and the largest 1-SM MMA always occupies half so double buffering is available.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma","revision":"PTX ISA 9.0","locator":"m64nNk16 D","observation":"N=256 gives N/2=128 D registers/thread on Hopper."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory","revision":"PTX ISA 9.0","locator":"TMEM and tcgen05.ld","observation":"Blackwell resident D is in TMEM, but epilogue loads and other state use registers; shape/packing do not imply exact relief or a universal half-capacity rule."}],"strongest_alternative":"The arithmetic illustrates a common BF16 case.","discriminating_result":"Only the Hopper fragment arithmetic is exact; all Blackwell resource/performance deductions are overgeneralized.","action":"correct","action_reason":"Keep the exact comparison while naming residual registers and configuration-specific capacity.","proposed_replacement":"Contrast resident D location and verify compiled registers, TMEM columns, loads, occupancy, and time for each kernel.","replacement_claim_ids":["pattern-register-pressure-current-c003"]} +{"receipt_id":"rem-pattern-register-pressure-005","receipt_scope":"wiki","path":"wiki/patterns/register-pressure.md","original_body_sha256":"6fd15f8aae2d1ca78d038b8af9ec545760eac38cdbb46e46985f19c35a84a491","original_unit_ids":["u011"],"locator":"body:L36","original_claim_id":"pattern-register-pressure-original-c009","original_claim":"TMEM-to-register transfer latency is offset by the registers freed from accumulators.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"PTX ISA 9.0","locator":"load/wait","observation":"Loads are asynchronous and require wait, but the ISA provides no fixed latency or performance offset guarantee."}],"strongest_alternative":"Offset describes the intended design tradeoff, not a guarantee.","discriminating_result":"The statement asserts a favorable balance without a measured workload or configuration.","action":"correct","action_reason":"Make it a controlled whole-kernel tradeoff.","proposed_replacement":"Measure added load/epilogue work against changed registers, spills, occupancy, and time.","replacement_claim_ids":["pattern-register-pressure-current-c004"]} +{"receipt_id":"rem-pattern-tail-effect-001","receipt_scope":"wiki","path":"wiki/patterns/tail-effect.md","original_body_sha256":"796c1dd09a9e76eff0996bf41283998fce447e492dd300571596ec89ad147ed9","original_unit_ids":["u001"],"locator":"body:L3","original_claim_id":"pattern-tail-effect-original-c001","original_claim":"Performance necessarily drops whenever total_tiles modulo num_SMs is nonzero because the last wave idles many SMs.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"workload distribution and occupancy","observation":"One-wave worker count, residency, durations, scheduling, and remainder determine impact; physical SM count alone does not."}],"strongest_alternative":"The modulo rule is an acceptable equal-duration model.","discriminating_result":"The page states an observed performance rule without declaring the model assumptions.","action":"correct","action_reason":"Define and compute the simplified model, then require measurement for real kernels.","proposed_replacement":"Use T=qW+r with measured available worker capacity W and separately test duration imbalance.","replacement_claim_ids":["pattern-tail-effect-current-c001"]} +{"receipt_id":"rem-pattern-tail-effect-002","receipt_scope":"wiki","path":"wiki/patterns/tail-effect.md","original_body_sha256":"796c1dd09a9e76eff0996bf41283998fce447e492dd300571596ec89ad147ed9","original_unit_ids":["u003","u004"],"locator":"body:L8-L9","original_claim_id":"pattern-tail-effect-original-c003","original_claim":"A static grid-stride schedule leaves remainder tiles on a few fixed SMs, while a non-persistent fixed grid has no dynamic block scheduling.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html","revision":"retrieved 2026-08-09","locator":"block scheduling","observation":"CUDA dynamically schedules blocks on available SMs; grid stride fixes logical worker indices, not blockIdx-to-SM placement."}],"strongest_alternative":"The claim means logical workers rather than physical SM identity.","discriminating_result":"It explicitly attributes work to few SMs and denies ordinary dynamic scheduling.","action":"correct","action_reason":"Separate logical worker assignment from hardware block placement and persistence.","proposed_replacement":"Instrument work per logical worker/SM instead of inferring placement from blockIdx.","replacement_claim_ids":["pattern-tail-effect-current-c002"]} +{"receipt_id":"rem-pattern-tail-effect-003","receipt_scope":"wiki","path":"wiki/patterns/tail-effect.md","original_body_sha256":"796c1dd09a9e76eff0996bf41283998fce447e492dd300571596ec89ad147ed9","original_unit_ids":["u005"],"locator":"body:L15","original_claim_id":"pattern-tail-effect-original-c004","original_claim":"CLC is hardware dynamic scheduling in which idle SMs grab tiles on demand.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"PTX ISA 9.0","locator":"request semantics","observation":"A selected thread requests cancellation of an unspecified unlaunched grid entity and processes the returned coordinate on success."}],"strongest_alternative":"Grab is harmless shorthand for worker-driven cancellation.","discriminating_result":"The shorthand hides request failure, unspecified choice, existing-grid scope, and thread/cluster protocol.","action":"correct","action_reason":"Describe exact work-reassignment lifecycle and limits.","proposed_replacement":"Test CLC as redistribution of existing unlaunched IDs, not work creation.","replacement_claim_ids":["pattern-tail-effect-current-c003"]} +{"receipt_id":"rem-pattern-tail-effect-004","receipt_scope":"wiki","path":"wiki/patterns/tail-effect.md","original_body_sha256":"796c1dd09a9e76eff0996bf41283998fce447e492dd300571596ec89ad147ed9","original_unit_ids":["u006"],"locator":"body:L16","original_claim_id":"pattern-tail-effect-original-c005","original_claim":"Persistent kernels use an SM-count grid, iterate over tiles, and have no wave boundary.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html","revision":"retrieved 2026-08-09","locator":"fixed worker model","observation":"Launch size is software-chosen and repeated acquisition still ends with a final subset; no elimination guarantee exists."}],"strongest_alternative":"Persistence removes hardware waves conceptually.","discriminating_result":"It changes the scheduling model but cannot remove finite-work termination or imbalance.","action":"correct","action_reason":"Present static persistence as a baseline with its own final work distribution.","proposed_replacement":"Choose worker count from resources and measure work counts/durations.","replacement_claim_ids":["pattern-tail-effect-current-c003"]} +{"receipt_id":"rem-pattern-tail-effect-005","receipt_scope":"wiki","path":"wiki/patterns/tail-effect.md","original_body_sha256":"796c1dd09a9e76eff0996bf41283998fce447e492dd300571596ec89ad147ed9","original_unit_ids":["u007"],"locator":"body:L17","original_claim_id":"pattern-tail-effect-original-c006","original_claim":"Raster order and swizzle patterns improve worker distribution.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0","locator":"raster/swizzle","observation":"These remap coordinates and may affect locality, not work count or guaranteed duration balance."}],"strongest_alternative":"Better locality reduces variable tile duration and thus distribution.","discriminating_result":"That indirect causal chain needs traffic and per-worker timing evidence.","action":"correct","action_reason":"Separate locality ordering from acquisition and decomposition.","proposed_replacement":"Test raster for cache effects while holding work acquisition constant.","replacement_claim_ids":["pattern-tail-effect-current-c003"]} +{"receipt_id":"rem-pattern-tail-effect-006","receipt_scope":"wiki","path":"wiki/patterns/tail-effect.md","original_body_sha256":"796c1dd09a9e76eff0996bf41283998fce447e492dd300571596ec89ad147ed9","original_unit_ids":["u008"],"locator":"body:L21-L28","original_claim_id":"pattern-tail-effect-original-c007","original_claim":"A B200 has 142 SMs; for 150 tiles CLC turns a 142+8 two-wave launch into one persistent wave with all SMs busy and causes the tutorial's 86%-to-98% result.","original_verdict":"refuted","evidence":[{"url":"https://gau-nernst.github.io/tcgen05/","revision":"retrieved 2026-08-09","locator":"persistent section","observation":"The source uses 148 SMs, static persistence without CLC, notes imperfect overlap, and has cumulative changes."}],"strongest_alternative":"142 is an availability-constrained SM count and the example is hypothetical.","discriminating_result":"It labels B200 hardware and tutorial data without such scope; the cited source contradicts both.","action":"correct","action_reason":"Use explicit hypothetical arithmetic separate from exact tutorial evidence.","proposed_replacement":"Show T=150,W=142 only as a declared model, then present v5/v6 as a non-CLC cumulative result.","replacement_claim_ids":["pattern-tail-effect-current-c001","pattern-tail-effect-current-c004"]} +{"receipt_id":"rem-pattern-tail-effect-007","receipt_scope":"wiki","path":"wiki/patterns/tail-effect.md","original_body_sha256":"796c1dd09a9e76eff0996bf41283998fce447e492dd300571596ec89ad147ed9","original_unit_ids":["u009"],"locator":"body:L31","original_claim_id":"pattern-tail-effect-original-c008","original_claim":"Tail effects are significant only for tile counts below four times the SM count.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"workload distribution","observation":"No four-wave threshold exists; significance depends on remainder, wave time, residency, imbalance, and whole-kernel fraction."}],"strongest_alternative":"Four waves is a practical cutoff for at most 25% fractional tail.","discriminating_result":"Remainder occupancy can make the bound much smaller or larger in effect, and unequal durations break it.","action":"correct","action_reason":"Give the analytical fractional bound under declared assumptions and measure real kernels.","proposed_replacement":"Under equal durations, amortization scales with wave count; use no universal cutoff.","replacement_claim_ids":["pattern-tail-effect-current-c001"]} +{"receipt_id":"rem-pattern-tail-effect-008","receipt_scope":"wiki","path":"wiki/patterns/tail-effect.md","original_body_sha256":"796c1dd09a9e76eff0996bf41283998fce447e492dd300571596ec89ad147ed9","original_unit_ids":["u011"],"locator":"body:L33","original_claim_id":"pattern-tail-effect-original-c010","original_claim":"CLC is available only on SM100 datacenter GPUs.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"PTX ISA 9.0","locator":"Target ISA Notes","observation":"CLC requires sm_100 or higher and explicitly supports SM120-family multicast targets."}],"strongest_alternative":"Current CUTLASS examples are SM100-only.","discriminating_result":"Library integration scope does not define ISA availability.","action":"correct","action_reason":"Separate ISA compatibility from pinned library examples.","proposed_replacement":"State sm_100-or-higher ISA support and scope CUTLASS scheduler examples to SM100.","replacement_claim_ids":["pattern-tail-effect-current-c003"]} +{"receipt_id":"rem-pattern-moe-load-imbalance-001","receipt_scope":"wiki","path":"wiki/patterns/moe-load-imbalance.md","original_body_sha256":"dd70d3638677328e71242ec9fa14891b9ed9465328f129378f5b835f52741275","original_unit_ids":["u001"],"locator":"body:L5","original_claim_id":"pattern-moe-load-imbalance-original-c001","original_claim":"Grouped MoE latency is necessarily the slowest expert because each SM owns an expert and may idle after finishing it.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html","revision":"retrieved 2026-08-09","locator":"block scheduling","observation":"CUDA exposes no fixed expert-to-SM ownership."}],"strongest_alternative":"The wording is shorthand for a particular expert-specialized implementation.","discriminating_result":"The page states it as the general grouped-GEMM symptom without an implementation trace.","action":"correct","action_reason":"Define imbalance across routing, logical work, worker time, and rank communication without assumed ownership.","proposed_replacement":"Diagnose measured token/tile/worker/rank imbalance for the exact implementation.","replacement_claim_ids":["pattern-moe-load-imbalance-current-c001"]} +{"receipt_id":"rem-pattern-moe-load-imbalance-002","receipt_scope":"wiki","path":"wiki/patterns/moe-load-imbalance.md","original_body_sha256":"dd70d3638677328e71242ec9fa14891b9ed9465328f129378f5b835f52741275","original_unit_ids":["u002"],"locator":"body:L9","original_claim_id":"pattern-moe-load-imbalance-original-c002","original_claim":"Trained MoE routers commonly send exactly 80 percent of tokens to 20 percent of experts.","original_verdict":"refuted","evidence":[{"url":"https://github.com/deepseek-ai/EPLB/blob/d52c72d5b2f2fb4c41afbf8eb21366820239913d/README.md","revision":"EPLB d52c72d5b2f2fb4c41afbf8eb21366820239913d","locator":"load input and examples","observation":"The official tool consumes measured loads and gives no universal 80/20 law."}],"strongest_alternative":"80/20 is an informal Pareto example.","discriminating_result":"It is labeled common in trained models without a scoped dataset.","action":"correct","action_reason":"Require exact observed count distributions.","proposed_replacement":"Record per-expert counts for the evaluated batch and phase.","replacement_claim_ids":["pattern-moe-load-imbalance-current-c001"]} +{"receipt_id":"rem-pattern-moe-load-imbalance-003","receipt_scope":"wiki","path":"wiki/patterns/moe-load-imbalance.md","original_body_sha256":"dd70d3638677328e71242ec9fa14891b9ed9465328f129378f5b835f52741275","original_unit_ids":["u003"],"locator":"body:L10","original_claim_id":"pattern-moe-load-imbalance-original-c003","original_claim":"Static assignment precomputes a tile-to-physical-SM mapping.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html","revision":"retrieved 2026-08-09","locator":"block scheduling","observation":"Logical block IDs are scheduled onto available SMs without a portable fixed mapping."}],"strongest_alternative":"The claim means logical persistent-worker ownership.","discriminating_result":"It explicitly names SM mapping rather than software worker mapping.","action":"correct","action_reason":"Separate logical assignment from physical placement.","proposed_replacement":"A static logical work rule cannot adapt after launch, although CUDA still places blocks dynamically.","replacement_claim_ids":["pattern-moe-load-imbalance-current-c003"]} +{"receipt_id":"rem-pattern-moe-load-imbalance-004","receipt_scope":"wiki","path":"wiki/patterns/moe-load-imbalance.md","original_body_sha256":"dd70d3638677328e71242ec9fa14891b9ed9465328f129378f5b835f52741275","original_unit_ids":["u004"],"locator":"body:L11","original_claim_id":"pattern-moe-load-imbalance-original-c004","original_claim":"Fixed M-max masked layout necessarily computes every padding row.","original_verdict":"refuted","evidence":[{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md","revision":"DeepGEMM 891d57b4","locator":"Masked grouped GEMM","observation":"masked_m supplies valid row counts and the implementation computes valid portions."}],"strongest_alternative":"Fixed storage allocation implies fixed compute extent.","discriminating_result":"The API explicitly separates allocated M-max from valid M.","action":"correct","action_reason":"Preserve graph-friendly allocation while removing false padding-compute attribution.","proposed_replacement":"Treat fixed allocation and valid work as separate quantities.","replacement_claim_ids":["pattern-moe-load-imbalance-current-c002"]} +{"receipt_id":"rem-pattern-moe-load-imbalance-005","receipt_scope":"wiki","path":"wiki/patterns/moe-load-imbalance.md","original_body_sha256":"dd70d3638677328e71242ec9fa14891b9ed9465328f129378f5b835f52741275","original_unit_ids":["u006"],"locator":"body:L18","original_claim_id":"pattern-moe-load-imbalance-original-c006","original_claim":"CLC makes the fastest SMs autonomously grab more tiles.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"PTX ISA 9.0","locator":"request semantics","observation":"A selected thread requests cancellation of unspecified unlaunched work and handles possible failure."}],"strongest_alternative":"Grab is concise shorthand for the CUTLASS scheduler.","discriminating_result":"The shorthand erases request, response, failure, and existing-grid limits.","action":"correct","action_reason":"State the exact reassignment protocol.","proposed_replacement":"Compare CLC-backed reassignment of existing unlaunched IDs with a matched static scheduler.","replacement_claim_ids":["pattern-moe-load-imbalance-current-c003"]} +{"receipt_id":"rem-pattern-moe-load-imbalance-006","receipt_scope":"wiki","path":"wiki/patterns/moe-load-imbalance.md","original_body_sha256":"dd70d3638677328e71242ec9fa14891b9ed9465328f129378f5b835f52741275","original_unit_ids":["u007"],"locator":"body:L19","original_claim_id":"pattern-moe-load-imbalance-original-c007","original_claim":"Persistent kernels inherently amortize launch overhead and loop over a dynamic queue.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0","locator":"static and dynamic schedulers","observation":"Persistent workers may advance statically or through CLC; performance depends on the workload."}],"strongest_alternative":"Most useful persistent kernels use a work queue.","discriminating_result":"The pinned implementation has a distinct zero-overhead static persistent scheduler.","action":"correct","action_reason":"Make persistence and acquisition policy independent choices.","proposed_replacement":"Compare static and dynamic persistent acquisition at matched resources.","replacement_claim_ids":["pattern-moe-load-imbalance-current-c003"]} +{"receipt_id":"rem-pattern-moe-load-imbalance-007","receipt_scope":"wiki","path":"wiki/patterns/moe-load-imbalance.md","original_body_sha256":"dd70d3638677328e71242ec9fa14891b9ed9465328f129378f5b835f52741275","original_unit_ids":["u008"],"locator":"body:L20","original_claim_id":"pattern-moe-load-imbalance-original-c008","original_claim":"Contiguous grouped layout always uses an offsets array for expert boundaries.","original_verdict":"refuted","evidence":[{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md","revision":"DeepGEMM 891d57b4","locator":"Contiguous grouped GEMM","observation":"It accepts per-row expert IDs or per-group prefix-sum ends and requires aligned segments."}],"strongest_alternative":"Both metadata forms can be described generically as offsets.","discriminating_result":"Per-row IDs are not boundary offsets and have a different shape and meaning.","action":"correct","action_reason":"Name both actual ABI forms.","proposed_replacement":"Describe packed A/D plus either per-row IDs or prefix-sum ends.","replacement_claim_ids":["pattern-moe-load-imbalance-current-c002"]} +{"receipt_id":"rem-pattern-moe-load-imbalance-008","receipt_scope":"wiki","path":"wiki/patterns/moe-load-imbalance.md","original_body_sha256":"dd70d3638677328e71242ec9fa14891b9ed9465328f129378f5b835f52741275","original_unit_ids":["u009"],"locator":"body:L21","original_claim_id":"pattern-moe-load-imbalance-original-c009","original_claim":"Masked grouped layout supports CUDA graphs but wastes compute on padding.","original_verdict":"refuted","evidence":[{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md","revision":"DeepGEMM 891d57b4","locator":"Masked grouped GEMM","observation":"The graph use is documented and valid-M counts limit computed portions."}],"strongest_alternative":"Predicated edge tiles still incur some instruction overhead.","discriminating_result":"That is not equivalent to computing every allocated padding row and requires generated-code measurement.","action":"correct","action_reason":"Retain graph scope and separate allocation from valid computation.","proposed_replacement":"Measure edge/predication cost rather than labeling all padding as GEMM work.","replacement_claim_ids":["pattern-moe-load-imbalance-current-c002"]} +{"receipt_id":"rem-pattern-moe-load-imbalance-009","receipt_scope":"wiki","path":"wiki/patterns/moe-load-imbalance.md","original_body_sha256":"dd70d3638677328e71242ec9fa14891b9ed9465328f129378f5b835f52741275","original_unit_ids":["u011"],"locator":"body:L23","original_claim_id":"pattern-moe-load-imbalance-original-c011","original_claim":"EPLB generally yields 1.49x prefill and 2.54x decode speedups.","original_verdict":"refuted","evidence":[{"url":"https://www.lmsys.org/blog/2025-05-05-large-scale-ep/","revision":"published 2025-05-05, retrieved 2026-08-09","locator":"EPLB overall results and limitations","observation":"The values belong to a 96-H100 large-scale SGLang study using in-distribution data and are not universal."}],"strongest_alternative":"The row only cites a known EPLB case study.","discriminating_result":"It omits hardware, system, workload, baseline, and stated distribution-shift limit.","action":"correct","action_reason":"Scope placement mechanism and measurement separately.","proposed_replacement":"Describe EPLB placement and report the figures only with the 96-H100 study scope.","replacement_claim_ids":["pattern-moe-load-imbalance-current-c004"]} +{"receipt_id":"rem-pattern-moe-load-imbalance-010","receipt_scope":"wiki","path":"wiki/patterns/moe-load-imbalance.md","original_body_sha256":"dd70d3638677328e71242ec9fa14891b9ed9465328f129378f5b835f52741275","original_unit_ids":["u015"],"locator":"body:L31","original_claim_id":"pattern-moe-load-imbalance-original-c014","original_claim":"The reward-hack incident prompted runtime isolation and subprocess evaluation in the MLSys 2026 FlashInfer contest.","original_verdict":"refuted","evidence":[{"url":"https://www.gpumode.com/news/reward-hacking-nvfp4","revision":"retrieved 2026-08-09","locator":"evaluation response","observation":"The postmortem identifies reference-kernels PR 104, not a FlashInfer/MLSys response."}],"strongest_alternative":"The later contest privately learned from the incident.","discriminating_result":"Neither first-party record establishes that causal link.","action":"delete","action_reason":"Preserve the verified incident mechanics without inventing historical causality.","proposed_replacement":null,"replacement_claim_ids":["pattern-moe-load-imbalance-current-c005"]} +{"receipt_id":"rem-pattern-moe-load-imbalance-011","receipt_scope":"wiki","path":"wiki/patterns/moe-load-imbalance.md","original_body_sha256":"dd70d3638677328e71242ec9fa14891b9ed9465328f129378f5b835f52741275","original_unit_ids":["u016"],"locator":"body:L35","original_claim_id":"pattern-moe-load-imbalance-original-c015","original_claim":"CLC is unavailable on SM120 consumer Blackwell.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"PTX ISA 9.0","locator":"Target ISA Notes","observation":"CLC requires sm_100 or higher and explicitly lists SM120-family multicast support."}],"strongest_alternative":"CUTLASS CLC scheduler examples are currently SM100-only.","discriminating_result":"That library scope does not negate ISA target support.","action":"correct","action_reason":"Separate PTX compatibility from a particular library integration.","proposed_replacement":"State the PTX target rule and scope CUTLASS examples to SM100.","replacement_claim_ids":["pattern-moe-load-imbalance-current-c003"]} +{"receipt_id":"rem-pattern-moe-load-imbalance-012","receipt_scope":"wiki","path":"wiki/patterns/moe-load-imbalance.md","original_body_sha256":"dd70d3638677328e71242ec9fa14891b9ed9465328f129378f5b835f52741275","original_unit_ids":["u017"],"locator":"body:L36","original_claim_id":"pattern-moe-load-imbalance-original-c016","original_claim":"Dynamic scheduling adds a small overhead to every tile.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0","locator":"CLC rules and pipeline","observation":"Initial work is static, later requests are pipelined and cluster-granular, and no small-latency bound is given."}],"strongest_alternative":"Small is qualitative and means amortizable.","discriminating_result":"The per-tile scope is false and the magnitude remains workload-dependent.","action":"correct","action_reason":"Require matched measurements of request activity and latency.","proposed_replacement":"Treat scheduler overhead and recovered imbalance as an empirical tradeoff.","replacement_claim_ids":["pattern-moe-load-imbalance-current-c003"]} +{"receipt_id":"rem-pattern-moe-load-imbalance-013","receipt_scope":"wiki","path":"wiki/patterns/moe-load-imbalance.md","original_body_sha256":"dd70d3638677328e71242ec9fa14891b9ed9465328f129378f5b835f52741275","original_unit_ids":["u018"],"locator":"body:L37","original_claim_id":"pattern-moe-load-imbalance-original-c017","original_claim":"Small experts have a universal minimum viable tile-size floor.","original_verdict":"refuted","evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/efficient_gemm.md","revision":"CUTLASS 4.5.0","locator":"small dimensions","observation":"Small-shape utilization depends gradually on predication, tile shape, and available blocks; no universal floor is defined."}],"strongest_alternative":"Floor means an informal profitability threshold.","discriminating_result":"Profitability thresholds vary with implementation, shape, and competing baseline.","action":"correct","action_reason":"Make small-M a measured risk rather than a fixed cutoff.","proposed_replacement":"Sweep token counts and tile shapes for the exact kernel.","replacement_claim_ids":["pattern-moe-load-imbalance-current-c001"]} +{"receipt_id":"rem-pattern-moe-load-imbalance-014","receipt_scope":"wiki","path":"wiki/patterns/moe-load-imbalance.md","original_body_sha256":"dd70d3638677328e71242ec9fa14891b9ed9465328f129378f5b835f52741275","original_unit_ids":["u019"],"locator":"body:L38","original_claim_id":"pattern-moe-load-imbalance-original-c018","original_claim":"EPLB is cluster-scale only and not single-device.","original_verdict":"refuted","evidence":[{"url":"https://github.com/deepseek-ai/EPLB/blob/d52c72d5b2f2fb4c41afbf8eb21366820239913d/eplb.py","revision":"EPLB d52c72d5b2f2fb4c41afbf8eb21366820239913d","locator":"rebalance_experts","observation":"The planner uses explicit node/GPU counts and supports intra-node placement; its relevant boundary is expert-parallel placement versus device-kernel scheduling."}],"strongest_alternative":"EPLB is intended for distributed expert parallelism.","discriminating_result":"That is true, but distributed does not mean cluster-only and the page needs the kernel/system boundary.","action":"correct","action_reason":"State intended multi-GPU placement scope without a false scale dichotomy.","proposed_replacement":"Treat EPLB as a host-side expert-placement complement, not a tile scheduler.","replacement_claim_ids":["pattern-moe-load-imbalance-current-c004"]} +{"receipt_id":"rem-pattern-moe-load-imbalance-015","receipt_scope":"wiki","path":"wiki/patterns/moe-load-imbalance.md","original_body_sha256":"dd70d3638677328e71242ec9fa14891b9ed9465328f129378f5b835f52741275","original_unit_ids":["u020","u021","u022"],"locator":"body:L42-L44","original_claim_id":"pattern-moe-load-imbalance-original-c019","original_claim":"Uniform routing, a large batch, or an auxiliary loss proves imbalance is absent.","original_verdict":"refuted","evidence":[{"url":"https://www.lmsys.org/blog/2025-05-05-large-scale-ep/","revision":"published 2025-05-05, retrieved 2026-08-09","locator":"EPLB case studies and limitations","observation":"Runtime load and throughput are measured directly and vary by phase, batch, rank count, and distribution; proxy conditions are not sufficient negative tests."}],"strongest_alternative":"Those conditions usually reduce expected routing skew.","discriminating_result":"Reducing expected skew does not establish balanced tiles, worker durations, or rank communication for the observed batch.","action":"correct","action_reason":"Replace proxies with an operational measurement criterion.","proposed_replacement":"Declare imbalance absent only when measured skew and a balanced variant do not materially affect time.","replacement_claim_ids":["pattern-moe-load-imbalance-current-c001"]} +{"receipt_id":"rem-kernel-sparse-mla-001","receipt_scope":"wiki","path":"wiki/kernels/sparse-mla.md","original_body_sha256":"3a4b0e533208cfbbe45f6c7c15396a5e66007ed4a56f0ca9699e034a5b36b628","original_unit_ids":["u001"],"locator":"body:L5","original_claim_id":"kernel-sparse-mla-original-c002","original_claim":"The complete decode attention path falls from O(sequence length) to O(topk=2048) per query.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-deepseek-v32/DeepSeek_V3_2.pdf","revision":"DeepSeek-V3.2-Exp 87e509a2","locator":"Inference Costs","observation":"Core attention becomes O(Lk) for the sequence, while the lightning indexer remains O(L squared)."}],"strongest_alternative":"Attention refers only to the MLA core after selection.","discriminating_result":"The sentence describes the whole two-stage pipeline without retaining the indexer scan.","action":"correct","action_reason":"Separate selector and selected-attention complexity.","proposed_replacement":"State O(Lk) for core attention and O(L squared) for the indexer across a length-L sequence.","replacement_claim_ids":["kernel-sparse-mla-current-c001"]} +{"receipt_id":"rem-kernel-sparse-mla-002","receipt_scope":"wiki","path":"wiki/kernels/sparse-mla.md","original_body_sha256":"3a4b0e533208cfbbe45f6c7c15396a5e66007ed4a56f0ca9699e034a5b36b628","original_unit_ids":["u002"],"locator":"body:L9-L27","original_claim_id":"kernel-sparse-mla-original-c003","original_claim":"The diagram gives a general 64-head indexer, 16-head MLA, page-64 shared-cache and selected-block contract.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-deepseek-v32/inference/config_671B_v3.2.json","revision":"87e509a2","locator":"index/MLA dimensions","observation":"The model has 64 indexer heads and 128 MLA heads; 16 is only an eight-way local shard."},{"path":"/tmp/kernelwiki-deepseek-v32/inference/model.py","revision":"same","locator":"separate caches","observation":"Indexer and MLA caches are distinct."}],"strongest_alternative":"The diagram depicts an eight-way production shard.","discriminating_result":"No shard scope is stated, and page/indices semantics remain wrong.","action":"correct","action_reason":"Replace invented deployment structure with paper equations and exact public API boundaries.","proposed_replacement":"Show selector outputs token positions consumed by separate sparse prefill/decode interfaces.","replacement_claim_ids":["kernel-sparse-mla-current-c001","kernel-sparse-mla-current-c002"]} +{"receipt_id":"rem-kernel-sparse-mla-003","receipt_scope":"wiki","path":"wiki/kernels/sparse-mla.md","original_body_sha256":"3a4b0e533208cfbbe45f6c7c15396a5e66007ed4a56f0ca9699e034a5b36b628","original_unit_ids":["u003","u004","u005","u006","u007"],"locator":"body:L31-L36","original_claim_id":"kernel-sparse-mla-original-c004","original_claim":"Every sparse-MLA entry is a fixed-page 656-byte cache with per-block scales and indexer RoPE.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py","revision":"FlashMLA 71c7379","locator":"V3 FP8+sparse decode layout","observation":"656 bytes is one decode layout with four per-128-value scales and attention RoPE."},{"path":"/tmp/kernelwiki-flashmla-pinned/tests/test_flash_mla_sparse_decoding.py","revision":"same","locator":"page-size cases","observation":"Tests exercise many page sizes, not only 64."}],"strongest_alternative":"The page intends the vLLM V3.2 production layout only.","discriminating_result":"It attributes the restriction to FlashMLA and misidentifies the scale and RoPE roles.","action":"correct","action_reason":"Scope the byte layout and make page size an explicit interface dimension.","proposed_replacement":"Document 656 bytes only for V3-family FP8 sparse decode and distinguish the separate indexer cache.","replacement_claim_ids":["kernel-sparse-mla-current-c003"]} +{"receipt_id":"rem-kernel-sparse-mla-004","receipt_scope":"wiki","path":"wiki/kernels/sparse-mla.md","original_body_sha256":"3a4b0e533208cfbbe45f6c7c15396a5e66007ed4a56f0ca9699e034a5b36b628","original_unit_ids":["u008"],"locator":"body:L41-L60","original_claim_id":"kernel-sparse-mla-original-c005","original_claim":"The invented CUDA indexer computes a maximum score per KV block and top-k blocks.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-deepseek-v32/DeepSeek_V3_2.pdf","revision":"87e509a2","locator":"Equation 1","observation":"DSA computes a weighted ReLU sum across indexer heads for each preceding token."},{"path":"/tmp/kernelwiki-deepseek-v32/inference/model.py","revision":"same","locator":"Indexer.forward","observation":"Top-k is over token scores and follows the logits operation."}],"strongest_alternative":"Block maxima approximate token selection in a paged kernel.","discriminating_result":"The released DSA reference selects token positions exactly and exposes no such approximation.","action":"delete","action_reason":"The snippet is not upstream code and is semantically misleading.","proposed_replacement":null,"replacement_claim_ids":["kernel-sparse-mla-current-c001","kernel-sparse-mla-current-c002"]} +{"receipt_id":"rem-kernel-sparse-mla-005","receipt_scope":"wiki","path":"wiki/kernels/sparse-mla.md","original_body_sha256":"3a4b0e533208cfbbe45f6c7c15396a5e66007ed4a56f0ca9699e034a5b36b628","original_unit_ids":["u009"],"locator":"body:L63-L79","original_claim_id":"kernel-sparse-mla-original-c006","original_claim":"The invented sparse-decode signature takes 2048 selected block indices and fixed [16,576] queries.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py","revision":"FlashMLA 71c7379","locator":"sparse decode signature","observation":"The real API uses batch/s_q/h_q-shaped Q and token page+offset indices; head count and topk are input dimensions."}],"strongest_alternative":"The sketch specializes the V3.2 deployment.","discriminating_result":"It labels token entries as blocks and omits the actual ABI and invalid-index rules.","action":"delete","action_reason":"Use the exact public contract instead of invented CUDA.","proposed_replacement":null,"replacement_claim_ids":["kernel-sparse-mla-current-c002"]} +{"receipt_id":"rem-kernel-sparse-mla-006","receipt_scope":"wiki","path":"wiki/kernels/sparse-mla.md","original_body_sha256":"3a4b0e533208cfbbe45f6c7c15396a5e66007ed4a56f0ca9699e034a5b36b628","original_unit_ids":["u010"],"locator":"body:L85","original_claim_id":"kernel-sparse-mla-original-c007","original_claim":"One dense H800 decode point simultaneously has 660 TFLOPS, 3000 GB/s, and compute-bound behavior.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"FlashMLA 71c7379","locator":"dense decode performance","observation":"The two maxima belong to separate memory-bound and compute-bound configurations."}],"strongest_alternative":"The row summarizes the variant rather than one point.","discriminating_result":"Its singular row and note conflate incompatible regimes.","action":"correct","action_reason":"Report the maxima as separate author-reported regimes.","proposed_replacement":"State up to 3000 GB/s in a memory-bound configuration and 660 TFLOPS in a compute-bound configuration.","replacement_claim_ids":["kernel-sparse-mla-current-c004"]} +{"receipt_id":"rem-kernel-sparse-mla-007","receipt_scope":"wiki","path":"wiki/kernels/sparse-mla.md","original_body_sha256":"3a4b0e533208cfbbe45f6c7c15396a5e66007ed4a56f0ca9699e034a5b36b628","original_unit_ids":["u011"],"locator":"body:L86","original_claim_id":"kernel-sparse-mla-original-c008","original_claim":"H800 sparse decode is an FP8 MMA at 410 TFLOPS.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"FlashMLA 71c7379","locator":"sparse decode performance","observation":"The cache is FP8 but the matrix multiplication is BF16."}],"strongest_alternative":"Dtype labels storage rather than arithmetic.","discriminating_result":"The table does not name that distinction and therefore misstates the kernel precision.","action":"correct","action_reason":"Separate storage and compute dtypes and retain source scope.","proposed_replacement":"FP8 KV storage, BF16 MMA, up to 410 TFLOPS on H800 SXM5/CUDA 12.8.","replacement_claim_ids":["kernel-sparse-mla-current-c004"]} +{"receipt_id":"rem-kernel-sparse-mla-008","receipt_scope":"wiki","path":"wiki/kernels/sparse-mla.md","original_body_sha256":"3a4b0e533208cfbbe45f6c7c15396a5e66007ed4a56f0ca9699e034a5b36b628","original_unit_ids":["u012"],"locator":"body:L87","original_claim_id":"kernel-sparse-mla-original-c009","original_claim":"B200 sparse decode is FP8 MMA at 350 TFLOPS and lower because bandwidth dominates.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"FlashMLA 71c7379","locator":"sparse decode performance","observation":"It reports up to 350 and says not really optimized; MMA is BF16 and no bandwidth causality is stated."}],"strongest_alternative":"The cause follows from a roofline model.","discriminating_result":"No shape, bytes, timing, or roofline record supports that attribution.","action":"correct","action_reason":"Retain only the scoped author report and explicit unoptimized caveat.","proposed_replacement":"Up to 350 TFLOPS on B200, labeled not really optimized by the source.","replacement_claim_ids":["kernel-sparse-mla-current-c004"]} +{"receipt_id":"rem-kernel-sparse-mla-009","receipt_scope":"wiki","path":"wiki/kernels/sparse-mla.md","original_body_sha256":"3a4b0e533208cfbbe45f6c7c15396a5e66007ed4a56f0ca9699e034a5b36b628","original_unit_ids":["u013"],"locator":"body:L88","original_claim_id":"kernel-sparse-mla-original-c010","original_claim":"The 1460-TFLOPS B200 result is dense MLA prefill and a tcgen05 peak.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"FlashMLA 71c7379","locator":"dense MHA prefill","observation":"The value is attributed to NVIDIA's dense MHA forward implementation, not MLA; no tcgen05-peak label is given."}],"strongest_alternative":"FlashMLA uses MLA as the library name for every operator.","discriminating_result":"The support matrix explicitly distinguishes MHA and MQA/MLA modes.","action":"correct","action_reason":"Name the correct operator and attribution.","proposed_replacement":"Dense MHA forward on B200, up to 1460 TFLOPS as reported by NVIDIA.","replacement_claim_ids":["kernel-sparse-mla-current-c004"]} +{"receipt_id":"rem-kernel-sparse-mla-010","receipt_scope":"wiki","path":"wiki/kernels/sparse-mla.md","original_body_sha256":"3a4b0e533208cfbbe45f6c7c15396a5e66007ed4a56f0ca9699e034a5b36b628","original_unit_ids":["u014"],"locator":"body:L89","original_claim_id":"kernel-sparse-mla-original-c011","original_claim":"B200 sparse prefill is FP8 and its 1450 TFLOPS matches BF16 dense prefill.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"FlashMLA 71c7379","locator":"sparse prefill API and performance","observation":"Inputs are BF16; 1450 is an incomplete author-reported maximum and no matched equivalence is claimed."}],"strongest_alternative":"1450 and 1460 are close enough to call a match.","discriminating_result":"They are different operators and undisclosed benchmark configurations, so numerical proximity is non-discriminating.","action":"correct","action_reason":"Report the value only with its dtype/environment/source limitations.","proposed_replacement":"Sparse MLA prefill with BF16 Q/KV, up to 1450 TFLOPS on B200/CUDA 12.9.","replacement_claim_ids":["kernel-sparse-mla-current-c004"]} +{"receipt_id":"rem-kernel-sparse-mla-011","receipt_scope":"wiki","path":"wiki/kernels/sparse-mla.md","original_body_sha256":"3a4b0e533208cfbbe45f6c7c15396a5e66007ed4a56f0ca9699e034a5b36b628","original_unit_ids":["u015"],"locator":"body:L93","original_claim_id":"kernel-sparse-mla-original-c012","original_claim":"32K is a verified universal context-length threshold for using sparse MLA.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-deepseek-v32/DeepSeek_V3_2.pdf","revision":"87e509a2","locator":"Inference Costs and Figure 3","observation":"The source gives a 0-128K curve and short-prefill special case, not a 32K crossover rule."}],"strongest_alternative":"32K is a common production heuristic.","discriminating_result":"No matched hardware/workload comparison establishes it as the boundary.","action":"correct","action_reason":"Replace threshold advice with measured selector-plus-attention crossover.","proposed_replacement":"Measure full pipeline latency across context lengths for the target deployment.","replacement_claim_ids":["kernel-sparse-mla-current-c005"]} +{"receipt_id":"rem-kernel-sparse-mla-012","receipt_scope":"wiki","path":"wiki/kernels/sparse-mla.md","original_body_sha256":"3a4b0e533208cfbbe45f6c7c15396a5e66007ed4a56f0ca9699e034a5b36b628","original_unit_ids":["u016"],"locator":"body:L94","original_claim_id":"kernel-sparse-mla-original-c014","original_claim":"Unspecified similar MLA architectures can use the same sparse kernel path.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py","revision":"FlashMLA 71c7379","locator":"decode and prefill contracts","observation":"Exact dimensions, dtypes, layouts, head relationships and index shapes are required."}],"strongest_alternative":"Similar means matching those exact contracts.","discriminating_result":"The page does not define similarity or require contract equality.","action":"correct","action_reason":"Make compatibility checklist-based.","proposed_replacement":"Use only when the model and cache/index tensors match the selected API contract.","replacement_claim_ids":["kernel-sparse-mla-current-c005"]} +{"receipt_id":"rem-kernel-sparse-mla-013","receipt_scope":"wiki","path":"wiki/kernels/sparse-mla.md","original_body_sha256":"3a4b0e533208cfbbe45f6c7c15396a5e66007ed4a56f0ca9699e034a5b36b628","original_unit_ids":["u018"],"locator":"frontmatter:performance_claims","original_claim_id":"kernel-sparse-mla-original-c016","original_claim":"A structured B200 record establishes 1450 FP8 TFLOPS at 32K/topk2048 and compute-bound utilization.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"FlashMLA 71c7379","locator":"sparse prefill performance","observation":"Only up to 1450 on B200/CUDA 12.9 is reported; inputs are BF16 and shape/timing/variance/utilization are absent."}],"strongest_alternative":"The production case in tests supplies 32K/topk2048.","discriminating_result":"The README maximum is not tied to that test case and lacks a reproducible tuple.","action":"delete","action_reason":"Do not encode an incomplete author maximum as a structured benchmark claim.","proposed_replacement":null,"replacement_claim_ids":["kernel-sparse-mla-current-c004"]} +{"receipt_id":"rem-kernel-sparse-mla-014","receipt_scope":"wiki","path":"wiki/kernels/sparse-mla.md","original_body_sha256":"3a4b0e533208cfbbe45f6c7c15396a5e66007ed4a56f0ca9699e034a5b36b628","original_unit_ids":["u001"],"locator":"frontmatter:blackwell_relevance","original_claim_id":"kernel-sparse-mla-original-c017","original_claim":"One SM100 FP8 block-scale tcgen05 mechanism enables both indexer and sparse attention.","original_verdict":"refuted","evidence":[{"path":"/tmp/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm100_fp8_mqa_logits.cuh","revision":"DeepGEMM 891d57b4","locator":"UMMA and scales","observation":"Indexer logits use FP8 UMMA with explicit scales."},{"path":"/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py","revision":"FlashMLA 71c7379","locator":"dtype contracts","observation":"Sparse decode dequantizes FP8 storage for BF16 MMA and sparse prefill uses BF16 Q/KV."}],"strongest_alternative":"All stages ultimately rely on tcgen05 on SM100.","discriminating_result":"Shared instruction generation does not make both stages FP8 block-scaled or one fused mechanism.","action":"correct","action_reason":"State separate SM100 indexer and attention implementations with their actual precisions.","proposed_replacement":"DeepGEMM provides FP8 indexer logits; FlashMLA provides BF16 sparse-attention math with mode-specific storage.","replacement_claim_ids":["kernel-sparse-mla-current-c002","kernel-sparse-mla-current-c003"]} +{"receipt_id":"rem-kernel-tensorrt-llm-blackwell-indexer-001","receipt_scope":"wiki","path":"wiki/kernels/tensorrt-llm-blackwell-indexer.md","original_body_sha256":"c6277510118240c58c46ecfd1a84c7978461c9a65c84b4f74cbd679662dae2d1","original_unit_ids":["u002"],"locator":"body:L8-L17","original_claim_id":"kernel-tensorrt-llm-blackwell-indexer-original-c003","original_claim":"The displayed candidate_indexer_probe is a meaningful implementation probe for the PR's sparse indexer path.","original_verdict":"refuted","evidence":[{"path":"artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheGather.cu","revision":"TensorRT-LLM PR 13340 merge 897c4bff","locator":"indexerKCacheGatherUnifiedKernel","observation":"The real operator handles separate payload/scale mappings, a strided paged cache, vectorized payload/scale copies, and negative slots; the sketch implements none of them."},{"path":"artifacts/prs/tensorrt-llm/PR-13340/key-files/tensorrt_llm/_torch/attention_backend/sparse/dsa.py","revision":"TensorRT-LLM PR 13340 merge 897c4bff","locator":"sparse_attn_indexer","observation":"The actual path has quantized Q/K preparation, cache movement, MQA logits, and separate top-k selection."}],"strongest_alternative":"The sketch is intentionally only a memory-safety smoke test rather than a functional implementation.","discriminating_result":"It has no bounds parameter or guard and merely casts one byte per thread to int, so even that narrower role is not established by the page or pinned source.","action":"delete","action_reason":"Remove invented CUDA that neither implements nor safely probes the pinned indexer contract.","proposed_replacement":null,"replacement_claim_ids":["kernel-tensorrt-llm-blackwell-indexer-current-c002","kernel-tensorrt-llm-blackwell-indexer-current-c003","kernel-tensorrt-llm-blackwell-indexer-current-c004"]} +{"receipt_id":"rem-kernel-tensorrt-llm-blackwell-indexer-002","receipt_scope":"wiki","path":"wiki/kernels/tensorrt-llm-blackwell-indexer.md","original_body_sha256":"c6277510118240c58c46ecfd1a84c7978461c9a65c84b4f74cbd679662dae2d1","original_unit_ids":["u001"],"locator":"frontmatter:blackwell_relevance","original_claim_id":"kernel-tensorrt-llm-blackwell-indexer-original-c007","original_claim":"The PR provides fused quantized gather/scatter kernels for the Blackwell FP4/FP8 indexer cache paths.","original_verdict":"refuted","evidence":[{"path":"artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/fusedCatFp4.cu","revision":"TensorRT-LLM PR 13340 merge 897c4bff","locator":"fusedCatFp4Kernel stages 1-7","observation":"FP4 quantization is fused with PE/NoPE concatenation before cache movement."},{"path":"artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheGather.cu","revision":"TensorRT-LLM PR 13340 merge 897c4bff","locator":"indexerKCacheGatherUnifiedKernel","observation":"Gather copies already-quantized payload and scale bytes and performs no quantization."},{"path":"artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheScatter.cu","revision":"TensorRT-LLM PR 13340 merge 897c4bff","locator":"indexerKCacheScatterUnifiedKernel","observation":"Scatter copies already-quantized payload and scale bytes and performs no quantization."}],"strongest_alternative":"Fused quantized gather/scatter could mean that both packed values and scales move in one copy kernel.","discriminating_result":"The wording attributes quantization to gather/scatter, but the pinned stage boundary places quantization exclusively in fusedCatFp4.","action":"correct","action_reason":"Name concat-plus-quantization and byte-copy cache movement as separate operations.","proposed_replacement":"On SM100 the PR adds fused FP4 concatenation/quantization, 64-byte packed cache movement, FP4 indexer logits, and separate top-k selection.","replacement_claim_ids":["kernel-tensorrt-llm-blackwell-indexer-current-c001","kernel-tensorrt-llm-blackwell-indexer-current-c004"]} +{"receipt_id":"rem-pattern-memory-bound-011","receipt_scope":"wiki","path":"wiki/patterns/memory-bound.md","original_body_sha256":"55576e61fe10aa796637b7056b452f0d94f5b0e727643e78a422e2b721eb30ff","original_unit_ids":["u011"],"locator":"body:L37","original_claim_id":"pattern-memory-bound-original-c011","original_claim":"One should always profile before optimizing because a wrong assumption wastes effort.","original_verdict":"refuted","evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"roofline, guided analysis, and profiling overhead","observation":"Profiling supplies concrete bottleneck evidence, but collection has replay/overhead costs and the guide does not make profiling a prerequisite for every optimization; correctness, static legality, and already-established invariants admit useful work before profiling."}],"strongest_alternative":"Always is harmless emphasis for the sound principle that measurement prevents speculative tuning.","discriminating_result":"The principle is useful, but the universal sequencing rule is stronger than the primary guidance and excludes correctness fixes, static analysis, and changes supported by an existing representative profile.","action":"correct","action_reason":"Replace an absolute workflow rule with an evidence-based diagnose, hypothesize, control, and measure loop.","proposed_replacement":"Use profiling when a performance hypothesis depends on an unknown bottleneck, and record the evidence or prior result that justifies each candidate.","replacement_claim_ids":["pattern-memory-bound-current-c004"]} +{"receipt_id":"rem-technique-chunk-parallelism-004","receipt_scope":"wiki","path":"wiki/techniques/chunk-parallelism.md","original_body_sha256":"f7d2c3d8c9327b5dfd8b4e5c47d71cc82ce97f8ba4c85aad30dbadf15aeb1e8c","original_unit_ids":["u006","u007"],"locator":"body:L46-L47","original_claim_id":"technique-chunk-parallelism-original-c004","original_claim":"Chunk parallelism should be used for all linear attention and recurrent-state models including Mamba, GatedDeltaNet, and the Delta Rule.","original_verdict":"refuted","evidence":[{"url":"https://arxiv.org/abs/2503.14376v3","revision":"TFLA v3","locator":"abstract and algorithm scope","observation":"TFLA establishes a two-level sequence-parallel construction for its linear-RNN/mLSTM formulation, not a universal transformation for every recurrent model."},{"url":"https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921","revision":"b53d6d3a161267432a79c1c04af69fa52bddc921","locator":"chunkwise training implementation","observation":"GatedDeltaNet uses a model-specific WY-form chunkwise algorithm and layouts."},{"url":"https://github.com/state-spaces/mamba","revision":"retrieved 2026-08-09","locator":"selective_scan_interface.py and selective_scan implementation","observation":"Mamba exposes its selective-scan recurrence rather than the page's generic chunk prescription."}],"strongest_alternative":"All named models have an associative or transformable recurrence, so some form of chunking is always appropriate.","discriminating_result":"Possible model-specific reformulations do not establish universal applicability or profitability; the direct implementations use different state equations, transforms, and execution paths.","action":"correct","action_reason":"Require a proven model-specific chunkwise-equivalent transform and measured benefit before transfer.","proposed_replacement":"Apply chunk parallelism only when the concrete recurrence has a validated chunkwise-equivalent formulation, correct boundary-state propagation, and a beneficial measured schedule.","replacement_claim_ids":["technique-chunk-parallelism-current-c001","technique-chunk-parallelism-current-c002","technique-chunk-parallelism-current-c003","technique-chunk-parallelism-current-c005"]} diff --git a/verification/reviewed/current/hw-2sm-cooperative.json b/verification/reviewed/current/hw-2sm-cooperative.json new file mode 100644 index 000000000..d521aea2a --- /dev/null +++ b/verification/reviewed/current/hw-2sm-cooperative.json @@ -0,0 +1,137 @@ +{ + "path": "wiki/hardware/2sm-cooperative.md", + "id": "hw-2sm-cooperative", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "hw-2sm-cooperative-current-c001", + "claim": "A cta_group::2 MMA is initiated by one thread from a valid CTA pair, requires the peer to remain active, and accesses current and peer CTA TMEM; pair allocation management instead requires one warp per CTA.", + "claim_type": "semantics", + "covers": ["u001", "u002"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-issue-granularity", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "9.7.16.5 Table 46, CTA Pair, and tcgen05.mma description", + "observation": "The table distinguishes one-thread pair-level MMA/commit issue from two-warp pair allocation management, defines the active peer condition, and the MMA description assigns both CTAs' TMEM to group 2." + } + ], + "challenge": { + "alternative": "Both CTAs must issue MMA collectively, while only one CTA allocates pair-level TMEM.", + "check": "Read the separate mma and alloc/dealloc rows of Table 46.", + "outcome": "The current page matches the rows; the alternative reverses their issue granularities." + } + }, + { + "claim_id": "hw-2sm-cooperative-current-c002", + "claim": "cta_group::2 selects pair-level resources while idesc and the instruction configuration select legal shape; m256xn256xk16 is an F16/BF16 maximum example, not the only group-2 shape.", + "claim_type": "semantics", + "covers": ["u003", "u004"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-data-path-layout-organization", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "instruction descriptor and Layout A-F, including M=128 + cta_group::2", + "observation": "idesc encodes M/N and other operation details, and the group-2 data-path layouts include values other than M=256." + } + ], + "challenge": { + "alternative": "The descriptor contains types only and cta_group alone fixes M=256.", + "check": "Inspect the idesc description and the explicitly titled M=128 + cta_group::2 layout.", + "outcome": "The primary source assigns shapes to idesc and directly falsifies a fixed-M alternative." + } + }, + { + "claim_id": "hw-2sm-cooperative-current-c003", + "claim": "The current page correctly points readers to the verified tcgen05 page for the cta_group::2 grammar and states that its lane-disable vector contains eight registers.", + "claim_type": "semantics", + "covers": ["u005"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "9.7.16.10.9.1 disable-output-lane operand description", + "observation": "The specification assigns four b32 elements for group 1 and eight for group 2; the local target exists and reproduces the normative grammar." + } + ], + "challenge": { + "alternative": "The eight-element count belongs to a different kind or sparse form.", + "check": "Inspect the operand-size rule immediately under the dense tcgen05.mma grammar.", + "outcome": "The size is selected by cta_group and applies to the dense grammar linked by the page." + } + }, + { + "claim_id": "hw-2sm-cooperative-current-c004", + "claim": "The correctness checklist contains the direct CTA-pair, allocation, descriptor, operand-lifetime, completion, and inter-thread ordering invariants required by PTX without asserting one universal data partition.", + "claim_type": "semantics", + "covers": ["u006", "u007", "u008", "u009", "u010", "u011"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-issue-granularity", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "issue granularity, MMA source lifetime, fence, and commit sections", + "observation": "The source defines pair lifetime and issue rules, prohibits modifying A/B source storage before MMA completion, assigns completion to commit/mbarrier, and assigns cross-thread ordering to tcgen05 fences composed with execution-ordering operations." + } + ], + "challenge": { + "alternative": "A CTA cluster barrier alone satisfies completion and lets the next TMA overwrite operands immediately.", + "check": "Compare the explicit async source-lifetime rule with fence and commit guarantees.", + "outcome": "The alternative lacks completion tracking; the current checklist keeps the mechanisms distinct." + } + }, + { + "claim_id": "hw-2sm-cooperative-current-c005", + "claim": "The page accurately scopes the tutorial's v4-to-v5 2-SM result to its B200 M=N=K=4096 environment and recommends benchmarking rather than a universal M threshold or persistence guarantee.", + "claim_type": "performance", + "covers": ["u012", "u013"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [ + { + "command": "conda run -n base python -c 'v4=1208.83; v5=1302.29; c=1506.74; print(100*v4/c,100*v5/c,100*(v5/v4-1),100*(v4/v4-1))'", + "environment": "Conda base, Python 3.13.14; source inputs from the article's B200 M=N=K=4096 table, PyTorch 2.9.1 + CUDA 13", + "reference": "https://gau-nernst.github.io/tcgen05/ final performance table", + "result": "80.228%, 86.431%, and 7.731% relative uplift, matching the current page's precise rounded values.", + "negative_control": "v4 compared with itself yields 0% uplift." + } + ], + "challenge": { + "alternative": "The percentages are portable peak-efficiency guarantees or prove that persistence is necessary for group 2.", + "check": "Trace the values and scheduling stages to the exact article setup and verify the current page's scope language.", + "outcome": "The page reports one experiment and explicitly withholds a universal rule." + } + }, + { + "claim_id": "hw-2sm-cooperative-current-c006", + "claim": "Both Related links resolve to the intended hardware pages.", + "claim_type": "other", + "covers": ["u014", "u015"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [ + { + "command": "for p in wiki/hardware/tcgen05-mma.md wiki/hardware/tmem.md wiki/hardware/does-not-exist.md; do test -f \"$p\" && echo \"$p present\" || echo \"$p missing\"; done", + "environment": "Current repository workspace on 2026-08-08", + "reference": "Targets resolved relative to wiki/hardware/", + "result": "tcgen05-mma.md and tmem.md are present.", + "negative_control": "does-not-exist.md reports missing." + } + ], + "challenge": { + "alternative": "One relative target has a case or directory mismatch.", + "check": "Resolve the exact case-sensitive paths and compare a broken control.", + "outcome": "Both intended paths resolve." + } + } + ], + "scope": "All 15 freshly rescanned current units were checked against CUDA 13.0.2 PTX ISA 9.0, the tutorial's exact source-reported setup, and local link resolution.", + "residual_risk": "No B200 or CUDA toolkit is present locally, so the reported performance was not reproduced; it remains explicitly source-reported and non-generalized." +} diff --git a/verification/reviewed/current/hw-clc.json b/verification/reviewed/current/hw-clc.json new file mode 100644 index 000000000..b5a77c9b5 --- /dev/null +++ b/verification/reviewed/current/hw-clc.json @@ -0,0 +1,388 @@ +{ + "path": "wiki/hardware/clc.md", + "id": "hw-clc", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "hw-clc-current-c001", + "claim": "CLC is a Blackwell compute-capability 10.0 work-stealing mechanism, and a CLC kernel launches the problem-sized grid.", + "claim_type": "mechanism", + "covers": ["u001"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", + "revision": "CUDA Programming Guide 13.2", + "locator": "4.12 introduction and Cluster Launch Control flow", + "observation": "The guide introduces CLC on Blackwell compute capability 10.0, defines it as work stealing, and describes canceling blocks from the launched problem grid." + } + ], + "challenge": { + "alternative": "CLC is a CUTLASS-only queue that replaces the CUDA grid.", + "check": "Inspect the architecture and launch model in the independent CUDA Programming Guide.", + "outcome": "The guide defines an architectural cancellation operation over not-yet-started grid blocks." + } + }, + { + "claim_id": "hw-clc-current-c002", + "claim": "Each ClcID is processed either by its normal launch or by a worker that cancels it; a worker uses blockIdx first and then may process a returned ClcID.", + "claim_type": "semantics", + "covers": ["u002", "u003", "u004", "u005"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", + "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "Dynamic Scheduler with Cluster Launch Control, rules 1-5 and pseudocode", + "observation": "The pinned documentation states both exclusive processing paths, uses blockIdx as first work, and uses the query response for subsequent work." + } + ], + "challenge": { + "alternative": "A ClcID may be processed both by a newly launched block and by the stealing worker, or the first tile also requires a query.", + "check": "Inspect rules 1-4 and the dynamic-persistent pseudocode.", + "outcome": "Rule 3 guarantees one of the two paths, and rule 4 assigns blockIdx first." + } + }, + { + "claim_id": "hw-clc-current-c003", + "claim": "CLC lets a persistent worker request subsequent work without a separate software queue and addresses load imbalance when real-time SM availability is uneven.", + "claim_type": "mechanism", + "covers": ["u006"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cutlass/4.3.3/media/docs/cpp/blackwell_cluster_launch_control.html", + "revision": "CUTLASS 4.3.3", + "locator": "Static Scheduler and Dynamic Scheduler with Cluster Launch Control", + "observation": "CUTLASS contrasts its static software tile scheduler with CLC queries and motivates dynamic scheduling with SMs unavailable because of other work." + } + ], + "challenge": { + "alternative": "CLC is useful only for equal-cost tail waves and provides no response to uneven SM availability.", + "check": "Inspect the scenario and motivation immediately preceding the documented CLC rules.", + "outcome": "The scenario explicitly varies SM availability and shows dynamic worker allocation." + } + }, + { + "claim_id": "hw-clc-current-c004", + "claim": "try_cancel asynchronously writes a 16-byte opaque response to shared memory, completes an mbarrier transaction, takes no target tile coordinate, and is decoded with the shown query_cancel forms.", + "claim_type": "code", + "covers": ["u007", "u008", "u009"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "9.7.13.17 try_cancel syntax and description; 9.7.13.18 query_cancel syntax and description", + "observation": "The page reproduces the normative b128 shared-response/mbarrier request and the predicate and first-ctaid decoder forms without inventing a target-coordinate operand." + } + ], + "challenge": { + "alternative": "The request returns coordinates and success directly in ordinary registers or accepts a caller-selected ClcID.", + "check": "Compare each displayed operand to the normative PTX grammar and decoder description.", + "outcome": "Coordinates and success exist only inside the later-loaded opaque response; request operands are shared addresses." + } + }, + { + "claim_id": "hw-clc-current-c005", + "claim": "A complete request loop needs response and mbarrier setup, a one-thread asynchronous request with 16-byte transaction accounting, completion wait, success-before-ID decode, and proxy ordering before reuse.", + "claim_type": "semantics", + "covers": ["u010", "u011", "u012", "u013", "u014", "u015", "u016"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", + "revision": "CUDA Programming Guide 13.2", + "locator": "4.12.1.1 Thread Block Cancellation, steps 1-6", + "observation": "The official programming sequence contains each listed setup, submission, synchronization, decode, and proxy-visibility step." + } + ], + "challenge": { + "alternative": "A CTA barrier alone makes a direct request result safe, so the transaction count and proxy fences are optional.", + "check": "Inspect the official step-by-step example and async-proxy note.", + "outcome": "The example separately performs mbarrier transaction tracking and async/generic proxy fences." + } + }, + { + "claim_id": "hw-clc-current-c006", + "claim": "After observing a failed request, another request is undefined; decoding an ID from failure is undefined; failure may indicate exhausted IDs or another scheduling reason such as higher-priority work.", + "claim_type": "semantics", + "covers": ["u017"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", + "revision": "CUDA Programming Guide 13.2", + "locator": "4.12 flow and 4.12.1.2 Constraints on Thread Block Cancellation", + "observation": "The guide lists both undefined behaviors and explains both end-of-grid and higher-priority scheduling as failure reasons." + } + ], + "challenge": { + "alternative": "Failure always and only means end-of-grid, and callers may safely retry or decode the stale ID.", + "check": "Inspect the failure explanation and both explicit constraints.", + "outcome": "The vendor guide names another failure cause and declares both retry-after-observation and failed-ID decode undefined." + } + }, + { + "claim_id": "hw-clc-current-c007", + "claim": "Cluster cancellation uses one cluster thread, a multicast response, per-CTA local barriers at cluster scope, local-rank adjustment, and prior cluster synchronization; a 2x2 query consumes a 2x2 ClcID group.", + "claim_type": "semantics", + "covers": ["u018", "u019"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", + "revision": "CUDA Programming Guide 13.2", + "locator": "4.12.2.2 Use-case: Thread Block Clusters", + "observation": "The guide specifies the one-thread request, multicasted first-ID response, per-CTA cluster-scope barriers, local-rank addition, and cluster.sync requirement." + }, + { + "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", + "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "Dynamic Scheduler rule 6", + "observation": "CUTLASS gives the exact 2x2 worker cluster consuming 2x2 ClcIDs example." + } + ], + "challenge": { + "alternative": "Each CTA submits an independent request and receives an unrelated coordinate.", + "check": "Compare the multicast first-ID rule and local-rank reconstruction in the official cluster example.", + "outcome": "One cluster request broadcasts the same base ID and CTAs derive their members by rank." + } + }, + { + "claim_id": "hw-clc-current-c008", + "claim": "In CUTLASS 4.5.0, PersistentScheduler on arch::Sm100 maps to PersistentTileSchedulerSm100; DynamicPersistentScheduler selects the dynamic implementation, whose advance and fetch methods use PipelineCLCFetchAsync.", + "claim_type": "compatibility", + "covers": ["u020"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/tile_scheduler.hpp", + "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "scheduler tags and Sm100 TileSchedulerSelector specializations", + "observation": "Both PersistentScheduler and DynamicPersistentScheduler select PersistentTileSchedulerSm100 for arch::Sm100." + }, + { + "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp", + "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "PersistentTileSchedulerSm100 Pipeline alias, advance_to_next_work, and fetch_next_work", + "observation": "The class aliases PipelineCLCFetchAsync and splits request submission from response consumption across the two named methods." + } + ], + "challenge": { + "alternative": "PersistentScheduler selects an SM90 static scheduler on SM100, or the named methods bypass the CLC pipeline.", + "check": "Inspect architecture-specific selector specializations and both method bodies at the pinned revision.", + "outcome": "The SM100 selector and implementation directly identify the dynamic CLC scheduler and pipeline." + } + }, + { + "claim_id": "hw-clc-current-c009", + "claim": "The schematic CUTLASS 3.x composition correctly places DynamicPersistentScheduler as the fourth GemmUniversal kernel template parameter after the problem shape and collective mainloop and epilogue.", + "claim_type": "code", + "covers": ["u021", "u022"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/95_blackwell_gemm_green_context/95_blackwell_gemm_green_context.cu", + "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "dynamic_kernel_config", + "observation": "The official SM100 example defines DynamicPersistentScheduler and instantiates kernel::GemmUniversal with Shape, collective mainloop, collective epilogue, and scheduler in exactly that order." + } + ], + "challenge": { + "alternative": "The scheduler belongs inside CollectiveMainloop or after a legacy device::GemmUniversal argument list.", + "check": "Compare the schematic's namespace, template arity, and order against the pinned example.", + "outcome": "All match the authoritative dynamic_kernel_config composition." + } + }, + { + "claim_id": "hw-clc-current-c010", + "claim": "CUTLASS applies software swizzle/rasterization to initial and returned ClcIDs using scheduler arguments; the CLC PTX interface has no swizzle-policy operand.", + "claim_type": "mechanism", + "covers": ["u023"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp", + "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "Arguments, initial_work_tile_info, fetch_next_work, and swizzle_and_rasterize", + "observation": "max_swizzle_size and raster_order feed a device-side transform invoked for blockIdx and decoded responses." + }, + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "9.7.13.17-18 CLC instruction syntax", + "observation": "The normative request and query operands contain no tile-order or swizzle policy." + } + ], + "challenge": { + "alternative": "CUTLASS forwards the scheduler arguments to a hidden launch-time CLC hardware policy.", + "check": "Trace both coordinate paths through the pinned swizzle_and_rasterize body and compare the PTX operands.", + "outcome": "CUTLASS performs the transform explicitly in software after obtaining coordinates." + } + }, + { + "claim_id": "hw-clc-current-c011", + "claim": "CLC cannot discard a caller-selected output tile.", + "claim_type": "semantics", + "covers": ["u024", "u025"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", + "revision": "CUDA Programming Guide 13.2", + "locator": "4.12 flow and 4.12.1 API Details", + "observation": "The request takes response and barrier addresses, selects an unspecified not-yet-started block, and transfers its ID to the requester for processing." + } + ], + "challenge": { + "alternative": "A desired ClcID is an input and success removes that output from the computation.", + "check": "Inspect the official function call operands and success path.", + "outcome": "No desired ID is supplied, and the successful caller executes the returned work." + } + }, + { + "claim_id": "hw-clc-current-c012", + "claim": "CLC only redistributes ClcIDs in the launched grid and cannot create more independent tiles than that grid contains.", + "claim_type": "mechanism", + "covers": ["u024", "u026"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", + "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "Dynamic Scheduler rules 1-6", + "observation": "The grid contains one ClcID per output block and every ID is consumed by normal launch or a successful cancellation; no operation creates IDs." + } + ], + "challenge": { + "alternative": "A successful request synthesizes a new work coordinate beyond the launched grid.", + "check": "Inspect the source and disposition of every ClcID in rules 1-6.", + "outcome": "Every returned ID is one whose original launch was atomically canceled." + } + }, + { + "claim_id": "hw-clc-current-c013", + "claim": "CLC does not guarantee full SM occupancy or universally eliminate GEMM wave quantization.", + "claim_type": "performance", + "covers": ["u024", "u027", "u028"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html", + "revision": "NVIDIA Matrix Multiplication Background User's Guide", + "locator": "2.3 tile parallelism and 3.2 Wave Quantization", + "observation": "Occupancy depends on the number of output tiles relative to SMs and on tile shape; an underfilled wave is possible whenever insufficient tiles remain." + }, + { + "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", + "revision": "CUDA Programming Guide 13.2", + "locator": "4.12 Cluster Launch Control flow", + "observation": "CLC redistributes pending grid blocks and its requests may decline; no full-occupancy or universal tail guarantee is made." + } + ], + "challenge": { + "alternative": "Work stealing guarantees all SMs remain busy for every grid size and tile duration distribution.", + "check": "Consider the documented model when fewer grid tiles than SMs exist and compare NVIDIA's wave-quantization definition.", + "outcome": "No redistribution can expose more simultaneous independent blocks than the grid contains." + } + }, + { + "claim_id": "hw-clc-current-c014", + "claim": "A 32-tile grid exposes at most 32 independent ClcIDs, while the referenced B200 configuration has 148 SMs.", + "claim_type": "numeric", + "covers": ["u029"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://developer.nvidia.com/blog/scaling-autonomous-ai-agents-and-workloads-with-nvidia-dgx-spark/", + "revision": "NVIDIA technical blog, 2026", + "locator": "Table 8, NVIDIA Blackwell B200 (Cloud) SM count column", + "observation": "The table lists 148 SMs for the B200 configuration." + }, + { + "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", + "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "Dynamic Scheduler rules 1-3", + "observation": "There is one ClcID per launched grid coordinate and every ID follows exactly one processing path, so a 32-tile grid has at most 32 independent IDs." + } + ], + "challenge": { + "alternative": "The B200 has 132 SMs or CLC expands 32 grid coordinates into 148 IDs.", + "check": "Read the exact B200 table cell and the ClcID cardinality rules.", + "outcome": "The vendor table gives 148, and CLC only transfers existing IDs." + } + }, + { + "claim_id": "hw-clc-current-c015", + "claim": "A meaningful CLC performance result must scope the GPU and available SMs, software, tile/cluster shape, data types, timed region, and measurement method.", + "claim_type": "performance", + "covers": ["u029"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html", + "revision": "NVIDIA Matrix Multiplication Background User's Guide", + "locator": "2.3 Typical Tile Dimensions and Performance; 3.2 Wave Quantization", + "observation": "NVIDIA's comparisons explicitly scope GPU, CUDA/cuBLAS versions, problem size, tile size, and the measured metric, and explain why tile and SM counts change performance." + }, + { + "url": "https://docs.nvidia.com/cutlass/4.3.3/media/docs/cpp/blackwell_cluster_launch_control.html", + "revision": "CUTLASS 4.3.3", + "locator": "Overview scenario", + "observation": "The CLC scheduling comparison fixes output tile grid, GPU SM count, unavailable SM count, and cluster shape before describing behavior." + } + ], + "challenge": { + "alternative": "CLC performance is an invariant feature property, so hardware, tile shape, software, and timed scope can be omitted.", + "check": "Compare NVIDIA's scoped examples and the variables they identify as controlling parallelism and scheduling.", + "outcome": "Changing those variables changes the number of tiles, waves, available workers, and measured result." + } + }, + { + "claim_id": "hw-clc-current-c016", + "claim": "All three Related links resolve to existing wiki pages.", + "claim_type": "other", + "covers": ["u030", "u031", "u032"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [ + { + "command": "conda run -n base python -c 'from pathlib import Path; base=Path(\"wiki/hardware\"); links=[\"../techniques/persistent-kernels.md\",\"../techniques/tile-scheduling.md\",\"2sm-cooperative.md\"]; resolved=[(x,(base/x).resolve().relative_to(Path.cwd().resolve()).as_posix(),(base/x).exists()) for x in links]; broken=\"missing-clc-target.md\"; print({\"links\":resolved,\"all_exist\":all(x[2] for x in resolved),\"negative_control\":(base/broken).exists()})'", + "environment": "Conda base, Python 3.13.14, repository root /users/Master/wiki-evolve/KernelWiki.", + "reference": "Each relative target must resolve from wiki/hardware and exist; a nonexistent target must fail.", + "result": "persistent-kernels.md, tile-scheduling.md, and 2sm-cooperative.md all resolved and existed.", + "negative_control": "wiki/hardware/missing-clc-target.md returned false." + } + ], + "challenge": { + "alternative": "A link appears valid textually but resolves relative to the wrong directory.", + "check": "Resolve every target from the actual page directory and compare a broken target.", + "outcome": "All three real targets exist at their resolved paths while the broken control does not." + } + } + ], + "commands": [ + "conda run -n base python scripts/validate.py", + "conda run -n base python /users/Master/.codex/skills/kernel-wiki-verifier/scripts/scan_wiki.py /users/Master/wiki-evolve/KernelWiki --output /users/Master/wiki-evolve/KernelWiki/verification/current-queue.jsonl", + "git clone --depth 1 --branch v4.5.0 --filter=blob:none https://github.com/NVIDIA/cutlass.git /tmp/kernelwiki-cutlass-dRC5i4/cutlass", + "conda run -n base python -c " + ], + "scope": "Independent current-text pass over all 32 coverage units. CLC semantics and constraints were checked against CUDA Programming Guide 13.2 and archived PTX ISA 9.0; CUTLASS API, pipeline, and coordinate transforms were checked at v4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5; the B200 count used the exact B200 column of NVIDIA's platform table; all current links were tested from the page directory.", + "residual_risk": "None for the claims retained on this page. No positive CLC throughput or latency number is claimed, so absence of a local NVIDIA GPU does not leave a performance verdict unresolved." +} diff --git a/verification/reviewed/current/hw-mbarrier.json b/verification/reviewed/current/hw-mbarrier.json new file mode 100644 index 000000000..7bfaa70fa --- /dev/null +++ b/verification/reviewed/current/hw-mbarrier.json @@ -0,0 +1,188 @@ +{ + "path": "wiki/hardware/mbarrier.md", + "id": "hw-mbarrier", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "hw-mbarrier-current-c001", + "claim": "mbarrier is a naturally aligned opaque 64-bit shared-memory object introduced in PTX 7.0 for sm_80; sm_90 added tx-count and cluster-scope support, and Blackwell uses it for TMA and tcgen05 completion without requiring it in every warp-specialized kernel.", + "claim_type": "compatibility", + "covers": ["u001", "u002"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "object size/alignment, PTX 7.0 target notes, PTX 8.0 change log, TMA and tcgen05 completion sections", + "observation": "The vendor history and operation definitions match each scoped generation claim; the page frames usage as optional design choice." + } + ], + "challenge": { + "alternative": "The sm_80 form is a different primitive and modern mbarrier began with Hopper.", + "check": "Compare base init/arrive/test_wait ISA notes with the later expect_tx/complete_tx notes.", + "outcome": "The same mbarrier object and base instructions require sm_80; only tx-count and cluster extensions begin with sm_90." + } + }, + { + "claim_id": "hw-mbarrier-current-c002", + "claim": "An mbarrier tracks pending arrivals, next-phase expected arrivals, and tx-count; a phase completes only when arrivals and tx-count are both zero, then advances atomically, and the displayed initialization/lifecycle rules and PTX form are exact.", + "claim_type": "code", + "covers": ["u003", "u004", "u005", "u006", "u007", "u008", "u009", "u010"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#contents-of-the-mbarrier-object", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "object contents, lifecycle, phase completion, and mbarrier.init syntax", + "observation": "All three fields, conjunctive completion, automatic next-phase initialization, wait-before-next-arrive rule, invalidation rule, and the exact CTA-shared init form are normative." + } + ], + "challenge": { + "alternative": "Either zero arrivals or zero tx-count independently completes the phase, or reinitialization may replace invalidation.", + "check": "Inspect the completion bullet conjunction and the undefined-behavior clause for init on a valid object.", + "outcome": "Both counters must be zero and a valid object must be invalidated before reinitialization." + } + }, + { + "claim_id": "hw-mbarrier-current-c003", + "claim": "The operation table and representative PTX correctly distinguish arrive, arrive.expect_tx, expect_tx, complete_tx, state/parity waits, retry behavior, and acquire/release versus relaxed semantics.", + "claim_type": "code", + "covers": ["u011", "u012", "u013", "u014", "u015", "u016", "u017", "u018", "u019"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "arrive/expect_tx/complete_tx and wait grammars and descriptions", + "observation": "The effects and exact CTA-shared forms match the ISA; try_wait may resume before completion and relaxed lacks ordering guarantees." + } + ], + "challenge": { + "alternative": "arrive.expect_tx only changes tx-count, while complete_tx also decrements arrivals.", + "check": "Trace each instruction to the separately defined arrive-on, expect-tx, and complete-tx operations.", + "outcome": "arrive.expect_tx composes arrive-on with expect-tx; complete_tx changes only tx-count, exactly as the table states." + } + }, + { + "claim_id": "hw-mbarrier-current-c004", + "claim": "Parity is each object's phase low bit, parity waits require lifetime tracking of that object, a stage ring therefore needs per-stage phase/state, and an opaque arrival token can identify the phase instead.", + "claim_type": "semantics", + "covers": ["u020", "u021", "u022"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "state and phaseParity operand semantics; parity tracking note", + "observation": "Waits accept either a returned state for the current/immediately preceding phase or that phase's even/odd parity, and the ISA requires parity tracking throughout object lifetime." + } + ], + "challenge": { + "alternative": "A global iteration parity works for any number of independently reused stage objects.", + "check": "Apply the per-object phase definition to a two-stage ring and compare a stage's first and second use.", + "outcome": "The global bit toggles twice before the same stage is reused and returns to its old value, while that stage advances once; per-object state is necessary." + } + }, + { + "claim_id": "hw-mbarrier-current-c005", + "claim": "The TMA sequence correctly uses arrive.expect_tx for one software arrival and the total expected bytes, TMA complete-tx for copied bytes, a consumer wait, and forbids unmatched extra arrival or complete_tx operations.", + "claim_type": "semantics", + "covers": ["u023", "u024", "u025", "u026", "u027", "u028", "u029"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "mbarrier::complete_tx::bytes mechanism and TMA operand conditions", + "observation": "Each TMA copy decrements tx-count by bytes copied and the mbarrier is the only guaranteed completion observation mechanism." + }, + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "arrive.expect_tx composed semantics", + "observation": "The instruction performs one arrive-on and increases tx-count, matching the accounting sequence and excluding a second unplanned arrival." + } + ], + "challenge": { + "alternative": "TMA completion performs an arrive-on, so pending-arrival count should include one hardware arrival per copy.", + "check": "Compare the TMA completion qualifier with the definitions of complete-tx and arrive-on.", + "outcome": "The qualifier explicitly requests complete-tx::bytes, which decrements tx-count and not pending arrivals." + } + }, + { + "claim_id": "hw-mbarrier-current-c006", + "claim": "tcgen05.commit tracks prior MMA/cp/shift operations with one completion arrive-on, distinct from TMA byte tx-count, and an applicable cross-thread consumer handoff combines the wait with after_thread_sync ordering rather than substituting the fence for completion.", + "claim_type": "semantics", + "covers": ["u030", "u031"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "mbarrier::arrive::one description and cross-thread synchronization examples", + "observation": "Commit signals one arrive-on after prior MMA/cp/shift completion, and the consumer example waits then executes after_thread_sync before issuing dependent tcgen05 work." + } + ], + "challenge": { + "alternative": "tcgen05.commit contributes a byte tx-count just like TMA, or the after fence itself waits.", + "check": "Inspect the completion_mechanism token and sequence in the different-thread example.", + "outcome": "The token is arrive::one and the example has a separate mbarrier wait before the fence." + } + }, + { + "claim_id": "hw-mbarrier-current-c007", + "claim": "The page correctly separates CTA/cluster shared address state space from synchronization scope, limits cluster-shared-object operations to arrive/expect_tx/complete_tx, and derives the eight-item correctness checklist from those lifecycle and ordering rules.", + "claim_type": "semantics", + "covers": ["u032", "u033", "u034", "u035", "u036", "u037", "u038", "u039", "u040", "u041", "u042", "u043", "u044"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#mbarrier-support-with-shared-memory", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "shared-location support table, wait scope description, and cluster Example 4", + "observation": "The table permits only the three stated operations on cluster-shared addresses; scope is a separate qualifier, and the example initializes an owner's CTA-shared object, maps it for remote arrival, and waits at the owner." + } + ], + "challenge": { + "alternative": ".shared::cluster implicitly supplies cluster scope and supports init/wait on the remote object.", + "check": "Compare grammar qualifiers with the operation-by-location support table.", + "outcome": "The grammar separates scope from state space and the table marks all other operations unsupported for .shared::cluster." + } + }, + { + "claim_id": "hw-mbarrier-current-c008", + "claim": "The three external references are pinned to the relevant PTX 9.0 sections and the two internal links resolve to the TMA and warp-specialization pages.", + "claim_type": "other", + "covers": ["u045", "u046", "u047", "u048", "u049"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [ + { + "command": "test -f wiki/hardware/tma.md && test -f wiki/techniques/warp-specialization.md; inspect three CUDA archive URLs", + "environment": "KernelWiki worktree; web retrieval on 2026-08-08", + "reference": "Internal targets exist and external anchors identify mbarrier lifecycle, waits, and TMA tensor-copy sections in CUDA 13.0.2.", + "result": "Both internal files resolve and all external links use the CUDA 13.0.2 archive with the intended anchors.", + "negative_control": "wiki/hardware/not-a-page.md does not exist, showing the file check rejects a broken relative target." + } + ], + "challenge": { + "alternative": "One link tracks rolling docs or resolves relative to the wrong directory.", + "check": "Inspect the archive segment and resolve relative paths from wiki/hardware.", + "outcome": "The three external URLs are archive-pinned, and both relative paths resolve to existing intended files." + } + } + ], + "commands": [ + "test -f wiki/hardware/tma.md && test -f wiki/techniques/warp-specialization.md", + "structural validation after remediation" + ], + "residual_risk": "No CUDA toolkit is installed locally, so the three short PTX forms were not assembled. Their qualifier ordering and operands match PTX ISA 9.0 grammar and examples exactly.", + "scope": "All 49 current units for body hash 4215a80fde138535a1d096475ad4d1b76c905d6cd8c3ab2b6198eae4c78180e3 were verified against CUDA 13.0.2 PTX ISA 9.0 and local link resolution." +} diff --git a/verification/reviewed/current/hw-nvfp4.json b/verification/reviewed/current/hw-nvfp4.json new file mode 100644 index 000000000..306e76b9b --- /dev/null +++ b/verification/reviewed/current/hw-nvfp4.json @@ -0,0 +1,179 @@ +{ + "path": "wiki/hardware/nvfp4.md", + "id": "hw-nvfp4", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "hw-nvfp4-current-c001", + "claim": "NVFP4 is a hierarchical quantization recipe whose 1D form reconstructs each E2M1 payload with a block-of-16 E4M3 local scale and a per-tensor FP32 scale; E2M1 has the stated bits, numeric set, and two-values-per-byte packing, and Transformer Engine 2.13 also defines 16-by-16 weight scaling.", + "claim_type": "semantics", + "covers": ["u001", "u002", "u003", "u004", "u005", "u006", "u007"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", + "revision": "Transformer Engine 2.13", + "locator": "E2M1 data format, two-level scaling, mathematical formulation, and 1D/2D scaling", + "observation": "The vendor defines the E2M1 bit allocation and range, the E2M1/local-scale/global-scale product, one E4M3 scale per 16 values, one FP32 scale per tensor, and a 16-by-16 block in 2D weight scaling. Exhaustive decoding yields the page's finite numeric set; two 4-bit values occupy one byte." + }, + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#alternate-floating-point-data-formats", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "ue4m3 alternate floating-point format", + "observation": "PTX names the scale representation .ue4m3 and defines its one padding/zero MSB followed by four exponent and three mantissa bits." + } + ], + "challenge": { + "alternative": "NVFP4 is only the E2M1 encoding, uses one scale level, or groups 32 values like MXFP4.", + "check": "Separate payload format from the two tensors of scale metadata in the version-pinned recipe and compare the 1D and 2D grouping definitions.", + "outcome": "The documentation explicitly gives E2M1 times local E4M3 times global FP32, with 16-value 1D and 16-by-16 2D groups." + } + }, + { + "claim_id": "hw-nvfp4-current-c002", + "claim": "The comparison table correctly distinguishes NVFP4 block-of-16 E4M3 local scaling plus a global scale from MXFP4 block-of-32 UE8M0 local scaling, and scopes the error statement to representational freedom rather than a universal outcome.", + "claim_type": "semantics", + "covers": ["u008", "u009", "u010", "u011", "u012", "u013", "u014"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", + "revision": "Transformer Engine 2.13", + "locator": "Comparison with MXFP8 and MXFP4", + "observation": "NVIDIA contrasts NVFP4's 16-value E4M3 local groups and per-tensor global scale with MXFP4's 32-value E8M0 power-of-two groups and describes accuracy benefits as recipe behavior rather than an input-independent theorem." + } + ], + "challenge": { + "alternative": "Smaller groups and fractional scales guarantee strictly lower error for all tensors and quantizers.", + "check": "Test the strict claim on exactly representable data and inspect whether the source quantifies every scale-selection method and metric.", + "outcome": "Exactly representable tensors can tie, and the source does not make a universal strict-error claim; the page now states only greater scale freedom and requires workload measurement." + } + }, + { + "claim_id": "hw-nvfp4-current-c003", + "claim": "The PTX table gives valid E2M1 block-scale combinations for kind::mxf4 block32 with UE8M0/scale_vec::2X and kind::mxf4nvf4 block16 with UE4M3/scale_vec::4X, and correctly warns that kind::mxf4nvf4 alone also permits UE8M0 modes and requires an architecture-specific target.", + "claim_type": "compatibility", + "covers": ["u015", "u016", "u017", "u018"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "tcgen05.mma block-scale grammar, valid combination tables, and target notes", + "observation": "The grammar and tables list the two displayed combinations, additional mxf4nvf4 UE8M0 combinations, and architecture-specific sm_100a-family targets." + } + ], + "challenge": { + "alternative": "kind::mxf4nvf4 uniquely selects block16 UE4M3, so the other qualifiers are documentation noise.", + "check": "Enumerate all rows with kind mxf4nvf4 in the PTX valid-combination tables.", + "outcome": "Rows include both UE4M3/block16 and UE8M0 modes; full qualifiers are behavior-selecting." + } + }, + { + "claim_id": "hw-nvfp4-current-c004", + "claim": "The displayed cvt.rn.f16x2.e2m1x2 form converts one byte-sized packed E2M1 pair to a 32-bit f16x2 result, and the typed mov.b32 scalar-to-four-byte capability and measurement caveat are accurate.", + "claim_type": "code", + "covers": ["u019", "u020", "u021"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "packed alternate-format conversion syntax and e2m1x2 example", + "observation": "The exact mnemonic, packed source width, and packed half result width match the page." + }, + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "scalar-to-vector move rules and b32/four-byte example", + "observation": "The ISA permits the described unpack when vector element declarations are byte-sized and explicitly provides semantics rather than a performance guarantee." + } + ], + "challenge": { + "alternative": "The source is a 32-bit scalar or the result contains four f16 values, and mov is unconditionally faster.", + "check": "Apply the x2 packed-type sizes in the cvt grammar and distinguish mov semantics from generated-instruction performance.", + "outcome": "e2m1x2 occupies 8 bits and f16x2 32 bits; neither PTX section promises a relative timing result." + } + }, + { + "claim_id": "hw-nvfp4-current-c005", + "claim": "The page rejects an architecture-independent 4x-over-Hopper number and enumerates the baseline and environment details needed to make an NVFP4 throughput comparison meaningful.", + "claim_type": "performance", + "covers": ["u022"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "mxf4/mxf4nvf4 target notes", + "observation": "Native tcgen05 FP4 forms require Blackwell architecture-specific targets, so Hopper cannot be a like-for-like native instruction baseline. The ISA contains no universal 4x result; shapes, layouts, target, and instruction combinations are explicit variables." + } + ], + "challenge": { + "alternative": "Four-bit operands automatically prove 4x throughput versus Hopper regardless of the chosen baseline.", + "check": "Check whether Hopper is a legal target for the FP4 instruction and whether operand width alone fixes MMA issue rate and end-to-end kernel work.", + "outcome": "Hopper is not a target for these instructions, and the comparison necessarily selects a different instruction/emulation path plus workload conditions." + } + }, + { + "claim_id": "hw-nvfp4-current-c006", + "claim": "The three external references are version-pinned to Transformer Engine 2.13 and CUDA 13.0.2 PTX ISA 9.0 sections that define NVFP4, tcgen05.mma, and cvt.", + "claim_type": "other", + "covers": ["u023", "u024", "u025"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", + "revision": "Transformer Engine 2.13", + "locator": "NVFP4 page", + "observation": "The URL includes the 2.13 release segment and resolves to the cited format/recipe material." + }, + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "tcgen05.mma and cvt anchors", + "observation": "Both links share the CUDA 13.0.2 archive and select the intended PTX sections." + } + ], + "challenge": { + "alternative": "One reference points to rolling documentation or to an unrelated conversion section.", + "check": "Inspect version path segments and resolve both PTX anchors in the archive.", + "outcome": "All version segments and section targets match the labels." + } + }, + { + "claim_id": "hw-nvfp4-current-c007", + "claim": "The three internal references resolve to the fine-grained quantization, NVFP4 GEMM, and NVFP4 GEMV pages.", + "claim_type": "other", + "covers": ["u026", "u027", "u028"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [ + { + "command": "test -f wiki/techniques/fine-grained-quantization.md && test -f wiki/kernels/nvfp4-gemm.md && test -f wiki/kernels/nvfp4-gemv.md", + "environment": "current KernelWiki worktree after NVFP4 remediation", + "reference": "Resolve targets from wiki/hardware and compare their frontmatter IDs with the link labels.", + "result": "All three targets exist and identify the intended technique and kernels.", + "negative_control": "wiki/kernels/not-an-nvfp4-kernel.md does not exist, so the check rejects a broken target." + } + ], + "challenge": { + "alternative": "At least one relative path resolves outside the intended wiki directory or to an unrelated page.", + "check": "Resolve each relative path and inspect its frontmatter ID.", + "outcome": "The paths resolve to technique-fine-grained-quantization, kernel-nvfp4-gemm, and kernel-nvfp4-gemv." + } + } + ], + "commands": [ + "test -f wiki/techniques/fine-grained-quantization.md && test -f wiki/kernels/nvfp4-gemm.md && test -f wiki/kernels/nvfp4-gemv.md", + "structural validation after remediation" + ], + "residual_risk": "No CUDA toolkit or NVIDIA GPU is installed locally, so the one-line PTX form was not assembled and no performance experiment was run. Exact syntax and compatibility come from PTX ISA 9.0; the page deliberately makes no numeric speed claim.", + "scope": "All 28 current units for body hash b574ab0127a35863b7ec1a2eb952aa93fe86bc145085a497ad64b9c3bc93a46a were checked against Transformer Engine 2.13, CUDA 13.0.2 PTX ISA 9.0, and local link resolution." +} diff --git a/verification/reviewed/current/hw-pdl-gdc.json b/verification/reviewed/current/hw-pdl-gdc.json new file mode 100644 index 000000000..45eb4e6ab --- /dev/null +++ b/verification/reviewed/current/hw-pdl-gdc.json @@ -0,0 +1,186 @@ +{ + "path": "wiki/hardware/pdl-gdc.md", + "id": "hw-pdl-gdc", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "hw-pdl-gdc-current-c001", + "claim": "PDL on compute capability 9.0 or later makes a same-stream secondary grid eligible for opportunistic early execution only after every primary CTA triggers or exits; dependent secondary work still waits for prerequisite completion and visibility, while omitted explicit triggers occur at CTA exit.", + "claim_type": "semantics", + "covers": ["u001", "u002", "u003", "u004", "u005", "u006", "u007"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#programmatic-dependent-launch-and-synchronization", + "revision": "CUDA 13.0.2", + "locator": "6.2.8.6 overview and API Description", + "observation": "The guide gives the compute-capability floor, same-stream requirement, all-CTA trigger/exit rule, implicit exit trigger, extensible launch attribute, mandatory dependency wait, and explicit warning that concurrency is opportunistic." + } + ], + "challenge": { + "alternative": "A single CTA trigger releases the secondary's dependent reads immediately and guarantees overlap.", + "check": "Separate grid scheduling eligibility from dependency completion and inspect the all-CTA and wait clauses.", + "outcome": "All CTAs must trigger or exit before early scheduling, and the secondary wait still blocks dependent work until prerequisites complete; overlap is not guaranteed." + } + }, + { + "claim_id": "hw-pdl-gdc-current-c002", + "claim": "The CUDA skeleton places a one-per-primary-CTA trigger, secondary-side dependency wait, and programmaticStreamSerializationAllowed launch attribute in their correct roles and same stream.", + "claim_type": "code", + "covers": ["u008", "u009", "u010"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#programmatic-dependent-launch-and-synchronization", + "revision": "CUDA 13.0.2", + "locator": "6.2.8.6.2 official CUDA example", + "observation": "The official example uses the same primary function, secondary function, launch attribute ID/value, config attrs, and cudaLaunchKernelEx role mapping. The page adds one-thread-per-CTA issue, which is sufficient because repeated same-CTA triggers have no additional effect." + }, + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-griddepcontrol", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "launch_dependents repeated-invocation rule", + "observation": "The ISA states that invocations after the first within a CTA have no extra side effect, supporting the single issuing thread per CTA." + } + ], + "challenge": { + "alternative": "cudaGridDependencySynchronize belongs in the primary or a normal secondary launch receives the dependency implicitly.", + "check": "Trace the two device functions and the host attribute in NVIDIA's official example.", + "outcome": "The trigger is primary-side, the wait is secondary-side, and the secondary is launched through the extensible API with explicit opt-in." + } + }, + { + "claim_id": "hw-pdl-gdc-current-c003", + "claim": "The exact PTX forms griddepcontrol.launch_dependents and griddepcontrol.wait, their per-CTA and visibility semantics, PTX 7.8 introduction, sm_90 floor, and mandatory wait pairing are correct.", + "claim_type": "code", + "covers": ["u011", "u012", "u013"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-griddepcontrol", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "syntax, description, note, ISA notes, and target notes", + "observation": "Every instruction spelling, behavioral qualification, history statement, and target requirement on the page matches the normative section." + } + ], + "challenge": { + "alternative": "launch_dependents itself makes prerequisite writes visible or the dependent may omit wait after early launch.", + "check": "Compare the two action descriptions and the explicit correctness note.", + "outcome": "Only wait provides prerequisite completion/visibility, and the note requires a wait when the prerequisite launches dependents." + } + }, + { + "claim_id": "hw-pdl-gdc-current-c004", + "claim": "Blackwell retains explicit CUDA PDL opt-in, while CUTLASS 4.5.0 separately defaults CUTLASS_ENABLE_GDC_FOR_SM100 ON and leaves its SM90 counterpart opt-in; this build setting is overridable and not device-wide behavior.", + "claim_type": "compatibility", + "covers": ["u014", "u015"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/CMakeLists.txt", + "revision": "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "lines 458-474", + "observation": "SM90 GDC is appended only if an externally set option is true; SM100 receives an overridable default variable set to ON and, when enabled, adds CUTLASS_ENABLE_GDC_FOR_SM100=1." + }, + { + "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/include/cutlass/arch/grid_dependency_control.h", + "revision": "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "compile guards and launch/wait wrappers", + "observation": "The macro gates instruction-emitting wrapper code for eligible architectures, demonstrating library compilation rather than a device-global launch mode." + } + ], + "challenge": { + "alternative": "The default-ON CMake option means every SM100 CUDA launch automatically participates in PDL.", + "check": "Follow the option into the guarded header and compare it with CUDA's host launch attribute contract.", + "outcome": "The option controls CUTLASS compilation only; ordinary kernels and launches do not acquire a dependency or host attribute." + } + }, + { + "claim_id": "hw-pdl-gdc-current-c005", + "claim": "The usage guidance correctly treats independent secondary preamble, post-trigger primary tail, co-residency headroom, and net saved time as benefit preconditions, lists candidate rather than guaranteed workloads, requires a scoped baseline, and forbids reliance on overlap for progress.", + "claim_type": "performance", + "covers": ["u016", "u017", "u018", "u019", "u020", "u021"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#programmatic-dependent-launch-and-synchronization", + "revision": "CUDA 13.0.2", + "locator": "background overlap diagram and opportunistic-concurrency warning", + "observation": "NVIDIA identifies primary tail and secondary preamble as the overlap opportunity, conditions execution on available hardware resources, declines to guarantee overlap, and warns that relying on it can deadlock. The page adds the necessary measurement controls rather than claiming a result." + } + ], + "challenge": { + "alternative": "PDL always shortens a chain because early scheduling has no resource or protocol cost.", + "check": "Apply the vendor's resource/opportunism caveats to a pair with no independent secondary preamble or no co-residency capacity.", + "outcome": "Such a pair has nothing useful to overlap or cannot co-reside, so only an experiment against the explicit baseline can establish benefit." + } + }, + { + "claim_id": "hw-pdl-gdc-current-c006", + "claim": "The external references are pinned to CUDA 13.0.2 PDL/PTX sections and the exact CUTLASS 4.5.0 commit document for dependent launch.", + "claim_type": "other", + "covers": ["u022", "u023", "u024"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#programmatic-dependent-launch-and-synchronization", + "revision": "CUDA 13.0.2", + "locator": "PDL section", + "observation": "The archive and anchor resolve to the cited API contract." + }, + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-griddepcontrol", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "griddepcontrol section", + "observation": "The archive and anchor resolve to the cited instruction contract." + }, + { + "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/dependent_kernel_launch.md", + "revision": "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "dependent kernel launch document", + "observation": "The immutable blob URL resolves to CUTLASS's GDC enablement and use documentation." + } + ], + "challenge": { + "alternative": "One link follows rolling documentation or a mutable branch.", + "check": "Inspect CUDA archive segments and the Git blob commit identifier.", + "outcome": "CUDA links include 13.0.2, and the CUTLASS link names the exact 40-character commit." + } + }, + { + "claim_id": "hw-pdl-gdc-current-c007", + "claim": "The internal references resolve to the persistent-kernel and Cluster Launch Control pages, whose scopes are distinct from PDL.", + "claim_type": "other", + "covers": ["u025", "u026"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [ + { + "command": "test -f wiki/techniques/persistent-kernels.md && test -f wiki/hardware/clc.md", + "environment": "current KernelWiki worktree after PDL remediation", + "reference": "Resolve both links from wiki/hardware and inspect page IDs and scopes.", + "result": "Both files exist; one covers grid-resident persistent scheduling and the other in-grid cancellation/work stealing.", + "negative_control": "wiki/hardware/not-clc.md does not exist, so the check rejects a broken target." + } + ], + "challenge": { + "alternative": "CLC is another name for PDL or one relative link is broken.", + "check": "Resolve both paths and inspect their frontmatter IDs and operation boundaries.", + "outcome": "Both links resolve, and CLC operates on not-yet-started blocks within a launch whereas PDL coordinates separate launches." + } + } + ], + "commands": [ + "test -f wiki/techniques/persistent-kernels.md && test -f wiki/hardware/clc.md", + "inspect CUTLASS 4.5.0 CMakeLists.txt and grid_dependency_control.h", + "structural validation after remediation" + ], + "residual_risk": "No CUDA toolkit or NVIDIA GPU is installed locally, so the CUDA skeleton was not compiled and an overlap timeline was not measured. Its roles mirror the CUDA 13.0.2 example and exact PTX forms; the page makes no quantitative benefit claim.", + "scope": "All 26 current units for body hash 4c2bc154856d1dcfd15bb4f6a12ae30d4dadc52ead1d4e5994b4da45e031cd98 were checked against CUDA 13.0.2, PTX ISA 9.0, CUTLASS 4.5.0 source, and current link resolution." +} diff --git a/verification/reviewed/current/hw-tcgen05-mma.json b/verification/reviewed/current/hw-tcgen05-mma.json new file mode 100644 index 000000000..8ab2756fe --- /dev/null +++ b/verification/reviewed/current/hw-tcgen05-mma.json @@ -0,0 +1,219 @@ +{ + "path": "wiki/hardware/tcgen05-mma.md", + "id": "hw-tcgen05-mma", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "hw-tcgen05-mma-current-c001", + "claim": "PTX defines tcgen05.mma as the fifth-generation TensorCore MMA family for sm_100a, and pinned CUTLASS uses UMMA as its SM100 wrapper namespace without defining the expanded acronym asserted by the old page.", + "claim_type": "compatibility", + "covers": ["u001"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "9.7.16.10.9.1 description and Target ISA Notes", + "observation": "The vendor ISA names the family, describes fifth-generation MMA, and lists sm_100a. Pinned CUTLASS source uses the UMMA namespace but contains no asserted expansion." + } + ], + "challenge": { + "alternative": "The page still promotes an unofficial acronym expansion or assigns tcgen05 to generic sm_100 rather than architecture-specific sm_100a PTX.", + "check": "Compare the wording to the pinned target notes and repository-wide CUTLASS nomenclature search.", + "outcome": "The page now states only the exact family, target, and namespace facts the primary material exposes." + } + }, + { + "claim_id": "hw-tcgen05-mma-current-c002", + "claim": "The WGMMA/tcgen05 comparison correctly distinguishes issue granularity, accumulator location, and A/B operand-location alternatives.", + "claim_type": "semantics", + "covers": ["u002", "u003", "u004", "u005", "u006"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-matrix-multiply-and-accumulate-operations", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "9.7.16.2 issue granularity; 9.7.16.10 overview; WGMMA syntax", + "observation": "tcgen05 mma is single-thread-issued, D is TMEM, A is SMEM or TMEM, and B is SMEM; WGMMA is warpgroup-scoped with register D and instruction-dependent register/descriptor A forms." + } + ], + "challenge": { + "alternative": "The table accidentally carries over the old SMEM-only A or mandatory-ldmatrix generalization.", + "check": "Compare every operand-location cell to both tcgen05 grammar alternatives and WGMMA A forms.", + "outcome": "Each current cell preserves the documented alternatives and makes no mandatory ldmatrix claim." + } + }, + { + "claim_id": "hw-tcgen05-mma-current-c003", + "claim": "Single-thread issue does not remove the explicit tcgen05 commit/mbarrier completion path or cross-thread ordering requirements.", + "claim_type": "semantics", + "covers": ["u007", "u008"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "9.7.16.12.1 tcgen05.commit", + "observation": "commit causes an mbarrier to track prior async operations by the issuer; fence semantics separately order tcgen05 across execution-ordering handoffs." + } + ], + "challenge": { + "alternative": "One issuer makes completion implicit at CTA synchronization.", + "check": "Inspect the explicit completion-mechanism definition and its scope.", + "outcome": "The ISA assigns completion tracking to tcgen05.commit and the mbarrier, independent of issue granularity." + } + }, + { + "claim_id": "hw-tcgen05-mma-current-c004", + "claim": "The page exactly partitions the seven dense kind qualifiers into three unscaled floating, three block-scaled MX, and one integer kind, including the required scale operands for block-scaled forms.", + "claim_type": "semantics", + "covers": ["u009", "u010", "u011", "u012", "u013"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "9.7.16.10.9.1 syntax groups 1, 2, and 5", + "observation": "The grammar lists f16/tf32/f8f6f4 without block scaling, mxf8f6f4/mxf4/mxf4nvf4 with block scaling and two TMEM scale operands, and i8 separately." + } + ], + "challenge": { + "alternative": "f8f6f4 is itself block-scaled or mxf8 is an accepted alias for mxf8f6f4.", + "check": "Compare the exact tokens and operands in the two floating grammar groups.", + "outcome": "The current taxonomy matches the exact tokens and keeps the unscaled and block-scaled forms separate." + } + }, + { + "claim_id": "hw-tcgen05-mma-current-c005", + "claim": "M and N are descriptor-selected under kind/layout/CTA-group/target constraints; cta_group::2 adds peer-CTA resource access, and all tcgen05 instructions in a kernel use one CTA-group value.", + "claim_type": "semantics", + "covers": ["u014"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instruction-descriptor", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "instruction descriptor, data-path layouts, operand access, and tcgen05.commit description", + "observation": "The ISA encodes shape in idesc, defines multiple layouts and peer accesses, and states that one cta_group qualifier value must be used throughout a kernel." + } + ], + "challenge": { + "alternative": "Each kind implies exactly one M/N pair and group 2 merely doubles M.", + "check": "Inspect descriptor-selected shapes and data-path layouts rather than tutorial maximum tiles.", + "outcome": "Multiple legal shapes disprove the fixed-shape alternative; the page now labels maxima as examples." + } + }, + { + "claim_id": "hw-tcgen05-mma-current-c006", + "claim": "Both displayed PTX blocks reproduce the normative unscaled and block-scaled tcgen05.mma operand grammar, with their intentionally non-standalone scope stated explicitly.", + "claim_type": "code", + "covers": ["u015", "u016", "u017", "u018", "u019", "u020"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "9.7.16.10.9.1 syntax, operand descriptions, and examples", + "observation": "The current excerpts retain the disable-output-lane vector and accumulation predicate for unscaled forms, and the block_scale qualifier plus two scale addresses for scaled forms." + } + ], + "challenge": { + "alternative": "The page repeats the old scalar-mask form or presents grammar metavariables as compilable inline PTX.", + "check": "Token-compare the blocks to the pinned grammar and inspect the paragraph that scopes the excerpts.", + "outcome": "The tokens align with the normative grammar and the page explicitly says the excerpts are not complete functions." + } + }, + { + "claim_id": "hw-tcgen05-mma-current-c007", + "claim": "The completion section correctly separates commit/mbarrier completion from tcgen05.fence ordering and preserves SMEM stage lifetime until asynchronous consumption completes.", + "claim_type": "semantics", + "covers": ["u021", "u022", "u023", "u024"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "9.7.16.11.1 fence description and 9.7.16.12.1 commit description", + "observation": "Fence orders operations around execution-ordering instructions and constrains code motion; commit makes an mbarrier track async completion. Operand storage cannot be reused while an async operation still consumes it." + } + ], + "challenge": { + "alternative": "A fence alone completes prior MMA, or an mbarrier wait alone establishes every cross-thread tcgen05 ordering edge.", + "check": "Compare the distinct guarantees in the adjacent fence and commit sections.", + "outcome": "Neither alternative supplies both guarantees; the current wording explicitly composes them only when applicable." + } + }, + { + "claim_id": "hw-tcgen05-mma-current-c008", + "claim": "The shared-memory descriptor section accurately describes the required field categories and the PTX 9.0 swizzle encodings, including 128B value 2 and invalid values 3, 5, and 7.", + "claim_type": "semantics", + "covers": ["u025", "u026"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "Table 40 shared-memory descriptor layout", + "observation": "Table 40 assigns the base/leading/stride/fixed/offset/mode fields and maps swizzles 0,1,2,4,6 while declaring 3,5,7 invalid." + } + ], + "challenge": { + "alternative": "The page accidentally uses WGMMA's older two-bit mode encoding, where value 3 has a meaning.", + "check": "Compare the page against the tcgen05-specific three-bit Table 40 rather than a WGMMA descriptor table.", + "outcome": "The current values exactly match the tcgen05 table and identify value 2, not 3, as ordinary 128B." + } + }, + { + "claim_id": "hw-tcgen05-mma-current-c009", + "claim": "The performance table exactly reports the tutorial's B200 M=N=K=4096 progression and correctly identifies v6 as static-scheduled persistence without CLC.", + "claim_type": "performance", + "covers": ["u028", "u029", "u030", "u031", "u032", "u033", "u034", "u035", "u036", "u037", "u038"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://gau-nernst.github.io/tcgen05/", + "revision": "Article dated 2025-12-21; linked code commit 3b90ac9b", + "locator": "experiment setup, Persistent kernel heading, and closing remarks table", + "observation": "The setup and all nine table values match the article; v6 is labeled persistent with static scheduling, and the author explicitly lists CLC as not implemented." + } + ], + "challenge": { + "alternative": "The 1475.93 result includes CLC or the rows mix benchmark environments.", + "check": "Trace the current table to the article's consolidated final table, setup, and explicit unimplemented-CLC remark.", + "outcome": "Every value shares the disclosed setup, and the source directly excludes CLC from v6." + } + }, + { + "claim_id": "hw-tcgen05-mma-current-c010", + "claim": "The register-pressure comparison and warp-specialization description are scoped to concrete pinned CUTLASS implementations rather than presented as universal occupancy guarantees.", + "claim_type": "mechanism", + "covers": ["u027"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/include/cute/arch/mma_sm90_gmma.hpp", + "revision": "CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "MMA_64x256x16_F32F16F16_SS CRegisters and Blackwell tutorial pipeline examples", + "observation": "The Hopper wrapper declares float[128] accumulators per thread, while pinned Blackwell examples keep D in TMEM and partition pipeline roles." + } + ], + "challenge": { + "alternative": "The 128-register statement is tile-wide or the page guarantees higher occupancy for every tcgen05 kernel.", + "check": "Inspect the wrapper typedef and the conditional wording of the current performance implication.", + "outcome": "The count is per wrapper participant and the current page says 'can' and 'kernel-dependent,' avoiding a guarantee." + } + } + ], + "scope": "All 38 units in the freshly rescanned current page were checked against CUDA 13.0.2 PTX ISA 9.0, pinned CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5, and the tutorial's source-reported environment and pinned code.", + "residual_risk": "No NVIDIA GPU or CUDA assembler is installed locally, so runtime performance and assembly acceptance were not rerun. The page confines code to normative grammar and labels benchmark values as source-reported." +} diff --git a/verification/reviewed/current/hw-tma.json b/verification/reviewed/current/hw-tma.json new file mode 100644 index 000000000..d7a4fba77 --- /dev/null +++ b/verification/reviewed/current/hw-tma.json @@ -0,0 +1,127 @@ +{ + "path": "wiki/hardware/tma.md", + "id": "hw-tma", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "hw-tma-current-c001", + "claim": "TMA on sm_90 and later permits one thread to issue non-blocking rank-1 through rank-5 tensor copies; global-to-shared loads can target CTA/cluster shared memory and multicast with mbarrier byte completion, shared-to-global stores use bulk groups, copied types are not generally converted, and OOB loads fill rather than clamp.", + "claim_type": "semantics", + "covers": ["u001", "u002", "u003", "u004", "u005"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"syntax, description, completion table, history, and target notes","observation":"The instruction contract supplies the rank set, directions, non-blocking behavior, multicast mode, direction-specific completion, datatype-copy semantics, and sm_90 floor."}, + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#asynchronous-data-copies-using-the-tensor-memory-accelerator-tma","revision":"CUDA 13.0.2","locator":"TMA overview and OOB behavior","observation":"The programming guide demonstrates one-thread issue and documents filling of out-of-bounds tensor-load elements rather than edge clamping."} + ], + "challenge": {"alternative":"All TMA directions use a barrier arrival and the descriptor requests source-to-destination numeric conversion.","check":"Compare load/store completion rows and inspect whether the tensor map contains distinct source and destination datatypes.","outcome":"Loads use complete_tx, stores use bulk_group, and one datatype describes the tensor representation; no general conversion pair exists."} + }, + { + "claim_id": "hw-tma-current-c002", + "claim": "The tensor-map field list, basic CUDA Driver API 13.0.97 tiled-encoder constraints, required CUresult handling, and Blackwell device-side encoding/proxy-publication qualification are accurate.", + "claim_type": "semantics", + "covers": ["u006", "u007", "u008", "u009", "u010", "u011", "u012", "u013", "u014", "u015", "u016", "u017", "u018", "u019", "u020", "u021"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html","revision":"CUDA Driver API 13.0.97","locator":"cuTensorMapEncodeTiled parameters, requirements, and returns","observation":"The API lists all summarized fields and the 64B tensor-map output alignment, basic 16B base/stride and inner-box rules, box dimensions 1..256, element strides 1..8, swizzle-span bound, type-specific exceptions, and CUresult failure contract."}, + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#encoding-a-tensor-map-on-device","revision":"CUDA 13.0.2","locator":"device-side encoding/modification and tensormap proxy fence","observation":"The Blackwell path and fence.proxy.tensormap::generic publication sequence match the page's qualification."} + ], + "challenge": {"alternative":"A tensor map is always host-created and 128B aligned, with unconstrained box dimensions.","check":"Inspect the Driver API object alignment/box bounds and the separate Blackwell device-encoding section.","outcome":"The host object alignment is 64B, box entries are bounded, and device-side construction/modification is documented with proxy ordering."} + }, + { + "claim_id": "hw-tma-current-c003", + "claim": "The representative 2D global-to-CTA-shared PTX is exact and the load lifecycle correctly separates initialization, one software arrive/expected-byte update, TMA complete_tx bytes, per-stage phase tracking, and an acquire wait with conjunctive phase completion.", + "claim_type": "code", + "covers": ["u022", "u023", "u024", "u025", "u026", "u027", "u028", "u029", "u030"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"global-to-shared grammar and mbarrier::complete_tx::bytes semantics","observation":"The shown instruction qualifier order/operands are accepted, the default load mode is tile, and completion decrements tx-count by copied bytes."}, + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#phase-completion-of-the-mbarrier-object","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"arrive.expect_tx, phase completion, and wait ordering","observation":"The page's separate arrival/expected-byte/complete-byte accounting and requirement for both counters to reach zero match the object lifecycle."} + ], + "challenge": {"alternative":"Each TMA completion supplies a software arrival, or zero tx-count alone releases the consumer.","check":"Trace complete-tx and arrive-on to their separate object fields and inspect the phase-completion conjunction.","outcome":"TMA changes only tx-count, and completion requires both tx-count and pending arrivals to be zero."} + }, + { + "claim_id": "hw-tma-current-c004", + "claim": "The shared-to-global store sequence uses the required bulk_group qualifier, commit_group, and wait_group 0, and correctly distinguishes grouping from completion and source-buffer reuse safety.", + "claim_type": "code", + "covers": ["u031", "u032", "u033"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"shared::cta-to-global form plus cp.async.bulk commit_group/wait_group sections","observation":"The exact instruction operands and completion qualifier match the grammar; commit creates a per-thread group and wait_group 0 waits for all prior committed groups and makes writes visible to the issuer."}], + "challenge": {"alternative":"commit_group is a blocking store completion operation, making wait_group redundant.","check":"Compare the definitions of commit_group and wait_group.","outcome":"Commit only batches uncommitted operations; the separately defined wait supplies completion."} + }, + { + "claim_id": "hw-tma-current-c005", + "claim": "The exact cluster-multicast form, same-offset destination and barrier signaling, cluster lifetime/initialization/election/mask/wait invariants, GEMM A/B index-reuse example, and non-guaranteed traffic caveat are correct.", + "claim_type": "code", + "covers": ["u034", "u035", "u036", "u037", "u038", "u039", "u040", "u041", "u042"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"multicast::cluster ctaMask, same-offset data, and mbarrier signal semantics","observation":"The instruction and same-offset semantics match exactly; cluster-shared lifetime makes initialization and liveness necessary, while one intended issue requires cluster-level rather than per-CTA election. The ISA gives no exact DRAM-saving factor."}], + "challenge": {"alternative":"threadIdx.x==0 elects one issuer for the whole cluster and a one-to-two multicast proves exactly half DRAM traffic.","check":"Count threadIdx.x==0 threads across CTAs and distinguish functional destination count from measured cache/DRAM transactions.","outcome":"There is one such thread per CTA, and the functional ISA does not specify baseline cache reuse or an exact bandwidth ratio."} + }, + { + "claim_id": "hw-tma-current-c006", + "claim": "TMA and tcgen05 expose multiple swizzle layouts; correctness requires the TMA map, base/leading dimensions, and consuming shared-memory descriptor to match, while bank-conflict benefit remains access-pattern-specific rather than universally requiring 128B.", + "claim_type": "semantics", + "covers": ["u043", "u044", "u045"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html","revision":"CUDA Driver API 13.0.97","locator":"CUtensorMapSwizzle enum and swizzle constraints","observation":"The tensor map supports none, 32B, 64B, 128B, and additional variants with inner-span/alignment/type rules."}, + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-layout","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"tcgen05 descriptor swizzling modes","observation":"tcgen05 descriptors likewise encode multiple modes and interpret addresses/layout accordingly; legal modes vary by instruction configuration."} + ], + "challenge": {"alternative":"128B is the one canonical tcgen05 layout and automatically makes any consumer conflict-free.","check":"Enumerate legal descriptor modes and apply a mismatched/alternate consumer access pattern.","outcome":"Non-128B modes are normative, and bank mapping still depends on consumer addresses; only a matched pattern can obtain the intended conflict behavior."} + }, + { + "claim_id": "hw-tma-current-c007", + "claim": "The Blackwell GEMM data path and per-stage full/empty ownership transitions correctly require load completion before MMA reads and asynchronous-MMA read completion before producer overwrite, with separate per-stage phases and no substitution of CTA sync or fence for completion.", + "claim_type": "semantics", + "covers": ["u046", "u047", "u048", "u049", "u050", "u051"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-dRC5i4/cutlass/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_6.py","revision":"CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"TMA atoms, PipelineTmaUmma full/empty producer-consumer loops, UMMA, TMEM retrieve, and epilogue","observation":"The complete implementation follows the stated data path and explicitly acquires/releases per-stage pipeline state around asynchronous TMA and UMMA lifetimes."}], + "challenge": {"alternative":"tcgen05 consumes shared operands at issue, so only a full barrier is needed and producers may overwrite immediately afterward.","check":"Trace the official pipeline's consumer release relative to asynchronous MMA issue/completion.","outcome":"Stage release is delayed through the consumer lifecycle; the implementation retains separate producer and consumer state, discriminating asynchronous operand lifetime."} + }, + { + "claim_id": "hw-tma-current-c008", + "claim": "The tuning guidance makes all layout, multicast, issue, and stage choices conditional on access pattern/resources, states the monotonic storage cost of distinct stages, rejects a universal 3-5 optimum, and specifies enough controls for a scoped profile.", + "claim_type": "performance", + "covers": ["u052"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"derive storage = fixed_bytes + stages * per_stage_bytes and inspect that no numeric speedup is asserted","environment":"symbolic resource accounting over the documented distinct-stage design","reference":"Each stage owns separate positive-size operand buffers and barrier state; performance depends on target execution/resource conditions.","result":"Adding a stage increases storage by per_stage_bytes; the page supplies no fixed optimum or outcome and requires controlled target measurement.","negative_control":"A single reused non-overlapped buffer does not satisfy the distinct-stage premise and therefore does not show the same linear allocation."}], + "challenge": {"alternative":"Three to five stages is an architecture constant that can be selected without profiling.","check":"Search the normative TMA contract for a stage-count field and compare resource use across different tile sizes.","outcome":"TMA exposes no such constant; tile bytes and occupancy change the feasible and useful count."} + }, + { + "claim_id": "hw-tma-current-c009", + "claim": "The four external references are pinned to CUDA 13.0.2/Driver API 13.0.97/PTX ISA 9.0 and an immutable CUTLASS 4.5.0 TMA tutorial.", + "claim_type": "other", + "covers": ["u053", "u054", "u055", "u056"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#asynchronous-data-copies-using-the-tensor-memory-accelerator-tma","revision":"CUDA 13.0.2","locator":"TMA section","observation":"The archive and anchor match the label."}, + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html","revision":"CUDA Driver API 13.0.97","locator":"tensor-map management","observation":"The archive resolves to the encoder constraints."}, + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"tensor copy section","observation":"The archive and anchor match the label."}, + {"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_tma/tma_v0.py","revision":"CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"complete TMA tutorial","observation":"The immutable blob exists at the pinned release commit."} + ], + "challenge": {"alternative":"One source follows rolling docs or a mutable Git branch.","check":"Inspect CUDA archive segments and the Git blob identifier.","outcome":"All CUDA links include 13.0.2 and CUTLASS names an exact commit."} + }, + { + "claim_id": "hw-tma-current-c010", + "claim": "The internal references resolve to the verified mbarrier and tcgen05 MMA pages.", + "claim_type": "other", + "covers": ["u057", "u058"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"test -f wiki/hardware/mbarrier.md && test -f wiki/hardware/tcgen05-mma.md","environment":"current KernelWiki worktree after TMA remediation","reference":"Resolve both relative paths and inspect their frontmatter IDs.","result":"Both files exist with IDs hw-mbarrier and hw-tcgen05-mma and cover the linked contracts.","negative_control":"wiki/hardware/not-tcgen05.md does not exist, so the path check rejects a broken target."}], + "challenge": {"alternative":"One path is broken or links to an unrelated generation.","check":"Resolve both paths and compare page IDs/titles.","outcome":"Both point to the intended verified hardware pages."} + } + ], + "commands": ["inspect CUTLASS 4.5.0 TMA/UMMA pipeline", "derive per-stage storage scaling", "test -f wiki/hardware/mbarrier.md && test -f wiki/hardware/tcgen05-mma.md", "structural validation after remediation"], + "residual_risk": "No CUDA toolkit or NVIDIA GPU is installed locally, so the three PTX snippets were not assembled and no TMA timing/traffic counters were collected. Syntax and ordering are version-pinned to PTX ISA 9.0; the page makes no numeric performance claim.", + "scope": "All 58 current units for body hash 1eb48a33cd77ce277ac4783d357f5784e487533159bbe155fc942f404f3655f1 were checked against CUDA 13.0.2, CUDA Driver API 13.0.97, PTX ISA 9.0, CUTLASS 4.5.0, symbolic resource accounting, and local link resolution." +} diff --git a/verification/reviewed/current/hw-tmem.json b/verification/reviewed/current/hw-tmem.json new file mode 100644 index 000000000..af2f1e9ad --- /dev/null +++ b/verification/reviewed/current/hw-tmem.json @@ -0,0 +1,253 @@ +{ + "path": "wiki/hardware/tmem.md", + "id": "hw-tmem", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "hw-tmem-current-c001", + "claim": "Blackwell TMEM is a 256 KiB per-SM, 128-lane by 512-column array of 32-bit cells used for tcgen05 D and some A operands; replacing a register-D WGMMA fragment changes rather than eliminates register pressure.", + "claim_type": "semantics", + "covers": ["u001", "u002"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "Tensor Memory organization and tcgen05 operand overview", + "observation": "The ISA specifies the capacity, lane-column geometry, and TMEM operand roles." + }, + { + "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cute/arch/mma_sm90_gmma.hpp", + "revision": "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "MMA_64x256x16_F32F16F16_SS::CRegisters", + "observation": "The representative SM90 wrapper declares 128 float accumulators per participating thread, while the corrected text expressly leaves other GPR uses in scope." + } + ], + "challenge": { + "alternative": "The 256 KiB figure is separate from the lane-column array, or moving D to TMEM makes the whole kernel register-free.", + "check": "Recompute the lane-column capacity and inspect both the exact WGMMA CRegisters typedef and the corrected residual-register wording.", + "outcome": "128 x 512 x 4 bytes equals 256 KiB, and the page limits the relief to the D fragment while preserving other GPR uses." + } + }, + { + "claim_id": "hw-tmem-current-c002", + "claim": "A b32 taddr encodes lane and column, while collective tcgen05 load/store access maps four warps to successive 32-lane chunks using shape-specific register layouts rather than per-thread row ownership.", + "claim_type": "semantics", + "covers": ["u003", "u004", "u005", "u006", "u007", "u008", "u009", "u010", "u011"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory-addressing", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "TMEM address format; tcgen05.ld/st access restrictions", + "observation": "The address field table assigns lane to bits 31:16 and column to 15:0; the access tables partition lanes into four warp chunks and define layouts by instruction shape." + } + ], + "challenge": { + "alternative": "The upper address half is a CUDA thread ID, making the corrected warning against row ownership unnecessary.", + "check": "Compare the address definition with the collective execution and access-restriction tables.", + "outcome": "The ISA calls it a TMEM lane and separately requires warp-collective execution with shape-dependent mappings." + } + }, + { + "claim_id": "hw-tmem-current-c003", + "claim": "TMEM allocation returns a b32 taddr through shared memory, uses one allocator warp for group 1 or one warp per peer CTA for group 2, requires one CTA-group value throughout the kernel, and every allocation must be collectively deallocated before exit.", + "claim_type": "semantics", + "covers": ["u012", "u013", "u014", "u015", "u016", "u017", "u018", "u019"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "alloc/dealloc syntax, issue granularity, constraints, and lifecycle", + "observation": "The section specifies the shared b32 result, group-specific warp collectives, same-group invariant, 32-512 power-of-two nCols, blocking availability behavior, and before-exit deallocation rule." + } + ], + "challenge": { + "alternative": "Allocation is a lane-0 instruction and CTA synchronization supplies the missing collective behavior.", + "check": "Inspect the mandatory .sync/.aligned and same-operand clauses for cta_group::1 and the paired-CTA issue table.", + "outcome": "The operation itself must execute uniformly across the allocator warp or warps; a later CTA barrier cannot repair divergent issue." + } + }, + { + "claim_id": "hw-tmem-current-c004", + "claim": "The 512-column pool admits the stated 128-, 256-, and 512-column whole-allocation budgets, while a 192-column logical region requires a legal 256-column reservation or suballocation.", + "claim_type": "semantics", + "covers": ["u020", "u021", "u022", "u023", "u024"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "nCols power-of-two constraint and 512-column maximum", + "observation": "Legal sizes are powers of two in [32,512]. Division of the single 512-column pool yields the displayed stage counts, and 192 itself is excluded." + } + ], + "challenge": { + "alternative": "Multiple-of-32 sizes such as 192 are legal even if the high-level CUTLASS allocator rounds them.", + "check": "Read the PTX operand constraint rather than inferring it from a library allocator.", + "outcome": "The ISA itself requires nCols to be a power of two, discriminating the 192-column case." + } + }, + { + "claim_id": "hw-tmem-current-c005", + "claim": "tcgen05 MMA/copy completion uses commit plus mbarrier, load/store completion uses the corresponding tcgen05.wait form, and before/after fences provide cross-thread ordering rather than completion; source lifetimes and CTA synchronization must respect those distinctions.", + "claim_type": "semantics", + "covers": ["u025", "u026", "u027", "u028", "u029", "u030"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-wait", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "wait::ld/wait::st semantics and asynchronous-operation table", + "observation": "The wait variants block on prior ld/st operations from the executing thread." + }, + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "commit completion and fence ordering sections", + "observation": "Commit attaches prior MMA/cp/shift completion to an mbarrier; fences compose around execution-ordering operations and provide no completion promise." + } + ], + "challenge": { + "alternative": "A CTA barrier or before_thread_sync fence implicitly waits for every prior asynchronous tcgen05 operation.", + "check": "Compare the normative effect of fence/barrier examples with the separate commit and wait completion mechanisms.", + "outcome": "Only commit/mbarrier and wait::* state completion; fences are explicitly ordering and code-motion constraints." + } + }, + { + "claim_id": "hw-tmem-current-c006", + "claim": "The page lists valid tcgen05.ld/st shapes, rejects nonexistent .32x1b scalar forms, and gives the exact valid group-1 128x256b shared-memory-to-TMEM tcgen05.cp instruction with correct operands and asynchronous caveat.", + "claim_type": "code", + "covers": ["u031", "u032", "u033", "u034"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "tcgen05.ld/st shape grammar", + "observation": "The five listed shapes appear in the grammar and .32x1b does not." + }, + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-cp", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "tcgen05.cp syntax, operand descriptions, and Example 1", + "observation": "The exact displayed instruction appears in the vendor example; taddr is the destination, sdesc is a b64 SMEM descriptor, and cp is asynchronous." + } + ], + "challenge": { + "alternative": "The copied shape is 128x256 elements rather than bits, or sdesc denotes another TMEM address.", + "check": "Inspect the shape suffix and operand-type description adjacent to the exact example.", + "outcome": "The suffix is explicitly 128x256b and the source is explicitly a shared-memory matrix descriptor." + } + }, + { + "claim_id": "hw-tmem-current-c007", + "claim": "A correct multi-stage TMEM accumulator pipeline must fit the column budget and use explicit producer/consumer barriers and completion before drain or recycle; alternating two addresses alone does not create overlap.", + "claim_type": "mechanism", + "covers": ["u035", "u036", "u037", "u038", "u039", "u040", "u041"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_6.py", + "revision": "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "TMEM allocation; PipelineUmmaAsync stages; producer/consumer acquire, commit, wait, and release", + "observation": "The pinned complete implementation allocates the shared TMEM pool and coordinates distinct accumulator producers/consumers with explicit stage barriers and completion state." + } + ], + "challenge": { + "alternative": "The hardware automatically detects alternating accumulator addresses and overlaps serial code without pipeline participants.", + "check": "Inspect the vendor tutorial for the control and synchronization state that accompanies multiple TMEM stages.", + "outcome": "The implementation explicitly acquires, commits, waits, and releases stages; address alternation is not sufficient." + } + }, + { + "claim_id": "hw-tmem-current-c008", + "claim": "CUTLASS 4.5.0's Python DSL uses TmemAllocator allocation/wait/retrieve/free operations, typed pointers, cute.make_tensor, tcgen05 copy atoms, and pipeline completion in the sequence summarized by the page.", + "claim_type": "code", + "covers": ["u042", "u043", "u044", "u045", "u046", "u047", "u048", "u049"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py", + "revision": "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "TmemAllocator construction and allocate/wait/retrieve/make_tensor/load/free flow", + "observation": "Every named operation and its ordering appears in the official tutorial." + }, + { + "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/python/CuTeDSL/cutlass/utils/tmem_allocator.py", + "revision": "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "TmemAllocator methods allocate, wait_for_alloc, retrieve_ptr, and free", + "observation": "The library implementation defines the exact high-level allocation API and collective allocator-warp behavior summarized." + } + ], + "challenge": { + "alternative": "The tutorial bypasses TmemAllocator and obtains an implicitly allocated TMEM tensor from the MMA atom.", + "check": "Trace the accumulator pointer from shared allocation-result storage through retrieve_ptr and make_tensor.", + "outcome": "The pointer is explicitly allocated, waited for, retrieved, rebound, drained, and freed." + } + }, + { + "claim_id": "hw-tmem-current-c009", + "claim": "The pinned links identify the exact CUTLASS 4.5.0 tutorial and allocator implementation, while the rejected Layout(memory_space=MemorySpace.TMEM), tcgen05_mma, and similar convenience calls are absent from that version.", + "claim_type": "compatibility", + "covers": ["u050"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass", + "revision": "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "repository-wide exact-token search; linked tutorial and allocator paths", + "observation": "The two linked files exist and implement the described API. Exact searches for MemorySpace.TMEM, tcgen05_mma, tcgen05_fence, and tmem_load as displayed API names return no definitions or uses." + } + ], + "challenge": { + "alternative": "The convenience calls are generated symbols or re-exported from a path not imported by the tutorial.", + "check": "Search the entire pinned repository, not only the tutorial directory, and inspect public arch and utility exports.", + "outcome": "The tokens remain absent repository-wide while the documented allocator and copy-atom interfaces are present." + } + }, + { + "claim_id": "hw-tmem-current-c010", + "claim": "The five primary-reference links resolve to the pinned PTX sections and pinned CUTLASS 4.5.0 SM100 tutorial tree that support the page.", + "claim_type": "other", + "covers": ["u051", "u052", "u053", "u054", "u055"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "Tensor Memory and linked alloc/ld/commit anchors", + "observation": "The archive resolves the listed normative sections at the pinned CUDA version." + }, + { + "url": "https://github.com/NVIDIA/cutlass/tree/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm", + "revision": "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "SM100 tutorial directory", + "observation": "The commit-pinned directory contains the referenced progressive SM100 Python GEMM tutorials." + } + ], + "challenge": { + "alternative": "The URLs silently track latest documentation or a moving CUTLASS branch.", + "check": "Inspect the CUDA archive segment and immutable Git commit embedded in each URL.", + "outcome": "Both source families are version-pinned in the links themselves." + } + } + ], + "commands": [ + "rg -n against the full pinned CUTLASS 4.5.0 worktree for exact API tokens", + "structural validation after remediation" + ], + "residual_risk": "No CUDA toolkit or NVIDIA GPU is installed locally, so the exact PTX example was not assembled and the pipeline was not benchmarked. Its syntax and semantics are matched to PTX ISA 9.0, and the high-level API is matched to immutable CUTLASS 4.5.0 source.", + "scope": "All 55 current scan units for body hash f6c0135c31b4600deecf3ee1dd972746a40bb9b3c0968b96e5ceb231a35ce816 were verified against CUDA 13.0.2 PTX ISA 9.0 and CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5." +} diff --git a/verification/reviewed/current/kernel-deepgemm.json b/verification/reviewed/current/kernel-deepgemm.json new file mode 100644 index 000000000..1f49926f5 --- /dev/null +++ b/verification/reviewed/current/kernel-deepgemm.json @@ -0,0 +1,110 @@ +{ + "path": "wiki/kernels/deepgemm.md", + "id": "kernel-deepgemm", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "kernel-deepgemm-current-c001", + "claim": "The page is scoped to DeepGEMM commit 891d57b, accurately identifies the project and its SM90/SM100 requirements, and distinguishes two representative local FP8 1D1D files from the full multi-kernel library.", + "claim_type": "compatibility", + "covers": ["u001", "u002"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-deepgemm-pinned/README.md","revision":"DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"overview, requirements, and interfaces","observation":"The README identifies the library, SM90/SM100 requirements, runtime-compiled design, and multiple dense/grouped/attention/MoE interfaces."},{"path":"artifacts/kernels/deepgemm/full/PROVENANCE.yaml","revision":"local byte-verified manifest","locator":"upstream_sha and file records","observation":"The two representative files are pinned to the same commit and carry matching SHA-256 values."}], + "challenge": {"alternative":"The commit link is only decorative and the local files come from a different revision or represent the whole repository.","check":"Compare the frontmatter, README tree, provenance upstream SHA, and exact file hashes.","outcome":"The revision matches across the page and manifest, hashes match, and the pinned tree visibly contains many other kernel/API files."} + }, + { + "claim_id": "kernel-deepgemm-current-c002", + "claim": "The DeepSeek-V3 1x128 activation and 128x128 weight scaling geometry, scoped outlier motivation, architecture-specific FP32 versus packed-UE8M0 representations, four-value packing, and pinned K-granularity statements are accurate.", + "claim_type": "semantics", + "covers": ["u003", "u004", "u005", "u006", "u007"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/abs/2412.19437v2","revision":"DeepSeek-V3 Technical Report, arXiv v2","locator":"section 3.3.2 and Appendix B.2","observation":"The paper specifies the two granularities and says smaller groups better accommodate outliers; it does not promise elimination of quantization error."},{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md","revision":"DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"interface notices","observation":"SM90 uses FP32 factors and SM100 uses packed UE8M0 with four values per torch.int."},{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh","revision":"DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"BLOCK_K and kGranKA/kGranKB static assertions","observation":"The SM100 template fixes BLOCK_K to 128 and permits scale K granularities 32 or 128; the corresponding SM90 file fixes BLOCK_K to 128."}], + "challenge": {"alternative":"Both operands use 128x128 blocks and every architecture consumes FP32 scales.","check":"Compare the paper's separate operand rules and the pinned README/kernel types.","outcome":"Activation grouping is per row, while the SM100 representation and accepted granularities differ explicitly from SM90."} + }, + { + "claim_id": "kernel-deepgemm-current-c003", + "claim": "The H800 retained-bit description, Nc=128 K-element/four-WGMMA interval, four-step walkthrough, and verbatim six-line excerpt of the pinned SM90 accum/final_accum promotion path match the paper and source.", + "claim_type": "code", + "covers": ["u008", "u009", "u010", "u011", "u012", "u013", "u014", "u041", "u042"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/abs/2412.19437v2","revision":"DeepSeek-V3 Technical Report, arXiv v2","locator":"sections 3.3.2 and 3.5.2","observation":"The report states about 14 retained bits, promotion every 128 inner-dimension elements, equivalence to four WGMMAs, and FP32 CUDA-core accumulation."},{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh","revision":"DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"math warp-group loop from accum declarations through Promote with scales and store","observation":"The code declares float accum/final_accum, loads FP32 A/B factors, issues and waits for WGMMA for one BLOCK_K=128 stage, scales into final_accum, and stages final values for store."}], + "challenge": {"alternative":"Nc=128 describes 128 output columns and the real accumulator is half precision as in the deleted sketch.","check":"Trace the k_block loop and declared C++ types and compare Nc's definition in the paper.","outcome":"The interval advances K, WGMMA output registers are declared float, and promotion targets a separate float array."} + }, + { + "claim_id": "kernel-deepgemm-current-c004", + "claim": "The SM100 walkthrough accurately describes the float/UE8M0 block-scaled instruction descriptor, TMA-to-shared and UTCCP-to-TMEM scale path, elected UMMA issue operands, TMEM accumulator lifecycle, absence of SM90 final_accum promotion, and precision qualification.", + "claim_type": "mechanism", + "covers": ["u015", "u016", "u017", "u018", "u019", "u020"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh","revision":"DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"TMA scale loads, make_instr_desc_block_scaled, UTCCP copies, mma_t::fma, tmem barriers, and epilogue","observation":"Each stated data-movement, descriptor, issue, and lifecycle element appears in the pinned source. The template selects accumulator and scale types rather than asserting a universal tcgen05 precision mode."}], + "challenge": {"alternative":"Packed scales remain in general registers and are immediate MMA operands, while TMEM holds only output.","check":"Follow smem_sfa/sfb through the UTCCP copy destinations and the final fma arguments.","outcome":"The copies target dedicated TMEM scale columns and fma receives their TMEM addresses separately from the TMEM accumulator column."} + }, + { + "claim_id": "kernel-deepgemm-current-c005", + "claim": "The grouped-interface table and surrounding text correctly distinguish M-grouped contiguous, M-grouped masked, and K-grouped contiguous shapes, metadata, purposes, alignment, and pinned SM90-NT/SM100-TN K-grouped variants.", + "claim_type": "semantics", + "covers": ["u021", "u022", "u023", "u024", "u025", "u031"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md","revision":"DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"Grouped GEMMs contiguous and masked sections","observation":"The README gives the M-grouped fixed-axis/use-case rules, segment alignment, masked decode role, and separate K-axis weight-backward API."},{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp","revision":"DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"m_grouped_fp8_fp4_gemm_* and k_grouped_fp8_gemm_* shape/layout checks","observation":"The API checks establish each tensor shape and metadata type; contiguous supports M-length or group-count metadata forms, masked_m is an int valid-length vector, and K-grouped dispatches TN only to SM100 and NT only to SM90."}], + "challenge": {"alternative":"Masked mode uses a Boolean element mask and K-grouped is merely a third M-grouped storage layout.","check":"Inspect masked_m dtype/shape and the K-grouped shape equations.","outcome":"masked_m has one int per group, while K-grouped sums per-group K lengths and fixes the output's M/N."} + }, + { + "claim_id": "kernel-deepgemm-current-c006", + "claim": "The pinned JIT defaults to NVCC, optionally selects NVRTC through DG_JIT_USE_NVRTC=1 with a performance caveat, keys the cache by kernel/compiler/flags/code, compiles and publishes misses, reuses hits, and makes no optimality guarantee.", + "claim_type": "mechanism", + "covers": ["u026", "u027", "u032"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md","revision":"DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"DG_JIT_USE_NVRTC environment variable","observation":"NVRTC is optional, defaults off, and may perform worse in some cases."},{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/jit/compiler.hpp","revision":"DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"Compiler::build and compiler LazyInit","observation":"The code constructs the stated signature, returns cache hits, compiles/publishes misses atomically, and selects NVCC unless the environment opt-in is nonzero."}], + "challenge": {"alternative":"NVRTC is the mandatory/default JIT and every invocation recompiles the specialization.","check":"Evaluate the compiler selection default and the early cache-hit return.","outcome":"The default expression is zero, selecting NVCC, and a valid cached runtime returns before compilation."} + }, + { + "claim_id": "kernel-deepgemm-current-c007", + "claim": "The pinned FP8 interface supports NT only on SM90 and NT/TN/NN/TT on SM100, with public wrappers and SM100 operand-major parameters implementing the additional layouts.", + "claim_type": "compatibility", + "covers": ["u028"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md","revision":"DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"Interfaces notices and normal dense GEMMs","observation":"The README states the exact architecture/layout support and naming convention."},{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp","revision":"DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"fp8_fp4_gemm_nt/nn/tn/tt and major checks","observation":"The wrappers transpose relevant views and pass major modes; the SM90 path asserts K-major inputs."}], + "challenge": {"alternative":"The four SM100 names all demand physically NT inputs and do not alter descriptors.","check":"Trace tensor-view transformations and kMajorA/kMajorB into the SM100 template.","outcome":"Different wrappers transform A and/or B and the selected major values reach descriptor construction."} + }, + { + "claim_id": "kernel-deepgemm-current-c008", + "claim": "The only retained numeric performance statement is the pinned project's source-reported up-to 1550 TFLOPS on H800, explicitly qualified as lacking the former 4096-cubed shape, 90% utilization, and a complete reproducibility envelope.", + "claim_type": "performance", + "covers": ["u029"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md","revision":"DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"2025-04-18 news item","observation":"The README says up to 1550 TFLOPS on H800 and supplies none of the deleted shape/utilization conditions."},{"url":"https://github.com/deepseek-ai/DeepGEMM/pull/86","revision":"merged 2025-04-14","locator":"performance report","observation":"A nearby disclosed peak result uses M=4096, N=7168, K=16384, helping discriminate it from the page's former unsupported 4096-cubed attribution."}], + "challenge": {"alternative":"The up-to result proves the former 4096-cubed and 90% row by implication.","check":"Search the pinned README and linked performance history for that condition tuple.","outcome":"The README supplies neither field, and the available shape-scoped peak evidence names a different tuple; the page now says exactly that."} + }, + { + "claim_id": "kernel-deepgemm-current-c009", + "claim": "The practical boundaries accurately separate input preparation from GEMM, state project warnings and grouped/JIT constraints, and avoid claiming a universal fine-grained-scaling cost or decision rule.", + "claim_type": "performance", + "covers": ["u030", "u033"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md","revision":"DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"interface notices","observation":"The project says transposition and FP8 casting are user-side operations and that its utility implementations may be slower than fusing them into prior kernels."},{"url":"https://arxiv.org/abs/2412.19437v2","revision":"DeepSeek-V3 Technical Report, arXiv v2","locator":"section 3.3.2","observation":"The paper describes the precision motivation and says the scale multiplication has minimal extra computational cost in its Hopper promotion strategy; it does not establish a universal cost ordering."}], + "challenge": {"alternative":"Every extra scale necessarily makes the end-to-end kernel slower than per-tensor scaling, so target/workload qualifications are evasive.","check":"Separate arithmetic count from native instruction support, scale preparation/fusion, and the measured scope in the primary sources.","outcome":"The sources establish architecture-dependent mechanisms and no controlled universal comparison; conditional wording is the strongest supported conclusion."} + }, + { + "claim_id": "kernel-deepgemm-current-c010", + "claim": "The four pinned external links, local provenance/hash statements, upstream-versus-derived labels, and page query command are correct and resolvable.", + "claim_type": "code", + "covers": ["u034", "u035", "u036", "u037", "u038", "u039", "u040"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"git -C /tmp/kernelwiki-deepgemm-pinned rev-parse HEAD; sha256sum artifacts/kernels/deepgemm/full/sm90_fp8_gemm_1d1d.cuh artifacts/kernels/deepgemm/full/sm100_fp8_gemm_1d1d.cuh; conda run -n base python scripts/get_page.py kernel-deepgemm --include-code","environment":"KernelWiki worktree; exact upstream clone; conda base Python 3.13.14","reference":"page links and artifacts/kernels/deepgemm/{full,variants}/PROVENANCE.yaml","result":"The clone is at 891d57b4db1071624b5c8fa0d1e51cb317fa709f, both hashes equal their manifest entries, modes are verbatim for full and derived for variants, and the query succeeds with attached code.","negative_control":"The non-existent id kernel-not-deepgemm is rejected, discriminating actual index resolution from unconditional output."}], + "challenge": {"alternative":"The links are rolling branches, the local files are untracked adaptations, or the query omits the assets.","check":"Inspect each URL for a full commit/version token, verify bytes/modes, and execute the lookup.","outcome":"All external references are immutable, local bytes and provenance agree, and the query returns the page plus its code bundle."} + } + ], + "commands": ["inspect DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f", "sha256sum pinned full assets", "conda run -n base python scripts/get_page.py kernel-deepgemm --include-code", "inspect DeepSeek-V3 Technical Report v2 and DeepGEMM performance history"], + "residual_risk": "No NVIDIA GPU or CUDA toolkit is installed locally, so the kernels were not compiled or executed and the source-reported H800 peak was not reproduced. The implementation statements are source-level conclusions tied to commit 891d57b; no performance portability beyond the explicit scope is asserted.", + "scope": "All 42 current units for body hash 5e2f98b996033253da1252cec281af40f1f858fa39675a8296b17564863d76fe were checked against the pinned DeepGEMM tree and local byte-verified assets, DeepSeek-V3 Technical Report v2, upstream performance history, and repository query behavior." +} diff --git a/verification/reviewed/current/kernel-flash-attention-4.json b/verification/reviewed/current/kernel-flash-attention-4.json new file mode 100644 index 000000000..56bf7762e --- /dev/null +++ b/verification/reviewed/current/kernel-flash-attention-4.json @@ -0,0 +1,100 @@ +{ + "path": "wiki/kernels/flash-attention-4.md", + "id": "kernel-flash-attention-4", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "kernel-flash-attention-4-current-c001", + "claim": "The page accurately separates the SM100/B200 scope of FA4 paper v1 from Dao-AILab commit a369df7 and reports the paper's CuTe DSL and single-kernel FA3 compilation comparison with exact times and scope.", + "claim_type": "performance", + "covers": ["u001", "u002", "u003", "u004"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/html/2603.05451v1","revision":"FlashAttention-4 paper v1, 2026-03-05","locator":"abstract, introduction, and section 5.1","observation":"The paper targets Blackwell/B200, identifies its CuTe DSL implementation, and reports FA3 versus FA4 compile times of 55 versus 2.5 seconds forward and 45 versus 1.4 seconds backward."},{"url":"https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute","revision":"Dao-AILab commit a369df707e1980fb328abcc1733e3457ec10155f","locator":"pinned package tree","observation":"The immutable URL resolves to the implementation snapshot named by the page."}], + "challenge": {"alternative":"The 20-30x number measures runtime or end-to-end build time, and the repository link follows a mutable branch.","check":"Inspect the paper's compilation units/labels and the Git URL's revision component.","outcome":"The values are seconds for individual kernel compilation, while the implementation URL embeds a full immutable commit."} + }, + { + "claim_id": "kernel-flash-attention-4-current-c002", + "claim": "FA4's forward ping-pong schedule alternates two 128-row output tiles, uses one MMA warp, two four-warp softmax groups and a correction warpgroup, stores score/output accumulators in separate TMEM regions, and is distinct from ordinary K/V double buffering.", + "claim_type": "mechanism", + "covers": ["u005", "u006"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/html/2603.05451v1","revision":"FA4 paper v1","locator":"section 3.1, Ping-pong Scheduling and forward pipeline figure","observation":"The paper describes two M=128 output tiles, their two softmax warpgroups, a correction warpgroup, one MMA warp, and distinct TMEM regions with alternating pipeline work."}], + "challenge": {"alternative":"The two objects are only K/V load buffers for one query tile, so the page overstates output-level concurrency.","check":"Follow the output-tile indices and softmax-warpgroup assignments in the paper's pipeline description.","outcome":"Each softmax group is assigned to a distinct output tile and the alternating state includes separate output accumulators, rejecting the K/V-only interpretation."} + }, + { + "claim_id": "kernel-flash-attention-4-current-c003", + "claim": "FA4 partially emulates about 10-25% of exponential entries; the displayed scalar Python implements the published floor-to-[0,1), degree-3 rounded-coefficient reference and is explicitly not the kernel selection, clamping, or scheduling implementation.", + "claim_type": "code", + "covers": ["u007", "u008", "u009", "u010"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/html/2603.05451v1","revision":"FA4 paper v1","locator":"section 3.2, Partial Emulation of Exponential","observation":"The source states the 10-25% selection, floor range reduction, [0,1) fraction, degree-3 polynomial, and hybrid FMA/MUFU execution. Direct comparison shows the displayed scalar performs those exact arithmetic steps and no kernel scheduling."},{"url":"https://tridao.me/blog/2026/flash4/","revision":"Tri Dao blog 2026-03-05","locator":"software exponential equations and coefficient row","observation":"The blog prints the rounded coefficients 1.0, 0.6951, 0.2276, and 0.0771 used verbatim by the scalar reference."}], + "challenge": {"alternative":"The original centered, degree-5 function is an equally valid representation of FA4, or the page now implies every entry uses the scalar path.","check":"Compare interval, polynomial degree, and entry-selection statements against the paper/blog and inspect the page qualification around the function.","outcome":"The primary sources select floor/[0,1)/degree 3 and only a fraction of entries; the current page and executable scalar match those boundaries exactly."} + }, + { + "claim_id": "kernel-flash-attention-4-current-c004", + "claim": "FA4 conditional rescaling compares running maxima, typically uses tau=8.0 base-2 units, retains the old reference maximum when skipping, tracks delayed normalization state, and renormalizes at the end; LSE-difference skipping is not equivalent.", + "claim_type": "mechanism", + "covers": ["u011", "u012"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/html/2603.05451v1","revision":"FA4 paper v1","locator":"section 3.3, Conditional Rescaling, equations 7-10","observation":"The paper supplies the one-direction max condition, typical log2(256)=8 threshold, retained-max exponent convention, auxiliary statistics, and final correction."}], + "challenge": {"alternative":"Checking absolute LSE change and then using the new maximum without a multiply is algebraically equivalent.","check":"Substitute the alternative state update into the paper's exponent and final-normalization equations.","outcome":"LSE contains the row sum and the alternative changes the exponent's reference without compensating state; it does not satisfy the published recurrence."} + }, + { + "claim_id": "kernel-flash-attention-4-current-c005", + "claim": "The backward section correctly scopes two-CTA M=256,N=128,K=128 use to five GEMMs, approximate shared-read halving to operand B, dS exchange to the dQ path, and halved global atomic reductions to the doubled dQ reduction width.", + "claim_type": "mechanism", + "covers": ["u013", "u014"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/html/2603.05451v1","revision":"FA4 paper v1","locator":"section 4, Two-CTA MMA for Backward Pass","observation":"The paper gives the five-GEMM shape, operand-B sharing scope, half-dS DSMEM exchange for dQ, doubled N reduction tile, and global-atomic effect exactly as separated on the page."}], + "challenge": {"alternative":"The factor applies to all shared-memory traffic, or paired CTAs simply split dK and dV into independent kernels.","check":"Trace each operand and the dS exchange in the backward dataflow figure/text.","outcome":"Only the B operand has the stated shared-read reuse and both CTAs participate in the coordinated GEMMs/dQ exchange, rejecting both broader alternatives."} + }, + { + "claim_id": "kernel-flash-attention-4-current-c006", + "claim": "The pinned implementation notes accurately name the SM100 forward/backward helpers, TMEM allocation and offsets, conditional TMA fallbacks, SM90/SM100/SM120 package modules, nvidia-cutlass-dsl 4.6.0.dev0 dependency, FP16/BF16 directions, and not-yet-implemented FP8 bring-up state.", + "claim_type": "code", + "covers": ["u015", "u016", "u017", "u018", "u019", "u020", "u021"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_fwd_sm100.py","revision":"Dao-AILab commit a369df707e1980fb328abcc1733e3457ec10155f","locator":"_setup_tiled_mma, TmemAllocator setup, copy-atom construction, and architecture dispatch inputs","observation":"The source uses make_trivial_tiled_mma, explicit TMEM allocation/offsets, and conditional Q/KV TMA or fallback paths as stated."},{"path":"/tmp/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_bwd_sm100.py","revision":"same commit","locator":"_setup_tiled_mma and two-CTA dS/dQ pipelines","observation":"The source constructs the five operations and coordinated two-CTA exchange/reduction flow without rank-partitioned dK/dV."},{"path":"/tmp/kernelwiki-flash-attention-pinned/flash_attn/cute/","revision":"same commit","locator":"README.md, pyproject.toml, flash_fwd.py, flash_bwd.py, architecture modules, and benchmark_flash_attention_fp8.py","observation":"The package identifies Hopper/Blackwell, includes SM90/SM100/SM120 modules, pins 4.6.0.dev0, exposes FP16/BF16 forward/backward, and says the FP8 call will raise until implemented."}], + "challenge": {"alternative":"Exact-token details are from a different branch, TMA is mandatory, or the FP8 benchmark proves supported FP8 execution.","check":"Confirm clone HEAD, inspect conditional copy construction and dependency manifest, then follow the FP8 benchmark exception path.","outcome":"HEAD equals a369df7, fallback branches are explicit, the exact dependency is pinned, and FP8 failure is expected rather than treated as success."} + }, + { + "claim_id": "kernel-flash-attention-4-current-c007", + "claim": "The performance section preserves the paper's up-to 1613-TFLOPS/s/71% result and the blog's distinct up-to 1605-TFLOPS/s/71% result with their source-specific scope, refuses the unsupported 8192-by-128 row, and makes no 29% time-breakdown inference.", + "claim_type": "performance", + "covers": ["u022", "u023"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/html/2603.05451v1","revision":"FA4 paper v1","locator":"abstract and section 5 benchmark setup/results","observation":"Paper v1 reports up to 1613 TFLOPS/s at 71% and up-to 1.3x/2.7x comparisons over a suite spanning sequence lengths 1K-32K and several head dimensions."},{"url":"https://tridao.me/blog/2026/flash4/","revision":"Tri Dao blog 2026-03-05","locator":"performance summary","observation":"The blog separately states up to 1605 TFLOPS/s at 71% and the up-to comparison factors. It does not establish the deleted full condition tuple or a complement-of-utilization time breakdown."}], + "challenge": {"alternative":"The two maxima can be merged into one exact seqlen=8192/head-dimension-128 row and 29% is necessarily non-MMA time.","check":"Search each first-party source for the complete condition tuple and an attributed time decomposition.","outcome":"Neither supplies the tuple or decomposition, while their textual peak values differ; the current page keeps them separate and explicitly unreproduced."} + }, + { + "claim_id": "kernel-flash-attention-4-current-c008", + "claim": "The practical boundaries correctly make use conditional on exact revision support and target measurement, reject universal sequence-length/head-dimension thresholds, separate compile from runtime speed, and list the missing controls needed before reproducing source-reported performance.", + "claim_type": "performance", + "covers": ["u024", "u025", "u026", "u027"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/html/2603.05451v1","revision":"FA4 paper v1","locator":"sections 5.1-5.3 benchmark methodology and results","observation":"The source separates compile timing from runtime evaluation and varies direction, causal mode, sequence length, and head dimensions under a defined B200 setup; it states no universal 1024/128 choice rule."},{"url":"https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/interface.py","revision":"Dao-AILab commit a369df707e1980fb328abcc1733e3457ec10155f","locator":"interface validation and architecture/configuration dispatch","observation":"Support depends on architecture, dtype, head dimensions, masking/features, and code revision, consistent with the page's conditional checklist."}], + "challenge": {"alternative":"Sequence length 1024 and head dimension 128 are architecture constants, and compile speed predicts runtime speed.","check":"Compare benchmark axes and separate compilation/runtime sections, then inspect configuration-dependent source dispatch.","outcome":"The paper evaluates multiple points and metrics while the source has condition-dependent paths, rejecting fixed universal thresholds and metric substitution."} + }, + { + "claim_id": "kernel-flash-attention-4-current-c009", + "claim": "The five pinned links resolve, local provenance accurately labels the full file as an adjacent CUTLASS reference and the variant as derived, all recorded hashes match, and the page query returns the declared local references while rejecting a false id.", + "claim_type": "code", + "covers": ["u028", "u029", "u030", "u031", "u032", "u033", "u034", "u035"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"git -C /tmp/kernelwiki-flash-attention-pinned rev-parse HEAD; sha256sum FA4 blog extracted files and kernel full/variant files; conda run -n base python scripts/extract_blog_code.py --check flash-attention-4; conda run -n base python scripts/get_page.py kernel-flash-attention-4 --include-code","environment":"KernelWiki worktree; exact upstream clone; conda base Python 3.13.14","reference":"page links plus blog and kernel PROVENANCE.yaml/MANIFEST.yaml files","result":"Clone HEAD is a369df707e1980fb328abcc1733e3457ec10155f; hashes are 89c085df..., 5747bb5e..., 2fa25bd8..., and 15df5285... and match manifests; extraction check and page query succeed with 2031 output lines.","negative_control":"kernel-not-flash-attention-4 exits 1, and the CUTLASS provenance repository/path visibly differs from Dao-AILab FA4."}], + "challenge": {"alternative":"The local full file is actually FA4 despite its repository label, the extracted blocks drifted from the summary, or get_page ignores artifact relationships.","check":"Compare owners/paths, verify every manifest hash, run the extraction idempotence check, and contrast a real versus false page id.","outcome":"The provenance proves the adjacent identity, all bytes agree, the extractor is idempotent, and only the real id resolves the attached bundle."} + } + ], + "commands": ["inspect FA4 paper v1 and Tri Dao's first-party blog", "inspect Dao-AILab/flash-attention commit a369df707e1980fb328abcc1733e3457ec10155f", "sample the published rounded exp2 polynomial with conda base Python", "verify artifact hashes and blog extraction idempotence", "conda run -n base python scripts/get_page.py kernel-flash-attention-4 --include-code", "conda run -n base python scripts/validate.py"], + "residual_risk": "No CUDA toolkit or NVIDIA GPU is installed, so no FA4 kernel was compiled or executed and neither B200 peak was reproduced. Source-level conclusions are tied to paper v1 and commit a369df7; blog/paper numbers remain explicitly source-reported, and the scalar polynomial test does not validate the production kernel's selective emulation, clamping, accuracy, or performance.", + "scope": "All 35 current units for body hash eb0beda6d21999eb5a9af51fbb7869f46f78dcb0a29c7d5df1fbde5a0b49b875 were checked against FA4 paper v1, Tri Dao's first-party blog, exact Dao-AILab commit a369df7, generated/local provenance and hashes, a deterministic scalar formula test, repository query behavior, and structural validation." +} diff --git a/verification/reviewed/current/kernel-flash-attention-sm100-mla-topk.json b/verification/reviewed/current/kernel-flash-attention-sm100-mla-topk.json new file mode 100644 index 000000000..f50cdcf5f --- /dev/null +++ b/verification/reviewed/current/kernel-flash-attention-sm100-mla-topk.json @@ -0,0 +1,60 @@ +{ + "path": "wiki/kernels/flash-attention-sm100-mla-topk.md", + "id": "kernel-flash-attention-sm100-mla-topk", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "kernel-flash-attention-sm100-mla-topk-current-c001", + "claim": "PR 2441 merge f219c89c adds the SM100 CuTe DSL MLA 64/512 top-k forward path for packed MQA with 128 query heads per KV head; top-k uses cp.async gather, defaults to 2048, requires a multiple of 256, and rejects page tables.", + "claim_type": "compatibility", + "covers": ["u001", "u002"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/prs/flash-attention/PR-2441/key-files/flash_attn/cute/flash_fwd_mla_sm100.py","revision":"PR 2441 merge f219c89c886c6ccbf9d3dbd9fe41b11ac64e9df8; SHA-256 f6512b25d901463cf62433b745e87ead30443f295ce313c03e363c886ece296e","locator":"FlashAttentionMLAForwardSm100 constructor and __call__ assertions","observation":"The source defaults hdim/hdimv to 64/512 and topk to 2048, requires pack_gqa, qhead_per_kvhead=128 and cp.async gather in top-k mode, checks topk_length modulo tile_n*2=256, and rejects mPageTable."}], + "challenge": {"alternative":"The code is a generic paged MLA path and the named dimensions/top-k are only benchmark examples.","check":"Inspect constructor defaults plus hard assertions and the page-table guard in the pinned merge.","outcome":"Several constraints are enforced in code, and page tables are explicitly rejected rather than merely unbenchmarked."} + }, + { + "claim_id": "kernel-flash-attention-sm100-mla-topk-current-c002", + "claim": "The pinned merge separates indexed top-k K/V gathering, tile scheduling, and their integration; the gather uses caller indices, cp.async copies, and an optional bounds bitmask, while the integrating kernel supplies two-CTA tcgen05/TMEM/softmax/output flow.", + "claim_type": "code", + "covers": ["u003", "u004", "u005", "u006", "u007"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/prs/flash-attention/PR-2441/key-files/flash_attn/cute/topk_gather_kv.py","revision":"PR 2441 merge f219c89c; SHA-256 58b10c0f75f789b80a163e976c4addd643665fc3b1498249da6e962fec673afb","locator":"CpasyncGatherKVManager create, load_index_topk, compute_bitmask, compute_X_ptr, and load_X","observation":"The dedicated manager reads indices, checks bounds when enabled, forms indexed global pointers, and issues cp.async K/V copies."},{"path":"artifacts/prs/flash-attention/PR-2441/key-files/flash_attn/cute/flash_fwd_mla_sm100.py","revision":"same merge and pinned hash","locator":"imports, scheduler selection, two-CTA/MMA/TMEM setup, and role dispatch","observation":"The kernel imports the gather manager and scheduler types independently and integrates them with cta_group TWO, explicit TMEM regions, softmax and epilogue roles."}], + "challenge": {"alternative":"Top-k is only a masking flag inside the softmax loop, and tile scheduling is inseparable from the gather implementation.","check":"Trace module ownership and dataflow from mIndexTopk to K/V pointers and separately from scheduler construction to work tiles.","outcome":"The gather manager owns indexed loads/validity, scheduler objects own work assignment, and the MLA module composes both, establishing separate evidence paths."} + }, + { + "claim_id": "kernel-flash-attention-sm100-mla-topk-current-c003", + "claim": "The performance table exactly transcribes the PR author's initial top-k=2048 decode rows—0.31 ms/955.47 TFLOPS without bitmask, 0.33 ms/898.08 with bitmask, and 1.98 ms/1180.70 vanilla—for batch 512, seqlen_q 1, seqlen_k 16384, 128 heads and the 64/512 shape, while explicitly recording the missing environment and non-reproduction.", + "claim_type": "performance", + "covers": ["u008", "u009", "u010", "u011", "u012"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://api.github.com/repos/Dao-AILab/flash-attention/pulls/2441","revision":"GitHub PR record observed 2026-08-08; merge f219c89c886c6ccbf9d3dbd9fe41b11ac64e9df8","locator":"PR body, initial saturating decode performance block and introductory sentence","observation":"The page's three rows, dimensions, top-k length, and no-bitmask bounds qualification exactly match the author text. That text does not name GPU model, dtype, detailed software/timing setup, sample count, or variance, matching the page caveat."}], + "challenge": {"alternative":"The 0.31 and 0.33 values can be rounded into one 0.3-ms B200 BF16 result, or the vanilla row is a throughput winner that contradicts sparse benefit.","check":"Compare exact row variants/columns and distinguish latency from reported TFLOPS and unreported environment fields.","outcome":"The two sparse rows differ by bounds handling and must remain distinct; lower work can yield lower latency with lower TFLOPS, and no absent environment field is inferred."} + }, + { + "claim_id": "kernel-flash-attention-sm100-mla-topk-current-c004", + "claim": "The transfer checklist follows enforced merge constraints: preserve exact MQA/dimension/top-k/index contracts, require in-bounds indices before disabling the bitmask, reject inferred page-table support, and measure indexed memory behavior separately from reduced arithmetic and full-workload correctness/latency.", + "claim_type": "performance", + "covers": ["u013", "u014", "u015", "u016"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/prs/flash-attention/PR-2441/key-files/flash_attn/cute/flash_fwd_mla_sm100.py","revision":"PR 2441 merge f219c89c","locator":"top-k constructor assertions, page-table guard, and gather/bitmask pipeline selection","observation":"The source enforces the listed compatibility and bounds distinctions and routes sparse loads through a dedicated indexed path."},{"path":"artifacts/prs/flash-attention/PR-2441/diff.patch","revision":"same merge; SHA-256 4525d7f9af76740692474ffc0266fe9cf72f894ce753fd9d18b70771c96b6810","locator":"benchmark effective-sequence FLOP accounting and gather-index setup","observation":"Arithmetic uses the effective gather length, while K/V access uses generated indices and separate bandwidth accounting, supporting independent measurement rather than a guaranteed proportional outcome."}], + "challenge": {"alternative":"Top-k length alone predicts latency and correctness, making bounds, page mode, and memory-path validation unnecessary.","check":"Compare hard feature guards, bitmask predicates, arbitrary indexed pointers, and the PR's differing bitmask/no-bitmask rows.","outcome":"Behavior and even reported latency vary with bounds handling, while an unsupported page mode is rejected; top-k length is insufficient."} + }, + { + "claim_id": "kernel-flash-attention-sm100-mla-topk-current-c005", + "claim": "The local source record and byte-verified PR-2441 provenance resolve the exact integration and gather files, and the displayed lookup succeeds while a false filename fails.", + "claim_type": "code", + "covers": ["u017", "u018"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python ; sha256sum artifacts/prs/flash-attention/PR-2441/diff.patch and key files","environment":"KernelWiki worktree; conda base Python 3.13.14","reference":"sources/prs/flash-attention/PR-2441.md and artifacts/prs/flash-attention/PR-2441/PROVENANCE.yaml","result":"The displayed assertions pass; provenance records merge f219c89c, flash_fwd_mla_sm100.py and topk_gather_kv.py with hashes that match the local files.","negative_control":"Replacing topk_gather_kv.py with nonexistent_topk.py raises AssertionError."}], + "challenge": {"alternative":"The page finds stale text but not the captured file, or the bundle belongs to the unrelated FA3 PR 1236.","check":"Resolve both paths, compare source_pr_id/upstream SHA, and test a deliberately absent filename.","outcome":"The manifest is explicitly PR 2441 at f219c89c, hash-matches its files, and rejects the absent token; PR 1236 is no longer a source relationship."} + } + ], + "commands": ["inspect PR 2441 GitHub API record", "inspect byte-verified PR-2441 diff and key files", "execute the page lookup and a failing filename negative control with conda base Python", "conda run -n base python scripts/validate.py"], + "residual_risk": "No NVIDIA GPU or CUDA toolkit is installed, so the kernel, numerical results, and author-reported latency/TFLOPS rows were not reproduced. Compatibility and mechanism statements are source-level conclusions tied to merge f219c89c; the performance rows remain deliberately outside structured metadata because the PR omits essential environment details.", + "scope": "All 18 current units for body hash 73e3ffa9038dafda7f4083be34a1dabae23531849cb738ba46e7914bfcff9857 were checked against the exact PR record, byte-verified merge artifacts, source constraints, a discriminating lookup test, and repository validation." +} diff --git a/verification/reviewed/current/kernel-flashmla.json b/verification/reviewed/current/kernel-flashmla.json new file mode 100644 index 000000000..a5b69396e --- /dev/null +++ b/verification/reviewed/current/kernel-flashmla.json @@ -0,0 +1,110 @@ +{ + "path": "wiki/kernels/flashmla.md", + "id": "kernel-flashmla", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "kernel-flashmla-current-c001", + "claim": "At commit 71c7379, FlashMLA powers DeepSeek-V3/V3.2-Exp and packages MLA-mode decode/sparse-prefill plus a distinct dense MHA prefill operator; the repository distinguishes MQA 576/512 from MHA 192/128 or 128/128 modes.", + "claim_type": "compatibility", + "covers": ["u001"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"DeepSeek FlashMLA commit 71c737929f2567bd0a094ae140f8f60f390b1232; SHA-256 46c001c53800f47148918f4fc2969102492dd5164f710fc015db1622f6f9929a","locator":"Introduction, support matrix, and MLA Mode footnote","observation":"The first-party snapshot names both model families, separates dense prefill as MHA, and defines the exact MQA/MHA dimension pairs transcribed on the page."}], + "challenge": {"alternative":"The package name makes dense prefill an MLA operator despite the table label.","check":"Compare the support-matrix mode column, dimension footnote, public API section, and test names.","outcome":"All four independently call the dense path MHA and reserve MQA dimensions for MLA calculation, so the page's distinction is deliberate and source-exact."} + }, + { + "claim_id": "kernel-flashmla-current-c002", + "claim": "DeepSeek-V2 specifies per-layer/token cache sizes of 2*n_h*d_h elements for MHA and d_c+d_h^R for MLA, approximately 4.5*d_h with d_c=4*d_h and d_h^R=d_h/2; byte totals require additional dtype/layer assumptions.", + "claim_type": "compatibility", + "covers": ["u002", "u003", "u004", "u005"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/html/2405.04434v5","revision":"DeepSeek-V2 paper v5","locator":"Section 2.1.1, equations 8-12, hyperparameters, and Table 1","observation":"The paper defines the low-rank cache, gives d_c=4d_h and d_h^R=d_h/2, and lists the exact element-count formulas. Because Table 1 counts elements rather than bytes, dtype and layer multiplication are necessary for whole-model byte totals."}], + "challenge": {"alternative":"Table 1 already defines an implementation byte count, so storage precision can be inferred.","check":"Inspect the table heading/units and whether a cache dtype or layer aggregation appears in the formula.","outcome":"The cells are symbolic numbers of cached elements and include an explicit layer factor only at model-total scope; no byte width is specified, preventing an unqualified KB conversion."} + }, + { + "claim_id": "kernel-flashmla-current-c003", + "claim": "The 656-byte ABI is limited to V3-family FP8 sparse decode and consists of 512 E4M3 NoPE bytes, four FP32 per-128-value scales, and 64 BF16 RoPE values; dense decode is BF16, another sparse layout exists, and page size is an input rather than a fixed 64.", + "claim_type": "code", + "covers": ["u006", "u007", "u008", "u009", "u010", "u011"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py","revision":"DeepSeek FlashMLA commit 71c7379","locator":"flash_mla_with_kvcache docstring, scheduler config, and dense/sparse dispatch","observation":"The source gives the exact V3 FP8+sparse byte fields, derives page_block_size from k_cache.shape[1], sends FP8 only to sparse dispatch, and sends dense calls to the BF16 path."},{"path":"/tmp/kernelwiki-flashmla-pinned/tests/quant.py","revision":"same commit","locator":"FP8KVCacheLayout.get_meta and quantize_k_cache","observation":"V32 uses (576,512,64,128,4) and 656 bytes, while MODEL1 uses (512,448,64,64,7) and a different E8M0 scale packing."}], + "challenge": {"alternative":"656 and page size 64 are compile-time invariants hidden below the Python interface.","check":"Trace interface assertions, quantizer arithmetic, SM90 656 assertion scope, and tests with multiple block sizes.","outcome":"The 656 assertion is inside the V32 sparse-FP8 specialization, while page_block_size is passed through metadata and sparse tests exercise values other than 64; the narrow scope survives both layers."} + }, + { + "claim_id": "kernel-flashmla-current-c004", + "claim": "The support table and requirements exactly distinguish dense decode SM90/BF16, sparse decode SM90+SM100/FP8-KV-to-BF16-MMA, dense MHA prefill SM100/BF16, sparse MLA prefill SM90+SM100/BF16, with CUDA 12.8+, CUDA 12.9+ for SM100, and PyTorch 2.0+.", + "claim_type": "compatibility", + "covers": ["u012", "u013", "u014", "u015", "u016"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"DeepSeek FlashMLA commit 71c7379","locator":"Requirements, support matrix, FP8 decode note, sparse-prefill equivalence, and Dense MHA Prefill","observation":"Every architecture/mode/format and version floor on the page is an exact transcription; the FP8 decode paragraph explicitly says dequantize/MMA/output bfloat16."}], + "challenge": {"alternative":"Global SM90/SM100 support means every operator supports both generations and FP8 is the arithmetic dtype throughout.","check":"Read each support-matrix row and the mode-specific dtype paragraphs rather than the global requirement line.","outcome":"Rows differ by architecture and cache format, and the source explicitly separates FP8 storage from BF16 compute, refuting a uniform interpretation."} + }, + { + "claim_id": "kernel-flashmla-current-c005", + "claim": "Sparse decode consumes page-encoded [batch,s_q,topk] indices with -1 invalid and no block table; sparse prefill consumes unbatched BF16 tensors and [s_q,h_kv,topk] indices, documents h_kv=1, accepts negative/out-of-range invalid indices, returns out/max_logits/lse, and neither attention interface produces top-k indices.", + "claim_type": "code", + "covers": ["u017", "u018", "u019", "u020"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py","revision":"DeepSeek FlashMLA commit 71c7379","locator":"flash_mla_with_kvcache indices documentation/dispatch and flash_mla_sparse_fwd signature","observation":"The exact tensor ranks, dtypes, physical page+offset formula, invalid values, absent block-table rule, and return tuple are encoded in the public interface; indices are caller inputs to both extension calls."},{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"same commit","locator":"Sparse Attention and Sparse MLA Prefill usage","observation":"The README independently states the index encoding, no-batch limitation, h_kv=1 equivalence, invalid-index rules, and return values."}], + "challenge": {"alternative":"The extension internally runs a selector before using optional indices, or sparse prefill shares decode paging semantics.","check":"Trace the Python dispatch arguments and compare the two documented equivalence calculations.","outcome":"Sparse extension calls receive indices directly, no selector output is returned or created, and prefill gathers flat kv[indices] without a page table, establishing two distinct caller contracts."} + }, + { + "claim_id": "kernel-flashmla-current-c006", + "claim": "The performance table precisely preserves the pinned README's author-reported maxima, operator identities, stated environments, and precision scopes while noting missing benchmark cells, non-reproduction, and the invalidity of cross-regime comparisons.", + "claim_type": "performance", + "covers": ["u021", "u022", "u023", "u024", "u025", "u026", "u027", "u028"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"DeepSeek FlashMLA commit 71c7379","locator":"three Performance subsections and support matrix","observation":"The page exactly transcribes dense-decode 3000 GB/s/660 TFLOPS H800 CUDA 12.8, sparse-decode 410 H800 and up to 350 B200, dense MHA prefill 1460/1000 B200 attributed to NVIDIA, and sparse-prefill 640 H800 CUDA 12.8/1450 B200 CUDA 12.9. Its precision labels follow the API/support text."}], + "challenge": {"alternative":"All TFLOPS rows can be ranked directly, and absent details can be inferred from nearby tests or GPU peaks.","check":"Compare headings, memory/compute-bound qualifications, phases, attribution, dtype contracts, and fields actually stated in the source.","outcome":"The observations cover decode versus prefill, MLA versus MHA, storage versus MMA precision, and different GPUs/software; the source omits complete shapes/timing statistics, so the page correctly keeps them out of structured records and warns against ranking."} + }, + { + "claim_id": "kernel-flashmla-current-c007", + "claim": "Pinned SM100 DeepSeek sources contain TMA, tcgen05, and TMEM paths, while local full artifacts are accurately labeled as separate CUTLASS 9baa06dd and FlashInfer 9a05c92a implementations and the derived helper is limited to verified layout/index arithmetic.", + "claim_type": "code", + "covers": ["u029", "u030", "u031", "u032", "u033"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashmla-pinned/csrc/sm100","revision":"DeepSeek FlashMLA commit 71c7379","locator":"decode/head64/kernel.cuh, prefill/sparse phase1 files, and prefill/dense kernel files","observation":"The pinned code includes TMEM allocators, tcgen05 synchronization/MMA helpers, and TMA-specialized paths across the named operator families."},{"path":"artifacts/kernels/flashmla/full/PROVENANCE.yaml","revision":"SHA-256 844d2083c1fc3098981334111fe48e41316a9c9b2b4abb448070379b8e0244f5","locator":"bundle_scope and per-file provenance","observation":"The manifest explicitly marks adjacency and records CUTLASS 9baa06dd for 77_blackwell_mla_fwd.cu and FlashInfer 9a05c92a for the header; file hashes match."},{"path":"artifacts/kernels/flashmla/variants/01-mla-layout-and-index-contract.cu","revision":"SHA-256 aa886373129daedc09b9dae7a23960b021256647e1002399db0dca82053c6f96","locator":"entire derived helper","observation":"The header says non-upstream and the file contains only compile-time 656-byte arithmetic plus physical-page index encode/decode, not a fabricated attention kernel."}], + "challenge": {"alternative":"Matching MLA names make the CUTLASS/FlashInfer files part of DeepSeek FlashMLA, or the derived helper still implies a portable kernel ABI.","check":"Compare repository/commit provenance and compile the helper independently of CUDA.","outcome":"Origins are disjoint and explicit; g++ C++17 syntax checking passes because the helper claims only arithmetic contracts and contains no GPU-kernel body."} + }, + { + "claim_id": "kernel-flashmla-current-c008", + "claim": "The selection checklist follows the verified compatibility boundaries: exact operator/dimensions/version/dtype/index matching, narrow 656-byte scope, external index generation, per-regime benchmarking, and reference output/LSE validation.", + "claim_type": "compatibility", + "covers": ["u034", "u035", "u036", "u037", "u038"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"DeepSeek FlashMLA commit 71c7379","locator":"support matrix, usage contracts, tests, and performance sections","observation":"The source exposes each listed compatibility field, requires caller indices, supplies reference/test paths, and reports performance only in distinct qualified regimes."}], + "challenge": {"alternative":"The library's compatibility wrapper makes exact dimensions/cache/index rules and numerical reference checks unnecessary.","check":"Inspect hard assertions and dispatch conditions in flash_mla_interface.py and compare the separate tests/reference implementations.","outcome":"Mode mismatches are asserted or route to different kernels, indices determine addresses, and performance maxima do not establish correctness; the checklist targets actual failure boundaries."} + }, + { + "claim_id": "kernel-flashmla-current-c009", + "claim": "The four source links resolve to the audited DeepSeek commit, DeepSeek-V2 v5 paper, and pinned CUTLASS forward/backward MLA-shape PRs, each used only for its stated subject.", + "claim_type": "other", + "covers": ["u039", "u040", "u041", "u042"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232","revision":"commit 71c7379","locator":"repository tree and README","observation":"Exact first-party FlashMLA snapshot."},{"url":"https://arxiv.org/html/2405.04434v5","revision":"DeepSeek-V2 v5","locator":"Section 2.1.1","observation":"First-party MLA mechanism and cache formula."},{"url":"https://github.com/NVIDIA/cutlass/pull/2466","revision":"merge 0e026982","locator":"PR title and changed paths","observation":"CUTLASS SM100 MLA-shape backward and reduction path."},{"url":"https://github.com/NVIDIA/cutlass/pull/2472","revision":"merge 9baa06dd","locator":"PR title and changed paths","observation":"CUTLASS SM100 MLA-shape forward path."}], + "challenge": {"alternative":"The CUTLASS PRs can substantiate DeepSeek FlashMLA performance/mechanisms directly, or the paper specifies the 656-byte ABI.","check":"Compare repositories, revision identities, claimed layers, and page wording around each citation.","outcome":"The page confines DeepSeek facts to its repository/paper and calls CUTLASS adjacent; it does not transfer CUTLASS numbers or conflate the model formula with the implementation ABI."} + }, + { + "claim_id": "kernel-flashmla-current-c010", + "claim": "The displayed conda-base get_page query resolves the corrected FlashMLA page and labeled artifacts, and the displayed mode-specific byte check evaluates to 656 without a GPU.", + "claim_type": "code", + "covers": ["u043", "u044", "u045", "u046"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python scripts/get_page.py kernel-flashmla --include-code; conda run -n base python -c ''","environment":"KernelWiki worktree; conda base Python 3.13.14","reference":"wiki/kernels/flashmla.md and artifacts/kernels/flashmla","result":"The query exits 0 and contains the audited commit, 'adjacent implementations', both full files, and variants/01-mla-layout-and-index-contract.cu; the byte assertion also exits 0 with value 656.","negative_control":"kernel-flashmla-does-not-exist exits 1 with 'Page not found'; changing expected bytes from 656 to 655 raises AssertionError."}], + "challenge": {"alternative":"The command prints only page Markdown or silently resolves a fuzzy unrelated ID, and the byte total depends on unavailable GPU behavior.","check":"Assert corrected artifact tokens in real output, repeat with a near-name known to be absent, execute the arithmetic, and alter its expected result.","outcome":"The exact page includes both artifact classes, the near-name fails, real arithmetic passes, and the altered result fails; the 656 total is a storage-layout identity rather than a GPU measurement."} + } + ], + "commands": ["inspect DeepSeek FlashMLA commit 71c737929f2567bd0a094ae140f8f60f390b1232 README, public interfaces, tests, quantizer, and SM100 code", "inspect DeepSeek-V2 paper v5 formulas", "execute both extracted Python contract checks", "compile the derived C++17 contract helper with g++ -fsyntax-only", "execute get_page with the real ID and a failing near-name", "conda run -n base python scripts/validate.py"], + "residual_risk": "No NVIDIA GPU or CUDA toolkit is installed, so attention kernels, numerical output, bandwidth, and throughput were not reproduced. The performance table is deliberately limited to qualified author reports and performance_claims is empty; implementation conclusions are source-level and tied to commit 71c7379.", + "scope": "All 46 current units for body hash 3c4ba2ede0a25c46871912580475e2e71786d53c38b8dea8330b7b017e42d0ae were checked against version-pinned primary sources, exact local provenance, executable contract checks, a compiled derived helper, and a discriminating repository query." +} diff --git a/verification/reviewed/current/kernel-fp8-block-scale-gemm.json b/verification/reviewed/current/kernel-fp8-block-scale-gemm.json new file mode 100644 index 000000000..1d1cf8600 --- /dev/null +++ b/verification/reviewed/current/kernel-fp8-block-scale-gemm.json @@ -0,0 +1,90 @@ +{ + "path": "wiki/kernels/fp8-block-scale-gemm.md", + "id": "kernel-fp8-block-scale-gemm", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "kernel-fp8-block-scale-gemm-current-c001", + "claim": "DeepSeek-V3 forward FP8 training uses 1x128 activation tiles and 128x128 weight blocks to adapt scales to local outliers, while other phases include exceptions such as 128x1 activation grouping.", + "claim_type": "compatibility", + "covers": ["u001", "u005"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/html/2412.19437v2","revision":"DeepSeek-V3 Technical Report v2","locator":"Sections 3.3.2 and Appendix B.2","observation":"The paper gives the exact forward groupings and outlier motivation, then explicitly distinguishes backward activation grouping and other higher-precision policies."}], + "challenge": {"alternative":"The 128x128 grouping is universal for every activation/weight/gradient and model phase.","check":"Compare forward quantization prose with backward exceptions and sensitivity ablations.","outcome":"The paper preserves 1x128 forward activations but changes/restricts other cases, matching the page's logical-format and non-universality boundary."} + }, + { + "claim_id": "kernel-fp8-block-scale-gemm-current-c002", + "claim": "DeepGEMM commit 891d57b4 implements related FP8 GEMMs on SM90 and SM100 with architecture-specific scale handling.", + "claim_type": "compatibility", + "covers": ["u002"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-deepgemm-pinned/README.md","revision":"commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"Introduction, requirements, and scale-factor notice","observation":"The project supports SM90/SM100 FP8 GEMMs and explicitly requires FP32 scale factors on SM90 versus packed UE8M0 on SM100."}], + "challenge": {"alternative":"The repository's current mixed FP8/FP4 scope makes the pinned FP8 paths obsolete or architecture-neutral.","check":"Inspect exact SM90/SM100 FP8 source inclusions and API dispatch at the pinned commit.","outcome":"Distinct 1D1D implementations remain selected by architecture and encode different scale paths, so the page's pinned split is current for that revision."} + }, + { + "claim_id": "kernel-fp8-block-scale-gemm-current-c003", + "claim": "For A[M,K] times B[N,K]^T under the DeepSeek-V3 forward recipe, logical activation-scale shape is [M,K/128] and logical weight-scale shape is [N/128,K/128]; the displayed derived check enforces divisibility and computes the 4096 example correctly without claiming an implementation layout.", + "claim_type": "code", + "covers": ["u003", "u004"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python ","environment":"KernelWiki worktree; conda base Python 3.13.14","reference":"wiki/kernels/fp8-block-scale-gemm.md and DeepSeek-V3 report Section 3.3.2","result":"The 4096-cubed example returns ((4096,32),(32,32)) and exits 0.","negative_control":"Changing N to 4095 triggers the divisibility assertion."}], + "challenge": {"alternative":"The example accidentally describes DeepGEMM's physical packed/TMA layout rather than only logical scale groups.","check":"Compare the function's returned ranks with the repository's transformed layouts and its own explicit prose disclaimer.","outcome":"The function only divides logical dimensions and the page says transformed layouts are excluded; the negative control discriminates the stated 128 grouping."} + }, + { + "claim_id": "kernel-fp8-block-scale-gemm-current-c004", + "claim": "Pinned SM90 1D1D requires FP32 factors and BLOCK_K=128, performs WGMMA partial accumulation, then scale-multiplies/adds into FP32 final_accum on CUDA cores; the paper equates the 128-element interval to four WGMMAs and describes the limiting additions as 14-bit precision.", + "claim_type": "code", + "covers": ["u006", "u007"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh","revision":"DeepGEMM commit 891d57b4","locator":"BLOCK_K assertion and math warp-group loop","observation":"The implementation declares float accum/final_accum, issues BLOCK_K/WGMMA::K operations, and then executes final_accum += scale_a*scale_b*accum."},{"path":"sources/docs/deepseek-v3-fp8.md","revision":"summary of DeepSeek-V3 report v2","locator":"Hopper Accumulation","observation":"The primary-paper capture records four WGMMAs per 128 elements and the 14-bit precision characterization."}], + "challenge": {"alternative":"Scale application is fused into WGMMA or occurs only once in the epilogue, and the paper's interval counts instructions.","check":"Trace one k_block iteration and compare it with the paper's explicit parenthetical equivalence.","outcome":"Scales are loaded per K block and combined after WGMMA completion; the paper says 128 elements equals four WGMMAs, ruling out both alternatives."} + }, + { + "claim_id": "kernel-fp8-block-scale-gemm-current-c005", + "claim": "Pinned SM100 uses packed four-per-int32 UE8M0 factors, TMA/UTCCP scale movement into TMEM, a block-scaled UMMA descriptor, and TMEM accumulation rather than the SM90 CUDA-core promotion loop; selected SM100 recipes include K granularities 32 and 128.", + "claim_type": "code", + "covers": ["u008", "u009", "u010"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-deepgemm-pinned/README.md","revision":"DeepGEMM commit 891d57b4","locator":"scale-factor notice","observation":"SM100 requires packed UE8M0 with four values per torch.int."},{"path":"/tmp/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh","revision":"same commit","locator":"TMA scale loads, UTCCP copies, make_instr_desc_block_scaled, and MMA loop","observation":"The exact dataflow moves factors to TMEM and selects float_ue8m0_t block-scaled UMMA; no SM90-style final_accum loop is present."},{"path":"/tmp/kernelwiki-deepgemm-pinned/tests/generators.py","revision":"same commit","locator":"gran_k_list for architecture 10","observation":"SM100 generator coverage includes gran_k 32 and 128, whereas SM90 uses 128."}], + "challenge": {"alternative":"Packed UE8M0 is merely an external storage optimization and the MMA still receives ordinary FP8 E4M3 scale values after unpacking.","check":"Follow scale type from API requirement through TMEM copy to the instruction descriptor template.","outcome":"The descriptor's scale type is cutlass::float_ue8m0_t and the hardware block-scaled operation consumes its descriptors; E4M3 is the matrix-element type, not the scale type."} + }, + { + "claim_id": "kernel-fp8-block-scale-gemm-current-c006", + "claim": "The only retained numeric performance observation is DeepGEMM's source-reported up-to 1550 TFLOPS on H800, explicitly missing shape/utilization/timing statistics and unreproduced; CUTLASS availability is not presented as a matched comparison.", + "claim_type": "performance", + "covers": ["u011", "u012"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-deepgemm-pinned/README.md","revision":"DeepGEMM commit 891d57b4","locator":"2025-04-18 news item","observation":"The page transcribes exactly the stated GPU and maximum while its caveat enumerates the fields absent from that sentence."},{"path":"sources/docs/cutlass-changelog-sm100.md","revision":"release capture through 4.5.0","locator":"blockscaled GEMM entries","observation":"CUTLASS availability is documented, but no matched CUTLASS/DeepGEMM comparison is present, agreeing with the page's refusal to rank them."}], + "challenge": {"alternative":"A shape and utilization can be recovered from linked optimization changes or theoretical peak and safely structured.","check":"Require one primary benchmark cell containing workload, environment, timing, samples, and metric conventions.","outcome":"No such cell is attached to the README maximum, so the qualified prose and empty structured list are the strongest supported form."} + }, + { + "claim_id": "kernel-fp8-block-scale-gemm-current-c007", + "claim": "The selection checklist correctly requires exact scale grouping/representation/layout/architecture matching, includes preprocessing and epilogue costs, and calls for numerical comparison on the real data distribution.", + "claim_type": "compatibility", + "covers": ["u013", "u014", "u015", "u016", "u017"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-deepgemm-pinned/README.md","revision":"DeepGEMM commit 891d57b4","locator":"interface notices and user-managed casting/layout warning","observation":"The README distinguishes scale ABIs and says transposition/FP8 casting are separate user responsibilities whose simple utilities may be slower."},{"path":"sources/docs/deepseek-v3-fp8.md","revision":"DeepSeek-V3 report v2 capture","locator":"Fine-Grained Quantization and Hopper Accumulation","observation":"The model source motivates group-specific numerical validation and documents scale/precision overheads and exceptions."}], + "challenge": {"alternative":"Matching FP8 dtype and peak GEMM throughput is enough to select the kernel.","check":"Enumerate API assertions and all work outside the GEMM call that affects compatibility, accuracy, and end-to-end time.","outcome":"Scale dtype/layout and recipe dispatch are hard requirements; casting/layout transforms and distribution-sensitive quantization error remain outside a bare throughput number."} + }, + { + "claim_id": "kernel-fp8-block-scale-gemm-current-c008", + "claim": "The source list points to the exact DeepSeek-V3 v2 FP8 report, audited DeepGEMM commit 891d57b4, and official NVIDIA PTX ISA, with each used only at its proper model, implementation, or instruction-contract scope.", + "claim_type": "other", + "covers": ["u018", "u019", "u020"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/html/2412.19437v2","revision":"DeepSeek-V3 report v2","locator":"FP8 Training","observation":"Model-level grouping/numerical source."},{"url":"https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f","revision":"commit 891d57b4","locator":"repository tree","observation":"Exact implementation snapshot."},{"url":"https://docs.nvidia.com/cuda/parallel-thread-execution/","revision":"official PTX ISA, consulted 2026-08-08","locator":"tcgen05.mma and alternate floating-point formats","observation":"Authoritative version-sensitive instruction and UE8M0 contract."}], + "challenge": {"alternative":"The paper can define exact current PTX syntax or the PTX manual can establish DeepGEMM's implementation choices.","check":"Map each page claim to its cited layer and inspect whether any behavior is transferred across sources.","outcome":"Model format comes from the paper, source dataflow from the pinned repository, and instruction caution from PTX; the page avoids cross-layer inference."} + } + ], + "commands": ["inspect DeepGEMM commit 891d57b4 README, SM90/SM100 kernels, PTX wrappers, and test recipes", "inspect DeepSeek-V3 Technical Report v2", "execute the displayed logical-shape check and a failing divisibility negative control", "conda run -n base python scripts/validate.py"], + "residual_risk": "No NVIDIA GPU or CUDA toolkit is installed, so kernel correctness, numerical error, and the H800 maximum were not reproduced. PTX behavior is described at contract level and exact assembly remains toolkit/version-sensitive.", + "scope": "All 20 current units for body hash 5c0369ba45f1bfcc220967e83d4765f7e8b56908a310a1bf0356e9bb2c996e70 were checked against the pinned implementation, primary paper, official PTX source, and a discriminating host check." +} diff --git a/verification/reviewed/current/kernel-fused-moe.json b/verification/reviewed/current/kernel-fused-moe.json new file mode 100644 index 000000000..d19e12481 --- /dev/null +++ b/verification/reviewed/current/kernel-fused-moe.json @@ -0,0 +1,156 @@ +{ + "path": "wiki/kernels/fused-moe.md", + "id": "kernel-fused-moe", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "kernel-fused-moe-current-c001", + "claim": "MLSys 2026 FlashInfer Track A targets B200, names Fused MoE with FP8 support, and links the exact DeepSeek-style definition whose scope is routing plus two grouped GEMMs rather than a guaranteed launch count.", + "claim_type": "compatibility", + "covers": ["u001", "u056", "u057"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://mlsys26.flashinfer.ai/","revision":"fetched 2026-08-08","locator":"Challenge and Track A","observation":"The organizer specifies NVIDIA Blackwell B200, calls Track A Fused MoE with FP8 support, and links the exact benchmark."},{"url":"https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","revision":"fetched 2026-08-08","locator":"description and tags","observation":"The definition says routing and two grouped GEMMs are included and is tagged deepseek-v3/deepseek-r1, FP8 E4M3FN, API trtllm_fp8_block_scale_moe, and EP=8."}], + "challenge": {"alternative":"The single public entry point proves that the whole operation is one GPU launch or that Hopper is an equal contest target.","check":"Separate organizer hardware scope and operation description from implementation internals.","outcome":"The primary pages specify B200 and an operation contract only; neither supplies a launch decomposition or Hopper target, matching the page's explicit boundary."} + }, + { + "claim_id": "kernel-fused-moe-current-c002", + "claim": "The fixed definition varies seq_len and uses 256 global experts, 32 local experts under EP=8, top-k 8, eight groups, four retained groups, H=7168, I=2048, W13 output 4096, and 128-element scale blocks; e32 denotes local experts.", + "claim_type": "compatibility", + "covers": ["u002", "u003", "u004", "u005", "u006", "u007", "u008", "u009", "u010", "u011", "u012", "u013"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","revision":"fetched 2026-08-08","locator":"definition name, Axes, and Reference Implementation constants","observation":"The primary definition exposes seq_len variable; axes 256 total/32 local/H7168/I2048/W13=4096 and block counts 56/16/32; its exact name/reference fix topk8/ng8/kg4 and block 128."}], + "challenge": {"alternative":"The slug's e32 is the total expert count and EP=8 merely replicates those weights.","check":"Compare routing_logits width, num_experts axis, num_local_experts axis, and local_expert_offset semantics.","outcome":"Routing logits have width 256 while local weight tensors have leading dimension 32 and a global offset; therefore 32 is local, not total."} + }, + { + "claim_id": "kernel-fused-moe-current-c003", + "claim": "The displayed FP8/FP32/BF16 tensor table, output, scale-storage boundary, and complete argument summary match the benchmark and pinned FlashInfer API.", + "claim_type": "code", + "covers": ["u014", "u015", "u016", "u017", "u018", "u019", "u020", "u021", "u022", "u023", "u024", "u025", "u028", "u059", "u060"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"trtllm_fp8_block_scale_moe_ds_routing_trace inputs/outputs and initializer","observation":"The source defines every tensor name, shape role, FP8/FP32 data construction, top-k/group scalars, local offset, routed factor, and BF16 output exactly as summarized."},{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/fused_moe/core.py","revision":"same commit","locator":"trtllm_fp8_block_scale_moe signature and docstring","observation":"The public API accepts the eight tensors and explicit expert/routing/intermediate/local parameters named on the page."}], + "challenge": {"alternative":"The table accidentally applies the current API's MXFP8 32-element UE8M0 variant or global-expert weight layout to the contest trace.","check":"Follow the trace's DeepSeekFp8 initializer and compare runner validation branches.","outcome":"This trace uses FP8 E4M3 tensors with FP32 128-block scales and 32 local weight rows; the separate MXFP8 branch uses different scale dtype/layout and is not imported into the table."} + }, + { + "claim_id": "kernel-fused-moe-current-c004", + "claim": "The official reference uses sigmoid+bias grouped selection, unbiased normalized combine weights, local expert filtering, one logical W13 projection split for SwiGLU, W2, and weighted accumulation; it mandates neither dual TMEM allocations nor a tcgen05 sequence.", + "claim_type": "code", + "covers": ["u029", "u030", "u031", "u032", "u033", "u034", "u035", "u036"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"_trtllm_fp8_block_scale_moe_ds_routing_reference and _fp8_moe_run_experts","observation":"The reference implements the six listed stages, applies bias only to selection, derives weights from unbiased sigmoid values, filters the global/local interval, and uses a single concatenated W13 matmul followed by a split."}], + "challenge": {"alternative":"The optimized runner may internally allocate separate accumulators and emit tcgen05, so the page should present those as contract requirements.","check":"Distinguish observable reference semantics from one possible optimized implementation.","outcome":"The source-level contract constrains inputs/results but not the internal TMEM or instruction schedule; the page makes exactly that distinction."} + }, + { + "claim_id": "kernel-fused-moe-current-c005", + "claim": "The derived standard-library reference implements the stated grouped selection and local W13/SwiGLU/W2 accumulation, passes its deterministic self-test, and rejects the biased-combine-weight alternative.", + "claim_type": "code", + "covers": ["u037"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python artifacts/kernels/fused-moe/variants/01-routing-plus-fusion-skeleton.py","environment":"KernelWiki worktree; conda base Python without third-party tensor libraries","reference":"official FlashInfer reference semantics at commit 7f614b8","result":"Exited 0 and printed 'fused-moe derived reference: PASS'; checks exact selected IDs, per-token weight sum 2.5, output shape, and finite output.","negative_control":"The self-test forms combine weights from biased selection scores and asserts they differ from the correct unbiased weights."}], + "challenge": {"alternative":"The variant repeats one vectorized implementation and therefore cannot discriminate selection bias from combine-weight semantics.","check":"Inspect and run the explicit wrong-weight negative control with nonzero routing bias.","outcome":"The control would fail if biased scores were reused for combination, while the correct implementation passes."} + }, + { + "claim_id": "kernel-fused-moe-current-c006", + "claim": "Starter-kit commit 75ccd05 records the stated B200 clocks/container/software stack, MoE correctness gates, and arithmetic-mean speedup scoring, while the current sources do not establish the removed TFLOPS/launch table or prefill/decode relabeling.", + "claim_type": "performance", + "covers": ["u038", "u039", "u040", "u041", "u042", "u043", "u044", "u058"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashinfer-bench-starter-kit/EVALUATION.md","revision":"flashinfer-bench-starter-kit 75ccd05cafceb0fd1f86be4cd0f2117249463c66","locator":"Environment, Run Evaluation, FlashInfer Baselines, and Scoring","observation":"The file gives B200 sm_100a, clocks 3996/1965, the exact image and tool versions, MoE tolerance command, baseline name, and mean per-workload speedup formula."},{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"trace axes","observation":"The variable axis is named seq_len and described as number of tokens, with no prefill/decode or request-batch label."}], + "challenge": {"alternative":"A former official table or common serving interpretation can restore 1262 TFLOPS and map seq_len endpoints to prefill/decode.","check":"Require an exact archived source cell and inspect the immutable trace axis description.","outcome":"No archived cell/revision is attached and the trace says number of tokens; the page correctly retains only the evaluation method, not a result or serving-phase inference."} + }, + { + "claim_id": "kernel-fused-moe-current-c007", + "claim": "Per-expert token counts create varying grouped-GEMM M dimensions, and pinned FlashInfer selects separate GEMM1/GEMM2 tactics over token buckets rather than establishing one universal bottleneck.", + "claim_type": "performance", + "covers": ["u045", "u047"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-deepgemm-pinned/README.md","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"Grouped GEMMs (contiguous layout)","observation":"The documentation says experts may process varying token counts and represents them as different M segments."},{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/fused_moe/core.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"MoERunner valid tactics, token bucketing, and choose_one calls","observation":"The runtime enumerates shape-valid tactics and tunes GEMM1/GEMM2 separately over token buckets."}], + "challenge": {"alternative":"Load imbalance is always the primary bottleneck and one fixed tactic is sufficient.","check":"Inspect the implementation's problem-dependent tactic enumeration and separate-stage selections.","outcome":"The runtime explicitly varies selection by shape/token bucket and stage; the page retains imbalance as a risk but rejects a universal ranking."} + }, + { + "claim_id": "kernel-fused-moe-current-c008", + "claim": "Small M or partially filled GEMM tiles can reduce utilization, as stated by NVIDIA's GEMM implementation guidance.", + "claim_type": "performance", + "covers": ["u046", "u062"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/NVIDIA/cutlass/blob/main/media/docs/cpp/efficient_gemm.md","revision":"CUTLASS documentation fetched 2026-08-08","locator":"Threadblock-level GEMM","observation":"NVIDIA explains that small M/N can leave threads outside the problem and that small M/N with large K may launch too few threadblocks to use all SMs."}], + "challenge": {"alternative":"Combining experts into a grouped GEMM guarantees full tiles and full-GPU occupancy.","check":"Apply the document's partial-tile and insufficient-grid alternatives to each variable-M group.","outcome":"Grouping adds independent problems but does not eliminate partial boundary tiles or guarantee enough total tiles; 'can reduce' is the supported scope."} + }, + { + "claim_id": "kernel-fused-moe-current-c009", + "claim": "Compatibility requires the complete routing/quantization/layout contract, and CUDA-graph layouts are backend-specific; DeepGEMM documents masked layout for one CPU-unknown decode case rather than as a universal requirement.", + "claim_type": "compatibility", + "covers": ["u048", "u049", "u063"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/fused_moe/runners.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"TrtllmFp8BlockRunner validation","observation":"The runner checks activation dtype, scale dtype/shape, local weight shapes, routing input mode, local expert interval, layout, and static options separately."},{"path":"/tmp/kernelwiki-deepgemm-pinned/README.md","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"Grouped GEMMs (masked layout)","observation":"Masked grouped GEMM is scoped to decoding with CUDA graph enabled when the CPU does not know expert token counts."}], + "challenge": {"alternative":"Matching FP8 E4M3 element type is sufficient, and masked layout is inherent to any graph capture.","check":"Compare the runner's independent ABI validations and the explicit conditional scope in DeepGEMM.","outcome":"Many fields beyond element type are enforced, and masked layout is presented as one backend scenario, matching the page."} + }, + { + "claim_id": "kernel-fused-moe-current-c010", + "claim": "CUDA 13.2.1 tensor-map alignment is configuration-specific: the object requires 64-byte alignment, ordinary globalAddress 16 bytes, and selected interleave/dtype cases 32 bytes, so 128 bytes is not universal.", + "claim_type": "compatibility", + "covers": ["u050", "u061"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html","revision":"CUDA Driver API 13.2.1","locator":"cuTensorMapEncodeTiled parameter requirements","observation":"The API states tensorMap address alignment 64 bytes, globalAddress 16 bytes normally, and 32-byte requirements for interleave-32B and selected packed data types."}], + "challenge": {"alternative":"The CUDA programming guide's 128-byte TMA-swizzle example makes every tensor base and descriptor require 128 bytes.","check":"Read the encoder's normative per-parameter requirements and distinguish a selected swizzle's shared/global layout guidance.","outcome":"The requirements vary by object, base, interleave, dtype, and swizzle; the page accurately rejects the blanket rule."} + }, + { + "claim_id": "kernel-fused-moe-current-c011", + "claim": "The local full bundle is accurately disclosed as adjacent MXFP4 vLLM, FP4 SGLang, and local pseudocode rather than the Track A implementation, with byte-pinned file scopes and corrected provenance.", + "claim_type": "code", + "covers": ["u051", "u052", "u053", "u054", "u055"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/kernels/fused-moe/full/vllm-PR-23696-dual-gemm.patch","revision":"vLLM merge 074854b24f6e0b1e237a004283e1f46d98c0d73c; SHA-256 033d30b15af4f2050189c2e76ba61959e1e5d4bd049dc901fd56f68890b891a0","locator":"five diff headers and MXFP4 backend code","observation":"The aggregate patch covers five vLLM files and implements MXFP4-weight expert compute."},{"path":"artifacts/kernels/fused-moe/full/flashinfer_cutedsl.py","revision":"SGLang c554dc5c64b661f2c53225b03a76359eaddc39e4; SHA-256 de6bd663382b5bf34dba077d90febc9cc80d9dda908d9947966a3cadbc21bf33","locator":"module and registered function","observation":"The byte-verified upstream file calls fp4_quantize and CuteDslMoEWrapper for the FP4 standard path."},{"path":"artifacts/kernels/fused-moe/full/moe-grouped-gemm-launch.cpp","revision":"SHA-256 1368d01910bce19816daf9af6963a24de3809a0b12162fce1e5dd9d79064e884","locator":"file header","observation":"The header explicitly labels the incomplete file as local illustrative pseudocode, not upstream or timed code."}], + "challenge": {"alternative":"Because two files originate upstream, the directory can still be called the full reference implementation of Track A.","check":"Compare each file's quantization/routing scope with the exact Track A signature.","outcome":"The upstream files use MXFP4/FP4 adjacent contracts and the third is pseudocode; the corrected page/provenance disclose this without overstating them."} + }, + { + "claim_id": "kernel-fused-moe-current-c012", + "claim": "The page's cited revisions and direct URLs resolve to the exact starter-kit evaluation, FlashInfer reference/API, CUDA tensor-map requirements, CUTLASS small-GEMM guidance, and DeepGEMM grouped-layout record used by its claims.", + "claim_type": "other", + "covers": ["u059", "u060", "u061", "u062", "u063"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/flashinfer-ai/flashinfer-bench-starter-kit/blob/75ccd05cafceb0fd1f86be4cd0f2117249463c66/EVALUATION.md","revision":"75ccd05cafceb0fd1f86be4cd0f2117249463c66","locator":"document","observation":"The pinned path resolves to the evaluation contract."},{"url":"https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/trace/templates/moe.py","revision":"7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"file","observation":"The pinned trace reference resolves."},{"url":"https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/fused_moe/core.py","revision":"7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"file","observation":"The pinned API implementation resolves."},{"url":"https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html","revision":"CUDA 13.2.1","locator":"tensor-memory API","observation":"The archived official requirements resolve."},{"url":"https://github.com/NVIDIA/cutlass/blob/main/media/docs/cpp/efficient_gemm.md","revision":"fetched 2026-08-08","locator":"file","observation":"The cited NVIDIA small-dimension discussion resolves."},{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md","revision":"891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"README","observation":"The pinned grouped-layout record resolves."}], + "challenge": {"alternative":"The prose may cite correct facts but the source links are mutable or point to neighboring revisions.","check":"Open each exact URL and verify commit/archive identifiers in the path.","outcome":"Implementation/evaluation links are commit-pinned and CUDA is archive-pinned; the two mutable documentation links are used only for current general guidance and were freshly checked."} + }, + { + "claim_id": "kernel-fused-moe-current-c013", + "claim": "The documented get_page command returns this page with labeled artifact content, while an unknown page ID fails distinctly.", + "claim_type": "code", + "covers": ["u064", "u065"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python scripts/get_page.py kernel-fused-moe --include-code","environment":"KernelWiki worktree; conda base","reference":"current wiki/kernels/fused-moe.md and artifact manifests","result":"Exited 0; output contains the corrected page, mixed-adjacent disclosure, MXFP4 labels, and derived reference.","negative_control":"kernel-fused-moe-broken exited 1 with 'No page found'."}], + "challenge": {"alternative":"The command resolves the page but omits or mislabels artifact content.","check":"Search the output for the corrected adjacent/derived scope and compare an unknown-ID control.","outcome":"The labeled artifacts appear in the positive output and the invalid ID is rejected."} + }, + { + "claim_id": "kernel-fused-moe-current-c014", + "claim": "The displayed derived helper computes all fixed Track A tensor shapes for a positive token count and rejects a zero-token negative control.", + "claim_type": "code", + "covers": ["u026", "u027"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python -c ","environment":"KernelWiki worktree; conda base Python","reference":"official definition axes and signature","result":"For tokens=7, returned routing [7,256], activations [7,7168], activation scales [56,7], W13 [32,4096,7168], W13 scales [32,32,56], W2 [32,7168,2048], and W2 scales [32,56,16].","negative_control":"Calling the same helper with tokens=0 triggers its assertion and exits nonzero."}], + "challenge": {"alternative":"The arithmetic accidentally treats 32 as global experts or transposes the published activation-scale layout.","check":"Compare every returned rank/order with the benchmark signature and exercise a non-default token count.","outcome":"The helper keeps routing width 256, local weight count 32, and activation scales [56,T], matching the primary contract."} + } + ], + "commands": [ + "inspect official contest and benchmark pages fetched 2026-08-08", + "inspect FlashInfer commit 7f614b8 trace template, API, validation, and autotuning", + "inspect starter-kit commit 75ccd05 evaluation contract and DeepGEMM commit 891d57b4 grouped layouts", + "run the standard-library derived reference self-test and biased-weight negative control under conda base", + "run get_page positive and unknown-ID negative controls under conda base" + ], + "residual_risk": "No B200 or CUDA toolkit is installed, so the optimized implementation, numeric tolerance on contest tensors, launch topology, and performance were not reproduced. The page therefore retains no result-level performance claim and labels local upstream files as adjacent rather than as the contest implementation.", + "scope": "All 65 current units for body hash 5b18f9bb5eb643757003bd25f3822609266ef79d9151f3fd7623e798928547ed were checked against primary web records, exact upstream revisions, and discriminating host tests." +} diff --git a/verification/reviewed/current/kernel-gated-delta-net.json b/verification/reviewed/current/kernel-gated-delta-net.json new file mode 100644 index 000000000..731625749 --- /dev/null +++ b/verification/reviewed/current/kernel-gated-delta-net.json @@ -0,0 +1,146 @@ +{ + "path": "wiki/kernels/gated-delta-net.md", + "id": "kernel-gated-delta-net", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "kernel-gated-delta-net-current-c001", + "claim": "FlashInfer commit 7f614b8 implements the displayed Gated DeltaNet recurrence with independent decay and sigmoid update gates, a delta correction against the decayed-state retrieval, output from the updated state, and a recurrent state whose decode shape has no prior-token axis.", + "claim_type": "code", + "covers": ["u001", "u002", "u003"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/gdn.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"_gdn_decode_reference and gated_delta_rule_decode_trace","observation":"The reference forms g=exp(-exp(A_log)*softplus(a+dt_bias)) and beta=sigmoid(b), decays state, reads with k, applies beta*(v-read) as an outer-product correction, and computes output from the new state. Decode state is [B,H,V,K] with no context-length axis."}], + "challenge": {"alternative":"The rule is additive-only, retrieves before decay, emits from the old state, or rereads all prior tokens during decode.","check":"Trace the reference operation order and input shapes, including the signs in the expanded remove/write expression.","outcome":"The subtraction, decay-first ordering, updated-state output, and fixed recurrent-state input reject each alternative; K*V work remains, so only context-length independence is claimed."} + }, + { + "claim_id": "kernel-gated-delta-net-current-c002", + "claim": "Pinned Qwen3-Next-80B-A3B-Instruct has 48 layers arranged as twelve repetitions of three GDN layers and one full Gated Attention layer, 16 QK and 32 value GDN heads of dimension 128, 512 experts with 10 routed plus one shared, 80B total/3B active parameters, and native context 262144.", + "claim_type": "compatibility", + "covers": ["u004", "u005", "u006", "u007", "u008", "u009", "u010", "u011"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/config.json","revision":"Qwen model revision 9c7f2fbe84465e40164a94cc16cd30b6999b0cc7","locator":"num_hidden_layers, full_attention_interval, linear-attention heads, expert fields, and max_position_embeddings","observation":"The immutable config records 48 layers, full attention every fourth layer, 16/32/128 GDN geometry, 512 experts, 10 routed experts, one shared expert, and 262144 positions."},{"url":"https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/tree/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7","revision":"same model revision","locator":"model card overview","observation":"The official card states 80B total and 3B activated parameters and writes the hybrid stack as 12 repetitions of three Gated DeltaNet blocks plus one Gated Attention block."}], + "challenge": {"alternative":"The 3:1 pattern describes heads rather than layers, 512 includes active experts, or fixed GDN state removes cache growth for the full model.","check":"Compare the explicit layer interval, expert-selection fields, and presence of full-attention layers in the immutable config.","outcome":"The interval yields 36 GDN and 12 full-attention layers, active experts are 10 routed plus one shared, and the full-attention quarter retains a context-dependent cache."} + }, + { + "claim_id": "kernel-gated-delta-net-current-c003", + "claim": "Qwen attributes attention-sink and massive-activation mitigation to an output gate in its full Gated Attention path and describes the effect as helpful, not as a required part of the GDN recurrence.", + "claim_type": "mechanism", + "covers": ["u012"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://qwen.ai/blog?id=e34c4305036ce60d55a0791b170337c2b70ae51d","revision":"Qwen3-Next architecture article fetched 2026-08-08","locator":"Gated Attention and Training-Stability-Friendly Designs","observation":"The official article places output gating in Gated Attention and says it helps eliminate attention sinks and massive activations; it does not make that gate part of the GDN kernel contract."}], + "challenge": {"alternative":"The article proves that output gating is necessary and belongs inside every GDN state update.","check":"Read the named architectural subsection and distinguish Gated Attention from Gated DeltaNet.","outcome":"The source uses a qualified benefit statement for the separate full-attention path, supporting the page's narrower wording."} + }, + { + "claim_id": "kernel-gated-delta-net-current-c004", + "claim": "MLSys 2026 Track C defines distinct decode and variable-length prefill contracts at TP=4 with Q/K/V heads 4/4/8, head size 128, BF16 Q/K/V and output, FP32 state [B or N,8,128,128], the listed gate inputs, and cu_seqlens only for prefill; the derived state sizes are 64 KiB per value head and 512 KiB across eight heads.", + "claim_type": "compatibility", + "covers": ["u013", "u014", "u015", "u016", "u017", "u018", "u019", "u020", "u021"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://mlsys26.flashinfer.ai/","revision":"fetched 2026-08-08","locator":"Track C","observation":"The organizer names Gated Delta Net as Track C and links separate decode and prefill definitions."},{"url":"https://bench.flashinfer.ai/kernels/gdn_decode_qk4_v8_d128_k_last","revision":"fetched 2026-08-08","locator":"Axes, Signature, and Outputs","observation":"Decode fixes q/k/v heads 4/4/8 and d128, uses BF16 token tensors, FP32 [B,8,128,128] state and four gate-related inputs, and returns BF16 [B,1,8,128] plus updated state."},{"url":"https://bench.flashinfer.ai/kernels/gdn_prefill_qk4_v8_d128_k_last","revision":"fetched 2026-08-08","locator":"Axes, Signature, and Outputs","observation":"Prefill uses total_seq_len, num_seqs, cu_seqlens[N+1], the same head geometry/dtypes, FP32 [N,8,128,128] state, BF16 token output, and one final state per sequence."}], + "challenge": {"alternative":"qk4/v8 are flattened dimensions, prefill is fixed-length, or 512 KiB applies to each head.","check":"Read the exact axis names and tensor ranks, then recompute 128*128*4 bytes and multiply by eight value heads.","outcome":"The definitions name head counts and variable-length metadata; arithmetic gives 64 KiB per head and 512 KiB across all eight, exactly as scoped."} + }, + { + "claim_id": "kernel-gated-delta-net-current-c005", + "claim": "NVlabs commit b53d6d3 is the authors' PyTorch/Triton research implementation and its README recommends FLA for faster kernels and variable-length functionality.", + "claim_type": "compatibility", + "covers": ["u022"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-gdn-pinned/README.md","revision":"NVlabs/GatedDeltaNet b53d6d3a161267432a79c1c04af69fa52bddc921","locator":"repository overview and FAQ questions 1 and 2","observation":"The authors provide PyTorch/Triton code, state that FLA kernels are faster, identify FLA variable-length support, and strongly recommend it."}], + "challenge": {"alternative":"FLA is an unendorsed fork and the authors' implementation already claims the same variable-length support.","check":"Use the authors' FAQ rather than a third-party comparison.","outcome":"The FAQ directly recommends FLA and distinguishes its faster and variable-length functionality."} + }, + { + "claim_id": "kernel-gated-delta-net-current-c006", + "claim": "At commit 7f614b8 FlashInfer provides the cited GDN trace contracts and a prefill dispatcher for SM90 and SM100/SM103, with the latter implemented in CuTe DSL and requiring CUDA 13 or newer and head size 128.", + "claim_type": "compatibility", + "covers": ["u023"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/gdn_prefill.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"chunk_gated_delta_rule dispatch and SM100 import guard","observation":"The wrapper dispatches SM90 separately and SM100/SM103 to chunk_gated_delta_rule_sm100; its checks document CuTe DSL, CUDA >=13, and head_size 128 for the SM100 path."},{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/gdn.py","revision":"same commit","locator":"decode and prefill trace definitions","observation":"The file supplies the exact reference contracts summarized on the page."}], + "challenge": {"alternative":"Blackwell prefill remains unimplemented or shares an unrestricted generic path with Hopper.","check":"Inspect architecture dispatch, imported implementation, and explicit runtime guards.","outcome":"A distinct SM100/103 implementation is dispatched and guarded by CUDA/head-size requirements, refuting both alternatives."} + }, + { + "claim_id": "kernel-gated-delta-net-current-c007", + "claim": "vLLM merge e1d85e5c supports uniform-batch full CUDA-graph capture for compatible recurrent-attention decode, while NVIDIA scopes CUDA-graph benefit to reduced CPU launch overhead rather than a universal workload speedup.", + "claim_type": "performance", + "covers": ["u024"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/vllm-project/vllm/blob/e1d85e5c2454bd8d349dbe676679380cbe0e920a/vllm/v1/attention/backends/mamba_attn.py","revision":"vLLM e1d85e5c2454bd8d349dbe676679380cbe0e920a","locator":"BaseMambaAttentionMetadataBuilder.build_for_cudagraph_capture","observation":"The backend declares uniform-batch support and implements decode-only full CUDA-graph capture."},{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cuda-graphs.html","revision":"CUDA Programming Guide fetched 2026-08-08","locator":"CUDA Graphs overview","observation":"NVIDIA explains that setup can dominate short kernels and graph replay reduces CPU launch costs."}], + "challenge": {"alternative":"Graphs remove GPU compute time or guarantee acceleration for every batch shape.","check":"Compare NVIDIA's overhead scope with vLLM's uniform-decode capture conditions.","outcome":"The sources support launch-cost reduction under compatible capture conditions, so the page correctly retains a workload-dependent boundary."} + }, + { + "claim_id": "kernel-gated-delta-net-current-c008", + "claim": "TFLA v3 and its official pinned code apply a second level of sequence parallelism to mLSTM for arbitrarily large chunks, but do not establish a Gated DeltaNet implementation or inline WGMMA/tcgen05 path.", + "claim_type": "compatibility", + "covers": ["u025"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-mlstm-kernels/README.md","revision":"NX-AI/mlstm_kernels 5b98ff8e2bec189b3d3c249405bab5149564d6f8","locator":"TFLA overview and implementation list","observation":"The official repository describes TFLA for mLSTM, arbitrary large chunks, and PyTorch/JAX/Triton implementations. A pinned-tree search finds no GatedDeltaNet, wgmma, tcgen05, inline PTX, or asm implementation."}], + "challenge": {"alternative":"A general linear-recurrence paper title or Hopper support is sufficient evidence of GDN-specific assembly.","check":"Inspect the published application and search the immutable implementation tree for GDN and instruction-family identifiers.","outcome":"The method and code are mLSTM-scoped and the claimed GDN/assembly identifiers are absent, supporting the explicit boundary."} + }, + { + "claim_id": "kernel-gated-delta-net-current-c009", + "claim": "The page accurately separates GDN's linear sequence scaling from Qwen's implementation-dependent 10x whole-model comparison, and correctly warns that GDN is a learned architectural layer rather than a weight-compatible runtime replacement for softmax attention.", + "claim_type": "performance", + "covers": ["u026", "u027"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/tree/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7","revision":"Qwen model revision 9c7f2fbe84465e40164a94cc16cd30b6999b0cc7","locator":"Highlights and implementation caveat","observation":"Qwen reports 10x whole-model inference throughput over Qwen3-32B for contexts above 32K and separately warns that efficiency depends strongly on implementation; it supplies no isolated GDN-kernel GPU result."},{"url":"https://github.com/NVlabs/GatedDeltaNet/blob/b53d6d3a161267432a79c1c04af69fa52bddc921/lit_gpt/gated_delta_net.py","revision":"NVlabs/GatedDeltaNet b53d6d3a161267432a79c1c04af69fa52bddc921","locator":"GatedDeltaNet initialization and forward","observation":"The layer owns learned projections, decay/update gates, convolution, output gating/norm, and recurrent state distinct from softmax attention."}], + "challenge": {"alternative":"O(n) complexity alone proves a 10x kernel speedup at 32K, or a serving runtime can swap GDN into arbitrary attention weights.","check":"Inspect the comparison's scope and implementation caveat, then compare GDN's learned parameter/state contract.","outcome":"The result is whole-model and implementation-dependent, and the layer requires GDN-specific learned components; neither alternative is supported."} + }, + { + "claim_id": "kernel-gated-delta-net-current-c010", + "claim": "The local full artifact is byte-pinned SGLang projection split/reshape/concatenation code from merge 5bdc07d and is accurately labeled as adjacent preprocessing rather than a GDN recurrence implementation.", + "claim_type": "code", + "covers": ["u028"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/kernels/gated-delta-net/full/gdn_fused_proj.py","revision":"SGLang 5bdc07d974f6cf236fa765a685453ea5e587a838; SHA-256 c4ef85167eb11065721e76dd5d63d6f346f309f84822578dd4a3874249acef89","locator":"module functions","observation":"The byte-verified file splits and reshapes QKVZBA projection output and concatenates fields; it contains no recurrent state update, prefill, or decode implementation."},{"path":"artifacts/kernels/gated-delta-net/full/PROVENANCE.yaml","revision":"current corrected provenance","locator":"scope and upstream_identity","observation":"Provenance records the exact SGLang merge/file hash and labels the code adjacent preprocessing."}], + "challenge": {"alternative":"An upstream file named gdn_fused_proj can be represented as the full GDN kernel.","check":"Inspect every function for state, gates, recurrence, token loop, or decode/prefill entry points.","outcome":"Only projection-output rearrangement is present, so the corrected adjacent label is precise."} + }, + { + "claim_id": "kernel-gated-delta-net-current-c011", + "claim": "The derived standard-library variant faithfully checks the one-head compact GDN recurrence against an independently expanded remove/write form and rejects an additive-only update.", + "claim_type": "code", + "covers": ["u029"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python artifacts/kernels/gated-delta-net/variants/01-gdn-recurrence-reference.py","environment":"KernelWiki worktree; conda base standard-library Python","reference":"FlashInfer _gdn_decode_reference at commit 7f614b8","result":"Exited 0 and printed 'gated-delta recurrence reference: PASS'; file SHA-256 is 11560c14d805f79dbc1025801beed373c0e17e712e054c6f85231071b8d8b971.","negative_control":"The self-test computes an additive-only state and asserts it differs from the correct delta-rule state."}], + "challenge": {"alternative":"The compact and expanded checks merely duplicate the same erroneous additive update.","check":"Inspect the expanded path's explicit removal of outer(k, beta*read) and run the additive-only negative control.","outcome":"The independently grouped remove/write form matches the compact correction while the no-removal alternative is rejected."} + }, + { + "claim_id": "kernel-gated-delta-net-current-c012", + "claim": "All ten primary-source links resolve to the named paper, immutable implementations/configuration, organizer, exact definitions, and official guidance.", + "claim_type": "other", + "covers": ["u030", "u031", "u032", "u033", "u034", "u035", "u036", "u037", "u038", "u039"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"curl -L -sS -o /dev/null -w '%{http_code}' ","environment":"KernelWiki host, 2026-08-08","reference":"HTTP success plus source identity and pinned revision in each target","result":"All ten targets returned HTTP 200; the implementation and model URLs retain the stated commit/revision paths.","negative_control":"https://mlsys26.flashinfer.ai/definitely-not-a-kernelwiki-page returned HTTP 404."}], + "challenge": {"alternative":"One or more citations are stale, redirect to a generic page, or lose the stated revision.","check":"Follow redirects for every target, inspect identity/revision, and compare a deliberately invalid same-site path.","outcome":"Each intended resource resolves and the invalid control is rejected; immutable code/model links preserve their revision identifiers."} + }, + { + "claim_id": "kernel-gated-delta-net-current-c013", + "claim": "The documented get_page command returns the corrected page and labeled local artifacts, while an unknown page ID fails distinctly.", + "claim_type": "code", + "covers": ["u040", "u041"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python scripts/get_page.py kernel-gated-delta-net --include-code","environment":"KernelWiki worktree; conda base","reference":"current page and corrected artifact manifests","result":"Exited 0; output contains 'Verified mechanism', the full artifact's 'adjacent preprocessing' disclosure, and the derived recurrence PASS marker.","negative_control":"kernel-gated-delta-net-broken exited nonzero with 'No page found'."}], + "challenge": {"alternative":"The query returns prose but silently omits or mislabels the artifacts.","check":"Search positive output for the corrected scope and derived file content, then compare an unknown-ID request.","outcome":"Both artifact classes are present with corrected labels and the invalid ID is rejected."} + } + ], + "commands": [ + "inspect FlashInfer commit 7f614b8, NVlabs/GatedDeltaNet commit b53d6d3, NX-AI/mlstm_kernels commit 5b98ff8, and vLLM merge e1d85e5c", + "inspect immutable Qwen3-Next configuration revision 9c7f2fbe and official Qwen architecture article", + "open the MLSys organizer and exact FlashInfer-Bench decode/prefill definitions on 2026-08-08", + "run the derived recurrence equivalence test and additive-only negative control under conda base", + "run ten-link and get_page positive/negative controls" + ], + "residual_risk": "No B200/H100 or CUDA toolkit is installed, so optimized GPU numeric behavior and performance were not reproduced. The page therefore retains no isolated kernel result and scopes implementation details to inspected primary revisions.", + "scope": "All 41 current coverage units for body hash 2ed46cc12dc31743da8142c69d5a8c6b3d03b077d1ddd624e2e4351206734b68 were checked against primary web records, immutable upstream source, and discriminating host tests." +} diff --git a/verification/reviewed/current/kernel-gated-dual-gemm.json b/verification/reviewed/current/kernel-gated-dual-gemm.json new file mode 100644 index 000000000..2d51a068d --- /dev/null +++ b/verification/reviewed/current/kernel-gated-dual-gemm.json @@ -0,0 +1,105 @@ +{ + "path": "wiki/kernels/gated-dual-gemm.md", + "id": "kernel-gated-dual-gemm", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "kernel-gated-dual-gemm-current-c001", + "claim": "Official NVIDIA rules identify NVFP4 Gated Dual GEMM as Blackwell Hackathon Kernel Challenge 3 for December 20, 2025 through January 16, 2026, and the challenge-opening GPU Mode task targets B200.", + "claim_type": "compatibility", + "covers": ["u001"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf","revision":"official rules fetched 2026-08-08","locator":"Section 3, Kernel #3","observation":"The sponsor's rules explicitly give the challenge number, name, and entry dates."},{"url":"https://github.com/gpu-mode/reference-kernels/blob/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm/task.yml","revision":"gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c","locator":"description","observation":"The immutable public task says the block-scaled dual matrix multiplication is optimized for NVIDIA B200."}], + "challenge": {"alternative":"The numbering/dates are inferred from repository history, or the target is generic Blackwell including Hopper/desktop parts.","check":"Compare the sponsor's numbered rules and the immutable task description.","outcome":"Both scope fields are explicit: Kernel #3 at the stated dates and NVIDIA B200; the page does not extend compatibility beyond that record."} + }, + { + "claim_id": "kernel-gated-dual-gemm-current-c002", + "claim": "The displayed result semantics match the pinned reference: two block-scaled products share A, SiLU applies only to the B1 branch, the result multiplies the B2 branch, and neither correctness interface nor reference mandates a single GPU launch or absence of intermediate storage.", + "claim_type": "code", + "covers": ["u002", "u003", "u004"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/reference.py","revision":"gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c","locator":"ref_kernel","observation":"The reference invokes torch._scaled_mm twice with a_ref and separate B/scales, then computes silu(ref1)*ref2; it allocates ref1 and ref2 explicitly."},{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/template.py","revision":"same commit","locator":"custom_kernel","observation":"The submission interface is a Python function returning c and contains no launch-count or intermediate-storage restriction."}], + "challenge": {"alternative":"The two products are multiplied before activation, the second branch is gated, or 'kernel' normatively means one launch.","check":"Trace the reference expression and inspect every constraint on custom_kernel.","outcome":"The expression fixes first-branch SiLU, while the interface constrains only output; no launch topology is stated."} + }, + { + "claim_id": "kernel-gated-dual-gemm-current-c003", + "claim": "The tensor table, reordered-scale inputs, divisibility requirements, 1e-3 tolerances, and disclosed E4M3FNUZ-versus-E4M3FN inconsistency accurately reproduce the challenge-opening task, template, and reference.", + "claim_type": "compatibility", + "covers": ["u005", "u006", "u007", "u008", "u009", "u010", "u011"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml","revision":"gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c","locator":"description and matrix constraints","observation":"The task publishes A/B1/B2 NVFP4 K-major shapes, logical FP8 scale shapes, FP16 C, K divisible by 256, and M/N tile divisibility; it labels scales e4m3fnuz."},{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/template.py","revision":"same commit","locator":"custom_kernel tuple documentation","observation":"The starter exposes raw and reordered copies of all three scale tensors plus preallocated C, and also labels scales float8_e4m3fnuz."},{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/reference.py","revision":"same commit","locator":"create_scale_factor_tensors, return tuple, and check_implementation","observation":"The generator actually constructs torch.float8_e4m3fn scale tensors, returns raw/reordered copies, and configures rtol=atol=1e-3."}], + "challenge": {"alternative":"FNUZ and FN are interchangeable spelling here, reordered scales are internal-only, or K/16 is a physical flat layout guarantee.","check":"Compare dtype constructors and the complete returned/submission tuple rather than normalizing the prose description.","outcome":"The primary files genuinely disagree on dtype suffix and expose both logical and reordered scales, so the page preserves the inconsistency and avoids a stronger physical-layout claim."} + }, + { + "claim_id": "kernel-gated-dual-gemm-current-c004", + "claim": "The four M/N/K/L workload rows, theoretical microsecond values, geometric-mean ranking, 1.5 GHz B200 model, and removal of the unmatched 18.5-us result exactly follow the official task's performance boundary.", + "claim_type": "performance", + "covers": ["u012", "u013", "u014", "u015", "u016"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml","revision":"gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c","locator":"speed-of-light table, benchmarks, and ranking_by","observation":"The task lists exactly the four rows and 4.708/8.714/2.125/6.535 us values, calls them speed-of-light analysis from max math/DRAM time at 1.5 GHz, and ranks by geometric mean. It has no M=1024 workload or 18.5-us measurement."}], + "challenge": {"alternative":"The listed values are measured baseline or winner latencies, and 18.5 us may be a hidden fifth benchmark.","check":"Read the table label and compare every benchmark dictionary in the immutable task.","outcome":"The values are explicitly theoretical and all four benchmark dictionaries are enumerated; no fifth shape or measured result exists in the source."} + }, + { + "claim_id": "kernel-gated-dual-gemm-current-c005", + "claim": "The page correctly treats shared-input reuse and fused epilogue traffic reduction as opportunities, not guarantees, rejects unsupported leaderboard/TMEM/TMA/CUTLASS/bottleneck details, and limits compatibility to the complete dual-NVFP4 contract rather than arbitrary dual-output or grouped-MoE operations.", + "claim_type": "compatibility", + "covers": ["u017", "u018", "u019"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml","revision":"gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c","locator":"complete task description and benchmark contract","observation":"A/SFA are logically shared and the operation has two same-shaped NVFP4 branches, but the task contains none of the removed implementation, leaderboard, launch-count, grouping, or bottleneck assertions."},{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/reference.py","revision":"same commit","locator":"ref_kernel and generate_input","observation":"The exact compatibility surface includes branch order, NVFP4 operands, distinct scale tensors/layouts, batch dimension, FP32 reference products, and FP16 output; the source is not grouped per expert."}], + "challenge": {"alternative":"Logical sharing forces one physical A load and dual TMEM accumulators, or any gated MLP can be adapted without additional evidence.","check":"Distinguish observable tensor semantics from internal scheduling and enumerate all dtype/layout/grouping preconditions.","outcome":"Several conforming decompositions remain possible and the ABI has material format/layout constraints; the page's scoped advice is the strongest supportable conclusion."} + }, + { + "claim_id": "kernel-gated-dual-gemm-current-c006", + "claim": "The local full bundle contains a byte-identical official task.yml from the pinned challenge-opening commit, is labeled as a specification rather than an optimized submission, and no longer presents unrelated blog/vLLM material as the contest implementation.", + "claim_type": "code", + "covers": ["u020"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/kernels/gated-dual-gemm/full/task.yml","revision":"gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c; SHA-256 2bf4e5c04cc8a9c283e519437c8324ca9265d9c12d0dfe2b76dc9f0542ada051","locator":"entire file","observation":"The local specification is byte-identical to the upstream task.yml blob at the pinned commit."},{"path":"artifacts/kernels/gated-dual-gemm/full/PROVENANCE.yaml","revision":"current corrected provenance","locator":"origin, upstream_sha, asset_mode, and file role","observation":"Provenance pins the full commit and labels the sole file an official problem specification; the former unrelated files are absent from the bundle."}], + "challenge": {"alternative":"A task specification plus directory name can still be mistaken for a winning/full optimized kernel.","check":"Inspect the page disclosure, provenance role, and actual bundle contents.","outcome":"All three explicitly call it a problem specification, and no optimized-submission file is present."} + }, + { + "claim_id": "kernel-gated-dual-gemm-current-c007", + "claim": "The derived standard-library variant performs both shared-A products, applies SiLU to the first branch, passes compact-versus-expanded equivalence, and rejects gating the second branch while making no NVFP4/GPU/performance claim.", + "claim_type": "code", + "covers": ["u021"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python artifacts/kernels/gated-dual-gemm/variants/01-gated-dual-gemm-semantics.py","environment":"KernelWiki worktree; conda base standard-library Python","reference":"gpu-mode/reference-kernels ref_kernel at c5b2f7c","result":"Exited 0 and printed 'gated dual GEMM semantic reference: PASS'; file SHA-256 ac13383732fc5c946ae1698cc0776131dd399c507736b28a1a7d4f45a976daeb.","negative_control":"The self-test computes SiLU(up)*gate and asserts that it differs from the required SiLU(gate)*up result."}], + "challenge": {"alternative":"The test duplicates the same branch-order mistake in both implementations or accidentally exercises symmetric data.","check":"Inspect the separately formed wrong_branch path and run it on asymmetric B1/B2 matrices.","outcome":"Compact and expanded correct forms match, while the explicit swapped-gate alternative differs, discriminating branch order."} + }, + { + "claim_id": "kernel-gated-dual-gemm-current-c008", + "claim": "The four primary-source links resolve to the official rules and the exact pinned task, starter, and reference files.", + "claim_type": "other", + "covers": ["u022", "u023", "u024", "u025"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"curl -L -sS -o /dev/null -w '%{http_code}' ","environment":"KernelWiki host, 2026-08-08","reference":"HTTP success and source identity/revision","result":"All four targets returned HTTP 200 and the three repository paths retain commit c5b2f7c062d5015f29c3a1043cfd04954397944c.","negative_control":"A nonexistent file under the same pinned GitHub problem path returned HTTP 404."}], + "challenge": {"alternative":"A source link is mutable, missing, or points to a neighboring problem/revision.","check":"Follow redirects and inspect problem path plus commit in every repository URL, with a same-path invalid control.","outcome":"Each link resolves to the intended record and all code links are immutable; the invalid control is rejected."} + }, + { + "claim_id": "kernel-gated-dual-gemm-current-c009", + "claim": "The documented get_page command returns the corrected page and both labeled artifact bundles, while an unknown page ID fails distinctly.", + "claim_type": "code", + "covers": ["u026", "u027"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python scripts/get_page.py kernel-gated-dual-gemm --include-code","environment":"KernelWiki worktree; conda base","reference":"current page and corrected provenance","result":"Exited 0; output contains 'Verified scope', the official task.yml disclosure, and the derived semantic reference PASS marker.","negative_control":"kernel-gated-dual-gemm-broken exited nonzero with 'No page found'."}], + "challenge": {"alternative":"The query resolves prose but omits the corrected full or derived artifact.","check":"Search positive output for content unique to each bundle and compare an invalid ID.","outcome":"Both artifact types appear with corrected scope and the unknown ID is rejected."} + } + ], + "commands": [ + "inspect official NVIDIA challenge rules and gpu-mode/reference-kernels commit c5b2f7c task/template/reference", + "compare the local task.yml byte-for-byte with its upstream blob", + "run the standard-library semantic reference and wrong-branch negative control under conda base", + "run four-link and get_page positive/negative controls" + ], + "residual_risk": "No B200, CUDA toolkit, or immutable public winning submission is available, so optimized numeric behavior, launch topology, and measured performance were not reproduced. The upstream FNUZ/FN scale-label inconsistency is disclosed rather than resolved editorially.", + "scope": "All 27 current coverage units for body hash 11f0038a8e9f681dac4ffd25654362cefa9f94a039e46745ac55f9488a405f80 were checked against official rules, immutable upstream source, byte comparison, and discriminating host tests." +} diff --git a/verification/reviewed/current/kernel-grouped-gemm.json b/verification/reviewed/current/kernel-grouped-gemm.json new file mode 100644 index 000000000..a84ddfdf9 --- /dev/null +++ b/verification/reviewed/current/kernel-grouped-gemm.json @@ -0,0 +1,106 @@ +{ + "path": "wiki/kernels/grouped-gemm.md", + "id": "kernel-grouped-gemm", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "kernel-grouped-gemm-current-c001", + "claim": "Official rules identify NVFP4 Grouped GEMM as Kernel Challenge 4 for January 17 through February 13, 2026 with 40% grand-prize weight; the corrected public task targets B200 and permits independent per-group M/N/K, while DeepGEMM separately defines fixed-N/K M-grouping and K-grouped backward.", + "claim_type": "compatibility", + "covers": ["u001", "u002", "u003", "u004", "u005"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf","revision":"official NVIDIA rules fetched 2026-08-08","locator":"Section 3 Kernel #4 and Grand Prize Judging Criteria","observation":"The rules provide the number, name, dates, and 40% weight."},{"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/task.yml","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"description, tests, and benchmarks","observation":"The task targets B200 and its correctness cases vary M, N, and K within group lists; the submission is a Python entry point with no one-launch rule."},{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"grouped GEMM sections","observation":"DeepGEMM explicitly fixes N/K for M-grouping and assigns K-grouping with fixed M/N to MoE weight backward."}], + "challenge": {"alternative":"All grouped GEMM APIs mean only M varies, or the contest's varied N/K values are documentation noise; 'kernel' requires one physical launch.","check":"Compare every official correctness dictionary and the unconstrained custom Python entry with DeepGEMM's explicit dimensional rules.","outcome":"Multiple required cases vary N/K, DeepGEMM expressly describes a narrower API, and neither interface constrains the number of launches."} + }, + { + "claim_id": "kernel-grouped-gemm-current-c002", + "claim": "The page's per-group packed-NVFP4 operands, FP16 output, logical scale shapes, four-list submission object, reordered scales, divisibility/tolerance rules, per-group reference semantics, and disclosed E4M3FNUZ-versus-E4M3FN inconsistency match the pinned GPU Mode task/template/reference.", + "claim_type": "compatibility", + "covers": ["u006", "u007", "u008", "u009", "u010", "u011", "u012", "u013", "u014"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/task.yml","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"description and constraints","observation":"The task publishes packed A/B, FP16 C, logical scale shapes, per-group sizes, K/256 and M/N tile divisibility, and E4M3FNUZ labels."},{"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/template.py","revision":"same commit","locator":"custom_kernel tuple documentation","observation":"The starter exposes logical and reordered scales as separate lists and states that each group can have different m/n/k/l, with l=1."},{"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/reference.py","revision":"same commit","locator":"ref_kernel, generate_input, and check_implementation","observation":"The reference processes each group with B transposed, constructs E4M3FN scales despite the prose label, returns four lists, writes FP16 results, and uses rtol=atol=1e-3."}], + "challenge": {"alternative":"The physical FP4 shape is [M,K,L], reordered scales are internal rather than submitted, all groups share dimensions, or FNUZ/FN is just one normalized dtype.","check":"Trace generated tensor storage, task.py/template tuple arity, correctness cases, and the actual dtype constructor.","outcome":"Packed FP4 uses K/2 bytes, the public object has four members, cases vary dimensions, and the primary files genuinely disagree on the dtype suffix; the page preserves each distinction."} + }, + { + "claim_id": "kernel-grouped-gemm-current-c009", + "claim": "The CPU grouped_reference code computes A_i times B_i-transpose independently for non-empty ordinary-number groups with different M/N/K, rejects ragged or incompatible per-group K, and explicitly makes no NVFP4, scaling, GPU, or performance claim.", + "claim_type": "code", + "covers": ["u015", "u016", "u017"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python -c ''","environment":"KernelWiki host; conda base standard-library Python","reference":"Per-group C_i=A_i@B_i.T semantics from gpu-mode/reference-kernels reference.py at ae67948685dfccf54ae8374dc9402addb7aae4f6","result":"Exited 0 and printed 'grouped independent-shape semantic reference: PASS; shared-K control rejected'; outputs were [[[17],[39]],[[1,5]]].","negative_control":"A deliberately wrong wrapper requiring every group to share the first group's K rejected the valid K=3 second group."}], + "challenge": {"alternative":"The nested loops accidentally compute A@B, reuse one group's dimensions, or imply that this is an NVFP4 implementation.","check":"Use non-square B row counts and different K across groups, compare exact expected products, and inspect the adjacent scope disclaimer.","outcome":"The asymmetric outputs require dotting each A row with each stored B row independently, the shared-K control fails, and the prose excludes packing/scales/GPU/performance."} + }, + { + "claim_id": "kernel-grouped-gemm-current-c003", + "claim": "The DeepGEMM table correctly records representative pinned M-contiguous, M-masked, and K-contiguous tensor shapes, group descriptors, valid-row behavior, fixed dimensions, alignment boundary, and documented MoE phase roles.", + "claim_type": "compatibility", + "covers": ["u018", "u019", "u020", "u021", "u022"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"Grouped GEMMs contiguous and masked sections","observation":"The README fixes N/K for M grouping, requires aligned expert segments, maps contiguous/masked/K-grouped modes to prefill/decode/weight backward, and says masked mode computes valid portions."},{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp","revision":"same commit","locator":"m_grouped_fp8_fp4_gemm_* and k_grouped_fp8_gemm_* functions","observation":"API assertions establish A/B/D ranks, int grouped_layout or masked_m inputs, packed sum-K operands, grouped D/C outputs, architecture/dtype/layout checks, and fixed dimensions."},{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/tests/generators.py","revision":"same commit","locator":"three grouped generators","observation":"Generators construct per-row IDs with -1 padding or prefix-sum ends, one masked-M count per group, and K-packed operands with D/C shaped [G,M,N]."}], + "challenge": {"alternative":"grouped_layout is always an offsets array, masked_m is a boolean [G,Mmax] mask that computes padding, or K-grouping returns one ungrouped M-by-N result.","check":"Compare both grouped-layout modes and concrete generator ranks against the C++ API assertions.","outcome":"The pinned code distinguishes map/prefix modes, uses one int count per group, and allocates one M-by-N result per K group, rejecting the former invented structs."} + }, + { + "claim_id": "kernel-grouped-gemm-current-c004", + "claim": "The four workload rows, group counts, M lists, N/K/L values, theoretical microsecond values, geometric-mean ranking, and 1.5-GHz B200 speed-of-light interpretation exactly reproduce the public task and are not presented as contestant measurements.", + "claim_type": "performance", + "covers": ["u023", "u024", "u025", "u026", "u027"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/task.yml","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"speed-of-light table, benchmarks, ranking_by, and timeout","observation":"The immutable task lists exactly the four rows and 18.833/10.667/2.406/1.525-us values, labels the analysis theoretical at 1.5 GHz, and ranks benchmark results by geometric mean."}], + "challenge": {"alternative":"The values are measured winner/baseline times, one row has 15 groups, or N equals K.","check":"Compare the table label and every benchmark dictionary at the pinned revision.","outcome":"The source explicitly calls them speed-of-light estimates; group counts are 8/8/2/2 and each row's N and K are distinct."} + }, + { + "claim_id": "kernel-grouped-gemm-current-c005", + "claim": "The reward-hack account correctly scopes 11.191 microseconds as a temporary number-one exploit report that was scrubbed, records the cloned-correctness and 120-group/15-call timing mechanism, and identifies reference-kernels PR #104 rather than a FlashInfer-Bench/MLSys change as the documented harness response.", + "claim_type": "performance", + "covers": ["u028", "u029", "u030"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://www.gpumode.com/news/reward-hacking-nvfp4","revision":"GPU Mode post dated 2026-03-09, fetched 2026-08-08","locator":"opening account, TL;DR, Correctness Check, Gaming the Timing Loop, and lessons","observation":"The official post supplies the temporary rank/report/scrub status, real padded 8-group correctness calls, one merged 120-group launch over 15 objects, 14 cached returns, division by 15, and a link to PR #104 as the response; it makes no FlashInfer-Bench/MLSys attribution."}], + "challenge": {"alternative":"The number is a legitimate per-call record, the exploit skips arithmetic, or the official remediation was in FlashInfer-Bench.","check":"Trace the timed call sequence, post-timing correctness statement, scrub notice, and named fix in the official account.","outcome":"All arithmetic is merged into one amortized launch, the leaderboard entry is invalidated, and the only concrete response identified is reference-kernels PR #104."} + }, + { + "claim_id": "kernel-grouped-gemm-current-c006", + "claim": "The implementation boundary is accurate: the task mandates no CUTLASS/CLC/TMA/TMEM/schedule/launch topology; the removed CUTLASS/CUDA sketches were not valid instances; small dimensions can underfill a GEMM; CLC requires an async request/mbarrier/decode protocol distinct from atomicAdd; and CUDA 13.2.1 does not define a universal 128-byte TMA alignment rule.", + "claim_type": "compatibility", + "covers": ["u031", "u032", "u033", "u034"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/task.yml","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"complete contract","observation":"The task specifies tensors, tests, workloads, scoring, and timeout but none of the named implementation requirements."},{"url":"https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/gemm/kernel/gemm_grouped.h","revision":"NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3","locator":"GemmGrouped declaration","observation":"The real template signature contradicts the former seven-argument invented type."},{"url":"https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/media/docs/cpp/efficient_gemm.md","revision":"same commit","locator":"Threadblock-level GEMM","observation":"CUTLASS documents partial threadblock work for small M/N and insufficient grid parallelism when M/N are small and K is large."},{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html","revision":"CUDA Programming Guide 13.2","locator":"4.12 flow and 4.12.1.1 API details","observation":"CLC's documented sequence includes initial block work, shared response/mbarrier, async cancellation, wait, decode, and proxy ordering, not a global atomic tile counter."},{"url":"https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html","revision":"CUDA Driver API 13.2.1","locator":"cuTensorMapEncodeTiled constraints","observation":"The API specifies a 64-byte descriptor and generally 16-byte global address/strides, plus feature-specific 32-byte constraints, rather than one universal 128-byte requirement."}], + "challenge": {"alternative":"The schedule tag alone makes the old kernel valid, atomic work stealing is CLC, small M always implies one dominant bottleneck, or all TMA paths inherit an undocumented 128-byte rule.","check":"Compare the exact CUTLASS type, CLC protocol, scoped efficiency wording, and versioned tensor-map constraints.","outcome":"Each stronger alternative is contradicted; the page limits performance conclusions to possibilities and exact implementation measurements."} + }, + { + "claim_id": "kernel-grouped-gemm-current-c007", + "claim": "All ten primary-source links resolve to the intended official rules, pinned GPU Mode and DeepGEMM files, GPU Mode postmortem, versioned CUDA documents, and pinned CUTLASS guide.", + "claim_type": "other", + "covers": ["u035", "u036", "u037", "u038", "u039", "u040", "u041", "u042", "u043", "u044"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"curl -L -sS -o /dev/null -w '%{http_code}' ","environment":"KernelWiki host, 2026-08-08","reference":"HTTP success plus source identity and immutable repository revision","result":"All ten targets returned HTTP 200; GPU Mode, DeepGEMM, and CUTLASS repository paths retain the claimed full commit IDs.","negative_control":"A nonexistent file under the same pinned GPU Mode grouped-GEMM path returned HTTP 404."}], + "challenge": {"alternative":"One link is dead, mutable where it claims a pin, or points to a neighboring problem/project.","check":"Follow redirects, inspect host/path/commit for every target, and compare a same-path invalid control.","outcome":"Every positive target resolves to its intended source identity/revision while the invalid control fails distinctly."} + }, + { + "claim_id": "kernel-grouped-gemm-current-c008", + "claim": "The documented query returns the corrected grouped-GEMM page, while an unknown page ID fails distinctly.", + "claim_type": "code", + "covers": ["u045", "u046"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python scripts/get_page.py kernel-grouped-gemm","environment":"KernelWiki worktree; conda base","reference":"current page and generated source index","result":"Exited 0 and returned the corrected title, independent-shape contract, DeepGEMM masked_m table, scrubbed exploit boundary, and TMA correction.","negative_control":"kernel-grouped-gemm-broken exited nonzero with 'No page found'."}], + "challenge": {"alternative":"The command resolves stale indexed content or accepts arbitrary page IDs.","check":"Search output for content unique to the corrected page and compare a near-name invalid ID.","outcome":"Corrected unique text is present and the unknown ID is rejected."} + } + ], + "commands": [ + "inspect the pinned task/template/reference and DeepGEMM README/API/generators", + "inspect versioned CUTLASS, CUDA CLC, and tensor-map primary documentation", + "run the CPU grouped-reference positive case and wrong shared-K negative control under conda base", + "run ten-link HTTP positive controls and one same-path 404 control", + "run get_page positive and near-name negative controls under conda base" + ], + "residual_risk": "No B200 or CUDA toolkit is available, so optimized numerical behavior and performance were not reproduced. The page intentionally contains no measured valid contestant record or claimed implementation topology; retained numeric values are source-labeled theoretical estimates.", + "scope": "All 46 current coverage units for body hash 733a51f41033ffcc0ff6fc42c4813e72da9b342f2e17aa629cfbee80177927ce were checked against immutable upstream source, versioned vendor documentation, and discriminating host controls." +} diff --git a/verification/reviewed/current/kernel-nsa.json b/verification/reviewed/current/kernel-nsa.json new file mode 100644 index 000000000..bab247af0 --- /dev/null +++ b/verification/reviewed/current/kernel-nsa.json @@ -0,0 +1,107 @@ +{ + "path": "wiki/kernels/nsa.md", + "id": "kernel-nsa", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "kernel-nsa-current-c001", + "claim": "NSA is the ACL 2025 three-branch natively trainable design; the authors report comparable-or-better quality on their evaluated tasks, and the paper's efficiency implementation is evidenced on A100/SM80 rather than Hopper or Blackwell.", + "claim_type": "compatibility", + "covers": ["u001", "u002"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://aclanthology.org/2025.acl-long.1126/","revision":"ACL 2025 anthology record fetched 2026-08-08","locator":"publication metadata and abstract","observation":"The official proceedings record identifies the paper, venue, authors, natively trainable hardware-aligned scope, and source-reported quality result."},{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings PDF fetched 2026-08-08","locator":"Section 4 experimental setup","observation":"The efficiency study specifies eight A100 GPUs and provides no Hopper/Blackwell compatibility or performance result."}], + "challenge": {"alternative":"Algorithmic portability is enough to list SM90/SM100, or the quality result is a universal guarantee.","check":"Separate tested hardware and evaluated tasks from plausible portability and untested models.","outcome":"Only A100 is directly evidenced and quality is explicitly scoped to the paper's evaluations."} + }, + { + "claim_id": "kernel-nsa-current-c002", + "claim": "The page accurately gives NSA's learned gated sum, compression/selection/window roles, reuse of compression scores, and main experimental hyperparameters 32/16, 64 with 16 selected blocks, and 512.", + "claim_type": "mechanism", + "covers": ["u003", "u004", "u005", "u006", "u007", "u008"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings PDF fetched 2026-08-08","locator":"Section 3.1, Equations 5-14, and Section 4.1 setup","observation":"The paper defines three input-gated branch outputs, learned block compression with position encoding, selection scores derived from compression attention, a recent-token window, and the listed experimental block/stride/count/window values."}], + "challenge": {"alternative":"The branches are simply concatenated or averaged, selection independently scores all fine blocks, or the numeric settings are architectural constants.","check":"Trace the equations and distinguish experimental setup from mechanism definition.","outcome":"The output is a learned gated sum, selection explicitly reuses compression scores, and the values are reported configuration choices rather than invariants."} + }, + { + "claim_id": "kernel-nsa-current-c003", + "claim": "The KernelWiki-derived CPU function implements the stated gated branch-vector sum, rejects missing/misaligned gates, and its ungated control produces a distinct result; the removed sparse and window sketches fail discriminating normalization and coverage cases.", + "claim_type": "code", + "covers": ["u009", "u010", "u011", "u016"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python -c ''","environment":"KernelWiki host; conda base standard-library Python","reference":"current page's CPU semantic function and gate equation","result":"Exited 0 and printed 'NSA gated branch semantic reference: PASS; ungated and missing-gate controls differ'; gated output was [1.0,10.0] and raw sum was [9.0,26.0].","negative_control":"The same inputs without gates differed, and a missing gate raised ValueError."},{"command":"conda run -n base python -c ''","environment":"KernelWiki host; conda base standard-library Python","reference":"immutable baseline u005/u006 listings","result":"Former selected-attention update returned 4.0 instead of normalized 2.0 with the wrong log-partition; former window positions were [0,2] instead of [0,1,2,3].","negative_control":"Stable normalized softmax and unit-stride enumeration returned the mathematically complete reference results."}], + "challenge": {"alternative":"The reference accidentally hard-codes the example, or the former errors disappear on realistic tiles.","check":"Inspect the generic loops/validation and choose one-block/equal-score cases where later iterations cannot repair normalization or skipped tokens.","outcome":"The current function operates generically over branch count and width; the former counterexamples isolate structural errors independent of realistic tensor size."} + }, + { + "claim_id": "kernel-nsa-current-c004", + "claim": "The page accurately scopes the paper's Triton design to group-centric query loading, shared sparse indices, contiguous selected KV blocks, and grid parallelism for nearly constant query/output loops, while declining to invent an exact grid or published source listing.", + "claim_type": "mechanism", + "covers": ["u012", "u013", "u014", "u015", "u016"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings PDF fetched 2026-08-08","locator":"Section 3.3 Hardware-aligned Sparse Attention","observation":"The authors describe existing-kernel compatibility for compression/window, a specialized Triton selected-attention kernel, all query heads in a GQA group resident in SRAM, shared sparse indices, continuous KV-block loads, and query/output grid loops. They do not publish the removed code or the formerly asserted exact grid tuple."}], + "challenge": {"alternative":"The paper mandates a three-dimensional query-block/head/batch grid or individual-token sparse gathers.","check":"Compare the exact implementation paragraph with the stronger alternatives.","outcome":"Only loop placement and contiguous blocks are specified; the page now preserves that evidence boundary."} + }, + { + "claim_id": "kernel-nsa-current-c005", + "claim": "The efficiency table correctly separates 9.0x/6.0x Figure 5 timing reports from Table 4's 11.6x expected memory-volume ratio, records the A100/GQA/dimension setup, lists missing reproducibility fields, and leaves structured performance metadata empty.", + "claim_type": "performance", + "covers": ["u017", "u018", "u019", "u020"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings PDF fetched 2026-08-08","locator":"Section 4.2, Figure 5, and Table 4","observation":"The paper reports 9.0x forward and 6.0x backward at 64K against Triton FlashAttention-2 on the stated A100 setup, and labels 11.6x expected based on 65,536 versus 5,632 memory-access tokens. The setup lists g=4, h=16, dk=192, dv=128 but omits the page-listed reproducibility fields."}], + "challenge": {"alternative":"11.6x is a measured decode latency from the same benchmark, or dtype and software can be safely inferred from typical A100 usage.","check":"Read the table heading/derivation and search the efficiency setup for each missing field.","outcome":"The decode value is explicitly expected rather than timed, and the omitted fields cannot be inferred into a structured record."} + }, + { + "claim_id": "kernel-nsa-current-c006", + "claim": "DeepSeek-V3.2-Exp deploys the later DSA mechanism with an indexer selecting up to 2,048 token positions and directs its CUDA paths to DeepGEMM and FlashMLA; this observable graph differs from NSA's gated compression/selection/window architecture.", + "claim_type": "compatibility", + "covers": ["u021", "u022"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-deepseek-v32/README.md","revision":"DeepSeek-V3.2-Exp 87e509a2e5a100d221c97df52c6e8be7835f0057","locator":"DSA release and high-performance CUDA kernels","observation":"The first-party release names DSA and links indexer logits to DeepGEMM and sparse attention to FlashMLA."},{"path":"/tmp/kernelwiki-deepseek-v32/inference/config_671B_v3.2.json","revision":"same commit","locator":"index_topk","observation":"The released configuration sets index_topk to 2048."},{"path":"/tmp/kernelwiki-deepseek-v32/inference/model.py","revision":"same commit","locator":"Indexer.forward and MLA.forward","observation":"The model performs learned top-k token selection and constructs an attention mask, without NSA's compression and sliding-window branch outputs."},{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232","locator":"Sparse Attention Kernels","observation":"FlashMLA states that its sparse kernels power DSA in V3.2-Exp."}], + "challenge": {"alternative":"DSA is merely a production rename of NSA and can inherit all paper claims.","check":"Compare the first-party inference graph and configuration with NSA Equations 5-14.","outcome":"DSA has a separate learned indexer/top-k mask and lacks the two other NSA outputs, so equivalence is observably false."} + }, + { + "claim_id": "kernel-nsa-current-c007", + "claim": "The applicability section correctly requires native training/model compatibility, preserves the GQA sharing mechanism, rejects a universal 32K threshold, scopes compression/Triton costs, and makes CUDA Graph use conditional on profiling a CPU-bound repeated workflow.", + "claim_type": "compatibility", + "covers": ["u023", "u024", "u025", "u026", "u027", "u028"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings PDF fetched 2026-08-08","locator":"Sections 3-4, Limitations, and Appendix B","observation":"The paper motivates native training, defines GQA-group sharing and learned compression, evaluates lengths without declaring a 32K cutoff, and notes Triton abstraction overhead versus native CUDA without publishing a CPU-launch profile."},{"url":"https://docs.nvidia.com/dl-cuda-graph/troubleshooting/performance-issues.html","revision":"NVIDIA CUDA Graphs guide fetched 2026-08-08","locator":"Performance Issues","observation":"NVIDIA scopes primary graph benefit to CPU-bound work and warns that GPU-bound workflows may see minimal benefit or regress."}], + "challenge": {"alternative":"Any long-context checkpoint can adopt NSA post hoc, or CUDA Graphs are always beneficial for decode.","check":"Compare the paper's training dependency and vendor graph bottleneck conditions with those universal claims.","outcome":"Both depend on preconditions the page now states explicitly."} + }, + { + "claim_id": "kernel-nsa-current-c008", + "claim": "All six cited source links resolve to the intended ACL record/PDF, pinned DeepSeek and FlashMLA revisions, NVIDIA CUDA Graph guidance, and explicitly third-party PyTorch repository.", + "claim_type": "other", + "covers": ["u029", "u030", "u031", "u032", "u033", "u034"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"curl -L -sS -o /dev/null -w '%{http_code}' ","environment":"KernelWiki host, 2026-08-08","reference":"HTTP resolution, source identity, and immutable repository revisions","result":"All six targets returned HTTP 200; repository URLs retained their full claimed commits and the lucidrains project identity is explicitly third-party.","negative_control":"A nonexistent sibling ACL Anthology path returned HTTP 404."}], + "challenge": {"alternative":"A link is dead, mutable where it claims a pin, or the third-party repository is presented as official.","check":"Follow redirects, inspect host/path/commit/owner, and compare a same-host invalid control.","outcome":"Every positive target has the stated identity/scope and the negative control fails distinctly."} + }, + { + "claim_id": "kernel-nsa-current-c009", + "claim": "The documented query returns the corrected NSA page, while a near-name unknown page ID fails distinctly.", + "claim_type": "code", + "covers": ["u035"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python scripts/get_page.py kernel-nsa","environment":"KernelWiki worktree; conda base","reference":"current page and regenerated indexes","result":"Exited 0 and returned the corrected gated three-branch mechanism, source-scoped efficiency table, and NSA-versus-DSA distinction.","negative_control":"kernel-nsa-broken exited nonzero with 'No page found'."}], + "challenge": {"alternative":"The query resolves stale generated content or accepts arbitrary near-name IDs.","check":"Search the positive output for newly unique text and compare the invalid ID.","outcome":"Corrected unique sections are present and the unknown ID is rejected."} + } + ], + "commands": [ + "inspect ACL paper equations, kernel-design section, efficiency setup/results, limitations, and native-training appendix", + "inspect pinned DeepSeek-V3.2-Exp and FlashMLA first-party source for DSA identity and execution graph", + "run the exact CPU gated-branch reference with ungated and missing-gate controls under conda base", + "run counterexamples against the immutable baseline's two removed pseudo-kernels", + "run six-link HTTP positive controls and one same-host 404 control", + "regenerate indexes and run get_page positive and near-name negative controls" + ], + "residual_risk": "No A100/Blackwell GPU, CUDA toolkit, or official NSA implementation is available locally. Source-reported performance was not reproduced, so no structured performance tuple is retained and the page explicitly records missing benchmark fields.", + "scope": "All 35 current coverage units for body hash 70483995c616c5db093c78410e006ee58b4ba3b50d6a9c3d4df495d17ee83b8b were checked against the ACL paper, immutable first-party deployment source, version-pinned FlashMLA source, vendor guidance, and discriminating host controls." +} diff --git a/verification/reviewed/current/kernel-nvfp4-gemm.json b/verification/reviewed/current/kernel-nvfp4-gemm.json new file mode 100644 index 000000000..6d6a7d574 --- /dev/null +++ b/verification/reviewed/current/kernel-nvfp4-gemm.json @@ -0,0 +1,150 @@ +{ + "path": "wiki/kernels/nvfp4-gemm.md", + "id": "kernel-nvfp4-gemm", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "kernel-nvfp4-gemm-current-c001", + "claim": "Official records identify this as B200-targeted Kernel Challenge 2, open November 29 through December 19, 2025 and worth 20 percent; the public task defines interface, correctness, cases, and ranking but no canonical implementation, mandatory CUTLASS schedule, contestant technique, or shape-independent bottleneck.", + "claim_type": "other", + "covers": ["u001", "u002"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url":"https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf","revision":"official terms fetched 2026-08-08","locator":"contest period, Kernel 2, B200 evaluation, and grand-prize weighting","observation":"The rules establish the stated identity, dates, device, and 20 percent weight."}, + {"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"complete task definition","observation":"The file contains tensor prose, tests, benchmark rows, and geometric-mean ranking; it mandates no implementation. Its speed-of-light text takes the maximum of math and DRAM time per shape rather than declaring one universal bottleneck."} + ], + "challenge": {"alternative":"The task implicitly standardizes the former CUTLASS/TMA/TMEM kernel and declares GEMM compute-bound by category.","check":"Enumerate normative inputs, outputs, checks, cases, and scoring, and search the pinned task for implementation requirements.","outcome":"Only observable task behavior is normative; implementation choice and achieved bottleneck are absent and the model includes both math and memory."} + }, + { + "claim_id": "kernel-nvfp4-gemm-current-c002", + "claim": "The generic one-dimensional NVFP4 recipe uses signed E2M1 payloads, block-of-16 E4M3 local scales, and a per-tensor FP32 global scale, whereas MXFP4 uses block-of-32 UE8M0 local scaling; the contest ABI exposes no FP32 global-scale tensors.", + "claim_type": "semantics", + "covers": ["u003", "u004"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url":"https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html","revision":"Transformer Engine 2.13","locator":"E2M1 format, NVFP4 two-level scaling, and MXFP4 comparison","observation":"The vendor documentation establishes the payload values, two-level recipe, 16-value groups, and MXFP4 contrast."}, + {"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"custom_kernel input tuple","observation":"The task interface lists packed A/B, logical and reordered SFA/SFB, and C, with no global FP32 scale arguments."} + ], + "challenge": {"alternative":"The contest's scale tensors implicitly include separately supplied FP32 globals, or NVFP4 and MXFP4 share the same local scale representation.","check":"Compare the complete seven-tensor input with the version-pinned recipe and MXFP4 table.","outcome":"No global argument is present in the task, while NVFP4 E4M3 and MXFP4 UE8M0 are explicitly distinct recipe fields."} + }, + { + "claim_id": "kernel-nvfp4-gemm-current-c003", + "claim": "The table exactly describes the actual seven task tensors, packed and logical/reordered shapes, per-L A by B-transpose FP16 reference, tolerance, K divisibility, tile-dependent M/N constraint, and the task-description and scale-dtype inconsistencies.", + "claim_type": "semantics", + "covers": ["u005", "u006", "u007", "u008", "u009", "u010", "u011", "u012", "u013", "u014", "u015", "u016"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [ + {"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/task.py","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"input_t","observation":"The type bound contains seven tensors."}, + {"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/reference.py","revision":"same commit","locator":"ref_kernel, generate_input, create_scale_factor_tensors, return tuple, and check_implementation","observation":"The code constructs physical packed A/B, logical scale shapes, reordered 32-by-4 layouts, preallocated FP16 C, computes each L slice with B transposed, and uses 1e-3 tolerances. It creates torch.float8_e4m3fn scales."}, + {"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/template.py","revision":"same commit","locator":"custom_kernel docstring and tuple unpack","observation":"The template exposes the seven fields and labels scale types e4m3fnuz."}, + {"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/task.yml","revision":"same commit","locator":"description and tests","observation":"The prose abbreviates five members, states K divisible by 256 and tile-relative M/N divisibility, and lists ten tests consistent with the table."} + ], + "challenge": {"alternative":"The reordered scales are an editorial optimization not supplied to entrants, physical FP4 shape keeps K rather than K/2, or FNUZ and FN labels agree upstream.","check":"Trace generate_input's allocations, permutations, return tuple, template unpack, and dtype constructors at the pinned commit.","outcome":"All seven objects are passed; packed allocations use K/2 bytes; the logical/reordered dimensions follow directly from atom_m=(32,4), atom_k=4, and K/16 scales; the two dtype suffixes visibly differ."} + }, + { + "claim_id": "kernel-nvfp4-gemm-current-c004", + "claim": "The Python shape helper reproduces the official task storage dimensions for valid positive shapes, enforces K divisible by 256, and correctly demonstrates that the former scale-array divisibility assertion rejects nine of ten official tests.", + "claim_type": "code", + "covers": ["u017", "u018"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [ + {"command":"conda run -n base python -c ''","environment":"KernelWiki host; conda base standard-library Python","reference":"current code block and pinned task.yml/reference.py","result":"Exited 0: current shape equations matched packed/logical/reordered construction; all ten official shapes passed the K rule; nine failed the old (K/16) modulo 128 assertion.","negative_control":"K=255 raised ValueError as required."} + ], + "challenge": {"alternative":"The helper accidentally models logical FP4 dimensions as physical bytes, or its scale layout works only for K divisible by 2048.","check":"Compare each returned dimension with the allocations/permutation in reference.py and exercise K=256, 512, 1536, 2048, 2304, 7168, and 16384.","outcome":"A/B use K/2, logical scales K/16, reordered rest_k K/64, and every published K succeeds, including those rejected by the former rule."} + }, + { + "claim_id": "kernel-nvfp4-gemm-current-c005", + "claim": "The implementation boundary correctly distinguishes NVFP4 block16 UE4M3 tcgen05 semantics from MXFP4 UE8M0, identifies the pointer-array CUTLASS schedule and current collective API without contestant attribution, states the tensor-map alignment contract rather than a universal 128-byte rule, and separates TMEM organization from logical tile limits.", + "claim_type": "compatibility", + "covers": ["u019", "u020", "u021", "u022"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"block-scale valid combinations and TMEM operands","observation":"The ISA lists mxf4nvf4 block16 with UE4M3 and describes TMEM destinations/scale addresses."}, + {"url":"https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512","revision":"NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3","locator":"mx_float4_t and nv_float4_t","observation":"The exact types select UE8M0 for MXFP4 and UE4M3 for NVFP4."}, + {"url":"https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu","revision":"same commit","locator":"operand types and lines 120-164 collective/kernel construction","observation":"The official single-GEMM example uses nv_float4 types, collective builders, kernel::GemmUniversal, and device::GemmUniversalAdapter."}, + {"url":"https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu","revision":"same commit","locator":"MMA1SMConfig and MMA2SMConfig","observation":"The named pointer-array schedule appears in the grouped example, which also defines a cooperative 256-by-256-by-256 logical MMA tile."}, + {"url":"https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html","revision":"CUDA Driver API 13.2.1","locator":"cuTensorMapEncodeTiled restrictions","observation":"General tiled-map alignment is 16 bytes for the base and byte strides, with mode-specific additional constraints; no universal 128-byte operand rule appears."}, + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"TMEM organization and allocation","observation":"The 128-lane by 512-column model describes tensor-memory addressing and allocation rather than a logical matrix-shape ceiling."} + ], + "challenge": {"alternative":"NVFP4 hardware consumes UE8M0 after a hidden conversion, the named schedule is the contest winner's single-GEMM kernel, TMA always needs 128 bytes, or TMEM hard-caps M at 128.","check":"Resolve exact scale typedefs, official schedule call site, Driver API restrictions, and the official two-SM tile configuration.","outcome":"Each alternative is contradicted by a direct type, API, or example locus; no contestant implementation source exists in the public task/leaderboard."} + }, + { + "claim_id": "kernel-nvfp4-gemm-current-c006", + "claim": "The three rows are the task's exact B200 1.5-GHz speed-of-light values for the listed M/N/K/L cases, ranking uses their geometric mean, and the values are theoretical max-of-math-and-DRAM comparisons rather than contestant or cuBLAS measurements.", + "claim_type": "performance", + "covers": ["u023", "u024", "u025", "u026", "u027"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"speed-of-light table, benchmarks, and ranking_by","observation":"The exact three shape/value rows are labeled speed-of-light analysis based on max FP4 Tensor Core math and DRAM throughput at 1.5 GHz, and ranking_by is geom. No measured contestant or cuBLAS row is present."} + ], + "challenge": {"alternative":"The values are observed contestant latencies or one cuBLAS benchmark summarized as a single score.","check":"Read the table label and compare rows with the benchmarks and ranking field.","outcome":"They are explicitly theoretical per-shape rows; the ranking is a later geometric aggregation, and no cuBLAS field exists."} + }, + { + "claim_id": "kernel-nvfp4-gemm-current-c007", + "claim": "The August 8, 2026 public leaderboard snapshot and score-times-one-million transformation reproduce ranks, users, scores, and timestamps for current ranks 1-3 and 8-10; the first three postdate the official cutoff, so the page correctly withholds winner, code, CUTLASS, cuBLAS, trials, and variance attributions.", + "claim_type": "performance", + "covers": ["u028", "u029", "u030", "u031", "u032", "u033", "u034", "u035"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url":"https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12","revision":"fetched 2026-08-08","locator":"LeaderboardRankedEntry rows 1-10","observation":"All displayed identities, ranks, floating-point scores, filenames, and UTC timestamps match after multiplying submission_score by one million. The response contains no source, method, comparison baseline, trials, or variance fields."}, + {"url":"https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf","revision":"official rules fetched 2026-08-08","locator":"contest period end","observation":"The rules end the contest on December 19, while the snapshot's current ranks 1-3 are timestamped December 20-21."} + ], + "challenge": {"alternative":"The current endpoint is automatically the immutable prize cutoff table, and participant techniques can be inferred from filenames.","check":"Compare row timestamps with the official deadline and inspect the complete API response schema/fields.","outcome":"Post-cutoff rows preclude that inference, and filenames/scores contain no implementation evidence."} + }, + { + "claim_id": "kernel-nvfp4-gemm-current-c008", + "claim": "Applicability is correctly limited to the exact packed payload, scale layouts, transpose/output/tolerance contract and B200 target; generic four-bit formats are not interchangeable, the native path is Blackwell-specific, software emulation is not excluded, and optimization choices require exact constraint checks and profiling.", + "claim_type": "compatibility", + "covers": ["u036", "u037"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"input and output contract","observation":"The implementation must accept the exact seven tensors and return the supplied FP16 output."}, + {"url":"https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html","revision":"Transformer Engine 2.13","locator":"NVFP4 recipe and MXFP4 comparison","observation":"NVFP4 and MXFP4 differ in scale representation and group size; arbitrary INT4 or FP4 data is not the same contract."}, + {"url":"https://developer.nvidia.com/blog/introducing-nvfp4-for-efficient-and-accurate-low-precision-inference/","revision":"NVIDIA technical blog fetched 2026-08-08","locator":"Blackwell fifth-generation Tensor Core discussion","observation":"Native NVFP4 compute is attributed to Blackwell hardware."}, + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"PTX ISA 9.0","locator":"target notes","observation":"The native block-scaled tcgen05 forms are architecture-specific; the ISA statement does not prohibit software conversion elsewhere."} + ], + "challenge": {"alternative":"Any 4-bit tensor can be reinterpreted losslessly, Hopper has the same native path under another mnemonic, or a particular scheduling choice is universally best.","check":"Compare payload/scale semantics, target tables, and normative task fields, then look for a source-backed universal schedule result.","outcome":"The format and target contracts differ, and no source provides a universal optimization ranking; the page states conditional boundaries only."} + }, + { + "claim_id": "kernel-nvfp4-gemm-current-c009", + "claim": "All twelve primary-source links resolve to their intended official rules, pinned task files, public leaderboard, versioned NVIDIA documents, pinned CUTLASS examples, and tensor-map reference; the 'Query via' label correctly introduces the local retrieval command.", + "claim_type": "other", + "covers": ["u038", "u039", "u040", "u041", "u042", "u043", "u044", "u045", "u046", "u047", "u048", "u049", "u050"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [ + {"command":"curl -L -sS -o /dev/null -w '%{http_code}' ","environment":"KernelWiki host, 2026-08-08","reference":"current Primary sources list and final target identities","result":"Every URL returned HTTP 200 and final content matched the named rules, file, API, documentation section, or example.","negative_control":"A nonexistent sibling file in the same pinned nvfp4_gemm GitHub tree returned HTTP 404."} + ], + "challenge": {"alternative":"A positive response is a soft 404, mutable unrelated page, or wrong commit path.","check":"Inspect final identities and exact commit strings, and compare the same-path invalid control.","outcome":"Pinned paths contain the named source files, vendor documents have the named sections, the API returns ranked JSON, and the invalid sibling is rejected."} + }, + { + "claim_id": "kernel-nvfp4-gemm-current-c010", + "claim": "The displayed conda-base get_page command retrieves the current page without the detached artifact bundle.", + "claim_type": "code", + "covers": ["u051"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [ + {"command":"conda run -n base python scripts/get_page.py kernel-nvfp4-gemm","environment":"KernelWiki repository; conda base, 2026-08-08","reference":"current query block and absence of artifact_dir","result":"Exited 0, emitted the current page, and contained no full/ artifact attachment markers or PR-2139 patch name.","negative_control":"The query with a nonexistent page ID exited nonzero."} + ], + "challenge": {"alternative":"Removing --include-code leaves the command invalid or artifacts still attach implicitly.","check":"Run the exact command and search output for former artifact filenames/section markers.","outcome":"The page resolves and the detached bundle is absent."} + } + ], + "commands": [ + "re-run the current CPU-only task shape helper and invalid-K control under conda base", + "fetch the public leaderboard and compare all six displayed rows field by field", + "resolve all twelve primary links with a pinned same-tree 404 control", + "run the current get_page command and verify detached artifact names do not appear", + "inspect the official task and CUTLASS/NVIDIA source loci used for every current mechanism and compatibility statement" + ], + "residual_risk": "No NVIDIA GPU or CUDA toolkit is available locally, so no GPU implementation or contestant performance was reproduced. The leaderboard is explicitly a dated current snapshot, not a cutoff or prize oracle; no contestant source is public through that endpoint.", + "scope": "All 51 current units for body hash c5bd73b9a4501849ca7bc91e4a94a7de62cd1bf395495ceb820d6ca9c8465a37 were checked against immutable task/CUTLASS revisions, version-pinned NVIDIA documentation, the dated public API response, and discriminating host controls." +} diff --git a/verification/reviewed/current/kernel-nvfp4-gemv.json b/verification/reviewed/current/kernel-nvfp4-gemv.json new file mode 100644 index 000000000..6432eddd3 --- /dev/null +++ b/verification/reviewed/current/kernel-nvfp4-gemv.json @@ -0,0 +1,104 @@ +{ + "path": "wiki/kernels/nvfp4-gemv.md", + "id": "kernel-nvfp4-gemv", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "kernel-nvfp4-gemv-current-c001", + "claim": "Problem 1 is a B200 block-scaled NVFP4 batched GEMV whose three published theoretical rows are DRAM-limited under the task model, while that task-specific result does not classify every GEMV implementation or shape.", + "claim_type": "performance", + "covers": ["u001"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/task.yml","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"description and speed-of-light analysis","observation":"The task names the B200 GEMV operation and uses max(FFMA, DRAM). An 8-TB/s payload calculation matched all three published rows within five percent, while a deliberately broken 16-TB/s control differed by more than 45 percent."},{"path":"/tmp/kernelwiki-amandeep-nvfp4.html","revision":"author post fetched 2026-08-08","locator":"Why 3x Off and takeaways","observation":"The author identifies this task and attempt-7 kernel as memory-bound while documenting implementation-dependent failed optimizations."}], + "challenge": {"alternative":"The theoretical rows are FFMA-limited or the task proves a datatype-wide bottleneck theorem.","check":"Compare the rows against the 8-TB/s byte model and retain only task-specific wording.","outcome":"The DRAM model reproduces the rows closely, while neither source supports extrapolation to every implementation."} + }, + { + "claim_id": "kernel-nvfp4-gemv-current-c002", + "claim": "At commit ae679486 the actual callable receives the seven displayed physical tensors, pads logical N=1 B/SFB to 128 rows, uses only output column zero, requires K divisible by 64, constructs E4M3FN scales despite E4M3FNUZ prose, and exposes no FP32 global-scale arguments.", + "claim_type": "code", + "covers": ["u002", "u003", "u004", "u005", "u006", "u007", "u008", "u009", "u010", "u011"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/task.py","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"input_t","observation":"The type is a seven-tensor tuple."},{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/reference.py","revision":"same commit","locator":"generate_input, create_scale_factor_tensors, ref_kernel, and return tuple","observation":"The exact physical payload, logical-scale, reordered-scale, and output shapes match; B is padded to 128, only res[:,0] is stored, scale dtype is float8_e4m3fn, and no global-scale scalar is returned."},{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/task.yml","revision":"same commit","locator":"description and K rule","observation":"The logical prose tuple, e4m3fnuz label, K-divisible-by-64 condition, and implementation-selected M-tile condition are explicit. A conda-base shape reconstruction matched all seven largest-row shapes, accepted K=64, and rejected K=63."}], + "challenge": {"alternative":"The extra two values are FP32 global scales or physical B really has one row.","check":"Trace every value returned by generate_input and every value unpacked by ref_kernel.","outcome":"The fifth and sixth inputs are reordered E4M3 scale tensors, and B/SFB are allocated with 128 rows; no scalar globals occur."} + }, + { + "claim_id": "kernel-nvfp4-gemv-current-c003", + "claim": "The task ranks the geometric mean of the three exact theoretical rows at a 1.5 GHz B200 clock; NVIDIA documents 8 TB/s HBM3e for one GB200 GPU, and the table values are model outputs rather than measured kernel results.", + "claim_type": "performance", + "covers": ["u012", "u013", "u014", "u015", "u016"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml","revision":"ae679486","locator":"speed-of-light table and ranking_by","observation":"The three shapes/times, 1.5 GHz assumption, and geometric-mean ranking match exactly."},{"url":"https://resources.nvidia.com/en-us-blackwell-architecture/blackwell-architecture-technical-brief","revision":"official brief fetched 2026-08-08","locator":"GB200 GPU memory subsystem","observation":"The brief specifies 8 TB/s HBM3e for one GPU."}], + "challenge": {"alternative":"The times are contestant measurements or 8 TB/s is a two-GPU aggregate.","check":"Read the task's speed-of-light label and the brief's one-GPU column.","outcome":"The task labels a theoretical analysis and the vendor scope is one GPU."} + }, + { + "claim_id": "kernel-nvfp4-gemv-current-c004", + "claim": "The dated public snapshot reports aggregate geometric-mean scores with the stated top-three and Yue rows; its ranks are mutable, and post-cutoff timestamps on current ranks 2-3 prevent treating it as an official prize-placement snapshot.", + "claim_type": "performance", + "covers": ["u017"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemv/NVIDIA?limit=100","revision":"snapshot fetched 2026-08-08","locator":"ranks 1-3 and 11 with submission_score and submission_time","observation":"Scores and names match the page; ranks 2-3 have November 30 UTC timestamps and Yue is rank 11 at 22.392217755."},{"url":"https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf","revision":"official rules fetched 2026-08-08","locator":"Kernel 1 end time","observation":"The official deadline is November 28 at 11:59 p.m. PT, before the current rank-2 and rank-3 timestamps."}], + "challenge": {"alternative":"The API freezes cutoff placements or every current top row predates the deadline.","check":"Compare its row timestamps with the exact rules deadline.","outcome":"At least ranks 2-3 postdate the deadline, so the page correctly avoids winner language."} + }, + { + "claim_id": "kernel-nvfp4-gemv-current-c005", + "claim": "PTX ISA 9.0 defines the displayed packed E2M1 conversion, typed b32-to-four-byte move, and v2/v4 u64 load widths; Yue used the conversion/move in an author-reported path, while the ISA makes no universal performance promise.", + "claim_type": "code", + "covers": ["u018", "u019", "u020"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"packed alternate-format cvt","observation":"The exact cvt.rn.f16x2.e2m1x2 form is documented."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov","revision":"same archive","locator":"scalar-to-vector mov","observation":"A b32 source may be decomposed into four byte destinations with matching total size."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld","revision":"same archive","locator":"vector loads","observation":"v2.u64 and v4.u64 move 128 and 256 bits; the ISA specifies semantics, not relative speed."},{"url":"https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html","revision":"author post fetched 2026-08-08","locator":"Optimization 3","observation":"The author shows mov.b32 and packed cvt in the reported vectorized PTX decode stage."}], + "challenge": {"alternative":"The suffix counts bytes, the move cannot target b8 lanes, or the author did not use these forms.","check":"Apply PTX vector/type-size rules and inspect the author's displayed asm.","outcome":"The widths and typed decomposition are exact, and both instructions appear in the author's code block; no speed theorem is inferred."} + }, + { + "claim_id": "kernel-nvfp4-gemv-current-c006", + "claim": "Amandeep reports the cache policies, wider PTX loads, and exact-K specializations observed in three solutions, while the author's own 64-bit wider-load and 80-to-64 register-cap experiments establish that these tuning effects are implementation-dependent.", + "claim_type": "performance", + "covers": ["u021"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"author post fetched 2026-08-08","locator":"attempts 9 and 11; What the Top Solutions Did Differently","observation":"The post reports A no-allocate, B evict-last, v2/v4 loads, exact-K templates, and register budgets as observations; it also reports the author's wider load 16-25 percent slower and register-cap change having zero effect."}], + "challenge": {"alternative":"The inspected code proves the techniques universally faster and the failed trials used identical mechanisms.","check":"Separate the author's retrospective observations from the controlled changes in the author's own kernel.","outcome":"The post itself presents shape/code-dependent successes and counterexamples, matching the page's conditional wording."} + }, + { + "claim_id": "kernel-nvfp4-gemv-current-c007", + "claim": "The ten-row table faithfully reports Yue's combined stage boundaries and author-reported timings, identifies 22.392 as an aggregate submitted score, and accurately notes that whole-B shared-memory loading did not improve one CuTe attempt.", + "claim_type": "performance", + "covers": ["u022", "u023", "u024", "u025", "u026", "u027", "u028", "u029", "u030", "u031", "u032", "u033"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html","revision":"author post fetched 2026-08-08","locator":"CuTe attempt and CUDA Optimizations 1-6","observation":"Every stage label, combined change, approximate timing, submitted score, and failed whole-B shared-memory idea matches the post."},{"url":"https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemv/NVIDIA?limit=100","revision":"snapshot fetched 2026-08-08","locator":"Yue row","observation":"The aggregate score is 22.392217755 microseconds."}], + "challenge": {"alternative":"The table isolates each listed technique or 22.392 is one benchmark row.","check":"Trace the author's headings and compare the endpoint's aggregate score field.","outcome":"The page explicitly labels combined changes and source reporting, and the endpoint/task identify a geometric-mean score."} + }, + { + "claim_id": "kernel-nvfp4-gemv-current-c008", + "claim": "The practical guidance correctly scopes use to matching N=1 NVFP4 operands, calls for profiling before bottleneck choices, describes the compiled-entry tradeoff of retained exact-K dispatch, requires version/target revalidation for PTX, records Transformer Engine 2.13 inference support as SM 10.0+, and notes that the task mandates results rather than implementation techniques.", + "claim_type": "compatibility", + "covers": ["u034", "u035", "u036", "u037", "u038"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml","revision":"ae679486","locator":"task contract and templates","observation":"The task fixes N=1 input/output semantics and checker criteria but does not require cache/load/register implementation choices."},{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"author post fetched 2026-08-08","locator":"attempts, exact-K specialization, and takeaways","observation":"The failed tuning attempts support profiling-dependent guidance, and launch-time exact-K dispatch entails retaining referenced specialization entries."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"versioning and target ISA notes","observation":"Instruction and target support are explicitly versioned."},{"url":"https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html","revision":"Transformer Engine 2.13","locator":"NVFP4 recipe and Supported devices","observation":"The exact recipe is required and inference support is listed as SM 10.0+."}], + "challenge": {"alternative":"Generic FP4 weights are compatible, lower resource settings are always better, PTX is permanently target-independent, or the contest prescribes a canonical kernel.","check":"Compare the exact recipe/task ABI, the author's counterexamples, archived PTX target notes, and task checker scope.","outcome":"Each alternative is excluded by the cited source; the page states conditional practice rather than performance guarantees."} + }, + { + "claim_id": "kernel-nvfp4-gemv-current-c009", + "claim": "All nine primary-source links resolve to the intended pinned task, rules, vendor documents, and author posts.", + "claim_type": "other", + "covers": ["u039", "u040", "u041", "u042", "u043", "u044", "u045", "u046"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"curl -L -A 'Mozilla/5.0' -sS -o /dev/null -w '%{http_code}' ","environment":"KernelWiki host, network fetched 2026-08-08","reference":"Each link should resolve to its named primary source.","result":"All nine linked targets returned HTTP 200 and their paths/titles matched the labels.","negative_control":"An invented same-site Simon article slug returned HTTP 404."}], + "challenge": {"alternative":"The sites return generic success for missing paths or a label points to an adjacent document.","check":"Inspect target paths/titles and compare a same-site missing-page control.","outcome":"The intended targets resolved while the fabricated slug was rejected."} + } + ], + "commands": [ + "conda run -n base python -c ''", + "conda run -n base python -c '<8-TB/s model and 16-TB/s negative control>'", + "curl current primary-source URLs with same-site 404 control" + ], + "scope": "All 46 current units for body hash 15a1de756e23c26b7938e4b0c5e12e918492d5f204e504a428fcb1573c5bbd26 were checked against gpu-mode/reference-kernels ae679486, official rules and NVIDIA versioned documentation, the public leaderboard snapshot fetched 2026-08-08, the author posts, and host controls.", + "residual_risk": "No B200 or CUDA toolkit is installed locally. Performance values remain explicitly author-reported or theoretical; no contestant implementation attribution is made from the leaderboard API, and no current rank is called an official prize placement." +} diff --git a/verification/reviewed/current/kernel-sparse-mla.json b/verification/reviewed/current/kernel-sparse-mla.json new file mode 100644 index 000000000..9d969fad9 --- /dev/null +++ b/verification/reviewed/current/kernel-sparse-mla.json @@ -0,0 +1,15 @@ +{ + "claims":[ + {"challenge":{"alternative":"DSA is ordinary dense MLA with a post-hoc mask, the indexer score is one dot product, or the complete path becomes O(k) per decode query.","check":"Compare equations 1-2, Figure 1, released dimensions, and the report's separate complexity statement.","outcome":"The paper defines a weighted multi-head ReLU score, top-k KV selection under MLA/MQA, and O(Lk) core attention while retaining an O(L squared) sequence-level indexer."},"claim":"The model section accurately defines DeepSeek-V3.2-Exp DSA's indexer and fine-grained selection, released 64x128 indexer/topk-2048 and 128-head/512+64 MLA dimensions, MQA sharing, and the separate O(Lk) core-attention versus O(L squared) indexer complexity.","claim_id":"kernel-sparse-mla-current-c001","claim_type":"mechanism","covers":["u001","u002","u003","u004","u005","u006","u007"],"evidence":[{"path":"/tmp/kernelwiki-deepseek-v32/DeepSeek_V3_2.pdf","revision":"DeepSeek-V3.2-Exp 87e509a2e5a100d221c97df52c6e8be7835f0057","locator":"Section 1 equations 1-2/Figure 1 and Section 3 Inference Costs","observation":"The mechanism, MLA/MQA instantiation, selected/core-attention complexity and retained indexer complexity match the page."},{"path":"/tmp/kernelwiki-deepseek-v32/inference/config_671B_v3.2.json","revision":"same commit","locator":"n_heads, kv_lora_rank, qk_rope_head_dim, index_n_heads, index_head_dim, index_topk","observation":"The exact released dimensions are 128, 512, 64, 64, 128 and 2048."}],"oracle":"local-primary-source","verdict":"verified"}, + {"challenge":{"alternative":"DeepGEMM fuses top-k and FlashMLA runs the indexer; sparse indices denote blocks and prefill/decode share one ABI.","check":"Trace the pinned indexer-logit output and both FlashMLA public signatures/reference gathers.","outcome":"DeepGEMM emits token logits, FlashMLA receives caller-produced indices, decode uses page+offset token encoding, and prefill has a separate unbatched BF16 contract."},"claim":"The implementation section correctly separates DeepGEMM indexer logits/top-k from FlashMLA attention and accurately records sparse decode and prefill tensor shapes, token-index encodings, invalid values, precision, outputs, page-size variability, and the SM100 precision boundary.","claim_id":"kernel-sparse-mla-current-c002","claim_type":"compatibility","covers":["u008","u009","u010","u011","u012","u013","u014","u015","u016"],"evidence":[{"path":"/tmp/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm100_fp8_mqa_logits.cuh","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"sm100_fp8_mqa_logits signature, UMMA, scales and logit store","observation":"The kernel consumes FP8 Q/K, scales and weights and emits logits; no top-k selector or FlashMLA attention is fused."},{"path":"/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py","revision":"FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232","locator":"flash_mla_with_kvcache and flash_mla_sparse_fwd","observation":"The two contracts, shapes, dtypes, invalid-index rules and outputs match the table; decode dequantizes FP8 storage for BF16 attention."},{"path":"/tmp/kernelwiki-flashmla-pinned/tests/test_flash_mla_sparse_decoding.py","revision":"same commit","locator":"correctness cases","observation":"Official cases include every listed page size, disproving a fixed-64 API."}],"oracle":"local-primary-source","verdict":"verified"}, + {"challenge":{"alternative":"656 bytes is a generic MLA or indexer cache and its four scales are per page/block.","check":"Inspect the mode-specific interface text, group boundaries, and released model cache declarations.","outcome":"The exact arithmetic and roles apply only to V3-family FP8 sparse decode; indexer K/scale caches are separate and the four scales cover successive 128-value groups within one token."},"claim":"The cache section correctly limits the 512+16+128=656-byte layout to V3-family FP8 sparse decode, identifies four per-128-value FP32 scales and attention RoPE, and distinguishes the separate indexer cache and other FlashMLA modes.","claim_id":"kernel-sparse-mla-current-c003","claim_type":"semantics","covers":["u017","u018","u019","u020","u021"],"evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py","revision":"FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232","locator":"V3-family FP8+sparse layout","observation":"The interface gives the exact element counts, bytes, scale grouping and mode restriction."},{"path":"/tmp/kernelwiki-deepseek-v32/inference/model.py","revision":"DeepSeek-V3.2-Exp 87e509a2","locator":"Indexer.k_cache/k_scale_cache and MLA kv_cache/pe_cache","observation":"The released reference declares separate indexer and attention caches."}],"oracle":"local-primary-source","verdict":"verified"}, + {"challenge":{"alternative":"The rows are reproducible matched benchmarks, use FP8 MMA throughout, or 1450 versus 1460 proves sparse/dense equivalence.","check":"Read every pinned README sentence, support row and API dtype, then check which benchmark fields are omitted.","outcome":"The page transcribes each environment/maximum/precision and caveat, labels them incomplete author reports, separates dense regimes, and rejects the unmatched cross-operator comparison."},"claim":"The performance section accurately transcribes and scopes FlashMLA's H800/B200 dense-decode and sparse prefill/decode maxima, storage-versus-MMA precision, B200 decode caveat, and NVIDIA dense-MHA values while explicitly withholding reproducible or matched-performance status.","claim_id":"kernel-sparse-mla-current-c004","claim_type":"performance","covers":["u022","u023","u024","u025","u026","u027","u028","u029"],"evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232","locator":"Performance, support matrix, and sparse API descriptions","observation":"All six sparse/dense rows, the separate regimes, BF16/FP8 distinction, CUDA environments, B200 caveat, and MHA attribution match; complete shapes/trials/variance are absent."}],"oracle":"local-primary-source","verdict":"verified"}, + {"challenge":{"alternative":"Architecture similarity or 32K context alone establishes compatibility and speedup, and kernel-only timing is sufficient.","check":"Apply the exact public tensor contracts and the paper's cost curve/retained indexer work to the checklist.","outcome":"The page requires contract equality, correctness against the model/reference, complete selector-plus-attention timing, a matched dense baseline, and an observed crossover rather than a fixed threshold."},"claim":"The evaluation procedure correctly gates compatibility on exact dimensions/layout/indices/semantics, checks selector and attention correctness including RoPE and invalid entries, and measures the entire pipeline with matched controls and no universal 32K crossover.","claim_id":"kernel-sparse-mla-current-c005","claim_type":"performance","covers":["u030","u031","u032","u033","u034","u035","u036","u037","u038"],"evidence":[{"path":"/tmp/kernelwiki-deepseek-v32/inference/model.py","revision":"DeepSeek-V3.2-Exp 87e509a2","locator":"Indexer.forward and MLA.forward","observation":"The model reference supplies score/top-k/mask behavior and the non-interleaved indexer versus interleaved MLA RoPE distinction."},{"path":"/tmp/kernelwiki-flashmla-pinned/tests/ref.py","revision":"FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232","locator":"prefill and decode references","observation":"The references define selected gather, attention outputs, LSE and invalid-entry behavior needed by the correctness step."},{"path":"/tmp/kernelwiki-deepseek-v32/DeepSeek_V3_2.pdf","revision":"same model commit","locator":"Inference Costs and Figure 3","observation":"The source measures deployed end-to-end costs over a range and defines no universal 32K threshold."}],"oracle":"local-primary-source","verdict":"verified"}, + {"challenge":{"alternative":"DSA is the deployment name for the ACL paper's NSA implementation.","check":"Compare DSA equations/model code with NSA's three branches and gates.","outcome":"DSA selects token positions with one learned indexer; NSA separately computes compressed, selected-block and sliding-window branches and combines them with gates."},"claim":"The page correctly distinguishes DeepSeek-V3.2-Exp DSA from ACL Native Sparse Attention and prevents transfer of NSA structure or A100 results.","claim_id":"kernel-sparse-mla-current-c006","claim_type":"compatibility","covers":["u039"],"evidence":[{"url":"https://github.com/deepseek-ai/DeepSeek-V3.2-Exp/blob/87e509a2e5a100d221c97df52c6e8be7835f0057/DeepSeek_V3_2.pdf","revision":"DeepSeek-V3.2-Exp 87e509a2","locator":"Section 1","observation":"DSA is a lightning indexer plus fine-grained token selection under MLA."},{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings","locator":"Sections 3.1-3.3 and 4","observation":"NSA has three gated branches and evaluates its efficiency implementation on A100."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"The links are mutable summaries or point to unrelated sparse-attention implementations.","check":"Resolve owners, full revisions and exact implementation/report paths.","outcome":"The first four links pin the DeepSeek report/model, FlashMLA repository/interface and DeepGEMM indexer source; the fifth is the official ACL proceedings paper used only for the boundary."},"claim":"The five primary references resolve to the exact first-party DSA model/report, pinned FlashMLA implementation/interface, pinned DeepGEMM indexer kernel, and official NSA paper used by the page.","claim_id":"kernel-sparse-mla-current-c007","claim_type":"semantics","covers":["u040","u041","u042","u043","u044"],"evidence":[{"url":"https://github.com/deepseek-ai/DeepSeek-V3.2-Exp/tree/87e509a2e5a100d221c97df52c6e8be7835f0057","revision":"87e509a2","locator":"tree","observation":"Pinned model/report source."},{"url":"https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232","revision":"71c7379","locator":"tree and interface","observation":"Pinned sparse-attention implementation."},{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_mqa_logits.cuh","revision":"891d57b4","locator":"file","observation":"Pinned indexer-logit kernel."},{"url":"https://aclanthology.org/2025.acl-long.1126/","revision":"ACL 2025","locator":"paper record","observation":"Official NSA paper."}],"oracle":"web-primary","verdict":"verified"} + ], + "commands":["Extract and inspect pinned DeepSeek-V3.2-Exp report/config/model","Inspect pinned FlashMLA interfaces/tests/references and source-reported performance","Inspect pinned DeepGEMM SM100 indexer-logit source"], + "id":"kernel-sparse-mla","page_verdict":"verified","path":"wiki/kernels/sparse-mla.md", + "residual_risk":"No CUDA toolkit or GPU is available, so source code was not built or profiled. Performance remains explicitly a transcription of incomplete author reports plus a required measurement protocol.", + "scope":"All 44 current units for hash 948b326db003b5db4472a9222cbd12a7ea8b28c706bb3c67abfc29cdc7846cca and the corrected Blackwell scope were challenged against pinned first-party model and kernel sources." +} diff --git a/verification/reviewed/current/kernel-tensorrt-llm-blackwell-indexer.json b/verification/reviewed/current/kernel-tensorrt-llm-blackwell-indexer.json new file mode 100644 index 000000000..7ef3855e1 --- /dev/null +++ b/verification/reviewed/current/kernel-tensorrt-llm-blackwell-indexer.json @@ -0,0 +1,186 @@ +{ + "claims": [ + { + "challenge": { + "alternative": "The FP4 change is one fused indexer kernel, gather/scatter also quantize or select top-k, or the code is a generic DSA ABI.", + "check": "Trace the pinned DSA call graph across fused preparation, cache operators, DeepGEMM logit dispatch, and top-k calls, then inspect each CUDA kernel's work.", + "outcome": "The source exposes distinct TensorRT-LLM-specific stages: fused FP4 concatenation/quantization, byte-copy cache movement, FP4 MQA logits, and later top-k selection." + }, + "claim": "The audited-scope section correctly limits the page to PR 13340 merge 897c4bff and accurately separates TensorRT-LLM's FP4 Q/K preparation, cache movement, MQA-logit, and top-k stages without claiming a generic ABI or fused quantized gather/scatter.", + "claim_id": "kernel-tensorrt-llm-blackwell-indexer-current-c001", + "claim_type": "mechanism", + "covers": ["u001", "u002", "u003", "u004", "u005", "u006", "u007"], + "evidence": [ + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/PROVENANCE.yaml", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "origin_url, upstream_sha, asset_mode, and file manifest", + "observation": "The verbatim bundle ties the audited CUDA and DSA files to PR 13340 and revision 897c4bff." + }, + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/fusedCatFp4.cu", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "fusedCatFp4Kernel", + "observation": "This stage concatenates and quantizes; it does not access the paged cache, calculate MQA logits, or select top-k." + }, + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/tensorrt_llm/_torch/attention_backend/sparse/dsa.py", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "_prep_q_or_k, _call_mqa_logits, _call_paged_mqa_logits, and sparse_attn_indexer", + "observation": "The TensorRT-LLM integration calls preparation, cache movement, FP4 logit dispatch, and prefill/decode top-k as distinct stages under its DSA metadata and cache-manager contracts." + } + ], + "oracle": "local-primary-source", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "The FP4 row uses one scale, arbitrary head dimension, conventional round-to-nearest boundaries, or a different nibble/byte order.", + "check": "Compare all constants, host assertions, scale construction, FP4 threshold table, nibble packing, and scale packing in the CUDA kernel and Torch wrapper.", + "outcome": "The implementation fixes 128 BF16 inputs, four groups of 32, 64 packed data bytes, four little-endian UE8M0 exponent bytes in one int32, strict-threshold E2M1 coding, and the documented wrapper constraints." + }, + "claim": "The FP4 contract section accurately states the 128-value input, 64-byte packed E2M1 payload, four-byte packed scale word, 68-byte cache footprint, wrapper constraints and outputs, scale formula, representable magnitudes, and nibble order.", + "claim_id": "kernel-tensorrt-llm-blackwell-indexer-current-c002", + "claim_type": "semantics", + "covers": ["u008", "u009", "u010", "u011", "u012", "u013", "u014"], + "evidence": [ + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/fusedCatFp4.cu", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "constants, quantizeFp4E2M1, fusedCatFp4Kernel stages 1-7, and invokeFusedCatFp4", + "observation": "The constants, reduction, power-of-two scale construction, FP4 table, low/high-nibble packing, little-endian scale packing, 128-head check, and stride restrictions match the page." + }, + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/thop/fusedCatFp4Op.cpp", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "fused_cat_fp4 validation and output allocation", + "observation": "The wrapper enforces CUDA/device/dtype/dimension/stride/alignment/row/head constraints and returns int8 [M,64] plus int32 [M,1]." + }, + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/tensorrt_llm/_torch/attention_backend/sparse/dsa.py", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "DSACacheManager cache-size calculation", + "observation": "At index_head_dim 128, the FP4 cache allocation uses 64 data bytes plus four scale bytes per token." + } + ], + "oracle": "local-primary-source", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "Gather/scatter require a contiguous cache, use one mapping, initialize invalid gather rows, or return numerically meaningful FP8/FP32 values in FP4 mode.", + "check": "Inspect cache shape/stride handling, separate mapping loads, negative-map branches, output allocation, view casts, and FP4 reinterpretation at the logit call.", + "outcome": "The kernels translate flat locations through four strides, consume separate payload/scale mappings, return before writes on either negative mapping, and transport raw bytes that FP4 code later views as int8/int32." + }, + "claim": "The gather/scatter section accurately specifies the strided four-dimensional cache, vector widths and FP4/FP8 byte counts, two int64 mapping arrays, whole-token negative-map skip, uninitialized skipped gather row, and historical typed-view boundary for FP4 data.", + "claim_id": "kernel-tensorrt-llm-blackwell-indexer-current-c003", + "claim_type": "semantics", + "covers": ["u015", "u016", "u017"], + "evidence": [ + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheGather.cu", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "flatIndexToMemoryOffset, indexerKCacheGatherUnifiedKernel, and invokeIndexerKCacheGather", + "observation": "Gather handles four-dimensional byte strides, copies four bytes per thread for 64- or 128-byte payloads plus a four-byte scale, and returns before writes when either mapping is negative." + }, + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheScatter.cu", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "indexerKCacheScatterUnifiedKernel and invokeIndexerKCacheScatter", + "observation": "Scatter implements the inverse byte movement with the same mapping, width, scale-size, and negative-map behavior." + }, + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/thop/IndexerKCacheGatherOp.cpp", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "output allocation and final view-casts", + "observation": "The wrapper allocates with empty, launches into uint8 buffers, then view-casts payload to float8 and scale to FP32 for the historical interface." + }, + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/tensorrt_llm/_torch/attention_backend/sparse/dsa.py", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "_call_mqa_logits FP4 branch", + "observation": "The FP4 call site reinterprets gathered payload as int8 and scales as int32 before dispatch." + } + ], + "oracle": "local-primary-source", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "Testing only round-trip cache bytes proves the complete indexer, kernel-only timing supports a speedup, or skipped empty rows are valid sentinel outputs.", + "check": "Map each procedure step to a distinct source contract and check whether the pinned PR supplies any benchmark record.", + "outcome": "Preparation, cache movement, logits, and top-k have separate correctness boundaries; skipped gather rows need explicit handling, complete performance needs stage and end-to-end timing, and neither the source card nor pinned implementation provides a result." + }, + "claim": "The procedure appropriately tests FP4 boundary/layout behavior, valid and invalid cache movement, paged/non-paged logits and selected indices, profiles each stage plus the complete indexer with reproducibility fields, and makes no unsupported performance claim.", + "claim_id": "kernel-tensorrt-llm-blackwell-indexer-current-c004", + "claim_type": "performance", + "covers": ["u018", "u019", "u020", "u021", "u022", "u023", "u024", "u025", "u026"], + "evidence": [ + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/fusedCatFp4.cu", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "quantizeFp4E2M1 and fusedCatFp4Kernel", + "observation": "The strict decision thresholds, saturation, row-stride loads, nibble packing, and scale packing are discriminating preparation tests." + }, + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheGather.cu", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "gather kernel and launch contract", + "observation": "Strided cache copies, both mapping arrays, exact payload/scale widths, and negative slots define the cache test cases." + }, + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/tensorrt_llm/_torch/attention_backend/sparse/dsa.py", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "non-paged/paged MQA logits and prefill/decode top-k dispatch", + "observation": "The call graph requires separate logit and selection checks and permits separate stage timing before complete-path timing." + }, + { + "path": "sources/prs/TensorRT-LLM/PR-13340.md", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "Summary, Description, and changed files", + "observation": "The source card describes the implementation change and records no performance measurement." + } + ], + "oracle": "local-primary-source", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "The links are mutable summaries, point outside the audited artifact, or fail to resolve to the implementation used for the claims.", + "check": "Resolve the PR link and each relative artifact path against the repository, then compare the manifest hashes and upstream paths.", + "outcome": "The official PR is the source-card origin, and all four relative links resolve to files listed as verbatim upstream assets for revision 897c4bff." + }, + "claim": "The primary-reference links resolve to the official TensorRT-LLM PR and the pinned verbatim fused preparation, gather, scatter, and DSA integration artifacts used by the page.", + "claim_id": "kernel-tensorrt-llm-blackwell-indexer-current-c005", + "claim_type": "semantics", + "covers": ["u027", "u028", "u029", "u030", "u031"], + "evidence": [ + { + "path": "sources/prs/TensorRT-LLM/PR-13340.md", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "url, merge_sha, and artifact_dir", + "observation": "The source card resolves the official PR and the repository's audited artifact directory." + }, + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/PROVENANCE.yaml", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "files manifest", + "observation": "The manifest lists fusedCatFp4.cu, both cache kernels, and dsa.py with upstream paths and SHA-256 hashes under verbatim mode." + } + ], + "oracle": "local-primary-source", + "verdict": "verified" + } + ], + "commands": [ + "Inspect the pinned PR provenance and source card", + "Inspect fusedCatFp4 CUDA and Torch wrapper contracts", + "Inspect gather/scatter CUDA and Torch wrapper contracts", + "Trace TensorRT-LLM DSA FP4 preparation, cache, logit, and top-k calls" + ], + "id": "kernel-tensorrt-llm-blackwell-indexer", + "page_verdict": "verified", + "path": "wiki/kernels/tensorrt-llm-blackwell-indexer.md", + "residual_risk": "No CUDA toolkit or GPU is available, so source semantics were not confirmed by build or execution. The page deliberately reports no performance result and specifies the GPU tests still required.", + "scope": "All 31 current units for hash 64b746e36ef7866e93cb1c9aca98703acc5008d485a0d9c96ab31f7a1504a964 and the corrected Blackwell frontmatter claim were challenged against the pinned PR's source card, provenance, CUDA kernels, wrappers, and DSA integration." +} diff --git a/verification/reviewed/current/lang-cuda-cpp.json b/verification/reviewed/current/lang-cuda-cpp.json new file mode 100644 index 000000000..47ee40b2c --- /dev/null +++ b/verification/reviewed/current/lang-cuda-cpp.json @@ -0,0 +1,90 @@ +{ + "path": "wiki/languages/cuda-cpp.md", + "id": "lang-cuda-cpp", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "lang-cuda-cpp-current-c001", + "claim": "The page faithfully scopes the tutorial's v6 result to M=N=K=4096 on the disclosed B200/PyTorch 2.9.1/CUDA 13 setup: 1475.93 TFLOP/s versus a 1506.74-TFLOP/s cuBLAS baseline is 97.96 percent and is not presented as portable performance.", + "claim_type": "performance", + "covers": ["u001"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/README.md","revision":"gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"benchmark environment and table","observation":"The author source states the shape, Modal B200, PyTorch 2.9.1/CUDA 13 environment, 1506.74 cuBLAS result, and 1475.93 v6 result; their ratio is 97.955 percent."}], + "challenge": {"alternative":"The percentage is against theoretical peak or is a general B200 result.","check":"Recompute the ratio using the adjacent author-table rows and inspect the stated environment.","outcome":"It rounds to 97.96 percent only for the disclosed cuBLAS comparator and exact setup, matching the page's narrow wording."} + }, + { + "claim_id": "lang-cuda-cpp-current-c002", + "claim": "The inline-PTX boundary guidance correctly states that the front end does not parse instruction text, r/l constraints select 32/64-bit integer registers, generic shared pointers need conversion, and hidden memory effects require a memory clobber.", + "claim_type": "code", + "covers": ["u002"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/inline-ptx-assembly/index.html","revision":"CUDA 13.0.2 Inline PTX Assembly guide","locator":"Assembler Syntax, Constraints, Memory Space Conflicts, and Incorrect Optimization","observation":"The guide states that front-end parsing excludes the template string, maps r/l to 32/64-bit registers, requires address-space-correct pointers, and prescribes a memory clobber for hidden memory effects."}], + "challenge": {"alternative":"asm volatile alone prevents both deletion and compiler memory motion, and generic pointers are accepted as shared addresses.","check":"Inspect the guide's separate volatile, memory-clobber, and memory-space sections.","outcome":"Volatile and memory clobbers solve different compiler problems, while shared instructions require a converted shared address."} + }, + { + "claim_id": "lang-cuda-cpp-current-c003", + "claim": "The allocation prose and code implement the PTX ISA 9.0 cta_group::1 contract: all lanes of one warp execute the same collective allocation, its b32 result is written to shared memory, and the CTA synchronizes before consuming it.", + "claim_type": "code", + "covers": ["u003", "u004"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"tcgen05.alloc grammar, semantics, and issue granularity","observation":"The exact cta_group::1 shared b32 form writes taddr to shared memory and requires identical execution by every thread in one warp."},{"url":"https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu","revision":"3b90ac9b","locator":"TMEM allocation and following synchronization","observation":"The pinned implementation executes allocation from all lanes of warp 1 and then synchronizes before loading taddr."}], + "challenge": {"alternative":"Only an elected lane should issue allocation because the result is scalar.","check":"Separate destination cardinality from documented instruction issue granularity.","outcome":"The scalar shared destination does not change the explicit warp-collective execution rule; lane-only execution is undefined."} + }, + { + "claim_id": "lang-cuda-cpp-current-c004", + "claim": "The unscaled f16 MMA wrapper has the legal five-operand structure: taddr, two matrix descriptors, one instruction descriptor, and an enable-input-D predicate constructed from its CUDA integer argument; an elected thread can issue this form.", + "claim_type": "code", + "covers": ["u005", "u006"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"unscaled floating-point MMA grammar and issue granularity","observation":"The displayed mandatory operands and single-thread issue rule match; enable-input-D is a predicate operand."},{"url":"https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/common.h#L66-L78","revision":"3b90ac9b","locator":"tcgen05_mma_f16","observation":"The pinned author wrapper uses one i_desc, declares a PTX predicate, converts the integer with setp.ne.b32, and emits the same five-operand MMA."}], + "challenge": {"alternative":"The last value is a second immediate descriptor or must use an r constraint directly as a predicate.","check":"Compare operand types in the normative grammar and pinned wrapper.","outcome":"Only one instruction descriptor exists; the last operand is .pred and is correctly materialized inside PTX."} + }, + { + "claim_id": "lang-cuda-cpp-current-c005", + "claim": "The TMEM load and teardown prose accurately requires collective warp execution and wait::ld before register consumption, synchronization after all readers, collective matching deallocation, and deallocation before exit.", + "claim_type": "mechanism", + "covers": ["u007"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"ld issue granularity, asynchronous operation, and wait::ld","observation":"All threads of a warp execute ld identically and wait::ld orders use of loaded registers."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-dealloc","revision":"same archive","locator":"deallocation semantics and issue granularity","observation":"Deallocation is warp-collective, must match the allocated columns/address, and every allocation must be deallocated before kernel exit."}], + "challenge": {"alternative":"One lane may load or deallocate after MMA issue because TMEM operations are synchronous.","check":"Inspect ld completion and collective granularity separately from MMA completion.","outcome":"ld has its own asynchronous completion wait and both ld/dealloc carry explicit collective participation rules."} + }, + { + "claim_id": "lang-cuda-cpp-current-c006", + "claim": "The six barrier invariants correctly describe the pinned staged pipeline: initialized and async-proxy-visible barriers, TMA full/byte completion, acquire waits, distinct tcgen completion, empty-stage ownership handoff, per-stage parity tracking, and compiler-visible memory effects.", + "claim_type": "mechanism", + "covers": ["u008", "u009", "u010", "u011", "u012", "u013", "u014", "u015"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-init","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"initialization visibility and fence.mbarrier_init","observation":"Barrier initialization requires publication before async-proxy use."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"same archive","locator":"mbarrier complete_tx byte tracking","observation":"TMA completion updates the supplied barrier transaction byte count."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit","revision":"same archive","locator":"commit mbarrier completion","observation":"tcgen05 commit separately attaches completion of prior tcgen operations to an mbarrier."},{"url":"https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu","revision":"3b90ac9b","locator":"barrier initialization plus load, compute, and epilogue loops","observation":"The pinned kernel implements distinct TMA/MMA/mainloop/epilogue barriers, phase variables, and full/empty ownership handoffs matching the page's lifecycle."}], + "challenge": {"alternative":"One expect_tx barrier and parity wait fully synchronize every producer, MMA, epilogue, and stage reuse edge.","check":"Trace who owns each stage before TMA, after TMA, after MMA, and after the epilogue.","outcome":"Those transitions have different producers, consumers, and completion mechanisms; collapsing them loses required lifetime and reuse edges."} + }, + { + "claim_id": "lang-cuda-cpp-current-c007", + "claim": "The stated six-warp role split exactly matches the pinned v6 implementation while being explicitly identified as implementation-specific.", + "claim_type": "code", + "covers": ["u016"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v6.cu","revision":"gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"NUM_WARPS, role branches, final synchronization and deallocation","observation":"The source uses warp 0 for elected TMA issue, warp 1 for collective alloc/elected MMA, four epilogue warps, a final reader synchronization, and collective warp-0 deallocation."}], + "challenge": {"alternative":"The six-warps split is required by tcgen05 or every lane in the producer/compute warps issues all operations.","check":"Inspect elected-thread predicates separately from collective instructions and compare with ISA issue rules.","outcome":"The source mixes elected and collective operations within its chosen split, and the page does not elevate that choice to an ISA rule."} + }, + { + "claim_id": "lang-cuda-cpp-current-c008", + "claim": "All three primary-reference URLs and both relative related links resolve to their stated archive, pinned source tree, and local pages; broken external and local controls do not resolve.", + "claim_type": "other", + "covers": ["u017", "u018", "u019", "u020", "u021"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"curl the three primary-reference URLs; test both relative local targets; repeat with fabricated external and local paths","environment":"current KernelWiki worktree with network, 2026-08-08","reference":"Every displayed reference should land on the named document/tree, while broken controls should fail.","result":"The NVIDIA archive links, pinned GitHub source tree, and both intended files resolved successfully.","negative_control":"A fabricated CUDA archive path returned HTTP 404 and wiki/languages/not-cuda.md was absent."}], + "challenge": {"alternative":"The commit URL points to the old nonexistent tcgen05 directory or a relative link escapes to an unrelated card.","check":"Resolve the exact commit tree path and inspect local target IDs.","outcome":"The URL names 02e_matmul_sm100 at the pinned commit; local targets identify lang-ptx and blog-tcgen05-tutorial."} + } + ], + "commands": ["conda run -n base python ratio check with broken comparator", "local and HTTP link checks with negative controls"], + "scope": "All 21 current units for body hash 280b0450a7c8a8d900672fd9664f15c0dfdc7f7464b794a771ca6275e0b23ead were checked against CUDA 13.0.2 PTX ISA 9.0, the CUDA 13.0.2 inline-PTX guide, Gau Nernst's pinned 3b90ac9b source, and host controls.", + "residual_risk": "No CUDA toolkit or Blackwell GPU is installed locally. Syntax and lifecycle guidance are source-verified, while the author-reported B200 performance has not been independently reproduced." +} diff --git a/verification/reviewed/current/lang-cute-dsl.json b/verification/reviewed/current/lang-cute-dsl.json new file mode 100644 index 000000000..7388dd74f --- /dev/null +++ b/verification/reviewed/current/lang-cute-dsl.json @@ -0,0 +1,110 @@ +{ + "path": "wiki/languages/cute-dsl.md", + "id": "lang-cute-dsl", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "lang-cute-dsl-current-c001", + "claim": "The page accurately scopes CuTe DSL as a Python-native interface shipped alongside CUTLASS C++ in exact release v4.5.0 at commit e406c186, and warns that rolling documentation can contain later names.", + "claim_type": "compatibility", + "covers": ["u001"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0/README.md","revision":"NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"DSL overview plus checked-out exact tag","observation":"The tag identifies both C++ and Python DSL interfaces, contains the CuTe DSL packages/examples, and resolves to the stated commit."}], + "challenge": {"alternative":"The page silently describes the later 4.5.2 API or treats CuTe DSL as the only Blackwell interface.","check":"Resolve the tag commit and compare the page's interface wording with the tagged README/source tree.","outcome":"The commit is exact and the page explicitly places the DSL alongside C++ rather than assigning exclusivity or primacy."} + }, + { + "claim_id": "lang-cute-dsl-current-c002", + "claim": "The FA4 paragraph faithfully reports that FA4 is entirely CuTe DSL, that its author reports about 20-30x shorter compile times than C++ templates and up to 1605 TFLOP/s on B200 BF16, and that 1605 is not a matched handwritten-C++ comparison.", + "claim_type": "performance", + "covers": ["u002"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://tridao.me/blog/2026/flash4/","revision":"Tri Dao first-party FA4 post, fetched 2026-08-08","locator":"Language and framework plus attention benchmark sections","observation":"The post gives the all-CuTe implementation, approximately 20-30x compile claim, and up-to-1605 B200 BF16 result; runtime comparators are FA2, Triton, Gluon, and cuDNN rather than a handwritten-C++ FA4 row."}], + "challenge": {"alternative":"1605 establishes general CuTe-versus-C++ parity or is a fully specified reproducible shape result.","check":"Inspect the benchmark comparator labels and scope of the maximum.","outcome":"The author supplies neither a handwritten-C++ comparator nor one fully specified row for the maximum, matching the page's limitation."} + }, + { + "claim_id": "lang-cute-dsl-current-c003", + "claim": "The F16/BF16 excerpt uses valid v4.5.0 Python API names and arguments for a one-CTA 128x256x16 SMEM-input operation; the two-CTA tutorial uses 256x256x16/CtaGroup.TWO and a TMEM-backed accumulator fragment.", + "claim_type": "code", + "covers": ["u003", "u004", "u005"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py","revision":"NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"imports, MmaF16BF16Op construction, make_tiled_mma, make_fragment_C, and TMEM rebind","observation":"The exact one-CTA operation shape, types, source, major modes, tiled-MMA construction, and accumulator rebind match."},{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_1.py","revision":"same tag","locator":"MmaF16BF16Op construction","observation":"The two-CTA tutorial passes (256,256,16) and CtaGroup.TWO."}], + "challenge": {"alternative":"The old nonexistent SM100_MMA aliases are still required, the major-mode namespace is invalid, or the dimensions are CTA tile shapes.","check":"Trace the exact symbols exported by nvgpu and the tuple passed to MmaF16BF16Op.","outcome":"nvgpu exports OperandMajorMode, the operation is constructed exactly with the displayed symbols, and the source separately names its larger MMA tiler."} + }, + { + "claim_id": "lang-cute-dsl-current-c004", + "claim": "The TMEM section accurately excerpts the tagged TmemAllocator allocation/wait/retrieve/rebind and typed ld-copy construction lifecycle, while explicitly retaining partitioning, synchronization, collective, two-CTA, and free preconditions.", + "claim_type": "code", + "covers": ["u006", "u007", "u008"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py","revision":"NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"TmemAllocator around lines 97-106, pointer retrieval around 174-177, copy setup around 190-200, and free around 269-270","observation":"Every displayed operation and the stated omitted partition/copy/sync steps correspond to the complete official example."},{"path":"/tmp/kernelwiki-cutlass-4.5.0/python/CuTeDSL/cutlass/cute/arch/tmem.py","revision":"same tag","locator":"alloc_tmem and dealloc_tmem contracts","observation":"The underlying helper validates column counts and emits architecture TMEM allocation/deallocation operations."}], + "challenge": {"alternative":"The excerpt can be copied as a standalone lane-local allocator or frees before epilogue readers complete.","check":"Follow the full official control flow from collective allocation through pipeline sync and free.","outcome":"The page marks missing context as required and places synchronization before free, rejecting standalone/lane-local interpretation."} + }, + { + "claim_id": "lang-cute-dsl-current-c005", + "claim": "The TMA section uses the exact one-CTA CopyBulkTensorTileG2SOp and make_tiled_tma_atom_A construction and correctly identifies the two-CTA multicast operation plus matching cluster/participant requirement.", + "claim_type": "code", + "covers": ["u009", "u010", "u011"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py","revision":"NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"TMA construction and tma_partition/cute.copy paths","observation":"The one-CTA names, arguments, partition, copy, and pipeline use match."},{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_1.py","revision":"same tag","locator":"CopyBulkTensorTileG2SMulticastOp and cluster launch","observation":"The two-CTA example uses the multicast operation with CtaGroup.TWO and derives TMA atoms from the cluster layout."}], + "challenge": {"alternative":"The invented SM100_TMA_LOAD_2D alias is needed or the multicast form can be used independently of launch topology.","check":"Search exact tag symbols and trace the cluster layout into atom construction/launch.","outcome":"The displayed typed operations are the actual API, and the multicast topology is constructed consistently rather than inferred automatically."} + }, + { + "claim_id": "lang-cute-dsl-current-c006", + "claim": "The warp-specialization paragraph correctly characterizes fp16_gemm_2.py's TMA/MMA/epilogue roles and its explicit PipelineTmaUmma, PipelineUmmaAsync, TMEM, TMA-store, tail, and teardown requirements.", + "claim_type": "mechanism", + "covers": ["u012"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py","revision":"NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"role dispatch, pipeline creation/states, TMEM epilogue, and final tails/free","observation":"The complete source contains every named role and lifecycle edge and cannot be reduced to an if-warp sketch safely."}], + "challenge": {"alternative":"A barrier arrive/wait around TMA is enough and the listed tails/empty states are optional optimization details.","check":"Trace stage ownership and termination in the official kernel.","outcome":"The full/empty pipelines protect reuse and tails drain outstanding work; omission changes lifetimes rather than only performance."} + }, + { + "claim_id": "lang-cute-dsl-current-c007", + "claim": "The layout paragraph accurately distinguishes helper-assisted construction from automatic policy: v4.5.0 supplies typed algebra and make_smem_layout_a/b while callers explicitly select tilers, types, major modes, alignments, cluster shape, and pipeline policy.", + "claim_type": "code", + "covers": ["u013"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py","revision":"NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"MMA construction, make_smem_layout_a/b, allocate_tensor, and pipeline creation","observation":"Helpers derive layouts from explicit inputs while the kernel author supplies all policy items named by the page."}], + "challenge": {"alternative":"The helpers autonomously select the full layout/swizzle and pipeline for arbitrary inputs.","check":"Enumerate parameters and subsequent explicit allocation/pipeline calls in the example.","outcome":"Substantial policy remains explicit, so helper-assisted is the evidence-supported scope."} + }, + { + "claim_id": "lang-cute-dsl-current-c008", + "claim": "The nine table links are untruncated verbatim upstream captures matching provenance SHA-256 records and purposes; the seven numeric line counts are exact.", + "claim_type": "other", + "covers": ["u014", "u015", "u016", "u017", "u018", "u019", "u020", "u021", "u022", "u023"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"parse all three PROVENANCE.yaml files; hash every upstream-file; count fp16_gemm_0..6 lines; compare representative files with raw merge-SHA content; alter one expected hash","environment":"current KernelWiki worktree, conda base Python, and network, 2026-08-09","reference":"Provenance requires mode=verbatim and size_cap_truncated=false; table supplies seven exact counts and nine purposes.","result":"All 11 captured upstream files matched provenance, all seven displayed counts matched [447,535,679,769,1065,919,1002], and representative remote bytes matched full PR merge SHAs.","negative_control":"Changing one expected SHA-256 to 64 zeros caused the check to reject the set."}], + "challenge": {"alternative":"The links are partial excerpts, descriptions are guessed, or short merge IDs point at mutable heads.","check":"Combine provenance fields/hashes with full merge SHAs from the GitHub PR API and file docstrings.","outcome":"The captures are complete and immutable, and the descriptions follow the upstream file text/PR summary."} + }, + { + "claim_id": "lang-cute-dsl-current-c009", + "claim": "PR 3106 and the v4.5.0 tag establish the stated progressive tutorial sequence and the tag's renamed cute/blackwell/tutorial/tutorial_gemm location.", + "claim_type": "code", + "covers": ["u024"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/prs/cutlass/PR-3106/diff.patch","revision":"NVIDIA/cutlass PR 3106 merge 772fbb264e0fe823d62dec10a44a67e1503afdfc","locator":"fp16_gemm_0.py through fp16_gemm_6.py docstrings and implementations","observation":"The merged source spells out and implements each progression step."},{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/README.md","revision":"NVIDIA/cutlass v4.5.0 e406c186","locator":"directory and tutorial README","observation":"The tagged official tutorial occupies the page's stated renamed path and contains the same numbered series."}], + "challenge": {"alternative":"The page conflates the older PR paths with current rolling locations or infers features from filenames.","check":"Compare PR merge paths/docstrings with the exact tag tree.","outcome":"The page labels both locations and derives steps from upstream descriptions/implementation, not filenames alone."} + }, + { + "claim_id": "lang-cute-dsl-current-c010", + "claim": "The three primary URLs, nine artifact links, and three related links resolve to the intended release/source/post or local file IDs, while fabricated external and local controls fail.", + "claim_type": "other", + "covers": ["u025", "u026", "u027", "u028", "u029", "u030"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"HTTP-check the tag, tutorial, and FA4 URLs; resolve all artifact/related links from wiki/languages/cute-dsl.md; repeat with fabricated HTTP and local targets","environment":"current KernelWiki worktree and network, 2026-08-09","reference":"Links should land on exact NVIDIA tag/tutorial, first-party FA4 post, nine artifacts, and three named local IDs.","result":"All intended external URLs returned HTTP 200; every artifact existed; related IDs matched hw-tcgen05-mma, kernel-flash-attention-4, and doc-cutlass-blackwell.","negative_control":"A fabricated CUTLASS commit path returned non-200 and wiki/languages/not-cute-dsl.md was absent."}], + "challenge": {"alternative":"GitHub redirects a missing tree to a generic page or relative artifact paths escape incorrectly.","check":"Require 200 for exact URLs and filesystem existence/ID matches, then compare broken controls.","outcome":"Every intended target resolved specifically and both broken controls failed."} + } + ], + "commands": ["NVIDIA/cutlass v4.5.0 sparse checkout at e406c186", "GitHub PR API full merge-SHA checks", "artifact hash/line-count verification with corrupted-hash control", "external and local link checks with negative controls"], + "scope": "All 30 current units for body hash 90ca9df7c219547d83ab1a5609935cb5d4b5fe4ebd1dee959da0fadaf0c0c5a9 were checked against CUTLASS v4.5.0, PR merge artifacts, first-party FA4 reporting, and host controls.", + "residual_risk": "No Blackwell GPU or installed CuTe DSL/CUDA toolchain is available locally. API and lifecycle claims are exact-source verified, artifacts are byte-verified, and FA4 numbers remain author-reported rather than independently reproduced." +} diff --git a/verification/reviewed/current/lang-ptx.json b/verification/reviewed/current/lang-ptx.json new file mode 100644 index 000000000..d8ce56bfe --- /dev/null +++ b/verification/reviewed/current/lang-ptx.json @@ -0,0 +1,100 @@ +{ + "path": "wiki/languages/ptx-sm100.md", + "id": "lang-ptx", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "lang-ptx-current-c001", + "claim": "The page is explicitly scoped to archived CUDA 13.0.2 PTX ISA 9.0 and correctly warns that representative forms still require typed operands, valid descriptors, collective participation, lifetimes, completion, and target/toolchain support.", + "claim_type": "compatibility", + "covers": ["u001"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"instruction grammar, target ISA notes, and tcgen05 memory-consistency model","observation":"The archive versions grammar/targets and places the listed operand, participation, lifetime, and completion conditions on these instruction families."}], + "challenge": {"alternative":"The snippets are complete inline-assembly functions or rolling PTX semantics can be substituted without a version check.","check":"Inspect operand declarations and target/version notes required by the archived grammar.","outcome":"The forms omit surrounding declarations by design, and support is explicitly versioned; the scope warning is necessary and accurate."} + }, + { + "claim_id": "lang-ptx-current-c002", + "claim": "The representative alloc/load-address/MMA/commit/dealloc forms and prose accurately describe a cta_group::1 unscaled F16 path: alloc writes a 32-bit taddr to shared memory and counts columns, allocation/deallocation are warp-collective, MMA D is in TMEM, commit attaches prior completion to an mbarrier, every allocation is freed, and one kernel uses one CTA-group mode.", + "claim_type": "code", + "covers": ["u002", "u003", "u004"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"alloc/dealloc grammar, issue granularity, and lifecycle","observation":"The exact forms, shared result, column unit, warp collectivity, and required deallocation match."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"same archive","locator":"unscaled floating-point grammar, CTA-group rule, and D semantics","observation":"The five-operand f16 form, TMEM D, group modes, and same-group-per-kernel restriction match."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit","revision":"same archive","locator":"mbarrier::arrive::one grammar","observation":"Commit tracks completion of prior asynchronous tcgen05 work with the displayed barrier form."}], + "challenge": {"alternative":"Alloc returns a column index directly in a register, ncols counts rows, MMA requires a lane-disable vector, or commit waits synchronously.","check":"Compare the exact unscaled grammar and optional operands with the alloc destination and commit description.","outcome":"Alloc stores a b32 taddr in shared memory and counts columns; the five-operand MMA is valid without the optional lane-disable vector; commit signals completion asynchronously."} + }, + { + "claim_id": "lang-ptx-current-c003", + "claim": "The displayed 32x32b.x1 ld/st and 128x256b cp forms are legal; ld/st are asynchronous warp-collective TMEM-register transfers with their matching waits, cp moves SMEM to TMEM, and cp/MMA source/destination lifetimes extend through their documented completion.", + "claim_type": "code", + "covers": ["u005", "u006", "u007"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"ld/st grammar, repetition table, collectivity, and wait","observation":"32x32b.x1 uses one register per thread, ld/st are warp-collective and asynchronous, and wait::ld/st provide completion."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-cp","revision":"same archive","locator":"cp grammar, 128x256b example, and asynchronous semantics","observation":"The exact cta_group::1 copy is shared-memory descriptor to TMEM and participates in tcgen05 completion/lifetime rules."}], + "challenge": {"alternative":"x1 is invalid, the waits are generic barriers, or cp copies TMEM to shared memory.","check":"Use the shape/repetition table, wait grammar, and cp operand direction.","outcome":"x1 is listed, waits target prior ld/st from the executing thread, and cp's destination is taddr while its source is sdesc."} + }, + { + "claim_id": "lang-ptx-current-c004", + "claim": "The four completion/ordering bullets correctly distinguish TMA mbarrier byte completion, tcgen05 commit completion, ld/st waits, and before/after_thread_sync cross-thread tcgen ordering; the fences are not completion substitutes.", + "claim_type": "mechanism", + "covers": ["u008", "u009", "u010", "u011"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"mbarrier complete_tx semantics","observation":"TMA load completion decrements byte transaction count on its barrier."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit","revision":"same archive","locator":"commit and cross-thread synchronization examples","observation":"Commit attaches prior MMA/cp/shift completion to an mbarrier."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence","revision":"same archive","locator":"before/after_thread_sync semantics","observation":"The fences constrain tcgen05 issue around execution ordering and do not wait for async completion."}], + "challenge": {"alternative":"One CTA barrier or after_thread_sync can replace every listed mechanism.","check":"Match each producer/consumer edge to the normative instruction that completes or orders it.","outcome":"The ISA assigns distinct state and scope to each edge, so none is a universal substitute."} + }, + { + "claim_id": "lang-ptx-current-c005", + "claim": "The packed conversion and typed move forms have the stated one-byte E2M1x2 source, 32-bit F16x2 result, and b32-to-four-b8 decomposition semantics, while PTX makes no universal speed promise.", + "claim_type": "code", + "covers": ["u012", "u013", "u014"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"packed alternate-format cvt","observation":"The E2M1x2/F16x2 types and widths match."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov","revision":"same archive","locator":"scalar-to-vector move type-size rule","observation":"Four byte destinations can receive a b32 source when declarations match; no performance comparison is specified."}], + "challenge": {"alternative":"The input is a 32-bit scalar FP4, mov changes numeric representation, or the ISA guarantees lower latency.","check":"Apply packed type widths and mov's bitwise typed-copy semantics.","outcome":"The input is one packed byte, mov preserves bits across equal total width, and the documentation contains no timing theorem."} + }, + { + "claim_id": "lang-ptx-current-c006", + "claim": "The three ld.global forms are legal 16/32-byte vector loads; no_allocate and evict_last are nonbinding L1 eviction-priority hints, so the page correctly rejects guaranteed bypass/residency and scopes performance to the exact access pattern.", + "claim_type": "code", + "covers": ["u015", "u016"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"ld grammar, vector examples, and cache operators","observation":"The forms and widths match; eviction-priority qualifiers are performance hints that may not be respected."}], + "challenge": {"alternative":"The qualifier names prescribe deterministic cache behavior or v4.u64 is four bytes.","check":"Read the cache-hint caveat and multiply vector arity by the 64-bit element width.","outcome":"The behavior is advisory, and v4.u64 is 256 bits (32 bytes)."} + }, + { + "claim_id": "lang-ptx-current-c007", + "claim": "The CLC fragment and prose accurately show an asynchronous 16-byte shared response completed on an mbarrier, followed by success and first-CTA-coordinate queries; the request cancels a not-yet-launched unit and takes no desired tile ID.", + "claim_type": "code", + "covers": ["u017", "u018", "u019"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"try_cancel grammar, response, completion, and query_cancel sections","observation":"The displayed request/query forms, 16-byte response, wait-before-query requirement, cancellation target, and no-coordinate request semantics match."}], + "challenge": {"alternative":"try_cancel returns coordinates directly or accepts the work ID to steal.","check":"Inspect request operands separately from query_cancel output operands.","outcome":"The request has only response and barrier addresses; coordinates are decoded from a completed successful response."} + }, + { + "claim_id": "lang-ptx-current-c008", + "claim": "The two TMA forms are valid 2D global-to-cluster-shared tile loads; multicast uses .multicast::cluster plus a 16-bit mask and sends data and barrier completion to corresponding offsets in selected live destination CTAs.", + "claim_type": "code", + "covers": ["u020", "u021", "u022", "u023", "u024"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"cluster destination and multicast grammar/semantics","observation":"Both exact forms, tensor-map coordinates, mbarrier, qualifier, mask width, corresponding offsets, and destination lifetime requirements match."}], + "challenge": {"alternative":"The shorter .multicast spelling is legal, the barrier is signaled only in the issuer, or destination CTAs need not be resident.","check":"Inspect the qualifier set and multicast destination/completion description.","outcome":"Only .multicast::cluster is legal, completion is multicast to matching offsets, and remote shared memory must remain live."} + }, + { + "claim_id": "lang-ptx-current-c009", + "claim": "The six internal links and archived PTX reference resolve to the intended pages, and a missing same-site/local control is rejected.", + "claim_type": "other", + "covers": ["u025", "u026", "u027", "u028", "u029", "u030", "u031"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"test six relative wiki targets; curl archived PTX URL; test missing local target and archived URL","environment":"current KernelWiki worktree and network, 2026-08-08","reference":"Each label should resolve to its named page/reference.","result":"All six files existed with matching IDs and the archived PTX URL returned HTTP 200.","negative_control":"A nonexistent local language page was absent and a fabricated archived PTX path returned HTTP 404."}], + "challenge": {"alternative":"A relative path escapes to an adjacent page or NVIDIA returns success for missing archive paths.","check":"Resolve exact files/IDs and compare the fabricated archive path.","outcome":"All intended targets matched and both broken controls failed."} + } + ], + "commands": ["local internal-link checks with missing-target control", "curl archived PTX URL with 404 control"], + "scope": "All 31 current units for body hash 4bbed1d5aa5094a031fdbe1e8349f5079263caf9887a77721a5ace82c565fad5 were checked against CUDA 13.0.2 PTX ISA 9.0 and host link controls.", + "residual_risk": "No CUDA assembler or Blackwell GPU is installed locally. The page makes no numeric performance claim; representative syntax and semantics come directly from the archived ISA, while full inline-assembly integration remains caller-specific." +} diff --git a/verification/reviewed/current/lang-triton.json b/verification/reviewed/current/lang-triton.json new file mode 100644 index 000000000..a7e57daa5 --- /dev/null +++ b/verification/reviewed/current/lang-triton.json @@ -0,0 +1,314 @@ +{ + "path": "wiki/languages/triton-blackwell.md", + "id": "lang-triton", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "lang-triton-current-c001", + "claim": "The checked native Blackwell backend boundary is between Triton v3.2.0 and v3.3.0: the corresponding TCGen5/TMEM/MMAv5 symbols are absent at the former tag, while the latter contains operations, allocation/lowering passes, and conversion tests for concrete output.", + "claim_type": "compatibility", + "covers": ["u001", "u002", "u003", "u035"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://github.com/triton-lang/triton/blob/819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c/include/triton/Dialect/TritonNvidiaGPU/IR/TritonNvidiaGPUOps.td", + "revision": "Triton v3.3.0 commit 819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c", + "locator": "TCGen5MMA, TCGen5MMAScaled, and TMEM operation definitions", + "observation": "The tagged dialect defines native TCGen5 MMA/scaled-MMA and tensor-memory operations." + }, + { + "url": "https://github.com/triton-lang/triton/blob/819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c/test/Conversion/tritongpu_to_llvm_blackwell.mlir", + "revision": "Triton v3.3.0 commit 819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c", + "locator": "FileCheck expectations for tcgen05 MMA, commit, TMEM, scaled MMA, and cta_group::2", + "observation": "The conversion suite requires concrete Blackwell-native output rather than merely declaring unused operations." + }, + { + "url": "https://github.com/triton-lang/triton/compare/v3.2.0...v3.3.0", + "revision": "v3.2.0 commit 9641643da6c52000c807b5eeed05edaec4402a67 to v3.3.0 commit 819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c", + "locator": "TritonNvidiaGPU operations, MMAv5 lowering, tensor-memory allocation, and Blackwell tests", + "observation": "The corresponding backend enters in this tag interval; exact-tree search finds no matching backend symbols at v3.2.0." + } + ], + "challenge": { + "alternative": "The v3.3 operations are unused scaffolding, and native lowering actually begins in v3.6.", + "check": "Compare the exact v3.2 and v3.3 trees and inspect the v3.3 conversion test expectations.", + "outcome": "The earlier tree lacks the backend symbols and the v3.3 tests require concrete TCGen5/TMEM output, discriminating real lowering from scaffolding." + } + }, + { + "claim_id": "lang-triton-current-c002", + "claim": "By v3.5.0 Triton ships explicit Gluon TCGen5/TMEM and Blackwell block-scaled matmul tutorials; v3.6.0 incrementally broadens layouts, copies, warp specialization, and initial multi-CTA/2-CTA Gluon support, while v3.7.0 continues 2-CTA/multicast/TMA work and v3.7.1 is a regression-only patch.", + "claim_type": "compatibility", + "covers": ["u004", "u005", "u006", "u008", "u009", "u010", "u011"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://github.com/triton-lang/triton/blob/c3c476f357f1e9768ea4e45aa5c17528449ab9ef/python/tutorials/gluon/06-tcgen05.py", + "revision": "Triton v3.5.0 commit c3c476f357f1e9768ea4e45aa5c17528449ab9ef", + "locator": "tutorial body", + "observation": "The pinned tutorial allocates, loads, and stores TMEM and invokes TCGen5 MMA through Gluon." + }, + { + "url": "https://github.com/triton-lang/triton/blob/c3c476f357f1e9768ea4e45aa5c17528449ab9ef/python/tutorials/10-block-scaled-matmul.py", + "revision": "Triton v3.5.0 commit c3c476f357f1e9768ea4e45aa5c17528449ab9ef", + "locator": "Blackwell block-scaled matmul tutorial", + "observation": "The v3.5 tag already contains the tl.dot_scaled Blackwell tutorial." + }, + { + "url": "https://github.com/triton-lang/triton/releases/tag/v3.6.0", + "revision": "Triton v3.6.0 commit 7c56a5e40f7fd928dfd5c72902d5def0097db73a", + "locator": "Blackwell, TMEM, tcgen05, warp-specialization, and Gluon release-note entries", + "observation": "The release enumerates generalization and initial multi-CTA/2-CTA work on top of earlier capabilities." + }, + { + "url": "https://github.com/triton-lang/triton/releases/tag/v3.7.0", + "revision": "Triton v3.7.0 commit 5f3f125e8f63c24613f1f73b937442864f263f94", + "locator": "2CTA, multicast, and TMA release-note entries", + "observation": "The next minor release contains further end-to-end Blackwell cluster-path work." + }, + { + "url": "https://github.com/triton-lang/triton/releases/tag/v3.7.1", + "revision": "Triton v3.7.1 commit f797708", + "locator": "release description", + "observation": "The patch release lists two regression fixes and explicitly advertises no new API or feature." + } + ], + "challenge": { + "alternative": "The tutorials and native surfaces first appear only in v3.6, and v3.7.1 adds another feature milestone.", + "check": "Open both tutorials at the exact v3.5 tag and compare the v3.6, v3.7, and v3.7.1 release descriptions.", + "outcome": "The tutorials exist at v3.5, the v3.6/v3.7 notes describe successive expansions, and v3.7.1 describes only regression fixes." + } + }, + { + "claim_id": "lang-triton-current-c003", + "claim": "vLLM PR 34597 adds FP8 KV-cache handling to a Triton MLA decode backend whose pinned kernel contains the displayed verbatim FP8-rescaling/tl.dot excerpt; the PR names SM120, while neither the PR nor code proves an SM100-only path or a particular TCGen5/TMEM lowering.", + "claim_type": "code", + "covers": ["u012", "u026", "u027", "u033", "u034"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://github.com/vllm-project/vllm/pull/34597", + "revision": "merged commit a1257fd1", + "locator": "PR summary and changed-files list", + "observation": "The PR adds FP8 cache support and identifies the Triton backend as the MLA option on SM120; it makes no SM100-only or instruction-lowering claim." + }, + { + "url": "https://github.com/vllm-project/vllm/blob/a1257fd1/vllm/v1/attention/ops/triton_decode_attention.py", + "revision": "commit a1257fd1", + "locator": "kernel definitions and tl.dot call sites", + "observation": "The code contains Triton JIT kernels and tl.dot but no architecture guard, Triton-version requirement, warp_specialize, TCGen5/TMEM name, or emitted PTX." + } + ], + "challenge": { + "alternative": "The local architecture field or the presence of tl.dot is enough to demonstrate SM100 TCGen5 emission.", + "check": "Inspect the primary PR motivation, exact code guards, and any emitted-IR/PTX evidence.", + "outcome": "The primary source names SM120 and supplies no target guard or lowering witness, so the page retains only the narrower implementation claim." + } + }, + { + "claim_id": "lang-triton-current-c004", + "claim": "On 2026-08-08 the live FlashInfer-Bench author leaderboard shows 0.628x/73.1% for Gemini 2.5 Pro, 0.467x/92.3% for GPT-5, and 0.456x/73.1% for Claude Opus 4.1, each across 660 workloads; it gives no Triton-release or Triton-only scope for those rows.", + "claim_type": "performance", + "covers": ["u019", "u020", "u021", "u022", "u023"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://bench.flashinfer.ai/", + "revision": "live leaderboard retrieved 2026-08-08", + "locator": "Author / Avg Speedup / % Resolved rows for the three named authors", + "observation": "The exact values and 660-workload denominators appear in the labeled rows; the page exposes no Triton version or Triton-only filter." + } + ], + "challenge": { + "alternative": "The values are historical Triton 3.5 results or belong to adjacent model/workload columns.", + "check": "Read each author row under the labeled columns and inspect the page for version and language metadata.", + "outcome": "The values map directly to the author rows, and no claimed Triton version/language scope is present." + } + }, + { + "claim_id": "lang-triton-current-c005", + "claim": "SGLang PR 5390 reports a scoped 10,447.34-versus-8,227.35 total-token-throughput comparison (26.98%), and PR 21595 changes the SM100 datacenter multimodal-attention default from triton_attn to FA4.", + "claim_type": "performance", + "covers": ["u017", "u018"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://github.com/sgl-project/sglang/pull/5390", + "revision": "merged commit 84810da4ae424d15ec06a3e32a03a839dd50a644", + "locator": "DeepSeek-R1 benchmark commands and CUTLASS/Triton result blocks", + "observation": "The PR records the two totals and the stated 3,000-prompt, TP8/DP8, float16, 1,000-input/1,000-output-token scope." + }, + { + "url": "https://github.com/sgl-project/sglang/pull/21595", + "revision": "merged commit 87a27682", + "locator": "summary and SM100-default change", + "observation": "The PR explicitly scopes the default switch to Blackwell SM100 datacenter multimodal attention." + } + ], + "challenge": { + "alternative": "The percentage compares different metrics, or the routing change applies to every GPU and workload.", + "check": "Use the total-token-throughput field in both PR 5390 result blocks and the architecture condition in PR 21595.", + "outcome": "The quotient is 26.98%, and the routing change is explicitly SM100-scoped." + } + }, + { + "claim_id": "lang-triton-current-c006", + "claim": "The page's triton-universe link resolves to the canonical ledger containing the tracked PR entries and their captured flags without duplicating its mutable count.", + "claim_type": "other", + "covers": ["u032"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [ + { + "command": "conda run -n base python -c ", + "environment": "Conda base, Python 3.13.14, repository filesystem", + "reference": "The relative link must resolve and the YAML must expose a PR collection with per-entry captured flags.", + "result": "The link resolves; the canonical YAML parses with 267 PR entries, each carrying a captured flag, while the wiki text states no copied count.", + "negative_control": "../../data/triton-universe-missing.yaml does not resolve." + } + ], + "challenge": { + "alternative": "The link points to a stale summary or the target lacks capture status.", + "check": "Resolve the exact relative path and inspect every PR entry for the captured key, alongside a broken-target control.", + "outcome": "The exact canonical target resolves and all entries expose the flag; the negative control fails." + } + }, + { + "claim_id": "lang-triton-current-c007", + "claim": "vLLM PR 29339 is an SM90/SM100 MXFP4 triton_kernels dispatch gate rather than a new Triton kernel or compiler-lowering implementation.", + "claim_type": "code", + "covers": ["u013"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://github.com/vllm-project/vllm/pull/29339", + "revision": "merged commit c17610e2", + "locator": "one-file dispatch diff", + "observation": "The PR gates triton_kernels MXFP4 use to SM90 and SM100 and does not add a Triton kernel." + } + ], + "challenge": { + "alternative": "The PR adds a target-specific Triton kernel or demonstrates a TCGen5 lowering.", + "check": "Inspect the exact one-file diff and changed-files list.", + "outcome": "Only dispatch logic changes; no Triton kernel, IR, or PTX is added." + } + }, + { + "claim_id": "lang-triton-current-c011", + "claim": "SGLang PR 21019 is a load/store GatedDeltaNet projection rearrangement without tl.dot, while PR 22079 is extend-attention code with tl.dot but no emitted-PTX witness.", + "claim_type": "code", + "covers": ["u014", "u015"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "artifacts/prs/sglang/PR-21019/key-files/python/sglang/jit_kernel/triton/gdn_fused_proj.py", + "revision": "upstream SHA 5bdc07d9, sha256 c4ef85167eb11065721e76dd5d63d6f346f309f84822578dd4a3874249acef89", + "locator": "Triton kernel load/store sites and whole-file tl.dot search", + "observation": "The verbatim kernel rearranges projection data with loads/stores and contains no tl.dot." + }, + { + "path": "artifacts/prs/sglang/PR-22079/key-files/python/sglang/srt/layers/attention/triton_ops/extend_attention.py", + "revision": "upstream SHA 5638d40f, sha256 d27789c32e180d725ed867f4f6cd05b09a3c47bfe37794ff60e150d000573edd", + "locator": "Triton extend-attention kernels and tl.dot sites", + "observation": "The verbatim code contains QK and probability/value dot products but no TCGen5 PTX dump." + } + ], + "challenge": { + "alternative": "The page swaps which SGLang artifact contains matrix products or either file contains an emitted TCGen5 witness.", + "check": "Inspect each exact file for decorators, tl.dot, load/store operations, TCGen5 names, and PTX.", + "outcome": "The GDN file is a load/store rearrangement, the attention file contains the dot products, and neither provides emitted TCGen5 PTX." + } + }, + { + "claim_id": "lang-triton-current-c008", + "claim": "CUDA Graphs reduce repeated CPU launch costs for short-kernel workflows; at vLLM commit a1257fd1 the v1 default is FULL_AND_PIECEWISE and the GatedDeltaNet backend supports uniform-batch graphs and decode full capture.", + "claim_type": "performance", + "covers": ["u024"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cuda-graphs.html", + "revision": "CUDA Programming Guide 13.2", + "locator": "4.2 CUDA Graphs introduction", + "observation": "The guide identifies host setup as significant for short kernels and explains that graphs reduce CPU launch costs by preparing work in advance." + }, + { + "url": "https://github.com/vllm-project/vllm/blob/a1257fd1/vllm/config/compilation.py", + "revision": "vLLM commit a1257fd1", + "locator": "CUDAGraphMode and CompilationConfig.cudagraph_mode", + "observation": "FULL_AND_PIECEWISE is documented as the v1 default." + }, + { + "url": "https://github.com/vllm-project/vllm/blob/a1257fd1/vllm/v1/attention/backends/gdn_attn.py", + "revision": "vLLM commit a1257fd1", + "locator": "graph-support methods and build_for_cudagraph_capture", + "observation": "The GDN backend advertises uniform-batch graph support and implements decode-only full graph capture." + } + ], + "challenge": { + "alternative": "Graphs do not address CPU launch cost, or GatedDeltaNet opts out of vLLM's graph path.", + "check": "Inspect the CUDA guide and the exact vLLM compilation default and GDN metadata builder.", + "outcome": "The primary documentation connects graphs to launch cost, and the pinned backend explicitly implements the capture path." + } + }, + { + "claim_id": "lang-triton-current-c009", + "claim": "All six linked example files resolve to provenance-pinned verbatim upstream assets and their role descriptions match the code at the recorded merge revisions.", + "claim_type": "code", + "covers": ["u025", "u026", "u027", "u028", "u029", "u030", "u031"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [ + { + "command": "conda run -n base python -c ", + "environment": "Conda base, Python 3.13.14, repository filesystem after remediation", + "reference": "Each link must exist, match a verbatim provenance entry, and contain the named kernel or wrapper behavior.", + "result": "All six links resolve; all five bundles mark the assets verbatim and their hashes match. Inspection finds the stated MLA kernel/backend, format conversion, normalization, GDN projection, and extend-attention roles.", + "negative_control": "artifacts/prs/vllm/PR-00000/key-files/missing.py does not resolve." + } + ], + "challenge": { + "alternative": "At least one link is a derived summary, stale target, hash mismatch, or mislabeled code role.", + "check": "Resolve every link, compare it with the owning provenance entry, hash the file, inspect relevant symbols, and test a broken path.", + "outcome": "Every real target and hash succeeds with asset_mode verbatim; the deliberately broken target fails." + } + }, + { + "claim_id": "lang-triton-current-c010", + "claim": "The checked compiler sources prove native TCGen5/TMEM paths exist, but the cited downstream kernels do not prove exact instruction selection for arbitrary tl.dot shapes.", + "claim_type": "compatibility", + "covers": ["u007", "u016"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [ + { + "command": "rg -n -i 'tcgen05|tmem|warp_specialize' artifacts/prs/vllm/PR-34597 artifacts/prs/sglang/PR-22079 artifacts/prs/sglang/PR-21019", + "environment": "GNU ripgrep over provenance-pinned downstream artifacts", + "reference": "An explicit lowering witness would name the backend mechanism or include emitted IR/PTX; a known upstream positive control must match the same search.", + "result": "No explicit lowering witness occurs in the three downstream bundles.", + "negative_control": "The identical terms match the pinned Triton v3.5.0 TCGen5 tutorial and the v3.3.0 compiler/test sources." + } + ], + "challenge": { + "alternative": "Every tl.dot in a post-boundary downstream kernel necessarily demonstrates TCGen5 emission.", + "check": "Search exact downstream source for target guards, backend names, IR, or PTX and compare to a known matching upstream control.", + "outcome": "The downstream files contain none of the discriminating witnesses while the controls do, so the page does not infer an unobserved lowering." + } + } + ], + "commands": [ + "git tag-tree comparison for Triton v3.2.0 through v3.7.1", + "conda run -n base python scripts/validate.py", + "conda run -n base python /users/Master/.codex/skills/kernel-wiki-verifier/scripts/scan_wiki.py . --output verification/current-queue.jsonl" + ], + "scope": "All 35 current units for body hash ea6b2fb7068dfa7b8a5d31fd9324937075bcbd313e86292a970f031a005a821c; Triton v3.2.0-v3.7.1 compiler/release boundary; exact downstream merge revisions; live FlashInfer-Bench rows retrieved 2026-08-08; local artifacts and links after remediation.", + "residual_risk": "No exact lowering is asserted for a downstream tl.dot kernel without matching IR/PTX; performance observations remain limited to their recorded source scopes." +} diff --git a/verification/reviewed/current/migration-register-to-tmem.json b/verification/reviewed/current/migration-register-to-tmem.json new file mode 100644 index 000000000..de6bacfcf --- /dev/null +++ b/verification/reviewed/current/migration-register-to-tmem.json @@ -0,0 +1,130 @@ +{ + "path": "wiki/migration/register-to-tmem.md", + "id": "migration-register-to-tmem", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "migration-register-to-tmem-current-c001", + "claim": "The Hopper description correctly gives the per-thread FP32 D fragment as N/2 registers for wgmma.m64nNk16, hence 128 at N=256, and identifies WGMMA asynchronous fence/commit/wait before dependent use.", + "claim_type": "mechanism", + "covers": ["u001"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"f32 D fragment and WGMMA async proxy operations","observation":"The fragment is N/2 f32 registers per thread and WGMMA defines fence, commit_group, and wait_group ordering/completion operations."}], + "challenge": {"alternative":"128 is a warpgroup total or WGMMA completes synchronously when issued.","check":"Inspect the per-thread fragment wording and async-proxy group operations.","outcome":"The count is explicitly per thread and dependent access requires the documented async lifecycle."} + }, + { + "claim_id": "migration-register-to-tmem-current-c002", + "claim": "The Blackwell description accurately gives TMEM's 128-by-512-by-32-bit CTA-visible structure and tcgen05 D residency while narrowing the benefit to removal of resident D registers, not elimination of all register, occupancy, spill, or tile constraints.", + "claim_type": "mechanism", + "covers": ["u002", "u003"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"TMEM organization and tcgen05 D","observation":"The dimensions/cell width and distinct TMEM D destination match; 128*512*4 bytes is 256 KiB."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2 Blackwell Tuning Guide","locator":"occupancy bullets","observation":"Compute capability 10.0 retains 64K 32-bit registers/SM and a 255-register/thread maximum, so the limitations are correctly preserved."}], + "challenge": {"alternative":"TMEM replaces the entire register file or guarantees a larger resident tile/CTA count.","check":"Separate the MMA D operand location from all other kernel live state and occupancy resources.","outcome":"Only D residency changes categorically; the page explicitly requires whole-kernel compiled evidence for the rest."} + }, + { + "claim_id": "migration-register-to-tmem-current-c003", + "claim": "All seven storage/lifecycle table rows correctly contrast WGMMA register D and scale-d/group completion with tcgen05 TMEM D, enable-input-d, commit/mbarrier, collective ld/wait, explicit deallocation, and disjoint storage for simultaneous live outputs.", + "claim_type": "mechanism", + "covers": ["u004", "u005", "u006", "u007", "u008", "u009", "u010", "u011"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-multiply-accumulate-instructions-wgmma-mma","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"WGMMA D, scale-d, commit/wait","observation":"Register D and false scale-d D=A*B semantics match."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"same archive","locator":"tcgen05 D, enable-input-d, commit, ld, waits, allocation","observation":"TMEM destination, false-predicate first write, completion/readback, and cleanup rows match the separate instruction contracts."}], + "challenge": {"alternative":"An instruction tile is always a CTA tile or two buffers can overlap the same TMEM columns safely.","check":"Inspect descriptor-defined instruction shapes and TMEM address/lifetime rules.","outcome":"The page explicitly rejects the tile equivalence and requires disjoint live regions."} + }, + { + "claim_id": "migration-register-to-tmem-current-c004", + "claim": "Lifecycle steps 1-2 accurately state one CTA-group mode, power-of-two column allocation in [32,512] across all 128 lanes, whole-warp cta_group::1 execution, shared taddr publication, and peer-liveness requirements for cta_group::2.", + "claim_type": "mechanism", + "covers": ["u012", "u013"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"allocation range/unit, issue granularity, shared result, peer and group rules","observation":"Every stated allocation and participation condition matches the normative section."}], + "challenge": {"alternative":"One elected lane can allocate because only one shared taddr is returned, or peer allocation can launch the second CTA implicitly.","check":"Read mandatory sync/aligned warp granularity and cta_group::2 peer activity conditions.","outcome":"All lanes of the designated warp participate and the peer must already be launched/live."} + }, + { + "claim_id": "migration-register-to-tmem-current-c005", + "claim": "Lifecycle steps 3-5 correctly distinguish barrier publication, false enable-input-d first-write semantics, valid single-thread MMA issue and descriptors, and tcgen05 commit/mbarrier completion from fence-only ordering.", + "claim_type": "mechanism", + "covers": ["u014", "u015", "u016"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"MMA grammar, issue granularity, D predicate and lifetimes","observation":"The descriptor/predicate/single-thread and D=A*B conditions match."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit","revision":"same archive","locator":"commit versus fence completion/ordering","observation":"Commit attaches completion to an mbarrier, while fences order execution without completing the asynchronous MMA."}], + "challenge": {"alternative":"Zero-storing all TMEM is mandatory or __syncthreads after a fence completes MMA.","check":"Use false enable-input-d semantics and the distinct commit/fence specifications.","outcome":"The predicate supplies first-write semantics and only the completion path licenses consumers/reuse."} + }, + { + "claim_id": "migration-register-to-tmem-current-c006", + "claim": "Lifecycle steps 6-7 correctly require collective legal tcgen05.ld plus wait::ld before register use, reader completion before overwrite/free, matching collective deallocation, and coverage of every kernel exit path.", + "claim_type": "mechanism", + "covers": ["u017", "u018"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"ld collectivity, asynchronous completion, wait and lifetimes","observation":"The readback requirements match."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"same archive","locator":"matching deallocation and before-exit obligation","observation":"All allocated TMEM must be deallocated collectively with matching address/count/group before exit."}], + "challenge": {"alternative":"Lane 0 may deallocate after issuing loads because wait::ld is warp-local and kernel exit releases the rest.","check":"Combine load lifetime with deallocation issue granularity and explicit-exit rule.","outcome":"All readers must finish and all lanes of the designated warp must execute deallocation; exit is not implicit cleanup."} + }, + { + "claim_id": "migration-register-to-tmem-current-c007", + "claim": "The six representative PTX forms are legal PTX ISA 9.0 forms and the surrounding warning accurately identifies the declarations, descriptors, participation, barrier, and inline-assembly context they omit.", + "claim_type": "code", + "covers": ["u019", "u020"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"alloc, unscaled f16 MMA, commit, 32x32b.x1 ld, wait::ld, and dealloc grammars","observation":"Every exact form is listed with the contextual constraints named by the page."}], + "challenge": {"alternative":"The forms compose into a runnable sequence as displayed or the MMA predicate may be a literal integer.","check":"Inspect required operand declarations/types and asynchronous/collective prerequisites.","outcome":"The page calls them representative forms only and explicitly denies standalone status."} + }, + { + "claim_id": "migration-register-to-tmem-current-c008", + "claim": "The overlap section gives the necessary compute-complete, read-complete, and nonaliasing stage-ownership edges and correctly rejects TMEM as a prerequisite for all tensor/non-matmul overlap by citing Hopper FA3.", + "claim_type": "mechanism", + "covers": ["u021", "u022", "u023", "u024", "u025"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"completion and operand lifetime rules","observation":"The three edges follow directly from asynchronous completion, TMEM read lifetime, and safe destination reuse."},{"url":"https://arxiv.org/abs/2407.08608","revision":"FlashAttention-3 paper","locator":"Hopper warp specialization and matmul/softmax interleaving","observation":"FA3 is a primary counterexample to the claim that register accumulators make overlap impossible."}], + "challenge": {"alternative":"Disjoint addresses alone make overlap safe or FA3 only overlaps TMA with MMA.","check":"Trace completion/reuse edges and read FA3's explicit matmul/softmax interleaving contribution.","outcome":"Disjoint storage does not supply completion, and FA3 explicitly overlaps tensor-core and non-matmul work on Hopper."} + }, + { + "claim_id": "migration-register-to-tmem-current-c009", + "claim": "The FA4 case study accurately reports TMEM score/output accumulators and specialized MMA/softmax/correction roles, software-selected polynomial exp2 work, and an author-reported up-to-1605-TFLOP/s B200 BF16 maximum labeled 71 percent without deriving universal resource claims.", + "claim_type": "performance", + "covers": ["u026"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/abs/2603.05451v1","revision":"FlashAttention-4 paper v1","locator":"forward architecture and TMEM mapping","observation":"The accumulator placement and warp roles match."},{"url":"https://tridao.me/blog/2026/flash4/","revision":"Tri Dao first-party post, fetched 2026-08-08","locator":"software exp and performance sections","observation":"The software-selected FMA polynomial and up-to-1605/71-percent author result match the scoped wording."}], + "challenge": {"alternative":"The result establishes a fixed register count or migration-only speedup for every FA workload.","check":"Inspect what the paper/post actually report versus omitted compiler resource rows and ablations.","outcome":"The page explicitly limits the number to a source-reported maximum and draws no universal resource conclusion."} + }, + { + "claim_id": "migration-register-to-tmem-current-c010", + "claim": "The retuning workflow correctly requires concrete compiler resource data, CUDA occupancy calculation, legal-shape and pipeline sweeps, controlled benchmarks, and generated-code inspection while rejecting a fixed 420-cycle tcgen05.ld theorem.", + "claim_type": "performance", + "covers": ["u027", "u028", "u029", "u030", "u031", "u032"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator","revision":"CUDA 13.0.2 Programming Guide","locator":"occupancy APIs and resource dependence","observation":"Occupancy requires actual launch/resource inputs rather than tile inference."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"PTX ISA 9.0","locator":"ld shape/semantics","observation":"The ISA defines legal collective mappings/completion but no universal latency."}], + "challenge": {"alternative":"ISA legality plus TMEM capacity is sufficient to choose the fastest tile without compiling or timing.","check":"Compare architecture legality with documented multi-resource occupancy and absence of timing guarantees.","outcome":"Legality cannot establish allocation, residency, spills, or workload performance; the measurement workflow is necessary."} + }, + { + "claim_id": "migration-register-to-tmem-current-c011", + "claim": "All six common-error bullets follow from the verified TMEM instruction-only address space, collective issue rules, distinct fence/completion semantics, enable-input-d first-write behavior, finite registers, and mandatory explicit deallocation.", + "claim_type": "mechanism", + "covers": ["u033", "u034", "u035", "u036", "u037", "u038"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"addressing, allocation, MMA, fence/commit, ld/st/wait and deallocation","observation":"The dedicated address model and each lifecycle rule directly support the warnings."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2 Blackwell Tuning Guide","locator":"finite register resources","observation":"Moving D does not establish whole-kernel no-spill behavior."}], + "challenge": {"alternative":"The bullets are merely conservative style preferences and lane-0/fence/implicit-exit forms are still legal shortcuts.","check":"Read the mandatory aligned collective, asynchronous completion, and explicit deallocation language.","outcome":"Those shortcuts violate normative behavior; only the performance-tuning warning remains empirical as stated."} + }, + { + "claim_id": "migration-register-to-tmem-current-c012", + "claim": "All eight primary URLs and three related local links resolve to the named archived sections, papers, and pages; fabricated external and local controls fail.", + "claim_type": "other", + "covers": ["u039", "u040", "u041", "u042", "u043", "u044", "u045", "u046", "u047", "u048", "u049"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"HTTP-check eight primary URLs; resolve three relative links and inspect target IDs; repeat with fabricated HTTP and local paths","environment":"current KernelWiki worktree and network, 2026-08-09","reference":"Targets should resolve to archived CUDA 13.0.2 sections, FA3/FA4 papers, hw-tmem, hw-tcgen05-mma, and pattern-register-pressure.","result":"All eight URLs returned HTTP 200 and all three files existed with expected IDs.","negative_control":"A fabricated CUDA archive page returned HTTP 404 and wiki/migration/not-register-to-tmem.md was absent."}], + "challenge": {"alternative":"Anchor redirects mask a missing document or a similarly named relative page is used.","check":"Require exact page HTTP success and inspect local frontmatter IDs against link intent.","outcome":"Every exact document/page resolved and both broken controls failed."} + } + ], + "commands": ["conda-base arithmetic checks for fragment and capacity formulas", "archived primary-source inspection", "external/local link checks with negative controls"], + "scope": "All 49 current units for body hash 7f0bb2f02e5a9f1f50c0c9003dc0baeeee3685876e3897726abbd12a14ae611a were checked against CUDA 13.0.2 PTX ISA 9.0, archived tuning guidance, FA3/FA4 primary sources, and host controls.", + "residual_risk": "No Hopper/Blackwell compiler or GPU is installed locally. The page intentionally contains no concrete occupancy or migration-speed claim; users must obtain compiler resources and target measurements for their kernel." +} diff --git a/verification/reviewed/current/migration-wgmma-to-tcgen05.json b/verification/reviewed/current/migration-wgmma-to-tcgen05.json new file mode 100644 index 000000000..78a6257aa --- /dev/null +++ b/verification/reviewed/current/migration-wgmma-to-tcgen05.json @@ -0,0 +1,177 @@ +{ + "path": "wiki/migration/wgmma-to-tcgen05.md", + "id": "migration-wgmma-to-tcgen05", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "migration-wgmma-to-tcgen05-current-c001", + "claim": "The migration comparison accurately maps issue granularity, accumulator and operand locations, completion, ordering, and narrow-format differences without assuming one WGMMA or tcgen05 A path.", + "claim_type": "compatibility", + "covers": ["u001", "u002", "u003", "u004", "u005", "u006", "u007", "u008", "u009"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-matrix-multiply-and-accumulate-operations", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "WGMMA operand/execution sections; tcgen05 issue, MMA grammar, kind, fence, and commit sections", + "observation": "Every current table cell matches the pinned instruction models, including both A-location alternatives and separate completion/ordering mechanisms." + } + ], + "challenge": { + "alternative": "The rewrite still assumes WGMMA register A or tcgen05 SMEM A universally.", + "check": "Compare the table and following warning with all SS/RS and SMEM/TMEM A grammar alternatives.", + "outcome": "The current page states both alternatives and makes porting conditional on the exact old form." + } + }, + { + "claim_id": "migration-wgmma-to-tcgen05-current-c002", + "claim": "The dependency-ordered checklist preserves the required operand inventory, legal instruction selection, collective TMEM lifecycle, source lifetime, completion, epilogue transfer, cta_group consistency, and workload retuning steps.", + "claim_type": "semantics", + "covers": ["u010", "u011", "u012", "u013", "u014", "u015", "u016", "u017"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "alloc/dealloc, MMA source lifetime, commit, ld, descriptor, and cta_group rules", + "observation": "The checklist follows the normative dependency order and confines performance-sensitive choices to explicit retuning." + } + ], + "challenge": { + "alternative": "The checklist releases A/B after issue, deallocates before epilogue, or treats fences as completion.", + "check": "Trace source lifetime, completion, load, and deallocation ordering through steps 3-7.", + "outcome": "Storage remains live through completion, epilogue transfer follows completion, and deallocation occurs before exit without an unsafe early placement." + } + }, + { + "claim_id": "migration-wgmma-to-tcgen05-current-c003", + "claim": "The TMEM lifecycle section correctly states the b32 shared-memory allocation result, power-of-two column constraints, warp-collective group-1 issue, two-warp group-2 issue, and mandatory deallocation before exit.", + "claim_type": "semantics", + "covers": ["u018", "u019"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "9.7.16.7.1 syntax, description, issue rules, and before-exit mandate", + "observation": "Each lifecycle statement follows directly from the allocation-management section and Table 46." + } + ], + "challenge": { + "alternative": "alloc is lane-0-issued like MMA, or CTA teardown implicitly handles deallocation.", + "check": "Inspect mandatory sync/aligned warp semantics and the explicit deallocate-before-exit sentence.", + "outcome": "The current page correctly rejects both alternatives." + } + }, + { + "claim_id": "migration-wgmma-to-tcgen05-current-c004", + "claim": "The synchronization section correctly assigns MMA completion to commit/mbarrier and cross-thread ordering/code-motion control to tcgen05 fences composed with execution-ordering operations.", + "claim_type": "semantics", + "covers": ["u020"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "mbarrier-based completion, specialized inter-thread synchronization, and fence description", + "observation": "The ISA gives the two mechanisms the distinct guarantees stated by the page." + } + ], + "challenge": { + "alternative": "fence plus __syncthreads drains the MMA pipeline and commit is optional.", + "check": "Compare the explicit completion list with fence's ordering-only contract.", + "outcome": "Only commit/mbarrier tracks MMA completion, matching the rewrite." + } + }, + { + "claim_id": "migration-wgmma-to-tcgen05-current-c005", + "claim": "The epilogue section accurately describes warp-access-restricted, asynchronous tcgen05.ld transfer into GPRs and the need for tcgen05.wait::ld when completion must be observed.", + "claim_type": "semantics", + "covers": ["u021"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "9.7.16.8.1 access restrictions, 9.7.16.8.3 tcgen05.ld, and wait-based completion", + "observation": "TMEM is lane-partitioned by warp, ld transfers to register vectors asynchronously, and wait::ld observes load completion." + } + ], + "challenge": { + "alternative": "Any single thread can read arbitrary TMEM lanes synchronously into scalar variables.", + "check": "Inspect issue/access restrictions and async completion for ld.", + "outcome": "The current warp-level description preserves the actual access model." + } + }, + { + "claim_id": "migration-wgmma-to-tcgen05-current-c006", + "claim": "The layout and shape guidance correctly treats 128B swizzling and maximum BF16 tiles as optional configuration examples rather than universal correctness or factor-of-two migration rules.", + "claim_type": "semantics", + "covers": ["u022", "u023"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "Table 40 swizzle modes; instruction descriptor and data-path layouts", + "observation": "The ISA defines multiple valid swizzles and shapes, with M/N selected through idesc under configuration-specific constraints." + } + ], + "challenge": { + "alternative": "Only 128B and M=128/256 are valid dense tcgen05 choices.", + "check": "Inspect alternative descriptor encodings and multiple layout organizations.", + "outcome": "The pinned tables directly support the current non-categorical wording." + } + }, + { + "claim_id": "migration-wgmma-to-tcgen05-current-c007", + "claim": "The role and register-budget guidance correctly separates single-thread MMA issue from total kernel thread needs and recommends retuning without guaranteeing occupancy gains.", + "claim_type": "mechanism", + "covers": ["u024", "u025"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/examples/cute/tutorial/blackwell/02_mma_tma_sm100.cu", + "revision": "CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "role partition, pipeline participants, and TMEM accumulator path", + "observation": "The example combines single-thread MMA issue with warp/CTA work for TMA, allocation, and accumulator consumption, demonstrating that total role counts remain a design variable." + } + ], + "challenge": { + "alternative": "Single-thread issue means a one-thread kernel is optimal or guarantees more resident CTAs.", + "check": "Compare issue granularity with all collective pipeline roles and resource constraints.", + "outcome": "The current page explicitly rejects that inference and recommends measurement." + } + }, + { + "claim_id": "migration-wgmma-to-tcgen05-current-c008", + "claim": "The CUTLASS guidance matches the pinned SM100 construction flow and exact schedule identifiers, and points to an existing complete example.", + "claim_type": "compatibility", + "covers": ["u026", "u027"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/examples/70_blackwell_gemm/70_blackwell_fp16_gemm.cu", + "revision": "CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "CollectiveBuilder, kernel::GemmUniversal, and device adapter; include/cutlass/gemm/dispatch_policy.hpp schedule tags", + "observation": "The current description follows the example's construction and both named 1SM/2SM schedule tags exist in the pinned dispatch-policy header." + } + ], + "challenge": { + "alternative": "A single nonexistent schedule alias still appears or the example uses device::GemmUniversal directly.", + "check": "Exact-token search the current names and trace the example's type chain.", + "outcome": "Only existing pinned names remain and the kernel/device layering is described accurately." + } + } + ], + "scope": "All 27 freshly rescanned current units were verified against CUDA 13.0.2 PTX ISA 9.0 and pinned CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5.", + "residual_risk": "No CUDA toolchain or NVIDIA GPU is available locally, so this is an evidence-scoped migration procedure rather than a compiled port. Workload-specific performance still requires target-hardware validation." +} diff --git a/verification/reviewed/current/pattern-compute-bound.json b/verification/reviewed/current/pattern-compute-bound.json new file mode 100644 index 000000000..24bcb582a --- /dev/null +++ b/verification/reviewed/current/pattern-compute-bound.json @@ -0,0 +1,13 @@ +{ + "claims":[ + {"challenge":{"alternative":"A fixed tensor-core utilization threshold and unsaturated DRAM are enough to classify the gap.","check":"Compare with official roofline and scheduler-analysis semantics.","outcome":"The guide requires relevant ceilings/intensity and distinguishes scheduler issue from sampled stalls; the page now does the same."},"claim":"Compute-ceiling under-attainment must be classified using the executed operation's relevant ceiling, arithmetic intensity, memory ceiling, whole-kernel time, and scheduler/source evidence rather than a fixed utilization threshold.","claim_id":"pattern-compute-bound-current-c001","claim_type":"performance","covers":["u001","u002"],"evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"roofline, Scheduler Statistics, Warp State Statistics","observation":"The guide defines roofline boundedness from intensity/ceilings and warns that sampled stalls alone need not limit performance."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"The inference table assigns causes from counter movement without controlling semantics or runtime.","check":"Inspect each row's counterfactual and the following control requirements.","outcome":"Every inference requires a matched change plus runtime improvement, and the page fixes math, coverage, environment, and records resource costs."},"claim":"The matched-control table makes only evidence-scoped inferences from one-variable changes that also improve end-to-end runtime, with fixed math/launch controls and explicit resource/profiler records.","claim_id":"pattern-compute-bound-current-c002","claim_type":"semantics","covers":["u003","u004","u005","u006","u007","u008"],"evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"baseline comparison, profiling sections, reproducibility","observation":"Nsight Compute supports matched baseline comparison and reports scheduler, resource, compute, and memory evidence; its documentation notes replay/reproducibility constraints."},{"url":"https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100","revision":"3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"v3-v6","observation":"Pinned variants embody the staged, role-separated, cooperative, and output-overlap mechanisms described as hypotheses."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"The candidate guidance still embeds universal stage counts, stall elimination, identical peer layouts, or 2x performance.","check":"Search the rewritten sections for absolutes and compare requirements with PTX and pinned source.","outcome":"The page rejects all four absolutes, gives the direct two-stage counterexample, and scopes cooperative mode to exact legal forms and measured aggregate/per-SM results."},"claim":"The candidate checks accurately scope pipeline depth, role specialization, TMEM output buffering, 2-SM MMA, fusion/interleaving, and approximation as controlled choices with correctness and resource costs rather than guaranteed fixes.","claim_id":"pattern-compute-bound-current-c003","claim_type":"compatibility","covers":["u009","u010","u011","u012"],"evidence":[{"url":"https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu","revision":"3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"two-stage launch","observation":"The beneficial cited pipeline uses two stages."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"PTX ISA 9.0","locator":"cta_group and constraints","observation":"Cooperative form legality depends on instruction kind/configuration and peer resources; the ISA supplies no universal shape or speedup."},{"url":"https://gau-nernst.github.io/tcgen05/","revision":"retrieved 2026-08-09","locator":"v4-v6","observation":"The tutorial shows role and output-overlap benefits in one setup alongside remaining idle time and added synchronization complexity."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"FA4 uses software exponentials universally or its peak isolates their benefit.","check":"Compare the selection algorithm, error analysis, hardware rates, and benchmark scope in paper/blog.","outcome":"The paper selects only a fraction for the cubic path and evaluates error; paper/blog peaks cover the coordinated full kernel with different stated maxima."},"claim":"The FA4 case accurately reports the B200/Hopper MMA and exponential rates, selected floor-reduced cubic software path, numerical-validation requirement, and full-kernel 1613/1605-TFLOP/s scope without single-technique attribution.","claim_id":"pattern-compute-bound-current-c004","claim_type":"performance","covers":["u013","u014","u015"],"evidence":[{"url":"https://arxiv.org/html/2603.05451v1","revision":"FlashAttention-4 paper v1","locator":"sections 2.2, partial software exponentiation, and results","observation":"The paper gives 8192/4096 MMA rates, equal 16-op exponential rates, the selected polynomial method/error analysis, and up to 1613 TFLOP/s at 71% for complete FA4."},{"url":"https://tridao.me/blog/2026/flash4/","revision":"Tri Dao FA4 blog, 2026","locator":"techniques and performance","observation":"The blog reports up to 1605 TFLOP/s at 71% for the complete design and lists multiple contributing mechanisms."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"The reference labels are unpinned or do not support the sections they follow.","check":"Resolve all five links and inspect their subject/revision.","outcome":"They lead to the official profiler guide, archived PTX ISA, full-SHA tutorial source, v1 paper, and author blog used above."},"claim":"The reference list resolves to the primary profiler, ISA, pinned implementation, paper, and author-blog evidence used by the page.","claim_id":"pattern-compute-bound-current-c005","claim_type":"semantics","covers":["u016","u017","u018","u019","u020"],"evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"2025.3","locator":"document","observation":"Official profiler guide."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html","revision":"PTX ISA 9.0","locator":"tcgen05","observation":"Official archived ISA."},{"url":"https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu","revision":"3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"file","observation":"Pinned tutorial source."},{"url":"https://arxiv.org/html/2603.05451v1","revision":"v1","locator":"paper","observation":"Versioned FA4 paper."},{"url":"https://tridao.me/blog/2026/flash4/","revision":"2026","locator":"post","observation":"FA4 author blog."}],"oracle":"web-primary","verdict":"verified"} + ], + "commands":["Inspect profiler roofline/scheduler guidance","Inspect PTX tcgen05 forms","Inspect tutorial v3-v6 and FA4 paper/blog"], + "id":"pattern-compute-bound","page_verdict":"verified","path":"wiki/patterns/compute-bound.md", + "residual_risk":"The decision procedure is source-verified but not run because no target GPU/profiler is available. Performance values remain author-reported.", + "scope":"All 20 current units for hash e08627fe8f79b7e82f8eb53531d3aa22ddb2a98858cae22ae1389d0084a145c3 were challenged against primary profiler, ISA, source, paper, and blog evidence." +} diff --git a/verification/reviewed/current/pattern-low-sm-utilization.json b/verification/reviewed/current/pattern-low-sm-utilization.json new file mode 100644 index 000000000..83d3aa681 --- /dev/null +++ b/verification/reviewed/current/pattern-low-sm-utilization.json @@ -0,0 +1,14 @@ +{ + "claims":[ + {"challenge":{"alternative":"The definition merely replaces one arbitrary threshold with subjective wording.","check":"Inspect whether the page requires measured useful work, profitable demand, time scope, and separates occupancy from activity.","outcome":"It requires all four and lists the records needed to distinguish shortage, tails, imbalance, residency, and phases."},"claim":"Low SM utilization is defined as materially insufficient useful active work relative to profitable demand, with theoretical occupancy explicitly separated from grid supply, achieved activity, balance, and temporal behavior.","claim_id":"pattern-low-sm-utilization-current-c001","claim_type":"semantics","covers":["u001","u002"],"evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"occupancy, PM sampling, scheduler analysis","observation":"The guide distinguishes occupancy limits, temporal sampling, and achieved scheduler/activity metrics, with no universal utilization cutoff."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"The four cases overlap so much that the table cannot discriminate them.","check":"Test each row's evidence against independent-work count, equal-duration remainder, per-worker duration, and resource/phase records.","outcome":"Each row uses a different observable/control, and the prose states that aggregate activity alone cannot choose among them."},"claim":"The page correctly separates insufficient independent work, equal-duration wave quantization, variable-duration imbalance, and residency/serial-phase limitations with discriminating evidence and controlled next experiments.","claim_id":"pattern-low-sm-utilization-current-c002","claim_type":"semantics","covers":["u003","u004","u005","u006","u007"],"evidence":[{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html","revision":"retrieved 2026-08-09","locator":"fixed-work and fixed-worker models","observation":"CUDA distinguishes problem-sized blocks from resident workers acquiring repeated work and motivates reassignment under uneven availability."},{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0","locator":"scheduler model","observation":"The scheduler documentation distinguishes grid IDs, persistent work acquisition, rasterization, and cluster behavior, supporting the page's case separation."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"CLC creates work or SM120 lacks it, while persistence/raster/decomposition are interchangeable remedies.","check":"Compare each mechanism and PTX target note with the page.","outcome":"The page keeps all four controls separate, limits CLC to existing unlaunched IDs, and accurately reports sm_100-or-higher plus explicit sm_120 multicast support."},"claim":"Persistence, coordinate order, CLC reassignment, and K decomposition are distinct controls; CLC redistributes existing unlaunched IDs without guaranteeing balance, and PTX ISA 9.0 does not categorically exclude SM120.","claim_id":"pattern-low-sm-utilization-current-c003","claim_type":"compatibility","covers":["u008","u009","u010","u011","u012","u013","u014"],"evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"PTX ISA 9.0","locator":"request semantics and Target ISA Notes","observation":"CLC cancels an unspecified not-yet-started grid entity, requires sm_100 or higher, and lists sm_120a/sm_120f for multicast."},{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0","locator":"SM100 scheduler","observation":"The pinned library route is SM100-specific and separates software rasterization from CLC acquisition."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"The v5-to-v6 improvement is a CLC or isolated tail-elimination result.","check":"Inspect source labels, closing remarks, and v6 roles/pipeline.","outcome":"The article labels v6 static persistence, explicitly excludes CLC, includes pipeline/role changes, and says overlap is imperfect."},"claim":"The tutorial case accurately reports v5 1302.29 and v6 1475.93 TFLOP/s, their 86.43% and 97.96% ratios, and scopes v6 to cumulative static persistence/output-pipeline changes without CLC or an all-SMs-busy claim.","claim_id":"pattern-low-sm-utilization-current-c004","claim_type":"performance","covers":["u015"],"evidence":[{"url":"https://gau-nernst.github.io/tcgen05/","revision":"retrieved 2026-08-09","locator":"v5/v6 results and closing remarks","observation":"The article reports the exact values against 1506.74 cuBLAS, names static scheduling, says CLC was not added, and describes imperfect overlap."},{"url":"https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu","revision":"3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"v6 implementation","observation":"The pinned source implements static work iteration and specialized output/mainloop coordination without CLC instructions."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"The checklist can pass while losing or duplicating CLC coordinates or conflating decomposition with acquisition.","check":"Inspect coverage, worker instrumentation, held controls, and reporting requirements.","outcome":"The procedure explicitly proves no missing/duplicate work, holds decomposition fixed for scheduler tests, and tests decomposition separately."},"claim":"The verification procedure establishes exact work coverage, worker demand and tail arithmetic, per-worker imbalance, matched scheduler controls, and end-to-end/resource reporting.","claim_id":"pattern-low-sm-utilization-current-c005","claim_type":"semantics","covers":["u016","u017","u018","u019","u020"],"evidence":[{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html","revision":"retrieved 2026-08-09","locator":"CLC work lifecycle","observation":"The programming model requires processing initial work and only successfully canceled returned work, making exact coverage and acquisition controls necessary."},{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0","locator":"CLC scheduler lifecycle","observation":"CUTLASS provides the initial/returned work paths and failure handling the checklist validates."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"The references mix PDL with CLC or omit the exact tutorial source.","check":"Resolve all five links.","outcome":"They point to CUDA CLC, PTX CLC, CUTLASS CLC, pinned v6, and the first-party tutorial; the unrelated PR-2161 source was removed."},"claim":"The reference list contains direct primary CLC contracts, pinned scheduler documentation/source, and the first-party tutorial result.","claim_id":"pattern-low-sm-utilization-current-c006","claim_type":"semantics","covers":["u021","u022","u023","u024","u025"],"evidence":[{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html","revision":"retrieved 2026-08-09","locator":"page","observation":"Official CLC guide."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"PTX ISA 9.0","locator":"section","observation":"Version-pinned CLC ISA."},{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0","locator":"file","observation":"Pinned CLC scheduler documentation."},{"url":"https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu","revision":"3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"file","observation":"Pinned v6 source."},{"url":"https://gau-nernst.github.io/tcgen05/","revision":"retrieved 2026-08-09","locator":"article","observation":"First-party progression."}],"oracle":"web-primary","verdict":"verified"} + ], + "commands":["Inspect PTX/CUDA CLC target and lifecycle","Inspect CUTLASS 4.5.0 scheduler docs","Inspect tutorial v5/v6 source and results"], + "id":"pattern-low-sm-utilization","page_verdict":"verified","path":"wiki/patterns/low-sm-utilization.md", + "residual_risk":"No target-side utilization trace was collected because no GPU/profiler is available. Tutorial performance is author-reported.", + "scope":"All 25 current units for hash 2c8aa46a8f96c06bc50d29d566037068eabf9c65e7c5d513b52cae6efc50c993 were challenged against primary CLC, scheduler, source, and profiling evidence." +} diff --git a/verification/reviewed/current/pattern-memory-bound.json b/verification/reviewed/current/pattern-memory-bound.json new file mode 100644 index 000000000..7c7048ea7 --- /dev/null +++ b/verification/reviewed/current/pattern-memory-bound.json @@ -0,0 +1,13 @@ +{ + "claims":[ + {"challenge":{"alternative":"The page still classifies from a vague high-bandwidth observation or algorithm name.","check":"Inspect the definition, required records, and four evidence cases against roofline semantics.","outcome":"Classification requires measured work/bytes, a relevant attained ceiling, traffic efficiency, issue evidence, and whole-kernel scope; labels are explicitly insufficient."},"claim":"The page correctly defines memory-bandwidth boundedness from measured arithmetic intensity and an attained relevant memory ceiling, and separates saturation, excess traffic, latency/issue limitation, and phase-local bandwidth using distinct evidence.","claim_id":"pattern-memory-bound-current-c001","claim_type":"performance","covers":["u001","u002","u003","u004","u005","u006","u007"],"evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"roofline, Memory Workload Analysis, scheduler analysis","observation":"The guide relates work/bytes to ceilings and exposes transferred traffic, cache, throughput, and scheduler evidence needed for these distinctions."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"Vector width, cache hints, register caps, multicast, or swizzle are presented as guaranteed bandwidth fixes.","check":"Search each section for guarantees and compare with ISA/compiler contracts.","outcome":"Each is conditional, controlled, and costed; hints remain hints, caps may spill, multicast requires shared operands, and swizzle is scoped to a concrete shared-memory trace."},"claim":"The candidate procedures accurately constrain vector loads, cache hints, register caps, TMA multicast, and swizzling to legal forms and matched experiments with their alignment, traffic, resource, reuse, cluster, and layout limitations.","claim_id":"pattern-memory-bound-current-c002","claim_type":"compatibility","covers":["u008","u009","u010","u011","u012"],"evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld","revision":"PTX ISA 9.0","locator":"natural alignment and cache operators","observation":"PTX specifies vector alignment and hint semantics without performance guarantees."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html","revision":"CUDA Compiler Driver 13.0.2","locator":"maxrregcount","observation":"Register caps are subject to ABI minima and may create spills."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"PTX ISA 9.0","locator":"multicast and tensor-map swizzle","observation":"PTX defines cluster destination masks, completion, swizzle modes, and their configuration constraints."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"The negative controls support a new universal claim that wide loads and caps are harmful.","check":"Inspect the page's inference scope.","outcome":"It reports exact source regressions/no-effect only for cited shapes and says they refute a fixed priority, not prove general harm."},"claim":"The Amandeep case accurately reports a 16-25% wider-load regression and no effect from an 80-to-64 register cap, using them only to reject category-level prescriptions and motivate measured compute/memory hypotheses.","claim_id":"pattern-memory-bound-current-c003","claim_type":"performance","covers":["u013","u014"],"evidence":[{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"retrieved 2026-08-08","locator":"wider-load and register-cap attempts, Takeaways","observation":"The first-party report gives those exact negative controls and recommends profiling the actual bottleneck."},{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"scheduler and memory sections","observation":"The profiler separates issue/latency evidence from achieved memory throughput, supporting the conditional compute-work discussion."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"The reproduction record omits a factor needed to distinguish cache state, nominal roof, or measurement noise.","check":"Compare it with the profiler and source experiment fields.","outcome":"It includes inputs, useful work/bytes, cache/warmup/statistic, hardware/software/clocks, generated code/resources, roof assumptions, achieved traffic/issue, correctness, and regressions."},"claim":"The reproduction record is sufficient to scope a memory-roof diagnosis and its controlled candidate variants without turning a nominal specification or one endpoint into causal evidence.","claim_id":"pattern-memory-bound-current-c004","claim_type":"semantics","covers":["u015"],"evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"reproducibility, roofline, memory and scheduler sections","observation":"The listed fields cover the environment, traffic, resource, issue, correctness, and comparison factors used by the official analyses."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"A reference label points to a secondary or unrelated page.","check":"Resolve all four URLs.","outcome":"They are the official profiler guide, archived PTX cache and multicast sections, and the cited first-party attempt report."},"claim":"The primary-reference list resolves directly to the profiler, version-pinned ISA, and first-party negative-control evidence used by the page.","claim_id":"pattern-memory-bound-current-c005","claim_type":"semantics","covers":["u016","u017","u018","u019"],"evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"2025.3","locator":"document","observation":"Official profiler guide."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html","revision":"PTX ISA 9.0","locator":"ld and cp.async.bulk.tensor","observation":"Official archived ISA contains both linked sections."},{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"retrieved 2026-08-08","locator":"article","observation":"First-party NVFP4 GEMV attempt report."}],"oracle":"web-primary","verdict":"verified"} + ], + "commands":["Inspect roofline/memory/scheduler profiler semantics","Inspect PTX cache, vector, TMA, and swizzle contracts","Inspect NVFP4 GEMV negative controls"], + "id":"pattern-memory-bound","page_verdict":"verified","path":"wiki/patterns/memory-bound.md", + "residual_risk":"The workflow was not run on target hardware because no GPU/profiler is available. Attempt results remain author-reported.", + "scope":"All 19 current units for hash df62414848150544dd22c908b1f015e7c3925bad2512fe91186cfc9edffc8e8b were challenged against profiler, ISA, compiler, and first-party experiment evidence." +} diff --git a/verification/reviewed/current/pattern-moe-load-imbalance.json b/verification/reviewed/current/pattern-moe-load-imbalance.json new file mode 100644 index 000000000..dbe2b536f --- /dev/null +++ b/verification/reviewed/current/pattern-moe-load-imbalance.json @@ -0,0 +1,14 @@ +{ + "claims":[ + {"challenge":{"alternative":"Token-count balance alone diagnoses the kernel, or one expert maps to one physical SM.","check":"Compare CUDA block placement and grouped tiling with every level in the measurement record.","outcome":"The page rejects fixed ownership and requires expert, tile, worker/SM, rank, and phase timing evidence before attribution."},"claim":"The diagnosis correctly separates routed-token, valid-row/tile, logical-worker/SM, rank-communication, and phase timing imbalance, treats small-M effects as configuration-specific, and rejects routing/loss/batch proxies as sufficient negative tests.","claim_id":"pattern-moe-load-imbalance-current-c001","claim_type":"performance","covers":["u001","u002","u003","u004","u005","u006","u007","u008","u009"],"evidence":[{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html","revision":"retrieved 2026-08-09","locator":"thread-block scheduling","observation":"Physical block placement is dynamic and does not imply one expert per SM."},{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/efficient_gemm.md","revision":"CUTLASS 4.5.0","locator":"small GEMM dimensions","observation":"Small dimensions can create predicated lanes and too few blocks, with no universal M threshold."},{"url":"https://www.lmsys.org/blog/2025-05-05-large-scale-ep/","revision":"published 2025-05-05, retrieved 2026-08-09","locator":"EPLB case studies and limitations","observation":"The study measures load/balance and throughput by phase and configuration rather than treating generic routing conditions as proof."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"Contiguous always means an offsets array, masked computes M-max padding, or K-grouped is a forward variable-M layout.","check":"Inspect all three pinned README contracts and their API assertions.","outcome":"The metadata forms, valid-M semantics, fixed axes, alignment, and weight-backward K grouping match the table exactly."},"claim":"The DeepGEMM table accurately distinguishes contiguous M grouping with two metadata forms and aligned segments, masked M grouping with fixed allocation but valid-portion compute for a graph decode case, and variable-K/fixed-MN grouping for weight backward, while prescribing appropriate controls.","claim_id":"pattern-moe-load-imbalance-current-c002","claim_type":"semantics","covers":["u010","u011","u012","u013","u014"],"evidence":[{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"Grouped GEMMs contiguous, masked, and K-grouped","observation":"The README gives each use, layout, fixed/variable axis, alignment, metadata, and valid-compute scope."},{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp","revision":"same commit","locator":"m_grouped and k_grouped API checks","observation":"The exact tensor ranks, axes, and integer metadata are enforced by the pinned API."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"Static ownership fixes a physical SM, CLC creates or steals arbitrary work, every tile pays a small request cost, or CLC excludes SM120.","check":"Trace CUDA placement, PTX request/target rules, and CUTLASS's initial and later work paths.","outcome":"The page preserves logical versus physical placement, existing-grid cancellation/failure, static initial work, pipelining/cluster granularity, and the broader ISA target boundary."},"claim":"The scheduler section correctly separates static persistence, CLC-backed reassignment, and work decomposition; states the exact request/failure/existing-grid limits and matched-test controls; and distinguishes PTX sm_100-or-higher/SM120-family support from CUTLASS 4.5.0's SM100 integration.","claim_id":"pattern-moe-load-imbalance-current-c003","claim_type":"compatibility","covers":["u015","u016","u017","u018","u019","u020","u021","u022","u023","u024"],"evidence":[{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html","revision":"retrieved 2026-08-09","locator":"thread-block scheduling","observation":"CUDA dynamically places blocks."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"request, response, and Target ISA Notes","observation":"The ISA defines cancellation of unlaunched work, possible failure, cluster forms, sm_100-or-higher support, and SM120-family multicast."},{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0","locator":"static scheduler, dynamic scheduler, and CLC pipeline","observation":"Initial blockIdx work is static; later requests are asynchronous, pipelined, and configuration-dependent."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"EPLB is a device scheduler or the two speedups transfer to arbitrary deployments and distributions.","check":"Inspect the pinned planner inputs/outputs and the measurement study's setup and limitations.","outcome":"The code performs host-side replica placement across configured GPUs/nodes; the values belong to a 96-H100 in-distribution ablation whose authors warn about distribution shifts."},"claim":"The EPLB section accurately scopes the pinned project as load-driven expert replication and GPU/node placement, not device tile scheduling, and scopes the 1.49x prefill and 2.54x decode values to the LMSYS/SGLang 96-H100 in-distribution study.","claim_id":"pattern-moe-load-imbalance-current-c004","claim_type":"performance","covers":["u025","u026"],"evidence":[{"url":"https://github.com/deepseek-ai/EPLB/blob/d52c72d5b2f2fb4c41afbf8eb21366820239913d/eplb.py","revision":"EPLB d52c72d5b2f2fb4c41afbf8eb21366820239913d","locator":"replicate_experts and rebalance_experts","observation":"The code takes logical-expert loads and emits physical/logical maps and replica counts across configured nodes and GPUs."},{"url":"https://www.lmsys.org/blog/2025-05-05-large-scale-ep/","revision":"published 2025-05-05, retrieved 2026-08-09","locator":"Ablation Study: EPLB and limitations","observation":"The source reports the two values in its large-scale deployment and explicitly notes in-distribution data and untested shifts."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"The exploit skipped correctness work, the reported time is a valid balancing result, or the incident is documented as causing the FlashInfer evaluator design.","check":"Separate the postmortem's correctness/timing paths and named response from the pinned later contest contract.","outcome":"The exact 15-object/120-group cache sequence is retained, the timing is rejected as invalid, PR 104 is the named response, and neither source states the later causality."},"claim":"The reward-hack section accurately records the real correctness path, fifteen-clone/120-group timing merge and cached calls, rejects the result as load-balancing performance evidence, and does not invent a causal link to the later FlashInfer contest.","claim_id":"pattern-moe-load-imbalance-current-c005","claim_type":"mechanism","covers":["u027","u028"],"evidence":[{"url":"https://www.gpumode.com/news/reward-hacking-nvfp4","revision":"retrieved 2026-08-09","locator":"correctness, timing exploit, and response","observation":"The official postmortem supplies the sequence and names reference-kernels PR 104."},{"url":"https://github.com/flashinfer-ai/flashinfer-bench-starter-kit/blob/75ccd05cafceb0fd1f86be4cd0f2117249463c66/EVALUATION.md","revision":"starter kit 75ccd05cafceb0fd1f86be4cd0f2117249463c66","locator":"evaluation contract","observation":"The later contract does not attribute its design to the GPU Mode event."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"One or more references are rolling, secondary, or omit the exact mechanism/revision claimed.","check":"Resolve every listed target and compare its revision and scope with the adjacent text.","outcome":"The list points to official CUDA/PTX, pinned CUTLASS/DeepGEMM/EPLB/evaluator revisions, and the two first-party incident/deployment reports used with explicit scope."},"claim":"The eight primary-reference entries resolve to the direct scheduling, CLC, grouped-layout, EPLB, scoped performance, reward-hack, and evaluator sources used by the page.","claim_id":"pattern-moe-load-imbalance-current-c006","claim_type":"semantics","covers":["u029","u030","u031","u032","u033","u034","u035","u036"],"evidence":[{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html","revision":"retrieved 2026-08-09","locator":"document","observation":"Official scheduling documentation."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"section","observation":"Archived CLC contract."},{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0","locator":"file","observation":"Pinned scheduler documentation."},{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md","revision":"891d57b4","locator":"file","observation":"Pinned grouped contracts."},{"url":"https://github.com/deepseek-ai/EPLB/tree/d52c72d5b2f2fb4c41afbf8eb21366820239913d","revision":"d52c72d5","locator":"tree","observation":"Pinned planner source."},{"url":"https://www.lmsys.org/blog/2025-05-05-large-scale-ep/","revision":"published 2025-05-05","locator":"article","observation":"Scoped deployment study."},{"url":"https://www.gpumode.com/news/reward-hacking-nvfp4","revision":"retrieved 2026-08-09","locator":"article","observation":"Official incident report."},{"url":"https://github.com/flashinfer-ai/flashinfer-bench-starter-kit/blob/75ccd05cafceb0fd1f86be4cd0f2117249463c66/EVALUATION.md","revision":"75ccd05c","locator":"file","observation":"Pinned contest evaluator."}],"oracle":"web-primary","verdict":"verified"} + ], + "commands":["Inspect CUDA/PTX scheduling and CLC contracts","Inspect pinned grouped-layout and EPLB sources","Challenge performance and history against scoped first-party records"], + "id":"pattern-moe-load-imbalance","page_verdict":"verified","path":"wiki/patterns/moe-load-imbalance.md", + "residual_risk":"No GPU or distributed deployment is available locally, so the page gives a discriminating measurement protocol and preserves external performance values only in their disclosed scope.", + "scope":"All 36 current units for hash 39a1e4e3fb4709726fa3f81e7e26b68f6c0936ca10fa151101aee424b9f283f0 were challenged against direct versioned scheduling, layout, placement, performance, and evaluator evidence." +} diff --git a/verification/reviewed/current/pattern-pipeline-stalls.json b/verification/reviewed/current/pattern-pipeline-stalls.json new file mode 100644 index 000000000..a0eaa8f5e --- /dev/null +++ b/verification/reviewed/current/pattern-pipeline-stalls.json @@ -0,0 +1,60 @@ +{ + "claims": [ + { + "challenge": {"alternative":"Low unit activity or barrier-wait samples are sufficient to diagnose a pipeline bottleneck.","check":"Compare the page's qualification with Nsight Compute's sampling and stall guidance.","outcome":"The profiler documents sampled readiness states and warns that stalls need not affect performance; the page requires scheduler under-issue, correlation, and a causal test."}, + "claim": "Low execution-unit activity and barrier-wait samples are diagnostic leads rather than causal proof; Nsight Compute stall data should be interpreted with scheduler issue evidence, available chip-specific metrics, and sampling limitations.", + "claim_id": "pattern-pipeline-stalls-current-c001", + "claim_type": "semantics", + "covers": ["u001", "u002"], + "evidence": [{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"Scheduler Statistics, Warp State Statistics, PM and warp sampling","observation":"The guide distinguishes issue/eligibility statistics from sampled warp states, cautions that stalls may not affect performance, documents queryable metric availability, and says warp sampling has no time resolution."}], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": {"alternative":"The page still confuses TMA transaction completion with tcgen05 ordering or prescribes one phase update for every wait.","check":"Trace each checklist item to PTX mbarrier counters, wait validity/visibility, tcgen05 completion, and fence semantics.","outcome":"The rewritten checklist assigns TMA bytes to complete_tx, phase to each reused object, MMA completion to commit/mbarrier, and tcgen05 fences only to applicable cross-thread execution-ordering handoffs."}, + "claim": "The correctness gate accurately separates mbarrier arrival and transaction-byte accounting, per-object phase tracking, TMA acquire visibility, asynchronous tcgen05 completion, cross-thread tcgen05 ordering, and pipeline boundary handling.", + "claim_id": "pattern-pipeline-stalls-current-c002", + "claim_type": "semantics", + "covers": ["u003", "u004", "u005", "u006", "u007", "u008", "u009", "u010", "u011"], + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier","revision":"PTX ISA 9.0","locator":"mbarrier lifecycle, arrive, complete_tx, and waits","observation":"PTX defines separate arrival/transaction counters, current-or-prior phase state/parity, and acquire visibility for associated bulk copies."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence","revision":"PTX ISA 9.0","locator":"tcgen05 completion and execution ordering","observation":"PTX distinguishes commit/mbarrier completion from before/after-thread-sync ordering across execution-ordering operations."}], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": {"alternative":"The workflow merely relabels pipeline stages, warp specialization, buffering, and ping-pong as guaranteed fixes.","check":"Inspect whether each row states a falsifiable target, controls, and costs and whether the workflow rejects failed predictions.","outcome":"Every change is conditional on a concrete edge, holds relevant variables fixed, records resource costs, and is rejected when predicted wait/time movement does not occur."}, + "claim": "The workflow diagnoses pipeline stalls through reproducible timing, profiler correlation, an edge-specific prediction, and one-variable tests of stage depth, role separation, TMEM buffering, or two-tile interleaving with explicit correctness and resource controls.", + "claim_id": "pattern-pipeline-stalls-current-c003", + "claim_type": "semantics", + "covers": ["u012", "u013", "u014", "u015", "u016", "u017", "u018", "u019", "u020", "u021"], + "evidence": [{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"profiling sections and reproducibility","observation":"The profiler exposes the stated section categories and documents replay, sampling, and reproducibility considerations."},{"url":"https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100","revision":"3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"v3-v6 implementations","observation":"The pinned versions demonstrate concrete stage, role, 2-SM, persistent, and output-overlap configurations with nontrivial storage and synchronization state."},{"url":"https://arxiv.org/html/2603.05451v1","revision":"FlashAttention-4 paper v1","locator":"section 3.1.2","observation":"The paper supplies the concrete two-output-tile attention case and describes maximized, not guaranteed, overlap."}], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": {"alternative":"The table still mislabels stage count, rounds the ratios incorrectly, or attributes v6 to CLC.","check":"Compare every value and version label with the article, inspect pinned v3 instantiation, and recompute each ratio from 1506.74.","outcome":"All six TFLOP/s values and labels match; v3 uses two stages, v6 says static scheduling, CLC is explicitly absent, and recomputation gives the displayed ratios to two decimals."}, + "claim": "For the tutorial's disclosed B200 4096-cubed experiment, the page accurately reports v1a through v6 values and cuBLAS ratios, identifies v3 as a two-stage source, and scopes the cumulative v6 endpoint to static persistence without CLC.", + "claim_id": "pattern-pipeline-stalls-current-c004", + "claim_type": "performance", + "covers": ["u022", "u023", "u024", "u025", "u026", "u027", "u028", "u029"], + "evidence": [{"url":"https://gau-nernst.github.io/tcgen05/","revision":"retrieved 2026-08-09","locator":"version result tables and closing progression","observation":"The article reports cuBLAS 1506.74; v1a 254.62; v2b 695.43; v3 939.61; v4 1208.83; v5 1302.29; and v6 static persistence 1475.93 TFLOP/s, and says CLC was not added. Dividing each kernel value by the stated cuBLAS value yields the displayed ratios."},{"url":"https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu","revision":"3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"matmul_v3 function","observation":"The v3 launch instantiates NUM_STAGES=2."}], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": {"alternative":"One or more primary-reference labels point to unrelated or unpinned material.","check":"Resolve each listed URL and compare its subject or revision with the label.","outcome":"The links resolve respectively to Nsight Compute 2025.3 profiling, PTX ISA 9.0 mbarrier, PTX ISA 9.0 tcgen05 fence, pinned tutorial v3 source, and the first-party tutorial progression."}, + "claim": "The primary-reference list links to the exact profiler guide, version-pinned mbarrier and tcgen05 contracts, pinned tutorial v3 source, and first-party progression used by the page.", + "claim_id": "pattern-pipeline-stalls-current-c005", + "claim_type": "semantics", + "covers": ["u030", "u031", "u032", "u033", "u034"], + "evidence": [{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"document title and contents","observation":"Official kernel profiling guide."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html","revision":"PTX ISA 9.0","locator":"mbarrier and tcgen05 fence sections","observation":"Official version-pinned ISA contains both linked contracts."},{"url":"https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu","revision":"3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"file","observation":"Pinned tutorial v3 implementation."},{"url":"https://gau-nernst.github.io/tcgen05/","revision":"retrieved 2026-08-09","locator":"article","observation":"First-party tutorial and result progression."}], + "oracle": "web-primary", + "verdict": "verified" + } + ], + "commands": ["Inspect PTX ISA 9.0 mbarrier and tcgen05 handoff contracts", "Inspect Nsight Compute 2025.3 profiler semantics", "Inspect pinned tutorial source and recompute ratios in Conda base"], + "id": "pattern-pipeline-stalls", + "page_verdict": "verified", + "path": "wiki/patterns/pipeline-stalls.md", + "residual_risk": "No target GPU or Nsight Compute installation is available locally, so the diagnostic procedure was source-verified but not executed. Tutorial performance remains author-reported.", + "scope": "All 34 current body units for hash 0c5294d705b6402ce30d3482e6d7f3bb2cb0ca6bf75e81712347ff1d23959d45 were challenged against profiler, ISA, paper, source, and arithmetic evidence." +} diff --git a/verification/reviewed/current/pattern-register-pressure.json b/verification/reviewed/current/pattern-register-pressure.json new file mode 100644 index 000000000..b318241a6 --- /dev/null +++ b/verification/reviewed/current/pattern-register-pressure.json @@ -0,0 +1,13 @@ +{ + "claims":[ + {"challenge":{"alternative":"Spill evidence alone proves register pressure, or maximum occupancy is always desirable.","check":"Compare the definition and required record with profiler occupancy/resource semantics.","outcome":"The page requires a register/spill constraint on useful scheduling or material traffic and a causal variant, while explicitly rejecting a universal occupancy target."},"claim":"The page correctly diagnoses performance-relevant register pressure from compiled allocation/spills, limiting residency and scheduler/traffic evidence for the exact binary, rather than from below-target occupancy or source complexity alone.","claim_id":"pattern-register-pressure-current-c001","claim_type":"performance","covers":["u001","u002","u003"],"evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"Launch Statistics, Occupancy, scheduler and source analysis","observation":"The profiler separates registers/spills, residency limit, achieved warps, scheduler states, and source-correlated generated instructions."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"Warp specialization is SM100-only and a register cap necessarily raises residency.","check":"Inspect Hopper specialization docs and compiler cap semantics.","outcome":"The page identifies Hopper support and makes both role separation and caps conditional on compiled-resource/runtime evidence, including spill and ABI risks."},"claim":"The live-range, role-specialization, and register-cap procedures correctly include Hopper availability, compiler/resource dependence, ABI minima, spill risk, and matched runtime validation.","claim_id":"pattern-register-pressure-current-c002","claim_type":"compatibility","covers":["u004","u005"],"evidence":[{"url":"https://docs.nvidia.com/cutlass/media/docs/cpp/hopper_gemm.html","revision":"retrieved 2026-08-09","locator":"warp-specialized Hopper GEMM","observation":"CUTLASS documents warp specialization on Hopper."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html","revision":"CUDA Compiler Driver 13.0.2","locator":"maxrregcount","observation":"The cap is subject to ABI minimums and can cause spills rather than a guaranteed residency result."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"The 128-register Hopper arithmetic or raw 256-KB TMEM size proves exactly 128 registers are freed and every large output double-buffers.","check":"Compare fragment mapping, TMEM organization, tcgen05 loads, packing, and lifecycle.","outcome":"Only the Hopper N/2 arithmetic is exact; the page explicitly retains Blackwell descriptor/barrier/load/epilogue registers and configuration-specific TMEM columns."},"claim":"The Hopper m64n256 FP32 WGMMA fragment is exactly 128 registers/thread for D, while the SM100 tcgen05 migration places resident D in 128x512 TMEM but retains configuration-specific columns, register work, collective lifecycle, completion, load/wait, and target constraints.","claim_id":"pattern-register-pressure-current-c003","claim_type":"semantics","covers":["u006","u007","u008"],"evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma","revision":"PTX ISA 9.0","locator":"m64nNk16 D mapping","observation":"FP32 D contributes N/2 registers per warpgroup thread."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory","revision":"PTX ISA 9.0","locator":"organization and lifecycle","observation":"TMEM is 128 lanes by512 32-bit cells with collective allocation/deallocation and kind/packing-sensitive use."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"PTX ISA 9.0","locator":"collective load and wait","observation":"TMEM-to-register loads are collective asynchronous operations with explicit completion."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"TMEM load cost is guaranteed to be offset by occupancy, so only register counts need reporting.","check":"Inspect whether the page claims or measures the balance.","outcome":"It calls the balance empirical and requires loads/waits, resource changes, achieved warps, stalls, correctness, and time including no-occupancy-change cases."},"claim":"The evaluation correctly treats TMEM-load/epilogue cost versus long-lived register relief as an empirical whole-kernel tradeoff requiring resource, scheduler, correctness, and timing evidence.","claim_id":"pattern-register-pressure-current-c004","claim_type":"performance","covers":["u009","u010"],"evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"PTX ISA 9.0","locator":"load completion","observation":"The ISA establishes asynchronous load/wait cost but no performance-offset guarantee."},{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"resource, occupancy, scheduler, and source sections","observation":"The requested record covers the observable tradeoff dimensions."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"The links do not pin the accumulator or load contracts.","check":"Resolve all five reference targets.","outcome":"They point to the official profiler, archived PTX WGMMA/TMEM/load sections, and a full-SHA tutorial tree."},"claim":"The references directly support profiler diagnosis, WGMMA fragment arithmetic, TMEM organization/lifecycle, tcgen05 loads, and the pinned SM100 implementation example.","claim_id":"pattern-register-pressure-current-c005","claim_type":"semantics","covers":["u011","u012","u013","u014","u015"],"evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"2025.3","locator":"document","observation":"Official profiler guide."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html","revision":"PTX ISA 9.0","locator":"WGMMA and tcgen05/TMEM","observation":"Official version-pinned instruction contracts."},{"url":"https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100","revision":"3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"tree","observation":"Pinned implementation source."}],"oracle":"web-primary","verdict":"verified"} + ], + "commands":["Inspect profiler register/occupancy semantics","Inspect WGMMA fragment and TMEM/tcgen05 contracts","Inspect Hopper specialization and compiler cap docs"], + "id":"pattern-register-pressure","page_verdict":"verified","path":"wiki/patterns/register-pressure.md", + "residual_risk":"No target compilation/profile was possible, so compiled resource outcomes remain a prescribed experiment rather than a local result.", + "scope":"All 15 current units for hash 6411aba77dd98964d25ec7fa3a3e7f97e70f5400886e930aaa77f6b2072c7621 were challenged against primary profiler, compiler, ISA, and pinned-source evidence." +} diff --git a/verification/reviewed/current/pattern-tail-effect.json b/verification/reviewed/current/pattern-tail-effect.json new file mode 100644 index 000000000..8a3814e77 --- /dev/null +++ b/verification/reviewed/current/pattern-tail-effect.json @@ -0,0 +1,13 @@ +{ + "claims":[ + {"challenge":{"alternative":"The r=0 case has an empty final wave, or 142 is asserted as B200 hardware.","check":"Apply Euclidean division and inspect the stated scope around W.","outcome":"The page explicitly treats r=0 as a full last wave, computes 8/142 correctly only as a declared model, and says W need not equal physical SM count."},"claim":"For equal-duration independent work, T=qW+r gives q full waves and, only when r>0, a final r-worker wave; T=150,W=142 yields 5.63%, while real worker capacity and fractional impact require measurement and no four-wave cutoff.","claim_id":"pattern-tail-effect-current-c001","claim_type":"code","covers":["u001","u002","u003","u004","u005"],"evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0","locator":"data-parallel work-ID model","observation":"The scheduler's independent grid-ID model supports the explicitly simplified arithmetic; Euclidean division verifies the counts and 8/142 value."},{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"occupancy and workload distribution","observation":"Actual residency/activity depends on launch resources and observed execution rather than physical SM count alone."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"Aggregate utilization and tile-count modulo alone are enough, and grid stride fixes physical-SM ownership.","check":"Inspect the required sweep/timeline and CUDA block-scheduling contract.","outcome":"The page requires a predicted sawtooth plus time-resolved end interval and separates stragglers; it correctly says blockIdx does not fix physical placement."},"claim":"The diagnosis procedure verifies a last-wave cause through worker/residency records, predicted remainder, an end-of-run underfilled interval, nearby-size sawtooth behavior, and separation from unequal-duration stragglers, while preserving CUDA's dynamic block placement.","claim_id":"pattern-tail-effect-current-c002","claim_type":"semantics","covers":["u006","u007","u008","u009","u010","u011"],"evidence":[{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html","revision":"retrieved 2026-08-09","locator":"thread-block scheduling","observation":"CUDA assigns blocks to available SMs without a programmer-visible blockIdx-to-SM mapping."},{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"PM sampling and workload distribution","observation":"Time-resolved metrics and resource/activity sections provide the evidence requested by the procedure."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"Persistence removes tails, CLC creates work, raster balances workers, or CLC excludes SM120.","check":"Compare every bullet and compatibility statement with PTX/CUTLASS semantics.","outcome":"All absolutes are rejected; mechanisms remain separate, CLC only redistributes unlaunched IDs, and SM120-family target support is stated accurately."},"claim":"The page accurately separates static persistence, CLC reassignment, coordinate transforms, and K decomposition, including their finite-work/reduction costs, matched-test controls, and PTX sm_100-or-higher/SM120-family compatibility scope.","claim_id":"pattern-tail-effect-current-c003","claim_type":"compatibility","covers":["u012","u013","u014","u015","u016"],"evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"PTX ISA 9.0","locator":"request and target semantics","observation":"CLC cancels existing unlaunched work, may fail, requires sm_100 or higher, and lists SM120-family multicast support."},{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0","locator":"scheduler, raster, and work-ID model","observation":"The pinned implementation distinguishes persistent acquisition and software coordinate transforms on SM100."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"The tutorial has 142 SMs or demonstrates CLC/all-SMs-busy causality.","check":"Inspect hardware count, v5/v6 labels, source, and closing remarks.","outcome":"It says 148 SMs, static persistence, no CLC, imperfect overlap, and cumulative pipeline changes; the page reports exactly that."},"claim":"The tutorial scope correctly identifies 148 SMs, v5 1302.29 and v6 1475.93 TFLOP/s (86.43% and 97.96%), and rejects CLC, perfect-overlap, and isolated-tail attribution.","claim_id":"pattern-tail-effect-current-c004","claim_type":"performance","covers":["u017"],"evidence":[{"url":"https://gau-nernst.github.io/tcgen05/","revision":"retrieved 2026-08-09","locator":"persistent section and closing progression","observation":"The source supplies the hardware count, values, static label, no-CLC note, and imperfect-overlap caveat."},{"url":"https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu","revision":"3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"v6","observation":"Pinned implementation uses static repeated work and contains no CLC request."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"The references omit the exact scheduling or result evidence.","check":"Resolve all five links.","outcome":"They point to CUDA scheduling, PTX CLC, pinned CUTLASS CLC, pinned v6, and the first-party results."},"claim":"The primary references directly support scheduling, CLC, pinned implementation, and tutorial performance scope.","claim_id":"pattern-tail-effect-current-c005","claim_type":"semantics","covers":["u018","u019","u020","u021","u022"],"evidence":[{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html","revision":"retrieved 2026-08-09","locator":"document","observation":"Official scheduling model."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"PTX ISA 9.0","locator":"section","observation":"Pinned CLC contract."},{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0","locator":"file","observation":"Pinned scheduler documentation."},{"url":"https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu","revision":"3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"file","observation":"Pinned v6 source."},{"url":"https://gau-nernst.github.io/tcgen05/","revision":"retrieved 2026-08-09","locator":"article","observation":"First-party results."}],"oracle":"web-primary","verdict":"verified"} + ], + "commands":["Check analytical wave arithmetic","Inspect CUDA/PTX/CUTLASS scheduling semantics","Inspect tutorial v6 and progression"], + "id":"pattern-tail-effect","page_verdict":"verified","path":"wiki/patterns/tail-effect.md", + "residual_risk":"The model is deliberately simplified and no target timeline was collected; real worker capacity and duration variance remain experiment inputs.", + "scope":"All 22 current units for hash 015951e658cf7ca107772efab40ffc056cd6152f9c5c7864a4d0b59dcc1df455 were challenged against arithmetic, scheduler, ISA, implementation, and result evidence." +} diff --git a/verification/reviewed/current/technique-cache-policy.json b/verification/reviewed/current/technique-cache-policy.json new file mode 100644 index 000000000..4b2f0da12 --- /dev/null +++ b/verification/reviewed/current/technique-cache-policy.json @@ -0,0 +1,110 @@ +{ + "claims": [ + { + "challenge": { + "alternative": "The qualifier names define mandatory bypass, immediate eviction, persistent residence, and completed prefetch behavior.", + "check": "Trace the load/store forms through the PTX grammar, description, alignment rule, and performance-hint language.", + "outcome": "All displayed forms are legal 16-byte accesses, while the normative contract remains eviction priority that may apply or a performance-only prefetch hint; the page preserves that distinction." + }, + "claim": "PTX 9.0 exposes load/store L1 eviction priorities and load-side L2 prefetch/cache-policy hints without guaranteeing placement, residence, eviction time, hit rate, or performance; the displayed v4.u32 fragments are legal and require 16-byte alignment.", + "claim_id": "technique-cache-policy-current-c001", + "claim_type": "code", + "covers": ["u001", "u002", "u003", "u004", "u005", "u006", "u007", "u008", "u009"], + "evidence": [ + { + "locator": "6.4.1.1 Addresses as Operands, 9.7.9.8 ld, and 9.7.9.12 st", + "observation": "The archived ISA grammar accepts the displayed vector and eviction forms, requires natural alignment to total access size, says eviction policies may be applied, and treats prefetch/cache policies as hints that do not alter consistency.", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "A memory-bound label, tensor role, or working-set-size threshold is sufficient to choose the policy without an ablation.", + "check": "Compare the ISA's absence of selection rules with interference/reuse counterexamples and the page's controlled procedure.", + "outcome": "The ISA supplies no tensor-role prescription, and concurrency changes effective reuse; the procedure holds co-varying choices fixed and measures each production shape." + }, + "claim": "The five-step policy-selection procedure validly starts from concrete warp addresses, reuse and interference, then controls correctness, generated code/resources, profiler evidence, and repeated timing; size or bottleneck labels alone do not predict a useful hint.", + "claim_id": "technique-cache-policy-current-c002", + "claim_type": "semantics", + "covers": ["u010", "u011", "u012", "u013", "u014", "u015", "u016"], + "evidence": [ + { + "locator": "ld cache qualifiers and performance scope", + "observation": "The ISA specifies legal cache controls and hint semantics but no workload category, cache-capacity threshold, tensor name, or GEMM-axis rule that establishes benefit.", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld" + }, + { + "locator": "Attempts 8-12 and Takeaways", + "observation": "The author's same-task experiments show ineffective and regressing plausible transformations and recommend profiling the concrete bottleneck before choosing one.", + "revision": "retrieved 2026-08-08", + "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "The post-event observations reveal rank-1 code and exact per-K policy choices.", + "check": "Separate what the author directly reports from missing contestant implementation and ablation evidence.", + "outcome": "The report names common techniques and co-varying differences but publishes neither the inspected contestant code nor rank-1 per-K qualifier mapping; the page states only that boundary." + }, + "claim": "Amandeep Singh reports A no_allocate and B evict_last among several co-varying techniques in three inspected solutions but provides no released contestant code, cache-only ablation, or evidence for the former rank-1 per-K attribution.", + "claim_id": "technique-cache-policy-current-c003", + "claim_type": "performance", + "covers": ["u017"], + "evidence": [ + { + "locator": "Post-event Observations", + "observation": "The cited report lists cache policies together with PTX decode, wide loads, specialization, register caps, and reuse; it supplies no source or isolated timings for the inspected contestants.", + "revision": "retrieved 2026-08-08", + "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/" + }, + { + "locator": "public task and leaderboard boundary", + "observation": "The public materials expose the task and scores rather than contestant source or technique metadata.", + "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6", + "url": "https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "The 39-to-27 microsecond endpoints are a cache-policy ablation or show that cache policy dominated.", + "check": "Compare both endpoint stage descriptions and enumerate every changed mechanism.", + "outcome": "The first stage already combines staging, tiling, float4, and conversion changes, while the second switches the vectorized PTX load/decode path; the page denies cache-only attribution." + }, + "claim": "Yue Zhang's approximately 39-to-27 microsecond endpoints change multiple load, staging, tiling, conversion, and PTX-decode mechanisms and therefore do not measure a cache-hint-only speedup or dominant cache-policy effect.", + "claim_id": "technique-cache-policy-current-c004", + "claim_type": "performance", + "covers": ["u018"], + "evidence": [ + { + "locator": "CUDA optimization 2 and 3", + "observation": "The author gives the same endpoints and full combined changes, with no cache-policy-only variant or causal decomposition.", + "revision": "retrieved 2026-08-08", + "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html" + } + ], + "oracle": "web-primary", + "verdict": "verified" + } + ], + "commands": [ + "Inspect archived PTX 9.0 load/store eviction, prefetch, cache-policy, vector, and alignment contracts", + "Challenge workload-selection rules against the ISA scope and same-task negative controls", + "Separate direct author reports from unsupported contestant and causal attribution" + ], + "id": "technique-cache-policy", + "page_verdict": "verified", + "path": "wiki/techniques/cache-policy.md", + "residual_risk": "The PTX fragments were checked against archived syntax but not assembled or profiled because this environment has no CUDA toolkit or GPU. Performance claims remain explicitly scoped to author reports and missing ablations.", + "scope": "All 18 current body units, the frontmatter hint claim, code fragments, procedure, and report-provenance boundaries were checked against direct versioned evidence." +} diff --git a/verification/reviewed/current/technique-cccl-memory-primitives.json b/verification/reviewed/current/technique-cccl-memory-primitives.json new file mode 100644 index 000000000..bcaf76acc --- /dev/null +++ b/verification/reviewed/current/technique-cccl-memory-primitives.json @@ -0,0 +1,40 @@ +{ + "path": "wiki/techniques/cccl-memory-primitives.md", + "id": "technique-cccl-memory-primitives", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "technique-cccl-memory-primitives-current-c001", + "claim": "The page accurately scopes PR 3559 to SM100 scan policies/dispatch/tests and PR 6152 to a debug-only DispatchTopK logging correction, excluding DSA or TopK performance conclusions from the latter.", + "claim_type": "compatibility", + "covers": ["u001", "u002", "u003", "u004"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/prs/cccl/PR-3559/diff.patch","revision":"CCCL captured merge 25523da2","locator":"complete changed-file patch","observation":"The patch adds scan type/policy plumbing and SM100 scan tunings across the stated files."},{"path":"artifacts/prs/cccl/PR-6152/diff.patch","revision":"CCCL captured merge 3fb05826","locator":"complete patch","observation":"Only the debug format, scalar grid argument and occupancy variable name change; no TopK algorithm or tuning changes."}], + "challenge": {"alternative":"The surrounding captured TopK key file makes PR 6152 valid evidence for the entire algorithm.","check":"Distinguish key-file context from added/deleted lines in the PR patch.","outcome":"The semantic delta is only logging; the page explicitly preserves that distinction."} + }, + { + "claim_id": "technique-cccl-memory-primitives-current-c002", + "claim": "The PR 3559 policy table correctly lists type/operator classification, threads/items, load/store/cache choices, delay constructors, and Policy1000-to-Policy900 fallback, and rejects items>=4 as proof of vectorization.", + "claim_type": "mechanism", + "covers": ["u005", "u006", "u007", "u008", "u009", "u010", "u011"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/prs/cccl/PR-3559/key-files/cub/cub/device/dispatch/tuning/tuning_scan.cuh","revision":"CCCL captured merge 25523da2","locator":"sm100_tuning specializations, select_agent_policy100 and Policy1000","observation":"Every listed policy field/classifier/fallback appears in the source; the double path explicitly derives from sm90_tuning and item count is independent of load/store/cache policy."}], + "challenge": {"alternative":"CUB may internally vectorize whenever items is at least four, making the deleted boolean a reliable proxy.","check":"Compare specializations with similar item counts and inspect independent BlockLoadAlgorithm/iterator/type dispatch.","outcome":"Item count neither encodes alignment/contiguity nor fixes the selected load algorithm; the page's non-implication is correct."} + }, + { + "claim_id": "technique-cccl-memory-primitives-current-c003", + "claim": "The transfer workflow follows the source's semantic/type/architecture dispatch dimensions, demands controlled correctness/performance evaluation, enumerates selection properties absent from PR 6152, and explicitly withholds conclusions for unrelated primitives.", + "claim_type": "performance", + "covers": ["u012", "u013", "u014", "u015", "u016", "u017"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/prs/cccl/PR-3559/key-files/cub/cub/device/dispatch/tuning/tuning_scan.cuh","revision":"CCCL captured merge 25523da2","locator":"classification, SM80/SM90/SM100 policies, tuning comments and fallbacks","observation":"The independent dimensions and scoped benchmark comments support exactly the proposed controlled matrix, without a portable speed guarantee."},{"path":"artifacts/prs/cccl/PR-6152/diff.patch","revision":"CCCL captured merge 3fb05826","locator":"all changed lines","observation":"No selection semantics, tie/ordering/NaN/repeatability or architecture result is changed or established, justifying the explicit evidence exclusions."}], + "challenge": {"alternative":"The workflow smuggles in unsupported claims by listing correctness cases not mentioned in either PR.","check":"Read each item as a verification requirement versus an asserted behavior and compare the final evidence exclusions.","outcome":"The page does not predict outcomes; it identifies what must be tested and states that PR 6152 establishes none of them."} + } + ], + "commands": ["inspect complete captured CCCL 3559/6152 diffs and scan/TopK key files"], + "scope": "All 17 current semantic units for body hash d1c2e2981a7cbb20812788c814381f9f5a67edcec521a6497f68034d83483b8f were checked against complete captured PR deltas and key files.", + "residual_risk": "No GPU benchmark was run; the page reports policy structure and a test protocol, not new performance results." +} diff --git a/verification/reviewed/current/technique-chunk-parallelism.json b/verification/reviewed/current/technique-chunk-parallelism.json new file mode 100644 index 000000000..ffec8004c --- /dev/null +++ b/verification/reviewed/current/technique-chunk-parallelism.json @@ -0,0 +1,132 @@ +{ + "claims": [ + { + "challenge": { + "alternative": "Chunk IDs in one GPU grid execute in increasing order, so every program can update one shared boundary state directly.", + "check": "Compare independent program-instance scheduling with the recurrence's ordered boundary dependency.", + "outcome": "The programming model supplies no grid order or implicit grid-wide barrier; the page requires an explicit scan or ordered stages and a token-reference oracle." + }, + "claim": "A chunkwise linear recurrence can expose parallel local matrix work only after a model-specific equivalent transform, while boundary states must preserve sequence order through a valid scan or explicitly ordered stages rather than an unordered shared-state grid.", + "claim_id": "technique-chunk-parallelism-current-c001", + "claim_type": "semantics", + "covers": ["u001", "u002", "u003", "u004", "u005", "u006", "u007"], + "evidence": [ + { + "locator": "Abstract", + "observation": "TFLA starts from a chunkwise-parallel formulation of linear RNNs and adds within-chunk sequence parallelism, supporting the separation between local work and recurrent boundary semantics.", + "revision": "arXiv:2503.14376v3", + "url": "https://arxiv.org/abs/2503.14376v3" + }, + { + "locator": "program instances and execution model", + "observation": "Triton programs are independent grid instances; the model does not promise increasing program-ID scheduling or an implicit whole-grid synchronization point.", + "revision": "Triton language documentation, retrieved 2026-08-09", + "url": "https://triton-lang.org/main/programming-guide/chapter-01/introduction.html" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "GatedDeltaNet chunking is equivalent to a generic softmax-style QK-transpose/V computation with an additive state.", + "check": "Inspect the pinned training implementation and recurrence representation.", + "outcome": "The repository uses model-specific chunkwise Triton paths and a WY representation of the gated delta rule; the page states that boundary and does not present generic substitute code." + }, + "claim": "The pinned GatedDeltaNet training implementation uses chunkwise Triton kernels and a WY representation, providing direct model-specific evidence but no universal generic-attention formula or chunk-size rule.", + "claim_id": "technique-chunk-parallelism-current-c002", + "claim_type": "mechanism", + "covers": ["u008"], + "evidence": [ + { + "locator": "chunkwise training paths and WY preparation", + "observation": "The repository implements the gated delta rule with dedicated chunkwise Triton kernels and WY transforms rather than the former additive pseudo-kernel.", + "revision": "NVlabs/GatedDeltaNet b53d6d3a161267432a79c1c04af69fa52bddc921", + "url": "https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "TFLA is a recursive Blackwell/GatedDeltaNet implementation whose arbitrary chunks come from TMEM capacity.", + "check": "Inspect the paper's stated algorithm, application, repository target, and benchmark scope.", + "outcome": "The paper adds one within-chunk sequence-parallel level, applies it to mLSTM, and reports H100 work; the page explicitly excludes the unsupported Blackwell, GDN, TMEM, and recursive claims." + }, + "claim": "TFLA adds an additional within-chunk sequence-parallel level for arbitrarily large chunks and reduced intermediate-state materialization, but its published application/code is mLSTM with H100 evidence rather than GatedDeltaNet or a Blackwell/TMEM implementation.", + "claim_id": "technique-chunk-parallelism-current-c003", + "claim_type": "compatibility", + "covers": ["u009"], + "evidence": [ + { + "locator": "Abstract and code link", + "observation": "The authors state the arbitrary-chunk, intensity, and materialization contributions, identify mLSTM as the application, and publish corresponding official code and H100 benchmarks.", + "revision": "arXiv:2503.14376v3; NX-AI/mlstm_kernels 5b98ff8e2bec189b3d3c249405bab5149564d6f8", + "url": "https://arxiv.org/abs/2503.14376v3" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "Qwen3-Next's 3:1 description is approximate, or the same chunkwise backend covers its full-attention layers.", + "check": "Expand the pinned 48-layer type pattern and distinguish linear_attention from full_attention entries.", + "outcome": "The pattern yields exactly 36 GDN and 12 full-attention layers, and the page assigns chunkwise GDN only to the former." + }, + "claim": "The pinned Qwen3-Next 48-layer configuration contains 36 Gated DeltaNet linear-attention and 12 full-attention layers in a repeated 3:1 pattern, so a chunkwise GDN backend does not replace the full-attention implementation or cache.", + "claim_id": "technique-chunk-parallelism-current-c004", + "claim_type": "compatibility", + "covers": ["u010"], + "evidence": [ + { + "locator": "config.json layer_types and num_hidden_layers", + "observation": "The immutable configuration repeats three linear_attention entries and one full_attention entry across 48 layers.", + "revision": "Qwen3-Next-80B-A3B-Instruct 9c7f2fbe84465e40164a94cc16cd30b6999b0cc7", + "url": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/config.json" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "C=32 and C=256-512 are architecture-independent decode/prefill optima, with larger Blackwell TMEM always favoring larger chunks.", + "check": "Search the cited algorithms for fixed prescriptions and enumerate correctness, resource, scan, and workload variables changed by chunk size.", + "outcome": "The cited sources provide no universal sizes or TMEM rule; the page removes them and defines a controlled multi-shape selection procedure." + }, + "claim": "No cited evidence establishes universal C=32 or C=256-512 optima; supported chunk sizes must be compared under fixed correctness and benchmarking controls across algorithm, resource, tail, scan, and workload costs, and TMEM availability alone predicts neither legality nor speed.", + "claim_id": "technique-chunk-parallelism-current-c005", + "claim_type": "performance", + "covers": ["u011", "u012", "u013", "u014", "u015", "u016", "u017", "u018"], + "evidence": [ + { + "locator": "Abstract and implementation scope", + "observation": "TFLA claims support for arbitrarily large chunks in its model-specific algorithm but supplies no fixed decode/prefill optimum or Blackwell/TMEM selection rule.", + "revision": "arXiv:2503.14376v3", + "url": "https://arxiv.org/abs/2503.14376v3" + }, + { + "locator": "pinned GDN implementation variants", + "observation": "The concrete GDN paths encode model dimensions, layouts, gates, and variable-length handling, demonstrating that these are implementation inputs rather than consequences of architecture name alone.", + "revision": "NVlabs/GatedDeltaNet b53d6d3a161267432a79c1c04af69fa52bddc921", + "url": "https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921" + } + ], + "oracle": "web-primary", + "verdict": "verified" + } + ], + "commands": [ + "Inspect TFLA v3 and pinned official mLSTM code scope", + "Inspect pinned GatedDeltaNet chunkwise/WY paths", + "Challenge unordered shared-state pseudocode against Triton's program model", + "Expand the immutable Qwen3-Next layer pattern" + ], + "id": "technique-chunk-parallelism", + "page_verdict": "verified", + "path": "wiki/techniques/chunk-parallelism.md", + "residual_risk": "No GPU was available to benchmark candidate chunk sizes. The page deliberately provides a verification procedure and source-scoped implementation boundaries rather than a tuned size or executable kernel.", + "scope": "All 18 current body units, the frontmatter Blackwell claim, sequence-ordering invariant, model/repository scopes, and strongest alternatives were checked against direct versioned sources." +} diff --git a/verification/reviewed/current/technique-double-buffering.json b/verification/reviewed/current/technique-double-buffering.json new file mode 100644 index 000000000..cb26be541 --- /dev/null +++ b/verification/reviewed/current/technique-double-buffering.json @@ -0,0 +1,100 @@ +{ + "path": "wiki/techniques/double-buffering.md", + "id": "technique-double-buffering", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "technique-double-buffering-current-c001", + "claim": "The page correctly defines buffering as disjoint stage ownership plus completion/reuse edges, distinguishes independent SMEM and TMEM uses, and scopes their simultaneous use to a concrete pinned SM100 kernel.", + "claim_type": "mechanism", + "covers": ["u001", "u002", "u003", "u004", "u005"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v6.cu","revision":"gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"NUM_STAGES arrays, BLOCK_N*2 allocation, TMA/MMA/epilogue role loops and barriers","observation":"The implementation contains separate multi-stage SMEM storage and disjoint ping-pong TMEM regions with explicit ownership handoffs."}], + "challenge": {"alternative":"The kernel only double-buffers one address space or uses the two copies sequentially without overlap.","check":"Trace stage indices, TMEM offsets, and independently executing warp-role loops.","outcome":"Different address spaces and regions are active under separate producer/MMA/epilogue roles; the page does not generalize beyond this example."} + }, + { + "claim_id": "technique-double-buffering-current-c002", + "claim": "The TMEM geometry, optional 256/256 address split, and disjoint-live-region invariant agree with PTX ISA 9.0 while avoiding a false universal logical-element capacity.", + "claim_type": "code", + "covers": ["u006", "u007", "u008"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"organization, addressing, allocation, and packing","observation":"The ISA defines 128 lanes, 512 columns, 32-bit cells, column-wide allocation and instruction-dependent packing; the displayed half-open ranges are arithmetically disjoint."}], + "challenge": {"alternative":"Every two-buffer implementation must divide the structure into equal halves or 65,536 cells always mean 65,536 FP32 outputs.","check":"Separate allocation/address geometry from implementation policy and instruction-kind packing.","outcome":"The page explicitly calls the split optional and refuses a universal logical-element count."} + }, + { + "claim_id": "technique-double-buffering-current-c003", + "claim": "The four TMEM state transitions and lifecycle paragraph correctly require compute completion before reads, collective load completion before reuse, phase-aware barriers, collective allocation/address publication, and matching deallocation before exit.", + "claim_type": "mechanism", + "covers": ["u009", "u010", "u011", "u012", "u013", "u014"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"allocation/deallocation collective and lifetime rules","observation":"The allocation, publication, matching collective cleanup, and before-exit conditions match."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"same archive","locator":"collective load and wait completion","observation":"tcgen05.ld is asynchronous and its register result requires wait::ld before use; source lifetimes extend through completion."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier","revision":"same archive","locator":"phase and arrival semantics","observation":"Reusable barriers operate in phases with expected arrivals, matching the page's reusable-protocol warning."}], + "challenge": {"alternative":"A fence or CTA barrier alone licenses readback, and kernel exit implicitly releases TMEM.","check":"Distinguish ordering from tcgen05 commit/completion and inspect the explicit deallocation obligation.","outcome":"The normative instructions require completion tracking and matching deallocation; the page retains both edges."} + }, + { + "claim_id": "technique-double-buffering-current-c004", + "claim": "The SMEM full/empty sequence and FlashInfer case accurately describe per-stage producer/consumer ownership; the example's three- and up-to-four-stage policies are source-scoped rather than universal defaults.", + "claim_type": "mechanism", + "covers": ["u015", "u016", "u017", "u018", "u019", "u023"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/prs/flashinfer/PR-2387/key-files/include/flashinfer/mamba/selective_state_update.cuh","revision":"flashinfer merge 18804cd51734cccf807356d017733bc757677f15; SHA-256 40a2de44b1f29b4b4c213e74e530131c9a43be5e602daa3c8bbd901a3b281446","locator":"SharedStorage/SharedStorageHorizontal, producer and consumer loops, dispatch near numStages","observation":"The merged source has numStages storage and full/empty barriers, waits before produce/consume/reuse, three stages in one path, and min(totalStages,4) in another."}], + "challenge": {"alternative":"The full/empty names are cosmetic and no transaction-byte or phase handoff occurs.","check":"Trace barrier_arrive_tx, waits, and arrive operations for one stage around its TMA and consumer accesses.","outcome":"The source explicitly attaches bytes to full, waits on matching stages, and returns empty ownership after consumption."} + }, + { + "claim_id": "technique-double-buffering-current-c005", + "claim": "The binary16 stage-size block computes exactly 16 KiB for A, 32 KiB for B, 48 KiB per stage, and 144 KiB for three unpadded stages while disclaiming a complete C++ layout.", + "claim_type": "code", + "covers": ["u020", "u021"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python exact integer computation for 128*64*2, 64*256*2, and three-stage totals","environment":"Conda base, host CPU","reference":"binary16 payload uses two bytes per element","result":"16384 B = 16 KiB, 32768 B = 32 KiB, 49152 B = 48 KiB, and 147456 B = 144 KiB.","negative_control":"Four-byte elements produce 32/64/96/288 KiB and therefore do not match the labeled binary16 example."}], + "challenge": {"alternative":"The total silently includes barriers, padding and epilogue storage.","check":"Compare the formula's named unpadded payload with the immediately following disclaimer.","outcome":"Only operand payload is counted, and all omitted allocation components are explicitly named."} + }, + { + "claim_id": "technique-double-buffering-current-c006", + "claim": "The resource caveat correctly scopes 228 KiB shared memory per SM to compute capability 10.0 and distinguishes it from per-block opt-in and other residency limits.", + "claim_type": "compatibility", + "covers": ["u022"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2 Blackwell Tuning Guide","locator":"compute capability 10.0 shared-memory and occupancy bullets","observation":"The guide reports 228 KiB per SM and describes occupancy as dependent on multiple resource limits."}], + "challenge": {"alternative":"228 KiB is available to every Blackwell CTA or applies to compute capability 12.0 as well.","check":"Read the table/bullet architecture scope and per-block opt-in discussion.","outcome":"The page restricts the figure to compute capability 10.0 and avoids a per-CTA or all-Blackwell guarantee."} + }, + { + "claim_id": "technique-double-buffering-current-c007", + "claim": "The Hopper/Blackwell comparison correctly computes WGMMA FP32 D as N/2 registers per thread, distinguishes resident D from whole-kernel register use, and rejects fixed occupancy conclusions from buffering alone.", + "claim_type": "mechanism", + "covers": ["u024", "u025", "u026", "u027", "u028", "u029"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"m64nNk16 f32 D fragment","observation":"Each thread has N/2 f32 D registers, giving 128 for N=256 and 256 for two simultaneous fragments."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"same archive","locator":"TMEM D and register load destination","observation":"tcgen05 D resides at a TMEM address and loads produce per-thread registers, supporting the scoped distinction."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator","revision":"CUDA 13.0.2 Programming Guide","locator":"occupancy resource calculation","observation":"Actual residency requires complete compiled/launch resource inputs, matching the table's resource-dependent wording."}], + "challenge": {"alternative":"Two fragments require only 64 registers per thread or TMEM removes every register associated with output handling.","check":"Substitute N=256 in the ISA fragment formula and inspect tcgen05.ld destination/state requirements.","outcome":"The exact result is 256 f32 D registers for two Hopper fragments; Blackwell removes resident D only, exactly as stated."} + }, + { + "claim_id": "technique-double-buffering-current-c008", + "claim": "The controlled-comparison workflow uses concrete compiler, occupancy, profiler, generated-code, wraparound, tail, and negative-test evidence and correctly denies that PTX defines a fixed mbarrier latency.", + "claim_type": "performance", + "covers": ["u030", "u031", "u032", "u033", "u034", "u035", "u036", "u037"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator","revision":"CUDA 13.0.2 Programming Guide","locator":"occupancy APIs","observation":"Occupancy is evaluated from the concrete kernel and launch resources, supporting the required resource record."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"mbarrier semantics","observation":"The ISA specifies ordering, arrival and phase behavior but no fixed cycle latency."}], + "challenge": {"alternative":"Correct ownership can be established by steady-state output tests alone or a universal barrier cost makes measurement unnecessary.","check":"Exercise first reuse and final-tail states and search the normative ISA for a latency guarantee.","outcome":"Those boundary paths are distinct protocol states, and the ISA contains no portable timing promise; the prescribed tests discriminate both failures."} + }, + { + "claim_id": "technique-double-buffering-current-c009", + "claim": "All seven primary links and three related local pages resolve to the intended pinned documents/files, while fabricated external and local controls fail.", + "claim_type": "other", + "covers": ["u038", "u039", "u040", "u041", "u042", "u043", "u044", "u045", "u046", "u047"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"HTTP-check seven exact primary URLs; resolve three relative wiki links; repeat with fabricated CUDA and wiki paths","environment":"current KernelWiki worktree and network, 2026-08-09","reference":"All named sources/pages should resolve and controls should fail.","result":"All seven exact URLs returned HTTP 200 and all three local files existed.","negative_control":"The fabricated CUDA document returned HTTP 404 and wiki/techniques/not-a-page.md was absent."}], + "challenge": {"alternative":"A redirect hides a missing revision or a similarly named local page satisfies the links.","check":"Require exact final HTTP success and exact relative target existence.","outcome":"Both immutable GitHub blob URLs and archived CUDA pages resolved; exact local paths existed while controls failed."} + } + ], + "commands": ["conda-base exact payload arithmetic with negative control", "PTX ISA 9.0 and archived tuning-guide inspection", "exact-revision source and artifact hash comparison", "external/local link checks with broken controls"], + "scope": "All 47 current units for body hash e32c3558c9bc6483db8f063a150cca5c88247952aa4d168c007310ad16e98baa were checked against archived primary documentation, exact-revision code, arithmetic, and link controls.", + "residual_risk": "No CUDA toolkit or SM90/SM100 GPU is installed locally. The page deliberately gives no performance crossover, occupancy result, or barrier latency; those remain measurements for each concrete kernel." +} diff --git a/verification/reviewed/current/technique-epilogue-fusion.json b/verification/reviewed/current/technique-epilogue-fusion.json new file mode 100644 index 000000000..c2a53ecbc --- /dev/null +++ b/verification/reviewed/current/technique-epilogue-fusion.json @@ -0,0 +1,90 @@ +{ + "path": "wiki/techniques/epilogue-fusion.md", + "id": "technique-epilogue-fusion", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "technique-epilogue-fusion-current-c001", + "claim": "The page correctly defines fusion as in-kernel output transformation/materialization and separates it from optional concurrent next-tile overlap and any fixed warp-role count.", + "claim_type": "mechanism", + "covers": ["u001", "u002"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/operations.hpp","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"fusion operation expression tags","observation":"Fusion operations define transformations of accumulator/source/auxiliary values independently of a next-tile overlap schedule."},{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py","revision":"same tag","locator":"four-warp epilogue group and completed-accumulator path","observation":"A concrete implementation uses four epilogue warps and a completed TMEM accumulator, disproving any architectural default role count while matching the scoped page."}], + "challenge": {"alternative":"Epilogue fusion necessarily uses double-buffered overlap or fourteen dedicated warps on SM100.","check":"Compare the fusion operation interface with a tagged SM100 role schedule.","outcome":"The operation graph is schedule-independent and the pinned schedule uses four epilogue warps; the page explicitly preserves that distinction."} + }, + { + "claim_id": "technique-epilogue-fusion-current-c002", + "claim": "The TMEM-to-output section accurately requires a legal collective TMEM-to-register copy/load and presents exact CUTLASS 4.5.0 CuTe DSL routing identifiers with an explicit non-standalone warning.", + "claim_type": "code", + "covers": ["u003", "u004", "u005", "u006"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"copy_atom_t2r through epilogue loop and tail","observation":"The displayed names and sequence occur in the official file; that file also supplies the layouts, participant groups, copy partitions, conversion, SMEM/TMA store pipeline, tail and TMEM free named by the warning."}], + "challenge": {"alternative":"The excerpt is runnable as written or silently invents the load shape.","check":"Exact-search every identifier and compare surrounding declarations/dependencies.","outcome":"All shown APIs and load shape occur at the tag, while ellipsis and the warning unambiguously prevent standalone interpretation."} + }, + { + "claim_id": "technique-epilogue-fusion-current-c003", + "claim": "All correctness-boundary rows and the multi-region paragraph match tcgen05 MMA completion, collective load completion/source lifetime, mbarrier phase, disjoint-address, output-edge, store-tail, and deallocation obligations.", + "claim_type": "mechanism", + "covers": ["u007", "u008", "u009", "u010", "u011", "u012"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"MMA completion and lifetimes","observation":"Commit and mbarrier completion are required before dependent TMEM access."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"same archive","locator":"collective load, wait, mapping and source lifetime","observation":"Loads are collective/asynchronous, require wait::ld, and preserve TMEM source lifetime through completion."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"same archive","locator":"allocation and explicit deallocation","observation":"TMEM must be collectively allocated and matching storage freed before exit."}], + "challenge": {"alternative":"Warp specialization, __syncthreads, or nonaliasing alone supplies every edge.","check":"Separate asynchronous proxy completion, register-load completion, address disjointness and lifecycle cleanup.","outcome":"Each is independent; the page names every required class rather than substituting one primitive for all of them."} + }, + { + "claim_id": "technique-epilogue-fusion-current-c004", + "claim": "The CUTLASS section gives the exact LinCombEltAct parameter order, accurately summarizes CollectiveBuilder's SM100 inputs, rejects a nonexistent type, scopes support to valid combinations, and correctly describes the pinned vLLM builder/carveout usage.", + "claim_type": "code", + "covers": ["u013", "u014", "u015", "u016", "u017"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/operations.hpp","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"LinCombEltAct declaration","observation":"The displayed six template parameters and defaults match exactly."},{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/collective_builder.hpp","revision":"same tag","locator":"CollectiveBuilder declaration","observation":"The described architecture through FusionOpOrCallbacks inputs match the template signature."},{"path":"artifacts/prs/vllm/PR-16032/key-files/csrc/quantization/fp4/nvfp4_scaled_mm_kernels.cu","revision":"vLLM merge ed7a29d9f8b48978e3bbf43599d21b4de65387e0","locator":"Fp4GemmSm100 CollectiveEpilogue/CollectiveMainloop","observation":"The wrapper uses EpilogueTileAuto, EpilogueScheduleAuto and StageCountAutoCarveout of CollectiveEpilogue::SharedStorage exactly as stated."}], + "challenge": {"alternative":"Sm100EpilogueTmaWarpSpecialized exists under an indirect include or the builder accepts every operation regardless of constraints.","check":"Search the entire exact-tag epilogue tree and inspect builder specializations/static assertions.","outcome":"The name is absent and support is selected through constrained builder/callback specializations; can_implement is correctly required."} + }, + { + "claim_id": "technique-epilogue-fusion-current-c005", + "claim": "The operation table correctly separates source linear combination from bias and describes activation, conversion/quantization, gated, online-softmax, and residual dependencies without claiming universal built-in composition.", + "claim_type": "mechanism", + "covers": ["u018", "u019", "u020", "u021", "u022", "u023", "u024", "u025"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/operations.hpp","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"LinearCombination, bias, activation, residual, block-scale and softmax tags","observation":"Source, explicit bias, activation, auxiliary/residual, quantized scale and reduction-capability distinctions match the table."},{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/activation.h","revision":"same tag","locator":"ReLU, SiLU, GELU functors","observation":"The named activation functions exist; the page correctly conditions compositions on supported/custom callbacks."}], + "challenge": {"alternative":"beta*C is itself always bias or every row is one built-in SM100 visitor.","check":"Compare distinct operation tags and callback capability flags.","outcome":"The page separates source and bias, and explicitly denies universal built-in composition."} + }, + { + "claim_id": "technique-epilogue-fusion-current-c006", + "claim": "The overlap/evaluation section requires both ownership directions, boundary/tail tests, controlled baselines, compiled/profiler resource evidence, and rejects fixed warp-bandwidth or guaranteed-speedup claims while preserving the conditional saved-intermediate benefit.", + "claim_type": "performance", + "covers": ["u026", "u027", "u028", "u029"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"load semantics without latency/bandwidth guarantee","observation":"The ISA supports correctness edges but publishes no equal-share bandwidth or universal performance result."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator","revision":"CUDA 13.0.2 Programming Guide","locator":"resource-dependent occupancy","observation":"Concrete registers, shared memory, block size and launch data are needed for occupancy, supporting the measurement workflow."}], + "challenge": {"alternative":"Removing an intermediate guarantees a win and steady-state correctness implies correct prologue/tail.","check":"Account for added resources/synchronization and separately exercise first/last/partial stages.","outcome":"Those costs and states can reverse or invalidate the result; the page explicitly requires controlled comparison and boundary faults."} + }, + { + "claim_id": "technique-epilogue-fusion-current-c007", + "claim": "The artifact section accurately distinguishes the byte-matched vLLM file, historical extracted snippet, and labeled incomplete derived skeleton, and the displayed Conda-base retrieval command succeeds while a nonexistent page fails.", + "claim_type": "code", + "covers": ["u030", "u031", "u032", "u033", "u034", "u035"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"sha256sum artifact files; compare nvfp4 file to raw merge revision; inspect headers/provenance; run Conda-base get_page command and nonexistent-page control","environment":"current worktree and network, 2026-08-09","reference":"Modes/hashes and command behavior should match the page.","result":"nvfp4 matches e8aed5... at the full merge SHA; the second full file says extracted; the variant says derived/not upstream; retrieval exited zero with 547 lines.","negative_control":"scripts/get_page.py no-such-epilogue-page --include-code exited 1."}], + "challenge": {"alternative":"Directory name full makes both files upstream-verbatim or the variant is safe production code.","check":"Use file-level provenance/header rather than directory inference.","outcome":"The file-level labels establish mixed modes exactly as the page warns."} + }, + { + "claim_id": "technique-epilogue-fusion-current-c008", + "claim": "All six primary URLs and three related local links resolve exactly; fabricated external and local controls fail.", + "claim_type": "other", + "covers": ["u036", "u037", "u038", "u039", "u040", "u041", "u042", "u043", "u044"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"HTTP-check six exact primary URLs; resolve three relative links; repeat with fabricated HTTP and local targets","environment":"current worktree/network, 2026-08-09","reference":"Exact pinned/archived sources and intended local pages should resolve.","result":"All six URLs returned HTTP 200 and all three local files existed.","negative_control":"A fabricated CUDA path returned HTTP 404 and wiki/techniques/not-real.md was absent."}], + "challenge": {"alternative":"Redirects hide missing revisions or similarly named local pages satisfy links.","check":"Require exact immutable GitHub blob/archive page success and exact resolved local paths.","outcome":"Every exact target resolved while both controls failed."} + } + ], + "commands": ["exact CUTLASS 4.5.0 and vLLM merge-source inspection", "artifact hash/provenance checks with negative controls", "Conda-base retrieval test", "external/local link checks"], + "scope": "All 44 current units for body hash 7b78dc20ff19f53642d3b695375cd2e64bb0f6f55abad1feba792f98a12ad994 were checked against exact-release code, archived PTX, byte hashes and discriminating host controls.", + "residual_risk": "No CUDA compiler or SM100 GPU is installed locally. The page intentionally contains no throughput, occupancy, participant-count, or TMEM-bandwidth prediction for an unmeasured kernel." +} diff --git a/verification/reviewed/current/technique-external-source-map-research.json b/verification/reviewed/current/technique-external-source-map-research.json new file mode 100644 index 000000000..33df21b24 --- /dev/null +++ b/verification/reviewed/current/technique-external-source-map-research.json @@ -0,0 +1,209 @@ +{ + "path": "wiki/techniques/external-source-map-research.md", + "id": "technique-external-source-map-research", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "technique-external-source-map-research-current-c001", + "claim": "The page correctly treats profiler results as search keys rather than root-cause proof and scopes the pinned modern source maps to Hopper SM90a, requiring separate evidence for Blackwell transfer.", + "claim_type": "compatibility", + "covers": ["u001"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "/tmp/kernelwiki-ext-cfx", + "revision": "ColfaxResearch/cfx-article-src fbecfed88de2e4246f104a023188ba722937c5fc", + "locator": "modern example Makefiles and SM90 source symbols", + "observation": "Relevant examples target SM90a and use SM90 facilities, while no profiler-causality or SM100 compatibility guarantee is stated." + }, + { + "path": "/tmp/kernelwiki-ext-colfax-cutlass", + "revision": "ColfaxResearch/cutlass-kernels 84f0802e2b4a1bf068ac70359f20ffdb368c8f6a", + "locator": "FMHA READMEs and compile scripts", + "observation": "The source explicitly identifies Hopper SM90/SM90a and CUDA 12.2/12.3 rather than Blackwell." + } + ], + "challenge": { + "alternative": "A post-Blackwell commit or a shared TMA concept makes these repositories direct SM100 evidence.", + "check": "Search pinned build targets and architecture-specific source symbols for SM100/tcgen05 versus SM90a/SM90.", + "outcome": "The implementations are explicitly SM90a/SM90; conceptual transfer does not establish SM100 compatibility or performance." + } + }, + { + "claim_id": "technique-external-source-map-research-current-c002", + "claim": "The displayed shell procedure obtains the exact cfx commit, verifies HEAD, and locates the stated TMA/barrier and persistent/Stream-K symbols.", + "claim_type": "code", + "covers": ["u002"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [ + { + "command": "Run the displayed init/remote/fetch/checkout/test/rg sequence in a fresh mktemp directory", + "environment": "2026-08-09 host with git network access", + "reference": "HEAD must equal fbecfed88de2e4246f104a023188ba722937c5fc and both mechanism searches must return pinned source lines.", + "result": "The fetch and detached checkout succeeded, HEAD matched exactly, and rg found SM90_TMA/mbarrier and PersistentTileScheduler/StreamK symbols.", + "negative_control": "Fetching the all-zero nonexistent revision failed with remote 'not our ref'." + } + ], + "challenge": { + "alternative": "The procedure still silently uses the moving default branch or accepts a wrong commit.", + "check": "Execute it from an empty repository and repeat fetch with a nonexistent SHA.", + "outcome": "The explicit object fetch/check pins the requested commit, and the invalid-SHA negative control is rejected." + } + }, + { + "claim_id": "technique-external-source-map-research-current-c003", + "claim": "The page's pinned repository inventory accurately distinguishes cfx SM90 mechanisms, simveit TMA transpose, simveit ldmatrix/stmatrix-only examples, Colfax Hopper GEMM/FMHA, and the cross-generation NVIDIA sample paths.", + "claim_type": "compatibility", + "covers": ["u003", "u004", "u005", "u006", "u007", "u008", "u009", "u010", "u011"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "/tmp/kernelwiki-ext-cfx", + "revision": "ColfaxResearch/cfx-article-src fbecfed88de2e4246f104a023188ba722937c5fc", + "locator": "tma/tma_copy.h, pipeline-gemm/, streamk/tile_scheduler.hpp", + "observation": "The exact TMA, pipeline, and three scheduler families listed by the page are present." + }, + { + "path": "/tmp/kernelwiki-ext-transpose", + "revision": "simveit/effective_transpose 994b2b5acaa67f80e411df3e8274b6ae13fd1949", + "locator": "Makefile and transpose_swizzle_batched.cu tensor-map encoder", + "observation": "The build target is sm_90a and the tensor map uses CU_TENSOR_MAP_SWIZZLE_128B." + }, + { + "path": "/tmp/kernelwiki-ext-load-store", + "revision": "simveit/load_and_store 05d828cf910dd43f0053ddbbe4744218a06e9d7f", + "locator": "README.md, Makefile and complete tracked file list", + "observation": "The six kernels are ldmatrix/stmatrix x1/x2/x4 and compile for sm_90a; the excluded broader mechanisms are absent." + }, + { + "path": "/tmp/kernelwiki-ext-colfax-cutlass", + "revision": "ColfaxResearch/cutlass-kernels 84f0802e2b4a1bf068ac70359f20ffdb368c8f6a", + "locator": "src/fmha*/README.md, compile scripts, source tree search", + "observation": "The repository contains SM90a GEMM/FMHA TMA/WGMMA pipelines and optional warp specialization, with no persistent/tile scheduler or SM100 implementation." + }, + { + "path": "/tmp/kernelwiki-ext-nvidia-samples", + "revision": "NVIDIA-developer-blog/code-samples 3350d216083a902ccbf5b31665e3b82096a75b55", + "locator": "complete file tree and the two page-listed CUDA paths", + "observation": "Both exact coalescing and transpose paths exist in a broad historical sample collection." + } + ], + "challenge": { + "alternative": "The exclusions are artifacts of shallow clones or case-sensitive searches rather than real scope limits.", + "check": "Inspect complete tracked file lists plus READMEs, build targets, and case-insensitive architecture/mechanism searches at each exact revision.", + "outcome": "The positive mechanisms have exact files/symbols, while the excluded scope has no target, implementation, or repository claim." + } + }, + { + "claim_id": "technique-external-source-map-research-current-c004", + "claim": "Coalescing, transpose, and matrix load/store examples are valid candidate sources for long-scoreboard or poor-sector investigations, but those metrics alone do not establish vector width or layout as the cause.", + "claim_type": "mechanism", + "covers": ["u012", "u013"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "/tmp/kernelwiki-ext-nvidia-samples", + "revision": "NVIDIA-developer-blog/code-samples 3350d216083a902ccbf5b31665e3b82096a75b55", + "locator": "coalescing-global/coalescing.cu and transpose/transpose.cu", + "observation": "The files isolate stride/offset coalescing and shared-memory transpose/padding mechanisms but do not define a unique mapping from profiler symptoms to cause." + }, + { + "path": "/tmp/kernelwiki-ext-load-store", + "revision": "simveit/load_and_store 05d828cf910dd43f0053ddbbe4744218a06e9d7f", + "locator": "ldmatrix/stmatrix examples", + "observation": "These are concrete matrix movement candidates, not evidence that a particular target kernel uses them or should change vector width." + } + ], + "challenge": { + "alternative": "Long scoreboard or poor sectors uniquely imply one of the listed memory-layout defects.", + "check": "Look for such causal semantics in the pinned sources and compare with their actual experimental variables.", + "outcome": "The sources demonstrate mechanisms only, matching the page's explicit non-causal wording." + } + }, + { + "claim_id": "technique-external-source-map-research-current-c005", + "claim": "Pinned SM90 TMA and pipelined GEMM paths are appropriate candidate implementations to inspect for barrier/TMA-wait stalls, without guaranteeing that stage or role changes improve another kernel.", + "claim_type": "mechanism", + "covers": ["u014", "u015"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "/tmp/kernelwiki-ext-cfx", + "revision": "ColfaxResearch/cfx-article-src fbecfed88de2e4246f104a023188ba722937c5fc", + "locator": "tma/tma_copy.h and pipeline-gemm/sm90_gemm_multistage.cu", + "observation": "The sources explicitly implement TMA transaction-barrier handling and staged producer/consumer GEMM flow." + }, + { + "path": "/tmp/kernelwiki-ext-colfax-cutlass/src/fmha-pipeline", + "revision": "ColfaxResearch/cutlass-kernels 84f0802e2b4a1bf068ac70359f20ffdb368c8f6a", + "locator": "README.md and fmha_pipe_ws.h", + "observation": "Stage count and optional warp-specialized execution are explicit configuration dimensions, not universal performance prescriptions." + } + ], + "challenge": { + "alternative": "Because the mechanisms match the stall labels, copying their stage count or role split must help.", + "check": "Inspect whether the pinned source results cover a different target kernel's shapes, resources, and timing.", + "outcome": "They do not; the page limits the action to inspection and explicitly withholds a transfer guarantee." + } + }, + { + "claim_id": "technique-external-source-map-research-current-c006", + "claim": "The cfx source contains non-persistent, data-parallel persistent, and Stream-K schedulers suitable for controlled tail-wave comparison, while persistence is not stated as a universal remedy.", + "claim_type": "mechanism", + "covers": ["u016", "u017"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "/tmp/kernelwiki-ext-cfx/streamk", + "revision": "ColfaxResearch/cfx-article-src fbecfed88de2e4246f104a023188ba722937c5fc", + "locator": "tile_scheduler.hpp classes and cli_options.h scheduler enumeration", + "observation": "All three named scheduler families and shape/scheduler command-line controls are present." + } + ], + "challenge": { + "alternative": "The Stream-K tutorial proves persistent scheduling should replace shape dispatch for every tail wave.", + "check": "Inspect the alternative schedulers and configurable problem shapes exposed by the source.", + "outcome": "The repository makes the alternatives comparable rather than declaring one universally optimal; the page requires target-shape profiling." + } + }, + { + "claim_id": "technique-external-source-map-research-current-c007", + "claim": "The provenance rule correctly treats local source pages as discovery leads and requires a full immutable upstream revision plus exact file/symbol locator, with separate SM100 evidence for Blackwell transfer.", + "claim_type": "other", + "covers": ["u018"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "plan.md", + "revision": "repository remediation contract", + "locator": "sections 5 and 8", + "observation": "The contract requires revision and exact locator, rejects summaries as primary evidence, and requires version-matched compatibility evidence." + }, + { + "path": "/tmp/kernelwiki-ext-cfx", + "revision": "ColfaxResearch/cfx-article-src fbecfed88de2e4246f104a023188ba722937c5fc", + "locator": "architecture-specific build/source files", + "observation": "Pinned Hopper source proves only its recorded scope and does not itself establish SM100 transfer." + } + ], + "challenge": { + "alternative": "A source-card URL or unversioned upstream path is enough because readers can inspect the latest version.", + "check": "Compare that alternative with repository drift and the overbroad baseline source-card metadata.", + "outcome": "Latest contents and local summaries can change or misstate scope; revision plus exact locator preserves the evidence actually reviewed." + } + } + ], + "commands": [ + "execute the displayed pinned git fetch/checkout/search procedure with an invalid-revision negative control", + "inspect exact revisions and complete tracked trees for all five linked repositories" + ], + "scope": "All 18 current semantic units for body hash e518329cc4548741f8dede0d4a60f5d493cc932595124606962a0e23358c5080 and the corrected linked source scopes were checked.", + "residual_risk": "No GPU benchmark was run. This page verifies source discovery, pinned scope, and evidence discipline; it makes no transferred correctness or performance claim." +} diff --git a/verification/reviewed/current/technique-fine-grained-quantization.json b/verification/reviewed/current/technique-fine-grained-quantization.json new file mode 100644 index 000000000..60118a4bf --- /dev/null +++ b/verification/reviewed/current/technique-fine-grained-quantization.json @@ -0,0 +1,109 @@ +{ + "path": "wiki/techniques/fine-grained-quantization.md", + "id": "technique-fine-grained-quantization", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "technique-fine-grained-quantization-current-c001", + "claim": "The overview accurately separates payload, scale type, geometry and layout, and correctly states the DeepSeek FP8, NVFP4 1D/2D, and MXFP4 recipe geometries without promising universal accuracy or speed.", + "claim_type": "semantics", + "covers": ["u001", "u002", "u003", "u004", "u005", "u006", "u007", "u008"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url":"https://arxiv.org/abs/2412.19437v2","revision":"DeepSeek-V3 Technical Report v2","locator":"section 3.3.2 and Appendix B.2","observation":"The report assigns 1x128 groups to activations and 128x128 groups to weights and scopes the outlier motivation."}, + {"url":"https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html","revision":"Transformer Engine 2.13","locator":"two-level scaling, 1D/2D scaling, comparison with MXFP4","observation":"The vendor defines NVFP4's E2M1/block16 E4M3/global-FP32 recipe, 16x16 weight mode, and MXFP4's block32 UE8M0 distinction."} + ], + "challenge": {"alternative":"All three are interchangeable block-scaled FP4/FP8 encodings and differ only in group size.","check":"Compare payload, local/global scale types and operand-specific geometry in both primary sources.","outcome":"The sources define independent recipe dimensions exactly matching the page's separation."} + }, + { + "claim_id": "technique-fine-grained-quantization-current-c002", + "claim": "The page correctly distinguishes one-factor-per-16384 count ratio (0.0061%) from the 0.0244% byte ratio for one FP32 factor and 16384 one-byte payloads.", + "claim_type": "semantics", + "covers": ["u009"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"awk arithmetic for 100/16384 and 100*4/16384","environment":"deterministic host arithmetic","reference":"One factor; FP32 factor is four bytes; FP8 payload is one byte.","result":"0.0061035% factor-count ratio and 0.0244141% byte ratio.","negative_control":"Replacing FP32 with a one-byte factor makes the byte ratio 0.0061035%, discriminating the missing-width error."}], + "challenge": {"alternative":"Both percentages describe the same overhead measure.","check":"Carry units through numerator and denominator.","outcome":"One divides object counts while the other divides bytes; they coincide only for a one-byte scale."} + }, + { + "claim_id": "technique-fine-grained-quantization-current-c003", + "claim": "The pinned DeepGEMM SM90 description and four-line fragment correctly give FP32 factors, BLOCK_K=128, about 14 retained bits/four-WGMMA source scope, and accum-to-final_accum scale promotion without universalizing Nc.", + "claim_type": "code", + "covers": ["u010", "u011", "u012", "u013"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh","path":"artifacts/kernels/deepgemm/full/sm90_fp8_gemm_1d1d.cuh","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f, byte-verified","locator":"BLOCK_K assertion and lines 250-323","observation":"The file fixes K=128, uses FP32 factor storage, declares float accum/final_accum, issues the K-block WGMMAs and contains the exact displayed four promotion lines."}, + {"url":"https://arxiv.org/abs/2412.19437v2","revision":"DeepSeek-V3 Technical Report v2","locator":"sections 3.3.2 and 3.5.2","observation":"The report states about 14 retained bits, Nc=128 inner-dimension elements, and equivalence to four WGMMAs in the described configuration."} + ], + "challenge": {"alternative":"Nc=128 is a general Hopper hardware rule or the excerpt uses a half-precision promotion target.","check":"Inspect declared C++ types and the source paper's scoped wording.","outcome":"Both arrays are float, and the report describes a specific H800 FP8 strategy rather than a hardware mandate."} + }, + { + "claim_id": "technique-fine-grained-quantization-current-c004", + "claim": "The pinned DeepGEMM SM100 description correctly gives K factor granularities 32/128, four-UE8M0 packing, TMA-to-SMEM and UTCCP-to-TMEM movement, TMEM scale operands, and absence of the SM90 final_accum loop; 0xff UE8M0 is NaN.", + "claim_type": "mechanism", + "covers": ["u014"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh","path":"artifacts/kernels/deepgemm/full/sm100_fp8_gemm_1d1d.cuh","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f, byte-verified","locator":"kGran assertions, factor sizes/TMA loads, UTCCP copies, block-scaled descriptor and mma_t::fma","observation":"Every stated representation and movement stage appears, and no final_accum symbol exists."}, + {"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md","revision":"DeepGEMM 891d57b","locator":"interface notices","observation":"SM100 factors are packed four UE8M0 values per torch.int."}, + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#alternate-floating-point-data-formats","revision":"PTX ISA 9.0","locator":"ue8m0 definition","observation":"The type is power-of-two for finite encodings and reserves 0xff for NaN."} + ], + "challenge": {"alternative":"Packed factors stay in ordinary registers or shared memory and tcgen05 reads them there directly.","check":"Trace factor buffers from TMA destination through UTCCP destination and final fma arguments.","outcome":"UTCCP targets dedicated TMEM columns and the fma wrapper receives those TMEM addresses."} + }, + { + "claim_id": "technique-fine-grained-quantization-current-c005", + "claim": "The displayed tcgen05 block-scale line matches PTX ISA 9.0 grammar for mxf8f6f4 scale_vec::1X, including idesc, TMEM scale operands and enable-input-d, and is accurately limited to an instruction-shape reference.", + "claim_type": "code", + "covers": ["u015", "u016", "u017"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"floating-point type with block scaling grammar and valid combination table","observation":"The exact qualifier/operand sequence is legal for mxf8f6f4 with UE8M0 scale_vec::1X, and the ISA separately defines descriptors, targets and asynchronous lifecycle requirements."}], + "challenge": {"alternative":"Scale operands are SMEM pointers or scale_d belongs before them as in unscaled tcgen05 forms.","check":"Compare block-scaled grammar with the unscaled floating-point grammar.","outcome":"Block-scaled syntax names TMEM SFA/SFB operands after idesc and does not use scale_d in that position."} + }, + { + "claim_id": "technique-fine-grained-quantization-current-c006", + "claim": "The Blackwell table exactly maps mxf4 block32 to E2M1/UE8M0/2X and mxf4nvf4 block16 to E2M1/UE4M3/4X, while correctly noting additional mxf4nvf4 UE8M0 modes and native UE4M3 support.", + "claim_type": "compatibility", + "covers": ["u018", "u019", "u020", "u021"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"valid scale_vectorsize/type/MMA-kind combinations","observation":"The table lists both displayed rows and the additional mxf4nvf4 UE8M0 block16/block32 possibilities."}], + "challenge": {"alternative":"The mxf4nvf4 kind alone uniquely means NVFP4 UE4M3 block16.","check":"Enumerate every mxf4nvf4 row in the valid-combination table.","outcome":"UE8M0 rows also exist, so full qualifiers and types are selection-critical."} + }, + { + "claim_id": "technique-fine-grained-quantization-current-c007", + "claim": "The transfer checklist and evidence boundaries correctly require exact recipe/ABI/instruction matching, include preprocessing in timing unless already produced, demand controlled accuracy/performance evaluation, and avoid universal block-size or Nc conclusions.", + "claim_type": "performance", + "covers": ["u022", "u023", "u024", "u025", "u026", "u027", "u028", "u029", "u030", "u031", "u032", "u033", "u034", "u035", "u036", "u037", "u038", "u039"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url":"https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html","revision":"Transformer Engine 2.13","locator":"recipe, 1D/2D grouping and MXFP4 comparison","observation":"Payload, local/global scaling and geometry are independent contract dimensions, and documented accuracy advantages are recipe-scoped rather than universal."}, + {"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md","revision":"DeepGEMM 891d57b","locator":"interface notices","observation":"The project explicitly warns that transpose/cast preparation is separate and utility implementations may be slower, and it distinguishes SM90/SM100 factor formats."}, + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"PTX ISA 9.0","locator":"block-scale grammar, combinations and target notes","observation":"Legality depends on kind, scale-vector, type and target; the ISA supplies no universal performance/accuracy ordering."} + ], + "challenge": {"alternative":"Native instruction support and smaller groups alone are enough to choose the fastest and most accurate recipe.","check":"Check whether the sources hold preparation/layout/workload constant or define an input-independent ordering.","outcome":"They do not; the checklist explicitly identifies those uncontrolled variables and requests measurement rather than predicting results."} + }, + { + "claim_id": "technique-fine-grained-quantization-current-c008", + "claim": "The four primary-reference links resolve to the exact DeepSeek-V3 v2 paper, pinned DeepGEMM commit, Transformer Engine 2.13 page, and archived PTX ISA 9.0 tcgen05 section.", + "claim_type": "other", + "covers": ["u040", "u041", "u042", "u043"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url":"https://arxiv.org/abs/2412.19437v2","revision":"v2","locator":"document landing/PDF","observation":"The URL is revision-pinned to v2."}, + {"url":"https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f","revision":"891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"repository tree","observation":"The URL embeds the full commit."}, + {"url":"https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html","revision":"Transformer Engine 2.13","locator":"NVFP4 page","observation":"The version path and topic match."}, + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"tcgen05.mma section","observation":"The archive and anchor match the label."} + ], + "challenge": {"alternative":"At least one link is rolling or points to a summary rather than the asserted primary source.","check":"Inspect URL version/commit segments and resolved section titles.","outcome":"All four are direct primary, revision-specific targets."} + } + ], + "commands": ["inspect byte-verified DeepGEMM SM90/SM100 sources", "compute scale count/byte ratios with a one-byte negative control", "resolve version-pinned primary references"], + "scope": "All 43 current semantic units for body hash af3e67a9f9a356ddf6111b4a87ae2ba28554f5f096b0b5a9e18e0749b82f6cf8 were checked.", + "residual_risk": "No CUDA toolkit or NVIDIA GPU is installed, so the grammar-level PTX was not assembled and no accuracy/performance benchmark was run. The page makes no new numeric speed or error claim." +} diff --git a/verification/reviewed/current/technique-kernel-fusion.json b/verification/reviewed/current/technique-kernel-fusion.json new file mode 100644 index 000000000..7655000b4 --- /dev/null +++ b/verification/reviewed/current/technique-kernel-fusion.json @@ -0,0 +1,99 @@ +{ + "path": "wiki/techniques/kernel-fusion.md", + "id": "technique-kernel-fusion", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "technique-kernel-fusion-current-c001", + "claim": "The page correctly defines fusion as implementing dependent operations in one launch and makes saved launches/traffic conditional on a pinned fused/unfused pair and actual on-chip lifetime.", + "claim_type": "performance", + "covers": ["u001", "u002"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/operations.hpp","revision":"CUTLASS v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"fusion operation expression types","observation":"The library composes operations in one kernel but makes no claim about a particular unfused launch count, global-memory baseline, or guaranteed speedup."}], + "challenge": {"alternative":"Any operation named fused necessarily removes every intermediate and is faster.","check":"Separate expression composition from storage/schedule choices and baseline behavior.","outcome":"A concrete kernel can stage, spill, or write values and incur added resources; the page requires those facts to be measured."} + }, + { + "claim_id": "technique-kernel-fusion-current-c002", + "claim": "The gated dual-GEMM formula, shared-A/separate-B scale contract, and output semantics match the pinned GPU Mode reference, while launch/TMEM/TMA schedules remain unspecified alternatives.", + "claim_type": "code", + "covers": ["u003", "u004", "u005"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/gpu-mode/reference-kernels/blob/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm/reference.py","revision":"gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c","locator":"correctness reference","observation":"It computes two scaled_mm calls sharing A/sfa with distinct B/sfb inputs and applies silu(gate)*up; no launch or TMEM decomposition is part of the contract."}], + "challenge": {"alternative":"The task requires two separately allocated TMEM accumulators because it names dual GEMM.","check":"Inspect observable reference semantics and task constraints for storage/instruction requirements.","outcome":"Only input/output math is fixed; a concatenated or separate internal representation can be compatible."} + }, + { + "claim_id": "technique-kernel-fusion-current-c003", + "claim": "The FlashInfer Track A description accurately includes routing, grouped selection, W13/SwiGLU/W2 and weighted accumulation, uses one logical concatenated W13 projection, and does not infer launch topology from the Fused MoE label.", + "claim_type": "mechanism", + "covers": ["u006"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"DeepSeek routing reference and _fp8_moe_run_experts","observation":"The reference implements the listed stages, makes one W13 matmul then chunks it into two halves, and constrains results rather than GPU launch decomposition."}], + "challenge": {"alternative":"The benchmark title proves submissions use one launch and no global intermediates.","check":"Distinguish definition/reference semantics from implementation traces or source of a named submission.","outcome":"The pinned reference contains no launch-count or on-chip-lifetime requirement; the page states that boundary."} + }, + { + "claim_id": "technique-kernel-fusion-current-c004", + "claim": "The implementation constraints correctly cover semantic preservation, scope-specific communication, resource accounting, SM100's 128x512-cell TMEM capacity and collective lifecycle, and capability-based fallbacks without imposing CTA-only fusion or a 256-column total.", + "claim_type": "mechanism", + "covers": ["u007", "u008", "u009", "u010", "u011", "u012", "u013", "u014", "u015", "u016"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"TMEM organization and access/lifetime model","observation":"TMEM is 128 lanes by 512 32-bit columns per SM and has collective, asynchronous access rules."}, + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"same archive","locator":"alloc/dealloc semantics","observation":"Allocation uses legal column counts, collective participation/result sharing, and matching deallocation."}, + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator","revision":"CUDA 13.0.2 Programming Guide","locator":"resource-dependent occupancy","observation":"Registers, shared memory and launch dimensions affect residency; deleting a global intermediate does not guarantee reduced on-chip pressure."} + ], + "challenge": {"alternative":"Fusion is legal whenever the operation graph is acyclic and two 256-column TMEM halves fit.","check":"Check communication scope, asynchronous lifetimes, collective allocation/free and all non-TMEM resource constraints.","outcome":"Acyclicity and one capacity partition are insufficient; the page enumerates the independent obligations."} + }, + { + "claim_id": "technique-kernel-fusion-current-c005", + "claim": "The correctness workflow covers exact semantic comparison and discriminating boundary, routing, scale/layout, completion, reuse and output-edge tests.", + "claim_type": "mechanism", + "covers": ["u017"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"routing and expert reference","observation":"Routing bias/ties, local filtering, scale dequantization, SwiGLU and weighted accumulation are independent semantic boundaries that a fused result must preserve."},{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py","revision":"CUTLASS v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"TMEM completion, epilogue edges, store tail and free","observation":"The exact schedule contains the completion/lifetime/output boundaries named by the negative tests."}], + "challenge": {"alternative":"Matching a few random dense outputs is enough to validate fusion.","check":"Exercise cases that isolate routing ties, empty groups, partial tiles and premature asynchronous reuse.","outcome":"Those cases discriminate semantic/synchronization errors that ordinary dense inputs may not expose."} + }, + { + "claim_id": "technique-kernel-fusion-current-c006", + "claim": "The performance protocol records the variables needed for a controlled fused/unfused comparison and correctly rejects guaranteed speedup from saved traffic alone.", + "claim_type": "performance", + "covers": ["u018"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator","revision":"CUDA 13.0.2 Programming Guide","locator":"occupancy/resource inputs","observation":"Resource and launch configuration can change residency and performance, supporting the page's required accounting rather than a traffic-only prediction."}], + "challenge": {"alternative":"Removing one intermediate is sufficient for a source-independent positive result.","check":"Include preprocessing, resource changes, synchronization, repeated trials and variance for identical semantics.","outcome":"Those costs can offset saved bytes; the page predicts no direction and requires measurement."} + }, + { + "claim_id": "technique-kernel-fusion-current-c007", + "claim": "The four primary-source links resolve to the pinned GPU Mode task, FlashInfer Track A definition, pinned FlashInfer reference, and archived PTX ISA 9.0 Tensor Memory section.", + "claim_type": "other", + "covers": ["u019", "u020", "u021", "u022"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url":"https://github.com/gpu-mode/reference-kernels/tree/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm","revision":"c5b2f7c062d5015f29c3a1043cfd04954397944c","locator":"task directory","observation":"The commit-pinned task resolves."}, + {"url":"https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","revision":"checked 2026-08-08","locator":"exact definition","observation":"The exact Track A definition resolves."}, + {"url":"https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/trace/templates/moe.py","revision":"7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"file","observation":"The pinned reference resolves."}, + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"Tensor Memory","observation":"The archived section resolves."} + ], + "challenge": {"alternative":"A source is mutable or points to a similarly named but different benchmark.","check":"Inspect commit/archive segments and the full benchmark definition ID.","outcome":"Three are immutable revision/archive URLs and the definition URL contains the exact Track A identity."} + }, + { + "claim_id": "technique-kernel-fusion-current-c008", + "claim": "The two related links resolve to the intended fused-MoE and epilogue-fusion pages.", + "claim_type": "other", + "covers": ["u023", "u024"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"test -f wiki/kernels/fused-moe.md && test -f wiki/techniques/epilogue-fusion.md","environment":"current worktree","reference":"Resolve links and compare frontmatter IDs.","result":"Targets exist with IDs kernel-fused-moe and technique-epilogue-fusion.","negative_control":"wiki/kernels/not-fused.md does not exist."}], + "challenge": {"alternative":"A relative target resolves to an unrelated page.","check":"Resolve from wiki/techniques and inspect IDs.","outcome":"Both labels and IDs agree."} + } + ], + "commands": ["inspect pinned task/reference sources", "resolve related links with broken-target negative control"], + "scope": "All 24 current units for body hash affee601d50f5bf6e00362f9728a695fb9a7a2806e5e295158ebf1ffe1d90954 were checked.", + "residual_risk": "No GPU profile was run. The page states correctness/resource protocols and source contracts, not a launch count, byte saving or speed result." +} diff --git a/verification/reviewed/current/technique-persistent-kernels.json b/verification/reviewed/current/technique-persistent-kernels.json new file mode 100644 index 000000000..d70968230 --- /dev/null +++ b/verification/reviewed/current/technique-persistent-kernels.json @@ -0,0 +1,100 @@ +{ + "path": "wiki/techniques/persistent-kernels.md", + "id": "technique-persistent-kernels", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "technique-persistent-kernels-current-c001", + "claim": "The persistence definition correctly requires repeated logical work by a resident worker without fixing launch count to physical SM count.", + "claim_type": "mechanism", + "covers": ["u001"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0 e406c186","locator":"persistent kernel and dynamic scheduler description","observation":"Workers can process multiple tiles; launched grid and resident execution are distinct, with grid/cluster shape determined by scheduler/problem policy."}], + "challenge": {"alternative":"Persistent necessarily means launch exactly one CTA per SM.","check":"Compare the launched problem-sized CLC grid with the number of simultaneously resident workers.","outcome":"The grid can exceed or differ from SM count while resident workers process multiple ClcIDs, matching the page's distinction."} + }, + { + "claim_id": "technique-persistent-kernels-current-c002", + "claim": "The CLC scope and two-path model correctly preserve the problem-sized grid, initial blockIdx, cancellation of another unlaunched entity, ClcID conservation, uneven-SM motivation, and exclusions for self-cancel/problem chaining.", + "claim_type": "mechanism", + "covers": ["u002", "u003", "u004", "u005"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0 e406c186","locator":"Dynamic Scheduler rules 1-4 and motivation","observation":"The official rules establish full-grid launch, initial blockIdx, success-only canceled-ID work and uneven-availability motivation."}], + "challenge": {"alternative":"CLC assigns every tile from a queue, creates new work, or cancels the requester when empty.","check":"Count the two documented ClcID paths and inspect the target/result of try_cancel.","outcome":"Every returned ID was already a not-yet-launched grid entity and the requester continues to process it; no other behavior is claimed."} + }, + { + "claim_id": "technique-persistent-kernels-current-c003", + "claim": "The exact PTX forms and request lifecycle correctly specify the 16-byte asynchronous response, mbarrier completion, success-only decode, proxy/lifetime requirements, terminal failed request, and cluster-rank derivation.", + "claim_type": "code", + "covers": ["u006", "u007", "u008", "u009", "u010"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"try_cancel/query grammars, transaction, ordering, failure and cluster semantics","observation":"All three displayed forms and every stated lifecycle/failure/cluster condition match the normative section."}], + "challenge": {"alternative":"A failed response may be retried or get_first_ctaid decoded regardless of is_canceled.","check":"Read the explicit undefined-behavior restrictions after request failure.","outcome":"Both operations are prohibited as described, and the page makes failure terminal."} + }, + { + "claim_id": "technique-persistent-kernels-current-c004", + "claim": "The static-stride code covers each valid flat tile exactly once for positive grid size, including nondivisible tails, and maps row-major coordinates by quotient/remainder.", + "claim_type": "code", + "covers": ["u011", "u012", "u013"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python enumerate CTA/iteration assignments for divisible and nondivisible total_tiles/gridDim.x pairs","environment":"Conda base, host CPU","reference":"Sorted assignments should equal range(total_tiles) with count one each.","result":"All tested positive grid sizes had exact unique coverage and correct quotient/remainder coordinates.","negative_control":"Replacing the stride with gridDim.x-1 produced duplicates."}], + "challenge": {"alternative":"The final partial wave skips tiles or assigns one tile to two CTAs.","check":"Enumerate cases with remainders 1 and gridDim.x-1.","outcome":"Only per-CTA loop lengths differ; coverage and uniqueness remain exact."} + }, + { + "claim_id": "technique-persistent-kernels-current-c005", + "claim": "The CUTLASS 4.5.0 route names the real scheduler/pipeline/work methods, assigns swizzle/raster to software policy, and rejects the invented clc_init/query_tile/self-cancel API.", + "claim_type": "compatibility", + "covers": ["u014", "u015", "u016"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp","revision":"CUTLASS 4.5.0 e406c186","locator":"PersistentTileSchedulerSm100 methods and swizzle_and_rasterize","observation":"WorkTileInfo, get_current_work, advance_to_next_work, fetch_next_work and the software transform are present; the rejected pseudo-helpers are absent."},{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"same tag","locator":"PipelineCLCFetchAsync integration","observation":"The documentation maps scheduler producer/consumer behavior through the named CLC pipeline."}], + "challenge": {"alternative":"Swizzle/raster is programmed into CLC hardware during an init call.","check":"Inspect raw PTX operands and scheduler coordinate transformation.","outcome":"Raw CLC receives response/mbarrier addresses only; CUTLASS transforms coordinates in software as stated."} + }, + { + "claim_id": "technique-persistent-kernels-current-c006", + "claim": "The tutorial paragraph reports the exact v3/v6 author numbers and correctly scopes v6 as a multi-change static-persistence endpoint that explicitly excludes CLC/swizzling and therefore is not an isolated CLC result.", + "claim_type": "performance", + "covers": ["u017"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"sources/blogs/tcgen05-tutorial.md","revision":"Gau Nernst article and source commit 3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"source-reported progression and supported takeaways","observation":"v3=939.61 and v6=1475.93 for the disclosed experiment; the source card records intermediate warp-specialization/2SM changes and the author's explicit omission of CLC/swizzling."}], + "challenge": {"alternative":"The endpoint delta can still be called the persistence speedup.","check":"List every change between v3 and v6 and inspect v6 scheduling code.","outcome":"Several performance-relevant changes intervene and v6 is statically scheduled, so the page correctly refuses causal attribution."} + }, + { + "claim_id": "technique-persistent-kernels-current-c007", + "claim": "The CLC evaluation/negative-control guidance follows the conserved-grid work model, requests controlled static/dynamic evidence, and correctly separates additional Stream-K decomposition from CLC.", + "claim_type": "performance", + "covers": ["u018", "u019", "u020"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0 e406c186","locator":"motivation, ClcID rules and software scheduler scope","observation":"Uneven availability, initial/returned coordinate consistency and finite launched ClcIDs motivate exactly the proposed controls."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"PTX ISA 9.0","locator":"failure/query restrictions","observation":"The negative controls intentionally violate explicit normative requirements and should be detected."}], + "challenge": {"alternative":"CLC alone can fill unused SMs by splitting K or generating coordinates.","check":"Compare raw request results with scheduler decomposition state.","outcome":"CLC returns only existing ClcIDs; Stream-K is correctly identified as a separate transformation."} + }, + { + "claim_id": "technique-persistent-kernels-current-c008", + "claim": "Artifact hashes/modes and the Conda-base retrieval command are accurate: the PR 2881 source byte-matches its full merge SHA, the skeleton is labeled derived, and a nonexistent page control fails.", + "claim_type": "code", + "covers": ["u021", "u022", "u023", "u024", "u025"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"sha256sum artifacts; raw-file byte comparison; inspect provenance/header; Conda-base get_page plus nonexistent ID","environment":"current worktree/network, 2026-08-09","reference":"Page hashes/modes and retrieval behavior","result":"All manifest hashes match; dense_gemm file equals merge 3f4c086d09bd1dc55defb955862f333893bbb28b; variant label is explicit; command exited zero with 2427 lines.","negative_control":"A nonexistent page ID exited nonzero."}], + "challenge": {"alternative":"The derived skeleton is safe upstream code or the exact source revision differs.","check":"Inspect first line and byte-compare the purported upstream file.","outcome":"The variant self-identifies as derived/incomplete while the upstream file hash matches exactly."} + }, + { + "claim_id": "technique-persistent-kernels-current-c009", + "claim": "All four primary links and three related local links resolve to exact intended targets; fabricated external/local controls fail.", + "claim_type": "other", + "covers": ["u026", "u027", "u028", "u029", "u030", "u031", "u032"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"HTTP-check four exact URLs; resolve three relative pages; run fabricated controls","environment":"current worktree/network, 2026-08-09","reference":"Pinned/archived and exact local targets should resolve.","result":"All four URLs returned HTTP 200 and all three related pages existed.","negative_control":"A fabricated local technique was absent and a fabricated CUDA URL returned HTTP 404."}], + "challenge": {"alternative":"Redirects or similarly named pages mask missing exact targets.","check":"Require exact immutable/archive URL success and exact relative file existence.","outcome":"Every target resolved and controls failed."} + } + ], + "commands": ["PTX ISA 9.0 and CUTLASS 4.5.0 exact-source inspection", "Conda-base static-stride enumeration", "author tutorial variant comparison", "artifact/link/retrieval checks with negative controls"], + "scope": "All 32 current units for body hash 5ea652650db0a472fb026f18b1244edf0e01a1cee16cfca178b5c2982608b1d2 were verified against pinned primary sources and discriminating host checks.", + "residual_risk": "No CC10.0 GPU is available locally. CLC benefit, scheduler overhead and work-distribution effects remain explicitly unquantified pending controlled target measurements." +} diff --git a/verification/reviewed/current/technique-ping-pong-scheduling.json b/verification/reviewed/current/technique-ping-pong-scheduling.json new file mode 100644 index 000000000..296534694 --- /dev/null +++ b/verification/reviewed/current/technique-ping-pong-scheduling.json @@ -0,0 +1,143 @@ +{ + "claims": [ + { + "challenge": { + "alternative": "The two-stage schedule is ordinary buffer ping-pong and the paper promises complete MMA/softmax overlap.", + "check": "Compare the paper's wording and the pinned implementation's per-stage state with the page's bounded definition.", + "outcome": "The paper says the schedule maximizes overlap, while the source gives each query stage distinct softmax/output state; the page expressly avoids a stall-free guarantee." + }, + "claim": "In the pinned FA4 SM100 forward implementation, q_stage=2 covers two 128-row query/output tiles and interleaves one tile's MMA with the other's softmax using one MMA warp, two softmax warpgroups, and a correction warpgroup.", + "claim_id": "technique-ping-pong-scheduling-current-c001", + "claim_type": "semantics", + "covers": ["u001", "u002"], + "evidence": [ + { + "locator": "section 3.1.2", + "observation": "The paper describes two output tiles per block, with tensor-core work for one tile overlapping softmax for the other, and identifies two softmax groups plus a correction group.", + "revision": "FlashAttention-4 paper v1, 2026-03-05", + "url": "https://arxiv.org/html/2603.05451v1" + }, + { + "locator": "q_stage default, cta_tiler, and warp-role definitions", + "observation": "The source defaults to q_stage=2 and m_block=128, extends the CTA query dimension by q_stage, and assigns the stated specialized roles.", + "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", + "url": "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "The role table and checklist overstate requirements that are merely CuTe implementation details.", + "check": "Separate source-specific role IDs from PTX lifecycle/completion requirements and inspect both against the page wording.", + "outcome": "The page labels role IDs as implementation choices; its completion and TMEM-lifetime requirements match the ISA, while its remaining handoffs match the pinned source's explicit pipelines." + }, + "claim": "The pinned default source uses the listed warp roles, one collective TMEM allocation with disjoint per-stage regions, and multiple producer/consumer pipelines; a correct adaptation must preserve completion, ownership, visibility, lifetime, boundary, and tail obligations.", + "claim_id": "technique-ping-pong-scheduling-current-c002", + "claim_type": "code", + "covers": ["u003", "u004", "u005", "u006", "u007", "u008", "u009", "u010", "u011", "u012", "u013", "u014", "u015", "u016", "u017", "u018", "u019"], + "evidence": [ + { + "locator": "warp IDs, TmemAllocator, TMEM offsets, and pipeline producer/consumer states", + "observation": "The implementation assigns softmax0 to warps 0-3, softmax1 to 4-7, correction to 8-11, MMA/load/epilogue to 12/14/13, leaves 15 empty in the default layout, allocates one TMEM region, and builds distinct pipelines and stage states.", + "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", + "url": "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py" + }, + { + "locator": "tcgen05 allocation, MMA, commit, fence, load, and deallocation contracts", + "observation": "PTX specifies collective TMEM lifetime and the completion and ordering machinery required around asynchronous tcgen05 operations; an unrelated barrier is not a substitute.", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "The B200-versus-Hopper throughput figures or the simplified cycle counts were transcribed from a secondary summary.", + "check": "Read the version-pinned paper's hardware table and worked roofline example directly.", + "outcome": "The primary paper gives 8192 versus 4096 BF16 MMA ops/clock/SM, 16 exponential ops/clock/SM for both, and the stated 1024/1024/768-cycle M=N=d=128 example." + }, + "claim": "FA4's paper models B200 at twice Hopper's BF16 tensor-core operations per clock per SM while exponential throughput is unchanged, and gives a simplified 128-by-128 example with 1024 MMA, 1024 exponential, and 768 shared-memory cycles.", + "claim_id": "technique-ping-pong-scheduling-current-c003", + "claim_type": "performance", + "covers": ["u020"], + "evidence": [ + { + "locator": "sections 2.2 and 3.1.1", + "observation": "The paper supplies the per-SM rates, example cycle counts, doubled-query scaling, and the caveat that its simplified roofline omits other resources.", + "revision": "FlashAttention-4 paper v1, 2026-03-05", + "url": "https://arxiv.org/html/2603.05451v1" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "The reported peak and utilization isolate ping-pong scheduling because it is the section's central technique.", + "check": "Inspect what system each source benchmarks and whether either includes a q_stage or ping-pong-only ablation.", + "outcome": "Both report complete FA4 forward kernels with multiple simultaneous techniques and no isolated ping-pong timing; the page states that limitation." + }, + "claim": "The FA4 paper reports up to 1613 TFLOP/s and 71% for complete forward kernels, while the author blog reports up to 1605 TFLOP/s and 71%; neither result isolates ping-pong scheduling or proves 100% unit utilization.", + "claim_id": "technique-ping-pong-scheduling-current-c004", + "claim_type": "performance", + "covers": ["u021"], + "evidence": [ + { + "locator": "abstract and performance results", + "observation": "The paper reports up to 1613 TFLOP/s and 71% for FA4 as a complete implementation.", + "revision": "FlashAttention-4 paper v1, 2026-03-05", + "url": "https://arxiv.org/html/2603.05451v1" + }, + { + "locator": "Performance and Key Techniques", + "observation": "The author blog reports up to 1605 TFLOP/s and 71% and lists several techniques contributing to the complete kernel, without a ping-pong-only ablation.", + "revision": "Tri Dao FA4 blog, 2026", + "url": "https://tridao.me/blog/2026/flash4/" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "q_stage=1 is not a supported control, or the pattern is Blackwell-only because the pinned file targets SM100.", + "check": "Inspect accepted q_stage values in source and the paper's stated scheduling lineage.", + "outcome": "The source accepts q_stage 1 or 2 and adjusts roles for one stage; the paper explicitly says the schedule is similar to FA3's Hopper schedule." + }, + "claim": "The pinned SM100 implementation supports q_stage=1 as a controlled comparison to q_stage=2, and the broader scheduling idea is not intrinsically Blackwell-only because the paper traces it to Hopper-era FA3.", + "claim_id": "technique-ping-pong-scheduling-current-c005", + "claim_type": "compatibility", + "covers": ["u022", "u023"], + "evidence": [ + { + "locator": "q_stage validation and role reassignment", + "observation": "The implementation restricts q_stage to 1 or 2 and conditionally changes its role layout for q_stage=1.", + "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", + "url": "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py" + }, + { + "locator": "section 3.1.2", + "observation": "The paper says FA4 follows a ping-pong schedule similar to FlashAttention-3, which targeted Hopper.", + "revision": "FlashAttention-4 paper v1, 2026-03-05", + "url": "https://arxiv.org/html/2603.05451v1" + } + ], + "oracle": "web-primary", + "verdict": "verified" + } + ], + "commands": [ + "Inspect FA4 paper v1 hardware model, pipeline, and performance scope", + "Inspect pinned Dao-AILab SM100 q_stage options, role assignments, TMEM offsets, and pipelines", + "Compare lifecycle requirements with PTX ISA 9.0 tcgen05 contracts" + ], + "id": "technique-ping-pong-scheduling", + "page_verdict": "verified", + "path": "wiki/techniques/ping-pong-scheduling.md", + "residual_risk": "The source was inspected but not compiled or profiled because this host has no CUDA toolkit or GPU. Performance remains author-reported and the recommended q_stage comparison is a procedure, not a measured local result.", + "scope": "All 23 current body units for hash 4553571158ca2be094f2a4198252e45b48298e0e318c60760f7f547f16ed438d were reviewed against the version-pinned paper, implementation, blog, and PTX lifecycle contract." +} diff --git a/verification/reviewed/current/technique-pipeline-stages.json b/verification/reviewed/current/technique-pipeline-stages.json new file mode 100644 index 000000000..63b81f4a8 --- /dev/null +++ b/verification/reviewed/current/technique-pipeline-stages.json @@ -0,0 +1,110 @@ +{ + "path": "wiki/techniques/pipeline-stages.md", + "id": "technique-pipeline-stages", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "technique-pipeline-stages-current-c001", + "claim": "The definition correctly makes stages disjoint SMEM buffers with independently transferred ownership and explicitly declines to guarantee full latency hiding or speedup.", + "claim_type": "mechanism", + "covers": ["u001"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/prs/flashinfer/PR-2387/key-files/include/flashinfer/mamba/selective_state_update.cuh","revision":"flashinfer merge 18804cd51734cccf807356d017733bc757677f15","locator":"state[numStages] and stage-specific full/empty barriers","observation":"The implementation allocates disjoint stages and transfers them between producer/consumer roles; source structure alone does not claim a universal performance result."}], + "challenge": {"alternative":"Modulo indices alias one buffer or stage count itself proves complete hiding.","check":"Trace storage dimensions and distinguish correctness structure from timing evidence.","outcome":"Storage is disjoint but no source-level invariant establishes hidden latency, exactly as the page states."} + }, + { + "claim_id": "technique-pipeline-stages-current-c002", + "claim": "The five-state table and lifecycle paragraphs correctly require prior-read completion, expected TMA transaction accounting, ready-phase observation, exclusive consumer ownership, tcgen05 completion before release, and explicit prologue/phase/tail handling.", + "claim_type": "mechanism", + "covers": ["u002", "u003", "u004", "u005", "u006", "u007", "u008", "u009"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"expected arrivals/transactions, phases and waits","observation":"Full/empty ownership and reusable phase obligations match."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit","revision":"same archive","locator":"MMA completion and operand lifetimes","observation":"SMEM operands cannot be released for overwrite merely after asynchronous MMA issue; completion must be established."}], + "challenge": {"alternative":"Modulo indexing and __syncthreads after issue are sufficient.","check":"Follow first use, first wraparound and asynchronous MMA operand lifetime.","outcome":"Modulo has no readiness state and CTA synchronization does not complete the async proxy; all extra edges named by the page are required."} + }, + { + "claim_id": "technique-pipeline-stages-current-c003", + "claim": "The TMA section correctly states expected-byte registration, autonomous mbarrier transaction completion and consumer wait, while narrowing the benefit and rejecting incomplete illustrative PTX as safe code.", + "claim_type": "mechanism", + "covers": ["u010", "u011", "u012"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"bulk tensor copy mbarrier complete_tx semantics","observation":"The hardware transaction completes on the supplied mbarrier after expected-byte accounting; full descriptors/grammar and proxy ordering are required."}], + "challenge": {"alternative":"TMA completion needs a manual producer arrival after transfer or proves the producer cannot be critical.","check":"Separate hardware complete_tx behavior from performance scheduling.","outcome":"Completion is autonomous, while issue work and load latency may still be exposed; the page states both precisely."} + }, + { + "claim_id": "technique-pipeline-stages-current-c004", + "claim": "The binary16 payload block and linear-growth statement compute 16/32/48/144/240 KiB exactly while distinguishing payload from complete allocation.", + "claim_type": "code", + "covers": ["u013", "u014", "u016"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python exact integer byte arithmetic and stage-count sweep","environment":"Conda base, host CPU","reference":"binary16=2 bytes and disjoint fixed-size operand stage payload","result":"A=16384 B, B=32768 B, stage=49152 B; 3=144 KiB, 5=240 KiB; stage payload is linear.","negative_control":"Four-byte elements or aliased stages fail the stated byte totals/simultaneous-ownership condition."}], + "challenge": {"alternative":"The calculation includes barriers/padding or total allocation is purely linear.","check":"Read the labels/disclaimer and add a fixed metadata term.","outcome":"The page explicitly says unpadded operand payload and notes fixed/stage-dependent metadata separately."} + }, + { + "claim_id": "technique-pipeline-stages-current-c005", + "claim": "The 228-KiB capacity and derived 63.2-percent/over-capacity statements are correctly scoped to CC10.0 per-SM capacity and preserve per-block/occupancy caveats.", + "claim_type": "compatibility", + "covers": ["u015"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2 Blackwell Tuning Guide","locator":"CC10.0 shared-memory capacity and occupancy","observation":"The guide reports 228 KiB/SM and resource-dependent residency; 144/228=63.16% and 240>228."}], + "challenge": {"alternative":"228 KiB is a universal Blackwell per-CTA allocation guarantee.","check":"Distinguish compute capability and per-SM versus per-block opt-in limits.","outcome":"The page states CC10.0 per SM and explicitly retains other limits."} + }, + { + "claim_id": "technique-pipeline-stages-current-c006", + "claim": "The stage-selection section correctly rejects universal depth labels and enumerates the concrete resource, transfer, compute, role, and loop-length variables that must be compiled and measured, including short/equal/long boundary cases.", + "claim_type": "performance", + "covers": ["u017", "u018", "u019", "u020", "u021", "u022", "u023"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/prs/flashinfer/PR-2387/key-files/include/flashinfer/mamba/selective_state_update.cuh","revision":"flashinfer merge 18804cd51734cccf807356d017733bc757677f15","locator":"three-stage and min(totalStages,4) dispatch policies","observation":"One exact kernel uses different stage policies by path/geometry, validating that depth depends on concrete constraints."},{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"pipeline stage parameters and resource construction","observation":"Stage choices interact with tile/storage/pipeline configuration rather than an architecture-only table."}], + "challenge": {"alternative":"Three stages is always standard/full and no compilation or short-K test is needed.","check":"Compare exact code paths and execute the state machine when K tiles are below stage count.","outcome":"Source policies differ and some stages are never produced in short loops, requiring the specified boundary tests."} + }, + { + "claim_id": "technique-pipeline-stages-current-c007", + "claim": "The Gau Nernst paragraph accurately reports/scopes v2b/v3 and later variant values, including the approximately 35.1-percent adjacent pipeline increase without attributing the v6 endpoint to warp specialization alone.", + "claim_type": "performance", + "covers": ["u024"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"sources/blogs/tcgen05-tutorial.md","revision":"article/source commit 3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"source-reported progression","observation":"All four numbers and variant labels match; 939.61/695.43-1 is about 35.1%."}], + "challenge": {"alternative":"1475.93 is the warp-specialization result or the 35.1% is portable.","check":"Inspect each intermediate version and benchmark scope.","outcome":"v4 is 1208.83, v6 adds two more changes, and the page limits the delta to the disclosed source progression."} + }, + { + "claim_id": "technique-pipeline-stages-current-c008", + "claim": "The Modular paragraph accurately scopes five input stages and the 85-percent endpoint to the full multi-step Part 3 design and correctly states that the source lacks the previously attributed fixed 400-cycle premise.", + "claim_type": "performance", + "covers": ["u025"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://www.modular.com/blog/matrix-multiplication-on-nvidias-blackwell-part-3-the-optimizations-behind-85-of-sota-performance","revision":"fetched 2026-08-09","locator":"five-stage input pipeline, warp specialization, output double buffering and 85% endpoint","observation":"The sequence and source-scoped endpoint match, and no fixed 400-cycle HBM statement appears."}], + "challenge": {"alternative":"Five stages alone produced 85% or the page omits a source-defined latency.","check":"Trace the article's kernel sequence and search its content for 400/cycles.","outcome":"Several changes precede the endpoint and the latency phrase is absent; the current wording is properly scoped."} + }, + { + "claim_id": "technique-pipeline-stages-current-c009", + "claim": "All eight validation steps follow from disjoint storage, initialization/publication, transaction, async MMA, phase, tail and boundary requirements and specify discriminating fault tests.", + "claim_type": "mechanism", + "covers": ["u026", "u027", "u028", "u029", "u030", "u031", "u032", "u033"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"init, expected transaction, phase and wait rules","observation":"Steps 1-4 and 6 directly exercise mandatory state/phase behavior."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit","revision":"same archive","locator":"MMA completion/lifetimes","observation":"Step 5 and early-release fault test follow from asynchronous operand lifetime; tail/boundary tests ensure no unproduced stage is waited upon."}], + "challenge": {"alternative":"Only steady-state happy-path output is needed.","check":"Inject wrong phase, early release and short-K states.","outcome":"Each targets a distinct normative edge and can expose deadlock or corruption hidden by steady state."} + }, + { + "claim_id": "technique-pipeline-stages-current-c010", + "claim": "All six primary URLs and three related local links resolve exactly, while fabricated external/local controls fail.", + "claim_type": "other", + "covers": ["u034", "u035", "u036", "u037", "u038", "u039", "u040", "u041", "u042"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"HTTP-check six exact references; resolve three relative links; repeat with fabricated targets","environment":"current worktree/network, 2026-08-09","reference":"Exact primary and related targets should resolve.","result":"All six URLs returned HTTP 200 and all three local files existed.","negative_control":"A fabricated CUDA page returned HTTP 404 and wiki/techniques/nope.md was absent."}], + "challenge": {"alternative":"Redirects or similar names mask missing exact targets.","check":"Require exact archived/immutable URLs and exact relative paths.","outcome":"Every intended target resolved and both controls failed."} + } + ], + "commands": ["archived PTX/tuning-guide inspection", "exact-revision CUTLASS/FlashInfer/tutorial inspection", "Conda-base byte/stage arithmetic", "primary article and link controls"], + "scope": "All 42 current units for body hash 7df323122ab18859d04251398e4b248898e5bca6a3c58fba24d67e3ba0fd70d2 were verified against pinned primary evidence and discriminating host checks.", + "residual_risk": "No CUDA compiler or SM100 GPU is available. The page intentionally leaves stage profitability, barrier overhead and exposed latency to target compilation/profiling." +} diff --git a/verification/reviewed/current/technique-register-budgeting.json b/verification/reviewed/current/technique-register-budgeting.json new file mode 100644 index 000000000..5448de5eb --- /dev/null +++ b/verification/reviewed/current/technique-register-budgeting.json @@ -0,0 +1,176 @@ +{ + "claims": [ + { + "challenge": { + "alternative": "Registers alone determine occupancy, so an inverse rule is sufficient.", + "check": "Compare the occupancy API inputs and launch-bounds threshold behavior with the page's resource model.", + "outcome": "The official API consumes a compiled function, block size, and dynamic shared memory, while the guide describes discrete resident-block thresholds; the page states both limits and avoids a performance guarantee." + }, + "claim": "Register use is one of several discrete residency constraints, and CUDA's occupancy API predicts active blocks for the compiled function and concrete launch inputs without predicting performance.", + "claim_id": "technique-register-budgeting-current-c001", + "claim_type": "semantics", + "covers": ["u001", "u002"], + "evidence": [ + { + "locator": "section 8.2.3.1", + "observation": "The guide defines active-block occupancy calculation from a kernel, block size, and shared-memory usage, and separately describes register thresholds between n and n+1 resident blocks.", + "revision": "CUDA C++ Programming Guide 13.0", + "url": "https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html" + }, + { + "locator": "cudaOccupancyMaxActiveBlocksPerMultiprocessor", + "observation": "The runtime contract returns maximum active blocks per SM for a function, block size, and dynamic shared-memory size.", + "revision": "CUDA Runtime API 13.0.2", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-runtime-api/group__CUDART__OCCUPANCY.html" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "The second launch-bounds argument forces that many blocks and therefore fixes the register count.", + "check": "Trace the compiler's documented derivation and both above-L and below-L cases.", + "outcome": "The compiler derives L, can reduce use with spill/instruction costs, and can increase use up to L; the page's table and prose preserve those exact limits." + }, + "claim": "maxrregcount is a maximum subject to ABI/compiler constraints, two-argument launch bounds guide an architecture-dependent limit rather than exact residency, resource-usage reports compiled resources, and the occupancy API does not measure runtime performance.", + "claim_id": "technique-register-budgeting-current-c002", + "claim_type": "mechanism", + "covers": ["u003", "u004", "u005", "u006", "u007"], + "evidence": [ + { + "locator": "--maxrregcount and --resource-usage", + "observation": "The driver documents the maximum, ABI bump, compiler-reserved registers, tradeoff, and the exact resource report contents.", + "revision": "CUDA Compiler Driver 13.0.2", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html" + }, + { + "locator": "Launch Bounds, lines 8688-8713", + "observation": "The guide defines L and both compiler response branches, and says the optimal values require per-kernel analysis and usually change across architectures.", + "revision": "CUDA C++ Programming Guide 13.0", + "url": "https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "A cap sweep can be interpreted from timings alone without recording compiler output or controlling correctness and launch inputs.", + "check": "Test whether a nominal cap necessarily changes the compiled allocation and whether occupancy APIs accept source-level categories.", + "outcome": "The compiler may naturally use fewer registers than the cap, while the API requires the concrete function and launch; the workflow records and discriminates both before attribution." + }, + "claim": "The shown nvcc commands validly compile an uncapped and 64-register-cap SM100a variant with resource reports, and the five-step procedure controls the evidence needed to attribute a cap effect.", + "claim_id": "technique-register-budgeting-current-c003", + "claim_type": "code", + "covers": ["u008", "u009", "u010", "u011", "u012", "u013", "u014", "u015"], + "evidence": [ + { + "locator": "--gpu-architecture, --maxrregcount and --resource-usage", + "observation": "The archived driver documents sm_100a as a target and the exact long-form options used by the two commands.", + "revision": "CUDA Compiler Driver 13.0.2", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html" + }, + { + "locator": "Occupancy Calculator", + "observation": "The official procedure queries the compiled function with block size and dynamic shared-memory bytes and converts returned blocks to active warps.", + "revision": "CUDA C++ Programming Guide 13.0", + "url": "https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "A memory-bound kernel can always hide spills, and any lower cap should increase occupancy.", + "check": "Inspect spill representation and a cap that remains above natural allocation.", + "outcome": "The compiler reports spills as stack/local-memory operations, and the cited author's 80-to-64 change had no effect because the allocation was already below 64; the page states both negative controls." + }, + "claim": "Spills add local-memory instructions and traffic, while a lower maxrregcount value can be inert when natural use is below the cap; neither a memory-bound label nor a nominal cap determines the outcome.", + "claim_id": "technique-register-budgeting-current-c004", + "claim_type": "performance", + "covers": ["u016"], + "evidence": [ + { + "locator": "Printing Code Generation Statistics", + "observation": "The driver identifies spill stores/loads and stack frame as per-thread stack/local-memory resource use.", + "revision": "CUDA Compiler Driver 13.0.2", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html" + }, + { + "locator": "Attempts 10-12", + "observation": "The author's memory-bound kernel regressed with spill-inducing pressure and saw no change from a cap that remained above natural use.", + "revision": "retrieved 2026-08-08", + "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "TMEM eliminates accumulator register pressure and yields a fixed per-thread saving.", + "check": "Trace D residence, epilogue transfer, and a concrete SM90 accumulator fragment.", + "outcome": "PTX puts tcgen05 D in TMEM, the pinned SM100 epilogue loads it to registers, and the 128-register comparison belongs to one m64n256 FP32 WGMMA shape; the page keeps that scope." + }, + "claim": "tcgen05 stores its resident D accumulator in TMEM, but epilogue batches and other kernel state still need registers; 128 registers per thread is a shape-specific SM90 m64n256 FP32 D example rather than a universal Blackwell saving.", + "claim_id": "technique-register-budgeting-current-c005", + "claim_type": "compatibility", + "covers": ["u017", "u018"], + "evidence": [ + { + "locator": "tcgen05.mma and Tensor Memory", + "observation": "The ISA defines D as a TMEM address and tcgen05.ld as the transfer from TMEM to registers.", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma" + }, + { + "locator": "epilogue path around lines 455-463", + "observation": "The pinned kernel loads Tensor Memory into registers before storing through shared memory.", + "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f", + "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh" + }, + { + "locator": "SM90 m64n256k16 FP32 WGMMA wrapper accumulator operand list", + "observation": "The wrapper exposes 128 float D registers per participating thread for this exact N=256 shape.", + "revision": "CUTLASS 4.5.0 e406c186d2cae5782a846f7280af282ca4fecec2", + "url": "https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/arch/mma_sm90_gmma.hpp" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "The reported 32 and 45 caps establish a causal ranking and a general memory-bound policy.", + "check": "List co-varying mechanisms and inspect the author's own cap experiment.", + "outcome": "The report lists PTX, cache, load-width, specialization, and reuse differences and supplies no cap ablation; its own above-natural cap change had no effect. The page explicitly limits the observation." + }, + "claim": "The author reported three top NVFP4 GEMV solutions around an 18.5 microsecond aggregate and caps of 32 and 45 in ranks 1 and 3, but the confounded implementations and missing ablation do not prove a causal register-cap or occupancy ordering.", + "claim_id": "technique-register-budgeting-current-c006", + "claim_type": "performance", + "covers": ["u019", "u020"], + "evidence": [ + { + "locator": "What the Top Solutions Did Differently and Attempt 11", + "observation": "The author reports the aggregate cluster and caps, enumerates multiple other differences, and reports a no-effect maxrregcount experiment in the author's own implementation.", + "revision": "retrieved 2026-08-08", + "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/" + } + ], + "oracle": "web-primary", + "verdict": "verified" + } + ], + "commands": [ + "Inspect archived CUDA 13 compiler, programming-guide, runtime, and PTX contracts", + "Inspect pinned SM100 DeepGEMM and CUTLASS 4.5.0 register-residence examples", + "Challenge the cited NVFP4 observation with its co-varying mechanisms and no-effect cap experiment" + ], + "id": "technique-register-budgeting", + "page_verdict": "verified", + "path": "wiki/techniques/register-budgeting.md", + "residual_risk": "The commands were syntax-verified against archived documentation but not executed because this environment has no CUDA toolkit or GPU. Cap performance remains target-, compiler-, kernel-, and workload-specific by design.", + "scope": "All 20 current body units, the revised frontmatter claim, command syntax, causal limits, and the cited performance observation were checked against direct versioned sources." +} diff --git a/verification/reviewed/current/technique-software-exp.json b/verification/reviewed/current/technique-software-exp.json new file mode 100644 index 000000000..bc8249ace --- /dev/null +++ b/verification/reviewed/current/technique-software-exp.json @@ -0,0 +1,159 @@ +{ + "claims": [ + { + "challenge": { + "alternative": "FA4 removes the hardware exponential path entirely and the polynomial alone defines the optimization.", + "check": "Trace fragment selection and the surrounding forward schedule in the pinned source.", + "outcome": "The implementation selects hardware or software per fragment and couples the split to two softmax warpgroups, TMEM staging, and correction; the page states that hybrid scope." + }, + "claim": "FA4's SM100 forward path evaluates a configuration-dependent fraction of exponentials with software FMA polynomial code while retaining hardware exp2 for the rest, inside a larger warp-specialized pipeline.", + "claim_id": "technique-software-exp-current-c001", + "claim_type": "mechanism", + "covers": ["u001", "u002"], + "evidence": [ + { + "locator": "forward pipeline and faster exponential sections", + "observation": "The first-party description states a tunable per-row split between hardware MUFU and software exponential and documents the two softmax warpgroups, TMEM stages, and correction role.", + "revision": "FlashAttention-4 blog, retrieved 2026-08-09", + "url": "https://tridao.me/blog/2026/flash4/" + }, + { + "locator": "softmax.py apply_exp2_convert and flash_fwd_sm100.py tuning table", + "observation": "The pinned code branches between cute.math.exp2 and ex2_emulation_2 by fragment and contains both zero-emulation and mixed configurations.", + "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", + "url": "https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "The feeds imply an architecture-wide SFU bottleneck and a fixed software speedup.", + "check": "Read the dimensions and distinguish analytical per-tile accounting from a measured software-exp ablation.", + "outcome": "The source fixes M=N=D=128 and reports modeled resource cycles; the page preserves that scope and explicitly rejects the unsupported universal inference." + }, + "claim": "The authors' B200 M=N=D=128 model uses 8192 BF16 tensor-core ops/cycle, 16 exp ops/cycle, 128 shared-memory bytes/cycle, and assigns 1024, 1024, and 768 forward cycles respectively, without defining universal software-exp throughput.", + "claim_id": "technique-software-exp-current-c002", + "claim_type": "performance", + "covers": ["u003", "u004"], + "evidence": [ + { + "locator": "Feeds and Speeds", + "observation": "The source lists the exact feeds and forward speeds for M=N=D=128 and motivates overlap rather than an isolated instruction-throughput result.", + "revision": "FlashAttention-4 blog, retrieved 2026-08-09", + "url": "https://tridao.me/blog/2026/flash4/" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "FA4 uses nearest-integer reduction, a symmetric interval, four Taylor FMAs, or hardware ex2 for reconstruction.", + "check": "Inspect constants, rounding modes, coefficient table, Horner evaluator, and combine function at the pinned revision.", + "outcome": "The exact code uses lower clamp -127, floor-style reduction via add.rm and 2^23+2^22, degree-3 packed Horner on [0,1), and integer exponent-field reconstruction; every page value matches." + }, + "claim": "Pinned FA4 clamps below -127 under an x<=127 assumption, derives floor and a fraction in [0,1), evaluates the listed degree-3 Sollya coefficient set with packed FMAs, reconstructs through exponent-field integer operations, and uses log2(e) to map natural exponential inputs.", + "claim_id": "technique-software-exp-current-c003", + "claim_type": "code", + "covers": ["u005", "u006", "u007", "u008", "u009", "u010", "u011", "u012", "u013", "u014", "u015", "u016"], + "evidence": [ + { + "locator": "POLY_EX2, evaluate_polynomial_2, combine_int_frac_ex2, ex2_emulation, ex2_emulation_2", + "observation": "The exact source contains the page's full coefficients, clamp/assumption, magic constant, rounding direction, three packed Horner steps, and integer reconstruction.", + "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", + "url": "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py" + }, + { + "locator": "Faster exponential section", + "observation": "The authors describe Cody-Waite floor reduction, degree-3 Horner coefficients selected with Sollya on [0,1), and exponent-bit combination.", + "revision": "FlashAttention-4 blog, retrieved 2026-08-09", + "url": "https://tridao.me/blog/2026/flash4/" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "The sampled error is a formal maximum or the old degree-4 coefficient result was accidentally reused.", + "check": "Recompute the current coefficient set over its stated interval and run the old coefficient set as a negative control.", + "outcome": "The current set reproduced about 8.763e-5 while the negative control produced about 5.568e-5; the page explicitly labels its value sampled rather than a proof." + }, + "claim": "A 1,000,002-point host grid over [0,1] gives approximately 8.763e-5 maximum sampled relative error for the pinned rounded degree-3 coefficients, without establishing a continuous error bound.", + "claim_id": "technique-software-exp-current-c004", + "claim_type": "semantics", + "covers": ["u017"], + "evidence": [ + { + "command": "Evaluate sum(p_i*x**i) / 2**x - 1 at x=i/1000001 for i=0..1000001", + "environment": "Conda base Python 3 host arithmetic", + "negative_control": "The removed degree-4 Taylor set on [-0.5,0.5] produced 5.5684318728932425e-5 instead of the current result.", + "reference": "Pinned POLY_EX2[3] coefficient tuple in flash_attn/cute/utils.py", + "result": "Maximum absolute sampled relative error 8.762961710306882e-5 at x approximately 0.1007928992." + } + ], + "oracle": "host-experiment", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "BF16 conversion makes scalar and end-to-end validation unnecessary, or an SFU utilization metric alone selects the winning split.", + "check": "Trace error propagation and configuration-dependent hybrid settings through the concrete softmax pipeline.", + "outcome": "Approximation affects numerator and denominator, special cases live outside the helper, and pinned settings include hardware-only and mixed paths; the procedure tests all discriminating layers." + }, + "claim": "Software-exp adoption requires scalar/special-case accuracy tests, complete attention correctness, generated-code inspection, resource profiling, and matched performance sweeps over the hybrid fraction rather than dtype or profiler-label assumptions.", + "claim_id": "technique-software-exp-current-c005", + "claim_type": "other", + "covers": ["u018", "u019", "u020", "u021", "u022", "u023", "u024", "u025"], + "evidence": [ + { + "locator": "utils.py helper assumptions; softmax.py special cases; flash_fwd_sm100.py tuning table", + "observation": "The pinned source separates helper-domain assumptions from softmax special handling and selects different emulation frequencies, including zero, by configuration.", + "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", + "url": "https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute" + }, + { + "locator": "forward pipeline and benchmark sections", + "observation": "The authors treat exponential selection as one component of a coupled pipeline and report configuration-level results, supporting the page's requirement for end-to-end controlled comparison.", + "revision": "FlashAttention-4 blog, retrieved 2026-08-09", + "url": "https://tridao.me/blog/2026/flash4/" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "The 1.1-1.3x figure is an isolated software-exp speedup.", + "check": "Inspect the benchmark label and the full list of forward design changes.", + "outcome": "The comparison is complete FA4 forward versus cuDNN 9.13 and includes multiple co-designed mechanisms; the page says exactly that." + }, + "claim": "FA4 reports complete forward speedups of 1.1-1.3x over cuDNN 9.13 on evaluated B200 BF16 cases, not a software-exponential-only ablation.", + "claim_id": "technique-software-exp-current-c006", + "claim_type": "performance", + "covers": ["u026"], + "evidence": [ + { + "locator": "Attention Benchmarks and forward-pass design", + "observation": "The first-party comparison is labeled for FA4 forward and the implementation includes pipelining, software/hardware exp, conditional rescaling, TMEM staging, and scheduling.", + "revision": "FlashAttention-4 blog, retrieved 2026-08-09", + "url": "https://tridao.me/blog/2026/flash4/" + } + ], + "oracle": "web-primary", + "verdict": "verified" + } + ], + "commands": [ + "Inspect pinned FA4 utils.py, softmax.py, and flash_fwd_sm100.py at a369df7", + "Compare first-party feeds-and-speeds and benchmark scope", + "Recompute the published rounded coefficient error with a negative control" + ], + "id": "technique-software-exp", + "page_verdict": "verified", + "path": "wiki/techniques/software-exp.md", + "residual_risk": "No B200 SASS or performance run was available. The numerical grid is a regression measurement, not a formal bound, and the page requires target-specific accuracy and performance validation.", + "scope": "All 26 current body units, pinned code links, numerical value, performance provenance, architecture scope, and frontmatter claim were checked." +} diff --git a/verification/reviewed/current/technique-swizzling.json b/verification/reviewed/current/technique-swizzling.json new file mode 100644 index 000000000..fd6da38a7 --- /dev/null +++ b/verification/reviewed/current/technique-swizzling.json @@ -0,0 +1,165 @@ +{ + "claims": [ + { + "challenge": { + "alternative": "A swizzle eliminates bank conflicts by construction, regardless of consumer access.", + "check": "Apply the documented bank mapping to different warp address sets and include same-word/broadcast cases.", + "outcome": "Conflicts are properties of the concrete request; the page defines that dependency and limits swizzle benefit to a target access pattern." + }, + "claim": "Shared memory maps successive 32-bit words to 32 banks, conflicts serialize different-word same-bank accesses within a warp request, and a swizzle may reduce but cannot universally eliminate conflicts.", + "claim_id": "technique-swizzling-current-c001", + "claim_type": "semantics", + "covers": ["u001", "u002"], + "evidence": [ + { + "locator": "Shared Memory and Bank Conflicts", + "observation": "The guide defines 32 banks, successive 32-bit-word mapping, conflict serialization, and same-word/broadcast exceptions.", + "revision": "Nsight Compute Profiling Guide 13.3", + "url": "https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "128B is the only legal Blackwell mapping, and a tensor-map enumerator alone determines correctness.", + "check": "Compare TMA enum mappings, encoder constraints, and tcgen05 descriptor layout values.", + "outcome": "Both APIs expose multiple modes and independent constraints, while the producer destination and consumer addressing must denote the same physical layout; the page lists exact common encodings." + }, + "claim": "A TMA-to-tcgen05 tile requires consistent tensor-map, base/alignment, and consumer-descriptor mappings; CUDA 13 defines the listed TMA chunk spans and PTX 9.0 defines legal no/32B/64B/128B tcgen05 modes with exact layout-field values and combination-specific constraints.", + "claim_id": "technique-swizzling-current-c002", + "claim_type": "compatibility", + "covers": ["u003", "u004", "u005", "u006", "u007", "u008", "u009", "u010", "u011", "u012", "u013", "u014", "u015"], + "evidence": [ + { + "locator": "CUtensorMapSwizzle and cuTensorMapEncodeTiled constraints", + "observation": "The API lists NONE, 32B/64B/128B 16B-chunk mappings and 128B atom variants, plus datatype/interleave/inner-box restrictions.", + "revision": "CUDA Driver API 13.0.97", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html" + }, + { + "locator": "tcgen05 shared-memory descriptor and valid combinations", + "observation": "PTX defines the descriptor fields, layout values, invalid values, and type/major/shape constraints quoted by the page.", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "Swizzle<3,4,3> means three base bits, four mask bits, and can be applied from row indices without base alignment.", + "check": "Evaluate the CUTLASS template masks and compare layout versus pointer swizzling.", + "outcome": "The template source defines the opposite parameter meanings, yields address bits 7:9 into 4:6, and explicitly distinguishes position-dependent hardware pointers; the page matches." + }, + "claim": "CUTLASS Swizzle uses mask width, invariant low bits, and field distance respectively; Swizzle<3,4,3> maps address bits 7:9 into 4:6, with equivalence to a logical row formula dependent on stride, byte units, composition, and base alignment.", + "claim_id": "technique-swizzling-current-c003", + "claim_type": "code", + "covers": ["u016", "u017", "u018", "u019", "u020", "u021"], + "evidence": [ + { + "locator": "Swizzle definition and apply", + "observation": "The header defines parameter meanings, mask positions, and XOR application exactly as summarized.", + "revision": "CUTLASS 4.5.0 e406c186d2cae5782a846f7280af282ca4fecec2", + "url": "https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/swizzle.hpp" + }, + { + "locator": "pointer_swizzle position dependence", + "observation": "CUTLASS states that hardware chose position-dependent swizzles and distinguishes swizzle_ptr from composed swizzle_layout.", + "revision": "CUTLASS 4.5.0 e406c186d2cae5782a846f7280af282ca4fecec2", + "url": "https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/pointer_swizzle.hpp" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "Supplying CU_TENSOR_MAP_SWIZZLE_128B is sufficient, and any mismatch always appears as a runtime error.", + "check": "Enumerate every encoder input/constraint and distinguish invalid encoding from a legal but logically inconsistent consumer.", + "outcome": "The driver validates encoder contracts through CUresult, while consumer semantics are separate; the checklist and two failure classes are correctly separated." + }, + "claim": "A reproducible tiled tensor map must validate alignment, rank-specific dimensions/strides, box and element strides, mode constraints, CUresult, and the matching consumer mapping; successful encoding alone does not validate logical MMA output.", + "claim_id": "technique-swizzling-current-c004", + "claim_type": "mechanism", + "covers": ["u022", "u023", "u024", "u025", "u026", "u027", "u028"], + "evidence": [ + { + "locator": "cuTensorMapEncodeTiled parameters, return values, and constraints", + "observation": "The archived API defines output/base alignment, rank-sized arrays, rank-minus-one byte strides, box/element constraints, swizzle restrictions, and CUresult failures.", + "revision": "CUDA Driver API 13.0.97", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html" + }, + { + "locator": "tcgen05 shared-memory descriptor", + "observation": "The consumer descriptor independently supplies base, leading/stride dimensions, offset, major mode, and swizzle layout.", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "The 2.75x result isolates bank-conflict removal by 128B swizzling.", + "check": "Diff the tutorial's v1b/v2b encoder, TMA issuance, and descriptors and read the author's causal discussion.", + "outcome": "The inner TMA width and loop structure change together with swizzle/descriptor mode, and the author explicitly proposes TMA efficiency as an alternative; the page reports a combined treatment." + }, + "claim": "The pinned tutorial's B200 4096-cube 3D TMA path reports 252.81 TFLOP/s for 16B/no-swizzle and 695.43 for 128B/128B-swizzle, approximately 2.75x and 46% of its cuBLAS result, without isolating bank-conflict or swizzle causality.", + "claim_id": "technique-swizzling-current-c005", + "claim_type": "performance", + "covers": ["u029", "u030", "u031", "u032"], + "evidence": [ + { + "locator": "128-byte global load and swizzling, v1/v2 diff, benchmark, and remarks", + "observation": "The first-party tutorial supplies the exact values, simultaneous 16B-to-128B TMA and swizzle changes, lack of ncu access, and alternative wider-load explanation.", + "revision": "tcgen05 for dummies; source commit 3b90ac9b3f624bdf1f6f78d02dcd533675d36573", + "url": "https://gau-nernst.github.io/tcgen05/" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "A zero conflict counter or successful tensor-map encode is sufficient to select and validate a swizzle.", + "check": "Separate encoding validity, output permutation correctness, per-access wavefronts, and end-to-end timing.", + "outcome": "Each oracle discriminates a different failure mode, and the procedure controls producer/consumer pairs, correctness, profiler-version metric discovery, resources, and timing." + }, + "claim": "Swizzle verification requires controlled matched-layout variants, explicit encoder and output checks, tool-discovered shared-memory request/wavefront/conflict metrics, end-to-end timing, and recorded environment rather than one counter or architecture heuristic.", + "claim_id": "technique-swizzling-current-c006", + "claim_type": "other", + "covers": ["u033", "u034", "u035", "u036", "u037", "u038", "u039"], + "evidence": [ + { + "locator": "Metrics Examples and Shared Memory table", + "observation": "Nsight Compute documents metric discovery/naming, l1tex bank-conflict aggregates, and requests/wavefronts/conflicts as distinct shared-memory observations.", + "revision": "Nsight Compute 13.3", + "url": "https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html" + }, + { + "locator": "cuTensorMapEncodeTiled and tcgen05 descriptor contracts", + "observation": "The producer and consumer interfaces have separate validity and layout requirements, supporting independent encoding and numerical checks.", + "revision": "CUDA 13.0.2", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html" + } + ], + "oracle": "web-primary", + "verdict": "verified" + } + ], + "commands": [ + "Inspect CUDA 13 tensor-map and PTX tcgen05 mapping contracts", + "Inspect CUTLASS 4.5.0 address-bit swizzle semantics", + "Trace tutorial v1b/v2b changes and causal caveat", + "Inspect current Nsight Compute bank-conflict measurement model" + ], + "id": "technique-swizzling", + "page_verdict": "verified", + "path": "wiki/techniques/swizzling.md", + "residual_risk": "No GPU experiment was available. The page intentionally requires per-consumer correctness and profiler comparison because legal mappings do not imply conflict-free or faster execution.", + "scope": "All 39 current body units, exact enum/descriptor values, CUTLASS semantics, performance provenance, validation workflow, and frontmatter claim were checked." +} diff --git a/verification/reviewed/current/technique-tile-scheduling.json b/verification/reviewed/current/technique-tile-scheduling.json new file mode 100644 index 000000000..0056d530f --- /dev/null +++ b/verification/reviewed/current/technique-tile-scheduling.json @@ -0,0 +1,83 @@ +{ + "path": "wiki/techniques/tile-scheduling.md", + "id": "technique-tile-scheduling", + "page_verdict": "verified", + "claims": [ + { + "claim_id": "technique-tile-scheduling-current-c001", + "claim": "The page correctly separates software coordinate order, resident-worker iteration, CLC reassignment of unlaunched IDs, and Stream-K/Split-K decomposition, and states that raw CLC has no raster/swizzle policy operand.", + "claim_type": "mechanism", + "covers": ["u001", "u002", "u003", "u004", "u005", "u006"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"sources/docs/cutlass-clc-documentation.md","revision":"CUTLASS 4.5.0 e406c186; checked against pinned documentation/source","locator":"Verified scope and CUTLASS integration","observation":"The source card distinguishes launched ClcIDs, software coordinate transforms and higher-level Stream-K decomposition exactly as the page does."}], + "challenge": {"alternative":"CUTLASS software integration makes raster/swizzle part of CLC itself, so the layer distinction is artificial.","check":"Compare raw PTX operands with CUTLASS scheduler transforms and Stream-K parameter state.","outcome":"Only response and mbarrier state reach the CLC instruction; raster and decomposition occur in software, making the distinction operationally necessary."} + }, + { + "claim_id": "technique-tile-scheduling-current-c002", + "claim": "The row-major and positive-grid static-stride snippets map quotient/remainder coordinates and cover every flat tile once, while the page correctly requires edge bijection and measurements before making cache claims.", + "claim_type": "code", + "covers": ["u007", "u008", "u009", "u010", "u011"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python enumerate quotient/remainder and CTA-stride mappings for divisible and nondivisible tile counts","environment":"Conda base, host CPU","reference":"All valid indices must occur once; coordinate order alone is not a cache simulation","result":"The displayed formulas produced exact unique coverage for positive grid sizes and correct row-major coordinates across tested edge cases.","negative_control":"Using gridDim.x-1 duplicated work, while the deleted baseline swizzle emitted out-of-range N on a 3x5 grid."}], + "challenge": {"alternative":"A nondivisible last iteration causes duplication, or the code silently assumes tiles_n divides total_tiles.","check":"Enumerate total_tiles/grid sizes with remainders one and grid-1 and use a partial final M row.","outcome":"The upper-bound check excludes invalid tail coordinates and every valid flat index still appears exactly once."} + }, + { + "claim_id": "technique-tile-scheduling-current-c003", + "claim": "The SM100 CLC lifecycle accurately states full-grid launch, initial blockIdx work, asynchronous 16-byte try_cancel response on mbarrier, success-only decode, cluster-rank coordinate derivation, terminal observed failure, and conservation of grid IDs.", + "claim_type": "mechanism", + "covers": ["u012", "u013", "u014", "u015", "u016", "u017"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"try_cancel/query semantics, completion, restrictions and cluster behavior","observation":"The normative rules match each stated request, response, failure and coordinate condition."},{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0 e406c186","locator":"Dynamic Scheduler rules","observation":"The full grid and two ClcID paths establish the page's conservation and uneven-availability scope."}], + "challenge": {"alternative":"CLC produces new queue work or assigns the initial tile through try_cancel.","check":"Trace every documented ClcID from grid launch through initial or canceled handling.","outcome":"Every ID exists in the launched grid and is handled on exactly one of the two paths; initial blockIdx precedes requests."} + }, + { + "claim_id": "technique-tile-scheduling-current-c004", + "claim": "The CUTLASS 4.5.0 table gives the exact SM100 public scheduler tags and selected implementation classes, identifies the group wrapper as SM90-style static scheduling, and accurately distinguishes deterministic turnstile from nondeterministic atomic-workspace Stream-K reduction.", + "claim_type": "compatibility", + "covers": ["u018", "u019", "u020", "u021", "u022", "u023", "u024"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/tile_scheduler.hpp","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"TileSchedulerSelector specializations for arch::Sm100","observation":"Each public tag maps to the implementation shown in the table, including default void, dynamic, static, Stream-K and group routes."},{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/sm100_tile_scheduler_group.hpp","revision":"same tag","locator":"class preamble and UnderlyingScheduler alias","observation":"The SM100 group class explicitly wraps PersistentTileSchedulerSm90Group and describes a static grouped scheduler."},{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/tile_scheduler_detail.hpp","revision":"same tag","locator":"ReductionMode enum","observation":"Deterministic uses lock/turnstile ordering; Nondeterministic uses mostly atomic workspace accumulation."}], + "challenge": {"alternative":"The tags are mere declarations with different effective mappings, or all Stream-K reductions are atomic.","check":"Follow every arch::Sm100 selector and inspect the reduction enum implementation comments.","outcome":"The specializations resolve to the listed classes and expose two materially different reduction modes."} + }, + { + "claim_id": "technique-tile-scheduling-current-c005", + "claim": "The tail model's Euclidean-division code and 150/142 example correctly handle both partial and exactly divisible final waves under the stated equal-duration, independent-tile, one-worker assumptions.", + "claim_type": "code", + "covers": ["u025", "u026", "u027", "u028"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python evaluate divmod for (150,142), (142,142), (284,142), and (143,142)","environment":"Conda base, host CPU","reference":"T=qW+r with 0<=r", + "observation": "The selector maps the PersistentScheduler tag for arch::Sm100 to PersistentTileSchedulerSm100." + } + ], + "challenge": { + "alternative": "PersistentScheduler maps to the older SM90 static scheduler on SM100.", + "check": "Inspect the architecture-specific specialization at the pinned v4.5.0 revision.", + "outcome": "The arch::Sm100 specialization names PersistentTileSchedulerSm100." + } + }, + { + "claim_id": "hw-clc-original-c012", + "claim": "The GemmUniversal typedef shown is a valid CUTLASS 4.5.0 SM100 CLC kernel instantiation.", + "claim_type": "code", + "covers": ["u017"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/95_blackwell_gemm_green_context/95_blackwell_gemm_green_context.cu", + "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "dynamic_kernel_config", + "observation": "The SM100 CLC example builds collective mainloop and epilogue types, uses cute::Shape tile/cluster shapes, sets DynamicPersistentScheduler, and instantiates cutlass::gemm::kernel::GemmUniversal with four template parameters. The wiki uses an incompatible legacy device::GemmUniversal signature and places PersistentScheduler as a trailing argument." + } + ], + "challenge": { + "alternative": "CUTLASS 4.5.0 retains a compatible legacy overload for exactly the shown SM100 form.", + "check": "Compare the complete pinned official SM100 dynamic-persistent example and scheduler position.", + "outcome": "The authoritative example uses the 3.x collective API and a different kernel template structure." + } + }, + { + "claim_id": "hw-clc-original-c013", + "claim": "The simplified ClcTileScheduler block accurately models a synchronous coordinate-returning CLC query.", + "claim_type": "code", + "covers": ["u018"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp", + "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "PersistentTileSchedulerSm100::advance_to_next_work and fetch_next_work", + "observation": "CUTLASS issues an async query through PipelineCLCFetchAsync, waits as a consumer, decodes a staged opaque response, releases the stage, and then swizzles/rasterizes. The wiki's synchronous clc_try_cancel call omits the required pipeline and response semantics." + } + ], + "challenge": { + "alternative": "The pipeline is only a latency optimization and synchronous direct coordinate return is semantically valid.", + "check": "Compare the PTX opaque-response completion model and the pinned CUTLASS implementation.", + "outcome": "Async shared-memory completion and explicit response decoding are part of the instruction semantics, not optional scheduling decoration." + } + }, + { + "claim_id": "hw-clc-original-c014", + "claim": "The three-row table gives generally valid exact SM-utilization values for 2048, 4096, and 8192 square GEMMs under static and CLC scheduling.", + "claim_type": "performance", + "covers": ["u019", "u020", "u021", "u022"], + "verdict": "refuted", + "oracle": "host-experiment", + "evidence": [ + { + "command": "conda run -n base python -c 'import math; cases=[(\"B200_128x256\",148,128,256),(\"B200_256x256\",148,256,256),(\"132SM_128x256\",132,128,256)]; out=[]\nfor name,sms,tm,tn in cases:\n tiles=math.ceil(2048/tm)*math.ceil(2048/tn); out.append((name,tiles,round(100*min(tiles,sms)/sms,2)))\nprint({\"same_2048x2048_different_scope\":out,\"negative_control_assume_invariant\":len({x[2] for x in out})==1})'", + "environment": "Conda base, Python 3.13.14, CPU-only tile-count upper-bound model; no GPU performance inference beyond active-SM capacity.", + "reference": "NVIDIA's wave-quantization model makes active-SM capacity depend on tile shape and SM count; exact utilization additionally requires hardware, dtype, software, occupancy, and timing scope.", + "result": "For the same 2048x2048 output, active-SM capacity is 86.49% for 128x256 tiles on 148 SMs, 43.24% for 256x256 tiles on 148 SMs, and 96.97% for 128x256 tiles on 132 SMs.", + "negative_control": "The deliberately false invariant-scope control evaluates to false because the three scoped capacities differ." + } + ], + "challenge": { + "alternative": "The table's percentages are invariant properties of GEMM dimension alone.", + "check": "Hold the 2048x2048 output constant while varying only documented scope variables: tile shape and SM count.", + "outcome": "The capacity changes materially, so unscoped exact utilization and improvement values are not valid factual measurements." + } + }, + { + "claim_id": "hw-clc-original-c015", + "claim": "The tcgen05 for dummies result improved from about 940 to 1476 TFLOPS because the final kernel used persistent scheduling plus CLC.", + "claim_type": "performance", + "covers": ["u023"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://gau-nernst.github.io/tcgen05/", + "revision": "Article dated 2025-12-21; code pinned at gau-nernst/learn-cuda commit 3b90ac9b", + "locator": "Persistent kernel (with static scheduling), final performance table and closing remarks", + "observation": "The author labels 1475.93 TFLOPS as persistent with static scheduling, reports 939.61 TFLOPS for pipelining, and explicitly lists CLC as a feature not added." + } + ], + "challenge": { + "alternative": "The section title is stale and the final code actually issues CLC instructions.", + "check": "Inspect the pinned final algorithm, performance row label, and the closing statement that CLC remains unimplemented.", + "outcome": "All three loci identify static scheduling and exclude CLC from the measured kernel." + } + }, + { + "claim_id": "hw-clc-original-c016", + "claim": "Small output dimensions can reduce GEMM tile parallelism and expose wave-quantization or under-utilization effects.", + "claim_type": "performance", + "covers": ["u024"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html", + "revision": "NVIDIA Matrix Multiplication Background User's Guide", + "locator": "2.3 Typical Tile Dimensions and 3.2 Wave Quantization", + "observation": "NVIDIA documents the tradeoff between tile parallelism and efficiency and shows that an underfilled tile wave reduces SM utilization." + } + ], + "challenge": { + "alternative": "Only K, not the output M-N footprint, controls parallel work and GPU utilization.", + "check": "Inspect the guide's output tiling model and wave-quantization example.", + "outcome": "Thread blocks tile the M-N output, and the resulting tile count directly controls wave occupancy." + } + }, + { + "claim_id": "hw-clc-original-c017", + "claim": "For the shown M=32, N=K=8192 and 128x256 tile, a B200 has 132 SMs and CLC lets 32 persistent CTAs handle the tiles while avoiding the static scheduler's idle-SM condition.", + "claim_type": "code", + "covers": ["u025"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://developer.nvidia.com/blog/scaling-autonomous-ai-agents-and-workloads-with-nvidia-dgx-spark/", + "revision": "NVIDIA technical blog, Table 8, 2026", + "locator": "Table 8 platform specifications", + "observation": "The NVIDIA B200 configuration has 148 SMs, not 132." + }, + { + "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", + "revision": "CUDA Programming Guide 13.2", + "locator": "4.12 Cluster Launch Control flow", + "observation": "CLC steals indices from the already launched problem-sized grid. With only 32 output tiles, it cannot create enough independent ClcIDs to occupy 148 SMs; a request can only cancel one of those 32 not-yet-started blocks." + } + ], + "challenge": { + "alternative": "CLC turns a 32-block problem into an SM-count-sized persistent grid with additional work IDs.", + "check": "Compare the 32-tile arithmetic with the official full-grid and cancellation model.", + "outcome": "The grid contains one ID per problem block; CLC redistributes those IDs but does not increase their number." + } + }, + { + "claim_id": "hw-clc-original-c018", + "claim": "CLC cancellation operates at cluster granularity, so a 2x2 worker cluster consumes a 2x2 set of ClcIDs on a successful query.", + "claim_type": "semantics", + "covers": ["u026"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", + "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "Dynamic Scheduler with Cluster Launch Control, rule 6", + "observation": "The documentation explicitly states cluster granularity and gives the 2x2 worker/2x2 ClcID example." + } + ], + "challenge": { + "alternative": "Each CTA in a cluster independently cancels exactly one unrelated CTA ID.", + "check": "Inspect rule 6 and the official cluster programming example's multicast response semantics.", + "outcome": "One cluster-level query consumes the matching cluster-sized group and broadcasts the first CTA ID." + } + }, + { + "claim_id": "hw-clc-original-c019", + "claim": "The cooperative_clc_loop code is a valid 2-CTA CLC plus cta_group::2 execution pattern.", + "claim_type": "code", + "covers": ["u027"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", + "revision": "CUDA Programming Guide 13.2", + "locator": "4.12.2.2 Use-case: Thread Block Clusters", + "observation": "The cluster form requires a single cluster thread, multicast try_cancel, per-CTA local mbarriers at cluster scope, cluster synchronization to guarantee all blocks exist, and local-rank adjustment of the returned first CTA ID. The wiki code implements none of these requirements." + } + ], + "challenge": { + "alternative": "clc_try_cancel_cluster encapsulates every required barrier, multicast, rank, and proxy operation.", + "check": "The helper is undefined and the surrounding code still derives rank from blockIdx.x modulo 2 without the documented returned-ID adjustment or cluster lifetime synchronization.", + "outcome": "The visible algorithm cannot establish the required cluster semantics even under that abstraction." + } + }, + { + "claim_id": "hw-clc-original-c020", + "claim": "CUTLASS can apply a software swizzle/rasterization transform to initial and CLC-returned work coordinates.", + "claim_type": "mechanism", + "covers": ["u028"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp", + "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "PersistentTileSchedulerSm100::fetch_next_work and swizzle_and_rasterize", + "observation": "The scheduler decodes a CLC response and passes its coordinates through swizzle_and_rasterize; Arguments exposes max_swizzle_size and raster_order." + } + ], + "challenge": { + "alternative": "Only the initial blockIdx is swizzled while CLC-returned IDs bypass the transform.", + "check": "Inspect fetch_next_work immediately after work_tile_info_from_clc_response.", + "outcome": "The decoded returned tile is explicitly passed to swizzle_and_rasterize." + } + }, + { + "claim_id": "hw-clc-original-c021", + "claim": "The apply_l2_swizzle code implements a collision-free Morton/Z-order mapping of all input tiles.", + "claim_type": "code", + "covers": ["u029"], + "verdict": "refuted", + "oracle": "host-experiment", + "evidence": [ + { + "command": "conda run -n base python -c 'N=4; bits=2; f=lambda m,n: ((m*N+n>>bits)//N,((m*N+n>>bits)%N)^(((m*N+n>>bits)//N)&((1<>b)&1)<<(2*b)|((m>>b)&1)<<(2*b+1) for b in range(2)); print({\"wiki_outputs\":out,\"unique\":len(set(out)),\"expected_unique\":16,\"claimed_morton_order\":sorted(((morton(m,n),m,n) for m in range(4) for n in range(4))),\"negative_control_identity_unique\":len(set((m,n) for m in range(4) for n in range(4)))})'", + "environment": "Conda base, Python 3.13.14; direct integer translation of the wiki function for a 4x4 grid with swizzle_bits=2.", + "reference": "A tile permutation, including a Morton order, must map all 16 input coordinates to 16 unique output coordinates.", + "result": "The function produces only four unique outputs; every four inputs collide. Its sequence differs from the 16-entry bit-interleaved Morton reference.", + "negative_control": "An identity mapping over the same grid produces all 16 unique outputs, showing the uniqueness check discriminates a valid permutation from the wiki function." + } + ], + "challenge": { + "alternative": "The repeated coordinates are expected Morton grouping rather than destructive collisions.", + "check": "Compare cardinality and exact order against bit-interleaved Morton indices.", + "outcome": "A traversal must retain one output per tile; collapsing 16 inputs to 4 outputs cannot be a traversal permutation." + } + }, + { + "claim_id": "hw-clc-original-c022", + "claim": "CLC scheduling overhead is universally near zero and CLC tail handling is universally optimal compared with a software atomic scheduler.", + "claim_type": "performance", + "covers": ["u030", "u031"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cutlass/4.3.3/media/docs/cpp/blackwell_cluster_launch_control.html", + "revision": "CUTLASS 4.3.3", + "locator": "Static Scheduler and Cluster Launch Control Pipeline Class", + "observation": "CUTLASS calls static scheduling zero-overhead and explicitly pipelines CLC queries to hide their latency. This contradicts an unqualified near-zero CLC cost, while no universal optimality guarantee is stated." + }, + { + "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", + "revision": "CUDA Programming Guide 13.2", + "locator": "4.12 Cluster Launch Control flow", + "observation": "CLC requests can decline for end-of-grid or other reasons such as a higher-priority kernel, so the API does not guarantee an optimal tail schedule." + } + ], + "challenge": { + "alternative": "Hardware CLC has no meaningful request latency or failure behavior under any workload.", + "check": "Inspect the CUTLASS latency-hiding pipeline rationale and the CUDA request-decline conditions.", + "outcome": "Both query latency and non-end-of-grid declines are documented, refuting the categorical comparison." + } + }, + { + "claim_id": "hw-clc-original-c023", + "claim": "CLC exposes try_cancel, while a software work queue must explicitly manage its atomic state and synchronization.", + "claim_type": "semantics", + "covers": ["u032"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "9.7.13.17 clusterlaunchcontrol.try_cancel", + "observation": "PTX defines the CLC request directly." + }, + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#atomic-functions", + "revision": "CUDA C++ Programming Guide 13.0.2", + "locator": "10.14 Atomic Functions", + "observation": "CUDA atomics provide read-modify-write primitives and defined scopes but do not constitute a tile scheduler; queue state, indexing, termination, and ordering remain software responsibilities." + } + ], + "challenge": { + "alternative": "CUDA atomicAdd itself supplies the same ClcID selection, cancellation, and completion protocol as CLC.", + "check": "Compare the normative APIs and operands of atomicAdd and clusterlaunchcontrol.try_cancel.", + "outcome": "atomicAdd changes a caller-provided word; CLC interacts with pending grid launches and returns an opaque cancellation response." + } + }, + { + "claim_id": "hw-clc-original-c024", + "claim": "L2 tile swizzling is a CLC hardware policy configured at kernel launch rather than a CUTLASS scheduler coordinate transform.", + "claim_type": "mechanism", + "covers": ["u033"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp", + "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "Arguments and swizzle_and_rasterize", + "observation": "max_swizzle_size and raster_order are CUTLASS scheduler arguments, and the transformation is ordinary scheduler code applied after CLC response decoding." + }, + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "9.7.13.17 and 9.7.13.18", + "observation": "The CLC PTX surface contains request and response operations and has no swizzle-policy operand or initialization instruction." + } + ], + "challenge": { + "alternative": "The CUTLASS arguments are forwarded to an undocumented CLC hardware launch attribute.", + "check": "Trace fetch_next_work from response decode through swizzle_and_rasterize at the pinned revision.", + "outcome": "The coordinates are transformed directly in device-side CUTLASS code." + } + }, + { + "claim_id": "hw-clc-original-c025", + "claim": "Base CLC try_cancel requires sm_100 or newer, while ordinary software atomic scheduling is available on sm_70 and newer.", + "claim_type": "compatibility", + "covers": ["u034"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "9.7.13.17 Target ISA Notes", + "observation": "The base instruction requires sm_100 or higher; the multicast qualifier has narrower architecture-specific targets." + }, + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#atomic-functions", + "revision": "CUDA C++ Programming Guide 13.0.2", + "locator": "10.14.1 atomicAdd availability", + "observation": "Common 32-bit atomicAdd variants predate sm_70, so an integer-atomic software scheduler is available throughout the stated sm_70+ range." + } + ], + "challenge": { + "alternative": "CLC is also available on Hopper sm_90, or software atomics first appear only on sm_100.", + "check": "Compare the PTX target note with the CUDA atomic availability notes.", + "outcome": "CLC starts at sm_100, while suitable integer atomics are supported well before sm_70." + } + }, + { + "claim_id": "hw-clc-original-c026", + "claim": "CUTLASS has a built-in SM100 CLC scheduler implementation; an atomic tile queue is a separate software scheduler design.", + "claim_type": "compatibility", + "covers": ["u035"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp", + "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "PersistentTileSchedulerSm100", + "observation": "CUTLASS implements the CLC pipeline, response decoding, coordinate mapping, and persistent-work interface in its SM100 scheduler." + } + ], + "challenge": { + "alternative": "CUTLASS only exposes raw CLC PTX and leaves the entire scheduler to each application.", + "check": "Inspect the complete PersistentTileSchedulerSm100 class at the pinned revision.", + "outcome": "The class provides a reusable built-in scheduler rather than only an instruction wrapper." + } + } + ], + "commands": [ + "conda run -n base python scripts/validate.py", + "conda run -n base python /users/Master/.codex/skills/kernel-wiki-verifier/scripts/scan_wiki.py /users/Master/wiki-evolve/KernelWiki --output /users/Master/wiki-evolve/KernelWiki/verification/baseline-queue.jsonl", + "nvidia-smi --query-gpu=name,compute_cap,driver_version --format=csv,noheader (not available)", + "nvcc --version (not available)", + "git ls-remote https://github.com/NVIDIA/cutlass.git refs/tags/v4.5.0", + "git clone --depth 1 --branch v4.5.0 --filter=blob:none https://github.com/NVIDIA/cutlass.git /tmp/kernelwiki-cutlass-dRC5i4/cutlass", + "conda run -n base python -c ", + "conda run -n base python -c ", + "conda run -n base python -c " + ], + "scope": "All 35 baseline coverage units were split into mechanism, semantics, compatibility, code, and performance claims. CLC semantics were checked against CUDA Programming Guide 13.2 and PTX ISA 9.0; CUTLASS behavior and API names were checked at v4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5; arithmetic-only examples were run in Conda base Python 3.13.14.", + "residual_risk": "No local NVIDIA GPU or CUDA compiler is installed. No positive CLC speedup is accepted from this page; its unscoped utilization table and CLC-attributed benchmark are refuted rather than replaced with new performance numbers." +} diff --git a/verification/reviewed/original/hw-mbarrier.json b/verification/reviewed/original/hw-mbarrier.json new file mode 100644 index 000000000..2d7d5082f --- /dev/null +++ b/verification/reviewed/original/hw-mbarrier.json @@ -0,0 +1,196 @@ +{ + "path": "wiki/hardware/mbarrier.md", + "id": "hw-mbarrier", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "hw-mbarrier-original-c001", + "claim": "mbarrier is a 64-bit shared-memory synchronization object for threads and asynchronous operations, introduced on Hopper and essential to Blackwell warp-specialized kernels.", + "claim_type": "compatibility", + "covers": ["u001"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "mbarrier overview, lifecycle, and ISA/target notes", + "observation": "The object is 64-bit shared-memory state for thread/async synchronization, but mbarrier instructions began in PTX 7.0 on sm_80. Hopper added transaction-count support, not mbarrier itself; 'essential' is also not a universal ISA requirement." + } + ] + }, + { + "claim_id": "hw-mbarrier-original-c002", + "claim": "The key-operation PTX correctly states that TMA completion performs an arrival with a transferred-byte count and then flips barrier parity.", + "claim_type": "code", + "covers": ["u002"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#contents-of-the-mbarrier-object", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "arrive-on, expect-tx, complete-tx, and phase-completion definitions", + "observation": "arrive.expect_tx performs an arrive-on and increments tx-count. A TMA complete-tx decrements tx-count rather than performing another arrival; phase advances only when both pending arrivals and tx-count reach zero." + } + ] + }, + { + "claim_id": "hw-mbarrier-original-c003", + "claim": "An mbarrier advances through phases, and parity is the low bit of the phase that a parity wait must track across reuse.", + "claim_type": "semantics", + "covers": ["u003"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "phase lifecycle and parity operand semantics", + "observation": "The ISA defines sequential phases, even/odd parity values, automatic reinitialization after completion, and a lifetime tracking requirement for parity waits." + } + ], + "challenge": { + "alternative": "Parity is toggled by every arrival rather than once per completed phase.", + "check": "Compare the arrive-on definition with the atomic phase-transition requirements.", + "outcome": "Arrivals only decrement pending count; parity changes with the phase transition after all completion conditions hold." + } + }, + { + "claim_id": "hw-mbarrier-original-c004", + "claim": "The pipeline pseudocode correctly uses one global phase bit toggled after every iteration for every stage barrier.", + "claim_type": "code", + "covers": ["u004"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "parity lifetime rule and Example 2", + "observation": "Parity belongs to each mbarrier object's phase. With multiple stage objects, a stage's expected parity changes only when that same object completes, not on every loop iteration; one globally toggled bit is wrong for typical even stage counts." + } + ] + }, + { + "claim_id": "hw-mbarrier-original-c005", + "claim": "TMA directly signals an mbarrier's transaction completion, eliminating the need for a producer thread to issue a separate completion arrival.", + "claim_type": "mechanism", + "covers": ["u005"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "mbarrier::complete_tx::bytes completion mechanism", + "observation": "On copy completion, TMA performs complete-tx for the copied byte count on the supplied mbarrier. The consumer still waits; only manual producer completion signaling is avoided." + } + ], + "challenge": { + "alternative": "The producer must issue mbarrier.complete_tx after the TMA instruction returns.", + "check": "Inspect the completion_mechanism definition for cp.async.bulk.tensor.", + "outcome": "The async operation itself performs complete-tx on completion, discriminating it from software-simulated complete_tx." + } + }, + { + "claim_id": "hw-mbarrier-original-c006", + "claim": "The displayed TMA integration code is a valid or reproducible CUDA example and its no-extra-arrive warning accurately describes TMA hardware as performing an arrival.", + "claim_type": "code", + "covers": ["u006", "u009"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "exact cp.async.bulk.tensor grammar and mbarrier completion", + "observation": "The page uses undefined helper calls and omits descriptor/coordinate/address details. More importantly, TMA performs complete-tx, not arrive-on; an extra arrive changes pending-arrival accounting rather than racing with a hardware arrival." + } + ] + }, + { + "claim_id": "hw-mbarrier-original-c007", + "claim": "Reusing a barrier stage with the wrong parity can let a wait observe an earlier phase rather than the intended new one.", + "claim_type": "semantics", + "covers": ["u007"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "phaseParity accepted phases and tracking note", + "observation": "A parity wait may refer to the current or immediately preceding phase, so software must track each object's phase throughout its lifetime." + } + ], + "challenge": { + "alternative": "The instruction derives the intended generation from the address and ignores a stale parity operand.", + "check": "Inspect the explicit phaseParity operand contract.", + "outcome": "The operand chooses the phase parity being tested; there is no separate generation tag supplied by the address." + } + }, + { + "claim_id": "hw-mbarrier-original-c008", + "claim": "If an mbarrier is initialized for N pending arrivals and only M","environment":"KernelWiki worktree; conda base Python 3.13.14","reference":"sources/prs/flash-attention/PR-2441.md","result":"The script exits 0 and both asserted file names are present in the local source page.","negative_control":"Changing topk_gather_kv.py to nonexistent_topk_gather.py raises AssertionError."}], + "challenge": {"alternative":"One filename appears only in an unrelated page or the code cannot resolve the relative path.","check":"Execute from repository root and replace one token with a known-absent name.","outcome":"The real page passes and the altered assertion fails, discriminating actual source-page content."} + }, + { + "claim_id": "kernel-flash-attention-sm100-mla-topk-original-c003", + "claim": "Top-k gather and tiled scheduling are separate implementation/evidence paths in PR 2441.", + "claim_type": "code", + "covers": ["u003"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/prs/flash-attention/PR-2441/PROVENANCE.yaml","revision":"PR 2441 merge f219c89c886c6ccbf9d3dbd9fe41b11ac64e9df8","locator":"key-file manifest","observation":"The byte-verified bundle separately captures topk_gather_kv.py, tile_scheduler.py, and the integrating flash_fwd_mla_sm100.py implementation."}], + "challenge": {"alternative":"Top-k selection is merely a scheduler flag with no distinct gather implementation.","check":"Inspect the separate files and imports in the pinned bundle.","outcome":"flash_fwd_mla_sm100 imports CpasyncGatherKVManager from a dedicated gather module while independently selecting tile-scheduler classes."} + }, + { + "claim_id": "kernel-flash-attention-sm100-mla-topk-original-c004", + "claim": "Top-k reduces the attention key extent used for arithmetic while indexed K/V gathering has a distinct memory-access path, so memory traffic/locality and tensor arithmetic should be evaluated separately.", + "claim_type": "performance", + "covers": ["u004", "u005"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/prs/flash-attention/PR-2441/diff.patch","revision":"PR 2441 merge f219c89c886c6ccbf9d3dbd9fe41b11ac64e9df8; SHA-256 4525d7f9af76740692474ffc0266fe9cf72f894ce753fd9d18b70771c96b6810","locator":"benchmark effective seqlen/FLOP accounting and topk_gather_kv.py addition","observation":"Benchmark arithmetic uses gather_kv_length as effective KV length, while the gather manager loads K/V through user-provided indices and optional bounds bitmasks rather than the contiguous path. These are separable arithmetic and memory mechanisms; no numeric locality gain is asserted."}], + "challenge": {"alternative":"Reducing effective KV length necessarily improves both compute and locality by the same factor.","check":"Compare FLOP accounting with the indexed gather address construction.","outcome":"Arithmetic count follows top-k length, but addresses follow arbitrary indices and require a separate gather/bitmask path, so locality does not follow algebraically."} + }, + { + "claim_id": "kernel-flash-attention-sm100-mla-topk-original-c005", + "claim": "The useful PR-2441 path is shape- and feature-specific and therefore warrants full-workload validation before transfer.", + "claim_type": "compatibility", + "covers": ["u006"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/prs/flash-attention/PR-2441/key-files/flash_attn/cute/flash_fwd_mla_sm100.py","revision":"PR 2441 merge f219c89c886c6ccbf9d3dbd9fe41b11ac64e9df8","locator":"constructor and __call__ assertions","observation":"The top-k path requires packed GQA/MQA with qhead_per_kvhead=128, cp.async K/V gather, hdim defaults 64/512, constrained top-k length, and no page table; other options dispatch differently or are rejected."}], + "challenge": {"alternative":"The merge is a generic sparse-attention primitive transferable unchanged to arbitrary MHA shapes and paging modes.","check":"Enumerate the constructor/interface assertions and unimplemented-feature checks.","outcome":"The source encodes exact MQA/head-dimension/gather/page-table constraints, rejecting generic compatibility."} + }, + { + "claim_id": "kernel-flash-attention-sm100-mla-topk-original-c006", + "claim": "PR 2441 reports exactly 0.3 ms on B200 BF16 for batch 512, seqlen_q 1, seqlen_k 16384, 128 heads, and top-k 2048.", + "claim_type": "performance", + "covers": ["u007"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://api.github.com/repos/Dao-AILab/flash-attention/pulls/2441","revision":"GitHub PR record observed 2026-08-08; merge f219c89c886c6ccbf9d3dbd9fe41b11ac64e9df8","locator":"PR body, initial saturating decode performance block","observation":"The author reports 0.31 ms and 955.47 TFLOPS for DSA without bitmask at the stated batch/sequence/head settings and top-k length 2048, plus 0.33 ms with bitmask. The PR body does not state B200 or BF16 for this row, so the structured record changes the value and supplies unsupported environment fields."}] + } + ], + "commands": ["inspect PR 2441 GitHub API record and merge f219c89c", "inspect byte-verified PR-2441 diff and key files", "execute the displayed source-page query with conda base Python and a failing negative control"], + "residual_risk": "No NVIDIA GPU or CUDA toolkit is installed, so the sparse MLA kernel and the author-reported decode timings were not reproduced. Code conclusions are tied to merge f219c89c; the PR performance block omits enough environment details that it cannot support a structured B200/BF16 benchmark record.", + "scope": "All 7 baseline units for body hash 260bbec945acf25551df5b9232d692574dbde84aca2aa0e9b3b73972a38e7239 were checked against the GitHub PR record, byte-verified merge artifacts, and a discriminating host execution of the page query." +} diff --git a/verification/reviewed/original/kernel-flashmla.json b/verification/reviewed/original/kernel-flashmla.json new file mode 100644 index 000000000..20bb15a2a --- /dev/null +++ b/verification/reviewed/original/kernel-flashmla.json @@ -0,0 +1,247 @@ +{ + "path": "wiki/kernels/flashmla.md", + "id": "kernel-flashmla", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "kernel-flashmla-original-c001", + "claim": "MLA compresses a standard-MHA KV cache from 327–516 KB/token to 70 KB/token, a 4.66–7.28x reduction.", + "claim_type": "performance", + "covers": ["u001"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/html/2405.04434v5","revision":"DeepSeek-V2 paper v5","locator":"Section 2.1.1, Table 1 and MLA hyperparameters","observation":"The paper gives cache elements/token as 2*n_h*d_h*l for MHA and (d_c+d_h^R)*l, approximately 4.5*d_h*l, for MLA, with d_c=4*d_h and d_h^R=d_h/2. It does not report these KB/token endpoints or ratios; for the stated n_h=128, the formula ratio is about 56.9x in equal-width elements, not 4.66–7.28x."}] + }, + { + "claim_id": "kernel-flashmla-original-c002", + "claim": "FlashMLA's four variants include dense MLA prefill on SM100 alongside dense/sparse MLA decoding and sparse MLA prefill.", + "claim_type": "compatibility", + "covers": ["u002"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"DeepSeek FlashMLA commit 71c737929f2567bd0a094ae140f8f60f390b1232; SHA-256 46c001c53800f47148918f4fc2969102492dd5164f710fc015db1622f6f9929a","locator":"support matrix and Dense MHA Prefill usage section","observation":"The repository classifies dense prefill on SM100 as MHA with d_qk 192/128 and d_v 128, not MLA/MQA; dense and sparse decode plus sparse prefill are MLA-mode MQA paths."}] + }, + { + "claim_id": "kernel-flashmla-original-c003", + "claim": "Every MLA KV-cache token is a 656-byte entry containing 512 FP8 bytes, 16 bytes of per-head FP32 scales, and 128 BF16 RoPE bytes, with 64-token dense pages and variable sparse pages.", + "claim_type": "code", + "covers": ["u003", "u004"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py","revision":"DeepSeek FlashMLA commit 71c7379","locator":"flash_mla_with_kvcache docstring and dispatch","observation":"The 656-byte layout is specifically DeepSeek V3/V3.1/V3.2 FP8+sparse decode; dense decode uses BF16. Its four FP32 scales are per 128-value group within each token, not per head. page_block_size is an input shape rather than a universal dense/sparse page rule."},{"path":"/tmp/kernelwiki-flashmla-pinned/tests/quant.py","revision":"same commit","locator":"FP8KVCacheLayout.get_meta and quantize_k_cache","observation":"The repository also implements a distinct 512-dimensional MODEL1_FP8Sparse format with seven E8M0 scales, disproving a universal 656-byte MLA layout."}] + }, + { + "claim_id": "kernel-flashmla-original-c004", + "claim": "Dense MLA decode uses a fixed 64-token page and reaches up to 3000 GB/s memory-bound and 660 TFLOPS compute-bound on H800.", + "claim_type": "performance", + "covers": ["u005"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"DeepSeek FlashMLA commit 71c7379","locator":"Performance, dense MLA decoding","observation":"The 3000-GB/s and 660-TFLOPS author-reported maxima on H800 SXM5/CUDA 12.8 are exact, but the public interface accepts page_block_size from the KV tensor and does not specify a fixed value of 64; 64 is a test default."}] + }, + { + "claim_id": "kernel-flashmla-original-c005", + "claim": "The displayed dense-decode CUDA template is a faithful SM90 BF16 FlashMLA structure using an FP8 cache, explicit scale/RoPE arrays, TMA, and one warpgroup per query head.", + "claim_type": "code", + "covers": ["u006"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashmla-pinned/csrc/sm90/decode","revision":"DeepSeek FlashMLA commit 71c7379","locator":"dense and sparse_fp8 implementations plus public interface","observation":"The snippet is not upstream code and mixes mutually exclusive modes: dense decode is BF16, while FP8 cache dispatch is sparse; the real implementation uses generated/specialized kernels, scheduler metadata, separate split/combine paths, and different tensor contracts."}] + }, + { + "claim_id": "kernel-flashmla-original-c006", + "claim": "Sparse MLA consumes token indices and computes attention only over the specified tokens.", + "claim_type": "code", + "covers": ["u007"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"DeepSeek FlashMLA commit 71c7379","locator":"Sparse Attention indices contract and sparse-prefill equivalence","observation":"Decode indices encode selected page/offset entries and sparse prefill gathers focused_kv=kv[indices] before QK and PV operations."}], + "challenge": {"alternative":"The indices merely mask scores after all KV entries have already been read.","check":"Trace the decode index-to-address logic and sparse-prefill reference gather.","outcome":"The SM90/SM100 kernels derive KV addresses from supplied indices, and the reference computation gathers only selected KV rows; this is not a post-read dense mask."} + }, + { + "claim_id": "kernel-flashmla-original-c007", + "claim": "The displayed sparse CUDA template accurately represents FlashMLA's sparse API and its 656-byte cache retrieval.", + "claim_type": "code", + "covers": ["u008"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py","revision":"DeepSeek FlashMLA commit 71c7379","locator":"flash_mla_with_kvcache and flash_mla_sparse_fwd signatures","observation":"The invented template is not an upstream kernel. Decode takes page-encoded [batch,s_q,topk] indices and the 656-byte layout only in one FP8 mode; sparse prefill instead takes BF16 [s_kv,h_kv,d_qk] KV and [s_q,h_kv,topk] indices."}] + }, + { + "claim_id": "kernel-flashmla-original-c008", + "claim": "FlashMLA's SM100 dense MLA prefill uses tcgen05/TMEM for forward and backward and reaches 1460/1000 TFLOPS on B200; the displayed five-stage sketch describes that path.", + "claim_type": "code", + "covers": ["u009", "u010"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"DeepSeek FlashMLA commit 71c7379","locator":"MHA prefill benchmark, support matrix, and Dense MHA Prefill section","observation":"The 1460/1000 figures belong to NVIDIA's dense MHA forward/backward implementation, not an MLA-prefill variant. Pinned source does use TMEM/TMA in that MHA implementation, but the page's pseudocode is invented and supplies unverified role/dataflow details."}] + }, + { + "claim_id": "kernel-flashmla-original-c009", + "claim": "Dense MLA decode is author-reported at up to 660 TFLOPS and 3000 GB/s on H800, and the support matrix labels its cache BF16.", + "claim_type": "performance", + "covers": ["u011"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"DeepSeek FlashMLA commit 71c7379","locator":"Performance and support matrix","observation":"The source states both maxima for H800 SXM5 with CUDA 12.8 and lists Dense Decoding / SM90 / MQA / BF16."}], + "challenge": {"alternative":"The 3000-GB/s and 660-TFLOPS figures describe one simultaneous operating point or an FP8 path.","check":"Read the exact benchmark qualification and support row.","outcome":"The source explicitly separates memory-bound and compute-bound configurations and labels dense-decode KV format BF16, so the table must preserve that distinction."} + }, + { + "claim_id": "kernel-flashmla-original-c010", + "claim": "Sparse decode runs at 410 TFLOPS on H800 and up to 350 TFLOPS on B200 with dtype FP8.", + "claim_type": "performance", + "covers": ["u012", "u013"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"DeepSeek FlashMLA commit 71c7379","locator":"sparse MLA decode performance paragraph","observation":"The values are exact author reports, but FP8 describes the KV-cache storage; the source explicitly says matrix multiplication is bfloat16. A single Dtype=FP8 column materially obscures the compute precision."}] + }, + { + "claim_id": "kernel-flashmla-original-c011", + "claim": "Dense prefill forward and backward are author-reported at up to 1460 and 1000 TFLOPS on B200 with BF16 tensors.", + "claim_type": "performance", + "covers": ["u014", "u015"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"DeepSeek FlashMLA commit 71c7379","locator":"dense MHA prefill performance and support matrix","observation":"The repository transcribes NVIDIA's B200 maxima and its test/API use BF16 Q/K/V. The rows are MHA prefill, not MLA decode."}], + "challenge": {"alternative":"The figures are MLA prefill results or independently reproduced DeepSeek benchmarks.","check":"Compare the benchmark heading, news entry, support matrix, and test name.","outcome":"Every locator calls this dense MHA on SM100 and attributes the numbers to NVIDIA, excluding an MLA or reproduced interpretation."} + }, + { + "claim_id": "kernel-flashmla-original-c012", + "claim": "Sparse prefill reaches up to 640 TFLOPS on H800 and 1450 TFLOPS on B200 with dtype FP8.", + "claim_type": "performance", + "covers": ["u016", "u017"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"DeepSeek FlashMLA commit 71c7379","locator":"sparse MLA prefill performance, API contract, and equivalent PyTorch code","observation":"The values and H800 CUDA 12.8/B200 CUDA 12.9 environments are source-reported, but sparse-prefill Q and KV tensors are explicitly bfloat16; the source does not label this path FP8."}] + }, + { + "claim_id": "kernel-flashmla-original-c013", + "claim": "FlashMLA powers the DeepSeek-V3 and DeepSeek-V3.2-Exp model families.", + "claim_type": "compatibility", + "covers": ["u018"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"DeepSeek FlashMLA commit 71c7379","locator":"Introduction","observation":"The first-party repository explicitly says its optimized attention kernels power DeepSeek-V3 and DeepSeek-V3.2-Exp."}], + "challenge": {"alternative":"The repository is only a standalone benchmark and is not connected to model deployments.","check":"Compare the first-party introduction and sparse-kernel release note with the linked model repositories.","outcome":"Both identify the kernels as powering the named model families and DSA release, although no broader serving-stack claim follows."} + }, + { + "claim_id": "kernel-flashmla-original-c014", + "claim": "SGLang and vLLM provide day-0 FlashMLA support.", + "claim_type": "compatibility", + "covers": ["u019"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"sources/prs/vllm/PR-39752.md","revision":"vLLM merge dc8df110","locator":"title, date, and purpose","observation":"The related vLLM source is a 2026 warning about FP8 prefill query quantization, not evidence of launch-day FlashMLA support; the FlashInfer PR is likewise a separate PDL change, and no SGLang source is attached. The unversioned 'day-0' statement is therefore a misleading source conflation."}] + }, + { + "claim_id": "kernel-flashmla-original-c015", + "claim": "CUTLASS has SM100 MLA attention kernels with a separate fused-reduction implementation.", + "claim_type": "code", + "covers": ["u020"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"sources/prs/cutlass/PR-2466.md","revision":"CUTLASS merge 0e026982","locator":"changed_paths and summary","observation":"Example 77 adds the SM100 MLA-shape backward path and includes kernel/sm100_fmha_mla_reduction.hpp; PR 2472 separately captures forward MLA shape support."}], + "challenge": {"alternative":"The reduction is merely generic FMHA code and no MLA specialization exists.","check":"Inspect the pinned PR paths and the d_qk=192/d_vo=128 specialization descriptions.","outcome":"The exact file and surrounding device/kernel paths are MLA-named and tied to the asymmetric MLA shape, distinguishing them from generic FMHA."} + }, + { + "claim_id": "kernel-flashmla-original-c016", + "claim": "FlashMLA sparse prefill/decode kernels power DeepSeek-V3.2-Exp's token-level DSA/NSA path and are applicable to long-sequence sparse attention.", + "claim_type": "compatibility", + "covers": ["u021", "u024"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"DeepSeek FlashMLA commit 71c7379","locator":"Introduction and 2025-09-29 news item","observation":"The repository identifies token-level sparse prefill and decode kernels as powering DeepSeek Sparse Attention in DeepSeek-V3.2-Exp."}], + "challenge": {"alternative":"The sparse kernels are generic pruning utilities unrelated to the released model's attention execution.","check":"Follow the first-party DSA release statement and API descriptions.","outcome":"The repository explicitly connects both sparse attention stages to DSA and exposes attention operators that consume selected-token indices."} + }, + { + "claim_id": "kernel-flashmla-original-c017", + "claim": "FlashMLA is relevant to serving DeepSeek MLA models and to decode workloads where KV-cache footprint or bandwidth is limiting.", + "claim_type": "performance", + "covers": ["u022", "u023"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/html/2405.04434v5","revision":"DeepSeek-V2 paper v5","locator":"Section 2.1.1","observation":"MLA reduces per-token KV elements via low-rank latent caching."},{"url":"https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232","revision":"DeepSeek FlashMLA commit 71c7379","locator":"README Introduction and dense-decode performance","observation":"The library targets DeepSeek models and reports a memory-bound dense-decode regime, supporting the stated selection context without promising a speedup."}], + "challenge": {"alternative":"KV compression is only a model-storage property and has no bearing on the decode kernel choice.","check":"Compare the MLA cache formula with FlashMLA's separately reported memory-bound decode benchmark and model linkage.","outcome":"The cache is an input to the paged decode operator and the source explicitly measures a bandwidth-bound regime; this supports relevance, not universal superiority."} + }, + { + "claim_id": "kernel-flashmla-original-c018", + "claim": "The 656-byte latent KV-cache format is generally tied to DeepSeek's MLA architecture.", + "claim_type": "compatibility", + "covers": ["u025"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py","revision":"DeepSeek FlashMLA commit 71c7379","locator":"DeepSeek V3/V3.1/V3.2 FP8+sparse docstring and dispatch","observation":"656 bytes is tied to one V3-family FP8+sparse decode mode, not MLA generally: dense decode is BF16 and another sparse model layout is 512-dimensional with a different scale representation."}] + }, + { + "claim_id": "kernel-flashmla-original-c019", + "claim": "FlashMLA's dense prefill support is SM100-only.", + "claim_type": "compatibility", + "covers": ["u026"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"DeepSeek FlashMLA commit 71c7379","locator":"support matrix","observation":"Dense Prefill is listed only for SM100 and in MHA mode."}], + "challenge": {"alternative":"SM90 dense prefill exists elsewhere in the tree or is implied by global SM90 support.","check":"Compare the support matrix with setup.py source lists and public prefill API dispatch.","outcome":"Dense-prefill build sources and dispatch are under sm100, while SM90 support applies to decode and sparse prefill, not this dense MHA path."} + }, + { + "claim_id": "kernel-flashmla-original-c020", + "claim": "Sparse MLA needs selected-token indices from an indexing stage outside the FlashMLA attention operator.", + "claim_type": "compatibility", + "covers": ["u027"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py","revision":"DeepSeek FlashMLA commit 71c7379","locator":"decode indices and flash_mla_sparse_fwd arguments","observation":"Both sparse APIs receive caller-produced int32 indices; no top-k selector is invoked by the attention interfaces."}], + "challenge": {"alternative":"FlashMLA computes top-k internally before attention.","check":"Trace public signatures, extension calls, and index address use.","outcome":"Indices are required inputs passed into sparse_decode_fwd/sparse_prefill_fwd and directly control gathers; the library code audited here does not produce them."} + }, + { + "claim_id": "kernel-flashmla-original-c021", + "claim": "The FlashMLA GitHub link and DeepSeek-V3 technical report are relevant first-party sources.", + "claim_type": "other", + "covers": ["u028", "u029"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232","revision":"commit 71c7379","locator":"repository README","observation":"This is the first-party implementation snapshot audited."},{"url":"https://arxiv.org/abs/2412.19437","revision":"DeepSeek-V3 technical report","locator":"paper record","observation":"This is the named model's first-party technical report and provides model-level MLA context."}], + "challenge": {"alternative":"The links refer to unrelated implementations or secondary summaries.","check":"Resolve authorship, repository owner, and paper title/model identity.","outcome":"Both are DeepSeek-authored primary sources for the implementation or model context."} + }, + { + "claim_id": "kernel-flashmla-original-c022", + "claim": "The CUTLASS SM100 attention changelog is relevant to the adjacent SM100 MLA implementation.", + "claim_type": "other", + "covers": ["u030"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"sources/prs/cutlass/PR-2472.md","revision":"CUTLASS merge 9baa06dd","locator":"summary and changed_paths","observation":"The version-pinned primary PR adds Example 77's SM100 MLA forward implementation; it is stronger and more stable evidence than the mutable latest changelog URL."}], + "challenge": {"alternative":"CUTLASS material is the DeepSeek FlashMLA implementation itself.","check":"Compare repositories and provenance.","outcome":"It is a separate adjacent NVIDIA implementation of an MLA-shaped prefill kernel, relevant only when explicitly labeled as such."} + }, + { + "claim_id": "kernel-flashmla-original-c023", + "claim": "The full artifact directory is a verbatim upstream FlashMLA implementation and its derived variant accurately represents FlashMLA decode.", + "claim_type": "code", + "covers": ["u031"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/kernels/flashmla/full/PROVENANCE.yaml","revision":"local provenance","locator":"origin_url, upstream_repo, and per-file records","observation":"The two files come from separate NVIDIA CUTLASS and FlashInfer repositories, not deepseek-ai/FlashMLA; the bundle-level origin records only CUTLASS."},{"path":"artifacts/kernels/flashmla/variants/01-mla-decode-inner-loop.cu","revision":"SHA-256 4f494dc2d282468966a1e2028bc50d0ef0c364b34d24f75ee881608ddef2df69","locator":"entire derived fragment","observation":"The fragment uses a false Dc=128 assumption and conflates SM90 wgmma with SM100 tcgen05, so it is not a faithful FlashMLA decode reference."}] + }, + { + "claim_id": "kernel-flashmla-original-c024", + "claim": "The displayed get_page command returns the FlashMLA page and included code artifacts.", + "claim_type": "code", + "covers": ["u032", "u033"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python scripts/get_page.py kernel-flashmla --include-code","environment":"KernelWiki worktree; conda base Python","reference":"wiki/kernels/flashmla.md and artifacts/kernels/flashmla","result":"Command exits successfully and returns the page plus both full and derived artifact sections.","negative_control":"Changing the ID to kernel-flashmla-does-not-exist exits nonzero with 'Page not found'."}], + "challenge": {"alternative":"The command is stale, requires bare python3, or omits the referenced artifact bundle.","check":"Execute it in the required conda environment and repeat with a known-absent page ID.","outcome":"The real ID resolves and includes artifacts; the false ID fails, discriminating actual page lookup."} + }, + { + "claim_id": "kernel-flashmla-original-c025", + "claim": "Structured B200 performance records support 1460 TFLOPS BF16 dense prefill and 1450 TFLOPS FP8 sparse prefill at approximately 65% utilization for the listed shapes.", + "claim_type": "performance", + "covers": ["u034"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"DeepSeek FlashMLA commit 71c7379","locator":"B200 prefill performance paragraphs","observation":"The repository reports the two throughput maxima but no ~65% utilization and no complete shape/timing record. Dense prefill is MHA; sparse prefill's documented inputs are BF16 rather than FP8. These omissions and dtype error make both structured records inadmissible."}] + } + ], + "commands": ["inspect DeepSeek FlashMLA README, public interfaces, tests, and SM90/SM100 source at commit 71c737929f2567bd0a094ae140f8f60f390b1232", "inspect DeepSeek-V2 paper v5 MLA cache formula", "inspect pinned CUTLASS PR 2466/2472 sources and local artifact provenance", "execute the displayed get_page query and a failing negative control with conda base Python"], + "residual_risk": "No NVIDIA GPU or CUDA toolkit is installed, so author-reported throughput/bandwidth and numerical kernels were not reproduced. Source-level conclusions are pinned to DeepSeek commit 71c7379; performance statements remain author reports and are not admitted as structured records without exact shapes, timing protocol, samples, and variance.", + "scope": "All 34 baseline units for body hash 997be90a5a387c776857d0d8257d24915a19a1aa829e9d76940c4de57a311691 were checked against the pinned DeepSeek implementation, first-party MLA paper, pinned CUTLASS records, local artifacts, and a discriminating host query." +} diff --git a/verification/reviewed/original/kernel-fp8-block-scale-gemm.json b/verification/reviewed/original/kernel-fp8-block-scale-gemm.json new file mode 100644 index 000000000..beea74f41 --- /dev/null +++ b/verification/reviewed/original/kernel-fp8-block-scale-gemm.json @@ -0,0 +1,123 @@ +{ + "path": "wiki/kernels/fp8-block-scale-gemm.md", + "id": "kernel-fp8-block-scale-gemm", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "kernel-fp8-block-scale-gemm-original-c001", + "claim": "DeepSeek-V3 FP8 training uses 1x128 activation scales and 128x128 weight scales to better accommodate outliers than coarser scaling.", + "claim_type": "compatibility", + "covers": ["u001"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/html/2412.19437v2","revision":"DeepSeek-V3 Technical Report v2","locator":"Section 3.3.2 Fine-Grained Quantization and Figure 7","observation":"The paper explicitly defines 1x128 activation tiles and 128x128 weight blocks and says smaller groups better accommodate feature outliers."}], + "challenge": {"alternative":"128 is only an illustrative diagram size or applies uniformly to all tensors/phases.","check":"Read the prose definitions and the appendix's forward/backward qualifications.","outcome":"The forward activation and weight groupings are explicit; the appendix separately notes 128x1 activation handling for backward, so the page's named forward pattern is real but not universal."} + }, + { + "claim_id": "kernel-fp8-block-scale-gemm-original-c002", + "claim": "DeepGEMM is a first-party implementation of these FP8 GEMM techniques, and CUTLASS supplies separate SM100 block-scaled schedules.", + "claim_type": "compatibility", + "covers": ["u002"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-deepgemm-pinned/README.md","revision":"DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f; SHA-256 tied to local clone","locator":"Introduction, requirements, and interfaces","observation":"DeepGEMM identifies itself as a tensor-core kernel library for FP8/FP4/BF16 on SM90/SM100."},{"path":"sources/docs/cutlass-changelog-sm100.md","revision":"CUTLASS release entries through 4.5.0","locator":"3.8.0-4.1.0 blockscaled GEMM entries","observation":"CUTLASS records SM100 blockscaled formats and persistent dense blockscaled GEMM examples as a separate implementation family."}], + "challenge": {"alternative":"DeepGEMM is merely a wrapper over CUTLASS schedules.","check":"Inspect the first-party README's dependency statement and pinned kernel sources.","outcome":"The project says it borrows concepts but avoids heavy CUTLASS templates and contains its own SM90/SM100 implementations; CUTLASS is adjacent, not its backend."} + }, + { + "claim_id": "kernel-fp8-block-scale-gemm-original-c003", + "claim": "Both architectures use the displayed common structure: activation scales may be FP32 or E4M3, weight scales are one per 128x128 block, FP8 products accumulate in FP32, and scales are applied at the MMA boundary.", + "claim_type": "code", + "covers": ["u003"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-deepgemm-pinned/README.md","revision":"DeepGEMM commit 891d57b4","locator":"scaling-factor interface notice","observation":"SM90 requires FP32 scales; SM100 requires packed UE8M0, not E4M3."},{"path":"/tmp/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh","revision":"same commit","locator":"BLOCK_K assertion and promotion loop","observation":"SM90 WGMMA partials are held in float registers and only then multiplied by scale_a/scale_b into final_accum after each K block; scales are not applied at the WGMMA boundary."}] + }, + { + "claim_id": "kernel-fp8-block-scale-gemm-original-c004", + "claim": "Hopper has an approximately 22-bit accumulator and DeepGEMM promotes every 128 WGMMAs to FP32 on CUDA cores, as represented by the displayed kernel.", + "claim_type": "code", + "covers": ["u004"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/html/2412.19437v2","revision":"DeepSeek-V3 Technical Report v2","locator":"Section 3.3.3 Increased Accumulation Precision and Figure 7(b)","observation":"The paper describes a 128-element K interval equivalent to four WGMMAs, not 128 WGMMA instructions; it describes 14-bit alignment/addition precision, not an FP22 accumulator."},{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh","revision":"commit 891d57b4","locator":"math warp-group K-block loop","observation":"The real kernel fixes BLOCK_K=128, issues BLOCK_K/WGMMA::K instructions, then combines scaled partials into final_accum; the displayed function/signature is invented."}] + }, + { + "claim_id": "kernel-fp8-block-scale-gemm-original-c005", + "claim": "The displayed SM100 kernel and inline tcgen05.mma string faithfully express DeepGEMM's native UE8M0 block-scaling path, with no promotion and a final global scale.", + "claim_type": "code", + "covers": ["u005"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh","revision":"DeepGEMM commit 891d57b4","locator":"scale TMA/UTCCP path, make_instr_desc_block_scaled, and MMA loop","observation":"The real source copies packed scale blocks into TMEM and constructs a block-scaled UMMA descriptor. It does not contain the displayed function, tmem_alloc helper, global_scale epilogue, or that inline instruction string."},{"path":"/tmp/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/ptx/tcgen05.cuh","revision":"same commit","locator":"tcgen05.mma mxf8f6f4 wrappers","observation":"The actual wrapper syntax is kind::mxf8f6f4.block_scale with bracketed scale descriptors and predicate; the displayed .scale_vec::1X form and operands do not match."}] + }, + { + "claim_id": "kernel-fp8-block-scale-gemm-original-c006", + "claim": "The displayed tensor layouts and scale dtypes are an accurate cross-architecture ABI, including FP32-or-E4M3 scales and four UE8M0 scales per int32 on Blackwell.", + "claim_type": "code", + "covers": ["u006"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-deepgemm-pinned/README.md","revision":"DeepGEMM commit 891d57b4","locator":"scaling-factor interface notice","observation":"Four packed UE8M0 values per torch.int is correct for SM100, but the alternative architecture dtype is FP32 on SM90, not E4M3. The current API also supports multiple granularity recipes and layouts, so the fixed shapes are a DeepSeek-V3 pattern rather than a universal ABI."}] + }, + { + "claim_id": "kernel-fp8-block-scale-gemm-original-c007", + "claim": "The pinned DeepGEMM README reports up to 1550 TFLOPS on H800.", + "claim_type": "performance", + "covers": ["u007"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-deepgemm-pinned/README.md","revision":"DeepGEMM commit 891d57b4","locator":"2025-04-18 news item","observation":"The source states exactly that DeepGEMM achieves up to 1550 TFLOPS on H800 and links the contributing changes."}], + "challenge": {"alternative":"1550 is tied to the page's 4096-cubed shape and ~90% utilization metadata.","check":"Read the news item and follow its explicit fields without importing nearby benchmark assumptions.","outcome":"Only GPU and up-to throughput are stated in the README sentence; shape, utilization, timing method, and variance are absent and must not be inferred."} + }, + { + "claim_id": "kernel-fp8-block-scale-gemm-original-c008", + "claim": "CUTLASS SM100 block-scaled schedules achieve a similar ratio to peak as the H800 DeepGEMM result.", + "claim_type": "performance", + "covers": ["u008"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"sources/docs/cutlass-changelog-sm100.md","revision":"captured changelog through CUTLASS 4.5.0","locator":"blockscaled GEMM release entries","observation":"The source establishes availability but contains no matched shape, GPU peak convention, or ratio comparable to DeepGEMM's H800 maximum."}] + }, + { + "claim_id": "kernel-fp8-block-scale-gemm-original-c009", + "claim": "This exact DeepGEMM-style block-scaled pattern should be used for FP8 inference in DeepSeek-V3, Qwen2-FP8, and similar models.", + "claim_type": "compatibility", + "covers": ["u009"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/html/2412.19437v2","revision":"DeepSeek-V3 Technical Report v2","locator":"Sections 3.3.1-3.3.2","observation":"The source establishes the exact format for DeepSeek-V3 training. It does not establish Qwen2-FP8 layout, scale dtype, kernel ABI, or compatibility; model-name transfer without a format contract is unsafe."}] + }, + { + "claim_id": "kernel-fp8-block-scale-gemm-original-c010", + "claim": "DeepSeek-V3 uses FP8 activations in its training framework.", + "claim_type": "compatibility", + "covers": ["u010"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/html/2412.19437v2","revision":"DeepSeek-V3 Technical Report v2","locator":"Section 3.3 FP8 Training","observation":"The paper documents FP8 activations, online tile-wise quantization, and phase-specific exceptions in the training framework."}], + "challenge": {"alternative":"FP8 applies only to stored weights or inference, not training activations.","check":"Inspect the mixed-precision framework and activation-quantization subsections.","outcome":"The source explicitly discusses forward and backward activation tiles and cached FP8 activations during training."} + }, + { + "claim_id": "kernel-fp8-block-scale-gemm-original-c011", + "claim": "Fine-grained scaling is a relevant option when per-tensor FP8 loses accuracy to outliers.", + "claim_type": "compatibility", + "covers": ["u011"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/html/2412.19437v2","revision":"DeepSeek-V3 Technical Report v2","locator":"Fine-Grained Quantization subsection","observation":"The authors motivate smaller scale groups specifically by activation/weight/gradient outliers and report that they better accommodate them."}], + "challenge": {"alternative":"Fine-grained scaling universally improves accuracy without cost and should always replace per-tensor scaling.","check":"Inspect the paper's overhead discussion and special-case policies.","outcome":"The paper documents scale-calculation/layout overhead and higher-precision exceptions, supporting conditional relevance rather than a universal rule."} + }, + { + "claim_id": "kernel-fp8-block-scale-gemm-original-c012", + "claim": "A structured benchmark records 1550 TFLOPS for M=N=K=4096 on H800 at approximately 90% utilization via CUDA-core promotion.", + "claim_type": "performance", + "covers": ["u012"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-deepgemm-pinned/README.md","revision":"DeepGEMM commit 891d57b4","locator":"2025-04-18 news item","observation":"The source reports only up to 1550 TFLOPS on H800. It does not attach M=N=K=4096, ~90%, timing details, samples, variance, or a single-kernel mechanism attribution to that maximum."}] + } + ], + "commands": ["inspect DeepGEMM commit 891d57b4 README and exact SM90/SM100 1D1D kernels", "inspect DeepSeek-V3 Technical Report v2 fine-grained quantization and accumulation sections", "inspect pinned CUTLASS SM100 changelog capture and DeepGEMM tcgen05 wrappers"], + "residual_risk": "No NVIDIA GPU or CUDA toolkit is installed, so kernel numerics and the 1550-TFLOPS author report were not reproduced. CUTLASS availability is source-level; no matched cross-library performance comparison was retained.", + "scope": "All 12 baseline units for body hash 9fc01ec2833f70fa337f1ec2f149ac72e477d8465563010c15b1aa7c0c2bcc50 were checked against the DeepSeek paper, pinned DeepGEMM source, and CUTLASS release record." +} diff --git a/verification/reviewed/original/kernel-fused-moe.json b/verification/reviewed/original/kernel-fused-moe.json new file mode 100644 index 000000000..c0ffbf1b5 --- /dev/null +++ b/verification/reviewed/original/kernel-fused-moe.json @@ -0,0 +1,295 @@ +{ + "path": "wiki/kernels/fused-moe.md", + "id": "kernel-fused-moe", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "kernel-fused-moe-original-c001", + "claim": "The FlashInfer Track A kernel covers the full MoE forward path in one to three launches and saves up to 21.9% of activation traffic relative to a five-to-seven-launch implementation.", + "claim_type": "performance", + "covers": ["u001"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","revision":"definition fetched 2026-08-08","locator":"description and signature","observation":"The official definition says only that routing and two grouped GEMMs are included. It gives no launch count or 21.9% traffic result, and its explicit inputs/outputs do not define a separate dispatch/combine benchmark contract."}] + }, + { + "claim_id": "kernel-fused-moe-original-c002", + "claim": "Fused MoE is Track A of the FlashInfer MLSys 2026 contest, which targets NVIDIA B200 and uses a DeepSeek-V3/R1-style FP8 MoE definition.", + "claim_type": "compatibility", + "covers": ["u002"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://mlsys26.flashinfer.ai/","revision":"site fetched 2026-08-08","locator":"Contest Overview and Track A","observation":"The organizer identifies B200 as the target and names Track A Fused MoE with FP8 support, linking the exact benchmark."},{"url":"https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","revision":"definition fetched 2026-08-08","locator":"tags and axes","observation":"The definition is tagged deepseek-v3/deepseek-r1 and records the fixed dimensions."}], + "challenge": {"alternative":"Track A is a generic MoE contest or targets Hopper as well as B200.","check":"Read the organizer's challenge target and follow its Track A link to the definition tags.","outcome":"The organizer says B200 and the linked definition names DeepSeek-V3/R1; no Hopper target is stated for this contest track."} + }, + { + "claim_id": "kernel-fused-moe-original-c003", + "claim": "The displayed pipeline routes each token to top-8 experts from a total of 32 and otherwise represents the benchmark's routing, W13/SwiGLU, W2, and weighted-combine semantics.", + "claim_type": "code", + "covers": ["u003"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","revision":"definition fetched 2026-08-08","locator":"Axes and Reference Implementation","observation":"Routing logits span 256 global experts; 32 is num_local_experts for EP=8. The reference selects top-8 globally after grouped routing, computes only the local expert interval, splits the 4096-row W13 result for SwiGLU, performs W2, and accumulates routing-weighted outputs."}] + }, + { + "claim_id": "kernel-fused-moe-original-c004", + "claim": "vLLM uses exactly seven launches, SGLang exactly five, and a fully fused implementation exactly one or two launches with the displayed operation boundaries.", + "claim_type": "code", + "covers": ["u004"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashinfer-bench-starter-kit/EVALUATION.md","revision":"flashinfer-bench-starter-kit 75ccd05cafceb0fd1f86be4cd0f2117249463c66","locator":"FlashInfer Baselines and MoE evaluation command","observation":"The official evaluation defines one FlashInfer wrapper baseline and measures submitted solution entry points; it records no framework launch-count decomposition."},{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/fused_moe/core.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"trtllm_fp8_block_scale_moe","observation":"One Python call enters a staged runner with routing and two separately tuned GEMM tactics; a call count is not a GPU launch count."}] + }, + { + "claim_id": "kernel-fused-moe-original-c005", + "claim": "The gated first projection is logically W13: a single 2I-wide GEMM result is split into up and gate halves, followed by a fused SwiGLU product.", + "claim_type": "code", + "covers": ["u005"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"_fp8_moe_run_experts","observation":"The official trace reference computes G1=A@W13.T, splits G1 into X1/X2, and computes silu(X2)*(X1+beta) before W2."}], + "challenge": {"alternative":"Gate and up must be two independently launched GEMMs with separate accumulator allocations.","check":"Inspect the reference's W13 tensor shape and GEMM1 expression.","outcome":"W13 is one [2I,H] tensor and the reference issues one logical matmul producing both halves; separate launches/allocations are not part of the contract."} + }, + { + "claim_id": "kernel-fused-moe-original-c006", + "claim": "The displayed CUDA function is a usable SM100 FP8 block-scale gated-dual-GEMM kernel with valid tcgen05 instructions, TMEM allocation, synchronization, indexing, and scale handling.", + "claim_type": "code", + "covers": ["u006"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/fused_moe/core.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"trtllm_fp8_block_scale_moe wrapper and runner call","observation":"The first-party implementation uses compiled TensorRT-LLM/FlashInfer runners and explicit tensor contracts, not the displayed helpers or inline assembly."},{"path":"wiki/hardware/tcgen05-mma.md","revision":"KernelWiki page already verified against PTX ISA 9.2","locator":"Instruction Forms and Ownership","observation":"Real tcgen05 forms use architecture-defined descriptors, predicates, and allocation ownership. The displayed instruction string omits required form/operands and the function also references undefined storage, rows, columns, and helpers."}] + }, + { + "claim_id": "kernel-fused-moe-original-c007", + "claim": "The displayed keyword call is the FlashInfer benchmark API: BF16 hidden states, 32-wide routing logits, fused w_gate_up/w_down keywords, and FP8 block-scale tensors in the shown shapes.", + "claim_type": "code", + "covers": ["u007"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","revision":"definition fetched 2026-08-08","locator":"Signature and baseline wrapper","observation":"The benchmark requires float8 hidden_states [T,7168], routing_logits [T,256], separate gemm1/gemm2 weight and FP32 scale tensors, plus local_expert_offset and routed_scaling_factor. Its wrapper passes the complete positional contract to trtllm_fp8_block_scale_moe; the displayed keyword names and shapes do not match."}] + }, + { + "claim_id": "kernel-fused-moe-original-c008", + "claim": "The displayed Triton kernel correctly implements a routed, block-scaled fused gate-up projection.", + "claim_type": "code", + "covers": ["u008"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"DeepSeek routing reference and _fp8_moe_run_experts","observation":"A correct path uses global expert IDs, a local expert offset, routing weights, all three FP32 scale tensors, bounds-safe gathering, W13 splitting, W2, and weighted accumulation. The displayed kernel leaves expert_ids and every scale pointer unused, lacks K/M/N masks and local offsets, and never performs W2 or combine."}] + }, + { + "claim_id": "kernel-fused-moe-original-c009", + "claim": "On B200 at the stated batch sizes, SGLang records 1262 TFLOPS/206.9 us with five launches, FlashInfer CuTe DSL 1225 TFLOPS/481.9 us with one-to-two launches, and vLLM 1117 TFLOPS/369.5 us with seven launches.", + "claim_type": "performance", + "covers": ["u009", "u010", "u011"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","revision":"definition and traces fetched 2026-08-08","locator":"definition, solutions, and per-workload traces","observation":"The official page contains no such framework table or launch counts, and searches of the complete serialized page find none of 1262, 206.9, 1225, 481.9, or 369.5. The isolated official traces report latency_ms per seq_len and solution, not the asserted mixed TFLOPS/batch table."}] + }, + { + "claim_id": "kernel-fused-moe-original-c010", + "claim": "There is no tuned FP8 MoE configuration for B200, so tile sizes and stages must be selected manually from scratch.", + "claim_type": "compatibility", + "covers": ["u012"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/fused_moe/core.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"MoERunner tactic enumeration and autotuner selection","observation":"The official implementation enumerates architecture/shape-valid GEMM1 and GEMM2 tactics and supplies token-bucketed runtime autotuning."},{"path":"/tmp/kernelwiki-flashinfer-pinned/benchmarks/bench_trtllm_gen_fused_moe_autotuner.py","revision":"same commit","locator":"bench_trtllm_gen_fused_moe_autotuner_fp8","observation":"A first-party B200-capable benchmark explicitly compares untuned and autotuned FP8-block tactics and passes tune_max_num_tokens."}] + }, + { + "claim_id": "kernel-fused-moe-original-c011", + "claim": "Block size 128 is required specifically to prevent FP8 numerical overflow and guarantee stability.", + "claim_type": "compatibility", + "covers": ["u013"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"trtllm_fp8_block_scale_moe_ds_routing_trace axes and inputs","observation":"The source defines 128 as this DeepSeek-FP8 trace's scale granularity and receives already-quantized FP8 tensors plus FP32 scales; it does not compute scales or state a stability requirement."},{"path":"sources/docs/deepseek-v3-fp8.md","revision":"DeepSeek-V3 Technical Report v2 capture","locator":"Fine-Grained Quantization","observation":"The paper motivates smaller groups as better accommodating local outliers and documents phase-specific exceptions; it does not state that 128 is a universal overflow-stability threshold."}] + }, + { + "claim_id": "kernel-fused-moe-original-c012", + "claim": "The benchmark's batch=1 workload is launch-overhead dominated and its batch=4096 workload is throughput dominated.", + "claim_type": "performance", + "covers": ["u014"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","revision":"definition fetched 2026-08-08","locator":"Axes","observation":"The varied axis is seq_len (number of tokens), not batch, and the definition supplies no profile attributing latency to launch overhead or compute throughput."}] + }, + { + "claim_id": "kernel-fused-moe-original-c013", + "claim": "Dynamic routing gives experts different M dimensions, which can create scheduling tail effects unless grouped work is balanced.", + "claim_type": "performance", + "covers": ["u015"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-deepgemm-pinned/README.md","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"Grouped GEMMs (contiguous layout)","observation":"The first-party documentation states that each expert may process a varying number of tokens and represents those expert problems with varying M segments."}], + "challenge": {"alternative":"A grouped or persistent scheduler makes expert imbalance disappear as a scheduling concern.","check":"Compare the logical per-group M sizes with the scheduler's finite assignment of tiles.","outcome":"Grouping can mitigate the tail, but it does not make unequal tile counts equal; the claim is conditional and does not say every implementation suffers the same magnitude."} + }, + { + "claim_id": "kernel-fused-moe-original-c014", + "claim": "Every TMA descriptor and expert tensor base must be aligned to 128 bytes.", + "claim_type": "compatibility", + "covers": ["u016"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html","revision":"CUDA Driver API 13.2.1","locator":"cuTensorMapEncodeTiled parameter requirements","observation":"The tensor-map object address is 64-byte aligned, the ordinary globalAddress requirement is 16 bytes, and selected interleave/data-type cases require 32 bytes. A 128-byte global alignment applies to particular swizzle considerations, not every descriptor or tensor base."}] + }, + { + "claim_id": "kernel-fused-moe-original-c015", + "claim": "Gate and up necessarily require two separate TMEM allocations that compete for the entire 256-KB SM budget.", + "claim_type": "code", + "covers": ["u017"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"_fp8_moe_run_experts","observation":"The benchmark contract represents gate and up as one concatenated W13 tensor and one logical 2I-wide GEMM result, then splits it for SwiGLU. It does not require two independent accumulator allocations."}] + }, + { + "claim_id": "kernel-fused-moe-original-c016", + "claim": "The exact kernel contract applies to DeepSeek-V3, Mixtral, and other MoE inference models without further compatibility checks.", + "claim_type": "compatibility", + "covers": ["u018"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","revision":"definition fetched 2026-08-08","locator":"tags, axes, and signature","observation":"The definition is specifically tagged DeepSeek-V3/R1 and fixes routing method, 256/32 expert geometry, FP8 type, block-scale layouts, and dimensions. It contains no Mixtral compatibility statement."}] + }, + { + "claim_id": "kernel-fused-moe-original-c017", + "claim": "The benchmark directly establishes both prefill and decode behavior by treating high batch as prefill and low batch as decode.", + "claim_type": "compatibility", + "covers": ["u019"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","revision":"definition fetched 2026-08-08","locator":"Axes","observation":"The trace varies seq_len/number of tokens and does not label workloads as serving prefill, decode, or request batch size."}] + }, + { + "claim_id": "kernel-fused-moe-original-c018", + "claim": "A W13/SwiGLU fusion is appropriate when a matched measurement shows it improves the intended workload over separate projections.", + "claim_type": "performance", + "covers": ["u020"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"_fp8_moe_run_experts","observation":"The reference contract permits one concatenated W13 matmul and immediate SwiGLU, so the candidate fusion preserves the logical operation; the page conditions use on an actual measured benefit rather than asserting one."}], + "challenge": {"alternative":"Combining gate and up always wins regardless of layout or workload.","check":"Read the conditional wording and compare it with the contract rather than an unqualified speed claim.","outcome":"The claim applies only after a matched measurement, so it does not assert universal benefit."} + }, + { + "claim_id": "kernel-fused-moe-original-c019", + "claim": "A routing-through-combine implementation must coordinate grouped routing, local expert selection, W13, activation, W2, and weighted accumulation.", + "claim_type": "code", + "covers": ["u021"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"DeepSeek routing reference and _fp8_moe_run_experts","observation":"The official reference spells out each of these dependent stages and their global/local expert mapping."}], + "challenge": {"alternative":"The single public API call means the implementation contract is only one simple matrix operation.","check":"Inspect the reference implementation behind the wrapper.","outcome":"The call is a wrapper; the reference separately implements routing, two GEMMs, activation, and weighted accumulation."} + }, + { + "claim_id": "kernel-fused-moe-original-c020", + "claim": "Expert load imbalance is the single primary bottleneck for all practical workloads of this definition.", + "claim_type": "performance", + "covers": ["u022"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/fused_moe/core.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"MoERunner tuning and separate GEMM1/GEMM2 tactic selection","observation":"The implementation selects shape- and architecture-valid tactics separately for the two GEMMs across token buckets, rather than encoding one universal primary bottleneck."},{"path":"/tmp/kernelwiki-flashinfer-bench-starter-kit/EVALUATION.md","revision":"commit 75ccd05cafceb0fd1f86be4cd0f2117249463c66","locator":"MoE evaluation and scoring","observation":"The contest measures whole-operation latency across workloads and publishes no profile that ranks load imbalance as the primary cause."}] + }, + { + "claim_id": "kernel-fused-moe-original-c021", + "claim": "CUDA graph compatibility universally requires a fixed masked expert layout.", + "claim_type": "compatibility", + "covers": ["u023"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-deepgemm-pinned/README.md","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"Grouped GEMMs (masked layout)","observation":"DeepGEMM presents masked layout as one supported decoding case when CUDA graphs are enabled and the CPU does not know expert token counts, not as a universal CUDA-graph requirement."},{"path":"artifacts/kernels/fused-moe/full/flashinfer_cutedsl.py","revision":"SGLang c554dc5c64b661f2c53225b03a76359eaddc39e4; SHA-256 de6bd663...","locator":"ensure_cutedsl_wrapper and standard dispatch function","observation":"This exact upstream FP4-adjacent runner preallocates CUDA-graph buffers while accepting StandardDispatchOutput; it does not impose DeepGEMM's masked M-grouped ABI."}] + }, + { + "claim_id": "kernel-fused-moe-original-c022", + "claim": "Small per-expert M dimensions can leave too few or partially filled GEMM tiles and reduce GPU utilization.", + "claim_type": "performance", + "covers": ["u024"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/NVIDIA/cutlass/blob/main/media/docs/cpp/efficient_gemm.md","revision":"CUTLASS documentation fetched 2026-08-08","locator":"Threadblock-level GEMM","observation":"NVIDIA states that small M or N can leave threads outside the problem and that small M/N with large K may launch too few threadblocks to use all SMs."}], + "challenge": {"alternative":"Grouped scheduling always combines enough small expert problems to eliminate thin-GEMM inefficiency.","check":"Read the stated small-dimension tile/wave conditions and distinguish mitigation from elimination.","outcome":"Grouping can increase aggregate parallel work but individual problem tiles may remain partial; the claim says can cause, not must dominate."} + }, + { + "claim_id": "kernel-fused-moe-original-c023", + "claim": "The benchmark's FP8 block-scale format stores additional scale-factor tensors beyond its quantized activations and weights.", + "claim_type": "compatibility", + "covers": ["u025"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","revision":"definition fetched 2026-08-08","locator":"Signature","observation":"The inputs explicitly include three non-empty FP32 scale tensors with shapes [56,T], [32,32,56], and [32,56,16] in addition to FP8 data tensors."}], + "challenge": {"alternative":"Scales are implicit in FP8 payloads and require no separate storage.","check":"Inspect the signature's independent tensor names, dtypes, and shapes.","outcome":"All three scale arrays are explicit inputs; the claim is limited to storage existence and does not quantify a performance penalty."} + }, + { + "claim_id": "kernel-fused-moe-original-c024", + "claim": "The FlashInfer MLSys 2026 contest link is the organizer page for Track A.", + "claim_type": "other", + "covers": ["u026"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://mlsys26.flashinfer.ai/","revision":"site fetched 2026-08-08","locator":"Track A Fused MoE","observation":"The live organizer page identifies Track A and links the exact FP8 MoE definition."}], + "challenge": {"alternative":"The URL is stale or belongs to a different contest track.","check":"Open the page and follow the Track A link.","outcome":"The page is live, identifies MLSys 2026 NVIDIA Track, and names Fused MoE as Track A."} + }, + { + "claim_id": "kernel-fused-moe-original-c025", + "claim": "GPU Mode NVFP4 Hackathon Problem 3 is a source for this FP8 routed MoE benchmark.", + "claim_type": "compatibility", + "covers": ["u027"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml","revision":"gpu-mode/reference-kernels 51e22db671d36c1c76091c43c36a44546ba324a1","locator":"task inputs and output","observation":"Problem 3 is an NVFP4 dual GEMM over A, B1, and B2; it contains no expert routing, local-expert offset, W2 down projection, or FP8 DeepSeek block-scale MoE contract."}] + }, + { + "claim_id": "kernel-fused-moe-original-c026", + "claim": "DeepGEMM's MoE implementation is direct evidence for this exact FlashInfer FP8 Track A contract.", + "claim_type": "compatibility", + "covers": ["u028"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-deepgemm-pinned/README.md","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"Grouped GEMMs and Mega MoE sections","observation":"The repository supplies generic grouped GEMMs and a distributed FP8xFP4 Mega MoE path with symmetric memory; it does not define the contest's routing-logit ABI, fixed FP8 tensors, or FlashInfer wrapper."}] + }, + { + "claim_id": "kernel-fused-moe-original-c027", + "claim": "The linked SGLang code is direct evidence for the exact FP8 Track A benchmark implementation and its asserted launch/performance table.", + "claim_type": "compatibility", + "covers": ["u029"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/kernels/fused-moe/full/flashinfer_cutedsl.py","revision":"SGLang c554dc5c64b661f2c53225b03a76359eaddc39e4; SHA-256 de6bd663...","locator":"module constants and registered fused function","observation":"The byte-matched upstream file is a standard-path FP4 CuteDSL runner: it quantizes activations with fp4_quantize and invokes CuteDslMoEWrapper. It is not the FP8 DeepSeek benchmark and contains no asserted table values or launch counts."}] + }, + { + "claim_id": "kernel-fused-moe-original-c028", + "claim": "The local full bundle is a verbatim upstream implementation of the page's FP8 fused MoE, and its derived variant faithfully represents production fusion.", + "claim_type": "code", + "covers": ["u030"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/kernels/fused-moe/full/PROVENANCE.yaml","revision":"SHA-256 3832ddeeeafe8055e44c890283926365817763af2c724c54cce0872c53699f17","locator":"bundle notes and file entries","observation":"The bundle mixes an MXFP4 vLLM PR diff, an FP4 SGLang runner, and locally extracted pseudocode, so it is not one upstream FP8 implementation."},{"path":"artifacts/kernels/fused-moe/variants/01-routing-plus-fusion-skeleton.py","revision":"SHA-256 2cc640b1baa1a6cd42718616ffe2cf78a9b6d4eb3317464f059c6fd442439342","locator":"fused_moe_forward","observation":"The derived function incorrectly boolean-multiplies masks for routing-weight indexing and asserts that three unrelated projects fuse all stages; it is neither a correct reference nor production code."}] + }, + { + "claim_id": "kernel-fused-moe-original-c029", + "claim": "scripts/get_page.py can retrieve kernel-fused-moe with its artifact code, while an unknown page ID fails.", + "claim_type": "code", + "covers": ["u031", "u032"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python scripts/get_page.py kernel-fused-moe --include-code","environment":"KernelWiki worktree; conda base","reference":"scripts/get_page.py","result":"Exited 0 and emitted the fused-moe page plus artifact content.","negative_control":"kernel-fused-moe-broken exited 1 with 'No page found'."}], + "challenge": {"alternative":"The command only prints Markdown and silently ignores --include-code or unknown IDs.","check":"Run the documented command and an unknown-ID negative control.","outcome":"The real ID emitted artifact-inclusive output and the control failed distinctly."} + }, + { + "claim_id": "kernel-fused-moe-original-c030", + "claim": "A reproducible structured B200 record establishes 1262 TFLOPS at topk=8, experts=32, H=7168, I=2048, batch=4096 and about 56% utilization.", + "claim_type": "performance", + "covers": ["u033"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","revision":"definition and traces fetched 2026-08-08","locator":"Axes, traces, and serialized solutions","observation":"The definition uses seq_len and distinguishes 256 global from 32 local experts. It has no 1262-TFLOPS or 56%-utilization record, and the repository page supplies no timing method, repetitions, variance, or exact source cell for the structured value."}] + } + ], + "commands": [ + "inspect FlashInfer-Bench definition and serialized official reference/baseline wrapper fetched 2026-08-08", + "inspect FlashInfer 7f614b8 trace template, API, runner validation, and autotuner", + "inspect flashinfer-bench-starter-kit 75ccd05 evaluation contract", + "inspect vLLM PR 23696 merge 074854b2, SGLang c554dc5c, DeepGEMM 891d57b4, GPU Mode reference-kernels 51e22db, and CUDA 13.2.1 TMA requirements", + "run get_page positive and unknown-ID negative controls under conda base" + ], + "residual_risk": "No B200, CUDA toolkit, or contest dataset is installed locally, so no candidate kernel, launch trace, numeric accuracy, or performance result was reproduced. Performance entries without a complete official record were removed rather than inferred from serialized historical traces.", + "scope": "All 33 baseline units for body hash c3f18f75c945dcd3475248cde9377aacfb323444fc25b68d9eef038acb26e09d were checked against the official contest/benchmark contract and pinned first-party implementations." +} diff --git a/verification/reviewed/original/kernel-gated-delta-net.json b/verification/reviewed/original/kernel-gated-delta-net.json new file mode 100644 index 000000000..cac96704e --- /dev/null +++ b/verification/reviewed/original/kernel-gated-delta-net.json @@ -0,0 +1,260 @@ +{ + "path": "wiki/kernels/gated-delta-net.md", + "id": "kernel-gated-delta-net", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "kernel-gated-delta-net-original-c001", + "claim": "Gated DeltaNet is an ICLR 2025 linear-attention architecture that combines adaptive exponential decay with a delta-rule memory update and has linear sequence-length complexity.", + "claim_type": "mechanism", + "covers": ["u001"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/abs/2412.06464","revision":"arXiv 2412.06464, fetched 2026-08-08","locator":"abstract and ICLR publication record","observation":"The paper introduces Gated DeltaNet by combining gating for adaptive memory control with the delta update rule; the official repository identifies the ICLR 2025 publication."}], + "challenge": {"alternative":"Gated DeltaNet is merely a renamed softmax-attention kernel or was not an ICLR 2025 result.","check":"Compare the paper abstract and official repository publication entry.","outcome":"Both identify a recurrent linear-transformer rule combining decay and delta updates, and the official record says ICLR 2025."} + }, + { + "claim_id": "kernel-gated-delta-net-original-c002", + "claim": "Gated DeltaNet is used in Qwen3-Next and Qwen3.5, whose cited official configurations use a 3:1 linear/full-attention pattern and a native 262,144-token context.", + "claim_type": "compatibility", + "covers": ["u001"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/config.json","revision":"Qwen model revision 9c7f2fbe84465e40164a94cc16cd30b6999b0cc7","locator":"full_attention_interval, max_position_embeddings, and linear-attention fields","observation":"The config fixes every fourth layer as full attention, GDN head fields, and max_position_embeddings 262144."},{"url":"https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/8472618112abcbd45acbcdc58436aff4233c23f7/config.json","revision":"Qwen3.5 model revision 8472618112abcbd45acbcdc58436aff4233c23f7","locator":"text_config.layer_types and max_position_embeddings","observation":"The config repeats three linear_attention entries followed by full_attention and fixes 262144 positions."}], + "challenge": {"alternative":"The 3:1 pattern and 262K context are editorial extrapolations from Qwen3-Next to Qwen3.5.","check":"Inspect both immutable model configurations independently.","outcome":"Each configuration directly records the hybrid layout and 262144 limit."} + }, + { + "claim_id": "kernel-gated-delta-net-original-c003", + "claim": "A fixed recurrent matrix gives constant-with-context decode cost and eliminates KV-cache growth entirely.", + "claim_type": "mechanism", + "covers": ["u002"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct","revision":"model card fetched 2026-08-08; config revision 9c7f2fbe84465e40164a94cc16cd30b6999b0cc7","locator":"Model Overview hybrid layout and config full_attention_interval","observation":"Qwen3-Next contains one full-attention layer for every three GDN layers. The GDN state is fixed with context length, but the full-attention layers still use cache; therefore the architecture does not eliminate KV-cache growth entirely."}] + }, + { + "claim_id": "kernel-gated-delta-net-original-c004", + "claim": "The Qwen3-Next-80B diagram is exact, including 48 layers, 36 GDN plus 12 full-attention layers, 512 experts, about 19 active experts per token, and 80B/3B parameter counts.", + "claim_type": "code", + "covers": ["u003"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct","revision":"model card fetched 2026-08-08; config revision 9c7f2fbe84465e40164a94cc16cd30b6999b0cc7","locator":"Model Overview and config.json","observation":"The 48-layer 3:1 layout and 80B/3B counts are correct, but the official card states 10 routed experts plus one shared expert, not about 19; it calls the fourth block Gated Attention, not plain GQA for global retrieval."}] + }, + { + "claim_id": "kernel-gated-delta-net-original-c005", + "claim": "The delta rule makes a targeted correction based on the difference between a value and the state's current retrieval for the key.", + "claim_type": "mechanism", + "covers": ["u004"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/gdn.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"_gdn_decode_reference","observation":"The reference decays the state, retrieves old_v with k, interpolates toward v with beta, removes the old association, and writes the new one."}], + "challenge": {"alternative":"The implementation only adds an outer product and never subtracts or corrects an existing association.","check":"Inspect old_v, state_remove, state_update, and their signs in the reference.","outcome":"The source explicitly subtracts k outer old_v before adding k outer new_v, rejecting the additive-only alternative."} + }, + { + "claim_id": "kernel-gated-delta-net-original-c006", + "claim": "The displayed delta_rule_step has the correct state orientation, gate meanings, decay order, and Gated DeltaNet recurrence.", + "claim_type": "code", + "covers": ["u005"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/gdn.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"_gdn_decode_reference","observation":"With state [K,V], the reference first forms old_state=g*S, retrieves k@old_state, and applies beta to v-old_v. The displayed function declares [d_k,d_v] but computes S@k, constructs a [d_v,d_k] outer product, and swaps beta/alpha roles."}] + }, + { + "claim_id": "kernel-gated-delta-net-original-c007", + "claim": "Chunk prefill resolves each chunk by a causal scalar recurrence while chunks are parallel except for sequential state propagation.", + "claim_type": "mechanism", + "covers": ["u006"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-gdn-pinned/lit_gpt/gated_delta_rule_ops/chunk.py","revision":"NVlabs/GatedDeltaNet b53d6d3a161267432a79c1c04af69fa52bddc921","locator":"chunk_gated_delta_rule_fwd_kernel_h and chunk_linear_attn_fwd_kernel_o","observation":"The chunkwise algorithm uses WY-derived and matrix-product kernels; the state kernel loops over chunks for a given state tile, while other chunk-local products are launched across chunk IDs. The prose's scalar-recurrence/parallel-chunk description does not match this decomposition."}] + }, + { + "claim_id": "kernel-gated-delta-net-original-c008", + "claim": "The displayed Triton prefill kernel correctly implements the Qwen/FlashInfer GDN contract.", + "claim_type": "code", + "covers": ["u007"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/gdn.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"gdn_prefill_trace signature and _gdn_prefill_reference","observation":"The contract uses separate q/k head counts 4, value heads 8, per-head dimension 128, state [N,8,128,128], g and beta gates, cu_seqlens, and output from the updated state. The sketch flattens heads to 512/1024, omits g and variable-length inputs, references undefined NUM_HEADS, and has inconsistent outer-product orientation."}] + }, + { + "claim_id": "kernel-gated-delta-net-original-c009", + "claim": "Single-token GDN decode performs a fixed-size state update whose work and storage do not grow with prior context length.", + "claim_type": "mechanism", + "covers": ["u008"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/gdn.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"gated_delta_rule_decode_trace and _gdn_decode_reference","observation":"seq_len is fixed to one; the input/output state shape depends on batch, heads, and head_size but not the number of prior tokens."}], + "challenge": {"alternative":"Decode rereads all prior keys and values, so work or storage grows with context length.","check":"Inspect the trace signature and reference loop bounds.","outcome":"The reference receives only one q/k/v token and a fixed recurrent state; there is no prior-sequence axis."} + }, + { + "claim_id": "kernel-gated-delta-net-original-c010", + "claim": "The displayed Triton decode kernel has the correct state size, gates, layout, ordering, and output for the benchmark.", + "claim_type": "code", + "covers": ["u009"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://bench.flashinfer.ai/kernels/gdn_decode_qk4_v8_d128_k_last","revision":"definition fetched 2026-08-08","locator":"Axes, Signature, and Outputs","observation":"The definition has state [B,8,128,128] with A_log, a, dt_bias, and b inputs. The sketch instead claims a 512x1024 per-head state, accepts one Beta pointer, omits the two-gate computation, and queries the state before the update whereas the reference outputs from the updated state."}] + }, + { + "claim_id": "kernel-gated-delta-net-original-c011", + "claim": "TFLA directly supplies Gated DeltaNet kernels and emits inline WGMMA on Hopper and tcgen05 on Blackwell.", + "claim_type": "compatibility", + "covers": ["u010"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-mlstm-kernels/README.md","revision":"NX-AI/mlstm_kernels 5b98ff8e2bec189b3d3c249405bab5149564d6f8","locator":"library overview and kernel lists","observation":"The official TFLA code is a PyTorch/JAX/Triton mLSTM library. Its README describes TFLA as an mLSTM method with a second level of sequence parallelism and arbitrarily large chunks; the pinned tree contains no GatedDeltaNet, wgmma, tcgen05, inline-PTX, or asm implementation."}] + }, + { + "claim_id": "kernel-gated-delta-net-original-c012", + "claim": "The displayed tcgen05 inline-assembly fragment is a valid TFLA Blackwell matrix multiply.", + "claim_type": "code", + "covers": ["u011"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"wiki/hardware/tcgen05-mma.md","revision":"KernelWiki page already verified against PTX ISA 9.2","locator":"Instruction Forms, Descriptors, and Ownership","observation":"Valid tcgen05.mma forms require architecture-defined operand descriptors, predicates, and TMEM ownership/synchronization. The fragment's mnemonic and three untyped address operands are not a valid instruction form, and no such fragment exists in the pinned TFLA repository."}] + }, + { + "claim_id": "kernel-gated-delta-net-original-c013", + "claim": "NVlabs provides Triton Gated DeltaNet code and its own README recommends the faster FLA implementation for variable-length functionality.", + "claim_type": "compatibility", + "covers": ["u012", "u013", "u014"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-gdn-pinned/README.md","revision":"NVlabs/GatedDeltaNet b53d6d3a161267432a79c1c04af69fa52bddc921","locator":"FAQ questions 1 and 2","observation":"The official README links its Triton implementation, says FLA kernels are faster, supports varlen training, and strongly recommends FLA."}], + "challenge": {"alternative":"FLA is merely a community fork with no recommendation or variable-length distinction from the authors.","check":"Read the authors' FAQ rather than a third-party comparison.","outcome":"The official NVlabs repository makes both statements directly."} + }, + { + "claim_id": "kernel-gated-delta-net-original-c014", + "claim": "Gated DeltaNet is MLSys 2026 FlashInfer Track C and its official decode/prefill definitions use qk heads 4, value heads 8, and head size 128.", + "claim_type": "compatibility", + "covers": ["u015", "u016", "u017"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://mlsys26.flashinfer.ai/","revision":"fetched 2026-08-08","locator":"Track C","observation":"The organizer names Gated Delta Net as Track C and links separate decode and prefill definitions."},{"url":"https://bench.flashinfer.ai/kernels/gdn_decode_qk4_v8_d128_k_last","revision":"fetched 2026-08-08","locator":"Axes","observation":"The decode axes are num_q_heads=4, num_k_heads=4, num_v_heads=8, head_size=128."},{"url":"https://bench.flashinfer.ai/kernels/gdn_prefill_qk4_v8_d128_k_last","revision":"fetched 2026-08-08","locator":"Axes","observation":"The prefill definition records the same fixed head geometry."}], + "challenge": {"alternative":"qk4 and v8 are flattened feature dimensions rather than head counts, or Track C is a single combined definition.","check":"Read the axis names and follow both organizer links.","outcome":"The definitions label them num_q_heads/num_k_heads/num_v_heads and expose distinct decode/prefill signatures."} + }, + { + "claim_id": "kernel-gated-delta-net-original-c015", + "claim": "FlashInfer Blackwell GDN prefill is still in progress while Hopper prefill and both decode paths are done.", + "claim_type": "compatibility", + "covers": ["u018"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/gdn_prefill.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"chunk_gated_delta_rule architecture dispatch","observation":"The current pinned implementation dispatches SM100/SM103 to chunk_gated_delta_rule_sm100 and documents CUDA 13 plus head_size 128 requirements, so Blackwell prefill is implemented rather than in progress."}] + }, + { + "claim_id": "kernel-gated-delta-net-original-c016", + "claim": "At 32K and longer contexts, GDN's O(n) scaling itself provides major throughput gains.", + "claim_type": "performance", + "covers": ["u019"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct","revision":"model card fetched 2026-08-08","locator":"Highlights and Quickstart caveat","observation":"Qwen reports 10x whole-model inference throughput over Qwen3-32B for contexts over 32K, but separately warns that efficiency depends highly on implementation. The result does not isolate the GDN kernel or establish a general causal threshold."}] + }, + { + "claim_id": "kernel-gated-delta-net-original-c017", + "claim": "Qwen3-Next uses a 75% GDN and 25% full-attention hybrid layout.", + "claim_type": "compatibility", + "covers": ["u020"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct","revision":"model card fetched 2026-08-08; config revision 9c7f2fbe84465e40164a94cc16cd30b6999b0cc7","locator":"Model Overview hybrid layout and full_attention_interval","observation":"The card specifies 12 repetitions of three GDN blocks followed by one Gated Attention block across 48 layers."}], + "challenge": {"alternative":"The 3:1 statement refers to heads within every layer rather than layer types.","check":"Inspect the explicit 48-layer layout and interval field.","outcome":"It is a layer sequence: three GDN layers then one full-attention layer."} + }, + { + "claim_id": "kernel-gated-delta-net-original-c018", + "claim": "Streaming GDN decode eliminates KV-cache growth for the whole hybrid model.", + "claim_type": "mechanism", + "covers": ["u021"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/config.json","revision":"9c7f2fbe84465e40164a94cc16cd30b6999b0cc7","locator":"full_attention_interval and use_cache","observation":"The model retains full-attention layers every fourth layer with caching enabled. Only the GDN layer's recurrent state is fixed with context length."}] + }, + { + "claim_id": "kernel-gated-delta-net-original-c019", + "claim": "Short GPU kernels can become host-launch sensitive; CUDA graphs reduce CPU launch cost, and pinned vLLM's Mamba/GDN scheduling backend supports full graph capture for uniform decode.", + "claim_type": "performance", + "covers": ["u022"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cuda-graphs.html","revision":"CUDA Programming Guide fetched 2026-08-08","locator":"CUDA Graphs overview","observation":"NVIDIA states that launch setup can be significant for short kernels and that graph replay reduces CPU launch costs."},{"url":"https://github.com/vllm-project/vllm/blob/e1d85e5c2454bd8d349dbe676679380cbe0e920a/vllm/v1/attention/backends/mamba_attn.py","revision":"vLLM e1d85e5c2454bd8d349dbe676679380cbe0e920a","locator":"BaseMambaAttentionMetadataBuilder.build_for_cudagraph_capture","observation":"The backend declares uniform-batch graph support and implements decode-only full CUDA graph capture."}], + "challenge": {"alternative":"CUDA graphs remove GPU compute cost or are guaranteed to accelerate every GDN batch.","check":"Read NVIDIA's scope and vLLM's capture preconditions.","outcome":"The sources limit the benefit to CPU launch cost and the vLLM path to compatible uniform decode batches; no universal speedup is asserted."} + }, + { + "claim_id": "kernel-gated-delta-net-original-c020", + "claim": "A typical recurrent state is 512 by 1024, or 512K floats per head.", + "claim_type": "compatibility", + "covers": ["u023"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://bench.flashinfer.ai/kernels/gdn_decode_qk4_v8_d128_k_last","revision":"definition fetched 2026-08-08","locator":"state input shape","observation":"The exact Qwen TP=4 definition uses [batch_size,8,128,128]. That is 16,384 floats per value head and 131,072 floats across eight heads, not 512K floats per head."}] + }, + { + "claim_id": "kernel-gated-delta-net-original-c021", + "claim": "GDN is a learned architectural layer and is not a drop-in runtime replacement for trained softmax-attention weights.", + "claim_type": "compatibility", + "covers": ["u024"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-gdn-pinned/lit_gpt/gated_delta_net.py","revision":"NVlabs/GatedDeltaNet b53d6d3a161267432a79c1c04af69fa52bddc921","locator":"GatedDeltaNet projections, gates, convolutions, and forward","observation":"The layer owns learned q/k/v, decay, update, convolution, output-gate, norm, and output-projection parameters with a recurrence distinct from softmax attention."}], + "challenge": {"alternative":"A serving framework can substitute GDN for an already-trained attention layer without changing weights or model behavior.","check":"Compare the required learned parameter set and recurrence with an attention layer's contract.","outcome":"The GDN-specific projections/gates and recurrent state have no weight-preserving drop-in mapping supplied by the source."} + }, + { + "claim_id": "kernel-gated-delta-net-original-c022", + "claim": "Qwen3.5 attention output gating is required to eliminate attention sinks and massive activations.", + "claim_type": "mechanism", + "covers": ["u025"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://qwen.ai/blog?id=e34c4305036ce60d55a0791b170337c2b70ae51d","revision":"Qwen3-Next blog fetched 2026-08-08","locator":"Gated Attention and Training-Stability-Friendly Designs","observation":"Qwen attributes output gating to the full-attention path in Qwen3-Next and says it helps eliminate those issues. It does not say the gate is a GDN-kernel requirement or prove necessity, and the original claim misattributes the statement to Qwen3.5."}] + }, + { + "claim_id": "kernel-gated-delta-net-original-c023", + "claim": "The four external source links resolve to the named GatedDeltaNet repository, TFLA paper, NVIDIA Qwen3-Next article, and MLSys contest page.", + "claim_type": "other", + "covers": ["u026", "u027", "u028", "u029"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"curl -L -sS -o /dev/null -w '%{http_code}' ","environment":"KernelWiki host, 2026-08-08","reference":"HTTP success and page identity","result":"All four URLs returned HTTP 200 and exposed the named upstream resource.","negative_control":"https://mlsys26.flashinfer.ai/definitely-not-a-kernelwiki-page returned HTTP 404."}], + "challenge": {"alternative":"One or more links silently redirect to a generic landing page or missing resource.","check":"Follow redirects, inspect page identity, and compare with a deliberately invalid same-site path.","outcome":"Each target resolves to the intended resource while the invalid control is rejected."} + }, + { + "claim_id": "kernel-gated-delta-net-original-c024", + "claim": "The local full bundle is a full GDN reference implementation, and both derived files are correctly labeled and semantically faithful GDN variants with complete provenance.", + "claim_type": "code", + "covers": ["u030"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/kernels/gated-delta-net/full/gdn_fused_proj.py","revision":"SGLang 5bdc07d974f6cf236fa765a685453ea5e587a838; SHA-256 c4ef85167eb11065721e76dd5d63d6f346f309f84822578dd4a3874249acef89","locator":"module functions","observation":"The verbatim file only splits and reshapes QKVZBA projection outputs; it does not implement the GDN recurrence, prefill, or decode."},{"path":"artifacts/kernels/gated-delta-net/variants/01-chunk-parallel-prefill-reference-pytorch.py","revision":"original SHA-256 d7e74b02df4974fb082cacbae8a5182dd2c40cb1c13fd1d74505b51cfb15481b","locator":"gated_delta_net_prefill","observation":"The derived file applies cumulative decay repeatedly, uses an additive update rather than the delta rule, and falsely says a per-token state update is paid once per chunk."},{"path":"artifacts/kernels/gated-delta-net/variants/02-triton-decode-step-kernel-streaming.py","revision":"original SHA-256 d5eef14bdbc44e50784c0ca34e7041dcad2151960a85feddba62b097e20079ad","locator":"gdn_decode_step_kernel","observation":"The file implements additive gated linear attention, not the gated delta recurrence. Its Python comments use # rather than the page's claimed required // header."}] + }, + { + "claim_id": "kernel-gated-delta-net-original-c025", + "claim": "The documented get_page command returns this page and its artifacts, while an unknown page ID fails distinctly.", + "claim_type": "code", + "covers": ["u031", "u032"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python scripts/get_page.py kernel-gated-delta-net --include-code","environment":"KernelWiki worktree; conda base","reference":"current original page and artifact manifests","result":"Exited 0 and returned the page plus full and derived artifact content.","negative_control":"kernel-gated-delta-net-broken exited nonzero with 'No page found'."}], + "challenge": {"alternative":"The command resolves only prose and silently omits code artifacts.","check":"Search positive output for gdn_fused_proj.py and both derived provenance headers, then compare an invalid ID.","outcome":"All artifacts appear in the positive output and the invalid ID is rejected."} + }, + { + "claim_id": "kernel-gated-delta-net-original-c026", + "claim": "A measured H100 BF16 workload at sequence length 8192 and qk4/v8/d128 establishes a 10x GDN speedup over Qwen3-32B at 32K-plus context.", + "claim_type": "performance", + "covers": ["u033"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct","revision":"model card fetched 2026-08-08","locator":"Highlights and Model Overview","observation":"The official claim is whole-model 10x inference throughput for Qwen3-Next-80B-A3B-Base versus Qwen3-32B-Base when context exceeds 32K. It specifies neither H100, BF16, sequence length 8192, nor the contest qk4/v8/d128 kernel shape."}] + } + ], + "commands": [ + "inspect GatedDeltaNet commit b53d6d3, FlashInfer commit 7f614b8, TFLA/mlstm_kernels commit 5b98ff8, and vLLM merge e1d85e5c", + "inspect official Qwen model configurations at revisions 9c7f2fbe and 84726181", + "open the organizer and exact FlashInfer-Bench decode/prefill definitions on 2026-08-08", + "run source-link and get_page positive/negative controls" + ], + "residual_risk": "No supported GPU or CUDA toolkit is installed, so no kernel performance or GPU numeric result was reproduced. The unsupported result-level and implementation-specific claims are refuted rather than retained.", + "scope": "All 33 original coverage units for immutable body hash af98b7d75e33581067427cda67ff21fbf228821cc926241a7539b6215cd781ec were atomically adjudicated against primary sources and discriminating host checks." +} diff --git a/verification/reviewed/original/kernel-gated-dual-gemm.json b/verification/reviewed/original/kernel-gated-dual-gemm.json new file mode 100644 index 000000000..05a2dabbc --- /dev/null +++ b/verification/reviewed/original/kernel-gated-dual-gemm.json @@ -0,0 +1,117 @@ +{ + "path": "wiki/kernels/gated-dual-gemm.md", + "id": "kernel-gated-dual-gemm", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "kernel-gated-dual-gemm-original-c001", + "claim": "The contest operation is necessarily a single fused GPU launch that eliminates exactly two global-memory roundtrips and is the canonical gate-up pattern for LLaMA, Qwen, DeepSeek, and most modern LLMs.", + "claim_type": "mechanism", + "covers": ["u001"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml","revision":"gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c","locator":"description and submission interface","observation":"The official task fixes the tensor contract and target operation but does not require a particular launch count, memory-traffic implementation, or model list. The custom Python entry point can launch any conforming implementation."},{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/reference.py","revision":"same commit","locator":"ref_kernel","observation":"The reference materializes two FP32 GEMM outputs before SiLU/multiply, demonstrating that observable correctness does not itself prove an in-kernel fusion or exact traffic reduction."}] + }, + { + "claim_id": "kernel-gated-dual-gemm-original-c002", + "claim": "NVFP4 Gated Dual GEMM was Kernel Challenge 3 of the NVIDIA/GPU Mode Blackwell NVFP4 Hackathon.", + "claim_type": "compatibility", + "covers": ["u002"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf","revision":"official rules fetched 2026-08-08","locator":"Section 3, Kernel #3","observation":"The official rules name Kernel #3 NVFP4 Gated Dual GEMM and give its December 20, 2025 through January 16, 2026 challenge window."}], + "challenge": {"alternative":"The numbering is an editorial inference from repository folder order.","check":"Read the sponsor's official numbered challenge list.","outcome":"The rules explicitly label it Kernel #3, independently of repository order."} + }, + { + "claim_id": "kernel-gated-dual-gemm-original-c003", + "claim": "The mathematical operation is two matrix products sharing the same left operand, followed by SiLU on the first result and elementwise multiplication with the second.", + "claim_type": "code", + "covers": ["u003"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/reference.py","revision":"gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c","locator":"ref_kernel","observation":"The official reference computes two torch._scaled_mm calls from a_ref with b1_ref and b2_ref, then returns silu(ref1) * ref2 as FP16."}], + "challenge": {"alternative":"SiLU applies after multiplying the two projections, or the second projection is the gated branch.","check":"Trace the exact reference expression and operand order.","outcome":"The reference applies SiLU only to ref1 before multiplying by ref2, matching the displayed algebra."} + }, + { + "claim_id": "kernel-gated-dual-gemm-original-c004", + "claim": "The displayed fused pseudocode proves that conforming submissions use one GPU kernel and never materialize intermediate gate/up tensors in global memory.", + "claim_type": "code", + "covers": ["u003"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml","revision":"gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c","locator":"files, description, and config","observation":"The benchmark calls a Python submission entry point and constrains result correctness/performance; it does not expose or require a one-launch decomposition."},{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/reference.py","revision":"same commit","locator":"ref_kernel allocations","observation":"The correctness reference explicitly creates ref1 and ref2 tensors, so the operation contract cannot serve as evidence that intermediates are universally absent."}] + }, + { + "claim_id": "kernel-gated-dual-gemm-original-c005", + "claim": "The displayed gated_dual_gemm_nvfp4 CUDA template is a valid Blackwell NVFP4 kernel implementing the contest contract.", + "claim_type": "code", + "covers": ["u004"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"PTX ISA 9.0 / CUDA 13.0.2","locator":"tcgen05.mma syntax for block-scaled floating-point types","observation":"NVFP4 MMA requires a qualified tcgen05.mma form with D TMEM address, A/B descriptors, instruction descriptor, SFA/SFB TMEM addresses, and enable-input-D. The sketch uses undefined helper calls with only two source placeholders and a destination."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"PTX ISA 9.0 / CUDA 13.0.2","locator":"allocation syntax and lifecycle","observation":"Allocation writes its base address into shared memory and must be issued/deallocated collectively with exact qualifiers; the sketch's scalar tmem_alloc/tmem_dealloc helpers and unqualified loads are not defined interfaces."}] + }, + { + "claim_id": "kernel-gated-dual-gemm-original-c006", + "claim": "The official contest/reference establishes one physical X load reused by both MMAs, two adjacent 256-column TMEM accumulators, an epilogue with no intermediate shared memory, and shared SFA as the winning Blackwell structure.", + "claim_type": "mechanism", + "covers": ["u005", "u006", "u007", "u008"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml","revision":"gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c","locator":"description, inputs, and benchmarks","observation":"The task makes A and SFA logical shared inputs and supplies separate B1/B2 scales, but specifies no global-load count, TMEM partition, shared-memory policy, winning kernel, or required internal schedule."},{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/reference.py","revision":"same commit","locator":"ref_kernel","observation":"The primary reference defines results, not an optimized Blackwell launch topology; it uses framework scaled GEMMs and separate intermediate tensors."}] + }, + { + "claim_id": "kernel-gated-dual-gemm-original-c007", + "claim": "The exact contest kernel can be used for any dual-output operation sharing one input and expands directly to per-expert MoE kernels across the named model families.", + "claim_type": "compatibility", + "covers": ["u009", "u010", "u011"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml","revision":"gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c","locator":"input dtypes, shapes, divisibility rules, and operation","observation":"The contract is narrow: two same-shaped NVFP4 block-scaled products, FP8 per-16 scales, fixed operand layouts, SiLU on the first branch, FP16 output, and shape divisibility. Arbitrary dual outputs or grouped per-expert execution do not satisfy it automatically."}] + }, + { + "claim_id": "kernel-gated-dual-gemm-original-c008", + "claim": "The local full bundle is a reference implementation of the contest kernel and its derived CUDA file is a faithful executable NVFP4 variant with complete provenance.", + "claim_type": "code", + "covers": ["u012"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/kernels/gated-dual-gemm/full/vllm-PR-23696-gated-dual-gemm.patch","revision":"SHA-256 033d30b15af4f2050189c2e76ba61959e1e5d4bd049dc901fd56f68890b891a0","locator":"patch subjects and backend code","observation":"The patch implements vLLM MXFP4 fused-MoE backend changes, not the GPU Mode NVFP4 dual-GEMM task."},{"path":"artifacts/kernels/gated-dual-gemm/full/blackwell-cutlass-schedules-and-tma.cu","revision":"SHA-256 e720037bbdb5ee5054259a331cdaf489e7135a44038d66d1fd9d9e5998b75255","locator":"entire extracted fragment","observation":"The fragment is commentary plus one offset-padding assignment extracted from a third-party MoE blog, not a dual GEMM implementation."},{"path":"artifacts/kernels/gated-dual-gemm/variants/01-fused-epilogue-swiglu-skeleton.cu","revision":"SHA-256 85e158c184546912579ac5270451471a96f135efa3584bafbf45064e5fb7e223","locator":"gated_dual_gemm_epilogue","observation":"The derived file computes only an epilogue over already-produced float arrays; it has no NVFP4 inputs, scales, matrix multiplication, launch, bounds, or self-test."}] + }, + { + "claim_id": "kernel-gated-dual-gemm-original-c009", + "claim": "The documented page query resolves the page and all listed artifacts, while an unknown page ID fails distinctly.", + "claim_type": "code", + "covers": ["u013", "u014"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python scripts/get_page.py kernel-gated-dual-gemm --include-code","environment":"KernelWiki worktree; conda base","reference":"original page and artifact manifests","result":"Exited 0 and returned the page, vLLM patch, extracted schedule fragment, and derived epilogue file.","negative_control":"kernel-gated-dual-gemm-broken exited nonzero with 'No page found'."}], + "challenge": {"alternative":"The query resolves only prose and silently omits artifact content.","check":"Search positive output for each named full/variant file and compare an unknown ID.","outcome":"All named artifacts appear and the invalid ID is rejected."} + }, + { + "claim_id": "kernel-gated-dual-gemm-original-c010", + "claim": "A B200 NVFP4 measurement of 18.5 microseconds for M=1024, N=4096, K=7168 establishes a compute-bound contest result.", + "claim_type": "performance", + "covers": ["u015"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml","revision":"gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c","locator":"benchmarks and speed-of-light table","observation":"The official benchmark M values are 256 or 512, not 1024. It publishes theoretical speed-of-light times 2.125-8.714 us at a stated 1.5 GHz model, not a measured 18.5 us result or compute-bound classification."}] + }, + { + "claim_id": "kernel-gated-dual-gemm-original-c011", + "claim": "The page's SM90 compatibility and claim that Blackwell TMEM necessarily holds two accumulators for a fusion Hopper registers could not handle efficiently are established for this kernel.", + "claim_type": "compatibility", + "covers": ["u004", "u006"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf","revision":"official rules fetched 2026-08-08","locator":"challenge title and Kernel #3","observation":"The challenge is explicitly the Blackwell NVFP4 hackathon and its task targets B200; the record does not establish a Hopper target."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory","revision":"PTX ISA 9.0 / CUDA 13.0.2","locator":"Tensor Memory organization and allocation","observation":"PTX documents 512 columns by 128 lanes/cells and legal allocation rules, but does not require this dual-GEMM implementation to partition two 256-column accumulators or compare its efficiency with a Hopper register design."}] + } + ], + "commands": [ + "inspect gpu-mode/reference-kernels commit c5b2f7c task.yml, reference.py, task.py, and repository license", + "inspect official NVIDIA hackathon rules and PTX ISA 9.0 Tensor Memory/tcgen05 instruction forms", + "inspect every local full and derived artifact with SHA-256", + "run get_page positive and unknown-ID negative controls under conda base" + ], + "residual_risk": "No B200, CUDA toolkit, or public immutable winning submission is available locally, so no optimized submission or measured contest result was reproduced. Unsupported implementation and performance claims are refuted rather than inferred.", + "scope": "All 15 original coverage units for immutable body hash 4aa8460e6ef7388eb59cdd6643cc8a05fecb149f9148104b8eec852b369e7373, including structured performance and semantic architecture metadata, were adjudicated atomically." +} diff --git a/verification/reviewed/original/kernel-grouped-gemm.json b/verification/reviewed/original/kernel-grouped-gemm.json new file mode 100644 index 000000000..ac7873bd0 --- /dev/null +++ b/verification/reviewed/original/kernel-grouped-gemm.json @@ -0,0 +1,213 @@ +{ + "path": "wiki/kernels/grouped-gemm.md", + "id": "kernel-grouped-gemm", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "kernel-grouped-gemm-original-c001", + "claim": "Grouped GEMM necessarily means variable M with shared N and K, batches MoE experts in one kernel launch, and is the most practically important MoE-serving kernel pattern.", + "claim_type": "compatibility", + "covers": ["u001"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_group_gemm/task.yml","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"description, tests, and benchmarks","observation":"The official grouped-GEMM task supplies an independent M, N, and K for each group; its tests vary all three. Its Python submission interface does not constrain launch count, and it contains no ranking of MoE kernel importance."}] + }, + { + "claim_id": "kernel-grouped-gemm-original-c002", + "claim": "NVFP4 Grouped GEMM was Hackathon Problem 4 with a 40% grand-prize weight, and DeepGEMM provides grouped GEMM interfaces tailored to MoE phases.", + "claim_type": "compatibility", + "covers": ["u002"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf","revision":"official NVIDIA rules fetched 2026-08-08","locator":"Section 3 Kernel #4 and Grand Prize Judging Criteria","observation":"The rules name Kernel #4 NVFP4 Grouped GEMM and weight kernels 1/2/3/4 at 10/20/30/40 percent."},{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"Grouped GEMMs (contiguous layout) and Grouped GEMMs (masked layout)","observation":"The project documents M-grouped contiguous, M-grouped masked, and K-grouped interfaces for MoE forward/prefill, decode, and weight backward."}], + "challenge": {"alternative":"Problem numbering/weight is inferred from repository order, or DeepGEMM only has ordinary dense GEMMs.","check":"Compare the sponsor's numbered rules with DeepGEMM's pinned grouped-GEMM sections.","outcome":"Both facts are explicit in independent primary sources."} + }, + { + "claim_id": "kernel-grouped-gemm-original-c003", + "claim": "The contest problem fixes N and K across groups so that only M varies, and the displayed DeepSeek-V3 256-expert token counts describe its input contract.", + "claim_type": "code", + "covers": ["u003"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_group_gemm/task.yml","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"ten tests and four benchmarks","observation":"Correctness tests include per-group changes to M, N, and K; benchmark group counts are 8, 8, 2, and 2. The task has no 256-expert DeepSeek token example."}] + }, + { + "claim_id": "kernel-grouped-gemm-original-c004", + "claim": "DeepGEMM provides M-grouped contiguous, M-grouped masked, and K-grouped layouts for different MoE phases.", + "claim_type": "compatibility", + "covers": ["u004"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-deepgemm-pinned/README.md","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"Grouped GEMMs (contiguous layout) and Grouped GEMMs (masked layout)","observation":"The README assigns contiguous M-grouping to training forward/prefill, masked M-grouping to graph-compatible decode, and K-grouping to MoE weight backward."}], + "challenge": {"alternative":"K-grouped is merely another alias for the M-grouped forward interface, or masked mode is not phase-specific.","check":"Read the three API roles in the pinned README and compare their fixed dimensions.","outcome":"The source distinguishes their axes, fixed dimensions, and MoE phases."} + }, + { + "claim_id": "kernel-grouped-gemm-original-c005", + "claim": "The three displayed C++ structs accurately model DeepGEMM's contiguous, masked, and K-grouped tensor layouts.", + "claim_type": "code", + "covers": ["u005"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-deepgemm-pinned/csrc/apis/gemm.hpp","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"m_grouped_fp8_fp4_gemm_nt_contiguous, m_grouped_fp8_fp4_gemm_nt_masked, and k_grouped_fp8_gemm_*_contiguous","observation":"Contiguous mode consumes a tensor-valued grouped_layout, masked mode consumes one integer valid-M count per group rather than a boolean matrix, and K-grouped produces one M-by-N result per group from packed K segments. These interfaces contradict the displayed offsets, mask, and single-output structs."},{"path":"/tmp/kernelwiki-deepgemm-pinned/tests/generators.py","revision":"same commit","locator":"generate_m_grouped_contiguous, generate_m_grouped_masked, and generate_k_grouped_contiguous","observation":"The generators construct per-row IDs or prefix-sum ends, an int32 masked_m vector, and grouped K result tensors, respectively."}] + }, + { + "claim_id": "kernel-grouped-gemm-original-c006", + "claim": "The displayed CUTLASS GemmGrouped instantiation and Arguments initializer are a valid SM100 NVFP4 grouped-GEMM API using the named pointer-array schedule and CLC.", + "claim_type": "code", + "covers": ["u006"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/cutlass-audit.8aj07j/repo/include/cutlass/gemm/kernel/gemm_grouped.h","revision":"NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3","locator":"kernel::GemmGrouped template declaration","observation":"The actual kernel::GemmGrouped takes Mma, Epilogue, ThreadblockSwizzle, GroupScheduleMode, and optional Transposed template parameters, not tile shape, architecture, element types, and layouts as shown."},{"path":"/tmp/cutlass-audit.8aj07j/repo/include/cutlass/gemm/dispatch_policy.hpp","revision":"same commit","locator":"SM100 BlockScaled Ptr Array Dense GEMM Dispatch Policies","observation":"The named schedule tag exists, but that does not make the unrelated seven-argument GemmGrouped sketch or its invented runtime initializer valid, nor does the tag itself establish CLC use."}] + }, + { + "claim_id": "kernel-grouped-gemm-original-c007", + "claim": "The static scheduling snippet is a complete schedule for the page's official grouped-GEMM contract.", + "claim_type": "code", + "covers": ["u007"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_group_gemm/task.yml","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"tests and problem_sizes contract","observation":"The official contract permits a different N and K per group, while the snippet accepts one global N and no per-group K. It also depends on undefined BLOCK_M and BLOCK_N constants and supplies no executable launch or correctness check."}] + }, + { + "claim_id": "kernel-grouped-gemm-original-c008", + "claim": "The displayed global-atomic persistent CUDA loop is a valid CLC grouped-GEMM kernel implementing TMA and tcgen05 MMA.", + "claim_type": "code", + "covers": ["u008"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel","revision":"CUDA 13.0.2 / PTX ISA 9.0","locator":"clusterlaunchcontrol.try_cancel and query_cancel syntax","observation":"CLC uses an asynchronous try_cancel request, shared opaque response, mbarrier completion, and query_cancel decoding. The snippet uses only atomicAdd and none of the CLC protocol; its TMA/MMA calls are undefined placeholders."}] + }, + { + "claim_id": "kernel-grouped-gemm-original-c009", + "claim": "A reward-hacking submission reached the number-one Problem 4 leaderboard position before it was scrubbed.", + "claim_type": "other", + "covers": ["u009"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://www.gpumode.com/news/reward-hacking-nvfp4","revision":"GPU Mode post dated 2026-03-09, fetched 2026-08-08","locator":"opening account and leaderboard figure caption","observation":"The official post says the 11.191-us submission surged to number one shortly before the deadline and was scrubbed minutes after the competition."}], + "challenge": {"alternative":"The entry was merely hypothetical or never appeared at the top of the live leaderboard.","check":"Inspect the official incident account and its contemporaneous leaderboard figure caption.","outcome":"Both explicitly record the temporary number-one position and later removal; the page must not call it a final winner."} + }, + { + "claim_id": "kernel-grouped-gemm-original-c010", + "claim": "During correctness the exploit ran an honest padded 8-group kernel on cloned objects; during timing it launched one merged 120-group kernel for 15 objects and returned cached pointers on calls 2 through 15.", + "claim_type": "code", + "covers": ["u010"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://www.gpumode.com/news/reward-hacking-nvfp4","revision":"GPU Mode post dated 2026-03-09, fetched 2026-08-08","locator":"TL;DR, Correctness Check, and Gaming the Timing Loop","observation":"The post documents cloned correctness objects, real padded 8-group calls, 15 reused timing objects, one 120-group launch, and 14 pointer-only returns."}], + "challenge": {"alternative":"The exploit skipped arithmetic, corrupted numerics, or launched 15 merged kernels.","check":"Trace the official call-by-call account and post-timing correctness description.","outcome":"The merged kernel computes all results once; the exploit is amortized timing/state reuse, not skipped math or 15 launches."} + }, + { + "claim_id": "kernel-grouped-gemm-original-c011", + "claim": "The exploit reported 11.191 microseconds, roughly 2 microseconds ahead of the next leaderboard entry.", + "claim_type": "performance", + "covers": ["u011"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://www.gpumode.com/news/reward-hacking-nvfp4","revision":"GPU Mode post dated 2026-03-09, fetched 2026-08-08","locator":"leaderboard figure caption","observation":"The official caption labels 11.191 us as the exploit's reported result and says it was roughly 2 us ahead."}], + "challenge": {"alternative":"11.191 us is an honest per-call kernel measurement or a final valid record.","check":"Read the result together with the scrub notice and timing-loop explanation.","outcome":"The number is verified only as an exploit-reported leaderboard value; the source explicitly invalidates its interpretation as honest per-call performance."} + }, + { + "claim_id": "kernel-grouped-gemm-original-c012", + "claim": "The reward-hacking incident led to FlashInfer-Bench evaluation-methodology improvements for the MLSys 2026 contest.", + "claim_type": "mechanism", + "covers": ["u011"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://www.gpumode.com/news/reward-hacking-nvfp4","revision":"GPU Mode post dated 2026-03-09, fetched 2026-08-08","locator":"What Can We Learn From This? and closing references","observation":"The official incident post says gpu-mode/reference-kernels PR #104 addressed the harness issue. It contains no FlashInfer-Bench or MLSys causation claim."}] + }, + { + "claim_id": "kernel-grouped-gemm-original-c013", + "claim": "DeepGEMM uses contiguous M-grouping for training forward or prefill, masked M-grouping for CUDA-graph decode, and grouped computation for MoE expert projections.", + "claim_type": "compatibility", + "covers": ["u012", "u014"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-deepgemm-pinned/README.md","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"Grouped GEMMs (contiguous layout) and Grouped GEMMs (masked layout)","observation":"The project explicitly maps variable expert-token M grouping to MoE forward/prefill and its masked interface to decode when CUDA graphs hide valid counts from the CPU."}], + "challenge": {"alternative":"Contiguous and masked are interchangeable storage names with no phase or graph distinction.","check":"Compare their stated scenarios and API inputs in the README and gemm.hpp.","outcome":"The contiguous API consumes a packed/group map, while masked mode consumes per-group valid-M counts under fixed graph shapes."} + }, + { + "claim_id": "kernel-grouped-gemm-original-c014", + "claim": "Grouped GEMM is directly usable for any workload whose GEMMs share N and K while M varies.", + "claim_type": "compatibility", + "covers": ["u013"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-deepgemm-pinned/csrc/apis/gemm.hpp","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"m_grouped_fp8_fp4_gemm_nt_contiguous type, shape, layout, architecture, and scale checks","observation":"The concrete interface additionally constrains architecture, FP8/FP4 or BF16 formats, operand major modes, scale representations, aligned segment layout, output dtype, and positive dimensions. Shared N/K and variable M alone are insufficient compatibility conditions."}] + }, + { + "claim_id": "kernel-grouped-gemm-original-c015", + "claim": "Expert load imbalance is universally the primary practical grouped-GEMM bottleneck.", + "claim_type": "performance", + "covers": ["u015"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://www.gpumode.com/news/reward-hacking-nvfp4","revision":"GPU Mode post dated 2026-03-09, fetched 2026-08-08","locator":"NCU Analysis: Direct Measurement of Fixed Kernel Startup Cost","observation":"For the studied B200 implementation, the post measures about 19.5 us of fixed kernel startup and attributes about 80% of a 148-tile runtime to setup, directly supplying a materially different dominant bottleneck. Primary bottlenecks are implementation- and workload-specific."}] + }, + { + "claim_id": "kernel-grouped-gemm-original-c016", + "claim": "Small GEMM M dimensions can leave threads outside the problem bounds or launch too few threadblocks to use all GPU multiprocessors.", + "claim_type": "performance", + "covers": ["u016"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/cutlass-audit.8aj07j/repo/media/docs/cpp/efficient_gemm.md","revision":"NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3","locator":"Threadblock-level GEMM","observation":"NVIDIA documents that small M or N can leave some threadblock threads without meaningful work, and small M and N with large K can produce too few threadblocks to use all multiprocessors."}], + "challenge": {"alternative":"Tensor-core tiling always maintains full utilization regardless of M.","check":"Inspect NVIDIA's documented boundary behavior for threadblock tiles and grid size.","outcome":"The primary documentation explicitly identifies partial tiles and insufficient grid parallelism; the claim is retained only as a possibility, not a universal measured outcome."} + }, + { + "claim_id": "kernel-grouped-gemm-original-c017", + "claim": "DeepGEMM's masked grouped layout necessarily wastes computation on padded rows when valid M counts are skewed.", + "claim_type": "mechanism", + "covers": ["u017"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-deepgemm-pinned/README.md","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"Grouped GEMMs (masked layout)","observation":"DeepGEMM states that the mask tensor makes the kernel compute only valid portions."},{"path":"/tmp/kernelwiki-deepgemm-pinned/tests/test_fp8_fp4.py","revision":"same commit","locator":"test_m_grouped_gemm_masked output checks","observation":"Correctness checks slice each group to masked_m[j] valid rows rather than asserting computation across every max-M row."}] + }, + { + "claim_id": "kernel-grouped-gemm-original-c018", + "claim": "CLC scheduling requires request/response and synchronization work that a precomputed static schedule does not perform.", + "claim_type": "mechanism", + "covers": ["u018"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html","revision":"CUDA Programming Guide 13.2","locator":"4.12.1.1 API Details, steps 1-6","observation":"The documented CLC path initializes an mbarrier, submits an asynchronous cancellation request, waits for completion, decodes success and coordinates, and performs proxy ordering; those operations are specific to the dynamic request path."}], + "challenge": {"alternative":"CLC tile acquisition is implicit and incurs no request or synchronization machinery beyond a static tile map.","check":"Enumerate the normative API sequence in the official guide.","outcome":"The sequence contains explicit request, transaction, wait, decode, and fence operations; end-to-end performance impact remains workload-specific."} + }, + { + "claim_id": "kernel-grouped-gemm-original-c019", + "claim": "TMA universally requires 128-byte alignment and therefore constrains grouped-GEMM minimum tile dimensions.", + "claim_type": "compatibility", + "covers": ["u019"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html","revision":"CUDA Driver API 13.2.1","locator":"cuTensorMapEncodeTiled parameter constraints","observation":"The API documents a 64-byte-aligned descriptor and generally 16-byte-aligned global address/strides, with feature-specific 32-byte constraints. It does not impose one universal 128-byte alignment or derive minimum tile dimensions from it."}] + }, + { + "claim_id": "kernel-grouped-gemm-original-c020", + "claim": "The four source links resolve to the intended official GPU Mode repository, DeepGEMM repository, GPU Mode incident post, and NVIDIA CUTLASS documentation.", + "claim_type": "other", + "covers": ["u020", "u021", "u022", "u023"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"curl -L -sS -o /dev/null -w '%{http_code}' ","environment":"KernelWiki host, 2026-08-08","reference":"HTTP resolution plus inspection of repository/site identity","result":"The repository, DeepGEMM, reward-hack, and CUTLASS targets resolved successfully; exact replacement links to pinned task/template/reference and DeepGEMM sources also returned HTTP 200.","negative_control":"A nonexistent file under the pinned GPU Mode grouped-GEMM directory returned HTTP 404."}], + "challenge": {"alternative":"A link is dead or resolves to an unrelated host/project.","check":"Follow redirects, inspect target identities, and compare a same-path nonexistent control.","outcome":"All positive targets resolved to the intended primary projects while the invalid target failed distinctly."} + }, + { + "claim_id": "kernel-grouped-gemm-original-c021", + "claim": "A valid B200 NVFP4 result is 11.2 microseconds for 15 groups with shared N=K, and the workload is compute-bound.", + "claim_type": "performance", + "covers": ["u024"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/task.yml","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"four benchmark rows and speed-of-light table","observation":"The official benchmarks contain 8, 8, 2, and 2 groups, with N and K distinct. Published 18.833/10.667/2.406/1.525-us values are theoretical speed-of-light estimates, not a measured 11.2-us result or compute-bound classification."},{"url":"https://www.gpumode.com/news/reward-hacking-nvfp4","revision":"GPU Mode post dated 2026-03-09, fetched 2026-08-08","locator":"leaderboard figure caption and Gaming the Timing Loop","observation":"11.191 us was an exploit-reported amortized value and the submission was scrubbed; it is not a valid performance record."}] + } + ], + "commands": [ + "inspect official NVIDIA rules and gpu-mode/reference-kernels grouped-GEMM task/template/reference at ae67948685dfccf54ae8374dc9402addb7aae4f6", + "inspect DeepGEMM grouped README, generators, tests, and API checks at 891d57b4db1071624b5c8fa0d1e51cb317fa709f", + "inspect NVIDIA CUTLASS grouped API and efficient-GEMM guide at e05f953a5b3d38adc240df2ff928e0421c2abba3", + "run primary-link positive/negative controls and get_page positive/negative controls" + ], + "residual_risk": "No B200 or CUDA toolkit is available, so no optimized kernel or performance result was reproduced. The public official task, exact library APIs, vendor semantics, and official incident account are sufficient to adjudicate the original interface, code, and record claims.", + "scope": "All 24 original coverage units for body hash 216f09e31b3f73503470760148850014df1540ad380997383317930a7b459e9e were checked against official rules, immutable upstream source, vendor documentation, and discriminating link/query controls." +} diff --git a/verification/reviewed/original/kernel-nsa.json b/verification/reviewed/original/kernel-nsa.json new file mode 100644 index 000000000..72bca7cff --- /dev/null +++ b/verification/reviewed/original/kernel-nsa.json @@ -0,0 +1,176 @@ +{ + "path": "wiki/kernels/nsa.md", + "id": "kernel-nsa", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "kernel-nsa-original-c001", + "claim": "Native Sparse Attention was published at ACL 2025 as a hardware-aligned, natively trainable long-context sparse-attention design with compression, selection, and sliding-window branches; the authors report quality comparable to or better than full attention in their evaluations.", + "claim_type": "mechanism", + "covers": ["u001"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings PDF fetched 2026-08-08","locator":"abstract, Sections 3 and 4","observation":"The paper describes NSA as hardware-aligned and natively trainable, defines the three branches, evaluates contexts through 64K, and reports accuracy comparable to or better than full attention on the studied tasks."}], + "challenge": {"alternative":"The three branches are a post-training pruning recipe, or the quality statement is a universal no-loss guarantee.","check":"Inspect the training formulation and the exact scope of the experimental results.","outcome":"The branches and gates are trained end-to-end; quality is an author-reported result on the paper's evaluations, not a universal guarantee."} + }, + { + "claim_id": "kernel-nsa-original-c002", + "claim": "The NSA paper reports 9.0x forward and 6.0x backward speedups at 64K versus its Triton FlashAttention-2 baseline and gives 11.6x as the 64K decoding memory-access-volume ratio labeled expected speedup.", + "claim_type": "performance", + "covers": ["u002", "u011"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings PDF fetched 2026-08-08","locator":"Section 4.2, Figure 5, and Table 4","observation":"Figure 5 reports 9.0x forward and 6.0x backward at 64K on an 8-GPU A100 system against a Triton FlashAttention-2 implementation. Table 4 labels 11.6x an expected speedup obtained from 65,536 versus 5,632 memory-access tokens."}], + "challenge": {"alternative":"All three values are matched wall-clock measurements from one benchmark record.","check":"Compare the Figure 5 experimental description with the Table 4 title and derivation.","outcome":"Forward/backward are reported benchmark speedups, while decoding is an expected memory-access ratio; they require separate labels."} + }, + { + "claim_id": "kernel-nsa-original-c003", + "claim": "The three-branch NSA mechanism is deployed in DeepSeek-V3.2-Exp using FlashMLA sparse kernels.", + "claim_type": "compatibility", + "covers": ["u002"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-deepseek-v32/README.md","revision":"DeepSeek-V3.2-Exp 87e509a2e5a100d221c97df52c6e8be7835f0057","locator":"DeepSeek Sparse Attention announcement and high-performance CUDA kernels","observation":"The release names its mechanism DeepSeek Sparse Attention (DSA), links indexer logits to DeepGEMM and sparse kernels to FlashMLA, and does not identify it as the ACL paper's three-branch NSA."},{"path":"/tmp/kernelwiki-deepseek-v32/inference/model.py","revision":"same commit","locator":"Indexer.forward and MLA.forward","observation":"The model selects top-k token positions with a learned indexer and masks full attention to those positions; it does not implement the NSA compression/selection/window gated sum."},{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232","locator":"Sparse Attention Kernels","observation":"FlashMLA says these kernels power DSA in DeepSeek-V3.2-Exp, not NSA."}] + }, + { + "claim_id": "kernel-nsa-original-c004", + "claim": "The displayed three-path diagram is an exact NSA output equation in which the three branch attention outputs are combined without learned gates.", + "claim_type": "code", + "covers": ["u003"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings PDF fetched 2026-08-08","locator":"Section 3.1, Equations 5 and 6","observation":"NSA multiplies each branch output by an input-dependent sigmoid gate produced by a learned MLP before summing the branches. The displayed Combine expression omits those gates."}] + }, + { + "claim_id": "kernel-nsa-original-c005", + "claim": "The paper implements the selected-attention training/prefill kernel in Triton using hardware-aligned contiguous KV blocks, loading all query heads in a GQA group together so they share sparse KV indices, and placing nearly constant query/output loops in the Triton grid.", + "claim_type": "mechanism", + "covers": ["u004", "u007", "u008", "u009"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings PDF fetched 2026-08-08","locator":"Section 3.3 Hardware-aligned Sparse Attention","observation":"The authors describe a Triton kernel that loads all query heads in one GQA group into SRAM, shares selected KV indices, loads continuous KV blocks, and parallelizes query/output loops through the Triton grid."}], + "challenge": {"alternative":"Group-centric loading is an editorial optimization not stated by the authors, or selected attention scatters individual tokens.","check":"Read the kernel-design paragraph and selection-block definition.","outcome":"Both group-level query loading and continuous block loading are explicit design choices in the paper."} + }, + { + "claim_id": "kernel-nsa-original-c006", + "claim": "The displayed nsa_sparse_attention_fwd function is a valid Triton implementation of NSA selected attention and correct online softmax.", + "claim_type": "code", + "covers": ["u005"], + "verdict": "refuted", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python -c ''","environment":"KernelWiki host; conda base standard-library Python","reference":"Displayed nsa_sparse_attention_fwd update and normalized softmax definition","result":"Exited 0: the displayed update produced 4.0 with lse=0.0; normalized softmax produced 2.0 with log-partition log(2). The listing also references undefined SEQ_LEN, ignores pid in query indexing, and loads one query head per program rather than the paper's group-centric query set.","negative_control":"The stable reference divided by the within-block partition sum and returned the expected average 2.0."}] + }, + { + "claim_id": "kernel-nsa-original-c007", + "claim": "The displayed nsa_sliding_window_fwd function is a valid Triton implementation of dense attention over every position in the local window.", + "claim_type": "code", + "covers": ["u006"], + "verdict": "refuted", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python -c ''","environment":"KernelWiki host; conda base standard-library Python","reference":"Displayed sliding-window loop","result":"Exited 0: the candidate loop visited [0,2], while a four-position window is [0,1,2,3].","negative_control":"A unit-stride reference enumerated all four positions."}], + "challenge": {"alternative":"BLOCK_SIZE denotes a vectorized block load hidden by the loop body.","check":"Inspect the addresses used inside the loop.","outcome":"The body loads only K[pos] and V[pos], not a BLOCK_SIZE-wide tile, and BLOCK_SIZE is not declared in the signature."} + }, + { + "claim_id": "kernel-nsa-original-c008", + "claim": "The paper specifies Triton grid dimensions exactly as (query_block, head, batch) and establishes that this mapping avoids dynamic scheduling overhead.", + "claim_type": "mechanism", + "covers": ["u010"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings PDF fetched 2026-08-08","locator":"Section 3.3 kernel design","observation":"The paper says outer query/output loops are implemented in the Triton grid because their loop counts are nearly constant. It does not give the asserted three-dimensional grid mapping or claim that it eliminates dynamic scheduling overhead."}] + }, + { + "claim_id": "kernel-nsa-original-c009", + "claim": "The 9.0x, 6.0x, and 11.6x values were all measured on H100 with BF16, and the structured 9.0x record is reproducible as H100 BF16 at sequence length 65,536.", + "claim_type": "performance", + "covers": ["u012", "u022"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings PDF fetched 2026-08-08","locator":"Section 4 experimental setup, Figure 5, and Table 4","observation":"The paper specifies an 8-GPU A100 system for efficiency experiments, not H100. It does not state BF16 for this benchmark, and 11.6x is labeled expected speedup from memory-access volume rather than a matched measurement. Batch, software versions, trials, and variance are also absent from the structured record."}] + }, + { + "claim_id": "kernel-nsa-original-c010", + "claim": "NSA should be used at 32K or longer because dense attention is categorically prohibitively expensive beyond that threshold.", + "claim_type": "compatibility", + "covers": ["u013"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings PDF fetched 2026-08-08","locator":"Sections 1 and 4","observation":"The paper motivates long-context efficiency and evaluates several lengths through 64K, but defines no universal 32K applicability threshold or categorical prohibition; suitability depends on model, quality target, hardware, and workload."}] + }, + { + "claim_id": "kernel-nsa-original-c011", + "claim": "NSA is designed for grouped-query or multi-query attention and makes its sparse selection differentiable for end-to-end training.", + "claim_type": "compatibility", + "covers": ["u014", "u015"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings PDF fetched 2026-08-08","locator":"Sections 3.2-3.3 and Appendix B","observation":"Selection scores are aggregated over query heads in a GQA/MQA group, and the design reuses differentiable compression-attention scores so sparse patterns can be learned during end-to-end training."}], + "challenge": {"alternative":"The top-n operation makes the entire selection mechanism non-trainable, or every query head selects independent blocks.","check":"Trace score construction, head aggregation, and gradient path in the paper.","outcome":"Block indices are shared at group level and importance comes from the trainable compression branch; the paper contrasts this with non-trainable post-hoc selection."} + }, + { + "claim_id": "kernel-nsa-original-c012", + "claim": "The compression branch uses a learned MLP with intra-block position encoding, adding trainable parameters and computation, although the page provides no quantified cost.", + "claim_type": "mechanism", + "covers": ["u016"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings PDF fetched 2026-08-08","locator":"Section 3.1 Compression of Coarse-grained Tokens","observation":"Compressed keys and values are produced by learned MLPs over token blocks with intra-block position encoding; this necessarily introduces parameters and operations, but the paper does not quantify a standalone overhead in the cited passage."}], + "challenge": {"alternative":"Compression is a fixed pooling transform with no learned parameters.","check":"Inspect the compression equations and parameter notation.","outcome":"The transformation is explicitly parameterized by learned MLP weights."} + }, + { + "claim_id": "kernel-nsa-original-c013", + "claim": "NSA token selection performs a separate first pass that scores every fine-grained block before a second top-n pass.", + "claim_type": "mechanism", + "covers": ["u017"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings PDF fetched 2026-08-08","locator":"Section 3.1 Selection of Important Tokens","observation":"NSA derives fine-grained block importance by reusing the compression-branch attention scores, aggregating them over related compressed blocks and query heads, then selecting top-n blocks. The paper does not define the claimed independent score-all-blocks pass."}] + }, + { + "claim_id": "kernel-nsa-original-c014", + "claim": "NSA's Triton implementation is established to be CPU-launch-bound for small-batch decode, so CUDA Graph mode is categorically recommended.", + "claim_type": "performance", + "covers": ["u018"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings PDF fetched 2026-08-08","locator":"Section 4.2 and Limitations","observation":"The paper analyzes decoding through memory-access volume and notes generic Triton abstraction overhead versus native CUDA; it provides no CPU-launch profile or CUDA Graph recommendation for NSA."},{"url":"https://docs.nvidia.com/dl-cuda-graph/troubleshooting/performance-issues.html","revision":"NVIDIA CUDA Graphs troubleshooting guide fetched 2026-08-08","locator":"Performance Issues","observation":"NVIDIA says graphs primarily benefit CPU-bound workloads and can give minimal benefit or regress when the workload is GPU-bound, so profiling is required before applying the recommendation."}] + }, + { + "claim_id": "kernel-nsa-original-c015", + "claim": "The paper's motivation and ablations support jointly training the sparse-selection mechanism with the model rather than applying a non-trainable post-hoc selector.", + "claim_type": "compatibility", + "covers": ["u019"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings PDF fetched 2026-08-08","locator":"Appendix B, Natively Trainable Sparse Attention","observation":"The authors argue that post-hoc sparsity can degrade pretrained models, contrast non-trainable selection with NSA's gradient path, and report that native training learns effective sparse patterns."}], + "challenge": {"alternative":"Joint training is a universal theorem and no pretrained model can tolerate post-hoc sparsity.","check":"Scope the statement to the paper's mechanism, motivation, and experiments.","outcome":"The evidence supports NSA's design choice and reported studies, not a universal impossibility result."} + }, + { + "claim_id": "kernel-nsa-original-c016", + "claim": "The ACL/arXiv paper link and the lucidrains repository link resolve to the intended NSA paper and a third-party PyTorch implementation, respectively.", + "claim_type": "other", + "covers": ["u020", "u021"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"curl -L -sS -o /dev/null -w '%{http_code}' ","environment":"KernelWiki host, 2026-08-08","reference":"Source-link identities and HTTP resolution","result":"ACL landing page, proceedings PDF, and lucidrains repository each returned HTTP 200 and matched their stated identities.","negative_control":"A nonexistent sibling path under aclanthology.org returned HTTP 404."}], + "challenge": {"alternative":"The repository is the authors' official implementation or one link silently redirects to an unrelated resource.","check":"Inspect repository owner/readme and final target identity, then compare a same-host invalid path.","outcome":"The links are live; lucidrains is explicitly third-party rather than an official paper artifact, and the negative control fails distinctly."} + }, + { + "claim_id": "kernel-nsa-original-c017", + "claim": "The NSA paper directly establishes SM90 and SM100 compatibility and that Blackwell's 126 MB L2 and bandwidth improve this Triton sparse-block implementation.", + "claim_type": "compatibility", + "covers": ["u004", "u007"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://aclanthology.org/2025.acl-long.1126.pdf","revision":"ACL 2025 proceedings PDF fetched 2026-08-08","locator":"Section 4 experimental setup and Section 3.3","observation":"The paper's efficiency evaluation uses A100 GPUs and does not report Hopper/Blackwell compatibility, a 126-MB L2 result, or a cross-architecture sparse-fetch benefit."},{"url":"https://github.com/deepseek-ai/FlashMLA/blob/71c737929f2567bd0a094ae140f8f60f390b1232/README.md","revision":"FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232","locator":"requirements and support matrix","observation":"SM90/SM100 support belongs to later FlashMLA DSA kernels, a different mechanism, and cannot be transferred to the ACL NSA implementation."}] + } + ], + "commands": [ + "inspect the ACL 2025 proceedings paper's equations, implementation section, efficiency experiment, tables, and appendix", + "inspect DeepSeek-V3.2-Exp 87e509a2 and FlashMLA 71c7379 source for the later DSA mechanism", + "run host counterexamples for the sparse-forward normalization and sliding-window coverage under conda base", + "run primary-link positive controls and a same-host 404 negative control" + ], + "residual_risk": "No NVIDIA GPU, CUDA toolkit, or paper implementation is available locally. The paper's performance values were not reproduced and its benchmark record omits dtype, software versions, batch details, raw trials, and variance; retained values remain explicitly source-reported.", + "scope": "All 22 baseline units for body hash 002fbf6ef7dba7c6bd6bfec4cdeccf4104b100309e09eae082faaf0aeec0e4c3, plus semantic architecture and Blackwell metadata, were checked against the ACL paper, immutable first-party deployment source, version-pinned FlashMLA source, and discriminating host controls." +} diff --git a/verification/reviewed/original/kernel-nvfp4-gemm.json b/verification/reviewed/original/kernel-nvfp4-gemm.json new file mode 100644 index 000000000..70a46e09b --- /dev/null +++ b/verification/reviewed/original/kernel-nvfp4-gemm.json @@ -0,0 +1,284 @@ +{ + "path": "wiki/kernels/nvfp4-gemm.md", + "id": "kernel-nvfp4-gemm", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "kernel-nvfp4-gemm-original-c001", + "claim": "NVFP4 GEMM multiplies block-scaled E2M1 inputs on Blackwell, whose native path includes block-scaled tcgen05.mma and TMEM accumulation and can be fed by asynchronous TMA pipelines.", + "claim_type": "mechanism", + "covers": ["u001"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"tcgen05.mma block-scale syntax, semantics, and target notes","observation":"The ISA defines Blackwell architecture-specific block-scaled mxf4nvf4 MMA with shared-memory descriptors, scale-factor TMEM addresses, and a TMEM accumulator destination."}, + {"url":"https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu","revision":"NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3","locator":"GEMM kernel configuration and collective builders","observation":"The official example instantiates nv_float4 E2M1 operands, block-scale factor layouts, a Blackwell collective mainloop, and a TMEM-backed universal kernel."} + ], + "challenge": {"alternative":"NVFP4 is merely a software decoding convention with no native Blackwell MMA path, or every implementation must use every listed mechanism.","check":"Inspect the ISA operand and target contract, then distinguish hardware availability from a universal implementation prescription.","outcome":"The native block-scaled instruction path is explicit; TMA and warp specialization are implementation choices rather than defining requirements."} + }, + { + "claim_id": "kernel-nvfp4-gemm-original-c002", + "claim": "Every NVFP4 GEMM is compute-bound, dominated by tensor-core throughput, and necessarily uses tcgen05.mma, TMA, TMEM, and warp specialization.", + "claim_type": "performance", + "covers": ["u001"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [ + {"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/task.yml","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"speed-of-light analysis and benchmark rows","observation":"The official task models each benchmark by the maximum of FP4 math time and DRAM time, so the controlling resource is shape-dependent; it defines an output contract rather than mandating a kernel mechanism."}, + {"path":"/tmp/cutlass-audit.8aj07j/repo/media/docs/cpp/efficient_gemm.md","revision":"NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3","locator":"threadblock rasterization, occupancy, and wave quantization discussion","observation":"The official guide makes achieved GEMM throughput conditional on problem shape, tiling, occupancy, and scheduling rather than datatype alone."} + ] + }, + { + "claim_id": "kernel-nvfp4-gemm-original-c003", + "claim": "NVFP4 GEMM was Problem 2 of the NVIDIA and GPU Mode contest, targeted B200, ran from November 29 through December 19, 2025, and carried 20 percent grand-prize weight.", + "claim_type": "other", + "covers": ["u002"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url":"https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf","revision":"official contest terms fetched 2026-08-08","locator":"contest period, Kernel 2 description, evaluation platform, and prize weighting","observation":"The rules give the November 29 to December 19 period, identify Kernel 2 as NVFP4 GEMM on NVIDIA B200, and assign it 20 percent of the grand-prize score."}, + {"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"description","observation":"The immutable official problem specification calls for a block-scaled matrix-matrix multiplication optimized for NVIDIA B200."} + ], + "challenge": {"alternative":"The page has conflated the GEMV or dual-GEMM problem with Problem 2, or the dates refer to a later public leaderboard period.","check":"Compare the exact Kernel 2 row and contest-period clauses with the pinned nvfp4_gemm directory.","outcome":"Both primary records identify the same GEMM task and B200 target; the rules, not later submissions, establish the contest window."} + }, + { + "claim_id": "kernel-nvfp4-gemm-original-c004", + "claim": "The page's asserted top entries were within one percent of cuBLAS and used CUTLASS SM100 schedules.", + "claim_type": "performance", + "covers": ["u002"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + {"url":"https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12","revision":"public Popcorn leaderboard snapshot fetched 2026-08-08","locator":"rank, user_name, submission_score, and submission_time fields","observation":"The public top three are gau.nernst, s.am._, and billcarson, not the cohort later asserted by the page; the endpoint publishes scores and filenames but no cuBLAS ratio or CUTLASS implementation attribution."}, + {"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"ranking criteria and speed-of-light table","observation":"The task ranks geometric-mean benchmark time and compares grand-prize entries with a theoretical math/DRAM speed-of-light model; it supplies no cuBLAS baseline or contestant implementation claims."} + ] + }, + { + "claim_id": "kernel-nvfp4-gemm-original-c005", + "claim": "The generic NVFP4 recipe uses signed E2M1 payload values 0, 0.5, 1, 1.5, 2, 3, 4, and 6, one E4M3 local scale per 16 values, and a per-tensor FP32 global scale; MXFP4 instead groups 32 values with power-of-two UE8M0 local scales.", + "claim_type": "code", + "covers": ["u003"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url":"https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html","revision":"Transformer Engine 2.13","locator":"E2M1 format, two-level scaling, and comparison with MXFP4","observation":"The version-pinned vendor documentation defines the payload range, block-of-16 E4M3 local scaling, per-tensor FP32 scale, and MXFP4 contrast stated by the page."} + ], + "challenge": {"alternative":"NVFP4 is only the E2M1 encoding, or it uses the same block size and scale representation as MXFP4.","check":"Separate payload from recipe metadata and compare the two format tables in the pinned documentation.","outcome":"The two scale levels and block-of-16 versus block-of-32 distinction are explicit."} + }, + { + "claim_id": "kernel-nvfp4-gemm-original-c006", + "claim": "CUTLASS defines KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100 as a one-SM pointer-array NVFP4 block-scaled schedule and uses it in an official grouped-GEMM example.", + "claim_type": "compatibility", + "covers": ["u004"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [ + {"path":"/tmp/cutlass-audit.8aj07j/repo/include/cutlass/gemm/dispatch_policy.hpp","revision":"NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3","locator":"lines 842-848, schedule declarations","observation":"The exact symbol derives from the one-SM and pointer-array NVFP4 schedule marker types."}, + {"path":"/tmp/cutlass-audit.8aj07j/repo/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu","revision":"same commit","locator":"MMA1SMConfig","observation":"The official grouped pointer-array example selects this schedule with an MmaTileShape of 128 by 256 by 256."} + ], + "challenge": {"alternative":"The symbol is fabricated or is a general single-GEMM schedule independent of pointer arrays.","check":"Resolve the exact declaration and an upstream call site.","outcome":"Both the inheritance and representative call site identify it as the pointer-array grouped schedule."} + }, + { + "claim_id": "kernel-nvfp4-gemm-original-c007", + "claim": "The GPU Mode Problem 2 kernel and its top performers are established to use KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100.", + "claim_type": "compatibility", + "covers": ["u004"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [ + {"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/task.yml","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"entire task contract","observation":"The task defines tensors, tests, benchmarks, and scoring but no canonical implementation or CUTLASS schedule."}, + {"path":"/tmp/cutlass-audit.8aj07j/repo/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu","revision":"NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3","locator":"problem-shape and schedule declarations","observation":"The schedule's direct official example is a pointer-array grouped GEMM, not the contest's standard seven-tensor task contract; the page's bundled sources contain no contestant solution."} + ] + }, + { + "claim_id": "kernel-nvfp4-gemm-original-c008", + "claim": "The displayed CUTLASS typedef is a valid CUTLASS 4.x NVFP4 GEMM definition for the asserted schedule and types.", + "claim_type": "code", + "covers": ["u005"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [ + {"path":"/tmp/cutlass-audit.8aj07j/repo/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu","revision":"NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3","locator":"lines 95-164, operand types, collective builders, kernel, and adapter","observation":"The official API wraps E2M1 in nv_float4_t, builds mainloop and epilogue collectives, instantiates cutlass::gemm::kernel::GemmUniversal with those collectives, then wraps it in device::GemmUniversalAdapter. The page instead passes obsolete scalar/layout arguments to device::GemmUniversal and omits the scale-factor interface."} + ] + }, + { + "claim_id": "kernel-nvfp4-gemm-original-c009", + "claim": "The displayed hand-written CUDA is a valid warp-specialized NVFP4 kernel with correct TMA and tcgen05.mma instruction operands.", + "claim_type": "code", + "covers": ["u006"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + {"url":"https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cute/arch/mma_sm100_umma.hpp#L1647-L1659","revision":"NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3","locator":"lines 1647-1659, block16 NVFP4 asm wrapper","observation":"The exact form is kind::mxf4nvf4.block_scale.block16 and requires D, A/B descriptors, an instruction descriptor, separate SFA/SFB TMEM addresses, and a predicate. The page emits kind::f8f6f4 with a different and incomplete operand list."}, + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"tcgen05.mma syntax and restrictions","observation":"The authoritative grammar contradicts the displayed instruction. The surrounding listing also contains undefined types, descriptors, helpers, barriers, and tile symbols and therefore is not executable code."} + ] + }, + { + "claim_id": "kernel-nvfp4-gemm-original-c010", + "claim": "Every TMA operand requires 128-byte alignment, universally implying an FP4 K dimension divisible by 256.", + "claim_type": "mechanism", + "covers": ["u007", "u020"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + {"url":"https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html","revision":"CUDA Driver API 13.2.1","locator":"cuTensorMapEncodeTiled parameters and restrictions","observation":"The general tiled tensor-map contract uses 16-byte global-address and stride constraints, with mode-specific additional rules; it does not impose one universal 128-byte rule on all operands. Problem 2's K-divisible-by-256 rule is therefore an interface constraint, not a universal TMA theorem."} + ] + }, + { + "claim_id": "kernel-nvfp4-gemm-original-c022", + "claim": "The displayed scale-array assertion (K/16) divisible by 128 is valid for the official Problem 2 input space.", + "claim_type": "code", + "covers": ["u008"], + "verdict": "refuted", + "oracle": "host-experiment", + "evidence": [ + {"command":"conda run -n base python -c ''","environment":"KernelWiki host; conda base standard-library Python","reference":"gpu-mode/reference-kernels ae679486 task.yml and reference.py shape construction","result":"PASS: all ten official shapes satisfy K modulo 256; nine valid tests, including K=256, are rejected by (K/16) modulo 128.","negative_control":"K=255 was rejected by the task-specific K-divisible-by-256 validator."} + ] + }, + { + "claim_id": "kernel-nvfp4-gemm-original-c011", + "claim": "NVFP4 tcgen05.mma expects UE8M0 scales, so E4M3 block scales must be rounded to powers of two and packed before MMA, adding conversion overhead.", + "claim_type": "code", + "covers": ["u009", "u010", "u019"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + {"url":"https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512","revision":"NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3","locator":"lines 494-512, mx_float4_t and nv_float4_t","observation":"CUTLASS assigns UE8M0 to MXFP4 but UE4M3 to NVFP4, directly separating the two recipes."}, + {"url":"https://docs.nvidia.com/cuda/cublas/index.html#element-1d-block-scaling-for-fp8-and-fp4-data-types","revision":"cuBLAS documentation fetched 2026-08-08","locator":"FP4 one-dimensional block scaling and CUBLASLT_MATMUL_MATRIX_SCALE_VEC16_UE4M3","observation":"The documented FP4 NVFP4 block-scale mode uses UE4M3 for each group of 16. UE8M0 is a different scale format, so the proposed conversion would discard the defining fractional scale information."} + ] + }, + { + "claim_id": "kernel-nvfp4-gemm-original-c012", + "claim": "Simon, yue, and currybab were ranks 1, 2, and 3 with geometric-mean scores 10.807, 10.914, and 10.931 microseconds.", + "claim_type": "performance", + "covers": ["u011", "u012", "u013", "u014"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + {"url":"https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12","revision":"public Popcorn leaderboard snapshot fetched 2026-08-08","locator":"rows 1-10","observation":"The rounded values do match Simon, yue, and currybab scores, but the endpoint places them at ranks 8, 9, and 10. Ranks 1-3 are gau.nernst at 9.981888843 us, s.am._ at 10.060110139 us, and billcarson at 10.137410720 us."}, + {"url":"https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf","revision":"official rules fetched 2026-08-08","locator":"contest period","observation":"The contest ended December 19, whereas the public current top three have December 20-21 timestamps, so the snapshot cannot by itself be called the official prize-placement table."} + ] + }, + { + "claim_id": "kernel-nvfp4-gemm-original-c013", + "claim": "This NVFP4 task is generally suitable for inference with any 4-bit quantized weights on Blackwell.", + "claim_type": "compatibility", + "covers": ["u015"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + {"url":"https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html","revision":"Transformer Engine 2.13","locator":"NVFP4 recipe and comparison with MXFP4","observation":"NVFP4 requires a specific E2M1 payload, block-of-16 E4M3 scaling, and global scaling recipe; generic 4-bit weights may instead be INT4, MXFP4, or use incompatible layouts and scales."}, + {"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"custom_kernel tensor contract","observation":"The task requires exact packed E2M1 operands plus logical and reordered block-scale tensors, not an arbitrary 4-bit weight tensor."} + ] + }, + { + "claim_id": "kernel-nvfp4-gemm-original-c014", + "claim": "An NVFP4 GEMM can implement an MLP matrix multiplication when its weights and activation path satisfy the exact NVFP4 contract; if profiling shows the tensor-core path is the bottleneck, tensor-core scheduling is the relevant optimization target.", + "claim_type": "compatibility", + "covers": ["u016", "u017"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url":"https://developer.nvidia.com/blog/introducing-nvfp4-for-efficient-and-accurate-low-precision-inference/","revision":"NVIDIA technical blog fetched 2026-08-08","locator":"NVFP4 inference and GEMM discussion","observation":"NVIDIA describes NVFP4 as a Blackwell low-precision inference format used for matrix multiplications in transformer layers."}, + {"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"speed-of-light model","observation":"The task explicitly models both FP4 tensor-core math and DRAM throughput; the slower component determines the bound for a particular shape."} + ], + "challenge": {"alternative":"Every transformer MLP accepts the task ABI, or tensor-core tuning matters even when memory or launch overhead dominates.","check":"Scope applicability to matching payload/scales/layouts and the bottleneck to a measured or modeled shape.","outcome":"The sources support the conditional use, not a universal workload classification."} + }, + { + "claim_id": "kernel-nvfp4-gemm-original-c015", + "claim": "The documented native NVFP4 tensor-core path is Blackwell-specific (including SM100/SM100a targets) and Hopper has no native FP4 tensor-core instruction path.", + "claim_type": "compatibility", + "covers": ["u018"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url":"https://developer.nvidia.com/blog/introducing-nvfp4-for-efficient-and-accurate-low-precision-inference/","revision":"NVIDIA technical blog fetched 2026-08-08","locator":"Blackwell Tensor Core support","observation":"NVIDIA attributes native NVFP4 processing to Blackwell's fifth-generation Tensor Cores."}, + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"target ISA notes for block-scaled tcgen05.mma","observation":"The NVFP4 forms require architecture-specific Blackwell targets; Hopper's wgmma instruction family has no native FP4 form."} + ], + "challenge": {"alternative":"The statement forbids all software emulation or future library support on non-Blackwell devices.","check":"Distinguish native tensor-core instruction support from software conversion or emulation.","outcome":"The evidence proves only the native hardware path boundary, which is the scope of the claim."} + }, + { + "claim_id": "kernel-nvfp4-gemm-original-c016", + "claim": "TMEM's 128-lane by 512-column organization limits every logical output tile to exactly 128 rows by 512 columns.", + "claim_type": "compatibility", + "covers": ["u021"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + {"url":"https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu#L147-L159","revision":"NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3","locator":"MMA1SMConfig and MMA2SMConfig","observation":"The official NVFP4 grouped example defines 128 by 256 by 256 for one-SM MMA and 256 by 256 by 256 for cooperative two-SM MMA, demonstrating that a logical M tile need not be limited to 128."}, + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"tensor-memory organization and allocation","observation":"The 128-lane by 512-column description is a storage/addressing organization per CTA allocation, not a universal logical GEMM output-shape ceiling."} + ] + }, + { + "claim_id": "kernel-nvfp4-gemm-original-c017", + "claim": "The four listed source hyperlinks resolve to the intended GPU Mode repository, NVIDIA NVFP4 article, Harold Benoit NVFP4 notes, and CUTLASS changelog.", + "claim_type": "other", + "covers": ["u022", "u023", "u024", "u025"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [ + {"command":"curl -L -sS -o /dev/null -w '%{http_code}' ","environment":"KernelWiki host, 2026-08-08","reference":"link status and final document identity","result":"All four URLs returned HTTP 200 and resolved to the named resources.","negative_control":"A nonexistent sibling page under haroldbenoit.com returned HTTP 404."} + ], + "challenge": {"alternative":"A URL succeeds only through a soft 404 or redirects to an unrelated landing page.","check":"Inspect final page titles/content and compare a same-site invalid path.","outcome":"Each positive target has the stated identity and the negative path is discriminated as 404."} + }, + { + "claim_id": "kernel-nvfp4-gemm-original-c018", + "claim": "The artifact files and their provenance manifests exist at the stated paths and record the listed hashes, modes, source locations, and CUTLASS revision.", + "claim_type": "other", + "covers": ["u026"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [ + {"path":"artifacts/kernels/nvfp4-gemm/full/PROVENANCE.yaml","revision":"repository baseline","locator":"files entries","observation":"The manifest names the PR patch and extracted snippet with their SHA-256 values, modes, upstream location or source path, and ca4fdbea revision; the variants manifest labels the sketch as derived."} + ], + "challenge": {"alternative":"The path, hashes, or derived labels are absent even though the prose says they exist.","check":"Resolve every listed path and compare digest/provenance fields.","outcome":"The structural provenance metadata exists as described; that does not establish semantic relevance or code validity."} + }, + { + "claim_id": "kernel-nvfp4-gemm-original-c019", + "claim": "The artifact bundle is a full reference implementation of GPU Mode Problem 2 NVFP4 GEMM.", + "claim_type": "code", + "covers": ["u026"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [ + {"path":"artifacts/kernels/nvfp4-gemm/full/PR-2139-blockwise-groupwise-gemm.patch","revision":"CUTLASS merge ca4fdbea; artifact SHA-256 e6c54c39519ee3ac609efdef6f8a9b23a56deaed34153e69ae75cf5892d93fc1","locator":"examples 67, 68, and 81 and their ElementA/ElementB declarations","observation":"The patch implements FP8 E4M3 blockwise/groupwise examples and contains no NVFP4 contest implementation."}, + {"path":"artifacts/kernels/nvfp4-gemm/full/blackwell-cutlass-schedules-and-tma.cu","revision":"artifact SHA-256 e720037bbdb5ee5054259a331cdaf489e7135a44038d66d1fd9d9e5998b75255","locator":"entire extracted file","observation":"The file is comments plus one blockscale_offsets assignment extracted from a third-party blog summary, not a callable GEMM."}, + {"path":"artifacts/kernels/nvfp4-gemm/variants/01-nvfp4-gemm-skeleton.cu","revision":"repository baseline derived variant","locator":"entire file","observation":"The explicitly derived sketch uses undefined helpers and incorrect tcgen05 scale/type syntax, so it is neither upstream code nor a working reference."} + ] + }, + { + "claim_id": "kernel-nvfp4-gemm-original-c020", + "claim": "The displayed get_page command retrieves this page and includes the attached artifact bundle.", + "claim_type": "code", + "covers": ["u027", "u028"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [ + {"command":"conda run -n base python scripts/get_page.py kernel-nvfp4-gemm --include-code","environment":"KernelWiki repository; conda base, 2026-08-08","reference":"repository query utility","result":"Exited 0 and emitted the page plus both full-bundle files and provenance records.","negative_control":"The same command with a nonexistent page ID exited nonzero."} + ], + "challenge": {"alternative":"The flag is unsupported or the artifact_dir does not resolve.","check":"Run the exact command and inspect artifact filenames, then use an invalid ID.","outcome":"The valid path and flag resolve; the invalid identity is rejected."} + }, + { + "claim_id": "kernel-nvfp4-gemm-original-c021", + "claim": "The structured 10.807 record is one B200 standard-shape latency in microseconds and establishes near-cuBLAS utilization.", + "claim_type": "performance", + "covers": ["u029"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + {"url":"https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12","revision":"public Popcorn leaderboard snapshot fetched 2026-08-08","locator":"Simon row, submission_score 1.0806750492229971e-05","observation":"The rounded value comes from Simon's leaderboard score and current rank 8, not a latency for one identified shape."}, + {"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"three benchmark rows and ranking_by geom","observation":"The score is the geometric mean across three explicit shapes. The source provides theoretical speed-of-light rows but no cuBLAS measurement, software stack, trials, variance, or near-cuBLAS utilization field."} + ] + } + ], + "commands": [ + "inspect the immutable official Problem 2 task.yml, task.py, reference.py, and template.py at gpu-mode/reference-kernels ae679486", + "fetch the public Popcorn leaderboard API and inspect rank, timestamp, identity, and score fields", + "inspect CUTLASS e05f953a declarations, official examples, NVFP4 scale types, and exact tcgen05 asm wrappers", + "run the official ten task shapes through a host shape oracle and challenge the old scale alignment assertion", + "run primary-link positive controls, same-host 404 controls, and the get_page positive/negative query", + "inspect the artifact patch, extracted snippet, derived variant, and provenance manifests" + ], + "residual_risk": "No NVIDIA GPU or CUDA toolkit is available locally. No contestant source is exposed by the public leaderboard, so no implementation or performance result was reproduced. The current leaderboard includes submissions after the rules' December 19 cutoff and is retained only as a dated public snapshot, not as a prize-placement oracle.", + "scope": "All 29 baseline units for body hash d176748702ac353341b6d4887c74d36b98deccce13fd42849ac545957b212fa0 were checked against official contest/task records, the public leaderboard API, version-pinned CUTLASS source, version-pinned NVIDIA documentation, artifact contents, and discriminating host controls." +} diff --git a/verification/reviewed/original/kernel-nvfp4-gemv.json b/verification/reviewed/original/kernel-nvfp4-gemv.json new file mode 100644 index 000000000..ca9fc3e89 --- /dev/null +++ b/verification/reviewed/original/kernel-nvfp4-gemv.json @@ -0,0 +1,329 @@ +{ + "path": "wiki/kernels/nvfp4-gemv.md", + "id": "kernel-nvfp4-gemv", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "kernel-nvfp4-gemv-original-c001", + "claim": "The official Problem 1 task is an NVFP4 block-scaled batched matrix-vector product evaluated on NVIDIA B200, and its published benchmark model treats the listed cases as DRAM-limited relative to FFMA throughput.", + "claim_type": "mechanism", + "covers": ["u001"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/task.yml","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"description and speed-of-light analysis","observation":"The task names batched NVFP4 GEMV on B200 and publishes a speed-of-light table based on max(FFMA math time, DRAM time); the author worklog independently identifies the submitted CUDA kernel as memory-bound."}], + "challenge": {"alternative":"The task is a general GEMM or its bound is necessarily tensor-core compute throughput.","check":"Inspect the pinned task operation, N dimension, and stated max-of-FFMA-and-DRAM model.","outcome":"The output N dimension is one and the task explicitly describes batched GEMV; its stated benchmark numbers use the DRAM side of the model."} + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c002", + "claim": "Every FP4 element is used only once and all NVFP4 GEMV kernels are dominated by bandwidth, making PTX cache control and register budgeting universally critical.", + "claim_type": "performance", + "covers": ["u001"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/reference.py","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"ref_kernel matrix multiplication and result-column selection","observation":"Each logical B value participates in every one of the M output rows, so not every FP4 value is used once; the task constrains results rather than prescribing PTX or register controls."},{"path":"/tmp/kernelwiki-amandeep-nvfp4.html","revision":"author URL fetched 2026-08-08","locator":"attempts 9-12 and takeaways","observation":"The author reports wider loads becoming 16-25 percent slower and a register-limit change having zero effect, contradicting the universal lever claims."}] + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c003", + "claim": "This was Problem 1 of the November 2025 hackathon; the M=7168, K=16384, L=1 row has an 8.622 microsecond theoretical time, and NVIDIA documents 8 TB/s HBM3e bandwidth for one GB200 GPU.", + "claim_type": "performance", + "covers": ["u002"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf","revision":"official rules fetched 2026-08-08","locator":"Kernel 1 description and contest period","observation":"The rules identify NVFP4 batched GEMV as Kernel 1 and give November 10 through November 28, 2025."},{"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml","revision":"ae679486","locator":"speed-of-light table","observation":"The row for M=7168, K=16384, L=1 is 8.622 microseconds at the stated B200 clock."},{"url":"https://resources.nvidia.com/en-us-blackwell-architecture/blackwell-architecture-technical-brief","revision":"NVIDIA Blackwell Architecture Technical Brief fetched 2026-08-08","locator":"GB200 GPU memory subsystem","observation":"The brief specifies 8 TB/s HBM3e bandwidth for one GB200 GPU."}], + "challenge": {"alternative":"The 8.6 value belongs to a different benchmark row or the 8 TB/s number is a dual-GPU aggregate.","check":"Match the exact task table row and the technical brief's one-GPU wording.","outcome":"Both sources scope the numbers to the asserted row and one GPU respectively."} + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c004", + "claim": "Top performers achieved about 22.4 microseconds, 2.6 times the 8.6 microsecond bound, and the structured 22.4 value is the latency of the M=7168, K=16384, L=1 shape.", + "claim_type": "performance", + "covers": ["u002", "u042"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemv/NVIDIA?limit=100","revision":"public leaderboard snapshot fetched 2026-08-08","locator":"ranks 1-3 and Yue row","observation":"The current rank-1 geometric-mean score is 18.549562452 microseconds; Yue's 22.392217755 score is rank 11. The API reports aggregate scores, not per-shape timings or causal attribution."},{"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml","revision":"ae679486","locator":"benchmarks and ranking_by","observation":"The task ranks the geometric mean of three benchmark rows, so 22.392 cannot be assigned to the one frontmatter shape."}] + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c005", + "claim": "The task's logical operation has packed E2M1 A and B operands, one E4M3 scale per 16 logical values, FP16 output shaped M by 1 by L, and the three displayed benchmark configurations.", + "claim_type": "code", + "covers": ["u003"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/task.yml","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"description and benchmarks","observation":"The logical operation, block size, FP16 output, and all three benchmark dimensions match."}], + "challenge": {"alternative":"The page copied the GEMM task or a prior benchmark set.","check":"Compare the pinned nvfp4_gemv directory's N=1 output and every benchmark row.","outcome":"The operation and all three rows match Problem 1's pinned task."} + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c006", + "claim": "The actual task ABI consists of five tensors plus two FP32 per-tensor global scales, with B and SFB physically having one row and the displayed global-scale dequantization formula.", + "claim_type": "code", + "covers": ["u003"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/task.py","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"input_t","observation":"The callable input is typed as a seven-tensor tuple."},{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/reference.py","revision":"same commit","locator":"generate_input return tuple and create_scale_factor_tensors","observation":"The harness returns A, B, logical SFA/SFB, reordered SFA/SFB, and C; B and both B-scale forms are padded to 128 rows, and no FP32 global-scale arguments exist. A conda-base host reconstruction matched all seven largest-row shapes, accepted K=64, and rejected K=63."}] + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c007", + "claim": "PTX ISA 9.0 defines cvt.rn.f16x2.e2m1x2 for packed FP4-pair conversion and typed mov.b32 scalar-to-vector decomposition can split a 32-bit register into four bytes.", + "claim_type": "code", + "covers": ["u005"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"packed alternate-format cvt syntax","observation":"The exact E2M1x2-to-F16x2 instruction form is documented."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"scalar-to-vector move rules","observation":"The typed move rules permit b32 decomposition into four byte-sized vector elements."}], + "challenge": {"alternative":"The source is a scalar FP4 or mov.b32 cannot target byte lanes.","check":"Check packed-type widths and the scalar-to-vector type-size rule.","outcome":"The source is e2m1x2 and four b8 destinations total the b32 source size."} + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c008", + "claim": "Raw PTX itself produced the stated critical gain over C intrinsics and mov.b32 alone is faster than a shift-mask chain.", + "claim_type": "performance", + "covers": ["u004", "u005"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html","revision":"author post fetched 2026-08-08","locator":"Optimization 2 and Optimization 3","observation":"The 443-to-39 change bundled removal of shared memory, per-thread tiles, float4 loads, and hardware intrinsics; the 39-to-about-27 change bundled vectorized PTX decode. No isolated mov or C-versus-PTX trial is reported."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"mov semantics","observation":"The ISA specifies semantics but no timing advantage over compiler-generated bit operations."}] + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c009", + "claim": "Amandeep reports that the solutions he studied used A loads with L1::no_allocate, B loads with L1::evict_last, and per-K load-qualifier tuning.", + "claim_type": "code", + "covers": ["u006", "u007"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"author post fetched 2026-08-08","locator":"What the Top Solutions Did Differently","observation":"The author explicitly reports the two cache policies and per-K qualifier selection as observations from solutions studied after the event."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"ld.global cache eviction priority hints","observation":"The qualifier spellings and their cache-hint meanings are valid PTX."}], + "challenge": {"alternative":"The qualifiers are invented syntax or were recommendations made by the wiki rather than observations in the source.","check":"Resolve both qualifier forms in PTX and inspect the author's top-solution section.","outcome":"Both syntax and the evidence scope as an author-reported observation are explicit."} + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c010", + "claim": "Lower register limits necessarily raise occupancy and performance; ranks 1 and 3 prove 32 versus 45 registers makes occupancy dominant, and the shown launch bounds are part of that evidence.", + "claim_type": "code", + "covers": ["u008", "u009"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"author post fetched 2026-08-08","locator":"attempt 11 and top-solution register budgets","observation":"The author reports 32 and 45 limits in inspected solutions but also reports that reducing his own limit from 80 to 64 had zero effect because natural use was already below 64. He supplies no controlled 32-versus-45 comparison or the displayed launch-bounds declaration."}] + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c011", + "claim": "Amandeep reports that inspected solutions templated on exact K and dispatched separate specializations so the compiler could unroll known trip counts and tune allocation per size.", + "claim_type": "mechanism", + "covers": ["u010"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"author post fetched 2026-08-08","locator":"What the Top Solutions Did Differently, per-K paragraph","observation":"The author describes exact-K templates, launch-time dispatch, full unrolling, and size-specific register allocation."}], + "challenge": {"alternative":"The implementations used one runtime K loop and the wiki inferred templates.","check":"Inspect the author's explicit description of templating and dispatch.","outcome":"The source directly states separate exact-K specializations."} + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c012", + "claim": "The displayed CUDA template is a usable representation of the official task ABI and its per-K dispatch values.", + "claim_type": "code", + "covers": ["u011"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/template.py","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"custom_kernel signature","observation":"The official entry receives seven tensors, including two reordered scale tensors; it receives no global FP32 scales."},{"path":"wiki/kernels/nvfp4-gemv.md","revision":"baseline body 55d84be3","locator":"u011","observation":"The listing contains undefined THREADS, MIN_BLOCKS, ELEMENTS_PER_LOAD, load_fp4, and conversion helpers, uses false global-scale parameters, and gives K dispatch values inconsistent with the official benchmark K values."}] + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c013", + "claim": "128- and 256-bit loads maximize bandwidth and are effective only with PTX byte unpacking.", + "claim_type": "performance", + "covers": ["u012", "u013"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"author post fetched 2026-08-08","locator":"attempt 9 and top-solution load-width discussion","observation":"The author's controlled widening from two uchar4 loads to one uint2 load was 16-25 percent slower because unpacking added instructions; inspected solutions used wider PTX loads, but the source does not establish a universal maximizing width or PTX mov as the only usable decode path."}] + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c014", + "claim": "The displayed v2.u64 and v4.u64 PTX forms move 16 and 32 bytes, enough to carry 32 and 64 packed FP4 values respectively.", + "claim_type": "code", + "covers": ["u013"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"vector ld syntax and type sizes","observation":"Two or four 64-bit destinations carry 128 or 256 bits; two packed 4-bit values occupy each byte."}], + "challenge": {"alternative":"The vector suffix counts bytes or FP4 lanes rather than 64-bit registers.","check":"Apply PTX vector arity and u64 width.","outcome":"The total widths are 128 and 256 bits, yielding the stated packed-element capacities."} + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c015", + "claim": "The logical N=1 B vector is reused across all M output rows, although the official reference physically pads B and its scales to 128 rows.", + "claim_type": "mechanism", + "covers": ["u014"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/reference.py","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"n=1, n_padded_128, scaled_mm, and c_ref[:,0] assignment","observation":"The harness pads B to 128 for scaled_mm but stores only result column zero, so one logical B row contributes to each M output row."}], + "challenge": {"alternative":"All 128 physical B rows contribute to an M-by-128 output contract.","check":"Trace the result slice copied into C.","outcome":"Only res[:,0] is copied into an M-by-1 output."} + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c016", + "claim": "The shown shared-memory B kernel is the rank-2 approach and reduces global traffic by exactly BLOCK_M.", + "claim_type": "code", + "covers": ["u015"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"author post fetched 2026-08-08","locator":"rank-2 reuse paragraph","observation":"The author reports shared B reads across rows but publishes neither this CUDA listing nor an exact BLOCK_M traffic ratio."},{"url":"https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html","revision":"author post fetched 2026-08-08","locator":"failed CuTe ideas and CUDA Optimization 2","observation":"Loading the whole B vector into shared memory gave no improvement in one attempt, and Yue's faster 443-to-39 change explicitly removed B shared memory."}] + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c017", + "claim": "Yue reports a CuTe baseline around 100 microseconds, a 39 microsecond CUDA stage, a 39-to-about-27 microsecond vectorized PTX-decode stage, 26-to-about-22.9 microseconds from two-tile ILP, and a final submitted geometric-mean score of 22.392 microseconds.", + "claim_type": "performance", + "covers": ["u016", "u017", "u019", "u020", "u021", "u022"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html","revision":"author post fetched 2026-08-08","locator":"CuTe attempt, CUDA Optimizations 2-6, and introduction","observation":"The author reports each stated stage and final submission score; the timings are explicitly scoped to the author's journey, not independent reproduction."},{"url":"https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemv/NVIDIA?limit=100","revision":"snapshot fetched 2026-08-08","locator":"Yue row","observation":"The public endpoint reports Yue's aggregate score as 22.392217755 microseconds."}], + "challenge": {"alternative":"22.392 is one shape latency or the intermediate numbers belong to another contestant.","check":"Match the author page's headings and compare the leaderboard score field.","outcome":"The intermediate values are in Yue's own progression and 22.392 is her three-row geometric-mean score."} + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c018", + "claim": "Coalescing alone produced the 443-to-39 microsecond change.", + "claim_type": "performance", + "covers": ["u018"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html","revision":"author post fetched 2026-08-08","locator":"Optimization 1 and Optimization 2 headings","observation":"Coalescing, collaboration, and reduction produced the earlier roughly 2000-to-443 step. The 443-to-39 step bundled removal of shared memory, vectorized loads, per-thread tiles, and hardware intrinsics."}] + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c019", + "claim": "For memory-bound kernels arithmetic optimization has minimal impact and optimization should focus on bandwidth, cache policy, and vectorized loads.", + "claim_type": "performance", + "covers": ["u023"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html","revision":"author post fetched 2026-08-08","locator":"Optimization 2, NCU observation, and PTX decode progression","observation":"The author attributes a major stage to reducing roughly tenfold excess instruction issue and accelerating FP4/scale decode, demonstrating that instruction-side work was material even in this bandwidth-limited task."}] + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c020", + "claim": "The 443-to-27 microsecond difference measures C intrinsics versus hand-written PTX.", + "claim_type": "performance", + "covers": ["u024"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html","revision":"author post fetched 2026-08-08","locator":"CUDA Optimization 2 and Optimization 3","observation":"The first stage combines four material changes and ends at 39; the next combined vectorized PTX decode stage reaches about 27. There is no isolated 443-to-27 C-versus-PTX trial."}] + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c021", + "claim": "Amandeep's retrospective lesson is to profile with Nsight Compute early and confirm the kernel bottleneck before choosing optimizations.", + "claim_type": "other", + "covers": ["u025"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"author post fetched 2026-08-08","locator":"takeaways","observation":"The author explicitly says the most important missed step after attempt 7 was running Nsight Compute to confirm the kernel was memory-bound."}], + "challenge": {"alternative":"The quote is invented or describes a different tool and task.","check":"Inspect the takeaways in the named NVFP4 GEMV worklog.","outcome":"The tool, task, and retrospective lesson all match."} + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c022", + "claim": "Reducing register limits reliably improves this kernel by increasing occupancy and hiding latency.", + "claim_type": "performance", + "covers": ["u026"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"author post fetched 2026-08-08","locator":"attempt 11","observation":"Reducing maxrregcount from 80 to 64 had zero effect because the kernel already used fewer than 64 registers; occupancy and performance are conditional on realized resource use."}] + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c023", + "claim": "TMEM is irrelevant to every memory-bound kernel and can help compute-bound kernels only.", + "claim_type": "compatibility", + "covers": ["u027"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"Tensor Memory and tcgen05 instruction overview","observation":"TMEM is an explicitly managed storage space used by tcgen05 operations and data movement; its availability and semantics are not conditioned on a whole-kernel compute-bound classification."},{"url":"https://arxiv.org/html/2603.05451v1","revision":"FlashAttention-4 paper v1","locator":"section 3.1 forward pipeline","observation":"A fused attention kernel uses separate TMEM regions as part of an overlap schedule involving MMA and non-MMA softmax work, contradicting the compute-bound-only categorical scope."}] + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c024", + "claim": "A batch-one decode matrix-vector operation is a plausible use of this kernel pattern when operands satisfy the exact NVFP4 contract and profiling or the task model shows low arithmetic intensity relative to available math throughput.", + "claim_type": "compatibility", + "covers": ["u028", "u029"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml","revision":"ae679486","locator":"operation and speed-of-light model","observation":"The exact task is an N=1 matrix-vector operation and explicitly compares arithmetic with DRAM time."},{"url":"https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html","revision":"Transformer Engine 2.13","locator":"NVFP4 inference support and recipe","observation":"The vendor documents NVFP4 for inference but requires its exact E2M1, block-scale, and global-scale recipe."}], + "challenge": {"alternative":"Any batch-one matrix-vector tensor can be passed directly regardless of quantization recipe or measured bottleneck.","check":"Compare the task ABI and vendor NVFP4 recipe with the conditional wording.","outcome":"The sources support the use only for matching packed payloads/scales/layouts and a diagnosed low-intensity case."} + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c025", + "claim": "The pattern applies to memory-bound operations with arbitrary FP4-quantized weights.", + "claim_type": "compatibility", + "covers": ["u030"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html","revision":"Transformer Engine 2.13","locator":"data format and MXFP4 comparison","observation":"NVFP4 is a specific E2M1 plus E4M3-block plus FP32-global recipe; other FP4 formats and scale layouts are not ABI-compatible."}] + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c026", + "claim": "Native FP4 decode is limited to SM100 and SM100a only.", + "claim_type": "compatibility", + "covers": ["u031"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html","revision":"Transformer Engine 2.13","locator":"Supported devices","observation":"The vendor lists NVFP4 inference support as SM 10.0 and later rather than only two targets."},{"url":"https://docs.nvidia.com/cuda/archive/12.9.2/parallel-thread-execution/index.html","revision":"CUDA 12.9.2, PTX ISA 8.8","locator":"release feature table for packed alternate-format cvt","observation":"The E2M1x2 conversion is listed for sm_100f, sm_101f, and sm_120f targets, directly exceeding SM100/SM100a-only scope."}] + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c027", + "claim": "Inline PTX optimizations require revalidation across toolkit and target versions because instruction grammar and legal targets are versioned.", + "claim_type": "compatibility", + "covers": ["u032"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"PTX versioning and target ISA notes","observation":"The ISA versions instruction introduction, syntax, and legal target architecture features."},{"url":"https://docs.nvidia.com/cuda/archive/13.2.0/parallel-thread-execution/index.html","revision":"CUDA 13.2.0, PTX ISA 9.2","locator":"target architecture notes","observation":"A neighboring release changes target naming and adds target families, demonstrating why embedded PTX should be rechecked."}], + "challenge": {"alternative":"PTX is a target-independent stable assembly language and the same inline text is guaranteed indefinitely.","check":"Compare version and target notes in adjacent archived PTX releases.","outcome":"The archive explicitly records versioned feature and target changes."} + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c028", + "claim": "Keeping separately dispatchable exact-K kernel specializations creates one retained kernel entry per selected K and trades a generic loop for additional compiled code.", + "claim_type": "mechanism", + "covers": ["u033"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"author post fetched 2026-08-08","locator":"per-K specialization paragraph","observation":"The author describes exact-K templates dispatched at launch, which requires retaining the selected specializations rather than one runtime-K body."}], + "challenge": {"alternative":"Only one K specialization is emitted and all other variants are dead-stripped.","check":"Apply the source's launch-time dispatch across multiple supported K values.","outcome":"Dispatch requires each referenced K entry to be available; the page's caveat correctly identifies the code-size tradeoff, not a measured byte count."} + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c029", + "claim": "For this B200 task the published speed-of-light model is bounded by DRAM throughput, with 8 TB/s as the documented one-GPU GB200 bandwidth assumption.", + "claim_type": "performance", + "covers": ["u034"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml","revision":"ae679486","locator":"speed-of-light analysis","observation":"The task states its B200 max-of-FFMA-and-DRAM model and the three theoretical rows."},{"url":"https://resources.nvidia.com/en-us-blackwell-architecture/blackwell-architecture-technical-brief","revision":"official technical brief fetched 2026-08-08","locator":"one GB200 GPU memory bandwidth","observation":"The vendor gives 8 TB/s HBM3e per GPU."}], + "challenge": {"alternative":"The task's table is a universal B200 limit or 8 TB/s belongs to a two-GPU superchip.","check":"Keep the assertion scoped to this task model and inspect the brief's per-GPU label.","outcome":"The page's caveat is supportable only in the narrowed task-specific scope."} + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c030", + "claim": "The four source links resolve to the intended official task repository and the Yue, Amandeep, and Simon author posts.", + "claim_type": "other", + "covers": ["u035", "u036", "u037", "u038"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"curl -L -A 'Mozilla/5.0' -sS -o /dev/null -w '%{http_code}' ","environment":"KernelWiki host, network fetched 2026-08-08","reference":"Each label should resolve to its named source.","result":"All official/pinned task and three author URLs returned HTTP 200.","negative_control":"Invented same-site Simon and Yue article slugs returned HTTP 404."}], + "challenge": {"alternative":"A link redirects to an unrelated page or succeeds only for arbitrary missing slugs.","check":"Resolve with redirects and compare against same-site missing-page controls.","outcome":"The named pages returned 200 while both missing-page controls returned 404."} + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c031", + "claim": "The artifact directories exist and the full directory contains a byte-verified verbatim vLLM source file at the recorded upstream revision, while the variants are explicitly labeled derived.", + "claim_type": "other", + "covers": ["u039"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/kernels/nvfp4-gemv/full/PROVENANCE.yaml","revision":"local provenance record retrieved 2026-04-17","locator":"origin, asset_mode, upstream_sha, and files","observation":"The record pins vLLM PR 16032 revision ed7a29d9 and SHA-256 e8aed5cc for nvfp4_scaled_mm_kernels.cu; the computed file digest matches."},{"path":"artifacts/kernels/nvfp4-gemv/variants/PROVENANCE.yaml","revision":"local provenance record","locator":"asset_mode and files","observation":"The variants are labeled derived rather than verbatim upstream implementations."}], + "challenge": {"alternative":"The paths are missing or the supposedly verbatim file differs from its ledger digest.","check":"Resolve both directories and recompute SHA-256.","outcome":"The paths exist and the file digest equals the provenance value."} + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c032", + "claim": "The attached full artifact is a reference implementation of the contest batched GEMV task.", + "claim_type": "code", + "covers": ["u039"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/kernels/nvfp4-gemv/full/nvfp4_scaled_mm_kernels.cu","revision":"vLLM ed7a29d9; SHA-256 e8aed5ccb3dd9de26c3aeff159a242a46dcb7c7d8d0351b6c44ff1f8d2f7effa","locator":"Fp4GemmSm100, args_from_options, and runGemm","observation":"The file implements a general M,N,K CUTLASS NVFP4 GEMM adapter and does not implement the contest's seven-tensor N=1 callable or checker."},{"path":"/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/template.py","revision":"gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6","locator":"custom_kernel contract","observation":"The contest entry point receives seven task tensors, which the vLLM source does not expose."}] + }, + { + "claim_id": "kernel-nvfp4-gemv-original-c033", + "claim": "The documented get_page command returns this page with attached code, while an invalid page ID is rejected.", + "claim_type": "code", + "covers": ["u040", "u041"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python scripts/get_page.py kernel-nvfp4-gemv --include-code","environment":"KernelWiki worktree before GEMV remediation, conda base","reference":"Output should identify wiki/kernels/nvfp4-gemv.md and include attached artifact text.","result":"Exit 0; 27,170-byte output began with the expected path and page frontmatter.","negative_control":"kernel-does-not-exist returned nonzero with 'No page found'."}], + "challenge": {"alternative":"The command silently returns a similarly named page or accepts any identifier.","check":"Inspect the returned path and run an invalid-ID control.","outcome":"The exact path was returned and the invalid ID was rejected."} + } + ], + "commands": [ + "conda run -n base python -c ''", + "conda run -n base python scripts/get_page.py kernel-nvfp4-gemv --include-code", + "curl link checks with same-site 404 controls", + "sha256sum artifacts/kernels/nvfp4-gemv/full/nvfp4_scaled_mm_kernels.cu" + ], + "scope": "All 42 baseline units for body hash 55d84be3408d0ab5d0adfed5c555998569f294e99d23cfc0c3fd582b32841d7d were adjudicated against gpu-mode/reference-kernels ae679486, official contest rules and NVIDIA documentation, the public leaderboard snapshot fetched 2026-08-08, the three author posts, pinned local artifacts, and host controls.", + "residual_risk": "No B200 or CUDA toolkit is installed locally. Author-reported implementation techniques and timings were checked for faithful attribution but were not independently performance-reproduced; the public leaderboard is a dated current snapshot, not an immutable official cutoff/prize-placement record." +} diff --git a/verification/reviewed/original/kernel-sparse-mla.json b/verification/reviewed/original/kernel-sparse-mla.json new file mode 100644 index 000000000..ea36bef6f --- /dev/null +++ b/verification/reviewed/original/kernel-sparse-mla.json @@ -0,0 +1,25 @@ +{ + "claims":[ + {"challenge":{"alternative":"The mechanism is the ACL Native Sparse Attention architecture or an unmodified dense MLA kernel.","check":"Compare the first-party DSA paper's equations/Figure 1 and released configuration/model path with both alternatives.","outcome":"The paper names DeepSeek Sparse Attention, gives a learned lightning indexer plus top-k token selection, instantiates it under MLA's MQA mode, and the release fixes index_topk=2048."},"claim":"DeepSeek-V3.2-Exp introduces DeepSeek Sparse Attention with a lightning indexer and fine-grained top-k token selection, instantiates it under MLA's MQA mode, and its released 671B configuration selects 2048 entries per query.","claim_id":"kernel-sparse-mla-original-c001","claim_type":"mechanism","covers":["u001"],"evidence":[{"path":"/tmp/kernelwiki-deepseek-v32/DeepSeek_V3_2.pdf","revision":"DeepSeek-V3.2-Exp 87e509a2e5a100d221c97df52c6e8be7835f0057","locator":"Section 1, equations 1-2 and Figure 1","observation":"The first-party report defines both components and the MLA/MQA instantiation."},{"path":"/tmp/kernelwiki-deepseek-v32/inference/config_671B_v3.2.json","revision":"same commit","locator":"index_topk","observation":"The released configuration sets 2048."}],"oracle":"local-primary-source","verdict":"verified"}, + {"claim":"The two-stage pipeline reduces total per-query decode attention computation from O(sequence length) to O(topk=2048).","claim_id":"kernel-sparse-mla-original-c002","claim_type":"performance","covers":["u001"],"evidence":[{"path":"/tmp/kernelwiki-deepseek-v32/DeepSeek_V3_2.pdf","revision":"DeepSeek-V3.2-Exp 87e509a2e5a100d221c97df52c6e8be7835f0057","locator":"Section 3, Inference Costs","observation":"Only core/main attention becomes O(Lk) over a sequence; the lightning indexer remains O(L squared), or O(L) work for one decode query, so the whole two-stage per-query path is not O(k)."}],"oracle":"local-primary-source","verdict":"refuted"}, + {"claim":"The architecture diagram is an accurate general kernel contract: a 64-head/128-dimensional FP8 indexer with a 64-token page selects 2048 block indices, then a 16-head sparse MLA gathers them from one shared cache.","claim_id":"kernel-sparse-mla-original-c003","claim_type":"code","covers":["u002"],"evidence":[{"path":"/tmp/kernelwiki-deepseek-v32/inference/config_671B_v3.2.json","revision":"DeepSeek-V3.2-Exp 87e509a2","locator":"index and MLA dimensions","observation":"The model has 64x128 indexer heads, 128 MLA query heads, latent 512 and RoPE 64; 16 MLA heads is only an eight-way local shard, not the architecture contract."},{"path":"/tmp/kernelwiki-deepseek-v32/inference/model.py","revision":"same commit","locator":"Indexer and MLA caches","observation":"The indexer maintains a separate 128-dimensional FP8 K cache and scale cache, then emits token positions; MLA maintains separate latent and positional caches."},{"path":"/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py","revision":"FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232","locator":"sparse decode indices","observation":"Decode indices encode selected token positions as physical-page-plus-offset values; they are not one index per KV block."}],"oracle":"local-primary-source","verdict":"refuted"}, + {"claim":"Every sparse-MLA KV cache entry is universally 656 bytes with 512 FP8 bytes, 16 bytes of per-block FP32 scales, 128 BF16 bytes for indexer positional encoding, and FlashMLA requires a fixed page size of 64.","claim_id":"kernel-sparse-mla-original-c004","claim_type":"compatibility","covers":["u003","u004","u005","u006","u007"],"evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py","revision":"FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232","locator":"DeepSeek V3-family FP8+sparse decode layout","observation":"656 bytes applies to that decode mode: 512 E4M3 NoPE values, four FP32 scales for successive 128-value groups, and 64 BF16 RoPE values for attention."},{"path":"/tmp/kernelwiki-flashmla-pinned/tests/test_flash_mla_sparse_decoding.py","revision":"same commit","locator":"correctness case block_size list","observation":"The official tests cover page sizes 2, 53, 61, 64, 69, 256 and 576, refuting a FlashMLA-wide fixed-64 requirement."},{"path":"/tmp/kernelwiki-deepseek-v32/inference/model.py","revision":"DeepSeek-V3.2-Exp 87e509a2","locator":"Indexer.k_cache versus MLA caches","observation":"Indexer K storage is separate from the 656-byte attention cache."}],"oracle":"local-primary-source","verdict":"refuted"}, + {"claim":"The displayed lightning-indexer CUDA sketch accurately computes one maximum score per KV block using FP8 block-scale MMA and then runs top-k over blocks.","claim_id":"kernel-sparse-mla-original-c005","claim_type":"code","covers":["u008"],"evidence":[{"path":"/tmp/kernelwiki-deepseek-v32/DeepSeek_V3_2.pdf","revision":"DeepSeek-V3.2-Exp 87e509a2","locator":"Equation 1","observation":"The index score is per preceding token and is a weighted sum across indexer heads of ReLU-applied dot products, not a per-block maximum."},{"path":"/tmp/kernelwiki-deepseek-v32/inference/model.py","revision":"same commit","locator":"Indexer.forward","observation":"The reference forms q/k, applies non-interleaved RoPE and rotations, quantizes FP8, computes token logits, masks them, and calls topk over token positions; it does not implement the displayed signature or block-max reduction."},{"path":"/tmp/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm100_fp8_mqa_logits.cuh","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"sm100_fp8_mqa_logits","observation":"The high-performance kernel emits logits for Q-head/KV ranges using explicit weights and scales; top-k selection is outside this kernel."}],"oracle":"local-primary-source","verdict":"refuted"}, + {"claim":"The displayed sparse-decode CUDA sketch accurately takes 2048 selected block indices, a generic FP8 paged cache and a [16,576] query, gathers blocks, and runs standard MLA.","claim_id":"kernel-sparse-mla-original-c006","claim_type":"code","covers":["u009"],"evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py","revision":"FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232","locator":"flash_mla_with_kvcache signature and sparse dispatch","observation":"The real interface accepts q[batch,s_q,h_q,576], the V3-family 656-byte FP8+sparse cache, and indices[batch,s_q,topk] whose values encode token page+offset; head count and topk are shape parameters, not the invented fixed signature."},{"path":"/tmp/kernelwiki-flashmla-pinned/tests/ref.py","revision":"same commit","locator":"ref_sparse_attn_decode","observation":"The reference gathers token rows selected by encoded indices, applies attention across query heads, and masks invalid entries; it does not interpret each index as a selected block."}],"oracle":"local-primary-source","verdict":"refuted"}, + {"claim":"One H800 dense-MLA decode operating point simultaneously reaches 660 BF16 TFLOPS and 3000 GB/s and is compute-bound.","claim_id":"kernel-sparse-mla-original-c007","claim_type":"performance","covers":["u010"],"evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232","locator":"dense MLA decoding performance","observation":"The author reports separate maxima for memory-bound and compute-bound configurations; the row merges them into one operating point and label."}],"oracle":"local-primary-source","verdict":"refuted"}, + {"claim":"Sparse MLA decode on H800 performs FP8 matrix multiplication at 410 TFLOPS.","claim_id":"kernel-sparse-mla-original-c008","claim_type":"performance","covers":["u011"],"evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232","locator":"sparse MLA decode performance","observation":"The 410-TFLOPS author report is for H800 SXM5/CUDA 12.8, but only KV storage is FP8; the matrix multiplication is explicitly BF16."}],"oracle":"local-primary-source","verdict":"refuted"}, + {"claim":"Sparse MLA decode on B200 performs FP8 matrix multiplication at 350 TFLOPS and is lower than H800 because decode is bandwidth-dominated.","claim_id":"kernel-sparse-mla-original-c009","claim_type":"performance","covers":["u012"],"evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232","locator":"sparse MLA decode performance","observation":"The source says up to 350 TFLOPS and not really optimized yet; it specifies BF16 MMA over an FP8 cache and does not attribute the cross-GPU difference to bandwidth."}],"oracle":"local-primary-source","verdict":"refuted"}, + {"claim":"Dense MLA prefill reaches 1460 BF16 TFLOPS on B200 through tcgen05.","claim_id":"kernel-sparse-mla-original-c010","claim_type":"performance","covers":["u013"],"evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232","locator":"dense prefill support matrix and performance","observation":"The 1460 author-reported value belongs to NVIDIA's dense MHA forward implementation on SM100, not a dense MLA/MQA prefill result; the README does not label it tcgen05 peak."}],"oracle":"local-primary-source","verdict":"refuted"}, + {"claim":"Sparse MLA prefill on B200 is FP8, reaches 1450 TFLOPS, and therefore matches BF16 dense prefill.","claim_id":"kernel-sparse-mla-original-c011","claim_type":"performance","covers":["u014"],"evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232","locator":"sparse MLA prefill performance and equivalent code","observation":"The 1450 maximum on B200/CUDA 12.9 is author-reported, but the sparse-prefill Q/KV interface is BF16 and the source makes no matched-performance equivalence to the separate dense-MHA maximum."}],"oracle":"local-primary-source","verdict":"refuted"}, + {"claim":"Sparse MLA has a verified use threshold at context lengths of 32K or greater.","claim_id":"kernel-sparse-mla-original-c012","claim_type":"performance","covers":["u015"],"evidence":[{"path":"/tmp/kernelwiki-deepseek-v32/DeepSeek_V3_2.pdf","revision":"DeepSeek-V3.2-Exp 87e509a2","locator":"Inference Costs and Figure 3","observation":"The report presents H800 cost curves from 0K to 128K and even uses masked MHA for short prefill, but defines no universal 32K crossover or use threshold."}],"oracle":"local-primary-source","verdict":"refuted"}, + {"challenge":{"alternative":"The released DSA kernels are unrelated to the DeepSeek-V3.2 model.","check":"Follow the model repository's high-performance-kernel links and FlashMLA's release statement.","outcome":"Both first-party repositories explicitly connect the indexer logits and sparse attention kernels to DeepSeek-V3.2-Exp DSA."},"claim":"DeepSeek-V3.2-Exp is a directly supported use of the released indexer and FlashMLA sparse-attention kernels.","claim_id":"kernel-sparse-mla-original-c013","claim_type":"compatibility","covers":["u016"],"evidence":[{"path":"/tmp/kernelwiki-deepseek-v32/README.md","revision":"DeepSeek-V3.2-Exp 87e509a2","locator":"Open-Source Kernels","observation":"The release links indexer logits to DeepGEMM and sparse attention to FlashMLA."},{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232","locator":"Introduction","observation":"The repository says the sparse kernels power DSA in DeepSeek-V3.2-Exp."}],"oracle":"local-primary-source","verdict":"verified"}, + {"claim":"The same sparse-kernel path can be recommended for unspecified similar MLA architectures.","claim_id":"kernel-sparse-mla-original-c014","claim_type":"compatibility","covers":["u016"],"evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py","revision":"FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232","locator":"V3-family decode and sparse-prefill contracts","observation":"The APIs impose exact head dimensions, cache layout, dtype, head-count, index-shape, and output contracts; architectural similarity alone does not establish compatibility."}],"oracle":"local-primary-source","verdict":"refuted"}, + {"challenge":{"alternative":"Sparse decode cannot affect per-token latency because the indexer still scans the prefix.","check":"Compare the main-attention complexity reduction and first-party H800 service cost curve with the retained caveat about the indexer.","outcome":"The main attention reads and computes over selected entries and the first-party deployment reports long-context end-to-end savings, supporting relevance but not universal superiority."},"claim":"The DSA/FlashMLA path is relevant to serving measurements where long-context per-token decode latency matters, provided indexer and sparse-attention time are both included.","claim_id":"kernel-sparse-mla-original-c015","claim_type":"performance","covers":["u017"],"evidence":[{"path":"/tmp/kernelwiki-deepseek-v32/DeepSeek_V3_2.pdf","revision":"DeepSeek-V3.2-Exp 87e509a2","locator":"Inference Costs and Figure 3","observation":"The paper reports reduced H800 service costs in long-context decoding while retaining the indexer's complexity."},{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232","locator":"sparse decode interface","observation":"The attention kernel consumes selected-token indices and therefore addresses the selected-attention portion of decode."}],"oracle":"local-primary-source","verdict":"verified"}, + {"claim":"A structured B200 performance record establishes 1450 FP8 TFLOPS for sparse prefill at sequence length 32K and topk 2048 and classifies it FP8 sparse compute-bound.","claim_id":"kernel-sparse-mla-original-c016","claim_type":"performance","covers":["u018"],"evidence":[{"path":"/tmp/kernelwiki-flashmla-pinned/README.md","revision":"FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232","locator":"sparse MLA prefill performance and API","observation":"The source reports only up to 1450 TFLOPS on B200/CUDA 12.9; the inputs are BF16 and it supplies no 32K/topk-2048 tuple, compute-bound classification, timed region, repetitions, or variance."}],"oracle":"local-primary-source","verdict":"refuted"}, + {"claim":"SM100 tcgen05 FP8 block-scale MMA is the common mechanism enabling both the lightning indexer and sparse-attention stages.","claim_id":"kernel-sparse-mla-original-c017","claim_type":"mechanism","covers":["u001"],"evidence":[{"path":"/tmp/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm100_fp8_mqa_logits.cuh","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"UMMA descriptor and scale handling","observation":"The indexer logit kernel uses FP8 UMMA with explicit Q/K scale handling, not the page's claimed shared FP8 block-scale attention pipeline."},{"path":"/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py","revision":"FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232","locator":"sparse decode and prefill dtype contracts","observation":"Sparse decode dequantizes FP8 KV to BF16 for MMA; sparse prefill accepts BF16 Q/KV. SM100 source uses tcgen05/TMEM, but not one FP8 block-scale MMA mechanism for both stages."}],"oracle":"local-primary-source","verdict":"refuted"} + ], + "commands":["Extract and inspect the first-party DeepSeek-V3.2-Exp report and model","Inspect pinned FlashMLA sparse interfaces, tests, and reported maxima","Inspect pinned DeepGEMM SM100 indexer-logit implementation"], + "id":"kernel-sparse-mla","page_verdict":"refuted","path":"wiki/kernels/sparse-mla.md", + "residual_risk":"No CUDA toolkit or GPU is present; code semantics were checked from pinned primary source, while performance was audited only as author-reported scope.", + "scope":"All 18 baseline units for hash 3a4b0e533208cfbbe45f6c7c15396a5e66007ed4a56f0ca9699e034a5b36b628 plus the material Blackwell frontmatter claim were audited against the first-party model report/configuration and pinned kernel sources." +} diff --git a/verification/reviewed/original/kernel-tensorrt-llm-blackwell-indexer.json b/verification/reviewed/original/kernel-tensorrt-llm-blackwell-indexer.json new file mode 100644 index 000000000..ef031172a --- /dev/null +++ b/verification/reviewed/original/kernel-tensorrt-llm-blackwell-indexer.json @@ -0,0 +1,187 @@ +{ + "claims": [ + { + "challenge": { + "alternative": "The PR only changes configuration or Python dispatch and does not provide the stated CUDA data path.", + "check": "Inspect the pinned PR artifact for the CUDA kernels, their Torch operators, and the DSA call sites.", + "outcome": "The artifact contains fusedCatFp4.cu plus indexerKCacheGather.cu and indexerKCacheScatter.cu, and dsa.py calls the corresponding operators in the FP4 indexer path." + }, + "claim": "TensorRT-LLM PR 13340 adds an SM100 FP4 DSA indexer path with native CUDA fused FP4 concatenation/quantization and K-cache gather/scatter operators.", + "claim_id": "kernel-tensorrt-llm-blackwell-indexer-original-c001", + "claim_type": "mechanism", + "covers": ["u001"], + "evidence": [ + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/fusedCatFp4.cu", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "fusedCatFp4Kernel and invokeFusedCatFp4", + "observation": "The CUDA implementation concatenates BF16 PE/NoPE inputs, quantizes 128 values to packed FP4 E2M1, and emits packed UE8M0 scales." + }, + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheGather.cu", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "indexerKCacheGatherUnifiedKernel", + "observation": "The CUDA gather copies 64-byte FP4 or 128-byte FP8 payloads and a four-byte scale word from the paged cache." + }, + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheScatter.cu", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "indexerKCacheScatterUnifiedKernel", + "observation": "The CUDA scatter copies the payload and scale word into the paged cache." + } + ], + "oracle": "local-primary-source", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "The PR establishes a generic sparse-indexer ABI or a reusable FlashInfer-Bench implementation.", + "check": "Compare the operator assertions and TensorRT-LLM-specific cache/dispatch contracts with that broader interpretation.", + "outcome": "The path is tied to TensorRT-LLM's DSA metadata, cache manager, SM100 dispatch, 128-element index head, and DeepGEMM calls, so it is evidence to adapt rather than a drop-in external kernel." + }, + "claim": "The PR is implementation evidence for quantized indexer cache movement and layout, but it is not a drop-in FlashInfer-Bench solution.", + "claim_id": "kernel-tensorrt-llm-blackwell-indexer-original-c002", + "claim_type": "compatibility", + "covers": ["u001"], + "evidence": [ + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/tensorrt_llm/_torch/attention_backend/sparse/dsa.py", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "Indexer._prep_q_or_k, sparse_attn_indexer, and DSACacheManager", + "observation": "The operators are embedded in TensorRT-LLM's DSA projection, cache-management, DeepGEMM-logit, and top-k pipeline rather than exposing a generic benchmark contract." + } + ], + "oracle": "local-primary-source", + "verdict": "verified" + }, + { + "claim": "The displayed candidate_indexer_probe is a meaningful implementation probe for the PR's sparse indexer path.", + "claim_id": "kernel-tensorrt-llm-blackwell-indexer-original-c003", + "claim_type": "code", + "covers": ["u002"], + "evidence": [ + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheGather.cu", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "indexerKCacheGatherUnifiedKernel", + "observation": "The real gather consumes separate int64 payload and scale slot mappings, handles a strided four-dimensional cache, copies vectorized payload and scale bytes, and skips negative mappings; the sketch does none of these and also lacks a bounds guard." + }, + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/tensorrt_llm/_torch/attention_backend/sparse/dsa.py", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "sparse_attn_indexer", + "observation": "The actual indexer performs quantized Q/K preparation, cache scatter/gather, MQA logit computation, and a separate top-k selection; assigning one cache byte to an index represents none of this behavior." + } + ], + "oracle": "local-primary-source", + "verdict": "refuted" + }, + { + "challenge": { + "alternative": "Packed data alone is sufficient for a correctness reference and negative slot mappings produce initialized sentinel outputs.", + "check": "Trace scale packing and the early-return behavior in both copy kernels and their allocating wrapper.", + "outcome": "FP4 requires four per-32-value UE8M0 exponents packed into one int32, and a negative payload or scale mapping makes the whole token return; the gather wrapper allocates empty output, so a skipped row is not initialized by the kernel." + }, + "claim": "A correctness reference must explicitly model FP4 packing, scale placement, and invalid-token behavior.", + "claim_id": "kernel-tensorrt-llm-blackwell-indexer-original-c004", + "claim_type": "semantics", + "covers": ["u003", "u004"], + "evidence": [ + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/fusedCatFp4.cu", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "FP4 block constants and stages 5-7", + "observation": "Each 128-value row becomes 64 packed bytes and four UE8M0 exponent bytes packed little-endian into one int32." + }, + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheGather.cu", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "negative slot check in indexerKCacheGatherUnifiedKernel", + "observation": "If either slot mapping is negative, the token returns before writing its allocated output row." + } + ], + "oracle": "local-primary-source", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "The PR only exposes the complete indexer, making gather/scatter traffic impossible to isolate.", + "check": "Inspect the registered Torch operators and DSA call graph for independent gather and scatter launches.", + "outcome": "Gather and scatter are separately registered operators and are launched independently of MQA logits and top-k, so their traffic can be profiled separately with valid inputs." + }, + "claim": "Gather/scatter traffic is a valid separate profiling target.", + "claim_id": "kernel-tensorrt-llm-blackwell-indexer-original-c005", + "claim_type": "performance", + "covers": ["u005"], + "evidence": [ + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/tensorrt_llm/_torch/attention_backend/sparse/dsa.py", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "cache scatter/gather calls and subsequent MQA-logit/top-k calls", + "observation": "The pinned path invokes cache movement, logit calculation, and top-k as distinct operations, permitting stage-specific profiling." + } + ], + "oracle": "local-primary-source", + "verdict": "verified" + }, + { + "challenge": { + "alternative": "The PR fuses gather/scatter with top-k, so separating them would misrepresent the implementation.", + "check": "Follow data from cache gather or paged-cache input through logits to top-k dispatch.", + "outcome": "The copy kernels do not calculate scores or select indices; dsa.py calls DeepGEMM MQA logits and then a distinct prefill/decode top-k operator." + }, + "claim": "Top-k selection should remain separate from the gather/scatter memory path while evaluating this pinned implementation.", + "claim_id": "kernel-tensorrt-llm-blackwell-indexer-original-c006", + "claim_type": "mechanism", + "covers": ["u006"], + "evidence": [ + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/tensorrt_llm/_torch/attention_backend/sparse/dsa.py", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "_call_mqa_logits, _call_paged_mqa_logits, indexer_topk_prefill, and indexer_topk_decode", + "observation": "Logit generation and top-k selection occur after and outside cache copy operations in the pinned pipeline." + } + ], + "oracle": "local-primary-source", + "verdict": "verified" + }, + { + "claim": "The PR provides fused quantized gather/scatter kernels for the Blackwell FP4/FP8 indexer cache paths.", + "claim_id": "kernel-tensorrt-llm-blackwell-indexer-original-c007", + "claim_type": "mechanism", + "covers": ["u001"], + "evidence": [ + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/fusedCatFp4.cu", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "fusedCatFp4Kernel stages 1-7", + "observation": "Quantization is fused with PE/NoPE concatenation in fusedCatFp4, before cache movement." + }, + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheGather.cu", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "indexerKCacheGatherUnifiedKernel", + "observation": "Gather only copies already-quantized payload and scale bytes; it performs no quantization." + }, + { + "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheScatter.cu", + "revision": "TensorRT-LLM PR 13340 merge 897c4bff", + "locator": "indexerKCacheScatterUnifiedKernel", + "observation": "Scatter only copies already-quantized payload and scale bytes; it performs no quantization." + } + ], + "oracle": "local-primary-source", + "verdict": "refuted" + } + ], + "commands": [ + "Inspect pinned fusedCatFp4 CUDA kernel and Torch wrapper", + "Inspect pinned K-cache gather/scatter CUDA kernels and Torch wrappers", + "Trace the pinned TensorRT-LLM DSA FP4 call path through logits and top-k" + ], + "id": "kernel-tensorrt-llm-blackwell-indexer", + "page_verdict": "refuted", + "path": "wiki/kernels/tensorrt-llm-blackwell-indexer.md", + "residual_risk": "No CUDA toolkit or GPU is present, so the pinned source was not built or executed and the page's profiling recommendations have no measured result.", + "scope": "All six baseline units for hash c6277510118240c58c46ecfd1a84c7978461c9a65c84b4f74cbd679662dae2d1 and the material Blackwell frontmatter claim were audited against the PR's pinned verbatim CUDA, wrapper, and DSA sources." +} diff --git a/verification/reviewed/original/lang-cuda-cpp.json b/verification/reviewed/original/lang-cuda-cpp.json new file mode 100644 index 000000000..623bf2e57 --- /dev/null +++ b/verification/reviewed/original/lang-cuda-cpp.json @@ -0,0 +1,87 @@ +{ + "path": "wiki/languages/cuda-cpp.md", + "id": "lang-cuda-cpp", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "lang-cuda-cpp-original-c001", + "claim": "Gau Nernst's hand-written CUDA C++ and inline-PTX tutorial reports 1475.93 TFLOP/s for its final 4096-cubed B200 kernel versus 1506.74 TFLOP/s for its PyTorch 2.9.1/CUDA 13 cuBLAS baseline, which rounds to 98 percent.", + "claim_type": "performance", + "covers": ["u001"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/README.md","revision":"gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"benchmark table and environment","observation":"The pinned author source gives M=N=K=4096, Modal B200, PyTorch 2.9.1/CUDA 13, cuBLAS 1506.74, and v6 1475.93 TFLOP/s. A conda-base ratio check produced 97.955 percent and rejected a broken 1600-TFLOP/s baseline."}], + "challenge": {"alternative":"The 98 percent compares with theoretical peak or another shape/environment.","check":"Use the exact cuBLAS and v6 rows and disclosed shape/software.","outcome":"The ratio rounds to 98 only against the stated same-table cuBLAS row; the claim is source-reported for that setup."} + }, + { + "claim_id": "lang-cuda-cpp-original-c002", + "claim": "The displayed tmem_alloc_cta helper correctly issues tcgen05.alloc from thread 0 alone.", + "claim_type": "code", + "covers": ["u002"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"cta_group::1 issue granularity","observation":"Allocation is warp-collective: all threads of one warp must execute the same sync.aligned instruction, and lane-0-only execution is undefined."},{"url":"https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu","revision":"gau-nernst/learn-cuda 3b90ac9b","locator":"TMEM allocation branch","observation":"The pinned author code executes alloc in every lane of warp 1, not thread 0."}] + }, + { + "claim_id": "lang-cuda-cpp-original-c003", + "claim": "The displayed tcgen05_mma helper supplies two immediate descriptors for accumulator C and output D as the final two operands of an unscaled f16 MMA.", + "claim_type": "code", + "covers": ["u002"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/common.h","revision":"gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"tcgen05_mma_f16","observation":"The real wrapper takes one instruction descriptor and an enable-input-D integer, converts the latter to a local predicate, then passes idesc and predicate as the final operands."},{"path":"/tmp/kernelwiki-ptx-1302.html","revision":"CUDA 13.0.2, PTX ISA 9.0 fetched 2026-08-08","locator":"tcgen05.mma unscaled f16 grammar","observation":"The required final operands are idesc and enable-input-D predicate, optionally preceded by a lane-disable vector; there are no C/D immediate descriptors."}] + }, + { + "claim_id": "lang-cuda-cpp-original-c004", + "claim": "The literal alloc, 32x32b.x1 load, and dealloc instruction spellings are legal PTX ISA 9.0 forms when issued with their required collective and completion context.", + "claim_type": "code", + "covers": ["u002"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"allocation and deallocation grammar","observation":"The exact shared b32 allocation and taddr deallocation forms match."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"same archive","locator":"32x32b repetition table","observation":"32x32b.x1.b32 is legal and produces one 32-bit register per participating thread."}], + "challenge": {"alternative":"The text strings themselves are fabricated even though wrapper participation is wrong.","check":"Separate instruction grammar from CUDA control flow and compare exact qualifiers.","outcome":"The instruction strings are valid; the allocation wrapper is wrong because of how it predicates them."} + }, + { + "claim_id": "lang-cuda-cpp-original-c005", + "claim": "The two mbarrier instructions have valid default release-CTA arrive-expect-tx and acquire-CTA parity-wait semantics for a correctly initialized CTA-shared barrier and phase.", + "claim_type": "code", + "covers": ["u003"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"arrive.expect_tx grammar and default semantics/scope","observation":"Absent qualifiers default to release and CTA; expect_tx adds byte count and performs one arrival."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait","revision":"same archive","locator":"try_wait.parity grammar and defaults","observation":"Absent qualifiers default to acquire and CTA; parity must track the current or immediately preceding phase."}], + "challenge": {"alternative":"The omitted semantic/scope qualifiers make both instructions relaxed or system-scoped.","check":"Read the documented defaults for each grammar.","outcome":"The defaults are release/acquire at CTA scope as required for a CTA-local stage."} + }, + { + "claim_id": "lang-cuda-cpp-original-c006", + "claim": "The two inline helpers alone are a safe complete TMA-to-MMA synchronization implementation.", + "claim_type": "code", + "covers": ["u003"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/inline-ptx-assembly/index.html#incorrect-optimization","revision":"CUDA 13.0.2 Inline PTX guide","locator":"volatile and memory clobber pitfall","observation":"Hidden memory effects or a need to stop compiler memory motion require a memory clobber; neither helper declares one."},{"url":"https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu","revision":"gau-nernst/learn-cuda 3b90ac9b","locator":"barrier initialization and staged pipeline","observation":"The complete source initializes every barrier, publishes initialization to the async proxy, tracks separate TMA/MMA/mainloop/epilogue phases, and uses paired full/empty handoffs; the page shows none of that lifecycle."}] + }, + { + "claim_id": "lang-cuda-cpp-original-c007", + "claim": "The warp-role skeleton matches the tutorial's final source: warp 0 elects one TMA issuer, CTA 0 warp 1 elects one MMA issuer, and warps 2-5 collectively drain TMEM in the epilogue.", + "claim_type": "code", + "covers": ["u004"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v6.cu","revision":"gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"NUM_WARPS and role branches around lines 227-297","observation":"The source declares one TMA warp, one MMA warp, four epilogue warps and uses elected single issuers for TMA/MMA while all epilogue lanes participate in TMEM loads."}], + "challenge": {"alternative":"Every lane in the TMA/MMA warps issues or the epilogue is lane-0-only.","check":"Inspect elect_sync predicates and warp-range branches.","outcome":"Only elected threads issue TMA/MMA; all warps at IDs 2 and above enter the epilogue path."} + }, + { + "claim_id": "lang-cuda-cpp-original-c008", + "claim": "The two related links resolve to the local PTX language page and evidence-scoped tcgen05 tutorial source card.", + "claim_type": "other", + "covers": ["u005", "u006"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"test -f wiki/languages/ptx-sm100.md && test -f sources/blogs/tcgen05-tutorial.md","environment":"current KernelWiki worktree","reference":"Resolve links from wiki/languages and inspect IDs.","result":"Both targets exist with IDs lang-ptx and blog-tcgen05-tutorial.","negative_control":"wiki/languages/not-cuda.md does not exist."}], + "challenge": {"alternative":"The ../../ source path escapes to a missing or unrelated file.","check":"Resolve it from wiki/languages and inspect frontmatter.","outcome":"It lands on the intended tutorial card; the broken local control fails."} + } + ], + "commands": ["conda run -n base python -c '<1475.93/1506.74 ratio with broken baseline control>'", "local related-link resolution"], + "scope": "All six baseline units for body hash 7a58194cb413f25af6db71a4e0d90828273f461ee66232063fb6e6a96176d836 were checked against PTX ISA 9.0, CUDA 13.0.2 inline-PTX rules, Gau Nernst's pinned commit 3b90ac9b, and host controls.", + "residual_risk": "No CUDA toolkit or B200 is installed locally. The author benchmark is verified as faithful source reporting, not independently reproduced performance." +} diff --git a/verification/reviewed/original/lang-cute-dsl.json b/verification/reviewed/original/lang-cute-dsl.json new file mode 100644 index 000000000..5c5e35212 --- /dev/null +++ b/verification/reviewed/original/lang-cute-dsl.json @@ -0,0 +1,173 @@ +{ + "path": "wiki/languages/cute-dsl.md", + "id": "lang-cute-dsl", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "lang-cute-dsl-original-c001", + "claim": "CUTLASS 4.5.0 includes the Python-native CuTe DSL and Blackwell support.", + "claim_type": "compatibility", + "covers": ["u001"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0/CHANGELOG.md","revision":"NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"4.5.0 CuTe DSL section and repository tree","observation":"The exact release contains CuTe DSL Python packages plus SM100/Blackwell operations and examples."}], + "challenge": {"alternative":"CuTe DSL arrived after 4.5.0 or only targets pre-Blackwell architectures.","check":"Inspect the v4.5.0 tag rather than rolling documentation.","outcome":"The tagged Python package implements tcgen05/TMEM and the tag contains Blackwell CuTe DSL examples."} + }, + { + "claim_id": "lang-cute-dsl-original-c002", + "claim": "CuTe DSL is the primary abstraction layer in CUTLASS 4.5.0 for Blackwell kernels.", + "claim_type": "other", + "covers": ["u001"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0/README.md","revision":"NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"Overview and repository organization","observation":"CUTLASS describes C++ template abstractions and Python DSLs as parallel interfaces; the release also ships extensive SM100 C++ kernels. It does not designate CuTe DSL as the primary Blackwell layer."}] + }, + { + "claim_id": "lang-cute-dsl-original-c003", + "claim": "FlashAttention-4 is implemented entirely in CuTe DSL, and its author reports approximately 20-30 times shorter compile times than C++ templates.", + "claim_type": "performance", + "covers": ["u001", "u006"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://tridao.me/blog/2026/flash4/","revision":"Tri Dao first-party FA4 post, fetched 2026-08-08","locator":"Language and framework: CuTe-DSL","observation":"The author says FA4 is entirely CuTe DSL and reports about 20-30x lower compile time than C++ templates."}], + "challenge": {"alternative":"The factor is an execution-speed result or applies to all CuTe DSL programs.","check":"Read the metric and subject in the first-party language/framework section.","outcome":"It is explicitly compile time for FA4's development comparison, not kernel runtime or a universal bound."} + }, + { + "claim_id": "lang-cute-dsl-original-c004", + "claim": "The displayed SM100_MMA_F16BF16_SS and SM100_MMA_F16BF16_SS_2SM symbols are usable CUTLASS 4.5.0 CuTe DSL Python MMA atoms.", + "claim_type": "code", + "covers": ["u002"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0/python/CuTeDSL/cutlass/cute/nvgpu/tcgen05/mma.py","revision":"NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"MmaF16BF16Op and CtaGroup definitions","observation":"The Python API constructs tcgen05.MmaF16BF16Op with datatypes, instruction shape, CTA group, operand source, and major modes; neither displayed SM100_MMA symbol exists in the tag."}] + }, + { + "claim_id": "lang-cute-dsl-original-c005", + "claim": "CUTLASS 4.5.0's F16/BF16 CuTe DSL operation represents one-CTA 128x256x16 and two-CTA 256x256x16 tcgen05 MMA shapes with TMEM accumulator fragments.", + "claim_type": "code", + "covers": ["u002"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py","revision":"NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"mma_inst_shape_mnk, MmaF16BF16Op, make_fragment_C, and TMEM pointer replacement","observation":"The 1-CTA example uses (128,256,16), CtaGroup.ONE, and replaces the C fragment pointer with allocated TMEM."},{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_1.py","revision":"same tag","locator":"mma_inst_shape_mnk and CtaGroup.TWO","observation":"The 2-CTA example uses (256,256,16) and CtaGroup.TWO."}], + "challenge": {"alternative":"Those are CTA tile shapes rather than instruction shapes, or the accumulator remains in registers.","check":"Trace mma_inst_shape_mnk into MmaF16BF16Op and make_fragment_C into the retrieved TMEM pointer.","outcome":"The source names and passes them as instruction shapes and explicitly swaps the accumulator tensor to TMEM."} + }, + { + "claim_id": "lang-cute-dsl-original-c006", + "claim": "The displayed make_tmem_ptr/make_tensor/copy fragment is a usable CUTLASS 4.5.0 CuTe DSL TMEM allocation and epilogue example.", + "claim_type": "code", + "covers": ["u003"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py","revision":"NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"TmemAllocator, retrieve_ptr, tmem copy atom/partition, and free","observation":"The release uses a shared holding buffer, TmemAllocator allocation/wait/retrieve/free lifecycle, Ld32x32bOp copy atom, make_tmem_copy, per-thread partitions, and a register tensor. make_tmem_ptr is absent."}] + }, + { + "claim_id": "lang-cute-dsl-original-c007", + "claim": "The displayed SM100_TMA_LOAD_2D and make_tma_copy call form are usable CUTLASS 4.5.0 CuTe DSL Python APIs for global-to-shared TMA.", + "claim_type": "code", + "covers": ["u004"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py","revision":"NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"host_function TMA atom construction and kernel TMA partition/copy","observation":"The tag uses CopyBulkTensorTileG2SOp and make_tiled_tma_atom_A/B, then tma_partition and cute.copy; neither displayed symbol/call signature occurs."}] + }, + { + "claim_id": "lang-cute-dsl-original-c008", + "claim": "The displayed @cute_kernel warp-role skeleton is a safe usable CuTe DSL kernel skeleton with correct TMEM and pipeline ownership.", + "claim_type": "code", + "covers": ["u005"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py","revision":"NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"kernel decorator, role dispatch, PipelineTmaUmma/PipelineUmmaAsync, TmemAllocator, and teardown","observation":"The real warp-specialized kernel uses @cute.kernel, explicit participant groups and states, collective TMEM allocation/free, tiled-copy partitioning, and pipeline tails. The displayed decorator/helpers are absent and deallocation from every epilogue warp is not the source lifecycle."}] + }, + { + "claim_id": "lang-cute-dsl-original-c009", + "claim": "CuTe DSL is Python-native and CUTLASS presents its development model as easier to learn, iterate, debug, and integrate than template-heavy C++.", + "claim_type": "other", + "covers": ["u007"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0/README.md","revision":"NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"CUTLASS DSL overview","observation":"NVIDIA describes Python-native interfaces, faster compile times, framework integration without glue code, and more intuitive metaprogramming than deep C++ expertise."}], + "challenge": {"alternative":"Python is only a host-side generator and users still author the kernel in C++ templates.","check":"Inspect the tagged CuTe DSL examples and decorators.","outcome":"Kernel bodies are authored in Python with @cute.kernel and lowered by the DSL."} + }, + { + "claim_id": "lang-cute-dsl-original-c010", + "claim": "The FlashAttention-4 post reports a peak 1605 TFLOP/s B200 BF16 result.", + "claim_type": "performance", + "covers": ["u008"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://tridao.me/blog/2026/flash4/","revision":"Tri Dao first-party FA4 post, fetched 2026-08-08","locator":"performance summary and attention benchmarks","observation":"The post reports up to 1605 TFLOP/s on B200 BF16 and labels it 71 percent; it is a maximum over evaluated configurations."}], + "challenge": {"alternative":"1605 is a single named shape or an independently reproduced median.","check":"Inspect how the post scopes the number and benchmark plots.","outcome":"It is an author-reported maximum, so only that scope is verified."} + }, + { + "claim_id": "lang-cute-dsl-original-c011", + "claim": "FlashAttention-4's 1605-TFLOP/s number establishes that CuTe DSL has the same performance as handwritten C++.", + "claim_type": "performance", + "covers": ["u008"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://tridao.me/blog/2026/flash4/","revision":"Tri Dao first-party FA4 post, fetched 2026-08-08","locator":"Language/framework and benchmark comparators","observation":"The post's runtime comparisons are with FA2, Triton, Gluon, and cuDNN; it gives no matched handwritten-C++ FA4 comparator. The 1605 number is FA4's own peak, not an equality measurement."}] + }, + { + "claim_id": "lang-cute-dsl-original-c012", + "claim": "CUTLASS 4.5.0 has first-class tcgen05 and TMEM Python APIs, including typed MMA/copy operations and allocation management.", + "claim_type": "compatibility", + "covers": ["u009"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0/python/CuTeDSL/cutlass/cute/nvgpu/tcgen05/mma.py","revision":"NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"MMA operation/trait implementations","observation":"The tag provides typed tcgen05 operations and traits."},{"path":"/tmp/kernelwiki-cutlass-4.5.0/python/CuTeDSL/cutlass/cute/arch/tmem.py","revision":"same tag","locator":"retrieve_tmem_ptr, alloc_tmem, and dealloc_tmem","observation":"The tag exposes TMEM allocation, pointer retrieval, and deallocation APIs."}], + "challenge": {"alternative":"Blackwell Python support is only an example-layer inline-PTX escape hatch.","check":"Inspect the installed package tree rather than examples alone.","outcome":"The architecture and nvgpu packages contain public typed operations and allocation helpers."} + }, + { + "claim_id": "lang-cute-dsl-original-c013", + "claim": "CuTe DSL automatically computes layouts and handles swizzles without explicit user configuration.", + "claim_type": "code", + "covers": ["u010"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py","revision":"NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"make_smem_layout_a/b and allocate_tensor","observation":"The example uses layout algebra and Blackwell helper functions but explicitly supplies MMA tilers, datatypes, major modes, layouts, alignment, and swizzle objects. Handling is typed/composable, not universally automatic."}] + }, + { + "claim_id": "lang-cute-dsl-original-c014", + "claim": "The nine linked local artifact files exist, match their provenance-recorded SHA-256 values, and are untruncated verbatim captures of the stated PR merge revisions; the seven tutorial files have the displayed line counts.", + "claim_type": "other", + "covers": ["u011", "u012", "u013", "u014", "u015", "u016", "u017", "u018", "u019", "u020"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"verify each local file against PROVENANCE.yaml SHA-256 and line count, compare representative files to raw.githubusercontent.com at full merge SHAs, then corrupt one expected hash in memory","environment":"current KernelWiki worktree and network, 2026-08-09","reference":"All nine file records declare mode=verbatim and size_cap_truncated=false; tutorial rows state exact line counts.","result":"All local hashes and seven numeric line counts matched; representative remote hashes at PR 3106/2881/3021 merge commits matched byte-for-byte.","negative_control":"The deliberately altered expected hash was rejected."}], + "challenge": {"alternative":"The files are reconstructed snippets, truncated captures, or point to mutable PR heads.","check":"Use provenance mode/truncation fields, content hashes, full merge SHAs from GitHub's PR API, and remote raw bytes.","outcome":"All recorded properties and sampled remote bytes agree; a mismatched hash fails."} + }, + { + "claim_id": "lang-cute-dsl-original-c015", + "claim": "The seven PR-3106 tutorial files form the stated progression from FP16 software-pipelined baseline through 2-CTA multicast, warp specialization, static persistence, preferred/dynamic clusters, TMA prefetch, and PDL.", + "claim_type": "code", + "covers": ["u012", "u013", "u014", "u015", "u016", "u017", "u018", "u021"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/prs/cutlass/PR-3106/diff.patch","revision":"NVIDIA/cutlass PR 3106 merge 772fbb264e0fe823d62dec10a44a67e1503afdfc","locator":"file docstrings and implementations for fp16_gemm_0.py through fp16_gemm_6.py","observation":"The upstream patch explicitly labels and implements the listed progression; the 4.5.0 official tree retains it under the renamed cute/blackwell/tutorial path."}], + "challenge": {"alternative":"The row labels were inferred from filenames and step 4/6 do not implement dynamic clusters or PDL.","check":"Inspect each file's upstream docstring and the corresponding launch/scheduler code.","outcome":"The descriptions and mechanisms occur in their named files, including preferred/fallback cluster launch and grid dependency control."} + }, + { + "claim_id": "lang-cute-dsl-original-c016", + "claim": "PR-2881's captured dense_gemm_persistent_prefetch.py is a full persistent GEMM with TMA prefetch, and PR-3021's captured clc.py is a full CLC Python binding revision.", + "claim_type": "code", + "covers": ["u019", "u020"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/prs/cutlass/PR-2881/key-files/examples/python/CuTeDSL/blackwell/dense_gemm_persistent_prefetch.py","revision":"NVIDIA/cutlass PR 2881 merge 3f4c086d09bd1dc55defb955862f333893bbb28b","locator":"full file and provenance","observation":"The complete 2147-line captured file implements persistent scheduling and TMA prefetch."},{"path":"artifacts/prs/cutlass/PR-3021/key-files/python/CuTeDSL/cutlass/cute/arch/clc.py","revision":"NVIDIA/cutlass PR 3021 merge 5c42d0f28c9dc728d874fd552492a355036a53e1","locator":"full file and provenance","observation":"The complete 124-line captured module defines the CLC issue/query binding API."}], + "challenge": {"alternative":"The files are merely diffs or excerpts whose omitted context changes their purpose.","check":"Check provenance mode and truncation, then inspect file entry points/imports.","outcome":"Both are recorded as full verbatim upstream files and contain the stated implementations."} + }, + { + "claim_id": "lang-cute-dsl-original-c017", + "claim": "The three related links resolve to the intended local tcgen05 hardware page, verified FlashAttention-4 page, and CUTLASS source card.", + "claim_type": "other", + "covers": ["u022", "u023", "u024"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"resolve the three relative links from wiki/languages/cute-dsl.md and inspect target IDs; test a missing neighbor","environment":"current KernelWiki worktree","reference":"Targets should identify hw-tcgen05-mma, kernel-flash-attention-4, and doc-cutlass-blackwell.","result":"All three files existed with the expected IDs.","negative_control":"wiki/languages/not-cute-dsl.md was absent."}], + "challenge": {"alternative":"A relative path lands on a similarly named but unrelated page.","check":"Resolve paths and inspect frontmatter IDs.","outcome":"Every ID matches the link label and the missing control fails."} + } + ], + "commands": ["git clone --depth 1 --branch v4.5.0 NVIDIA/cutlass and inspect tag e406c186", "GitHub PR API merge-SHA checks", "artifact SHA-256/line-count verification with corrupted-hash control", "local related-link checks with missing-target control"], + "scope": "All 24 baseline units for body hash 4e09e1f2220eed702b5a7c8e6d7318de541e65faf5660835318c6916f5f1625b were checked against NVIDIA/cutlass v4.5.0, first-party FA4 reporting, full PR merge revisions, provenance hashes, and host controls.", + "residual_risk": "No Blackwell GPU or installed CuTe DSL toolchain is available locally. Exact APIs and artifact contents are source-verified; FA4 compile/runtime numbers remain author-reported rather than independently reproduced." +} diff --git a/verification/reviewed/original/lang-ptx.json b/verification/reviewed/original/lang-ptx.json new file mode 100644 index 000000000..edc3ac8c4 --- /dev/null +++ b/verification/reviewed/original/lang-ptx.json @@ -0,0 +1,115 @@ +{ + "path": "wiki/languages/ptx-sm100.md", + "id": "lang-ptx", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "lang-ptx-original-c001", + "claim": "The page is an overview of PTX instruction families relevant to Blackwell SM100 hardware features.", + "claim_type": "other", + "covers": ["u001"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"tcgen05, cluster launch control, cp.async.bulk.tensor, packed cvt, and ld sections","observation":"The archive defines each instruction family represented by the page and gives Blackwell target notes for the architecture-specific forms."}], + "challenge": {"alternative":"These mnemonics belong to CUDA C++ rather than PTX or to Hopper only.","check":"Resolve each mnemonic in the archived PTX ISA and inspect target notes.","outcome":"They are PTX instructions; tcgen05 and CLC forms have Blackwell target scope while TMA also has Hopper forms."} + }, + { + "claim_id": "lang-ptx-original-c002", + "claim": "The displayed tcgen05 alloc/dealloc, unscaled f16 MMA, ld/st 32x32b x1, and shared-to-TMEM cp forms are legal PTX ISA 9.0 instruction shapes with the described operand locations; all allocated TMEM must be deallocated before exit.", + "claim_type": "code", + "covers": ["u002"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"alloc/dealloc grammar and lifecycle","observation":"The shown shared b32 allocation destination, taddr deallocation source, column count, and before-exit rule match."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"same archive","locator":"unscaled floating-point grammar and examples","observation":"The f16 forms use TMEM D, A/B descriptors, idesc, and enable-input-D predicate for both CTA-group modes."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"same archive","locator":"ld/st grammar and repetition table","observation":"32x32b.x1 is valid for warp-collective TMEM-register transfers."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-cp","revision":"same archive","locator":"cp grammar and 128x256b example","observation":"The exact cta_group::1 128x256b shared-descriptor-to-TMEM form is documented."}], + "challenge": {"alternative":"The allocation counts rows, MMA requires block-scale operands, x1 is illegal for 32x32b, or cp copies in the opposite direction.","check":"Compare each exact grammar and operand description rather than adjacent forms.","outcome":"Allocation counts columns, the unscaled f16 grammar accepts the shown five operands, x1 is listed, and cp is SMEM to TMEM."} + }, + { + "claim_id": "lang-ptx-original-c003", + "claim": "tcgen05.fence::after_thread_sync is the critical fence between TMA completion and MMA.", + "claim_type": "mechanism", + "covers": ["u002"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"after_thread_sync semantics and canonical handoff patterns","observation":"The fence orders subsequent asynchronous tcgen05 instructions across a documented execution-ordering handoff; it is not a TMA completion wait."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"same archive","locator":"global-to-shared mbarrier completion","observation":"TMA load completion is tracked through complete_tx on an mbarrier and must be awaited before consuming the destination."}] + }, + { + "claim_id": "lang-ptx-original-c004", + "claim": "PTX ISA 9.0 defines the displayed E2M1x2-to-F16x2 conversion and typed b32 scalar-to-four-byte vector move.", + "claim_type": "code", + "covers": ["u003"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"packed alternate-format conversion","observation":"The cvt mnemonic and packed source/result types match."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov","revision":"same archive","locator":"scalar-to-vector move","observation":"A b32 source can be decomposed into four b8 destinations when types are declared accordingly."}], + "challenge": {"alternative":"The conversion consumes one scalar FP4 or the move cannot write byte-sized destinations.","check":"Apply packed x2 widths and the scalar/vector total-size rule.","outcome":"The source is an 8-bit pair and four byte destinations exactly match b32."} + }, + { + "claim_id": "lang-ptx-original-c005", + "claim": "The mov.b32 byte decomposition is categorically faster than bitwise extraction.", + "claim_type": "performance", + "covers": ["u003"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"mov semantics","observation":"The ISA provides no relative latency or throughput guarantee."},{"url":"https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html","revision":"author post fetched 2026-08-08","locator":"vectorized PTX decode stage","observation":"The author's reported improvement bundles mov, conversions, vectorized decode, and arithmetic; no isolated mov-versus-bitwise trial is reported."}] + }, + { + "claim_id": "lang-ptx-original-c006", + "claim": "The displayed ld.global load forms are legal vector widths and L1 eviction-priority qualifiers in PTX ISA 9.0.", + "claim_type": "code", + "covers": ["u004"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"ld grammar, eviction priorities, and vector examples","observation":"L1::no_allocate and L1::evict_last are legal hints; v2.u64 and v4.u64 move 128 and 256 bits."}], + "challenge": {"alternative":"v4.u64 is unsupported or the qualifiers are L2-only.","check":"Inspect the exact ld vector/type and L1 eviction-priority sets.","outcome":"Both qualifiers are in the L1 set and v4.u64 is an explicit 256-bit example."} + }, + { + "claim_id": "lang-ptx-original-c007", + "claim": "L1::no_allocate necessarily bypasses L1 for one-use data and L1::evict_last keeps reused data resident.", + "claim_type": "performance", + "covers": ["u004"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#cache-operators","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"cache eviction priority semantics","observation":"These qualifiers are performance hints that may not always be respected and do not guarantee bypass, residence, or workload suitability."},{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"author post fetched 2026-08-08","locator":"top-solution observations and failed wider-load experiment","observation":"The policies are source-reported for particular implementations, not universal cache guarantees."}] + }, + { + "claim_id": "lang-ptx-original-c008", + "claim": "clusterlaunchcontrol.try_cancel takes one ClcID output and directly returns a valid tile ID or decline.", + "claim_type": "code", + "covers": ["u005"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"try_cancel grammar and completion","observation":"The asynchronous instruction takes shared response and mbarrier addresses, writes a 16-byte opaque response, and requires query_cancel instructions after completion to determine success and decode coordinates."}] + }, + { + "claim_id": "lang-ptx-original-c009", + "claim": "The first displayed cp.async.bulk.tensor form is a legal 2D global-to-cluster-shared TMA load completed in bytes on an mbarrier.", + "claim_type": "code", + "covers": ["u006"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"global-to-shared::cluster grammar","observation":"The dimensions, state spaces, optional tile mode, tensor-map coordinates, and mbarrier completion operands match."}], + "challenge": {"alternative":"Cluster shared memory requires multicast or the tensor map must be a raw global address.","check":"Compare non-multicast cluster-destination grammar and operand descriptions.","outcome":"A cluster destination is legal without multicast and uses an opaque tensor-map address plus coordinates."} + }, + { + "claim_id": "lang-ptx-original-c010", + "claim": "Appending .multicast is the legal multicast qualifier for the displayed TMA load.", + "claim_type": "code", + "covers": ["u006"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"global-to-shared::cluster multicast grammar and example","observation":"The required qualifier is .multicast::cluster, followed by a 16-bit CTA mask; .multicast alone is not in the grammar."}] + }, + { + "claim_id": "lang-ptx-original-c011", + "claim": "The three related links resolve to the CUDA C++, tcgen05 MMA, and NVFP4 pages named by their labels.", + "claim_type": "other", + "covers": ["u007", "u008", "u009"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"test -f wiki/languages/cuda-cpp.md && test -f wiki/hardware/tcgen05-mma.md && test -f wiki/hardware/nvfp4.md","environment":"current KernelWiki worktree","reference":"Resolve each relative path and compare frontmatter IDs with labels.","result":"All three files exist with IDs lang-cuda-cpp, hw-tcgen05-mma, and hw-nvfp4.","negative_control":"wiki/languages/not-a-real-target.md does not exist."}], + "challenge": {"alternative":"A relative path resolves to an adjacent directory or unrelated ID.","check":"Resolve from wiki/languages and inspect target frontmatter.","outcome":"Each exact target and ID matches; the missing-target control fails."} + } + ], + "commands": ["local path resolution with a missing-target control"], + "scope": "All nine baseline units for body hash 7ef09348591994e947177b5ee3a4a04af46cd573163a183b47b016ff51d1fec9 were checked against CUDA 13.0.2 PTX ISA 9.0, the version-scoped author evidence, and local link controls.", + "residual_risk": "No CUDA assembler or Blackwell GPU is installed locally. Legal forms and ordering semantics come from PTX ISA 9.0; the original page's performance prescriptions were not retained without measurements." +} diff --git a/verification/reviewed/original/lang-triton.json b/verification/reviewed/original/lang-triton.json new file mode 100644 index 000000000..b1648e0e0 --- /dev/null +++ b/verification/reviewed/original/lang-triton.json @@ -0,0 +1,266 @@ +{ + "path": "wiki/languages/triton-blackwell.md", + "id": "lang-triton", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "lang-triton-original-c001", + "claim": "The GitHub v3.6.0 release is commit 7c56a5e40f7fd928dfd5c72902d5def0097db73a and was published on 2026-01-21 UTC.", + "claim_type": "compatibility", + "covers": ["u001", "u046"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url": "https://github.com/triton-lang/triton/releases/tag/v3.6.0", "revision": "v3.6.0, commit 7c56a5e40f7fd928dfd5c72902d5def0097db73a", "locator": "release header", "observation": "The upstream release page identifies v3.6.0, short commit 7c56a5e, and publication at 21 Jan 23:27."}], + "challenge": {"alternative": "The date refers only to a later documentation update or another tag.", "check": "Inspect the signed upstream v3.6.0 release header and resolve the tag with git ls-remote.", "outcome": "Both identify the same v3.6.0 commit and release publication."} + }, + { + "claim_id": "lang-triton-original-c002", + "claim": "Triton 3.6 was the first release with native Blackwell tcgen05/TMEM lowering; through Triton 3.5 the compiler only emitted Hopper WGMMA and kept accumulators in registers.", + "claim_type": "compatibility", + "covers": ["u001", "u011", "u017", "u020", "u022", "u024", "u040", "u041", "u042", "u043", "u046"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + {"url": "https://github.com/triton-lang/triton/blob/819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c/include/triton/Dialect/TritonNvidiaGPU/IR/TritonNvidiaGPUOps.td", "revision": "Triton v3.3.0, commit 819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c", "locator": "TTNG_TCGen5MMAOp, TTNG_TCGen5MMAScaledOp, TTNG_TMEMLoadOp, TTNG_TMEMStoreOp, TTNG_TMEMAllocOp, TTNG_TMEMCopyOp", "observation": "The v3.3.0 dialect defines native TCGen5 MMA and TMEM operations; TMEMCopy explicitly lowers to tcgen05.cp."}, + {"url": "https://github.com/triton-lang/triton/blob/819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c/test/Conversion/tritongpu_to_llvm_blackwell.mlir", "revision": "Triton v3.3.0, commit 819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c", "locator": "Blackwell lowering CHECK directives", "observation": "Pinned tests require tcgen05.mma, tcgen05.commit, TMEM encodings, scaled MMA, and cta_group::2 output."}, + {"url": "https://github.com/triton-lang/triton/compare/v3.2.0...v3.3.0", "revision": "v3.2.0 9641643da6c52000c807b5eeed05edaec4402a67 to v3.3.0 819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c", "locator": "new Blackwell dialect, lowering, allocation, and test files", "observation": "The version boundary adds MMAv5 lowering, tensor-memory allocation, and Blackwell lowering tests; the corresponding symbols are absent at v3.2.0."} + ], + "challenge": {"alternative": "The v3.3 symbols are dead scaffolding while real compilation still emits only WGMMA.", "check": "Inspect v3.3's conversion tests and MMAv5 lowering files, and compare their absence at v3.2.", "outcome": "The pinned v3.3 tests check concrete tcgen05 PTX forms, directly contradicting the claimed 3.6 first-support boundary."} + }, + { + "claim_id": "lang-triton-original-c003", + "claim": "Triton 3.6 incrementally expands the existing Blackwell backend with generic tcgen05.cp, more generic TMEM/ld/st layouts, tcgen05.mma layout generalization, aref warp-specialization work, and initial Gluon multi-CTA/2CTA support.", + "claim_type": "compatibility", + "covers": ["u011", "u012", "u013", "u014", "u015", "u017", "u019", "u022"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url": "https://github.com/triton-lang/triton/releases/tag/v3.6.0", "revision": "Triton v3.6.0, commit 7c56a5e40f7fd928dfd5c72902d5def0097db73a", "locator": "Backend & Compiler lines 221-225; NVIDIA Backend lines 310-340; Gluon lines 351-389", "observation": "The release enumerates generic tcgen05.cp, TMEM layout changes, tcgen05.mma generalization, warp-specialization aref work, initial 2CTA/multi-CTA, num_ctas, and scaled MMA support."}], + "challenge": {"alternative": "These items are a wholesale first implementation rather than refinements.", "check": "Compare the 3.6 release items with the pinned v3.3 and v3.5 dialect/tests and release notes.", "outcome": "Earlier releases already contain TCGen5/TMEM lowering; the listed 3.6 changes are narrower additions and generalizations."} + }, + { + "claim_id": "lang-triton-original-c004", + "claim": "By Triton 3.5, Blackwell paths include persistent-matmul and fused-attention warp specialization, generic tcgen05.ld/st exposure, block-scaled matmul, and an explicit Gluon tcgen05/TMEM tutorial.", + "claim_type": "compatibility", + "covers": ["u006", "u007", "u013", "u014"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url": "https://github.com/triton-lang/triton/releases/tag/v3.5.0", "revision": "Triton v3.5.0, commit c3c476f357f1e9768ea4e45aa5c17528449ab9ef", "locator": "NVIDIA Backend, Hopper/Blackwell Features and Blackwell-Specific", "observation": "The release notes enable warp specialization for persistent matmul and FA and list TMEM, tcgen05_copy, generic tcgen05.ld/st, and tcgen05.commit work."}, + {"url": "https://github.com/triton-lang/triton/blob/c3c476f357f1e9768ea4e45aa5c17528449ab9ef/python/tutorials/gluon/06-tcgen05.py", "revision": "Triton v3.5.0, commit c3c476f357f1e9768ea4e45aa5c17528449ab9ef", "locator": "tutorial introduction, tmem_example_kernel, and small_mma_kernel", "observation": "The tutorial allocates/loads/stores TMEM and invokes tcgen05_mma on Blackwell."}, + {"url": "https://github.com/triton-lang/triton/blob/c3c476f357f1e9768ea4e45aa5c17528449ab9ef/python/tutorials/10-block-scaled-matmul.py", "revision": "Triton v3.5.0, commit c3c476f357f1e9768ea4e45aa5c17528449ab9ef", "locator": "Blackwell block-scaled matmul tutorial", "observation": "The v3.5 tree already ships the tl.dot_scaled Blackwell tutorial."} + ], + "challenge": {"alternative": "Those user-visible examples appear only after the v3.5 tag.", "check": "Open the files and release notes at the exact v3.5 commit.", "outcome": "The tagged v3.5 tree contains the tutorial files and implementation references."} + }, + { + "claim_id": "lang-triton-original-c005", + "claim": "The page's 'when to use' bullets are recommendations; their concrete examples are grounded by actual Triton attention, GatedDeltaNet projection, format-conversion, normalization, dot_scaled, and warp-specialized matmul code, but the page provides no benchmark scope for its CuTe-DSL turnaround comparison.", + "claim_type": "other", + "covers": ["u001", "u002", "u003", "u004", "u005"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path": "artifacts/prs/", "revision": "verbatim bundles at SHAs a1257fd1, 890df95b, 766d225f, 5bdc07d9, and 5638d40f", "locator": "the six files linked in original units u048-u053 and their PROVENANCE.yaml entries", "observation": "Pinned code contains Triton JIT kernels for the named workload categories. No unit or cited benchmark defines or measures 'turnaround' against CuTe DSL."}], + "challenge": {"alternative": "The bullets report a scoped comparative benchmark rather than editorial guidance.", "check": "Inspect the bullets, cited source trail, and performance table columns.", "outcome": "They contain no workload, machine, software revision, timed region, trials, or turnaround measurements."} + }, + { + "claim_id": "lang-triton-original-c006", + "claim": "The 'use alternatives' introduction is editorial guidance rather than a universal benchmark result; the two following bullets give specific downstream cases.", + "claim_type": "other", + "covers": ["u008"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path": "verification/baseline-queue.jsonl", "revision": "immutable baseline body SHA 6b9394376db86dd1090ef2012a0d599ad3ac0703f7df8c9bbf9c61c4c24888f1", "locator": "u008-u010", "observation": "The imperative heading is followed by one SGLang benchmark report and one routing decision, not a universal performance table."}], + "challenge": {"alternative": "The heading itself is an independently quantified claim for all workloads.", "check": "Inspect its grammatical form and the scope of the two supporting units.", "outcome": "It is a recommendation whose factual support is limited to the two named cases."} + }, + { + "claim_id": "lang-triton-original-c007", + "claim": "SGLang PR 5390 reports 10,447.34 total tok/s for its CUTLASS MLA run versus 8,227.35 tok/s for its Triton run (26.98% higher), and PR 21595 changes the SM100 multimodal-attention default from triton_attn to FA4.", + "claim_type": "performance", + "covers": ["u009", "u010", "u032"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url": "https://github.com/sgl-project/sglang/pull/5390", "revision": "merged commit 84810da4ae424d15ec06a3e32a03a839dd50a644", "locator": "Deepseek-R1 Benchmarks, CUTLASS and Triton result blocks", "observation": "The PR records the commands, 3,000 prompts, token lengths, dtype, TP/DP settings, and the two reported throughput totals; this verifies only what that PR reports, not a universal speedup."}, + {"url": "https://github.com/sgl-project/sglang/pull/21595", "revision": "merged commit 87a27682", "locator": "Summary and commit 'Restrict fa4 default to Blackwell SM100 only'", "observation": "The PR explicitly changes the Blackwell datacenter SM100 default from triton_attn to fa4 while retaining other defaults by architecture."} + ], + "challenge": {"alternative": "The percentage compares different metrics or the routing change applies to every GPU.", "check": "Use the total-token-throughput rows for both PR 5390 runs and the architecture-scoped summary of PR 21595.", "outcome": "10447.34/8227.35-1 is 26.98%, and the routing PR explicitly restricts the change to SM100."} + }, + { + "claim_id": "lang-triton-original-c008", + "claim": "The page correctly cautions that plain tl.dot lowering and every tutorial/downstream shape are not proved by the cited downstream artifacts.", + "claim_type": "compatibility", + "covers": ["u016", "u018", "u019", "u025", "u026", "u027"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path": "artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/ops/triton_decode_attention.py", "revision": "upstream SHA a1257fd1, sha256 16cba0b3e98d2829df6d235e3d368ffaa66d72cc2a100ac53f96542e30472796", "locator": "whole file; tl.dot sites around lines 358, 372, and 399", "observation": "The artifact contains tl.dot kernels but no PTX dump, tcgen05 name, TMEM operation, warp_specialize argument, or lowering record."}], + "challenge": {"alternative": "The downstream artifact itself contains inspectable tcgen05 PTX proving every tl.dot path.", "check": "Search the provenance-pinned artifact for tcgen05, TMEM, PTX, and warp_specialize and compare to a positive-control Triton v3.5 tcgen05 tutorial.", "outcome": "The downstream file has none of those lowering witnesses; the positive control does."} + }, + { + "claim_id": "lang-triton-original-c009", + "claim": "vLLM PR 34597 is an SM100-only production-decode anchor that demonstrates its tl.dot kernels lower through Triton 3.6's tcgen05/TMEM path.", + "claim_type": "compatibility", + "covers": ["u017", "u023", "u024", "u026", "u027", "u048", "u049", "u054"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + {"url": "https://github.com/vllm-project/vllm/pull/34597", "revision": "merged commit a1257fd1", "locator": "PR summary and changed files", "observation": "The PR adds FP8 KV-cache support to a generic Triton MLA backend and describes that backend as the only MLA option on SM120; it does not identify an SM100-only path or tcgen05 lowering."}, + {"url": "https://github.com/vllm-project/vllm/blob/a1257fd1/vllm/v1/attention/ops/triton_decode_attention.py", "revision": "commit a1257fd1", "locator": "kernel definitions and tl.dot call sites", "observation": "The code contains tl.dot but no architecture guard, Triton-version requirement, warp_specialize, tcgen05, TMEM, or emitted PTX evidence."} + ], + "challenge": {"alternative": "Architecture metadata outside the code proves it is SM100-only and its tl.dot necessarily emits tcgen05.", "check": "Inspect the primary PR body, exact changed code, and its guards/imports.", "outcome": "The primary source mentions SM120 and supplies no lowering witness, contradicting the SM100-only/demonstration framing."} + }, + { + "claim_id": "lang-triton-original-c010", + "claim": "vLLM PR 34597 genuinely modifies an @triton.jit MLA decode kernel with tl.dot operations and adds FP8 KV-cache support, with two verbatim files pinned locally at merge SHA a1257fd1.", + "claim_type": "code", + "covers": ["u017", "u021", "u023", "u024", "u026", "u027", "u048", "u049", "u054"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path": "artifacts/prs/vllm/PR-34597/", "revision": "upstream SHA a1257fd1; asset_mode verbatim", "locator": "PROVENANCE.yaml; triton_decode_attention.py decorators and tl.dot call sites; triton_mla.py supported_kv_cache_dtypes", "observation": "The provenance lists exact hashes; the code contains Triton kernels and the stated tl.dot operations, while the backend file adds FP8 cache dtypes."}], + "challenge": {"alternative": "The bundle is a derived excerpt or contains only a dispatch gate.", "check": "Verify asset_mode, upstream paths, SHA-256 values, decorators, and dot call sites.", "outcome": "Both files are marked verbatim and contain the implementation described, though they do not prove a particular backend lowering."} + }, + { + "claim_id": "lang-triton-original-c011", + "claim": "vLLM PR 29339 gates triton_kernels MXFP4 use to SM90 and SM100 and changes only the dispatch file mxfp4.py.", + "claim_type": "compatibility", + "covers": ["u028"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url": "https://github.com/vllm-project/vllm/pull/29339", "revision": "merged commit c17610e2", "locator": "PR title, motivation, and changed files", "observation": "The upstream PR limits triton_kernels to SM90/SM100 for MXFP4 and lists only vllm/model_executor/layers/quantization/mxfp4.py."}], + "challenge": {"alternative": "The PR adds a new Triton kernel or proves TCGen5 code generation.", "check": "Inspect its one-file diff and changed-files list.", "outcome": "It is a dispatch gate, not a kernel or lowering proof."} + }, + { + "claim_id": "lang-triton-original-c012", + "claim": "Pinned SGLang artifacts show a GatedDeltaNet fused projection implemented with Triton loads/stores and an extend-attention kernel implemented with Triton dot products.", + "claim_type": "code", + "covers": ["u029", "u030", "u031", "u052", "u053"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [ + {"path": "artifacts/prs/sglang/PR-21019/key-files/python/sglang/jit_kernel/triton/gdn_fused_proj.py", "revision": "upstream SHA 5bdc07d9, sha256 c4ef85167eb11065721e76dd5d63d6f346f309f84822578dd4a3874249acef89", "locator": "@triton.jit kernels and tl.load/tl.store sites", "observation": "The verbatim file implements fused QKVZBA split/reshape/cat projection through Triton memory operations and contains no tl.dot."}, + {"path": "artifacts/prs/sglang/PR-22079/key-files/python/sglang/srt/layers/attention/triton_ops/extend_attention.py", "revision": "upstream SHA 5638d40f, sha256 d27789c32e180d725ed867f4f6cd05b09a3c47bfe37794ff60e150d000573edd", "locator": "@triton.jit extend-attention kernels and tl.dot call sites", "observation": "The verbatim file performs QK, QPE/KPE, and probability/value dot products."} + ], + "challenge": {"alternative": "The page swapped which artifact contains matrix products.", "check": "Search each exact file for decorators, tl.dot, tl.load, and tl.store.", "outcome": "The GDN projection is load/store rearrangement; extend_attention contains the dot products, matching the narrowed claim."} + }, + { + "claim_id": "lang-triton-original-c013", + "claim": "The local downstream artifacts do not contain an explicit tcgen05 PTX dump or a warp-specialized descriptor/TMA lowering witness.", + "claim_type": "code", + "covers": ["u018", "u019", "u026", "u027"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command": "rg -n -i 'tcgen05|tmem|warp_specialize' artifacts/prs/vllm/PR-34597 artifacts/prs/sglang/PR-22079 artifacts/prs/sglang/PR-21019", "environment": "GNU ripgrep over provenance-pinned local artifacts", "reference": "A lowering witness would contain a TCGen5/TMEM/PTX term or explicit warp-specialize surface.", "result": "No matching lowering witness was found in the three cited downstream bundles.", "negative_control": "The same terms match Triton v3.5.0's python/tutorials/gluon/06-tcgen05.py and compiler sources."}], + "challenge": {"alternative": "The absence result is caused by a broken search pattern.", "check": "Run the identical search against the pinned upstream positive control.", "outcome": "The positive control matches, so the search discriminates the downstream artifacts."} + }, + { + "claim_id": "lang-triton-original-c014", + "claim": "The live FlashInfer-Bench author leaderboard displays 0.628x/73.1% for gemini-2.5-pro, 0.467x/92.3% for gpt-5-2025-08-07, and 0.456x/73.1% for claude-opus-4-1-20250805 across 660 workloads.", + "claim_type": "performance", + "covers": ["u033", "u034", "u035", "u036"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url": "https://bench.flashinfer.ai/", "revision": "live leaderboard retrieved 2026-08-08", "locator": "Leaderboard, Author / Avg Speedup / % Resolved rows 1-3", "observation": "The three exact values and resolution percentages appear in the named rows; each percentage is labeled over 660 workloads."}], + "challenge": {"alternative": "The values belong to model workloads rather than leaderboard authors or adjacent columns.", "check": "Read the author row and its two labeled columns without flattening neighboring rows.", "outcome": "Each value maps to the same author row and the exact Avg Speedup and % Resolved columns."} + }, + { + "claim_id": "lang-triton-original-c015", + "claim": "Those FlashInfer-Bench rows are a preserved Triton-3.5-era snapshot that specifically measures AI-generated Triton kernels, and no Triton 3.6 rerun is available.", + "claim_type": "performance", + "covers": ["u033", "u037"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url": "https://bench.flashinfer.ai/", "revision": "live leaderboard retrieved 2026-08-08", "locator": "Leaderboard rows and labels", "observation": "The current live page still shows the values but labels them only by author, speedup, resolved percentage, and workload count; it supplies no Triton version, historical-snapshot date, or Triton-only language filter."}], + "challenge": {"alternative": "Hidden table metadata pins these exact rows to Triton 3.5 and marks a missing 3.6 rerun.", "check": "Inspect the leaderboard labels and linked public page for version/language scope.", "outcome": "The source exposes neither asserted scope, while the identical values are current, so the historical Triton-version framing is false."} + }, + { + "claim_id": "lang-triton-original-c016", + "claim": "The gated_delta_net_decode block is a usable Triton kernel implementing its stated recurrent-state update.", + "claim_type": "code", + "covers": ["u038"], + "verdict": "refuted", + "oracle": "host-experiment", + "evidence": [{"command": "conda run -n base python -c ", "environment": "Conda base, Python 3.13.14, host AST analysis", "reference": "A self-contained kernel must define every loaded local name or receive it as an argument.", "result": "gated_delta_net_decode loads undefined name offsets; its state/output address expressions are also inconsistent between load and store.", "negative_control": "A well-scoped control function taking offsets and d as arguments reports no undefined names."}], + "challenge": {"alternative": "offsets is an implicit Triton builtin.", "check": "Compare the function arguments and Triton language namespace with the AST load set.", "outcome": "offsets is neither an argument, assignment, nor tl builtin, so the displayed function cannot compile as written."} + }, + { + "claim_id": "lang-triton-original-c017", + "claim": "The sparse_attention_fwd block is a usable Triton sparse-attention kernel.", + "claim_type": "code", + "covers": ["u039"], + "verdict": "refuted", + "oracle": "host-experiment", + "evidence": [{"command": "conda run -n base python -c ", "environment": "Conda base, Python 3.13.14, host AST analysis", "reference": "A self-contained kernel must define every loaded local name or receive it as an argument and must write its promised output.", "result": "sparse_attention_fwd loads undefined d and offsets, stops at an unfinished softmax comment, and never stores Output.", "negative_control": "A well-scoped control function taking offsets and d as arguments reports no undefined names."}], + "challenge": {"alternative": "d and offsets are compiler-provided names and the omitted store is outside the excerpt by design.", "check": "Inspect the complete fenced block, signature, and loaded names.", "outcome": "The block closes without defining the names or producing Output, so it is incomplete executable-looking code."} + }, + { + "claim_id": "lang-triton-original-c018", + "claim": "CPU launch overhead can materially affect short-kernel workflows; CUDA Graphs reduce repeated CPU launch costs, and vLLM at commit a1257fd1 defaults to FULL_AND_PIECEWISE graphs and has GDN decode graph support.", + "claim_type": "performance", + "covers": ["u044", "u045", "u046"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + {"url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cuda-graphs.html", "revision": "CUDA Programming Guide 13.2", "locator": "4.2 CUDA Graphs introduction", "observation": "The guide states that per-kernel host setup is significant for short kernels and graphs reduce CPU launch costs by preparing work in advance."}, + {"url": "https://github.com/vllm-project/vllm/blob/a1257fd1/vllm/config/compilation.py", "revision": "vLLM commit a1257fd1", "locator": "CUDAGraphMode and CompilationConfig.cudagraph_mode documentation", "observation": "FULL_AND_PIECEWISE is documented as the v1 default and most performant mode for most models."}, + {"url": "https://github.com/vllm-project/vllm/blob/a1257fd1/vllm/v1/attention/backends/gdn_attn.py", "revision": "vLLM commit a1257fd1", "locator": "GDNAttentionMetadataBuilder and build_for_cudagraph_capture", "observation": "The GDN backend advertises uniform-batch graph support and implements decode-only full graph capture."} + ], + "challenge": {"alternative": "CUDA Graphs are unrelated to launch overhead or GDN opts out of vLLM's default graph path.", "check": "Inspect the CUDA guide plus the pinned compilation default and GDN metadata builder.", "outcome": "The sources directly connect graphs to CPU launch cost and explicitly implement GDN decode graph capture."} + }, + { + "claim_id": "lang-triton-original-c019", + "claim": "All six artifact links in the examples table resolve to provenance-pinned verbatim upstream files, and their file-purpose descriptions match the code at the recorded revisions.", + "claim_type": "other", + "covers": ["u047", "u048", "u049", "u050", "u051", "u052", "u053", "u054"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command": "conda run -n base python -c ", "environment": "Conda base, Python 3.13.14, repository filesystem", "reference": "Each relative link must resolve and its hash must match a verbatim provenance entry.", "result": "All six paths exist; their five bundles declare asset_mode verbatim and exact upstream SHAs/hashes. Code inspection finds the described MLA, FP8-cache, format-conversion, normalization, GDN projection, and extend-attention roles.", "negative_control": "artifacts/prs/vllm/PR-00000/key-files/missing.py does not resolve."}], + "challenge": {"alternative": "At least one link is a summary, derived asset, or stale target.", "check": "Resolve paths, read bundle asset_mode/upstream_sha, verify hashes, and inspect the referenced symbols.", "outcome": "Every linked file is present and provenance marks it verbatim; the broken control fails as expected."} + }, + { + "claim_id": "lang-triton-original-c020", + "claim": "data/triton-universe.yaml enumerates exactly 42 pull requests.", + "claim_type": "numeric", + "covers": ["u055"], + "verdict": "refuted", + "oracle": "host-experiment", + "evidence": [{"command": "conda run -n base python -c ", "environment": "Conda base, Python 3.13.14, PyYAML", "reference": "The declared total and actual list length should equal the wiki's 42.", "result": "The file declares total=267 and contains 267 PR entries: 117 captured and 150 skipped.", "negative_control": "The equality check actual_count == 42 evaluates false while actual_count == declared total evaluates true."}], + "challenge": {"alternative": "42 is a filtered sublist explicitly identified by the page.", "check": "Inspect the file's top-level lane, total, captured, skipped, and prs fields.", "outcome": "The page says full universe, while the canonical full list has 267 entries."} + }, + { + "claim_id": "lang-triton-original-c021", + "claim": "Entries marked captured:false in data/triton-universe.yaml are intentionally not shipped; the five table bundles are captured:true.", + "claim_type": "other", + "covers": ["u047", "u055"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command": "conda run -n base python -c ", "environment": "Conda base, Python 3.13.14, PyYAML", "reference": "Linked bundles should be captured and skipped entries should lack shipped bundles by policy.", "result": "All five linked PR IDs are captured:true; the universe has 150 captured:false entries and the table files all exist.", "negative_control": "A nonexistent PR ID has no universe entry or artifact path."}], + "challenge": {"alternative": "The linked artifacts violate the capture ledger.", "check": "Join their source PR IDs against data/triton-universe.yaml.", "outcome": "Every linked bundle is marked captured:true."} + }, + { + "claim_id": "lang-triton-original-c022", + "claim": "PR 34597 and PR 29339 are post-refresh while PR 22079 and PR 21019 are pre-refresh according to data/refresh-cutoff.yaml::previous_pages_manifest.", + "claim_type": "semantics", + "covers": ["u017", "u023", "u028", "u029", "u030", "u031", "u054"], + "verdict": "refuted", + "oracle": "host-experiment", + "evidence": [{"command": "conda run -n base python -c ", "environment": "Conda base, Python 3.13.14, PyYAML", "reference": "The asserted manifest should exist and distinguish the named entries relative to the active cutoff.", "result": "The file contains only cutoff_date=2026-05-20 and no previous_pages_manifest; all four PR dates precede the active cutoff.", "negative_control": "A synthetic 2026-05-21 date is correctly classified after the cutoff."}], + "challenge": {"alternative": "The manifest exists under another key and still supports the labels.", "check": "Parse every top-level key and compare the named PR dates to the sole cutoff.", "outcome": "There is no manifest field and no named PR is post-cutoff, so the administrative classifications are stale."} + }, + { + "claim_id": "lang-triton-original-c023", + "claim": "The page's version_sensitive pointer resolves to a registry entry that applies back to this page.", + "claim_type": "other", + "covers": ["u056"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command": "conda run -n base python scripts/validate.py", "environment": "Conda base, Python 3.13.14", "reference": "The validator requires the pointer ID to resolve and the registry entry to include the page in applies_to.", "result": "Validation passes with vs-triton-3.6-blackwell-tcgen05 resolving to an entry that lists wiki/languages/triton-blackwell.md.", "negative_control": "The registry validator's unresolved-pointer rule rejects an unknown ID in its test path."}], + "challenge": {"alternative": "The pointer is dangling even though the prose mentions a registry.", "check": "Resolve it through the repository validator and inspect applies_to.", "outcome": "The structural pointer is valid; its semantic lower bound is separately refuted by c002."} + } + ], + "commands": [ + "git compare and grep across exact Triton v3.2.0, v3.3.0, v3.4.0, v3.5.0, and v3.6.0 tags", + "conda run -n base python AST audit of both wiki code blocks with a well-scoped negative control", + "conda run -n base python YAML/link/provenance/count audits with broken controls", + "conda run -n base python scripts/validate.py" + ], + "scope": "Original body hash 6b9394376db86dd1090ef2012a0d599ad3ac0703f7df8c9bbf9c61c4c24888f; all 56 units. Version boundary checked from v3.2.0 through v3.7.1; downstream claims pinned to merge SHAs; leaderboard retrieved 2026-08-08.", + "residual_risk": "No GPU was available, so no new PTX or performance run was attempted. Claims requiring such proof were either limited to what pinned sources report or refuted when the page asserted the absent proof." +} diff --git a/verification/reviewed/original/migration-register-to-tmem.json b/verification/reviewed/original/migration-register-to-tmem.json new file mode 100644 index 000000000..32c925055 --- /dev/null +++ b/verification/reviewed/original/migration-register-to-tmem.json @@ -0,0 +1,263 @@ +{ + "path": "wiki/migration/register-to-tmem.md", + "id": "migration-register-to-tmem", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "migration-register-to-tmem-original-c001", + "claim": "For Hopper wgmma.mma_async.m64n256k16 with an f32 accumulator, each of the 128 warpgroup threads holds N/2 = 128 f32 accumulator registers; high per-thread register use can constrain occupancy.", + "claim_type": "mechanism", + "covers": ["u001"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"m64nNk16 accumulator D fragment table","observation":"The f32 accumulator is N/2 registers per thread, hence 128 when N=256."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/hopper-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2 Hopper Tuning Guide","locator":"occupancy limits","observation":"The guide identifies register usage as one factor limiting active blocks/warps."}], + "challenge": {"alternative":"The 128 values are distributed across the whole warpgroup rather than held by every thread.","check":"Read the fragment table's per-thread vector expression and substitute N=256.","outcome":"The table explicitly describes each thread's fragment as N/2 f32 registers."} + }, + { + "claim_id": "migration-register-to-tmem-original-c002", + "claim": "On sm_100a, tcgen05 MMA writes D in Tensor Memory, whose CTA-visible structure is 128 lanes by 512 columns of 32-bit cells (256 KiB total addressable structure).", + "claim_type": "mechanism", + "covers": ["u002"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"Tensor Memory organization and tcgen05 MMA D operand","observation":"The ISA defines 512 columns, 128 rows/lanes, 32-bit cells and TMEM destination addresses for D."}], + "challenge": {"alternative":"TMEM is merely a name for accumulator registers or has 512 bytes rather than columns.","check":"Inspect the address-space organization and MMA destination operand type.","outcome":"It is a distinct dynamically allocated space and column allocation covers all 128 lanes."} + }, + { + "claim_id": "migration-register-to-tmem-original-c003", + "claim": "Putting the resident MMA accumulator in TMEM eliminates accumulator register pressure entirely and necessarily frees the full register file for epilogues, enables larger tiles, and removes registers as an occupancy bottleneck.", + "claim_type": "performance", + "covers": ["u002"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2 Blackwell Tuning Guide","locator":"occupancy limits","observation":"Blackwell still has a 64K-register file and 255-register per-thread limit; all nonresident accumulator fragments, addresses, loop state, and epilogue batches still consume registers. Occupancy remains resource- and launch-specific."}] + }, + { + "claim_id": "migration-register-to-tmem-original-c004", + "claim": "An SM90 SM has 65,536 32-bit registers (256 KiB).", + "claim_type": "compatibility", + "covers": ["u003"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/hopper-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2 Hopper Tuning Guide","locator":"Streaming Multiprocessor occupancy bullets","observation":"The official guide states 64K 32-bit registers per SM; multiplying by four bytes gives 256 KiB."}], + "challenge": {"alternative":"64K denotes bytes or registers per processing block.","check":"Read the unit and scope in the occupancy bullet.","outcome":"It explicitly says 32-bit registers per SM."} + }, + { + "claim_id": "migration-register-to-tmem-original-c005", + "claim": "The detailed 174-register Hopper budget and its conclusion of exactly two warpgroups/equivalently two CTAs per SM are generally valid for m64n256k16 BF16.", + "claim_type": "code", + "covers": ["u004"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"A and D register fragment tables","observation":"The register-A BF16 fragment is four f16x2 registers, not the claimed 16; only the 128-register f32 D count is ISA-defined."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator","revision":"CUDA 13.0.2 Programming Guide","locator":"occupancy calculation","observation":"Active blocks depend on block size, allocated registers, shared memory, barriers, and allocation granularity; a warpgroup is not inherently one CTA."}] + }, + { + "claim_id": "migration-register-to-tmem-original-c006", + "claim": "Changing the example to k32 and double-buffering necessarily yields 300 registers per thread, 38,400 per warpgroup, and exactly one CTA per SM.", + "claim_type": "code", + "covers": ["u005", "u006"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/hopper-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2 Hopper Tuning Guide","locator":"maximum registers per thread and occupancy","observation":"The architectural maximum is 255 registers per thread, and actual allocation/spilling is compiler- and kernel-specific."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16","revision":"PTX ISA 9.0","locator":"D fragment table","observation":"Accumulator register count depends on N and dtype, not K; double buffering is an implementation choice rather than an m64n256k32 consequence."}] + }, + { + "claim_id": "migration-register-to-tmem-original-c007", + "claim": "Compiler register spills use per-thread local memory, which resides in device memory and may be cached in L2 and, depending on architecture, L1; spills can materially hurt performance.", + "claim_type": "mechanism", + "covers": ["u007"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#local-memory","revision":"CUDA 13.0.2 Programming Guide","locator":"Local Memory","observation":"Automatic variables may be placed in local memory, which is off-chip device memory and follows documented cache behavior; the guide warns that access is expensive."}], + "challenge": {"alternative":"Local memory means on-chip storage private to the thread and therefore register-speed.","check":"Inspect the programming guide's physical placement and caching description.","outcome":"Local is a scope/name, not on-chip placement; accesses use device-memory paths."} + }, + { + "claim_id": "migration-register-to-tmem-original-c008", + "claim": "Register, L1-local, and L2-spill accesses universally cost about 4, 30, and 200 cycles, so each spilled accumulator adds exactly 200 times K_tiles cycles.", + "claim_type": "code", + "covers": ["u008"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-best-practices-guide/index.html#local-memory","revision":"CUDA 13.0.2 Best Practices Guide","locator":"Local memory performance","observation":"Official guidance does not define these fixed latencies; caching, dependencies, coalescing, occupancy, compiler placement, and overlap determine cost. One spill access does not imply a serialized 200-cycle addition per K tile."}] + }, + { + "claim_id": "migration-register-to-tmem-original-c009", + "claim": "The 29-register Blackwell budget, 7,424-register CTA total, eight-CTA register limit, two-CTA TMEM limit, and categorical TMEM-bound occupancy conclusion are generally valid for m128n256k16.", + "claim_type": "code", + "covers": ["u009", "u010"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2 Blackwell Tuning Guide","locator":"occupancy limits","observation":"The guide requires occupancy calculation from actual block size, compiled register allocation, shared memory, cluster shape and other resources; none of the page's per-thread category counts are architectural constants."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory-allocation","revision":"PTX ISA 9.0","locator":"dynamic column allocation","observation":"TMEM allocations are dynamic in power-of-two column units; the ISA does not derive the page's universal CTA residency limit from an MMA N dimension."}] + }, + { + "claim_id": "migration-register-to-tmem-original-c010", + "claim": "The Hopper pseudocode captures the central register lifecycle: an f32 m64n256 accumulator is a 128-register fragment per thread, wgmma updates register D asynchronously, commit/wait completes a group, and the registers are then directly available to the issuing threads.", + "claim_type": "code", + "covers": ["u011"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-multiply-accumulate-instructions-wgmma-mma","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"wgmma register fragments, mma_async, commit_group, and wait_group","observation":"The page's high-level register-D lifecycle and 128-register count agree, although helper names are explicitly pseudocode."}], + "challenge": {"alternative":"wgmma writes D to shared memory or TMEM and only later transfers it to registers.","check":"Inspect the D operand class and fragment table.","outcome":"D is an explicit per-thread register vector in the wgmma instruction."} + }, + { + "claim_id": "migration-register-to-tmem-original-c011", + "claim": "The Hopper pseudocode proves its arrays remain in registers and that no registers remain for epilogue temporaries.", + "claim_type": "code", + "covers": ["u011"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#local-memory","revision":"CUDA 13.0.2 Programming Guide","locator":"automatic arrays and local memory","observation":"A C++ array's placement is compiler-dependent; dynamic indexing or pressure can place it in local memory. Register headroom also depends on the compiled kernel, not the illustrative tally."}] + }, + { + "claim_id": "migration-register-to-tmem-original-c012", + "claim": "The displayed Blackwell CUDA/PTX block is a correct accumulator lifecycle: lane 0 allocates/deallocates, 32x1b stores/loads initialize/read f32, a literal enables input D, no completion is needed, and a fence plus syncthreads makes results readable.", + "claim_type": "code", + "covers": ["u012"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"alloc/dealloc issue granularity","observation":"cta_group::1 allocation and deallocation require all lanes of one warp, not lane 0."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"same archive","locator":"ld/st shape grammar and completion","observation":"The displayed 32x1b forms are not legal; ld/st are warp-collective asynchronous operations with matching waits."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"same archive","locator":"f16 grammar, predicate and completion semantics","observation":"enable-input-D is a predicate, and MMA completion must be tracked before readers/reuse; fence is ordering, not completion."}] + }, + { + "claim_id": "migration-register-to-tmem-original-c013", + "claim": "With TMEM, tile size is no longer constrained by registers; Hopper m128n256 necessarily spills, while Blackwell m128n256/m128n512/m256n256 have the stated universal fit and occupancy properties.", + "claim_type": "code", + "covers": ["u013", "u014"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2 Blackwell Tuning Guide","locator":"occupancy resources","observation":"Registers remain a kernel resource and practical tile choice depends on all resources and measured workload behavior."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"PTX ISA 9.0","locator":"supported matrix shapes","observation":"The common single instruction has N up to 256; an m128n512 output would require an implementation decomposition, not the claimed one-instruction fit."}] + }, + { + "claim_id": "migration-register-to-tmem-original-c014", + "claim": "A Blackwell epilogue has the full register file available, and the displayed rich epilogue necessarily uses about 24 temporaries without spilling whereas the Hopper equivalent often spills.", + "claim_type": "code", + "covers": ["u015", "u016"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2 Blackwell Tuning Guide","locator":"register limits","observation":"Blackwell has 64K registers/SM and a 255-register/thread limit shared by all live variables; TMEM residency reduces one source of pressure but does not reserve the full file for an epilogue or prove allocation/spilling for pseudocode."}] + }, + { + "claim_id": "migration-register-to-tmem-original-c015", + "claim": "Overlapping one tile's epilogue with another tile's MMA is impossible with register accumulators, and the two-allocation sketch is a complete safe tcgen05 overlap protocol.", + "claim_type": "code", + "covers": ["u017", "u018"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/abs/2407.08608","revision":"FlashAttention-3 paper, arXiv 2407.08608","locator":"Hopper asynchrony and warp-specialization contributions","observation":"FA3 overlaps Hopper tensor-core work with non-matmul work via warp specialization/interleaving despite register accumulators, disproving impossibility."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit","revision":"PTX ISA 9.0","locator":"commit, waits, lifetimes, and allocation rules","observation":"Safe overlap requires explicit completion and no conflicting read/write/reuse; the sketch's opaque fence_and_sync and helper calls do not establish those edges."}] + }, + { + "claim_id": "migration-register-to-tmem-original-c016", + "claim": "FlashAttention-4 is a concrete Blackwell case using TMEM-resident score/output accumulators and scheduling non-matmul work around asynchronous tensor-core work.", + "claim_type": "mechanism", + "covers": ["u019"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/abs/2603.05451v1","revision":"FlashAttention-4 paper v1","locator":"forward kernel architecture and ping-pong scheduling","observation":"The paper places attention accumulators in TMEM and assigns MMA, softmax, and correction roles to overlap asymmetric units."}], + "challenge": {"alternative":"FA4 keeps both GEMM accumulators resident in ordinary registers as on Hopper.","check":"Inspect the paper's thread-memory mapping and forward pipeline.","outcome":"Its Blackwell design explicitly uses TMEM for the score/output accumulator storage."} + }, + { + "claim_id": "migration-register-to-tmem-original-c017", + "claim": "The Hopper and Blackwell FlashAttention register budgets of about 254 and 20 registers per thread, two- and 236-register margins, and resulting spill/no-spill conclusions are measured or source-defined.", + "claim_type": "code", + "covers": ["u020", "u021"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/abs/2603.05451v1","revision":"FlashAttention-4 paper v1","locator":"architecture and evaluation","observation":"The paper explains TMEM and warp roles but does not publish these per-thread category totals or margins; actual register allocation is variant/compiler-specific and the claimed arithmetic omits live epilogue/pipeline state."}] + }, + { + "claim_id": "migration-register-to-tmem-original-c018", + "claim": "The first-party FA4 report uses software-selected polynomial exp2 work on CUDA cores and reports up to 1605 TFLOP/s on B200 BF16 at a labeled 71 percent.", + "claim_type": "performance", + "covers": ["u022"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://tridao.me/blog/2026/flash4/","revision":"Tri Dao first-party FA4 post, fetched 2026-08-08","locator":"software exponential and performance sections","observation":"The author describes selective polynomial/FMA exp2 computation and reports up to 1605 TFLOP/s on B200 BF16, labeled 71 percent."}], + "challenge": {"alternative":"All exponentials use the polynomial path or 1605 is an independently reproduced fixed-shape median.","check":"Read the selection description and maximum-performance wording.","outcome":"Only a selected fraction uses software exp, and the number is an author-reported maximum; the page's wording is correspondingly source-scoped only in this claim."} + }, + { + "claim_id": "migration-register-to-tmem-original-c019", + "claim": "TMEM is not directly addressable by ordinary CUDA expressions; tcgen05 data-movement instructions transfer values between TMEM and registers for scalar/epilogue computation.", + "claim_type": "code", + "covers": ["u023"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"TMEM addressing and tcgen05.ld/st","observation":"TMEM uses a special 32-bit lane/column address and is accessed by the tcgen05 instruction family; ld transfers TMEM data to per-thread registers."}], + "challenge": {"alternative":"A TMEM address can be cast to a CUDA pointer and dereferenced like shared/global memory.","check":"Inspect defined state spaces and the instruction-only addressing model.","outcome":"No CUDA/PTX generic load/store state space maps to TMEM; dedicated tcgen05 movement is required."} + }, + { + "claim_id": "migration-register-to-tmem-original-c020", + "claim": "Every fine-grained TMEM load costs 420 cycles while an x4 vector form fetches four adjacent values in one 420-cycle access, so the displayed loop is a valid optimization.", + "claim_type": "code", + "covers": ["u024"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"ld shape/repetition/register mapping","observation":"The instruction has collective shape- and repetition-specific mapping; it is not a scalar tmem_load_f32 API, and x4 does not establish the page's adjacent-scalar abstraction or latency."},{"url":"https://arxiv.org/abs/2512.02189","revision":"Blackwell microbenchmark paper v1","locator":"memory hierarchy latency experiment","observation":"The cited approximately 420-cycle figure concerns a measured cache-miss path in that methodology, not a normative per-tcgen05.ld latency for every shape."}] + }, + { + "claim_id": "migration-register-to-tmem-original-c021", + "claim": "A Blackwell migration should use at minimum m128n256 for one-SM or m256n256 for two-SM MMA, and those choices carry no spilling risk.", + "claim_type": "code", + "covers": ["u025"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"shape constraints","observation":"The ISA supports shape choices through descriptors but does not prescribe a performance minimum."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy","revision":"Blackwell Tuning Guide","locator":"occupancy guidance","observation":"Tile selection and spills depend on the full kernel, shape, resources and measurements; TMEM does not guarantee spill-free code."}] + }, + { + "claim_id": "migration-register-to-tmem-original-c022", + "claim": "Every TMEM allocation must be explicitly deallocated before kernel exit, including persistent-kernel termination paths.", + "claim_type": "code", + "covers": ["u026"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"allocation/deallocation description","observation":"The ISA requires all allocated Tensor Memory to be deallocated with tcgen05.dealloc before the kernel exits."}], + "challenge": {"alternative":"Kernel exit implicitly releases TMEM as it does local registers.","check":"Read the explicit lifecycle requirement in the allocation section.","outcome":"The ISA states the opposite: explicit deallocation is mandatory."} + }, + { + "claim_id": "migration-register-to-tmem-original-c023", + "claim": "Omitting deallocation merely leaks TMEM to the next CTA, which then cannot allocate, and making one source path reach a helper is sufficient cleanup.", + "claim_type": "mechanism", + "covers": ["u026"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"deallocation and collective issue rules","observation":"The ISA specifies a must-deallocate contract and collective matching operation; it does not define omission as a recoverable cross-CTA leak. Every actual exit path and every lane of the designated warp must satisfy the protocol."}] + }, + { + "claim_id": "migration-register-to-tmem-original-c024", + "claim": "The summary correctly contrasts register-resident wgmma D with TMEM-resident tcgen05 D, direct register result use with collective tcgen05.ld readback, implicit register lifetime with explicit TMEM deallocation, and the fact that double-buffered resident accumulators consume two register fragments versus two TMEM regions.", + "claim_type": "mechanism", + "covers": ["u027", "u029", "u030", "u031", "u032", "u034"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"wgmma D register fragments","observation":"Hopper D is a per-thread register vector."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"same archive","locator":"tcgen05 D, ld, allocation/deallocation","observation":"Blackwell D is addressed in TMEM, read through tcgen05.ld, and explicitly deallocated; separate live buffers consume separate storage regions."}], + "challenge": {"alternative":"tcgen05 D is still an implicit register vector or TMEM disappears automatically on scope exit.","check":"Compare operand classes and lifecycle sections for both instruction families.","outcome":"The architectures expose distinct storage and cleanup contracts exactly as summarized in these rows."} + }, + { + "claim_id": "migration-register-to-tmem-original-c025", + "claim": "A migrated accumulator must be zero-initialized with a tcgen05.st zero pattern before its first MMA.", + "claim_type": "code", + "covers": ["u028"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"enable-input-D semantics","observation":"When enable-input-D is false, the operation computes D=A*B without reading prior D, so an explicit zero store is not universally required."}] + }, + { + "claim_id": "migration-register-to-tmem-original-c026", + "claim": "TMEM gives every epilogue ample access to the full register file with only minimal spill risk.", + "claim_type": "performance", + "covers": ["u033"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2 Blackwell Tuning Guide","locator":"register file and per-thread limits","observation":"All other live values and loaded TMEM batches consume the same finite register file; the spill outcome requires compiled-resource evidence for a concrete kernel."}] + }, + { + "claim_id": "migration-register-to-tmem-original-c027", + "claim": "m64n256 is Hopper's universal maximum practical tile, while m128n256 or m256n256 are Blackwell's universal practical maxima determined only by TMEM columns.", + "claim_type": "performance", + "covers": ["u035"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-multiply-accumulate-instructions-wgmma-mma","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"instruction shapes","observation":"Instruction shape is not a complete CTA tile or practical-kernel maximum; kernels compose multiple instructions."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy","revision":"Blackwell Tuning Guide","locator":"resource limits","observation":"Practical tiles depend on workload and all resources rather than a universal architecture-only maximum."}] + } + ], + "commands": ["direct archived PTX/tuning-guide inspection", "FA3/FA4 primary-paper comparison", "host arithmetic checks for N/2 and TMEM byte capacity"], + "scope": "All 35 baseline units for body hash d18a8529e0c9cab4d1711c15938892a04771dbccb5548fc61e5298b0fd6c0ed4 were checked against CUDA 13.0.2 PTX ISA 9.0, archived Hopper/Blackwell guidance, and author-primary FA3/FA4 papers.", + "residual_risk": "No Hopper/Blackwell compiler or GPU is installed locally. Architectural storage and instruction contracts are source-verified; no unsourced register allocation, latency, occupancy, or performance estimate is accepted." +} diff --git a/verification/reviewed/original/migration-wgmma-to-tcgen05.json b/verification/reviewed/original/migration-wgmma-to-tcgen05.json new file mode 100644 index 000000000..b2688149e --- /dev/null +++ b/verification/reviewed/original/migration-wgmma-to-tcgen05.json @@ -0,0 +1,369 @@ +{ + "path": "wiki/migration/wgmma-to-tcgen05.md", + "id": "migration-wgmma-to-tcgen05", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "migration-wgmma-to-tcgen05-original-c001", + "claim": "Porting an SM90 WGMMA kernel to SM100 tcgen05 changes MMA issue from a warpgroup to one thread and moves the D accumulator from per-thread registers to TMEM.", + "claim_type": "compatibility", + "covers": ["u001", "u002", "u003"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-matrix-multiply-and-accumulate-operations", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "WGMMA execution model; tcgen05 issue granularity and MMA overview", + "observation": "WGMMA is a warpgroup collective with register D; tcgen05 MMA has single-thread issue and TMEM D." + } + ], + "challenge": { + "alternative": "tcgen05 is only a renamed WGMMA encoding and retains warpgroup-issued register fragments.", + "check": "Compare issue granularity and D operand locations in the two pinned ISA sections.", + "outcome": "Both properties change explicitly, discriminating the programming models." + } + }, + { + "claim_id": "migration-wgmma-to-tcgen05-original-c002", + "claim": "The migration universally changes ldmatrix-plus-register A into direct-SMEM A and replaces WGMMA synchronization with async fences.", + "claim_type": "semantics", + "covers": ["u004", "u005"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "WGMMA A operand forms; tcgen05 A grammar; fence and commit sections", + "observation": "WGMMA has both descriptor-A and register-A forms; tcgen05 A can be SMEM or TMEM. tcgen05 fences order cross-thread operations, while commit plus mbarrier provides MMA completion." + } + ] + }, + { + "claim_id": "migration-wgmma-to-tcgen05-original-c003", + "claim": "Compared with Hopper WGMMA FP8 support, the tcgen05 family adds native FP6/FP4 and block-scaled MX/NVFP4 kinds.", + "claim_type": "compatibility", + "covers": ["u006"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-matrix-multiply-and-accumulate-operations", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "WGMMA alternate floating formats and tcgen05 kind inventory", + "observation": "The WGMMA section includes FP8 formats, while tcgen05 adds f8f6f4 and the block-scaled mxf8f6f4, mxf4, and mxf4nvf4 kinds." + } + ], + "challenge": { + "alternative": "FP4/FP6 appear only in software conversion and not in tcgen05 MMA grammar.", + "check": "Inspect the exact tcgen05 kind and block-scale grammar tokens.", + "outcome": "The native grammar directly names the narrow-precision kinds." + } + }, + { + "claim_id": "migration-wgmma-to-tcgen05-original-c004", + "claim": "The migration checklist is correct as written, including mandatory ldmatrix removal, 64B-to-128B swizzle conversion, and replacing commit/wait with fences.", + "claim_type": "code", + "covers": ["u007"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "Table 40, MMA operand grammar, and completion mechanism", + "observation": "tcgen05 supports several valid swizzles and SMEM/TMEM A; MMA completion requires commit/mbarrier. The checklist's categorical substitutions are false." + } + ] + }, + { + "claim_id": "migration-wgmma-to-tcgen05-original-c005", + "claim": "The Hopper kernel block is valid or structurally faithful WGMMA CUDA for the migration baseline.", + "claim_type": "code", + "covers": ["u008"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/include/cute/arch/mma_sm90_gmma.hpp", + "revision": "CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "MMA_64x256x16 WGMMA wrapper signatures", + "observation": "The wiki block contains literal ellipses and undefined names and combines an ldmatrix fragment with a made-up WGMMA operand list. The pinned wrappers show exact descriptor-A or register-A signatures instead." + } + ] + }, + { + "claim_id": "migration-wgmma-to-tcgen05-original-c006", + "claim": "The Blackwell kernel block correctly demonstrates TMEM allocation, tcgen05 MMA, completion, TMEM load, and deallocation.", + "claim_type": "code", + "covers": ["u009"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "alloc/dealloc issue semantics, mma grammar, commit, and ld access restrictions", + "observation": "alloc/dealloc are warp-collective rather than lane-0 operations; the MMA omits its lane mask; fence/syncthreads is not completion; tcgen05.ld is warp-collective. Multiple undefined helpers further prevent execution." + } + ] + }, + { + "claim_id": "migration-wgmma-to-tcgen05-original-c007", + "claim": "The accumulator examples correctly contrast a 128-register per-thread Hopper m64n256 FP32 fragment with a TMEM-addressed Blackwell accumulator allocation concept.", + "claim_type": "code", + "covers": ["u010", "u011", "u012", "u013"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/include/cute/arch/mma_sm90_gmma.hpp", + "revision": "CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "MMA_64x256x16_F32F16F16_SS CRegisters; PTX tcgen05.alloc semantics", + "observation": "The wrapper declares float CRegisters[128]; PTX alloc returns a 32-bit TMEM address via shared memory and uses it for TMEM-backed D. The tmem_alloc name is plainly pseudocode, consistent with page metadata." + } + ], + "challenge": { + "alternative": "128 is a tile-wide count or TMEM allocation returns a pointer-sized generic address.", + "check": "Inspect the per-thread wrapper typedef and PTX's b32 taddr allocation result.", + "outcome": "The original contrast is correct at the stated conceptual level." + } + }, + { + "claim_id": "migration-wgmma-to-tcgen05-original-c008", + "claim": "A WGMMA port should always remove ldmatrix because Hopper necessarily register-stages A while tcgen05 necessarily uses an SMEM A descriptor.", + "claim_type": "code", + "covers": ["u014", "u015", "u016", "u017"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "WGMMA SS/RS forms and tcgen05 SMEM-A/TMEM-A grammar", + "observation": "Some WGMMA forms already source A through an SMEM descriptor, and tcgen05 can source A from TMEM. Migration depends on the exact source instruction, not a universal ldmatrix step." + } + ] + }, + { + "claim_id": "migration-wgmma-to-tcgen05-original-c009", + "claim": "The Hopper swizzle example accurately notes that both 64B and 128B swizzled shared-memory layouts are supported WGMMA/TMA choices.", + "claim_type": "code", + "covers": ["u018", "u019"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-shared-memory-layout-matrix-descriptor", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "WGMMA shared-memory layouts and descriptor swizzle modes", + "observation": "The WGMMA layout section documents both 64-byte and 128-byte swizzle modes." + } + ], + "challenge": { + "alternative": "64B swizzle is only a tcgen05 descriptor value and is invalid for WGMMA.", + "check": "Use the WGMMA-specific layout section rather than the adjacent tcgen05 table.", + "outcome": "The Hopper section directly documents 64B layouts." + } + }, + { + "claim_id": "migration-wgmma-to-tcgen05-original-c010", + "claim": "Blackwell tcgen05 correctness requires converting every operand layout to 128-byte swizzling.", + "claim_type": "code", + "covers": ["u020", "u021"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "Table 40 and tcgen05 shared-memory layout sections", + "observation": "tcgen05 defines valid no-swizzle, 128B, 64B, and 32B modes subject to their layout constraints." + } + ] + }, + { + "claim_id": "migration-wgmma-to-tcgen05-original-c011", + "claim": "The WGMMA-to-tcgen05 issue snippets are valid pseudocode and tcgen05 needs no commit because it is fully asynchronous.", + "claim_type": "code", + "covers": ["u022", "u023", "u024", "u025"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "MMA grammar and mbarrier-based completion mechanism", + "observation": "Both snippets contain literal ellipses or omit required operands; tcgen05 MMA completion is explicitly observed by tcgen05.commit plus mbarrier wait." + } + ] + }, + { + "claim_id": "migration-wgmma-to-tcgen05-original-c012", + "claim": "wgmma.wait_group.sync.aligned 0 waits until all committed WGMMA groups are complete before their accumulator registers are consumed.", + "claim_type": "code", + "covers": ["u026", "u027"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-wait-group", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "wgmma.wait_group syntax and description", + "observation": "The immediate 0 form waits for completion of all prior committed WGMMA groups." + } + ], + "challenge": { + "alternative": "Immediate 0 allows one group to remain outstanding.", + "check": "Compare the immediate-count definition and the adjacent example values.", + "outcome": "The zero value leaves no committed groups pending before use." + } + }, + { + "claim_id": "migration-wgmma-to-tcgen05-original-c013", + "claim": "A tcgen05.fence before __syncthreads replaces wgmma.wait_group and makes the TMEM accumulator ready.", + "claim_type": "code", + "covers": ["u028", "u029"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "mbarrier completion and tcgen05.fence descriptions", + "observation": "commit/mbarrier observes MMA completion; fence only orders operations around execution-ordering handoffs and constrains code motion." + } + ] + }, + { + "claim_id": "migration-wgmma-to-tcgen05-original-c014", + "claim": "A Hopper epilogue can consume its register accumulator directly, whereas a tcgen05 epilogue first transfers TMEM values into registers before scalar post-ops and stores.", + "claim_type": "code", + "covers": ["u030", "u031", "u032", "u033"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "WGMMA D fragments and 9.7.16.8.3 tcgen05.ld", + "observation": "WGMMA returns D in per-thread registers; tcgen05.ld collectively transfers TMEM data into a register vector for further computation." + } + ], + "challenge": { + "alternative": "CUDA scalar arithmetic can address TMEM directly without tcgen05.ld.", + "check": "Inspect the TMEM access-instruction model and destination of tcgen05.ld.", + "outcome": "TMEM-to-GPR transfer is the documented path before ordinary register computation." + } + }, + { + "claim_id": "migration-wgmma-to-tcgen05-original-c015", + "claim": "The two warp-specialization diagrams describe typical required Hopper and Blackwell allocations of 13+ and 5 warps, including a dedicated CLC warp on Blackwell.", + "claim_type": "code", + "covers": ["u034", "u035"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/examples/cute/tutorial/blackwell/02_mma_tma_sm100.cu", + "revision": "CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "thread-role partition and pipeline definitions; Hopper warp-specialized examples", + "observation": "Vendor examples choose workload-specific role counts and do not impose the diagrams' totals or a mandatory CLC scheduler warp." + } + ] + }, + { + "claim_id": "migration-wgmma-to-tcgen05-original-c016", + "claim": "Single-thread MMA issue permits other threads or warps to specialize in prefetch, epilogue, or reduction work, but the profitable role partition is kernel-dependent.", + "claim_type": "mechanism", + "covers": ["u036", "u037", "u038", "u039"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/examples/cute/tutorial/blackwell/02_mma_tma_sm100.cu", + "revision": "CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "warp-specialized producer/consumer pipeline", + "observation": "The pinned example assigns distinct data movement, MMA, and accumulator-consumer roles around a single-thread-issued MMA." + } + ], + "challenge": { + "alternative": "The rest of the issuing warp is forced idle and cannot perform any other pipeline role.", + "check": "Inspect the vendor example's role branching and pipelines.", + "outcome": "The implementation uses non-issuing execution resources for other work, while leaving the exact partition configurable." + } + }, + { + "claim_id": "migration-wgmma-to-tcgen05-original-c017", + "claim": "Blackwell's base MMA tile universally doubles Hopper M from 64 to 128, and migration should mechanically double CTA M or choose the two fixed table shapes.", + "claim_type": "semantics", + "covers": ["u040", "u041", "u042", "u043", "u044", "u045"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-data-path-layout-organization", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "WGMMA shape inventory; tcgen05 idesc and data-path layouts", + "observation": "Both instruction families expose multiple legal shapes. tcgen05 M/N are descriptor- and configuration-selected, so tile migration is not a fixed two-row lookup or mandatory doubling rule." + } + ] + }, + { + "claim_id": "migration-wgmma-to-tcgen05-original-c018", + "claim": "The pitfall list correctly says missing 128B swizzle silently corrupts tcgen05, fence-plus-syncthreads is the required completion path, and retaining Hopper thread counts necessarily wastes resources.", + "claim_type": "semantics", + "covers": ["u046", "u048", "u049"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "descriptor modes, issue granularity, commit, and fence", + "observation": "Several swizzles are valid, completion uses commit/mbarrier, and one-thread MMA issue does not determine the thread requirements of loads, epilogue, scheduling, or other kernel work." + } + ] + }, + { + "claim_id": "migration-wgmma-to-tcgen05-original-c019", + "claim": "All dynamically allocated TMEM must be explicitly deallocated before kernel exit, and removing register-resident accumulators warrants retuning rather than assuming Hopper's register budget remains optimal.", + "claim_type": "semantics", + "covers": ["u047", "u050"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [ + { + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "locator": "9.7.16.7.1 allocation lifecycle; WGMMA and tcgen05 D locations", + "observation": "PTX requires every allocation to be deallocated before exit. Moving D from GPRs to TMEM changes one resource constraint, supporting a retuning recommendation without guaranteeing a particular optimum." + } + ], + "challenge": { + "alternative": "CTA teardown implicitly releases TMEM, or identical register limits dominate both kernels.", + "check": "Inspect the explicit before-exit mandate and compare accumulator storage locations.", + "outcome": "The ISA requires deallocation and the accumulator resource changes, so both scoped statements hold." + } + }, + { + "claim_id": "migration-wgmma-to-tcgen05-original-c020", + "claim": "CUTLASS migration is largely an arch-tag and schedule-token substitution using the shown device::GemmUniversal and KernelScheduleSm100CpAsyncWarpSpecialized types, after which MMA_Atom/MMA_Traits automatically handle all differences.", + "claim_type": "code", + "covers": ["u051", "u052", "u053"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/examples/70_blackwell_gemm/70_blackwell_fp16_gemm.cu", + "revision": "CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "locator": "SM100 CollectiveBuilder and GemmUniversal construction; gemm/dispatch_policy.hpp", + "observation": "The displayed schedule type does not exist in pinned CUTLASS. SM100 examples configure collective builders, tile/cluster shapes, stages, and supported schedule tags such as KernelTmaWarpSpecialized1SmSm100 before instantiating kernel::GemmUniversal and a device adapter." + } + ] + } + ], + "scope": "All 53 immutable baseline units were checked against CUDA 13.0.2 PTX ISA 9.0 and pinned CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5. Code was assessed according to the page's pseudocode reproducibility label while executable-looking PTX was still required to preserve real operand and ordering semantics.", + "residual_risk": "No CUDA toolkit or NVIDIA GPU is installed locally, so no migrated kernel was compiled or run. The discriminating ISA and pinned-library checks are sufficient to refute syntax, lifecycle, and API claims but do not predict performance for a specific port." +} diff --git a/verification/reviewed/original/pattern-compute-bound.json b/verification/reviewed/original/pattern-compute-bound.json new file mode 100644 index 000000000..28d7c29df --- /dev/null +++ b/verification/reviewed/original/pattern-compute-bound.json @@ -0,0 +1,20 @@ +{ + "claims": [ + {"claim":"Tensor-core utilization below 70% plus unsaturated memory bandwidth proves a kernel is compute-bound but below peak.","claim_id":"pattern-compute-bound-original-c001","claim_type":"performance","covers":["u001"],"evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline","revision":"Nsight Compute 2025.3","locator":"roofline analysis","observation":"Boundedness depends on arithmetic intensity relative to measured compute and memory ceilings; neither an architecture-wide 70% threshold nor unsaturated DRAM alone establishes it."}],"oracle":"web-primary","verdict":"refuted"}, + {"challenge":{"alternative":"The listed events cannot explain under-attainment unless a profiler names them as a stall reason.","check":"Inspect direct implementations where TMA readiness, non-MMA work, or epilogue occupancy lies on a dependency path.","outcome":"Pinned tutorial and FA4 sources contain these dependency structures, but the page correctly labels them only as likely causes."},"claim":"TMA-readiness bubbles, non-matmul work, and an epilogue that delays mainloop reuse are plausible causes of compute-throughput under-attainment.","claim_id":"pattern-compute-bound-original-c002","claim_type":"semantics","covers":["u002","u003","u005"],"evidence":[{"url":"https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100","revision":"3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"v3-v6 producer/consumer pipelines","observation":"The implementations contain TMA-to-MMA and mainloop-to-epilogue dependencies; v6 adds overlap after profiling epilogue/setup idle periods."},{"url":"https://arxiv.org/html/2603.05451v1","revision":"FlashAttention-4 paper v1","locator":"section 2.2 and 3","observation":"The paper analyzes exponentials, correction, and SMEM work as non-MMA contributors that become material relative to faster tensor cores."}],"oracle":"web-primary","verdict":"verified"}, + {"claim":"Single-SM MMA tiles are inherently too small to utilize available compute.","claim_id":"pattern-compute-bound-original-c003","claim_type":"performance","covers":["u004"],"evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"PTX ISA 9.0","locator":"cta_group and shape constraints","observation":"PTX defines multiple legal single-CTA and paired-CTA shapes; it gives no utilization conclusion from CTA-group mode alone."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"2-SM cooperative MMA always doubles the effective tile to m256n256 and provides twice the compute per cycle.","claim_id":"pattern-compute-bound-original-c004","claim_type":"performance","covers":["u006"],"evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"PTX ISA 9.0","locator":"cta_group::2 shapes","observation":"Legal shapes depend on kind, descriptors, layouts, target, and CTA group; cooperative work occupies two CTAs/SMs and does not promise a universal 2x throughput per cycle."}],"oracle":"web-primary","verdict":"refuted"}, + {"challenge":{"alternative":"The candidate rows promise these transformations will improve every compute-bound kernel.","check":"Read each row as a structural effect and compare it to pinned implementations.","outcome":"The rows state possible overlap/offload mechanisms without a numeric guarantee; pinned sources demonstrate each structure in a scoped kernel."},"claim":"Operand stages can overlap TMA and MMA, output buffering/fusion can overlap epilogue and later MMA, and FA4 moves selected exponential evaluations to CUDA-core FMA code.","claim_id":"pattern-compute-bound-original-c005","claim_type":"semantics","covers":["u007","u009","u010"],"evidence":[{"url":"https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100","revision":"3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"v3 and v6","observation":"v3 stages TMA/MMA work and v6 uses output buffering/epilogue specialization for overlap."},{"url":"https://arxiv.org/html/2603.05451v1","revision":"FlashAttention-4 paper v1","locator":"partial software exponentiation","observation":"FA4 selects a fraction of exponentials for a cubic polynomial evaluated with FMA while retaining hardware ex2 for others."}],"oracle":"web-primary","verdict":"verified"}, + {"claim":"Warp specialization provides dedicated TMA/MMA/epilogue warps with no stalls.","claim_id":"pattern-compute-bound-original-c006","claim_type":"performance","covers":["u008"],"evidence":[{"url":"https://gau-nernst.github.io/tcgen05/","revision":"retrieved 2026-08-09","locator":"warp specialization and persistent sections","observation":"The tutorial reports a scoped improvement but later profiling still observes idle periods and imperfect overlap; role separation does not guarantee no stalls."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"FA4 uses Cody-Waite plus a Horner polynomial for every software exponential, thereby multiplying exponential throughput and causing the full-kernel 1605-TFLOP/s result.","claim_id":"pattern-compute-bound-original-c007","claim_type":"code","covers":["u011"],"evidence":[{"url":"https://arxiv.org/html/2603.05451v1","revision":"FlashAttention-4 paper v1","locator":"partial software exponentiation","observation":"FA4 uses floor-based base-2 range reduction and a cubic polynomial for a selected fraction of values, with other values using ex2; it does not call this Cody-Waite or isolate a multiplied-throughput result."},{"url":"https://tridao.me/blog/2026/flash4/","revision":"Tri Dao FA4 blog, 2026","locator":"performance and techniques","observation":"1605 TFLOP/s and 71% describe the complete forward kernel with several techniques, not a software-exp-only causal result."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"2-SM cooperative MMA requires cluster configuration and identical shared-memory layouts in the two CTAs.","claim_id":"pattern-compute-bound-original-c008","claim_type":"compatibility","covers":["u012"],"evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"PTX ISA 9.0","locator":"cta_group::2 requirements","observation":"The CTAs must be in a cluster and satisfy instruction-specific peer/resource requirements, but PTX does not impose the page's universal identical-SMEM-layout rule."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"Pipeline depth is workload-dependent, with three to five stages typical.","claim_id":"pattern-compute-bound-original-c009","claim_type":"performance","covers":["u013"],"evidence":[{"url":"https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu","revision":"3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"matmul_v3 launch","observation":"The cited beneficial pipelined variant uses two stages, directly defeating the unqualified typical range even though depth is configuration-dependent."}],"oracle":"web-primary","verdict":"refuted"}, + {"challenge":{"alternative":"A software transcendental can be exactly rounded and therefore need not trade accuracy.","check":"Inspect the specific software exponential linked by this page.","outcome":"FA4's cubic approximation has measured error and is selected only where the algorithm accepts it, making accuracy and throughput explicit design variables in this cited technique."},"claim":"The cited FA4 software-exponential technique trades approximation accuracy against throughput.","claim_id":"pattern-compute-bound-original-c010","claim_type":"semantics","covers":["u014"],"evidence":[{"url":"https://arxiv.org/html/2603.05451v1","revision":"FlashAttention-4 paper v1","locator":"software exponential accuracy and selection","observation":"The paper reports approximation error, selects only a subset of values, and evaluates accuracy/performance together."}],"oracle":"web-primary","verdict":"verified"} + ], + "commands":["Inspect Nsight Compute roofline semantics","Inspect PTX ISA 9.0 cta_group::2 contracts","Inspect FA4 paper/blog and pinned tutorial implementations"], + "id":"pattern-compute-bound", + "page_verdict":"refuted", + "path":"wiki/patterns/compute-bound.md", + "residual_risk":"No B200 or profiler is available locally. Compatibility and algorithms are source-inspected; performance is author-reported.", + "scope":"All 14 baseline units for hash 1250f2c8d05fdd1ec4dcb30133b71caf29617b7fe0c0131fcafbe9ca9a4fe320 were reviewed, including the pseudo-code block and every candidate-effect claim." +} diff --git a/verification/reviewed/original/pattern-low-sm-utilization.json b/verification/reviewed/original/pattern-low-sm-utilization.json new file mode 100644 index 000000000..9aadbe2b6 --- /dev/null +++ b/verification/reviewed/original/pattern-low-sm-utilization.json @@ -0,0 +1,17 @@ +{ + "claims":[ + {"claim":"SM utilization below 60% despite sufficient occupancy, plus idle-SM intervals, defines the low-utilization symptom.","claim_id":"pattern-low-sm-utilization-original-c001","claim_type":"performance","covers":["u001"],"evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"occupancy, PM sampling, scheduler statistics","observation":"Occupancy is a residency limit rather than achieved activity, PM samples are temporal observations, and no official 60% threshold defines the pattern."}],"oracle":"web-primary","verdict":"refuted"}, + {"challenge":{"alternative":"Only the last-wave remainder can leave SMs idle; imbalance, static assignment, and a small grid cannot.","check":"Apply the scheduler semantics and basic independent-block count to each listed condition.","outcome":"Each condition can reduce concurrent useful work, though none is proven by aggregate utilization alone."},"claim":"A partial final wave, variable tile duration, nonadaptive static assignment, or fewer independent blocks than available workers can each leave SMs without useful work.","claim_id":"pattern-low-sm-utilization-original-c002","claim_type":"semantics","covers":["u002","u003","u004","u005"],"evidence":[{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html","revision":"CUDA Programming Guide, retrieved 2026-08-09","locator":"fixed-work and fixed-worker scheduling motivation","observation":"The guide distinguishes fixed work per block from persistent work acquisition and motivates CLC with uneven runtime SM availability."},{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0","locator":"CLC scheduling model","observation":"CUTLASS documents initial grid work and redistribution of not-yet-started IDs, consistent with these possible shortage/imbalance mechanisms."}],"oracle":"web-primary","verdict":"verified"}, + {"claim":"CLC is SM100-only dynamic tile assignment that eliminates load imbalance.","claim_id":"pattern-low-sm-utilization-original-c003","claim_type":"performance","covers":["u006"],"evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"PTX ISA 9.0","locator":"target and request semantics","observation":"The instruction requires sm_100 or higher and redistributes an unspecified not-yet-started ID; it cannot create work or guarantee elimination of imbalance."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"Persistent kernels apply on SM90+ and eliminate the tail effect while making launch overhead one-time.","claim_id":"pattern-low-sm-utilization-original-c004","claim_type":"performance","covers":["u007"],"evidence":[{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html","revision":"CUDA Programming Guide, retrieved 2026-08-09","locator":"fixed number of thread blocks","observation":"Persistent workers can process multiple work items, but the guide does not promise elimination of tails; launch count and uncovered parallelism remain workload/scheduler dependent."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"Tile scheduling on SM90+ necessarily improves L2 locality and reduces load variance.","claim_id":"pattern-low-sm-utilization-original-c005","claim_type":"performance","covers":["u008"],"evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0","locator":"scheduler and swizzle options","observation":"Scheduling exposes choices, but coordinate order and work reassignment have no guaranteed cache or variance effect."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"The tcgen05 tutorial moves from 86% without persistence/CLC to 98% with persistence plus CLC, keeping all SMs busy.","claim_id":"pattern-low-sm-utilization-original-c006","claim_type":"code","covers":["u009"],"evidence":[{"url":"https://gau-nernst.github.io/tcgen05/","revision":"retrieved 2026-08-09","locator":"v5, v6, and closing remarks","observation":"v5 is 1302.29/1506.74=86.43%; v6 static persistence is 1475.93/1506.74=97.96%. The author explicitly says CLC was not added and notes overlap remains imperfect."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"CLC is available only on SM100 datacenter GPUs and not SM120 consumer GPUs.","claim_id":"pattern-low-sm-utilization-original-c007","claim_type":"compatibility","covers":["u010"],"evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"PTX ISA 9.0","locator":"Target ISA Notes","observation":"The base instruction requires sm_100 or higher, and the multicast qualifier explicitly lists sm_120a/sm_120f support, refuting the page's SM120 exclusion."}],"oracle":"web-primary","verdict":"refuted"}, + {"challenge":{"alternative":"A persistent worker loop is no harder to inspect than a one-work-item block because the profiler still observes one kernel launch.","check":"Compare the fixed-work and fixed-worker execution models and identify what must be attributed to reproduce a logical work item.","outcome":"A persistent block repeatedly acquires and processes logical work, so debugging and profiling must distinguish worker identity, acquired work identity, and loop iteration; this is additional attribution state, not a claim that tools cannot profile the kernel."},"claim":"Persistent kernels complicate debugging and profiling.","claim_id":"pattern-low-sm-utilization-original-c008","claim_type":"semantics","covers":["u011"],"evidence":[{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html","revision":"CUDA Programming Guide, retrieved 2026-08-09","locator":"fixed-work and fixed-worker scheduling models","observation":"The fixed-worker model has a resident block repeatedly request and process work rather than binding one block to one logical item, introducing a worker-to-work mapping and repeated lifetime that diagnostics must attribute."}],"oracle":"web-primary","verdict":"verified"}, + {"claim":"A non-persistent kernel should always use a grid much larger than the SM count.","claim_id":"pattern-low-sm-utilization-original-c009","claim_type":"performance","covers":["u012"],"evidence":[{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html","revision":"CUDA Programming Guide, retrieved 2026-08-09","locator":"thread-block scheduling and occupancy","observation":"Useful grid size follows problem decomposition, block resources, and residency; the guide provides no grid-much-greater-than-SM rule, and small problems may not expose that work."}],"oracle":"web-primary","verdict":"refuted"} + ], + "commands":["Inspect CUDA/PTX CLC compatibility and work semantics","Inspect CUTLASS 4.5.0 scheduler model","Inspect tutorial v5/v6 progression"], + "id":"pattern-low-sm-utilization","page_verdict":"refuted","path":"wiki/patterns/low-sm-utilization.md", + "residual_risk":"No GPU profiling was possible. Scheduler compatibility is source-verified and tutorial performance remains author-reported.", + "scope":"All 12 baseline units for hash ffe8f3f021f4946f96b95bfc41d6745d38c52fd887cb7171751f8f4e9934e91b were reviewed, including the code block and applicability claims." +} diff --git a/verification/reviewed/original/pattern-memory-bound.json b/verification/reviewed/original/pattern-memory-bound.json new file mode 100644 index 000000000..066ffb68e --- /dev/null +++ b/verification/reviewed/original/pattern-memory-bound.json @@ -0,0 +1,21 @@ +{ + "claims":[ + {"claim":"High DRAM throughput, low tensor-core utilization, and arithmetic intensity below a roofline knee establish that a kernel is memory-bandwidth bound.","claim_id":"pattern-memory-bound-original-c001","claim_type":"performance","covers":["u001"],"evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline","revision":"Nsight Compute 2025.3","locator":"roofline analysis","observation":"The roofline needs achieved work, bytes, and relevant measured ceilings; vague 'high' bandwidth and low tensor activity do not establish proximity to the attainable memory roof or exclude latency/instruction limits."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"GEMV, small-batch decode, and reduction kernels are intrinsically low-arithmetic-intensity causes of memory boundedness.","claim_id":"pattern-memory-bound-original-c002","claim_type":"performance","covers":["u002"],"evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline","revision":"Nsight Compute 2025.3","locator":"arithmetic intensity","observation":"Intensity is operations divided by transferred bytes for the concrete implementation/input; an algorithm-category label cannot substitute for that measurement."}],"oracle":"web-primary","verdict":"refuted"}, + {"challenge":{"alternative":"Using each element once need not be poor reuse if the algorithm has no reuse to exploit.","check":"Interpret the row as a possible traffic/intensity mechanism rather than an optimization judgment.","outcome":"Single-use traffic lowers operations per transferred byte relative to reusable data, although it does not by itself prove a reachable bandwidth ceiling."},"claim":"Using each transferred data element only once is a possible source of low arithmetic intensity.","claim_id":"pattern-memory-bound-original-c003","claim_type":"semantics","covers":["u003"],"evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline","revision":"Nsight Compute 2025.3","locator":"arithmetic-intensity definition","observation":"Arithmetic intensity is the ratio of operations to bytes; reuse can raise work per transferred byte."}],"oracle":"web-primary","verdict":"verified"}, + {"claim":"Uncoalesced loads or L1 thrashing are causes of a memory-bandwidth-bound diagnosis.","claim_id":"pattern-memory-bound-original-c004","claim_type":"performance","covers":["u004"],"evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"Memory Workload Analysis","observation":"Inefficient transactions and cache behavior can create excess traffic or latency, but can also leave bandwidth below its ceiling; the observations require separate requested/transferred-byte and throughput analysis."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"128- or 256-bit vectorized loads maximize memory-bandwidth utilization.","claim_id":"pattern-memory-bound-original-c005","claim_type":"performance","covers":["u005"],"evidence":[{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"retrieved 2026-08-08","locator":"Attempt 9","observation":"The cited NVFP4 GEMV author reports that changing to wider uint2 loads made the kernel 16-25% slower, a direct negative control."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"L1::no_allocate is the correct policy for streaming data and L1::evict_last for reused data.","claim_id":"pattern-memory-bound-original-c006","claim_type":"performance","covers":["u006"],"evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld","revision":"PTX ISA 9.0","locator":"eviction priorities","observation":"These are performance hints that may be applied, not guaranteed placement rules or tensor-role prescriptions."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"Setting maxrregcount increases occupancy.","claim_id":"pattern-memory-bound-original-c007","claim_type":"performance","covers":["u007"],"evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html","revision":"CUDA Compiler Driver 13.0.2","locator":"maxrregcount","observation":"The option limits allocated registers subject to ABI minima; it may introduce spills and need not change the limiting occupancy resource."},{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"retrieved 2026-08-08","locator":"register cap attempt","observation":"The cited author reports that reducing maxrregcount from 80 to 64 had no performance effect."}],"oracle":"web-primary","verdict":"refuted"}, + {"challenge":{"alternative":"TMA multicast merely duplicates separate global loads and therefore shares no loaded data.","check":"Inspect the normative multicast form and destination mask.","outcome":"One tensor-copy operation can write selected cluster CTAs' shared-memory destinations, sharing the global operand transfer across the cluster."},"claim":"TMA multicast can distribute one global-to-shared tensor transfer to selected CTAs in a cluster.","claim_id":"pattern-memory-bound-original-c008","claim_type":"semantics","covers":["u008"],"evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"PTX ISA 9.0","locator":"multicast::cluster form","observation":"PTX defines a global-to-cluster-shared tensor copy with a CTA mask and per-destination mbarrier completion."}],"oracle":"web-primary","verdict":"verified"}, + {"claim":"Shared-memory swizzling eliminates bank conflicts.","claim_id":"pattern-memory-bound-original-c009","claim_type":"performance","covers":["u009"],"evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-map-swizzle","revision":"PTX ISA 9.0","locator":"swizzle modes and layout constraints","observation":"Swizzling remaps addresses; conflict behavior depends on access pattern and legal layout, and no mode universally eliminates conflicts."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"For memory-bound NVFP4 GEMV, the correct priority is wide/coalesced loads, lower register count, differentiated cache policy, and never optimizing compute.","claim_id":"pattern-memory-bound-original-c010","claim_type":"code","covers":["u010"],"evidence":[{"url":"https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/","revision":"retrieved 2026-08-08","locator":"Attempts and Takeaways","observation":"The author's wider-load and lower-register-cap attempts failed or regressed, and the retrospective recommends profiling the actual bottleneck rather than this fixed priority list."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"One should always profile before optimizing because a wrong assumption wastes effort.","claim_id":"pattern-memory-bound-original-c011","claim_type":"semantics","covers":["u011"],"evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"roofline, guided analysis, and profiling overhead","observation":"Profiling supplies concrete bottleneck evidence, but collection has replay/overhead costs and the guide does not make profiling a prerequisite for every optimization; correctness, static legality, and already-established representative evidence are counterexamples to the absolute sequence."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"B200 has 8 TB/s of bandwidth, and a speed-of-light calculation determines achievable performance.","claim_id":"pattern-memory-bound-original-c012","claim_type":"performance","covers":["u012"],"evidence":[{"url":"https://www.nvidia.com/en-us/data-center/blackwell/","revision":"retrieved 2026-08-09","locator":"B200 specifications","observation":"A platform specification can give a nominal memory-bandwidth value, but roofline supplies an upper-bound model; achieved performance also depends on access efficiency, clocks, operation mix, and measured ceilings."},{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline","revision":"Nsight Compute 2025.3","locator":"roofline limitations","observation":"Roofline classifies observed work/traffic against ceilings; it does not determine one exact achievable result."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"Instruction-level parallelism and compute optimizations necessarily have diminishing returns for memory-bound kernels.","claim_id":"pattern-memory-bound-original-c013","claim_type":"performance","covers":["u013"],"evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"scheduler and memory analysis","observation":"A memory-roof classification does not show whether instruction issue, address/decode work, latency hiding, or transaction inefficiency keeps achieved bandwidth below the attainable ceiling."}],"oracle":"web-primary","verdict":"refuted"} + ], + "commands":["Inspect Nsight Compute roofline/memory semantics","Inspect PTX load hints, multicast, and swizzle contracts","Inspect cited NVFP4 GEMV negative controls"], + "id":"pattern-memory-bound","page_verdict":"refuted","path":"wiki/patterns/memory-bound.md", + "residual_risk":"No target memory counters were collected locally. Hardware behavior is source-scoped and cited performance is author-reported.", + "scope":"All 13 baseline units for hash 55576e61fe10aa796637b7056b452f0d94f5b0e727643e78a422e2b721eb30ff were audited, including the code-comment prescription and numerical roof claim." +} diff --git a/verification/reviewed/original/pattern-moe-load-imbalance.json b/verification/reviewed/original/pattern-moe-load-imbalance.json new file mode 100644 index 000000000..816e23743 --- /dev/null +++ b/verification/reviewed/original/pattern-moe-load-imbalance.json @@ -0,0 +1,27 @@ +{ + "claims":[ + {"claim":"MoE grouped GEMM imbalance is necessarily per-expert compute-time imbalance in which each SM owns one expert and latency is set by the slowest expert.","claim_id":"pattern-moe-load-imbalance-original-c001","claim_type":"mechanism","covers":["u001"],"evidence":[{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"grouped GEMM interfaces","observation":"Grouped work is represented as multiple variable-size GEMMs; the interface does not assign one expert to one physical SM or define latency as the maximum expert time."},{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html","revision":"retrieved 2026-08-09","locator":"thread-block scheduling","observation":"CUDA schedules thread blocks to available SMs and exposes no fixed expert-to-SM ownership."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"Trained MoE routers commonly send exactly 80 percent of tokens to 20 percent of experts.","claim_id":"pattern-moe-load-imbalance-original-c002","claim_type":"performance","covers":["u002"],"evidence":[{"url":"https://github.com/deepseek-ai/EPLB/blob/d52c72d5b2f2fb4c41afbf8eb21366820239913d/README.md","revision":"EPLB d52c72d5b2f2fb4c41afbf8eb21366820239913d","locator":"README policies and example","observation":"The official project consumes measured expert loads and supplies no universal 80/20 distribution or frequency claim."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"Static tile assignment means a precomputed tile-to-physical-SM mapping that cannot rebalance at runtime.","claim_id":"pattern-moe-load-imbalance-original-c003","claim_type":"semantics","covers":["u003"],"evidence":[{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html","revision":"retrieved 2026-08-09","locator":"thread-block scheduling","observation":"CUDA dynamically places blocks on available SMs; software can precompute logical work ownership but not a portable tile-to-physical-SM map."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"A fixed M-max masked grouped layout necessarily executes computation on every padding row.","claim_id":"pattern-moe-load-imbalance-original-c004","claim_type":"mechanism","covers":["u004"],"evidence":[{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"Masked grouped GEMM","observation":"The documented masked interface takes valid M counts and says it computes only valid portions while retaining fixed allocation for CUDA graphs."}],"oracle":"web-primary","verdict":"refuted"}, + {"challenge":{"alternative":"Every M smaller than the output tile has full tensor-core utilization because the instruction shape is unchanged.","check":"Inspect NVIDIA's small-dimension GEMM discussion and the grouped valid-M representation.","outcome":"Partial tiles can leave threads predicated out and small problems can expose too little parallel work, although the effect remains kernel- and shape-specific."},"claim":"Small per-expert M or partial output tiles can reduce useful parallel work and utilization, but this is a configuration-specific risk rather than an M less than BLOCK_M rule.","claim_id":"pattern-moe-load-imbalance-original-c005","claim_type":"performance","covers":["u005"],"evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/efficient_gemm.md","revision":"CUTLASS 4.5.0","locator":"small GEMM dimensions and tile quantization","observation":"CUTLASS explains that small dimensions can leave threads out of bounds and insufficient threadblocks, without defining a universal threshold."}],"oracle":"web-primary","verdict":"verified"}, + {"claim":"CLC is hardware dynamic tile assignment in which the fastest SMs autonomously grab more tiles.","claim_id":"pattern-moe-load-imbalance-original-c006","claim_type":"semantics","covers":["u006"],"evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"request and response semantics","observation":"A selected thread asynchronously requests cancellation of an unspecified unlaunched block or cluster and handles success/failure; the ISA does not define autonomous fastest-SM work stealing."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"Persistent kernels inherently amortize launch overhead by looping over a dynamic work queue.","claim_id":"pattern-moe-load-imbalance-original-c007","claim_type":"performance","covers":["u007"],"evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0","locator":"static and CLC-backed persistent schedulers","observation":"CUTLASS documents both a zero-overhead static next-work rule and a CLC-backed dynamic rule; persistence does not imply a dynamic queue, and launch benefit is workload-dependent."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"DeepGEMM contiguous grouped layout always uses an offsets array to delimit packed variable-M experts.","claim_id":"pattern-moe-load-imbalance-original-c008","claim_type":"semantics","covers":["u008"],"evidence":[{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"Contiguous grouped GEMM","observation":"The packed M layout accepts either a per-row expert index or a per-group prefix-sum end representation and requires aligned segments; it is not a single offsets-array ABI."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"DeepGEMM masked grouped layout supports a CUDA-graph decode case but necessarily wastes compute on padding.","claim_id":"pattern-moe-load-imbalance-original-c009","claim_type":"mechanism","covers":["u009"],"evidence":[{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"Masked grouped GEMM","observation":"The README documents CUDA-graph-friendly decode and valid-M counts, explicitly limiting computation to valid portions rather than all allocated rows."}],"oracle":"web-primary","verdict":"refuted"}, + {"challenge":{"alternative":"K-grouped is another forward/prefill representation or also varies M and N.","check":"Inspect the pinned README and API shape assertions.","outcome":"They assign K-grouped to MoE weight backward, pack the K axis, and retain fixed M/N with grouped outputs."},"claim":"DeepGEMM K-grouped layout is documented for MoE weight-gradient computation with variable K and fixed M and N.","claim_id":"pattern-moe-load-imbalance-original-c010","claim_type":"semantics","covers":["u010"],"evidence":[{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f","locator":"K-grouped GEMM","observation":"The README gives the weight-backward use and fixed-axis contract."},{"url":"https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp","revision":"same commit","locator":"k_grouped_fp8_gemm functions","observation":"API checks establish packed variable K operands and fixed M/N grouped outputs."}],"oracle":"web-primary","verdict":"verified"}, + {"claim":"EPLB replication across GPUs generically provides 1.49x prefill and 2.54x decode speedups.","claim_id":"pattern-moe-load-imbalance-original-c011","claim_type":"performance","covers":["u011"],"evidence":[{"url":"https://www.lmsys.org/blog/2025-05-05-large-scale-ep/","revision":"LMSYS post published 2025-05-05, retrieved 2026-08-09","locator":"Ablation Study: EPLB, Overall Results and limitations","observation":"The figures are from a large-scale 96-H100 SGLang deployment using in-distribution data; the authors explicitly limit generalization to production distribution shifts."},{"url":"https://github.com/deepseek-ai/EPLB/blob/d52c72d5b2f2fb4c41afbf8eb21366820239913d/eplb.py","revision":"EPLB d52c72d5b2f2fb4c41afbf8eb21366820239913d","locator":"replicate_experts and rebalance_experts","observation":"The library replicates loaded logical experts and packs physical experts across configured GPUs/nodes but supplies no universal speedup contract."}],"oracle":"web-primary","verdict":"refuted"}, + {"challenge":{"alternative":"The submission only skipped work and never produced correct outputs.","check":"Inspect the official postmortem's correctness and timing paths separately.","outcome":"It ran a real padded eight-group kernel on cloned objects for correctness, then merged and cached work only under timing."},"claim":"GPU Mode's official postmortem says the temporarily first-place submission combined a real correct kernel path with a timing-harness exploit.","claim_id":"pattern-moe-load-imbalance-original-c012","claim_type":"mechanism","covers":["u012","u013"],"evidence":[{"url":"https://www.gpumode.com/news/reward-hacking-nvfp4","revision":"retrieved 2026-08-09","locator":"correctness and timing analysis","observation":"The postmortem distinguishes real correctness computation from the timing-specific exploit."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"The exploit merely reused one result without performing all timed inputs.","check":"Inspect object counts, merged group count, return behavior, and timing division in the official postmortem.","outcome":"Fifteen cloned eight-group inputs became one 120-group call, calls 2-15 returned cached pointers, and the harness divided by fifteen."},"claim":"During timing the exploit merged fifteen eight-group inputs into one 120-group call, returned cached pointers for calls 2 through 15, and benefited from division by fifteen.","claim_id":"pattern-moe-load-imbalance-original-c013","claim_type":"mechanism","covers":["u014"],"evidence":[{"url":"https://www.gpumode.com/news/reward-hacking-nvfp4","revision":"retrieved 2026-08-09","locator":"timing exploit sequence","observation":"The official account gives the same counts and amortization behavior."}],"oracle":"web-primary","verdict":"verified"}, + {"claim":"The GPU Mode reward-hack incident prompted the MLSys 2026 FlashInfer contest to add runtime isolation and subprocess evaluation.","claim_id":"pattern-moe-load-imbalance-original-c014","claim_type":"mechanism","covers":["u015"],"evidence":[{"url":"https://www.gpumode.com/news/reward-hacking-nvfp4","revision":"retrieved 2026-08-09","locator":"evaluation response","observation":"The postmortem points to gpu-mode/reference-kernels PR 104 and does not claim a FlashInfer/MLSys response."},{"url":"https://github.com/flashinfer-ai/flashinfer-bench-starter-kit/blob/75ccd05cafceb0fd1f86be4cd0f2117249463c66/EVALUATION.md","revision":"starter kit 75ccd05cafceb0fd1f86be4cd0f2117249463c66","locator":"evaluation contract","observation":"The contest contract documents its evaluator but attributes no design change to the GPU Mode incident."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"CLC is available only on SM100 data-center GPUs and not on SM120 consumer Blackwell.","claim_id":"pattern-moe-load-imbalance-original-c015","claim_type":"compatibility","covers":["u016"],"evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"Target ISA Notes","observation":"The instruction requires sm_100 or higher and the multicast form explicitly lists sm_120a and the SM120 family."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"Dynamic scheduling adds a small overhead to every tile relative to a static precomputed schedule.","claim_id":"pattern-moe-load-imbalance-original-c016","claim_type":"performance","covers":["u017"],"evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0","locator":"CLC rules and pipeline","observation":"The initial blockIdx is fully static and incurs no CLC request; later asynchronous requests are pipelined and cluster-granular. The source gives no universal small latency bound."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"Small experts have a universal minimum viable tile-size floor below which a scheduling technique cannot benefit.","claim_id":"pattern-moe-load-imbalance-original-c017","claim_type":"performance","covers":["u018"],"evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/efficient_gemm.md","revision":"CUTLASS 4.5.0","locator":"small GEMM dimensions","observation":"The document describes gradual, shape- and tiling-dependent underutilization from predication and insufficient blocks, not a universal viability threshold."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"EPLB works only at cluster scale and is categorically not a single-device mechanism.","claim_id":"pattern-moe-load-imbalance-original-c018","claim_type":"compatibility","covers":["u019"],"evidence":[{"url":"https://github.com/deepseek-ai/EPLB/blob/d52c72d5b2f2fb4c41afbf8eb21366820239913d/eplb.py","revision":"EPLB d52c72d5b2f2fb4c41afbf8eb21366820239913d","locator":"rebalance_experts parameters and policies","observation":"The planner accepts explicit node and GPU counts and operates on load tensors; its intended use is expert-parallel placement across GPUs, which can be intra-node and is not synonymous with cluster scale. It is distinct from a device-kernel tile scheduler."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"Uniform routing, a very large batch, or an auxiliary load-balancing loss by itself proves that runtime MoE load imbalance is not an issue.","claim_id":"pattern-moe-load-imbalance-original-c019","claim_type":"performance","covers":["u020","u021","u022"],"evidence":[{"url":"https://www.lmsys.org/blog/2025-05-05-large-scale-ep/","revision":"published 2025-05-05, retrieved 2026-08-09","locator":"EPLB case studies and limitations","observation":"The deployment diagnoses imbalance from measured per-expert/rank loads and throughput, notes phase/distribution differences, and treats benefit as batch- and distribution-dependent; none of the three proxies is a sufficient runtime negative test."}],"oracle":"web-primary","verdict":"refuted"} + ], + "commands":["Inspect CUDA/PTX block scheduling and CLC semantics","Inspect pinned DeepGEMM grouped-layout contracts","Inspect pinned EPLB placement code and scoped 96-H100 ablation","Inspect official reward-hack and FlashInfer evaluation records"], + "id":"pattern-moe-load-imbalance","page_verdict":"refuted","path":"wiki/patterns/moe-load-imbalance.md", + "residual_risk":"No GPU timeline or expert trace was available; utilization and scheduler-benefit claims are retained only as measurement-dependent risks.", + "scope":"All 22 baseline units for hash dd70d3638677328e71242ec9fa14891b9ed9465328f129378f5b835f52741275 were audited across routing, layouts, schedulers, distributed placement, performance, compatibility, and reward-hack history." +} diff --git a/verification/reviewed/original/pattern-pipeline-stalls.json b/verification/reviewed/original/pattern-pipeline-stalls.json new file mode 100644 index 000000000..0ab6d9bf7 --- /dev/null +++ b/verification/reviewed/original/pattern-pipeline-stalls.json @@ -0,0 +1,130 @@ +{ + "claims": [ + { + "claim": "Low TMA/tcgen05 activity, phase-local tensor-core drops, and warps observed at mbarrier.try_wait establish pipeline stalls in an otherwise compute-bound kernel.", + "claim_id": "pattern-pipeline-stalls-original-c001", + "claim_type": "semantics", + "covers": ["u001"], + "evidence": [{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"Warp State Statistics and sampling","observation":"Warp states are sampled symptoms, stalls are not necessarily performance-impacting, and NVIDIA says to focus on them when schedulers fail to issue; source correlation does not by itself prove the causal pipeline or a compute-bound classification."}], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "Two stages cannot hide a three-cycle latency chain.", + "claim_id": "pattern-pipeline-stalls-original-c002", + "claim_type": "performance", + "covers": ["u002"], + "evidence": [{"url":"https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu","revision":"3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"matmul_v3 launch and pipeline","observation":"The cited tutorial's pipelined v3 instantiates NUM_STAGES=2 and reports a benefit; neither it nor the tuning guide defines a generic three-cycle chain or a universal depth rule."}], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "challenge": {"alternative":"A stale phase can be harmless because parity repeats every other reuse.","check":"Inspect the PTX validity window for state/parity waits and the per-iteration example.","outcome":"A wait may identify only the current or immediately preceding phase, so incorrect per-object phase tracking can wait for the wrong reuse or invoke an invalid state."}, + "claim": "Incorrect mbarrier phase tracking can make a consumer wait for or observe the wrong reuse of a barrier.", + "claim_id": "pattern-pipeline-stalls-original-c003", + "claim_type": "semantics", + "covers": ["u003"], + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait","revision":"PTX ISA 9.0","locator":"state and phaseParity validity","observation":"test_wait/try_wait accepts state or parity for the current or immediately preceding phase, requiring phase state to be maintained for each reused object."}], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "claim": "Missing tcgen05.fence::after_thread_sync between a TMA wait and MMA issue lets MMA read SMEM before the TMA transfer is visible.", + "claim_id": "pattern-pipeline-stalls-original-c004", + "claim_type": "semantics", + "covers": ["u004"], + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait","revision":"PTX ISA 9.0","locator":"acquire wait ordering","observation":"A successful acquire wait provides visibility for prior cp.async.bulk operations associated with the barrier."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence","revision":"PTX ISA 9.0","locator":"tcgen05 fence description","observation":"after_thread_sync orders subsequent asynchronous tcgen05 operations after prior tcgen05/execution-ordering operations across threads; it is not the general TMA-to-SMEM visibility primitive asserted by the page."}], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "Using one tile serializes every warp on that tile's softmax or epilogue.", + "claim_id": "pattern-pipeline-stalls-original-c005", + "claim_type": "performance", + "covers": ["u005"], + "evidence": [{"url":"https://arxiv.org/html/2603.05451v1","revision":"FlashAttention-4 paper v1","locator":"sections 3.1.1-3.1.2","observation":"FA4 motivates two-tile overlap for its specific attention schedule, but does not establish that every single-tile kernel serializes all warps or that this is a general stall cause."}], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "After a TMA operation, hardware performs an mbarrier arrival in addition to the producer's manual arrival, so the producer over-arrives and releases the next stage stale.", + "claim_id": "pattern-pipeline-stalls-original-c006", + "claim_type": "semantics", + "covers": ["u006"], + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive","revision":"PTX ISA 9.0","locator":"arrive.expect_tx and complete_tx semantics","observation":"arrive.expect_tx performs the software arrival and increases tx-count; a TMA .complete_tx operation decrements transaction bytes rather than performing a second arrival. An unmatched extra arrive can still break a phase, but the stated mechanism is false."}], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "Three to five pipeline stages are typical on Blackwell, and warp specialization eliminates role-switching stalls.", + "claim_id": "pattern-pipeline-stalls-original-c007", + "claim_type": "performance", + "covers": ["u007", "u008"], + "evidence": [{"url":"https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu","revision":"3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"NUM_STAGES instantiation","observation":"The cited v3 uses two stages, which is a direct counterexample to the proposed typical range."},{"url":"https://gau-nernst.github.io/tcgen05/","revision":"retrieved 2026-08-09","locator":"warp specialization","observation":"The tutorial reports a source-specific speedup and distinct role loops, not elimination of all role-switching or pipeline stalls."}], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "challenge": {"alternative":"The candidate descriptions imply guaranteed speedups.","check":"Compare them with the exact TMEM ownership and FA4 scheduling contracts.","outcome":"The rows only identify the structural overlap each technique can test; the underlying sources require disjoint state and synchronization and do not guarantee speedup."}, + "claim": "Double-buffering can use separate TMEM output regions while MMA and output consumption overlap, and FA4's ping-pong schedule alternates two query/output tiles between MMA and softmax work.", + "claim_id": "pattern-pipeline-stalls-original-c008", + "claim_type": "semantics", + "covers": ["u009", "u010"], + "evidence": [{"url":"https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu","revision":"3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"TMEM output stages and epilogue handoff","observation":"The pinned tutorial source uses distinct output buffers so epilogue and MMA work can overlap."},{"url":"https://arxiv.org/html/2603.05451v1","revision":"FlashAttention-4 paper v1","locator":"section 3.1.2","observation":"FA4 assigns two output tiles per block and overlaps one tile's MMA with the other's softmax."}], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "claim": "The six-step diagnosis checklist gives generally correct profiler metrics, phase progression, TMA accounting, tcgen05 fencing, and stage-count advice.", + "claim_id": "pattern-pipeline-stalls-original-c009", + "claim_type": "code", + "covers": ["u011"], + "evidence": [{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"metric collection and Warp State Statistics","observation":"Available metrics and sampled stall states are tool/chip dependent and must be interpreted with scheduler issue evidence."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence","revision":"PTX ISA 9.0","locator":"after_thread_sync semantics","observation":"The checklist's mandatory after_thread_sync placement for TMA-to-MMA is a semantic conflation, and parity changes per barrier phase/reuse rather than after every wait globally."}], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "The tcgen05 tutorial progresses from a one-stage 62%-of-cuBLAS kernel to three-stage 70%, warp-specialized 80%, 2-SM 86%, and persistent-plus-CLC 98%, with the stated isolated causes.", + "claim_id": "pattern-pipeline-stalls-original-c010", + "claim_type": "performance", + "covers": ["u012", "u013", "u014", "u015", "u016"], + "evidence": [{"url":"https://gau-nernst.github.io/tcgen05/","revision":"retrieved 2026-08-09","locator":"final progression and version sections","observation":"The exact results are v1a 254.62, v2b 695.43, v3 pipelining 939.61, v4 warp specialization 1208.83, v5 2-SM 1302.29, and v6 static persistence 1475.93 TFLOP/s versus 1506.74 cuBLAS. The pinned v3 source uses two stages, and the author explicitly did not add CLC."}], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "Too many stages exceed a 228-KiB shared-memory budget.", + "claim_id": "pattern-pipeline-stalls-original-c011", + "claim_type": "compatibility", + "covers": ["u017"], + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2 Blackwell Tuning Guide","locator":"occupancy and shared memory","observation":"Compute capability 10.0 has a 228-KiB per-SM limit, but whether a stage count exceeds it depends on bytes per stage, fixed storage, per-block limits, and configuration; the unqualified statement has no tile payload."}], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "Phase-tracking bugs are notoriously hard to debug, so development assertions should be added.", + "claim_id": "pattern-pipeline-stalls-original-c012", + "claim_type": "semantics", + "covers": ["u018"], + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait","revision":"PTX ISA 9.0","locator":"state and phaseParity validity","observation":"A phase-aware wait is valid only for the current or immediately preceding phase, so the program must maintain the correct per-object phase across reuse."},{"url":"https://docs.nvidia.com/compute-sanitizer/ComputeSanitizer/index.html","revision":"Compute Sanitizer documentation, retrieved 2026-08-09","locator":"Racecheck and Synccheck synchronization tracing and deadlock detection","observation":"NVIDIA provides specialized development diagnostics for asynchronous-copy synchronization, barrier misuse, synchronization tracing, and deadlocks, confirming that these state protocols need explicit development-time checks beyond output timing."}], + "oracle": "web-primary", + "verdict": "verified", + "challenge": {"alternative":"Output comparison alone is enough, so assertions on phase invariants add no debugging value.","check":"Inspect the phase validity window and the failure modes addressed by NVIDIA's synchronization diagnostics.","outcome":"A stale or wrong per-object phase can wait on the wrong reuse or deadlock; checking the application's expected phase/ownership state during development localizes that protocol error before a final-value mismatch or hang."} + }, + { + "claim": "Pipelining is a waste of effort on memory-bound kernels.", + "claim_id": "pattern-pipeline-stalls-original-c013", + "claim_type": "performance", + "covers": ["u019"], + "evidence": [{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline","revision":"Nsight Compute 2025.3","locator":"roofline analysis","observation":"A memory-bound classification identifies a limiting ceiling; it does not imply that latency-hiding, issue overlap, or a staged transfer change cannot improve the measured kernel."}], + "oracle": "web-primary", + "verdict": "refuted" + } + ], + "commands": ["Inspect PTX ISA 9.0 mbarrier and tcgen05 ordering contracts", "Inspect Nsight Compute 2025.3 profiling semantics", "Inspect tcgen05 tutorial article and pinned v3/v6 source"], + "id": "pattern-pipeline-stalls", + "page_verdict": "refuted", + "path": "wiki/patterns/pipeline-stalls.md", + "residual_risk": "No Blackwell GPU or Nsight Compute installation is available locally, so no target-side profiler experiment was performed. The tutorial measurements remain author-reported.", + "scope": "All 19 baseline body units for hash ec21f3208711a7056069ab687fce39dd6bc96f6ee8fd093b42542af45799b592 were reviewed, including diagnostic semantics, synchronization claims, candidates, the checklist, and every progression number." +} diff --git a/verification/reviewed/original/pattern-register-pressure.json b/verification/reviewed/original/pattern-register-pressure.json new file mode 100644 index 000000000..195fa38bd --- /dev/null +++ b/verification/reviewed/original/pattern-register-pressure.json @@ -0,0 +1,17 @@ +{ + "claims":[ + {"claim":"Below-target occupancy plus local-memory spills establishes that high registers per thread are the limiting cause.","claim_id":"pattern-register-pressure-original-c001","claim_type":"performance","covers":["u001"],"evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"Launch Statistics and Occupancy","observation":"The profiler reports registers, spills, theoretical/achieved occupancy, and limiting resources separately; spilling does not prove registers are the occupancy limiter or that a target occupancy is useful."}],"oracle":"web-primary","verdict":"refuted"}, + {"challenge":{"alternative":"Accumulator, epilogue, or control state cannot contribute to register pressure because the compiler always shortens their lifetimes.","check":"Inspect WGMMA fragment ownership and compiled-source liveness/resource reporting.","outcome":"WGMMA D is a per-thread register fragment and extra live epilogue/control values can overlap its lifetime; exact allocation remains compiler/configuration dependent."},"claim":"Hopper WGMMA accumulators and simultaneously live epilogue or control-flow state can contribute to per-thread register use.","claim_id":"pattern-register-pressure-original-c002","claim_type":"semantics","covers":["u002","u003","u004"],"evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma","revision":"PTX ISA 9.0","locator":"WGMMA D operands","observation":"WGMMA accumulator fragments are distributed across warpgroup threads in registers."},{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"Launch Statistics and source analysis","observation":"Compiled registers and spills reflect live program state and can be correlated with source/generated code."}],"oracle":"web-primary","verdict":"verified"}, + {"claim":"SM100 TMEM is a dedicated 256-KB memory that simply moves accumulators out of registers.","claim_id":"pattern-register-pressure-original-c003","claim_type":"compatibility","covers":["u005"],"evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory","revision":"PTX ISA 9.0","locator":"organization and access","observation":"TMEM is organized as 128 lanes by 512 32-bit columns and tcgen05 D resides there, but access/lifecycle and logical capacity depend on instruction kind/packing; 'SM100 only' and a simple byte-memory migration omit target and access constraints."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"Warp specialization applies only to SM100+ and reduces each warp's register requirement.","claim_id":"pattern-register-pressure-original-c004","claim_type":"compatibility","covers":["u006"],"evidence":[{"url":"https://docs.nvidia.com/cutlass/media/docs/cpp/hopper_gemm.html","revision":"CUTLASS documentation, retrieved 2026-08-09","locator":"Hopper warp-specialized kernels","observation":"CUTLASS documents warp-specialized GEMM on Hopper, refuting SM100+ applicability; register effects depend on role code, compiler allocation, and optional register reallocation."}],"oracle":"web-primary","verdict":"refuted"}, + {"challenge":{"alternative":"Register-to-TMEM migration means replacing every accumulator register reference mechanically.","check":"Compare the migration page and PTX completion/load lifecycle.","outcome":"The cited approach explicitly replaces resident D storage while adding allocation, completion, collective load, wait, and epilogue handoffs."},"claim":"Register-to-TMEM migration is a systematic SM90-WGMMA to SM100-tcgen05 change for resident D accumulators, with new lifecycle and synchronization requirements.","claim_id":"pattern-register-pressure-original-c005","claim_type":"semantics","covers":["u007"],"evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions","revision":"PTX ISA 9.0","locator":"tcgen05 MMA, allocation, completion, and load","observation":"tcgen05 uses TMEM D and specifies allocation, asynchronous completion, collective load, wait, and deallocation mechanisms."}],"oracle":"web-primary","verdict":"verified"}, + {"claim":"A Hopper m64n256 FP32 WGMMA accumulator costs about 128 registers/thread, while Blackwell frees exactly all 128, uses zero accumulator registers, has 256 KB TMEM, and the largest 1-SM MMA always occupies half so double buffering is available.","claim_id":"pattern-register-pressure-original-c006","claim_type":"code","covers":["u008"],"evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma","revision":"PTX ISA 9.0","locator":"m64nNk16 D fragment","observation":"At N=256 the FP32 D fragment is N/2=128 registers per thread, validating only the Hopper arithmetic."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory","revision":"PTX ISA 9.0","locator":"organization and tcgen05 access","observation":"Resident D is in 128x512 TMEM cells, but epilogue tcgen05.ld destinations and other state still use registers; capacity/shape/packing constraints do not imply a universal half-capacity largest MMA or exactly 128 freed registers."}],"oracle":"web-primary","verdict":"refuted"}, + {"challenge":{"alternative":"Consumer Blackwell SM120 also exposes TMEM/tcgen05.","check":"Inspect PTX target notes for tensor-memory instructions.","outcome":"The tcgen05 TMEM instruction family is restricted to architecture-specific data-center targets such as sm_100a and is not a generic sm_120 path."},"claim":"The TMEM/tcgen05 path described is available on SM100-class data-center targets rather than SM120 consumer Blackwell.","claim_id":"pattern-register-pressure-original-c007","claim_type":"compatibility","covers":["u009"],"evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory","revision":"PTX ISA 9.0","locator":"Target ISA Notes","observation":"Tensor-memory instructions list architecture-specific data-center targets and do not provide the described SM120 consumer path."}],"oracle":"web-primary","verdict":"verified"}, + {"challenge":{"alternative":"TMEM is automatically managed at kernel entry/exit.","check":"Inspect alloc/dealloc semantics and exit rule.","outcome":"PTX requires collective allocation and deallocation, and exiting without deallocation is invalid."},"claim":"TMEM requires an explicit allocation and deallocation lifecycle.","claim_id":"pattern-register-pressure-original-c008","claim_type":"semantics","covers":["u010"],"evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"PTX ISA 9.0","locator":"allocation/deallocation lifecycle","observation":"PTX defines collective alloc/dealloc and requires deallocation before CTA exit."}],"oracle":"web-primary","verdict":"verified"}, + {"claim":"TMEM-to-register transfer latency is offset by the registers freed from accumulators.","claim_id":"pattern-register-pressure-original-c009","claim_type":"performance","covers":["u011"],"evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"PTX ISA 9.0","locator":"load and wait semantics","observation":"tcgen05.ld is asynchronous and requires its load wait before use, but the ISA gives no fixed latency or guarantee that freed resident state offsets its cost."}],"oracle":"web-primary","verdict":"refuted"} + ], + "commands":["Inspect Nsight Compute register/occupancy semantics","Inspect WGMMA fragment and TMEM lifecycle contracts","Inspect Hopper warp-specialization documentation"], + "id":"pattern-register-pressure","page_verdict":"refuted","path":"wiki/patterns/register-pressure.md", + "residual_risk":"No target compilation or occupancy experiment is possible without a CUDA toolkit/GPU. Register effects beyond ISA fragment arithmetic remain compiler-specific.", + "scope":"All 11 baseline units for hash 6fd15f8aae2d1ca78d038b8af9ec545760eac38cdbb46e46985f19c35a84a491 were reviewed, including the arithmetic code block and target claims." +} diff --git a/verification/reviewed/original/pattern-tail-effect.json b/verification/reviewed/original/pattern-tail-effect.json new file mode 100644 index 000000000..f97f8f409 --- /dev/null +++ b/verification/reviewed/original/pattern-tail-effect.json @@ -0,0 +1,18 @@ +{ + "claims":[ + {"claim":"Performance necessarily drops whenever total_tiles modulo num_SMs is nonzero because the last wave idles many SMs.","claim_id":"pattern-tail-effect-original-c001","claim_type":"performance","covers":["u001"],"evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"occupancy and workload distribution","observation":"Physical SM count is not automatically the one-wave worker count; residency, concurrent work, tile durations, scheduling, and the remainder magnitude determine observed impact."}],"oracle":"web-primary","verdict":"refuted"}, + {"challenge":{"alternative":"When N is divisible by M, N%M=0 means the final wave has zero active workers.","check":"Apply Euclidean division N=qM+r and account for the r=0 case.","outcome":"For r>0 the partial wave has r workers; for r=0 and N>0 the final wave is full, not empty."},"claim":"In an equal-duration, one-tile-per-worker model, N tiles on M workers require ceil(N/M) waves and a nonzero remainder r=N%M makes the final partial wave use r workers.","claim_id":"pattern-tail-effect-original-c002","claim_type":"semantics","covers":["u002"],"evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0","locator":"problem grid and persistent scheduling model","observation":"The documented model treats launched grid IDs as work acquired by workers, consistent with the page's simplified wave arithmetic when its assumptions hold."}],"oracle":"web-primary","verdict":"verified"}, + {"claim":"A static grid-stride schedule leaves remainder tiles on a few fixed SMs, while a non-persistent fixed grid has no dynamic block scheduling.","claim_id":"pattern-tail-effect-original-c003","claim_type":"semantics","covers":["u003","u004"],"evidence":[{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html","revision":"retrieved 2026-08-09","locator":"thread-block scheduling","observation":"CUDA schedules blocks onto available SMs; blockIdx-to-SM assignment is not fixed by a grid stride. Persistence changes repeated work acquisition, not whether the ordinary grid is dynamically scheduled by hardware."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"CLC is hardware dynamic scheduling in which idle SMs grab tiles on demand.","claim_id":"pattern-tail-effect-original-c004","claim_type":"semantics","covers":["u005"],"evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"PTX ISA 9.0","locator":"request semantics","observation":"A designated thread asynchronously requests cancellation of an unspecified not-yet-started block/cluster and processes a successful response; SMs do not autonomously grab arbitrary tiles."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"Persistent kernels use an SM-count grid, iterate over tiles, and have no wave boundary.","claim_id":"pattern-tail-effect-original-c005","claim_type":"performance","covers":["u006"],"evidence":[{"url":"https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html","revision":"retrieved 2026-08-09","locator":"fixed number of thread blocks","observation":"Persistent launch size is a software choice and repeated work still terminates with a final subset; persistence does not guarantee one CTA per physical SM or removal of all tail behavior."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"Raster order and swizzle patterns improve worker distribution.","claim_id":"pattern-tail-effect-original-c006","claim_type":"performance","covers":["u007"],"evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0","locator":"rasterization and swizzle","observation":"These are software coordinate transforms applied to work IDs; they can affect locality but do not create IDs or guarantee a more even worker-duration distribution."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"A B200 has 142 SMs; for 150 tiles CLC turns a 142+8 two-wave launch into one persistent wave with all SMs busy and causes the tutorial's 86%-to-98% result.","claim_id":"pattern-tail-effect-original-c007","claim_type":"code","covers":["u008"],"evidence":[{"url":"https://gau-nernst.github.io/tcgen05/","revision":"retrieved 2026-08-09","locator":"persistent section and results","observation":"The author's Modal B200 has 148 SMs, v6 uses static persistence without CLC, overlap remains imperfect, and the 86.43%-to-97.96% cumulative step includes other pipeline changes."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"PTX ISA 9.0","locator":"work semantics","observation":"CLC cannot turn eight independent IDs into 142 or 148 independent pieces of work."}],"oracle":"web-primary","verdict":"refuted"}, + {"claim":"Tail effects are significant only for tile counts below four times the SM count.","claim_id":"pattern-tail-effect-original-c008","claim_type":"performance","covers":["u009"],"evidence":[{"url":"https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html","revision":"Nsight Compute 2025.3","locator":"workload distribution","observation":"No four-wave cutoff is defined; impact depends on remainder, wave duration, residency, imbalance, and whole-kernel fraction."}],"oracle":"web-primary","verdict":"refuted"}, + {"challenge":{"alternative":"The absolute last-wave cost grows with problem size, so it is not amortized.","check":"For equal-duration work, divide at most one partial-wave duration by q full waves plus the final wave.","outcome":"The fractional contribution is bounded by roughly one over the number of waves and shrinks as q grows, under the simplified assumptions."},"claim":"For equal-duration independent tiles and fixed worker capacity, a single final partial wave is fractionally amortized as the number of full waves becomes large.","claim_id":"pattern-tail-effect-original-c009","claim_type":"performance","covers":["u010"],"evidence":[{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0","locator":"data-parallel tile scheduling","observation":"The scheduler model supports the independent-work assumptions; the amortization follows directly from the wave-count arithmetic and is explicitly scoped."}],"oracle":"web-primary","verdict":"verified"}, + {"claim":"CLC is available only on SM100 datacenter GPUs.","claim_id":"pattern-tail-effect-original-c010","claim_type":"compatibility","covers":["u011"],"evidence":[{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"PTX ISA 9.0","locator":"Target ISA Notes","observation":"The instruction requires sm_100 or higher and its multicast qualifier explicitly supports sm_120a and the SM120 family."}],"oracle":"web-primary","verdict":"refuted"} + ], + "commands":["Audit wave arithmetic and CUDA block scheduling","Inspect PTX/CUDA CLC semantics and targets","Inspect tutorial hardware/result scope"], + "id":"pattern-tail-effect","page_verdict":"refuted","path":"wiki/patterns/tail-effect.md", + "residual_risk":"No GPU trace was available; the equal-duration model is analytical and real duration/residency must be measured.", + "scope":"All 11 baseline units for hash 796c1dd09a9e76eff0996bf41283998fce447e492dd300571596ec89ad147ed9 were audited, including arithmetic, scheduler, compatibility, and example claims." +} diff --git a/verification/reviewed/original/technique-cache-policy.json b/verification/reviewed/original/technique-cache-policy.json new file mode 100644 index 000000000..2dc2c02ed --- /dev/null +++ b/verification/reviewed/original/technique-cache-policy.json @@ -0,0 +1,106 @@ +{ + "claims": [ + { + "claim": "L1 eviction qualifiers control cache admission and are critical whenever a memory-bound kernel's L1 working set matters more than compute.", + "claim_id": "technique-cache-policy-original-c001", + "claim_type": "semantics", + "covers": ["u001"], + "evidence": [ + { + "locator": "9.7.9.8 ld, eviction priorities", + "observation": "The ISA says eviction policies may be applied during access; it does not define these qualifiers as mandatory admission controls or prescribe them by bottleneck category.", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "The PTX pattern makes A bypass L1, keeps B resident, and evicts C immediately, thereby preventing cache pollution.", + "claim_id": "technique-cache-policy-original-c002", + "claim_type": "code", + "covers": ["u002"], + "evidence": [ + { + "locator": "9.7.9.8 ld and 9.7.9.12 st", + "observation": "The displayed vector load/store forms are legal on SM100 with natural alignment, but L1 qualifiers are eviction priorities that may apply. Their names do not guarantee bypass, residence, immediate eviction, or performance.", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-st" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "The rank-1 NVFP4 GEMV submission used an aggressive A no_allocate policy for K=16384 and a relaxed policy for K=2048.", + "claim_id": "technique-cache-policy-original-c003", + "claim_type": "performance", + "covers": ["u003", "u004", "u005"], + "evidence": [ + { + "locator": "Post-event Observations", + "observation": "The cited retrospective reports A no_allocate, B evict_last, exact-K specializations, and caps among inspected solutions, but publishes no contestant code and does not assign the page's per-K qualifier policies to rank 1.", + "revision": "retrieved 2026-08-08", + "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/" + }, + { + "locator": "Public leaderboard and task directory", + "observation": "The public contest materials expose scores and a task contract, not contestant source or technique metadata.", + "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6", + "url": "https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "Cache policy plus PTX byte unpacking partly caused a 16x 443-to-27 microsecond gain and cache policy can dominate memory-bound kernels.", + "claim_id": "technique-cache-policy-original-c004", + "claim_type": "performance", + "covers": ["u006", "u007"], + "evidence": [ + { + "locator": "CUDA optimization 1 through 3", + "observation": "The 443-to-27 endpoints include coalescing, shared-memory changes, collaboration, reductions, per-thread tiles, vector width, hardware conversion, and PTX decode. No cache-policy-only run or dominant-lever result is reported.", + "revision": "retrieved 2026-08-08", + "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "Cache differentiation should be used for memory-bound kernels, clear streaming/reuse patterns, inputs larger than B200's 126 MB L2, and separate GEMM M/N loading patterns.", + "claim_id": "technique-cache-policy-original-c005", + "claim_type": "performance", + "covers": ["u008", "u009", "u010", "u011"], + "evidence": [ + { + "locator": "9.7.9.8 ld", + "observation": "The ISA defines legal hints but no input-size, tensor-role, GEMM-axis, or bottleneck rule that predicts benefit. Cache behavior depends on the exact address stream, reuse distance, concurrency, and compiled access path.", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld" + }, + { + "locator": "Attempts 8-12 and Takeaways", + "observation": "The author's task-specific conclusion is to profile the actual bottleneck and run controlled variants; workload categories alone did not predict beneficial transformations.", + "revision": "retrieved 2026-08-08", + "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + } + ], + "commands": [ + "Inspect archived PTX 9.0 ld/st eviction-priority grammar and semantics", + "Compare the cited author reports with the page's contestant and causal attributions", + "Check the pinned public contest boundary for source and technique visibility" + ], + "id": "technique-cache-policy", + "page_verdict": "refuted", + "path": "wiki/techniques/cache-policy.md", + "residual_risk": "No B200 or released rank-1 source was available; unsupported contestant attribution was therefore removed rather than guessed.", + "scope": "All 11 baseline body units and the frontmatter performance claim were checked against archived PTX, the cited reports, and pinned public contest materials." +} diff --git a/verification/reviewed/original/technique-cccl-memory-primitives.json b/verification/reviewed/original/technique-cccl-memory-primitives.json new file mode 100644 index 000000000..975041251 --- /dev/null +++ b/verification/reviewed/original/technique-cccl-memory-primitives.json @@ -0,0 +1,58 @@ +{ + "path": "wiki/techniques/cccl-memory-primitives.md", + "id": "technique-cccl-memory-primitives", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "technique-cccl-memory-primitives-original-c001", + "claim": "The two linked CCCL PRs are broadly reusable SM100/SM90 policy and dispatch sources for scan, top-k, fill, histogram, reduce and block load/store, including DSA TopK tuning.", + "claim_type": "compatibility", + "covers": ["u001"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/prs/cccl/PR-3559/diff.patch","revision":"CCCL PR 3559 captured merge 25523da2","locator":"changed files and sm100_tuning additions","observation":"PR 3559 specifically adds B200/SM100 exclusive-sum scan policies and dispatch/type plumbing; it does not cover top-k, fill, histogram or reduce."},{"path":"artifacts/prs/cccl/PR-6152/diff.patch","revision":"CCCL PR 6152 captured merge 3fb05826","locator":"complete 13-line debug-log fix","observation":"PR 6152 only fixes stale CUB_DEBUG_LOG formatting/variable names in DispatchTopK; it is not an SM100 tuning or DSA integration PR."}] + }, + { + "claim_id": "technique-cccl-memory-primitives-original-c002", + "claim": "The displayed PrimitivePolicy is a meaningful vectorization probe because ITEMS_PER_THREAD >= 4 implies vectorized memory access.", + "claim_type": "code", + "covers": ["u002"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/prs/cccl/PR-3559/key-files/cub/cub/device/dispatch/tuning/tuning_scan.cuh","revision":"CCCL PR 3559 captured merge 25523da2","locator":"sm100_tuning specializations and ScanPolicyWrapper","observation":"SM100 scan policies independently choose threads, items, delay constructor, load/store algorithms and cache modifier by input/accumulator/offset/operator classification. Item count alone neither specifies nor proves vectorized access."}] + }, + { + "claim_id": "technique-cccl-memory-primitives-original-c003", + "claim": "A transfer review should determine whether an upstream change has a distinct SM100 policy or merely inherits an earlier architecture's policy.", + "claim_type": "compatibility", + "covers": ["u003"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/prs/cccl/PR-3559/key-files/cub/cub/device/dispatch/tuning/tuning_scan.cuh","revision":"CCCL PR 3559 captured merge 25523da2","locator":"sm90_tuning, sm100_tuning and policy_hub Policy900/Policy1000","observation":"The source has explicit SM90 and SM100 policy layers; some SM100 types use new specializations while double falls back to an SM90 tuning."}], + "challenge": {"alternative":"The PR's B200 title means every path automatically uses a new SM100 policy.","check":"Follow type/operator classification through select_agent_policy100 and inspect explicit fallbacks.","outcome":"Coverage is selective, so the page's architecture-specific check is necessary."} + }, + { + "claim_id": "technique-cccl-memory-primitives-original-c004", + "claim": "Atomic or relaxed variants require determinism and tie-breaking validation before transfer.", + "claim_type": "semantics", + "covers": ["u004"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://nvidia.github.io/cccl/unstable/cccl/determinism.html","revision":"CCCL documentation, retrieved 2026-08-09","locator":"determinism definitions and TopK example","observation":"CCCL distinguishes selected-set determinism from output-order determinism for parallel algorithms, so transfer validation must state which result and tie behavior are required."},{"url":"https://nvidia.github.io/cccl/unstable/cub/environment.html","revision":"CCCL documentation, retrieved 2026-08-09","locator":"DeviceTopK execution environments","observation":"DeviceTopK exposes algorithm-specific determinism and ordering controls rather than promising one result order for every execution environment."}], + "challenge": {"alternative":"TopK ties and relaxed atomic ordering are harmless because any equal-valued item is semantically interchangeable.","check":"Compare set membership, output order, and application identity requirements under equal keys.","outcome":"CCCL treats these as distinct determinism contracts; an application that consumes indices or stable order must validate the required contract before transferring a relaxed or atomic variant."} + }, + { + "claim_id": "technique-cccl-memory-primitives-original-c005", + "claim": "For DSA TopK, radix/select cost should be isolated from score-computation cost.", + "claim_type": "performance", + "covers": ["u005"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/prs/tensorrt-llm/PR-13340/key-files/tensorrt_llm/_torch/attention_backend/sparse/dsa.py","revision":"TensorRT-LLM PR 13340 captured merge 897c4bff","locator":"sparse_attn_indexer logits dispatch and prefill/decode TopK selection","observation":"The pinned DSA path first computes indexer logits through separate MQA-logits kernels, then dispatches distinct prefill/decode TopK selection, making score computation and selection separable measured stages."}], + "challenge": {"alternative":"Only combined indexer latency matters because logits and TopK are fused into one inseparable operation.","check":"Trace the pinned DSA call path from score generation to selected indices.","outcome":"The implementation has separate logits and TopK dispatch boundaries, so isolating their costs is both possible and necessary to attribute a TopK-policy change."} + } + ], + "commands": ["inspect complete captured diffs and key files for CCCL PRs 3559 and 6152", "inspect official CCCL determinism and DeviceTopK execution-environment contracts", "trace pinned TensorRT-LLM DSA logits and TopK dispatch boundaries"], + "scope": "All five baseline units and linked source scopes were checked against the complete captured PR diffs/key files.", + "residual_risk": "No GPU benchmark was run. Determinism requirements are source-verified at the CCCL contract level, and DSA cost isolation is source-verified from a pinned implementation boundary rather than benchmarked on target hardware." +} diff --git a/verification/reviewed/original/technique-chunk-parallelism.json b/verification/reviewed/original/technique-chunk-parallelism.json new file mode 100644 index 000000000..7a8a4e536 --- /dev/null +++ b/verification/reviewed/original/technique-chunk-parallelism.json @@ -0,0 +1,124 @@ +{ + "claims": [ + { + "claim": "GatedDeltaNet, RetNet, and Mamba are all O(n) linear-attention variants whose naive implementations are sequential, and Blackwell TMEM permits larger chunks with better tensor-core utilization.", + "claim_id": "technique-chunk-parallelism-original-c001", + "claim_type": "semantics", + "covers": ["u001"], + "evidence": [ + { + "locator": "Abstract", + "observation": "TFLA concerns linear RNNs and mLSTM, says practical benefits require optimized kernels, and introduces within-chunk sequence parallelization. It does not define Mamba as linear attention or connect chunk size to Blackwell TMEM.", + "revision": "arXiv:2503.14376v3", + "url": "https://arxiv.org/abs/2503.14376v3" + }, + { + "locator": "training implementation", + "observation": "The pinned GatedDeltaNet training path uses chunkwise Triton kernels with a WY representation, not the page's generic additive state update; its architecture parameters do not imply a TMEM capacity rule.", + "revision": "NVlabs/GatedDeltaNet b53d6d3a161267432a79c1c04af69fa52bddc921", + "url": "https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "The displayed Triton kernel correctly implements chunk-parallel linear attention with sequential inter-chunk state propagation.", + "claim_id": "technique-chunk-parallelism-original-c002", + "claim_type": "code", + "covers": ["u002"], + "evidence": [ + { + "locator": "Triton programming model, program instances", + "observation": "Separate Triton program instances in one launch have no grid-wide execution order or synchronization. Every chunk here reads and writes the same State address, so the claimed previous-chunk dependency is a data race rather than a sequential propagation.", + "revision": "Triton language documentation, retrieved 2026-08-09", + "url": "https://triton-lang.org/main/programming-guide/chapter-01/introduction.html" + }, + { + "locator": "chunkwise training implementation", + "observation": "The pinned GDN implementation uses a multi-stage WY/chunkwise algorithm; the snippet has undefined offsets, shapes, masking, update_state, head/batch addressing, and state ordering and is not an executable equivalent.", + "revision": "NVlabs/GatedDeltaNet b53d6d3a161267432a79c1c04af69fa52bddc921", + "url": "https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "Chunks of 32 are a low-latency decode choice, chunks of 256-512 improve prefill tensor-core throughput, and TFLA uses recursive two-level tiling to allow arbitrary chunk sizes.", + "claim_id": "technique-chunk-parallelism-original-c003", + "claim_type": "performance", + "covers": ["u003", "u004", "u005"], + "evidence": [ + { + "locator": "Abstract and algorithm scope", + "observation": "TFLA introduces an additional level of sequence parallelization within each chunk and claims arbitrarily large chunks for its mLSTM algorithm. It does not prescribe 32 for decode, 256-512 for prefill, call the construction recursive, or establish Blackwell results.", + "revision": "arXiv:2503.14376v3", + "url": "https://arxiv.org/abs/2503.14376v3" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "Chunk parallelism should be used for all linear attention and recurrent-state models including Mamba, GatedDeltaNet, and the Delta Rule.", + "claim_id": "technique-chunk-parallelism-original-c004", + "claim_type": "compatibility", + "covers": ["u006", "u007"], + "evidence": [ + { + "locator": "Abstract and algorithm scope", + "observation": "TFLA establishes its two-level sequence-parallel construction for a specific linear-RNN/mLSTM formulation, not a universal rule for every recurrent model.", + "revision": "arXiv:2503.14376v3", + "url": "https://arxiv.org/abs/2503.14376v3" + }, + { + "locator": "chunkwise training implementation", + "observation": "GatedDeltaNet uses a model-specific WY-form chunkwise algorithm and layouts rather than the page's universal generic update.", + "revision": "b53d6d3a161267432a79c1c04af69fa52bddc921", + "url": "https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921" + }, + { + "locator": "selective_scan interface and implementation", + "observation": "Mamba exposes a selective-scan recurrence and implementation, so transfer requires a model-specific equivalent transformation and profitability test.", + "revision": "retrieved 2026-08-09", + "url": "https://github.com/state-spaces/mamba" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "challenge": { + "alternative": "The 3:1 ratio is an informal description rather than an exact model configuration.", + "check": "Expand the pinned layer_types pattern over num_hidden_layers.", + "outcome": "The 48-layer configuration repeats three linear-attention entries and one full-attention entry twelve times, yielding 36 and 12 layers." + }, + "claim": "Qwen3-Next uses a 3:1 hybrid ratio of Gated DeltaNet linear-attention layers to full-attention layers.", + "claim_id": "technique-chunk-parallelism-original-c005", + "claim_type": "compatibility", + "covers": ["u008"], + "evidence": [ + { + "locator": "config.json layer_types and num_hidden_layers", + "observation": "The immutable 48-layer configuration repeats three linear_attention entries followed by one full_attention entry.", + "revision": "Qwen3-Next-80B-A3B-Instruct 9c7f2fbe84465e40164a94cc16cd30b6999b0cc7", + "url": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/config.json" + } + ], + "oracle": "web-primary", + "verdict": "verified" + } + ], + "commands": [ + "Inspect TFLA v3 algorithm and scope", + "Inspect pinned GatedDeltaNet chunkwise implementation boundary", + "Discriminate the Triton snippet's inter-program state race", + "Expand the immutable Qwen3-Next layer pattern" + ], + "id": "technique-chunk-parallelism", + "page_verdict": "refuted", + "path": "wiki/techniques/chunk-parallelism.md", + "residual_risk": "No GPU execution was available. The universal prescription is refuted by the model-specific scope and distinct direct implementations; profitability for any concrete transformed model remains hardware- and workload-dependent.", + "scope": "All eight baseline body units and the frontmatter TMEM/chunk-size claim were checked against the cited papers, pinned repository, Triton execution model, and immutable model configuration." +} diff --git a/verification/reviewed/original/technique-double-buffering.json b/verification/reviewed/original/technique-double-buffering.json new file mode 100644 index 000000000..475f52d03 --- /dev/null +++ b/verification/reviewed/original/technique-double-buffering.json @@ -0,0 +1,188 @@ +{ + "path": "wiki/techniques/double-buffering.md", + "id": "technique-double-buffering", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "technique-double-buffering-original-c001", + "claim": "Double- or multi-buffering uses disjoint copies so a producer can fill one stage while a consumer reads another; a Blackwell kernel can combine SMEM operand staging with separate TMEM accumulator regions.", + "claim_type": "mechanism", + "covers": ["u001"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v6.cu","revision":"gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"NUM_STAGES SMEM arrays, BLOCK_N*2 TMEM allocation, mainloop/epilogue barriers","observation":"The pinned kernel has multiple SMEM stages and two disjoint BLOCK_N-wide TMEM accumulator regions with producer, MMA, and epilogue handoffs."}], + "challenge": {"alternative":"The two mechanisms cannot coexist because both consume the same storage or synchronization object.","check":"Trace the address spaces and barrier arrays in the pinned implementation.","outcome":"SMEM operand stages and TMEM accumulator regions are separately allocated and coordinated by distinct pipeline/barrier state."} + }, + { + "claim_id": "technique-double-buffering-original-c002", + "claim": "The CTA-visible TMEM address structure has 128 lanes by 512 columns of 32-bit cells; a symmetric 512-column allocation can place two 256-column accumulator regions at offsets 0 and 256.", + "claim_type": "mechanism", + "covers": ["u002"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"Tensor Memory organization and allocation","observation":"TMEM is organized as 128 lanes by 512 columns of 32-bit cells, and allocations reserve columns across all lanes; offsets 0 and 256 therefore name disjoint halves of a 512-column allocation."}], + "challenge": {"alternative":"The 256 KiB figure means one independently private physical array per CTA or mandates two equal buffers.","check":"Separate the ISA-visible structure from allocation policy and inspect the example's chosen offsets.","outcome":"The dimensions are architectural, but equal 256-column partitioning is an implementation choice; the claim is scoped accordingly."} + }, + { + "claim_id": "technique-double-buffering-original-c003", + "claim": "The displayed tmem_double_buffer_mainloop is a safe CUDA implementation of two-buffer TMEM accumulation and epilogue reuse.", + "claim_type": "code", + "covers": ["u003"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"collective allocation/deallocation and lifetime rules","observation":"The block contains no TMEM allocation or mandatory deallocation and does not establish publication of a returned address."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"same archive","locator":"tcgen05.ld collective execution, completion, and address mapping","observation":"TMEM readback is a warp-collective asynchronous instruction followed by wait::ld, not a per-row scalar load callable independently by lanes."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier","revision":"same archive","locator":"mbarrier phase and initialization semantics","observation":"Reusable barriers require correct expected-arrival counts and phase/parity tracking; the block initializes drained barriers to one despite claiming fourteen arrivals and reuses waits without tokens/phases."}] + }, + { + "claim_id": "technique-double-buffering-original-c004", + "claim": "Multiple SMEM copies of A/B tiles can overlap asynchronous TMA production with MMA consumption when each stage has a correct full/empty ownership protocol.", + "claim_type": "mechanism", + "covers": ["u004"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/prs/flashinfer/PR-2387/key-files/include/flashinfer/mamba/selective_state_update.cuh","revision":"flashinfer merge 18804cd51734cccf807356d017733bc757677f15","locator":"SharedStorageHorizontal and producer/consumer stage loops","observation":"The merged code uses numStages SMEM arrays with per-stage bar_empty/bar_full handoffs around TMA copies and consumption."}], + "challenge": {"alternative":"Multiple arrays alone guarantee overlap without a stage-state protocol.","check":"Remove or invert a full/empty handoff in the source control-flow graph.","outcome":"The producer would overwrite unread data or the consumer would read incomplete data; synchronization and ownership are essential parts of the verified claim."} + }, + { + "claim_id": "technique-double-buffering-original-c005", + "claim": "For TILE_M=128, TILE_N=256, TILE_K=64 and binary16 operands, one conceptual A/B stage occupies 16 KiB plus 32 KiB, so three stages occupy 144 KiB before barriers, padding, alignment, and other shared storage.", + "claim_type": "code", + "covers": ["u005"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python integer arithmetic for 128*64*2, 64*256*2, and three-stage total","environment":"Conda base, host CPU","reference":"binary16 is two bytes; compare exact byte totals","result":"16384 + 32768 = 49152 bytes per stage and 147456 bytes = 144 KiB for three stages.","negative_control":"Treating binary16 as four bytes doubles all totals and fails the stated example."}], + "challenge": {"alternative":"The struct is a complete drop-in shared-memory layout and its payload total is the allocation total.","check":"Account for barriers, padding, descriptor alignment, swizzled layouts, and placement in dynamic/static shared memory.","outcome":"Only the payload arithmetic and conceptual staging are established; the displayed type is not sufficient production code."} + }, + { + "claim_id": "technique-double-buffering-original-c006", + "claim": "Every fully optimized Blackwell GEMM uses simultaneous three-stage SMEM and two-buffer TMEM.", + "claim_type": "performance", + "covers": ["u006"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"first Blackwell GEMM tutorial stage","observation":"The official tutorial progression includes valid Blackwell GEMMs before the later pipelined variants; neither exact stage count nor double-buffer policy is a universal optimization requirement."}] + }, + { + "claim_id": "technique-double-buffering-original-c007", + "claim": "The displayed combined CUDA block and timeline form a complete safe implementation in which the previous tile's epilogue overlaps the next tile's MMA.", + "claim_type": "code", + "covers": ["u007"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v6.cu","revision":"gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"full kernel role loops, ready/empty barriers, parity, prologue and tail","observation":"The real implementation requires independent persistent role loops, phase-aware ready/empty handoffs, prologue seeding, read completion, final drain, and TMEM cleanup; the block leaves these as opaque helpers/comments and its per-warp common outer loop does not prove overlap."}] + }, + { + "claim_id": "technique-double-buffering-original-c008", + "claim": "SM90 has no TMEM, and WGMMA's D accumulator is a per-thread register fragment, so keeping two simultaneous live D fragments increases register demand.", + "claim_type": "mechanism", + "covers": ["u008"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"WGMMA D fragment tables","observation":"WGMMA D is explicitly distributed as register vectors among warpgroup threads; two simultaneously live vectors require twice that D-fragment storage."}], + "challenge": {"alternative":"Double buffering merely renames one register fragment and therefore has no live-range cost.","check":"Require simultaneous producer and consumer ownership of different output tiles.","outcome":"Overlap requires two disjoint live results; reuse of the same fragment would overwrite the result still being drained."} + }, + { + "claim_id": "technique-double-buffering-original-c009", + "claim": "For a Hopper m64n256 FP32 accumulator, two buffers consume only 64 registers (256 bytes) per thread, whereas two Blackwell TMEM buffers have zero register cost and dedicate all 256 KiB to accumulator space.", + "claim_type": "code", + "covers": ["u009"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"m64nNk16 f32 D fragment","observation":"Each thread holds N/2 f32 registers: 128 registers for N=256, hence 256 registers (1024 bytes) for two simultaneous fragments, not 64 (256 bytes)."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2 Blackwell Tuning Guide","locator":"register and TMEM resource limits","observation":"TMEM removes resident D from registers but addresses, descriptors, synchronization state, readback batches, and epilogue values still consume registers; TMEM can also serve non-accumulator uses."}] + }, + { + "claim_id": "technique-double-buffering-original-c010", + "claim": "The storage and access rows correctly contrast WGMMA register-resident D with tcgen05 TMEM-resident D that must be transferred to registers for an epilogue.", + "claim_type": "mechanism", + "covers": ["u010", "u013"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-multiply-accumulate-instructions-wgmma-mma","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"WGMMA D operand and fragments","observation":"D is a register operand distributed among threads."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"same archive","locator":"TMEM-to-register load","observation":"tcgen05.ld transfers a TMEM region into per-thread registers before ordinary scalar processing."}], + "challenge": {"alternative":"Both architectures expose D as an ordinary pointer-addressable buffer.","check":"Compare the operand classes and legal load operations.","outcome":"WGMMA names register fragments, while tcgen05 names a TMEM address and uses dedicated collective movement instructions."} + }, + { + "claim_id": "technique-double-buffering-original-c011", + "claim": "Doubling an accumulator costs exactly twice the Hopper registers but zero Blackwell registers.", + "claim_type": "performance", + "covers": ["u011"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"tcgen05.ld destination operands","observation":"TMEM readback places values in register vectors, so the resident D vector is absent but total register cost is not zero."}] + }, + { + "claim_id": "technique-double-buffering-original-c012", + "claim": "Accumulator double-buffering typically changes Hopper occupancy from two to one CTA per SM and has no Blackwell occupancy impact.", + "claim_type": "performance", + "covers": ["u012"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator","revision":"CUDA 13.0.2 Programming Guide","locator":"occupancy calculation","observation":"Resident CTAs depend on block size, compiled registers, shared memory, barriers, clusters, and allocation granularities; neither architecture implies the displayed transition."}] + }, + { + "claim_id": "technique-double-buffering-original-c013", + "claim": "The maximum practical accumulator is about 16K elements on Hopper and exactly 128 by 512 elements on Blackwell.", + "claim_type": "performance", + "covers": ["u014"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-multiply-accumulate-instructions-wgmma-mma","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"WGMMA instruction shapes","observation":"An instruction fragment is not a universal kernel accumulator maximum; kernels may compose instructions and distribute output ownership."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory","revision":"same archive","locator":"TMEM structure and packing","observation":"128 by 512 is the cell-address structure, not a universal count of logical accumulator elements; packing and descriptor shape depend on dtype/instruction kind."}] + }, + { + "claim_id": "technique-double-buffering-original-c014", + "claim": "TMEM double-buffering should always be used for any nontrivial Blackwell epilogue and its only cost is plentiful TMEM capacity.", + "claim_type": "performance", + "covers": ["u015"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2 Blackwell Tuning Guide","locator":"resource-dependent occupancy and TMEM","observation":"TMEM allocation can constrain residency and buffering also adds synchronization, address/lifetime complexity, and readback; profitability is kernel- and workload-dependent."}] + }, + { + "claim_id": "technique-double-buffering-original-c015", + "claim": "Every GEMM or attention mainloop should use SMEM multi-staging, with three stages as the universal default and four or five only for long high-latency K loops.", + "claim_type": "performance", + "covers": ["u016"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/prs/flashinfer/PR-2387/key-files/include/flashinfer/mamba/selective_state_update.cuh","revision":"flashinfer merge 18804cd51734cccf807356d017733bc757677f15","locator":"numStages selections near kernel dispatch","observation":"One merged kernel selects three stages in one path and min(totalStages,4) in another, demonstrating shape/path-specific policy rather than the page's universal rule."},{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"tutorial progression and stage construction","observation":"Official examples derive or configure stages from tile and resource policy rather than declaring three universal."}] + }, + { + "claim_id": "technique-double-buffering-original-c016", + "claim": "Simultaneous SMEM and TMEM buffering is the standard production Blackwell GEMM approach used by all CUTLASS and CuTe DSL kernels.", + "claim_type": "compatibility", + "covers": ["u017"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"multiple tutorial GEMM variants","observation":"CUTLASS supplies multiple Blackwell schedules and incremental examples; the sources establish concrete combined implementations, not universality across all kernels."}] + }, + { + "claim_id": "technique-double-buffering-original-c017", + "claim": "TMEM double-buffering always requires every accumulator to fit in exactly half of the 512 columns, and every FP32 tile wider than N=256 must be split.", + "claim_type": "compatibility", + "covers": ["u018"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory-allocation","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"column allocation and addressing","observation":"The ISA specifies allocation units and address structure, not symmetric two-way partitioning; implementations may use unequal regions, more/fewer buffers, packed formats, or decomposed ownership."}] + }, + { + "claim_id": "technique-double-buffering-original-c018", + "claim": "Compute capability 10.0 supports up to 228 KiB of shared memory per SM, so operand stages and epilogue scratch can compete for the shared-memory budget.", + "claim_type": "compatibility", + "covers": ["u019"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy","revision":"CUDA 13.0.2 Blackwell Tuning Guide","locator":"SM shared-memory capacity","observation":"The guide reports up to 228 KiB per SM for compute capability 10.0; all dynamic/static shared objects contribute to the relevant allocation constraints."}], + "challenge": {"alternative":"The 228 KiB figure applies equally to every Blackwell compute capability or is all available to one CTA without reservation/configuration limits.","check":"Read the architecture scope and distinguish per-SM capacity from per-block opt-in limits.","outcome":"The claim is valid only when scoped to compute capability 10.0 and as a shared budget, not a universal Blackwell per-block guarantee."} + }, + { + "claim_id": "technique-double-buffering-original-c019", + "claim": "The mbarrier handoff between TMEM buffers costs a universal few cycles per tile, making it proportionally worse for tiles with few K iterations.", + "claim_type": "performance", + "covers": ["u020"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"mbarrier semantics","observation":"The ISA specifies ordering, arrivals, phases, and waits but no fixed cycle latency; exposed cost depends on contention, polling, dependencies, schedule, and target."}] + } + ], + "commands": ["archived CUDA 13.0.2 PTX/tuning-guide inspection", "exact-revision author and CUTLASS source inspection", "FlashInfer PR artifact hash comparison against the full merge SHA", "conda-base byte arithmetic and negative control"], + "scope": "All 20 baseline units for body hash 787b870b0080a00805a0601a72d16367df1fd5ff21f1043c92acaf6e2d91a387 were reviewed; code blocks were judged both for their scoped arithmetic/concepts and for whether they establish a safe runnable lifecycle.", + "residual_risk": "No CUDA toolkit or SM90/SM100 GPU is installed locally. No throughput or instruction-latency claim is accepted without a pinned benchmark, and illustrative layouts are not treated as compiled kernels." +} diff --git a/verification/reviewed/original/technique-epilogue-fusion.json b/verification/reviewed/original/technique-epilogue-fusion.json new file mode 100644 index 000000000..b7ffada4a --- /dev/null +++ b/verification/reviewed/original/technique-epilogue-fusion.json @@ -0,0 +1,179 @@ +{ + "path": "wiki/techniques/epilogue-fusion.md", + "id": "technique-epilogue-fusion", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "technique-epilogue-fusion-original-c001", + "claim": "Epilogue fusion inherently means overlapping post-MMA work with ongoing MMA, and Blackwell normally assigns warps 2-15 to drain one TMEM half while warp 1 writes the other.", + "claim_type": "mechanism", + "covers": ["u001"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"warp-role dispatch and epilogue path","observation":"The official example assigns a 128-thread epilogue group (four warps), reads one completed TMEM accumulator, converts through registers/SMEM, and stores; fusion and concurrent next-tile MMA are separate scheduling choices."}] + }, + { + "claim_id": "technique-epilogue-fusion-original-c002", + "claim": "An SM100 tcgen05 accumulator resides in TMEM and an ordinary elementwise epilogue must first transfer its relevant values into registers through a legal TMEM copy/load operation.", + "claim_type": "mechanism", + "covers": ["u002"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"tcgen05.ld TMEM-to-register operation","observation":"The instruction collectively copies TMEM data into per-thread register vectors and requires load completion before use."}], + "challenge": {"alternative":"TMEM can be cast to a CUDA pointer and used directly by scalar arithmetic.","check":"Inspect PTX state spaces and legal TMEM movement instructions.","outcome":"TMEM is instruction-addressed and tcgen05.ld/copy is required before ordinary register computation."} + }, + { + "claim_id": "technique-epilogue-fusion-original-c003", + "claim": "The displayed epilogue_warp_fn is safe usable CUDA for fourteen warps to load TMEM scalars, apply scale/bias/ReLU, and store a tile.", + "claim_type": "code", + "covers": ["u003"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"shape, collective execution, mapping, and completion","observation":"There is no scalar tmem_load_f32(r,c) operation; loads are warp-collective shaped operations with mapped register vectors and wait::ld completion. The block also omits edge predicates and source/result layout mapping."}] + }, + { + "claim_id": "technique-epilogue-fusion-original-c004", + "claim": "TMEM double-buffering with a mandatory 256/256 column split is the key required mechanism for Blackwell epilogue fusion.", + "claim_type": "mechanism", + "covers": ["u004"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"TMEM allocation and epilogue","observation":"The official warp-specialized GEMM allocates 512 columns and performs a fused conversion/store epilogue without establishing the page's mandatory equal-half ping-pong design."}] + }, + { + "claim_id": "technique-epilogue-fusion-original-c005", + "claim": "The displayed mma_epilogue_overlap is a complete phase-correct two-region protocol.", + "claim_type": "code", + "covers": ["u005"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"expected arrivals and reusable phases","observation":"The epilogue-done barriers are initialized with count one while comments require all epilogue warps, and repeated waits have no token/parity state."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit","revision":"same archive","locator":"tcgen05 completion","observation":"A raw barrier arrival after an opaque MMA helper does not attach asynchronous MMA completion as tcgen05.commit does; allocation, load completion, tail drain, and deallocation are also absent."}] + }, + { + "claim_id": "technique-epilogue-fusion-original-c006", + "claim": "CUTLASS 4.5.0 defines composable epilogue fusion operation tags and a CollectiveBuilder that accepts a fusion operation or callbacks.", + "claim_type": "compatibility", + "covers": ["u006"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/operations.hpp","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"FusionOperation and LinComb* operation tags","observation":"The release defines linear combination, activation, row/column bias, auxiliary, block-scale, residual, and top-k softmax tags."},{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/collective_builder.hpp","revision":"same tag","locator":"CollectiveBuilder template","observation":"The final policy parameter accepts FusionOpOrCallbacks with a LinearCombination default."}], + "challenge": {"alternative":"The names are documentation-only and not accepted by the SM100 builder.","check":"Trace operation tags through sm100_builder.inl and sm100 callback specializations.","outcome":"SM100 builder specializations instantiate callbacks for supported operation tags; support remains combination-dependent rather than arbitrary."} + }, + { + "claim_id": "technique-epilogue-fusion-original-c007", + "claim": "CUTLASS 4.5.0 can fuse arbitrary element-wise operations without support or schedule constraints.", + "claim_type": "compatibility", + "covers": ["u006"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/builders/sm100_builder.inl","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"SM100 CollectiveBuilder specializations and static constraints","observation":"Support is selected by architecture, operator class, schedule, layouts, alignments, datatypes, tile and recognized fusion operation/callback combinations; it is not arbitrary."}] + }, + { + "claim_id": "technique-epilogue-fusion-original-c008", + "claim": "The displayed LinCombEltAct and Sm100EpilogueTmaWarpSpecialized typedefs are valid CUTLASS 4.5.0 code for a scale+bias+ReLU SM100 epilogue.", + "claim_type": "code", + "covers": ["u007"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/operations.hpp","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"LinCombEltAct template signature","observation":"The actual parameters are ActivationFn template, ElementOutput, ElementCompute, ElementSource, ElementScalar, and rounding; the shown order/types do not represent the claimed output/source policy."},{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue","revision":"same tag","locator":"full tree symbol search","observation":"No Sm100EpilogueTmaWarpSpecialized type exists; construction uses CollectiveBuilder, EpilogueTileAuto, EpilogueScheduleAuto and an optional fusion op/callback."}] + }, + { + "claim_id": "technique-epilogue-fusion-original-c009", + "claim": "The row labeled Scale + Bias gives the standard GEMM source epilogue equation y=alpha*acc+beta*C.", + "claim_type": "mechanism", + "covers": ["u008"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/operations.hpp","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"LinearCombination versus LinCombPerRowBias/LinCombPerColBias","observation":"alpha*acc+beta*C is a linear combination with a source tensor C; bias is a distinct row/column input in separate operation tags."}] + }, + { + "claim_id": "technique-epilogue-fusion-original-c010", + "claim": "Activation, output conversion/quantization, gated activation/product, online-softmax rescaling, and residual addition are legitimate categories of work that may be fused around a matrix/attention output when their data dependencies and supported implementation permit it.", + "claim_type": "mechanism", + "covers": ["u009", "u010", "u011", "u012", "u013"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/operations.hpp","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"activation, residual, bias, block-scale and softmax operation tags","observation":"The release directly implements several listed categories and marks source/residual/activation capabilities."},{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/activation.h","revision":"same tag","locator":"ReLU, SiLU and GELU functors","observation":"The activation functions named in the table are present as epilogue-compatible functors."}], + "challenge": {"alternative":"Every listed formula is automatically one CUTLASS SM100 visitor and therefore always legal/profitable.","check":"Search exact operation tags and builder constraints for each composition.","outcome":"Only the operation categories are established; exact combinations require supported callbacks or custom code and benchmarking."} + }, + { + "claim_id": "technique-epilogue-fusion-original-c011", + "claim": "Every Blackwell GEMM with a nontrivial epilogue should use fourteen epilogue warps, which are available by default, and fusion necessarily removes one launch and one global-memory round trip.", + "claim_type": "performance", + "covers": ["u014"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"epilogue CooperativeGroup and dispatch","observation":"This official example uses a 128-thread/four-warp epilogue group, not a default fourteen. Whether fusion removes a launch/intermediate depends on the unfused baseline and graph."}] + }, + { + "claim_id": "technique-epilogue-fusion-original-c012", + "claim": "Attention implementations can interleave softmax rescaling/output accumulation with tensor-core work when their schedule provides correct dependencies.", + "claim_type": "mechanism", + "covers": ["u015"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/abs/2407.08608","revision":"FlashAttention-3 paper","locator":"asynchrony and interleaving design","observation":"FA3 describes overlapping/interleaving GEMM and softmax/non-GEMM work on Hopper through warp specialization and instruction scheduling."}], + "challenge": {"alternative":"Softmax rescaling must wait for the entire attention matrix and cannot overlap any subsequent tensor-core work.","check":"Inspect the tiled online-softmax schedule and producer/consumer dependencies.","outcome":"The cited kernel operates tilewise and interleaves independent tensor/non-tensor work; exact next-tile timing remains implementation-specific."} + }, + { + "claim_id": "technique-epilogue-fusion-original-c013", + "claim": "Fusing FP32-to-FP8/FP16 output conversion can avoid materializing an FP32 intermediate in global memory.", + "claim_type": "mechanism", + "covers": ["u016"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/sm100_visitor_store_tma_warpspecialized.hpp","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"compute_quantized_value and store visitor","observation":"The SM100 visitor computes quantized output fragments before the output store instead of storing an FP32 intermediate tensor."}], + "challenge": {"alternative":"Conversion still necessarily writes and rereads an FP32 tensor between kernels.","check":"Trace accumulator fragment conversion into the visitor's output/store path.","outcome":"The conversion occurs in the epilogue path before the final output write, so no separate FP32 global intermediate is required."} + }, + { + "claim_id": "technique-epilogue-fusion-original-c014", + "claim": "An epilogue may read a TMEM accumulator only after the corresponding tcgen05 MMA work completes; synchronization must also prevent overwrite before all reads complete.", + "claim_type": "mechanism", + "covers": ["u017"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"MMA completion and operand lifetimes","observation":"Commit associates asynchronous work completion with an mbarrier; destination/source lifetimes prohibit conflicting access before completion."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"same archive","locator":"load completion and source lifetime","observation":"TMEM source data must remain valid until the collective load completes."}], + "challenge": {"alternative":"Disjoint warp roles or __syncthreads alone makes incomplete asynchronous work visible.","check":"Distinguish thread synchronization from async-proxy completion and source lifetime.","outcome":"Role separation does not complete asynchronous operations; explicit completion edges are required."} + }, + { + "claim_id": "technique-epilogue-fusion-original-c015", + "claim": "Fourteen simultaneous epilogue warps receive equal proportional shares of TMEM bandwidth, and sufficiently wide TILE_N therefore bottlenecks on TMEM read bandwidth.", + "claim_type": "performance", + "covers": ["u018"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"load semantics and shapes","observation":"The ISA specifies mappings and completion but no equal-share bandwidth model or TILE_N bottleneck threshold; the assumed fourteen-warp configuration is not architectural."}] + }, + { + "claim_id": "technique-epilogue-fusion-original-c016", + "claim": "A store-only epilogue wastes fourteen warps and should reduce CTA size or repurpose those exact warps for next-tile TMA prefetch.", + "claim_type": "performance", + "covers": ["u019"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"role groups and epilogue TMA-store pipeline","observation":"The official design uses four epilogue warps for conversion/staging/store; changing CTA roles affects collectives, pipeline participation, and occupancy and cannot be inferred from epilogue simplicity alone."}] + }, + { + "claim_id": "technique-epilogue-fusion-original-c017", + "claim": "Every file under artifacts/kernels/epilogue-fusion/full is verbatim upstream code.", + "claim_type": "compatibility", + "covers": ["u020"], + "verdict": "refuted", + "oracle": "host-experiment", + "evidence": [{"command":"inspect full/PROVENANCE.yaml and the first lines of every full artifact; compare nvfp4 file with raw merge revision","environment":"current worktree, 2026-08-09","reference":"Every full file should be upstream-verbatim if the page claim is true.","result":"nvfp4_scaled_mm_kernels.cu byte-matches vLLM merge ed7a29d9..., but tmem-load-into-registers-for-epilogue.cu declares itself an extracted block from a local blog card and is not upstream-verbatim.","negative_control":"The separately labeled variants file declares derived/not upstream, demonstrating the repository can distinguish modes."}] + }, + { + "claim_id": "technique-epilogue-fusion-original-c018", + "claim": "The page and its artifact bundle are returned by scripts/get_page.py with --include-code, and variant files carry the required derived/not-upstream header.", + "claim_type": "code", + "covers": ["u020", "u021", "u022"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python scripts/get_page.py technique-epilogue-fusion --include-code; inspect output and variant headers","environment":"Conda base, current worktree","reference":"Command should exit zero, include the page/artifacts, and variants should be labeled.","result":"Command exited zero with 628 lines and included artifact content; 01-double-buffered-tmem-epilogue-skeleton.cu starts with the required derived/not-upstream provenance line.","negative_control":"A nonexistent page ID is rejected by the retrieval script."}], + "challenge": {"alternative":"The command omits artifacts or treats the derived skeleton as upstream.","check":"Search captured output for artifact paths and inspect the first source line.","outcome":"Artifacts are included and the variant's provenance label is explicit; this does not rescue the separate false claim about every full file."} + } + ], + "commands": ["exact CUTLASS 4.5.0 source search", "PTX ISA 9.0 lifecycle inspection", "vLLM PR full-SHA and artifact byte-hash comparison", "conda-base page retrieval with provenance controls"], + "scope": "All 22 baseline units for body hash f37f59190f56273dbd2ec10059fd85e03af89ea72f6cf5e9898fb14e1f3fd450 were reviewed, including every code block and artifact-routing claim.", + "residual_risk": "No CUDA compiler or SM100 GPU is available locally. Unsupported scheduling and throughput claims were refuted rather than inferred from source structure." +} diff --git a/verification/reviewed/original/technique-external-source-map-research.json b/verification/reviewed/original/technique-external-source-map-research.json new file mode 100644 index 000000000..79b9484ff --- /dev/null +++ b/verification/reviewed/original/technique-external-source-map-research.json @@ -0,0 +1,193 @@ +{ + "path": "wiki/techniques/external-source-map-research.md", + "id": "technique-external-source-map-research", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "technique-external-source-map-research-original-c001", + "claim": "Once profiling has identified an edit family, searching a small upstream source repository for the measured mechanism and inspecting exact files is a valid way to find candidate implementations.", + "claim_type": "mechanism", + "covers": ["u001"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "/tmp/kernelwiki-ext-cfx", + "revision": "ColfaxResearch/cfx-article-src fbecfed88de2e4246f104a023188ba722937c5fc", + "locator": "tma/tma_copy.h, pipeline-gemm/sm90_gemm_multistage.cu, streamk/tile_scheduler.hpp, transpose-cute/", + "observation": "The pinned repository contains compact, mechanism-specific examples whose exact paths can be inspected after a profiler has narrowed the candidate class." + }, + { + "path": "/tmp/kernelwiki-ext-load-store", + "revision": "simveit/load_and_store 05d828cf910dd43f0053ddbbe4744218a06e9d7f", + "locator": "ld_matrix_x{1,2,4}.cu and st_matrix_x{1,2,4}.cu", + "observation": "The repository is a small set of direct ldmatrix/stmatrix instruction examples rather than a general optimization authority." + } + ], + "challenge": { + "alternative": "The source cards alone are sufficient implementation evidence, so exact upstream files do not need inspection.", + "check": "Compare the source-card summaries with the pinned repository file trees and implementations.", + "outcome": "Several source-card architecture and mechanism labels exceed the code's actual scope, while exact files discriminate what is present." + } + }, + { + "claim_id": "technique-external-source-map-research-original-c002", + "claim": "The displayed clone-and-grep snippet is reproducible enough to support exact implementation citations.", + "claim_type": "code", + "covers": ["u002"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "/tmp/kernelwiki-ext-cfx", + "revision": "ColfaxResearch/cfx-article-src fbecfed88de2e4246f104a023188ba722937c5fc", + "locator": "git revision and repository tree", + "observation": "The snippet clones the moving default branch and records neither this immutable revision nor an exact file; future results can change." + }, + { + "path": "/tmp/kernelwiki-ext-load-store/README.md", + "revision": "simveit/load_and_store 05d828cf910dd43f0053ddbbe4744218a06e9d7f", + "locator": "repository scope and six .cu files", + "observation": "This repository only demonstrates ldmatrix/stmatrix, so the shared broad regular expression does not establish the page's wider TMA, swizzle, Stream-K, or vector-width scope." + } + ] + }, + { + "claim_id": "technique-external-source-map-research-original-c003", + "claim": "Load/store and transpose examples are relevant candidate sources after profiling identifies long-scoreboard or sector-utilization symptoms, without proving that vector width or layout is the cause.", + "claim_type": "mechanism", + "covers": ["u003", "u004"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "/tmp/kernelwiki-ext-transpose", + "revision": "simveit/effective_transpose 994b2b5acaa67f80e411df3e8274b6ae13fd1949", + "locator": "transpose_naive.cu, transpose_swizzle.cu and transpose_swizzle_batched.cu", + "observation": "The pinned SM90a examples compare TMA transpose variants and explicitly encode 128-byte tensor-map swizzling." + }, + { + "path": "/tmp/kernelwiki-ext-nvidia-samples", + "revision": "NVIDIA-developer-blog/code-samples 3350d216083a902ccbf5b31665e3b82096a75b55", + "locator": "series/cuda-cpp/coalescing-global/coalescing.cu and series/cuda-cpp/transpose/transpose.cu", + "observation": "The source tree contains direct coalescing and shared-memory transpose examples, including a bank-conflict-free padded layout." + } + ], + "challenge": { + "alternative": "Those profiler symptoms uniquely diagnose a layout or vector-width defect.", + "check": "Inspect whether the repositories define profiler-counter causality or instead provide candidate mechanisms.", + "outcome": "They provide implementation examples but no symptom-to-cause theorem; the page only recommends searching before editing." + } + }, + { + "claim_id": "technique-external-source-map-research-original-c004", + "claim": "The Colfax repositories contain Hopper pipelined GEMM and TMA/barrier examples relevant to investigating barrier or TMA-wait stalls before changing stages or producer/consumer roles.", + "claim_type": "mechanism", + "covers": ["u005", "u006"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "/tmp/kernelwiki-ext-cfx", + "revision": "ColfaxResearch/cfx-article-src fbecfed88de2e4246f104a023188ba722937c5fc", + "locator": "pipeline-gemm/README.md, pipeline-gemm/sm90_gemm_multistage.cu and tma/tma_copy.h", + "observation": "The pinned files implement SM90 TMA copies, transaction barriers, multistage GEMM, and a warp-specialized alternative." + }, + { + "path": "/tmp/kernelwiki-ext-colfax-cutlass/src/fmha-pipeline", + "revision": "ColfaxResearch/cutlass-kernels 84f0802e2b4a1bf068ac70359f20ffdb368c8f6a", + "locator": "README.md and fmha_pipe_ws.h", + "observation": "The source documents and implements SM90a software pipelining with optional warp specialization and CUTLASS PipelineTmaAsync." + } + ], + "challenge": { + "alternative": "The examples establish that changing stage count or role split will improve any kernel with these stalls.", + "check": "Inspect the stated target/version and configuration scope of each example.", + "outcome": "The sources are Hopper-specific candidate implementations and make no portable performance guarantee; the page says to search them before changing code." + } + }, + { + "claim_id": "technique-external-source-map-research-original-c005", + "claim": "The pinned cfx repository contains persistent data-parallel and Stream-K schedulers relevant to investigating tail waves before adding shape-specific dispatch.", + "claim_type": "mechanism", + "covers": ["u007", "u008"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "/tmp/kernelwiki-ext-cfx/streamk", + "revision": "ColfaxResearch/cfx-article-src fbecfed88de2e4246f104a023188ba722937c5fc", + "locator": "tile_scheduler.hpp classes and cli_options.h scheduler values", + "observation": "The source implements SingleTileScheduler, DataParallelPersistentTileScheduler, and StreamKPersistentTileScheduler, selectable as non-persistent, persistent, Stream-K, or heuristic modes." + } + ], + "challenge": { + "alternative": "A tail wave proves that a persistent or Stream-K scheduler is the best fix.", + "check": "Inspect whether the repository connects profiler tail metrics to a universal scheduler choice.", + "outcome": "It exposes alternatives and shape controls rather than a universal choice; the page treats them as candidates to inspect." + } + }, + { + "claim_id": "technique-external-source-map-research-original-c006", + "claim": "Citing a local source page plus an exact upstream path, commit, or URL is sufficient implementation provenance.", + "claim_type": "other", + "covers": ["u009"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "plan.md", + "revision": "repository remediation contract", + "locator": "sections 5 and 8", + "observation": "Evidence must record a revision and an exact section, symbol, line, or table locator; a moving repository URL by itself is insufficient." + }, + { + "path": "sources/blogs/colfax-cutlass-kernels.md", + "revision": "baseline body 4fa361c619785bf4e42b8fd58ce26e98c998de467797e6cea6b778a328fe3184", + "locator": "frontmatter and body", + "observation": "The local summary assigns SM100 and persistent/tile-scheduling scope although the pinned source is SM90a and contains no such implementation, demonstrating why the source page is only a lead." + } + ] + }, + { + "claim_id": "technique-external-source-map-research-original-c007", + "claim": "The linked repositories provide TMA, swizzle, Stream-K, persistence, and load/store mechanics on both Hopper- and Blackwell-era kernels.", + "claim_type": "compatibility", + "covers": ["u001"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [ + { + "path": "/tmp/kernelwiki-ext-cfx", + "revision": "ColfaxResearch/cfx-article-src fbecfed88de2e4246f104a023188ba722937c5fc", + "locator": "pipeline-gemm, streamk, tma and transpose-cute build files/source symbols", + "observation": "The relevant modern examples compile for SM90a and use SM90 TMA/GMMA facilities; no SM100 target or tcgen05 implementation appears." + }, + { + "path": "/tmp/kernelwiki-ext-colfax-cutlass", + "revision": "ColfaxResearch/cutlass-kernels 84f0802e2b4a1bf068ac70359f20ffdb368c8f6a", + "locator": "FMHA READMEs, compile scripts and CUTLASS GEMM source", + "observation": "The repository explicitly targets Hopper SM90/SM90a with CUDA 12.2/12.3 and SM90 TMA/WGMMA." + }, + { + "path": "/tmp/kernelwiki-ext-transpose", + "revision": "simveit/effective_transpose 994b2b5acaa67f80e411df3e8274b6ae13fd1949", + "locator": "README.md and Makefile", + "observation": "The repository calls itself a Hopper transpose example and compiles with -arch=sm_90a." + }, + { + "path": "/tmp/kernelwiki-ext-load-store", + "revision": "simveit/load_and_store 05d828cf910dd43f0053ddbbe4744218a06e9d7f", + "locator": "README.md and Makefile", + "observation": "The repository is six ldmatrix/stmatrix examples compiled for sm_90a, not a Blackwell source map." + } + ] + } + ], + "commands": [ + "git ls-remote each linked repository and inspect pinned shallow clones", + "rg exact architecture, mechanism and scheduler symbols in each pinned tree" + ], + "scope": "All nine baseline body units, frontmatter architecture claims, and all five linked source-card scopes were checked against exact upstream repository revisions.", + "residual_risk": "No GPU benchmark was run. The verified workflow claims are candidate-discovery rules, not claims that a profiler symptom proves a root cause or that a transferred implementation improves performance." +} diff --git a/verification/reviewed/original/technique-fine-grained-quantization.json b/verification/reviewed/original/technique-fine-grained-quantization.json new file mode 100644 index 000000000..c19c6e091 --- /dev/null +++ b/verification/reviewed/original/technique-fine-grained-quantization.json @@ -0,0 +1,173 @@ +{ + "path": "wiki/techniques/fine-grained-quantization.md", + "id": "technique-fine-grained-quantization", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "technique-fine-grained-quantization-original-c001", + "claim": "Fine-grained scaling prevents an outlier from destroying tensor precision, and Blackwell universally uses native UE8M0 while Hopper requires Nc=128 CUDA-core promotion.", + "claim_type": "compatibility", + "covers": ["u001"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [ + {"url":"https://arxiv.org/abs/2412.19437v2","revision":"DeepSeek-V3 Technical Report v2","locator":"sections 3.3.2 and 3.5.2","observation":"The paper presents 1x128/128x128 scaling and Nc=128 as its scoped FP8 training design; it does not guarantee that local scaling prevents all outlier-induced error or prescribe this Hopper implementation universally."}, + {"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"block-scale valid-combination table","observation":"Blackwell block-scaled tcgen05 supports multiple kinds and both UE8M0 and UE4M3 combinations, not only UE8M0."} + ] + }, + { + "claim_id": "technique-fine-grained-quantization-original-c002", + "claim": "The scaling diagram's 128x128 FP32 factor has 0.006% overhead and a 1x128 activation group captures a per-channel distribution.", + "claim_type": "code", + "covers": ["u002"], + "verdict": "refuted", + "oracle": "host-experiment", + "evidence": [ + {"command":"Compute scale-count ratio and storage ratio for one FP32 factor per 128x128 FP8 payload block","environment":"deterministic host arithmetic","reference":"Distinguish factor count from bytes and distinguish a row-by-128-channel group from a channel-wide group.","result":"1/16384 is 0.0061035% as a count ratio, but 4/(16384*1) is 0.024414% storage overhead; a 1x128 group is one token/row across 128 channels, not a per-channel distribution.","negative_control":"Using a one-byte factor yields the stated 0.0061035% byte ratio, confirming that the missing factor width causes the ambiguity."} + ] + }, + { + "claim_id": "technique-fine-grained-quantization-original-c003", + "claim": "DeepSeek's fine-grained FP8 scheme uses 1x128 activation groups and 128x128 weight groups.", + "claim_type": "semantics", + "covers": ["u003"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/abs/2412.19437v2","revision":"DeepSeek-V3 Technical Report v2","locator":"section 3.3.2 and Appendix B.2","observation":"The report states tile-wise 1x128 activation scaling and block-wise 128x128 weight scaling."}], + "challenge": {"alternative":"Both operands use the same 128x128 block.","check":"Read the separate activation and weight granularity definitions.","outcome":"The paper explicitly assigns different geometries to the two operands."} + }, + { + "claim_id": "technique-fine-grained-quantization-original-c004", + "claim": "The displayed deepgemm_fp8_tile function is a representative DeepGEMM CUDA implementation with the stated scale tensors and FP32 output accumulator.", + "claim_type": "code", + "covers": ["u004"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/kernels/deepgemm/full/sm90_fp8_gemm_1d1d.cuh","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f, byte-verified","locator":"BLOCK_K assertion and math warp-group loop","observation":"The upstream kernel uses WGMMA descriptors, staged TMA scale loads, float accum/final_accum arrays, and a templated epilogue. The displayed function uses undefined types/constants/partial and is neither upstream nor compilable."}] + }, + { + "claim_id": "technique-fine-grained-quantization-original-c005", + "claim": "Hopper FP8 WGMMA accumulates with approximately FP22 precision and DeepGEMM universally promotes every four instructions.", + "claim_type": "semantics", + "covers": ["u005"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/abs/2412.19437v2","revision":"DeepSeek-V3 Technical Report v2","locator":"sections 3.3.2 and 3.5.2","observation":"The report characterizes H800 FP8 Tensor Core accumulation as retaining about 14 bits, not FP22, and describes Nc=128/four-WGMMA promotion for its specific scheme."}] + }, + { + "claim_id": "technique-fine-grained-quantization-original-c006", + "claim": "The displayed hopper_fp8_gemm_with_promotion function is a valid DeepGEMM implementation of Nc=128 promotion.", + "claim_type": "code", + "covers": ["u006"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/kernels/deepgemm/full/sm90_fp8_gemm_1d1d.cuh","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f, byte-verified","locator":"lines 250-323 accumulator, WGMMA, promotion and store path","observation":"The exact source differs materially and the sketch contains undefined types, dimensions, helper calls, scale indices, and partial/tail behavior; it cannot serve as reproducible code."}] + }, + { + "claim_id": "technique-fine-grained-quantization-original-c007", + "claim": "Nc=128 was selected because four WGMMA operations cover 128 K elements, bound relative error near 0.1%, make smaller intervals too costly, and make Nc=256 unacceptably inaccurate.", + "claim_type": "performance", + "covers": ["u007", "u008", "u009", "u010", "u011"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/abs/2412.19437v2","revision":"DeepSeek-V3 Technical Report v2","locator":"sections 3.3.2 and 3.5.2","observation":"The source supports Nc=128 and equivalence to four WGMMAs in the described configuration, but provides none of the page's 0.1%, 'too much overhead', or 'unacceptable Nc=256' thresholds."}] + }, + { + "claim_id": "technique-fine-grained-quantization-original-c008", + "claim": "DeepGEMM's pinned SM100 FP8 path uses native tcgen05 block scaling with packed UE8M0 factors instead of the SM90 final_accum promotion loop.", + "claim_type": "mechanism", + "covers": ["u012"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/kernels/deepgemm/full/sm100_fp8_gemm_1d1d.cuh","revision":"DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f, byte-verified","locator":"block-scaled descriptor, TMA/UTCCP scale path and mma_t::fma","observation":"The file selects float_ue8m0_t block scaling and TMEM scale operands and has no SM90 final_accum loop."}], + "challenge": {"alternative":"SM100 still performs the same CUDA-core promotion after every K block.","check":"Search the pinned SM100 kernel for final_accum and trace scale addresses into the UMMA wrapper.","outcome":"No final_accum promotion exists; the scales are operands of block-scaled UMMA."} + }, + { + "claim_id": "technique-fine-grained-quantization-original-c009", + "claim": "The UE8M0 pseudo-structure correctly decodes every byte as 2^(e-127), including all 256 encodings.", + "claim_type": "code", + "covers": ["u013"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#alternate-floating-point-data-formats","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"ue8m0 alternate floating-point format","observation":"Encoding 0xff is NaN, so the displayed exponent formula incorrectly treats it as a finite 2^128 value. PTX also specifies packed ue8m0x2 register use rather than this invented descriptor API."}] + }, + { + "claim_id": "technique-fine-grained-quantization-original-c010", + "claim": "The displayed tcgen05.mma PTX is valid native block-scale grammar with UE8M0 factors read directly from shared memory.", + "claim_type": "code", + "covers": ["u014"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"floating-point block-scale syntax","observation":"Valid grammar requires a block-scaled kind such as mxf8f6f4, the .block_scale qualifier, an instruction descriptor, two TMEM scale addresses, and enable-input-d. The displayed f8f6f4 form instead supplies nonexistent SMEM scale operands and a misplaced scale_d/mask sequence."}] + }, + { + "claim_id": "technique-fine-grained-quantization-original-c011", + "claim": "UE8M0 and E4M3 scale elements occupy 8 bits and FP32 scale elements occupy 32 bits.", + "claim_type": "semantics", + "covers": ["u015"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#alternate-floating-point-data-formats","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"ue8m0 and ue4m3 format definitions","observation":"Both alternate scale formats are byte-sized; FP32 is a 32-bit floating type."}], + "challenge": {"alternative":"The four-value packing makes each UE8M0 scale 32 bits.","check":"Distinguish element width from container width.","outcome":"Four byte-sized values can occupy one 32-bit container without changing element width."} + }, + { + "claim_id": "technique-fine-grained-quantization-original-c012", + "claim": "UE8M0 has the finite range 2^-127 through 2^128 and E4M3 scale values are accurately summarized as about 0 through 448 with 240 distinct values.", + "claim_type": "semantics", + "covers": ["u016", "u017"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#alternate-floating-point-data-formats","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"ue8m0 and ue4m3 definitions","observation":"UE8M0 reserves 0xff for NaN, invalidating the claimed finite 2^128 endpoint. PTX's unsigned UE4M3 scale encoding is not the signed E4M3 payload/range summarized by the table."}] + }, + { + "claim_id": "technique-fine-grained-quantization-original-c013", + "claim": "The compared recipes use 32-value MXFP4 groups, 16-value NVFP4 1D groups, and 128-element DeepGEMM SM90 activation groups.", + "claim_type": "semantics", + "covers": ["u018"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html","revision":"Transformer Engine 2.13","locator":"NVFP4 quantization and comparison with MXFP4","observation":"The documentation distinguishes NVFP4 block 16 from MXFP4 block 32."},{"url":"https://arxiv.org/abs/2412.19437v2","revision":"DeepSeek-V3 Technical Report v2","locator":"section 3.3.2","observation":"DeepSeek activation scaling is one scale per 128 channels for each token."}], + "challenge": {"alternative":"All low-precision formats use one universal block size.","check":"Compare the recipe-specific group definitions.","outcome":"The primary sources explicitly define different sizes and geometries."} + }, + { + "claim_id": "technique-fine-grained-quantization-original-c014", + "claim": "E4M3-scaled NVFP4 requires software decode on Blackwell, and FP32 scaling is categorically the best-precision option while UE8M0 is categorically coarser.", + "claim_type": "compatibility", + "covers": ["u019", "u020"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"mxf4nvf4 valid combinations","observation":"Blackwell tcgen05 natively accepts the block16 UE4M3 NVFP4 combination. Element format alone does not establish a universal error ranking across different grouping and quantizer recipes."}] + }, + { + "claim_id": "technique-fine-grained-quantization-original-c015", + "claim": "NVFP4's 1D recipe reconstructs an E2M1 payload with a block-of-16 E4M3 scale and one per-tensor FP32 scale.", + "claim_type": "code", + "covers": ["u021", "u022"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html","revision":"Transformer Engine 2.13","locator":"E2M1 format, two-level scaling and 1D grouping","observation":"The recipe is exactly payload times local block scale times global scale; the displayed function explicitly decodes only the selected low nibble."}], + "challenge": {"alternative":"NVFP4 uses only one scale or groups 32 payloads like MXFP4.","check":"Inspect both levels and the 1D group size in the versioned recipe.","outcome":"The source defines two levels and 16-value local groups."} + }, + { + "claim_id": "technique-fine-grained-quantization-original-c016", + "claim": "The three prescriptive use rules are generally validated: DeepGEMM geometry for 671B+ training, NVFP4 as highest precision versus MXFP4 native acceleration, and Nc=128 promotion for Hopper FP8 inference.", + "claim_type": "performance", + "covers": ["u023", "u024", "u025"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/abs/2412.19437v2","revision":"DeepSeek-V3 Technical Report v2","locator":"model and FP8 training sections","observation":"This validates one model/training design, not a general size threshold or inference rule."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"block-scale valid combinations","observation":"Both MXFP4 UE8M0 and NVFP4-compatible UE4M3 forms have native block-scaled instruction support; the ISA makes no highest-precision recommendation."}] + }, + { + "claim_id": "technique-fine-grained-quantization-original-c017", + "claim": "The caveats establish universal accuracy/performance orderings for scale format, block size and Nc, and say Blackwell native block scaling works only with UE8M0.", + "claim_type": "performance", + "covers": ["u026", "u027", "u028", "u029"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html","revision":"Transformer Engine 2.13","locator":"NVFP4/MXFP4 comparison","observation":"Finer grouping and fractional scales provide more representational freedom and metadata, but do not prove strictly better precision for every tensor or implementation."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"mxf4nvf4 block16 UE4M3 row","observation":"Native UE4M3 block scaling directly contradicts the UE8M0-only claim; the ISA supplies no Nc performance/accuracy curve."}] + } + ], + "commands": ["inspect byte-verified DeepGEMM SM90/SM100 FP8 sources", "check exact PTX ISA 9.0 block-scale grammar and valid combinations"], + "scope": "All 29 immutable baseline units and frontmatter compatibility claims were checked against DeepSeek-V3 v2, pinned DeepGEMM 891d57b, Transformer Engine 2.13, and PTX ISA 9.0.", + "residual_risk": "No GPU experiment was run. Unsupported universal accuracy and performance comparisons are refuted rather than replaced with estimates." +} diff --git a/verification/reviewed/original/technique-kernel-fusion.json b/verification/reviewed/original/technique-kernel-fusion.json new file mode 100644 index 000000000..977a350a8 --- /dev/null +++ b/verification/reviewed/original/technique-kernel-fusion.json @@ -0,0 +1,75 @@ +{ + "path": "wiki/techniques/kernel-fusion.md", + "id": "technique-kernel-fusion", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "technique-kernel-fusion-original-c001", + "claim": "Kernel fusion always eliminates intermediate global-memory round trips and is critical because MoE/attention pipelines generally use five to seven launches.", + "claim_type": "performance", + "covers": ["u001"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"Track A reference operation","observation":"The operation includes routing and two grouped GEMMs but defines observable semantics, not a launch count or guarantee that every intermediate remains on chip."},{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/operations.hpp","revision":"CUTLASS v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"fusion operation graph types","observation":"Fusion describes in-kernel expression composition; whether a specific unfused baseline would materialize an intermediate and whether the fused schedule avoids all traffic are separate implementation facts."}] + }, + { + "claim_id": "technique-kernel-fusion-original-c002", + "claim": "The displayed CUDA kernel is a valid fused dual-GEMM implementation using two independent 256-column TMEM allocations.", + "claim_type": "code", + "covers": ["u002"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"TMEM allocation, address and capacity rules","observation":"TMEM allocation is collective, returns a shared result, must obey legal column counts and lifecycle rules, and the SM has 512 columns total. The sketch's tmem_alloc/tmem_load/tcgen05_mma calls are invented and omit completion/free semantics."},{"url":"https://github.com/gpu-mode/reference-kernels/blob/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm/reference.py","revision":"gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c","locator":"dual-GEMM correctness reference","observation":"The operation requires two scaled matrix products and SwiGLU but does not prescribe two TMEM allocations or this instruction schedule."}] + }, + { + "claim_id": "technique-kernel-fusion-original-c003", + "claim": "vLLM uses the listed seven-kernel graph, SGLang the listed five-kernel graph, and an ideal one-to-two-launch implementation saves exactly 21.9% activation traffic.", + "claim_type": "performance", + "covers": ["u003", "u004", "u005"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","revision":"FlashInfer Track A benchmark definition checked 2026-08-08","locator":"operation definition and traces","observation":"The primary benchmark defines routing and expert computations but provides no framework launch counts, ideal launch topology, or 21.9% traffic result."},{"url":"https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/trace/templates/moe.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"MoE reference","observation":"The reference uses one logical W13 projection split for SwiGLU and does not support the page's gate/up launch decomposition or cross-framework counts."}] + }, + { + "claim_id": "technique-kernel-fusion-original-c004", + "claim": "SM100 has only 256 TMEM columns total, which directly limits the number of fused accumulators.", + "claim_type": "compatibility", + "covers": ["u006"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"TMEM organization","observation":"Each SM exposes 512 columns of 128 lanes, not 256 total. Accumulator feasibility also depends on mapping, datatype, shapes and simultaneous lifetimes."}] + }, + { + "claim_id": "technique-kernel-fusion-original-c005", + "claim": "Fusing a complex epilogue can increase register/resource pressure and reduce occupancy, so the resource impact must be measured.", + "claim_type": "performance", + "covers": ["u007"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator","revision":"CUDA 13.0.2 Programming Guide","locator":"resource-dependent occupancy","observation":"Registers and shared memory per block constrain resident blocks; adding live epilogue state can therefore affect occupancy, without guaranteeing that it does."}], + "challenge": {"alternative":"Fusion always lowers pressure because it deletes intermediate buffers.","check":"Distinguish global buffers from per-thread registers and per-block shared state used by the fused expression.","outcome":"Deleting a global tensor does not remove the extra values that must remain live inside the fused kernel."} + }, + { + "claim_id": "technique-kernel-fusion-original-c006", + "claim": "A fusion opportunity requires a dependency graph that is DAG-compatible with CTA scope.", + "claim_type": "mechanism", + "covers": ["u008"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py","revision":"FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6","locator":"routing and local-expert reference","observation":"The logical operation includes cross-token/expert selection and accumulation; an implementation may use multiple CTAs, multiple launches, atomics, clusters or workspace. 'DAG-compatible with CTA scope' is neither defined nor a necessary universal fusion criterion."}] + }, + { + "claim_id": "technique-kernel-fusion-original-c007", + "claim": "The fused-MoE and epilogue-fusion related links resolve to the intended pages.", + "claim_type": "other", + "covers": ["u009", "u010"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"test -f wiki/kernels/fused-moe.md && test -f wiki/techniques/epilogue-fusion.md","environment":"baseline worktree","reference":"Resolve relative links and inspect frontmatter IDs.","result":"Both files exist with IDs kernel-fused-moe and technique-epilogue-fusion.","negative_control":"wiki/kernels/not-fused.md does not exist."}], + "challenge": {"alternative":"One relative link escapes its directory or resolves to an unrelated page.","check":"Resolve from wiki/techniques and inspect target IDs.","outcome":"Both targets match their labels."} + } + ], + "commands": ["inspect pinned FlashInfer Track A reference and GPU Mode gated dual-GEMM reference", "resolve related links with a broken-target control"], + "scope": "All ten immutable baseline units and frontmatter TMEM claim were checked against PTX ISA 9.0 and pinned primary implementations/contracts.", + "residual_risk": "No GPU profile was run; launch-count, traffic and speed claims lacking a reproducible primary record are refuted rather than estimated." +} diff --git a/verification/reviewed/original/technique-persistent-kernels.json b/verification/reviewed/original/technique-persistent-kernels.json new file mode 100644 index 000000000..c5c67bba7 --- /dev/null +++ b/verification/reviewed/original/technique-persistent-kernels.json @@ -0,0 +1,142 @@ +{ + "path": "wiki/techniques/persistent-kernels.md", + "id": "technique-persistent-kernels", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "technique-persistent-kernels-original-c001", + "claim": "Persistent kernels always launch exactly one CTA per SM, and CLC replaces their software scheduler with a query that returns new work or lets a CTA cancel itself.", + "claim_type": "mechanism", + "covers": ["u001"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0 e406c186","locator":"Dynamic Scheduler rules and request flow","observation":"CLC launches the problem-sized grid; each worker processes its initial blockIdx then cancels a not-yet-launched ClcID and processes the returned coordinate. try_cancel does not cancel the caller, and persistent grid size is policy/resource dependent."}] + }, + { + "claim_id": "technique-persistent-kernels-original-c002", + "claim": "CLC can let a running compute-capability-10.0 worker acquire a not-yet-started grid coordinate after its initial blockIdx work, enabling hardware-assisted dynamic work redistribution.", + "claim_type": "mechanism", + "covers": ["u002"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0 e406c186","locator":"CLC dynamic scheduler rules","observation":"The documented worker handles blockIdx first and can cancel/receive an unlaunched ClcID for subsequent work."}], + "challenge": {"alternative":"CLC creates new tiles or assigns every tile through a separate hardware queue.","check":"Trace the initial and canceled-ID paths against the launched grid.","outcome":"Every work ID already exists in the grid and is handled either by ordinary launch or cancellation, exactly as scoped."} + }, + { + "claim_id": "technique-persistent-kernels-original-c003", + "claim": "The displayed persistent_gemm_clc loop correctly queries work first, retries when none exists, and then uses try_cancel for self-termination.", + "claim_type": "code", + "covers": ["u003"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"try_cancel/query semantics and restrictions","observation":"try_cancel is itself the asynchronous request to cancel another not-yet-started block/cluster; query decodes that response. After a thread observes failure, another request from that thread is undefined. The block reverses the protocol and invents self-cancellation."}] + }, + { + "claim_id": "technique-persistent-kernels-original-c004", + "claim": "The text control-flow block completely describes a CLC loop as repeated cancel/wait/query, returning immediately on failure.", + "claim_type": "code", + "covers": ["u004", "u005"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"16-byte response, mbarrier transaction, proxy fences, query and cluster rules","observation":"The sequence omits initial blockIdx work, expect-tx/publication/fences, the exact failure restriction, and cluster-rank coordinate derivation. It is not a complete protocol even though cancel-before-query is directionally correct."}] + }, + { + "claim_id": "technique-persistent-kernels-original-c005", + "claim": "The Hopper-style static-stride example maps CTA i to indices i + iteration*gridDim.x and converts a flat index into row-major tile coordinates.", + "claim_type": "code", + "covers": ["u006", "u007"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python enumerate static-stride assignments for multiple grid/tile counts and test uniqueness/coverage","environment":"Conda base, host CPU","reference":"Each valid flat tile index should occur exactly once and map by quotient/remainder.","result":"All indices [0,total_tiles) were covered once with tile_m=idx//tiles_n and tile_n=idx%tiles_n.","negative_control":"Using stride total_ctas-1 produced duplicate assignments."}], + "challenge": {"alternative":"The loop skips or duplicates tiles when total_tiles is not divisible by gridDim.x.","check":"Enumerate nondivisible sizes and compare the sorted union with the exact index range.","outcome":"The final iteration counts differ by at most one and the union remains exact."} + }, + { + "claim_id": "technique-persistent-kernels-original-c006", + "claim": "The comparison table accurately states that CLC assigns arbitrary tiles, automatically balances variable tile costs, minimizes ordinary tail waves, chains problems, uses try_cancel for self-termination, and applies a hardware swizzled raster.", + "claim_type": "mechanism", + "covers": ["u008", "u009", "u010", "u011", "u012", "u013"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0 e406c186","locator":"dynamic scheduler and software swizzle rules","observation":"CLC redistributes existing unlaunched ClcIDs to address uneven SM availability. CUTLASS applies swizzle/raster as a software coordinate transform; CLC neither chains distinct problems nor guarantees ordinary tail elimination."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"request/query result","observation":"try_cancel targets another not-yet-started grid entity; failure means no returned work, not caller cancellation."}] + }, + { + "claim_id": "technique-persistent-kernels-original-c007", + "claim": "CUTLASS 4.5.0 provides PersistentTileSchedulerSm100 and PipelineCLCFetchAsync integration for CLC-backed scheduling.", + "claim_type": "compatibility", + "covers": ["u014"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0 e406c186","locator":"CUTLASS implementation links and scheduler pipeline","observation":"The official tagged documentation identifies PersistentTileSchedulerSm100 and PipelineCLCFetchAsync and describes their producer/consumer roles."}], + "challenge": {"alternative":"The scheduler is a generic software queue with no CLC instruction path.","check":"Follow the linked scheduler/pipeline sources to the arch CLC wrappers.","outcome":"The fetch pipeline submits and decodes clusterlaunchcontrol operations for the SM100 scheduler."} + }, + { + "claim_id": "technique-persistent-kernels-original-c008", + "claim": "The displayed PersistentTileSchedulerSm100 class is a valid simplified CUTLASS 4.5.0 API with clc_init, clc_query_tile and clc_try_cancel helpers and a thread-0 CTA broadcast.", + "claim_type": "code", + "covers": ["u015"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp","revision":"CUTLASS 4.5.0 e406c186","locator":"PersistentTileSchedulerSm100 interface and CLC pipeline calls","observation":"The displayed helper API and init policy do not exist. Real code uses WorkTileInfo, get_current_work/fetch_next_work/advance_to_next_work, PipelineCLCFetchAsync state, cluster participation and software swizzle/raster transforms."}] + }, + { + "claim_id": "technique-persistent-kernels-original-c009", + "claim": "The tutorial isolates a 940-to-1476 TFLOP/s, 57-percent improvement caused by CLC persistence, specifically through eliminated tail, reduced launches and CLC-controlled L2 swizzle.", + "claim_type": "code", + "covers": ["u016", "u017", "u018", "u019", "u020", "u021"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v6.cu","revision":"gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"v6 persistent kernel and static tile assignment","observation":"v6 uses persistent static scheduling, not CLC."},{"path":"sources/blogs/tcgen05-tutorial.md","revision":"pinned source card verified against author article/commit","locator":"progression table and supported takeaways","observation":"939.61 is v3 pipelining and 1475.93 is v6 after warp specialization, 2-SM MMA and persistence; the article explicitly says CLC and threadblock swizzling were not added, so neither isolation nor causal breakdown follows."}] + }, + { + "claim_id": "technique-persistent-kernels-original-c010", + "claim": "Persistent kernels are most beneficial only above two-to-three tiles per SM, CLC chains grouped problems without launch gaps, and CLC naturally handles arbitrary variable-cost tiles such as triangular masks.", + "claim_type": "performance", + "covers": ["u022", "u023", "u024"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0 e406c186","locator":"motivation and dynamic scheduler limits","observation":"The documented motivation is uneven SM availability and redistribution of existing ClcIDs. It supplies no 2-3x threshold, multi-problem chaining, or cost-aware selection guarantee."}] + }, + { + "claim_id": "technique-persistent-kernels-original-c011", + "claim": "CLC is a compute-capability-10.0 Blackwell feature unavailable on Hopper, where persistent work scheduling is software-managed.", + "claim_type": "compatibility", + "covers": ["u025"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#cluster-launch-control","revision":"CUDA 13.0.2 Programming Guide","locator":"availability","observation":"CLC is documented for compute capability 10.0; Hopper compute capability 9.0 does not expose it."}], + "challenge": {"alternative":"Hopper supports the same PTX operation under a different name.","check":"Inspect architecture requirements for the CLC API/PTX instruction.","outcome":"The feature is gated to CC 10.0, so Hopper requires a software schedule or another mechanism."} + }, + { + "claim_id": "technique-persistent-kernels-original-c012", + "claim": "A failed CLC request represents a race and the same thread must retry until another CTA generates work.", + "claim_type": "mechanism", + "covers": ["u026"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"failed-request restriction","observation":"After a thread observes a failed request, a subsequent request from that thread is undefined; other CTAs do not generate new ClcIDs."}] + }, + { + "claim_id": "technique-persistent-kernels-original-c013", + "claim": "When independent tile count is below available SM count, a scheduler cannot create enough extra parallel tiles to fill the idle SMs, so added dynamic scheduling may not repay its complexity.", + "claim_type": "performance", + "covers": ["u027"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0 e406c186","locator":"problem-sized grid and ClcID conservation","observation":"CLC only redistributes existing grid IDs, so fewer independent tiles than SMs remain fewer work items."}], + "challenge": {"alternative":"CLC synthesizes extra K-splits or duplicate tiles to fill all SMs.","check":"Trace whether try_cancel returns anything outside the launched grid.","outcome":"It can return only an existing unlaunched ClcID; decomposition is a separate scheduler policy."} + }, + { + "claim_id": "technique-persistent-kernels-original-c014", + "claim": "The artifact directory contains declared upstream patch/source material, a labeled derived skeleton, matching recorded file hashes, and is retrievable with the shown get_page command.", + "claim_type": "code", + "covers": ["u028", "u029", "u030"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"sha256sum all artifact files; inspect provenance/header modes; byte-compare dense_gemm_persistent_prefetch.py to CUTLASS PR 2881 merge; run Conda-base get_page","environment":"current worktree/network, 2026-08-09","reference":"Hashes/modes and retrieval should agree with manifests and page.","result":"All declared hashes match; dense_gemm file byte-matches merge 3f4c086d09bd1dc55defb955862f333893bbb28b; variant is labeled derived/not upstream; retrieval exited zero.","negative_control":"A nonexistent page ID exits nonzero."}], + "challenge": {"alternative":"The derived skeleton is silently treated as upstream or the page command omits artifacts.","check":"Inspect first-line labels and captured retrieval output.","outcome":"Mode labeling is explicit and artifact content is included."} + } + ], + "commands": ["archived CLC/PTX and CUTLASS 4.5.0 documentation inspection", "exact author tutorial source comparison", "Conda-base static-stride enumeration", "artifact hash/provenance and retrieval controls"], + "scope": "All 30 baseline units for body hash 7d1d16d68cfae0d699d1b73b4d10ac71a12c878966081788134245f3861fc283 were reviewed, including all code, table, performance-attribution, and artifact claims.", + "residual_risk": "No CC10.0 GPU is available locally. No CLC performance claim is accepted without a controlled implementation-specific benchmark." +} diff --git a/verification/reviewed/original/technique-ping-pong-scheduling.json b/verification/reviewed/original/technique-ping-pong-scheduling.json new file mode 100644 index 000000000..d27d90d12 --- /dev/null +++ b/verification/reviewed/original/technique-ping-pong-scheduling.json @@ -0,0 +1,150 @@ +{ + "claims": [ + { + "claim": "FlashAttention-4 introduced a two-query-tile single-CTA ping-pong schedule that ensures its softmax warpgroup never waits for MMA.", + "claim_id": "technique-ping-pong-scheduling-original-c001", + "claim_type": "semantics", + "covers": ["u001"], + "evidence": [ + { + "locator": "section 3.1.2", + "observation": "The paper says FA4 follows a ping-pong schedule similar to FA3. It maximizes overlap between one tile's MMA and the other's softmax but does not claim a no-stall guarantee.", + "revision": "FlashAttention-4 paper v1, 2026-03-05", + "url": "https://arxiv.org/html/2603.05451v1" + }, + { + "locator": "warp IDs, q_stage, TMEM offsets, and pipeline construction", + "observation": "The pinned implementation defaults to q_stage=2, assigns separate four-warp softmax groups plus correction/MMA roles, and uses explicit multi-stage barriers; it does not expose the page's one-barrier loop.", + "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", + "url": "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "The displayed CUDA kernel is a valid two-tile FA4 ping-pong implementation with correct TMEM allocation, warpgroup roles, asynchronous MMA completion, and synchronization.", + "claim_id": "technique-ping-pong-scheduling-original-c002", + "claim_type": "code", + "covers": ["u002"], + "evidence": [ + { + "locator": "flash_fwd_sm100.py setup, kernel roles, and pipeline loops", + "observation": "The real CuTe DSL kernel uses one MMA warp, two four-warp softmax groups, a correction warpgroup, load/epilogue roles, one TMEM allocation with disjoint offsets, and multiple producer/consumer barriers. The snippet's helpers are undefined and its generic arrive/wait is not tied to tcgen05 completion or safe buffer ownership.", + "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", + "url": "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py" + }, + { + "locator": "tcgen05 allocation, MMA, commit, fence, and mbarrier contracts", + "observation": "TMEM lifecycle is collective and asynchronous MMA requires its documented completion/order protocol; an unrelated mbarrier arrive/wait pair does not establish completion.", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "challenge": { + "alternative": "The doubled tensor-core and unchanged exponential-throughput comparison is only qualitative marketing.", + "check": "Compare the paper's per-SM throughput figures for B200 and Hopper.", + "outcome": "The paper gives 8192 versus 4096 BF16 MMA ops/clock/SM and 16 exponential ops/clock/SM on both B200 and Hopper." + }, + "claim": "The FA4 paper models B200 BF16 tensor-core throughput as double Hopper while B200 exponential-unit throughput remains the same as Hopper.", + "claim_id": "technique-ping-pong-scheduling-original-c003", + "claim_type": "performance", + "covers": ["u003"], + "evidence": [ + { + "locator": "section 2.2 Shifting bottlenecks", + "observation": "The paper lists 8192 versus 4096 BF16 MMA operations per clock per SM and 16 exponential operations per clock per SM on B200, the same exponential rate as Hopper.", + "revision": "FlashAttention-4 paper v1, 2026-03-05", + "url": "https://arxiv.org/html/2603.05451v1" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "claim": "A single-tile schedule alternately leaves SFU and tensor cores idle, whereas ping-pong keeps both units 100% busy.", + "claim_id": "technique-ping-pong-scheduling-original-c004", + "claim_type": "performance", + "covers": ["u004", "u005"], + "evidence": [ + { + "locator": "sections 3.1.1-3.1.2", + "observation": "The paper gives a simplified roofline and says the schedule maximizes overlap; it explicitly notes the analysis omits resources and never claims universal exclusive idleness or measured 100% utilization of both units.", + "revision": "FlashAttention-4 paper v1, 2026-03-05", + "url": "https://arxiv.org/html/2603.05451v1" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "The ping-pong pattern itself accounts for FA4's 1605 TFLOPS/s and 71% BF16 result.", + "claim_id": "technique-ping-pong-scheduling-original-c005", + "claim_type": "performance", + "covers": ["u006"], + "evidence": [ + { + "locator": "Performance and Key Techniques", + "observation": "The blog reports up to 1605 TFLOPS/s and 71% for the complete FA4 forward design, which also uses partial software exponentials, conditional rescaling, TMEM partitioning, register tuning, and other pipeline choices; it provides no ping-pong-only attribution for the peak.", + "revision": "Tri Dao FA4 blog, 2026", + "url": "https://tridao.me/blog/2026/flash4/" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "Ping-pong should be used for compute-bound or SFU-heavy Blackwell attention kernels and is not useful on Hopper.", + "claim_id": "technique-ping-pong-scheduling-original-c006", + "claim_type": "compatibility", + "covers": ["u007", "u008", "u009"], + "evidence": [ + { + "locator": "section 3.1.2", + "observation": "The paper explicitly says FA4 follows a schedule similar to FA3, a Hopper kernel, refuting 'not useful on Hopper.' It motivates FA4 from a concrete balance but does not prescribe the technique solely from compute-bound or SFU-heavy labels.", + "revision": "FlashAttention-4 paper v1, 2026-03-05", + "url": "https://arxiv.org/html/2603.05451v1" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "The local full bundle is a verbatim full reference implementation of FA4 ping-pong scheduling and the derived variant faithfully demonstrates the overlap; the displayed query retrieves it.", + "claim_id": "technique-ping-pong-scheduling-original-c007", + "claim_type": "code", + "covers": ["u010", "u011", "u012"], + "evidence": [ + { + "path": "artifacts/kernels/ping-pong-scheduling/full/PROVENANCE.yaml", + "revision": "local provenance", + "locator": "origin_url, upstream_repo, source_pr_id, files", + "observation": "The full file comes from NVIDIA CUTLASS PR 2466 at merge 0e026982 and is a Blackwell FMHA backward kernel, not Dao-AILab FA4 forward source or a full implementation of its two-output-tile schedule." + }, + { + "path": "artifacts/kernels/ping-pong-scheduling/variants/01-two-tile-ping-pong-skeleton.cu", + "revision": "derived local variant", + "locator": "entire function", + "observation": "The skeleton calls wait_mma, then completes softmax A, and only afterward issues MMA B; its visible single-thread order does not implement the claimed overlap and all helpers are undefined." + } + ], + "oracle": "local-primary-source", + "verdict": "refuted" + } + ], + "commands": [ + "Inspect FA4 paper v1 pipeline and hardware model", + "Inspect pinned Dao-AILab SM100 forward q_stage, warp roles, TMEM offsets, and barriers", + "Inspect local full/variant provenance and visible execution order", + "Run the repository page query through conda base Python" + ], + "id": "technique-ping-pong-scheduling", + "page_verdict": "refuted", + "path": "wiki/techniques/ping-pong-scheduling.md", + "residual_risk": "The pinned FA4 implementation was source-inspected but not compiled or profiled because no CUDA toolkit or GPU is available. The paper/blog performance remains author-reported.", + "scope": "All 12 baseline body units, artifact references, code claims, hardware motivation, performance attribution, and architecture prescription were checked against direct paper, source, ISA, and provenance evidence." +} diff --git a/verification/reviewed/original/technique-pipeline-stages.json b/verification/reviewed/original/technique-pipeline-stages.json new file mode 100644 index 000000000..7bf66643a --- /dev/null +++ b/verification/reviewed/original/technique-pipeline-stages.json @@ -0,0 +1,162 @@ +{ + "path": "wiki/techniques/pipeline-stages.md", + "id": "technique-pipeline-stages", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "technique-pipeline-stages-original-c001", + "claim": "A three-to-five-stage SMEM pipeline hides global-memory latency entirely and is critical for high utilization on every Hopper/Blackwell kernel.", + "claim_type": "performance", + "covers": ["u001"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-best-practices-guide/index.html#asynchronous-copy-from-global-memory-to-shared-memory","revision":"CUDA 13.0.2 Best Practices Guide","locator":"asynchronous-copy pipeline tradeoffs","observation":"Overlap and benefit depend on work balance, issue/completion, occupancy, resource use and workload; documentation provides no universal stage range or complete-latency-hiding guarantee."}] + }, + { + "claim_id": "technique-pipeline-stages-original-c002", + "claim": "The tutorial progression is 695 TFLOP/s without pipelining, 940 with three stages, and 1476 immediately after adding warp specialization.", + "claim_type": "code", + "covers": ["u002", "u003"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"sources/blogs/tcgen05-tutorial.md","revision":"Gau Nernst article/source commit 3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"source-reported progression","observation":"v2b=695.43, v3 pipelining=939.61, v4 warp specialization=1208.83, v5 2-SM=1302.29 and v6 static persistence=1475.93; the last line skips two changes."}] + }, + { + "claim_id": "technique-pipeline-stages-original-c003", + "claim": "For the author's exact 4096-cubed B200 setup, the pipelining step changes 695.43 to 939.61 TFLOP/s, approximately a 35.1-percent increase, and the author attributes that step to TMA/MMA overlap.", + "claim_type": "performance", + "covers": ["u002", "u004"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"sources/blogs/tcgen05-tutorial.md","revision":"Gau Nernst article/source commit 3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"v2b/v3 progression and pipelining description","observation":"The source reports those values for consecutive variants, with v3 adding pipelining."}], + "challenge": {"alternative":"The delta is a portable B200 theorem or independently reproduced isolated latency percentage.","check":"Inspect benchmark scope and adjacent source changes.","outcome":"It is only an author-reported result for the disclosed shape/environment; the verified claim is scoped accordingly."} + }, + { + "claim_id": "technique-pipeline-stages-original-c004", + "claim": "A circular SMEM pipeline allocates multiple operand stages and cycles producer/consumer ownership through them.", + "claim_type": "mechanism", + "covers": ["u005"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/prs/flashinfer/PR-2387/key-files/include/flashinfer/mamba/selective_state_update.cuh","revision":"flashinfer merge 18804cd51734cccf807356d017733bc757677f15","locator":"state[numStages], stage modulo loops and full/empty barriers","observation":"The merged implementation allocates per-stage storage and cycles stage indices through explicit producer/consumer handoffs."}], + "challenge": {"alternative":"Multiple copies are used only sequentially and therefore do not form a pipeline.","check":"Trace producer and consumer loops around the same modulo-indexed stages.","outcome":"Separate roles advance with stage-specific full/empty waits, enabling simultaneous ownership of different stages."} + }, + { + "claim_id": "technique-pipeline-stages-original-c005", + "claim": "The displayed pipelined_gemm is a complete safe three-stage TMA/tcgen05 implementation.", + "claim_type": "code", + "covers": ["u006"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"tcgen05 MMA completion and operand lifetimes","observation":"The consumer releases SMEM immediately after an opaque asynchronous MMA issue without attaching/waiting for completion; syncwarp does not complete tcgen05."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier","revision":"same archive","locator":"initialization, phase/tokens and transaction completion","observation":"The code omits async-proxy publication fencing, reusable barrier phase/token state, initial empty-stage seeding, and a correct tail protocol."}] + }, + { + "claim_id": "technique-pipeline-stages-original-c006", + "claim": "The optimal stage count depends on per-stage work/latency and resource constraints rather than being an architectural constant.", + "claim_type": "performance", + "covers": ["u007"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"artifacts/prs/flashinfer/PR-2387/key-files/include/flashinfer/mamba/selective_state_update.cuh","revision":"flashinfer merge 18804cd51734cccf807356d017733bc757677f15","locator":"numStages dispatch choices","observation":"Different code paths select three stages or cap a geometry-derived total at four, demonstrating implementation dependence."}], + "challenge": {"alternative":"One architecture-wide stage count is always optimal regardless of tile or K depth.","check":"Compare source-selected counts across paths with different geometry.","outcome":"The exact merged kernel uses different policies, rejecting the constant-stage alternative."} + }, + { + "claim_id": "technique-pipeline-stages-original-c007", + "claim": "Two stages give partial hiding, three fully hide latency for most GEMMs, four-to-five give full hiding with margin, and more than five are universally excessive.", + "claim_type": "performance", + "covers": ["u008", "u009", "u010", "u011"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"pipeline construction and stage policies across tutorial variants","observation":"Stage counts derive from tile, shared-memory capacity and schedule; no tagged source defines the table's universal hiding classifications or thresholds."}] + }, + { + "claim_id": "technique-pipeline-stages-original-c008", + "claim": "For CC10.0's 228-KiB per-SM shared-memory budget, binary16 A[128,64] and B[64,256] payloads occupy 16 and 32 KiB; three unpadded stages are 144 KiB and five are 240 KiB.", + "claim_type": "compatibility", + "covers": ["u012", "u013", "u014", "u015", "u016", "u017"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python exact byte/KiB and percentage arithmetic; compare 5-stage total with 228 KiB","environment":"Conda base, host CPU","reference":"binary16 is 2 bytes and CC10.0 guide reports 228 KiB/SM","result":"16+32=48 KiB/stage; 3=144 KiB=63.16% of 228; 5=240 KiB>228.","negative_control":"Treating KB as decimal or binary16 as 4 bytes fails the listed values."}], + "challenge": {"alternative":"144 KiB is the complete dynamic shared-memory allocation or 228 KiB is guaranteed per CTA.","check":"Account for barriers/padding/other storage and distinguish per-SM from opt-in per-block limits.","outcome":"Only unpadded operand payload and per-SM capacity are verified; the original caveat that other storage remains is retained in scope."} + }, + { + "claim_id": "technique-pipeline-stages-original-c009", + "claim": "mbarrier supports phase-based asymmetric producer/consumer synchronization without requiring every CTA thread to execute each wait.", + "claim_type": "mechanism", + "covers": ["u018"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"mbarrier object, phases, arrivals and waits","observation":"mbarrier is addressable state with expected arrivals/transactions and phase-aware waits usable by designated producer/consumer participants."}], + "challenge": {"alternative":"mbarrier is just a spelling of CTA-wide bar.sync and all threads must participate identically.","check":"Compare participant/arrival semantics and transaction completion with bar.sync.","outcome":"mbarrier permits explicitly counted asymmetric arrivals and waits, matching the scoped claim."} + }, + { + "claim_id": "technique-pipeline-stages-original-c010", + "claim": "The displayed PTX block is an accurate complete phase protocol, and autonomous TMA completion removes the producer from the critical path.", + "claim_type": "code", + "covers": ["u019", "u020"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"tensor-copy grammar and mbarrier complete_tx","observation":"The shown copy form omits required `.tile` and full operand/context details; transaction expectation and proxy ordering are not represented."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier","revision":"same archive","locator":"phase completion","observation":"An arrival does not generically 'flip phase' at issue; phase changes after expected arrivals/transactions complete. Autonomous completion reduces explicit producer work but does not prove the producer is absent from the performance critical path."}] + }, + { + "claim_id": "technique-pipeline-stages-original-c011", + "claim": "The Modular series establishes that a five-stage input pipeline alone reaches 85 percent of SOTA, was selected to hide an approximately 400-cycle B200 HBM latency, and has the displayed exact steady-state timeline.", + "claim_type": "code", + "covers": ["u021", "u022"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://www.modular.com/blog/matrix-multiplication-on-nvidias-blackwell-part-3-the-optimizations-behind-85-of-sota-performance","revision":"Modular Part 3, fetched 2026-08-09","locator":"Kernel 6 pipeline, warp specialization, write-out double buffering and final result","observation":"The article uses five stages but attributes the eventual 85% endpoint to a sequence including 2-SM, pipelining/warp specialization and output write-back double buffering; it does not provide the page's 400-cycle premise or exact fabricated stage timeline."}] + }, + { + "claim_id": "technique-pipeline-stages-original-c012", + "claim": "Pipelining is never harmful for all compute- and memory-bound GEMMs, and it is almost always paired with warp specialization.", + "claim_type": "performance", + "covers": ["u023", "u025"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-best-practices-guide/index.html#asynchronous-copy-from-global-memory-to-shared-memory","revision":"CUDA 13.0.2 Best Practices Guide","locator":"pipeline/performance considerations","observation":"Extra stages consume shared memory and coordination state and can reduce occupancy; profitability and whether warp roles are specialized depend on the concrete schedule."}] + }, + { + "claim_id": "technique-pipeline-stages-original-c013", + "claim": "Attention mainloops can pipeline KV tile production with computation when their dependencies and buffer ownership permit it.", + "claim_type": "mechanism", + "covers": ["u024"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://arxiv.org/abs/2407.08608","revision":"FlashAttention-3 paper","locator":"asynchronous pipeline and warp specialization","observation":"The paper pipelines asynchronous data movement and tensor-core/non-matmul work across attention tiles."}], + "challenge": {"alternative":"Every KV tile must be loaded synchronously immediately before its MMA.","check":"Inspect the staged attention schedule and producer/consumer roles.","outcome":"Independent next-tile movement overlaps current work under explicit synchronization."} + }, + { + "claim_id": "technique-pipeline-stages-original-c014", + "claim": "Unpadded operand SMEM payload grows linearly with stage count and therefore competes with tile size and other shared storage.", + "claim_type": "mechanism", + "covers": ["u026"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python sweep stage_count*per_stage_bytes","environment":"Conda base, host CPU","reference":"Each independent stage contains one fixed-size operand payload.","result":"Payload is exactly linear for stage counts 1-16.","negative_control":"Adding a fixed per-kernel overhead makes total allocation affine, confirming the page limits the linear statement to stage payload."}], + "challenge": {"alternative":"Stages alias the same SMEM and consume no additional payload.","check":"Require producer and consumer simultaneous ownership of different stages.","outcome":"Safe simultaneous ownership needs disjoint buffers, producing the stated linear payload term."} + }, + { + "claim_id": "technique-pipeline-stages-original-c015", + "claim": "Barrier initialization overhead is negligible and placing initialization in a syncthreads block is a sufficient pre-TMA publication protocol.", + "claim_type": "performance", + "covers": ["u027"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-init","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"initialization and visibility","observation":"The ISA defines initialization plus required memory/proxy ordering and participant conditions but no negligible-cost guarantee; __syncthreads alone is not a generic substitute for every required proxy fence."}] + }, + { + "claim_id": "technique-pipeline-stages-original-c016", + "claim": "Incorrect phase/reuse tracking can deadlock or corrupt a reusable pipeline, and short loops require guarded prologue/tail handling because steady state may never be reached.", + "claim_type": "mechanism", + "covers": ["u028", "u029"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"phase and wait semantics","observation":"Waiting on the wrong phase may never complete or may observe the wrong reuse; bounded loops must avoid waiting for stages that were never produced."}], + "challenge": {"alternative":"Modulo indexing alone makes phases and short-loop boundaries automatic.","check":"Run the state machine for num_k_tiles below stage count and for first stage reuse.","outcome":"Some stages are never produced in the short case and reuse changes expected phase, requiring the guards stated."} + } + ], + "commands": ["author progression and exact-revision source inspection", "archived PTX mbarrier/TMA inspection", "Conda-base payload arithmetic and state controls", "Modular primary article comparison"], + "scope": "All 29 baseline units for body hash d990987f96b0c765ab7753798f189a2125cbab0619c1671b450e305a4c628966 were reviewed, including code, ordering, table, arithmetic and performance-attribution claims.", + "residual_risk": "No CUDA compiler or SM100 GPU is available locally. Stage profitability and exposed latency remain target measurements." +} diff --git a/verification/reviewed/original/technique-register-budgeting.json b/verification/reviewed/original/technique-register-budgeting.json new file mode 100644 index 000000000..d00131197 --- /dev/null +++ b/verification/reviewed/original/technique-register-budgeting.json @@ -0,0 +1,157 @@ +{ + "claims": [ + { + "claim": "SM occupancy is inversely proportional to registers per thread, higher occupancy is the governing goal for memory-bound kernels, register controls force a budget, and Blackwell TMEM frees about 100 registers per thread.", + "claim_id": "technique-register-budgeting-original-c001", + "claim_type": "semantics", + "covers": ["u001"], + "evidence": [ + { + "locator": "sections 8.2.3.1 and 10.38", + "observation": "Occupancy is calculated from the compiled function, block size, and shared-memory use; register thresholds cause discrete resident-block transitions. Launch bounds guide compiler heuristics rather than imposing an exact register count.", + "revision": "CUDA C++ Programming Guide 13.0", + "url": "https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html" + }, + { + "locator": "epilogue TMEM load path around lines 455-463", + "observation": "The pinned SM100 implementation loads completed TMEM values into registers for its epilogue, so TMEM does not eliminate all accumulator-related register demand or establish a fixed saving.", + "path": "artifacts/kernels/deepgemm/full/sm100_fp8_gemm_1d1d.cuh", + "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f, byte-verified", + "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "The displayed launch-bounds example forces 32 registers per thread, yields about four 256-thread blocks per SM, and spills if necessary.", + "claim_id": "technique-register-budgeting-original-c002", + "claim_type": "code", + "covers": ["u002"], + "evidence": [ + { + "locator": "Launch Bounds, lines 8688-8713", + "observation": "The two-argument qualifier supplies a maximum thread count and desired minimum resident blocks. The compiler derives an architecture-dependent upper limit L and may reduce or increase register use; it does not set 32 registers or guarantee residency when another resource limits the launch.", + "revision": "CUDA C++ Programming Guide 13.0", + "url": "https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "Lowering the register budget has the fixed qualitative consequences shown: spills are bad, recomputation is neutral, and reduced unrolling is bad for compute-bound kernels.", + "claim_id": "technique-register-budgeting-original-c003", + "claim_type": "performance", + "covers": ["u003", "u004", "u005", "u006"], + "evidence": [ + { + "locator": "Launch Bounds, lines 8700-8706", + "observation": "The documented consequence of forcing usage below the compiler's initial choice is usually more local-memory use and/or more instructions; it does not assign workload-independent good, neutral, or bad labels to individual transformations.", + "revision": "CUDA C++ Programming Guide 13.0", + "url": "https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html" + }, + { + "locator": "Attempts 9-12", + "observation": "In the author's kernel, wider loads, extra accumulator chains, higher unrolling, and software pipelining all regressed or had no effect, showing that individual compiler/code-shape changes need a concrete comparison.", + "revision": "retrieved 2026-08-08", + "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "Memory-bound kernels can hide spill traffic behind memory latency, so aggressive register budgeting often wins.", + "claim_id": "technique-register-budgeting-original-c004", + "claim_type": "performance", + "covers": ["u007"], + "evidence": [ + { + "locator": "--resource-usage and launch-bounds descriptions", + "observation": "Spill stores and loads are stack/local-memory traffic, and the compiler guide describes register limits as a tradeoff rather than a generally winning policy.", + "revision": "CUDA Compiler Driver 13.0.2", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html" + }, + { + "locator": "Attempts 10-12", + "observation": "This memory-bound B200 kernel regressed when register pressure caused spills, and reducing maxrregcount from 80 to 64 had no effect because the natural allocation was already below the cap.", + "revision": "retrieved 2026-08-08", + "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "GPU Mode rank 1 used 32 registers at exactly 18.5 microseconds and rank 3 used 45 registers at about 20 microseconds.", + "claim_id": "technique-register-budgeting-original-c005", + "claim_type": "performance", + "covers": ["u008", "u009"], + "evidence": [ + { + "locator": "What the Top Solutions Did Differently", + "observation": "The author says all three inspected solutions clustered around an 18.5 microsecond geometric mean and separately reports caps of 32 for rank 1 and 45 for rank 3. The source does not assign the table's exact 18.5 and approximately 20 microsecond values to those two rows or publish controlled runs.", + "revision": "retrieved 2026-08-08", + "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "The difference between the 32- and 45-register solutions proves that occupancy dominates NVFP4 GEMV performance.", + "claim_id": "technique-register-budgeting-original-c006", + "claim_type": "performance", + "covers": ["u010"], + "evidence": [ + { + "locator": "What the Top Solutions Did Differently", + "observation": "The compared solutions also differed in PTX decode, cache policy, load width, exact-K specialization, and reuse. No one-variable register-cap or occupancy ablation is provided.", + "revision": "retrieved 2026-08-08", + "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/" + }, + { + "locator": "Attempt 11", + "observation": "The author's own cap change from 80 to 64 produced no effect, directly demonstrating that a nominal cap does not identify the compiled register count or isolate occupancy.", + "revision": "retrieved 2026-08-08", + "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "Occupancy-first register budgeting should be used for memory-bound, inner-loop-pressure, and sub-byte kernels, but not for compute-bound GEMM or kernels whose spills serialize.", + "claim_id": "technique-register-budgeting-original-c007", + "claim_type": "performance", + "covers": ["u011", "u012", "u013", "u014", "u015"], + "evidence": [ + { + "locator": "sections 8.2.3.1 and 10.38", + "observation": "CUDA exposes occupancy prediction and compiler constraints but does not prescribe these categories; optimal launch bounds require detailed per-kernel analysis and generally differ across architecture revisions.", + "revision": "CUDA C++ Programming Guide 13.0", + "url": "https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html" + }, + { + "locator": "Attempts 10-12 and Takeaways", + "observation": "Even within one memory-bound sub-byte kernel, only measurements of natural register use, spills, and variants discriminated useful from ineffective or harmful changes.", + "revision": "retrieved 2026-08-08", + "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + } + ], + "commands": [ + "Inspect CUDA 13.0 launch-bounds, occupancy, maxrregcount, spill, and resource-usage contracts", + "Compare the author-reported NVFP4 observations and failed attempts", + "Inspect the pinned SM100 DeepGEMM TMEM epilogue path" + ], + "id": "technique-register-budgeting", + "page_verdict": "refuted", + "path": "wiki/techniques/register-budgeting.md", + "residual_risk": "No B200 compiler or profiler run was available. Exact cap effects and performance remain compiler-, kernel-, launch-, and workload-specific.", + "scope": "All 15 baseline body units plus the frontmatter TMEM/register claim were checked against archived NVIDIA documentation, the cited author's report, and a pinned SM100 implementation." +} diff --git a/verification/reviewed/original/technique-software-exp.json b/verification/reviewed/original/technique-software-exp.json new file mode 100644 index 000000000..2838f4c89 --- /dev/null +++ b/verification/reviewed/original/technique-software-exp.json @@ -0,0 +1,236 @@ +{ + "claims": [ + { + "claim": "FlashAttention-4 replaces hardware ex2 with a four-FMA degree-4 software exponential over a symmetric reduced interval.", + "claim_id": "technique-software-exp-original-c001", + "claim_type": "mechanism", + "covers": ["u001"], + "evidence": [ + { + "locator": "Faster exponential section", + "observation": "The authors run a selected software degree-3 path alongside hardware MUFU.EX2, with floor reduction and a fractional interval [0,1); they do not replace all hardware exponentials.", + "revision": "FlashAttention-4 blog, retrieved 2026-08-09", + "url": "https://tridao.me/blog/2026/flash4/" + }, + { + "locator": "flash_attn/cute/utils.py POLY_EX2 and ex2_emulation_2; softmax.py apply_exp2_convert", + "observation": "The pinned implementation uses the degree-3 coefficient set by default and chooses between hardware exp2 and software emulation by fragment.", + "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", + "url": "https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "The Hopper/Blackwell resource table gives exact architecture counts and implies that any exp-per-MMA kernel is SFU-bound and software exp has eight times hardware throughput.", + "claim_id": "technique-software-exp-original-c002", + "claim_type": "performance", + "covers": ["u002", "u003", "u004", "u005", "u006", "u007", "u008"], + "evidence": [ + { + "locator": "Feeds and Speeds", + "observation": "For one M=N=D=128 B200 model, the authors use 8192 BF16 tensor-core ops/cycle, 16 exponential ops/cycle, and derive 1024 cycles for each of tensor cores and exponentials. They do not publish the page's architecture table, universal bottleneck rule, or 8x software throughput.", + "revision": "FlashAttention-4 blog, retrieved 2026-08-09", + "url": "https://tridao.me/blog/2026/flash4/" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "The displayed software_exp2 function implements FlashAttention-4's Cody-Waite reduction and minimax polynomial with about 22 bits of accuracy.", + "claim_id": "technique-software-exp-original-c003", + "claim_type": "code", + "covers": ["u009", "u010"], + "evidence": [ + { + "locator": "Faster exponential section", + "observation": "The published method uses floor reduction to f in [0,1), degree-3 Sollya coefficients 1, 0.6951, 0.2276, 0.0771, and exponent-bit reconstruction. The snippet uses nearest-integer reduction, a symmetric interval, degree-4 Taylor coefficients, and ldexpf.", + "revision": "FlashAttention-4 blog, retrieved 2026-08-09", + "url": "https://tridao.me/blog/2026/flash4/" + }, + { + "locator": "flash_attn/cute/utils.py lines 32-58 and 756-793", + "observation": "The exact pinned code confirms the published degree-3 path, clamps below -127, assumes inputs no larger than 127, and uses packed operations plus bit reconstruction.", + "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", + "url": "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "The displayed softmax CUDA function is a valid FlashAttention-4 implementation and its FMA throughput is fully available beside tensor-core work.", + "claim_id": "technique-software-exp-original-c004", + "claim_type": "code", + "covers": ["u011", "u012"], + "evidence": [ + { + "locator": "forward pipeline and faster exponential sections", + "observation": "FA4 has two synchronized softmax warpgroups, a correction warpgroup, TMEM staging, conditional rescaling, and a tunable hybrid exponential split. The standalone warp function implements none of that schedule and the source only says FMA units would otherwise be underutilized, not fully available.", + "revision": "FlashAttention-4 blog, retrieved 2026-08-09", + "url": "https://tridao.me/blog/2026/flash4/" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "The displayed PTX is a valid SFU-free FlashAttention-4 Horner implementation.", + "claim_id": "technique-software-exp-original-c005", + "claim_type": "code", + "covers": ["u013", "u014"], + "evidence": [ + { + "locator": "floating-point instructions and ex2.approx", + "observation": "The fragment uses undeclared %ni and %scale registers and reconstructs with ex2.approx, so it is neither self-contained nor SFU-free. Its coefficients and reduction also differ from FA4.", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html" + }, + { + "locator": "flash_attn/cute/utils.py combine_int_frac_ex2 and ex2_emulation_2", + "observation": "The exact software path uses integer exponent-field reconstruction rather than a hardware ex2 instruction.", + "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", + "url": "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "The page's degree-4 Taylor polynomial has about 22 mantissa bits of accuracy on the stated reduced interval.", + "claim_id": "technique-software-exp-original-c006", + "claim_type": "semantics", + "covers": ["u015"], + "evidence": [ + { + "command": "Dense-grid relative-error evaluation of the displayed polynomial on 1,000,002 points in [-0.5,0.5]", + "environment": "Conda base Python 3 host arithmetic", + "negative_control": "The same evaluator applied to the published rounded degree-3 coefficients on [0,1) produced approximately 8.763e-5, confirming the procedure distinguishes coefficient sets.", + "reference": "Compare the displayed polynomial to math.pow(2,x) and report maximum sampled relative error.", + "result": "Maximum sampled relative error was 5.5684e-5 at -0.5, approximately 14.13 bits, not 22 bits. A grid is not a rigorous global bound." + } + ], + "oracle": "host-experiment", + "verdict": "refuted" + }, + { + "claim": "FlashAttention-4 measured a 1.1-1.3x cuDNN speedup from software exponential alone.", + "claim_id": "technique-software-exp-original-c007", + "claim_type": "performance", + "covers": ["u020"], + "evidence": [ + { + "locator": "Attention Benchmarks", + "observation": "The 1.1-1.3x forward comparison is for the complete FA4 implementation, which combines pipelining, hybrid exp, conditional rescaling, TMEM staging, scheduling, and other choices; no exp-only ablation is reported there.", + "revision": "FlashAttention-4 blog, retrieved 2026-08-09", + "url": "https://tridao.me/blog/2026/flash4/" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "Software exp should be used for any transcendental-throughput-limited kernel and is not recommended on Hopper because four FMAs are unjustified there.", + "claim_id": "technique-software-exp-original-c008", + "claim_type": "compatibility", + "covers": ["u021", "u022"], + "evidence": [ + { + "locator": "FA4 README and implementation scope", + "observation": "The current FA4 package says it is optimized for Hopper and Blackwell, while the cited software-exp rationale and tuning are for a concrete Blackwell forward pipeline. Neither source defines a universal transcendental substitution rule or the stated Hopper prohibition.", + "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", + "url": "https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "Four dependent FMAs take about 16 cycles versus about 20 cycles for ex2.approx, and the win follows from 128 FMA versus 16 SFU units.", + "claim_id": "technique-software-exp-original-c009", + "claim_type": "performance", + "covers": ["u023"], + "evidence": [ + { + "locator": "Feeds and Speeds and Faster exponential sections", + "observation": "The source provides a tile-level 16-exp-ops/cycle model and motivates a hybrid path, but does not provide the page's instruction latency, FMA-unit count, or derived 8x comparison. The actual default polynomial is degree 3 plus range reduction and reconstruction.", + "revision": "FlashAttention-4 blog, retrieved 2026-08-09", + "url": "https://tridao.me/blog/2026/flash4/" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "challenge": { + "alternative": "Natural exponential needs different polynomial coefficients rather than input conversion.", + "check": "Apply the identity e^x = 2^(x log2(e)).", + "outcome": "The identity exactly supports the stated input conversion, independent of the approximation chosen for exp2." + }, + "claim": "A base-2 exponential implementation can approximate e^x by first multiplying x by log2(e).", + "claim_id": "technique-software-exp-original-c010", + "claim_type": "semantics", + "covers": ["u024"], + "evidence": [ + { + "locator": "softmax scale_log2 construction and exp2 calls", + "observation": "FA4 converts the natural softmax scale to base-2 before evaluating exp2.", + "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", + "url": "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/softmax.py" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "claim": "Attention inputs are always non-positive, so software reconstruction only needs to handle underflow and not special values or an explicit input contract.", + "claim_id": "technique-software-exp-original-c011", + "claim_type": "semantics", + "covers": ["u025"], + "evidence": [ + { + "locator": "flash_attn/cute/utils.py ex2_emulation and ex2_emulation_2", + "observation": "The exact helpers state x<=127 assumptions, clamp values below -127, and rely on particular rounding modes. The wider softmax implementation separately handles all-masked -infinity and dtype scaling cases.", + "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", + "url": "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "Softmax approximation errors cancel under normalization, BF16 conversion makes the polynomial automatically sufficient, and degree 6 gives near-ULP accuracy across the full float range.", + "claim_id": "technique-software-exp-original-c012", + "claim_type": "semantics", + "covers": ["u016", "u017", "u018", "u019"], + "evidence": [ + { + "locator": "Faster exponential section and forward pipeline", + "observation": "The authors specify a degree-3 relative-error approximation and BF16 conversion inside one validated attention design; they make none of the page's error-cancellation, universal sufficiency, or degree-6 full-range near-ULP claims.", + "revision": "FlashAttention-4 blog, retrieved 2026-08-09", + "url": "https://tridao.me/blog/2026/flash4/" + }, + { + "locator": "flash_attn/cute/utils.py POLY_EX2 and softmax implementation", + "observation": "The pinned implementation contains degree 0 through 5 coefficient sets, not the claimed degree-6 construction, and validates approximation only as part of its concrete dtype/softmax pipeline.", + "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", + "url": "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + } + ], + "commands": [ + "Inspect FlashAttention-4 first-party blog and pinned source revision a369df7", + "Inspect archived PTX ISA 9.0 instruction declarations", + "Evaluate both displayed and published coefficient sets on dense host grids" + ], + "id": "technique-software-exp", + "page_verdict": "refuted", + "path": "wiki/techniques/software-exp.md", + "residual_risk": "No B200 SASS inspection or performance run was available. The dense numerical experiment is discriminating evidence, not a formal error bound.", + "scope": "All 25 baseline body units and the page's architecture, code, accuracy, compatibility, and performance implications were checked." +} diff --git a/verification/reviewed/original/technique-swizzling.json b/verification/reviewed/original/technique-swizzling.json new file mode 100644 index 000000000..2f47abb21 --- /dev/null +++ b/verification/reviewed/original/technique-swizzling.json @@ -0,0 +1,228 @@ +{ + "claims": [ + { + "claim": "Swizzling makes consecutive row/column accesses hit different 32-byte banks, eliminates conflicts, and 128-byte swizzling is mandatory for all SM100 TMA/tcgen05 inputs with unswizzled GEMM limited to 46% throughput.", + "claim_id": "technique-swizzling-original-c001", + "claim_type": "semantics", + "covers": ["u001", "u002", "u003"], + "evidence": [ + { + "locator": "Shared Memory and bank-conflict descriptions", + "observation": "Shared memory has 32 banks with successive 32-bit words mapped to successive banks; conflicts depend on the concrete warp addresses, with broadcasts and same-word accesses as exceptions. Banks are not 32 bytes wide.", + "revision": "Nsight Compute Profiling Guide 13.3", + "url": "https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html" + }, + { + "locator": "tcgen05 shared-memory descriptor swizzling modes", + "observation": "PTX defines legal no-swizzle, 32B, 64B, and 128B descriptors, so 128B is not a universal correctness requirement.", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "The tutorial demonstrates a 2.7x improvement from swizzling alone, from 255 to 695 TFLOP/s.", + "claim_id": "technique-swizzling-original-c002", + "claim_type": "code", + "covers": ["u004", "u005"], + "evidence": [ + { + "locator": "128-byte global load and swizzling, benchmark table and remarks", + "observation": "The exact 3D endpoints are 252.81 and 695.43 TFLOP/s, but the revision also changes the inner TMA tile from 16B to 128B. The author explicitly says the speedup may be wider/faster TMA rather than removed bank conflicts and had no ncu access to discriminate it.", + "revision": "tcgen05 for dummies, pinned source 3b90ac9b", + "url": "https://gau-nernst.github.io/tcgen05/" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "The manual row-XOR function and bank-rotation diagram define the general 128B TMA/tcgen05 swizzle and guarantee conflict-free column access.", + "claim_id": "technique-swizzling-original-c003", + "claim_type": "code", + "covers": ["u006", "u007", "u008", "u009"], + "evidence": [ + { + "locator": "include/cute/swizzle.hpp Swizzle", + "observation": "CuTe defines swizzling over address bits, with BBits as mask width, MBase as invariant low bits, and SShift as mask distance. Mapping depends on byte address and alignment, not an arbitrary row parameter; Swizzle<3,4,3> XORs address bits 7:9 into 4:6.", + "revision": "CUTLASS 4.5.0 e406c186d2cae5782a846f7280af282ca4fecec2", + "url": "https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/swizzle.hpp" + }, + { + "locator": "CUtensorMapSwizzle enum", + "observation": "Driver documentation specifies 16B chunks within a 128B span for ordinary 128B swizzle; it does not define the page's one-bank-per-row rotation or conflict-free guarantee for every consumer access.", + "revision": "CUDA Driver API 13.0.97", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "challenge": { + "alternative": "TMA writes a canonical layout and the MMA descriptor can choose swizzle independently.", + "check": "Compare tensor-map swizzle semantics with tcgen05 descriptor layout selection.", + "outcome": "Both describe the destination/consumer layout; disagreement changes physical-to-logical addressing." + }, + "claim": "A TMA tensor map encodes a shared-memory swizzle and a TMA-produced tile must be consumed with a descriptor/layout that describes the same mapping.", + "claim_id": "technique-swizzling-original-c004", + "claim_type": "mechanism", + "covers": ["u010"], + "evidence": [ + { + "locator": "cuTensorMapEncodeTiled and CUtensorMapSwizzle", + "observation": "The driver API defines swizzle as the shared-memory bank-shuffling pattern in the tensor map.", + "revision": "CUDA Driver API 13.0.97", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html" + }, + { + "locator": "shared-memory descriptor", + "observation": "PTX encodes the tcgen05 shared-memory layout and mode in the consumer descriptor.", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma" + } + ], + "oracle": "web-primary", + "verdict": "verified" + }, + { + "claim": "The displayed cuTensorMapEncodeTiled call is a reproducible 2D FP16 128B-swizzle encoder.", + "claim_id": "technique-swizzling-original-c005", + "claim_type": "code", + "covers": ["u011"], + "evidence": [ + { + "locator": "cuTensorMapEncodeTiled signature and parameter constraints", + "observation": "The snippet leaves all dimension/stride arrays and pointer alignment undefined, describes a two-entry global stride although rank-2 uses rank-1 byte strides, and does not check CUresult or the 128B inner-box constraint.", + "revision": "CUDA Driver API 13.0.97", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "The no/32B/64B/128B swizzle modes have the page's universal non-MMA, narrow-type, medium-tile, and standard BF16/FP16 use cases.", + "claim_id": "technique-swizzling-original-c006", + "claim_type": "compatibility", + "covers": ["u012", "u013", "u014", "u015", "u016"], + "evidence": [ + { + "locator": "CUtensorMapSwizzle enum and constraints", + "observation": "The API defines chunk/span mappings and datatype/interleave/box restrictions, not the page's qualitative use-case table.", + "revision": "CUDA Driver API 13.0.97", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html" + }, + { + "locator": "tcgen05 valid combinations of type, major-ness, and swizzling", + "observation": "Legality is kind-, type-, shape-, major-mode-, and target-specific; no single 128B default covers all BF16/FP16 MMA operands.", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "The CuTe snippet correctly defines Swizzle, and Swizzle<3,4,3> with the shown layout is a generally valid 128B TMA/tcgen05 tile.", + "claim_id": "technique-swizzling-original-c007", + "claim_type": "code", + "covers": ["u017", "u018"], + "evidence": [ + { + "locator": "include/cute/swizzle.hpp lines 42-82", + "observation": "The page reverses the parameter meanings: BBits is mask width and MBase is low invariant bits. CUTLASS also distinguishes position-independent swizzle layouts from the hardware's position-dependent swizzle pointers, so alignment/composition cannot be omitted.", + "revision": "CUTLASS 4.5.0 e406c186d2cae5782a846f7280af282ca4fecec2", + "url": "https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/swizzle.hpp" + }, + { + "locator": "include/cute/pointer_swizzle.hpp lines 39-56", + "observation": "The source explicitly distinguishes position-independent swizzle_layout from position-dependent hardware swizzle pointers.", + "revision": "CUTLASS 4.5.0 e406c186d2cae5782a846f7280af282ca4fecec2", + "url": "https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/pointer_swizzle.hpp" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "nvprof or the displayed ncu metric proves swizzling eliminated conflicts when the result is exactly zero.", + "claim_id": "technique-swizzling-original-c008", + "claim_type": "code", + "covers": ["u019", "u020"], + "evidence": [ + { + "locator": "Metrics Examples and Shared Memory table", + "observation": "Current Nsight Compute documents l1tex__data_bank_conflicts_pipe_lsu.sum and derived/shared wavefront analysis, not the page's qualified metric as a portable contract. Zero conflicts is neither guaranteed nor sufficient to validate a layout or output.", + "revision": "Nsight Compute 13.3", + "url": "https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html" + }, + { + "locator": "tutorial swizzle remarks", + "observation": "The tutorial author lacked ncu access and explicitly could not determine whether bank conflicts caused the measured change.", + "revision": "tcgen05 for dummies, retrieved 2026-08-09", + "url": "https://gau-nernst.github.io/tcgen05/" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "Every Blackwell tensor-core and Hopper WGMMA kernel requires 128B swizzling, and swizzling prevents serialization for non-MMA column accesses.", + "claim_id": "technique-swizzling-original-c009", + "claim_type": "compatibility", + "covers": ["u021", "u022", "u023"], + "evidence": [ + { + "locator": "tcgen05 descriptor modes and valid combinations", + "observation": "No-swizzle, 32B, 64B, and 128B layouts are legal for tcgen05 under their respective constraints.", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma" + }, + { + "locator": "shared-memory conflict definition", + "observation": "Conflict behavior follows the exact warp addresses, including same-word and broadcast exceptions; a swizzle can improve one access and conflict for another.", + "revision": "Nsight Compute Profiling Guide 13.3", + "url": "https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "A mismatched descriptor always fails silently, CuTe avoids indexing errors, and element width determines the optimal swizzle while TMEM removes the issue.", + "claim_id": "technique-swizzling-original-c010", + "claim_type": "semantics", + "covers": ["u024", "u025", "u026"], + "evidence": [ + { + "locator": "cuTensorMapEncodeTiled return and constraints", + "observation": "Invalid tensor-map combinations can return an encoding error, while a legal but logically mismatched producer/consumer layout requires output validation. The API does not select swizzle from element width alone.", + "revision": "CUDA Driver API 13.0.97", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html" + }, + { + "locator": "CuTe swizzle and pointer implementations", + "observation": "CuTe expresses mappings but cannot guarantee the programmer composed the matching tensor map, alignment, layout, and consumer descriptor.", + "revision": "CUTLASS 4.5.0 e406c186d2cae5782a846f7280af282ca4fecec2", + "url": "https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/swizzle.hpp" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + } + ], + "commands": [ + "Inspect archived CUDA Driver/PTX swizzle enumerations and constraints", + "Inspect CUTLASS 4.5.0 Swizzle and pointer semantics", + "Compare the tutorial's exact v1/v2 changes and causal caveat", + "Check current Nsight Compute shared-memory metrics" + ], + "id": "technique-swizzling", + "page_verdict": "refuted", + "path": "wiki/techniques/swizzling.md", + "residual_risk": "No GPU profiler run was available. Exact bank-conflict behavior and performance remain properties of a concrete instruction access pattern and layout pair.", + "scope": "All 26 baseline body units plus mandatory/correctness frontmatter implications, code, profiler guidance, and performance attribution were checked." +} diff --git a/verification/reviewed/original/technique-tile-scheduling.json b/verification/reviewed/original/technique-tile-scheduling.json new file mode 100644 index 000000000..8bf3a90e4 --- /dev/null +++ b/verification/reviewed/original/technique-tile-scheduling.json @@ -0,0 +1,152 @@ +{ + "path": "wiki/techniques/tile-scheduling.md", + "id": "technique-tile-scheduling", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "technique-tile-scheduling-original-c001", + "claim": "Tile order can affect locality and utilization, and Blackwell CLC itself supplies dynamic policies including a swizzled raster while Hopper uses only launch-time static mappings.", + "claim_type": "mechanism", + "covers": ["u001"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md","revision":"CUTLASS 4.5.0 e406c186","locator":"Dynamic Scheduler and software swizzle rules","observation":"CLC redistributes already-launched ClcIDs; CUTLASS applies raster order and swizzle as software coordinate transforms, not as a CLC hardware scheduling policy. The broad locality/utilization premise is plausible, but the architecture contrast is false."}] + }, + { + "claim_id": "technique-tile-scheduling-original-c002", + "claim": "The displayed linear-raster code maps flat indices row-major, and consecutive tiles have categorically poor L2 locality because B column zero is evicted before tile four.", + "claim_type": "code", + "covers": ["u002", "u003"], + "verdict": "refuted", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python enumerate quotient/remainder coordinates and reuse sets","environment":"Conda base, host CPU; no GPU","reference":"The code's tile_m=i//tiles_n and tile_n=i%tiles_n mapping, with cache behavior left unmodeled","result":"The mapping is row-major, but adjacent tiles reuse the same A tile row; eviction of B column zero cannot be inferred from four coordinate assignments without cache capacity, residency and interleaving information.","negative_control":"Treating every distinct B column as an eviction event reproduces the page's unsupported conclusion and ignores simultaneous residency."}] + }, + { + "claim_id": "technique-tile-scheduling-original-c003", + "claim": "The displayed blocked-raster function is a valid all-shape swizzle that iterates column-major within groups and maximizes both A and B L2 reuse.", + "claim_type": "code", + "covers": ["u004", "u005"], + "verdict": "refuted", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python enumerate the displayed swizzled_raster for 4x4 and nondivisible 3x5 grids","environment":"Conda base, host CPU","reference":"Every input index must map bijectively into [0,tiles_m) x [0,tiles_n), and comments must match quotient/remainder order","result":"4x4 happens to cover exactly, but 3x5 with swizzle 2 returns (0,5), an out-of-range coordinate, and the within-group order is row-major rather than the claimed column-major traversal.","negative_control":"Testing only grids whose N dimension is divisible by swizzle_size hides the final-group bug."}] + }, + { + "claim_id": "technique-tile-scheduling-original-c004", + "claim": "For positive total_ctas, the static-stride snippet assigns CTA i the flat indices i + iteration*total_ctas and converts each valid index to row-major coordinates.", + "claim_type": "code", + "covers": ["u006", "u007"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python enumerate static-stride assignments for divisible and nondivisible tile counts","environment":"Conda base, host CPU","reference":"The sorted union should equal range(total_tiles) exactly once","result":"For positive grid sizes, all valid flat indices were covered once and quotient/remainder produced the stated row-major coordinates.","negative_control":"Using total_ctas-1 as the stride produced duplicate assignments."}], + "challenge": {"alternative":"A nondivisible final iteration skips or duplicates a tile.","check":"Enumerate remainders one and total_ctas-1 and count every flat index.","outcome":"Only CTA loop lengths differ; the assignment remains exact."} + }, + { + "claim_id": "technique-tile-scheduling-original-c005", + "claim": "CLC exposes an initialization-time hardware schedule policy with LinearRaster, SwizzledRaster, ColumnFirst and Hilbert choices through the displayed enum and helper functions.", + "claim_type": "code", + "covers": ["u008", "u009"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"clusterlaunchcontrol.try_cancel and query instruction syntax","observation":"The raw interface submits a cancel request to a response buffer/mbarrier and queries the response; it has no policy operand or scheduler initialization API. The enum and helper calls are invented."}] + }, + { + "claim_id": "technique-tile-scheduling-original-c006", + "claim": "CUTLASS 4.5.0 provides persistent, Stream-K and grouped scheduler abstractions for SM100.", + "claim_type": "compatibility", + "covers": ["u010"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/tile_scheduler.hpp","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"scheduler tags and SM100 TileSchedulerSelector specializations","observation":"PersistentScheduler, DynamicPersistentScheduler, StreamKScheduler and GroupScheduler tags map to SM100 scheduler implementations."}], + "challenge": {"alternative":"The named categories exist only for SM90 and have no SM100 selection route.","check":"Inspect the arch::Sm100 selector specializations and included implementation headers.","outcome":"The tag-to-class mappings explicitly instantiate SM100 persistent, Stream-K and group implementations."} + }, + { + "claim_id": "technique-tile-scheduling-original-c007", + "claim": "All SM100 persistent schedulers inherit PersistentTileSchedulerSm100, the public types are StreamKSchedulerSm100 and GroupedTileSchedulerSm100, default scheduling is a hardware swizzled raster, and the displayed GemmUniversal signature is valid CUTLASS 4.5.0.", + "claim_type": "code", + "covers": ["u011"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/tile_scheduler.hpp","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"public scheduler tags and SM100 selector mappings","observation":"The public tags are DynamicPersistentScheduler, StreamKScheduler and GroupScheduler; implementation classes are detail types. The group wrapper reuses an SM90 static grouped scheduler rather than inheriting the SM100 CLC class, and modern GemmUniversal composition does not use the shown legacy argument list."}] + }, + { + "claim_id": "technique-tile-scheduling-original-c008", + "claim": "The displayed B200 cache calculation correctly counts the first linear wave's A/B working set and proves that swizzling halves L2 misses for the stated GEMM.", + "claim_type": "code", + "covers": ["u012", "u013"], + "verdict": "refuted", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python enumerate the first 142 row-major coordinates in a 64x32 tile grid and count unique M and N coordinates","environment":"Conda base, host CPU; no cache simulator or GPU","reference":"Unique operand panels follow directly from the coordinate set; miss rates require measurement or a validated cache model","result":"The first 142 indices span 5 M rows and all 32 N columns, not approximately 20 A rows or 5 B groups. The arithmetic therefore cannot establish the claimed 2x miss reduction.","negative_control":"Multiplying two unrelated 4.4 ratios recreates the page's approximately-20 value but does not count coordinate reuse."}] + }, + { + "claim_id": "technique-tile-scheduling-original-c009", + "claim": "A tail wave is the final scheduling wave when fewer independent work units remain than available workers.", + "claim_type": "mechanism", + "covers": ["u014"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python compute quotient/remainder wave occupancy for total work and worker counts","environment":"Conda base, host CPU","reference":"For equal-duration independent tiles and one resident CTA per worker, total=q*workers+r","result":"When 0= M return before __syncthreads, so a partial final block can deadlock or exhibit undefined synchronization behavior; several helper types/functions are also undefined.", + "revision": "CUDA C++ Programming Guide 13.0", + "url": "https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "The optimization table isolates coalescing, FP4 intrinsics, vector loads plus cache policy, ILP plus maxrregcount, and an 8.6-microsecond speed-of-light into a single 2000-to-22.4-microsecond progression.", + "claim_id": "technique-vectorized-loads-original-c009", + "claim_type": "performance", + "covers": ["u018", "u019", "u020", "u021", "u022", "u023"], + "evidence": [ + { + "locator": "Reported Progression", + "observation": "Every author stage has more changes than the wiki's label, and the final 22.392 value is a three-case geometric-mean score. The 8.622 theoretical value belongs only to the first benchmark shape, not to the aggregate endpoint.", + "revision": "retrieved 2026-08-08", + "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html" + }, + { + "locator": "Pinned task benchmark table", + "observation": "The official model gives 8.622, 17.275, and 4.317 microseconds for the three different shapes and ranks submissions by their geometric mean.", + "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6", + "url": "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "Wide loads and cache policies are essential for all GEMV/memory-bound or sub-byte kernels, and single-token decode GEMV is always bandwidth-bound.", + "claim_id": "technique-vectorized-loads-original-c010", + "claim_type": "performance", + "covers": ["u024", "u025", "u026"], + "evidence": [ + { + "locator": "Attempt 9 and Takeaways", + "observation": "A wider load regressed this sub-byte GEMV by 16-25%, and the author recommends diagnosing the concrete bottleneck rather than choosing a transformation from a workload label.", + "revision": "retrieved 2026-08-08", + "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/" + }, + { + "locator": "Roofline Analysis", + "observation": "The official method determines memory- versus compute-boundedness from measured arithmetic intensity and achieved ceilings, not solely from an algorithm or decode label.", + "revision": "Nsight Compute 13.3", + "url": "https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html#roofline" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "A misaligned 256-bit PTX load safely falls back to narrower transactions, while no_allocate is necessarily harmful for reused data.", + "claim_id": "technique-vectorized-loads-original-c011", + "claim_type": "semantics", + "covers": ["u027", "u028"], + "evidence": [ + { + "locator": "6.4.1.1 Addresses as Operands and 9.7.9.8 ld", + "observation": "An address must be naturally aligned to the access size; misalignment is undefined and may mask low bits or fault. L1::no_allocate is an eviction-priority hint whose performance effect must be measured, not a universal harm rule.", + "revision": "CUDA 13.0.2, PTX ISA 9.0", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#addresses-as-operands" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + }, + { + "claim": "A cap that is too low necessarily spills and loses more than occupancy gains, while inline PTX bypasses the compiler register allocator.", + "claim_id": "technique-vectorized-loads-original-c012", + "claim_type": "semantics", + "covers": ["u029", "u030"], + "evidence": [ + { + "locator": "--maxrregcount and inline PTX constraints", + "observation": "A cap is a requested maximum subject to ABI/compiler constraints and can add spills or instructions, but the performance balance is empirical. Inline-assembly operands are represented by compiler-selected registers through constraints, so the construct does not bypass register allocation.", + "revision": "CUDA 13.0.2", + "url": "https://docs.nvidia.com/cuda/archive/13.0.2/inline-ptx-assembly/index.html" + } + ], + "oracle": "web-primary", + "verdict": "refuted" + } + ], + "commands": [ + "Inspect archived PTX 9.0 vector-load, alignment, cache-hint, and mov contracts", + "Compare the cited NVFP4 author reports, including negative controls and co-varying stages", + "Compare the example ABI and theoretical times with the pinned contest task" + ], + "id": "technique-vectorized-loads", + "page_verdict": "refuted", + "path": "wiki/techniques/vectorized-loads.md", + "residual_risk": "No CUDA toolkit or B200 was available. Author-reported performance was therefore checked for attribution and scope but not independently reproduced.", + "scope": "All 30 baseline body units and the frontmatter performance claim were checked against archived NVIDIA documentation, the pinned task, and the cited authors' reports." +} diff --git a/verification/reviewed/original/technique-warp-specialization.json b/verification/reviewed/original/technique-warp-specialization.json new file mode 100644 index 000000000..02dfb7eb7 --- /dev/null +++ b/verification/reviewed/original/technique-warp-specialization.json @@ -0,0 +1,163 @@ +{ + "path": "wiki/techniques/warp-specialization.md", + "id": "technique-warp-specialization", + "page_verdict": "refuted", + "claims": [ + { + "claim_id": "technique-warp-specialization-original-c001", + "claim": "SM100 replaces Hopper's four-warp model with a fixed 16-warp CTA in which one warp issues tcgen05 MMA and all remaining warps are freed for producer/consumer roles.", + "claim_type": "mechanism", + "covers": ["u001"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v4.cu","revision":"gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"NUM_WARPS, role branches and epilogue","observation":"The source-reported SM100 warp-specialization example launches four warps: elected lanes from warps 0 and 1 issue TMA/MMA, and all four warps later load/store the output. Sixteen warps is not an architectural replacement rule."}] + }, + { + "claim_id": "technique-warp-specialization-original-c002", + "claim": "The canonical SM100 GEMM role assignment is exactly warp 0 for TMA, warp 1 for MMA, and warps 2-15 for epilogue work.", + "claim_type": "mechanism", + "covers": ["u002", "u003", "u004", "u005"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/sm100_gemm_tma_warpspecialized.hpp","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"thread counts and WarpCategory","observation":"The generic SM100 kernel assigns warp categories MMA=0, Sched=1, MainloopLoad=2, EpilogueLoad=3, then a configuration-derived number of epilogue warps. It is neither the stated IDs nor a fixed 14-warp epilogue."}] + }, + { + "claim_id": "technique-warp-specialization-original-c003", + "claim": "Hopper WGMMA is a warpgroup operation whereas SM100 tcgen05.mma can be issued by one thread, but the page also implies that this issuing thread removes explicit operand movement and makes a whole MMA warp the architectural unit.", + "claim_type": "mechanism", + "covers": ["u006"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"tcgen05.mma description, operands and thread semantics","observation":"tcgen05.mma has one-thread issue and TMEM D, but it still consumes explicit A/B SMEM descriptors or TMEM addresses with lifecycle/ordering requirements. PTX defines the issuing thread, not a mandatory dedicated warp role."}] + }, + { + "claim_id": "technique-warp-specialization-original-c004", + "claim": "The Hopper/Blackwell table gives universal producer, epilogue, synchronization and register-pressure arrangements, including a fixed 14-warp SM100 epilogue and uniformly low register pressure.", + "claim_type": "mechanism", + "covers": ["u007", "u008", "u009", "u010", "u011", "u012"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/sm100_gemm_tma_warpspecialized.hpp","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"NumEpilogueThreads, WarpCategory and pipeline construction","observation":"Role count and pipeline participation are collective/epilogue dependent. SM100 has scheduler, mainloop-load and optional epilogue-load roles in addition to MMA/epilogue, invalidating the fixed comparison. TMEM changes accumulator placement but does not guarantee low GPR pressure for every role."}] + }, + { + "claim_id": "technique-warp-specialization-original-c005", + "claim": "The displayed 512-thread CUDA skeleton is a safe implementation of a three-role SM100 GEMM with reusable TMA stages and a correct MMA-to-epilogue handoff.", + "claim_type": "code", + "covers": ["u013", "u014"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"tcgen05.commit completion semantics","observation":"tcgen05 MMA is asynchronous. The skeleton releases each SMEM stage after issue using syncwarp and signals accumulator completion with ordinary mbarrier.arrive, never committing the MMA sequence to an mbarrier and waiting for completion."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-load","revision":"same archive","locator":"tcgen05.ld grammar and completion","observation":"tmem_load is not a CUDA/PTX API and the snippet omits legal TMEM addressing, load instruction shape, wait and output tile offsets."}] + }, + { + "claim_id": "technique-warp-specialization-original-c006", + "claim": "A reusable async producer/consumer stage needs a full/data-ready handoff after transfer completion and an empty/buffer-free handoff only after the consumer has finished using the stage.", + "claim_type": "mechanism", + "covers": ["u015", "u016", "u017"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"mbarrier phases, expected transactions, waits and reuse","observation":"Phase-aware full/empty handoffs protect async transaction completion and prevent stage overwrite before consumer completion."}], + "challenge": {"alternative":"One data-ready barrier is sufficient because TMA and tcgen05 consume SMEM immediately at issue.","check":"Inspect asynchronous completion and operand-lifetime requirements for both operations.","outcome":"The producer must not overwrite operands while MMA can still consume them, so a reverse release/completion path is required for reuse."} + }, + { + "claim_id": "technique-warp-specialization-original-c007", + "claim": "The displayed PTX accurately maps a TMA data-ready and tcgen05 buffer-release protocol using only ordinary mbarrier.arrive and one try_wait.parity instruction per handoff.", + "claim_type": "code", + "covers": ["u018", "u019"], + "verdict": "refuted", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"TMA complete_tx and expected transaction protocol","observation":"TMA completion decrements expected transaction bytes; a manual generic arrival does not represent the full copy/expect-tx protocol."},{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit","revision":"same archive","locator":"commit mbarrier arrival","observation":"Safe release after MMA requires completion tracking, which the PTX block omits. try_wait also returns a predicate and is not an unconditional blocking wait by itself."}] + }, + { + "claim_id": "technique-warp-specialization-original-c008", + "claim": "CUTLASS 4.5.0 implements the page's exact 1/1/14 partition in a type named CollectiveMma_1SM with WarpCount Shape<1,1,14> and the displayed producer/math/epilogue methods.", + "claim_type": "code", + "covers": ["u020", "u021"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/sm100_gemm_tma_warpspecialized.hpp","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"SM100 GemmUniversal kernel role dispatch","observation":"The alleged type/methods/shape are absent. Real kernel dispatch uses WarpCategory with four control roles plus collective-derived epilogue threads; CollectiveMma is a templated collective specialization, not CollectiveMma_1SM."}] + }, + { + "claim_id": "technique-warp-specialization-original-c009", + "claim": "Warp specialization is always required for Blackwell GEMMs because the tcgen05 model assumes that scheduling structure.", + "claim_type": "performance", + "covers": ["u022"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v3.cu","revision":"gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"pre-specialization pipelined tcgen05 kernel","observation":"The tutorial has a functioning tcgen05 GEMM before its v4 warp-specialization optimization. PTX constrains issue/operands, not a universal software role partition."}] + }, + { + "claim_id": "technique-warp-specialization-original-c010", + "claim": "The pinned FlashAttention-4 SM100 forward implementation uses a 16-warp partition with two groups of four softmax warps and dedicated MMA/load/correction/epilogue roles.", + "claim_type": "mechanism", + "covers": ["u023"], + "verdict": "verified", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_fwd_sm100.py","revision":"Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f","locator":"softmax0_warp_ids, softmax1_warp_ids and role IDs","observation":"The source assigns warps 0-3 and 4-7 to two softmax groups, 8-11 correction, 12 MMA, 13 epilogue, 14 load and 15 initially empty, with configuration-dependent adjustments."}], + "challenge": {"alternative":"The two groups are conceptual query tiles but not distinct warp-role groups.","check":"Trace the explicit warp ID tuples and stage-specific softmax dispatch.","outcome":"Separate ID sets execute stage 0/1 softmax work, so the scoped implementation claim holds."} + }, + { + "claim_id": "technique-warp-specialization-original-c011", + "claim": "Warp specialization is the cleanest decomposition for every producer-consumer kernel whenever TMA and MMA can overlap.", + "claim_type": "performance", + "covers": ["u024"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v3.cu","revision":"gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"pipelined non-specialized implementation","observation":"A valid temporal producer/consumer pipeline can overlap work before static role specialization. Profit depends on loop balance, register allocation, participant counts and workload; no primary source defines a universal cleanest choice."}] + }, + { + "claim_id": "technique-warp-specialization-original-c012", + "claim": "Fourteen epilogue warps are generally underutilized for simple stores and become more beneficial as epilogue complexity increases.", + "claim_type": "performance", + "covers": ["u025"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/sm100_gemm_tma_warpspecialized.hpp","revision":"NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5","locator":"CollectiveEpilogue::ThreadCount-derived role sizing","observation":"The implementation does not prescribe 14 epilogue warps; the count is collective-dependent. The source supplies no monotonic performance result relating epilogue complexity to a fixed count."}] + }, + { + "claim_id": "technique-warp-specialization-original-c013", + "claim": "A single MMA-control warp prevents overlap of independent MMA streams in one CTA, and 2-SM cooperative mode should therefore be used for larger tiles.", + "claim_type": "mechanism", + "covers": ["u026"], + "verdict": "refuted", + "oracle": "local-primary-source", + "evidence": [{"path":"/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v4.cu","revision":"gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573","locator":"elected MMA lane nested issue loops","observation":"One elected issuing lane submits many asynchronous tcgen05 MMA operations before a commit; one control warp does not imply only one outstanding stream. The later 2-SM variant is a separate source-specific tile choice, not a consequence of that alleged prohibition."}] + }, + { + "claim_id": "technique-warp-specialization-original-c014", + "claim": "mbarrier objects must be initialized and made visible before any participant or async mechanism uses or waits on them; CTA synchronization can be part of that publication protocol.", + "claim_type": "semantics", + "covers": ["u027"], + "verdict": "verified", + "oracle": "web-primary", + "evidence": [{"url":"https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-init","revision":"CUDA 13.0.2, PTX ISA 9.0","locator":"initialization and visibility requirements","observation":"Use before initialization or required visibility ordering is invalid; appropriate synchronization/fences must publish the object before participants proceed."}], + "challenge": {"alternative":"mbarrier.init alone is immediately visible to all threads and async proxies without synchronization.","check":"Inspect memory/proxy visibility requirements and the tutorial's release fence plus syncthreads sequence.","outcome":"Initialization must be ordered before use; the page's conditional synchronization advice is directionally correct, though exact fences depend on scope and proxy."} + }, + { + "claim_id": "technique-warp-specialization-original-c015", + "claim": "The full and variants artifact directories declare respectively a verbatim PR-1039 file and a derived teaching skeleton, and their files match the SHA-256 values recorded in provenance.", + "claim_type": "code", + "covers": ["u028"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"sha256sum artifact files; compare the full file with PR-1039 key-file and its new-file diff content; inspect both PROVENANCE.yaml modes; conda run -n base python scripts/verify_verbatim.py --strict","environment":"Host shell and Conda base, no GPU","reference":"Recorded hashes 792ac63e... and 2c2ac9a2..., verbatim/derived labels, and exact upstream merged-PR content","result":"Both local hashes match their manifests, the full file byte-matches the captured key-file/new-file patch content, and the strict verifier validates the upstream PR merge and verbatim bundle.","negative_control":"Comparing the derived skeleton with the full file fails, preserving the provenance-mode distinction."}], + "challenge": {"alternative":"Matching local duplicate files proves only internal consistency, not the declared upstream verbatim lineage.","check":"Run the strict verifier against the public PR merge metadata/content and separately compare the derived variant with the verbatim file.","outcome":"The strict upstream check passes for the verbatim bundle, while the derived skeleton intentionally differs and remains labeled derived."} + }, + { + "claim_id": "technique-warp-specialization-original-c016", + "claim": "scripts/get_page.py can retrieve the warp-specialization page and include its linked artifact bundle.", + "claim_type": "code", + "covers": ["u029", "u030"], + "verdict": "verified", + "oracle": "host-experiment", + "evidence": [{"command":"conda run -n base python scripts/get_page.py technique-warp-specialization --include-code","environment":"Conda base, repository root","reference":"Output should contain the page, full artifact/provenance and derived variant","result":"Command exited zero and produced 1,443 lines containing both artifact modes and the full SM100 FMHA source.","negative_control":"A nonexistent page ID exits nonzero."}], + "challenge": {"alternative":"--include-code is ignored and only emits Markdown.","check":"Search output after the Artifact Bundle heading for provenance and source filenames.","outcome":"Both full and variant file contents are present after the page body."} + } + ], + "commands": [ + "conda run -n base python scripts/get_page.py technique-warp-specialization --include-code", + "rg exact role dispatch and thread counts in CUTLASS v4.5.0 and pinned tutorial/FA4 source", + "sha256sum and cmp local artifact files" + ], + "scope": "All 30 baseline units were checked against CUDA 13.0.2 PTX, CUTLASS 4.5.0, the pinned tutorial and FA4 implementations, plus local artifact/CLI checks.", + "residual_risk": "No CUDA compiler or SM100 GPU is available. Artifact hashes, captured patch content, provenance modes, and the public merged-PR lineage pass strict verification." +} diff --git a/verification/tools/build_before_after_report.py b/verification/tools/build_before_after_report.py new file mode 100644 index 000000000..9952c4bb7 --- /dev/null +++ b/verification/tools/build_before_after_report.py @@ -0,0 +1,329 @@ +#!/usr/bin/env python3 +"""Build an exact-text Wiki before/after/reason JSONL report.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import subprocess +from pathlib import Path +from typing import Any + + +FRONTMATTER = re.compile(r"^---\s*\r?\n(.*?)\r?\n---\s*\r?\n", re.DOTALL) +BODY_LOCATOR = re.compile(r"^body:L(\d+)-L(\d+)$") + + +def load_json(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"{path}: expected a JSON object") + return value + + +def load_jsonl(path: Path) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if not line.strip(): + continue + try: + row = json.loads(line) + except json.JSONDecodeError as error: + raise ValueError(f"{path}:{line_number}: invalid JSON: {error}") from error + if not isinstance(row, dict): + raise ValueError(f"{path}:{line_number}: expected a JSON object") + rows.append(row) + return rows + + +def queue_index(rows: list[dict[str, Any]], source: Path) -> dict[str, dict[str, Any]]: + result: dict[str, dict[str, Any]] = {} + for row in rows: + path = row.get("path") + if not isinstance(path, str) or not path: + raise ValueError(f"{source}: queue entry is missing path") + if path in result: + raise ValueError(f"{source}: duplicate path {path!r}") + result[path] = row + return result + + +def claim_index( + rows: list[dict[str, Any]], source: Path +) -> dict[str, tuple[str, dict[str, Any]]]: + result: dict[str, tuple[str, dict[str, Any]]] = {} + for page in rows: + path = page.get("path") + for claim in page.get("claims", []): + claim_id = claim.get("claim_id") + if not isinstance(path, str) or not isinstance(claim_id, str): + raise ValueError(f"{source}: claim is missing path or claim_id") + if claim_id in result: + raise ValueError(f"{source}: duplicate claim_id {claim_id!r}") + result[claim_id] = (path, claim) + return result + + +def split_page(text: str) -> tuple[str, str]: + match = FRONTMATTER.match(text) + if not match: + raise ValueError("Wiki page is missing YAML frontmatter") + return match.group(1), text[match.end() :] + + +def top_level_fields(frontmatter: str) -> list[str]: + fields: list[str] = [] + for line in frontmatter.splitlines(): + match = re.match(r"^([A-Za-z_][A-Za-z0-9_-]*):(?:\s|$)", line) + if match: + fields.append(match.group(1)) + return fields + + +def extract_frontmatter_field(frontmatter: str, field: str) -> str | None: + lines = frontmatter.splitlines() + start: int | None = None + for index, line in enumerate(lines): + if re.match(rf"^{re.escape(field)}:(?:\s|$)", line): + start = index + break + if start is None: + return None + end = start + 1 + while end < len(lines): + if re.match(r"^[A-Za-z_][A-Za-z0-9_-]*:(?:\s|$)", lines[end]): + break + end += 1 + return "\n".join(lines[start:end]).strip() + + +def extract_fields(text: str, fields: list[str]) -> str | None: + frontmatter, _ = split_page(text) + parts = [extract_frontmatter_field(frontmatter, field) for field in fields] + return "\n\n".join(part for part in parts if part) or None + + +def locator_fields(locator: str, before_text: str, after_text: str) -> list[str]: + if not locator.startswith("frontmatter:"): + return [] + description = locator.split(":", 1)[1] + before_frontmatter, _ = split_page(before_text) + after_frontmatter, _ = split_page(after_text) + candidates = top_level_fields(before_frontmatter) + for field in top_level_fields(after_frontmatter): + if field not in candidates: + candidates.append(field) + fields = [ + field + for field in candidates + if re.search( + rf"(? dict[str, dict[str, Any]]: + return {unit["id"]: unit for unit in queue_page.get("claim_units", [])} + + +def ordered_unit_ids(queue_page: dict[str, Any], requested: list[str]) -> list[str]: + requested_set = set(requested) + known = [unit["id"] for unit in queue_page.get("claim_units", [])] + missing = requested_set.difference(known) + if missing: + raise ValueError( + f"{queue_page['path']}: unknown coverage units {', '.join(sorted(missing))}" + ) + return [unit_id for unit_id in known if unit_id in requested_set] + + +def extract_units( + text: str, queue_page: dict[str, Any], requested: list[str] +) -> tuple[str | None, list[str]]: + frontmatter, body = split_page(text) + body_lines = body.splitlines() + units = unit_index(queue_page) + metadata_parts: list[str] = [] + metadata_fields: list[str] = [] + body_ranges: list[tuple[int, int]] = [] + for unit_id in ordered_unit_ids(queue_page, requested): + locator = units[unit_id]["locator"] + body_match = BODY_LOCATOR.match(locator) + if body_match: + start, end = map(int, body_match.groups()) + if body_ranges and start <= body_ranges[-1][1] + 1: + body_ranges[-1] = (body_ranges[-1][0], max(body_ranges[-1][1], end)) + else: + body_ranges.append((start, end)) + continue + if locator.startswith("frontmatter:"): + field = locator.split(":", 1)[1] + metadata_fields.append(field) + part = extract_frontmatter_field(frontmatter, field) + if part and part not in metadata_parts: + metadata_parts.append(part) + continue + raise ValueError(f"{queue_page['path']}: unsupported locator {locator!r}") + body_parts = [ + "\n".join(body_lines[start - 1 : end]).strip() + for start, end in body_ranges + ] + parts = metadata_parts + [part for part in body_parts if part] + return "\n\n".join(parts) or None, metadata_fields + + +def git_show(repository_root: Path, revision: str, path: str) -> str: + result = subprocess.run( + ["git", "show", f"{revision}:{path}"], + cwd=repository_root, + check=False, + capture_output=True, + text=True, + encoding="utf-8", + ) + if result.returncode: + raise ValueError(f"cannot read {path!r} at {revision}: {result.stderr.strip()}") + return result.stdout + + +def validate_body(text: str, queue_page: dict[str, Any], label: str) -> None: + _, body = split_page(text) + observed = hashlib.sha256(body.encode("utf-8")).hexdigest() + expected = queue_page.get("body_sha256") + if observed != expected: + raise ValueError( + f"{queue_page['path']}: {label} body hash {observed} != queue {expected}" + ) + + +def evidence_urls( + receipt: dict[str, Any], + original_claims: dict[str, tuple[str, dict[str, Any]]], + current_claims: dict[str, tuple[str, dict[str, Any]]], +) -> list[str]: + evidence = list(receipt.get("evidence", [])) + original = original_claims.get(receipt["original_claim_id"]) + if original: + evidence.extend(original[1].get("evidence", [])) + for claim_id in receipt.get("replacement_claim_ids", []): + evidence.extend(current_claims[claim_id][1].get("evidence", [])) + urls: list[str] = [] + for item in evidence: + url = item.get("url") + if isinstance(url, str) and url and url not in urls: + urls.append(url) + return urls + + +def main() -> int: + repository_root = Path(__file__).resolve().parents[2] + verification_dir = repository_root / "verification" + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--output", + type=Path, + default=verification_dir / "before-after-report.jsonl", + ) + args = parser.parse_args() + + metadata = load_json(verification_dir / "baseline-metadata.json") + baseline_revision = metadata["repository_commit"] + baseline_queue = queue_index( + load_jsonl(verification_dir / "baseline-queue.jsonl"), + verification_dir / "baseline-queue.jsonl", + ) + current_queue = queue_index( + load_jsonl(verification_dir / "current-queue.jsonl"), + verification_dir / "current-queue.jsonl", + ) + original_claims = claim_index( + load_jsonl(verification_dir / "original-report.jsonl"), + verification_dir / "original-report.jsonl", + ) + current_claims = claim_index( + load_jsonl(verification_dir / "verification-report.jsonl"), + verification_dir / "verification-report.jsonl", + ) + receipts = [ + receipt + for receipt in load_jsonl(verification_dir / "remediation-log.jsonl") + if str(receipt.get("path", "")).startswith("wiki/") + ] + + baseline_texts: dict[str, str] = {} + current_texts: dict[str, str] = {} + records: list[dict[str, Any]] = [] + for receipt in receipts: + path = receipt["path"] + before_page = baseline_queue[path] + after_page = current_queue[path] + if path not in baseline_texts: + baseline_texts[path] = git_show(repository_root, baseline_revision, path) + if path not in current_texts: + current_texts[path] = (repository_root / path).read_text(encoding="utf-8") + before_text = baseline_texts[path] + after_text = current_texts[path] + validate_body(before_text, before_page, "baseline") + validate_body(after_text, after_page, "current") + + original_unit_ids = list(receipt.get("original_unit_ids", [])) + if original_unit_ids: + before_statement, metadata_fields = extract_units( + before_text, before_page, original_unit_ids + ) + after_unit_ids: list[str] = [] + for claim_id in receipt.get("replacement_claim_ids", []): + if claim_id not in current_claims: + raise ValueError(f"replacement claim {claim_id!r} is missing") + claim_path, claim = current_claims[claim_id] + if claim_path != path: + raise ValueError( + f"{receipt['receipt_id']}: replacement claim belongs to {claim_path}" + ) + after_unit_ids.extend(claim.get("covers", [])) + after_statement, _ = extract_units(after_text, after_page, after_unit_ids) + current_metadata = extract_fields(after_text, metadata_fields) + if current_metadata: + after_statement = ( + f"{current_metadata}\n\n{after_statement}" + if after_statement + else current_metadata + ) + else: + fields = locator_fields(receipt["locator"], before_text, after_text) + before_statement = extract_fields(before_text, fields) + after_statement = extract_fields(after_text, fields) + + if before_statement is None: + raise ValueError(f"{receipt['receipt_id']}: no exact baseline text found") + + records.append( + { + "path": path, + "before": {"statement": before_statement}, + "after": {"statement": after_statement}, + "reason": { + "statement": receipt["action_reason"], + "urls": evidence_urls(receipt, original_claims, current_claims), + }, + } + ) + + args.output.write_text( + "".join(json.dumps(record, ensure_ascii=False) + "\n" for record in records), + encoding="utf-8", + ) + print( + f"wrote records={len(records)} paths={len({record['path'] for record in records})} " + f"output={args.output}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/verification/tools/replace_report_entry.py b/verification/tools/replace_report_entry.py new file mode 100644 index 000000000..dd80b0e9f --- /dev/null +++ b/verification/tools/replace_report_entry.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Replace one JSONL report entry with a reviewed JSON object.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +def load_jsonl(path: Path) -> list[dict]: + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("report", type=Path) + parser.add_argument("entry", type=Path) + args = parser.parse_args() + + replacement = json.loads(args.entry.read_text(encoding="utf-8")) + path = replacement.get("path") + if not isinstance(path, str) or not path: + parser.error("entry must contain a non-empty path") + + rows = load_jsonl(args.report) + matches = [index for index, row in enumerate(rows) if row.get("path") == path] + if len(matches) != 1: + parser.error(f"expected exactly one report entry for {path!r}, found {len(matches)}") + + rows[matches[0]] = replacement + args.report.write_text( + "".join(json.dumps(row, sort_keys=True) + "\n" for row in rows), + encoding="utf-8", + ) + print(f"replaced path={path} report={args.report}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/verification/verification-report.jsonl b/verification/verification-report.jsonl new file mode 100644 index 000000000..f2a0ce195 --- /dev/null +++ b/verification/verification-report.jsonl @@ -0,0 +1,52 @@ +{"claims": [{"challenge": {"alternative": "Both CTAs must issue MMA collectively, while only one CTA allocates pair-level TMEM.", "check": "Read the separate mma and alloc/dealloc rows of Table 46.", "outcome": "The current page matches the rows; the alternative reverses their issue granularities."}, "claim": "A cta_group::2 MMA is initiated by one thread from a valid CTA pair, requires the peer to remain active, and accesses current and peer CTA TMEM; pair allocation management instead requires one warp per CTA.", "claim_id": "hw-2sm-cooperative-current-c001", "claim_type": "semantics", "covers": ["u001", "u002"], "evidence": [{"locator": "9.7.16.5 Table 46, CTA Pair, and tcgen05.mma description", "observation": "The table distinguishes one-thread pair-level MMA/commit issue from two-warp pair allocation management, defines the active peer condition, and the MMA description assigns both CTAs' TMEM to group 2.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-issue-granularity"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The descriptor contains types only and cta_group alone fixes M=256.", "check": "Inspect the idesc description and the explicitly titled M=128 + cta_group::2 layout.", "outcome": "The primary source assigns shapes to idesc and directly falsifies a fixed-M alternative."}, "claim": "cta_group::2 selects pair-level resources while idesc and the instruction configuration select legal shape; m256xn256xk16 is an F16/BF16 maximum example, not the only group-2 shape.", "claim_id": "hw-2sm-cooperative-current-c002", "claim_type": "semantics", "covers": ["u003", "u004"], "evidence": [{"locator": "instruction descriptor and Layout A-F, including M=128 + cta_group::2", "observation": "idesc encodes M/N and other operation details, and the group-2 data-path layouts include values other than M=256.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-data-path-layout-organization"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The eight-element count belongs to a different kind or sparse form.", "check": "Inspect the operand-size rule immediately under the dense tcgen05.mma grammar.", "outcome": "The size is selected by cta_group and applies to the dense grammar linked by the page."}, "claim": "The current page correctly points readers to the verified tcgen05 page for the cta_group::2 grammar and states that its lane-disable vector contains eight registers.", "claim_id": "hw-2sm-cooperative-current-c003", "claim_type": "semantics", "covers": ["u005"], "evidence": [{"locator": "9.7.16.10.9.1 disable-output-lane operand description", "observation": "The specification assigns four b32 elements for group 1 and eight for group 2; the local target exists and reproduces the normative grammar.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A CTA cluster barrier alone satisfies completion and lets the next TMA overwrite operands immediately.", "check": "Compare the explicit async source-lifetime rule with fence and commit guarantees.", "outcome": "The alternative lacks completion tracking; the current checklist keeps the mechanisms distinct."}, "claim": "The correctness checklist contains the direct CTA-pair, allocation, descriptor, operand-lifetime, completion, and inter-thread ordering invariants required by PTX without asserting one universal data partition.", "claim_id": "hw-2sm-cooperative-current-c004", "claim_type": "semantics", "covers": ["u006", "u007", "u008", "u009", "u010", "u011"], "evidence": [{"locator": "issue granularity, MMA source lifetime, fence, and commit sections", "observation": "The source defines pair lifetime and issue rules, prohibits modifying A/B source storage before MMA completion, assigns completion to commit/mbarrier, and assigns cross-thread ordering to tcgen05 fences composed with execution-ordering operations.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-issue-granularity"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The percentages are portable peak-efficiency guarantees or prove that persistence is necessary for group 2.", "check": "Trace the values and scheduling stages to the exact article setup and verify the current page's scope language.", "outcome": "The page reports one experiment and explicitly withholds a universal rule."}, "claim": "The page accurately scopes the tutorial's v4-to-v5 2-SM result to its B200 M=N=K=4096 environment and recommends benchmarking rather than a universal M threshold or persistence guarantee.", "claim_id": "hw-2sm-cooperative-current-c005", "claim_type": "performance", "covers": ["u012", "u013"], "evidence": [{"command": "conda run -n base python -c 'v4=1208.83; v5=1302.29; c=1506.74; print(100*v4/c,100*v5/c,100*(v5/v4-1),100*(v4/v4-1))'", "environment": "Conda base, Python 3.13.14; source inputs from the article's B200 M=N=K=4096 table, PyTorch 2.9.1 + CUDA 13", "negative_control": "v4 compared with itself yields 0% uplift.", "reference": "https://gau-nernst.github.io/tcgen05/ final performance table", "result": "80.228%, 86.431%, and 7.731% relative uplift, matching the current page's precise rounded values."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "One relative target has a case or directory mismatch.", "check": "Resolve the exact case-sensitive paths and compare a broken control.", "outcome": "Both intended paths resolve."}, "claim": "Both Related links resolve to the intended hardware pages.", "claim_id": "hw-2sm-cooperative-current-c006", "claim_type": "other", "covers": ["u014", "u015"], "evidence": [{"command": "for p in wiki/hardware/tcgen05-mma.md wiki/hardware/tmem.md wiki/hardware/does-not-exist.md; do test -f \"$p\" && echo \"$p present\" || echo \"$p missing\"; done", "environment": "Current repository workspace on 2026-08-08", "negative_control": "does-not-exist.md reports missing.", "reference": "Targets resolved relative to wiki/hardware/", "result": "tcgen05-mma.md and tmem.md are present."}], "oracle": "host-experiment", "verdict": "verified"}], "id": "hw-2sm-cooperative", "page_verdict": "verified", "path": "wiki/hardware/2sm-cooperative.md", "residual_risk": "No B200 or CUDA toolkit is present locally, so the reported performance was not reproduced; it remains explicitly source-reported and non-generalized.", "scope": "All 15 freshly rescanned current units were checked against CUDA 13.0.2 PTX ISA 9.0, the tutorial's exact source-reported setup, and local link resolution."} +{"claims": [{"challenge": {"alternative": "CLC is a CUTLASS-only queue that replaces the CUDA grid.", "check": "Inspect the architecture and launch model in the independent CUDA Programming Guide.", "outcome": "The guide defines an architectural cancellation operation over not-yet-started grid blocks."}, "claim": "CLC is a Blackwell compute-capability 10.0 work-stealing mechanism, and a CLC kernel launches the problem-sized grid.", "claim_id": "hw-clc-current-c001", "claim_type": "mechanism", "covers": ["u001"], "evidence": [{"locator": "4.12 introduction and Cluster Launch Control flow", "observation": "The guide introduces CLC on Blackwell compute capability 10.0, defines it as work stealing, and describes canceling blocks from the launched problem grid.", "revision": "CUDA Programming Guide 13.2", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A ClcID may be processed both by a newly launched block and by the stealing worker, or the first tile also requires a query.", "check": "Inspect rules 1-4 and the dynamic-persistent pseudocode.", "outcome": "Rule 3 guarantees one of the two paths, and rule 4 assigns blockIdx first."}, "claim": "Each ClcID is processed either by its normal launch or by a worker that cancels it; a worker uses blockIdx first and then may process a returned ClcID.", "claim_id": "hw-clc-current-c002", "claim_type": "semantics", "covers": ["u002", "u003", "u004", "u005"], "evidence": [{"locator": "Dynamic Scheduler with Cluster Launch Control, rules 1-5 and pseudocode", "observation": "The pinned documentation states both exclusive processing paths, uses blockIdx as first work, and uses the query response for subsequent work.", "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "CLC is useful only for equal-cost tail waves and provides no response to uneven SM availability.", "check": "Inspect the scenario and motivation immediately preceding the documented CLC rules.", "outcome": "The scenario explicitly varies SM availability and shows dynamic worker allocation."}, "claim": "CLC lets a persistent worker request subsequent work without a separate software queue and addresses load imbalance when real-time SM availability is uneven.", "claim_id": "hw-clc-current-c003", "claim_type": "mechanism", "covers": ["u006"], "evidence": [{"locator": "Static Scheduler and Dynamic Scheduler with Cluster Launch Control", "observation": "CUTLASS contrasts its static software tile scheduler with CLC queries and motivates dynamic scheduling with SMs unavailable because of other work.", "revision": "CUTLASS 4.3.3", "url": "https://docs.nvidia.com/cutlass/4.3.3/media/docs/cpp/blackwell_cluster_launch_control.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The request returns coordinates and success directly in ordinary registers or accepts a caller-selected ClcID.", "check": "Compare each displayed operand to the normative PTX grammar and decoder description.", "outcome": "Coordinates and success exist only inside the later-loaded opaque response; request operands are shared addresses."}, "claim": "try_cancel asynchronously writes a 16-byte opaque response to shared memory, completes an mbarrier transaction, takes no target tile coordinate, and is decoded with the shown query_cancel forms.", "claim_id": "hw-clc-current-c004", "claim_type": "code", "covers": ["u007", "u008", "u009"], "evidence": [{"locator": "9.7.13.17 try_cancel syntax and description; 9.7.13.18 query_cancel syntax and description", "observation": "The page reproduces the normative b128 shared-response/mbarrier request and the predicate and first-ctaid decoder forms without inventing a target-coordinate operand.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A CTA barrier alone makes a direct request result safe, so the transaction count and proxy fences are optional.", "check": "Inspect the official step-by-step example and async-proxy note.", "outcome": "The example separately performs mbarrier transaction tracking and async/generic proxy fences."}, "claim": "A complete request loop needs response and mbarrier setup, a one-thread asynchronous request with 16-byte transaction accounting, completion wait, success-before-ID decode, and proxy ordering before reuse.", "claim_id": "hw-clc-current-c005", "claim_type": "semantics", "covers": ["u010", "u011", "u012", "u013", "u014", "u015", "u016"], "evidence": [{"locator": "4.12.1.1 Thread Block Cancellation, steps 1-6", "observation": "The official programming sequence contains each listed setup, submission, synchronization, decode, and proxy-visibility step.", "revision": "CUDA Programming Guide 13.2", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Failure always and only means end-of-grid, and callers may safely retry or decode the stale ID.", "check": "Inspect the failure explanation and both explicit constraints.", "outcome": "The vendor guide names another failure cause and declares both retry-after-observation and failed-ID decode undefined."}, "claim": "After observing a failed request, another request is undefined; decoding an ID from failure is undefined; failure may indicate exhausted IDs or another scheduling reason such as higher-priority work.", "claim_id": "hw-clc-current-c006", "claim_type": "semantics", "covers": ["u017"], "evidence": [{"locator": "4.12 flow and 4.12.1.2 Constraints on Thread Block Cancellation", "observation": "The guide lists both undefined behaviors and explains both end-of-grid and higher-priority scheduling as failure reasons.", "revision": "CUDA Programming Guide 13.2", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Each CTA submits an independent request and receives an unrelated coordinate.", "check": "Compare the multicast first-ID rule and local-rank reconstruction in the official cluster example.", "outcome": "One cluster request broadcasts the same base ID and CTAs derive their members by rank."}, "claim": "Cluster cancellation uses one cluster thread, a multicast response, per-CTA local barriers at cluster scope, local-rank adjustment, and prior cluster synchronization; a 2x2 query consumes a 2x2 ClcID group.", "claim_id": "hw-clc-current-c007", "claim_type": "semantics", "covers": ["u018", "u019"], "evidence": [{"locator": "4.12.2.2 Use-case: Thread Block Clusters", "observation": "The guide specifies the one-thread request, multicasted first-ID response, per-CTA cluster-scope barriers, local-rank addition, and cluster.sync requirement.", "revision": "CUDA Programming Guide 13.2", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"}, {"locator": "Dynamic Scheduler rule 6", "observation": "CUTLASS gives the exact 2x2 worker cluster consuming 2x2 ClcIDs example.", "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "PersistentScheduler selects an SM90 static scheduler on SM100, or the named methods bypass the CLC pipeline.", "check": "Inspect architecture-specific selector specializations and both method bodies at the pinned revision.", "outcome": "The SM100 selector and implementation directly identify the dynamic CLC scheduler and pipeline."}, "claim": "In CUTLASS 4.5.0, PersistentScheduler on arch::Sm100 maps to PersistentTileSchedulerSm100; DynamicPersistentScheduler selects the dynamic implementation, whose advance and fetch methods use PipelineCLCFetchAsync.", "claim_id": "hw-clc-current-c008", "claim_type": "compatibility", "covers": ["u020"], "evidence": [{"locator": "scheduler tags and Sm100 TileSchedulerSelector specializations", "observation": "Both PersistentScheduler and DynamicPersistentScheduler select PersistentTileSchedulerSm100 for arch::Sm100.", "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/tile_scheduler.hpp"}, {"locator": "PersistentTileSchedulerSm100 Pipeline alias, advance_to_next_work, and fetch_next_work", "observation": "The class aliases PipelineCLCFetchAsync and splits request submission from response consumption across the two named methods.", "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The scheduler belongs inside CollectiveMainloop or after a legacy device::GemmUniversal argument list.", "check": "Compare the schematic's namespace, template arity, and order against the pinned example.", "outcome": "All match the authoritative dynamic_kernel_config composition."}, "claim": "The schematic CUTLASS 3.x composition correctly places DynamicPersistentScheduler as the fourth GemmUniversal kernel template parameter after the problem shape and collective mainloop and epilogue.", "claim_id": "hw-clc-current-c009", "claim_type": "code", "covers": ["u021", "u022"], "evidence": [{"locator": "dynamic_kernel_config", "observation": "The official SM100 example defines DynamicPersistentScheduler and instantiates kernel::GemmUniversal with Shape, collective mainloop, collective epilogue, and scheduler in exactly that order.", "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/95_blackwell_gemm_green_context/95_blackwell_gemm_green_context.cu"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "CUTLASS forwards the scheduler arguments to a hidden launch-time CLC hardware policy.", "check": "Trace both coordinate paths through the pinned swizzle_and_rasterize body and compare the PTX operands.", "outcome": "CUTLASS performs the transform explicitly in software after obtaining coordinates."}, "claim": "CUTLASS applies software swizzle/rasterization to initial and returned ClcIDs using scheduler arguments; the CLC PTX interface has no swizzle-policy operand.", "claim_id": "hw-clc-current-c010", "claim_type": "mechanism", "covers": ["u023"], "evidence": [{"locator": "Arguments, initial_work_tile_info, fetch_next_work, and swizzle_and_rasterize", "observation": "max_swizzle_size and raster_order feed a device-side transform invoked for blockIdx and decoded responses.", "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp"}, {"locator": "9.7.13.17-18 CLC instruction syntax", "observation": "The normative request and query operands contain no tile-order or swizzle policy.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A desired ClcID is an input and success removes that output from the computation.", "check": "Inspect the official function call operands and success path.", "outcome": "No desired ID is supplied, and the successful caller executes the returned work."}, "claim": "CLC cannot discard a caller-selected output tile.", "claim_id": "hw-clc-current-c011", "claim_type": "semantics", "covers": ["u024", "u025"], "evidence": [{"locator": "4.12 flow and 4.12.1 API Details", "observation": "The request takes response and barrier addresses, selects an unspecified not-yet-started block, and transfers its ID to the requester for processing.", "revision": "CUDA Programming Guide 13.2", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A successful request synthesizes a new work coordinate beyond the launched grid.", "check": "Inspect the source and disposition of every ClcID in rules 1-6.", "outcome": "Every returned ID is one whose original launch was atomically canceled."}, "claim": "CLC only redistributes ClcIDs in the launched grid and cannot create more independent tiles than that grid contains.", "claim_id": "hw-clc-current-c012", "claim_type": "mechanism", "covers": ["u024", "u026"], "evidence": [{"locator": "Dynamic Scheduler rules 1-6", "observation": "The grid contains one ClcID per output block and every ID is consumed by normal launch or a successful cancellation; no operation creates IDs.", "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Work stealing guarantees all SMs remain busy for every grid size and tile duration distribution.", "check": "Consider the documented model when fewer grid tiles than SMs exist and compare NVIDIA's wave-quantization definition.", "outcome": "No redistribution can expose more simultaneous independent blocks than the grid contains."}, "claim": "CLC does not guarantee full SM occupancy or universally eliminate GEMM wave quantization.", "claim_id": "hw-clc-current-c013", "claim_type": "performance", "covers": ["u024", "u027", "u028"], "evidence": [{"locator": "2.3 tile parallelism and 3.2 Wave Quantization", "observation": "Occupancy depends on the number of output tiles relative to SMs and on tile shape; an underfilled wave is possible whenever insufficient tiles remain.", "revision": "NVIDIA Matrix Multiplication Background User's Guide", "url": "https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html"}, {"locator": "4.12 Cluster Launch Control flow", "observation": "CLC redistributes pending grid blocks and its requests may decline; no full-occupancy or universal tail guarantee is made.", "revision": "CUDA Programming Guide 13.2", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The B200 has 132 SMs or CLC expands 32 grid coordinates into 148 IDs.", "check": "Read the exact B200 table cell and the ClcID cardinality rules.", "outcome": "The vendor table gives 148, and CLC only transfers existing IDs."}, "claim": "A 32-tile grid exposes at most 32 independent ClcIDs, while the referenced B200 configuration has 148 SMs.", "claim_id": "hw-clc-current-c014", "claim_type": "numeric", "covers": ["u029"], "evidence": [{"locator": "Table 8, NVIDIA Blackwell B200 (Cloud) SM count column", "observation": "The table lists 148 SMs for the B200 configuration.", "revision": "NVIDIA technical blog, 2026", "url": "https://developer.nvidia.com/blog/scaling-autonomous-ai-agents-and-workloads-with-nvidia-dgx-spark/"}, {"locator": "Dynamic Scheduler rules 1-3", "observation": "There is one ClcID per launched grid coordinate and every ID follows exactly one processing path, so a 32-tile grid has at most 32 independent IDs.", "revision": "CUTLASS 4.5.0, commit e406c186f510a15091cce01f782020ceb7ba8eb5", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "CLC performance is an invariant feature property, so hardware, tile shape, software, and timed scope can be omitted.", "check": "Compare NVIDIA's scoped examples and the variables they identify as controlling parallelism and scheduling.", "outcome": "Changing those variables changes the number of tiles, waves, available workers, and measured result."}, "claim": "A meaningful CLC performance result must scope the GPU and available SMs, software, tile/cluster shape, data types, timed region, and measurement method.", "claim_id": "hw-clc-current-c015", "claim_type": "performance", "covers": ["u029"], "evidence": [{"locator": "2.3 Typical Tile Dimensions and Performance; 3.2 Wave Quantization", "observation": "NVIDIA's comparisons explicitly scope GPU, CUDA/cuBLAS versions, problem size, tile size, and the measured metric, and explain why tile and SM counts change performance.", "revision": "NVIDIA Matrix Multiplication Background User's Guide", "url": "https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html"}, {"locator": "Overview scenario", "observation": "The CLC scheduling comparison fixes output tile grid, GPU SM count, unavailable SM count, and cluster shape before describing behavior.", "revision": "CUTLASS 4.3.3", "url": "https://docs.nvidia.com/cutlass/4.3.3/media/docs/cpp/blackwell_cluster_launch_control.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A link appears valid textually but resolves relative to the wrong directory.", "check": "Resolve every target from the actual page directory and compare a broken target.", "outcome": "All three real targets exist at their resolved paths while the broken control does not."}, "claim": "All three Related links resolve to existing wiki pages.", "claim_id": "hw-clc-current-c016", "claim_type": "other", "covers": ["u030", "u031", "u032"], "evidence": [{"command": "conda run -n base python -c 'from pathlib import Path; base=Path(\"wiki/hardware\"); links=[\"../techniques/persistent-kernels.md\",\"../techniques/tile-scheduling.md\",\"2sm-cooperative.md\"]; resolved=[(x,(base/x).resolve().relative_to(Path.cwd().resolve()).as_posix(),(base/x).exists()) for x in links]; broken=\"missing-clc-target.md\"; print({\"links\":resolved,\"all_exist\":all(x[2] for x in resolved),\"negative_control\":(base/broken).exists()})'", "environment": "Conda base, Python 3.13.14, repository root /users/Master/wiki-evolve/KernelWiki.", "negative_control": "wiki/hardware/missing-clc-target.md returned false.", "reference": "Each relative target must resolve from wiki/hardware and exist; a nonexistent target must fail.", "result": "persistent-kernels.md, tile-scheduling.md, and 2sm-cooperative.md all resolved and existed."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["conda run -n base python scripts/validate.py", "conda run -n base python /users/Master/.codex/skills/kernel-wiki-verifier/scripts/scan_wiki.py /users/Master/wiki-evolve/KernelWiki --output /users/Master/wiki-evolve/KernelWiki/verification/current-queue.jsonl", "git clone --depth 1 --branch v4.5.0 --filter=blob:none https://github.com/NVIDIA/cutlass.git /tmp/kernelwiki-cutlass-dRC5i4/cutlass", "conda run -n base python -c "], "id": "hw-clc", "page_verdict": "verified", "path": "wiki/hardware/clc.md", "residual_risk": "None for the claims retained on this page. No positive CLC throughput or latency number is claimed, so absence of a local NVIDIA GPU does not leave a performance verdict unresolved.", "scope": "Independent current-text pass over all 32 coverage units. CLC semantics and constraints were checked against CUDA Programming Guide 13.2 and archived PTX ISA 9.0; CUTLASS API, pipeline, and coordinate transforms were checked at v4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5; the B200 count used the exact B200 column of NVIDIA's platform table; all current links were tested from the page directory."} +{"claims": [{"challenge": {"alternative": "The sm_80 form is a different primitive and modern mbarrier began with Hopper.", "check": "Compare base init/arrive/test_wait ISA notes with the later expect_tx/complete_tx notes.", "outcome": "The same mbarrier object and base instructions require sm_80; only tx-count and cluster extensions begin with sm_90."}, "claim": "mbarrier is a naturally aligned opaque 64-bit shared-memory object introduced in PTX 7.0 for sm_80; sm_90 added tx-count and cluster-scope support, and Blackwell uses it for TMA and tcgen05 completion without requiring it in every warp-specialized kernel.", "claim_id": "hw-mbarrier-current-c001", "claim_type": "compatibility", "covers": ["u001", "u002"], "evidence": [{"locator": "object size/alignment, PTX 7.0 target notes, PTX 8.0 change log, TMA and tcgen05 completion sections", "observation": "The vendor history and operation definitions match each scoped generation claim; the page frames usage as optional design choice.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Either zero arrivals or zero tx-count independently completes the phase, or reinitialization may replace invalidation.", "check": "Inspect the completion bullet conjunction and the undefined-behavior clause for init on a valid object.", "outcome": "Both counters must be zero and a valid object must be invalidated before reinitialization."}, "claim": "An mbarrier tracks pending arrivals, next-phase expected arrivals, and tx-count; a phase completes only when arrivals and tx-count are both zero, then advances atomically, and the displayed initialization/lifecycle rules and PTX form are exact.", "claim_id": "hw-mbarrier-current-c002", "claim_type": "code", "covers": ["u003", "u004", "u005", "u006", "u007", "u008", "u009", "u010"], "evidence": [{"locator": "object contents, lifecycle, phase completion, and mbarrier.init syntax", "observation": "All three fields, conjunctive completion, automatic next-phase initialization, wait-before-next-arrive rule, invalidation rule, and the exact CTA-shared init form are normative.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#contents-of-the-mbarrier-object"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "arrive.expect_tx only changes tx-count, while complete_tx also decrements arrivals.", "check": "Trace each instruction to the separately defined arrive-on, expect-tx, and complete-tx operations.", "outcome": "arrive.expect_tx composes arrive-on with expect-tx; complete_tx changes only tx-count, exactly as the table states."}, "claim": "The operation table and representative PTX correctly distinguish arrive, arrive.expect_tx, expect_tx, complete_tx, state/parity waits, retry behavior, and acquire/release versus relaxed semantics.", "claim_id": "hw-mbarrier-current-c003", "claim_type": "code", "covers": ["u011", "u012", "u013", "u014", "u015", "u016", "u017", "u018", "u019"], "evidence": [{"locator": "arrive/expect_tx/complete_tx and wait grammars and descriptions", "observation": "The effects and exact CTA-shared forms match the ISA; try_wait may resume before completion and relaxed lacks ordering guarantees.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A global iteration parity works for any number of independently reused stage objects.", "check": "Apply the per-object phase definition to a two-stage ring and compare a stage's first and second use.", "outcome": "The global bit toggles twice before the same stage is reused and returns to its old value, while that stage advances once; per-object state is necessary."}, "claim": "Parity is each object's phase low bit, parity waits require lifetime tracking of that object, a stage ring therefore needs per-stage phase/state, and an opaque arrival token can identify the phase instead.", "claim_id": "hw-mbarrier-current-c004", "claim_type": "semantics", "covers": ["u020", "u021", "u022"], "evidence": [{"locator": "state and phaseParity operand semantics; parity tracking note", "observation": "Waits accept either a returned state for the current/immediately preceding phase or that phase's even/odd parity, and the ISA requires parity tracking throughout object lifetime.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "TMA completion performs an arrive-on, so pending-arrival count should include one hardware arrival per copy.", "check": "Compare the TMA completion qualifier with the definitions of complete-tx and arrive-on.", "outcome": "The qualifier explicitly requests complete-tx::bytes, which decrements tx-count and not pending arrivals."}, "claim": "The TMA sequence correctly uses arrive.expect_tx for one software arrival and the total expected bytes, TMA complete-tx for copied bytes, a consumer wait, and forbids unmatched extra arrival or complete_tx operations.", "claim_id": "hw-mbarrier-current-c005", "claim_type": "semantics", "covers": ["u023", "u024", "u025", "u026", "u027", "u028", "u029"], "evidence": [{"locator": "mbarrier::complete_tx::bytes mechanism and TMA operand conditions", "observation": "Each TMA copy decrements tx-count by bytes copied and the mbarrier is the only guaranteed completion observation mechanism.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"}, {"locator": "arrive.expect_tx composed semantics", "observation": "The instruction performs one arrive-on and increases tx-count, matching the accounting sequence and excluding a second unplanned arrival.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "tcgen05.commit contributes a byte tx-count just like TMA, or the after fence itself waits.", "check": "Inspect the completion_mechanism token and sequence in the different-thread example.", "outcome": "The token is arrive::one and the example has a separate mbarrier wait before the fence."}, "claim": "tcgen05.commit tracks prior MMA/cp/shift operations with one completion arrive-on, distinct from TMA byte tx-count, and an applicable cross-thread consumer handoff combines the wait with after_thread_sync ordering rather than substituting the fence for completion.", "claim_id": "hw-mbarrier-current-c006", "claim_type": "semantics", "covers": ["u030", "u031"], "evidence": [{"locator": "mbarrier::arrive::one description and cross-thread synchronization examples", "observation": "Commit signals one arrive-on after prior MMA/cp/shift completion, and the consumer example waits then executes after_thread_sync before issuing dependent tcgen05 work.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": ".shared::cluster implicitly supplies cluster scope and supports init/wait on the remote object.", "check": "Compare grammar qualifiers with the operation-by-location support table.", "outcome": "The grammar separates scope from state space and the table marks all other operations unsupported for .shared::cluster."}, "claim": "The page correctly separates CTA/cluster shared address state space from synchronization scope, limits cluster-shared-object operations to arrive/expect_tx/complete_tx, and derives the eight-item correctness checklist from those lifecycle and ordering rules.", "claim_id": "hw-mbarrier-current-c007", "claim_type": "semantics", "covers": ["u032", "u033", "u034", "u035", "u036", "u037", "u038", "u039", "u040", "u041", "u042", "u043", "u044"], "evidence": [{"locator": "shared-location support table, wait scope description, and cluster Example 4", "observation": "The table permits only the three stated operations on cluster-shared addresses; scope is a separate qualifier, and the example initializes an owner's CTA-shared object, maps it for remote arrival, and waits at the owner.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#mbarrier-support-with-shared-memory"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "One link tracks rolling docs or resolves relative to the wrong directory.", "check": "Inspect the archive segment and resolve relative paths from wiki/hardware.", "outcome": "The three external URLs are archive-pinned, and both relative paths resolve to existing intended files."}, "claim": "The three external references are pinned to the relevant PTX 9.0 sections and the two internal links resolve to the TMA and warp-specialization pages.", "claim_id": "hw-mbarrier-current-c008", "claim_type": "other", "covers": ["u045", "u046", "u047", "u048", "u049"], "evidence": [{"command": "test -f wiki/hardware/tma.md && test -f wiki/techniques/warp-specialization.md; inspect three CUDA archive URLs", "environment": "KernelWiki worktree; web retrieval on 2026-08-08", "negative_control": "wiki/hardware/not-a-page.md does not exist, showing the file check rejects a broken relative target.", "reference": "Internal targets exist and external anchors identify mbarrier lifecycle, waits, and TMA tensor-copy sections in CUDA 13.0.2.", "result": "Both internal files resolve and all external links use the CUDA 13.0.2 archive with the intended anchors."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["test -f wiki/hardware/tma.md && test -f wiki/techniques/warp-specialization.md", "structural validation after remediation"], "id": "hw-mbarrier", "page_verdict": "verified", "path": "wiki/hardware/mbarrier.md", "residual_risk": "No CUDA toolkit is installed locally, so the three short PTX forms were not assembled. Their qualifier ordering and operands match PTX ISA 9.0 grammar and examples exactly.", "scope": "All 49 current units for body hash 4215a80fde138535a1d096475ad4d1b76c905d6cd8c3ab2b6198eae4c78180e3 were verified against CUDA 13.0.2 PTX ISA 9.0 and local link resolution."} +{"claims": [{"challenge": {"alternative": "NVFP4 is only the E2M1 encoding, uses one scale level, or groups 32 values like MXFP4.", "check": "Separate payload format from the two tensors of scale metadata in the version-pinned recipe and compare the 1D and 2D grouping definitions.", "outcome": "The documentation explicitly gives E2M1 times local E4M3 times global FP32, with 16-value 1D and 16-by-16 2D groups."}, "claim": "NVFP4 is a hierarchical quantization recipe whose 1D form reconstructs each E2M1 payload with a block-of-16 E4M3 local scale and a per-tensor FP32 scale; E2M1 has the stated bits, numeric set, and two-values-per-byte packing, and Transformer Engine 2.13 also defines 16-by-16 weight scaling.", "claim_id": "hw-nvfp4-current-c001", "claim_type": "semantics", "covers": ["u001", "u002", "u003", "u004", "u005", "u006", "u007"], "evidence": [{"locator": "E2M1 data format, two-level scaling, mathematical formulation, and 1D/2D scaling", "observation": "The vendor defines the E2M1 bit allocation and range, the E2M1/local-scale/global-scale product, one E4M3 scale per 16 values, one FP32 scale per tensor, and a 16-by-16 block in 2D weight scaling. Exhaustive decoding yields the page's finite numeric set; two 4-bit values occupy one byte.", "revision": "Transformer Engine 2.13", "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"}, {"locator": "ue4m3 alternate floating-point format", "observation": "PTX names the scale representation .ue4m3 and defines its one padding/zero MSB followed by four exponent and three mantissa bits.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#alternate-floating-point-data-formats"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Smaller groups and fractional scales guarantee strictly lower error for all tensors and quantizers.", "check": "Test the strict claim on exactly representable data and inspect whether the source quantifies every scale-selection method and metric.", "outcome": "Exactly representable tensors can tie, and the source does not make a universal strict-error claim; the page now states only greater scale freedom and requires workload measurement."}, "claim": "The comparison table correctly distinguishes NVFP4 block-of-16 E4M3 local scaling plus a global scale from MXFP4 block-of-32 UE8M0 local scaling, and scopes the error statement to representational freedom rather than a universal outcome.", "claim_id": "hw-nvfp4-current-c002", "claim_type": "semantics", "covers": ["u008", "u009", "u010", "u011", "u012", "u013", "u014"], "evidence": [{"locator": "Comparison with MXFP8 and MXFP4", "observation": "NVIDIA contrasts NVFP4's 16-value E4M3 local groups and per-tensor global scale with MXFP4's 32-value E8M0 power-of-two groups and describes accuracy benefits as recipe behavior rather than an input-independent theorem.", "revision": "Transformer Engine 2.13", "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "kind::mxf4nvf4 uniquely selects block16 UE4M3, so the other qualifiers are documentation noise.", "check": "Enumerate all rows with kind mxf4nvf4 in the PTX valid-combination tables.", "outcome": "Rows include both UE4M3/block16 and UE8M0 modes; full qualifiers are behavior-selecting."}, "claim": "The PTX table gives valid E2M1 block-scale combinations for kind::mxf4 block32 with UE8M0/scale_vec::2X and kind::mxf4nvf4 block16 with UE4M3/scale_vec::4X, and correctly warns that kind::mxf4nvf4 alone also permits UE8M0 modes and requires an architecture-specific target.", "claim_id": "hw-nvfp4-current-c003", "claim_type": "compatibility", "covers": ["u015", "u016", "u017", "u018"], "evidence": [{"locator": "tcgen05.mma block-scale grammar, valid combination tables, and target notes", "observation": "The grammar and tables list the two displayed combinations, additional mxf4nvf4 UE8M0 combinations, and architecture-specific sm_100a-family targets.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The source is a 32-bit scalar or the result contains four f16 values, and mov is unconditionally faster.", "check": "Apply the x2 packed-type sizes in the cvt grammar and distinguish mov semantics from generated-instruction performance.", "outcome": "e2m1x2 occupies 8 bits and f16x2 32 bits; neither PTX section promises a relative timing result."}, "claim": "The displayed cvt.rn.f16x2.e2m1x2 form converts one byte-sized packed E2M1 pair to a 32-bit f16x2 result, and the typed mov.b32 scalar-to-four-byte capability and measurement caveat are accurate.", "claim_id": "hw-nvfp4-current-c004", "claim_type": "code", "covers": ["u019", "u020", "u021"], "evidence": [{"locator": "packed alternate-format conversion syntax and e2m1x2 example", "observation": "The exact mnemonic, packed source width, and packed half result width match the page.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt"}, {"locator": "scalar-to-vector move rules and b32/four-byte example", "observation": "The ISA permits the described unpack when vector element declarations are byte-sized and explicitly provides semantics rather than a performance guarantee.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Four-bit operands automatically prove 4x throughput versus Hopper regardless of the chosen baseline.", "check": "Check whether Hopper is a legal target for the FP4 instruction and whether operand width alone fixes MMA issue rate and end-to-end kernel work.", "outcome": "Hopper is not a target for these instructions, and the comparison necessarily selects a different instruction/emulation path plus workload conditions."}, "claim": "The page rejects an architecture-independent 4x-over-Hopper number and enumerates the baseline and environment details needed to make an NVFP4 throughput comparison meaningful.", "claim_id": "hw-nvfp4-current-c005", "claim_type": "performance", "covers": ["u022"], "evidence": [{"locator": "mxf4/mxf4nvf4 target notes", "observation": "Native tcgen05 FP4 forms require Blackwell architecture-specific targets, so Hopper cannot be a like-for-like native instruction baseline. The ISA contains no universal 4x result; shapes, layouts, target, and instruction combinations are explicit variables.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "One reference points to rolling documentation or to an unrelated conversion section.", "check": "Inspect version path segments and resolve both PTX anchors in the archive.", "outcome": "All version segments and section targets match the labels."}, "claim": "The three external references are version-pinned to Transformer Engine 2.13 and CUDA 13.0.2 PTX ISA 9.0 sections that define NVFP4, tcgen05.mma, and cvt.", "claim_id": "hw-nvfp4-current-c006", "claim_type": "other", "covers": ["u023", "u024", "u025"], "evidence": [{"locator": "NVFP4 page", "observation": "The URL includes the 2.13 release segment and resolves to the cited format/recipe material.", "revision": "Transformer Engine 2.13", "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"}, {"locator": "tcgen05.mma and cvt anchors", "observation": "Both links share the CUDA 13.0.2 archive and select the intended PTX sections.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "At least one relative path resolves outside the intended wiki directory or to an unrelated page.", "check": "Resolve each relative path and inspect its frontmatter ID.", "outcome": "The paths resolve to technique-fine-grained-quantization, kernel-nvfp4-gemm, and kernel-nvfp4-gemv."}, "claim": "The three internal references resolve to the fine-grained quantization, NVFP4 GEMM, and NVFP4 GEMV pages.", "claim_id": "hw-nvfp4-current-c007", "claim_type": "other", "covers": ["u026", "u027", "u028"], "evidence": [{"command": "test -f wiki/techniques/fine-grained-quantization.md && test -f wiki/kernels/nvfp4-gemm.md && test -f wiki/kernels/nvfp4-gemv.md", "environment": "current KernelWiki worktree after NVFP4 remediation", "negative_control": "wiki/kernels/not-an-nvfp4-kernel.md does not exist, so the check rejects a broken target.", "reference": "Resolve targets from wiki/hardware and compare their frontmatter IDs with the link labels.", "result": "All three targets exist and identify the intended technique and kernels."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["test -f wiki/techniques/fine-grained-quantization.md && test -f wiki/kernels/nvfp4-gemm.md && test -f wiki/kernels/nvfp4-gemv.md", "structural validation after remediation"], "id": "hw-nvfp4", "page_verdict": "verified", "path": "wiki/hardware/nvfp4.md", "residual_risk": "No CUDA toolkit or NVIDIA GPU is installed locally, so the one-line PTX form was not assembled and no performance experiment was run. Exact syntax and compatibility come from PTX ISA 9.0; the page deliberately makes no numeric speed claim.", "scope": "All 28 current units for body hash b574ab0127a35863b7ec1a2eb952aa93fe86bc145085a497ad64b9c3bc93a46a were checked against Transformer Engine 2.13, CUDA 13.0.2 PTX ISA 9.0, and local link resolution."} +{"claims": [{"challenge": {"alternative": "A single CTA trigger releases the secondary's dependent reads immediately and guarantees overlap.", "check": "Separate grid scheduling eligibility from dependency completion and inspect the all-CTA and wait clauses.", "outcome": "All CTAs must trigger or exit before early scheduling, and the secondary wait still blocks dependent work until prerequisites complete; overlap is not guaranteed."}, "claim": "PDL on compute capability 9.0 or later makes a same-stream secondary grid eligible for opportunistic early execution only after every primary CTA triggers or exits; dependent secondary work still waits for prerequisite completion and visibility, while omitted explicit triggers occur at CTA exit.", "claim_id": "hw-pdl-gdc-current-c001", "claim_type": "semantics", "covers": ["u001", "u002", "u003", "u004", "u005", "u006", "u007"], "evidence": [{"locator": "6.2.8.6 overview and API Description", "observation": "The guide gives the compute-capability floor, same-stream requirement, all-CTA trigger/exit rule, implicit exit trigger, extensible launch attribute, mandatory dependency wait, and explicit warning that concurrency is opportunistic.", "revision": "CUDA 13.0.2", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#programmatic-dependent-launch-and-synchronization"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "cudaGridDependencySynchronize belongs in the primary or a normal secondary launch receives the dependency implicitly.", "check": "Trace the two device functions and the host attribute in NVIDIA's official example.", "outcome": "The trigger is primary-side, the wait is secondary-side, and the secondary is launched through the extensible API with explicit opt-in."}, "claim": "The CUDA skeleton places a one-per-primary-CTA trigger, secondary-side dependency wait, and programmaticStreamSerializationAllowed launch attribute in their correct roles and same stream.", "claim_id": "hw-pdl-gdc-current-c002", "claim_type": "code", "covers": ["u008", "u009", "u010"], "evidence": [{"locator": "6.2.8.6.2 official CUDA example", "observation": "The official example uses the same primary function, secondary function, launch attribute ID/value, config attrs, and cudaLaunchKernelEx role mapping. The page adds one-thread-per-CTA issue, which is sufficient because repeated same-CTA triggers have no additional effect.", "revision": "CUDA 13.0.2", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#programmatic-dependent-launch-and-synchronization"}, {"locator": "launch_dependents repeated-invocation rule", "observation": "The ISA states that invocations after the first within a CTA have no extra side effect, supporting the single issuing thread per CTA.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-griddepcontrol"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "launch_dependents itself makes prerequisite writes visible or the dependent may omit wait after early launch.", "check": "Compare the two action descriptions and the explicit correctness note.", "outcome": "Only wait provides prerequisite completion/visibility, and the note requires a wait when the prerequisite launches dependents."}, "claim": "The exact PTX forms griddepcontrol.launch_dependents and griddepcontrol.wait, their per-CTA and visibility semantics, PTX 7.8 introduction, sm_90 floor, and mandatory wait pairing are correct.", "claim_id": "hw-pdl-gdc-current-c003", "claim_type": "code", "covers": ["u011", "u012", "u013"], "evidence": [{"locator": "syntax, description, note, ISA notes, and target notes", "observation": "Every instruction spelling, behavioral qualification, history statement, and target requirement on the page matches the normative section.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-griddepcontrol"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The default-ON CMake option means every SM100 CUDA launch automatically participates in PDL.", "check": "Follow the option into the guarded header and compare it with CUDA's host launch attribute contract.", "outcome": "The option controls CUTLASS compilation only; ordinary kernels and launches do not acquire a dependency or host attribute."}, "claim": "Blackwell retains explicit CUDA PDL opt-in, while CUTLASS 4.5.0 separately defaults CUTLASS_ENABLE_GDC_FOR_SM100 ON and leaves its SM90 counterpart opt-in; this build setting is overridable and not device-wide behavior.", "claim_id": "hw-pdl-gdc-current-c004", "claim_type": "compatibility", "covers": ["u014", "u015"], "evidence": [{"locator": "lines 458-474", "observation": "SM90 GDC is appended only if an externally set option is true; SM100 receives an overridable default variable set to ON and, when enabled, adds CUTLASS_ENABLE_GDC_FOR_SM100=1.", "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/CMakeLists.txt", "revision": "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5"}, {"locator": "compile guards and launch/wait wrappers", "observation": "The macro gates instruction-emitting wrapper code for eligible architectures, demonstrating library compilation rather than a device-global launch mode.", "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/include/cutlass/arch/grid_dependency_control.h", "revision": "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "PDL always shortens a chain because early scheduling has no resource or protocol cost.", "check": "Apply the vendor's resource/opportunism caveats to a pair with no independent secondary preamble or no co-residency capacity.", "outcome": "Such a pair has nothing useful to overlap or cannot co-reside, so only an experiment against the explicit baseline can establish benefit."}, "claim": "The usage guidance correctly treats independent secondary preamble, post-trigger primary tail, co-residency headroom, and net saved time as benefit preconditions, lists candidate rather than guaranteed workloads, requires a scoped baseline, and forbids reliance on overlap for progress.", "claim_id": "hw-pdl-gdc-current-c005", "claim_type": "performance", "covers": ["u016", "u017", "u018", "u019", "u020", "u021"], "evidence": [{"locator": "background overlap diagram and opportunistic-concurrency warning", "observation": "NVIDIA identifies primary tail and secondary preamble as the overlap opportunity, conditions execution on available hardware resources, declines to guarantee overlap, and warns that relying on it can deadlock. The page adds the necessary measurement controls rather than claiming a result.", "revision": "CUDA 13.0.2", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#programmatic-dependent-launch-and-synchronization"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "One link follows rolling documentation or a mutable branch.", "check": "Inspect CUDA archive segments and the Git blob commit identifier.", "outcome": "CUDA links include 13.0.2, and the CUTLASS link names the exact 40-character commit."}, "claim": "The external references are pinned to CUDA 13.0.2 PDL/PTX sections and the exact CUTLASS 4.5.0 commit document for dependent launch.", "claim_id": "hw-pdl-gdc-current-c006", "claim_type": "other", "covers": ["u022", "u023", "u024"], "evidence": [{"locator": "PDL section", "observation": "The archive and anchor resolve to the cited API contract.", "revision": "CUDA 13.0.2", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#programmatic-dependent-launch-and-synchronization"}, {"locator": "griddepcontrol section", "observation": "The archive and anchor resolve to the cited instruction contract.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-griddepcontrol"}, {"locator": "dependent kernel launch document", "observation": "The immutable blob URL resolves to CUTLASS's GDC enablement and use documentation.", "revision": "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/dependent_kernel_launch.md"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "CLC is another name for PDL or one relative link is broken.", "check": "Resolve both paths and inspect their frontmatter IDs and operation boundaries.", "outcome": "Both links resolve, and CLC operates on not-yet-started blocks within a launch whereas PDL coordinates separate launches."}, "claim": "The internal references resolve to the persistent-kernel and Cluster Launch Control pages, whose scopes are distinct from PDL.", "claim_id": "hw-pdl-gdc-current-c007", "claim_type": "other", "covers": ["u025", "u026"], "evidence": [{"command": "test -f wiki/techniques/persistent-kernels.md && test -f wiki/hardware/clc.md", "environment": "current KernelWiki worktree after PDL remediation", "negative_control": "wiki/hardware/not-clc.md does not exist, so the check rejects a broken target.", "reference": "Resolve both links from wiki/hardware and inspect page IDs and scopes.", "result": "Both files exist; one covers grid-resident persistent scheduling and the other in-grid cancellation/work stealing."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["test -f wiki/techniques/persistent-kernels.md && test -f wiki/hardware/clc.md", "inspect CUTLASS 4.5.0 CMakeLists.txt and grid_dependency_control.h", "structural validation after remediation"], "id": "hw-pdl-gdc", "page_verdict": "verified", "path": "wiki/hardware/pdl-gdc.md", "residual_risk": "No CUDA toolkit or NVIDIA GPU is installed locally, so the CUDA skeleton was not compiled and an overlap timeline was not measured. Its roles mirror the CUDA 13.0.2 example and exact PTX forms; the page makes no quantitative benefit claim.", "scope": "All 26 current units for body hash 4c2bc154856d1dcfd15bb4f6a12ae30d4dadc52ead1d4e5994b4da45e031cd98 were checked against CUDA 13.0.2, PTX ISA 9.0, CUTLASS 4.5.0 source, and current link resolution."} +{"claims": [{"challenge": {"alternative": "The page still promotes an unofficial acronym expansion or assigns tcgen05 to generic sm_100 rather than architecture-specific sm_100a PTX.", "check": "Compare the wording to the pinned target notes and repository-wide CUTLASS nomenclature search.", "outcome": "The page now states only the exact family, target, and namespace facts the primary material exposes."}, "claim": "PTX defines tcgen05.mma as the fifth-generation TensorCore MMA family for sm_100a, and pinned CUTLASS uses UMMA as its SM100 wrapper namespace without defining the expanded acronym asserted by the old page.", "claim_id": "hw-tcgen05-mma-current-c001", "claim_type": "compatibility", "covers": ["u001"], "evidence": [{"locator": "9.7.16.10.9.1 description and Target ISA Notes", "observation": "The vendor ISA names the family, describes fifth-generation MMA, and lists sm_100a. Pinned CUTLASS source uses the UMMA namespace but contains no asserted expansion.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The table accidentally carries over the old SMEM-only A or mandatory-ldmatrix generalization.", "check": "Compare every operand-location cell to both tcgen05 grammar alternatives and WGMMA A forms.", "outcome": "Each current cell preserves the documented alternatives and makes no mandatory ldmatrix claim."}, "claim": "The WGMMA/tcgen05 comparison correctly distinguishes issue granularity, accumulator location, and A/B operand-location alternatives.", "claim_id": "hw-tcgen05-mma-current-c002", "claim_type": "semantics", "covers": ["u002", "u003", "u004", "u005", "u006"], "evidence": [{"locator": "9.7.16.2 issue granularity; 9.7.16.10 overview; WGMMA syntax", "observation": "tcgen05 mma is single-thread-issued, D is TMEM, A is SMEM or TMEM, and B is SMEM; WGMMA is warpgroup-scoped with register D and instruction-dependent register/descriptor A forms.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-matrix-multiply-and-accumulate-operations"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "One issuer makes completion implicit at CTA synchronization.", "check": "Inspect the explicit completion-mechanism definition and its scope.", "outcome": "The ISA assigns completion tracking to tcgen05.commit and the mbarrier, independent of issue granularity."}, "claim": "Single-thread issue does not remove the explicit tcgen05 commit/mbarrier completion path or cross-thread ordering requirements.", "claim_id": "hw-tcgen05-mma-current-c003", "claim_type": "semantics", "covers": ["u007", "u008"], "evidence": [{"locator": "9.7.16.12.1 tcgen05.commit", "observation": "commit causes an mbarrier to track prior async operations by the issuer; fence semantics separately order tcgen05 across execution-ordering handoffs.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "f8f6f4 is itself block-scaled or mxf8 is an accepted alias for mxf8f6f4.", "check": "Compare the exact tokens and operands in the two floating grammar groups.", "outcome": "The current taxonomy matches the exact tokens and keeps the unscaled and block-scaled forms separate."}, "claim": "The page exactly partitions the seven dense kind qualifiers into three unscaled floating, three block-scaled MX, and one integer kind, including the required scale operands for block-scaled forms.", "claim_id": "hw-tcgen05-mma-current-c004", "claim_type": "semantics", "covers": ["u009", "u010", "u011", "u012", "u013"], "evidence": [{"locator": "9.7.16.10.9.1 syntax groups 1, 2, and 5", "observation": "The grammar lists f16/tf32/f8f6f4 without block scaling, mxf8f6f4/mxf4/mxf4nvf4 with block scaling and two TMEM scale operands, and i8 separately.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Each kind implies exactly one M/N pair and group 2 merely doubles M.", "check": "Inspect descriptor-selected shapes and data-path layouts rather than tutorial maximum tiles.", "outcome": "Multiple legal shapes disprove the fixed-shape alternative; the page now labels maxima as examples."}, "claim": "M and N are descriptor-selected under kind/layout/CTA-group/target constraints; cta_group::2 adds peer-CTA resource access, and all tcgen05 instructions in a kernel use one CTA-group value.", "claim_id": "hw-tcgen05-mma-current-c005", "claim_type": "semantics", "covers": ["u014"], "evidence": [{"locator": "instruction descriptor, data-path layouts, operand access, and tcgen05.commit description", "observation": "The ISA encodes shape in idesc, defines multiple layouts and peer accesses, and states that one cta_group qualifier value must be used throughout a kernel.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instruction-descriptor"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The page repeats the old scalar-mask form or presents grammar metavariables as compilable inline PTX.", "check": "Token-compare the blocks to the pinned grammar and inspect the paragraph that scopes the excerpts.", "outcome": "The tokens align with the normative grammar and the page explicitly says the excerpts are not complete functions."}, "claim": "Both displayed PTX blocks reproduce the normative unscaled and block-scaled tcgen05.mma operand grammar, with their intentionally non-standalone scope stated explicitly.", "claim_id": "hw-tcgen05-mma-current-c006", "claim_type": "code", "covers": ["u015", "u016", "u017", "u018", "u019", "u020"], "evidence": [{"locator": "9.7.16.10.9.1 syntax, operand descriptions, and examples", "observation": "The current excerpts retain the disable-output-lane vector and accumulation predicate for unscaled forms, and the block_scale qualifier plus two scale addresses for scaled forms.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A fence alone completes prior MMA, or an mbarrier wait alone establishes every cross-thread tcgen05 ordering edge.", "check": "Compare the distinct guarantees in the adjacent fence and commit sections.", "outcome": "Neither alternative supplies both guarantees; the current wording explicitly composes them only when applicable."}, "claim": "The completion section correctly separates commit/mbarrier completion from tcgen05.fence ordering and preserves SMEM stage lifetime until asynchronous consumption completes.", "claim_id": "hw-tcgen05-mma-current-c007", "claim_type": "semantics", "covers": ["u021", "u022", "u023", "u024"], "evidence": [{"locator": "9.7.16.11.1 fence description and 9.7.16.12.1 commit description", "observation": "Fence orders operations around execution-ordering instructions and constrains code motion; commit makes an mbarrier track async completion. Operand storage cannot be reused while an async operation still consumes it.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The page accidentally uses WGMMA's older two-bit mode encoding, where value 3 has a meaning.", "check": "Compare the page against the tcgen05-specific three-bit Table 40 rather than a WGMMA descriptor table.", "outcome": "The current values exactly match the tcgen05 table and identify value 2, not 3, as ordinary 128B."}, "claim": "The shared-memory descriptor section accurately describes the required field categories and the PTX 9.0 swizzle encodings, including 128B value 2 and invalid values 3, 5, and 7.", "claim_id": "hw-tcgen05-mma-current-c008", "claim_type": "semantics", "covers": ["u025", "u026"], "evidence": [{"locator": "Table 40 shared-memory descriptor layout", "observation": "Table 40 assigns the base/leading/stride/fixed/offset/mode fields and maps swizzles 0,1,2,4,6 while declaring 3,5,7 invalid.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The 1475.93 result includes CLC or the rows mix benchmark environments.", "check": "Trace the current table to the article's consolidated final table, setup, and explicit unimplemented-CLC remark.", "outcome": "Every value shares the disclosed setup, and the source directly excludes CLC from v6."}, "claim": "The performance table exactly reports the tutorial's B200 M=N=K=4096 progression and correctly identifies v6 as static-scheduled persistence without CLC.", "claim_id": "hw-tcgen05-mma-current-c009", "claim_type": "performance", "covers": ["u028", "u029", "u030", "u031", "u032", "u033", "u034", "u035", "u036", "u037", "u038"], "evidence": [{"locator": "experiment setup, Persistent kernel heading, and closing remarks table", "observation": "The setup and all nine table values match the article; v6 is labeled persistent with static scheduling, and the author explicitly lists CLC as not implemented.", "revision": "Article dated 2025-12-21; linked code commit 3b90ac9b", "url": "https://gau-nernst.github.io/tcgen05/"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The 128-register statement is tile-wide or the page guarantees higher occupancy for every tcgen05 kernel.", "check": "Inspect the wrapper typedef and the conditional wording of the current performance implication.", "outcome": "The count is per wrapper participant and the current page says 'can' and 'kernel-dependent,' avoiding a guarantee."}, "claim": "The register-pressure comparison and warp-specialization description are scoped to concrete pinned CUTLASS implementations rather than presented as universal occupancy guarantees.", "claim_id": "hw-tcgen05-mma-current-c010", "claim_type": "mechanism", "covers": ["u027"], "evidence": [{"locator": "MMA_64x256x16_F32F16F16_SS CRegisters and Blackwell tutorial pipeline examples", "observation": "The Hopper wrapper declares float[128] accumulators per thread, while pinned Blackwell examples keep D in TMEM and partition pipeline roles.", "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/include/cute/arch/mma_sm90_gmma.hpp", "revision": "CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "verified"}], "id": "hw-tcgen05-mma", "page_verdict": "verified", "path": "wiki/hardware/tcgen05-mma.md", "residual_risk": "No NVIDIA GPU or CUDA assembler is installed locally, so runtime performance and assembly acceptance were not rerun. The page confines code to normative grammar and labels benchmark values as source-reported.", "scope": "All 38 units in the freshly rescanned current page were checked against CUDA 13.0.2 PTX ISA 9.0, pinned CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5, and the tutorial's source-reported environment and pinned code."} +{"claims": [{"challenge": {"alternative": "All TMA directions use a barrier arrival and the descriptor requests source-to-destination numeric conversion.", "check": "Compare load/store completion rows and inspect whether the tensor map contains distinct source and destination datatypes.", "outcome": "Loads use complete_tx, stores use bulk_group, and one datatype describes the tensor representation; no general conversion pair exists."}, "claim": "TMA on sm_90 and later permits one thread to issue non-blocking rank-1 through rank-5 tensor copies; global-to-shared loads can target CTA/cluster shared memory and multicast with mbarrier byte completion, shared-to-global stores use bulk groups, copied types are not generally converted, and OOB loads fill rather than clamp.", "claim_id": "hw-tma-current-c001", "claim_type": "semantics", "covers": ["u001", "u002", "u003", "u004", "u005"], "evidence": [{"locator": "syntax, description, completion table, history, and target notes", "observation": "The instruction contract supplies the rank set, directions, non-blocking behavior, multicast mode, direction-specific completion, datatype-copy semantics, and sm_90 floor.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"}, {"locator": "TMA overview and OOB behavior", "observation": "The programming guide demonstrates one-thread issue and documents filling of out-of-bounds tensor-load elements rather than edge clamping.", "revision": "CUDA 13.0.2", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#asynchronous-data-copies-using-the-tensor-memory-accelerator-tma"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A tensor map is always host-created and 128B aligned, with unconstrained box dimensions.", "check": "Inspect the Driver API object alignment/box bounds and the separate Blackwell device-encoding section.", "outcome": "The host object alignment is 64B, box entries are bounded, and device-side construction/modification is documented with proxy ordering."}, "claim": "The tensor-map field list, basic CUDA Driver API 13.0.97 tiled-encoder constraints, required CUresult handling, and Blackwell device-side encoding/proxy-publication qualification are accurate.", "claim_id": "hw-tma-current-c002", "claim_type": "semantics", "covers": ["u006", "u007", "u008", "u009", "u010", "u011", "u012", "u013", "u014", "u015", "u016", "u017", "u018", "u019", "u020", "u021"], "evidence": [{"locator": "cuTensorMapEncodeTiled parameters, requirements, and returns", "observation": "The API lists all summarized fields and the 64B tensor-map output alignment, basic 16B base/stride and inner-box rules, box dimensions 1..256, element strides 1..8, swizzle-span bound, type-specific exceptions, and CUresult failure contract.", "revision": "CUDA Driver API 13.0.97", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"}, {"locator": "device-side encoding/modification and tensormap proxy fence", "observation": "The Blackwell path and fence.proxy.tensormap::generic publication sequence match the page's qualification.", "revision": "CUDA 13.0.2", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#encoding-a-tensor-map-on-device"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Each TMA completion supplies a software arrival, or zero tx-count alone releases the consumer.", "check": "Trace complete-tx and arrive-on to their separate object fields and inspect the phase-completion conjunction.", "outcome": "TMA changes only tx-count, and completion requires both tx-count and pending arrivals to be zero."}, "claim": "The representative 2D global-to-CTA-shared PTX is exact and the load lifecycle correctly separates initialization, one software arrive/expected-byte update, TMA complete_tx bytes, per-stage phase tracking, and an acquire wait with conjunctive phase completion.", "claim_id": "hw-tma-current-c003", "claim_type": "code", "covers": ["u022", "u023", "u024", "u025", "u026", "u027", "u028", "u029", "u030"], "evidence": [{"locator": "global-to-shared grammar and mbarrier::complete_tx::bytes semantics", "observation": "The shown instruction qualifier order/operands are accepted, the default load mode is tile, and completion decrements tx-count by copied bytes.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"}, {"locator": "arrive.expect_tx, phase completion, and wait ordering", "observation": "The page's separate arrival/expected-byte/complete-byte accounting and requirement for both counters to reach zero match the object lifecycle.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#phase-completion-of-the-mbarrier-object"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "commit_group is a blocking store completion operation, making wait_group redundant.", "check": "Compare the definitions of commit_group and wait_group.", "outcome": "Commit only batches uncommitted operations; the separately defined wait supplies completion."}, "claim": "The shared-to-global store sequence uses the required bulk_group qualifier, commit_group, and wait_group 0, and correctly distinguishes grouping from completion and source-buffer reuse safety.", "claim_id": "hw-tma-current-c004", "claim_type": "code", "covers": ["u031", "u032", "u033"], "evidence": [{"locator": "shared::cta-to-global form plus cp.async.bulk commit_group/wait_group sections", "observation": "The exact instruction operands and completion qualifier match the grammar; commit creates a per-thread group and wait_group 0 waits for all prior committed groups and makes writes visible to the issuer.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "threadIdx.x==0 elects one issuer for the whole cluster and a one-to-two multicast proves exactly half DRAM traffic.", "check": "Count threadIdx.x==0 threads across CTAs and distinguish functional destination count from measured cache/DRAM transactions.", "outcome": "There is one such thread per CTA, and the functional ISA does not specify baseline cache reuse or an exact bandwidth ratio."}, "claim": "The exact cluster-multicast form, same-offset destination and barrier signaling, cluster lifetime/initialization/election/mask/wait invariants, GEMM A/B index-reuse example, and non-guaranteed traffic caveat are correct.", "claim_id": "hw-tma-current-c005", "claim_type": "code", "covers": ["u034", "u035", "u036", "u037", "u038", "u039", "u040", "u041", "u042"], "evidence": [{"locator": "multicast::cluster ctaMask, same-offset data, and mbarrier signal semantics", "observation": "The instruction and same-offset semantics match exactly; cluster-shared lifetime makes initialization and liveness necessary, while one intended issue requires cluster-level rather than per-CTA election. The ISA gives no exact DRAM-saving factor.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "128B is the one canonical tcgen05 layout and automatically makes any consumer conflict-free.", "check": "Enumerate legal descriptor modes and apply a mismatched/alternate consumer access pattern.", "outcome": "Non-128B modes are normative, and bank mapping still depends on consumer addresses; only a matched pattern can obtain the intended conflict behavior."}, "claim": "TMA and tcgen05 expose multiple swizzle layouts; correctness requires the TMA map, base/leading dimensions, and consuming shared-memory descriptor to match, while bank-conflict benefit remains access-pattern-specific rather than universally requiring 128B.", "claim_id": "hw-tma-current-c006", "claim_type": "semantics", "covers": ["u043", "u044", "u045"], "evidence": [{"locator": "CUtensorMapSwizzle enum and swizzle constraints", "observation": "The tensor map supports none, 32B, 64B, 128B, and additional variants with inner-span/alignment/type rules.", "revision": "CUDA Driver API 13.0.97", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"}, {"locator": "tcgen05 descriptor swizzling modes", "observation": "tcgen05 descriptors likewise encode multiple modes and interpret addresses/layout accordingly; legal modes vary by instruction configuration.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-layout"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "tcgen05 consumes shared operands at issue, so only a full barrier is needed and producers may overwrite immediately afterward.", "check": "Trace the official pipeline's consumer release relative to asynchronous MMA issue/completion.", "outcome": "Stage release is delayed through the consumer lifecycle; the implementation retains separate producer and consumer state, discriminating asynchronous operand lifetime."}, "claim": "The Blackwell GEMM data path and per-stage full/empty ownership transitions correctly require load completion before MMA reads and asynchronous-MMA read completion before producer overwrite, with separate per-stage phases and no substitution of CTA sync or fence for completion.", "claim_id": "hw-tma-current-c007", "claim_type": "semantics", "covers": ["u046", "u047", "u048", "u049", "u050", "u051"], "evidence": [{"locator": "TMA atoms, PipelineTmaUmma full/empty producer-consumer loops, UMMA, TMEM retrieve, and epilogue", "observation": "The complete implementation follows the stated data path and explicitly acquires/releases per-stage pipeline state around asynchronous TMA and UMMA lifetimes.", "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_6.py", "revision": "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Three to five stages is an architecture constant that can be selected without profiling.", "check": "Search the normative TMA contract for a stage-count field and compare resource use across different tile sizes.", "outcome": "TMA exposes no such constant; tile bytes and occupancy change the feasible and useful count."}, "claim": "The tuning guidance makes all layout, multicast, issue, and stage choices conditional on access pattern/resources, states the monotonic storage cost of distinct stages, rejects a universal 3-5 optimum, and specifies enough controls for a scoped profile.", "claim_id": "hw-tma-current-c008", "claim_type": "performance", "covers": ["u052"], "evidence": [{"command": "derive storage = fixed_bytes + stages * per_stage_bytes and inspect that no numeric speedup is asserted", "environment": "symbolic resource accounting over the documented distinct-stage design", "negative_control": "A single reused non-overlapped buffer does not satisfy the distinct-stage premise and therefore does not show the same linear allocation.", "reference": "Each stage owns separate positive-size operand buffers and barrier state; performance depends on target execution/resource conditions.", "result": "Adding a stage increases storage by per_stage_bytes; the page supplies no fixed optimum or outcome and requires controlled target measurement."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "One source follows rolling docs or a mutable Git branch.", "check": "Inspect CUDA archive segments and the Git blob identifier.", "outcome": "All CUDA links include 13.0.2 and CUTLASS names an exact commit."}, "claim": "The four external references are pinned to CUDA 13.0.2/Driver API 13.0.97/PTX ISA 9.0 and an immutable CUTLASS 4.5.0 TMA tutorial.", "claim_id": "hw-tma-current-c009", "claim_type": "other", "covers": ["u053", "u054", "u055", "u056"], "evidence": [{"locator": "TMA section", "observation": "The archive and anchor match the label.", "revision": "CUDA 13.0.2", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#asynchronous-data-copies-using-the-tensor-memory-accelerator-tma"}, {"locator": "tensor-map management", "observation": "The archive resolves to the encoder constraints.", "revision": "CUDA Driver API 13.0.97", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"}, {"locator": "tensor copy section", "observation": "The archive and anchor match the label.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"}, {"locator": "complete TMA tutorial", "observation": "The immutable blob exists at the pinned release commit.", "revision": "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_tma/tma_v0.py"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "One path is broken or links to an unrelated generation.", "check": "Resolve both paths and compare page IDs/titles.", "outcome": "Both point to the intended verified hardware pages."}, "claim": "The internal references resolve to the verified mbarrier and tcgen05 MMA pages.", "claim_id": "hw-tma-current-c010", "claim_type": "other", "covers": ["u057", "u058"], "evidence": [{"command": "test -f wiki/hardware/mbarrier.md && test -f wiki/hardware/tcgen05-mma.md", "environment": "current KernelWiki worktree after TMA remediation", "negative_control": "wiki/hardware/not-tcgen05.md does not exist, so the path check rejects a broken target.", "reference": "Resolve both relative paths and inspect their frontmatter IDs.", "result": "Both files exist with IDs hw-mbarrier and hw-tcgen05-mma and cover the linked contracts."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["inspect CUTLASS 4.5.0 TMA/UMMA pipeline", "derive per-stage storage scaling", "test -f wiki/hardware/mbarrier.md && test -f wiki/hardware/tcgen05-mma.md", "structural validation after remediation"], "id": "hw-tma", "page_verdict": "verified", "path": "wiki/hardware/tma.md", "residual_risk": "No CUDA toolkit or NVIDIA GPU is installed locally, so the three PTX snippets were not assembled and no TMA timing/traffic counters were collected. Syntax and ordering are version-pinned to PTX ISA 9.0; the page makes no numeric performance claim.", "scope": "All 58 current units for body hash 1eb48a33cd77ce277ac4783d357f5784e487533159bbe155fc942f404f3655f1 were checked against CUDA 13.0.2, CUDA Driver API 13.0.97, PTX ISA 9.0, CUTLASS 4.5.0, symbolic resource accounting, and local link resolution."} +{"claims": [{"challenge": {"alternative": "The 256 KiB figure is separate from the lane-column array, or moving D to TMEM makes the whole kernel register-free.", "check": "Recompute the lane-column capacity and inspect both the exact WGMMA CRegisters typedef and the corrected residual-register wording.", "outcome": "128 x 512 x 4 bytes equals 256 KiB, and the page limits the relief to the D fragment while preserving other GPR uses."}, "claim": "Blackwell TMEM is a 256 KiB per-SM, 128-lane by 512-column array of 32-bit cells used for tcgen05 D and some A operands; replacing a register-D WGMMA fragment changes rather than eliminates register pressure.", "claim_id": "hw-tmem-current-c001", "claim_type": "semantics", "covers": ["u001", "u002"], "evidence": [{"locator": "Tensor Memory organization and tcgen05 operand overview", "observation": "The ISA specifies the capacity, lane-column geometry, and TMEM operand roles.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"}, {"locator": "MMA_64x256x16_F32F16F16_SS::CRegisters", "observation": "The representative SM90 wrapper declares 128 float accumulators per participating thread, while the corrected text expressly leaves other GPR uses in scope.", "revision": "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cute/arch/mma_sm90_gmma.hpp"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The upper address half is a CUDA thread ID, making the corrected warning against row ownership unnecessary.", "check": "Compare the address definition with the collective execution and access-restriction tables.", "outcome": "The ISA calls it a TMEM lane and separately requires warp-collective execution with shape-dependent mappings."}, "claim": "A b32 taddr encodes lane and column, while collective tcgen05 load/store access maps four warps to successive 32-lane chunks using shape-specific register layouts rather than per-thread row ownership.", "claim_id": "hw-tmem-current-c002", "claim_type": "semantics", "covers": ["u003", "u004", "u005", "u006", "u007", "u008", "u009", "u010", "u011"], "evidence": [{"locator": "TMEM address format; tcgen05.ld/st access restrictions", "observation": "The address field table assigns lane to bits 31:16 and column to 15:0; the access tables partition lanes into four warp chunks and define layouts by instruction shape.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory-addressing"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Allocation is a lane-0 instruction and CTA synchronization supplies the missing collective behavior.", "check": "Inspect the mandatory .sync/.aligned and same-operand clauses for cta_group::1 and the paired-CTA issue table.", "outcome": "The operation itself must execute uniformly across the allocator warp or warps; a later CTA barrier cannot repair divergent issue."}, "claim": "TMEM allocation returns a b32 taddr through shared memory, uses one allocator warp for group 1 or one warp per peer CTA for group 2, requires one CTA-group value throughout the kernel, and every allocation must be collectively deallocated before exit.", "claim_id": "hw-tmem-current-c003", "claim_type": "semantics", "covers": ["u012", "u013", "u014", "u015", "u016", "u017", "u018", "u019"], "evidence": [{"locator": "alloc/dealloc syntax, issue granularity, constraints, and lifecycle", "observation": "The section specifies the shared b32 result, group-specific warp collectives, same-group invariant, 32-512 power-of-two nCols, blocking availability behavior, and before-exit deallocation rule.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Multiple-of-32 sizes such as 192 are legal even if the high-level CUTLASS allocator rounds them.", "check": "Read the PTX operand constraint rather than inferring it from a library allocator.", "outcome": "The ISA itself requires nCols to be a power of two, discriminating the 192-column case."}, "claim": "The 512-column pool admits the stated 128-, 256-, and 512-column whole-allocation budgets, while a 192-column logical region requires a legal 256-column reservation or suballocation.", "claim_id": "hw-tmem-current-c004", "claim_type": "semantics", "covers": ["u020", "u021", "u022", "u023", "u024"], "evidence": [{"locator": "nCols power-of-two constraint and 512-column maximum", "observation": "Legal sizes are powers of two in [32,512]. Division of the single 512-column pool yields the displayed stage counts, and 192 itself is excluded.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A CTA barrier or before_thread_sync fence implicitly waits for every prior asynchronous tcgen05 operation.", "check": "Compare the normative effect of fence/barrier examples with the separate commit and wait completion mechanisms.", "outcome": "Only commit/mbarrier and wait::* state completion; fences are explicitly ordering and code-motion constraints."}, "claim": "tcgen05 MMA/copy completion uses commit plus mbarrier, load/store completion uses the corresponding tcgen05.wait form, and before/after fences provide cross-thread ordering rather than completion; source lifetimes and CTA synchronization must respect those distinctions.", "claim_id": "hw-tmem-current-c005", "claim_type": "semantics", "covers": ["u025", "u026", "u027", "u028", "u029", "u030"], "evidence": [{"locator": "wait::ld/wait::st semantics and asynchronous-operation table", "observation": "The wait variants block on prior ld/st operations from the executing thread.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-wait"}, {"locator": "commit completion and fence ordering sections", "observation": "Commit attaches prior MMA/cp/shift completion to an mbarrier; fences compose around execution-ordering operations and provide no completion promise.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The copied shape is 128x256 elements rather than bits, or sdesc denotes another TMEM address.", "check": "Inspect the shape suffix and operand-type description adjacent to the exact example.", "outcome": "The suffix is explicitly 128x256b and the source is explicitly a shared-memory matrix descriptor."}, "claim": "The page lists valid tcgen05.ld/st shapes, rejects nonexistent .32x1b scalar forms, and gives the exact valid group-1 128x256b shared-memory-to-TMEM tcgen05.cp instruction with correct operands and asynchronous caveat.", "claim_id": "hw-tmem-current-c006", "claim_type": "code", "covers": ["u031", "u032", "u033", "u034"], "evidence": [{"locator": "tcgen05.ld/st shape grammar", "observation": "The five listed shapes appear in the grammar and .32x1b does not.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"}, {"locator": "tcgen05.cp syntax, operand descriptions, and Example 1", "observation": "The exact displayed instruction appears in the vendor example; taddr is the destination, sdesc is a b64 SMEM descriptor, and cp is asynchronous.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-cp"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The hardware automatically detects alternating accumulator addresses and overlaps serial code without pipeline participants.", "check": "Inspect the vendor tutorial for the control and synchronization state that accompanies multiple TMEM stages.", "outcome": "The implementation explicitly acquires, commits, waits, and releases stages; address alternation is not sufficient."}, "claim": "A correct multi-stage TMEM accumulator pipeline must fit the column budget and use explicit producer/consumer barriers and completion before drain or recycle; alternating two addresses alone does not create overlap.", "claim_id": "hw-tmem-current-c007", "claim_type": "mechanism", "covers": ["u035", "u036", "u037", "u038", "u039", "u040", "u041"], "evidence": [{"locator": "TMEM allocation; PipelineUmmaAsync stages; producer/consumer acquire, commit, wait, and release", "observation": "The pinned complete implementation allocates the shared TMEM pool and coordinates distinct accumulator producers/consumers with explicit stage barriers and completion state.", "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_6.py", "revision": "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The tutorial bypasses TmemAllocator and obtains an implicitly allocated TMEM tensor from the MMA atom.", "check": "Trace the accumulator pointer from shared allocation-result storage through retrieve_ptr and make_tensor.", "outcome": "The pointer is explicitly allocated, waited for, retrieved, rebound, drained, and freed."}, "claim": "CUTLASS 4.5.0's Python DSL uses TmemAllocator allocation/wait/retrieve/free operations, typed pointers, cute.make_tensor, tcgen05 copy atoms, and pipeline completion in the sequence summarized by the page.", "claim_id": "hw-tmem-current-c008", "claim_type": "code", "covers": ["u042", "u043", "u044", "u045", "u046", "u047", "u048", "u049"], "evidence": [{"locator": "TmemAllocator construction and allocate/wait/retrieve/make_tensor/load/free flow", "observation": "Every named operation and its ordering appears in the official tutorial.", "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py", "revision": "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5"}, {"locator": "TmemAllocator methods allocate, wait_for_alloc, retrieve_ptr, and free", "observation": "The library implementation defines the exact high-level allocation API and collective allocator-warp behavior summarized.", "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/python/CuTeDSL/cutlass/utils/tmem_allocator.py", "revision": "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The convenience calls are generated symbols or re-exported from a path not imported by the tutorial.", "check": "Search the entire pinned repository, not only the tutorial directory, and inspect public arch and utility exports.", "outcome": "The tokens remain absent repository-wide while the documented allocator and copy-atom interfaces are present."}, "claim": "The pinned links identify the exact CUTLASS 4.5.0 tutorial and allocator implementation, while the rejected Layout(memory_space=MemorySpace.TMEM), tcgen05_mma, and similar convenience calls are absent from that version.", "claim_id": "hw-tmem-current-c009", "claim_type": "compatibility", "covers": ["u050"], "evidence": [{"locator": "repository-wide exact-token search; linked tutorial and allocator paths", "observation": "The two linked files exist and implement the described API. Exact searches for MemorySpace.TMEM, tcgen05_mma, tcgen05_fence, and tmem_load as displayed API names return no definitions or uses.", "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass", "revision": "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The URLs silently track latest documentation or a moving CUTLASS branch.", "check": "Inspect the CUDA archive segment and immutable Git commit embedded in each URL.", "outcome": "Both source families are version-pinned in the links themselves."}, "claim": "The five primary-reference links resolve to the pinned PTX sections and pinned CUTLASS 4.5.0 SM100 tutorial tree that support the page.", "claim_id": "hw-tmem-current-c010", "claim_type": "other", "covers": ["u051", "u052", "u053", "u054", "u055"], "evidence": [{"locator": "Tensor Memory and linked alloc/ld/commit anchors", "observation": "The archive resolves the listed normative sections at the pinned CUDA version.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"}, {"locator": "SM100 tutorial directory", "observation": "The commit-pinned directory contains the referenced progressive SM100 Python GEMM tutorials.", "revision": "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5", "url": "https://github.com/NVIDIA/cutlass/tree/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm"}], "oracle": "web-primary", "verdict": "verified"}], "commands": ["rg -n against the full pinned CUTLASS 4.5.0 worktree for exact API tokens", "structural validation after remediation"], "id": "hw-tmem", "page_verdict": "verified", "path": "wiki/hardware/tmem.md", "residual_risk": "No CUDA toolkit or NVIDIA GPU is installed locally, so the exact PTX example was not assembled and the pipeline was not benchmarked. Its syntax and semantics are matched to PTX ISA 9.0, and the high-level API is matched to immutable CUTLASS 4.5.0 source.", "scope": "All 55 current scan units for body hash f6c0135c31b4600deecf3ee1dd972746a40bb9b3c0968b96e5ceb231a35ce816 were verified against CUDA 13.0.2 PTX ISA 9.0 and CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5."} +{"claims": [{"challenge": {"alternative": "The commit link is only decorative and the local files come from a different revision or represent the whole repository.", "check": "Compare the frontmatter, README tree, provenance upstream SHA, and exact file hashes.", "outcome": "The revision matches across the page and manifest, hashes match, and the pinned tree visibly contains many other kernel/API files."}, "claim": "The page is scoped to DeepGEMM commit 891d57b, accurately identifies the project and its SM90/SM100 requirements, and distinguishes two representative local FP8 1D1D files from the full multi-kernel library.", "claim_id": "kernel-deepgemm-current-c001", "claim_type": "compatibility", "covers": ["u001", "u002"], "evidence": [{"locator": "overview, requirements, and interfaces", "observation": "The README identifies the library, SM90/SM100 requirements, runtime-compiled design, and multiple dense/grouped/attention/MoE interfaces.", "path": "/tmp/kernelwiki-deepgemm-pinned/README.md", "revision": "DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f"}, {"locator": "upstream_sha and file records", "observation": "The two representative files are pinned to the same commit and carry matching SHA-256 values.", "path": "artifacts/kernels/deepgemm/full/PROVENANCE.yaml", "revision": "local byte-verified manifest"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Both operands use 128x128 blocks and every architecture consumes FP32 scales.", "check": "Compare the paper's separate operand rules and the pinned README/kernel types.", "outcome": "Activation grouping is per row, while the SM100 representation and accepted granularities differ explicitly from SM90."}, "claim": "The DeepSeek-V3 1x128 activation and 128x128 weight scaling geometry, scoped outlier motivation, architecture-specific FP32 versus packed-UE8M0 representations, four-value packing, and pinned K-granularity statements are accurate.", "claim_id": "kernel-deepgemm-current-c002", "claim_type": "semantics", "covers": ["u003", "u004", "u005", "u006", "u007"], "evidence": [{"locator": "section 3.3.2 and Appendix B.2", "observation": "The paper specifies the two granularities and says smaller groups better accommodate outliers; it does not promise elimination of quantization error.", "revision": "DeepSeek-V3 Technical Report, arXiv v2", "url": "https://arxiv.org/abs/2412.19437v2"}, {"locator": "interface notices", "observation": "SM90 uses FP32 factors and SM100 uses packed UE8M0 with four values per torch.int.", "revision": "DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"}, {"locator": "BLOCK_K and kGranKA/kGranKB static assertions", "observation": "The SM100 template fixes BLOCK_K to 128 and permits scale K granularities 32 or 128; the corresponding SM90 file fixes BLOCK_K to 128.", "revision": "DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Nc=128 describes 128 output columns and the real accumulator is half precision as in the deleted sketch.", "check": "Trace the k_block loop and declared C++ types and compare Nc's definition in the paper.", "outcome": "The interval advances K, WGMMA output registers are declared float, and promotion targets a separate float array."}, "claim": "The H800 retained-bit description, Nc=128 K-element/four-WGMMA interval, four-step walkthrough, and verbatim six-line excerpt of the pinned SM90 accum/final_accum promotion path match the paper and source.", "claim_id": "kernel-deepgemm-current-c003", "claim_type": "code", "covers": ["u008", "u009", "u010", "u011", "u012", "u013", "u014", "u041", "u042"], "evidence": [{"locator": "sections 3.3.2 and 3.5.2", "observation": "The report states about 14 retained bits, promotion every 128 inner-dimension elements, equivalence to four WGMMAs, and FP32 CUDA-core accumulation.", "revision": "DeepSeek-V3 Technical Report, arXiv v2", "url": "https://arxiv.org/abs/2412.19437v2"}, {"locator": "math warp-group loop from accum declarations through Promote with scales and store", "observation": "The code declares float accum/final_accum, loads FP32 A/B factors, issues and waits for WGMMA for one BLOCK_K=128 stage, scales into final_accum, and stages final values for store.", "revision": "DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Packed scales remain in general registers and are immediate MMA operands, while TMEM holds only output.", "check": "Follow smem_sfa/sfb through the UTCCP copy destinations and the final fma arguments.", "outcome": "The copies target dedicated TMEM scale columns and fma receives their TMEM addresses separately from the TMEM accumulator column."}, "claim": "The SM100 walkthrough accurately describes the float/UE8M0 block-scaled instruction descriptor, TMA-to-shared and UTCCP-to-TMEM scale path, elected UMMA issue operands, TMEM accumulator lifecycle, absence of SM90 final_accum promotion, and precision qualification.", "claim_id": "kernel-deepgemm-current-c004", "claim_type": "mechanism", "covers": ["u015", "u016", "u017", "u018", "u019", "u020"], "evidence": [{"locator": "TMA scale loads, make_instr_desc_block_scaled, UTCCP copies, mma_t::fma, tmem barriers, and epilogue", "observation": "Each stated data-movement, descriptor, issue, and lifecycle element appears in the pinned source. The template selects accumulator and scale types rather than asserting a universal tcgen05 precision mode.", "revision": "DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Masked mode uses a Boolean element mask and K-grouped is merely a third M-grouped storage layout.", "check": "Inspect masked_m dtype/shape and the K-grouped shape equations.", "outcome": "masked_m has one int per group, while K-grouped sums per-group K lengths and fixes the output's M/N."}, "claim": "The grouped-interface table and surrounding text correctly distinguish M-grouped contiguous, M-grouped masked, and K-grouped contiguous shapes, metadata, purposes, alignment, and pinned SM90-NT/SM100-TN K-grouped variants.", "claim_id": "kernel-deepgemm-current-c005", "claim_type": "semantics", "covers": ["u021", "u022", "u023", "u024", "u025", "u031"], "evidence": [{"locator": "Grouped GEMMs contiguous and masked sections", "observation": "The README gives the M-grouped fixed-axis/use-case rules, segment alignment, masked decode role, and separate K-axis weight-backward API.", "revision": "DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"}, {"locator": "m_grouped_fp8_fp4_gemm_* and k_grouped_fp8_gemm_* shape/layout checks", "observation": "The API checks establish each tensor shape and metadata type; contiguous supports M-length or group-count metadata forms, masked_m is an int valid-length vector, and K-grouped dispatches TN only to SM100 and NT only to SM90.", "revision": "DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "NVRTC is the mandatory/default JIT and every invocation recompiles the specialization.", "check": "Evaluate the compiler selection default and the early cache-hit return.", "outcome": "The default expression is zero, selecting NVCC, and a valid cached runtime returns before compilation."}, "claim": "The pinned JIT defaults to NVCC, optionally selects NVRTC through DG_JIT_USE_NVRTC=1 with a performance caveat, keys the cache by kernel/compiler/flags/code, compiles and publishes misses, reuses hits, and makes no optimality guarantee.", "claim_id": "kernel-deepgemm-current-c006", "claim_type": "mechanism", "covers": ["u026", "u027", "u032"], "evidence": [{"locator": "DG_JIT_USE_NVRTC environment variable", "observation": "NVRTC is optional, defaults off, and may perform worse in some cases.", "revision": "DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"}, {"locator": "Compiler::build and compiler LazyInit", "observation": "The code constructs the stated signature, returns cache hits, compiles/publishes misses atomically, and selects NVCC unless the environment opt-in is nonzero.", "revision": "DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/jit/compiler.hpp"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The four SM100 names all demand physically NT inputs and do not alter descriptors.", "check": "Trace tensor-view transformations and kMajorA/kMajorB into the SM100 template.", "outcome": "Different wrappers transform A and/or B and the selected major values reach descriptor construction."}, "claim": "The pinned FP8 interface supports NT only on SM90 and NT/TN/NN/TT on SM100, with public wrappers and SM100 operand-major parameters implementing the additional layouts.", "claim_id": "kernel-deepgemm-current-c007", "claim_type": "compatibility", "covers": ["u028"], "evidence": [{"locator": "Interfaces notices and normal dense GEMMs", "observation": "The README states the exact architecture/layout support and naming convention.", "revision": "DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"}, {"locator": "fp8_fp4_gemm_nt/nn/tn/tt and major checks", "observation": "The wrappers transpose relevant views and pass major modes; the SM90 path asserts K-major inputs.", "revision": "DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The up-to result proves the former 4096-cubed and 90% row by implication.", "check": "Search the pinned README and linked performance history for that condition tuple.", "outcome": "The README supplies neither field, and the available shape-scoped peak evidence names a different tuple; the page now says exactly that."}, "claim": "The only retained numeric performance statement is the pinned project's source-reported up-to 1550 TFLOPS on H800, explicitly qualified as lacking the former 4096-cubed shape, 90% utilization, and a complete reproducibility envelope.", "claim_id": "kernel-deepgemm-current-c008", "claim_type": "performance", "covers": ["u029"], "evidence": [{"locator": "2025-04-18 news item", "observation": "The README says up to 1550 TFLOPS on H800 and supplies none of the deleted shape/utilization conditions.", "revision": "DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"}, {"locator": "performance report", "observation": "A nearby disclosed peak result uses M=4096, N=7168, K=16384, helping discriminate it from the page's former unsupported 4096-cubed attribution.", "revision": "merged 2025-04-14", "url": "https://github.com/deepseek-ai/DeepGEMM/pull/86"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Every extra scale necessarily makes the end-to-end kernel slower than per-tensor scaling, so target/workload qualifications are evasive.", "check": "Separate arithmetic count from native instruction support, scale preparation/fusion, and the measured scope in the primary sources.", "outcome": "The sources establish architecture-dependent mechanisms and no controlled universal comparison; conditional wording is the strongest supported conclusion."}, "claim": "The practical boundaries accurately separate input preparation from GEMM, state project warnings and grouped/JIT constraints, and avoid claiming a universal fine-grained-scaling cost or decision rule.", "claim_id": "kernel-deepgemm-current-c009", "claim_type": "performance", "covers": ["u030", "u033"], "evidence": [{"locator": "interface notices", "observation": "The project says transposition and FP8 casting are user-side operations and that its utility implementations may be slower than fusing them into prior kernels.", "revision": "DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"}, {"locator": "section 3.3.2", "observation": "The paper describes the precision motivation and says the scale multiplication has minimal extra computational cost in its Hopper promotion strategy; it does not establish a universal cost ordering.", "revision": "DeepSeek-V3 Technical Report, arXiv v2", "url": "https://arxiv.org/abs/2412.19437v2"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The links are rolling branches, the local files are untracked adaptations, or the query omits the assets.", "check": "Inspect each URL for a full commit/version token, verify bytes/modes, and execute the lookup.", "outcome": "All external references are immutable, local bytes and provenance agree, and the query returns the page plus its code bundle."}, "claim": "The four pinned external links, local provenance/hash statements, upstream-versus-derived labels, and page query command are correct and resolvable.", "claim_id": "kernel-deepgemm-current-c010", "claim_type": "code", "covers": ["u034", "u035", "u036", "u037", "u038", "u039", "u040"], "evidence": [{"command": "git -C /tmp/kernelwiki-deepgemm-pinned rev-parse HEAD; sha256sum artifacts/kernels/deepgemm/full/sm90_fp8_gemm_1d1d.cuh artifacts/kernels/deepgemm/full/sm100_fp8_gemm_1d1d.cuh; conda run -n base python scripts/get_page.py kernel-deepgemm --include-code", "environment": "KernelWiki worktree; exact upstream clone; conda base Python 3.13.14", "negative_control": "The non-existent id kernel-not-deepgemm is rejected, discriminating actual index resolution from unconditional output.", "reference": "page links and artifacts/kernels/deepgemm/{full,variants}/PROVENANCE.yaml", "result": "The clone is at 891d57b4db1071624b5c8fa0d1e51cb317fa709f, both hashes equal their manifest entries, modes are verbatim for full and derived for variants, and the query succeeds with attached code."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["inspect DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f", "sha256sum pinned full assets", "conda run -n base python scripts/get_page.py kernel-deepgemm --include-code", "inspect DeepSeek-V3 Technical Report v2 and DeepGEMM performance history"], "id": "kernel-deepgemm", "page_verdict": "verified", "path": "wiki/kernels/deepgemm.md", "residual_risk": "No NVIDIA GPU or CUDA toolkit is installed locally, so the kernels were not compiled or executed and the source-reported H800 peak was not reproduced. The implementation statements are source-level conclusions tied to commit 891d57b; no performance portability beyond the explicit scope is asserted.", "scope": "All 42 current units for body hash 5e2f98b996033253da1252cec281af40f1f858fa39675a8296b17564863d76fe were checked against the pinned DeepGEMM tree and local byte-verified assets, DeepSeek-V3 Technical Report v2, upstream performance history, and repository query behavior."} +{"claims": [{"challenge": {"alternative": "The 20-30x number measures runtime or end-to-end build time, and the repository link follows a mutable branch.", "check": "Inspect the paper's compilation units/labels and the Git URL's revision component.", "outcome": "The values are seconds for individual kernel compilation, while the implementation URL embeds a full immutable commit."}, "claim": "The page accurately separates the SM100/B200 scope of FA4 paper v1 from Dao-AILab commit a369df7 and reports the paper's CuTe DSL and single-kernel FA3 compilation comparison with exact times and scope.", "claim_id": "kernel-flash-attention-4-current-c001", "claim_type": "performance", "covers": ["u001", "u002", "u003", "u004"], "evidence": [{"locator": "abstract, introduction, and section 5.1", "observation": "The paper targets Blackwell/B200, identifies its CuTe DSL implementation, and reports FA3 versus FA4 compile times of 55 versus 2.5 seconds forward and 45 versus 1.4 seconds backward.", "revision": "FlashAttention-4 paper v1, 2026-03-05", "url": "https://arxiv.org/html/2603.05451v1"}, {"locator": "pinned package tree", "observation": "The immutable URL resolves to the implementation snapshot named by the page.", "revision": "Dao-AILab commit a369df707e1980fb328abcc1733e3457ec10155f", "url": "https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The two objects are only K/V load buffers for one query tile, so the page overstates output-level concurrency.", "check": "Follow the output-tile indices and softmax-warpgroup assignments in the paper's pipeline description.", "outcome": "Each softmax group is assigned to a distinct output tile and the alternating state includes separate output accumulators, rejecting the K/V-only interpretation."}, "claim": "FA4's forward ping-pong schedule alternates two 128-row output tiles, uses one MMA warp, two four-warp softmax groups and a correction warpgroup, stores score/output accumulators in separate TMEM regions, and is distinct from ordinary K/V double buffering.", "claim_id": "kernel-flash-attention-4-current-c002", "claim_type": "mechanism", "covers": ["u005", "u006"], "evidence": [{"locator": "section 3.1, Ping-pong Scheduling and forward pipeline figure", "observation": "The paper describes two M=128 output tiles, their two softmax warpgroups, a correction warpgroup, one MMA warp, and distinct TMEM regions with alternating pipeline work.", "revision": "FA4 paper v1", "url": "https://arxiv.org/html/2603.05451v1"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The original centered, degree-5 function is an equally valid representation of FA4, or the page now implies every entry uses the scalar path.", "check": "Compare interval, polynomial degree, and entry-selection statements against the paper/blog and inspect the page qualification around the function.", "outcome": "The primary sources select floor/[0,1)/degree 3 and only a fraction of entries; the current page and executable scalar match those boundaries exactly."}, "claim": "FA4 partially emulates about 10-25% of exponential entries; the displayed scalar Python implements the published floor-to-[0,1), degree-3 rounded-coefficient reference and is explicitly not the kernel selection, clamping, or scheduling implementation.", "claim_id": "kernel-flash-attention-4-current-c003", "claim_type": "code", "covers": ["u007", "u008", "u009", "u010"], "evidence": [{"locator": "section 3.2, Partial Emulation of Exponential", "observation": "The source states the 10-25% selection, floor range reduction, [0,1) fraction, degree-3 polynomial, and hybrid FMA/MUFU execution. Direct comparison shows the displayed scalar performs those exact arithmetic steps and no kernel scheduling.", "revision": "FA4 paper v1", "url": "https://arxiv.org/html/2603.05451v1"}, {"locator": "software exponential equations and coefficient row", "observation": "The blog prints the rounded coefficients 1.0, 0.6951, 0.2276, and 0.0771 used verbatim by the scalar reference.", "revision": "Tri Dao blog 2026-03-05", "url": "https://tridao.me/blog/2026/flash4/"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Checking absolute LSE change and then using the new maximum without a multiply is algebraically equivalent.", "check": "Substitute the alternative state update into the paper's exponent and final-normalization equations.", "outcome": "LSE contains the row sum and the alternative changes the exponent's reference without compensating state; it does not satisfy the published recurrence."}, "claim": "FA4 conditional rescaling compares running maxima, typically uses tau=8.0 base-2 units, retains the old reference maximum when skipping, tracks delayed normalization state, and renormalizes at the end; LSE-difference skipping is not equivalent.", "claim_id": "kernel-flash-attention-4-current-c004", "claim_type": "mechanism", "covers": ["u011", "u012"], "evidence": [{"locator": "section 3.3, Conditional Rescaling, equations 7-10", "observation": "The paper supplies the one-direction max condition, typical log2(256)=8 threshold, retained-max exponent convention, auxiliary statistics, and final correction.", "revision": "FA4 paper v1", "url": "https://arxiv.org/html/2603.05451v1"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The factor applies to all shared-memory traffic, or paired CTAs simply split dK and dV into independent kernels.", "check": "Trace each operand and the dS exchange in the backward dataflow figure/text.", "outcome": "Only the B operand has the stated shared-read reuse and both CTAs participate in the coordinated GEMMs/dQ exchange, rejecting both broader alternatives."}, "claim": "The backward section correctly scopes two-CTA M=256,N=128,K=128 use to five GEMMs, approximate shared-read halving to operand B, dS exchange to the dQ path, and halved global atomic reductions to the doubled dQ reduction width.", "claim_id": "kernel-flash-attention-4-current-c005", "claim_type": "mechanism", "covers": ["u013", "u014"], "evidence": [{"locator": "section 4, Two-CTA MMA for Backward Pass", "observation": "The paper gives the five-GEMM shape, operand-B sharing scope, half-dS DSMEM exchange for dQ, doubled N reduction tile, and global-atomic effect exactly as separated on the page.", "revision": "FA4 paper v1", "url": "https://arxiv.org/html/2603.05451v1"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Exact-token details are from a different branch, TMA is mandatory, or the FP8 benchmark proves supported FP8 execution.", "check": "Confirm clone HEAD, inspect conditional copy construction and dependency manifest, then follow the FP8 benchmark exception path.", "outcome": "HEAD equals a369df7, fallback branches are explicit, the exact dependency is pinned, and FP8 failure is expected rather than treated as success."}, "claim": "The pinned implementation notes accurately name the SM100 forward/backward helpers, TMEM allocation and offsets, conditional TMA fallbacks, SM90/SM100/SM120 package modules, nvidia-cutlass-dsl 4.6.0.dev0 dependency, FP16/BF16 directions, and not-yet-implemented FP8 bring-up state.", "claim_id": "kernel-flash-attention-4-current-c006", "claim_type": "code", "covers": ["u015", "u016", "u017", "u018", "u019", "u020", "u021"], "evidence": [{"locator": "_setup_tiled_mma, TmemAllocator setup, copy-atom construction, and architecture dispatch inputs", "observation": "The source uses make_trivial_tiled_mma, explicit TMEM allocation/offsets, and conditional Q/KV TMA or fallback paths as stated.", "path": "/tmp/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_fwd_sm100.py", "revision": "Dao-AILab commit a369df707e1980fb328abcc1733e3457ec10155f"}, {"locator": "_setup_tiled_mma and two-CTA dS/dQ pipelines", "observation": "The source constructs the five operations and coordinated two-CTA exchange/reduction flow without rank-partitioned dK/dV.", "path": "/tmp/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_bwd_sm100.py", "revision": "same commit"}, {"locator": "README.md, pyproject.toml, flash_fwd.py, flash_bwd.py, architecture modules, and benchmark_flash_attention_fp8.py", "observation": "The package identifies Hopper/Blackwell, includes SM90/SM100/SM120 modules, pins 4.6.0.dev0, exposes FP16/BF16 forward/backward, and says the FP8 call will raise until implemented.", "path": "/tmp/kernelwiki-flash-attention-pinned/flash_attn/cute/", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The two maxima can be merged into one exact seqlen=8192/head-dimension-128 row and 29% is necessarily non-MMA time.", "check": "Search each first-party source for the complete condition tuple and an attributed time decomposition.", "outcome": "Neither supplies the tuple or decomposition, while their textual peak values differ; the current page keeps them separate and explicitly unreproduced."}, "claim": "The performance section preserves the paper's up-to 1613-TFLOPS/s/71% result and the blog's distinct up-to 1605-TFLOPS/s/71% result with their source-specific scope, refuses the unsupported 8192-by-128 row, and makes no 29% time-breakdown inference.", "claim_id": "kernel-flash-attention-4-current-c007", "claim_type": "performance", "covers": ["u022", "u023"], "evidence": [{"locator": "abstract and section 5 benchmark setup/results", "observation": "Paper v1 reports up to 1613 TFLOPS/s at 71% and up-to 1.3x/2.7x comparisons over a suite spanning sequence lengths 1K-32K and several head dimensions.", "revision": "FA4 paper v1", "url": "https://arxiv.org/html/2603.05451v1"}, {"locator": "performance summary", "observation": "The blog separately states up to 1605 TFLOPS/s at 71% and the up-to comparison factors. It does not establish the deleted full condition tuple or a complement-of-utilization time breakdown.", "revision": "Tri Dao blog 2026-03-05", "url": "https://tridao.me/blog/2026/flash4/"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Sequence length 1024 and head dimension 128 are architecture constants, and compile speed predicts runtime speed.", "check": "Compare benchmark axes and separate compilation/runtime sections, then inspect configuration-dependent source dispatch.", "outcome": "The paper evaluates multiple points and metrics while the source has condition-dependent paths, rejecting fixed universal thresholds and metric substitution."}, "claim": "The practical boundaries correctly make use conditional on exact revision support and target measurement, reject universal sequence-length/head-dimension thresholds, separate compile from runtime speed, and list the missing controls needed before reproducing source-reported performance.", "claim_id": "kernel-flash-attention-4-current-c008", "claim_type": "performance", "covers": ["u024", "u025", "u026", "u027"], "evidence": [{"locator": "sections 5.1-5.3 benchmark methodology and results", "observation": "The source separates compile timing from runtime evaluation and varies direction, causal mode, sequence length, and head dimensions under a defined B200 setup; it states no universal 1024/128 choice rule.", "revision": "FA4 paper v1", "url": "https://arxiv.org/html/2603.05451v1"}, {"locator": "interface validation and architecture/configuration dispatch", "observation": "Support depends on architecture, dtype, head dimensions, masking/features, and code revision, consistent with the page's conditional checklist.", "revision": "Dao-AILab commit a369df707e1980fb328abcc1733e3457ec10155f", "url": "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/interface.py"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The local full file is actually FA4 despite its repository label, the extracted blocks drifted from the summary, or get_page ignores artifact relationships.", "check": "Compare owners/paths, verify every manifest hash, run the extraction idempotence check, and contrast a real versus false page id.", "outcome": "The provenance proves the adjacent identity, all bytes agree, the extractor is idempotent, and only the real id resolves the attached bundle."}, "claim": "The five pinned links resolve, local provenance accurately labels the full file as an adjacent CUTLASS reference and the variant as derived, all recorded hashes match, and the page query returns the declared local references while rejecting a false id.", "claim_id": "kernel-flash-attention-4-current-c009", "claim_type": "code", "covers": ["u028", "u029", "u030", "u031", "u032", "u033", "u034", "u035"], "evidence": [{"command": "git -C /tmp/kernelwiki-flash-attention-pinned rev-parse HEAD; sha256sum FA4 blog extracted files and kernel full/variant files; conda run -n base python scripts/extract_blog_code.py --check flash-attention-4; conda run -n base python scripts/get_page.py kernel-flash-attention-4 --include-code", "environment": "KernelWiki worktree; exact upstream clone; conda base Python 3.13.14", "negative_control": "kernel-not-flash-attention-4 exits 1, and the CUTLASS provenance repository/path visibly differs from Dao-AILab FA4.", "reference": "page links plus blog and kernel PROVENANCE.yaml/MANIFEST.yaml files", "result": "Clone HEAD is a369df707e1980fb328abcc1733e3457ec10155f; hashes are 89c085df..., 5747bb5e..., 2fa25bd8..., and 15df5285... and match manifests; extraction check and page query succeed with 2031 output lines."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["inspect FA4 paper v1 and Tri Dao's first-party blog", "inspect Dao-AILab/flash-attention commit a369df707e1980fb328abcc1733e3457ec10155f", "sample the published rounded exp2 polynomial with conda base Python", "verify artifact hashes and blog extraction idempotence", "conda run -n base python scripts/get_page.py kernel-flash-attention-4 --include-code", "conda run -n base python scripts/validate.py"], "id": "kernel-flash-attention-4", "page_verdict": "verified", "path": "wiki/kernels/flash-attention-4.md", "residual_risk": "No CUDA toolkit or NVIDIA GPU is installed, so no FA4 kernel was compiled or executed and neither B200 peak was reproduced. Source-level conclusions are tied to paper v1 and commit a369df7; blog/paper numbers remain explicitly source-reported, and the scalar polynomial test does not validate the production kernel's selective emulation, clamping, accuracy, or performance.", "scope": "All 35 current units for body hash eb0beda6d21999eb5a9af51fbb7869f46f78dcb0a29c7d5df1fbde5a0b49b875 were checked against FA4 paper v1, Tri Dao's first-party blog, exact Dao-AILab commit a369df7, generated/local provenance and hashes, a deterministic scalar formula test, repository query behavior, and structural validation."} +{"claims": [{"challenge": {"alternative": "The code is a generic paged MLA path and the named dimensions/top-k are only benchmark examples.", "check": "Inspect constructor defaults plus hard assertions and the page-table guard in the pinned merge.", "outcome": "Several constraints are enforced in code, and page tables are explicitly rejected rather than merely unbenchmarked."}, "claim": "PR 2441 merge f219c89c adds the SM100 CuTe DSL MLA 64/512 top-k forward path for packed MQA with 128 query heads per KV head; top-k uses cp.async gather, defaults to 2048, requires a multiple of 256, and rejects page tables.", "claim_id": "kernel-flash-attention-sm100-mla-topk-current-c001", "claim_type": "compatibility", "covers": ["u001", "u002"], "evidence": [{"locator": "FlashAttentionMLAForwardSm100 constructor and __call__ assertions", "observation": "The source defaults hdim/hdimv to 64/512 and topk to 2048, requires pack_gqa, qhead_per_kvhead=128 and cp.async gather in top-k mode, checks topk_length modulo tile_n*2=256, and rejects mPageTable.", "path": "artifacts/prs/flash-attention/PR-2441/key-files/flash_attn/cute/flash_fwd_mla_sm100.py", "revision": "PR 2441 merge f219c89c886c6ccbf9d3dbd9fe41b11ac64e9df8; SHA-256 f6512b25d901463cf62433b745e87ead30443f295ce313c03e363c886ece296e"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Top-k is only a masking flag inside the softmax loop, and tile scheduling is inseparable from the gather implementation.", "check": "Trace module ownership and dataflow from mIndexTopk to K/V pointers and separately from scheduler construction to work tiles.", "outcome": "The gather manager owns indexed loads/validity, scheduler objects own work assignment, and the MLA module composes both, establishing separate evidence paths."}, "claim": "The pinned merge separates indexed top-k K/V gathering, tile scheduling, and their integration; the gather uses caller indices, cp.async copies, and an optional bounds bitmask, while the integrating kernel supplies two-CTA tcgen05/TMEM/softmax/output flow.", "claim_id": "kernel-flash-attention-sm100-mla-topk-current-c002", "claim_type": "code", "covers": ["u003", "u004", "u005", "u006", "u007"], "evidence": [{"locator": "CpasyncGatherKVManager create, load_index_topk, compute_bitmask, compute_X_ptr, and load_X", "observation": "The dedicated manager reads indices, checks bounds when enabled, forms indexed global pointers, and issues cp.async K/V copies.", "path": "artifacts/prs/flash-attention/PR-2441/key-files/flash_attn/cute/topk_gather_kv.py", "revision": "PR 2441 merge f219c89c; SHA-256 58b10c0f75f789b80a163e976c4addd643665fc3b1498249da6e962fec673afb"}, {"locator": "imports, scheduler selection, two-CTA/MMA/TMEM setup, and role dispatch", "observation": "The kernel imports the gather manager and scheduler types independently and integrates them with cta_group TWO, explicit TMEM regions, softmax and epilogue roles.", "path": "artifacts/prs/flash-attention/PR-2441/key-files/flash_attn/cute/flash_fwd_mla_sm100.py", "revision": "same merge and pinned hash"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The 0.31 and 0.33 values can be rounded into one 0.3-ms B200 BF16 result, or the vanilla row is a throughput winner that contradicts sparse benefit.", "check": "Compare exact row variants/columns and distinguish latency from reported TFLOPS and unreported environment fields.", "outcome": "The two sparse rows differ by bounds handling and must remain distinct; lower work can yield lower latency with lower TFLOPS, and no absent environment field is inferred."}, "claim": "The performance table exactly transcribes the PR author's initial top-k=2048 decode rows\u20140.31 ms/955.47 TFLOPS without bitmask, 0.33 ms/898.08 with bitmask, and 1.98 ms/1180.70 vanilla\u2014for batch 512, seqlen_q 1, seqlen_k 16384, 128 heads and the 64/512 shape, while explicitly recording the missing environment and non-reproduction.", "claim_id": "kernel-flash-attention-sm100-mla-topk-current-c003", "claim_type": "performance", "covers": ["u008", "u009", "u010", "u011", "u012"], "evidence": [{"locator": "PR body, initial saturating decode performance block and introductory sentence", "observation": "The page's three rows, dimensions, top-k length, and no-bitmask bounds qualification exactly match the author text. That text does not name GPU model, dtype, detailed software/timing setup, sample count, or variance, matching the page caveat.", "revision": "GitHub PR record observed 2026-08-08; merge f219c89c886c6ccbf9d3dbd9fe41b11ac64e9df8", "url": "https://api.github.com/repos/Dao-AILab/flash-attention/pulls/2441"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Top-k length alone predicts latency and correctness, making bounds, page mode, and memory-path validation unnecessary.", "check": "Compare hard feature guards, bitmask predicates, arbitrary indexed pointers, and the PR's differing bitmask/no-bitmask rows.", "outcome": "Behavior and even reported latency vary with bounds handling, while an unsupported page mode is rejected; top-k length is insufficient."}, "claim": "The transfer checklist follows enforced merge constraints: preserve exact MQA/dimension/top-k/index contracts, require in-bounds indices before disabling the bitmask, reject inferred page-table support, and measure indexed memory behavior separately from reduced arithmetic and full-workload correctness/latency.", "claim_id": "kernel-flash-attention-sm100-mla-topk-current-c004", "claim_type": "performance", "covers": ["u013", "u014", "u015", "u016"], "evidence": [{"locator": "top-k constructor assertions, page-table guard, and gather/bitmask pipeline selection", "observation": "The source enforces the listed compatibility and bounds distinctions and routes sparse loads through a dedicated indexed path.", "path": "artifacts/prs/flash-attention/PR-2441/key-files/flash_attn/cute/flash_fwd_mla_sm100.py", "revision": "PR 2441 merge f219c89c"}, {"locator": "benchmark effective-sequence FLOP accounting and gather-index setup", "observation": "Arithmetic uses the effective gather length, while K/V access uses generated indices and separate bandwidth accounting, supporting independent measurement rather than a guaranteed proportional outcome.", "path": "artifacts/prs/flash-attention/PR-2441/diff.patch", "revision": "same merge; SHA-256 4525d7f9af76740692474ffc0266fe9cf72f894ce753fd9d18b70771c96b6810"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The page finds stale text but not the captured file, or the bundle belongs to the unrelated FA3 PR 1236.", "check": "Resolve both paths, compare source_pr_id/upstream SHA, and test a deliberately absent filename.", "outcome": "The manifest is explicitly PR 2441 at f219c89c, hash-matches its files, and rejects the absent token; PR 1236 is no longer a source relationship."}, "claim": "The local source record and byte-verified PR-2441 provenance resolve the exact integration and gather files, and the displayed lookup succeeds while a false filename fails.", "claim_id": "kernel-flash-attention-sm100-mla-topk-current-c005", "claim_type": "code", "covers": ["u017", "u018"], "evidence": [{"command": "conda run -n base python ; sha256sum artifacts/prs/flash-attention/PR-2441/diff.patch and key files", "environment": "KernelWiki worktree; conda base Python 3.13.14", "negative_control": "Replacing topk_gather_kv.py with nonexistent_topk.py raises AssertionError.", "reference": "sources/prs/flash-attention/PR-2441.md and artifacts/prs/flash-attention/PR-2441/PROVENANCE.yaml", "result": "The displayed assertions pass; provenance records merge f219c89c, flash_fwd_mla_sm100.py and topk_gather_kv.py with hashes that match the local files."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["inspect PR 2441 GitHub API record", "inspect byte-verified PR-2441 diff and key files", "execute the page lookup and a failing filename negative control with conda base Python", "conda run -n base python scripts/validate.py"], "id": "kernel-flash-attention-sm100-mla-topk", "page_verdict": "verified", "path": "wiki/kernels/flash-attention-sm100-mla-topk.md", "residual_risk": "No NVIDIA GPU or CUDA toolkit is installed, so the kernel, numerical results, and author-reported latency/TFLOPS rows were not reproduced. Compatibility and mechanism statements are source-level conclusions tied to merge f219c89c; the performance rows remain deliberately outside structured metadata because the PR omits essential environment details.", "scope": "All 18 current units for body hash 73e3ffa9038dafda7f4083be34a1dabae23531849cb738ba46e7914bfcff9857 were checked against the exact PR record, byte-verified merge artifacts, source constraints, a discriminating lookup test, and repository validation."} +{"claims": [{"challenge": {"alternative": "The package name makes dense prefill an MLA operator despite the table label.", "check": "Compare the support-matrix mode column, dimension footnote, public API section, and test names.", "outcome": "All four independently call the dense path MHA and reserve MQA dimensions for MLA calculation, so the page's distinction is deliberate and source-exact."}, "claim": "At commit 71c7379, FlashMLA powers DeepSeek-V3/V3.2-Exp and packages MLA-mode decode/sparse-prefill plus a distinct dense MHA prefill operator; the repository distinguishes MQA 576/512 from MHA 192/128 or 128/128 modes.", "claim_id": "kernel-flashmla-current-c001", "claim_type": "compatibility", "covers": ["u001"], "evidence": [{"locator": "Introduction, support matrix, and MLA Mode footnote", "observation": "The first-party snapshot names both model families, separates dense prefill as MHA, and defines the exact MQA/MHA dimension pairs transcribed on the page.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "DeepSeek FlashMLA commit 71c737929f2567bd0a094ae140f8f60f390b1232; SHA-256 46c001c53800f47148918f4fc2969102492dd5164f710fc015db1622f6f9929a"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Table 1 already defines an implementation byte count, so storage precision can be inferred.", "check": "Inspect the table heading/units and whether a cache dtype or layer aggregation appears in the formula.", "outcome": "The cells are symbolic numbers of cached elements and include an explicit layer factor only at model-total scope; no byte width is specified, preventing an unqualified KB conversion."}, "claim": "DeepSeek-V2 specifies per-layer/token cache sizes of 2*n_h*d_h elements for MHA and d_c+d_h^R for MLA, approximately 4.5*d_h with d_c=4*d_h and d_h^R=d_h/2; byte totals require additional dtype/layer assumptions.", "claim_id": "kernel-flashmla-current-c002", "claim_type": "compatibility", "covers": ["u002", "u003", "u004", "u005"], "evidence": [{"locator": "Section 2.1.1, equations 8-12, hyperparameters, and Table 1", "observation": "The paper defines the low-rank cache, gives d_c=4d_h and d_h^R=d_h/2, and lists the exact element-count formulas. Because Table 1 counts elements rather than bytes, dtype and layer multiplication are necessary for whole-model byte totals.", "revision": "DeepSeek-V2 paper v5", "url": "https://arxiv.org/html/2405.04434v5"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "656 and page size 64 are compile-time invariants hidden below the Python interface.", "check": "Trace interface assertions, quantizer arithmetic, SM90 656 assertion scope, and tests with multiple block sizes.", "outcome": "The 656 assertion is inside the V32 sparse-FP8 specialization, while page_block_size is passed through metadata and sparse tests exercise values other than 64; the narrow scope survives both layers."}, "claim": "The 656-byte ABI is limited to V3-family FP8 sparse decode and consists of 512 E4M3 NoPE bytes, four FP32 per-128-value scales, and 64 BF16 RoPE values; dense decode is BF16, another sparse layout exists, and page size is an input rather than a fixed 64.", "claim_id": "kernel-flashmla-current-c003", "claim_type": "code", "covers": ["u006", "u007", "u008", "u009", "u010", "u011"], "evidence": [{"locator": "flash_mla_with_kvcache docstring, scheduler config, and dense/sparse dispatch", "observation": "The source gives the exact V3 FP8+sparse byte fields, derives page_block_size from k_cache.shape[1], sends FP8 only to sparse dispatch, and sends dense calls to the BF16 path.", "path": "/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py", "revision": "DeepSeek FlashMLA commit 71c7379"}, {"locator": "FP8KVCacheLayout.get_meta and quantize_k_cache", "observation": "V32 uses (576,512,64,128,4) and 656 bytes, while MODEL1 uses (512,448,64,64,7) and a different E8M0 scale packing.", "path": "/tmp/kernelwiki-flashmla-pinned/tests/quant.py", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Global SM90/SM100 support means every operator supports both generations and FP8 is the arithmetic dtype throughout.", "check": "Read each support-matrix row and the mode-specific dtype paragraphs rather than the global requirement line.", "outcome": "Rows differ by architecture and cache format, and the source explicitly separates FP8 storage from BF16 compute, refuting a uniform interpretation."}, "claim": "The support table and requirements exactly distinguish dense decode SM90/BF16, sparse decode SM90+SM100/FP8-KV-to-BF16-MMA, dense MHA prefill SM100/BF16, sparse MLA prefill SM90+SM100/BF16, with CUDA 12.8+, CUDA 12.9+ for SM100, and PyTorch 2.0+.", "claim_id": "kernel-flashmla-current-c004", "claim_type": "compatibility", "covers": ["u012", "u013", "u014", "u015", "u016"], "evidence": [{"locator": "Requirements, support matrix, FP8 decode note, sparse-prefill equivalence, and Dense MHA Prefill", "observation": "Every architecture/mode/format and version floor on the page is an exact transcription; the FP8 decode paragraph explicitly says dequantize/MMA/output bfloat16.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "DeepSeek FlashMLA commit 71c7379"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The extension internally runs a selector before using optional indices, or sparse prefill shares decode paging semantics.", "check": "Trace the Python dispatch arguments and compare the two documented equivalence calculations.", "outcome": "Sparse extension calls receive indices directly, no selector output is returned or created, and prefill gathers flat kv[indices] without a page table, establishing two distinct caller contracts."}, "claim": "Sparse decode consumes page-encoded [batch,s_q,topk] indices with -1 invalid and no block table; sparse prefill consumes unbatched BF16 tensors and [s_q,h_kv,topk] indices, documents h_kv=1, accepts negative/out-of-range invalid indices, returns out/max_logits/lse, and neither attention interface produces top-k indices.", "claim_id": "kernel-flashmla-current-c005", "claim_type": "code", "covers": ["u017", "u018", "u019", "u020"], "evidence": [{"locator": "flash_mla_with_kvcache indices documentation/dispatch and flash_mla_sparse_fwd signature", "observation": "The exact tensor ranks, dtypes, physical page+offset formula, invalid values, absent block-table rule, and return tuple are encoded in the public interface; indices are caller inputs to both extension calls.", "path": "/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py", "revision": "DeepSeek FlashMLA commit 71c7379"}, {"locator": "Sparse Attention and Sparse MLA Prefill usage", "observation": "The README independently states the index encoding, no-batch limitation, h_kv=1 equivalence, invalid-index rules, and return values.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "All TFLOPS rows can be ranked directly, and absent details can be inferred from nearby tests or GPU peaks.", "check": "Compare headings, memory/compute-bound qualifications, phases, attribution, dtype contracts, and fields actually stated in the source.", "outcome": "The observations cover decode versus prefill, MLA versus MHA, storage versus MMA precision, and different GPUs/software; the source omits complete shapes/timing statistics, so the page correctly keeps them out of structured records and warns against ranking."}, "claim": "The performance table precisely preserves the pinned README's author-reported maxima, operator identities, stated environments, and precision scopes while noting missing benchmark cells, non-reproduction, and the invalidity of cross-regime comparisons.", "claim_id": "kernel-flashmla-current-c006", "claim_type": "performance", "covers": ["u021", "u022", "u023", "u024", "u025", "u026", "u027", "u028"], "evidence": [{"locator": "three Performance subsections and support matrix", "observation": "The page exactly transcribes dense-decode 3000 GB/s/660 TFLOPS H800 CUDA 12.8, sparse-decode 410 H800 and up to 350 B200, dense MHA prefill 1460/1000 B200 attributed to NVIDIA, and sparse-prefill 640 H800 CUDA 12.8/1450 B200 CUDA 12.9. Its precision labels follow the API/support text.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "DeepSeek FlashMLA commit 71c7379"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Matching MLA names make the CUTLASS/FlashInfer files part of DeepSeek FlashMLA, or the derived helper still implies a portable kernel ABI.", "check": "Compare repository/commit provenance and compile the helper independently of CUDA.", "outcome": "Origins are disjoint and explicit; g++ C++17 syntax checking passes because the helper claims only arithmetic contracts and contains no GPU-kernel body."}, "claim": "Pinned SM100 DeepSeek sources contain TMA, tcgen05, and TMEM paths, while local full artifacts are accurately labeled as separate CUTLASS 9baa06dd and FlashInfer 9a05c92a implementations and the derived helper is limited to verified layout/index arithmetic.", "claim_id": "kernel-flashmla-current-c007", "claim_type": "code", "covers": ["u029", "u030", "u031", "u032", "u033"], "evidence": [{"locator": "decode/head64/kernel.cuh, prefill/sparse phase1 files, and prefill/dense kernel files", "observation": "The pinned code includes TMEM allocators, tcgen05 synchronization/MMA helpers, and TMA-specialized paths across the named operator families.", "path": "/tmp/kernelwiki-flashmla-pinned/csrc/sm100", "revision": "DeepSeek FlashMLA commit 71c7379"}, {"locator": "bundle_scope and per-file provenance", "observation": "The manifest explicitly marks adjacency and records CUTLASS 9baa06dd for 77_blackwell_mla_fwd.cu and FlashInfer 9a05c92a for the header; file hashes match.", "path": "artifacts/kernels/flashmla/full/PROVENANCE.yaml", "revision": "SHA-256 844d2083c1fc3098981334111fe48e41316a9c9b2b4abb448070379b8e0244f5"}, {"locator": "entire derived helper", "observation": "The header says non-upstream and the file contains only compile-time 656-byte arithmetic plus physical-page index encode/decode, not a fabricated attention kernel.", "path": "artifacts/kernels/flashmla/variants/01-mla-layout-and-index-contract.cu", "revision": "SHA-256 aa886373129daedc09b9dae7a23960b021256647e1002399db0dca82053c6f96"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The library's compatibility wrapper makes exact dimensions/cache/index rules and numerical reference checks unnecessary.", "check": "Inspect hard assertions and dispatch conditions in flash_mla_interface.py and compare the separate tests/reference implementations.", "outcome": "Mode mismatches are asserted or route to different kernels, indices determine addresses, and performance maxima do not establish correctness; the checklist targets actual failure boundaries."}, "claim": "The selection checklist follows the verified compatibility boundaries: exact operator/dimensions/version/dtype/index matching, narrow 656-byte scope, external index generation, per-regime benchmarking, and reference output/LSE validation.", "claim_id": "kernel-flashmla-current-c008", "claim_type": "compatibility", "covers": ["u034", "u035", "u036", "u037", "u038"], "evidence": [{"locator": "support matrix, usage contracts, tests, and performance sections", "observation": "The source exposes each listed compatibility field, requires caller indices, supplies reference/test paths, and reports performance only in distinct qualified regimes.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "DeepSeek FlashMLA commit 71c7379"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The CUTLASS PRs can substantiate DeepSeek FlashMLA performance/mechanisms directly, or the paper specifies the 656-byte ABI.", "check": "Compare repositories, revision identities, claimed layers, and page wording around each citation.", "outcome": "The page confines DeepSeek facts to its repository/paper and calls CUTLASS adjacent; it does not transfer CUTLASS numbers or conflate the model formula with the implementation ABI."}, "claim": "The four source links resolve to the audited DeepSeek commit, DeepSeek-V2 v5 paper, and pinned CUTLASS forward/backward MLA-shape PRs, each used only for its stated subject.", "claim_id": "kernel-flashmla-current-c009", "claim_type": "other", "covers": ["u039", "u040", "u041", "u042"], "evidence": [{"locator": "repository tree and README", "observation": "Exact first-party FlashMLA snapshot.", "revision": "commit 71c7379", "url": "https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232"}, {"locator": "Section 2.1.1", "observation": "First-party MLA mechanism and cache formula.", "revision": "DeepSeek-V2 v5", "url": "https://arxiv.org/html/2405.04434v5"}, {"locator": "PR title and changed paths", "observation": "CUTLASS SM100 MLA-shape backward and reduction path.", "revision": "merge 0e026982", "url": "https://github.com/NVIDIA/cutlass/pull/2466"}, {"locator": "PR title and changed paths", "observation": "CUTLASS SM100 MLA-shape forward path.", "revision": "merge 9baa06dd", "url": "https://github.com/NVIDIA/cutlass/pull/2472"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The command prints only page Markdown or silently resolves a fuzzy unrelated ID, and the byte total depends on unavailable GPU behavior.", "check": "Assert corrected artifact tokens in real output, repeat with a near-name known to be absent, execute the arithmetic, and alter its expected result.", "outcome": "The exact page includes both artifact classes, the near-name fails, real arithmetic passes, and the altered result fails; the 656 total is a storage-layout identity rather than a GPU measurement."}, "claim": "The displayed conda-base get_page query resolves the corrected FlashMLA page and labeled artifacts, and the displayed mode-specific byte check evaluates to 656 without a GPU.", "claim_id": "kernel-flashmla-current-c010", "claim_type": "code", "covers": ["u043", "u044", "u045", "u046"], "evidence": [{"command": "conda run -n base python scripts/get_page.py kernel-flashmla --include-code; conda run -n base python -c ''", "environment": "KernelWiki worktree; conda base Python 3.13.14", "negative_control": "kernel-flashmla-does-not-exist exits 1 with 'Page not found'; changing expected bytes from 656 to 655 raises AssertionError.", "reference": "wiki/kernels/flashmla.md and artifacts/kernels/flashmla", "result": "The query exits 0 and contains the audited commit, 'adjacent implementations', both full files, and variants/01-mla-layout-and-index-contract.cu; the byte assertion also exits 0 with value 656."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["inspect DeepSeek FlashMLA commit 71c737929f2567bd0a094ae140f8f60f390b1232 README, public interfaces, tests, quantizer, and SM100 code", "inspect DeepSeek-V2 paper v5 formulas", "execute both extracted Python contract checks", "compile the derived C++17 contract helper with g++ -fsyntax-only", "execute get_page with the real ID and a failing near-name", "conda run -n base python scripts/validate.py"], "id": "kernel-flashmla", "page_verdict": "verified", "path": "wiki/kernels/flashmla.md", "residual_risk": "No NVIDIA GPU or CUDA toolkit is installed, so attention kernels, numerical output, bandwidth, and throughput were not reproduced. The performance table is deliberately limited to qualified author reports and performance_claims is empty; implementation conclusions are source-level and tied to commit 71c7379.", "scope": "All 46 current units for body hash 3c4ba2ede0a25c46871912580475e2e71786d53c38b8dea8330b7b017e42d0ae were checked against version-pinned primary sources, exact local provenance, executable contract checks, a compiled derived helper, and a discriminating repository query."} +{"claims": [{"challenge": {"alternative": "The 128x128 grouping is universal for every activation/weight/gradient and model phase.", "check": "Compare forward quantization prose with backward exceptions and sensitivity ablations.", "outcome": "The paper preserves 1x128 forward activations but changes/restricts other cases, matching the page's logical-format and non-universality boundary."}, "claim": "DeepSeek-V3 forward FP8 training uses 1x128 activation tiles and 128x128 weight blocks to adapt scales to local outliers, while other phases include exceptions such as 128x1 activation grouping.", "claim_id": "kernel-fp8-block-scale-gemm-current-c001", "claim_type": "compatibility", "covers": ["u001", "u005"], "evidence": [{"locator": "Sections 3.3.2 and Appendix B.2", "observation": "The paper gives the exact forward groupings and outlier motivation, then explicitly distinguishes backward activation grouping and other higher-precision policies.", "revision": "DeepSeek-V3 Technical Report v2", "url": "https://arxiv.org/html/2412.19437v2"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The repository's current mixed FP8/FP4 scope makes the pinned FP8 paths obsolete or architecture-neutral.", "check": "Inspect exact SM90/SM100 FP8 source inclusions and API dispatch at the pinned commit.", "outcome": "Distinct 1D1D implementations remain selected by architecture and encode different scale paths, so the page's pinned split is current for that revision."}, "claim": "DeepGEMM commit 891d57b4 implements related FP8 GEMMs on SM90 and SM100 with architecture-specific scale handling.", "claim_id": "kernel-fp8-block-scale-gemm-current-c002", "claim_type": "compatibility", "covers": ["u002"], "evidence": [{"locator": "Introduction, requirements, and scale-factor notice", "observation": "The project supports SM90/SM100 FP8 GEMMs and explicitly requires FP32 scale factors on SM90 versus packed UE8M0 on SM100.", "path": "/tmp/kernelwiki-deepgemm-pinned/README.md", "revision": "commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The example accidentally describes DeepGEMM's physical packed/TMA layout rather than only logical scale groups.", "check": "Compare the function's returned ranks with the repository's transformed layouts and its own explicit prose disclaimer.", "outcome": "The function only divides logical dimensions and the page says transformed layouts are excluded; the negative control discriminates the stated 128 grouping."}, "claim": "For A[M,K] times B[N,K]^T under the DeepSeek-V3 forward recipe, logical activation-scale shape is [M,K/128] and logical weight-scale shape is [N/128,K/128]; the displayed derived check enforces divisibility and computes the 4096 example correctly without claiming an implementation layout.", "claim_id": "kernel-fp8-block-scale-gemm-current-c003", "claim_type": "code", "covers": ["u003", "u004"], "evidence": [{"command": "conda run -n base python ", "environment": "KernelWiki worktree; conda base Python 3.13.14", "negative_control": "Changing N to 4095 triggers the divisibility assertion.", "reference": "wiki/kernels/fp8-block-scale-gemm.md and DeepSeek-V3 report Section 3.3.2", "result": "The 4096-cubed example returns ((4096,32),(32,32)) and exits 0."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "Scale application is fused into WGMMA or occurs only once in the epilogue, and the paper's interval counts instructions.", "check": "Trace one k_block iteration and compare it with the paper's explicit parenthetical equivalence.", "outcome": "Scales are loaded per K block and combined after WGMMA completion; the paper says 128 elements equals four WGMMAs, ruling out both alternatives."}, "claim": "Pinned SM90 1D1D requires FP32 factors and BLOCK_K=128, performs WGMMA partial accumulation, then scale-multiplies/adds into FP32 final_accum on CUDA cores; the paper equates the 128-element interval to four WGMMAs and describes the limiting additions as 14-bit precision.", "claim_id": "kernel-fp8-block-scale-gemm-current-c004", "claim_type": "code", "covers": ["u006", "u007"], "evidence": [{"locator": "BLOCK_K assertion and math warp-group loop", "observation": "The implementation declares float accum/final_accum, issues BLOCK_K/WGMMA::K operations, and then executes final_accum += scale_a*scale_b*accum.", "path": "/tmp/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh", "revision": "DeepGEMM commit 891d57b4"}, {"locator": "Hopper Accumulation", "observation": "The primary-paper capture records four WGMMAs per 128 elements and the 14-bit precision characterization.", "path": "sources/docs/deepseek-v3-fp8.md", "revision": "summary of DeepSeek-V3 report v2"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Packed UE8M0 is merely an external storage optimization and the MMA still receives ordinary FP8 E4M3 scale values after unpacking.", "check": "Follow scale type from API requirement through TMEM copy to the instruction descriptor template.", "outcome": "The descriptor's scale type is cutlass::float_ue8m0_t and the hardware block-scaled operation consumes its descriptors; E4M3 is the matrix-element type, not the scale type."}, "claim": "Pinned SM100 uses packed four-per-int32 UE8M0 factors, TMA/UTCCP scale movement into TMEM, a block-scaled UMMA descriptor, and TMEM accumulation rather than the SM90 CUDA-core promotion loop; selected SM100 recipes include K granularities 32 and 128.", "claim_id": "kernel-fp8-block-scale-gemm-current-c005", "claim_type": "code", "covers": ["u008", "u009", "u010"], "evidence": [{"locator": "scale-factor notice", "observation": "SM100 requires packed UE8M0 with four values per torch.int.", "path": "/tmp/kernelwiki-deepgemm-pinned/README.md", "revision": "DeepGEMM commit 891d57b4"}, {"locator": "TMA scale loads, UTCCP copies, make_instr_desc_block_scaled, and MMA loop", "observation": "The exact dataflow moves factors to TMEM and selects float_ue8m0_t block-scaled UMMA; no SM90-style final_accum loop is present.", "path": "/tmp/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh", "revision": "same commit"}, {"locator": "gran_k_list for architecture 10", "observation": "SM100 generator coverage includes gran_k 32 and 128, whereas SM90 uses 128.", "path": "/tmp/kernelwiki-deepgemm-pinned/tests/generators.py", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "A shape and utilization can be recovered from linked optimization changes or theoretical peak and safely structured.", "check": "Require one primary benchmark cell containing workload, environment, timing, samples, and metric conventions.", "outcome": "No such cell is attached to the README maximum, so the qualified prose and empty structured list are the strongest supported form."}, "claim": "The only retained numeric performance observation is DeepGEMM's source-reported up-to 1550 TFLOPS on H800, explicitly missing shape/utilization/timing statistics and unreproduced; CUTLASS availability is not presented as a matched comparison.", "claim_id": "kernel-fp8-block-scale-gemm-current-c006", "claim_type": "performance", "covers": ["u011", "u012"], "evidence": [{"locator": "2025-04-18 news item", "observation": "The page transcribes exactly the stated GPU and maximum while its caveat enumerates the fields absent from that sentence.", "path": "/tmp/kernelwiki-deepgemm-pinned/README.md", "revision": "DeepGEMM commit 891d57b4"}, {"locator": "blockscaled GEMM entries", "observation": "CUTLASS availability is documented, but no matched CUTLASS/DeepGEMM comparison is present, agreeing with the page's refusal to rank them.", "path": "sources/docs/cutlass-changelog-sm100.md", "revision": "release capture through 4.5.0"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Matching FP8 dtype and peak GEMM throughput is enough to select the kernel.", "check": "Enumerate API assertions and all work outside the GEMM call that affects compatibility, accuracy, and end-to-end time.", "outcome": "Scale dtype/layout and recipe dispatch are hard requirements; casting/layout transforms and distribution-sensitive quantization error remain outside a bare throughput number."}, "claim": "The selection checklist correctly requires exact scale grouping/representation/layout/architecture matching, includes preprocessing and epilogue costs, and calls for numerical comparison on the real data distribution.", "claim_id": "kernel-fp8-block-scale-gemm-current-c007", "claim_type": "compatibility", "covers": ["u013", "u014", "u015", "u016", "u017"], "evidence": [{"locator": "interface notices and user-managed casting/layout warning", "observation": "The README distinguishes scale ABIs and says transposition/FP8 casting are separate user responsibilities whose simple utilities may be slower.", "path": "/tmp/kernelwiki-deepgemm-pinned/README.md", "revision": "DeepGEMM commit 891d57b4"}, {"locator": "Fine-Grained Quantization and Hopper Accumulation", "observation": "The model source motivates group-specific numerical validation and documents scale/precision overheads and exceptions.", "path": "sources/docs/deepseek-v3-fp8.md", "revision": "DeepSeek-V3 report v2 capture"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The paper can define exact current PTX syntax or the PTX manual can establish DeepGEMM's implementation choices.", "check": "Map each page claim to its cited layer and inspect whether any behavior is transferred across sources.", "outcome": "Model format comes from the paper, source dataflow from the pinned repository, and instruction caution from PTX; the page avoids cross-layer inference."}, "claim": "The source list points to the exact DeepSeek-V3 v2 FP8 report, audited DeepGEMM commit 891d57b4, and official NVIDIA PTX ISA, with each used only at its proper model, implementation, or instruction-contract scope.", "claim_id": "kernel-fp8-block-scale-gemm-current-c008", "claim_type": "other", "covers": ["u018", "u019", "u020"], "evidence": [{"locator": "FP8 Training", "observation": "Model-level grouping/numerical source.", "revision": "DeepSeek-V3 report v2", "url": "https://arxiv.org/html/2412.19437v2"}, {"locator": "repository tree", "observation": "Exact implementation snapshot.", "revision": "commit 891d57b4", "url": "https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f"}, {"locator": "tcgen05.mma and alternate floating-point formats", "observation": "Authoritative version-sensitive instruction and UE8M0 contract.", "revision": "official PTX ISA, consulted 2026-08-08", "url": "https://docs.nvidia.com/cuda/parallel-thread-execution/"}], "oracle": "web-primary", "verdict": "verified"}], "commands": ["inspect DeepGEMM commit 891d57b4 README, SM90/SM100 kernels, PTX wrappers, and test recipes", "inspect DeepSeek-V3 Technical Report v2", "execute the displayed logical-shape check and a failing divisibility negative control", "conda run -n base python scripts/validate.py"], "id": "kernel-fp8-block-scale-gemm", "page_verdict": "verified", "path": "wiki/kernels/fp8-block-scale-gemm.md", "residual_risk": "No NVIDIA GPU or CUDA toolkit is installed, so kernel correctness, numerical error, and the H800 maximum were not reproduced. PTX behavior is described at contract level and exact assembly remains toolkit/version-sensitive.", "scope": "All 20 current units for body hash 5c0369ba45f1bfcc220967e83d4765f7e8b56908a310a1bf0356e9bb2c996e70 were checked against the pinned implementation, primary paper, official PTX source, and a discriminating host check."} +{"claims": [{"challenge": {"alternative": "The single public entry point proves that the whole operation is one GPU launch or that Hopper is an equal contest target.", "check": "Separate organizer hardware scope and operation description from implementation internals.", "outcome": "The primary pages specify B200 and an operation contract only; neither supplies a launch decomposition or Hopper target, matching the page's explicit boundary."}, "claim": "MLSys 2026 FlashInfer Track A targets B200, names Fused MoE with FP8 support, and links the exact DeepSeek-style definition whose scope is routing plus two grouped GEMMs rather than a guaranteed launch count.", "claim_id": "kernel-fused-moe-current-c001", "claim_type": "compatibility", "covers": ["u001", "u056", "u057"], "evidence": [{"locator": "Challenge and Track A", "observation": "The organizer specifies NVIDIA Blackwell B200, calls Track A Fused MoE with FP8 support, and links the exact benchmark.", "revision": "fetched 2026-08-08", "url": "https://mlsys26.flashinfer.ai/"}, {"locator": "description and tags", "observation": "The definition says routing and two grouped GEMMs are included and is tagged deepseek-v3/deepseek-r1, FP8 E4M3FN, API trtllm_fp8_block_scale_moe, and EP=8.", "revision": "fetched 2026-08-08", "url": "https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The slug's e32 is the total expert count and EP=8 merely replicates those weights.", "check": "Compare routing_logits width, num_experts axis, num_local_experts axis, and local_expert_offset semantics.", "outcome": "Routing logits have width 256 while local weight tensors have leading dimension 32 and a global offset; therefore 32 is local, not total."}, "claim": "The fixed definition varies seq_len and uses 256 global experts, 32 local experts under EP=8, top-k 8, eight groups, four retained groups, H=7168, I=2048, W13 output 4096, and 128-element scale blocks; e32 denotes local experts.", "claim_id": "kernel-fused-moe-current-c002", "claim_type": "compatibility", "covers": ["u002", "u003", "u004", "u005", "u006", "u007", "u008", "u009", "u010", "u011", "u012", "u013"], "evidence": [{"locator": "definition name, Axes, and Reference Implementation constants", "observation": "The primary definition exposes seq_len variable; axes 256 total/32 local/H7168/I2048/W13=4096 and block counts 56/16/32; its exact name/reference fix topk8/ng8/kg4 and block 128.", "revision": "fetched 2026-08-08", "url": "https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The table accidentally applies the current API's MXFP8 32-element UE8M0 variant or global-expert weight layout to the contest trace.", "check": "Follow the trace's DeepSeekFp8 initializer and compare runner validation branches.", "outcome": "This trace uses FP8 E4M3 tensors with FP32 128-block scales and 32 local weight rows; the separate MXFP8 branch uses different scale dtype/layout and is not imported into the table."}, "claim": "The displayed FP8/FP32/BF16 tensor table, output, scale-storage boundary, and complete argument summary match the benchmark and pinned FlashInfer API.", "claim_id": "kernel-fused-moe-current-c003", "claim_type": "code", "covers": ["u014", "u015", "u016", "u017", "u018", "u019", "u020", "u021", "u022", "u023", "u024", "u025", "u028", "u059", "u060"], "evidence": [{"locator": "trtllm_fp8_block_scale_moe_ds_routing_trace inputs/outputs and initializer", "observation": "The source defines every tensor name, shape role, FP8/FP32 data construction, top-k/group scalars, local offset, routed factor, and BF16 output exactly as summarized.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}, {"locator": "trtllm_fp8_block_scale_moe signature and docstring", "observation": "The public API accepts the eight tensors and explicit expert/routing/intermediate/local parameters named on the page.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/fused_moe/core.py", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The optimized runner may internally allocate separate accumulators and emit tcgen05, so the page should present those as contract requirements.", "check": "Distinguish observable reference semantics from one possible optimized implementation.", "outcome": "The source-level contract constrains inputs/results but not the internal TMEM or instruction schedule; the page makes exactly that distinction."}, "claim": "The official reference uses sigmoid+bias grouped selection, unbiased normalized combine weights, local expert filtering, one logical W13 projection split for SwiGLU, W2, and weighted accumulation; it mandates neither dual TMEM allocations nor a tcgen05 sequence.", "claim_id": "kernel-fused-moe-current-c004", "claim_type": "code", "covers": ["u029", "u030", "u031", "u032", "u033", "u034", "u035", "u036"], "evidence": [{"locator": "_trtllm_fp8_block_scale_moe_ds_routing_reference and _fp8_moe_run_experts", "observation": "The reference implements the six listed stages, applies bias only to selection, derives weights from unbiased sigmoid values, filters the global/local interval, and uses a single concatenated W13 matmul followed by a split.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The variant repeats one vectorized implementation and therefore cannot discriminate selection bias from combine-weight semantics.", "check": "Inspect and run the explicit wrong-weight negative control with nonzero routing bias.", "outcome": "The control would fail if biased scores were reused for combination, while the correct implementation passes."}, "claim": "The derived standard-library reference implements the stated grouped selection and local W13/SwiGLU/W2 accumulation, passes its deterministic self-test, and rejects the biased-combine-weight alternative.", "claim_id": "kernel-fused-moe-current-c005", "claim_type": "code", "covers": ["u037"], "evidence": [{"command": "conda run -n base python artifacts/kernels/fused-moe/variants/01-routing-plus-fusion-skeleton.py", "environment": "KernelWiki worktree; conda base Python without third-party tensor libraries", "negative_control": "The self-test forms combine weights from biased selection scores and asserts they differ from the correct unbiased weights.", "reference": "official FlashInfer reference semantics at commit 7f614b8", "result": "Exited 0 and printed 'fused-moe derived reference: PASS'; checks exact selected IDs, per-token weight sum 2.5, output shape, and finite output."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "A former official table or common serving interpretation can restore 1262 TFLOPS and map seq_len endpoints to prefill/decode.", "check": "Require an exact archived source cell and inspect the immutable trace axis description.", "outcome": "No archived cell/revision is attached and the trace says number of tokens; the page correctly retains only the evaluation method, not a result or serving-phase inference."}, "claim": "Starter-kit commit 75ccd05 records the stated B200 clocks/container/software stack, MoE correctness gates, and arithmetic-mean speedup scoring, while the current sources do not establish the removed TFLOPS/launch table or prefill/decode relabeling.", "claim_id": "kernel-fused-moe-current-c006", "claim_type": "performance", "covers": ["u038", "u039", "u040", "u041", "u042", "u043", "u044", "u058"], "evidence": [{"locator": "Environment, Run Evaluation, FlashInfer Baselines, and Scoring", "observation": "The file gives B200 sm_100a, clocks 3996/1965, the exact image and tool versions, MoE tolerance command, baseline name, and mean per-workload speedup formula.", "path": "/tmp/kernelwiki-flashinfer-bench-starter-kit/EVALUATION.md", "revision": "flashinfer-bench-starter-kit 75ccd05cafceb0fd1f86be4cd0f2117249463c66"}, {"locator": "trace axes", "observation": "The variable axis is named seq_len and described as number of tokens, with no prefill/decode or request-batch label.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Load imbalance is always the primary bottleneck and one fixed tactic is sufficient.", "check": "Inspect the implementation's problem-dependent tactic enumeration and separate-stage selections.", "outcome": "The runtime explicitly varies selection by shape/token bucket and stage; the page retains imbalance as a risk but rejects a universal ranking."}, "claim": "Per-expert token counts create varying grouped-GEMM M dimensions, and pinned FlashInfer selects separate GEMM1/GEMM2 tactics over token buckets rather than establishing one universal bottleneck.", "claim_id": "kernel-fused-moe-current-c007", "claim_type": "performance", "covers": ["u045", "u047"], "evidence": [{"locator": "Grouped GEMMs (contiguous layout)", "observation": "The documentation says experts may process varying token counts and represents them as different M segments.", "path": "/tmp/kernelwiki-deepgemm-pinned/README.md", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f"}, {"locator": "MoERunner valid tactics, token bucketing, and choose_one calls", "observation": "The runtime enumerates shape-valid tactics and tunes GEMM1/GEMM2 separately over token buckets.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/fused_moe/core.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Combining experts into a grouped GEMM guarantees full tiles and full-GPU occupancy.", "check": "Apply the document's partial-tile and insufficient-grid alternatives to each variable-M group.", "outcome": "Grouping adds independent problems but does not eliminate partial boundary tiles or guarantee enough total tiles; 'can reduce' is the supported scope."}, "claim": "Small M or partially filled GEMM tiles can reduce utilization, as stated by NVIDIA's GEMM implementation guidance.", "claim_id": "kernel-fused-moe-current-c008", "claim_type": "performance", "covers": ["u046", "u062"], "evidence": [{"locator": "Threadblock-level GEMM", "observation": "NVIDIA explains that small M/N can leave threads outside the problem and that small M/N with large K may launch too few threadblocks to use all SMs.", "revision": "CUTLASS documentation fetched 2026-08-08", "url": "https://github.com/NVIDIA/cutlass/blob/main/media/docs/cpp/efficient_gemm.md"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Matching FP8 E4M3 element type is sufficient, and masked layout is inherent to any graph capture.", "check": "Compare the runner's independent ABI validations and the explicit conditional scope in DeepGEMM.", "outcome": "Many fields beyond element type are enforced, and masked layout is presented as one backend scenario, matching the page."}, "claim": "Compatibility requires the complete routing/quantization/layout contract, and CUDA-graph layouts are backend-specific; DeepGEMM documents masked layout for one CPU-unknown decode case rather than as a universal requirement.", "claim_id": "kernel-fused-moe-current-c009", "claim_type": "compatibility", "covers": ["u048", "u049", "u063"], "evidence": [{"locator": "TrtllmFp8BlockRunner validation", "observation": "The runner checks activation dtype, scale dtype/shape, local weight shapes, routing input mode, local expert interval, layout, and static options separately.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/fused_moe/runners.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}, {"locator": "Grouped GEMMs (masked layout)", "observation": "Masked grouped GEMM is scoped to decoding with CUDA graph enabled when the CPU does not know expert token counts.", "path": "/tmp/kernelwiki-deepgemm-pinned/README.md", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The CUDA programming guide's 128-byte TMA-swizzle example makes every tensor base and descriptor require 128 bytes.", "check": "Read the encoder's normative per-parameter requirements and distinguish a selected swizzle's shared/global layout guidance.", "outcome": "The requirements vary by object, base, interleave, dtype, and swizzle; the page accurately rejects the blanket rule."}, "claim": "CUDA 13.2.1 tensor-map alignment is configuration-specific: the object requires 64-byte alignment, ordinary globalAddress 16 bytes, and selected interleave/dtype cases 32 bytes, so 128 bytes is not universal.", "claim_id": "kernel-fused-moe-current-c010", "claim_type": "compatibility", "covers": ["u050", "u061"], "evidence": [{"locator": "cuTensorMapEncodeTiled parameter requirements", "observation": "The API states tensorMap address alignment 64 bytes, globalAddress 16 bytes normally, and 32-byte requirements for interleave-32B and selected packed data types.", "revision": "CUDA Driver API 13.2.1", "url": "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Because two files originate upstream, the directory can still be called the full reference implementation of Track A.", "check": "Compare each file's quantization/routing scope with the exact Track A signature.", "outcome": "The upstream files use MXFP4/FP4 adjacent contracts and the third is pseudocode; the corrected page/provenance disclose this without overstating them."}, "claim": "The local full bundle is accurately disclosed as adjacent MXFP4 vLLM, FP4 SGLang, and local pseudocode rather than the Track A implementation, with byte-pinned file scopes and corrected provenance.", "claim_id": "kernel-fused-moe-current-c011", "claim_type": "code", "covers": ["u051", "u052", "u053", "u054", "u055"], "evidence": [{"locator": "five diff headers and MXFP4 backend code", "observation": "The aggregate patch covers five vLLM files and implements MXFP4-weight expert compute.", "path": "artifacts/kernels/fused-moe/full/vllm-PR-23696-dual-gemm.patch", "revision": "vLLM merge 074854b24f6e0b1e237a004283e1f46d98c0d73c; SHA-256 033d30b15af4f2050189c2e76ba61959e1e5d4bd049dc901fd56f68890b891a0"}, {"locator": "module and registered function", "observation": "The byte-verified upstream file calls fp4_quantize and CuteDslMoEWrapper for the FP4 standard path.", "path": "artifacts/kernels/fused-moe/full/flashinfer_cutedsl.py", "revision": "SGLang c554dc5c64b661f2c53225b03a76359eaddc39e4; SHA-256 de6bd663382b5bf34dba077d90febc9cc80d9dda908d9947966a3cadbc21bf33"}, {"locator": "file header", "observation": "The header explicitly labels the incomplete file as local illustrative pseudocode, not upstream or timed code.", "path": "artifacts/kernels/fused-moe/full/moe-grouped-gemm-launch.cpp", "revision": "SHA-256 1368d01910bce19816daf9af6963a24de3809a0b12162fce1e5dd9d79064e884"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The prose may cite correct facts but the source links are mutable or point to neighboring revisions.", "check": "Open each exact URL and verify commit/archive identifiers in the path.", "outcome": "Implementation/evaluation links are commit-pinned and CUDA is archive-pinned; the two mutable documentation links are used only for current general guidance and were freshly checked."}, "claim": "The page's cited revisions and direct URLs resolve to the exact starter-kit evaluation, FlashInfer reference/API, CUDA tensor-map requirements, CUTLASS small-GEMM guidance, and DeepGEMM grouped-layout record used by its claims.", "claim_id": "kernel-fused-moe-current-c012", "claim_type": "other", "covers": ["u059", "u060", "u061", "u062", "u063"], "evidence": [{"locator": "document", "observation": "The pinned path resolves to the evaluation contract.", "revision": "75ccd05cafceb0fd1f86be4cd0f2117249463c66", "url": "https://github.com/flashinfer-ai/flashinfer-bench-starter-kit/blob/75ccd05cafceb0fd1f86be4cd0f2117249463c66/EVALUATION.md"}, {"locator": "file", "observation": "The pinned trace reference resolves.", "revision": "7f614b86470180bab2d22e36fd1775791c6bf3e6", "url": "https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/trace/templates/moe.py"}, {"locator": "file", "observation": "The pinned API implementation resolves.", "revision": "7f614b86470180bab2d22e36fd1775791c6bf3e6", "url": "https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/fused_moe/core.py"}, {"locator": "tensor-memory API", "observation": "The archived official requirements resolve.", "revision": "CUDA 13.2.1", "url": "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"}, {"locator": "file", "observation": "The cited NVIDIA small-dimension discussion resolves.", "revision": "fetched 2026-08-08", "url": "https://github.com/NVIDIA/cutlass/blob/main/media/docs/cpp/efficient_gemm.md"}, {"locator": "README", "observation": "The pinned grouped-layout record resolves.", "revision": "891d57b4db1071624b5c8fa0d1e51cb317fa709f", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The command resolves the page but omits or mislabels artifact content.", "check": "Search the output for the corrected adjacent/derived scope and compare an unknown-ID control.", "outcome": "The labeled artifacts appear in the positive output and the invalid ID is rejected."}, "claim": "The documented get_page command returns this page with labeled artifact content, while an unknown page ID fails distinctly.", "claim_id": "kernel-fused-moe-current-c013", "claim_type": "code", "covers": ["u064", "u065"], "evidence": [{"command": "conda run -n base python scripts/get_page.py kernel-fused-moe --include-code", "environment": "KernelWiki worktree; conda base", "negative_control": "kernel-fused-moe-broken exited 1 with 'No page found'.", "reference": "current wiki/kernels/fused-moe.md and artifact manifests", "result": "Exited 0; output contains the corrected page, mixed-adjacent disclosure, MXFP4 labels, and derived reference."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "The arithmetic accidentally treats 32 as global experts or transposes the published activation-scale layout.", "check": "Compare every returned rank/order with the benchmark signature and exercise a non-default token count.", "outcome": "The helper keeps routing width 256, local weight count 32, and activation scales [56,T], matching the primary contract."}, "claim": "The displayed derived helper computes all fixed Track A tensor shapes for a positive token count and rejects a zero-token negative control.", "claim_id": "kernel-fused-moe-current-c014", "claim_type": "code", "covers": ["u026", "u027"], "evidence": [{"command": "conda run -n base python -c ", "environment": "KernelWiki worktree; conda base Python", "negative_control": "Calling the same helper with tokens=0 triggers its assertion and exits nonzero.", "reference": "official definition axes and signature", "result": "For tokens=7, returned routing [7,256], activations [7,7168], activation scales [56,7], W13 [32,4096,7168], W13 scales [32,32,56], W2 [32,7168,2048], and W2 scales [32,56,16]."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["inspect official contest and benchmark pages fetched 2026-08-08", "inspect FlashInfer commit 7f614b8 trace template, API, validation, and autotuning", "inspect starter-kit commit 75ccd05 evaluation contract and DeepGEMM commit 891d57b4 grouped layouts", "run the standard-library derived reference self-test and biased-weight negative control under conda base", "run get_page positive and unknown-ID negative controls under conda base"], "id": "kernel-fused-moe", "page_verdict": "verified", "path": "wiki/kernels/fused-moe.md", "residual_risk": "No B200 or CUDA toolkit is installed, so the optimized implementation, numeric tolerance on contest tensors, launch topology, and performance were not reproduced. The page therefore retains no result-level performance claim and labels local upstream files as adjacent rather than as the contest implementation.", "scope": "All 65 current units for body hash 5b18f9bb5eb643757003bd25f3822609266ef79d9151f3fd7623e798928547ed were checked against primary web records, exact upstream revisions, and discriminating host tests."} +{"claims": [{"challenge": {"alternative": "The rule is additive-only, retrieves before decay, emits from the old state, or rereads all prior tokens during decode.", "check": "Trace the reference operation order and input shapes, including the signs in the expanded remove/write expression.", "outcome": "The subtraction, decay-first ordering, updated-state output, and fixed recurrent-state input reject each alternative; K*V work remains, so only context-length independence is claimed."}, "claim": "FlashInfer commit 7f614b8 implements the displayed Gated DeltaNet recurrence with independent decay and sigmoid update gates, a delta correction against the decayed-state retrieval, output from the updated state, and a recurrent state whose decode shape has no prior-token axis.", "claim_id": "kernel-gated-delta-net-current-c001", "claim_type": "code", "covers": ["u001", "u002", "u003"], "evidence": [{"locator": "_gdn_decode_reference and gated_delta_rule_decode_trace", "observation": "The reference forms g=exp(-exp(A_log)*softplus(a+dt_bias)) and beta=sigmoid(b), decays state, reads with k, applies beta*(v-read) as an outer-product correction, and computes output from the new state. Decode state is [B,H,V,K] with no context-length axis.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/gdn.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The 3:1 pattern describes heads rather than layers, 512 includes active experts, or fixed GDN state removes cache growth for the full model.", "check": "Compare the explicit layer interval, expert-selection fields, and presence of full-attention layers in the immutable config.", "outcome": "The interval yields 36 GDN and 12 full-attention layers, active experts are 10 routed plus one shared, and the full-attention quarter retains a context-dependent cache."}, "claim": "Pinned Qwen3-Next-80B-A3B-Instruct has 48 layers arranged as twelve repetitions of three GDN layers and one full Gated Attention layer, 16 QK and 32 value GDN heads of dimension 128, 512 experts with 10 routed plus one shared, 80B total/3B active parameters, and native context 262144.", "claim_id": "kernel-gated-delta-net-current-c002", "claim_type": "compatibility", "covers": ["u004", "u005", "u006", "u007", "u008", "u009", "u010", "u011"], "evidence": [{"locator": "num_hidden_layers, full_attention_interval, linear-attention heads, expert fields, and max_position_embeddings", "observation": "The immutable config records 48 layers, full attention every fourth layer, 16/32/128 GDN geometry, 512 experts, 10 routed experts, one shared expert, and 262144 positions.", "revision": "Qwen model revision 9c7f2fbe84465e40164a94cc16cd30b6999b0cc7", "url": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/config.json"}, {"locator": "model card overview", "observation": "The official card states 80B total and 3B activated parameters and writes the hybrid stack as 12 repetitions of three Gated DeltaNet blocks plus one Gated Attention block.", "revision": "same model revision", "url": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/tree/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The article proves that output gating is necessary and belongs inside every GDN state update.", "check": "Read the named architectural subsection and distinguish Gated Attention from Gated DeltaNet.", "outcome": "The source uses a qualified benefit statement for the separate full-attention path, supporting the page's narrower wording."}, "claim": "Qwen attributes attention-sink and massive-activation mitigation to an output gate in its full Gated Attention path and describes the effect as helpful, not as a required part of the GDN recurrence.", "claim_id": "kernel-gated-delta-net-current-c003", "claim_type": "mechanism", "covers": ["u012"], "evidence": [{"locator": "Gated Attention and Training-Stability-Friendly Designs", "observation": "The official article places output gating in Gated Attention and says it helps eliminate attention sinks and massive activations; it does not make that gate part of the GDN kernel contract.", "revision": "Qwen3-Next architecture article fetched 2026-08-08", "url": "https://qwen.ai/blog?id=e34c4305036ce60d55a0791b170337c2b70ae51d"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "qk4/v8 are flattened dimensions, prefill is fixed-length, or 512 KiB applies to each head.", "check": "Read the exact axis names and tensor ranks, then recompute 128*128*4 bytes and multiply by eight value heads.", "outcome": "The definitions name head counts and variable-length metadata; arithmetic gives 64 KiB per head and 512 KiB across all eight, exactly as scoped."}, "claim": "MLSys 2026 Track C defines distinct decode and variable-length prefill contracts at TP=4 with Q/K/V heads 4/4/8, head size 128, BF16 Q/K/V and output, FP32 state [B or N,8,128,128], the listed gate inputs, and cu_seqlens only for prefill; the derived state sizes are 64 KiB per value head and 512 KiB across eight heads.", "claim_id": "kernel-gated-delta-net-current-c004", "claim_type": "compatibility", "covers": ["u013", "u014", "u015", "u016", "u017", "u018", "u019", "u020", "u021"], "evidence": [{"locator": "Track C", "observation": "The organizer names Gated Delta Net as Track C and links separate decode and prefill definitions.", "revision": "fetched 2026-08-08", "url": "https://mlsys26.flashinfer.ai/"}, {"locator": "Axes, Signature, and Outputs", "observation": "Decode fixes q/k/v heads 4/4/8 and d128, uses BF16 token tensors, FP32 [B,8,128,128] state and four gate-related inputs, and returns BF16 [B,1,8,128] plus updated state.", "revision": "fetched 2026-08-08", "url": "https://bench.flashinfer.ai/kernels/gdn_decode_qk4_v8_d128_k_last"}, {"locator": "Axes, Signature, and Outputs", "observation": "Prefill uses total_seq_len, num_seqs, cu_seqlens[N+1], the same head geometry/dtypes, FP32 [N,8,128,128] state, BF16 token output, and one final state per sequence.", "revision": "fetched 2026-08-08", "url": "https://bench.flashinfer.ai/kernels/gdn_prefill_qk4_v8_d128_k_last"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "FLA is an unendorsed fork and the authors' implementation already claims the same variable-length support.", "check": "Use the authors' FAQ rather than a third-party comparison.", "outcome": "The FAQ directly recommends FLA and distinguishes its faster and variable-length functionality."}, "claim": "NVlabs commit b53d6d3 is the authors' PyTorch/Triton research implementation and its README recommends FLA for faster kernels and variable-length functionality.", "claim_id": "kernel-gated-delta-net-current-c005", "claim_type": "compatibility", "covers": ["u022"], "evidence": [{"locator": "repository overview and FAQ questions 1 and 2", "observation": "The authors provide PyTorch/Triton code, state that FLA kernels are faster, identify FLA variable-length support, and strongly recommend it.", "path": "/tmp/kernelwiki-gdn-pinned/README.md", "revision": "NVlabs/GatedDeltaNet b53d6d3a161267432a79c1c04af69fa52bddc921"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Blackwell prefill remains unimplemented or shares an unrestricted generic path with Hopper.", "check": "Inspect architecture dispatch, imported implementation, and explicit runtime guards.", "outcome": "A distinct SM100/103 implementation is dispatched and guarded by CUDA/head-size requirements, refuting both alternatives."}, "claim": "At commit 7f614b8 FlashInfer provides the cited GDN trace contracts and a prefill dispatcher for SM90 and SM100/SM103, with the latter implemented in CuTe DSL and requiring CUDA 13 or newer and head size 128.", "claim_id": "kernel-gated-delta-net-current-c006", "claim_type": "compatibility", "covers": ["u023"], "evidence": [{"locator": "chunk_gated_delta_rule dispatch and SM100 import guard", "observation": "The wrapper dispatches SM90 separately and SM100/SM103 to chunk_gated_delta_rule_sm100; its checks document CuTe DSL, CUDA >=13, and head_size 128 for the SM100 path.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/gdn_prefill.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}, {"locator": "decode and prefill trace definitions", "observation": "The file supplies the exact reference contracts summarized on the page.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/gdn.py", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Graphs remove GPU compute time or guarantee acceleration for every batch shape.", "check": "Compare NVIDIA's overhead scope with vLLM's uniform-decode capture conditions.", "outcome": "The sources support launch-cost reduction under compatible capture conditions, so the page correctly retains a workload-dependent boundary."}, "claim": "vLLM merge e1d85e5c supports uniform-batch full CUDA-graph capture for compatible recurrent-attention decode, while NVIDIA scopes CUDA-graph benefit to reduced CPU launch overhead rather than a universal workload speedup.", "claim_id": "kernel-gated-delta-net-current-c007", "claim_type": "performance", "covers": ["u024"], "evidence": [{"locator": "BaseMambaAttentionMetadataBuilder.build_for_cudagraph_capture", "observation": "The backend declares uniform-batch support and implements decode-only full CUDA-graph capture.", "revision": "vLLM e1d85e5c2454bd8d349dbe676679380cbe0e920a", "url": "https://github.com/vllm-project/vllm/blob/e1d85e5c2454bd8d349dbe676679380cbe0e920a/vllm/v1/attention/backends/mamba_attn.py"}, {"locator": "CUDA Graphs overview", "observation": "NVIDIA explains that setup can dominate short kernels and graph replay reduces CPU launch costs.", "revision": "CUDA Programming Guide fetched 2026-08-08", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cuda-graphs.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A general linear-recurrence paper title or Hopper support is sufficient evidence of GDN-specific assembly.", "check": "Inspect the published application and search the immutable implementation tree for GDN and instruction-family identifiers.", "outcome": "The method and code are mLSTM-scoped and the claimed GDN/assembly identifiers are absent, supporting the explicit boundary."}, "claim": "TFLA v3 and its official pinned code apply a second level of sequence parallelism to mLSTM for arbitrarily large chunks, but do not establish a Gated DeltaNet implementation or inline WGMMA/tcgen05 path.", "claim_id": "kernel-gated-delta-net-current-c008", "claim_type": "compatibility", "covers": ["u025"], "evidence": [{"locator": "TFLA overview and implementation list", "observation": "The official repository describes TFLA for mLSTM, arbitrary large chunks, and PyTorch/JAX/Triton implementations. A pinned-tree search finds no GatedDeltaNet, wgmma, tcgen05, inline PTX, or asm implementation.", "path": "/tmp/kernelwiki-mlstm-kernels/README.md", "revision": "NX-AI/mlstm_kernels 5b98ff8e2bec189b3d3c249405bab5149564d6f8"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "O(n) complexity alone proves a 10x kernel speedup at 32K, or a serving runtime can swap GDN into arbitrary attention weights.", "check": "Inspect the comparison's scope and implementation caveat, then compare GDN's learned parameter/state contract.", "outcome": "The result is whole-model and implementation-dependent, and the layer requires GDN-specific learned components; neither alternative is supported."}, "claim": "The page accurately separates GDN's linear sequence scaling from Qwen's implementation-dependent 10x whole-model comparison, and correctly warns that GDN is a learned architectural layer rather than a weight-compatible runtime replacement for softmax attention.", "claim_id": "kernel-gated-delta-net-current-c009", "claim_type": "performance", "covers": ["u026", "u027"], "evidence": [{"locator": "Highlights and implementation caveat", "observation": "Qwen reports 10x whole-model inference throughput over Qwen3-32B for contexts above 32K and separately warns that efficiency depends strongly on implementation; it supplies no isolated GDN-kernel GPU result.", "revision": "Qwen model revision 9c7f2fbe84465e40164a94cc16cd30b6999b0cc7", "url": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/tree/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7"}, {"locator": "GatedDeltaNet initialization and forward", "observation": "The layer owns learned projections, decay/update gates, convolution, output gating/norm, and recurrent state distinct from softmax attention.", "revision": "NVlabs/GatedDeltaNet b53d6d3a161267432a79c1c04af69fa52bddc921", "url": "https://github.com/NVlabs/GatedDeltaNet/blob/b53d6d3a161267432a79c1c04af69fa52bddc921/lit_gpt/gated_delta_net.py"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "An upstream file named gdn_fused_proj can be represented as the full GDN kernel.", "check": "Inspect every function for state, gates, recurrence, token loop, or decode/prefill entry points.", "outcome": "Only projection-output rearrangement is present, so the corrected adjacent label is precise."}, "claim": "The local full artifact is byte-pinned SGLang projection split/reshape/concatenation code from merge 5bdc07d and is accurately labeled as adjacent preprocessing rather than a GDN recurrence implementation.", "claim_id": "kernel-gated-delta-net-current-c010", "claim_type": "code", "covers": ["u028"], "evidence": [{"locator": "module functions", "observation": "The byte-verified file splits and reshapes QKVZBA projection output and concatenates fields; it contains no recurrent state update, prefill, or decode implementation.", "path": "artifacts/kernels/gated-delta-net/full/gdn_fused_proj.py", "revision": "SGLang 5bdc07d974f6cf236fa765a685453ea5e587a838; SHA-256 c4ef85167eb11065721e76dd5d63d6f346f309f84822578dd4a3874249acef89"}, {"locator": "scope and upstream_identity", "observation": "Provenance records the exact SGLang merge/file hash and labels the code adjacent preprocessing.", "path": "artifacts/kernels/gated-delta-net/full/PROVENANCE.yaml", "revision": "current corrected provenance"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The compact and expanded checks merely duplicate the same erroneous additive update.", "check": "Inspect the expanded path's explicit removal of outer(k, beta*read) and run the additive-only negative control.", "outcome": "The independently grouped remove/write form matches the compact correction while the no-removal alternative is rejected."}, "claim": "The derived standard-library variant faithfully checks the one-head compact GDN recurrence against an independently expanded remove/write form and rejects an additive-only update.", "claim_id": "kernel-gated-delta-net-current-c011", "claim_type": "code", "covers": ["u029"], "evidence": [{"command": "conda run -n base python artifacts/kernels/gated-delta-net/variants/01-gdn-recurrence-reference.py", "environment": "KernelWiki worktree; conda base standard-library Python", "negative_control": "The self-test computes an additive-only state and asserts it differs from the correct delta-rule state.", "reference": "FlashInfer _gdn_decode_reference at commit 7f614b8", "result": "Exited 0 and printed 'gated-delta recurrence reference: PASS'; file SHA-256 is 11560c14d805f79dbc1025801beed373c0e17e712e054c6f85231071b8d8b971."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "One or more citations are stale, redirect to a generic page, or lose the stated revision.", "check": "Follow redirects for every target, inspect identity/revision, and compare a deliberately invalid same-site path.", "outcome": "Each intended resource resolves and the invalid control is rejected; immutable code/model links preserve their revision identifiers."}, "claim": "All ten primary-source links resolve to the named paper, immutable implementations/configuration, organizer, exact definitions, and official guidance.", "claim_id": "kernel-gated-delta-net-current-c012", "claim_type": "other", "covers": ["u030", "u031", "u032", "u033", "u034", "u035", "u036", "u037", "u038", "u039"], "evidence": [{"command": "curl -L -sS -o /dev/null -w '%{http_code}' ", "environment": "KernelWiki host, 2026-08-08", "negative_control": "https://mlsys26.flashinfer.ai/definitely-not-a-kernelwiki-page returned HTTP 404.", "reference": "HTTP success plus source identity and pinned revision in each target", "result": "All ten targets returned HTTP 200; the implementation and model URLs retain the stated commit/revision paths."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "The query returns prose but silently omits or mislabels the artifacts.", "check": "Search positive output for the corrected scope and derived file content, then compare an unknown-ID request.", "outcome": "Both artifact classes are present with corrected labels and the invalid ID is rejected."}, "claim": "The documented get_page command returns the corrected page and labeled local artifacts, while an unknown page ID fails distinctly.", "claim_id": "kernel-gated-delta-net-current-c013", "claim_type": "code", "covers": ["u040", "u041"], "evidence": [{"command": "conda run -n base python scripts/get_page.py kernel-gated-delta-net --include-code", "environment": "KernelWiki worktree; conda base", "negative_control": "kernel-gated-delta-net-broken exited nonzero with 'No page found'.", "reference": "current page and corrected artifact manifests", "result": "Exited 0; output contains 'Verified mechanism', the full artifact's 'adjacent preprocessing' disclosure, and the derived recurrence PASS marker."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["inspect FlashInfer commit 7f614b8, NVlabs/GatedDeltaNet commit b53d6d3, NX-AI/mlstm_kernels commit 5b98ff8, and vLLM merge e1d85e5c", "inspect immutable Qwen3-Next configuration revision 9c7f2fbe and official Qwen architecture article", "open the MLSys organizer and exact FlashInfer-Bench decode/prefill definitions on 2026-08-08", "run the derived recurrence equivalence test and additive-only negative control under conda base", "run ten-link and get_page positive/negative controls"], "id": "kernel-gated-delta-net", "page_verdict": "verified", "path": "wiki/kernels/gated-delta-net.md", "residual_risk": "No B200/H100 or CUDA toolkit is installed, so optimized GPU numeric behavior and performance were not reproduced. The page therefore retains no isolated kernel result and scopes implementation details to inspected primary revisions.", "scope": "All 41 current coverage units for body hash 2ed46cc12dc31743da8142c69d5a8c6b3d03b077d1ddd624e2e4351206734b68 were checked against primary web records, immutable upstream source, and discriminating host tests."} +{"claims": [{"challenge": {"alternative": "The numbering/dates are inferred from repository history, or the target is generic Blackwell including Hopper/desktop parts.", "check": "Compare the sponsor's numbered rules and the immutable task description.", "outcome": "Both scope fields are explicit: Kernel #3 at the stated dates and NVIDIA B200; the page does not extend compatibility beyond that record."}, "claim": "Official NVIDIA rules identify NVFP4 Gated Dual GEMM as Blackwell Hackathon Kernel Challenge 3 for December 20, 2025 through January 16, 2026, and the challenge-opening GPU Mode task targets B200.", "claim_id": "kernel-gated-dual-gemm-current-c001", "claim_type": "compatibility", "covers": ["u001"], "evidence": [{"locator": "Section 3, Kernel #3", "observation": "The sponsor's rules explicitly give the challenge number, name, and entry dates.", "revision": "official rules fetched 2026-08-08", "url": "https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf"}, {"locator": "description", "observation": "The immutable public task says the block-scaled dual matrix multiplication is optimized for NVIDIA B200.", "revision": "gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c", "url": "https://github.com/gpu-mode/reference-kernels/blob/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm/task.yml"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The two products are multiplied before activation, the second branch is gated, or 'kernel' normatively means one launch.", "check": "Trace the reference expression and inspect every constraint on custom_kernel.", "outcome": "The expression fixes first-branch SiLU, while the interface constrains only output; no launch topology is stated."}, "claim": "The displayed result semantics match the pinned reference: two block-scaled products share A, SiLU applies only to the B1 branch, the result multiplies the B2 branch, and neither correctness interface nor reference mandates a single GPU launch or absence of intermediate storage.", "claim_id": "kernel-gated-dual-gemm-current-c002", "claim_type": "code", "covers": ["u002", "u003", "u004"], "evidence": [{"locator": "ref_kernel", "observation": "The reference invokes torch._scaled_mm twice with a_ref and separate B/scales, then computes silu(ref1)*ref2; it allocates ref1 and ref2 explicitly.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/reference.py", "revision": "gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c"}, {"locator": "custom_kernel", "observation": "The submission interface is a Python function returning c and contains no launch-count or intermediate-storage restriction.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/template.py", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "FNUZ and FN are interchangeable spelling here, reordered scales are internal-only, or K/16 is a physical flat layout guarantee.", "check": "Compare dtype constructors and the complete returned/submission tuple rather than normalizing the prose description.", "outcome": "The primary files genuinely disagree on dtype suffix and expose both logical and reordered scales, so the page preserves the inconsistency and avoids a stronger physical-layout claim."}, "claim": "The tensor table, reordered-scale inputs, divisibility requirements, 1e-3 tolerances, and disclosed E4M3FNUZ-versus-E4M3FN inconsistency accurately reproduce the challenge-opening task, template, and reference.", "claim_id": "kernel-gated-dual-gemm-current-c003", "claim_type": "compatibility", "covers": ["u005", "u006", "u007", "u008", "u009", "u010", "u011"], "evidence": [{"locator": "description and matrix constraints", "observation": "The task publishes A/B1/B2 NVFP4 K-major shapes, logical FP8 scale shapes, FP16 C, K divisible by 256, and M/N tile divisibility; it labels scales e4m3fnuz.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml", "revision": "gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c"}, {"locator": "custom_kernel tuple documentation", "observation": "The starter exposes raw and reordered copies of all three scale tensors plus preallocated C, and also labels scales float8_e4m3fnuz.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/template.py", "revision": "same commit"}, {"locator": "create_scale_factor_tensors, return tuple, and check_implementation", "observation": "The generator actually constructs torch.float8_e4m3fn scale tensors, returns raw/reordered copies, and configures rtol=atol=1e-3.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/reference.py", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The listed values are measured baseline or winner latencies, and 18.5 us may be a hidden fifth benchmark.", "check": "Read the table label and compare every benchmark dictionary in the immutable task.", "outcome": "The values are explicitly theoretical and all four benchmark dictionaries are enumerated; no fifth shape or measured result exists in the source."}, "claim": "The four M/N/K/L workload rows, theoretical microsecond values, geometric-mean ranking, 1.5 GHz B200 model, and removal of the unmatched 18.5-us result exactly follow the official task's performance boundary.", "claim_id": "kernel-gated-dual-gemm-current-c004", "claim_type": "performance", "covers": ["u012", "u013", "u014", "u015", "u016"], "evidence": [{"locator": "speed-of-light table, benchmarks, and ranking_by", "observation": "The task lists exactly the four rows and 4.708/8.714/2.125/6.535 us values, calls them speed-of-light analysis from max math/DRAM time at 1.5 GHz, and ranks by geometric mean. It has no M=1024 workload or 18.5-us measurement.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml", "revision": "gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Logical sharing forces one physical A load and dual TMEM accumulators, or any gated MLP can be adapted without additional evidence.", "check": "Distinguish observable tensor semantics from internal scheduling and enumerate all dtype/layout/grouping preconditions.", "outcome": "Several conforming decompositions remain possible and the ABI has material format/layout constraints; the page's scoped advice is the strongest supportable conclusion."}, "claim": "The page correctly treats shared-input reuse and fused epilogue traffic reduction as opportunities, not guarantees, rejects unsupported leaderboard/TMEM/TMA/CUTLASS/bottleneck details, and limits compatibility to the complete dual-NVFP4 contract rather than arbitrary dual-output or grouped-MoE operations.", "claim_id": "kernel-gated-dual-gemm-current-c005", "claim_type": "compatibility", "covers": ["u017", "u018", "u019"], "evidence": [{"locator": "complete task description and benchmark contract", "observation": "A/SFA are logically shared and the operation has two same-shaped NVFP4 branches, but the task contains none of the removed implementation, leaderboard, launch-count, grouping, or bottleneck assertions.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml", "revision": "gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c"}, {"locator": "ref_kernel and generate_input", "observation": "The exact compatibility surface includes branch order, NVFP4 operands, distinct scale tensors/layouts, batch dimension, FP32 reference products, and FP16 output; the source is not grouped per expert.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/reference.py", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "A task specification plus directory name can still be mistaken for a winning/full optimized kernel.", "check": "Inspect the page disclosure, provenance role, and actual bundle contents.", "outcome": "All three explicitly call it a problem specification, and no optimized-submission file is present."}, "claim": "The local full bundle contains a byte-identical official task.yml from the pinned challenge-opening commit, is labeled as a specification rather than an optimized submission, and no longer presents unrelated blog/vLLM material as the contest implementation.", "claim_id": "kernel-gated-dual-gemm-current-c006", "claim_type": "code", "covers": ["u020"], "evidence": [{"locator": "entire file", "observation": "The local specification is byte-identical to the upstream task.yml blob at the pinned commit.", "path": "artifacts/kernels/gated-dual-gemm/full/task.yml", "revision": "gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c; SHA-256 2bf4e5c04cc8a9c283e519437c8324ca9265d9c12d0dfe2b76dc9f0542ada051"}, {"locator": "origin, upstream_sha, asset_mode, and file role", "observation": "Provenance pins the full commit and labels the sole file an official problem specification; the former unrelated files are absent from the bundle.", "path": "artifacts/kernels/gated-dual-gemm/full/PROVENANCE.yaml", "revision": "current corrected provenance"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The test duplicates the same branch-order mistake in both implementations or accidentally exercises symmetric data.", "check": "Inspect the separately formed wrong_branch path and run it on asymmetric B1/B2 matrices.", "outcome": "Compact and expanded correct forms match, while the explicit swapped-gate alternative differs, discriminating branch order."}, "claim": "The derived standard-library variant performs both shared-A products, applies SiLU to the first branch, passes compact-versus-expanded equivalence, and rejects gating the second branch while making no NVFP4/GPU/performance claim.", "claim_id": "kernel-gated-dual-gemm-current-c007", "claim_type": "code", "covers": ["u021"], "evidence": [{"command": "conda run -n base python artifacts/kernels/gated-dual-gemm/variants/01-gated-dual-gemm-semantics.py", "environment": "KernelWiki worktree; conda base standard-library Python", "negative_control": "The self-test computes SiLU(up)*gate and asserts that it differs from the required SiLU(gate)*up result.", "reference": "gpu-mode/reference-kernels ref_kernel at c5b2f7c", "result": "Exited 0 and printed 'gated dual GEMM semantic reference: PASS'; file SHA-256 ac13383732fc5c946ae1698cc0776131dd399c507736b28a1a7d4f45a976daeb."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "A source link is mutable, missing, or points to a neighboring problem/revision.", "check": "Follow redirects and inspect problem path plus commit in every repository URL, with a same-path invalid control.", "outcome": "Each link resolves to the intended record and all code links are immutable; the invalid control is rejected."}, "claim": "The four primary-source links resolve to the official rules and the exact pinned task, starter, and reference files.", "claim_id": "kernel-gated-dual-gemm-current-c008", "claim_type": "other", "covers": ["u022", "u023", "u024", "u025"], "evidence": [{"command": "curl -L -sS -o /dev/null -w '%{http_code}' ", "environment": "KernelWiki host, 2026-08-08", "negative_control": "A nonexistent file under the same pinned GitHub problem path returned HTTP 404.", "reference": "HTTP success and source identity/revision", "result": "All four targets returned HTTP 200 and the three repository paths retain commit c5b2f7c062d5015f29c3a1043cfd04954397944c."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "The query resolves prose but omits the corrected full or derived artifact.", "check": "Search positive output for content unique to each bundle and compare an invalid ID.", "outcome": "Both artifact types appear with corrected scope and the unknown ID is rejected."}, "claim": "The documented get_page command returns the corrected page and both labeled artifact bundles, while an unknown page ID fails distinctly.", "claim_id": "kernel-gated-dual-gemm-current-c009", "claim_type": "code", "covers": ["u026", "u027"], "evidence": [{"command": "conda run -n base python scripts/get_page.py kernel-gated-dual-gemm --include-code", "environment": "KernelWiki worktree; conda base", "negative_control": "kernel-gated-dual-gemm-broken exited nonzero with 'No page found'.", "reference": "current page and corrected provenance", "result": "Exited 0; output contains 'Verified scope', the official task.yml disclosure, and the derived semantic reference PASS marker."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["inspect official NVIDIA challenge rules and gpu-mode/reference-kernels commit c5b2f7c task/template/reference", "compare the local task.yml byte-for-byte with its upstream blob", "run the standard-library semantic reference and wrong-branch negative control under conda base", "run four-link and get_page positive/negative controls"], "id": "kernel-gated-dual-gemm", "page_verdict": "verified", "path": "wiki/kernels/gated-dual-gemm.md", "residual_risk": "No B200, CUDA toolkit, or immutable public winning submission is available, so optimized numeric behavior, launch topology, and measured performance were not reproduced. The upstream FNUZ/FN scale-label inconsistency is disclosed rather than resolved editorially.", "scope": "All 27 current coverage units for body hash 11f0038a8e9f681dac4ffd25654362cefa9f94a039e46745ac55f9488a405f80 were checked against official rules, immutable upstream source, byte comparison, and discriminating host tests."} +{"claims": [{"challenge": {"alternative": "All grouped GEMM APIs mean only M varies, or the contest's varied N/K values are documentation noise; 'kernel' requires one physical launch.", "check": "Compare every official correctness dictionary and the unconstrained custom Python entry with DeepGEMM's explicit dimensional rules.", "outcome": "Multiple required cases vary N/K, DeepGEMM expressly describes a narrower API, and neither interface constrains the number of launches."}, "claim": "Official rules identify NVFP4 Grouped GEMM as Kernel Challenge 4 for January 17 through February 13, 2026 with 40% grand-prize weight; the corrected public task targets B200 and permits independent per-group M/N/K, while DeepGEMM separately defines fixed-N/K M-grouping and K-grouped backward.", "claim_id": "kernel-grouped-gemm-current-c001", "claim_type": "compatibility", "covers": ["u001", "u002", "u003", "u004", "u005"], "evidence": [{"locator": "Section 3 Kernel #4 and Grand Prize Judging Criteria", "observation": "The rules provide the number, name, dates, and 40% weight.", "revision": "official NVIDIA rules fetched 2026-08-08", "url": "https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf"}, {"locator": "description, tests, and benchmarks", "observation": "The task targets B200 and its correctness cases vary M, N, and K within group lists; the submission is a Python entry point with no one-launch rule.", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6", "url": "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/task.yml"}, {"locator": "grouped GEMM sections", "observation": "DeepGEMM explicitly fixes N/K for M-grouping and assigns K-grouping with fixed M/N to MoE weight backward.", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The physical FP4 shape is [M,K,L], reordered scales are internal rather than submitted, all groups share dimensions, or FNUZ/FN is just one normalized dtype.", "check": "Trace generated tensor storage, task.py/template tuple arity, correctness cases, and the actual dtype constructor.", "outcome": "Packed FP4 uses K/2 bytes, the public object has four members, cases vary dimensions, and the primary files genuinely disagree on the dtype suffix; the page preserves each distinction."}, "claim": "The page's per-group packed-NVFP4 operands, FP16 output, logical scale shapes, four-list submission object, reordered scales, divisibility/tolerance rules, per-group reference semantics, and disclosed E4M3FNUZ-versus-E4M3FN inconsistency match the pinned GPU Mode task/template/reference.", "claim_id": "kernel-grouped-gemm-current-c002", "claim_type": "compatibility", "covers": ["u006", "u007", "u008", "u009", "u010", "u011", "u012", "u013", "u014"], "evidence": [{"locator": "description and constraints", "observation": "The task publishes packed A/B, FP16 C, logical scale shapes, per-group sizes, K/256 and M/N tile divisibility, and E4M3FNUZ labels.", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6", "url": "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/task.yml"}, {"locator": "custom_kernel tuple documentation", "observation": "The starter exposes logical and reordered scales as separate lists and states that each group can have different m/n/k/l, with l=1.", "revision": "same commit", "url": "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/template.py"}, {"locator": "ref_kernel, generate_input, and check_implementation", "observation": "The reference processes each group with B transposed, constructs E4M3FN scales despite the prose label, returns four lists, writes FP16 results, and uses rtol=atol=1e-3.", "revision": "same commit", "url": "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/reference.py"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The nested loops accidentally compute A@B, reuse one group's dimensions, or imply that this is an NVFP4 implementation.", "check": "Use non-square B row counts and different K across groups, compare exact expected products, and inspect the adjacent scope disclaimer.", "outcome": "The asymmetric outputs require dotting each A row with each stored B row independently, the shared-K control fails, and the prose excludes packing/scales/GPU/performance."}, "claim": "The CPU grouped_reference code computes A_i times B_i-transpose independently for non-empty ordinary-number groups with different M/N/K, rejects ragged or incompatible per-group K, and explicitly makes no NVFP4, scaling, GPU, or performance claim.", "claim_id": "kernel-grouped-gemm-current-c009", "claim_type": "code", "covers": ["u015", "u016", "u017"], "evidence": [{"command": "conda run -n base python -c ''", "environment": "KernelWiki host; conda base standard-library Python", "negative_control": "A deliberately wrong wrapper requiring every group to share the first group's K rejected the valid K=3 second group.", "reference": "Per-group C_i=A_i@B_i.T semantics from gpu-mode/reference-kernels reference.py at ae67948685dfccf54ae8374dc9402addb7aae4f6", "result": "Exited 0 and printed 'grouped independent-shape semantic reference: PASS; shared-K control rejected'; outputs were [[[17],[39]],[[1,5]]]."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "grouped_layout is always an offsets array, masked_m is a boolean [G,Mmax] mask that computes padding, or K-grouping returns one ungrouped M-by-N result.", "check": "Compare both grouped-layout modes and concrete generator ranks against the C++ API assertions.", "outcome": "The pinned code distinguishes map/prefix modes, uses one int count per group, and allocates one M-by-N result per K group, rejecting the former invented structs."}, "claim": "The DeepGEMM table correctly records representative pinned M-contiguous, M-masked, and K-contiguous tensor shapes, group descriptors, valid-row behavior, fixed dimensions, alignment boundary, and documented MoE phase roles.", "claim_id": "kernel-grouped-gemm-current-c003", "claim_type": "compatibility", "covers": ["u018", "u019", "u020", "u021", "u022"], "evidence": [{"locator": "Grouped GEMMs contiguous and masked sections", "observation": "The README fixes N/K for M grouping, requires aligned expert segments, maps contiguous/masked/K-grouped modes to prefill/decode/weight backward, and says masked mode computes valid portions.", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"}, {"locator": "m_grouped_fp8_fp4_gemm_* and k_grouped_fp8_gemm_* functions", "observation": "API assertions establish A/B/D ranks, int grouped_layout or masked_m inputs, packed sum-K operands, grouped D/C outputs, architecture/dtype/layout checks, and fixed dimensions.", "revision": "same commit", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp"}, {"locator": "three grouped generators", "observation": "Generators construct per-row IDs with -1 padding or prefix-sum ends, one masked-M count per group, and K-packed operands with D/C shaped [G,M,N].", "revision": "same commit", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/tests/generators.py"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The values are measured winner/baseline times, one row has 15 groups, or N equals K.", "check": "Compare the table label and every benchmark dictionary at the pinned revision.", "outcome": "The source explicitly calls them speed-of-light estimates; group counts are 8/8/2/2 and each row's N and K are distinct."}, "claim": "The four workload rows, group counts, M lists, N/K/L values, theoretical microsecond values, geometric-mean ranking, and 1.5-GHz B200 speed-of-light interpretation exactly reproduce the public task and are not presented as contestant measurements.", "claim_id": "kernel-grouped-gemm-current-c004", "claim_type": "performance", "covers": ["u023", "u024", "u025", "u026", "u027"], "evidence": [{"locator": "speed-of-light table, benchmarks, ranking_by, and timeout", "observation": "The immutable task lists exactly the four rows and 18.833/10.667/2.406/1.525-us values, labels the analysis theoretical at 1.5 GHz, and ranks benchmark results by geometric mean.", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6", "url": "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/task.yml"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The number is a legitimate per-call record, the exploit skips arithmetic, or the official remediation was in FlashInfer-Bench.", "check": "Trace the timed call sequence, post-timing correctness statement, scrub notice, and named fix in the official account.", "outcome": "All arithmetic is merged into one amortized launch, the leaderboard entry is invalidated, and the only concrete response identified is reference-kernels PR #104."}, "claim": "The reward-hack account correctly scopes 11.191 microseconds as a temporary number-one exploit report that was scrubbed, records the cloned-correctness and 120-group/15-call timing mechanism, and identifies reference-kernels PR #104 rather than a FlashInfer-Bench/MLSys change as the documented harness response.", "claim_id": "kernel-grouped-gemm-current-c005", "claim_type": "performance", "covers": ["u028", "u029", "u030"], "evidence": [{"locator": "opening account, TL;DR, Correctness Check, Gaming the Timing Loop, and lessons", "observation": "The official post supplies the temporary rank/report/scrub status, real padded 8-group correctness calls, one merged 120-group launch over 15 objects, 14 cached returns, division by 15, and a link to PR #104 as the response; it makes no FlashInfer-Bench/MLSys attribution.", "revision": "GPU Mode post dated 2026-03-09, fetched 2026-08-08", "url": "https://www.gpumode.com/news/reward-hacking-nvfp4"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The schedule tag alone makes the old kernel valid, atomic work stealing is CLC, small M always implies one dominant bottleneck, or all TMA paths inherit an undocumented 128-byte rule.", "check": "Compare the exact CUTLASS type, CLC protocol, scoped efficiency wording, and versioned tensor-map constraints.", "outcome": "Each stronger alternative is contradicted; the page limits performance conclusions to possibilities and exact implementation measurements."}, "claim": "The implementation boundary is accurate: the task mandates no CUTLASS/CLC/TMA/TMEM/schedule/launch topology; the removed CUTLASS/CUDA sketches were not valid instances; small dimensions can underfill a GEMM; CLC requires an async request/mbarrier/decode protocol distinct from atomicAdd; and CUDA 13.2.1 does not define a universal 128-byte TMA alignment rule.", "claim_id": "kernel-grouped-gemm-current-c006", "claim_type": "compatibility", "covers": ["u031", "u032", "u033", "u034"], "evidence": [{"locator": "complete contract", "observation": "The task specifies tensors, tests, workloads, scoring, and timeout but none of the named implementation requirements.", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6", "url": "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/task.yml"}, {"locator": "GemmGrouped declaration", "observation": "The real template signature contradicts the former seven-argument invented type.", "revision": "NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3", "url": "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/gemm/kernel/gemm_grouped.h"}, {"locator": "Threadblock-level GEMM", "observation": "CUTLASS documents partial threadblock work for small M/N and insufficient grid parallelism when M/N are small and K is large.", "revision": "same commit", "url": "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/media/docs/cpp/efficient_gemm.md"}, {"locator": "4.12 flow and 4.12.1.1 API details", "observation": "CLC's documented sequence includes initial block work, shared response/mbarrier, async cancellation, wait, decode, and proxy ordering, not a global atomic tile counter.", "revision": "CUDA Programming Guide 13.2", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"}, {"locator": "cuTensorMapEncodeTiled constraints", "observation": "The API specifies a 64-byte descriptor and generally 16-byte global address/strides, plus feature-specific 32-byte constraints, rather than one universal 128-byte requirement.", "revision": "CUDA Driver API 13.2.1", "url": "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "One link is dead, mutable where it claims a pin, or points to a neighboring problem/project.", "check": "Follow redirects, inspect host/path/commit for every target, and compare a same-path invalid control.", "outcome": "Every positive target resolves to its intended source identity/revision while the invalid control fails distinctly."}, "claim": "All ten primary-source links resolve to the intended official rules, pinned GPU Mode and DeepGEMM files, GPU Mode postmortem, versioned CUDA documents, and pinned CUTLASS guide.", "claim_id": "kernel-grouped-gemm-current-c007", "claim_type": "other", "covers": ["u035", "u036", "u037", "u038", "u039", "u040", "u041", "u042", "u043", "u044"], "evidence": [{"command": "curl -L -sS -o /dev/null -w '%{http_code}' ", "environment": "KernelWiki host, 2026-08-08", "negative_control": "A nonexistent file under the same pinned GPU Mode grouped-GEMM path returned HTTP 404.", "reference": "HTTP success plus source identity and immutable repository revision", "result": "All ten targets returned HTTP 200; GPU Mode, DeepGEMM, and CUTLASS repository paths retain the claimed full commit IDs."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "The command resolves stale indexed content or accepts arbitrary page IDs.", "check": "Search output for content unique to the corrected page and compare a near-name invalid ID.", "outcome": "Corrected unique text is present and the unknown ID is rejected."}, "claim": "The documented query returns the corrected grouped-GEMM page, while an unknown page ID fails distinctly.", "claim_id": "kernel-grouped-gemm-current-c008", "claim_type": "code", "covers": ["u045", "u046"], "evidence": [{"command": "conda run -n base python scripts/get_page.py kernel-grouped-gemm", "environment": "KernelWiki worktree; conda base", "negative_control": "kernel-grouped-gemm-broken exited nonzero with 'No page found'.", "reference": "current page and generated source index", "result": "Exited 0 and returned the corrected title, independent-shape contract, DeepGEMM masked_m table, scrubbed exploit boundary, and TMA correction."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["inspect the pinned task/template/reference and DeepGEMM README/API/generators", "inspect versioned CUTLASS, CUDA CLC, and tensor-map primary documentation", "run the CPU grouped-reference positive case and wrong shared-K negative control under conda base", "run ten-link HTTP positive controls and one same-path 404 control", "run get_page positive and near-name negative controls under conda base"], "id": "kernel-grouped-gemm", "page_verdict": "verified", "path": "wiki/kernels/grouped-gemm.md", "residual_risk": "No B200 or CUDA toolkit is available, so optimized numerical behavior and performance were not reproduced. The page intentionally contains no measured valid contestant record or claimed implementation topology; retained numeric values are source-labeled theoretical estimates.", "scope": "All 46 current coverage units for body hash 733a51f41033ffcc0ff6fc42c4813e72da9b342f2e17aa629cfbee80177927ce were checked against immutable upstream source, versioned vendor documentation, and discriminating host controls."} +{"claims": [{"challenge": {"alternative": "Algorithmic portability is enough to list SM90/SM100, or the quality result is a universal guarantee.", "check": "Separate tested hardware and evaluated tasks from plausible portability and untested models.", "outcome": "Only A100 is directly evidenced and quality is explicitly scoped to the paper's evaluations."}, "claim": "NSA is the ACL 2025 three-branch natively trainable design; the authors report comparable-or-better quality on their evaluated tasks, and the paper's efficiency implementation is evidenced on A100/SM80 rather than Hopper or Blackwell.", "claim_id": "kernel-nsa-current-c001", "claim_type": "compatibility", "covers": ["u001", "u002"], "evidence": [{"locator": "publication metadata and abstract", "observation": "The official proceedings record identifies the paper, venue, authors, natively trainable hardware-aligned scope, and source-reported quality result.", "revision": "ACL 2025 anthology record fetched 2026-08-08", "url": "https://aclanthology.org/2025.acl-long.1126/"}, {"locator": "Section 4 experimental setup", "observation": "The efficiency study specifies eight A100 GPUs and provides no Hopper/Blackwell compatibility or performance result.", "revision": "ACL 2025 proceedings PDF fetched 2026-08-08", "url": "https://aclanthology.org/2025.acl-long.1126.pdf"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The branches are simply concatenated or averaged, selection independently scores all fine blocks, or the numeric settings are architectural constants.", "check": "Trace the equations and distinguish experimental setup from mechanism definition.", "outcome": "The output is a learned gated sum, selection explicitly reuses compression scores, and the values are reported configuration choices rather than invariants."}, "claim": "The page accurately gives NSA's learned gated sum, compression/selection/window roles, reuse of compression scores, and main experimental hyperparameters 32/16, 64 with 16 selected blocks, and 512.", "claim_id": "kernel-nsa-current-c002", "claim_type": "mechanism", "covers": ["u003", "u004", "u005", "u006", "u007", "u008"], "evidence": [{"locator": "Section 3.1, Equations 5-14, and Section 4.1 setup", "observation": "The paper defines three input-gated branch outputs, learned block compression with position encoding, selection scores derived from compression attention, a recent-token window, and the listed experimental block/stride/count/window values.", "revision": "ACL 2025 proceedings PDF fetched 2026-08-08", "url": "https://aclanthology.org/2025.acl-long.1126.pdf"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The reference accidentally hard-codes the example, or the former errors disappear on realistic tiles.", "check": "Inspect the generic loops/validation and choose one-block/equal-score cases where later iterations cannot repair normalization or skipped tokens.", "outcome": "The current function operates generically over branch count and width; the former counterexamples isolate structural errors independent of realistic tensor size."}, "claim": "The KernelWiki-derived CPU function implements the stated gated branch-vector sum, rejects missing/misaligned gates, and its ungated control produces a distinct result; the removed sparse and window sketches fail discriminating normalization and coverage cases.", "claim_id": "kernel-nsa-current-c003", "claim_type": "code", "covers": ["u009", "u010", "u011", "u016"], "evidence": [{"command": "conda run -n base python -c ''", "environment": "KernelWiki host; conda base standard-library Python", "negative_control": "The same inputs without gates differed, and a missing gate raised ValueError.", "reference": "current page's CPU semantic function and gate equation", "result": "Exited 0 and printed 'NSA gated branch semantic reference: PASS; ungated and missing-gate controls differ'; gated output was [1.0,10.0] and raw sum was [9.0,26.0]."}, {"command": "conda run -n base python -c ''", "environment": "KernelWiki host; conda base standard-library Python", "negative_control": "Stable normalized softmax and unit-stride enumeration returned the mathematically complete reference results.", "reference": "immutable baseline u005/u006 listings", "result": "Former selected-attention update returned 4.0 instead of normalized 2.0 with the wrong log-partition; former window positions were [0,2] instead of [0,1,2,3]."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "The paper mandates a three-dimensional query-block/head/batch grid or individual-token sparse gathers.", "check": "Compare the exact implementation paragraph with the stronger alternatives.", "outcome": "Only loop placement and contiguous blocks are specified; the page now preserves that evidence boundary."}, "claim": "The page accurately scopes the paper's Triton design to group-centric query loading, shared sparse indices, contiguous selected KV blocks, and grid parallelism for nearly constant query/output loops, while declining to invent an exact grid or published source listing.", "claim_id": "kernel-nsa-current-c004", "claim_type": "mechanism", "covers": ["u012", "u013", "u014", "u015", "u016"], "evidence": [{"locator": "Section 3.3 Hardware-aligned Sparse Attention", "observation": "The authors describe existing-kernel compatibility for compression/window, a specialized Triton selected-attention kernel, all query heads in a GQA group resident in SRAM, shared sparse indices, continuous KV-block loads, and query/output grid loops. They do not publish the removed code or the formerly asserted exact grid tuple.", "revision": "ACL 2025 proceedings PDF fetched 2026-08-08", "url": "https://aclanthology.org/2025.acl-long.1126.pdf"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "11.6x is a measured decode latency from the same benchmark, or dtype and software can be safely inferred from typical A100 usage.", "check": "Read the table heading/derivation and search the efficiency setup for each missing field.", "outcome": "The decode value is explicitly expected rather than timed, and the omitted fields cannot be inferred into a structured record."}, "claim": "The efficiency table correctly separates 9.0x/6.0x Figure 5 timing reports from Table 4's 11.6x expected memory-volume ratio, records the A100/GQA/dimension setup, lists missing reproducibility fields, and leaves structured performance metadata empty.", "claim_id": "kernel-nsa-current-c005", "claim_type": "performance", "covers": ["u017", "u018", "u019", "u020"], "evidence": [{"locator": "Section 4.2, Figure 5, and Table 4", "observation": "The paper reports 9.0x forward and 6.0x backward at 64K against Triton FlashAttention-2 on the stated A100 setup, and labels 11.6x expected based on 65,536 versus 5,632 memory-access tokens. The setup lists g=4, h=16, dk=192, dv=128 but omits the page-listed reproducibility fields.", "revision": "ACL 2025 proceedings PDF fetched 2026-08-08", "url": "https://aclanthology.org/2025.acl-long.1126.pdf"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "DSA is merely a production rename of NSA and can inherit all paper claims.", "check": "Compare the first-party inference graph and configuration with NSA Equations 5-14.", "outcome": "DSA has a separate learned indexer/top-k mask and lacks the two other NSA outputs, so equivalence is observably false."}, "claim": "DeepSeek-V3.2-Exp deploys the later DSA mechanism with an indexer selecting up to 2,048 token positions and directs its CUDA paths to DeepGEMM and FlashMLA; this observable graph differs from NSA's gated compression/selection/window architecture.", "claim_id": "kernel-nsa-current-c006", "claim_type": "compatibility", "covers": ["u021", "u022"], "evidence": [{"locator": "DSA release and high-performance CUDA kernels", "observation": "The first-party release names DSA and links indexer logits to DeepGEMM and sparse attention to FlashMLA.", "path": "/tmp/kernelwiki-deepseek-v32/README.md", "revision": "DeepSeek-V3.2-Exp 87e509a2e5a100d221c97df52c6e8be7835f0057"}, {"locator": "index_topk", "observation": "The released configuration sets index_topk to 2048.", "path": "/tmp/kernelwiki-deepseek-v32/inference/config_671B_v3.2.json", "revision": "same commit"}, {"locator": "Indexer.forward and MLA.forward", "observation": "The model performs learned top-k token selection and constructs an attention mask, without NSA's compression and sliding-window branch outputs.", "path": "/tmp/kernelwiki-deepseek-v32/inference/model.py", "revision": "same commit"}, {"locator": "Sparse Attention Kernels", "observation": "FlashMLA states that its sparse kernels power DSA in V3.2-Exp.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Any long-context checkpoint can adopt NSA post hoc, or CUDA Graphs are always beneficial for decode.", "check": "Compare the paper's training dependency and vendor graph bottleneck conditions with those universal claims.", "outcome": "Both depend on preconditions the page now states explicitly."}, "claim": "The applicability section correctly requires native training/model compatibility, preserves the GQA sharing mechanism, rejects a universal 32K threshold, scopes compression/Triton costs, and makes CUDA Graph use conditional on profiling a CPU-bound repeated workflow.", "claim_id": "kernel-nsa-current-c007", "claim_type": "compatibility", "covers": ["u023", "u024", "u025", "u026", "u027", "u028"], "evidence": [{"locator": "Sections 3-4, Limitations, and Appendix B", "observation": "The paper motivates native training, defines GQA-group sharing and learned compression, evaluates lengths without declaring a 32K cutoff, and notes Triton abstraction overhead versus native CUDA without publishing a CPU-launch profile.", "revision": "ACL 2025 proceedings PDF fetched 2026-08-08", "url": "https://aclanthology.org/2025.acl-long.1126.pdf"}, {"locator": "Performance Issues", "observation": "NVIDIA scopes primary graph benefit to CPU-bound work and warns that GPU-bound workflows may see minimal benefit or regress.", "revision": "NVIDIA CUDA Graphs guide fetched 2026-08-08", "url": "https://docs.nvidia.com/dl-cuda-graph/troubleshooting/performance-issues.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A link is dead, mutable where it claims a pin, or the third-party repository is presented as official.", "check": "Follow redirects, inspect host/path/commit/owner, and compare a same-host invalid control.", "outcome": "Every positive target has the stated identity/scope and the negative control fails distinctly."}, "claim": "All six cited source links resolve to the intended ACL record/PDF, pinned DeepSeek and FlashMLA revisions, NVIDIA CUDA Graph guidance, and explicitly third-party PyTorch repository.", "claim_id": "kernel-nsa-current-c008", "claim_type": "other", "covers": ["u029", "u030", "u031", "u032", "u033", "u034"], "evidence": [{"command": "curl -L -sS -o /dev/null -w '%{http_code}' ", "environment": "KernelWiki host, 2026-08-08", "negative_control": "A nonexistent sibling ACL Anthology path returned HTTP 404.", "reference": "HTTP resolution, source identity, and immutable repository revisions", "result": "All six targets returned HTTP 200; repository URLs retained their full claimed commits and the lucidrains project identity is explicitly third-party."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "The query resolves stale generated content or accepts arbitrary near-name IDs.", "check": "Search the positive output for newly unique text and compare the invalid ID.", "outcome": "Corrected unique sections are present and the unknown ID is rejected."}, "claim": "The documented query returns the corrected NSA page, while a near-name unknown page ID fails distinctly.", "claim_id": "kernel-nsa-current-c009", "claim_type": "code", "covers": ["u035"], "evidence": [{"command": "conda run -n base python scripts/get_page.py kernel-nsa", "environment": "KernelWiki worktree; conda base", "negative_control": "kernel-nsa-broken exited nonzero with 'No page found'.", "reference": "current page and regenerated indexes", "result": "Exited 0 and returned the corrected gated three-branch mechanism, source-scoped efficiency table, and NSA-versus-DSA distinction."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["inspect ACL paper equations, kernel-design section, efficiency setup/results, limitations, and native-training appendix", "inspect pinned DeepSeek-V3.2-Exp and FlashMLA first-party source for DSA identity and execution graph", "run the exact CPU gated-branch reference with ungated and missing-gate controls under conda base", "run counterexamples against the immutable baseline's two removed pseudo-kernels", "run six-link HTTP positive controls and one same-host 404 control", "regenerate indexes and run get_page positive and near-name negative controls"], "id": "kernel-nsa", "page_verdict": "verified", "path": "wiki/kernels/nsa.md", "residual_risk": "No A100/Blackwell GPU, CUDA toolkit, or official NSA implementation is available locally. Source-reported performance was not reproduced, so no structured performance tuple is retained and the page explicitly records missing benchmark fields.", "scope": "All 35 current coverage units for body hash 70483995c616c5db093c78410e006ee58b4ba3b50d6a9c3d4df495d17ee83b8b were checked against the ACL paper, immutable first-party deployment source, version-pinned FlashMLA source, vendor guidance, and discriminating host controls."} +{"claims": [{"challenge": {"alternative": "The task implicitly standardizes the former CUTLASS/TMA/TMEM kernel and declares GEMM compute-bound by category.", "check": "Enumerate normative inputs, outputs, checks, cases, and scoring, and search the pinned task for implementation requirements.", "outcome": "Only observable task behavior is normative; implementation choice and achieved bottleneck are absent and the model includes both math and memory."}, "claim": "Official records identify this as B200-targeted Kernel Challenge 2, open November 29 through December 19, 2025 and worth 20 percent; the public task defines interface, correctness, cases, and ranking but no canonical implementation, mandatory CUTLASS schedule, contestant technique, or shape-independent bottleneck.", "claim_id": "kernel-nvfp4-gemm-current-c001", "claim_type": "other", "covers": ["u001", "u002"], "evidence": [{"locator": "contest period, Kernel 2, B200 evaluation, and grand-prize weighting", "observation": "The rules establish the stated identity, dates, device, and 20 percent weight.", "revision": "official terms fetched 2026-08-08", "url": "https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf"}, {"locator": "complete task definition", "observation": "The file contains tensor prose, tests, benchmark rows, and geometric-mean ranking; it mandates no implementation. Its speed-of-light text takes the maximum of math and DRAM time per shape rather than declaring one universal bottleneck.", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6", "url": "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The contest's scale tensors implicitly include separately supplied FP32 globals, or NVFP4 and MXFP4 share the same local scale representation.", "check": "Compare the complete seven-tensor input with the version-pinned recipe and MXFP4 table.", "outcome": "No global argument is present in the task, while NVFP4 E4M3 and MXFP4 UE8M0 are explicitly distinct recipe fields."}, "claim": "The generic one-dimensional NVFP4 recipe uses signed E2M1 payloads, block-of-16 E4M3 local scales, and a per-tensor FP32 global scale, whereas MXFP4 uses block-of-32 UE8M0 local scaling; the contest ABI exposes no FP32 global-scale tensors.", "claim_id": "kernel-nvfp4-gemm-current-c002", "claim_type": "semantics", "covers": ["u003", "u004"], "evidence": [{"locator": "E2M1 format, NVFP4 two-level scaling, and MXFP4 comparison", "observation": "The vendor documentation establishes the payload values, two-level recipe, 16-value groups, and MXFP4 contrast.", "revision": "Transformer Engine 2.13", "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"}, {"locator": "custom_kernel input tuple", "observation": "The task interface lists packed A/B, logical and reordered SFA/SFB, and C, with no global FP32 scale arguments.", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6", "url": "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The reordered scales are an editorial optimization not supplied to entrants, physical FP4 shape keeps K rather than K/2, or FNUZ and FN labels agree upstream.", "check": "Trace generate_input's allocations, permutations, return tuple, template unpack, and dtype constructors at the pinned commit.", "outcome": "All seven objects are passed; packed allocations use K/2 bytes; the logical/reordered dimensions follow directly from atom_m=(32,4), atom_k=4, and K/16 scales; the two dtype suffixes visibly differ."}, "claim": "The table exactly describes the actual seven task tensors, packed and logical/reordered shapes, per-L A by B-transpose FP16 reference, tolerance, K divisibility, tile-dependent M/N constraint, and the task-description and scale-dtype inconsistencies.", "claim_id": "kernel-nvfp4-gemm-current-c003", "claim_type": "semantics", "covers": ["u005", "u006", "u007", "u008", "u009", "u010", "u011", "u012", "u013", "u014", "u015", "u016"], "evidence": [{"locator": "input_t", "observation": "The type bound contains seven tensors.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/task.py", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6"}, {"locator": "ref_kernel, generate_input, create_scale_factor_tensors, return tuple, and check_implementation", "observation": "The code constructs physical packed A/B, logical scale shapes, reordered 32-by-4 layouts, preallocated FP16 C, computes each L slice with B transposed, and uses 1e-3 tolerances. It creates torch.float8_e4m3fn scales.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/reference.py", "revision": "same commit"}, {"locator": "custom_kernel docstring and tuple unpack", "observation": "The template exposes the seven fields and labels scale types e4m3fnuz.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/template.py", "revision": "same commit"}, {"locator": "description and tests", "observation": "The prose abbreviates five members, states K divisible by 256 and tile-relative M/N divisibility, and lists ten tests consistent with the table.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/task.yml", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The helper accidentally models logical FP4 dimensions as physical bytes, or its scale layout works only for K divisible by 2048.", "check": "Compare each returned dimension with the allocations/permutation in reference.py and exercise K=256, 512, 1536, 2048, 2304, 7168, and 16384.", "outcome": "A/B use K/2, logical scales K/16, reordered rest_k K/64, and every published K succeeds, including those rejected by the former rule."}, "claim": "The Python shape helper reproduces the official task storage dimensions for valid positive shapes, enforces K divisible by 256, and correctly demonstrates that the former scale-array divisibility assertion rejects nine of ten official tests.", "claim_id": "kernel-nvfp4-gemm-current-c004", "claim_type": "code", "covers": ["u017", "u018"], "evidence": [{"command": "conda run -n base python -c ''", "environment": "KernelWiki host; conda base standard-library Python", "negative_control": "K=255 raised ValueError as required.", "reference": "current code block and pinned task.yml/reference.py", "result": "Exited 0: current shape equations matched packed/logical/reordered construction; all ten official shapes passed the K rule; nine failed the old (K/16) modulo 128 assertion."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "NVFP4 hardware consumes UE8M0 after a hidden conversion, the named schedule is the contest winner's single-GEMM kernel, TMA always needs 128 bytes, or TMEM hard-caps M at 128.", "check": "Resolve exact scale typedefs, official schedule call site, Driver API restrictions, and the official two-SM tile configuration.", "outcome": "Each alternative is contradicted by a direct type, API, or example locus; no contestant implementation source exists in the public task/leaderboard."}, "claim": "The implementation boundary correctly distinguishes NVFP4 block16 UE4M3 tcgen05 semantics from MXFP4 UE8M0, identifies the pointer-array CUTLASS schedule and current collective API without contestant attribution, states the tensor-map alignment contract rather than a universal 128-byte rule, and separates TMEM organization from logical tile limits.", "claim_id": "kernel-nvfp4-gemm-current-c005", "claim_type": "compatibility", "covers": ["u019", "u020", "u021", "u022"], "evidence": [{"locator": "block-scale valid combinations and TMEM operands", "observation": "The ISA lists mxf4nvf4 block16 with UE4M3 and describes TMEM destinations/scale addresses.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}, {"locator": "mx_float4_t and nv_float4_t", "observation": "The exact types select UE8M0 for MXFP4 and UE4M3 for NVFP4.", "revision": "NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3", "url": "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512"}, {"locator": "operand types and lines 120-164 collective/kernel construction", "observation": "The official single-GEMM example uses nv_float4 types, collective builders, kernel::GemmUniversal, and device::GemmUniversalAdapter.", "revision": "same commit", "url": "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu"}, {"locator": "MMA1SMConfig and MMA2SMConfig", "observation": "The named pointer-array schedule appears in the grouped example, which also defines a cooperative 256-by-256-by-256 logical MMA tile.", "revision": "same commit", "url": "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu"}, {"locator": "cuTensorMapEncodeTiled restrictions", "observation": "General tiled-map alignment is 16 bytes for the base and byte strides, with mode-specific additional constraints; no universal 128-byte operand rule appears.", "revision": "CUDA Driver API 13.2.1", "url": "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"}, {"locator": "TMEM organization and allocation", "observation": "The 128-lane by 512-column model describes tensor-memory addressing and allocation rather than a logical matrix-shape ceiling.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The values are observed contestant latencies or one cuBLAS benchmark summarized as a single score.", "check": "Read the table label and compare rows with the benchmarks and ranking field.", "outcome": "They are explicitly theoretical per-shape rows; the ranking is a later geometric aggregation, and no cuBLAS field exists."}, "claim": "The three rows are the task's exact B200 1.5-GHz speed-of-light values for the listed M/N/K/L cases, ranking uses their geometric mean, and the values are theoretical max-of-math-and-DRAM comparisons rather than contestant or cuBLAS measurements.", "claim_id": "kernel-nvfp4-gemm-current-c006", "claim_type": "performance", "covers": ["u023", "u024", "u025", "u026", "u027"], "evidence": [{"locator": "speed-of-light table, benchmarks, and ranking_by", "observation": "The exact three shape/value rows are labeled speed-of-light analysis based on max FP4 Tensor Core math and DRAM throughput at 1.5 GHz, and ranking_by is geom. No measured contestant or cuBLAS row is present.", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6", "url": "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The current endpoint is automatically the immutable prize cutoff table, and participant techniques can be inferred from filenames.", "check": "Compare row timestamps with the official deadline and inspect the complete API response schema/fields.", "outcome": "Post-cutoff rows preclude that inference, and filenames/scores contain no implementation evidence."}, "claim": "The August 8, 2026 public leaderboard snapshot and score-times-one-million transformation reproduce ranks, users, scores, and timestamps for current ranks 1-3 and 8-10; the first three postdate the official cutoff, so the page correctly withholds winner, code, CUTLASS, cuBLAS, trials, and variance attributions.", "claim_id": "kernel-nvfp4-gemm-current-c007", "claim_type": "performance", "covers": ["u028", "u029", "u030", "u031", "u032", "u033", "u034", "u035"], "evidence": [{"locator": "LeaderboardRankedEntry rows 1-10", "observation": "All displayed identities, ranks, floating-point scores, filenames, and UTC timestamps match after multiplying submission_score by one million. The response contains no source, method, comparison baseline, trials, or variance fields.", "revision": "fetched 2026-08-08", "url": "https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12"}, {"locator": "contest period end", "observation": "The rules end the contest on December 19, while the snapshot's current ranks 1-3 are timestamped December 20-21.", "revision": "official rules fetched 2026-08-08", "url": "https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Any 4-bit tensor can be reinterpreted losslessly, Hopper has the same native path under another mnemonic, or a particular scheduling choice is universally best.", "check": "Compare payload/scale semantics, target tables, and normative task fields, then look for a source-backed universal schedule result.", "outcome": "The format and target contracts differ, and no source provides a universal optimization ranking; the page states conditional boundaries only."}, "claim": "Applicability is correctly limited to the exact packed payload, scale layouts, transpose/output/tolerance contract and B200 target; generic four-bit formats are not interchangeable, the native path is Blackwell-specific, software emulation is not excluded, and optimization choices require exact constraint checks and profiling.", "claim_id": "kernel-nvfp4-gemm-current-c008", "claim_type": "compatibility", "covers": ["u036", "u037"], "evidence": [{"locator": "input and output contract", "observation": "The implementation must accept the exact seven tensors and return the supplied FP16 output.", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6", "url": "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py"}, {"locator": "NVFP4 recipe and MXFP4 comparison", "observation": "NVFP4 and MXFP4 differ in scale representation and group size; arbitrary INT4 or FP4 data is not the same contract.", "revision": "Transformer Engine 2.13", "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"}, {"locator": "Blackwell fifth-generation Tensor Core discussion", "observation": "Native NVFP4 compute is attributed to Blackwell hardware.", "revision": "NVIDIA technical blog fetched 2026-08-08", "url": "https://developer.nvidia.com/blog/introducing-nvfp4-for-efficient-and-accurate-low-precision-inference/"}, {"locator": "target notes", "observation": "The native block-scaled tcgen05 forms are architecture-specific; the ISA statement does not prohibit software conversion elsewhere.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A positive response is a soft 404, mutable unrelated page, or wrong commit path.", "check": "Inspect final identities and exact commit strings, and compare the same-path invalid control.", "outcome": "Pinned paths contain the named source files, vendor documents have the named sections, the API returns ranked JSON, and the invalid sibling is rejected."}, "claim": "All twelve primary-source links resolve to their intended official rules, pinned task files, public leaderboard, versioned NVIDIA documents, pinned CUTLASS examples, and tensor-map reference; the 'Query via' label correctly introduces the local retrieval command.", "claim_id": "kernel-nvfp4-gemm-current-c009", "claim_type": "other", "covers": ["u038", "u039", "u040", "u041", "u042", "u043", "u044", "u045", "u046", "u047", "u048", "u049", "u050"], "evidence": [{"command": "curl -L -sS -o /dev/null -w '%{http_code}' ", "environment": "KernelWiki host, 2026-08-08", "negative_control": "A nonexistent sibling file in the same pinned nvfp4_gemm GitHub tree returned HTTP 404.", "reference": "current Primary sources list and final target identities", "result": "Every URL returned HTTP 200 and final content matched the named rules, file, API, documentation section, or example."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "Removing --include-code leaves the command invalid or artifacts still attach implicitly.", "check": "Run the exact command and search output for former artifact filenames/section markers.", "outcome": "The page resolves and the detached bundle is absent."}, "claim": "The displayed conda-base get_page command retrieves the current page without the detached artifact bundle.", "claim_id": "kernel-nvfp4-gemm-current-c010", "claim_type": "code", "covers": ["u051"], "evidence": [{"command": "conda run -n base python scripts/get_page.py kernel-nvfp4-gemm", "environment": "KernelWiki repository; conda base, 2026-08-08", "negative_control": "The query with a nonexistent page ID exited nonzero.", "reference": "current query block and absence of artifact_dir", "result": "Exited 0, emitted the current page, and contained no full/ artifact attachment markers or PR-2139 patch name."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["re-run the current CPU-only task shape helper and invalid-K control under conda base", "fetch the public leaderboard and compare all six displayed rows field by field", "resolve all twelve primary links with a pinned same-tree 404 control", "run the current get_page command and verify detached artifact names do not appear", "inspect the official task and CUTLASS/NVIDIA source loci used for every current mechanism and compatibility statement"], "id": "kernel-nvfp4-gemm", "page_verdict": "verified", "path": "wiki/kernels/nvfp4-gemm.md", "residual_risk": "No NVIDIA GPU or CUDA toolkit is available locally, so no GPU implementation or contestant performance was reproduced. The leaderboard is explicitly a dated current snapshot, not a cutoff or prize oracle; no contestant source is public through that endpoint.", "scope": "All 51 current units for body hash c5bd73b9a4501849ca7bc91e4a94a7de62cd1bf395495ceb820d6ca9c8465a37 were checked against immutable task/CUTLASS revisions, version-pinned NVIDIA documentation, the dated public API response, and discriminating host controls."} +{"claims": [{"challenge": {"alternative": "The theoretical rows are FFMA-limited or the task proves a datatype-wide bottleneck theorem.", "check": "Compare the rows against the 8-TB/s byte model and retain only task-specific wording.", "outcome": "The DRAM model reproduces the rows closely, while neither source supports extrapolation to every implementation."}, "claim": "Problem 1 is a B200 block-scaled NVFP4 batched GEMV whose three published theoretical rows are DRAM-limited under the task model, while that task-specific result does not classify every GEMV implementation or shape.", "claim_id": "kernel-nvfp4-gemv-current-c001", "claim_type": "performance", "covers": ["u001"], "evidence": [{"locator": "description and speed-of-light analysis", "observation": "The task names the B200 GEMV operation and uses max(FFMA, DRAM). An 8-TB/s payload calculation matched all three published rows within five percent, while a deliberately broken 16-TB/s control differed by more than 45 percent.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/task.yml", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6"}, {"locator": "Why 3x Off and takeaways", "observation": "The author identifies this task and attempt-7 kernel as memory-bound while documenting implementation-dependent failed optimizations.", "path": "/tmp/kernelwiki-amandeep-nvfp4.html", "revision": "author post fetched 2026-08-08"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The extra two values are FP32 global scales or physical B really has one row.", "check": "Trace every value returned by generate_input and every value unpacked by ref_kernel.", "outcome": "The fifth and sixth inputs are reordered E4M3 scale tensors, and B/SFB are allocated with 128 rows; no scalar globals occur."}, "claim": "At commit ae679486 the actual callable receives the seven displayed physical tensors, pads logical N=1 B/SFB to 128 rows, uses only output column zero, requires K divisible by 64, constructs E4M3FN scales despite E4M3FNUZ prose, and exposes no FP32 global-scale arguments.", "claim_id": "kernel-nvfp4-gemv-current-c002", "claim_type": "code", "covers": ["u002", "u003", "u004", "u005", "u006", "u007", "u008", "u009", "u010", "u011"], "evidence": [{"locator": "input_t", "observation": "The type is a seven-tensor tuple.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/task.py", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6"}, {"locator": "generate_input, create_scale_factor_tensors, ref_kernel, and return tuple", "observation": "The exact physical payload, logical-scale, reordered-scale, and output shapes match; B is padded to 128, only res[:,0] is stored, scale dtype is float8_e4m3fn, and no global-scale scalar is returned.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/reference.py", "revision": "same commit"}, {"locator": "description and K rule", "observation": "The logical prose tuple, e4m3fnuz label, K-divisible-by-64 condition, and implementation-selected M-tile condition are explicit. A conda-base shape reconstruction matched all seven largest-row shapes, accepted K=64, and rejected K=63.", "path": "/tmp/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/task.yml", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The times are contestant measurements or 8 TB/s is a two-GPU aggregate.", "check": "Read the task's speed-of-light label and the brief's one-GPU column.", "outcome": "The task labels a theoretical analysis and the vendor scope is one GPU."}, "claim": "The task ranks the geometric mean of the three exact theoretical rows at a 1.5 GHz B200 clock; NVIDIA documents 8 TB/s HBM3e for one GB200 GPU, and the table values are model outputs rather than measured kernel results.", "claim_id": "kernel-nvfp4-gemv-current-c003", "claim_type": "performance", "covers": ["u012", "u013", "u014", "u015", "u016"], "evidence": [{"locator": "speed-of-light table and ranking_by", "observation": "The three shapes/times, 1.5 GHz assumption, and geometric-mean ranking match exactly.", "revision": "ae679486", "url": "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml"}, {"locator": "GB200 GPU memory subsystem", "observation": "The brief specifies 8 TB/s HBM3e for one GPU.", "revision": "official brief fetched 2026-08-08", "url": "https://resources.nvidia.com/en-us-blackwell-architecture/blackwell-architecture-technical-brief"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The API freezes cutoff placements or every current top row predates the deadline.", "check": "Compare its row timestamps with the exact rules deadline.", "outcome": "At least ranks 2-3 postdate the deadline, so the page correctly avoids winner language."}, "claim": "The dated public snapshot reports aggregate geometric-mean scores with the stated top-three and Yue rows; its ranks are mutable, and post-cutoff timestamps on current ranks 2-3 prevent treating it as an official prize-placement snapshot.", "claim_id": "kernel-nvfp4-gemv-current-c004", "claim_type": "performance", "covers": ["u017"], "evidence": [{"locator": "ranks 1-3 and 11 with submission_score and submission_time", "observation": "Scores and names match the page; ranks 2-3 have November 30 UTC timestamps and Yue is rank 11 at 22.392217755.", "revision": "snapshot fetched 2026-08-08", "url": "https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemv/NVIDIA?limit=100"}, {"locator": "Kernel 1 end time", "observation": "The official deadline is November 28 at 11:59 p.m. PT, before the current rank-2 and rank-3 timestamps.", "revision": "official rules fetched 2026-08-08", "url": "https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The suffix counts bytes, the move cannot target b8 lanes, or the author did not use these forms.", "check": "Apply PTX vector/type-size rules and inspect the author's displayed asm.", "outcome": "The widths and typed decomposition are exact, and both instructions appear in the author's code block; no speed theorem is inferred."}, "claim": "PTX ISA 9.0 defines the displayed packed E2M1 conversion, typed b32-to-four-byte move, and v2/v4 u64 load widths; Yue used the conversion/move in an author-reported path, while the ISA makes no universal performance promise.", "claim_id": "kernel-nvfp4-gemv-current-c005", "claim_type": "code", "covers": ["u018", "u019", "u020"], "evidence": [{"locator": "packed alternate-format cvt", "observation": "The exact cvt.rn.f16x2.e2m1x2 form is documented.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt"}, {"locator": "scalar-to-vector mov", "observation": "A b32 source may be decomposed into four byte destinations with matching total size.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov"}, {"locator": "vector loads", "observation": "v2.u64 and v4.u64 move 128 and 256 bits; the ISA specifies semantics, not relative speed.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld"}, {"locator": "Optimization 3", "observation": "The author shows mov.b32 and packed cvt in the reported vectorized PTX decode stage.", "revision": "author post fetched 2026-08-08", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The inspected code proves the techniques universally faster and the failed trials used identical mechanisms.", "check": "Separate the author's retrospective observations from the controlled changes in the author's own kernel.", "outcome": "The post itself presents shape/code-dependent successes and counterexamples, matching the page's conditional wording."}, "claim": "Amandeep reports the cache policies, wider PTX loads, and exact-K specializations observed in three solutions, while the author's own 64-bit wider-load and 80-to-64 register-cap experiments establish that these tuning effects are implementation-dependent.", "claim_id": "kernel-nvfp4-gemv-current-c006", "claim_type": "performance", "covers": ["u021"], "evidence": [{"locator": "attempts 9 and 11; What the Top Solutions Did Differently", "observation": "The post reports A no-allocate, B evict-last, v2/v4 loads, exact-K templates, and register budgets as observations; it also reports the author's wider load 16-25 percent slower and register-cap change having zero effect.", "revision": "author post fetched 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The table isolates each listed technique or 22.392 is one benchmark row.", "check": "Trace the author's headings and compare the endpoint's aggregate score field.", "outcome": "The page explicitly labels combined changes and source reporting, and the endpoint/task identify a geometric-mean score."}, "claim": "The ten-row table faithfully reports Yue's combined stage boundaries and author-reported timings, identifies 22.392 as an aggregate submitted score, and accurately notes that whole-B shared-memory loading did not improve one CuTe attempt.", "claim_id": "kernel-nvfp4-gemv-current-c007", "claim_type": "performance", "covers": ["u022", "u023", "u024", "u025", "u026", "u027", "u028", "u029", "u030", "u031", "u032", "u033"], "evidence": [{"locator": "CuTe attempt and CUDA Optimizations 1-6", "observation": "Every stage label, combined change, approximate timing, submitted score, and failed whole-B shared-memory idea matches the post.", "revision": "author post fetched 2026-08-08", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}, {"locator": "Yue row", "observation": "The aggregate score is 22.392217755 microseconds.", "revision": "snapshot fetched 2026-08-08", "url": "https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemv/NVIDIA?limit=100"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Generic FP4 weights are compatible, lower resource settings are always better, PTX is permanently target-independent, or the contest prescribes a canonical kernel.", "check": "Compare the exact recipe/task ABI, the author's counterexamples, archived PTX target notes, and task checker scope.", "outcome": "Each alternative is excluded by the cited source; the page states conditional practice rather than performance guarantees."}, "claim": "The practical guidance correctly scopes use to matching N=1 NVFP4 operands, calls for profiling before bottleneck choices, describes the compiled-entry tradeoff of retained exact-K dispatch, requires version/target revalidation for PTX, records Transformer Engine 2.13 inference support as SM 10.0+, and notes that the task mandates results rather than implementation techniques.", "claim_id": "kernel-nvfp4-gemv-current-c008", "claim_type": "compatibility", "covers": ["u034", "u035", "u036", "u037", "u038"], "evidence": [{"locator": "task contract and templates", "observation": "The task fixes N=1 input/output semantics and checker criteria but does not require cache/load/register implementation choices.", "revision": "ae679486", "url": "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml"}, {"locator": "attempts, exact-K specialization, and takeaways", "observation": "The failed tuning attempts support profiling-dependent guidance, and launch-time exact-K dispatch entails retaining referenced specialization entries.", "revision": "author post fetched 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}, {"locator": "versioning and target ISA notes", "observation": "Instruction and target support are explicitly versioned.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"locator": "NVFP4 recipe and Supported devices", "observation": "The exact recipe is required and inference support is listed as SM 10.0+.", "revision": "Transformer Engine 2.13", "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The sites return generic success for missing paths or a label points to an adjacent document.", "check": "Inspect target paths/titles and compare a same-site missing-page control.", "outcome": "The intended targets resolved while the fabricated slug was rejected."}, "claim": "All nine primary-source links resolve to the intended pinned task, rules, vendor documents, and author posts.", "claim_id": "kernel-nvfp4-gemv-current-c009", "claim_type": "other", "covers": ["u039", "u040", "u041", "u042", "u043", "u044", "u045", "u046"], "evidence": [{"command": "curl -L -A 'Mozilla/5.0' -sS -o /dev/null -w '%{http_code}' ", "environment": "KernelWiki host, network fetched 2026-08-08", "negative_control": "An invented same-site Simon article slug returned HTTP 404.", "reference": "Each link should resolve to its named primary source.", "result": "All nine linked targets returned HTTP 200 and their paths/titles matched the labels."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["conda run -n base python -c ''", "conda run -n base python -c '<8-TB/s model and 16-TB/s negative control>'", "curl current primary-source URLs with same-site 404 control"], "id": "kernel-nvfp4-gemv", "page_verdict": "verified", "path": "wiki/kernels/nvfp4-gemv.md", "residual_risk": "No B200 or CUDA toolkit is installed locally. Performance values remain explicitly author-reported or theoretical; no contestant implementation attribution is made from the leaderboard API, and no current rank is called an official prize placement.", "scope": "All 46 current units for body hash 15a1de756e23c26b7938e4b0c5e12e918492d5f204e504a428fcb1573c5bbd26 were checked against gpu-mode/reference-kernels ae679486, official rules and NVIDIA versioned documentation, the public leaderboard snapshot fetched 2026-08-08, the author posts, and host controls."} +{"claims": [{"challenge": {"alternative": "DSA is ordinary dense MLA with a post-hoc mask, the indexer score is one dot product, or the complete path becomes O(k) per decode query.", "check": "Compare equations 1-2, Figure 1, released dimensions, and the report's separate complexity statement.", "outcome": "The paper defines a weighted multi-head ReLU score, top-k KV selection under MLA/MQA, and O(Lk) core attention while retaining an O(L squared) sequence-level indexer."}, "claim": "The model section accurately defines DeepSeek-V3.2-Exp DSA's indexer and fine-grained selection, released 64x128 indexer/topk-2048 and 128-head/512+64 MLA dimensions, MQA sharing, and the separate O(Lk) core-attention versus O(L squared) indexer complexity.", "claim_id": "kernel-sparse-mla-current-c001", "claim_type": "mechanism", "covers": ["u001", "u002", "u003", "u004", "u005", "u006", "u007"], "evidence": [{"locator": "Section 1 equations 1-2/Figure 1 and Section 3 Inference Costs", "observation": "The mechanism, MLA/MQA instantiation, selected/core-attention complexity and retained indexer complexity match the page.", "path": "/tmp/kernelwiki-deepseek-v32/DeepSeek_V3_2.pdf", "revision": "DeepSeek-V3.2-Exp 87e509a2e5a100d221c97df52c6e8be7835f0057"}, {"locator": "n_heads, kv_lora_rank, qk_rope_head_dim, index_n_heads, index_head_dim, index_topk", "observation": "The exact released dimensions are 128, 512, 64, 64, 128 and 2048.", "path": "/tmp/kernelwiki-deepseek-v32/inference/config_671B_v3.2.json", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "DeepGEMM fuses top-k and FlashMLA runs the indexer; sparse indices denote blocks and prefill/decode share one ABI.", "check": "Trace the pinned indexer-logit output and both FlashMLA public signatures/reference gathers.", "outcome": "DeepGEMM emits token logits, FlashMLA receives caller-produced indices, decode uses page+offset token encoding, and prefill has a separate unbatched BF16 contract."}, "claim": "The implementation section correctly separates DeepGEMM indexer logits/top-k from FlashMLA attention and accurately records sparse decode and prefill tensor shapes, token-index encodings, invalid values, precision, outputs, page-size variability, and the SM100 precision boundary.", "claim_id": "kernel-sparse-mla-current-c002", "claim_type": "compatibility", "covers": ["u008", "u009", "u010", "u011", "u012", "u013", "u014", "u015", "u016"], "evidence": [{"locator": "sm100_fp8_mqa_logits signature, UMMA, scales and logit store", "observation": "The kernel consumes FP8 Q/K, scales and weights and emits logits; no top-k selector or FlashMLA attention is fused.", "path": "/tmp/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm100_fp8_mqa_logits.cuh", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f"}, {"locator": "flash_mla_with_kvcache and flash_mla_sparse_fwd", "observation": "The two contracts, shapes, dtypes, invalid-index rules and outputs match the table; decode dequantizes FP8 storage for BF16 attention.", "path": "/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py", "revision": "FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232"}, {"locator": "correctness cases", "observation": "Official cases include every listed page size, disproving a fixed-64 API.", "path": "/tmp/kernelwiki-flashmla-pinned/tests/test_flash_mla_sparse_decoding.py", "revision": "same commit"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "656 bytes is a generic MLA or indexer cache and its four scales are per page/block.", "check": "Inspect the mode-specific interface text, group boundaries, and released model cache declarations.", "outcome": "The exact arithmetic and roles apply only to V3-family FP8 sparse decode; indexer K/scale caches are separate and the four scales cover successive 128-value groups within one token."}, "claim": "The cache section correctly limits the 512+16+128=656-byte layout to V3-family FP8 sparse decode, identifies four per-128-value FP32 scales and attention RoPE, and distinguishes the separate indexer cache and other FlashMLA modes.", "claim_id": "kernel-sparse-mla-current-c003", "claim_type": "semantics", "covers": ["u017", "u018", "u019", "u020", "u021"], "evidence": [{"locator": "V3-family FP8+sparse layout", "observation": "The interface gives the exact element counts, bytes, scale grouping and mode restriction.", "path": "/tmp/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py", "revision": "FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232"}, {"locator": "Indexer.k_cache/k_scale_cache and MLA kv_cache/pe_cache", "observation": "The released reference declares separate indexer and attention caches.", "path": "/tmp/kernelwiki-deepseek-v32/inference/model.py", "revision": "DeepSeek-V3.2-Exp 87e509a2"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The rows are reproducible matched benchmarks, use FP8 MMA throughout, or 1450 versus 1460 proves sparse/dense equivalence.", "check": "Read every pinned README sentence, support row and API dtype, then check which benchmark fields are omitted.", "outcome": "The page transcribes each environment/maximum/precision and caveat, labels them incomplete author reports, separates dense regimes, and rejects the unmatched cross-operator comparison."}, "claim": "The performance section accurately transcribes and scopes FlashMLA's H800/B200 dense-decode and sparse prefill/decode maxima, storage-versus-MMA precision, B200 decode caveat, and NVIDIA dense-MHA values while explicitly withholding reproducible or matched-performance status.", "claim_id": "kernel-sparse-mla-current-c004", "claim_type": "performance", "covers": ["u022", "u023", "u024", "u025", "u026", "u027", "u028", "u029"], "evidence": [{"locator": "Performance, support matrix, and sparse API descriptions", "observation": "All six sparse/dense rows, the separate regimes, BF16/FP8 distinction, CUDA environments, B200 caveat, and MHA attribution match; complete shapes/trials/variance are absent.", "path": "/tmp/kernelwiki-flashmla-pinned/README.md", "revision": "FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Architecture similarity or 32K context alone establishes compatibility and speedup, and kernel-only timing is sufficient.", "check": "Apply the exact public tensor contracts and the paper's cost curve/retained indexer work to the checklist.", "outcome": "The page requires contract equality, correctness against the model/reference, complete selector-plus-attention timing, a matched dense baseline, and an observed crossover rather than a fixed threshold."}, "claim": "The evaluation procedure correctly gates compatibility on exact dimensions/layout/indices/semantics, checks selector and attention correctness including RoPE and invalid entries, and measures the entire pipeline with matched controls and no universal 32K crossover.", "claim_id": "kernel-sparse-mla-current-c005", "claim_type": "performance", "covers": ["u030", "u031", "u032", "u033", "u034", "u035", "u036", "u037", "u038"], "evidence": [{"locator": "Indexer.forward and MLA.forward", "observation": "The model reference supplies score/top-k/mask behavior and the non-interleaved indexer versus interleaved MLA RoPE distinction.", "path": "/tmp/kernelwiki-deepseek-v32/inference/model.py", "revision": "DeepSeek-V3.2-Exp 87e509a2"}, {"locator": "prefill and decode references", "observation": "The references define selected gather, attention outputs, LSE and invalid-entry behavior needed by the correctness step.", "path": "/tmp/kernelwiki-flashmla-pinned/tests/ref.py", "revision": "FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232"}, {"locator": "Inference Costs and Figure 3", "observation": "The source measures deployed end-to-end costs over a range and defines no universal 32K threshold.", "path": "/tmp/kernelwiki-deepseek-v32/DeepSeek_V3_2.pdf", "revision": "same model commit"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "DSA is the deployment name for the ACL paper's NSA implementation.", "check": "Compare DSA equations/model code with NSA's three branches and gates.", "outcome": "DSA selects token positions with one learned indexer; NSA separately computes compressed, selected-block and sliding-window branches and combines them with gates."}, "claim": "The page correctly distinguishes DeepSeek-V3.2-Exp DSA from ACL Native Sparse Attention and prevents transfer of NSA structure or A100 results.", "claim_id": "kernel-sparse-mla-current-c006", "claim_type": "compatibility", "covers": ["u039"], "evidence": [{"locator": "Section 1", "observation": "DSA is a lightning indexer plus fine-grained token selection under MLA.", "revision": "DeepSeek-V3.2-Exp 87e509a2", "url": "https://github.com/deepseek-ai/DeepSeek-V3.2-Exp/blob/87e509a2e5a100d221c97df52c6e8be7835f0057/DeepSeek_V3_2.pdf"}, {"locator": "Sections 3.1-3.3 and 4", "observation": "NSA has three gated branches and evaluates its efficiency implementation on A100.", "revision": "ACL 2025 proceedings", "url": "https://aclanthology.org/2025.acl-long.1126.pdf"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The links are mutable summaries or point to unrelated sparse-attention implementations.", "check": "Resolve owners, full revisions and exact implementation/report paths.", "outcome": "The first four links pin the DeepSeek report/model, FlashMLA repository/interface and DeepGEMM indexer source; the fifth is the official ACL proceedings paper used only for the boundary."}, "claim": "The five primary references resolve to the exact first-party DSA model/report, pinned FlashMLA implementation/interface, pinned DeepGEMM indexer kernel, and official NSA paper used by the page.", "claim_id": "kernel-sparse-mla-current-c007", "claim_type": "semantics", "covers": ["u040", "u041", "u042", "u043", "u044"], "evidence": [{"locator": "tree", "observation": "Pinned model/report source.", "revision": "87e509a2", "url": "https://github.com/deepseek-ai/DeepSeek-V3.2-Exp/tree/87e509a2e5a100d221c97df52c6e8be7835f0057"}, {"locator": "tree and interface", "observation": "Pinned sparse-attention implementation.", "revision": "71c7379", "url": "https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232"}, {"locator": "file", "observation": "Pinned indexer-logit kernel.", "revision": "891d57b4", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_mqa_logits.cuh"}, {"locator": "paper record", "observation": "Official NSA paper.", "revision": "ACL 2025", "url": "https://aclanthology.org/2025.acl-long.1126/"}], "oracle": "web-primary", "verdict": "verified"}], "commands": ["Extract and inspect pinned DeepSeek-V3.2-Exp report/config/model", "Inspect pinned FlashMLA interfaces/tests/references and source-reported performance", "Inspect pinned DeepGEMM SM100 indexer-logit source"], "id": "kernel-sparse-mla", "page_verdict": "verified", "path": "wiki/kernels/sparse-mla.md", "residual_risk": "No CUDA toolkit or GPU is available, so source code was not built or profiled. Performance remains explicitly a transcription of incomplete author reports plus a required measurement protocol.", "scope": "All 44 current units for hash 948b326db003b5db4472a9222cbd12a7ea8b28c706bb3c67abfc29cdc7846cca and the corrected Blackwell scope were challenged against pinned first-party model and kernel sources."} +{"claims": [{"challenge": {"alternative": "The FP4 change is one fused indexer kernel, gather/scatter also quantize or select top-k, or the code is a generic DSA ABI.", "check": "Trace the pinned DSA call graph across fused preparation, cache operators, DeepGEMM logit dispatch, and top-k calls, then inspect each CUDA kernel's work.", "outcome": "The source exposes distinct TensorRT-LLM-specific stages: fused FP4 concatenation/quantization, byte-copy cache movement, FP4 MQA logits, and later top-k selection."}, "claim": "The audited-scope section correctly limits the page to PR 13340 merge 897c4bff and accurately separates TensorRT-LLM's FP4 Q/K preparation, cache movement, MQA-logit, and top-k stages without claiming a generic ABI or fused quantized gather/scatter.", "claim_id": "kernel-tensorrt-llm-blackwell-indexer-current-c001", "claim_type": "mechanism", "covers": ["u001", "u002", "u003", "u004", "u005", "u006", "u007"], "evidence": [{"locator": "origin_url, upstream_sha, asset_mode, and file manifest", "observation": "The verbatim bundle ties the audited CUDA and DSA files to PR 13340 and revision 897c4bff.", "path": "artifacts/prs/tensorrt-llm/PR-13340/PROVENANCE.yaml", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}, {"locator": "fusedCatFp4Kernel", "observation": "This stage concatenates and quantizes; it does not access the paged cache, calculate MQA logits, or select top-k.", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/fusedCatFp4.cu", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}, {"locator": "_prep_q_or_k, _call_mqa_logits, _call_paged_mqa_logits, and sparse_attn_indexer", "observation": "The TensorRT-LLM integration calls preparation, cache movement, FP4 logit dispatch, and prefill/decode top-k as distinct stages under its DSA metadata and cache-manager contracts.", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/tensorrt_llm/_torch/attention_backend/sparse/dsa.py", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The FP4 row uses one scale, arbitrary head dimension, conventional round-to-nearest boundaries, or a different nibble/byte order.", "check": "Compare all constants, host assertions, scale construction, FP4 threshold table, nibble packing, and scale packing in the CUDA kernel and Torch wrapper.", "outcome": "The implementation fixes 128 BF16 inputs, four groups of 32, 64 packed data bytes, four little-endian UE8M0 exponent bytes in one int32, strict-threshold E2M1 coding, and the documented wrapper constraints."}, "claim": "The FP4 contract section accurately states the 128-value input, 64-byte packed E2M1 payload, four-byte packed scale word, 68-byte cache footprint, wrapper constraints and outputs, scale formula, representable magnitudes, and nibble order.", "claim_id": "kernel-tensorrt-llm-blackwell-indexer-current-c002", "claim_type": "semantics", "covers": ["u008", "u009", "u010", "u011", "u012", "u013", "u014"], "evidence": [{"locator": "constants, quantizeFp4E2M1, fusedCatFp4Kernel stages 1-7, and invokeFusedCatFp4", "observation": "The constants, reduction, power-of-two scale construction, FP4 table, low/high-nibble packing, little-endian scale packing, 128-head check, and stride restrictions match the page.", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/fusedCatFp4.cu", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}, {"locator": "fused_cat_fp4 validation and output allocation", "observation": "The wrapper enforces CUDA/device/dtype/dimension/stride/alignment/row/head constraints and returns int8 [M,64] plus int32 [M,1].", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/thop/fusedCatFp4Op.cpp", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}, {"locator": "DSACacheManager cache-size calculation", "observation": "At index_head_dim 128, the FP4 cache allocation uses 64 data bytes plus four scale bytes per token.", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/tensorrt_llm/_torch/attention_backend/sparse/dsa.py", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Gather/scatter require a contiguous cache, use one mapping, initialize invalid gather rows, or return numerically meaningful FP8/FP32 values in FP4 mode.", "check": "Inspect cache shape/stride handling, separate mapping loads, negative-map branches, output allocation, view casts, and FP4 reinterpretation at the logit call.", "outcome": "The kernels translate flat locations through four strides, consume separate payload/scale mappings, return before writes on either negative mapping, and transport raw bytes that FP4 code later views as int8/int32."}, "claim": "The gather/scatter section accurately specifies the strided four-dimensional cache, vector widths and FP4/FP8 byte counts, two int64 mapping arrays, whole-token negative-map skip, uninitialized skipped gather row, and historical typed-view boundary for FP4 data.", "claim_id": "kernel-tensorrt-llm-blackwell-indexer-current-c003", "claim_type": "semantics", "covers": ["u015", "u016", "u017"], "evidence": [{"locator": "flatIndexToMemoryOffset, indexerKCacheGatherUnifiedKernel, and invokeIndexerKCacheGather", "observation": "Gather handles four-dimensional byte strides, copies four bytes per thread for 64- or 128-byte payloads plus a four-byte scale, and returns before writes when either mapping is negative.", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheGather.cu", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}, {"locator": "indexerKCacheScatterUnifiedKernel and invokeIndexerKCacheScatter", "observation": "Scatter implements the inverse byte movement with the same mapping, width, scale-size, and negative-map behavior.", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheScatter.cu", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}, {"locator": "output allocation and final view-casts", "observation": "The wrapper allocates with empty, launches into uint8 buffers, then view-casts payload to float8 and scale to FP32 for the historical interface.", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/thop/IndexerKCacheGatherOp.cpp", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}, {"locator": "_call_mqa_logits FP4 branch", "observation": "The FP4 call site reinterprets gathered payload as int8 and scales as int32 before dispatch.", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/tensorrt_llm/_torch/attention_backend/sparse/dsa.py", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Testing only round-trip cache bytes proves the complete indexer, kernel-only timing supports a speedup, or skipped empty rows are valid sentinel outputs.", "check": "Map each procedure step to a distinct source contract and check whether the pinned PR supplies any benchmark record.", "outcome": "Preparation, cache movement, logits, and top-k have separate correctness boundaries; skipped gather rows need explicit handling, complete performance needs stage and end-to-end timing, and neither the source card nor pinned implementation provides a result."}, "claim": "The procedure appropriately tests FP4 boundary/layout behavior, valid and invalid cache movement, paged/non-paged logits and selected indices, profiles each stage plus the complete indexer with reproducibility fields, and makes no unsupported performance claim.", "claim_id": "kernel-tensorrt-llm-blackwell-indexer-current-c004", "claim_type": "performance", "covers": ["u018", "u019", "u020", "u021", "u022", "u023", "u024", "u025", "u026"], "evidence": [{"locator": "quantizeFp4E2M1 and fusedCatFp4Kernel", "observation": "The strict decision thresholds, saturation, row-stride loads, nibble packing, and scale packing are discriminating preparation tests.", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/fusedCatFp4.cu", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}, {"locator": "gather kernel and launch contract", "observation": "Strided cache copies, both mapping arrays, exact payload/scale widths, and negative slots define the cache test cases.", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheGather.cu", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}, {"locator": "non-paged/paged MQA logits and prefill/decode top-k dispatch", "observation": "The call graph requires separate logit and selection checks and permits separate stage timing before complete-path timing.", "path": "artifacts/prs/tensorrt-llm/PR-13340/key-files/tensorrt_llm/_torch/attention_backend/sparse/dsa.py", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}, {"locator": "Summary, Description, and changed files", "observation": "The source card describes the implementation change and records no performance measurement.", "path": "sources/prs/TensorRT-LLM/PR-13340.md", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The links are mutable summaries, point outside the audited artifact, or fail to resolve to the implementation used for the claims.", "check": "Resolve the PR link and each relative artifact path against the repository, then compare the manifest hashes and upstream paths.", "outcome": "The official PR is the source-card origin, and all four relative links resolve to files listed as verbatim upstream assets for revision 897c4bff."}, "claim": "The primary-reference links resolve to the official TensorRT-LLM PR and the pinned verbatim fused preparation, gather, scatter, and DSA integration artifacts used by the page.", "claim_id": "kernel-tensorrt-llm-blackwell-indexer-current-c005", "claim_type": "semantics", "covers": ["u027", "u028", "u029", "u030", "u031"], "evidence": [{"locator": "url, merge_sha, and artifact_dir", "observation": "The source card resolves the official PR and the repository's audited artifact directory.", "path": "sources/prs/TensorRT-LLM/PR-13340.md", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}, {"locator": "files manifest", "observation": "The manifest lists fusedCatFp4.cu, both cache kernels, and dsa.py with upstream paths and SHA-256 hashes under verbatim mode.", "path": "artifacts/prs/tensorrt-llm/PR-13340/PROVENANCE.yaml", "revision": "TensorRT-LLM PR 13340 merge 897c4bff"}], "oracle": "local-primary-source", "verdict": "verified"}], "commands": ["Inspect the pinned PR provenance and source card", "Inspect fusedCatFp4 CUDA and Torch wrapper contracts", "Inspect gather/scatter CUDA and Torch wrapper contracts", "Trace TensorRT-LLM DSA FP4 preparation, cache, logit, and top-k calls"], "id": "kernel-tensorrt-llm-blackwell-indexer", "page_verdict": "verified", "path": "wiki/kernels/tensorrt-llm-blackwell-indexer.md", "residual_risk": "No CUDA toolkit or GPU is available, so source semantics were not confirmed by build or execution. The page deliberately reports no performance result and specifies the GPU tests still required.", "scope": "All 31 current units for hash 64b746e36ef7866e93cb1c9aca98703acc5008d485a0d9c96ab31f7a1504a964 and the corrected Blackwell frontmatter claim were challenged against the pinned PR's source card, provenance, CUDA kernels, wrappers, and DSA integration."} +{"claims": [{"challenge": {"alternative": "The percentage is against theoretical peak or is a general B200 result.", "check": "Recompute the ratio using the adjacent author-table rows and inspect the stated environment.", "outcome": "It rounds to 97.96 percent only for the disclosed cuBLAS comparator and exact setup, matching the page's narrow wording."}, "claim": "The page faithfully scopes the tutorial's v6 result to M=N=K=4096 on the disclosed B200/PyTorch 2.9.1/CUDA 13 setup: 1475.93 TFLOP/s versus a 1506.74-TFLOP/s cuBLAS baseline is 97.96 percent and is not presented as portable performance.", "claim_id": "lang-cuda-cpp-current-c001", "claim_type": "performance", "covers": ["u001"], "evidence": [{"locator": "benchmark environment and table", "observation": "The author source states the shape, Modal B200, PyTorch 2.9.1/CUDA 13 environment, 1506.74 cuBLAS result, and 1475.93 v6 result; their ratio is 97.955 percent.", "path": "/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/README.md", "revision": "gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "asm volatile alone prevents both deletion and compiler memory motion, and generic pointers are accepted as shared addresses.", "check": "Inspect the guide's separate volatile, memory-clobber, and memory-space sections.", "outcome": "Volatile and memory clobbers solve different compiler problems, while shared instructions require a converted shared address."}, "claim": "The inline-PTX boundary guidance correctly states that the front end does not parse instruction text, r/l constraints select 32/64-bit integer registers, generic shared pointers need conversion, and hidden memory effects require a memory clobber.", "claim_id": "lang-cuda-cpp-current-c002", "claim_type": "code", "covers": ["u002"], "evidence": [{"locator": "Assembler Syntax, Constraints, Memory Space Conflicts, and Incorrect Optimization", "observation": "The guide states that front-end parsing excludes the template string, maps r/l to 32/64-bit registers, requires address-space-correct pointers, and prescribes a memory clobber for hidden memory effects.", "revision": "CUDA 13.0.2 Inline PTX Assembly guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/inline-ptx-assembly/index.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Only an elected lane should issue allocation because the result is scalar.", "check": "Separate destination cardinality from documented instruction issue granularity.", "outcome": "The scalar shared destination does not change the explicit warp-collective execution rule; lane-only execution is undefined."}, "claim": "The allocation prose and code implement the PTX ISA 9.0 cta_group::1 contract: all lanes of one warp execute the same collective allocation, its b32 result is written to shared memory, and the CTA synchronizes before consuming it.", "claim_id": "lang-cuda-cpp-current-c003", "claim_type": "code", "covers": ["u003", "u004"], "evidence": [{"locator": "tcgen05.alloc grammar, semantics, and issue granularity", "observation": "The exact cta_group::1 shared b32 form writes taddr to shared memory and requires identical execution by every thread in one warp.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"}, {"locator": "TMEM allocation and following synchronization", "observation": "The pinned implementation executes allocation from all lanes of warp 1 and then synchronizes before loading taddr.", "revision": "3b90ac9b", "url": "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The last value is a second immediate descriptor or must use an r constraint directly as a predicate.", "check": "Compare operand types in the normative grammar and pinned wrapper.", "outcome": "Only one instruction descriptor exists; the last operand is .pred and is correctly materialized inside PTX."}, "claim": "The unscaled f16 MMA wrapper has the legal five-operand structure: taddr, two matrix descriptors, one instruction descriptor, and an enable-input-D predicate constructed from its CUDA integer argument; an elected thread can issue this form.", "claim_id": "lang-cuda-cpp-current-c004", "claim_type": "code", "covers": ["u005", "u006"], "evidence": [{"locator": "unscaled floating-point MMA grammar and issue granularity", "observation": "The displayed mandatory operands and single-thread issue rule match; enable-input-D is a predicate operand.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}, {"locator": "tcgen05_mma_f16", "observation": "The pinned author wrapper uses one i_desc, declares a PTX predicate, converts the integer with setp.ne.b32, and emits the same five-operand MMA.", "revision": "3b90ac9b", "url": "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/common.h#L66-L78"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "One lane may load or deallocate after MMA issue because TMEM operations are synchronous.", "check": "Inspect ld completion and collective granularity separately from MMA completion.", "outcome": "ld has its own asynchronous completion wait and both ld/dealloc carry explicit collective participation rules."}, "claim": "The TMEM load and teardown prose accurately requires collective warp execution and wait::ld before register consumption, synchronization after all readers, collective matching deallocation, and deallocation before exit.", "claim_id": "lang-cuda-cpp-current-c005", "claim_type": "mechanism", "covers": ["u007"], "evidence": [{"locator": "ld issue granularity, asynchronous operation, and wait::ld", "observation": "All threads of a warp execute ld identically and wait::ld orders use of loaded registers.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"}, {"locator": "deallocation semantics and issue granularity", "observation": "Deallocation is warp-collective, must match the allocated columns/address, and every allocation must be deallocated before kernel exit.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-dealloc"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "One expect_tx barrier and parity wait fully synchronize every producer, MMA, epilogue, and stage reuse edge.", "check": "Trace who owns each stage before TMA, after TMA, after MMA, and after the epilogue.", "outcome": "Those transitions have different producers, consumers, and completion mechanisms; collapsing them loses required lifetime and reuse edges."}, "claim": "The six barrier invariants correctly describe the pinned staged pipeline: initialized and async-proxy-visible barriers, TMA full/byte completion, acquire waits, distinct tcgen completion, empty-stage ownership handoff, per-stage parity tracking, and compiler-visible memory effects.", "claim_id": "lang-cuda-cpp-current-c006", "claim_type": "mechanism", "covers": ["u008", "u009", "u010", "u011", "u012", "u013", "u014", "u015"], "evidence": [{"locator": "initialization visibility and fence.mbarrier_init", "observation": "Barrier initialization requires publication before async-proxy use.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-init"}, {"locator": "mbarrier complete_tx byte tracking", "observation": "TMA completion updates the supplied barrier transaction byte count.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"}, {"locator": "commit mbarrier completion", "observation": "tcgen05 commit separately attaches completion of prior tcgen operations to an mbarrier.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"}, {"locator": "barrier initialization plus load, compute, and epilogue loops", "observation": "The pinned kernel implements distinct TMA/MMA/mainloop/epilogue barriers, phase variables, and full/empty ownership handoffs matching the page's lifecycle.", "revision": "3b90ac9b", "url": "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The six-warps split is required by tcgen05 or every lane in the producer/compute warps issues all operations.", "check": "Inspect elected-thread predicates separately from collective instructions and compare with ISA issue rules.", "outcome": "The source mixes elected and collective operations within its chosen split, and the page does not elevate that choice to an ISA rule."}, "claim": "The stated six-warp role split exactly matches the pinned v6 implementation while being explicitly identified as implementation-specific.", "claim_id": "lang-cuda-cpp-current-c007", "claim_type": "code", "covers": ["u016"], "evidence": [{"locator": "NUM_WARPS, role branches, final synchronization and deallocation", "observation": "The source uses warp 0 for elected TMA issue, warp 1 for collective alloc/elected MMA, four epilogue warps, a final reader synchronization, and collective warp-0 deallocation.", "path": "/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v6.cu", "revision": "gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The commit URL points to the old nonexistent tcgen05 directory or a relative link escapes to an unrelated card.", "check": "Resolve the exact commit tree path and inspect local target IDs.", "outcome": "The URL names 02e_matmul_sm100 at the pinned commit; local targets identify lang-ptx and blog-tcgen05-tutorial."}, "claim": "All three primary-reference URLs and both relative related links resolve to their stated archive, pinned source tree, and local pages; broken external and local controls do not resolve.", "claim_id": "lang-cuda-cpp-current-c008", "claim_type": "other", "covers": ["u017", "u018", "u019", "u020", "u021"], "evidence": [{"command": "curl the three primary-reference URLs; test both relative local targets; repeat with fabricated external and local paths", "environment": "current KernelWiki worktree with network, 2026-08-08", "negative_control": "A fabricated CUDA archive path returned HTTP 404 and wiki/languages/not-cuda.md was absent.", "reference": "Every displayed reference should land on the named document/tree, while broken controls should fail.", "result": "The NVIDIA archive links, pinned GitHub source tree, and both intended files resolved successfully."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["conda run -n base python ratio check with broken comparator", "local and HTTP link checks with negative controls"], "id": "lang-cuda-cpp", "page_verdict": "verified", "path": "wiki/languages/cuda-cpp.md", "residual_risk": "No CUDA toolkit or Blackwell GPU is installed locally. Syntax and lifecycle guidance are source-verified, while the author-reported B200 performance has not been independently reproduced.", "scope": "All 21 current units for body hash 280b0450a7c8a8d900672fd9664f15c0dfdc7f7464b794a771ca6275e0b23ead were checked against CUDA 13.0.2 PTX ISA 9.0, the CUDA 13.0.2 inline-PTX guide, Gau Nernst's pinned 3b90ac9b source, and host controls."} +{"claims": [{"challenge": {"alternative": "The page silently describes the later 4.5.2 API or treats CuTe DSL as the only Blackwell interface.", "check": "Resolve the tag commit and compare the page's interface wording with the tagged README/source tree.", "outcome": "The commit is exact and the page explicitly places the DSL alongside C++ rather than assigning exclusivity or primacy."}, "claim": "The page accurately scopes CuTe DSL as a Python-native interface shipped alongside CUTLASS C++ in exact release v4.5.0 at commit e406c186, and warns that rolling documentation can contain later names.", "claim_id": "lang-cute-dsl-current-c001", "claim_type": "compatibility", "covers": ["u001"], "evidence": [{"locator": "DSL overview plus checked-out exact tag", "observation": "The tag identifies both C++ and Python DSL interfaces, contains the CuTe DSL packages/examples, and resolves to the stated commit.", "path": "/tmp/kernelwiki-cutlass-4.5.0/README.md", "revision": "NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "1605 establishes general CuTe-versus-C++ parity or is a fully specified reproducible shape result.", "check": "Inspect the benchmark comparator labels and scope of the maximum.", "outcome": "The author supplies neither a handwritten-C++ comparator nor one fully specified row for the maximum, matching the page's limitation."}, "claim": "The FA4 paragraph faithfully reports that FA4 is entirely CuTe DSL, that its author reports about 20-30x shorter compile times than C++ templates and up to 1605 TFLOP/s on B200 BF16, and that 1605 is not a matched handwritten-C++ comparison.", "claim_id": "lang-cute-dsl-current-c002", "claim_type": "performance", "covers": ["u002"], "evidence": [{"locator": "Language and framework plus attention benchmark sections", "observation": "The post gives the all-CuTe implementation, approximately 20-30x compile claim, and up-to-1605 B200 BF16 result; runtime comparators are FA2, Triton, Gluon, and cuDNN rather than a handwritten-C++ FA4 row.", "revision": "Tri Dao first-party FA4 post, fetched 2026-08-08", "url": "https://tridao.me/blog/2026/flash4/"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The old nonexistent SM100_MMA aliases are still required, the major-mode namespace is invalid, or the dimensions are CTA tile shapes.", "check": "Trace the exact symbols exported by nvgpu and the tuple passed to MmaF16BF16Op.", "outcome": "nvgpu exports OperandMajorMode, the operation is constructed exactly with the displayed symbols, and the source separately names its larger MMA tiler."}, "claim": "The F16/BF16 excerpt uses valid v4.5.0 Python API names and arguments for a one-CTA 128x256x16 SMEM-input operation; the two-CTA tutorial uses 256x256x16/CtaGroup.TWO and a TMEM-backed accumulator fragment.", "claim_id": "lang-cute-dsl-current-c003", "claim_type": "code", "covers": ["u003", "u004", "u005"], "evidence": [{"locator": "imports, MmaF16BF16Op construction, make_tiled_mma, make_fragment_C, and TMEM rebind", "observation": "The exact one-CTA operation shape, types, source, major modes, tiled-MMA construction, and accumulator rebind match.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py", "revision": "NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5"}, {"locator": "MmaF16BF16Op construction", "observation": "The two-CTA tutorial passes (256,256,16) and CtaGroup.TWO.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_1.py", "revision": "same tag"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The excerpt can be copied as a standalone lane-local allocator or frees before epilogue readers complete.", "check": "Follow the full official control flow from collective allocation through pipeline sync and free.", "outcome": "The page marks missing context as required and places synchronization before free, rejecting standalone/lane-local interpretation."}, "claim": "The TMEM section accurately excerpts the tagged TmemAllocator allocation/wait/retrieve/rebind and typed ld-copy construction lifecycle, while explicitly retaining partitioning, synchronization, collective, two-CTA, and free preconditions.", "claim_id": "lang-cute-dsl-current-c004", "claim_type": "code", "covers": ["u006", "u007", "u008"], "evidence": [{"locator": "TmemAllocator around lines 97-106, pointer retrieval around 174-177, copy setup around 190-200, and free around 269-270", "observation": "Every displayed operation and the stated omitted partition/copy/sync steps correspond to the complete official example.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py", "revision": "NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5"}, {"locator": "alloc_tmem and dealloc_tmem contracts", "observation": "The underlying helper validates column counts and emits architecture TMEM allocation/deallocation operations.", "path": "/tmp/kernelwiki-cutlass-4.5.0/python/CuTeDSL/cutlass/cute/arch/tmem.py", "revision": "same tag"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The invented SM100_TMA_LOAD_2D alias is needed or the multicast form can be used independently of launch topology.", "check": "Search exact tag symbols and trace the cluster layout into atom construction/launch.", "outcome": "The displayed typed operations are the actual API, and the multicast topology is constructed consistently rather than inferred automatically."}, "claim": "The TMA section uses the exact one-CTA CopyBulkTensorTileG2SOp and make_tiled_tma_atom_A construction and correctly identifies the two-CTA multicast operation plus matching cluster/participant requirement.", "claim_id": "lang-cute-dsl-current-c005", "claim_type": "code", "covers": ["u009", "u010", "u011"], "evidence": [{"locator": "TMA construction and tma_partition/cute.copy paths", "observation": "The one-CTA names, arguments, partition, copy, and pipeline use match.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py", "revision": "NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5"}, {"locator": "CopyBulkTensorTileG2SMulticastOp and cluster launch", "observation": "The two-CTA example uses the multicast operation with CtaGroup.TWO and derives TMA atoms from the cluster layout.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_1.py", "revision": "same tag"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "A barrier arrive/wait around TMA is enough and the listed tails/empty states are optional optimization details.", "check": "Trace stage ownership and termination in the official kernel.", "outcome": "The full/empty pipelines protect reuse and tails drain outstanding work; omission changes lifetimes rather than only performance."}, "claim": "The warp-specialization paragraph correctly characterizes fp16_gemm_2.py's TMA/MMA/epilogue roles and its explicit PipelineTmaUmma, PipelineUmmaAsync, TMEM, TMA-store, tail, and teardown requirements.", "claim_id": "lang-cute-dsl-current-c006", "claim_type": "mechanism", "covers": ["u012"], "evidence": [{"locator": "role dispatch, pipeline creation/states, TMEM epilogue, and final tails/free", "observation": "The complete source contains every named role and lifecycle edge and cannot be reduced to an if-warp sketch safely.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py", "revision": "NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The helpers autonomously select the full layout/swizzle and pipeline for arbitrary inputs.", "check": "Enumerate parameters and subsequent explicit allocation/pipeline calls in the example.", "outcome": "Substantial policy remains explicit, so helper-assisted is the evidence-supported scope."}, "claim": "The layout paragraph accurately distinguishes helper-assisted construction from automatic policy: v4.5.0 supplies typed algebra and make_smem_layout_a/b while callers explicitly select tilers, types, major modes, alignments, cluster shape, and pipeline policy.", "claim_id": "lang-cute-dsl-current-c007", "claim_type": "code", "covers": ["u013"], "evidence": [{"locator": "MMA construction, make_smem_layout_a/b, allocate_tensor, and pipeline creation", "observation": "Helpers derive layouts from explicit inputs while the kernel author supplies all policy items named by the page.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py", "revision": "NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The links are partial excerpts, descriptions are guessed, or short merge IDs point at mutable heads.", "check": "Combine provenance fields/hashes with full merge SHAs from the GitHub PR API and file docstrings.", "outcome": "The captures are complete and immutable, and the descriptions follow the upstream file text/PR summary."}, "claim": "The nine table links are untruncated verbatim upstream captures matching provenance SHA-256 records and purposes; the seven numeric line counts are exact.", "claim_id": "lang-cute-dsl-current-c008", "claim_type": "other", "covers": ["u014", "u015", "u016", "u017", "u018", "u019", "u020", "u021", "u022", "u023"], "evidence": [{"command": "parse all three PROVENANCE.yaml files; hash every upstream-file; count fp16_gemm_0..6 lines; compare representative files with raw merge-SHA content; alter one expected hash", "environment": "current KernelWiki worktree, conda base Python, and network, 2026-08-09", "negative_control": "Changing one expected SHA-256 to 64 zeros caused the check to reject the set.", "reference": "Provenance requires mode=verbatim and size_cap_truncated=false; table supplies seven exact counts and nine purposes.", "result": "All 11 captured upstream files matched provenance, all seven displayed counts matched [447,535,679,769,1065,919,1002], and representative remote bytes matched full PR merge SHAs."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "The page conflates the older PR paths with current rolling locations or infers features from filenames.", "check": "Compare PR merge paths/docstrings with the exact tag tree.", "outcome": "The page labels both locations and derives steps from upstream descriptions/implementation, not filenames alone."}, "claim": "PR 3106 and the v4.5.0 tag establish the stated progressive tutorial sequence and the tag's renamed cute/blackwell/tutorial/tutorial_gemm location.", "claim_id": "lang-cute-dsl-current-c009", "claim_type": "code", "covers": ["u024"], "evidence": [{"locator": "fp16_gemm_0.py through fp16_gemm_6.py docstrings and implementations", "observation": "The merged source spells out and implements each progression step.", "path": "artifacts/prs/cutlass/PR-3106/diff.patch", "revision": "NVIDIA/cutlass PR 3106 merge 772fbb264e0fe823d62dec10a44a67e1503afdfc"}, {"locator": "directory and tutorial README", "observation": "The tagged official tutorial occupies the page's stated renamed path and contains the same numbered series.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/README.md", "revision": "NVIDIA/cutlass v4.5.0 e406c186"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "GitHub redirects a missing tree to a generic page or relative artifact paths escape incorrectly.", "check": "Require 200 for exact URLs and filesystem existence/ID matches, then compare broken controls.", "outcome": "Every intended target resolved specifically and both broken controls failed."}, "claim": "The three primary URLs, nine artifact links, and three related links resolve to the intended release/source/post or local file IDs, while fabricated external and local controls fail.", "claim_id": "lang-cute-dsl-current-c010", "claim_type": "other", "covers": ["u025", "u026", "u027", "u028", "u029", "u030"], "evidence": [{"command": "HTTP-check the tag, tutorial, and FA4 URLs; resolve all artifact/related links from wiki/languages/cute-dsl.md; repeat with fabricated HTTP and local targets", "environment": "current KernelWiki worktree and network, 2026-08-09", "negative_control": "A fabricated CUTLASS commit path returned non-200 and wiki/languages/not-cute-dsl.md was absent.", "reference": "Links should land on exact NVIDIA tag/tutorial, first-party FA4 post, nine artifacts, and three named local IDs.", "result": "All intended external URLs returned HTTP 200; every artifact existed; related IDs matched hw-tcgen05-mma, kernel-flash-attention-4, and doc-cutlass-blackwell."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["NVIDIA/cutlass v4.5.0 sparse checkout at e406c186", "GitHub PR API full merge-SHA checks", "artifact hash/line-count verification with corrupted-hash control", "external and local link checks with negative controls"], "id": "lang-cute-dsl", "page_verdict": "verified", "path": "wiki/languages/cute-dsl.md", "residual_risk": "No Blackwell GPU or installed CuTe DSL/CUDA toolchain is available locally. API and lifecycle claims are exact-source verified, artifacts are byte-verified, and FA4 numbers remain author-reported rather than independently reproduced.", "scope": "All 30 current units for body hash 90ca9df7c219547d83ab1a5609935cb5d4b5fe4ebd1dee959da0fadaf0c0c5a9 were checked against CUTLASS v4.5.0, PR merge artifacts, first-party FA4 reporting, and host controls."} +{"claims": [{"challenge": {"alternative": "The snippets are complete inline-assembly functions or rolling PTX semantics can be substituted without a version check.", "check": "Inspect operand declarations and target/version notes required by the archived grammar.", "outcome": "The forms omit surrounding declarations by design, and support is explicitly versioned; the scope warning is necessary and accurate."}, "claim": "The page is explicitly scoped to archived CUDA 13.0.2 PTX ISA 9.0 and correctly warns that representative forms still require typed operands, valid descriptors, collective participation, lifetimes, completion, and target/toolchain support.", "claim_id": "lang-ptx-current-c001", "claim_type": "compatibility", "covers": ["u001"], "evidence": [{"locator": "instruction grammar, target ISA notes, and tcgen05 memory-consistency model", "observation": "The archive versions grammar/targets and places the listed operand, participation, lifetime, and completion conditions on these instruction families.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Alloc returns a column index directly in a register, ncols counts rows, MMA requires a lane-disable vector, or commit waits synchronously.", "check": "Compare the exact unscaled grammar and optional operands with the alloc destination and commit description.", "outcome": "Alloc stores a b32 taddr in shared memory and counts columns; the five-operand MMA is valid without the optional lane-disable vector; commit signals completion asynchronously."}, "claim": "The representative alloc/load-address/MMA/commit/dealloc forms and prose accurately describe a cta_group::1 unscaled F16 path: alloc writes a 32-bit taddr to shared memory and counts columns, allocation/deallocation are warp-collective, MMA D is in TMEM, commit attaches prior completion to an mbarrier, every allocation is freed, and one kernel uses one CTA-group mode.", "claim_id": "lang-ptx-current-c002", "claim_type": "code", "covers": ["u002", "u003", "u004"], "evidence": [{"locator": "alloc/dealloc grammar, issue granularity, and lifecycle", "observation": "The exact forms, shared result, column unit, warp collectivity, and required deallocation match.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"}, {"locator": "unscaled floating-point grammar, CTA-group rule, and D semantics", "observation": "The five-operand f16 form, TMEM D, group modes, and same-group-per-kernel restriction match.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}, {"locator": "mbarrier::arrive::one grammar", "observation": "Commit tracks completion of prior asynchronous tcgen05 work with the displayed barrier form.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "x1 is invalid, the waits are generic barriers, or cp copies TMEM to shared memory.", "check": "Use the shape/repetition table, wait grammar, and cp operand direction.", "outcome": "x1 is listed, waits target prior ld/st from the executing thread, and cp's destination is taddr while its source is sdesc."}, "claim": "The displayed 32x32b.x1 ld/st and 128x256b cp forms are legal; ld/st are asynchronous warp-collective TMEM-register transfers with their matching waits, cp moves SMEM to TMEM, and cp/MMA source/destination lifetimes extend through their documented completion.", "claim_id": "lang-ptx-current-c003", "claim_type": "code", "covers": ["u005", "u006", "u007"], "evidence": [{"locator": "ld/st grammar, repetition table, collectivity, and wait", "observation": "32x32b.x1 uses one register per thread, ld/st are warp-collective and asynchronous, and wait::ld/st provide completion.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"}, {"locator": "cp grammar, 128x256b example, and asynchronous semantics", "observation": "The exact cta_group::1 copy is shared-memory descriptor to TMEM and participates in tcgen05 completion/lifetime rules.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-cp"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "One CTA barrier or after_thread_sync can replace every listed mechanism.", "check": "Match each producer/consumer edge to the normative instruction that completes or orders it.", "outcome": "The ISA assigns distinct state and scope to each edge, so none is a universal substitute."}, "claim": "The four completion/ordering bullets correctly distinguish TMA mbarrier byte completion, tcgen05 commit completion, ld/st waits, and before/after_thread_sync cross-thread tcgen ordering; the fences are not completion substitutes.", "claim_id": "lang-ptx-current-c004", "claim_type": "mechanism", "covers": ["u008", "u009", "u010", "u011"], "evidence": [{"locator": "mbarrier complete_tx semantics", "observation": "TMA load completion decrements byte transaction count on its barrier.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"}, {"locator": "commit and cross-thread synchronization examples", "observation": "Commit attaches prior MMA/cp/shift completion to an mbarrier.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"}, {"locator": "before/after_thread_sync semantics", "observation": "The fences constrain tcgen05 issue around execution ordering and do not wait for async completion.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The input is a 32-bit scalar FP4, mov changes numeric representation, or the ISA guarantees lower latency.", "check": "Apply packed type widths and mov's bitwise typed-copy semantics.", "outcome": "The input is one packed byte, mov preserves bits across equal total width, and the documentation contains no timing theorem."}, "claim": "The packed conversion and typed move forms have the stated one-byte E2M1x2 source, 32-bit F16x2 result, and b32-to-four-b8 decomposition semantics, while PTX makes no universal speed promise.", "claim_id": "lang-ptx-current-c005", "claim_type": "code", "covers": ["u012", "u013", "u014"], "evidence": [{"locator": "packed alternate-format cvt", "observation": "The E2M1x2/F16x2 types and widths match.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt"}, {"locator": "scalar-to-vector move type-size rule", "observation": "Four byte destinations can receive a b32 source when declarations match; no performance comparison is specified.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The qualifier names prescribe deterministic cache behavior or v4.u64 is four bytes.", "check": "Read the cache-hint caveat and multiply vector arity by the 64-bit element width.", "outcome": "The behavior is advisory, and v4.u64 is 256 bits (32 bytes)."}, "claim": "The three ld.global forms are legal 16/32-byte vector loads; no_allocate and evict_last are nonbinding L1 eviction-priority hints, so the page correctly rejects guaranteed bypass/residency and scopes performance to the exact access pattern.", "claim_id": "lang-ptx-current-c006", "claim_type": "code", "covers": ["u015", "u016"], "evidence": [{"locator": "ld grammar, vector examples, and cache operators", "observation": "The forms and widths match; eviction-priority qualifiers are performance hints that may not be respected.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "try_cancel returns coordinates directly or accepts the work ID to steal.", "check": "Inspect request operands separately from query_cancel output operands.", "outcome": "The request has only response and barrier addresses; coordinates are decoded from a completed successful response."}, "claim": "The CLC fragment and prose accurately show an asynchronous 16-byte shared response completed on an mbarrier, followed by success and first-CTA-coordinate queries; the request cancels a not-yet-launched unit and takes no desired tile ID.", "claim_id": "lang-ptx-current-c007", "claim_type": "code", "covers": ["u017", "u018", "u019"], "evidence": [{"locator": "try_cancel grammar, response, completion, and query_cancel sections", "observation": "The displayed request/query forms, 16-byte response, wait-before-query requirement, cancellation target, and no-coordinate request semantics match.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The shorter .multicast spelling is legal, the barrier is signaled only in the issuer, or destination CTAs need not be resident.", "check": "Inspect the qualifier set and multicast destination/completion description.", "outcome": "Only .multicast::cluster is legal, completion is multicast to matching offsets, and remote shared memory must remain live."}, "claim": "The two TMA forms are valid 2D global-to-cluster-shared tile loads; multicast uses .multicast::cluster plus a 16-bit mask and sends data and barrier completion to corresponding offsets in selected live destination CTAs.", "claim_id": "lang-ptx-current-c008", "claim_type": "code", "covers": ["u020", "u021", "u022", "u023", "u024"], "evidence": [{"locator": "cluster destination and multicast grammar/semantics", "observation": "Both exact forms, tensor-map coordinates, mbarrier, qualifier, mask width, corresponding offsets, and destination lifetime requirements match.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A relative path escapes to an adjacent page or NVIDIA returns success for missing archive paths.", "check": "Resolve exact files/IDs and compare the fabricated archive path.", "outcome": "All intended targets matched and both broken controls failed."}, "claim": "The six internal links and archived PTX reference resolve to the intended pages, and a missing same-site/local control is rejected.", "claim_id": "lang-ptx-current-c009", "claim_type": "other", "covers": ["u025", "u026", "u027", "u028", "u029", "u030", "u031"], "evidence": [{"command": "test six relative wiki targets; curl archived PTX URL; test missing local target and archived URL", "environment": "current KernelWiki worktree and network, 2026-08-08", "negative_control": "A nonexistent local language page was absent and a fabricated archived PTX path returned HTTP 404.", "reference": "Each label should resolve to its named page/reference.", "result": "All six files existed with matching IDs and the archived PTX URL returned HTTP 200."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["local internal-link checks with missing-target control", "curl archived PTX URL with 404 control"], "id": "lang-ptx", "page_verdict": "verified", "path": "wiki/languages/ptx-sm100.md", "residual_risk": "No CUDA assembler or Blackwell GPU is installed locally. The page makes no numeric performance claim; representative syntax and semantics come directly from the archived ISA, while full inline-assembly integration remains caller-specific.", "scope": "All 31 current units for body hash 4bbed1d5aa5094a031fdbe1e8349f5079263caf9887a77721a5ace82c565fad5 were checked against CUDA 13.0.2 PTX ISA 9.0 and host link controls."} +{"claims": [{"challenge": {"alternative": "The v3.3 operations are unused scaffolding, and native lowering actually begins in v3.6.", "check": "Compare the exact v3.2 and v3.3 trees and inspect the v3.3 conversion test expectations.", "outcome": "The earlier tree lacks the backend symbols and the v3.3 tests require concrete TCGen5/TMEM output, discriminating real lowering from scaffolding."}, "claim": "The checked native Blackwell backend boundary is between Triton v3.2.0 and v3.3.0: the corresponding TCGen5/TMEM/MMAv5 symbols are absent at the former tag, while the latter contains operations, allocation/lowering passes, and conversion tests for concrete output.", "claim_id": "lang-triton-current-c001", "claim_type": "compatibility", "covers": ["u001", "u002", "u003", "u035"], "evidence": [{"locator": "TCGen5MMA, TCGen5MMAScaled, and TMEM operation definitions", "observation": "The tagged dialect defines native TCGen5 MMA/scaled-MMA and tensor-memory operations.", "revision": "Triton v3.3.0 commit 819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c", "url": "https://github.com/triton-lang/triton/blob/819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c/include/triton/Dialect/TritonNvidiaGPU/IR/TritonNvidiaGPUOps.td"}, {"locator": "FileCheck expectations for tcgen05 MMA, commit, TMEM, scaled MMA, and cta_group::2", "observation": "The conversion suite requires concrete Blackwell-native output rather than merely declaring unused operations.", "revision": "Triton v3.3.0 commit 819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c", "url": "https://github.com/triton-lang/triton/blob/819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c/test/Conversion/tritongpu_to_llvm_blackwell.mlir"}, {"locator": "TritonNvidiaGPU operations, MMAv5 lowering, tensor-memory allocation, and Blackwell tests", "observation": "The corresponding backend enters in this tag interval; exact-tree search finds no matching backend symbols at v3.2.0.", "revision": "v3.2.0 commit 9641643da6c52000c807b5eeed05edaec4402a67 to v3.3.0 commit 819e9c8c29ad2ae96cbd93a1d3b8a3a0f4c8f09c", "url": "https://github.com/triton-lang/triton/compare/v3.2.0...v3.3.0"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The tutorials and native surfaces first appear only in v3.6, and v3.7.1 adds another feature milestone.", "check": "Open both tutorials at the exact v3.5 tag and compare the v3.6, v3.7, and v3.7.1 release descriptions.", "outcome": "The tutorials exist at v3.5, the v3.6/v3.7 notes describe successive expansions, and v3.7.1 describes only regression fixes."}, "claim": "By v3.5.0 Triton ships explicit Gluon TCGen5/TMEM and Blackwell block-scaled matmul tutorials; v3.6.0 incrementally broadens layouts, copies, warp specialization, and initial multi-CTA/2-CTA Gluon support, while v3.7.0 continues 2-CTA/multicast/TMA work and v3.7.1 is a regression-only patch.", "claim_id": "lang-triton-current-c002", "claim_type": "compatibility", "covers": ["u004", "u005", "u006", "u008", "u009", "u010", "u011"], "evidence": [{"locator": "tutorial body", "observation": "The pinned tutorial allocates, loads, and stores TMEM and invokes TCGen5 MMA through Gluon.", "revision": "Triton v3.5.0 commit c3c476f357f1e9768ea4e45aa5c17528449ab9ef", "url": "https://github.com/triton-lang/triton/blob/c3c476f357f1e9768ea4e45aa5c17528449ab9ef/python/tutorials/gluon/06-tcgen05.py"}, {"locator": "Blackwell block-scaled matmul tutorial", "observation": "The v3.5 tag already contains the tl.dot_scaled Blackwell tutorial.", "revision": "Triton v3.5.0 commit c3c476f357f1e9768ea4e45aa5c17528449ab9ef", "url": "https://github.com/triton-lang/triton/blob/c3c476f357f1e9768ea4e45aa5c17528449ab9ef/python/tutorials/10-block-scaled-matmul.py"}, {"locator": "Blackwell, TMEM, tcgen05, warp-specialization, and Gluon release-note entries", "observation": "The release enumerates generalization and initial multi-CTA/2-CTA work on top of earlier capabilities.", "revision": "Triton v3.6.0 commit 7c56a5e40f7fd928dfd5c72902d5def0097db73a", "url": "https://github.com/triton-lang/triton/releases/tag/v3.6.0"}, {"locator": "2CTA, multicast, and TMA release-note entries", "observation": "The next minor release contains further end-to-end Blackwell cluster-path work.", "revision": "Triton v3.7.0 commit 5f3f125e8f63c24613f1f73b937442864f263f94", "url": "https://github.com/triton-lang/triton/releases/tag/v3.7.0"}, {"locator": "release description", "observation": "The patch release lists two regression fixes and explicitly advertises no new API or feature.", "revision": "Triton v3.7.1 commit f797708", "url": "https://github.com/triton-lang/triton/releases/tag/v3.7.1"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The local architecture field or the presence of tl.dot is enough to demonstrate SM100 TCGen5 emission.", "check": "Inspect the primary PR motivation, exact code guards, and any emitted-IR/PTX evidence.", "outcome": "The primary source names SM120 and supplies no target guard or lowering witness, so the page retains only the narrower implementation claim."}, "claim": "vLLM PR 34597 adds FP8 KV-cache handling to a Triton MLA decode backend whose pinned kernel contains the displayed verbatim FP8-rescaling/tl.dot excerpt; the PR names SM120, while neither the PR nor code proves an SM100-only path or a particular TCGen5/TMEM lowering.", "claim_id": "lang-triton-current-c003", "claim_type": "code", "covers": ["u012", "u026", "u027", "u033", "u034"], "evidence": [{"locator": "PR summary and changed-files list", "observation": "The PR adds FP8 cache support and identifies the Triton backend as the MLA option on SM120; it makes no SM100-only or instruction-lowering claim.", "revision": "merged commit a1257fd1", "url": "https://github.com/vllm-project/vllm/pull/34597"}, {"locator": "kernel definitions and tl.dot call sites", "observation": "The code contains Triton JIT kernels and tl.dot but no architecture guard, Triton-version requirement, warp_specialize, TCGen5/TMEM name, or emitted PTX.", "revision": "commit a1257fd1", "url": "https://github.com/vllm-project/vllm/blob/a1257fd1/vllm/v1/attention/ops/triton_decode_attention.py"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The values are historical Triton 3.5 results or belong to adjacent model/workload columns.", "check": "Read each author row under the labeled columns and inspect the page for version and language metadata.", "outcome": "The values map directly to the author rows, and no claimed Triton version/language scope is present."}, "claim": "On 2026-08-08 the live FlashInfer-Bench author leaderboard shows 0.628x/73.1% for Gemini 2.5 Pro, 0.467x/92.3% for GPT-5, and 0.456x/73.1% for Claude Opus 4.1, each across 660 workloads; it gives no Triton-release or Triton-only scope for those rows.", "claim_id": "lang-triton-current-c004", "claim_type": "performance", "covers": ["u019", "u020", "u021", "u022", "u023"], "evidence": [{"locator": "Author / Avg Speedup / % Resolved rows for the three named authors", "observation": "The exact values and 660-workload denominators appear in the labeled rows; the page exposes no Triton version or Triton-only filter.", "revision": "live leaderboard retrieved 2026-08-08", "url": "https://bench.flashinfer.ai/"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The percentage compares different metrics, or the routing change applies to every GPU and workload.", "check": "Use the total-token-throughput field in both PR 5390 result blocks and the architecture condition in PR 21595.", "outcome": "The quotient is 26.98%, and the routing change is explicitly SM100-scoped."}, "claim": "SGLang PR 5390 reports a scoped 10,447.34-versus-8,227.35 total-token-throughput comparison (26.98%), and PR 21595 changes the SM100 datacenter multimodal-attention default from triton_attn to FA4.", "claim_id": "lang-triton-current-c005", "claim_type": "performance", "covers": ["u017", "u018"], "evidence": [{"locator": "DeepSeek-R1 benchmark commands and CUTLASS/Triton result blocks", "observation": "The PR records the two totals and the stated 3,000-prompt, TP8/DP8, float16, 1,000-input/1,000-output-token scope.", "revision": "merged commit 84810da4ae424d15ec06a3e32a03a839dd50a644", "url": "https://github.com/sgl-project/sglang/pull/5390"}, {"locator": "summary and SM100-default change", "observation": "The PR explicitly scopes the default switch to Blackwell SM100 datacenter multimodal attention.", "revision": "merged commit 87a27682", "url": "https://github.com/sgl-project/sglang/pull/21595"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The link points to a stale summary or the target lacks capture status.", "check": "Resolve the exact relative path and inspect every PR entry for the captured key, alongside a broken-target control.", "outcome": "The exact canonical target resolves and all entries expose the flag; the negative control fails."}, "claim": "The page's triton-universe link resolves to the canonical ledger containing the tracked PR entries and their captured flags without duplicating its mutable count.", "claim_id": "lang-triton-current-c006", "claim_type": "other", "covers": ["u032"], "evidence": [{"command": "conda run -n base python -c ", "environment": "Conda base, Python 3.13.14, repository filesystem", "negative_control": "../../data/triton-universe-missing.yaml does not resolve.", "reference": "The relative link must resolve and the YAML must expose a PR collection with per-entry captured flags.", "result": "The link resolves; the canonical YAML parses with 267 PR entries, each carrying a captured flag, while the wiki text states no copied count."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "The PR adds a target-specific Triton kernel or demonstrates a TCGen5 lowering.", "check": "Inspect the exact one-file diff and changed-files list.", "outcome": "Only dispatch logic changes; no Triton kernel, IR, or PTX is added."}, "claim": "vLLM PR 29339 is an SM90/SM100 MXFP4 triton_kernels dispatch gate rather than a new Triton kernel or compiler-lowering implementation.", "claim_id": "lang-triton-current-c007", "claim_type": "code", "covers": ["u013"], "evidence": [{"locator": "one-file dispatch diff", "observation": "The PR gates triton_kernels MXFP4 use to SM90 and SM100 and does not add a Triton kernel.", "revision": "merged commit c17610e2", "url": "https://github.com/vllm-project/vllm/pull/29339"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The page swaps which SGLang artifact contains matrix products or either file contains an emitted TCGen5 witness.", "check": "Inspect each exact file for decorators, tl.dot, load/store operations, TCGen5 names, and PTX.", "outcome": "The GDN file is a load/store rearrangement, the attention file contains the dot products, and neither provides emitted TCGen5 PTX."}, "claim": "SGLang PR 21019 is a load/store GatedDeltaNet projection rearrangement without tl.dot, while PR 22079 is extend-attention code with tl.dot but no emitted-PTX witness.", "claim_id": "lang-triton-current-c011", "claim_type": "code", "covers": ["u014", "u015"], "evidence": [{"locator": "Triton kernel load/store sites and whole-file tl.dot search", "observation": "The verbatim kernel rearranges projection data with loads/stores and contains no tl.dot.", "path": "artifacts/prs/sglang/PR-21019/key-files/python/sglang/jit_kernel/triton/gdn_fused_proj.py", "revision": "upstream SHA 5bdc07d9, sha256 c4ef85167eb11065721e76dd5d63d6f346f309f84822578dd4a3874249acef89"}, {"locator": "Triton extend-attention kernels and tl.dot sites", "observation": "The verbatim code contains QK and probability/value dot products but no TCGen5 PTX dump.", "path": "artifacts/prs/sglang/PR-22079/key-files/python/sglang/srt/layers/attention/triton_ops/extend_attention.py", "revision": "upstream SHA 5638d40f, sha256 d27789c32e180d725ed867f4f6cd05b09a3c47bfe37794ff60e150d000573edd"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Graphs do not address CPU launch cost, or GatedDeltaNet opts out of vLLM's graph path.", "check": "Inspect the CUDA guide and the exact vLLM compilation default and GDN metadata builder.", "outcome": "The primary documentation connects graphs to launch cost, and the pinned backend explicitly implements the capture path."}, "claim": "CUDA Graphs reduce repeated CPU launch costs for short-kernel workflows; at vLLM commit a1257fd1 the v1 default is FULL_AND_PIECEWISE and the GatedDeltaNet backend supports uniform-batch graphs and decode full capture.", "claim_id": "lang-triton-current-c008", "claim_type": "performance", "covers": ["u024"], "evidence": [{"locator": "4.2 CUDA Graphs introduction", "observation": "The guide identifies host setup as significant for short kernels and explains that graphs reduce CPU launch costs by preparing work in advance.", "revision": "CUDA Programming Guide 13.2", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cuda-graphs.html"}, {"locator": "CUDAGraphMode and CompilationConfig.cudagraph_mode", "observation": "FULL_AND_PIECEWISE is documented as the v1 default.", "revision": "vLLM commit a1257fd1", "url": "https://github.com/vllm-project/vllm/blob/a1257fd1/vllm/config/compilation.py"}, {"locator": "graph-support methods and build_for_cudagraph_capture", "observation": "The GDN backend advertises uniform-batch graph support and implements decode-only full graph capture.", "revision": "vLLM commit a1257fd1", "url": "https://github.com/vllm-project/vllm/blob/a1257fd1/vllm/v1/attention/backends/gdn_attn.py"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "At least one link is a derived summary, stale target, hash mismatch, or mislabeled code role.", "check": "Resolve every link, compare it with the owning provenance entry, hash the file, inspect relevant symbols, and test a broken path.", "outcome": "Every real target and hash succeeds with asset_mode verbatim; the deliberately broken target fails."}, "claim": "All six linked example files resolve to provenance-pinned verbatim upstream assets and their role descriptions match the code at the recorded merge revisions.", "claim_id": "lang-triton-current-c009", "claim_type": "code", "covers": ["u025", "u026", "u027", "u028", "u029", "u030", "u031"], "evidence": [{"command": "conda run -n base python -c ", "environment": "Conda base, Python 3.13.14, repository filesystem after remediation", "negative_control": "artifacts/prs/vllm/PR-00000/key-files/missing.py does not resolve.", "reference": "Each link must exist, match a verbatim provenance entry, and contain the named kernel or wrapper behavior.", "result": "All six links resolve; all five bundles mark the assets verbatim and their hashes match. Inspection finds the stated MLA kernel/backend, format conversion, normalization, GDN projection, and extend-attention roles."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "Every tl.dot in a post-boundary downstream kernel necessarily demonstrates TCGen5 emission.", "check": "Search exact downstream source for target guards, backend names, IR, or PTX and compare to a known matching upstream control.", "outcome": "The downstream files contain none of the discriminating witnesses while the controls do, so the page does not infer an unobserved lowering."}, "claim": "The checked compiler sources prove native TCGen5/TMEM paths exist, but the cited downstream kernels do not prove exact instruction selection for arbitrary tl.dot shapes.", "claim_id": "lang-triton-current-c010", "claim_type": "compatibility", "covers": ["u007", "u016"], "evidence": [{"command": "rg -n -i 'tcgen05|tmem|warp_specialize' artifacts/prs/vllm/PR-34597 artifacts/prs/sglang/PR-22079 artifacts/prs/sglang/PR-21019", "environment": "GNU ripgrep over provenance-pinned downstream artifacts", "negative_control": "The identical terms match the pinned Triton v3.5.0 TCGen5 tutorial and the v3.3.0 compiler/test sources.", "reference": "An explicit lowering witness would name the backend mechanism or include emitted IR/PTX; a known upstream positive control must match the same search.", "result": "No explicit lowering witness occurs in the three downstream bundles."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["git tag-tree comparison for Triton v3.2.0 through v3.7.1", "conda run -n base python scripts/validate.py", "conda run -n base python /users/Master/.codex/skills/kernel-wiki-verifier/scripts/scan_wiki.py . --output verification/current-queue.jsonl"], "id": "lang-triton", "page_verdict": "verified", "path": "wiki/languages/triton-blackwell.md", "residual_risk": "No exact lowering is asserted for a downstream tl.dot kernel without matching IR/PTX; performance observations remain limited to their recorded source scopes.", "scope": "All 35 current units for body hash ea6b2fb7068dfa7b8a5d31fd9324937075bcbd313e86292a970f031a005a821c; Triton v3.2.0-v3.7.1 compiler/release boundary; exact downstream merge revisions; live FlashInfer-Bench rows retrieved 2026-08-08; local artifacts and links after remediation."} +{"claims": [{"challenge": {"alternative": "128 is a warpgroup total or WGMMA completes synchronously when issued.", "check": "Inspect the per-thread fragment wording and async-proxy group operations.", "outcome": "The count is explicitly per thread and dependent access requires the documented async lifecycle."}, "claim": "The Hopper description correctly gives the per-thread FP32 D fragment as N/2 registers for wgmma.m64nNk16, hence 128 at N=256, and identifies WGMMA asynchronous fence/commit/wait before dependent use.", "claim_id": "migration-register-to-tmem-current-c001", "claim_type": "mechanism", "covers": ["u001"], "evidence": [{"locator": "f32 D fragment and WGMMA async proxy operations", "observation": "The fragment is N/2 f32 registers per thread and WGMMA defines fence, commit_group, and wait_group ordering/completion operations.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "TMEM replaces the entire register file or guarantees a larger resident tile/CTA count.", "check": "Separate the MMA D operand location from all other kernel live state and occupancy resources.", "outcome": "Only D residency changes categorically; the page explicitly requires whole-kernel compiled evidence for the rest."}, "claim": "The Blackwell description accurately gives TMEM's 128-by-512-by-32-bit CTA-visible structure and tcgen05 D residency while narrowing the benefit to removal of resident D registers, not elimination of all register, occupancy, spill, or tile constraints.", "claim_id": "migration-register-to-tmem-current-c002", "claim_type": "mechanism", "covers": ["u002", "u003"], "evidence": [{"locator": "TMEM organization and tcgen05 D", "observation": "The dimensions/cell width and distinct TMEM D destination match; 128*512*4 bytes is 256 KiB.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"}, {"locator": "occupancy bullets", "observation": "Compute capability 10.0 retains 64K 32-bit registers/SM and a 255-register/thread maximum, so the limitations are correctly preserved.", "revision": "CUDA 13.0.2 Blackwell Tuning Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "An instruction tile is always a CTA tile or two buffers can overlap the same TMEM columns safely.", "check": "Inspect descriptor-defined instruction shapes and TMEM address/lifetime rules.", "outcome": "The page explicitly rejects the tile equivalence and requires disjoint live regions."}, "claim": "All seven storage/lifecycle table rows correctly contrast WGMMA register D and scale-d/group completion with tcgen05 TMEM D, enable-input-d, commit/mbarrier, collective ld/wait, explicit deallocation, and disjoint storage for simultaneous live outputs.", "claim_id": "migration-register-to-tmem-current-c003", "claim_type": "mechanism", "covers": ["u004", "u005", "u006", "u007", "u008", "u009", "u010", "u011"], "evidence": [{"locator": "WGMMA D, scale-d, commit/wait", "observation": "Register D and false scale-d D=A*B semantics match.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-multiply-accumulate-instructions-wgmma-mma"}, {"locator": "tcgen05 D, enable-input-d, commit, ld, waits, allocation", "observation": "TMEM destination, false-predicate first write, completion/readback, and cleanup rows match the separate instruction contracts.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "One elected lane can allocate because only one shared taddr is returned, or peer allocation can launch the second CTA implicitly.", "check": "Read mandatory sync/aligned warp granularity and cta_group::2 peer activity conditions.", "outcome": "All lanes of the designated warp participate and the peer must already be launched/live."}, "claim": "Lifecycle steps 1-2 accurately state one CTA-group mode, power-of-two column allocation in [32,512] across all 128 lanes, whole-warp cta_group::1 execution, shared taddr publication, and peer-liveness requirements for cta_group::2.", "claim_id": "migration-register-to-tmem-current-c004", "claim_type": "mechanism", "covers": ["u012", "u013"], "evidence": [{"locator": "allocation range/unit, issue granularity, shared result, peer and group rules", "observation": "Every stated allocation and participation condition matches the normative section.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Zero-storing all TMEM is mandatory or __syncthreads after a fence completes MMA.", "check": "Use false enable-input-d semantics and the distinct commit/fence specifications.", "outcome": "The predicate supplies first-write semantics and only the completion path licenses consumers/reuse."}, "claim": "Lifecycle steps 3-5 correctly distinguish barrier publication, false enable-input-d first-write semantics, valid single-thread MMA issue and descriptors, and tcgen05 commit/mbarrier completion from fence-only ordering.", "claim_id": "migration-register-to-tmem-current-c005", "claim_type": "mechanism", "covers": ["u014", "u015", "u016"], "evidence": [{"locator": "MMA grammar, issue granularity, D predicate and lifetimes", "observation": "The descriptor/predicate/single-thread and D=A*B conditions match.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}, {"locator": "commit versus fence completion/ordering", "observation": "Commit attaches completion to an mbarrier, while fences order execution without completing the asynchronous MMA.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Lane 0 may deallocate after issuing loads because wait::ld is warp-local and kernel exit releases the rest.", "check": "Combine load lifetime with deallocation issue granularity and explicit-exit rule.", "outcome": "All readers must finish and all lanes of the designated warp must execute deallocation; exit is not implicit cleanup."}, "claim": "Lifecycle steps 6-7 correctly require collective legal tcgen05.ld plus wait::ld before register use, reader completion before overwrite/free, matching collective deallocation, and coverage of every kernel exit path.", "claim_id": "migration-register-to-tmem-current-c006", "claim_type": "mechanism", "covers": ["u017", "u018"], "evidence": [{"locator": "ld collectivity, asynchronous completion, wait and lifetimes", "observation": "The readback requirements match.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"}, {"locator": "matching deallocation and before-exit obligation", "observation": "All allocated TMEM must be deallocated collectively with matching address/count/group before exit.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The forms compose into a runnable sequence as displayed or the MMA predicate may be a literal integer.", "check": "Inspect required operand declarations/types and asynchronous/collective prerequisites.", "outcome": "The page calls them representative forms only and explicitly denies standalone status."}, "claim": "The six representative PTX forms are legal PTX ISA 9.0 forms and the surrounding warning accurately identifies the declarations, descriptors, participation, barrier, and inline-assembly context they omit.", "claim_id": "migration-register-to-tmem-current-c007", "claim_type": "code", "covers": ["u019", "u020"], "evidence": [{"locator": "alloc, unscaled f16 MMA, commit, 32x32b.x1 ld, wait::ld, and dealloc grammars", "observation": "Every exact form is listed with the contextual constraints named by the page.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Disjoint addresses alone make overlap safe or FA3 only overlaps TMA with MMA.", "check": "Trace completion/reuse edges and read FA3's explicit matmul/softmax interleaving contribution.", "outcome": "Disjoint storage does not supply completion, and FA3 explicitly overlaps tensor-core and non-matmul work on Hopper."}, "claim": "The overlap section gives the necessary compute-complete, read-complete, and nonaliasing stage-ownership edges and correctly rejects TMEM as a prerequisite for all tensor/non-matmul overlap by citing Hopper FA3.", "claim_id": "migration-register-to-tmem-current-c008", "claim_type": "mechanism", "covers": ["u021", "u022", "u023", "u024", "u025"], "evidence": [{"locator": "completion and operand lifetime rules", "observation": "The three edges follow directly from asynchronous completion, TMEM read lifetime, and safe destination reuse.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"}, {"locator": "Hopper warp specialization and matmul/softmax interleaving", "observation": "FA3 is a primary counterexample to the claim that register accumulators make overlap impossible.", "revision": "FlashAttention-3 paper", "url": "https://arxiv.org/abs/2407.08608"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The result establishes a fixed register count or migration-only speedup for every FA workload.", "check": "Inspect what the paper/post actually report versus omitted compiler resource rows and ablations.", "outcome": "The page explicitly limits the number to a source-reported maximum and draws no universal resource conclusion."}, "claim": "The FA4 case study accurately reports TMEM score/output accumulators and specialized MMA/softmax/correction roles, software-selected polynomial exp2 work, and an author-reported up-to-1605-TFLOP/s B200 BF16 maximum labeled 71 percent without deriving universal resource claims.", "claim_id": "migration-register-to-tmem-current-c009", "claim_type": "performance", "covers": ["u026"], "evidence": [{"locator": "forward architecture and TMEM mapping", "observation": "The accumulator placement and warp roles match.", "revision": "FlashAttention-4 paper v1", "url": "https://arxiv.org/abs/2603.05451v1"}, {"locator": "software exp and performance sections", "observation": "The software-selected FMA polynomial and up-to-1605/71-percent author result match the scoped wording.", "revision": "Tri Dao first-party post, fetched 2026-08-08", "url": "https://tridao.me/blog/2026/flash4/"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "ISA legality plus TMEM capacity is sufficient to choose the fastest tile without compiling or timing.", "check": "Compare architecture legality with documented multi-resource occupancy and absence of timing guarantees.", "outcome": "Legality cannot establish allocation, residency, spills, or workload performance; the measurement workflow is necessary."}, "claim": "The retuning workflow correctly requires concrete compiler resource data, CUDA occupancy calculation, legal-shape and pipeline sweeps, controlled benchmarks, and generated-code inspection while rejecting a fixed 420-cycle tcgen05.ld theorem.", "claim_id": "migration-register-to-tmem-current-c010", "claim_type": "performance", "covers": ["u027", "u028", "u029", "u030", "u031", "u032"], "evidence": [{"locator": "occupancy APIs and resource dependence", "observation": "Occupancy requires actual launch/resource inputs rather than tile inference.", "revision": "CUDA 13.0.2 Programming Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"}, {"locator": "ld shape/semantics", "observation": "The ISA defines legal collective mappings/completion but no universal latency.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The bullets are merely conservative style preferences and lane-0/fence/implicit-exit forms are still legal shortcuts.", "check": "Read the mandatory aligned collective, asynchronous completion, and explicit deallocation language.", "outcome": "Those shortcuts violate normative behavior; only the performance-tuning warning remains empirical as stated."}, "claim": "All six common-error bullets follow from the verified TMEM instruction-only address space, collective issue rules, distinct fence/completion semantics, enable-input-d first-write behavior, finite registers, and mandatory explicit deallocation.", "claim_id": "migration-register-to-tmem-current-c011", "claim_type": "mechanism", "covers": ["u033", "u034", "u035", "u036", "u037", "u038"], "evidence": [{"locator": "addressing, allocation, MMA, fence/commit, ld/st/wait and deallocation", "observation": "The dedicated address model and each lifecycle rule directly support the warnings.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"}, {"locator": "finite register resources", "observation": "Moving D does not establish whole-kernel no-spill behavior.", "revision": "CUDA 13.0.2 Blackwell Tuning Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Anchor redirects mask a missing document or a similarly named relative page is used.", "check": "Require exact page HTTP success and inspect local frontmatter IDs against link intent.", "outcome": "Every exact document/page resolved and both broken controls failed."}, "claim": "All eight primary URLs and three related local links resolve to the named archived sections, papers, and pages; fabricated external and local controls fail.", "claim_id": "migration-register-to-tmem-current-c012", "claim_type": "other", "covers": ["u039", "u040", "u041", "u042", "u043", "u044", "u045", "u046", "u047", "u048", "u049"], "evidence": [{"command": "HTTP-check eight primary URLs; resolve three relative links and inspect target IDs; repeat with fabricated HTTP and local paths", "environment": "current KernelWiki worktree and network, 2026-08-09", "negative_control": "A fabricated CUDA archive page returned HTTP 404 and wiki/migration/not-register-to-tmem.md was absent.", "reference": "Targets should resolve to archived CUDA 13.0.2 sections, FA3/FA4 papers, hw-tmem, hw-tcgen05-mma, and pattern-register-pressure.", "result": "All eight URLs returned HTTP 200 and all three files existed with expected IDs."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["conda-base arithmetic checks for fragment and capacity formulas", "archived primary-source inspection", "external/local link checks with negative controls"], "id": "migration-register-to-tmem", "page_verdict": "verified", "path": "wiki/migration/register-to-tmem.md", "residual_risk": "No Hopper/Blackwell compiler or GPU is installed locally. The page intentionally contains no concrete occupancy or migration-speed claim; users must obtain compiler resources and target measurements for their kernel.", "scope": "All 49 current units for body hash 7f0bb2f02e5a9f1f50c0c9003dc0baeeee3685876e3897726abbd12a14ae611a were checked against CUDA 13.0.2 PTX ISA 9.0, archived tuning guidance, FA3/FA4 primary sources, and host controls."} +{"claims": [{"challenge": {"alternative": "The rewrite still assumes WGMMA register A or tcgen05 SMEM A universally.", "check": "Compare the table and following warning with all SS/RS and SMEM/TMEM A grammar alternatives.", "outcome": "The current page states both alternatives and makes porting conditional on the exact old form."}, "claim": "The migration comparison accurately maps issue granularity, accumulator and operand locations, completion, ordering, and narrow-format differences without assuming one WGMMA or tcgen05 A path.", "claim_id": "migration-wgmma-to-tcgen05-current-c001", "claim_type": "compatibility", "covers": ["u001", "u002", "u003", "u004", "u005", "u006", "u007", "u008", "u009"], "evidence": [{"locator": "WGMMA operand/execution sections; tcgen05 issue, MMA grammar, kind, fence, and commit sections", "observation": "Every current table cell matches the pinned instruction models, including both A-location alternatives and separate completion/ordering mechanisms.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-matrix-multiply-and-accumulate-operations"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The checklist releases A/B after issue, deallocates before epilogue, or treats fences as completion.", "check": "Trace source lifetime, completion, load, and deallocation ordering through steps 3-7.", "outcome": "Storage remains live through completion, epilogue transfer follows completion, and deallocation occurs before exit without an unsafe early placement."}, "claim": "The dependency-ordered checklist preserves the required operand inventory, legal instruction selection, collective TMEM lifecycle, source lifetime, completion, epilogue transfer, cta_group consistency, and workload retuning steps.", "claim_id": "migration-wgmma-to-tcgen05-current-c002", "claim_type": "semantics", "covers": ["u010", "u011", "u012", "u013", "u014", "u015", "u016", "u017"], "evidence": [{"locator": "alloc/dealloc, MMA source lifetime, commit, ld, descriptor, and cta_group rules", "observation": "The checklist follows the normative dependency order and confines performance-sensitive choices to explicit retuning.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "alloc is lane-0-issued like MMA, or CTA teardown implicitly handles deallocation.", "check": "Inspect mandatory sync/aligned warp semantics and the explicit deallocate-before-exit sentence.", "outcome": "The current page correctly rejects both alternatives."}, "claim": "The TMEM lifecycle section correctly states the b32 shared-memory allocation result, power-of-two column constraints, warp-collective group-1 issue, two-warp group-2 issue, and mandatory deallocation before exit.", "claim_id": "migration-wgmma-to-tcgen05-current-c003", "claim_type": "semantics", "covers": ["u018", "u019"], "evidence": [{"locator": "9.7.16.7.1 syntax, description, issue rules, and before-exit mandate", "observation": "Each lifecycle statement follows directly from the allocation-management section and Table 46.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "fence plus __syncthreads drains the MMA pipeline and commit is optional.", "check": "Compare the explicit completion list with fence's ordering-only contract.", "outcome": "Only commit/mbarrier tracks MMA completion, matching the rewrite."}, "claim": "The synchronization section correctly assigns MMA completion to commit/mbarrier and cross-thread ordering/code-motion control to tcgen05 fences composed with execution-ordering operations.", "claim_id": "migration-wgmma-to-tcgen05-current-c004", "claim_type": "semantics", "covers": ["u020"], "evidence": [{"locator": "mbarrier-based completion, specialized inter-thread synchronization, and fence description", "observation": "The ISA gives the two mechanisms the distinct guarantees stated by the page.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Any single thread can read arbitrary TMEM lanes synchronously into scalar variables.", "check": "Inspect issue/access restrictions and async completion for ld.", "outcome": "The current warp-level description preserves the actual access model."}, "claim": "The epilogue section accurately describes warp-access-restricted, asynchronous tcgen05.ld transfer into GPRs and the need for tcgen05.wait::ld when completion must be observed.", "claim_id": "migration-wgmma-to-tcgen05-current-c005", "claim_type": "semantics", "covers": ["u021"], "evidence": [{"locator": "9.7.16.8.1 access restrictions, 9.7.16.8.3 tcgen05.ld, and wait-based completion", "observation": "TMEM is lane-partitioned by warp, ld transfers to register vectors asynchronously, and wait::ld observes load completion.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Only 128B and M=128/256 are valid dense tcgen05 choices.", "check": "Inspect alternative descriptor encodings and multiple layout organizations.", "outcome": "The pinned tables directly support the current non-categorical wording."}, "claim": "The layout and shape guidance correctly treats 128B swizzling and maximum BF16 tiles as optional configuration examples rather than universal correctness or factor-of-two migration rules.", "claim_id": "migration-wgmma-to-tcgen05-current-c006", "claim_type": "semantics", "covers": ["u022", "u023"], "evidence": [{"locator": "Table 40 swizzle modes; instruction descriptor and data-path layouts", "observation": "The ISA defines multiple valid swizzles and shapes, with M/N selected through idesc under configuration-specific constraints.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Single-thread issue means a one-thread kernel is optimal or guarantees more resident CTAs.", "check": "Compare issue granularity with all collective pipeline roles and resource constraints.", "outcome": "The current page explicitly rejects that inference and recommends measurement."}, "claim": "The role and register-budget guidance correctly separates single-thread MMA issue from total kernel thread needs and recommends retuning without guaranteeing occupancy gains.", "claim_id": "migration-wgmma-to-tcgen05-current-c007", "claim_type": "mechanism", "covers": ["u024", "u025"], "evidence": [{"locator": "role partition, pipeline participants, and TMEM accumulator path", "observation": "The example combines single-thread MMA issue with warp/CTA work for TMA, allocation, and accumulator consumption, demonstrating that total role counts remain a design variable.", "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/examples/cute/tutorial/blackwell/02_mma_tma_sm100.cu", "revision": "CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "A single nonexistent schedule alias still appears or the example uses device::GemmUniversal directly.", "check": "Exact-token search the current names and trace the example's type chain.", "outcome": "Only existing pinned names remain and the kernel/device layering is described accurately."}, "claim": "The CUTLASS guidance matches the pinned SM100 construction flow and exact schedule identifiers, and points to an existing complete example.", "claim_id": "migration-wgmma-to-tcgen05-current-c008", "claim_type": "compatibility", "covers": ["u026", "u027"], "evidence": [{"locator": "CollectiveBuilder, kernel::GemmUniversal, and device adapter; include/cutlass/gemm/dispatch_policy.hpp schedule tags", "observation": "The current description follows the example's construction and both named 1SM/2SM schedule tags exist in the pinned dispatch-policy header.", "path": "/tmp/kernelwiki-cutlass-dRC5i4/cutlass/examples/70_blackwell_gemm/70_blackwell_fp16_gemm.cu", "revision": "CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "verified"}], "id": "migration-wgmma-to-tcgen05", "page_verdict": "verified", "path": "wiki/migration/wgmma-to-tcgen05.md", "residual_risk": "No CUDA toolchain or NVIDIA GPU is available locally, so this is an evidence-scoped migration procedure rather than a compiled port. Workload-specific performance still requires target-hardware validation.", "scope": "All 27 freshly rescanned current units were verified against CUDA 13.0.2 PTX ISA 9.0 and pinned CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5."} +{"claims": [{"challenge": {"alternative": "A fixed tensor-core utilization threshold and unsaturated DRAM are enough to classify the gap.", "check": "Compare with official roofline and scheduler-analysis semantics.", "outcome": "The guide requires relevant ceilings/intensity and distinguishes scheduler issue from sampled stalls; the page now does the same."}, "claim": "Compute-ceiling under-attainment must be classified using the executed operation's relevant ceiling, arithmetic intensity, memory ceiling, whole-kernel time, and scheduler/source evidence rather than a fixed utilization threshold.", "claim_id": "pattern-compute-bound-current-c001", "claim_type": "performance", "covers": ["u001", "u002"], "evidence": [{"locator": "roofline, Scheduler Statistics, Warp State Statistics", "observation": "The guide defines roofline boundedness from intensity/ceilings and warns that sampled stalls alone need not limit performance.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The inference table assigns causes from counter movement without controlling semantics or runtime.", "check": "Inspect each row's counterfactual and the following control requirements.", "outcome": "Every inference requires a matched change plus runtime improvement, and the page fixes math, coverage, environment, and records resource costs."}, "claim": "The matched-control table makes only evidence-scoped inferences from one-variable changes that also improve end-to-end runtime, with fixed math/launch controls and explicit resource/profiler records.", "claim_id": "pattern-compute-bound-current-c002", "claim_type": "semantics", "covers": ["u003", "u004", "u005", "u006", "u007", "u008"], "evidence": [{"locator": "baseline comparison, profiling sections, reproducibility", "observation": "Nsight Compute supports matched baseline comparison and reports scheduler, resource, compute, and memory evidence; its documentation notes replay/reproducibility constraints.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}, {"locator": "v3-v6", "observation": "Pinned variants embody the staged, role-separated, cooperative, and output-overlap mechanisms described as hypotheses.", "revision": "3b90ac9b3f624bdf1f6f78d02dcd533675d36573", "url": "https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The candidate guidance still embeds universal stage counts, stall elimination, identical peer layouts, or 2x performance.", "check": "Search the rewritten sections for absolutes and compare requirements with PTX and pinned source.", "outcome": "The page rejects all four absolutes, gives the direct two-stage counterexample, and scopes cooperative mode to exact legal forms and measured aggregate/per-SM results."}, "claim": "The candidate checks accurately scope pipeline depth, role specialization, TMEM output buffering, 2-SM MMA, fusion/interleaving, and approximation as controlled choices with correctness and resource costs rather than guaranteed fixes.", "claim_id": "pattern-compute-bound-current-c003", "claim_type": "compatibility", "covers": ["u009", "u010", "u011", "u012"], "evidence": [{"locator": "two-stage launch", "observation": "The beneficial cited pipeline uses two stages.", "revision": "3b90ac9b3f624bdf1f6f78d02dcd533675d36573", "url": "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu"}, {"locator": "cta_group and constraints", "observation": "Cooperative form legality depends on instruction kind/configuration and peer resources; the ISA supplies no universal shape or speedup.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}, {"locator": "v4-v6", "observation": "The tutorial shows role and output-overlap benefits in one setup alongside remaining idle time and added synchronization complexity.", "revision": "retrieved 2026-08-09", "url": "https://gau-nernst.github.io/tcgen05/"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "FA4 uses software exponentials universally or its peak isolates their benefit.", "check": "Compare the selection algorithm, error analysis, hardware rates, and benchmark scope in paper/blog.", "outcome": "The paper selects only a fraction for the cubic path and evaluates error; paper/blog peaks cover the coordinated full kernel with different stated maxima."}, "claim": "The FA4 case accurately reports the B200/Hopper MMA and exponential rates, selected floor-reduced cubic software path, numerical-validation requirement, and full-kernel 1613/1605-TFLOP/s scope without single-technique attribution.", "claim_id": "pattern-compute-bound-current-c004", "claim_type": "performance", "covers": ["u013", "u014", "u015"], "evidence": [{"locator": "sections 2.2, partial software exponentiation, and results", "observation": "The paper gives 8192/4096 MMA rates, equal 16-op exponential rates, the selected polynomial method/error analysis, and up to 1613 TFLOP/s at 71% for complete FA4.", "revision": "FlashAttention-4 paper v1", "url": "https://arxiv.org/html/2603.05451v1"}, {"locator": "techniques and performance", "observation": "The blog reports up to 1605 TFLOP/s at 71% for the complete design and lists multiple contributing mechanisms.", "revision": "Tri Dao FA4 blog, 2026", "url": "https://tridao.me/blog/2026/flash4/"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The reference labels are unpinned or do not support the sections they follow.", "check": "Resolve all five links and inspect their subject/revision.", "outcome": "They lead to the official profiler guide, archived PTX ISA, full-SHA tutorial source, v1 paper, and author blog used above."}, "claim": "The reference list resolves to the primary profiler, ISA, pinned implementation, paper, and author-blog evidence used by the page.", "claim_id": "pattern-compute-bound-current-c005", "claim_type": "semantics", "covers": ["u016", "u017", "u018", "u019", "u020"], "evidence": [{"locator": "document", "observation": "Official profiler guide.", "revision": "2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}, {"locator": "tcgen05", "observation": "Official archived ISA.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"locator": "file", "observation": "Pinned tutorial source.", "revision": "3b90ac9b3f624bdf1f6f78d02dcd533675d36573", "url": "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu"}, {"locator": "paper", "observation": "Versioned FA4 paper.", "revision": "v1", "url": "https://arxiv.org/html/2603.05451v1"}, {"locator": "post", "observation": "FA4 author blog.", "revision": "2026", "url": "https://tridao.me/blog/2026/flash4/"}], "oracle": "web-primary", "verdict": "verified"}], "commands": ["Inspect profiler roofline/scheduler guidance", "Inspect PTX tcgen05 forms", "Inspect tutorial v3-v6 and FA4 paper/blog"], "id": "pattern-compute-bound", "page_verdict": "verified", "path": "wiki/patterns/compute-bound.md", "residual_risk": "The decision procedure is source-verified but not run because no target GPU/profiler is available. Performance values remain author-reported.", "scope": "All 20 current units for hash e08627fe8f79b7e82f8eb53531d3aa22ddb2a98858cae22ae1389d0084a145c3 were challenged against primary profiler, ISA, source, paper, and blog evidence."} +{"claims": [{"challenge": {"alternative": "The definition merely replaces one arbitrary threshold with subjective wording.", "check": "Inspect whether the page requires measured useful work, profitable demand, time scope, and separates occupancy from activity.", "outcome": "It requires all four and lists the records needed to distinguish shortage, tails, imbalance, residency, and phases."}, "claim": "Low SM utilization is defined as materially insufficient useful active work relative to profitable demand, with theoretical occupancy explicitly separated from grid supply, achieved activity, balance, and temporal behavior.", "claim_id": "pattern-low-sm-utilization-current-c001", "claim_type": "semantics", "covers": ["u001", "u002"], "evidence": [{"locator": "occupancy, PM sampling, scheduler analysis", "observation": "The guide distinguishes occupancy limits, temporal sampling, and achieved scheduler/activity metrics, with no universal utilization cutoff.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The four cases overlap so much that the table cannot discriminate them.", "check": "Test each row's evidence against independent-work count, equal-duration remainder, per-worker duration, and resource/phase records.", "outcome": "Each row uses a different observable/control, and the prose states that aggregate activity alone cannot choose among them."}, "claim": "The page correctly separates insufficient independent work, equal-duration wave quantization, variable-duration imbalance, and residency/serial-phase limitations with discriminating evidence and controlled next experiments.", "claim_id": "pattern-low-sm-utilization-current-c002", "claim_type": "semantics", "covers": ["u003", "u004", "u005", "u006", "u007"], "evidence": [{"locator": "fixed-work and fixed-worker models", "observation": "CUDA distinguishes problem-sized blocks from resident workers acquiring repeated work and motivates reassignment under uneven availability.", "revision": "retrieved 2026-08-09", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"}, {"locator": "scheduler model", "observation": "The scheduler documentation distinguishes grid IDs, persistent work acquisition, rasterization, and cluster behavior, supporting the page's case separation.", "revision": "CUTLASS 4.5.0", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "CLC creates work or SM120 lacks it, while persistence/raster/decomposition are interchangeable remedies.", "check": "Compare each mechanism and PTX target note with the page.", "outcome": "The page keeps all four controls separate, limits CLC to existing unlaunched IDs, and accurately reports sm_100-or-higher plus explicit sm_120 multicast support."}, "claim": "Persistence, coordinate order, CLC reassignment, and K decomposition are distinct controls; CLC redistributes existing unlaunched IDs without guaranteeing balance, and PTX ISA 9.0 does not categorically exclude SM120.", "claim_id": "pattern-low-sm-utilization-current-c003", "claim_type": "compatibility", "covers": ["u008", "u009", "u010", "u011", "u012", "u013", "u014"], "evidence": [{"locator": "request semantics and Target ISA Notes", "observation": "CLC cancels an unspecified not-yet-started grid entity, requires sm_100 or higher, and lists sm_120a/sm_120f for multicast.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"}, {"locator": "SM100 scheduler", "observation": "The pinned library route is SM100-specific and separates software rasterization from CLC acquisition.", "revision": "CUTLASS 4.5.0", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The v5-to-v6 improvement is a CLC or isolated tail-elimination result.", "check": "Inspect source labels, closing remarks, and v6 roles/pipeline.", "outcome": "The article labels v6 static persistence, explicitly excludes CLC, includes pipeline/role changes, and says overlap is imperfect."}, "claim": "The tutorial case accurately reports v5 1302.29 and v6 1475.93 TFLOP/s, their 86.43% and 97.96% ratios, and scopes v6 to cumulative static persistence/output-pipeline changes without CLC or an all-SMs-busy claim.", "claim_id": "pattern-low-sm-utilization-current-c004", "claim_type": "performance", "covers": ["u015"], "evidence": [{"locator": "v5/v6 results and closing remarks", "observation": "The article reports the exact values against 1506.74 cuBLAS, names static scheduling, says CLC was not added, and describes imperfect overlap.", "revision": "retrieved 2026-08-09", "url": "https://gau-nernst.github.io/tcgen05/"}, {"locator": "v6 implementation", "observation": "The pinned source implements static work iteration and specialized output/mainloop coordination without CLC instructions.", "revision": "3b90ac9b3f624bdf1f6f78d02dcd533675d36573", "url": "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The checklist can pass while losing or duplicating CLC coordinates or conflating decomposition with acquisition.", "check": "Inspect coverage, worker instrumentation, held controls, and reporting requirements.", "outcome": "The procedure explicitly proves no missing/duplicate work, holds decomposition fixed for scheduler tests, and tests decomposition separately."}, "claim": "The verification procedure establishes exact work coverage, worker demand and tail arithmetic, per-worker imbalance, matched scheduler controls, and end-to-end/resource reporting.", "claim_id": "pattern-low-sm-utilization-current-c005", "claim_type": "semantics", "covers": ["u016", "u017", "u018", "u019", "u020"], "evidence": [{"locator": "CLC work lifecycle", "observation": "The programming model requires processing initial work and only successfully canceled returned work, making exact coverage and acquisition controls necessary.", "revision": "retrieved 2026-08-09", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"}, {"locator": "CLC scheduler lifecycle", "observation": "CUTLASS provides the initial/returned work paths and failure handling the checklist validates.", "revision": "CUTLASS 4.5.0", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The references mix PDL with CLC or omit the exact tutorial source.", "check": "Resolve all five links.", "outcome": "They point to CUDA CLC, PTX CLC, CUTLASS CLC, pinned v6, and the first-party tutorial; the unrelated PR-2161 source was removed."}, "claim": "The reference list contains direct primary CLC contracts, pinned scheduler documentation/source, and the first-party tutorial result.", "claim_id": "pattern-low-sm-utilization-current-c006", "claim_type": "semantics", "covers": ["u021", "u022", "u023", "u024", "u025"], "evidence": [{"locator": "page", "observation": "Official CLC guide.", "revision": "retrieved 2026-08-09", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html"}, {"locator": "section", "observation": "Version-pinned CLC ISA.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"}, {"locator": "file", "observation": "Pinned CLC scheduler documentation.", "revision": "CUTLASS 4.5.0", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}, {"locator": "file", "observation": "Pinned v6 source.", "revision": "3b90ac9b3f624bdf1f6f78d02dcd533675d36573", "url": "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu"}, {"locator": "article", "observation": "First-party progression.", "revision": "retrieved 2026-08-09", "url": "https://gau-nernst.github.io/tcgen05/"}], "oracle": "web-primary", "verdict": "verified"}], "commands": ["Inspect PTX/CUDA CLC target and lifecycle", "Inspect CUTLASS 4.5.0 scheduler docs", "Inspect tutorial v5/v6 source and results"], "id": "pattern-low-sm-utilization", "page_verdict": "verified", "path": "wiki/patterns/low-sm-utilization.md", "residual_risk": "No target-side utilization trace was collected because no GPU/profiler is available. Tutorial performance is author-reported.", "scope": "All 25 current units for hash 2c8aa46a8f96c06bc50d29d566037068eabf9c65e7c5d513b52cae6efc50c993 were challenged against primary CLC, scheduler, source, and profiling evidence."} +{"claims": [{"challenge": {"alternative": "The page still classifies from a vague high-bandwidth observation or algorithm name.", "check": "Inspect the definition, required records, and four evidence cases against roofline semantics.", "outcome": "Classification requires measured work/bytes, a relevant attained ceiling, traffic efficiency, issue evidence, and whole-kernel scope; labels are explicitly insufficient."}, "claim": "The page correctly defines memory-bandwidth boundedness from measured arithmetic intensity and an attained relevant memory ceiling, and separates saturation, excess traffic, latency/issue limitation, and phase-local bandwidth using distinct evidence.", "claim_id": "pattern-memory-bound-current-c001", "claim_type": "performance", "covers": ["u001", "u002", "u003", "u004", "u005", "u006", "u007"], "evidence": [{"locator": "roofline, Memory Workload Analysis, scheduler analysis", "observation": "The guide relates work/bytes to ceilings and exposes transferred traffic, cache, throughput, and scheduler evidence needed for these distinctions.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Vector width, cache hints, register caps, multicast, or swizzle are presented as guaranteed bandwidth fixes.", "check": "Search each section for guarantees and compare with ISA/compiler contracts.", "outcome": "Each is conditional, controlled, and costed; hints remain hints, caps may spill, multicast requires shared operands, and swizzle is scoped to a concrete shared-memory trace."}, "claim": "The candidate procedures accurately constrain vector loads, cache hints, register caps, TMA multicast, and swizzling to legal forms and matched experiments with their alignment, traffic, resource, reuse, cluster, and layout limitations.", "claim_id": "pattern-memory-bound-current-c002", "claim_type": "compatibility", "covers": ["u008", "u009", "u010", "u011", "u012"], "evidence": [{"locator": "natural alignment and cache operators", "observation": "PTX specifies vector alignment and hint semantics without performance guarantees.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld"}, {"locator": "maxrregcount", "observation": "Register caps are subject to ABI minima and may create spills.", "revision": "CUDA Compiler Driver 13.0.2", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html"}, {"locator": "multicast and tensor-map swizzle", "observation": "PTX defines cluster destination masks, completion, swizzle modes, and their configuration constraints.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The negative controls support a new universal claim that wide loads and caps are harmful.", "check": "Inspect the page's inference scope.", "outcome": "It reports exact source regressions/no-effect only for cited shapes and says they refute a fixed priority, not prove general harm."}, "claim": "The Amandeep case accurately reports a 16-25% wider-load regression and no effect from an 80-to-64 register cap, using them only to reject category-level prescriptions and motivate measured compute/memory hypotheses.", "claim_id": "pattern-memory-bound-current-c003", "claim_type": "performance", "covers": ["u013", "u014"], "evidence": [{"locator": "wider-load and register-cap attempts, Takeaways", "observation": "The first-party report gives those exact negative controls and recommends profiling the actual bottleneck.", "revision": "retrieved 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}, {"locator": "scheduler and memory sections", "observation": "The profiler separates issue/latency evidence from achieved memory throughput, supporting the conditional compute-work discussion.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The reproduction record omits a factor needed to distinguish cache state, nominal roof, or measurement noise.", "check": "Compare it with the profiler and source experiment fields.", "outcome": "It includes inputs, useful work/bytes, cache/warmup/statistic, hardware/software/clocks, generated code/resources, roof assumptions, achieved traffic/issue, correctness, and regressions."}, "claim": "The reproduction record is sufficient to scope a memory-roof diagnosis and its controlled candidate variants without turning a nominal specification or one endpoint into causal evidence.", "claim_id": "pattern-memory-bound-current-c004", "claim_type": "semantics", "covers": ["u015"], "evidence": [{"locator": "reproducibility, roofline, memory and scheduler sections", "observation": "The listed fields cover the environment, traffic, resource, issue, correctness, and comparison factors used by the official analyses.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A reference label points to a secondary or unrelated page.", "check": "Resolve all four URLs.", "outcome": "They are the official profiler guide, archived PTX cache and multicast sections, and the cited first-party attempt report."}, "claim": "The primary-reference list resolves directly to the profiler, version-pinned ISA, and first-party negative-control evidence used by the page.", "claim_id": "pattern-memory-bound-current-c005", "claim_type": "semantics", "covers": ["u016", "u017", "u018", "u019"], "evidence": [{"locator": "document", "observation": "Official profiler guide.", "revision": "2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}, {"locator": "ld and cp.async.bulk.tensor", "observation": "Official archived ISA contains both linked sections.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"locator": "article", "observation": "First-party NVFP4 GEMV attempt report.", "revision": "retrieved 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}], "oracle": "web-primary", "verdict": "verified"}], "commands": ["Inspect roofline/memory/scheduler profiler semantics", "Inspect PTX cache, vector, TMA, and swizzle contracts", "Inspect NVFP4 GEMV negative controls"], "id": "pattern-memory-bound", "page_verdict": "verified", "path": "wiki/patterns/memory-bound.md", "residual_risk": "The workflow was not run on target hardware because no GPU/profiler is available. Attempt results remain author-reported.", "scope": "All 19 current units for hash df62414848150544dd22c908b1f015e7c3925bad2512fe91186cfc9edffc8e8b were challenged against profiler, ISA, compiler, and first-party experiment evidence."} +{"claims": [{"challenge": {"alternative": "Token-count balance alone diagnoses the kernel, or one expert maps to one physical SM.", "check": "Compare CUDA block placement and grouped tiling with every level in the measurement record.", "outcome": "The page rejects fixed ownership and requires expert, tile, worker/SM, rank, and phase timing evidence before attribution."}, "claim": "The diagnosis correctly separates routed-token, valid-row/tile, logical-worker/SM, rank-communication, and phase timing imbalance, treats small-M effects as configuration-specific, and rejects routing/loss/batch proxies as sufficient negative tests.", "claim_id": "pattern-moe-load-imbalance-current-c001", "claim_type": "performance", "covers": ["u001", "u002", "u003", "u004", "u005", "u006", "u007", "u008", "u009"], "evidence": [{"locator": "thread-block scheduling", "observation": "Physical block placement is dynamic and does not imply one expert per SM.", "revision": "retrieved 2026-08-09", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html"}, {"locator": "small GEMM dimensions", "observation": "Small dimensions can create predicated lanes and too few blocks, with no universal M threshold.", "revision": "CUTLASS 4.5.0", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/efficient_gemm.md"}, {"locator": "EPLB case studies and limitations", "observation": "The study measures load/balance and throughput by phase and configuration rather than treating generic routing conditions as proof.", "revision": "published 2025-05-05, retrieved 2026-08-09", "url": "https://www.lmsys.org/blog/2025-05-05-large-scale-ep/"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Contiguous always means an offsets array, masked computes M-max padding, or K-grouped is a forward variable-M layout.", "check": "Inspect all three pinned README contracts and their API assertions.", "outcome": "The metadata forms, valid-M semantics, fixed axes, alignment, and weight-backward K grouping match the table exactly."}, "claim": "The DeepGEMM table accurately distinguishes contiguous M grouping with two metadata forms and aligned segments, masked M grouping with fixed allocation but valid-portion compute for a graph decode case, and variable-K/fixed-MN grouping for weight backward, while prescribing appropriate controls.", "claim_id": "pattern-moe-load-imbalance-current-c002", "claim_type": "semantics", "covers": ["u010", "u011", "u012", "u013", "u014"], "evidence": [{"locator": "Grouped GEMMs contiguous, masked, and K-grouped", "observation": "The README gives each use, layout, fixed/variable axis, alignment, metadata, and valid-compute scope.", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"}, {"locator": "m_grouped and k_grouped API checks", "observation": "The exact tensor ranks, axes, and integer metadata are enforced by the pinned API.", "revision": "same commit", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Static ownership fixes a physical SM, CLC creates or steals arbitrary work, every tile pays a small request cost, or CLC excludes SM120.", "check": "Trace CUDA placement, PTX request/target rules, and CUTLASS's initial and later work paths.", "outcome": "The page preserves logical versus physical placement, existing-grid cancellation/failure, static initial work, pipelining/cluster granularity, and the broader ISA target boundary."}, "claim": "The scheduler section correctly separates static persistence, CLC-backed reassignment, and work decomposition; states the exact request/failure/existing-grid limits and matched-test controls; and distinguishes PTX sm_100-or-higher/SM120-family support from CUTLASS 4.5.0's SM100 integration.", "claim_id": "pattern-moe-load-imbalance-current-c003", "claim_type": "compatibility", "covers": ["u015", "u016", "u017", "u018", "u019", "u020", "u021", "u022", "u023", "u024"], "evidence": [{"locator": "thread-block scheduling", "observation": "CUDA dynamically places blocks.", "revision": "retrieved 2026-08-09", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html"}, {"locator": "request, response, and Target ISA Notes", "observation": "The ISA defines cancellation of unlaunched work, possible failure, cluster forms, sm_100-or-higher support, and SM120-family multicast.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"}, {"locator": "static scheduler, dynamic scheduler, and CLC pipeline", "observation": "Initial blockIdx work is static; later requests are asynchronous, pipelined, and configuration-dependent.", "revision": "CUTLASS 4.5.0", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "EPLB is a device scheduler or the two speedups transfer to arbitrary deployments and distributions.", "check": "Inspect the pinned planner inputs/outputs and the measurement study's setup and limitations.", "outcome": "The code performs host-side replica placement across configured GPUs/nodes; the values belong to a 96-H100 in-distribution ablation whose authors warn about distribution shifts."}, "claim": "The EPLB section accurately scopes the pinned project as load-driven expert replication and GPU/node placement, not device tile scheduling, and scopes the 1.49x prefill and 2.54x decode values to the LMSYS/SGLang 96-H100 in-distribution study.", "claim_id": "pattern-moe-load-imbalance-current-c004", "claim_type": "performance", "covers": ["u025", "u026"], "evidence": [{"locator": "replicate_experts and rebalance_experts", "observation": "The code takes logical-expert loads and emits physical/logical maps and replica counts across configured nodes and GPUs.", "revision": "EPLB d52c72d5b2f2fb4c41afbf8eb21366820239913d", "url": "https://github.com/deepseek-ai/EPLB/blob/d52c72d5b2f2fb4c41afbf8eb21366820239913d/eplb.py"}, {"locator": "Ablation Study: EPLB and limitations", "observation": "The source reports the two values in its large-scale deployment and explicitly notes in-distribution data and untested shifts.", "revision": "published 2025-05-05, retrieved 2026-08-09", "url": "https://www.lmsys.org/blog/2025-05-05-large-scale-ep/"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The exploit skipped correctness work, the reported time is a valid balancing result, or the incident is documented as causing the FlashInfer evaluator design.", "check": "Separate the postmortem's correctness/timing paths and named response from the pinned later contest contract.", "outcome": "The exact 15-object/120-group cache sequence is retained, the timing is rejected as invalid, PR 104 is the named response, and neither source states the later causality."}, "claim": "The reward-hack section accurately records the real correctness path, fifteen-clone/120-group timing merge and cached calls, rejects the result as load-balancing performance evidence, and does not invent a causal link to the later FlashInfer contest.", "claim_id": "pattern-moe-load-imbalance-current-c005", "claim_type": "mechanism", "covers": ["u027", "u028"], "evidence": [{"locator": "correctness, timing exploit, and response", "observation": "The official postmortem supplies the sequence and names reference-kernels PR 104.", "revision": "retrieved 2026-08-09", "url": "https://www.gpumode.com/news/reward-hacking-nvfp4"}, {"locator": "evaluation contract", "observation": "The later contract does not attribute its design to the GPU Mode event.", "revision": "starter kit 75ccd05cafceb0fd1f86be4cd0f2117249463c66", "url": "https://github.com/flashinfer-ai/flashinfer-bench-starter-kit/blob/75ccd05cafceb0fd1f86be4cd0f2117249463c66/EVALUATION.md"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "One or more references are rolling, secondary, or omit the exact mechanism/revision claimed.", "check": "Resolve every listed target and compare its revision and scope with the adjacent text.", "outcome": "The list points to official CUDA/PTX, pinned CUTLASS/DeepGEMM/EPLB/evaluator revisions, and the two first-party incident/deployment reports used with explicit scope."}, "claim": "The eight primary-reference entries resolve to the direct scheduling, CLC, grouped-layout, EPLB, scoped performance, reward-hack, and evaluator sources used by the page.", "claim_id": "pattern-moe-load-imbalance-current-c006", "claim_type": "semantics", "covers": ["u029", "u030", "u031", "u032", "u033", "u034", "u035", "u036"], "evidence": [{"locator": "document", "observation": "Official scheduling documentation.", "revision": "retrieved 2026-08-09", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html"}, {"locator": "section", "observation": "Archived CLC contract.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"}, {"locator": "file", "observation": "Pinned scheduler documentation.", "revision": "CUTLASS 4.5.0", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}, {"locator": "file", "observation": "Pinned grouped contracts.", "revision": "891d57b4", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"}, {"locator": "tree", "observation": "Pinned planner source.", "revision": "d52c72d5", "url": "https://github.com/deepseek-ai/EPLB/tree/d52c72d5b2f2fb4c41afbf8eb21366820239913d"}, {"locator": "article", "observation": "Scoped deployment study.", "revision": "published 2025-05-05", "url": "https://www.lmsys.org/blog/2025-05-05-large-scale-ep/"}, {"locator": "article", "observation": "Official incident report.", "revision": "retrieved 2026-08-09", "url": "https://www.gpumode.com/news/reward-hacking-nvfp4"}, {"locator": "file", "observation": "Pinned contest evaluator.", "revision": "75ccd05c", "url": "https://github.com/flashinfer-ai/flashinfer-bench-starter-kit/blob/75ccd05cafceb0fd1f86be4cd0f2117249463c66/EVALUATION.md"}], "oracle": "web-primary", "verdict": "verified"}], "commands": ["Inspect CUDA/PTX scheduling and CLC contracts", "Inspect pinned grouped-layout and EPLB sources", "Challenge performance and history against scoped first-party records"], "id": "pattern-moe-load-imbalance", "page_verdict": "verified", "path": "wiki/patterns/moe-load-imbalance.md", "residual_risk": "No GPU or distributed deployment is available locally, so the page gives a discriminating measurement protocol and preserves external performance values only in their disclosed scope.", "scope": "All 36 current units for hash 39a1e4e3fb4709726fa3f81e7e26b68f6c0936ca10fa151101aee424b9f283f0 were challenged against direct versioned scheduling, layout, placement, performance, and evaluator evidence."} +{"claims": [{"challenge": {"alternative": "Low unit activity or barrier-wait samples are sufficient to diagnose a pipeline bottleneck.", "check": "Compare the page's qualification with Nsight Compute's sampling and stall guidance.", "outcome": "The profiler documents sampled readiness states and warns that stalls need not affect performance; the page requires scheduler under-issue, correlation, and a causal test."}, "claim": "Low execution-unit activity and barrier-wait samples are diagnostic leads rather than causal proof; Nsight Compute stall data should be interpreted with scheduler issue evidence, available chip-specific metrics, and sampling limitations.", "claim_id": "pattern-pipeline-stalls-current-c001", "claim_type": "semantics", "covers": ["u001", "u002"], "evidence": [{"locator": "Scheduler Statistics, Warp State Statistics, PM and warp sampling", "observation": "The guide distinguishes issue/eligibility statistics from sampled warp states, cautions that stalls may not affect performance, documents queryable metric availability, and says warp sampling has no time resolution.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The page still confuses TMA transaction completion with tcgen05 ordering or prescribes one phase update for every wait.", "check": "Trace each checklist item to PTX mbarrier counters, wait validity/visibility, tcgen05 completion, and fence semantics.", "outcome": "The rewritten checklist assigns TMA bytes to complete_tx, phase to each reused object, MMA completion to commit/mbarrier, and tcgen05 fences only to applicable cross-thread execution-ordering handoffs."}, "claim": "The correctness gate accurately separates mbarrier arrival and transaction-byte accounting, per-object phase tracking, TMA acquire visibility, asynchronous tcgen05 completion, cross-thread tcgen05 ordering, and pipeline boundary handling.", "claim_id": "pattern-pipeline-stalls-current-c002", "claim_type": "semantics", "covers": ["u003", "u004", "u005", "u006", "u007", "u008", "u009", "u010", "u011"], "evidence": [{"locator": "mbarrier lifecycle, arrive, complete_tx, and waits", "observation": "PTX defines separate arrival/transaction counters, current-or-prior phase state/parity, and acquire visibility for associated bulk copies.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"}, {"locator": "tcgen05 completion and execution ordering", "observation": "PTX distinguishes commit/mbarrier completion from before/after-thread-sync ordering across execution-ordering operations.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The workflow merely relabels pipeline stages, warp specialization, buffering, and ping-pong as guaranteed fixes.", "check": "Inspect whether each row states a falsifiable target, controls, and costs and whether the workflow rejects failed predictions.", "outcome": "Every change is conditional on a concrete edge, holds relevant variables fixed, records resource costs, and is rejected when predicted wait/time movement does not occur."}, "claim": "The workflow diagnoses pipeline stalls through reproducible timing, profiler correlation, an edge-specific prediction, and one-variable tests of stage depth, role separation, TMEM buffering, or two-tile interleaving with explicit correctness and resource controls.", "claim_id": "pattern-pipeline-stalls-current-c003", "claim_type": "semantics", "covers": ["u012", "u013", "u014", "u015", "u016", "u017", "u018", "u019", "u020", "u021"], "evidence": [{"locator": "profiling sections and reproducibility", "observation": "The profiler exposes the stated section categories and documents replay, sampling, and reproducibility considerations.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}, {"locator": "v3-v6 implementations", "observation": "The pinned versions demonstrate concrete stage, role, 2-SM, persistent, and output-overlap configurations with nontrivial storage and synchronization state.", "revision": "3b90ac9b3f624bdf1f6f78d02dcd533675d36573", "url": "https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100"}, {"locator": "section 3.1.2", "observation": "The paper supplies the concrete two-output-tile attention case and describes maximized, not guaranteed, overlap.", "revision": "FlashAttention-4 paper v1", "url": "https://arxiv.org/html/2603.05451v1"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The table still mislabels stage count, rounds the ratios incorrectly, or attributes v6 to CLC.", "check": "Compare every value and version label with the article, inspect pinned v3 instantiation, and recompute each ratio from 1506.74.", "outcome": "All six TFLOP/s values and labels match; v3 uses two stages, v6 says static scheduling, CLC is explicitly absent, and recomputation gives the displayed ratios to two decimals."}, "claim": "For the tutorial's disclosed B200 4096-cubed experiment, the page accurately reports v1a through v6 values and cuBLAS ratios, identifies v3 as a two-stage source, and scopes the cumulative v6 endpoint to static persistence without CLC.", "claim_id": "pattern-pipeline-stalls-current-c004", "claim_type": "performance", "covers": ["u022", "u023", "u024", "u025", "u026", "u027", "u028", "u029"], "evidence": [{"locator": "version result tables and closing progression", "observation": "The article reports cuBLAS 1506.74; v1a 254.62; v2b 695.43; v3 939.61; v4 1208.83; v5 1302.29; and v6 static persistence 1475.93 TFLOP/s, and says CLC was not added. Dividing each kernel value by the stated cuBLAS value yields the displayed ratios.", "revision": "retrieved 2026-08-09", "url": "https://gau-nernst.github.io/tcgen05/"}, {"locator": "matmul_v3 function", "observation": "The v3 launch instantiates NUM_STAGES=2.", "revision": "3b90ac9b3f624bdf1f6f78d02dcd533675d36573", "url": "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "One or more primary-reference labels point to unrelated or unpinned material.", "check": "Resolve each listed URL and compare its subject or revision with the label.", "outcome": "The links resolve respectively to Nsight Compute 2025.3 profiling, PTX ISA 9.0 mbarrier, PTX ISA 9.0 tcgen05 fence, pinned tutorial v3 source, and the first-party tutorial progression."}, "claim": "The primary-reference list links to the exact profiler guide, version-pinned mbarrier and tcgen05 contracts, pinned tutorial v3 source, and first-party progression used by the page.", "claim_id": "pattern-pipeline-stalls-current-c005", "claim_type": "semantics", "covers": ["u030", "u031", "u032", "u033", "u034"], "evidence": [{"locator": "document title and contents", "observation": "Official kernel profiling guide.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}, {"locator": "mbarrier and tcgen05 fence sections", "observation": "Official version-pinned ISA contains both linked contracts.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"locator": "file", "observation": "Pinned tutorial v3 implementation.", "revision": "3b90ac9b3f624bdf1f6f78d02dcd533675d36573", "url": "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu"}, {"locator": "article", "observation": "First-party tutorial and result progression.", "revision": "retrieved 2026-08-09", "url": "https://gau-nernst.github.io/tcgen05/"}], "oracle": "web-primary", "verdict": "verified"}], "commands": ["Inspect PTX ISA 9.0 mbarrier and tcgen05 handoff contracts", "Inspect Nsight Compute 2025.3 profiler semantics", "Inspect pinned tutorial source and recompute ratios in Conda base"], "id": "pattern-pipeline-stalls", "page_verdict": "verified", "path": "wiki/patterns/pipeline-stalls.md", "residual_risk": "No target GPU or Nsight Compute installation is available locally, so the diagnostic procedure was source-verified but not executed. Tutorial performance remains author-reported.", "scope": "All 34 current body units for hash 0c5294d705b6402ce30d3482e6d7f3bb2cb0ca6bf75e81712347ff1d23959d45 were challenged against profiler, ISA, paper, source, and arithmetic evidence."} +{"claims": [{"challenge": {"alternative": "Spill evidence alone proves register pressure, or maximum occupancy is always desirable.", "check": "Compare the definition and required record with profiler occupancy/resource semantics.", "outcome": "The page requires a register/spill constraint on useful scheduling or material traffic and a causal variant, while explicitly rejecting a universal occupancy target."}, "claim": "The page correctly diagnoses performance-relevant register pressure from compiled allocation/spills, limiting residency and scheduler/traffic evidence for the exact binary, rather than from below-target occupancy or source complexity alone.", "claim_id": "pattern-register-pressure-current-c001", "claim_type": "performance", "covers": ["u001", "u002", "u003"], "evidence": [{"locator": "Launch Statistics, Occupancy, scheduler and source analysis", "observation": "The profiler separates registers/spills, residency limit, achieved warps, scheduler states, and source-correlated generated instructions.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Warp specialization is SM100-only and a register cap necessarily raises residency.", "check": "Inspect Hopper specialization docs and compiler cap semantics.", "outcome": "The page identifies Hopper support and makes both role separation and caps conditional on compiled-resource/runtime evidence, including spill and ABI risks."}, "claim": "The live-range, role-specialization, and register-cap procedures correctly include Hopper availability, compiler/resource dependence, ABI minima, spill risk, and matched runtime validation.", "claim_id": "pattern-register-pressure-current-c002", "claim_type": "compatibility", "covers": ["u004", "u005"], "evidence": [{"locator": "warp-specialized Hopper GEMM", "observation": "CUTLASS documents warp specialization on Hopper.", "revision": "retrieved 2026-08-09", "url": "https://docs.nvidia.com/cutlass/media/docs/cpp/hopper_gemm.html"}, {"locator": "maxrregcount", "observation": "The cap is subject to ABI minimums and can cause spills rather than a guaranteed residency result.", "revision": "CUDA Compiler Driver 13.0.2", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The 128-register Hopper arithmetic or raw 256-KB TMEM size proves exactly 128 registers are freed and every large output double-buffers.", "check": "Compare fragment mapping, TMEM organization, tcgen05 loads, packing, and lifecycle.", "outcome": "Only the Hopper N/2 arithmetic is exact; the page explicitly retains Blackwell descriptor/barrier/load/epilogue registers and configuration-specific TMEM columns."}, "claim": "The Hopper m64n256 FP32 WGMMA fragment is exactly 128 registers/thread for D, while the SM100 tcgen05 migration places resident D in 128x512 TMEM but retains configuration-specific columns, register work, collective lifecycle, completion, load/wait, and target constraints.", "claim_id": "pattern-register-pressure-current-c003", "claim_type": "semantics", "covers": ["u006", "u007", "u008"], "evidence": [{"locator": "m64nNk16 D mapping", "observation": "FP32 D contributes N/2 registers per warpgroup thread.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma"}, {"locator": "organization and lifecycle", "observation": "TMEM is 128 lanes by512 32-bit cells with collective allocation/deallocation and kind/packing-sensitive use.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"}, {"locator": "collective load and wait", "observation": "TMEM-to-register loads are collective asynchronous operations with explicit completion.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "TMEM load cost is guaranteed to be offset by occupancy, so only register counts need reporting.", "check": "Inspect whether the page claims or measures the balance.", "outcome": "It calls the balance empirical and requires loads/waits, resource changes, achieved warps, stalls, correctness, and time including no-occupancy-change cases."}, "claim": "The evaluation correctly treats TMEM-load/epilogue cost versus long-lived register relief as an empirical whole-kernel tradeoff requiring resource, scheduler, correctness, and timing evidence.", "claim_id": "pattern-register-pressure-current-c004", "claim_type": "performance", "covers": ["u009", "u010"], "evidence": [{"locator": "load completion", "observation": "The ISA establishes asynchronous load/wait cost but no performance-offset guarantee.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"}, {"locator": "resource, occupancy, scheduler, and source sections", "observation": "The requested record covers the observable tradeoff dimensions.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The links do not pin the accumulator or load contracts.", "check": "Resolve all five reference targets.", "outcome": "They point to the official profiler, archived PTX WGMMA/TMEM/load sections, and a full-SHA tutorial tree."}, "claim": "The references directly support profiler diagnosis, WGMMA fragment arithmetic, TMEM organization/lifecycle, tcgen05 loads, and the pinned SM100 implementation example.", "claim_id": "pattern-register-pressure-current-c005", "claim_type": "semantics", "covers": ["u011", "u012", "u013", "u014", "u015"], "evidence": [{"locator": "document", "observation": "Official profiler guide.", "revision": "2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}, {"locator": "WGMMA and tcgen05/TMEM", "observation": "Official version-pinned instruction contracts.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"locator": "tree", "observation": "Pinned implementation source.", "revision": "3b90ac9b3f624bdf1f6f78d02dcd533675d36573", "url": "https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100"}], "oracle": "web-primary", "verdict": "verified"}], "commands": ["Inspect profiler register/occupancy semantics", "Inspect WGMMA fragment and TMEM/tcgen05 contracts", "Inspect Hopper specialization and compiler cap docs"], "id": "pattern-register-pressure", "page_verdict": "verified", "path": "wiki/patterns/register-pressure.md", "residual_risk": "No target compilation/profile was possible, so compiled resource outcomes remain a prescribed experiment rather than a local result.", "scope": "All 15 current units for hash 6411aba77dd98964d25ec7fa3a3e7f97e70f5400886e930aaa77f6b2072c7621 were challenged against primary profiler, compiler, ISA, and pinned-source evidence."} +{"claims": [{"challenge": {"alternative": "The r=0 case has an empty final wave, or 142 is asserted as B200 hardware.", "check": "Apply Euclidean division and inspect the stated scope around W.", "outcome": "The page explicitly treats r=0 as a full last wave, computes 8/142 correctly only as a declared model, and says W need not equal physical SM count."}, "claim": "For equal-duration independent work, T=qW+r gives q full waves and, only when r>0, a final r-worker wave; T=150,W=142 yields 5.63%, while real worker capacity and fractional impact require measurement and no four-wave cutoff.", "claim_id": "pattern-tail-effect-current-c001", "claim_type": "code", "covers": ["u001", "u002", "u003", "u004", "u005"], "evidence": [{"locator": "data-parallel work-ID model", "observation": "The scheduler's independent grid-ID model supports the explicitly simplified arithmetic; Euclidean division verifies the counts and 8/142 value.", "revision": "CUTLASS 4.5.0", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}, {"locator": "occupancy and workload distribution", "observation": "Actual residency/activity depends on launch resources and observed execution rather than physical SM count alone.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Aggregate utilization and tile-count modulo alone are enough, and grid stride fixes physical-SM ownership.", "check": "Inspect the required sweep/timeline and CUDA block-scheduling contract.", "outcome": "The page requires a predicted sawtooth plus time-resolved end interval and separates stragglers; it correctly says blockIdx does not fix physical placement."}, "claim": "The diagnosis procedure verifies a last-wave cause through worker/residency records, predicted remainder, an end-of-run underfilled interval, nearby-size sawtooth behavior, and separation from unequal-duration stragglers, while preserving CUDA's dynamic block placement.", "claim_id": "pattern-tail-effect-current-c002", "claim_type": "semantics", "covers": ["u006", "u007", "u008", "u009", "u010", "u011"], "evidence": [{"locator": "thread-block scheduling", "observation": "CUDA assigns blocks to available SMs without a programmer-visible blockIdx-to-SM mapping.", "revision": "retrieved 2026-08-09", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html"}, {"locator": "PM sampling and workload distribution", "observation": "Time-resolved metrics and resource/activity sections provide the evidence requested by the procedure.", "revision": "Nsight Compute 2025.3", "url": "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Persistence removes tails, CLC creates work, raster balances workers, or CLC excludes SM120.", "check": "Compare every bullet and compatibility statement with PTX/CUTLASS semantics.", "outcome": "All absolutes are rejected; mechanisms remain separate, CLC only redistributes unlaunched IDs, and SM120-family target support is stated accurately."}, "claim": "The page accurately separates static persistence, CLC reassignment, coordinate transforms, and K decomposition, including their finite-work/reduction costs, matched-test controls, and PTX sm_100-or-higher/SM120-family compatibility scope.", "claim_id": "pattern-tail-effect-current-c003", "claim_type": "compatibility", "covers": ["u012", "u013", "u014", "u015", "u016"], "evidence": [{"locator": "request and target semantics", "observation": "CLC cancels existing unlaunched work, may fail, requires sm_100 or higher, and lists SM120-family multicast support.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"}, {"locator": "scheduler, raster, and work-ID model", "observation": "The pinned implementation distinguishes persistent acquisition and software coordinate transforms on SM100.", "revision": "CUTLASS 4.5.0", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The tutorial has 142 SMs or demonstrates CLC/all-SMs-busy causality.", "check": "Inspect hardware count, v5/v6 labels, source, and closing remarks.", "outcome": "It says 148 SMs, static persistence, no CLC, imperfect overlap, and cumulative pipeline changes; the page reports exactly that."}, "claim": "The tutorial scope correctly identifies 148 SMs, v5 1302.29 and v6 1475.93 TFLOP/s (86.43% and 97.96%), and rejects CLC, perfect-overlap, and isolated-tail attribution.", "claim_id": "pattern-tail-effect-current-c004", "claim_type": "performance", "covers": ["u017"], "evidence": [{"locator": "persistent section and closing progression", "observation": "The source supplies the hardware count, values, static label, no-CLC note, and imperfect-overlap caveat.", "revision": "retrieved 2026-08-09", "url": "https://gau-nernst.github.io/tcgen05/"}, {"locator": "v6", "observation": "Pinned implementation uses static repeated work and contains no CLC request.", "revision": "3b90ac9b3f624bdf1f6f78d02dcd533675d36573", "url": "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The references omit the exact scheduling or result evidence.", "check": "Resolve all five links.", "outcome": "They point to CUDA scheduling, PTX CLC, pinned CUTLASS CLC, pinned v6, and the first-party results."}, "claim": "The primary references directly support scheduling, CLC, pinned implementation, and tutorial performance scope.", "claim_id": "pattern-tail-effect-current-c005", "claim_type": "semantics", "covers": ["u018", "u019", "u020", "u021", "u022"], "evidence": [{"locator": "document", "observation": "Official scheduling model.", "revision": "retrieved 2026-08-09", "url": "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html"}, {"locator": "section", "observation": "Pinned CLC contract.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"}, {"locator": "file", "observation": "Pinned scheduler documentation.", "revision": "CUTLASS 4.5.0", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}, {"locator": "file", "observation": "Pinned v6 source.", "revision": "3b90ac9b3f624bdf1f6f78d02dcd533675d36573", "url": "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu"}, {"locator": "article", "observation": "First-party results.", "revision": "retrieved 2026-08-09", "url": "https://gau-nernst.github.io/tcgen05/"}], "oracle": "web-primary", "verdict": "verified"}], "commands": ["Check analytical wave arithmetic", "Inspect CUDA/PTX/CUTLASS scheduling semantics", "Inspect tutorial v6 and progression"], "id": "pattern-tail-effect", "page_verdict": "verified", "path": "wiki/patterns/tail-effect.md", "residual_risk": "The model is deliberately simplified and no target timeline was collected; real worker capacity and duration variance remain experiment inputs.", "scope": "All 22 current units for hash 015951e658cf7ca107772efab40ffc056cd6152f9c5c7864a4d0b59dcc1df455 were challenged against arithmetic, scheduler, ISA, implementation, and result evidence."} +{"claims": [{"challenge": {"alternative": "The qualifier names define mandatory bypass, immediate eviction, persistent residence, and completed prefetch behavior.", "check": "Trace the load/store forms through the PTX grammar, description, alignment rule, and performance-hint language.", "outcome": "All displayed forms are legal 16-byte accesses, while the normative contract remains eviction priority that may apply or a performance-only prefetch hint; the page preserves that distinction."}, "claim": "PTX 9.0 exposes load/store L1 eviction priorities and load-side L2 prefetch/cache-policy hints without guaranteeing placement, residence, eviction time, hit rate, or performance; the displayed v4.u32 fragments are legal and require 16-byte alignment.", "claim_id": "technique-cache-policy-current-c001", "claim_type": "code", "covers": ["u001", "u002", "u003", "u004", "u005", "u006", "u007", "u008", "u009"], "evidence": [{"locator": "6.4.1.1 Addresses as Operands, 9.7.9.8 ld, and 9.7.9.12 st", "observation": "The archived ISA grammar accepts the displayed vector and eviction forms, requires natural alignment to total access size, says eviction policies may be applied, and treats prefetch/cache policies as hints that do not alter consistency.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A memory-bound label, tensor role, or working-set-size threshold is sufficient to choose the policy without an ablation.", "check": "Compare the ISA's absence of selection rules with interference/reuse counterexamples and the page's controlled procedure.", "outcome": "The ISA supplies no tensor-role prescription, and concurrency changes effective reuse; the procedure holds co-varying choices fixed and measures each production shape."}, "claim": "The five-step policy-selection procedure validly starts from concrete warp addresses, reuse and interference, then controls correctness, generated code/resources, profiler evidence, and repeated timing; size or bottleneck labels alone do not predict a useful hint.", "claim_id": "technique-cache-policy-current-c002", "claim_type": "semantics", "covers": ["u010", "u011", "u012", "u013", "u014", "u015", "u016"], "evidence": [{"locator": "ld cache qualifiers and performance scope", "observation": "The ISA specifies legal cache controls and hint semantics but no workload category, cache-capacity threshold, tensor name, or GEMM-axis rule that establishes benefit.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld"}, {"locator": "Attempts 8-12 and Takeaways", "observation": "The author's same-task experiments show ineffective and regressing plausible transformations and recommend profiling the concrete bottleneck before choosing one.", "revision": "retrieved 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The post-event observations reveal rank-1 code and exact per-K policy choices.", "check": "Separate what the author directly reports from missing contestant implementation and ablation evidence.", "outcome": "The report names common techniques and co-varying differences but publishes neither the inspected contestant code nor rank-1 per-K qualifier mapping; the page states only that boundary."}, "claim": "Amandeep Singh reports A no_allocate and B evict_last among several co-varying techniques in three inspected solutions but provides no released contestant code, cache-only ablation, or evidence for the former rank-1 per-K attribution.", "claim_id": "technique-cache-policy-current-c003", "claim_type": "performance", "covers": ["u017"], "evidence": [{"locator": "Post-event Observations", "observation": "The cited report lists cache policies together with PTX decode, wide loads, specialization, register caps, and reuse; it supplies no source or isolated timings for the inspected contestants.", "revision": "retrieved 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}, {"locator": "public task and leaderboard boundary", "observation": "The public materials expose the task and scores rather than contestant source or technique metadata.", "revision": "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6", "url": "https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The 39-to-27 microsecond endpoints are a cache-policy ablation or show that cache policy dominated.", "check": "Compare both endpoint stage descriptions and enumerate every changed mechanism.", "outcome": "The first stage already combines staging, tiling, float4, and conversion changes, while the second switches the vectorized PTX load/decode path; the page denies cache-only attribution."}, "claim": "Yue Zhang's approximately 39-to-27 microsecond endpoints change multiple load, staging, tiling, conversion, and PTX-decode mechanisms and therefore do not measure a cache-hint-only speedup or dominant cache-policy effect.", "claim_id": "technique-cache-policy-current-c004", "claim_type": "performance", "covers": ["u018"], "evidence": [{"locator": "CUDA optimization 2 and 3", "observation": "The author gives the same endpoints and full combined changes, with no cache-policy-only variant or causal decomposition.", "revision": "retrieved 2026-08-08", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}], "oracle": "web-primary", "verdict": "verified"}], "commands": ["Inspect archived PTX 9.0 load/store eviction, prefetch, cache-policy, vector, and alignment contracts", "Challenge workload-selection rules against the ISA scope and same-task negative controls", "Separate direct author reports from unsupported contestant and causal attribution"], "id": "technique-cache-policy", "page_verdict": "verified", "path": "wiki/techniques/cache-policy.md", "residual_risk": "The PTX fragments were checked against archived syntax but not assembled or profiled because this environment has no CUDA toolkit or GPU. Performance claims remain explicitly scoped to author reports and missing ablations.", "scope": "All 18 current body units, the frontmatter hint claim, code fragments, procedure, and report-provenance boundaries were checked against direct versioned evidence."} +{"claims": [{"challenge": {"alternative": "The surrounding captured TopK key file makes PR 6152 valid evidence for the entire algorithm.", "check": "Distinguish key-file context from added/deleted lines in the PR patch.", "outcome": "The semantic delta is only logging; the page explicitly preserves that distinction."}, "claim": "The page accurately scopes PR 3559 to SM100 scan policies/dispatch/tests and PR 6152 to a debug-only DispatchTopK logging correction, excluding DSA or TopK performance conclusions from the latter.", "claim_id": "technique-cccl-memory-primitives-current-c001", "claim_type": "compatibility", "covers": ["u001", "u002", "u003", "u004"], "evidence": [{"locator": "complete changed-file patch", "observation": "The patch adds scan type/policy plumbing and SM100 scan tunings across the stated files.", "path": "artifacts/prs/cccl/PR-3559/diff.patch", "revision": "CCCL captured merge 25523da2"}, {"locator": "complete patch", "observation": "Only the debug format, scalar grid argument and occupancy variable name change; no TopK algorithm or tuning changes.", "path": "artifacts/prs/cccl/PR-6152/diff.patch", "revision": "CCCL captured merge 3fb05826"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "CUB may internally vectorize whenever items is at least four, making the deleted boolean a reliable proxy.", "check": "Compare specializations with similar item counts and inspect independent BlockLoadAlgorithm/iterator/type dispatch.", "outcome": "Item count neither encodes alignment/contiguity nor fixes the selected load algorithm; the page's non-implication is correct."}, "claim": "The PR 3559 policy table correctly lists type/operator classification, threads/items, load/store/cache choices, delay constructors, and Policy1000-to-Policy900 fallback, and rejects items>=4 as proof of vectorization.", "claim_id": "technique-cccl-memory-primitives-current-c002", "claim_type": "mechanism", "covers": ["u005", "u006", "u007", "u008", "u009", "u010", "u011"], "evidence": [{"locator": "sm100_tuning specializations, select_agent_policy100 and Policy1000", "observation": "Every listed policy field/classifier/fallback appears in the source; the double path explicitly derives from sm90_tuning and item count is independent of load/store/cache policy.", "path": "artifacts/prs/cccl/PR-3559/key-files/cub/cub/device/dispatch/tuning/tuning_scan.cuh", "revision": "CCCL captured merge 25523da2"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The workflow smuggles in unsupported claims by listing correctness cases not mentioned in either PR.", "check": "Read each item as a verification requirement versus an asserted behavior and compare the final evidence exclusions.", "outcome": "The page does not predict outcomes; it identifies what must be tested and states that PR 6152 establishes none of them."}, "claim": "The transfer workflow follows the source's semantic/type/architecture dispatch dimensions, demands controlled correctness/performance evaluation, enumerates selection properties absent from PR 6152, and explicitly withholds conclusions for unrelated primitives.", "claim_id": "technique-cccl-memory-primitives-current-c003", "claim_type": "performance", "covers": ["u012", "u013", "u014", "u015", "u016", "u017"], "evidence": [{"locator": "classification, SM80/SM90/SM100 policies, tuning comments and fallbacks", "observation": "The independent dimensions and scoped benchmark comments support exactly the proposed controlled matrix, without a portable speed guarantee.", "path": "artifacts/prs/cccl/PR-3559/key-files/cub/cub/device/dispatch/tuning/tuning_scan.cuh", "revision": "CCCL captured merge 25523da2"}, {"locator": "all changed lines", "observation": "No selection semantics, tie/ordering/NaN/repeatability or architecture result is changed or established, justifying the explicit evidence exclusions.", "path": "artifacts/prs/cccl/PR-6152/diff.patch", "revision": "CCCL captured merge 3fb05826"}], "oracle": "local-primary-source", "verdict": "verified"}], "commands": ["inspect complete captured CCCL 3559/6152 diffs and scan/TopK key files"], "id": "technique-cccl-memory-primitives", "page_verdict": "verified", "path": "wiki/techniques/cccl-memory-primitives.md", "residual_risk": "No GPU benchmark was run; the page reports policy structure and a test protocol, not new performance results.", "scope": "All 17 current semantic units for body hash d1c2e2981a7cbb20812788c814381f9f5a67edcec521a6497f68034d83483b8f were checked against complete captured PR deltas and key files."} +{"claims": [{"challenge": {"alternative": "Chunk IDs in one GPU grid execute in increasing order, so every program can update one shared boundary state directly.", "check": "Compare independent program-instance scheduling with the recurrence's ordered boundary dependency.", "outcome": "The programming model supplies no grid order or implicit grid-wide barrier; the page requires an explicit scan or ordered stages and a token-reference oracle."}, "claim": "A chunkwise linear recurrence can expose parallel local matrix work only after a model-specific equivalent transform, while boundary states must preserve sequence order through a valid scan or explicitly ordered stages rather than an unordered shared-state grid.", "claim_id": "technique-chunk-parallelism-current-c001", "claim_type": "semantics", "covers": ["u001", "u002", "u003", "u004", "u005", "u006", "u007"], "evidence": [{"locator": "Abstract", "observation": "TFLA starts from a chunkwise-parallel formulation of linear RNNs and adds within-chunk sequence parallelism, supporting the separation between local work and recurrent boundary semantics.", "revision": "arXiv:2503.14376v3", "url": "https://arxiv.org/abs/2503.14376v3"}, {"locator": "program instances and execution model", "observation": "Triton programs are independent grid instances; the model does not promise increasing program-ID scheduling or an implicit whole-grid synchronization point.", "revision": "Triton language documentation, retrieved 2026-08-09", "url": "https://triton-lang.org/main/programming-guide/chapter-01/introduction.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "GatedDeltaNet chunking is equivalent to a generic softmax-style QK-transpose/V computation with an additive state.", "check": "Inspect the pinned training implementation and recurrence representation.", "outcome": "The repository uses model-specific chunkwise Triton paths and a WY representation of the gated delta rule; the page states that boundary and does not present generic substitute code."}, "claim": "The pinned GatedDeltaNet training implementation uses chunkwise Triton kernels and a WY representation, providing direct model-specific evidence but no universal generic-attention formula or chunk-size rule.", "claim_id": "technique-chunk-parallelism-current-c002", "claim_type": "mechanism", "covers": ["u008"], "evidence": [{"locator": "chunkwise training paths and WY preparation", "observation": "The repository implements the gated delta rule with dedicated chunkwise Triton kernels and WY transforms rather than the former additive pseudo-kernel.", "revision": "NVlabs/GatedDeltaNet b53d6d3a161267432a79c1c04af69fa52bddc921", "url": "https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "TFLA is a recursive Blackwell/GatedDeltaNet implementation whose arbitrary chunks come from TMEM capacity.", "check": "Inspect the paper's stated algorithm, application, repository target, and benchmark scope.", "outcome": "The paper adds one within-chunk sequence-parallel level, applies it to mLSTM, and reports H100 work; the page explicitly excludes the unsupported Blackwell, GDN, TMEM, and recursive claims."}, "claim": "TFLA adds an additional within-chunk sequence-parallel level for arbitrarily large chunks and reduced intermediate-state materialization, but its published application/code is mLSTM with H100 evidence rather than GatedDeltaNet or a Blackwell/TMEM implementation.", "claim_id": "technique-chunk-parallelism-current-c003", "claim_type": "compatibility", "covers": ["u009"], "evidence": [{"locator": "Abstract and code link", "observation": "The authors state the arbitrary-chunk, intensity, and materialization contributions, identify mLSTM as the application, and publish corresponding official code and H100 benchmarks.", "revision": "arXiv:2503.14376v3; NX-AI/mlstm_kernels 5b98ff8e2bec189b3d3c249405bab5149564d6f8", "url": "https://arxiv.org/abs/2503.14376v3"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Qwen3-Next's 3:1 description is approximate, or the same chunkwise backend covers its full-attention layers.", "check": "Expand the pinned 48-layer type pattern and distinguish linear_attention from full_attention entries.", "outcome": "The pattern yields exactly 36 GDN and 12 full-attention layers, and the page assigns chunkwise GDN only to the former."}, "claim": "The pinned Qwen3-Next 48-layer configuration contains 36 Gated DeltaNet linear-attention and 12 full-attention layers in a repeated 3:1 pattern, so a chunkwise GDN backend does not replace the full-attention implementation or cache.", "claim_id": "technique-chunk-parallelism-current-c004", "claim_type": "compatibility", "covers": ["u010"], "evidence": [{"locator": "config.json layer_types and num_hidden_layers", "observation": "The immutable configuration repeats three linear_attention entries and one full_attention entry across 48 layers.", "revision": "Qwen3-Next-80B-A3B-Instruct 9c7f2fbe84465e40164a94cc16cd30b6999b0cc7", "url": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7/config.json"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "C=32 and C=256-512 are architecture-independent decode/prefill optima, with larger Blackwell TMEM always favoring larger chunks.", "check": "Search the cited algorithms for fixed prescriptions and enumerate correctness, resource, scan, and workload variables changed by chunk size.", "outcome": "The cited sources provide no universal sizes or TMEM rule; the page removes them and defines a controlled multi-shape selection procedure."}, "claim": "No cited evidence establishes universal C=32 or C=256-512 optima; supported chunk sizes must be compared under fixed correctness and benchmarking controls across algorithm, resource, tail, scan, and workload costs, and TMEM availability alone predicts neither legality nor speed.", "claim_id": "technique-chunk-parallelism-current-c005", "claim_type": "performance", "covers": ["u011", "u012", "u013", "u014", "u015", "u016", "u017", "u018"], "evidence": [{"locator": "Abstract and implementation scope", "observation": "TFLA claims support for arbitrarily large chunks in its model-specific algorithm but supplies no fixed decode/prefill optimum or Blackwell/TMEM selection rule.", "revision": "arXiv:2503.14376v3", "url": "https://arxiv.org/abs/2503.14376v3"}, {"locator": "pinned GDN implementation variants", "observation": "The concrete GDN paths encode model dimensions, layouts, gates, and variable-length handling, demonstrating that these are implementation inputs rather than consequences of architecture name alone.", "revision": "NVlabs/GatedDeltaNet b53d6d3a161267432a79c1c04af69fa52bddc921", "url": "https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921"}], "oracle": "web-primary", "verdict": "verified"}], "commands": ["Inspect TFLA v3 and pinned official mLSTM code scope", "Inspect pinned GatedDeltaNet chunkwise/WY paths", "Challenge unordered shared-state pseudocode against Triton's program model", "Expand the immutable Qwen3-Next layer pattern"], "id": "technique-chunk-parallelism", "page_verdict": "verified", "path": "wiki/techniques/chunk-parallelism.md", "residual_risk": "No GPU was available to benchmark candidate chunk sizes. The page deliberately provides a verification procedure and source-scoped implementation boundaries rather than a tuned size or executable kernel.", "scope": "All 18 current body units, the frontmatter Blackwell claim, sequence-ordering invariant, model/repository scopes, and strongest alternatives were checked against direct versioned sources."} +{"claims": [{"challenge": {"alternative": "The kernel only double-buffers one address space or uses the two copies sequentially without overlap.", "check": "Trace stage indices, TMEM offsets, and independently executing warp-role loops.", "outcome": "Different address spaces and regions are active under separate producer/MMA/epilogue roles; the page does not generalize beyond this example."}, "claim": "The page correctly defines buffering as disjoint stage ownership plus completion/reuse edges, distinguishes independent SMEM and TMEM uses, and scopes their simultaneous use to a concrete pinned SM100 kernel.", "claim_id": "technique-double-buffering-current-c001", "claim_type": "mechanism", "covers": ["u001", "u002", "u003", "u004", "u005"], "evidence": [{"locator": "NUM_STAGES arrays, BLOCK_N*2 allocation, TMA/MMA/epilogue role loops and barriers", "observation": "The implementation contains separate multi-stage SMEM storage and disjoint ping-pong TMEM regions with explicit ownership handoffs.", "path": "/tmp/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v6.cu", "revision": "gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Every two-buffer implementation must divide the structure into equal halves or 65,536 cells always mean 65,536 FP32 outputs.", "check": "Separate allocation/address geometry from implementation policy and instruction-kind packing.", "outcome": "The page explicitly calls the split optional and refuses a universal logical-element count."}, "claim": "The TMEM geometry, optional 256/256 address split, and disjoint-live-region invariant agree with PTX ISA 9.0 while avoiding a false universal logical-element capacity.", "claim_id": "technique-double-buffering-current-c002", "claim_type": "code", "covers": ["u006", "u007", "u008"], "evidence": [{"locator": "organization, addressing, allocation, and packing", "observation": "The ISA defines 128 lanes, 512 columns, 32-bit cells, column-wide allocation and instruction-dependent packing; the displayed half-open ranges are arithmetically disjoint.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A fence or CTA barrier alone licenses readback, and kernel exit implicitly releases TMEM.", "check": "Distinguish ordering from tcgen05 commit/completion and inspect the explicit deallocation obligation.", "outcome": "The normative instructions require completion tracking and matching deallocation; the page retains both edges."}, "claim": "The four TMEM state transitions and lifecycle paragraph correctly require compute completion before reads, collective load completion before reuse, phase-aware barriers, collective allocation/address publication, and matching deallocation before exit.", "claim_id": "technique-double-buffering-current-c003", "claim_type": "mechanism", "covers": ["u009", "u010", "u011", "u012", "u013", "u014"], "evidence": [{"locator": "allocation/deallocation collective and lifetime rules", "observation": "The allocation, publication, matching collective cleanup, and before-exit conditions match.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"}, {"locator": "collective load and wait completion", "observation": "tcgen05.ld is asynchronous and its register result requires wait::ld before use; source lifetimes extend through completion.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"}, {"locator": "phase and arrival semantics", "observation": "Reusable barriers operate in phases with expected arrivals, matching the page's reusable-protocol warning.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The full/empty names are cosmetic and no transaction-byte or phase handoff occurs.", "check": "Trace barrier_arrive_tx, waits, and arrive operations for one stage around its TMA and consumer accesses.", "outcome": "The source explicitly attaches bytes to full, waits on matching stages, and returns empty ownership after consumption."}, "claim": "The SMEM full/empty sequence and FlashInfer case accurately describe per-stage producer/consumer ownership; the example's three- and up-to-four-stage policies are source-scoped rather than universal defaults.", "claim_id": "technique-double-buffering-current-c004", "claim_type": "mechanism", "covers": ["u015", "u016", "u017", "u018", "u019", "u023"], "evidence": [{"locator": "SharedStorage/SharedStorageHorizontal, producer and consumer loops, dispatch near numStages", "observation": "The merged source has numStages storage and full/empty barriers, waits before produce/consume/reuse, three stages in one path, and min(totalStages,4) in another.", "path": "artifacts/prs/flashinfer/PR-2387/key-files/include/flashinfer/mamba/selective_state_update.cuh", "revision": "flashinfer merge 18804cd51734cccf807356d017733bc757677f15; SHA-256 40a2de44b1f29b4b4c213e74e530131c9a43be5e602daa3c8bbd901a3b281446"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The total silently includes barriers, padding and epilogue storage.", "check": "Compare the formula's named unpadded payload with the immediately following disclaimer.", "outcome": "Only operand payload is counted, and all omitted allocation components are explicitly named."}, "claim": "The binary16 stage-size block computes exactly 16 KiB for A, 32 KiB for B, 48 KiB per stage, and 144 KiB for three unpadded stages while disclaiming a complete C++ layout.", "claim_id": "technique-double-buffering-current-c005", "claim_type": "code", "covers": ["u020", "u021"], "evidence": [{"command": "conda run -n base python exact integer computation for 128*64*2, 64*256*2, and three-stage totals", "environment": "Conda base, host CPU", "negative_control": "Four-byte elements produce 32/64/96/288 KiB and therefore do not match the labeled binary16 example.", "reference": "binary16 payload uses two bytes per element", "result": "16384 B = 16 KiB, 32768 B = 32 KiB, 49152 B = 48 KiB, and 147456 B = 144 KiB."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "228 KiB is available to every Blackwell CTA or applies to compute capability 12.0 as well.", "check": "Read the table/bullet architecture scope and per-block opt-in discussion.", "outcome": "The page restricts the figure to compute capability 10.0 and avoids a per-CTA or all-Blackwell guarantee."}, "claim": "The resource caveat correctly scopes 228 KiB shared memory per SM to compute capability 10.0 and distinguishes it from per-block opt-in and other residency limits.", "claim_id": "technique-double-buffering-current-c006", "claim_type": "compatibility", "covers": ["u022"], "evidence": [{"locator": "compute capability 10.0 shared-memory and occupancy bullets", "observation": "The guide reports 228 KiB per SM and describes occupancy as dependent on multiple resource limits.", "revision": "CUDA 13.0.2 Blackwell Tuning Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Two fragments require only 64 registers per thread or TMEM removes every register associated with output handling.", "check": "Substitute N=256 in the ISA fragment formula and inspect tcgen05.ld destination/state requirements.", "outcome": "The exact result is 256 f32 D registers for two Hopper fragments; Blackwell removes resident D only, exactly as stated."}, "claim": "The Hopper/Blackwell comparison correctly computes WGMMA FP32 D as N/2 registers per thread, distinguishes resident D from whole-kernel register use, and rejects fixed occupancy conclusions from buffering alone.", "claim_id": "technique-double-buffering-current-c007", "claim_type": "mechanism", "covers": ["u024", "u025", "u026", "u027", "u028", "u029"], "evidence": [{"locator": "m64nNk16 f32 D fragment", "observation": "Each thread has N/2 f32 D registers, giving 128 for N=256 and 256 for two simultaneous fragments.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16"}, {"locator": "TMEM D and register load destination", "observation": "tcgen05 D resides at a TMEM address and loads produce per-thread registers, supporting the scoped distinction.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"}, {"locator": "occupancy resource calculation", "observation": "Actual residency requires complete compiled/launch resource inputs, matching the table's resource-dependent wording.", "revision": "CUDA 13.0.2 Programming Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Correct ownership can be established by steady-state output tests alone or a universal barrier cost makes measurement unnecessary.", "check": "Exercise first reuse and final-tail states and search the normative ISA for a latency guarantee.", "outcome": "Those boundary paths are distinct protocol states, and the ISA contains no portable timing promise; the prescribed tests discriminate both failures."}, "claim": "The controlled-comparison workflow uses concrete compiler, occupancy, profiler, generated-code, wraparound, tail, and negative-test evidence and correctly denies that PTX defines a fixed mbarrier latency.", "claim_id": "technique-double-buffering-current-c008", "claim_type": "performance", "covers": ["u030", "u031", "u032", "u033", "u034", "u035", "u036", "u037"], "evidence": [{"locator": "occupancy APIs", "observation": "Occupancy is evaluated from the concrete kernel and launch resources, supporting the required resource record.", "revision": "CUDA 13.0.2 Programming Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"}, {"locator": "mbarrier semantics", "observation": "The ISA specifies ordering, arrival and phase behavior but no fixed cycle latency.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A redirect hides a missing revision or a similarly named local page satisfies the links.", "check": "Require exact final HTTP success and exact relative target existence.", "outcome": "Both immutable GitHub blob URLs and archived CUDA pages resolved; exact local paths existed while controls failed."}, "claim": "All seven primary links and three related local pages resolve to the intended pinned documents/files, while fabricated external and local controls fail.", "claim_id": "technique-double-buffering-current-c009", "claim_type": "other", "covers": ["u038", "u039", "u040", "u041", "u042", "u043", "u044", "u045", "u046", "u047"], "evidence": [{"command": "HTTP-check seven exact primary URLs; resolve three relative wiki links; repeat with fabricated CUDA and wiki paths", "environment": "current KernelWiki worktree and network, 2026-08-09", "negative_control": "The fabricated CUDA document returned HTTP 404 and wiki/techniques/not-a-page.md was absent.", "reference": "All named sources/pages should resolve and controls should fail.", "result": "All seven exact URLs returned HTTP 200 and all three local files existed."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["conda-base exact payload arithmetic with negative control", "PTX ISA 9.0 and archived tuning-guide inspection", "exact-revision source and artifact hash comparison", "external/local link checks with broken controls"], "id": "technique-double-buffering", "page_verdict": "verified", "path": "wiki/techniques/double-buffering.md", "residual_risk": "No CUDA toolkit or SM90/SM100 GPU is installed locally. The page deliberately gives no performance crossover, occupancy result, or barrier latency; those remain measurements for each concrete kernel.", "scope": "All 47 current units for body hash e32c3558c9bc6483db8f063a150cca5c88247952aa4d168c007310ad16e98baa were checked against archived primary documentation, exact-revision code, arithmetic, and link controls."} +{"claims": [{"challenge": {"alternative": "Epilogue fusion necessarily uses double-buffered overlap or fourteen dedicated warps on SM100.", "check": "Compare the fusion operation interface with a tagged SM100 role schedule.", "outcome": "The operation graph is schedule-independent and the pinned schedule uses four epilogue warps; the page explicitly preserves that distinction."}, "claim": "The page correctly defines fusion as in-kernel output transformation/materialization and separates it from optional concurrent next-tile overlap and any fixed warp-role count.", "claim_id": "technique-epilogue-fusion-current-c001", "claim_type": "mechanism", "covers": ["u001", "u002"], "evidence": [{"locator": "fusion operation expression tags", "observation": "Fusion operations define transformations of accumulator/source/auxiliary values independently of a next-tile overlap schedule.", "path": "/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/operations.hpp", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}, {"locator": "four-warp epilogue group and completed-accumulator path", "observation": "A concrete implementation uses four epilogue warps and a completed TMEM accumulator, disproving any architectural default role count while matching the scoped page.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py", "revision": "same tag"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The excerpt is runnable as written or silently invents the load shape.", "check": "Exact-search every identifier and compare surrounding declarations/dependencies.", "outcome": "All shown APIs and load shape occur at the tag, while ellipsis and the warning unambiguously prevent standalone interpretation."}, "claim": "The TMEM-to-output section accurately requires a legal collective TMEM-to-register copy/load and presents exact CUTLASS 4.5.0 CuTe DSL routing identifiers with an explicit non-standalone warning.", "claim_id": "technique-epilogue-fusion-current-c002", "claim_type": "code", "covers": ["u003", "u004", "u005", "u006"], "evidence": [{"locator": "copy_atom_t2r through epilogue loop and tail", "observation": "The displayed names and sequence occur in the official file; that file also supplies the layouts, participant groups, copy partitions, conversion, SMEM/TMA store pipeline, tail and TMEM free named by the warning.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Warp specialization, __syncthreads, or nonaliasing alone supplies every edge.", "check": "Separate asynchronous proxy completion, register-load completion, address disjointness and lifecycle cleanup.", "outcome": "Each is independent; the page names every required class rather than substituting one primitive for all of them."}, "claim": "All correctness-boundary rows and the multi-region paragraph match tcgen05 MMA completion, collective load completion/source lifetime, mbarrier phase, disjoint-address, output-edge, store-tail, and deallocation obligations.", "claim_id": "technique-epilogue-fusion-current-c003", "claim_type": "mechanism", "covers": ["u007", "u008", "u009", "u010", "u011", "u012"], "evidence": [{"locator": "MMA completion and lifetimes", "observation": "Commit and mbarrier completion are required before dependent TMEM access.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"}, {"locator": "collective load, wait, mapping and source lifetime", "observation": "Loads are collective/asynchronous, require wait::ld, and preserve TMEM source lifetime through completion.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"}, {"locator": "allocation and explicit deallocation", "observation": "TMEM must be collectively allocated and matching storage freed before exit.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Sm100EpilogueTmaWarpSpecialized exists under an indirect include or the builder accepts every operation regardless of constraints.", "check": "Search the entire exact-tag epilogue tree and inspect builder specializations/static assertions.", "outcome": "The name is absent and support is selected through constrained builder/callback specializations; can_implement is correctly required."}, "claim": "The CUTLASS section gives the exact LinCombEltAct parameter order, accurately summarizes CollectiveBuilder's SM100 inputs, rejects a nonexistent type, scopes support to valid combinations, and correctly describes the pinned vLLM builder/carveout usage.", "claim_id": "technique-epilogue-fusion-current-c004", "claim_type": "code", "covers": ["u013", "u014", "u015", "u016", "u017"], "evidence": [{"locator": "LinCombEltAct declaration", "observation": "The displayed six template parameters and defaults match exactly.", "path": "/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/operations.hpp", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}, {"locator": "CollectiveBuilder declaration", "observation": "The described architecture through FusionOpOrCallbacks inputs match the template signature.", "path": "/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/collective_builder.hpp", "revision": "same tag"}, {"locator": "Fp4GemmSm100 CollectiveEpilogue/CollectiveMainloop", "observation": "The wrapper uses EpilogueTileAuto, EpilogueScheduleAuto and StageCountAutoCarveout of CollectiveEpilogue::SharedStorage exactly as stated.", "path": "artifacts/prs/vllm/PR-16032/key-files/csrc/quantization/fp4/nvfp4_scaled_mm_kernels.cu", "revision": "vLLM merge ed7a29d9f8b48978e3bbf43599d21b4de65387e0"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "beta*C is itself always bias or every row is one built-in SM100 visitor.", "check": "Compare distinct operation tags and callback capability flags.", "outcome": "The page separates source and bias, and explicitly denies universal built-in composition."}, "claim": "The operation table correctly separates source linear combination from bias and describes activation, conversion/quantization, gated, online-softmax, and residual dependencies without claiming universal built-in composition.", "claim_id": "technique-epilogue-fusion-current-c005", "claim_type": "mechanism", "covers": ["u018", "u019", "u020", "u021", "u022", "u023", "u024", "u025"], "evidence": [{"locator": "LinearCombination, bias, activation, residual, block-scale and softmax tags", "observation": "Source, explicit bias, activation, auxiliary/residual, quantized scale and reduction-capability distinctions match the table.", "path": "/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/operations.hpp", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}, {"locator": "ReLU, SiLU, GELU functors", "observation": "The named activation functions exist; the page correctly conditions compositions on supported/custom callbacks.", "path": "/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/activation.h", "revision": "same tag"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Removing an intermediate guarantees a win and steady-state correctness implies correct prologue/tail.", "check": "Account for added resources/synchronization and separately exercise first/last/partial stages.", "outcome": "Those costs and states can reverse or invalidate the result; the page explicitly requires controlled comparison and boundary faults."}, "claim": "The overlap/evaluation section requires both ownership directions, boundary/tail tests, controlled baselines, compiled/profiler resource evidence, and rejects fixed warp-bandwidth or guaranteed-speedup claims while preserving the conditional saved-intermediate benefit.", "claim_id": "technique-epilogue-fusion-current-c006", "claim_type": "performance", "covers": ["u026", "u027", "u028", "u029"], "evidence": [{"locator": "load semantics without latency/bandwidth guarantee", "observation": "The ISA supports correctness edges but publishes no equal-share bandwidth or universal performance result.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"}, {"locator": "resource-dependent occupancy", "observation": "Concrete registers, shared memory, block size and launch data are needed for occupancy, supporting the measurement workflow.", "revision": "CUDA 13.0.2 Programming Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Directory name full makes both files upstream-verbatim or the variant is safe production code.", "check": "Use file-level provenance/header rather than directory inference.", "outcome": "The file-level labels establish mixed modes exactly as the page warns."}, "claim": "The artifact section accurately distinguishes the byte-matched vLLM file, historical extracted snippet, and labeled incomplete derived skeleton, and the displayed Conda-base retrieval command succeeds while a nonexistent page fails.", "claim_id": "technique-epilogue-fusion-current-c007", "claim_type": "code", "covers": ["u030", "u031", "u032", "u033", "u034", "u035"], "evidence": [{"command": "sha256sum artifact files; compare nvfp4 file to raw merge revision; inspect headers/provenance; run Conda-base get_page command and nonexistent-page control", "environment": "current worktree and network, 2026-08-09", "negative_control": "scripts/get_page.py no-such-epilogue-page --include-code exited 1.", "reference": "Modes/hashes and command behavior should match the page.", "result": "nvfp4 matches e8aed5... at the full merge SHA; the second full file says extracted; the variant says derived/not upstream; retrieval exited zero with 547 lines."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "Redirects hide missing revisions or similarly named local pages satisfy links.", "check": "Require exact immutable GitHub blob/archive page success and exact resolved local paths.", "outcome": "Every exact target resolved while both controls failed."}, "claim": "All six primary URLs and three related local links resolve exactly; fabricated external and local controls fail.", "claim_id": "technique-epilogue-fusion-current-c008", "claim_type": "other", "covers": ["u036", "u037", "u038", "u039", "u040", "u041", "u042", "u043", "u044"], "evidence": [{"command": "HTTP-check six exact primary URLs; resolve three relative links; repeat with fabricated HTTP and local targets", "environment": "current worktree/network, 2026-08-09", "negative_control": "A fabricated CUDA path returned HTTP 404 and wiki/techniques/not-real.md was absent.", "reference": "Exact pinned/archived sources and intended local pages should resolve.", "result": "All six URLs returned HTTP 200 and all three local files existed."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["exact CUTLASS 4.5.0 and vLLM merge-source inspection", "artifact hash/provenance checks with negative controls", "Conda-base retrieval test", "external/local link checks"], "id": "technique-epilogue-fusion", "page_verdict": "verified", "path": "wiki/techniques/epilogue-fusion.md", "residual_risk": "No CUDA compiler or SM100 GPU is installed locally. The page intentionally contains no throughput, occupancy, participant-count, or TMEM-bandwidth prediction for an unmeasured kernel.", "scope": "All 44 current units for body hash 7b78dc20ff19f53642d3b695375cd2e64bb0f6f55abad1feba792f98a12ad994 were checked against exact-release code, archived PTX, byte hashes and discriminating host controls."} +{"claims": [{"challenge": {"alternative": "A post-Blackwell commit or a shared TMA concept makes these repositories direct SM100 evidence.", "check": "Search pinned build targets and architecture-specific source symbols for SM100/tcgen05 versus SM90a/SM90.", "outcome": "The implementations are explicitly SM90a/SM90; conceptual transfer does not establish SM100 compatibility or performance."}, "claim": "The page correctly treats profiler results as search keys rather than root-cause proof and scopes the pinned modern source maps to Hopper SM90a, requiring separate evidence for Blackwell transfer.", "claim_id": "technique-external-source-map-research-current-c001", "claim_type": "compatibility", "covers": ["u001"], "evidence": [{"locator": "modern example Makefiles and SM90 source symbols", "observation": "Relevant examples target SM90a and use SM90 facilities, while no profiler-causality or SM100 compatibility guarantee is stated.", "path": "/tmp/kernelwiki-ext-cfx", "revision": "ColfaxResearch/cfx-article-src fbecfed88de2e4246f104a023188ba722937c5fc"}, {"locator": "FMHA READMEs and compile scripts", "observation": "The source explicitly identifies Hopper SM90/SM90a and CUDA 12.2/12.3 rather than Blackwell.", "path": "/tmp/kernelwiki-ext-colfax-cutlass", "revision": "ColfaxResearch/cutlass-kernels 84f0802e2b4a1bf068ac70359f20ffdb368c8f6a"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The procedure still silently uses the moving default branch or accepts a wrong commit.", "check": "Execute it from an empty repository and repeat fetch with a nonexistent SHA.", "outcome": "The explicit object fetch/check pins the requested commit, and the invalid-SHA negative control is rejected."}, "claim": "The displayed shell procedure obtains the exact cfx commit, verifies HEAD, and locates the stated TMA/barrier and persistent/Stream-K symbols.", "claim_id": "technique-external-source-map-research-current-c002", "claim_type": "code", "covers": ["u002"], "evidence": [{"command": "Run the displayed init/remote/fetch/checkout/test/rg sequence in a fresh mktemp directory", "environment": "2026-08-09 host with git network access", "negative_control": "Fetching the all-zero nonexistent revision failed with remote 'not our ref'.", "reference": "HEAD must equal fbecfed88de2e4246f104a023188ba722937c5fc and both mechanism searches must return pinned source lines.", "result": "The fetch and detached checkout succeeded, HEAD matched exactly, and rg found SM90_TMA/mbarrier and PersistentTileScheduler/StreamK symbols."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "The exclusions are artifacts of shallow clones or case-sensitive searches rather than real scope limits.", "check": "Inspect complete tracked file lists plus READMEs, build targets, and case-insensitive architecture/mechanism searches at each exact revision.", "outcome": "The positive mechanisms have exact files/symbols, while the excluded scope has no target, implementation, or repository claim."}, "claim": "The page's pinned repository inventory accurately distinguishes cfx SM90 mechanisms, simveit TMA transpose, simveit ldmatrix/stmatrix-only examples, Colfax Hopper GEMM/FMHA, and the cross-generation NVIDIA sample paths.", "claim_id": "technique-external-source-map-research-current-c003", "claim_type": "compatibility", "covers": ["u003", "u004", "u005", "u006", "u007", "u008", "u009", "u010", "u011"], "evidence": [{"locator": "tma/tma_copy.h, pipeline-gemm/, streamk/tile_scheduler.hpp", "observation": "The exact TMA, pipeline, and three scheduler families listed by the page are present.", "path": "/tmp/kernelwiki-ext-cfx", "revision": "ColfaxResearch/cfx-article-src fbecfed88de2e4246f104a023188ba722937c5fc"}, {"locator": "Makefile and transpose_swizzle_batched.cu tensor-map encoder", "observation": "The build target is sm_90a and the tensor map uses CU_TENSOR_MAP_SWIZZLE_128B.", "path": "/tmp/kernelwiki-ext-transpose", "revision": "simveit/effective_transpose 994b2b5acaa67f80e411df3e8274b6ae13fd1949"}, {"locator": "README.md, Makefile and complete tracked file list", "observation": "The six kernels are ldmatrix/stmatrix x1/x2/x4 and compile for sm_90a; the excluded broader mechanisms are absent.", "path": "/tmp/kernelwiki-ext-load-store", "revision": "simveit/load_and_store 05d828cf910dd43f0053ddbbe4744218a06e9d7f"}, {"locator": "src/fmha*/README.md, compile scripts, source tree search", "observation": "The repository contains SM90a GEMM/FMHA TMA/WGMMA pipelines and optional warp specialization, with no persistent/tile scheduler or SM100 implementation.", "path": "/tmp/kernelwiki-ext-colfax-cutlass", "revision": "ColfaxResearch/cutlass-kernels 84f0802e2b4a1bf068ac70359f20ffdb368c8f6a"}, {"locator": "complete file tree and the two page-listed CUDA paths", "observation": "Both exact coalescing and transpose paths exist in a broad historical sample collection.", "path": "/tmp/kernelwiki-ext-nvidia-samples", "revision": "NVIDIA-developer-blog/code-samples 3350d216083a902ccbf5b31665e3b82096a75b55"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Long scoreboard or poor sectors uniquely imply one of the listed memory-layout defects.", "check": "Look for such causal semantics in the pinned sources and compare with their actual experimental variables.", "outcome": "The sources demonstrate mechanisms only, matching the page's explicit non-causal wording."}, "claim": "Coalescing, transpose, and matrix load/store examples are valid candidate sources for long-scoreboard or poor-sector investigations, but those metrics alone do not establish vector width or layout as the cause.", "claim_id": "technique-external-source-map-research-current-c004", "claim_type": "mechanism", "covers": ["u012", "u013"], "evidence": [{"locator": "coalescing-global/coalescing.cu and transpose/transpose.cu", "observation": "The files isolate stride/offset coalescing and shared-memory transpose/padding mechanisms but do not define a unique mapping from profiler symptoms to cause.", "path": "/tmp/kernelwiki-ext-nvidia-samples", "revision": "NVIDIA-developer-blog/code-samples 3350d216083a902ccbf5b31665e3b82096a75b55"}, {"locator": "ldmatrix/stmatrix examples", "observation": "These are concrete matrix movement candidates, not evidence that a particular target kernel uses them or should change vector width.", "path": "/tmp/kernelwiki-ext-load-store", "revision": "simveit/load_and_store 05d828cf910dd43f0053ddbbe4744218a06e9d7f"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Because the mechanisms match the stall labels, copying their stage count or role split must help.", "check": "Inspect whether the pinned source results cover a different target kernel's shapes, resources, and timing.", "outcome": "They do not; the page limits the action to inspection and explicitly withholds a transfer guarantee."}, "claim": "Pinned SM90 TMA and pipelined GEMM paths are appropriate candidate implementations to inspect for barrier/TMA-wait stalls, without guaranteeing that stage or role changes improve another kernel.", "claim_id": "technique-external-source-map-research-current-c005", "claim_type": "mechanism", "covers": ["u014", "u015"], "evidence": [{"locator": "tma/tma_copy.h and pipeline-gemm/sm90_gemm_multistage.cu", "observation": "The sources explicitly implement TMA transaction-barrier handling and staged producer/consumer GEMM flow.", "path": "/tmp/kernelwiki-ext-cfx", "revision": "ColfaxResearch/cfx-article-src fbecfed88de2e4246f104a023188ba722937c5fc"}, {"locator": "README.md and fmha_pipe_ws.h", "observation": "Stage count and optional warp-specialized execution are explicit configuration dimensions, not universal performance prescriptions.", "path": "/tmp/kernelwiki-ext-colfax-cutlass/src/fmha-pipeline", "revision": "ColfaxResearch/cutlass-kernels 84f0802e2b4a1bf068ac70359f20ffdb368c8f6a"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The Stream-K tutorial proves persistent scheduling should replace shape dispatch for every tail wave.", "check": "Inspect the alternative schedulers and configurable problem shapes exposed by the source.", "outcome": "The repository makes the alternatives comparable rather than declaring one universally optimal; the page requires target-shape profiling."}, "claim": "The cfx source contains non-persistent, data-parallel persistent, and Stream-K schedulers suitable for controlled tail-wave comparison, while persistence is not stated as a universal remedy.", "claim_id": "technique-external-source-map-research-current-c006", "claim_type": "mechanism", "covers": ["u016", "u017"], "evidence": [{"locator": "tile_scheduler.hpp classes and cli_options.h scheduler enumeration", "observation": "All three named scheduler families and shape/scheduler command-line controls are present.", "path": "/tmp/kernelwiki-ext-cfx/streamk", "revision": "ColfaxResearch/cfx-article-src fbecfed88de2e4246f104a023188ba722937c5fc"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "A source-card URL or unversioned upstream path is enough because readers can inspect the latest version.", "check": "Compare that alternative with repository drift and the overbroad baseline source-card metadata.", "outcome": "Latest contents and local summaries can change or misstate scope; revision plus exact locator preserves the evidence actually reviewed."}, "claim": "The provenance rule correctly treats local source pages as discovery leads and requires a full immutable upstream revision plus exact file/symbol locator, with separate SM100 evidence for Blackwell transfer.", "claim_id": "technique-external-source-map-research-current-c007", "claim_type": "other", "covers": ["u018"], "evidence": [{"locator": "sections 5 and 8", "observation": "The contract requires revision and exact locator, rejects summaries as primary evidence, and requires version-matched compatibility evidence.", "path": "plan.md", "revision": "repository remediation contract"}, {"locator": "architecture-specific build/source files", "observation": "Pinned Hopper source proves only its recorded scope and does not itself establish SM100 transfer.", "path": "/tmp/kernelwiki-ext-cfx", "revision": "ColfaxResearch/cfx-article-src fbecfed88de2e4246f104a023188ba722937c5fc"}], "oracle": "local-primary-source", "verdict": "verified"}], "commands": ["execute the displayed pinned git fetch/checkout/search procedure with an invalid-revision negative control", "inspect exact revisions and complete tracked trees for all five linked repositories"], "id": "technique-external-source-map-research", "page_verdict": "verified", "path": "wiki/techniques/external-source-map-research.md", "residual_risk": "No GPU benchmark was run. This page verifies source discovery, pinned scope, and evidence discipline; it makes no transferred correctness or performance claim.", "scope": "All 18 current semantic units for body hash e518329cc4548741f8dede0d4a60f5d493cc932595124606962a0e23358c5080 and the corrected linked source scopes were checked."} +{"claims": [{"challenge": {"alternative": "All three are interchangeable block-scaled FP4/FP8 encodings and differ only in group size.", "check": "Compare payload, local/global scale types and operand-specific geometry in both primary sources.", "outcome": "The sources define independent recipe dimensions exactly matching the page's separation."}, "claim": "The overview accurately separates payload, scale type, geometry and layout, and correctly states the DeepSeek FP8, NVFP4 1D/2D, and MXFP4 recipe geometries without promising universal accuracy or speed.", "claim_id": "technique-fine-grained-quantization-current-c001", "claim_type": "semantics", "covers": ["u001", "u002", "u003", "u004", "u005", "u006", "u007", "u008"], "evidence": [{"locator": "section 3.3.2 and Appendix B.2", "observation": "The report assigns 1x128 groups to activations and 128x128 groups to weights and scopes the outlier motivation.", "revision": "DeepSeek-V3 Technical Report v2", "url": "https://arxiv.org/abs/2412.19437v2"}, {"locator": "two-level scaling, 1D/2D scaling, comparison with MXFP4", "observation": "The vendor defines NVFP4's E2M1/block16 E4M3/global-FP32 recipe, 16x16 weight mode, and MXFP4's block32 UE8M0 distinction.", "revision": "Transformer Engine 2.13", "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Both percentages describe the same overhead measure.", "check": "Carry units through numerator and denominator.", "outcome": "One divides object counts while the other divides bytes; they coincide only for a one-byte scale."}, "claim": "The page correctly distinguishes one-factor-per-16384 count ratio (0.0061%) from the 0.0244% byte ratio for one FP32 factor and 16384 one-byte payloads.", "claim_id": "technique-fine-grained-quantization-current-c002", "claim_type": "semantics", "covers": ["u009"], "evidence": [{"command": "awk arithmetic for 100/16384 and 100*4/16384", "environment": "deterministic host arithmetic", "negative_control": "Replacing FP32 with a one-byte factor makes the byte ratio 0.0061035%, discriminating the missing-width error.", "reference": "One factor; FP32 factor is four bytes; FP8 payload is one byte.", "result": "0.0061035% factor-count ratio and 0.0244141% byte ratio."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "Nc=128 is a general Hopper hardware rule or the excerpt uses a half-precision promotion target.", "check": "Inspect declared C++ types and the source paper's scoped wording.", "outcome": "Both arrays are float, and the report describes a specific H800 FP8 strategy rather than a hardware mandate."}, "claim": "The pinned DeepGEMM SM90 description and four-line fragment correctly give FP32 factors, BLOCK_K=128, about 14 retained bits/four-WGMMA source scope, and accum-to-final_accum scale promotion without universalizing Nc.", "claim_id": "technique-fine-grained-quantization-current-c003", "claim_type": "code", "covers": ["u010", "u011", "u012", "u013"], "evidence": [{"locator": "BLOCK_K assertion and lines 250-323", "observation": "The file fixes K=128, uses FP32 factor storage, declares float accum/final_accum, issues the K-block WGMMAs and contains the exact displayed four promotion lines.", "path": "artifacts/kernels/deepgemm/full/sm90_fp8_gemm_1d1d.cuh", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f, byte-verified", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh"}, {"locator": "sections 3.3.2 and 3.5.2", "observation": "The report states about 14 retained bits, Nc=128 inner-dimension elements, and equivalence to four WGMMAs in the described configuration.", "revision": "DeepSeek-V3 Technical Report v2", "url": "https://arxiv.org/abs/2412.19437v2"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Packed factors stay in ordinary registers or shared memory and tcgen05 reads them there directly.", "check": "Trace factor buffers from TMA destination through UTCCP destination and final fma arguments.", "outcome": "UTCCP targets dedicated TMEM columns and the fma wrapper receives those TMEM addresses."}, "claim": "The pinned DeepGEMM SM100 description correctly gives K factor granularities 32/128, four-UE8M0 packing, TMA-to-SMEM and UTCCP-to-TMEM movement, TMEM scale operands, and absence of the SM90 final_accum loop; 0xff UE8M0 is NaN.", "claim_id": "technique-fine-grained-quantization-current-c004", "claim_type": "mechanism", "covers": ["u014"], "evidence": [{"locator": "kGran assertions, factor sizes/TMA loads, UTCCP copies, block-scaled descriptor and mma_t::fma", "observation": "Every stated representation and movement stage appears, and no final_accum symbol exists.", "path": "artifacts/kernels/deepgemm/full/sm100_fp8_gemm_1d1d.cuh", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f, byte-verified", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh"}, {"locator": "interface notices", "observation": "SM100 factors are packed four UE8M0 values per torch.int.", "revision": "DeepGEMM 891d57b", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"}, {"locator": "ue8m0 definition", "observation": "The type is power-of-two for finite encodings and reserves 0xff for NaN.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#alternate-floating-point-data-formats"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Scale operands are SMEM pointers or scale_d belongs before them as in unscaled tcgen05 forms.", "check": "Compare block-scaled grammar with the unscaled floating-point grammar.", "outcome": "Block-scaled syntax names TMEM SFA/SFB operands after idesc and does not use scale_d in that position."}, "claim": "The displayed tcgen05 block-scale line matches PTX ISA 9.0 grammar for mxf8f6f4 scale_vec::1X, including idesc, TMEM scale operands and enable-input-d, and is accurately limited to an instruction-shape reference.", "claim_id": "technique-fine-grained-quantization-current-c005", "claim_type": "code", "covers": ["u015", "u016", "u017"], "evidence": [{"locator": "floating-point type with block scaling grammar and valid combination table", "observation": "The exact qualifier/operand sequence is legal for mxf8f6f4 with UE8M0 scale_vec::1X, and the ISA separately defines descriptors, targets and asynchronous lifecycle requirements.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The mxf4nvf4 kind alone uniquely means NVFP4 UE4M3 block16.", "check": "Enumerate every mxf4nvf4 row in the valid-combination table.", "outcome": "UE8M0 rows also exist, so full qualifiers and types are selection-critical."}, "claim": "The Blackwell table exactly maps mxf4 block32 to E2M1/UE8M0/2X and mxf4nvf4 block16 to E2M1/UE4M3/4X, while correctly noting additional mxf4nvf4 UE8M0 modes and native UE4M3 support.", "claim_id": "technique-fine-grained-quantization-current-c006", "claim_type": "compatibility", "covers": ["u018", "u019", "u020", "u021"], "evidence": [{"locator": "valid scale_vectorsize/type/MMA-kind combinations", "observation": "The table lists both displayed rows and the additional mxf4nvf4 UE8M0 block16/block32 possibilities.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Native instruction support and smaller groups alone are enough to choose the fastest and most accurate recipe.", "check": "Check whether the sources hold preparation/layout/workload constant or define an input-independent ordering.", "outcome": "They do not; the checklist explicitly identifies those uncontrolled variables and requests measurement rather than predicting results."}, "claim": "The transfer checklist and evidence boundaries correctly require exact recipe/ABI/instruction matching, include preprocessing in timing unless already produced, demand controlled accuracy/performance evaluation, and avoid universal block-size or Nc conclusions.", "claim_id": "technique-fine-grained-quantization-current-c007", "claim_type": "performance", "covers": ["u022", "u023", "u024", "u025", "u026", "u027", "u028", "u029", "u030", "u031", "u032", "u033", "u034", "u035", "u036", "u037", "u038", "u039"], "evidence": [{"locator": "recipe, 1D/2D grouping and MXFP4 comparison", "observation": "Payload, local/global scaling and geometry are independent contract dimensions, and documented accuracy advantages are recipe-scoped rather than universal.", "revision": "Transformer Engine 2.13", "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"}, {"locator": "interface notices", "observation": "The project explicitly warns that transpose/cast preparation is separate and utility implementations may be slower, and it distinguishes SM90/SM100 factor formats.", "revision": "DeepGEMM 891d57b", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"}, {"locator": "block-scale grammar, combinations and target notes", "observation": "Legality depends on kind, scale-vector, type and target; the ISA supplies no universal performance/accuracy ordering.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "At least one link is rolling or points to a summary rather than the asserted primary source.", "check": "Inspect URL version/commit segments and resolved section titles.", "outcome": "All four are direct primary, revision-specific targets."}, "claim": "The four primary-reference links resolve to the exact DeepSeek-V3 v2 paper, pinned DeepGEMM commit, Transformer Engine 2.13 page, and archived PTX ISA 9.0 tcgen05 section.", "claim_id": "technique-fine-grained-quantization-current-c008", "claim_type": "other", "covers": ["u040", "u041", "u042", "u043"], "evidence": [{"locator": "document landing/PDF", "observation": "The URL is revision-pinned to v2.", "revision": "v2", "url": "https://arxiv.org/abs/2412.19437v2"}, {"locator": "repository tree", "observation": "The URL embeds the full commit.", "revision": "891d57b4db1071624b5c8fa0d1e51cb317fa709f", "url": "https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f"}, {"locator": "NVFP4 page", "observation": "The version path and topic match.", "revision": "Transformer Engine 2.13", "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"}, {"locator": "tcgen05.mma section", "observation": "The archive and anchor match the label.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "verified"}], "commands": ["inspect byte-verified DeepGEMM SM90/SM100 sources", "compute scale count/byte ratios with a one-byte negative control", "resolve version-pinned primary references"], "id": "technique-fine-grained-quantization", "page_verdict": "verified", "path": "wiki/techniques/fine-grained-quantization.md", "residual_risk": "No CUDA toolkit or NVIDIA GPU is installed, so the grammar-level PTX was not assembled and no accuracy/performance benchmark was run. The page makes no new numeric speed or error claim.", "scope": "All 43 current semantic units for body hash af3e67a9f9a356ddf6111b4a87ae2ba28554f5f096b0b5a9e18e0749b82f6cf8 were checked."} +{"claims": [{"challenge": {"alternative": "Any operation named fused necessarily removes every intermediate and is faster.", "check": "Separate expression composition from storage/schedule choices and baseline behavior.", "outcome": "A concrete kernel can stage, spill, or write values and incur added resources; the page requires those facts to be measured."}, "claim": "The page correctly defines fusion as implementing dependent operations in one launch and makes saved launches/traffic conditional on a pinned fused/unfused pair and actual on-chip lifetime.", "claim_id": "technique-kernel-fusion-current-c001", "claim_type": "performance", "covers": ["u001", "u002"], "evidence": [{"locator": "fusion operation expression types", "observation": "The library composes operations in one kernel but makes no claim about a particular unfused launch count, global-memory baseline, or guaranteed speedup.", "path": "/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/operations.hpp", "revision": "CUTLASS v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The task requires two separately allocated TMEM accumulators because it names dual GEMM.", "check": "Inspect observable reference semantics and task constraints for storage/instruction requirements.", "outcome": "Only input/output math is fixed; a concatenated or separate internal representation can be compatible."}, "claim": "The gated dual-GEMM formula, shared-A/separate-B scale contract, and output semantics match the pinned GPU Mode reference, while launch/TMEM/TMA schedules remain unspecified alternatives.", "claim_id": "technique-kernel-fusion-current-c002", "claim_type": "code", "covers": ["u003", "u004", "u005"], "evidence": [{"locator": "correctness reference", "observation": "It computes two scaled_mm calls sharing A/sfa with distinct B/sfb inputs and applies silu(gate)*up; no launch or TMEM decomposition is part of the contract.", "revision": "gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c", "url": "https://github.com/gpu-mode/reference-kernels/blob/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm/reference.py"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The benchmark title proves submissions use one launch and no global intermediates.", "check": "Distinguish definition/reference semantics from implementation traces or source of a named submission.", "outcome": "The pinned reference contains no launch-count or on-chip-lifetime requirement; the page states that boundary."}, "claim": "The FlashInfer Track A description accurately includes routing, grouped selection, W13/SwiGLU/W2 and weighted accumulation, uses one logical concatenated W13 projection, and does not infer launch topology from the Fused MoE label.", "claim_id": "technique-kernel-fusion-current-c003", "claim_type": "mechanism", "covers": ["u006"], "evidence": [{"locator": "DeepSeek routing reference and _fp8_moe_run_experts", "observation": "The reference implements the listed stages, makes one W13 matmul then chunks it into two halves, and constrains results rather than GPU launch decomposition.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Fusion is legal whenever the operation graph is acyclic and two 256-column TMEM halves fit.", "check": "Check communication scope, asynchronous lifetimes, collective allocation/free and all non-TMEM resource constraints.", "outcome": "Acyclicity and one capacity partition are insufficient; the page enumerates the independent obligations."}, "claim": "The implementation constraints correctly cover semantic preservation, scope-specific communication, resource accounting, SM100's 128x512-cell TMEM capacity and collective lifecycle, and capability-based fallbacks without imposing CTA-only fusion or a 256-column total.", "claim_id": "technique-kernel-fusion-current-c004", "claim_type": "mechanism", "covers": ["u007", "u008", "u009", "u010", "u011", "u012", "u013", "u014", "u015", "u016"], "evidence": [{"locator": "TMEM organization and access/lifetime model", "observation": "TMEM is 128 lanes by 512 32-bit columns per SM and has collective, asynchronous access rules.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"}, {"locator": "alloc/dealloc semantics", "observation": "Allocation uses legal column counts, collective participation/result sharing, and matching deallocation.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"}, {"locator": "resource-dependent occupancy", "observation": "Registers, shared memory and launch dimensions affect residency; deleting a global intermediate does not guarantee reduced on-chip pressure.", "revision": "CUDA 13.0.2 Programming Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Matching a few random dense outputs is enough to validate fusion.", "check": "Exercise cases that isolate routing ties, empty groups, partial tiles and premature asynchronous reuse.", "outcome": "Those cases discriminate semantic/synchronization errors that ordinary dense inputs may not expose."}, "claim": "The correctness workflow covers exact semantic comparison and discriminating boundary, routing, scale/layout, completion, reuse and output-edge tests.", "claim_id": "technique-kernel-fusion-current-c005", "claim_type": "mechanism", "covers": ["u017"], "evidence": [{"locator": "routing and expert reference", "observation": "Routing bias/ties, local filtering, scale dequantization, SwiGLU and weighted accumulation are independent semantic boundaries that a fused result must preserve.", "path": "/tmp/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py", "revision": "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6"}, {"locator": "TMEM completion, epilogue edges, store tail and free", "observation": "The exact schedule contains the completion/lifetime/output boundaries named by the negative tests.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py", "revision": "CUTLASS v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Removing one intermediate is sufficient for a source-independent positive result.", "check": "Include preprocessing, resource changes, synchronization, repeated trials and variance for identical semantics.", "outcome": "Those costs can offset saved bytes; the page predicts no direction and requires measurement."}, "claim": "The performance protocol records the variables needed for a controlled fused/unfused comparison and correctly rejects guaranteed speedup from saved traffic alone.", "claim_id": "technique-kernel-fusion-current-c006", "claim_type": "performance", "covers": ["u018"], "evidence": [{"locator": "occupancy/resource inputs", "observation": "Resource and launch configuration can change residency and performance, supporting the page's required accounting rather than a traffic-only prediction.", "revision": "CUDA 13.0.2 Programming Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A source is mutable or points to a similarly named but different benchmark.", "check": "Inspect commit/archive segments and the full benchmark definition ID.", "outcome": "Three are immutable revision/archive URLs and the definition URL contains the exact Track A identity."}, "claim": "The four primary-source links resolve to the pinned GPU Mode task, FlashInfer Track A definition, pinned FlashInfer reference, and archived PTX ISA 9.0 Tensor Memory section.", "claim_id": "technique-kernel-fusion-current-c007", "claim_type": "other", "covers": ["u019", "u020", "u021", "u022"], "evidence": [{"locator": "task directory", "observation": "The commit-pinned task resolves.", "revision": "c5b2f7c062d5015f29c3a1043cfd04954397944c", "url": "https://github.com/gpu-mode/reference-kernels/tree/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm"}, {"locator": "exact definition", "observation": "The exact Track A definition resolves.", "revision": "checked 2026-08-08", "url": "https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048"}, {"locator": "file", "observation": "The pinned reference resolves.", "revision": "7f614b86470180bab2d22e36fd1775791c6bf3e6", "url": "https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/trace/templates/moe.py"}, {"locator": "Tensor Memory", "observation": "The archived section resolves.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A relative target resolves to an unrelated page.", "check": "Resolve from wiki/techniques and inspect IDs.", "outcome": "Both labels and IDs agree."}, "claim": "The two related links resolve to the intended fused-MoE and epilogue-fusion pages.", "claim_id": "technique-kernel-fusion-current-c008", "claim_type": "other", "covers": ["u023", "u024"], "evidence": [{"command": "test -f wiki/kernels/fused-moe.md && test -f wiki/techniques/epilogue-fusion.md", "environment": "current worktree", "negative_control": "wiki/kernels/not-fused.md does not exist.", "reference": "Resolve links and compare frontmatter IDs.", "result": "Targets exist with IDs kernel-fused-moe and technique-epilogue-fusion."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["inspect pinned task/reference sources", "resolve related links with broken-target negative control"], "id": "technique-kernel-fusion", "page_verdict": "verified", "path": "wiki/techniques/kernel-fusion.md", "residual_risk": "No GPU profile was run. The page states correctness/resource protocols and source contracts, not a launch count, byte saving or speed result.", "scope": "All 24 current units for body hash affee601d50f5bf6e00362f9728a695fb9a7a2806e5e295158ebf1ffe1d90954 were checked."} +{"claims": [{"challenge": {"alternative": "Persistent necessarily means launch exactly one CTA per SM.", "check": "Compare the launched problem-sized CLC grid with the number of simultaneously resident workers.", "outcome": "The grid can exceed or differ from SM count while resident workers process multiple ClcIDs, matching the page's distinction."}, "claim": "The persistence definition correctly requires repeated logical work by a resident worker without fixing launch count to physical SM count.", "claim_id": "technique-persistent-kernels-current-c001", "claim_type": "mechanism", "covers": ["u001"], "evidence": [{"locator": "persistent kernel and dynamic scheduler description", "observation": "Workers can process multiple tiles; launched grid and resident execution are distinct, with grid/cluster shape determined by scheduler/problem policy.", "revision": "CUTLASS 4.5.0 e406c186", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "CLC assigns every tile from a queue, creates new work, or cancels the requester when empty.", "check": "Count the two documented ClcID paths and inspect the target/result of try_cancel.", "outcome": "Every returned ID was already a not-yet-launched grid entity and the requester continues to process it; no other behavior is claimed."}, "claim": "The CLC scope and two-path model correctly preserve the problem-sized grid, initial blockIdx, cancellation of another unlaunched entity, ClcID conservation, uneven-SM motivation, and exclusions for self-cancel/problem chaining.", "claim_id": "technique-persistent-kernels-current-c002", "claim_type": "mechanism", "covers": ["u002", "u003", "u004", "u005"], "evidence": [{"locator": "Dynamic Scheduler rules 1-4 and motivation", "observation": "The official rules establish full-grid launch, initial blockIdx, success-only canceled-ID work and uneven-availability motivation.", "revision": "CUTLASS 4.5.0 e406c186", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A failed response may be retried or get_first_ctaid decoded regardless of is_canceled.", "check": "Read the explicit undefined-behavior restrictions after request failure.", "outcome": "Both operations are prohibited as described, and the page makes failure terminal."}, "claim": "The exact PTX forms and request lifecycle correctly specify the 16-byte asynchronous response, mbarrier completion, success-only decode, proxy/lifetime requirements, terminal failed request, and cluster-rank derivation.", "claim_id": "technique-persistent-kernels-current-c003", "claim_type": "code", "covers": ["u006", "u007", "u008", "u009", "u010"], "evidence": [{"locator": "try_cancel/query grammars, transaction, ordering, failure and cluster semantics", "observation": "All three displayed forms and every stated lifecycle/failure/cluster condition match the normative section.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The final partial wave skips tiles or assigns one tile to two CTAs.", "check": "Enumerate cases with remainders 1 and gridDim.x-1.", "outcome": "Only per-CTA loop lengths differ; coverage and uniqueness remain exact."}, "claim": "The static-stride code covers each valid flat tile exactly once for positive grid size, including nondivisible tails, and maps row-major coordinates by quotient/remainder.", "claim_id": "technique-persistent-kernels-current-c004", "claim_type": "code", "covers": ["u011", "u012", "u013"], "evidence": [{"command": "conda run -n base python enumerate CTA/iteration assignments for divisible and nondivisible total_tiles/gridDim.x pairs", "environment": "Conda base, host CPU", "negative_control": "Replacing the stride with gridDim.x-1 produced duplicates.", "reference": "Sorted assignments should equal range(total_tiles) with count one each.", "result": "All tested positive grid sizes had exact unique coverage and correct quotient/remainder coordinates."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "Swizzle/raster is programmed into CLC hardware during an init call.", "check": "Inspect raw PTX operands and scheduler coordinate transformation.", "outcome": "Raw CLC receives response/mbarrier addresses only; CUTLASS transforms coordinates in software as stated."}, "claim": "The CUTLASS 4.5.0 route names the real scheduler/pipeline/work methods, assigns swizzle/raster to software policy, and rejects the invented clc_init/query_tile/self-cancel API.", "claim_id": "technique-persistent-kernels-current-c005", "claim_type": "compatibility", "covers": ["u014", "u015", "u016"], "evidence": [{"locator": "PersistentTileSchedulerSm100 methods and swizzle_and_rasterize", "observation": "WorkTileInfo, get_current_work, advance_to_next_work, fetch_next_work and the software transform are present; the rejected pseudo-helpers are absent.", "revision": "CUTLASS 4.5.0 e406c186", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp"}, {"locator": "PipelineCLCFetchAsync integration", "observation": "The documentation maps scheduler producer/consumer behavior through the named CLC pipeline.", "revision": "same tag", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The endpoint delta can still be called the persistence speedup.", "check": "List every change between v3 and v6 and inspect v6 scheduling code.", "outcome": "Several performance-relevant changes intervene and v6 is statically scheduled, so the page correctly refuses causal attribution."}, "claim": "The tutorial paragraph reports the exact v3/v6 author numbers and correctly scopes v6 as a multi-change static-persistence endpoint that explicitly excludes CLC/swizzling and therefore is not an isolated CLC result.", "claim_id": "technique-persistent-kernels-current-c006", "claim_type": "performance", "covers": ["u017"], "evidence": [{"locator": "source-reported progression and supported takeaways", "observation": "v3=939.61 and v6=1475.93 for the disclosed experiment; the source card records intermediate warp-specialization/2SM changes and the author's explicit omission of CLC/swizzling.", "path": "sources/blogs/tcgen05-tutorial.md", "revision": "Gau Nernst article and source commit 3b90ac9b3f624bdf1f6f78d02dcd533675d36573"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "CLC alone can fill unused SMs by splitting K or generating coordinates.", "check": "Compare raw request results with scheduler decomposition state.", "outcome": "CLC returns only existing ClcIDs; Stream-K is correctly identified as a separate transformation."}, "claim": "The CLC evaluation/negative-control guidance follows the conserved-grid work model, requests controlled static/dynamic evidence, and correctly separates additional Stream-K decomposition from CLC.", "claim_id": "technique-persistent-kernels-current-c007", "claim_type": "performance", "covers": ["u018", "u019", "u020"], "evidence": [{"locator": "motivation, ClcID rules and software scheduler scope", "observation": "Uneven availability, initial/returned coordinate consistency and finite launched ClcIDs motivate exactly the proposed controls.", "revision": "CUTLASS 4.5.0 e406c186", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}, {"locator": "failure/query restrictions", "observation": "The negative controls intentionally violate explicit normative requirements and should be detected.", "revision": "PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The derived skeleton is safe upstream code or the exact source revision differs.", "check": "Inspect first line and byte-compare the purported upstream file.", "outcome": "The variant self-identifies as derived/incomplete while the upstream file hash matches exactly."}, "claim": "Artifact hashes/modes and the Conda-base retrieval command are accurate: the PR 2881 source byte-matches its full merge SHA, the skeleton is labeled derived, and a nonexistent page control fails.", "claim_id": "technique-persistent-kernels-current-c008", "claim_type": "code", "covers": ["u021", "u022", "u023", "u024", "u025"], "evidence": [{"command": "sha256sum artifacts; raw-file byte comparison; inspect provenance/header; Conda-base get_page plus nonexistent ID", "environment": "current worktree/network, 2026-08-09", "negative_control": "A nonexistent page ID exited nonzero.", "reference": "Page hashes/modes and retrieval behavior", "result": "All manifest hashes match; dense_gemm file equals merge 3f4c086d09bd1dc55defb955862f333893bbb28b; variant label is explicit; command exited zero with 2427 lines."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "Redirects or similarly named pages mask missing exact targets.", "check": "Require exact immutable/archive URL success and exact relative file existence.", "outcome": "Every target resolved and controls failed."}, "claim": "All four primary links and three related local links resolve to exact intended targets; fabricated external/local controls fail.", "claim_id": "technique-persistent-kernels-current-c009", "claim_type": "other", "covers": ["u026", "u027", "u028", "u029", "u030", "u031", "u032"], "evidence": [{"command": "HTTP-check four exact URLs; resolve three relative pages; run fabricated controls", "environment": "current worktree/network, 2026-08-09", "negative_control": "A fabricated local technique was absent and a fabricated CUDA URL returned HTTP 404.", "reference": "Pinned/archived and exact local targets should resolve.", "result": "All four URLs returned HTTP 200 and all three related pages existed."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["PTX ISA 9.0 and CUTLASS 4.5.0 exact-source inspection", "Conda-base static-stride enumeration", "author tutorial variant comparison", "artifact/link/retrieval checks with negative controls"], "id": "technique-persistent-kernels", "page_verdict": "verified", "path": "wiki/techniques/persistent-kernels.md", "residual_risk": "No CC10.0 GPU is available locally. CLC benefit, scheduler overhead and work-distribution effects remain explicitly unquantified pending controlled target measurements.", "scope": "All 32 current units for body hash 5ea652650db0a472fb026f18b1244edf0e01a1cee16cfca178b5c2982608b1d2 were verified against pinned primary sources and discriminating host checks."} +{"claims": [{"challenge": {"alternative": "The two-stage schedule is ordinary buffer ping-pong and the paper promises complete MMA/softmax overlap.", "check": "Compare the paper's wording and the pinned implementation's per-stage state with the page's bounded definition.", "outcome": "The paper says the schedule maximizes overlap, while the source gives each query stage distinct softmax/output state; the page expressly avoids a stall-free guarantee."}, "claim": "In the pinned FA4 SM100 forward implementation, q_stage=2 covers two 128-row query/output tiles and interleaves one tile's MMA with the other's softmax using one MMA warp, two softmax warpgroups, and a correction warpgroup.", "claim_id": "technique-ping-pong-scheduling-current-c001", "claim_type": "semantics", "covers": ["u001", "u002"], "evidence": [{"locator": "section 3.1.2", "observation": "The paper describes two output tiles per block, with tensor-core work for one tile overlapping softmax for the other, and identifies two softmax groups plus a correction group.", "revision": "FlashAttention-4 paper v1, 2026-03-05", "url": "https://arxiv.org/html/2603.05451v1"}, {"locator": "q_stage default, cta_tiler, and warp-role definitions", "observation": "The source defaults to q_stage=2 and m_block=128, extends the CTA query dimension by q_stage, and assigns the stated specialized roles.", "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", "url": "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The role table and checklist overstate requirements that are merely CuTe implementation details.", "check": "Separate source-specific role IDs from PTX lifecycle/completion requirements and inspect both against the page wording.", "outcome": "The page labels role IDs as implementation choices; its completion and TMEM-lifetime requirements match the ISA, while its remaining handoffs match the pinned source's explicit pipelines."}, "claim": "The pinned default source uses the listed warp roles, one collective TMEM allocation with disjoint per-stage regions, and multiple producer/consumer pipelines; a correct adaptation must preserve completion, ownership, visibility, lifetime, boundary, and tail obligations.", "claim_id": "technique-ping-pong-scheduling-current-c002", "claim_type": "code", "covers": ["u003", "u004", "u005", "u006", "u007", "u008", "u009", "u010", "u011", "u012", "u013", "u014", "u015", "u016", "u017", "u018", "u019"], "evidence": [{"locator": "warp IDs, TmemAllocator, TMEM offsets, and pipeline producer/consumer states", "observation": "The implementation assigns softmax0 to warps 0-3, softmax1 to 4-7, correction to 8-11, MMA/load/epilogue to 12/14/13, leaves 15 empty in the default layout, allocates one TMEM region, and builds distinct pipelines and stage states.", "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", "url": "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py"}, {"locator": "tcgen05 allocation, MMA, commit, fence, load, and deallocation contracts", "observation": "PTX specifies collective TMEM lifetime and the completion and ordering machinery required around asynchronous tcgen05 operations; an unrelated barrier is not a substitute.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The B200-versus-Hopper throughput figures or the simplified cycle counts were transcribed from a secondary summary.", "check": "Read the version-pinned paper's hardware table and worked roofline example directly.", "outcome": "The primary paper gives 8192 versus 4096 BF16 MMA ops/clock/SM, 16 exponential ops/clock/SM for both, and the stated 1024/1024/768-cycle M=N=d=128 example."}, "claim": "FA4's paper models B200 at twice Hopper's BF16 tensor-core operations per clock per SM while exponential throughput is unchanged, and gives a simplified 128-by-128 example with 1024 MMA, 1024 exponential, and 768 shared-memory cycles.", "claim_id": "technique-ping-pong-scheduling-current-c003", "claim_type": "performance", "covers": ["u020"], "evidence": [{"locator": "sections 2.2 and 3.1.1", "observation": "The paper supplies the per-SM rates, example cycle counts, doubled-query scaling, and the caveat that its simplified roofline omits other resources.", "revision": "FlashAttention-4 paper v1, 2026-03-05", "url": "https://arxiv.org/html/2603.05451v1"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The reported peak and utilization isolate ping-pong scheduling because it is the section's central technique.", "check": "Inspect what system each source benchmarks and whether either includes a q_stage or ping-pong-only ablation.", "outcome": "Both report complete FA4 forward kernels with multiple simultaneous techniques and no isolated ping-pong timing; the page states that limitation."}, "claim": "The FA4 paper reports up to 1613 TFLOP/s and 71% for complete forward kernels, while the author blog reports up to 1605 TFLOP/s and 71%; neither result isolates ping-pong scheduling or proves 100% unit utilization.", "claim_id": "technique-ping-pong-scheduling-current-c004", "claim_type": "performance", "covers": ["u021"], "evidence": [{"locator": "abstract and performance results", "observation": "The paper reports up to 1613 TFLOP/s and 71% for FA4 as a complete implementation.", "revision": "FlashAttention-4 paper v1, 2026-03-05", "url": "https://arxiv.org/html/2603.05451v1"}, {"locator": "Performance and Key Techniques", "observation": "The author blog reports up to 1605 TFLOP/s and 71% and lists several techniques contributing to the complete kernel, without a ping-pong-only ablation.", "revision": "Tri Dao FA4 blog, 2026", "url": "https://tridao.me/blog/2026/flash4/"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "q_stage=1 is not a supported control, or the pattern is Blackwell-only because the pinned file targets SM100.", "check": "Inspect accepted q_stage values in source and the paper's stated scheduling lineage.", "outcome": "The source accepts q_stage 1 or 2 and adjusts roles for one stage; the paper explicitly says the schedule is similar to FA3's Hopper schedule."}, "claim": "The pinned SM100 implementation supports q_stage=1 as a controlled comparison to q_stage=2, and the broader scheduling idea is not intrinsically Blackwell-only because the paper traces it to Hopper-era FA3.", "claim_id": "technique-ping-pong-scheduling-current-c005", "claim_type": "compatibility", "covers": ["u022", "u023"], "evidence": [{"locator": "q_stage validation and role reassignment", "observation": "The implementation restricts q_stage to 1 or 2 and conditionally changes its role layout for q_stage=1.", "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", "url": "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py"}, {"locator": "section 3.1.2", "observation": "The paper says FA4 follows a ping-pong schedule similar to FlashAttention-3, which targeted Hopper.", "revision": "FlashAttention-4 paper v1, 2026-03-05", "url": "https://arxiv.org/html/2603.05451v1"}], "oracle": "web-primary", "verdict": "verified"}], "commands": ["Inspect FA4 paper v1 hardware model, pipeline, and performance scope", "Inspect pinned Dao-AILab SM100 q_stage options, role assignments, TMEM offsets, and pipelines", "Compare lifecycle requirements with PTX ISA 9.0 tcgen05 contracts"], "id": "technique-ping-pong-scheduling", "page_verdict": "verified", "path": "wiki/techniques/ping-pong-scheduling.md", "residual_risk": "The source was inspected but not compiled or profiled because this host has no CUDA toolkit or GPU. Performance remains author-reported and the recommended q_stage comparison is a procedure, not a measured local result.", "scope": "All 23 current body units for hash 4553571158ca2be094f2a4198252e45b48298e0e318c60760f7f547f16ed438d were reviewed against the version-pinned paper, implementation, blog, and PTX lifecycle contract."} +{"claims": [{"challenge": {"alternative": "Modulo indices alias one buffer or stage count itself proves complete hiding.", "check": "Trace storage dimensions and distinguish correctness structure from timing evidence.", "outcome": "Storage is disjoint but no source-level invariant establishes hidden latency, exactly as the page states."}, "claim": "The definition correctly makes stages disjoint SMEM buffers with independently transferred ownership and explicitly declines to guarantee full latency hiding or speedup.", "claim_id": "technique-pipeline-stages-current-c001", "claim_type": "mechanism", "covers": ["u001"], "evidence": [{"locator": "state[numStages] and stage-specific full/empty barriers", "observation": "The implementation allocates disjoint stages and transfers them between producer/consumer roles; source structure alone does not claim a universal performance result.", "path": "artifacts/prs/flashinfer/PR-2387/key-files/include/flashinfer/mamba/selective_state_update.cuh", "revision": "flashinfer merge 18804cd51734cccf807356d017733bc757677f15"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Modulo indexing and __syncthreads after issue are sufficient.", "check": "Follow first use, first wraparound and asynchronous MMA operand lifetime.", "outcome": "Modulo has no readiness state and CTA synchronization does not complete the async proxy; all extra edges named by the page are required."}, "claim": "The five-state table and lifecycle paragraphs correctly require prior-read completion, expected TMA transaction accounting, ready-phase observation, exclusive consumer ownership, tcgen05 completion before release, and explicit prologue/phase/tail handling.", "claim_id": "technique-pipeline-stages-current-c002", "claim_type": "mechanism", "covers": ["u002", "u003", "u004", "u005", "u006", "u007", "u008", "u009"], "evidence": [{"locator": "expected arrivals/transactions, phases and waits", "observation": "Full/empty ownership and reusable phase obligations match.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"}, {"locator": "MMA completion and operand lifetimes", "observation": "SMEM operands cannot be released for overwrite merely after asynchronous MMA issue; completion must be established.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "TMA completion needs a manual producer arrival after transfer or proves the producer cannot be critical.", "check": "Separate hardware complete_tx behavior from performance scheduling.", "outcome": "Completion is autonomous, while issue work and load latency may still be exposed; the page states both precisely."}, "claim": "The TMA section correctly states expected-byte registration, autonomous mbarrier transaction completion and consumer wait, while narrowing the benefit and rejecting incomplete illustrative PTX as safe code.", "claim_id": "technique-pipeline-stages-current-c003", "claim_type": "mechanism", "covers": ["u010", "u011", "u012"], "evidence": [{"locator": "bulk tensor copy mbarrier complete_tx semantics", "observation": "The hardware transaction completes on the supplied mbarrier after expected-byte accounting; full descriptors/grammar and proxy ordering are required.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The calculation includes barriers/padding or total allocation is purely linear.", "check": "Read the labels/disclaimer and add a fixed metadata term.", "outcome": "The page explicitly says unpadded operand payload and notes fixed/stage-dependent metadata separately."}, "claim": "The binary16 payload block and linear-growth statement compute 16/32/48/144/240 KiB exactly while distinguishing payload from complete allocation.", "claim_id": "technique-pipeline-stages-current-c004", "claim_type": "code", "covers": ["u013", "u014", "u016"], "evidence": [{"command": "conda run -n base python exact integer byte arithmetic and stage-count sweep", "environment": "Conda base, host CPU", "negative_control": "Four-byte elements or aliased stages fail the stated byte totals/simultaneous-ownership condition.", "reference": "binary16=2 bytes and disjoint fixed-size operand stage payload", "result": "A=16384 B, B=32768 B, stage=49152 B; 3=144 KiB, 5=240 KiB; stage payload is linear."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "228 KiB is a universal Blackwell per-CTA allocation guarantee.", "check": "Distinguish compute capability and per-SM versus per-block opt-in limits.", "outcome": "The page states CC10.0 per SM and explicitly retains other limits."}, "claim": "The 228-KiB capacity and derived 63.2-percent/over-capacity statements are correctly scoped to CC10.0 per-SM capacity and preserve per-block/occupancy caveats.", "claim_id": "technique-pipeline-stages-current-c005", "claim_type": "compatibility", "covers": ["u015"], "evidence": [{"locator": "CC10.0 shared-memory capacity and occupancy", "observation": "The guide reports 228 KiB/SM and resource-dependent residency; 144/228=63.16% and 240>228.", "revision": "CUDA 13.0.2 Blackwell Tuning Guide", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Three stages is always standard/full and no compilation or short-K test is needed.", "check": "Compare exact code paths and execute the state machine when K tiles are below stage count.", "outcome": "Source policies differ and some stages are never produced in short loops, requiring the specified boundary tests."}, "claim": "The stage-selection section correctly rejects universal depth labels and enumerates the concrete resource, transfer, compute, role, and loop-length variables that must be compiled and measured, including short/equal/long boundary cases.", "claim_id": "technique-pipeline-stages-current-c006", "claim_type": "performance", "covers": ["u017", "u018", "u019", "u020", "u021", "u022", "u023"], "evidence": [{"locator": "three-stage and min(totalStages,4) dispatch policies", "observation": "One exact kernel uses different stage policies by path/geometry, validating that depth depends on concrete constraints.", "path": "artifacts/prs/flashinfer/PR-2387/key-files/include/flashinfer/mamba/selective_state_update.cuh", "revision": "flashinfer merge 18804cd51734cccf807356d017733bc757677f15"}, {"locator": "pipeline stage parameters and resource construction", "observation": "Stage choices interact with tile/storage/pipeline configuration rather than an architecture-only table.", "path": "/tmp/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "1475.93 is the warp-specialization result or the 35.1% is portable.", "check": "Inspect each intermediate version and benchmark scope.", "outcome": "v4 is 1208.83, v6 adds two more changes, and the page limits the delta to the disclosed source progression."}, "claim": "The Gau Nernst paragraph accurately reports/scopes v2b/v3 and later variant values, including the approximately 35.1-percent adjacent pipeline increase without attributing the v6 endpoint to warp specialization alone.", "claim_id": "technique-pipeline-stages-current-c007", "claim_type": "performance", "covers": ["u024"], "evidence": [{"locator": "source-reported progression", "observation": "All four numbers and variant labels match; 939.61/695.43-1 is about 35.1%.", "path": "sources/blogs/tcgen05-tutorial.md", "revision": "article/source commit 3b90ac9b3f624bdf1f6f78d02dcd533675d36573"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "Five stages alone produced 85% or the page omits a source-defined latency.", "check": "Trace the article's kernel sequence and search its content for 400/cycles.", "outcome": "Several changes precede the endpoint and the latency phrase is absent; the current wording is properly scoped."}, "claim": "The Modular paragraph accurately scopes five input stages and the 85-percent endpoint to the full multi-step Part 3 design and correctly states that the source lacks the previously attributed fixed 400-cycle premise.", "claim_id": "technique-pipeline-stages-current-c008", "claim_type": "performance", "covers": ["u025"], "evidence": [{"locator": "five-stage input pipeline, warp specialization, output double buffering and 85% endpoint", "observation": "The sequence and source-scoped endpoint match, and no fixed 400-cycle HBM statement appears.", "revision": "fetched 2026-08-09", "url": "https://www.modular.com/blog/matrix-multiplication-on-nvidias-blackwell-part-3-the-optimizations-behind-85-of-sota-performance"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Only steady-state happy-path output is needed.", "check": "Inject wrong phase, early release and short-K states.", "outcome": "Each targets a distinct normative edge and can expose deadlock or corruption hidden by steady state."}, "claim": "All eight validation steps follow from disjoint storage, initialization/publication, transaction, async MMA, phase, tail and boundary requirements and specify discriminating fault tests.", "claim_id": "technique-pipeline-stages-current-c009", "claim_type": "mechanism", "covers": ["u026", "u027", "u028", "u029", "u030", "u031", "u032", "u033"], "evidence": [{"locator": "init, expected transaction, phase and wait rules", "observation": "Steps 1-4 and 6 directly exercise mandatory state/phase behavior.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"}, {"locator": "MMA completion/lifetimes", "observation": "Step 5 and early-release fault test follow from asynchronous operand lifetime; tail/boundary tests ensure no unproduced stage is waited upon.", "revision": "same archive", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Redirects or similar names mask missing exact targets.", "check": "Require exact archived/immutable URLs and exact relative paths.", "outcome": "Every intended target resolved and both controls failed."}, "claim": "All six primary URLs and three related local links resolve exactly, while fabricated external/local controls fail.", "claim_id": "technique-pipeline-stages-current-c010", "claim_type": "other", "covers": ["u034", "u035", "u036", "u037", "u038", "u039", "u040", "u041", "u042"], "evidence": [{"command": "HTTP-check six exact references; resolve three relative links; repeat with fabricated targets", "environment": "current worktree/network, 2026-08-09", "negative_control": "A fabricated CUDA page returned HTTP 404 and wiki/techniques/nope.md was absent.", "reference": "Exact primary and related targets should resolve.", "result": "All six URLs returned HTTP 200 and all three local files existed."}], "oracle": "host-experiment", "verdict": "verified"}], "commands": ["archived PTX/tuning-guide inspection", "exact-revision CUTLASS/FlashInfer/tutorial inspection", "Conda-base byte/stage arithmetic", "primary article and link controls"], "id": "technique-pipeline-stages", "page_verdict": "verified", "path": "wiki/techniques/pipeline-stages.md", "residual_risk": "No CUDA compiler or SM100 GPU is available. The page intentionally leaves stage profitability, barrier overhead and exposed latency to target compilation/profiling.", "scope": "All 42 current units for body hash 7df323122ab18859d04251398e4b248898e5bca6a3c58fba24d67e3ba0fd70d2 were verified against pinned primary evidence and discriminating host checks."} +{"claims": [{"challenge": {"alternative": "Registers alone determine occupancy, so an inverse rule is sufficient.", "check": "Compare the occupancy API inputs and launch-bounds threshold behavior with the page's resource model.", "outcome": "The official API consumes a compiled function, block size, and dynamic shared memory, while the guide describes discrete resident-block thresholds; the page states both limits and avoids a performance guarantee."}, "claim": "Register use is one of several discrete residency constraints, and CUDA's occupancy API predicts active blocks for the compiled function and concrete launch inputs without predicting performance.", "claim_id": "technique-register-budgeting-current-c001", "claim_type": "semantics", "covers": ["u001", "u002"], "evidence": [{"locator": "section 8.2.3.1", "observation": "The guide defines active-block occupancy calculation from a kernel, block size, and shared-memory usage, and separately describes register thresholds between n and n+1 resident blocks.", "revision": "CUDA C++ Programming Guide 13.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html"}, {"locator": "cudaOccupancyMaxActiveBlocksPerMultiprocessor", "observation": "The runtime contract returns maximum active blocks per SM for a function, block size, and dynamic shared-memory size.", "revision": "CUDA Runtime API 13.0.2", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-runtime-api/group__CUDART__OCCUPANCY.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The second launch-bounds argument forces that many blocks and therefore fixes the register count.", "check": "Trace the compiler's documented derivation and both above-L and below-L cases.", "outcome": "The compiler derives L, can reduce use with spill/instruction costs, and can increase use up to L; the page's table and prose preserve those exact limits."}, "claim": "maxrregcount is a maximum subject to ABI/compiler constraints, two-argument launch bounds guide an architecture-dependent limit rather than exact residency, resource-usage reports compiled resources, and the occupancy API does not measure runtime performance.", "claim_id": "technique-register-budgeting-current-c002", "claim_type": "mechanism", "covers": ["u003", "u004", "u005", "u006", "u007"], "evidence": [{"locator": "--maxrregcount and --resource-usage", "observation": "The driver documents the maximum, ABI bump, compiler-reserved registers, tradeoff, and the exact resource report contents.", "revision": "CUDA Compiler Driver 13.0.2", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html"}, {"locator": "Launch Bounds, lines 8688-8713", "observation": "The guide defines L and both compiler response branches, and says the optimal values require per-kernel analysis and usually change across architectures.", "revision": "CUDA C++ Programming Guide 13.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A cap sweep can be interpreted from timings alone without recording compiler output or controlling correctness and launch inputs.", "check": "Test whether a nominal cap necessarily changes the compiled allocation and whether occupancy APIs accept source-level categories.", "outcome": "The compiler may naturally use fewer registers than the cap, while the API requires the concrete function and launch; the workflow records and discriminates both before attribution."}, "claim": "The shown nvcc commands validly compile an uncapped and 64-register-cap SM100a variant with resource reports, and the five-step procedure controls the evidence needed to attribute a cap effect.", "claim_id": "technique-register-budgeting-current-c003", "claim_type": "code", "covers": ["u008", "u009", "u010", "u011", "u012", "u013", "u014", "u015"], "evidence": [{"locator": "--gpu-architecture, --maxrregcount and --resource-usage", "observation": "The archived driver documents sm_100a as a target and the exact long-form options used by the two commands.", "revision": "CUDA Compiler Driver 13.0.2", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html"}, {"locator": "Occupancy Calculator", "observation": "The official procedure queries the compiled function with block size and dynamic shared-memory bytes and converts returned blocks to active warps.", "revision": "CUDA C++ Programming Guide 13.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A memory-bound kernel can always hide spills, and any lower cap should increase occupancy.", "check": "Inspect spill representation and a cap that remains above natural allocation.", "outcome": "The compiler reports spills as stack/local-memory operations, and the cited author's 80-to-64 change had no effect because the allocation was already below 64; the page states both negative controls."}, "claim": "Spills add local-memory instructions and traffic, while a lower maxrregcount value can be inert when natural use is below the cap; neither a memory-bound label nor a nominal cap determines the outcome.", "claim_id": "technique-register-budgeting-current-c004", "claim_type": "performance", "covers": ["u016"], "evidence": [{"locator": "Printing Code Generation Statistics", "observation": "The driver identifies spill stores/loads and stack frame as per-thread stack/local-memory resource use.", "revision": "CUDA Compiler Driver 13.0.2", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html"}, {"locator": "Attempts 10-12", "observation": "The author's memory-bound kernel regressed with spill-inducing pressure and saw no change from a cap that remained above natural use.", "revision": "retrieved 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "TMEM eliminates accumulator register pressure and yields a fixed per-thread saving.", "check": "Trace D residence, epilogue transfer, and a concrete SM90 accumulator fragment.", "outcome": "PTX puts tcgen05 D in TMEM, the pinned SM100 epilogue loads it to registers, and the 128-register comparison belongs to one m64n256 FP32 WGMMA shape; the page keeps that scope."}, "claim": "tcgen05 stores its resident D accumulator in TMEM, but epilogue batches and other kernel state still need registers; 128 registers per thread is a shape-specific SM90 m64n256 FP32 D example rather than a universal Blackwell saving.", "claim_id": "technique-register-budgeting-current-c005", "claim_type": "compatibility", "covers": ["u017", "u018"], "evidence": [{"locator": "tcgen05.mma and Tensor Memory", "observation": "The ISA defines D as a TMEM address and tcgen05.ld as the transfer from TMEM to registers.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}, {"locator": "epilogue path around lines 455-463", "observation": "The pinned kernel loads Tensor Memory into registers before storing through shared memory.", "revision": "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f", "url": "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh"}, {"locator": "SM90 m64n256k16 FP32 WGMMA wrapper accumulator operand list", "observation": "The wrapper exposes 128 float D registers per participating thread for this exact N=256 shape.", "revision": "CUTLASS 4.5.0 e406c186d2cae5782a846f7280af282ca4fecec2", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/arch/mma_sm90_gmma.hpp"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The reported 32 and 45 caps establish a causal ranking and a general memory-bound policy.", "check": "List co-varying mechanisms and inspect the author's own cap experiment.", "outcome": "The report lists PTX, cache, load-width, specialization, and reuse differences and supplies no cap ablation; its own above-natural cap change had no effect. The page explicitly limits the observation."}, "claim": "The author reported three top NVFP4 GEMV solutions around an 18.5 microsecond aggregate and caps of 32 and 45 in ranks 1 and 3, but the confounded implementations and missing ablation do not prove a causal register-cap or occupancy ordering.", "claim_id": "technique-register-budgeting-current-c006", "claim_type": "performance", "covers": ["u019", "u020"], "evidence": [{"locator": "What the Top Solutions Did Differently and Attempt 11", "observation": "The author reports the aggregate cluster and caps, enumerates multiple other differences, and reports a no-effect maxrregcount experiment in the author's own implementation.", "revision": "retrieved 2026-08-08", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}], "oracle": "web-primary", "verdict": "verified"}], "commands": ["Inspect archived CUDA 13 compiler, programming-guide, runtime, and PTX contracts", "Inspect pinned SM100 DeepGEMM and CUTLASS 4.5.0 register-residence examples", "Challenge the cited NVFP4 observation with its co-varying mechanisms and no-effect cap experiment"], "id": "technique-register-budgeting", "page_verdict": "verified", "path": "wiki/techniques/register-budgeting.md", "residual_risk": "The commands were syntax-verified against archived documentation but not executed because this environment has no CUDA toolkit or GPU. Cap performance remains target-, compiler-, kernel-, and workload-specific by design.", "scope": "All 20 current body units, the revised frontmatter claim, command syntax, causal limits, and the cited performance observation were checked against direct versioned sources."} +{"claims": [{"challenge": {"alternative": "FA4 removes the hardware exponential path entirely and the polynomial alone defines the optimization.", "check": "Trace fragment selection and the surrounding forward schedule in the pinned source.", "outcome": "The implementation selects hardware or software per fragment and couples the split to two softmax warpgroups, TMEM staging, and correction; the page states that hybrid scope."}, "claim": "FA4's SM100 forward path evaluates a configuration-dependent fraction of exponentials with software FMA polynomial code while retaining hardware exp2 for the rest, inside a larger warp-specialized pipeline.", "claim_id": "technique-software-exp-current-c001", "claim_type": "mechanism", "covers": ["u001", "u002"], "evidence": [{"locator": "forward pipeline and faster exponential sections", "observation": "The first-party description states a tunable per-row split between hardware MUFU and software exponential and documents the two softmax warpgroups, TMEM stages, and correction role.", "revision": "FlashAttention-4 blog, retrieved 2026-08-09", "url": "https://tridao.me/blog/2026/flash4/"}, {"locator": "softmax.py apply_exp2_convert and flash_fwd_sm100.py tuning table", "observation": "The pinned code branches between cute.math.exp2 and ex2_emulation_2 by fragment and contains both zero-emulation and mixed configurations.", "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", "url": "https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The feeds imply an architecture-wide SFU bottleneck and a fixed software speedup.", "check": "Read the dimensions and distinguish analytical per-tile accounting from a measured software-exp ablation.", "outcome": "The source fixes M=N=D=128 and reports modeled resource cycles; the page preserves that scope and explicitly rejects the unsupported universal inference."}, "claim": "The authors' B200 M=N=D=128 model uses 8192 BF16 tensor-core ops/cycle, 16 exp ops/cycle, 128 shared-memory bytes/cycle, and assigns 1024, 1024, and 768 forward cycles respectively, without defining universal software-exp throughput.", "claim_id": "technique-software-exp-current-c002", "claim_type": "performance", "covers": ["u003", "u004"], "evidence": [{"locator": "Feeds and Speeds", "observation": "The source lists the exact feeds and forward speeds for M=N=D=128 and motivates overlap rather than an isolated instruction-throughput result.", "revision": "FlashAttention-4 blog, retrieved 2026-08-09", "url": "https://tridao.me/blog/2026/flash4/"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "FA4 uses nearest-integer reduction, a symmetric interval, four Taylor FMAs, or hardware ex2 for reconstruction.", "check": "Inspect constants, rounding modes, coefficient table, Horner evaluator, and combine function at the pinned revision.", "outcome": "The exact code uses lower clamp -127, floor-style reduction via add.rm and 2^23+2^22, degree-3 packed Horner on [0,1), and integer exponent-field reconstruction; every page value matches."}, "claim": "Pinned FA4 clamps below -127 under an x<=127 assumption, derives floor and a fraction in [0,1), evaluates the listed degree-3 Sollya coefficient set with packed FMAs, reconstructs through exponent-field integer operations, and uses log2(e) to map natural exponential inputs.", "claim_id": "technique-software-exp-current-c003", "claim_type": "code", "covers": ["u005", "u006", "u007", "u008", "u009", "u010", "u011", "u012", "u013", "u014", "u015", "u016"], "evidence": [{"locator": "POLY_EX2, evaluate_polynomial_2, combine_int_frac_ex2, ex2_emulation, ex2_emulation_2", "observation": "The exact source contains the page's full coefficients, clamp/assumption, magic constant, rounding direction, three packed Horner steps, and integer reconstruction.", "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", "url": "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py"}, {"locator": "Faster exponential section", "observation": "The authors describe Cody-Waite floor reduction, degree-3 Horner coefficients selected with Sollya on [0,1), and exponent-bit combination.", "revision": "FlashAttention-4 blog, retrieved 2026-08-09", "url": "https://tridao.me/blog/2026/flash4/"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The sampled error is a formal maximum or the old degree-4 coefficient result was accidentally reused.", "check": "Recompute the current coefficient set over its stated interval and run the old coefficient set as a negative control.", "outcome": "The current set reproduced about 8.763e-5 while the negative control produced about 5.568e-5; the page explicitly labels its value sampled rather than a proof."}, "claim": "A 1,000,002-point host grid over [0,1] gives approximately 8.763e-5 maximum sampled relative error for the pinned rounded degree-3 coefficients, without establishing a continuous error bound.", "claim_id": "technique-software-exp-current-c004", "claim_type": "semantics", "covers": ["u017"], "evidence": [{"command": "Evaluate sum(p_i*x**i) / 2**x - 1 at x=i/1000001 for i=0..1000001", "environment": "Conda base Python 3 host arithmetic", "negative_control": "The removed degree-4 Taylor set on [-0.5,0.5] produced 5.5684318728932425e-5 instead of the current result.", "reference": "Pinned POLY_EX2[3] coefficient tuple in flash_attn/cute/utils.py", "result": "Maximum absolute sampled relative error 8.762961710306882e-5 at x approximately 0.1007928992."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "BF16 conversion makes scalar and end-to-end validation unnecessary, or an SFU utilization metric alone selects the winning split.", "check": "Trace error propagation and configuration-dependent hybrid settings through the concrete softmax pipeline.", "outcome": "Approximation affects numerator and denominator, special cases live outside the helper, and pinned settings include hardware-only and mixed paths; the procedure tests all discriminating layers."}, "claim": "Software-exp adoption requires scalar/special-case accuracy tests, complete attention correctness, generated-code inspection, resource profiling, and matched performance sweeps over the hybrid fraction rather than dtype or profiler-label assumptions.", "claim_id": "technique-software-exp-current-c005", "claim_type": "other", "covers": ["u018", "u019", "u020", "u021", "u022", "u023", "u024", "u025"], "evidence": [{"locator": "utils.py helper assumptions; softmax.py special cases; flash_fwd_sm100.py tuning table", "observation": "The pinned source separates helper-domain assumptions from softmax special handling and selects different emulation frequencies, including zero, by configuration.", "revision": "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", "url": "https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute"}, {"locator": "forward pipeline and benchmark sections", "observation": "The authors treat exponential selection as one component of a coupled pipeline and report configuration-level results, supporting the page's requirement for end-to-end controlled comparison.", "revision": "FlashAttention-4 blog, retrieved 2026-08-09", "url": "https://tridao.me/blog/2026/flash4/"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The 1.1-1.3x figure is an isolated software-exp speedup.", "check": "Inspect the benchmark label and the full list of forward design changes.", "outcome": "The comparison is complete FA4 forward versus cuDNN 9.13 and includes multiple co-designed mechanisms; the page says exactly that."}, "claim": "FA4 reports complete forward speedups of 1.1-1.3x over cuDNN 9.13 on evaluated B200 BF16 cases, not a software-exponential-only ablation.", "claim_id": "technique-software-exp-current-c006", "claim_type": "performance", "covers": ["u026"], "evidence": [{"locator": "Attention Benchmarks and forward-pass design", "observation": "The first-party comparison is labeled for FA4 forward and the implementation includes pipelining, software/hardware exp, conditional rescaling, TMEM staging, and scheduling.", "revision": "FlashAttention-4 blog, retrieved 2026-08-09", "url": "https://tridao.me/blog/2026/flash4/"}], "oracle": "web-primary", "verdict": "verified"}], "commands": ["Inspect pinned FA4 utils.py, softmax.py, and flash_fwd_sm100.py at a369df7", "Compare first-party feeds-and-speeds and benchmark scope", "Recompute the published rounded coefficient error with a negative control"], "id": "technique-software-exp", "page_verdict": "verified", "path": "wiki/techniques/software-exp.md", "residual_risk": "No B200 SASS or performance run was available. The numerical grid is a regression measurement, not a formal bound, and the page requires target-specific accuracy and performance validation.", "scope": "All 26 current body units, pinned code links, numerical value, performance provenance, architecture scope, and frontmatter claim were checked."} +{"claims": [{"challenge": {"alternative": "A swizzle eliminates bank conflicts by construction, regardless of consumer access.", "check": "Apply the documented bank mapping to different warp address sets and include same-word/broadcast cases.", "outcome": "Conflicts are properties of the concrete request; the page defines that dependency and limits swizzle benefit to a target access pattern."}, "claim": "Shared memory maps successive 32-bit words to 32 banks, conflicts serialize different-word same-bank accesses within a warp request, and a swizzle may reduce but cannot universally eliminate conflicts.", "claim_id": "technique-swizzling-current-c001", "claim_type": "semantics", "covers": ["u001", "u002"], "evidence": [{"locator": "Shared Memory and Bank Conflicts", "observation": "The guide defines 32 banks, successive 32-bit-word mapping, conflict serialization, and same-word/broadcast exceptions.", "revision": "Nsight Compute Profiling Guide 13.3", "url": "https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "128B is the only legal Blackwell mapping, and a tensor-map enumerator alone determines correctness.", "check": "Compare TMA enum mappings, encoder constraints, and tcgen05 descriptor layout values.", "outcome": "Both APIs expose multiple modes and independent constraints, while the producer destination and consumer addressing must denote the same physical layout; the page lists exact common encodings."}, "claim": "A TMA-to-tcgen05 tile requires consistent tensor-map, base/alignment, and consumer-descriptor mappings; CUDA 13 defines the listed TMA chunk spans and PTX 9.0 defines legal no/32B/64B/128B tcgen05 modes with exact layout-field values and combination-specific constraints.", "claim_id": "technique-swizzling-current-c002", "claim_type": "compatibility", "covers": ["u003", "u004", "u005", "u006", "u007", "u008", "u009", "u010", "u011", "u012", "u013", "u014", "u015"], "evidence": [{"locator": "CUtensorMapSwizzle and cuTensorMapEncodeTiled constraints", "observation": "The API lists NONE, 32B/64B/128B 16B-chunk mappings and 128B atom variants, plus datatype/interleave/inner-box restrictions.", "revision": "CUDA Driver API 13.0.97", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"}, {"locator": "tcgen05 shared-memory descriptor and valid combinations", "observation": "PTX defines the descriptor fields, layout values, invalid values, and type/major/shape constraints quoted by the page.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Swizzle<3,4,3> means three base bits, four mask bits, and can be applied from row indices without base alignment.", "check": "Evaluate the CUTLASS template masks and compare layout versus pointer swizzling.", "outcome": "The template source defines the opposite parameter meanings, yields address bits 7:9 into 4:6, and explicitly distinguishes position-dependent hardware pointers; the page matches."}, "claim": "CUTLASS Swizzle uses mask width, invariant low bits, and field distance respectively; Swizzle<3,4,3> maps address bits 7:9 into 4:6, with equivalence to a logical row formula dependent on stride, byte units, composition, and base alignment.", "claim_id": "technique-swizzling-current-c003", "claim_type": "code", "covers": ["u016", "u017", "u018", "u019", "u020", "u021"], "evidence": [{"locator": "Swizzle definition and apply", "observation": "The header defines parameter meanings, mask positions, and XOR application exactly as summarized.", "revision": "CUTLASS 4.5.0 e406c186d2cae5782a846f7280af282ca4fecec2", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/swizzle.hpp"}, {"locator": "pointer_swizzle position dependence", "observation": "CUTLASS states that hardware chose position-dependent swizzles and distinguishes swizzle_ptr from composed swizzle_layout.", "revision": "CUTLASS 4.5.0 e406c186d2cae5782a846f7280af282ca4fecec2", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/pointer_swizzle.hpp"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "Supplying CU_TENSOR_MAP_SWIZZLE_128B is sufficient, and any mismatch always appears as a runtime error.", "check": "Enumerate every encoder input/constraint and distinguish invalid encoding from a legal but logically inconsistent consumer.", "outcome": "The driver validates encoder contracts through CUresult, while consumer semantics are separate; the checklist and two failure classes are correctly separated."}, "claim": "A reproducible tiled tensor map must validate alignment, rank-specific dimensions/strides, box and element strides, mode constraints, CUresult, and the matching consumer mapping; successful encoding alone does not validate logical MMA output.", "claim_id": "technique-swizzling-current-c004", "claim_type": "mechanism", "covers": ["u022", "u023", "u024", "u025", "u026", "u027", "u028"], "evidence": [{"locator": "cuTensorMapEncodeTiled parameters, return values, and constraints", "observation": "The archived API defines output/base alignment, rank-sized arrays, rank-minus-one byte strides, box/element constraints, swizzle restrictions, and CUresult failures.", "revision": "CUDA Driver API 13.0.97", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"}, {"locator": "tcgen05 shared-memory descriptor", "observation": "The consumer descriptor independently supplies base, leading/stride dimensions, offset, major mode, and swizzle layout.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The 2.75x result isolates bank-conflict removal by 128B swizzling.", "check": "Diff the tutorial's v1b/v2b encoder, TMA issuance, and descriptors and read the author's causal discussion.", "outcome": "The inner TMA width and loop structure change together with swizzle/descriptor mode, and the author explicitly proposes TMA efficiency as an alternative; the page reports a combined treatment."}, "claim": "The pinned tutorial's B200 4096-cube 3D TMA path reports 252.81 TFLOP/s for 16B/no-swizzle and 695.43 for 128B/128B-swizzle, approximately 2.75x and 46% of its cuBLAS result, without isolating bank-conflict or swizzle causality.", "claim_id": "technique-swizzling-current-c005", "claim_type": "performance", "covers": ["u029", "u030", "u031", "u032"], "evidence": [{"locator": "128-byte global load and swizzling, v1/v2 diff, benchmark, and remarks", "observation": "The first-party tutorial supplies the exact values, simultaneous 16B-to-128B TMA and swizzle changes, lack of ncu access, and alternative wider-load explanation.", "revision": "tcgen05 for dummies; source commit 3b90ac9b3f624bdf1f6f78d02dcd533675d36573", "url": "https://gau-nernst.github.io/tcgen05/"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "A zero conflict counter or successful tensor-map encode is sufficient to select and validate a swizzle.", "check": "Separate encoding validity, output permutation correctness, per-access wavefronts, and end-to-end timing.", "outcome": "Each oracle discriminates a different failure mode, and the procedure controls producer/consumer pairs, correctness, profiler-version metric discovery, resources, and timing."}, "claim": "Swizzle verification requires controlled matched-layout variants, explicit encoder and output checks, tool-discovered shared-memory request/wavefront/conflict metrics, end-to-end timing, and recorded environment rather than one counter or architecture heuristic.", "claim_id": "technique-swizzling-current-c006", "claim_type": "other", "covers": ["u033", "u034", "u035", "u036", "u037", "u038", "u039"], "evidence": [{"locator": "Metrics Examples and Shared Memory table", "observation": "Nsight Compute documents metric discovery/naming, l1tex bank-conflict aggregates, and requests/wavefronts/conflicts as distinct shared-memory observations.", "revision": "Nsight Compute 13.3", "url": "https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html"}, {"locator": "cuTensorMapEncodeTiled and tcgen05 descriptor contracts", "observation": "The producer and consumer interfaces have separate validity and layout requirements, supporting independent encoding and numerical checks.", "revision": "CUDA 13.0.2", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"}], "oracle": "web-primary", "verdict": "verified"}], "commands": ["Inspect CUDA 13 tensor-map and PTX tcgen05 mapping contracts", "Inspect CUTLASS 4.5.0 address-bit swizzle semantics", "Trace tutorial v1b/v2b changes and causal caveat", "Inspect current Nsight Compute bank-conflict measurement model"], "id": "technique-swizzling", "page_verdict": "verified", "path": "wiki/techniques/swizzling.md", "residual_risk": "No GPU experiment was available. The page intentionally requires per-consumer correctness and profiler comparison because legal mappings do not imply conflict-free or faster execution.", "scope": "All 39 current body units, exact enum/descriptor values, CUTLASS semantics, performance provenance, validation workflow, and frontmatter claim were checked."} +{"claims": [{"challenge": {"alternative": "CUTLASS software integration makes raster/swizzle part of CLC itself, so the layer distinction is artificial.", "check": "Compare raw PTX operands with CUTLASS scheduler transforms and Stream-K parameter state.", "outcome": "Only response and mbarrier state reach the CLC instruction; raster and decomposition occur in software, making the distinction operationally necessary."}, "claim": "The page correctly separates software coordinate order, resident-worker iteration, CLC reassignment of unlaunched IDs, and Stream-K/Split-K decomposition, and states that raw CLC has no raster/swizzle policy operand.", "claim_id": "technique-tile-scheduling-current-c001", "claim_type": "mechanism", "covers": ["u001", "u002", "u003", "u004", "u005", "u006"], "evidence": [{"locator": "Verified scope and CUTLASS integration", "observation": "The source card distinguishes launched ClcIDs, software coordinate transforms and higher-level Stream-K decomposition exactly as the page does.", "path": "sources/docs/cutlass-clc-documentation.md", "revision": "CUTLASS 4.5.0 e406c186; checked against pinned documentation/source"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "A nondivisible last iteration causes duplication, or the code silently assumes tiles_n divides total_tiles.", "check": "Enumerate total_tiles/grid sizes with remainders one and grid-1 and use a partial final M row.", "outcome": "The upper-bound check excludes invalid tail coordinates and every valid flat index still appears exactly once."}, "claim": "The row-major and positive-grid static-stride snippets map quotient/remainder coordinates and cover every flat tile once, while the page correctly requires edge bijection and measurements before making cache claims.", "claim_id": "technique-tile-scheduling-current-c002", "claim_type": "code", "covers": ["u007", "u008", "u009", "u010", "u011"], "evidence": [{"command": "conda run -n base python enumerate quotient/remainder and CTA-stride mappings for divisible and nondivisible tile counts", "environment": "Conda base, host CPU", "negative_control": "Using gridDim.x-1 duplicated work, while the deleted baseline swizzle emitted out-of-range N on a 3x5 grid.", "reference": "All valid indices must occur once; coordinate order alone is not a cache simulation", "result": "The displayed formulas produced exact unique coverage for positive grid sizes and correct row-major coordinates across tested edge cases."}], "oracle": "host-experiment", "verdict": "verified"}, {"challenge": {"alternative": "CLC produces new queue work or assigns the initial tile through try_cancel.", "check": "Trace every documented ClcID from grid launch through initial or canceled handling.", "outcome": "Every ID exists in the launched grid and is handled on exactly one of the two paths; initial blockIdx precedes requests."}, "claim": "The SM100 CLC lifecycle accurately states full-grid launch, initial blockIdx work, asynchronous 16-byte try_cancel response on mbarrier, success-only decode, cluster-rank coordinate derivation, terminal observed failure, and conservation of grid IDs.", "claim_id": "technique-tile-scheduling-current-c003", "claim_type": "mechanism", "covers": ["u012", "u013", "u014", "u015", "u016", "u017"], "evidence": [{"locator": "try_cancel/query semantics, completion, restrictions and cluster behavior", "observation": "The normative rules match each stated request, response, failure and coordinate condition.", "revision": "CUDA 13.0.2, PTX ISA 9.0", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"}, {"locator": "Dynamic Scheduler rules", "observation": "The full grid and two ClcID paths establish the page's conservation and uneven-availability scope.", "revision": "CUTLASS 4.5.0 e406c186", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "oracle": "web-primary", "verdict": "verified"}, {"challenge": {"alternative": "The tags are mere declarations with different effective mappings, or all Stream-K reductions are atomic.", "check": "Follow every arch::Sm100 selector and inspect the reduction enum implementation comments.", "outcome": "The specializations resolve to the listed classes and expose two materially different reduction modes."}, "claim": "The CUTLASS 4.5.0 table gives the exact SM100 public scheduler tags and selected implementation classes, identifies the group wrapper as SM90-style static scheduling, and accurately distinguishes deterministic turnstile from nondeterministic atomic-workspace Stream-K reduction.", "claim_id": "technique-tile-scheduling-current-c004", "claim_type": "compatibility", "covers": ["u018", "u019", "u020", "u021", "u022", "u023", "u024"], "evidence": [{"locator": "TileSchedulerSelector specializations for arch::Sm100", "observation": "Each public tag maps to the implementation shown in the table, including default void, dynamic, static, Stream-K and group routes.", "path": "/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/tile_scheduler.hpp", "revision": "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5"}, {"locator": "class preamble and UnderlyingScheduler alias", "observation": "The SM100 group class explicitly wraps PersistentTileSchedulerSm90Group and describes a static grouped scheduler.", "path": "/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/sm100_tile_scheduler_group.hpp", "revision": "same tag"}, {"locator": "ReductionMode enum", "observation": "Deterministic uses lock/turnstile ordering; Nondeterministic uses mostly atomic workspace accumulation.", "path": "/tmp/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/tile_scheduler_detail.hpp", "revision": "same tag"}], "oracle": "local-primary-source", "verdict": "verified"}, {"challenge": {"alternative": "The example implicitly counts multiple resident CTAs or unequal tile duration, invalidating the arithmetic.", "check": "Read the declared model assumptions before evaluating the formula.", "outcome": "The page explicitly limits the arithmetic to equal-duration independent tiles and available one-tile workers; broader runtime behavior is not claimed."}, "claim": "The tail model's Euclidean-division code and 150/142 example correctly handle both partial and exactly divisible final waves under the stated equal-duration, independent-tile, one-worker assumptions.", "claim_id": "technique-tile-scheduling-current-c005", "claim_type": "code", "covers": ["u025", "u026", "u027", "u028"], "evidence": [{"command": "conda run -n base python evaluate divmod for (150,142), (142,142), (284,142), and (143,142)", "environment": "Conda base, host CPU", "negative_control": "Using r/W without the r=0 special case reports zero last-wave occupancy for 142 and 284 tiles.", "reference": "T=qW+r with 0<=r Date: Tue, 11 Aug 2026 15:43:46 -0500 Subject: [PATCH 3/3] update verification --- sources/blogs/colfax-cutlass-blackwell.md | 2 +- sources/blogs/tcgen05-tutorial.md | 2 +- sources/docs/cutlass-clc-documentation.md | 2 +- sources/docs/nvidia-ptx-isa-sm100.md | 2 +- verification/before-after-report.jsonl | 296 +- verification/current-queue.jsonl | 32 +- verification/evidence/MANIFEST.json | 1200 + verification/evidence/README.md | 30 + .../historical-artifacts/MANIFEST.json | 48 + .../variants/01-mla-decode-inner-loop.cu | 34 + ...hunk-parallel-prefill-reference-pytorch.py | 32 + .../02-triton-decode-step-kernel-streaming.py | 36 + .../blackwell-cutlass-schedules-and-tma.cu | 24 + .../full/vllm-PR-23696-gated-dual-gemm.patch | 872 + .../01-fused-epilogue-swiglu-skeleton.cu | 25 + .../72b_blackwell_nvfp4_nvfp4_gemm.cu | 597 + .../75_blackwell_grouped_gemm_block_scaled.cu | 944 + .../include/cutlass/gemm/dispatch_policy.hpp | 1590 + .../cutlass/gemm/kernel/gemm_grouped.h | 457 + .../repo/media/docs/cpp/efficient_gemm.md | 290 + .../kernelwiki-amandeep-nvfp4.html | 221 + .../nvfp4/gemv/attempt1.py | 252 + .../nvfp4/gemv/attempt10.py | 263 + .../nvfp4/gemv/attempt11.py | 243 + .../nvfp4/gemv/attempt12.py | 280 + .../nvfp4/gemv/attempt2.py | 257 + .../nvfp4/gemv/attempt3.py | 269 + .../nvfp4/gemv/attempt4.py | 269 + .../nvfp4/gemv/attempt5.py | 266 + .../nvfp4/gemv/attempt6.py | 285 + .../nvfp4/gemv/attempt7.py | 238 + .../nvfp4/gemv/attempt8.py | 262 + .../nvfp4/gemv/attempt9.py | 268 + .../nvfp4/gemv/reference.py | 165 + .../nvfp4/gemv/task.py | 58 + .../nvfp4/gemv/test_harness.py | 99 + .../collective/builders/sm100_builder.inl | 1819 + .../collective/builders/sm103_builder.inl | 108 + .../collective/builders/sm120_builder.inl | 517 + .../collective/builders/sm120_common.inl | 80 + .../collective/builders/sm90_builder.inl | 888 + .../collective/builders/sm90_common.inl | 104 + .../collective/collective_builder.hpp | 126 + .../collective/collective_epilogue.hpp | 75 + .../epilogue/collective/default_epilogue.hpp | 265 + .../collective/default_epilogue_array.hpp | 287 + .../cutlass/epilogue/collective/detail.hpp | 950 + .../collective/epilogue_tensor_broadcast.hpp | 271 + .../sm100_epilogue_array_nosmem.hpp | 980 + ...0_epilogue_array_planar_complex_nosmem.hpp | 345 + ...ray_planar_complex_tma_warpspecialized.hpp | 1163 + ...100_epilogue_array_tma_warpspecialized.hpp | 1609 + .../collective/sm100_epilogue_nosmem.hpp | 864 + ...gue_planar_complex_tma_warpspecialized.hpp | 897 + .../sm100_epilogue_tma_warpspecialized.hpp | 1299 + .../collective/sm70_epilogue_vectorized.hpp | 549 + .../sm70_epilogue_vectorized_array.hpp | 412 + ...m90_epilogue_array_tma_warpspecialized.hpp | 1245 + .../sm90_epilogue_tma_warpspecialized.hpp | 963 + ...e_tma_warpspecialized_bias_elementwise.hpp | 164 + .../cutlass/epilogue/dispatch_policy.hpp | 344 + .../cutlass/epilogue/fusion/callbacks.hpp | 91 + .../cutlass/epilogue/fusion/operations.hpp | 645 + .../sm100_callbacks_tma_warpspecialized.hpp | 1322 + ...00_visitor_compute_tma_warpspecialized.hpp | 500 + ...m100_visitor_store_tma_warpspecialized.hpp | 666 + .../sm120_callbacks_tma_warpspecialized.hpp | 1593 + ...m120_visitor_store_tma_warpspecialized.hpp | 899 + .../sm90_callbacks_tma_warpspecialized.hpp | 2792 + ...90_visitor_compute_tma_warpspecialized.hpp | 842 + .../sm90_visitor_load_tma_warpspecialized.hpp | 1492 + ...sm90_visitor_store_tma_warpspecialized.hpp | 1722 + .../sm90_visitor_tma_warpspecialized.hpp | 1149 + .../fusion/sm90_visitor_topk_softmax.hpp | 763 + .../cutlass/epilogue/thread/activation.h | 949 + .../cutlass/epilogue/thread/conversion_op.h | 148 + .../cutlass/epilogue/thread/detail.hpp | 52 + .../epilogue/thread/linear_combination.h | 527 + .../linear_combination_bias_elementwise.h | 985 + .../thread/linear_combination_bias_relu.h | 610 + .../thread/linear_combination_clamp.h | 684 + .../thread/linear_combination_dgelu.h | 250 + .../thread/linear_combination_drelu.h | 452 + .../epilogue/thread/linear_combination_gelu.h | 70 + .../thread/linear_combination_generic.h | 265 + .../linear_combination_generic_with_scaling.h | 325 + .../thread/linear_combination_hardswish.h | 69 + .../thread/linear_combination_leaky_relu.h | 231 + .../thread/linear_combination_params.h | 75 + .../linear_combination_planar_complex.h | 236 + .../epilogue/thread/linear_combination_relu.h | 572 + .../thread/linear_combination_relu0.h | 543 + .../linear_combination_residual_block.h | 301 + .../thread/linear_combination_sigmoid.h | 70 + .../epilogue/thread/linear_combination_silu.h | 69 + .../linear_combination_tensor_broadcast.hpp | 253 + .../linear_combination_with_elementwise.h | 234 + .../cutlass/epilogue/thread/reduction_op.h | 97 + .../cutlass/epilogue/thread/scale_type.h | 66 + .../default_epilogue_complex_tensor_op.h | 255 + ...default_epilogue_complex_tensor_op_blas3.h | 264 + .../default_epilogue_direct_store.h | 74 + .../default_epilogue_planar_complex.h | 241 + .../threadblock/default_epilogue_simt.h | 443 + .../threadblock/default_epilogue_tensor_op.h | 904 + .../default_epilogue_tensor_op_blas3.h | 175 + .../default_epilogue_volta_tensor_op.h | 337 + .../default_epilogue_with_absmax.h | 126 + .../default_epilogue_with_broadcast.h | 376 + .../default_epilogue_with_reduction.h | 177 + .../default_epilogue_wmma_tensor_op.h | 165 + .../threadblock/default_thread_map_simt.h | 127 + .../default_thread_map_tensor_op.h | 208 + .../default_thread_map_volta_tensor_op.h | 228 + .../default_thread_map_wmma_tensor_op.h | 113 + .../direct_store_epilogue_iterator.h | 142 + .../cutlass/epilogue/threadblock/epilogue.h | 550 + .../epilogue/threadblock/epilogue_base.h | 236 + .../threadblock/epilogue_base_streamk.h | 197 + .../epilogue/threadblock/epilogue_depthwise.h | 335 + .../threadblock/epilogue_direct_store.h | 347 + .../threadblock/epilogue_gemm_k_reduction.h | 206 + .../threadblock/epilogue_planar_complex.h | 401 + .../threadblock/epilogue_smem_accumulator.h | 224 + .../epilogue_streamk_with_broadcast.h | 443 + .../epilogue_visitor_with_softmax.h | 513 + .../threadblock/epilogue_with_absmax.h | 922 + .../threadblock/epilogue_with_broadcast.h | 1717 + .../threadblock/epilogue_with_reduction.h | 819 + .../epilogue_with_scaling_factor.h | 231 + .../threadblock/epilogue_with_visitor.h | 409 + .../epilogue_with_visitor_callbacks.h | 526 + .../epilogue/threadblock/epilogue_workspace.h | 197 + .../threadblock/fusion/visitor_2x.hpp | 433 + .../threadblock/fusion/visitor_compute.hpp | 109 + .../threadblock/fusion/visitor_load.hpp | 597 + .../threadblock/fusion/visitor_store.hpp | 802 + .../epilogue/threadblock/fusion/visitors.hpp | 38 + .../threadblock/interleaved_epilogue.h | 407 + .../threadblock/output_iterator_parameter.h | 223 + .../threadblock/output_tile_thread_map.h | 628 + .../threadblock/predicated_tile_iterator.h | 1387 + .../predicated_tile_iterator_affine.h | 615 + ...cated_tile_iterator_affine_layout_params.h | 156 + .../predicated_tile_iterator_blas3.h | 633 + .../predicated_tile_iterator_conv.h | 562 + .../predicated_tile_iterator_direct_conv.h | 445 + .../predicated_tile_iterator_params.h | 483 + .../predicated_tile_iterator_predicates.h | 309 + .../predicated_tile_iterator_strided_dgrad.h | 479 + .../threadblock/shared_load_iterator.h | 223 + .../threadblock/shared_load_iterator_mixed.h | 594 + .../shared_load_iterator_pitch_linear.h | 194 + .../fragment_iterator_complex_tensor_op.h | 187 + ...ment_iterator_gaussian_complex_tensor_op.h | 194 + .../epilogue/warp/fragment_iterator_simt.h | 164 + .../warp/fragment_iterator_tensor_op.h | 378 + .../warp/fragment_iterator_volta_tensor_op.h | 269 + .../warp/fragment_iterator_wmma_tensor_op.h | 158 + .../cutlass/epilogue/warp/simt_policy.h | 107 + .../cutlass/epilogue/warp/tensor_op_policy.h | 189 + .../epilogue/warp/tile_iterator_simt.h | 785 + .../epilogue/warp/tile_iterator_tensor_op.h | 671 + .../warp/tile_iterator_tensor_op_mixed.h | 1089 + .../warp/tile_iterator_volta_tensor_op.h | 440 + .../warp/tile_iterator_wmma_tensor_op.h | 224 + .../epilogue/warp/volta_tensor_op_policy.h | 195 + .../epilogue/warp/wmma_tensor_op_policy.h | 101 + .../kernel/sm100_gemm_tma_warpspecialized.hpp | 963 + .../kernel/sm100_tile_scheduler_group.hpp | 337 + .../kernel/sm100_tile_scheduler_stream_k.hpp | 1012 + .../cutlass/gemm/kernel/tile_scheduler.hpp | 423 + .../gemm/kernel/tile_scheduler_detail.hpp | 88 + .../gemm/kernel/tile_scheduler_params.h | 2579 + .../kernelwiki-cutlass-4.5.0/CHANGELOG.md | 1138 + .../kernelwiki-cutlass-4.5.0/README.md | 656 + .../tutorial/tutorial_gemm/README.md | 25 + .../tutorial/tutorial_gemm/fp16_gemm_0.py | 441 + .../tutorial/tutorial_gemm/fp16_gemm_1.py | 533 + .../tutorial/tutorial_gemm/fp16_gemm_2.py | 687 + .../tutorial/tutorial_gemm/fp16_gemm_3.py | 779 + .../tutorial/tutorial_gemm/fp16_gemm_3_1.py | 891 + .../tutorial/tutorial_gemm/fp16_gemm_4.py | 1074 + .../tutorial/tutorial_gemm/fp16_gemm_5.py | 929 + .../tutorial/tutorial_gemm/fp16_gemm_6.py | 1011 + .../tutorial/tutorial_gemm/nvfp4_gemm_0.py | 778 + .../tutorial/tutorial_gemm/nvfp4_gemm_1.py | 934 + .../blackwell/tutorial/tutorial_gemm/utils.py | 366 + .../python/CuTeDSL/cutlass/cute/arch/tmem.py | 220 + .../CuTeDSL/cutlass/cute/nvgpu/tcgen05/mma.py | 1869 + .../cutlass/CMakeLists.txt | 1215 + .../70_blackwell_fp16_gemm.cu | 486 + .../tutorial/blackwell/02_mma_tma_sm100.cu | 692 + .../tutorial/tutorial_gemm/fp16_gemm_0.py | 441 + .../tutorial/tutorial_gemm/fp16_gemm_6.py | 1011 + .../blackwell/tutorial/tutorial_tma/tma_v0.py | 409 + .../include/cute/arch/mma_sm90_gmma.hpp | 20974 +++++ .../cutlass/arch/grid_dependency_control.h | 109 + .../CuTeDSL/cutlass/utils/tmem_allocator.py | 691 + .../kernelwiki-deepgemm-pinned/README.md | 207 + .../csrc/apis/gemm.hpp | 715 + .../csrc/jit/compiler.hpp | 362 + .../deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh | 567 + .../deep_gemm/impls/sm100_fp8_mqa_logits.cuh | 403 + .../deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh | 346 + .../include/deep_gemm/ptx/tcgen05.cuh | 168 + .../tests/generators.py | 407 + .../tests/test_fp8_fp4.py | 222 + .../kernelwiki-deepseek-v32/DeepSeek_V3_2.pdf | Bin 0 -> 526189 bytes .../kernelwiki-deepseek-v32/README.md | 146 + .../inference/config_671B_v3.2.json | 26 + .../inference/model.py | 923 + .../kernelwiki-ext-cfx/.gitmodules | 3 + .../kernelwiki-ext-cfx/README.md | 4 + .../kernelwiki-ext-cfx/cutlass_gemm/Makefile | 11 + .../kernelwiki-ext-cfx/cutlass_gemm/README.md | 27 + .../cutlass_gemm/cutlass_gemm/cutlass_gemm.cu | 88 + .../cutlass_gemm/cutlass_gemm.hpp | 200 + .../cutlass_gemm/cutlass_gemm/setup.py | 33 + .../kernelwiki-ext-cfx/cutlass_gemm/gemm.py | 21 + .../kernelwiki-ext-cfx/evt/Makefile | 19 + .../kernelwiki-ext-cfx/evt/README.md | 27 + .../kernelwiki-ext-cfx/evt/evt_gemm_cute.cu | 763 + .../kernelwiki-ext-cfx/evt/node_types.md | 203 + .../kernelwiki-ext-cfx/evt/reference.h | 345 + .../include/utils/cuda_launch.hpp | 19 + .../kernelwiki-ext-cfx/pipeline-gemm/Makefile | 25 + .../pipeline-gemm/README.md | 36 + .../hopper-gemm-ws/convert_util.h | 30 + .../hopper-gemm-ws/epilogue_sm90_tma_ws.hpp | 151 + .../hopper-gemm-ws/hopper_gemm_kernel.h | 163 + .../hopper_gemm_kernel_launch.h | 223 + .../hopper-gemm-ws/kernel_traits.h | 124 + .../mainloop_sm90_tma_gmma_ws.hpp | 256 + .../hopper-gemm-ws/tile_scheduler.hpp | 169 + .../pipeline-gemm/sm90_gemm_multistage.cu | 598 + .../pipeline-gemm/sm90_gemm_ws.cu | 24 + .../kernelwiki-ext-cfx/streamk/Makefile | 24 + .../kernelwiki-ext-cfx/streamk/README.md | 25 + .../kernelwiki-ext-cfx/streamk/cli_options.h | 76 + .../kernelwiki-ext-cfx/streamk/convert_util.h | 26 + .../streamk/cutlass_streamk/Makefile | 22 + .../streamk/cutlass_streamk/benchmark.cu | 558 + .../streamk/cutlass_streamk/profile.sh | 19 + .../streamk/epilogue_sm90_tma_ws.hpp | 156 + .../streamk/hopper_gemm_kernel.h | 168 + .../streamk/hopper_gemm_kernel_launch.h | 406 + .../streamk/kernel_traits.h | 119 + .../streamk/mainloop_sm90_tma_gmma_ws.hpp | 275 + .../kernelwiki-ext-cfx/streamk/streamk.cu | 97 + .../streamk/tile_scheduler.hpp | 624 + .../kernelwiki-ext-cfx/tma/Makefile | 25 + .../kernelwiki-ext-cfx/tma/README.md | 20 + .../kernelwiki-ext-cfx/tma/main.cu | 27 + .../kernelwiki-ext-cfx/tma/scale_tma_kernel.h | 260 + .../kernelwiki-ext-cfx/tma/shared_storage.h | 11 + .../kernelwiki-ext-cfx/tma/smem_helper.hpp | 54 + .../kernelwiki-ext-cfx/tma/tma_copy.h | 228 + .../tma/tma_copy_multicast.h | 385 + .../transpose-cute/Makefile | 29 + .../transpose-cute/README.md | 29 + .../transpose-cute/include/copy.h | 156 + .../transpose-cute/include/copy_direct.h | 139 + .../transpose-cute/include/copy_smem.h | 162 + .../include/fma_inner_product_naive.h | 183 + .../transpose-cute/include/shared_storage.h | 28 + .../transpose-cute/include/smem_helper.hpp | 54 + .../transpose-cute/include/transpose_naive.h | 88 + .../transpose-cute/include/transpose_smem.h | 128 + .../include/transpose_tmastore_vectorized.h | 155 + .../transpose-cute/include/util.h | 92 + .../kernelwiki-ext-cfx/transpose-cute/main.cu | 47 + .../transpose-cute/python/copy_cute.cu | 66 + .../transpose-cute/python/setup.py | 40 + .../transpose-cute/python/transpose_cute.cu | 102 + .../transpose-cute/torch_benchmark.py | 51 + .../kernelwiki-ext-colfax-cutlass/LICENSE | 21 + .../kernelwiki-ext-colfax-cutlass/README.md | 21 + .../include/utils/cuda_launch.hpp | 22 + .../include/utils/fmha_cutlass.hpp | 318 + .../include/utils/random.hpp | 92 + .../lib/gemm/copy_tensor.hpp | 102 + .../lib/gemm/gemm_tensor.hpp | 46 + .../src/cute-gemm-tma-gma/compile.sh | 1 + .../src/cute-gemm-tma-gma/gemm.cu | 691 + .../src/cutlass-gemm/compile.sh | 1 + .../src/cutlass-gemm/compile_ws.sh | 1 + .../src/cutlass-gemm/gemm.cu | 471 + .../src/fmha-pipeline/README.md | 44 + .../src/fmha-pipeline/compile.sh | 1 + .../src/fmha-pipeline/compile_H128.sh | 1 + .../src/fmha-pipeline/compile_H64.sh | 1 + .../fmha-pipeline/compile_run_all_config.sh | 63 + .../src/fmha-pipeline/fmha_consumer.h | 73 + .../src/fmha-pipeline/fmha_epilogue.h | 177 + .../src/fmha-pipeline/fmha_forward.cu | 794 + .../src/fmha-pipeline/fmha_nopipe.h | 342 + .../src/fmha-pipeline/fmha_pipe_nows.h | 331 + .../src/fmha-pipeline/fmha_pipe_ws.h | 365 + .../src/fmha-pipeline/fmha_producer.h | 79 + .../src/fmha-pipeline/online_softmax.h | 207 + .../src/fmha-pipeline/reg2reg.h | 347 + .../src/fmha-pipeline/shared_storage.h | 85 + .../src/fmha-pipeline/ss_helper.h | 94 + .../src/fmha/README.md | 26 + .../src/fmha/compile.sh | 1 + .../src/fmha/compile_run_all_config.sh | 46 + .../src/fmha/fmha_forward.cu | 866 + .../src/fmha/online_softmax.h | 207 + .../kernelwiki-ext-load-store/LICENSE | 21 + .../kernelwiki-ext-load-store/Makefile | 50 + .../kernelwiki-ext-load-store/README.md | 2 + .../kernelwiki-ext-load-store/ld_matrix_x1.cu | 42 + .../kernelwiki-ext-load-store/ld_matrix_x2.cu | 50 + .../kernelwiki-ext-load-store/ld_matrix_x4.cu | 64 + .../kernelwiki-ext-load-store/st_matrix_x1.cu | 41 + .../kernelwiki-ext-load-store/st_matrix_x2.cu | 50 + .../kernelwiki-ext-load-store/st_matrix_x4.cu | 66 + .../kernelwiki-ext-nvidia-samples/.gitignore | 2 + .../kernelwiki-ext-nvidia-samples/.gitmodules | 9 + .../MATLAB_arrayfun/ArrayfunArticle.m | 474 + .../MATLAB_arrayfun/README.md | 9 + .../MATLAB_arrayfun/drawAntenna.m | 64 + .../MATLAB_arrayfun/drawMap.m | 22 + .../MATLAB_arrayfun/drawMultipleSignalMaps.m | 20 + .../MATLAB_arrayfun/drawSignalMap.m | 25 + .../MATLAB_arrayfun/multipleSignalMaps.m | 23 + .../powerCalculationWithGpuArray.m | 13 + .../MATLAB_arrayfun/powerKernel.m | 16 + .../MATLAB_arrayfun/reportTime.m | 12 + .../PetDetectionRecognitionScript.m | 111 + .../MATLAB_deeplearning/README.md | 7 + .../MATLAB_deeplearning/cnnPredict.m | 88 + .../MATLAB_deeplearning/filterRegions.m | 22 + .../MATLAB_deeplearning/findPet.m | 12 + .../MATLAB_deeplearning/getImageLabels.m | 5 + .../kernelwiki-ext-nvidia-samples/README.md | 13 + .../c++11_cuda/Makefile | 31 + .../c++11_cuda/README.md | 7 + .../c++11_cuda/c++11_cuda.cu | 144 + .../c++11_cuda/warandpeace.txt | 65340 ++++++++++++++++ .../kernelwiki-ext-nvidia-samples/license.txt | 25 + .../posts/002-openacc-example/common/timer.h | 78 + .../posts/002-openacc-example/step1/Makefile | 43 + .../002-openacc-example/step1/Makefile_f90 | 42 + .../002-openacc-example/step1/laplace2d.c | 114 + .../002-openacc-example/step1/laplace2d.f90 | 107 + .../posts/002-openacc-example/step2/Makefile | 43 + .../002-openacc-example/step2/Makefile_f90 | 42 + .../002-openacc-example/step2/laplace2d.c | 120 + .../002-openacc-example/step2/laplace2d.f90 | 113 + .../posts/002-openacc-example/step3/Makefile | 43 + .../002-openacc-example/step3/Makefile_f90 | 42 + .../002-openacc-example/step3/laplace2d.c | 123 + .../002-openacc-example/step3/laplace2d.f90 | 117 + .../posts/TensorRT-3.0/convert.ipynb | 178 + .../posts/TensorRT-3.0/test.ipynb | 236 + .../posts/TensorRT-3.0/train.ipynb | 236 + .../posts/TensorRT-introduction/Makefile | 50 + .../posts/TensorRT-introduction/cudaWrapper.h | 81 + .../posts/TensorRT-introduction/ioHelper.cpp | 119 + .../posts/TensorRT-introduction/ioHelper.h | 67 + .../posts/TensorRT-introduction/ioHelper.o | Bin 0 -> 31368 bytes .../TensorRT-introduction/simpleOnnx.cpp | 253 + .../TensorRT-introduction/simpleOnnx_1.cpp | 222 + .../TensorRT-introduction/simpleOnnx_2.cpp | 225 + .../LongstaffSchwartz_vs10.sln | 20 + .../LongstaffSchwartz_vs11.sln | 20 + .../posts/american-options/Makefile | 110 + .../posts/american-options/Readme.md | 14 + .../LongstaffSchwartzSvd2_vs10.vcxproj | 98 + ...LongstaffSchwartzSvd2_vs10.vcxproj.filters | 14 + .../LongstaffSchwartzSvd2_vs10.vcxproj.user | 3 + .../LongstaffSchwartzSvd2_vs11.vcxproj | 100 + ...LongstaffSchwartzSvd2_vs11.vcxproj.filters | 14 + .../LongstaffSchwartzSvd2_vs11.vcxproj.user | 3 + .../longstaff_schwartz_svd_2.cu | 1679 + .../doc/html/CUDA__Aware__MPI_8c.html | 228 + .../doc/html/CUDA__Aware__MPI_8c__incl.map | 3 + .../doc/html/CUDA__Aware__MPI_8c__incl.md5 | 1 + .../doc/html/CUDA__Aware__MPI_8c__incl.png | Bin 0 -> 9783 bytes .../doc/html/CUDA__Normal__MPI_8c.html | 247 + .../doc/html/CUDA__Normal__MPI_8c__incl.map | 3 + .../doc/html/CUDA__Normal__MPI_8c__incl.md5 | 1 + .../doc/html/CUDA__Normal__MPI_8c__incl.png | Bin 0 -> 9814 bytes .../doc/html/Device_8cu.html | 753 + .../doc/html/Device_8cu__incl.map | 3 + .../doc/html/Device_8cu__incl.md5 | 1 + .../doc/html/Device_8cu__incl.png | Bin 0 -> 8835 bytes ...8324d1c9b1ee7193242670ba4b97d88_cgraph.map | 3 + ...8324d1c9b1ee7193242670ba4b97d88_cgraph.md5 | 1 + ...8324d1c9b1ee7193242670ba4b97d88_cgraph.png | Bin 0 -> 3101 bytes ...b641f8b5d521a230416b64d8b809324_cgraph.map | 3 + ...b641f8b5d521a230416b64d8b809324_cgraph.md5 | 1 + ...b641f8b5d521a230416b64d8b809324_cgraph.png | Bin 0 -> 3750 bytes ...5a52e3c2e58d2d94403a0dce2e29686_cgraph.map | 3 + ...5a52e3c2e58d2d94403a0dce2e29686_cgraph.md5 | 1 + ...5a52e3c2e58d2d94403a0dce2e29686_cgraph.png | Bin 0 -> 4425 bytes ...aebe085afbffaa0d6bcd77fa16ceb97_cgraph.map | 3 + ...aebe085afbffaa0d6bcd77fa16ceb97_cgraph.md5 | 1 + ...aebe085afbffaa0d6bcd77fa16ceb97_cgraph.png | Bin 0 -> 3420 bytes .../doc/html/Host_8c.html | 824 + .../doc/html/Host_8c__incl.map | 3 + .../doc/html/Host_8c__incl.md5 | 1 + .../doc/html/Host_8c__incl.png | Bin 0 -> 12396 bytes ...ea9f99f85b380e912940f351366a862_cgraph.map | 3 + ...ea9f99f85b380e912940f351366a862_cgraph.md5 | 1 + ...ea9f99f85b380e912940f351366a862_cgraph.png | Bin 0 -> 2772 bytes ...2ea6f92f186de790a55fdf700026dd9_cgraph.map | 9 + ...2ea6f92f186de790a55fdf700026dd9_cgraph.md5 | 1 + ...2ea6f92f186de790a55fdf700026dd9_cgraph.png | Bin 0 -> 25855 bytes ...28696e075d2053b4de5662556c16f3d_cgraph.map | 6 + ...28696e075d2053b4de5662556c16f3d_cgraph.md5 | 1 + ...28696e075d2053b4de5662556c16f3d_cgraph.png | Bin 0 -> 12517 bytes ...f3118f365ec96030a022fa17b73fc42_cgraph.map | 3 + ...f3118f365ec96030a022fa17b73fc42_cgraph.md5 | 1 + ...f3118f365ec96030a022fa17b73fc42_cgraph.png | Bin 0 -> 3076 bytes .../doc/html/Input_8c.html | 200 + .../doc/html/Input_8c__incl.map | 3 + .../doc/html/Input_8c__incl.md5 | 1 + .../doc/html/Input_8c__incl.png | Bin 0 -> 10720 bytes .../doc/html/Jacobi_8c.html | 172 + .../doc/html/Jacobi_8c__incl.map | 3 + .../doc/html/Jacobi_8c__incl.md5 | 1 + .../doc/html/Jacobi_8c__incl.png | Bin 0 -> 8834 bytes ...c04138a5bfe5d72780bb7e82a18e627_cgraph.map | 19 + ...c04138a5bfe5d72780bb7e82a18e627_cgraph.md5 | 1 + ...c04138a5bfe5d72780bb7e82a18e627_cgraph.png | Bin 0 -> 54384 bytes .../doc/html/Jacobi_8h.html | 1361 + .../doc/html/Jacobi_8h__dep__incl.map | 8 + .../doc/html/Jacobi_8h__dep__incl.md5 | 1 + .../doc/html/Jacobi_8h__dep__incl.png | Bin 0 -> 12616 bytes .../doc/html/Jacobi_8h__incl.map | 2 + .../doc/html/Jacobi_8h__incl.md5 | 1 + .../doc/html/Jacobi_8h__incl.png | Bin 0 -> 7185 bytes ...8324d1c9b1ee7193242670ba4b97d88_cgraph.map | 3 + ...8324d1c9b1ee7193242670ba4b97d88_cgraph.md5 | 1 + ...8324d1c9b1ee7193242670ba4b97d88_cgraph.png | Bin 0 -> 3101 bytes ...ea9f99f85b380e912940f351366a862_cgraph.map | 3 + ...ea9f99f85b380e912940f351366a862_cgraph.md5 | 1 + ...ea9f99f85b380e912940f351366a862_cgraph.png | Bin 0 -> 2772 bytes ...b641f8b5d521a230416b64d8b809324_cgraph.map | 3 + ...b641f8b5d521a230416b64d8b809324_cgraph.md5 | 1 + ...b641f8b5d521a230416b64d8b809324_cgraph.png | Bin 0 -> 3750 bytes ...2ea6f92f186de790a55fdf700026dd9_cgraph.map | 9 + ...2ea6f92f186de790a55fdf700026dd9_cgraph.md5 | 1 + ...2ea6f92f186de790a55fdf700026dd9_cgraph.png | Bin 0 -> 25855 bytes ...5a52e3c2e58d2d94403a0dce2e29686_cgraph.map | 3 + ...5a52e3c2e58d2d94403a0dce2e29686_cgraph.md5 | 1 + ...5a52e3c2e58d2d94403a0dce2e29686_cgraph.png | Bin 0 -> 4425 bytes ...aebe085afbffaa0d6bcd77fa16ceb97_cgraph.map | 3 + ...aebe085afbffaa0d6bcd77fa16ceb97_cgraph.md5 | 1 + ...aebe085afbffaa0d6bcd77fa16ceb97_cgraph.png | Bin 0 -> 3420 bytes ...f3118f365ec96030a022fa17b73fc42_cgraph.map | 3 + ...f3118f365ec96030a022fa17b73fc42_cgraph.md5 | 1 + ...f3118f365ec96030a022fa17b73fc42_cgraph.png | Bin 0 -> 3076 bytes .../cuda-aware-mpi-example/doc/html/bc_s.png | Bin 0 -> 677 bytes .../doc/html/closed.png | Bin 0 -> 126 bytes .../doc/html/doxygen.css | 949 + .../doc/html/doxygen.png | Bin 0 -> 3942 bytes .../doc/html/files.html | 114 + .../doc/html/globals.html | 302 + .../doc/html/globals_defs.html | 137 + .../doc/html/globals_func.html | 265 + .../doc/html/graph_legend.html | 163 + .../doc/html/graph_legend.md5 | 1 + .../doc/html/graph_legend.png | Bin 0 -> 18827 bytes .../doc/html/index.html | 99 + .../doc/html/installdox | 112 + .../cuda-aware-mpi-example/doc/html/jquery.js | 64 + .../cuda-aware-mpi-example/doc/html/nav_f.png | Bin 0 -> 159 bytes .../cuda-aware-mpi-example/doc/html/nav_h.png | Bin 0 -> 97 bytes .../cuda-aware-mpi-example/doc/html/open.png | Bin 0 -> 118 bytes .../doc/html/search/all_61.html | 25 + .../doc/html/search/all_61.js | 5 + .../doc/html/search/all_62.html | 25 + .../doc/html/search/all_62.js | 4 + .../doc/html/search/all_63.html | 25 + .../doc/html/search/all_63.js | 11 + .../doc/html/search/all_64.html | 25 + .../doc/html/search/all_64.js | 5 + .../doc/html/search/all_65.html | 25 + .../doc/html/search/all_65.js | 5 + .../doc/html/search/all_66.html | 25 + .../doc/html/search/all_66.js | 4 + .../doc/html/search/all_68.html | 25 + .../doc/html/search/all_68.js | 5 + .../doc/html/search/all_69.html | 25 + .../doc/html/search/all_69.js | 6 + .../doc/html/search/all_6a.html | 25 + .../doc/html/search/all_6a.js | 8 + .../doc/html/search/all_6d.html | 25 + .../doc/html/search/all_6d.js | 6 + .../doc/html/search/all_70.html | 25 + .../doc/html/search/all_70.js | 6 + .../doc/html/search/all_72.html | 25 + .../doc/html/search/all_72.js | 4 + .../doc/html/search/all_73.html | 25 + .../doc/html/search/all_73.js | 7 + .../doc/html/search/all_74.html | 25 + .../doc/html/search/all_74.js | 4 + .../doc/html/search/all_75.html | 25 + .../doc/html/search/all_75.js | 4 + .../doc/html/search/close.png | Bin 0 -> 273 bytes .../doc/html/search/defines_64.html | 25 + .../doc/html/search/defines_64.js | 4 + .../doc/html/search/defines_65.html | 25 + .../doc/html/search/defines_65.js | 4 + .../doc/html/search/defines_6a.html | 25 + .../doc/html/search/defines_6a.js | 5 + .../doc/html/search/defines_6d.html | 25 + .../doc/html/search/defines_6d.js | 5 + .../doc/html/search/defines_73.html | 25 + .../doc/html/search/defines_73.js | 5 + .../doc/html/search/defines_75.html | 25 + .../doc/html/search/defines_75.js | 4 + .../doc/html/search/files_63.html | 25 + .../doc/html/search/files_63.js | 5 + .../doc/html/search/files_64.html | 25 + .../doc/html/search/files_64.js | 4 + .../doc/html/search/files_68.html | 25 + .../doc/html/search/files_68.js | 4 + .../doc/html/search/files_69.html | 25 + .../doc/html/search/files_69.js | 4 + .../doc/html/search/files_6a.html | 25 + .../doc/html/search/files_6a.js | 5 + .../doc/html/search/functions_61.html | 25 + .../doc/html/search/functions_61.js | 5 + .../doc/html/search/functions_62.html | 25 + .../doc/html/search/functions_62.js | 4 + .../doc/html/search/functions_63.html | 25 + .../doc/html/search/functions_63.js | 9 + .../doc/html/search/functions_65.html | 25 + .../doc/html/search/functions_65.js | 4 + .../doc/html/search/functions_66.html | 25 + .../doc/html/search/functions_66.js | 4 + .../doc/html/search/functions_68.html | 25 + .../doc/html/search/functions_68.js | 4 + .../doc/html/search/functions_69.html | 25 + .../doc/html/search/functions_69.js | 5 + .../doc/html/search/functions_6a.html | 25 + .../doc/html/search/functions_6a.js | 4 + .../doc/html/search/functions_6d.html | 25 + .../doc/html/search/functions_6d.js | 4 + .../doc/html/search/functions_70.html | 25 + .../doc/html/search/functions_70.js | 6 + .../doc/html/search/functions_72.html | 25 + .../doc/html/search/functions_72.js | 4 + .../doc/html/search/functions_73.html | 25 + .../doc/html/search/functions_73.js | 5 + .../doc/html/search/functions_74.html | 25 + .../doc/html/search/functions_74.js | 4 + .../doc/html/search/mag_sel.png | Bin 0 -> 563 bytes .../doc/html/search/nomatches.html | 12 + .../doc/html/search/search.css | 238 + .../doc/html/search/search.js | 801 + .../doc/html/search/search_l.png | Bin 0 -> 604 bytes .../doc/html/search/search_m.png | Bin 0 -> 158 bytes .../doc/html/search/search_r.png | Bin 0 -> 612 bytes .../cuda-aware-mpi-example/doc/html/tab_a.png | Bin 0 -> 140 bytes .../cuda-aware-mpi-example/doc/html/tab_b.png | Bin 0 -> 178 bytes .../cuda-aware-mpi-example/doc/html/tab_h.png | Bin 0 -> 192 bytes .../cuda-aware-mpi-example/doc/html/tab_s.png | Bin 0 -> 189 bytes .../cuda-aware-mpi-example/doc/html/tabs.css | 59 + .../src/CUDA_Aware_MPI.c | 91 + .../src/CUDA_Normal_MPI.c | 87 + .../cuda-aware-mpi-example/src/Device.cu | 341 + .../posts/cuda-aware-mpi-example/src/Host.c | 501 + .../posts/cuda-aware-mpi-example/src/Input.c | 185 + .../posts/cuda-aware-mpi-example/src/Jacobi.c | 87 + .../cuda-aware-mpi-example/src/Jacobi.doxygen | 1781 + .../posts/cuda-aware-mpi-example/src/Jacobi.h | 164 + .../posts/cuda-aware-mpi-example/src/Makefile | 87 + .../posts/cuda-aware-mpi-example/src/README | 110 + .../posts/cuda-vmm/Makefile | 38 + .../posts/cuda-vmm/cuvector.cpp | 362 + .../posts/cuda-vmm/cuvector.h | 146 + .../posts/cuda-vmm/sync_main.cu | 167 + .../posts/cuda-vmm/vector_main.cpp | 173 + .../posts/cufft-callbacks/Makefile | 44 + .../posts/cufft-callbacks/common.h | 154 + .../posts/cufft-callbacks/cufft_callbacks.cu | 172 + .../cufft-callbacks/cufft_no_callbacks.cu | 215 + .../egl_OpenGl_without_Xserver/EGL/egl.h | 303 + .../egl_OpenGl_without_Xserver/EGL/eglext.h | 821 + .../EGL/eglplatform.h | 125 + .../KHR/khrplatform.h | 282 + .../posts/egl_OpenGl_without_Xserver/Makefile | 18 + .../egl_OpenGl_without_Xserver/Readme.txt | 3 + .../egl_OpenGl_without_Xserver/tinyegl.cc | 187 + .../posts/grcuda/R/cuml_dbscan.R | 59 + .../posts/grcuda/R/run_r.sh | 43 + .../posts/grcuda/README.md | 129 + .../posts/grcuda/kernel/saxpy.cu | 34 + .../posts/grcuda/kernel/saxpy.py | 48 + .../posts/grcuda/kernel/saxpy_nvrtc.py | 58 + .../posts/grcuda/mandelbrot/app.js | 72 + .../posts/grcuda/mandelbrot/license.txt | 25 + .../posts/grcuda/mandelbrot/package.json | 18 + .../posts/gups/LICENSE | 28 + .../posts/gups/LICENSE.gups.cu | 12 + .../posts/gups/Makefile | 95 + .../posts/gups/README.md | 70 + .../posts/gups/gups.cu | 614 + .../posts/gups/run.py | 178 + .../posts/math-libraries-intro/Makefile | 42 + .../posts/math-libraries-intro/README.md | 128 + .../math-libraries-intro/cublas-example.cu | 107 + .../math-libraries-intro/openblas-example.cpp | 47 + .../posts/mixed-precision/.gitignore | 1 + .../posts/mixed-precision/Makefile | 37 + .../posts/mixed-precision/fp16_conversion.h | 114 + .../posts/mixed-precision/haxpy.cu | 101 + .../posts/nvml/Makefile | 36 + .../posts/nvml/matrixMul.cu | 359 + .../posts/nvml/nvml_test.cu | 157 + .../posts/nvtx/Makefile | 51 + .../posts/nvtx/compiler_inst_nvtx.cu | 126 + .../posts/nvtx/env_hpc_sdk.sh | 33 + .../posts/nvtx/inst_nvtx.cpp | 87 + .../posts/nvtx/manual_nvtx.cu | 160 + .../openacc-unified-memory-jacobi/Makefile | 33 + .../common/timer.h | 78 + .../openacc-unified-memory-jacobi/laplace2d.c | 101 + .../parallel_reduction_with_shfl/Makefile | 32 + .../block_reduce.h | 48 + .../device_reduce_atomic.h | 91 + .../device_reduce_block_atomic.h | 96 + .../device_reduce_stable.h | 95 + .../device_reduce_warp_atomic.h | 96 + .../fake_atomic.h | 36 + .../parallel_reduction_with_shfl/fake_shfl.h | 43 + .../parallel_reduction_with_shfl/getdata.sh | 43 + .../parallel_reduction_with_shfl/main.cu | 194 + .../parallel_reduction_with_shfl/readme.md | 5 + .../warp_reduce.h | 37 + .../posts/printf-alternative/CASerror.h | 191 + .../posts/printf-alternative/CMakeLists.txt | 17 + .../posts/printf-alternative/README.md | 3 + .../posts/printf-alternative/main.cu | 304 + .../posts/rnn/LSTM.cu | 682 + .../posts/rnn/Makefile | 34 + .../posts/rnn/README.md | 5 + .../separate-compilation-linking/main.cpp | 73 + .../separate-compilation-linking/makefile | 35 + .../separate-compilation-linking/particle.cpp | 49 + .../separate-compilation-linking/particle.h | 46 + .../posts/separate-compilation-linking/v3.cpp | 59 + .../posts/separate-compilation-linking/v3.h | 45 + .../posts/tensor-cores/Makefile | 35 + .../posts/tensor-cores/README.md | 3 + .../tensor-cores/simpleTensorCoreGEMM.cu | 309 + .../unified-memory-oversubscription/Makefile | 37 + .../unified-memory-oversubscription/README.md | 39 + .../uvm_oversubs.cu | 550 + .../posts/unified-memory/.gitignore | 4 + .../posts/unified-memory/Makefile | 38 + .../posts/unified-memory/README.md | 12 + .../posts/unified-memory/dataElem.cu | 86 + .../posts/unified-memory/dataElem_um.cu | 66 + .../posts/unified-memory/dataElem_um_c++_1.cu | 82 + .../posts/unified-memory/dataElem_um_c++_2.cu | 163 + .../cuda-cpp/coalescing-global/coalescing.cu | 137 + .../finite-difference/finite-difference.cu | 523 + .../optimize-data-transfers/bandwidthtest.cu | 135 + .../optimize-data-transfers/profile.cu | 41 + .../cuda-cpp/overlap-data-transfers/async.cu | 165 + .../cuda-cpp/shared-memory/shared-memory.cu | 76 + .../series/cuda-cpp/transpose/transpose.cu | 313 + .../series/cuda-fortran/async.cuf | 139 + .../series/cuda-fortran/finiteDifference.cuf | 638 + .../series/cuda-fortran/shared.cuf | 118 + .../series/cuda-fortran/transpose.cuf | 326 + .../kernelwiki-ext-transpose/LICENSE | 21 + .../kernelwiki-ext-transpose/Makefile | 46 + .../kernelwiki-ext-transpose/README.md | 13 + .../kernelwiki-ext-transpose/swizzle.cu | 148 + .../transpose_naive.cu | 197 + .../transpose_swizzle.cu | 224 + .../transpose_swizzle_batched.cu | 229 + .../transpose_swizzle_batched_for_profile.cu | 204 + .../kernelwiki-ext-transpose/utils.h | 188 + .../flash_attn/cute/utils.py | 1041 + .../flash_attn/cute/.flake8 | 4 + .../flash_attn/cute/AUTHORS | 8 + .../flash_attn/cute/LICENSE | 29 + .../flash_attn/cute/MANIFEST.in | 5 + .../flash_attn/cute/README.md | 33 + .../flash_attn/cute/__init__.py | 18 + .../flash_attn/cute/ampere_helpers.py | 103 + .../flash_attn/cute/barrier.py | 71 + .../flash_attn/cute/bench_utils.py | 243 + .../flash_attn/cute/benchmark.py | 268 + .../cute/benchmark_flash_attention_fp8.py | 434 + .../flash_attn/cute/blackwell_helpers.py | 1115 + .../flash_attn/cute/block_info.py | 169 + .../flash_attn/cute/block_sparse_utils.py | 1954 + .../flash_attn/cute/block_sparsity.py | 726 + .../flash_attn/cute/cache_utils.py | 281 + .../flash_attn/cute/compute_block_sparsity.py | 551 + .../flash_attn/cute/copy_utils.py | 372 + .../flash_attn/cute/cu_blocks_kernel.py | 181 + .../flash_attn/cute/cute_dsl_ptxas.py | 151 + .../flash_attn/cute/cute_dsl_utils.py | 255 + .../flash_attn/cute/fa_logging.py | 97 + .../flash_attn/cute/fast_math.py | 21 + .../flash_attn/cute/flash_bwd.py | 1346 + .../flash_attn/cute/flash_bwd_mla_dk_sm100.py | 1185 + .../cute/flash_bwd_mla_dq_dqv_sm100.py | 1297 + .../flash_attn/cute/flash_bwd_mla_sm100.py | 2135 + .../flash_attn/cute/flash_bwd_postprocess.py | 683 + .../flash_attn/cute/flash_bwd_preprocess.py | 467 + .../flash_attn/cute/flash_bwd_sm100.py | 4174 + .../flash_attn/cute/flash_bwd_sm120.py | 55 + .../flash_attn/cute/flash_bwd_sm90.py | 1955 + .../flash_attn/cute/flash_fwd.py | 1247 + .../flash_attn/cute/flash_fwd_combine.py | 738 + .../flash_attn/cute/flash_fwd_mla_sm100.py | 3160 + .../flash_attn/cute/flash_fwd_sm100.py | 3385 + .../flash_attn/cute/flash_fwd_sm120.py | 61 + .../flash_attn/cute/flash_fwd_sm90.py | 1550 + .../flash_attn/cute/interface.py | 4143 + .../flash_attn/cute/mask.py | 1711 + .../flash_attn/cute/mma_sm100_desc.py | 296 + .../flash_attn/cute/named_barrier.py | 63 + .../flash_attn/cute/pack_gqa.py | 263 + .../flash_attn/cute/paged_kv.py | 248 + .../flash_attn/cute/pipeline.py | 402 + .../flash_attn/cute/prepare_scheduler.py | 387 + .../flash_attn/cute/pyproject.toml | 75 + .../flash_attn/cute/seqlen_info.py | 304 + .../cute/sm100_hd256_2cta_fmha_backward.py | 225 + ...100_hd256_2cta_fmha_backward_dkdvkernel.py | 3411 + ...sm100_hd256_2cta_fmha_backward_dqkernel.py | 2317 + .../cute/sm100_hd256_2cta_fmha_forward.py | 1920 + .../flash_attn/cute/sm90_config_search.py | 402 + .../flash_attn/cute/softmax.py | 711 + .../flash_attn/cute/testing.py | 492 + .../flash_attn/cute/tile_scheduler.py | 2120 + .../flash_attn/cute/topk_gather_kv.py | 278 + .../flash_attn/cute/utils.py | 1041 + .../EVALUATION.md | 165 + .../bench_trtllm_gen_fused_moe_autotuner.py | 666 + .../flashinfer/fused_moe/core.py | 5235 ++ .../flashinfer/fused_moe/runners.py | 2155 + .../flashinfer/gdn_prefill.py | 581 + .../flashinfer/trace/templates/gdn.py | 758 + .../flashinfer/trace/templates/moe.py | 4052 + .../kernelwiki-flashmla-pinned/README.md | 234 + .../csrc/sm100/decode/head128/README.md | 1 + .../csrc/sm100/decode/head64/config.h | 212 + .../decode/head64/instantiations/model1.cu | 8 + .../sm100/decode/head64/instantiations/v32.cu | 8 + .../csrc/sm100/decode/head64/kernel.cuh | 968 + .../csrc/sm100/decode/head64/kernel.h | 11 + .../csrc/sm100/helpers.h | 35 + .../prefill/dense/collective/fmha_common.hpp | 127 + .../prefill/dense/collective/fmha_fusion.hpp | 396 + ..._fmha_fwd_epilogue_tma_warpspecialized.hpp | 237 + ..._fmha_fwd_mainloop_tma_warpspecialized.hpp | 1214 + .../sm100_fmha_load_tma_warpspecialized.hpp | 302 + ...a_mla_fwd_mainloop_tma_warpspecialized.hpp | 1221 + ...m100_fmha_mla_load_tma_warpspecialized.hpp | 326 + .../prefill/dense/common/gather_tensor.hpp | 215 + .../csrc/sm100/prefill/dense/common/helper.h | 72 + .../csrc/sm100/prefill/dense/common/mask.cuh | 8 + .../prefill/dense/common/pipeline_mla.hpp | 250 + .../csrc/sm100/prefill/dense/common/pow_2.hpp | 92 + .../csrc/sm100/prefill/dense/common/utils.hpp | 33 + .../csrc/sm100/prefill/dense/device/fmha.hpp | 281 + .../prefill/dense/device/fmha_device_bwd.hpp | 340 + .../prefill/dense/fmha_cutlass_bwd_sm100.cu | 83 + .../prefill/dense/fmha_cutlass_bwd_sm100.cuh | 203 + .../prefill/dense/fmha_cutlass_fwd_sm100.cu | 83 + .../prefill/dense/fmha_cutlass_fwd_sm100.cuh | 335 + .../csrc/sm100/prefill/dense/interface.h | 14 + .../kernel/fmha_causal_tile_scheduler.hpp | 197 + .../dense/kernel/fmha_kernel_bwd_convert.hpp | 160 + .../dense/kernel/fmha_kernel_bwd_sum_OdO.hpp | 168 + .../prefill/dense/kernel/fmha_options.hpp | 85 + .../dense/kernel/fmha_tile_scheduler.hpp | 162 + ...00_fmha_bwd_kernel_tma_warpspecialized.hpp | 1835 + ...mha_bwd_mla_kernel_tma_warpspecialized.hpp | 1829 + ...00_fmha_fwd_kernel_tma_warpspecialized.hpp | 643 + .../sm100/prefill/sparse/common_subroutine.h | 208 + .../sm100/prefill/sparse/fwd/head128/config.h | 140 + .../fwd/head128/instantiations/phase1_k512.cu | 8 + .../fwd/head128/instantiations/phase1_k576.cu | 8 + .../prefill/sparse/fwd/head128/phase1.cuh | 706 + .../sm100/prefill/sparse/fwd/head128/phase1.h | 10 + .../sm100/prefill/sparse/fwd/head64/config.h | 157 + .../fwd/head64/instantiations/phase1_k512.cu | 8 + .../fwd/head64/instantiations/phase1_k576.cu | 8 + .../prefill/sparse/fwd/head64/phase1.cuh | 673 + .../sm100/prefill/sparse/fwd/head64/phase1.h | 10 + .../fwd_for_small_topk/head128/config.h | 140 + .../instantiations/phase1_decode_k512.cu | 8 + .../instantiations/phase1_prefill_k512.cu | 8 + .../fwd_for_small_topk/head128/phase1.cuh | 1107 + .../fwd_for_small_topk/head128/phase1.h | 10 + .../csrc/sm90/decode/dense/config.h | 11 + .../sm90/decode/dense/instantiations/bf16.cu | 8 + .../sm90/decode/dense/instantiations/fp16.cu | 10 + .../csrc/sm90/decode/dense/splitkv_mla.cuh | 1355 + .../csrc/sm90/decode/dense/splitkv_mla.h | 10 + .../csrc/sm90/decode/dense/traits.h | 107 + .../decode/sparse_fp8/components/config.h | 21 + .../decode/sparse_fp8/components/dequant.h | 127 + .../decode/sparse_fp8/components/helpers.h | 109 + .../csrc/sm90/decode/sparse_fp8/config.h | 279 + .../instantiations/model1_persistent_h128.cu | 7 + .../instantiations/model1_persistent_h64.cu | 8 + .../instantiations/v32_persistent_h128.cu | 7 + .../instantiations/v32_persistent_h64.cu | 7 + .../sm90/decode/sparse_fp8/splitkv_mla.cuh | 787 + .../csrc/sm90/decode/sparse_fp8/splitkv_mla.h | 11 + .../flash_mla/flash_mla_interface.py | 435 + .../kernelwiki-flashmla-pinned/tests/quant.py | 158 + .../kernelwiki-flashmla-pinned/tests/ref.py | 103 + .../tests/test_flash_mla_sparse_decoding.py | 319 + .../kernelwiki-gdn-pinned/README.md | 225 + .../lit_gpt/gated_delta_net.py | 233 + .../lit_gpt/gated_delta_rule_ops/chunk.py | 748 + .../.clang-format | 1 + .../kernelwiki-learn-cuda-3b90ac9b/.gitignore | 3 + .../01_vector_addition/add.cu | 30 + .../01_vector_addition/main.py | 16 + .../02a_matmul_simt/README.md | 213 + .../02a_matmul_simt/main.cu | 84 + .../02a_matmul_simt/main.py | 56 + .../02a_matmul_simt/matmul.cpp | 39 + .../02a_matmul_simt/matmul.cu | 534 + .../02a_matmul_simt/matmul_triton.py | 183 + .../02b_matmul_sm80/README.md | 56 + .../02b_matmul_sm80/common.h | 117 + .../02b_matmul_sm80/cutedsl_v1.py | 458 + .../02b_matmul_sm80/main.py | 82 + .../02b_matmul_sm80/matmul.cpp | 45 + .../02b_matmul_sm80/matmul_v1.cu | 276 + .../02b_matmul_sm80/matmul_v5.cu | 148 + .../02b_matmul_sm80/matmul_v6.cu | 185 + .../02c_matmul_sm120/README.md | 20 + .../02c_matmul_sm120/common.h | 52 + .../02c_matmul_sm120/main.py | 78 + .../02c_matmul_sm120/matmul.cpp | 35 + .../02c_matmul_sm120/matmul_v0.cu | 196 + .../02c_matmul_sm120/matmul_v1.cu | 226 + .../02d_matmul_cdna3/.gitignore | 1 + .../02d_matmul_cdna3/README.md | 45 + .../02d_matmul_cdna3/common.h | 38 + .../02d_matmul_cdna3/main.py | 67 + .../02d_matmul_cdna3/matmul.cu | 51 + .../02d_matmul_cdna3/matmul_v1.cu | 191 + .../02d_matmul_cdna3/matmul_v2.cu | 148 + .../02d_matmul_cdna3/matmul_v3.cu | 248 + .../02e_matmul_sm100/README.md | 71 + .../02e_matmul_sm100/common.h | 127 + .../02e_matmul_sm100/main.py | 161 + .../02e_matmul_sm100/matmul.cpp | 81 + .../02e_matmul_sm100/matmul_v0.cu | 211 + .../02e_matmul_sm100/matmul_v1.cu | 224 + .../02e_matmul_sm100/matmul_v2.cu | 233 + .../02e_matmul_sm100/matmul_v3.cu | 243 + .../02e_matmul_sm100/matmul_v4.cu | 243 + .../02e_matmul_sm100/matmul_v5.cu | 315 + .../02e_matmul_sm100/matmul_v6.cu | 396 + .../02e_matmul_sm100/profiler.h | 50 + .../02e_matmul_sm100/trace_v5.json.gz | Bin 0 -> 488587 bytes .../02e_matmul_sm100/trace_v6.json.gz | Bin 0 -> 474510 bytes .../03_sum/README.md | 27 + .../03_sum/main.cu | 57 + .../03_sum/main.py | 49 + .../03_sum/sum.cpp | 84 + .../03_sum/sum.cu | 283 + .../04_softmax/README.md | 40 + .../04_softmax/main.py | 43 + .../04_softmax/softmax.cpp | 32 + .../04_softmax/softmax.cu | 362 + .../05_fp6/README.md | 86 + .../05_fp6/fp6.cu | 602 + .../06_box_blur/box_blur.cu | 115 + .../06_box_blur/main.py | 53 + .../07_attention/README.md | 18 + .../07_attention/attention.cpp | 49 + .../07_attention/attention_v1.cu | 258 + .../07_attention/attention_v2.cu | 280 + .../07_attention/attention_v3.cu | 299 + .../07_attention/attention_v4.cu | 308 + .../07_attention/attention_v5.cu | 308 + .../07_attention/common.h | 121 + .../07_attention/main.py | 112 + .../08_row_scaled_mm/README.md | 26 + .../08_row_scaled_mm/common.h | 134 + .../08_row_scaled_mm/ext.cpp | 62 + .../08_row_scaled_mm/main.py | 109 + .../08_row_scaled_mm/row_scaled_mm_v1.cu | 201 + .../09_block_scaled_mm/README.md | 4 + .../09_block_scaled_mm/common.h | 110 + .../09_block_scaled_mm/ext.cpp | 52 + .../09_block_scaled_mm/main.py | 133 + .../09_block_scaled_mm/mxfp8_mm_v1.cu | 182 + .../09_block_scaled_mm/mxfp8_mm_v2.cu | 153 + .../10_p2p/README.md | 13 + .../10_p2p/main.py | 92 + .../10_p2p/p2p.cu | 109 + .../11_gemv/main.py | 42 + .../11_gemv/triton_v1.py | 70 + .../kernelwiki-learn-cuda-3b90ac9b/README.md | 79 + .../pyproject.toml | 6 + .../kernelwiki-mlstm-kernels/README.md | 297 + .../local-snapshots/kernelwiki-ptx-1302.html | 44686 +++++++++++ .../nvidia/nvfp4_dual_gemm/reference.py | 199 + .../problems/nvidia/nvfp4_dual_gemm/task.yml | 64 + .../nvidia/nvfp4_dual_gemm/template.py | 28 + .../problems/nvidia/nvfp4_gemm/reference.py | 161 + .../problems/nvidia/nvfp4_gemm/task.py | 11 + .../problems/nvidia/nvfp4_gemm/task.yml | 58 + .../problems/nvidia/nvfp4_gemm/template.py | 25 + .../problems/nvidia/nvfp4_gemv/reference.py | 169 + .../problems/nvidia/nvfp4_gemv/task.py | 10 + .../problems/nvidia/nvfp4_gemv/task.yml | 60 + .../problems/nvidia/nvfp4_gemv/template.py | 25 + .../problems/nvidia/nvfp4_group_gemm/task.yml | 67 + verification/original-report.jsonl | 88 +- verification/remediation-log.jsonl | 472 +- .../reviewed/current/hw-2sm-cooperative.json | 2 +- verification/reviewed/current/hw-clc.json | 2 +- .../reviewed/current/hw-mbarrier.json | 6 +- verification/reviewed/current/hw-nvfp4.json | 4 +- verification/reviewed/current/hw-pdl-gdc.json | 4 +- .../reviewed/current/hw-tcgen05-mma.json | 16 +- verification/reviewed/current/hw-tma.json | 6 +- verification/reviewed/current/hw-tmem.json | 22 +- .../reviewed/current/kernel-deepgemm.json | 4 +- .../current/kernel-flash-attention-4.json | 4 +- ...kernel-flash-attention-sm100-mla-topk.json | 2 +- .../reviewed/current/kernel-flashmla.json | 16 +- .../current/kernel-fp8-block-scale-gemm.json | 12 +- .../reviewed/current/kernel-fused-moe.json | 12 +- .../current/kernel-gated-delta-net.json | 10 +- .../current/kernel-gated-dual-gemm.json | 10 +- .../reviewed/current/kernel-grouped-gemm.json | 4 +- verification/reviewed/current/kernel-nsa.json | 6 +- .../reviewed/current/kernel-nvfp4-gemm.json | 16 +- .../reviewed/current/kernel-nvfp4-gemv.json | 6 +- .../reviewed/current/kernel-sparse-mla.json | 10 +- .../reviewed/current/lang-cuda-cpp.json | 12 +- .../reviewed/current/lang-cute-dsl.json | 14 +- verification/reviewed/current/lang-ptx.json | 6 +- .../reviewed/current/lang-triton.json | 6 +- .../current/migration-register-to-tmem.json | 12 +- .../current/migration-wgmma-to-tcgen05.json | 16 +- .../current/pattern-compute-bound.json | 2 +- .../current/pattern-low-sm-utilization.json | 4 +- .../current/pattern-moe-load-imbalance.json | 4 +- .../current/pattern-pipeline-stalls.json | 2 +- .../current/pattern-register-pressure.json | 6 +- .../reviewed/current/pattern-tail-effect.json | 4 +- .../current/technique-chunk-parallelism.json | 2 +- .../current/technique-double-buffering.json | 6 +- .../current/technique-epilogue-fusion.json | 12 +- ...echnique-external-source-map-research.json | 26 +- .../technique-fine-grained-quantization.json | 8 +- .../current/technique-kernel-fusion.json | 8 +- .../current/technique-persistent-kernels.json | 4 +- .../current/technique-pipeline-stages.json | 6 +- .../current/technique-register-budgeting.json | 6 +- .../reviewed/current/technique-swizzling.json | 12 +- .../current/technique-tile-scheduling.json | 8 +- .../technique-warp-specialization.json | 8 +- .../reviewed/original/hw-2sm-cooperative.json | 4 +- verification/reviewed/original/hw-clc.json | 8 +- .../reviewed/original/hw-mbarrier.json | 4 +- verification/reviewed/original/hw-nvfp4.json | 4 +- .../reviewed/original/hw-tcgen05-mma.json | 38 +- verification/reviewed/original/hw-tma.json | 10 +- verification/reviewed/original/hw-tmem.json | 36 +- .../reviewed/original/kernel-deepgemm.json | 6 +- .../original/kernel-flash-attention-4.json | 10 +- ...kernel-flash-attention-sm100-mla-topk.json | 2 +- .../reviewed/original/kernel-flashmla.json | 36 +- .../original/kernel-fp8-block-scale-gemm.json | 12 +- .../reviewed/original/kernel-fused-moe.json | 28 +- .../original/kernel-gated-delta-net.json | 22 +- .../original/kernel-gated-dual-gemm.json | 16 +- .../original/kernel-grouped-gemm.json | 22 +- .../reviewed/original/kernel-nsa.json | 8 +- .../reviewed/original/kernel-nvfp4-gemm.json | 24 +- .../reviewed/original/kernel-nvfp4-gemv.json | 16 +- .../reviewed/original/kernel-sparse-mla.json | 34 +- .../reviewed/original/lang-cuda-cpp.json | 10 +- .../reviewed/original/lang-cute-dsl.json | 20 +- verification/reviewed/original/lang-ptx.json | 4 +- .../reviewed/original/lang-triton.json | 12 +- .../original/migration-register-to-tmem.json | 22 +- .../original/migration-wgmma-to-tcgen05.json | 34 +- .../original/pattern-compute-bound.json | 6 +- .../original/pattern-low-sm-utilization.json | 4 +- .../original/pattern-moe-load-imbalance.json | 4 +- .../original/pattern-pipeline-stalls.json | 4 +- .../original/pattern-register-pressure.json | 6 +- .../original/pattern-tail-effect.json | 6 +- .../original/technique-chunk-parallelism.json | 2 +- .../original/technique-double-buffering.json | 18 +- .../original/technique-epilogue-fusion.json | 30 +- ...echnique-external-source-map-research.json | 26 +- .../technique-fine-grained-quantization.json | 10 +- .../original/technique-kernel-fusion.json | 6 +- .../technique-persistent-kernels.json | 10 +- .../technique-ping-pong-scheduling.json | 2 +- .../original/technique-pipeline-stages.json | 4 +- .../original/technique-swizzling.json | 24 +- .../original/technique-tile-scheduling.json | 14 +- .../technique-warp-specialization.json | 24 +- .../tools/audit_verification_evidence.py | 235 + .../tools/check_verification_evidence.py | 677 + .../tools/migrate_verification_evidence.py | 308 + .../tools/normalize_verification_records.py | 325 + verification/verification-report.jsonl | 90 +- wiki/hardware/nvfp4.md | 2 +- wiki/hardware/tmem.md | 6 +- wiki/kernels/nvfp4-gemm.md | 2 +- wiki/migration/register-to-tmem.md | 4 +- wiki/patterns/compute-bound.md | 2 +- wiki/patterns/low-sm-utilization.md | 2 +- wiki/patterns/moe-load-imbalance.md | 2 +- wiki/patterns/pipeline-stalls.md | 2 +- wiki/patterns/register-pressure.md | 2 +- wiki/patterns/tail-effect.md | 2 +- wiki/techniques/double-buffering.md | 4 +- wiki/techniques/epilogue-fusion.md | 4 +- wiki/techniques/fine-grained-quantization.md | 2 +- wiki/techniques/persistent-kernels.md | 2 +- wiki/techniques/pipeline-stages.md | 2 +- wiki/techniques/swizzling.md | 2 +- 1035 files changed, 382984 insertions(+), 1019 deletions(-) create mode 100644 verification/evidence/MANIFEST.json create mode 100644 verification/evidence/README.md create mode 100644 verification/evidence/historical-artifacts/MANIFEST.json create mode 100644 verification/evidence/historical-artifacts/artifacts/kernels/flashmla/variants/01-mla-decode-inner-loop.cu create mode 100644 verification/evidence/historical-artifacts/artifacts/kernels/gated-delta-net/variants/01-chunk-parallel-prefill-reference-pytorch.py create mode 100644 verification/evidence/historical-artifacts/artifacts/kernels/gated-delta-net/variants/02-triton-decode-step-kernel-streaming.py create mode 100644 verification/evidence/historical-artifacts/artifacts/kernels/gated-dual-gemm/full/blackwell-cutlass-schedules-and-tma.cu create mode 100644 verification/evidence/historical-artifacts/artifacts/kernels/gated-dual-gemm/full/vllm-PR-23696-gated-dual-gemm.patch create mode 100644 verification/evidence/historical-artifacts/artifacts/kernels/gated-dual-gemm/variants/01-fused-epilogue-swiglu-skeleton.cu create mode 100644 verification/evidence/local-snapshots/cutlass-audit.8aj07j/repo/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu create mode 100644 verification/evidence/local-snapshots/cutlass-audit.8aj07j/repo/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu create mode 100644 verification/evidence/local-snapshots/cutlass-audit.8aj07j/repo/include/cutlass/gemm/dispatch_policy.hpp create mode 100644 verification/evidence/local-snapshots/cutlass-audit.8aj07j/repo/include/cutlass/gemm/kernel/gemm_grouped.h create mode 100644 verification/evidence/local-snapshots/cutlass-audit.8aj07j/repo/media/docs/cpp/efficient_gemm.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-amandeep-nvfp4.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-amandeepsp-cuda/nvfp4/gemv/attempt1.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-amandeepsp-cuda/nvfp4/gemv/attempt10.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-amandeepsp-cuda/nvfp4/gemv/attempt11.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-amandeepsp-cuda/nvfp4/gemv/attempt12.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-amandeepsp-cuda/nvfp4/gemv/attempt2.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-amandeepsp-cuda/nvfp4/gemv/attempt3.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-amandeepsp-cuda/nvfp4/gemv/attempt4.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-amandeepsp-cuda/nvfp4/gemv/attempt5.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-amandeepsp-cuda/nvfp4/gemv/attempt6.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-amandeepsp-cuda/nvfp4/gemv/attempt7.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-amandeepsp-cuda/nvfp4/gemv/attempt8.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-amandeepsp-cuda/nvfp4/gemv/attempt9.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-amandeepsp-cuda/nvfp4/gemv/reference.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-amandeepsp-cuda/nvfp4/gemv/task.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-amandeepsp-cuda/nvfp4/gemv/test_harness.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/builders/sm100_builder.inl create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/builders/sm103_builder.inl create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/builders/sm120_builder.inl create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/builders/sm120_common.inl create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/builders/sm90_builder.inl create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/builders/sm90_common.inl create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/collective_builder.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/collective_epilogue.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/default_epilogue.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/default_epilogue_array.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/detail.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/epilogue_tensor_broadcast.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/sm100_epilogue_array_nosmem.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/sm100_epilogue_array_planar_complex_nosmem.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/sm100_epilogue_array_planar_complex_tma_warpspecialized.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/sm100_epilogue_array_tma_warpspecialized.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/sm100_epilogue_nosmem.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/sm100_epilogue_planar_complex_tma_warpspecialized.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/sm100_epilogue_tma_warpspecialized.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/sm70_epilogue_vectorized.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/sm70_epilogue_vectorized_array.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/sm90_epilogue_array_tma_warpspecialized.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/sm90_epilogue_tma_warpspecialized.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/sm90_epilogue_tma_warpspecialized_bias_elementwise.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/dispatch_policy.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/callbacks.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/operations.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/sm100_callbacks_tma_warpspecialized.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/sm100_visitor_compute_tma_warpspecialized.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/sm100_visitor_store_tma_warpspecialized.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/sm120_callbacks_tma_warpspecialized.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/sm120_visitor_store_tma_warpspecialized.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/sm90_callbacks_tma_warpspecialized.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/sm90_visitor_compute_tma_warpspecialized.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/sm90_visitor_load_tma_warpspecialized.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/sm90_visitor_store_tma_warpspecialized.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/sm90_visitor_tma_warpspecialized.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/sm90_visitor_topk_softmax.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/activation.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/conversion_op.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/detail.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/linear_combination.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/linear_combination_bias_elementwise.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/linear_combination_bias_relu.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/linear_combination_clamp.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/linear_combination_dgelu.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/linear_combination_drelu.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/linear_combination_gelu.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/linear_combination_generic.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/linear_combination_generic_with_scaling.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/linear_combination_hardswish.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/linear_combination_leaky_relu.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/linear_combination_params.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/linear_combination_planar_complex.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/linear_combination_relu.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/linear_combination_relu0.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/linear_combination_residual_block.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/linear_combination_sigmoid.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/linear_combination_silu.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/linear_combination_tensor_broadcast.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/linear_combination_with_elementwise.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/reduction_op.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/scale_type.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/default_epilogue_complex_tensor_op.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/default_epilogue_complex_tensor_op_blas3.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/default_epilogue_direct_store.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/default_epilogue_planar_complex.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/default_epilogue_simt.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/default_epilogue_tensor_op.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/default_epilogue_tensor_op_blas3.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/default_epilogue_volta_tensor_op.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/default_epilogue_with_absmax.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/default_epilogue_with_broadcast.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/default_epilogue_with_reduction.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/default_epilogue_wmma_tensor_op.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/default_thread_map_simt.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/default_thread_map_tensor_op.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/default_thread_map_volta_tensor_op.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/default_thread_map_wmma_tensor_op.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/direct_store_epilogue_iterator.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/epilogue.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/epilogue_base.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/epilogue_base_streamk.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/epilogue_depthwise.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/epilogue_direct_store.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/epilogue_gemm_k_reduction.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/epilogue_planar_complex.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/epilogue_smem_accumulator.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/epilogue_streamk_with_broadcast.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/epilogue_visitor_with_softmax.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/epilogue_with_absmax.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/epilogue_with_broadcast.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/epilogue_with_reduction.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/epilogue_with_scaling_factor.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/epilogue_with_visitor.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/epilogue_with_visitor_callbacks.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/epilogue_workspace.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/fusion/visitor_2x.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/fusion/visitor_compute.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/fusion/visitor_load.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/fusion/visitor_store.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/fusion/visitors.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/interleaved_epilogue.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/output_iterator_parameter.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/output_tile_thread_map.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/predicated_tile_iterator.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/predicated_tile_iterator_affine.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/predicated_tile_iterator_affine_layout_params.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/predicated_tile_iterator_blas3.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/predicated_tile_iterator_conv.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/predicated_tile_iterator_direct_conv.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/predicated_tile_iterator_params.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/predicated_tile_iterator_predicates.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/predicated_tile_iterator_strided_dgrad.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/shared_load_iterator.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/shared_load_iterator_mixed.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/threadblock/shared_load_iterator_pitch_linear.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/warp/fragment_iterator_complex_tensor_op.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/warp/fragment_iterator_gaussian_complex_tensor_op.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/warp/fragment_iterator_simt.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/warp/fragment_iterator_tensor_op.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/warp/fragment_iterator_volta_tensor_op.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/warp/fragment_iterator_wmma_tensor_op.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/warp/simt_policy.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/warp/tensor_op_policy.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/warp/tile_iterator_simt.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/warp/tile_iterator_tensor_op.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/warp/tile_iterator_tensor_op_mixed.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/warp/tile_iterator_volta_tensor_op.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/warp/tile_iterator_wmma_tensor_op.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/warp/volta_tensor_op_policy.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/warp/wmma_tensor_op_policy.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/sm100_gemm_tma_warpspecialized.hpp create mode 100755 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/sm100_tile_scheduler_group.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/sm100_tile_scheduler_stream_k.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/tile_scheduler.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/tile_scheduler_detail.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/tile_scheduler_params.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0/CHANGELOG.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_1.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_3.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_3_1.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_4.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_5.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_6.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/nvfp4_gemm_0.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/nvfp4_gemm_1.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/utils.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0/python/CuTeDSL/cutlass/cute/arch/tmem.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0/python/CuTeDSL/cutlass/cute/nvgpu/tcgen05/mma.py create mode 100755 verification/evidence/local-snapshots/kernelwiki-cutlass-dRC5i4/cutlass/CMakeLists.txt create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-dRC5i4/cutlass/examples/70_blackwell_gemm/70_blackwell_fp16_gemm.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-dRC5i4/cutlass/examples/cute/tutorial/blackwell/02_mma_tma_sm100.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-dRC5i4/cutlass/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-dRC5i4/cutlass/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_6.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-dRC5i4/cutlass/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_tma/tma_v0.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-dRC5i4/cutlass/include/cute/arch/mma_sm90_gmma.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-dRC5i4/cutlass/include/cutlass/arch/grid_dependency_control.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-cutlass-dRC5i4/cutlass/python/CuTeDSL/cutlass/utils/tmem_allocator.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-deepgemm-pinned/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-deepgemm-pinned/csrc/apis/gemm.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-deepgemm-pinned/csrc/jit/compiler.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh create mode 100644 verification/evidence/local-snapshots/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm100_fp8_mqa_logits.cuh create mode 100644 verification/evidence/local-snapshots/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh create mode 100644 verification/evidence/local-snapshots/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/ptx/tcgen05.cuh create mode 100644 verification/evidence/local-snapshots/kernelwiki-deepgemm-pinned/tests/generators.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-deepgemm-pinned/tests/test_fp8_fp4.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-deepseek-v32/DeepSeek_V3_2.pdf create mode 100644 verification/evidence/local-snapshots/kernelwiki-deepseek-v32/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-deepseek-v32/inference/config_671B_v3.2.json create mode 100644 verification/evidence/local-snapshots/kernelwiki-deepseek-v32/inference/model.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/.gitmodules create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/cutlass_gemm/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/cutlass_gemm/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/cutlass_gemm/cutlass_gemm/cutlass_gemm.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/cutlass_gemm/cutlass_gemm/cutlass_gemm.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/cutlass_gemm/cutlass_gemm/setup.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/cutlass_gemm/gemm.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/evt/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/evt/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/evt/evt_gemm_cute.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/evt/node_types.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/evt/reference.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/include/utils/cuda_launch.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/pipeline-gemm/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/pipeline-gemm/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/pipeline-gemm/hopper-gemm-ws/convert_util.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/pipeline-gemm/hopper-gemm-ws/epilogue_sm90_tma_ws.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/pipeline-gemm/hopper-gemm-ws/hopper_gemm_kernel.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/pipeline-gemm/hopper-gemm-ws/hopper_gemm_kernel_launch.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/pipeline-gemm/hopper-gemm-ws/kernel_traits.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/pipeline-gemm/hopper-gemm-ws/mainloop_sm90_tma_gmma_ws.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/pipeline-gemm/hopper-gemm-ws/tile_scheduler.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/pipeline-gemm/sm90_gemm_multistage.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/pipeline-gemm/sm90_gemm_ws.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/streamk/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/streamk/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/streamk/cli_options.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/streamk/convert_util.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/streamk/cutlass_streamk/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/streamk/cutlass_streamk/benchmark.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/streamk/cutlass_streamk/profile.sh create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/streamk/epilogue_sm90_tma_ws.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/streamk/hopper_gemm_kernel.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/streamk/hopper_gemm_kernel_launch.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/streamk/kernel_traits.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/streamk/mainloop_sm90_tma_gmma_ws.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/streamk/streamk.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/streamk/tile_scheduler.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/tma/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/tma/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/tma/main.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/tma/scale_tma_kernel.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/tma/shared_storage.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/tma/smem_helper.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/tma/tma_copy.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/tma/tma_copy_multicast.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/transpose-cute/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/transpose-cute/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/transpose-cute/include/copy.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/transpose-cute/include/copy_direct.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/transpose-cute/include/copy_smem.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/transpose-cute/include/fma_inner_product_naive.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/transpose-cute/include/shared_storage.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/transpose-cute/include/smem_helper.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/transpose-cute/include/transpose_naive.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/transpose-cute/include/transpose_smem.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/transpose-cute/include/transpose_tmastore_vectorized.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/transpose-cute/include/util.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/transpose-cute/main.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/transpose-cute/python/copy_cute.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/transpose-cute/python/setup.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/transpose-cute/python/transpose_cute.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-cfx/transpose-cute/torch_benchmark.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/LICENSE create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/include/utils/cuda_launch.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/include/utils/fmha_cutlass.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/include/utils/random.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/lib/gemm/copy_tensor.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/lib/gemm/gemm_tensor.hpp create mode 100755 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/cute-gemm-tma-gma/compile.sh create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/cute-gemm-tma-gma/gemm.cu create mode 100755 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/cutlass-gemm/compile.sh create mode 100755 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/cutlass-gemm/compile_ws.sh create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/cutlass-gemm/gemm.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/fmha-pipeline/README.md create mode 100755 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/fmha-pipeline/compile.sh create mode 100755 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/fmha-pipeline/compile_H128.sh create mode 100755 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/fmha-pipeline/compile_H64.sh create mode 100755 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/fmha-pipeline/compile_run_all_config.sh create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/fmha-pipeline/fmha_consumer.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/fmha-pipeline/fmha_epilogue.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/fmha-pipeline/fmha_forward.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/fmha-pipeline/fmha_nopipe.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/fmha-pipeline/fmha_pipe_nows.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/fmha-pipeline/fmha_pipe_ws.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/fmha-pipeline/fmha_producer.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/fmha-pipeline/online_softmax.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/fmha-pipeline/reg2reg.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/fmha-pipeline/shared_storage.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/fmha-pipeline/ss_helper.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/fmha/README.md create mode 100755 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/fmha/compile.sh create mode 100755 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/fmha/compile_run_all_config.sh create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/fmha/fmha_forward.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/fmha/online_softmax.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-load-store/LICENSE create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-load-store/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-load-store/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-load-store/ld_matrix_x1.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-load-store/ld_matrix_x2.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-load-store/ld_matrix_x4.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-load-store/st_matrix_x1.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-load-store/st_matrix_x2.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-load-store/st_matrix_x4.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/.gitignore create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/.gitmodules create mode 100755 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/MATLAB_arrayfun/ArrayfunArticle.m create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/MATLAB_arrayfun/README.md create mode 100755 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/MATLAB_arrayfun/drawAntenna.m create mode 100755 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/MATLAB_arrayfun/drawMap.m create mode 100755 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/MATLAB_arrayfun/drawMultipleSignalMaps.m create mode 100755 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/MATLAB_arrayfun/drawSignalMap.m create mode 100755 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/MATLAB_arrayfun/multipleSignalMaps.m create mode 100755 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/MATLAB_arrayfun/powerCalculationWithGpuArray.m create mode 100755 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/MATLAB_arrayfun/powerKernel.m create mode 100755 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/MATLAB_arrayfun/reportTime.m create mode 100755 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/MATLAB_deeplearning/PetDetectionRecognitionScript.m create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/MATLAB_deeplearning/README.md create mode 100755 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/MATLAB_deeplearning/cnnPredict.m create mode 100755 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/MATLAB_deeplearning/filterRegions.m create mode 100755 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/MATLAB_deeplearning/findPet.m create mode 100755 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/MATLAB_deeplearning/getImageLabels.m create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/c++11_cuda/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/c++11_cuda/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/c++11_cuda/c++11_cuda.cu create mode 100755 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/c++11_cuda/warandpeace.txt create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/license.txt create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/002-openacc-example/common/timer.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/002-openacc-example/step1/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/002-openacc-example/step1/Makefile_f90 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/002-openacc-example/step1/laplace2d.c create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/002-openacc-example/step1/laplace2d.f90 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/002-openacc-example/step2/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/002-openacc-example/step2/Makefile_f90 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/002-openacc-example/step2/laplace2d.c create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/002-openacc-example/step2/laplace2d.f90 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/002-openacc-example/step3/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/002-openacc-example/step3/Makefile_f90 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/002-openacc-example/step3/laplace2d.c create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/002-openacc-example/step3/laplace2d.f90 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/TensorRT-3.0/convert.ipynb create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/TensorRT-3.0/test.ipynb create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/TensorRT-3.0/train.ipynb create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/TensorRT-introduction/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/TensorRT-introduction/cudaWrapper.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/TensorRT-introduction/ioHelper.cpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/TensorRT-introduction/ioHelper.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/TensorRT-introduction/ioHelper.o create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/TensorRT-introduction/simpleOnnx.cpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/TensorRT-introduction/simpleOnnx_1.cpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/TensorRT-introduction/simpleOnnx_2.cpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/american-options/LongstaffSchwartz_vs10.sln create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/american-options/LongstaffSchwartz_vs11.sln create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/american-options/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/american-options/Readme.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/american-options/VisualStudio/LongstaffSchwartzSvd2_vs10.vcxproj create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/american-options/VisualStudio/LongstaffSchwartzSvd2_vs10.vcxproj.filters create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/american-options/VisualStudio/LongstaffSchwartzSvd2_vs10.vcxproj.user create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/american-options/VisualStudio/LongstaffSchwartzSvd2_vs11.vcxproj create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/american-options/VisualStudio/LongstaffSchwartzSvd2_vs11.vcxproj.filters create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/american-options/VisualStudio/LongstaffSchwartzSvd2_vs11.vcxproj.user create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/american-options/longstaff_schwartz_svd_2.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/CUDA__Aware__MPI_8c.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/CUDA__Aware__MPI_8c__incl.map create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/CUDA__Aware__MPI_8c__incl.md5 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/CUDA__Aware__MPI_8c__incl.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/CUDA__Normal__MPI_8c.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/CUDA__Normal__MPI_8c__incl.map create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/CUDA__Normal__MPI_8c__incl.md5 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/CUDA__Normal__MPI_8c__incl.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Device_8cu.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Device_8cu__incl.map create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Device_8cu__incl.md5 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Device_8cu__incl.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Device_8cu_a08324d1c9b1ee7193242670ba4b97d88_cgraph.map create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Device_8cu_a08324d1c9b1ee7193242670ba4b97d88_cgraph.md5 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Device_8cu_a08324d1c9b1ee7193242670ba4b97d88_cgraph.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Device_8cu_a3b641f8b5d521a230416b64d8b809324_cgraph.map create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Device_8cu_a3b641f8b5d521a230416b64d8b809324_cgraph.md5 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Device_8cu_a3b641f8b5d521a230416b64d8b809324_cgraph.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Device_8cu_a95a52e3c2e58d2d94403a0dce2e29686_cgraph.map create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Device_8cu_a95a52e3c2e58d2d94403a0dce2e29686_cgraph.md5 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Device_8cu_a95a52e3c2e58d2d94403a0dce2e29686_cgraph.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Device_8cu_aeaebe085afbffaa0d6bcd77fa16ceb97_cgraph.map create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Device_8cu_aeaebe085afbffaa0d6bcd77fa16ceb97_cgraph.md5 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Device_8cu_aeaebe085afbffaa0d6bcd77fa16ceb97_cgraph.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Host_8c.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Host_8c__incl.map create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Host_8c__incl.md5 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Host_8c__incl.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Host_8c_a0ea9f99f85b380e912940f351366a862_cgraph.map create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Host_8c_a0ea9f99f85b380e912940f351366a862_cgraph.md5 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Host_8c_a0ea9f99f85b380e912940f351366a862_cgraph.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Host_8c_a52ea6f92f186de790a55fdf700026dd9_cgraph.map create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Host_8c_a52ea6f92f186de790a55fdf700026dd9_cgraph.md5 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Host_8c_a52ea6f92f186de790a55fdf700026dd9_cgraph.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Host_8c_ac28696e075d2053b4de5662556c16f3d_cgraph.map create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Host_8c_ac28696e075d2053b4de5662556c16f3d_cgraph.md5 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Host_8c_ac28696e075d2053b4de5662556c16f3d_cgraph.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Host_8c_aef3118f365ec96030a022fa17b73fc42_cgraph.map create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Host_8c_aef3118f365ec96030a022fa17b73fc42_cgraph.md5 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Host_8c_aef3118f365ec96030a022fa17b73fc42_cgraph.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Input_8c.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Input_8c__incl.map create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Input_8c__incl.md5 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Input_8c__incl.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8c.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8c__incl.map create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8c__incl.md5 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8c__incl.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8c_a3c04138a5bfe5d72780bb7e82a18e627_cgraph.map create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8c_a3c04138a5bfe5d72780bb7e82a18e627_cgraph.md5 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8c_a3c04138a5bfe5d72780bb7e82a18e627_cgraph.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h__dep__incl.map create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h__dep__incl.md5 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h__dep__incl.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h__incl.map create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h__incl.md5 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h__incl.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h_a08324d1c9b1ee7193242670ba4b97d88_cgraph.map create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h_a08324d1c9b1ee7193242670ba4b97d88_cgraph.md5 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h_a08324d1c9b1ee7193242670ba4b97d88_cgraph.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h_a0ea9f99f85b380e912940f351366a862_cgraph.map create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h_a0ea9f99f85b380e912940f351366a862_cgraph.md5 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h_a0ea9f99f85b380e912940f351366a862_cgraph.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h_a3b641f8b5d521a230416b64d8b809324_cgraph.map create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h_a3b641f8b5d521a230416b64d8b809324_cgraph.md5 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h_a3b641f8b5d521a230416b64d8b809324_cgraph.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h_a52ea6f92f186de790a55fdf700026dd9_cgraph.map create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h_a52ea6f92f186de790a55fdf700026dd9_cgraph.md5 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h_a52ea6f92f186de790a55fdf700026dd9_cgraph.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h_a95a52e3c2e58d2d94403a0dce2e29686_cgraph.map create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h_a95a52e3c2e58d2d94403a0dce2e29686_cgraph.md5 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h_a95a52e3c2e58d2d94403a0dce2e29686_cgraph.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h_aeaebe085afbffaa0d6bcd77fa16ceb97_cgraph.map create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h_aeaebe085afbffaa0d6bcd77fa16ceb97_cgraph.md5 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h_aeaebe085afbffaa0d6bcd77fa16ceb97_cgraph.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h_aef3118f365ec96030a022fa17b73fc42_cgraph.map create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h_aef3118f365ec96030a022fa17b73fc42_cgraph.md5 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/Jacobi_8h_aef3118f365ec96030a022fa17b73fc42_cgraph.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/bc_s.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/closed.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/doxygen.css create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/doxygen.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/files.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/globals.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/globals_defs.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/globals_func.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/graph_legend.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/graph_legend.md5 create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/graph_legend.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/index.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/installdox create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/jquery.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/nav_f.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/nav_h.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/open.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_61.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_61.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_62.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_62.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_63.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_63.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_64.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_64.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_65.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_65.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_66.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_66.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_68.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_68.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_69.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_69.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_6a.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_6a.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_6d.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_6d.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_70.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_70.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_72.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_72.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_73.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_73.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_74.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_74.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_75.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/all_75.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/close.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/defines_64.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/defines_64.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/defines_65.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/defines_65.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/defines_6a.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/defines_6a.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/defines_6d.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/defines_6d.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/defines_73.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/defines_73.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/defines_75.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/defines_75.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/files_63.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/files_63.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/files_64.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/files_64.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/files_68.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/files_68.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/files_69.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/files_69.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/files_6a.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/files_6a.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_61.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_61.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_62.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_62.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_63.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_63.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_65.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_65.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_66.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_66.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_68.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_68.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_69.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_69.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_6a.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_6a.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_6d.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_6d.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_70.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_70.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_72.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_72.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_73.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_73.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_74.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/functions_74.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/mag_sel.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/nomatches.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/search.css create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/search.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/search_l.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/search_m.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/search/search_r.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/tab_a.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/tab_b.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/tab_h.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/tab_s.png create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/doc/html/tabs.css create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/src/CUDA_Aware_MPI.c create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/src/CUDA_Normal_MPI.c create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/src/Device.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/src/Host.c create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/src/Input.c create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/src/Jacobi.c create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/src/Jacobi.doxygen create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/src/Jacobi.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/src/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-aware-mpi-example/src/README create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-vmm/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-vmm/cuvector.cpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-vmm/cuvector.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-vmm/sync_main.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cuda-vmm/vector_main.cpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cufft-callbacks/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cufft-callbacks/common.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cufft-callbacks/cufft_callbacks.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/cufft-callbacks/cufft_no_callbacks.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/egl_OpenGl_without_Xserver/EGL/egl.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/egl_OpenGl_without_Xserver/EGL/eglext.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/egl_OpenGl_without_Xserver/EGL/eglplatform.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/egl_OpenGl_without_Xserver/KHR/khrplatform.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/egl_OpenGl_without_Xserver/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/egl_OpenGl_without_Xserver/Readme.txt create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/egl_OpenGl_without_Xserver/tinyegl.cc create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/grcuda/R/cuml_dbscan.R create mode 100755 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/grcuda/R/run_r.sh create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/grcuda/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/grcuda/kernel/saxpy.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/grcuda/kernel/saxpy.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/grcuda/kernel/saxpy_nvrtc.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/grcuda/mandelbrot/app.js create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/grcuda/mandelbrot/license.txt create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/grcuda/mandelbrot/package.json create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/gups/LICENSE create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/gups/LICENSE.gups.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/gups/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/gups/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/gups/gups.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/gups/run.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/math-libraries-intro/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/math-libraries-intro/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/math-libraries-intro/cublas-example.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/math-libraries-intro/openblas-example.cpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/mixed-precision/.gitignore create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/mixed-precision/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/mixed-precision/fp16_conversion.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/mixed-precision/haxpy.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/nvml/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/nvml/matrixMul.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/nvml/nvml_test.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/nvtx/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/nvtx/compiler_inst_nvtx.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/nvtx/env_hpc_sdk.sh create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/nvtx/inst_nvtx.cpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/nvtx/manual_nvtx.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/openacc-unified-memory-jacobi/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/openacc-unified-memory-jacobi/common/timer.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/openacc-unified-memory-jacobi/laplace2d.c create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/parallel_reduction_with_shfl/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/parallel_reduction_with_shfl/block_reduce.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/parallel_reduction_with_shfl/device_reduce_atomic.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/parallel_reduction_with_shfl/device_reduce_block_atomic.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/parallel_reduction_with_shfl/device_reduce_stable.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/parallel_reduction_with_shfl/device_reduce_warp_atomic.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/parallel_reduction_with_shfl/fake_atomic.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/parallel_reduction_with_shfl/fake_shfl.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/parallel_reduction_with_shfl/getdata.sh create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/parallel_reduction_with_shfl/main.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/parallel_reduction_with_shfl/readme.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/parallel_reduction_with_shfl/warp_reduce.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/printf-alternative/CASerror.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/printf-alternative/CMakeLists.txt create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/printf-alternative/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/printf-alternative/main.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/rnn/LSTM.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/rnn/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/rnn/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/separate-compilation-linking/main.cpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/separate-compilation-linking/makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/separate-compilation-linking/particle.cpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/separate-compilation-linking/particle.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/separate-compilation-linking/v3.cpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/separate-compilation-linking/v3.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/tensor-cores/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/tensor-cores/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/tensor-cores/simpleTensorCoreGEMM.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/unified-memory-oversubscription/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/unified-memory-oversubscription/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/unified-memory-oversubscription/uvm_oversubs.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/unified-memory/.gitignore create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/unified-memory/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/unified-memory/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/unified-memory/dataElem.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/unified-memory/dataElem_um.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/unified-memory/dataElem_um_c++_1.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/posts/unified-memory/dataElem_um_c++_2.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/series/cuda-cpp/coalescing-global/coalescing.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/series/cuda-cpp/finite-difference/finite-difference.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/series/cuda-cpp/optimize-data-transfers/bandwidthtest.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/series/cuda-cpp/optimize-data-transfers/profile.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/series/cuda-cpp/overlap-data-transfers/async.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/series/cuda-cpp/shared-memory/shared-memory.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/series/cuda-cpp/transpose/transpose.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/series/cuda-fortran/async.cuf create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/series/cuda-fortran/finiteDifference.cuf create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/series/cuda-fortran/shared.cuf create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples/series/cuda-fortran/transpose.cuf create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-transpose/LICENSE create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-transpose/Makefile create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-transpose/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-transpose/swizzle.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-transpose/transpose_naive.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-transpose/transpose_swizzle.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-transpose/transpose_swizzle_batched.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-transpose/transpose_swizzle_batched_for_profile.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-ext-transpose/utils.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-fa4-current/flash_attn/cute/utils.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/.flake8 create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/AUTHORS create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/LICENSE create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/MANIFEST.in create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/__init__.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/ampere_helpers.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/barrier.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/bench_utils.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/benchmark.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/benchmark_flash_attention_fp8.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/blackwell_helpers.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/block_info.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/block_sparse_utils.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/block_sparsity.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/cache_utils.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/compute_block_sparsity.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/copy_utils.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/cu_blocks_kernel.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/cute_dsl_ptxas.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/cute_dsl_utils.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/fa_logging.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/fast_math.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_bwd.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_bwd_mla_dk_sm100.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_bwd_mla_dq_dqv_sm100.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_bwd_mla_sm100.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_bwd_postprocess.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_bwd_preprocess.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_bwd_sm100.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_bwd_sm120.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_bwd_sm90.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_fwd.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_fwd_combine.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_fwd_mla_sm100.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_fwd_sm100.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_fwd_sm120.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_fwd_sm90.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/interface.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/mask.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/mma_sm100_desc.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/named_barrier.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/pack_gqa.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/paged_kv.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/pipeline.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/prepare_scheduler.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/pyproject.toml create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/seqlen_info.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/sm100_hd256_2cta_fmha_backward.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/sm100_hd256_2cta_fmha_backward_dkdvkernel.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/sm100_hd256_2cta_fmha_backward_dqkernel.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/sm100_hd256_2cta_fmha_forward.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/sm90_config_search.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/softmax.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/testing.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/tile_scheduler.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/topk_gather_kv.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/utils.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashinfer-bench-starter-kit/EVALUATION.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashinfer-pinned/benchmarks/bench_trtllm_gen_fused_moe_autotuner.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashinfer-pinned/flashinfer/fused_moe/core.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashinfer-pinned/flashinfer/fused_moe/runners.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashinfer-pinned/flashinfer/gdn_prefill.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/gdn.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/decode/head128/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/decode/head64/config.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/decode/head64/instantiations/model1.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/decode/head64/instantiations/v32.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/decode/head64/kernel.cuh create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/decode/head64/kernel.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/helpers.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/collective/fmha_common.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/collective/fmha_fusion.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/collective/sm100_fmha_fwd_epilogue_tma_warpspecialized.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/collective/sm100_fmha_fwd_mainloop_tma_warpspecialized.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/collective/sm100_fmha_load_tma_warpspecialized.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/collective/sm100_fmha_mla_fwd_mainloop_tma_warpspecialized.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/collective/sm100_fmha_mla_load_tma_warpspecialized.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/common/gather_tensor.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/common/helper.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/common/mask.cuh create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/common/pipeline_mla.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/common/pow_2.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/common/utils.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/device/fmha.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/device/fmha_device_bwd.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/fmha_cutlass_bwd_sm100.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/fmha_cutlass_bwd_sm100.cuh create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/fmha_cutlass_fwd_sm100.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/fmha_cutlass_fwd_sm100.cuh create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/interface.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/kernel/fmha_causal_tile_scheduler.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/kernel/fmha_kernel_bwd_convert.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/kernel/fmha_kernel_bwd_sum_OdO.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/kernel/fmha_options.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/kernel/fmha_tile_scheduler.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/kernel/sm100_fmha_bwd_kernel_tma_warpspecialized.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/kernel/sm100_fmha_bwd_mla_kernel_tma_warpspecialized.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/dense/kernel/sm100_fmha_fwd_kernel_tma_warpspecialized.hpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/sparse/common_subroutine.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/sparse/fwd/head128/config.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/sparse/fwd/head128/instantiations/phase1_k512.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/sparse/fwd/head128/instantiations/phase1_k576.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/sparse/fwd/head128/phase1.cuh create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/sparse/fwd/head128/phase1.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/sparse/fwd/head64/config.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/sparse/fwd/head64/instantiations/phase1_k512.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/sparse/fwd/head64/instantiations/phase1_k576.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/sparse/fwd/head64/phase1.cuh create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/sparse/fwd/head64/phase1.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/sparse/fwd_for_small_topk/head128/config.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/sparse/fwd_for_small_topk/head128/instantiations/phase1_decode_k512.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/sparse/fwd_for_small_topk/head128/instantiations/phase1_prefill_k512.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/sparse/fwd_for_small_topk/head128/phase1.cuh create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100/prefill/sparse/fwd_for_small_topk/head128/phase1.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm90/decode/dense/config.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm90/decode/dense/instantiations/bf16.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm90/decode/dense/instantiations/fp16.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm90/decode/dense/splitkv_mla.cuh create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm90/decode/dense/splitkv_mla.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm90/decode/dense/traits.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm90/decode/sparse_fp8/components/config.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm90/decode/sparse_fp8/components/dequant.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm90/decode/sparse_fp8/components/helpers.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm90/decode/sparse_fp8/config.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm90/decode/sparse_fp8/instantiations/model1_persistent_h128.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm90/decode/sparse_fp8/instantiations/model1_persistent_h64.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm90/decode/sparse_fp8/instantiations/v32_persistent_h128.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm90/decode/sparse_fp8/instantiations/v32_persistent_h64.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm90/decode/sparse_fp8/splitkv_mla.cuh create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm90/decode/sparse_fp8/splitkv_mla.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/tests/quant.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/tests/ref.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/tests/test_flash_mla_sparse_decoding.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-gdn-pinned/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-gdn-pinned/lit_gpt/gated_delta_net.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-gdn-pinned/lit_gpt/gated_delta_rule_ops/chunk.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/.clang-format create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/.gitignore create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/01_vector_addition/add.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/01_vector_addition/main.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02a_matmul_simt/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02a_matmul_simt/main.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02a_matmul_simt/main.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02a_matmul_simt/matmul.cpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02a_matmul_simt/matmul.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02a_matmul_simt/matmul_triton.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02b_matmul_sm80/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02b_matmul_sm80/common.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02b_matmul_sm80/cutedsl_v1.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02b_matmul_sm80/main.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02b_matmul_sm80/matmul.cpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02b_matmul_sm80/matmul_v1.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02b_matmul_sm80/matmul_v5.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02b_matmul_sm80/matmul_v6.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02c_matmul_sm120/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02c_matmul_sm120/common.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02c_matmul_sm120/main.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02c_matmul_sm120/matmul.cpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02c_matmul_sm120/matmul_v0.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02c_matmul_sm120/matmul_v1.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02d_matmul_cdna3/.gitignore create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02d_matmul_cdna3/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02d_matmul_cdna3/common.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02d_matmul_cdna3/main.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02d_matmul_cdna3/matmul.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02d_matmul_cdna3/matmul_v1.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02d_matmul_cdna3/matmul_v2.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02d_matmul_cdna3/matmul_v3.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/common.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/main.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul.cpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v0.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v1.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v2.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v3.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v4.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v5.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v6.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/profiler.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/trace_v5.json.gz create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/trace_v6.json.gz create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/03_sum/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/03_sum/main.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/03_sum/main.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/03_sum/sum.cpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/03_sum/sum.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/04_softmax/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/04_softmax/main.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/04_softmax/softmax.cpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/04_softmax/softmax.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/05_fp6/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/05_fp6/fp6.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/06_box_blur/box_blur.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/06_box_blur/main.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/07_attention/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/07_attention/attention.cpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/07_attention/attention_v1.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/07_attention/attention_v2.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/07_attention/attention_v3.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/07_attention/attention_v4.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/07_attention/attention_v5.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/07_attention/common.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/07_attention/main.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/08_row_scaled_mm/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/08_row_scaled_mm/common.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/08_row_scaled_mm/ext.cpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/08_row_scaled_mm/main.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/08_row_scaled_mm/row_scaled_mm_v1.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/09_block_scaled_mm/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/09_block_scaled_mm/common.h create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/09_block_scaled_mm/ext.cpp create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/09_block_scaled_mm/main.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/09_block_scaled_mm/mxfp8_mm_v1.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/09_block_scaled_mm/mxfp8_mm_v2.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/10_p2p/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/10_p2p/main.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/10_p2p/p2p.cu create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/11_gemv/main.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/11_gemv/triton_v1.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/pyproject.toml create mode 100644 verification/evidence/local-snapshots/kernelwiki-mlstm-kernels/README.md create mode 100644 verification/evidence/local-snapshots/kernelwiki-ptx-1302.html create mode 100644 verification/evidence/local-snapshots/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/reference.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml create mode 100644 verification/evidence/local-snapshots/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/template.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/reference.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/task.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/task.yml create mode 100644 verification/evidence/local-snapshots/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/template.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/reference.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/task.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/task.yml create mode 100644 verification/evidence/local-snapshots/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/template.py create mode 100644 verification/evidence/local-snapshots/kernelwiki-reference-kernels/problems/nvidia/nvfp4_group_gemm/task.yml create mode 100644 verification/tools/audit_verification_evidence.py create mode 100644 verification/tools/check_verification_evidence.py create mode 100644 verification/tools/migrate_verification_evidence.py create mode 100644 verification/tools/normalize_verification_records.py diff --git a/sources/blogs/colfax-cutlass-blackwell.md b/sources/blogs/colfax-cutlass-blackwell.md index 6d09d653f..eb0fb6ae1 100644 --- a/sources/blogs/colfax-cutlass-blackwell.md +++ b/sources/blogs/colfax-cutlass-blackwell.md @@ -44,5 +44,5 @@ substitute for the ISA grammar: choose the atom from the version-pinned library rather than reconstructing a template signature from prose. -For executable references, use the [PTX ISA 9.0 tcgen05 sections](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma) +For executable references, use the [PTX ISA 9.0 tcgen05 sections](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma) and the [CUTLASS 4.5.0 SM100 examples](https://github.com/NVIDIA/cutlass/tree/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/70_blackwell_gemm). diff --git a/sources/blogs/tcgen05-tutorial.md b/sources/blogs/tcgen05-tutorial.md index 1952114c4..96491bd0c 100644 --- a/sources/blogs/tcgen05-tutorial.md +++ b/sources/blogs/tcgen05-tutorial.md @@ -54,4 +54,4 @@ These values are reports for the author's exact environment; they are not portab - [Article](https://gau-nernst.github.io/tcgen05/) - [Pinned source tree](https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100) -- [PTX ISA 9.0 tcgen05 reference](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma) +- [PTX ISA 9.0 tcgen05 reference](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma) diff --git a/sources/docs/cutlass-clc-documentation.md b/sources/docs/cutlass-clc-documentation.md index 2c366b319..9688e2724 100644 --- a/sources/docs/cutlass-clc-documentation.md +++ b/sources/docs/cutlass-clc-documentation.md @@ -30,4 +30,4 @@ Swizzle size and raster order are CUTLASS software coordinate-transform policy a - [Pinned CLC documentation](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md) - [Pinned SM100 scheduler](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp) -- [PTX ISA 9.0 CLC instructions](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel) +- [PTX ISA 9.0 CLC instructions](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel) diff --git a/sources/docs/nvidia-ptx-isa-sm100.md b/sources/docs/nvidia-ptx-isa-sm100.md index 7bcecc493..be51264bf 100644 --- a/sources/docs/nvidia-ptx-isa-sm100.md +++ b/sources/docs/nvidia-ptx-isa-sm100.md @@ -32,7 +32,7 @@ The ISA defines legal behavior, not a universal performance ranking between inst ## Direct Links - [PTX ISA 9.0 contents](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/contents.html) -- [`tcgen05.mma`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma) +- [`tcgen05.mma`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma) - [Tensor Memory](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory) - [`clusterlaunchcontrol.try_cancel`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel) - [`cp.async.bulk.tensor`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor) diff --git a/verification/before-after-report.jsonl b/verification/before-after-report.jsonl index 71ae08449..c52de8b73 100644 --- a/verification/before-after-report.jsonl +++ b/verification/before-after-report.jsonl @@ -20,73 +20,73 @@ {"path": "wiki/languages/triton-blackwell.md", "before": {"statement": "```python\n@triton.jit\ndef sparse_attention_fwd(Q, K, V, Output, TopK_Indices,\n block_size: tl.constexpr, topk: tl.constexpr):\n \"\"\"Attend to top-k sparse token blocks only.\"\"\"\n qid = tl.program_id(0)\n q = tl.load(Q + qid * d + tl.arange(0, d))\n acc = tl.zeros([d], dtype=tl.float32)\n for i in range(topk):\n bidx = tl.load(TopK_Indices + qid * topk + i)\n k_block = tl.load(K + bidx * block_size * d + offsets)\n v_block = tl.load(V + bidx * block_size * d + offsets)\n scores = tl.dot(q[None, :], tl.trans(k_block))\n # softmax + accumulate...\n```"}, "after": {"statement": null}, "reason": {"statement": "Replace the incomplete executable-looking block with links to complete provenance-pinned attention kernels.", "urls": []}} {"path": "wiki/languages/triton-blackwell.md", "before": {"statement": "The full 42-PR universe is enumerated in `data/triton-universe.yaml`. Entries marked `captured: false` do not ship locally because they fall outside the three in-policy sub-scopes (see the policy file for reasons)."}, "after": {"statement": "The complete tracked PR universe and its captured/skipped flags are recorded in [`data/triton-universe.yaml`](../../data/triton-universe.yaml); this page does not duplicate that changing count."}, "reason": {"statement": "Remove the stale count and retain a stable pointer to the canonical ledger and capture flags.", "urls": []}} {"path": "wiki/languages/triton-blackwell.md", "before": {"statement": "- **What is verified**: the Triton 3.6 release adds tcgen05 + TMEM lowering infrastructure (per `doc-triton-3.6-blackwell`), and tracked downstream repos are landing real Triton kernel changes for SM100 post-3.6 (per `pr-vllm-34597`, the post-refresh primary anchor — vLLM's \"[Kernel] Add FP8 KV cache support to Triton MLA decode attention\" PR, which directly modifies `@triton.jit`-decorated kernels in `vllm/v1/attention/ops/triton_decode_attention.py` doing `tl.dot(q, k)` / `tl.dot(qpe, kpe)` / `tl.dot(p, v)` matmul on `[sm100]`, shipped verbatim under `artifacts/prs/vllm/PR-34597/`). Supplementary post-refresh evidence: `pr-vllm-29339` shows the Triton 3.6 `triton_kernels` library being explicitly scoped to Blackwell production codepaths. Pre-refresh historical evidence: `pr-sglang-22079` (the Gemma4 NVFP4 attention kernel merged 2026-04-03). Together these establish that Triton 3.6+ on Blackwell is materially different from the pre-3.6 \"lacked-tcgen05 / lacked-TMEM\" story.\n\n- **`pr-vllm-34597`** (`source_category: upstream-code`) — **post-refresh primary anchor**: vLLM's \"[Kernel] Add FP8 KV cache support to Triton MLA decode attention\" PR (merged 2026-02-16, post-Triton-3.6.0 release; `architectures: [sm100]`; tags `attention`, `decode`, `fp8`, `mla`). Directly modifies actual Triton kernel files: `vllm/v1/attention/ops/triton_decode_attention.py` (the `@triton.jit`-decorated MLA decode kernel doing `tl.dot(q, k)` for attention scores, `tl.dot(qpe, kpe)` for positional contributions, and `acc += tl.dot(p, v)` for the output projection) and `vllm/v1/attention/backends/mla/triton_mla.py` (the backend wrapping it). The Triton kernel itself is shipped verbatim under [`artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/ops/triton_decode_attention.py`](../../artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/ops/triton_decode_attention.py) (756 lines, pinned at upstream SHA `a1257fd1`). This PR page is post-refresh — it is NOT in `data/refresh-cutoff.yaml::previous_pages_manifest`. This is the \"downstream adoption is happening\" half of AC-1.2 and the AC-1.1 \"new tracked-repo PR page demonstrating a kernel\" anchor.\n\nSupplementary post-refresh anchor: [`pr-vllm-29339`](../../sources/prs/vllm/PR-29339.md) — vLLM bugfix that scopes the upstream `triton_kernels` library (the `triton-lang/triton/python/triton_kernels` collection shipped with Triton 3.6) to `[sm100, sm90]` for the MXFP4 quantization path. Modifies only a dispatch gate (`vllm/model_executor/layers/quantization/mxfp4.py`), which is why `pr-vllm-34597` is the primary anchor and `pr-vllm-29339` is supplementary.\n\nPre-refresh historical anchors (retained as supplementary context, not as AC-1.1 \"new tracked-repo PR page\" evidence on their own):\n\n- [`pr-sglang-22079`](../../sources/prs/sglang/PR-22079.md) — Gemma4 NVFP4 SGLang `extend_attention` Triton kernel doing actual `tl.dot(q,k)` / `tl.dot(p,v)` matmul on `[sm100, sm90]`, merged 2026-04-03. Strongest in-corpus example of a real `tl.dot` Triton matmul landing for SM100 post-3.6, but pre-refresh per `previous_pages_manifest`.\n- [`pr-sglang-21019`](../../sources/prs/sglang/PR-21019.md) — Qwen3.5 GDN projection fused split/reshape/cat kernel merged 2026-03-20. `tl.load`/`tl.store` only (memory rearrangement, no `tl.dot`); demonstrates \"Triton on SM100 post-3.6\" but not the matmul lowering path.\n\nThe current AC-1.1 **post-refresh primary upstream-code anchor** is `pr-vllm-34597` (above), with the actual Triton decode-attention kernel shipped verbatim. Supplementary post-refresh anchor: `pr-vllm-29339` ([`sources/prs/vllm/PR-29339.md`](../../sources/prs/vllm/PR-29339.md)), a vLLM bugfix scoping the Triton 3.6 `triton_kernels` library to `[sm100, sm90]` for the MXFP4 quantization path (no artifact bundle because the change is dispatch-gate-only)."}, "after": {"statement": null}, "reason": {"statement": "Refresh-round administration is stale and irrelevant to the durable technical claim.", "urls": []}} -{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "`tcgen05.mma` is the Blackwell (SM100/SM100a) matrix-multiply-accumulate instruction that replaces Hopper's `wgmma.mma_async`. The name stands for **Tensor Core Generation 05**. NVIDIA also refers to the higher-level abstraction as **UMMA** (Unified Matrix Multiply-Accumulate) in the CUTLASS framework."}, "after": {"statement": null}, "reason": {"statement": "Retain the official tcgen05 name and CUTLASS namespace without inventing an acronym expansion.", "urls": []}} -{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "1. **No warpgroup synchronization overhead** -- one elected thread (typically lane 0 of warp 0) issues the MMA."}, "after": {"statement": "| Completion | WGMMA commit/wait groups | `tcgen05.commit` plus an mbarrier wait |\n\nSingle-thread issue reduces the number of issuing threads, but it does not remove asynchronous completion, operand-lifetime, or inter-thread ordering requirements.\n\n`tcgen05.mma` is asynchronous. Two mechanisms serve different purposes:\n\n1. `tcgen05.commit.cta_group::N.mbarrier::arrive::one.b64` makes an mbarrier track completion of prior asynchronous tcgen05 operations issued by the executing thread. Waiting on that mbarrier observes completion.\n2. `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` order tcgen05 operations across an execution-ordering handoff and constrain code motion. A fence is not a completion wait.\n\nA cross-thread result handoff therefore needs both the applicable completion protocol and the applicable fence/execution-ordering protocol. Likewise, a pipelined mainloop must not release or overwrite an SMEM stage until the asynchronous MMA has finished consuming it."}, "reason": {"statement": "Separate issue granularity from synchronization and completion requirements.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence"]}} -{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "| Operand A source | Registers or SMEM | Shared memory only |\n\n| Matrix load | ldmatrix to registers | Direct from SMEM (no ldmatrix) |"}, "after": {"statement": "The programming-model shift from Hopper WGMMA is precise:\n\n| Issue granularity | Warpgroup | One thread for `cta_group::1` or `cta_group::2` |\n| D accumulator | Per-thread registers | Tensor Memory (TMEM) |\n| A location | Register or SMEM forms, depending on instruction | SMEM descriptor or TMEM address |\n| B location | SMEM descriptor | SMEM descriptor |"}, "reason": {"statement": "Restore the actual operand-location alternatives and avoid a mandatory ldmatrix claim.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-matrix-multiply-and-accumulate-operations"]}} -{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "| Synchronization | Warpgroup-scoped barriers | Fully async, fence-based |\n\n3. **Fence-based completion** -- the producer must insert explicit fences before reading results from TMEM.\n\nFences are **mandatory** for correctness. The hardware does not implicitly synchronize between MMA and TMEM reads/writes.\n\nInsert before reading MMA results from TMEM:\n\n```cuda\n// Fence before CTA sync and TMEM reads after the completion mechanism.\n__device__ void fence_before_tmem_read() {\n asm volatile(\"tcgen05.fence::before_thread_sync;\");\n __syncthreads(); // Ensure all threads see the fence\n}\n```\n\n```ptx\n// Fence before reading TMEM (most common)\ntcgen05.fence::before_thread_sync;\n\n// Fence after CTA sync before issuing dependent tcgen05 operations\ntcgen05.fence::after_thread_sync;\n```"}, "after": {"statement": "`tcgen05.mma` is asynchronous. Two mechanisms serve different purposes:\n\n1. `tcgen05.commit.cta_group::N.mbarrier::arrive::one.b64` makes an mbarrier track completion of prior asynchronous tcgen05 operations issued by the executing thread. Waiting on that mbarrier observes completion.\n2. `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` order tcgen05 operations across an execution-ordering handoff and constrain code motion. A fence is not a completion wait.\n\nA cross-thread result handoff therefore needs both the applicable completion protocol and the applicable fence/execution-ordering protocol. Likewise, a pipelined mainloop must not release or overwrite an SMEM stage until the asynchronous MMA has finished consuming it."}, "reason": {"statement": "Teach completion and ordering as separate protocols.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"]}} -{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "| `tcgen05.mma.kind::f16` | FP16/BF16 | FP16/BF16 | FP32 | None | m128n256k16 | Standard half-precision |\n\n| `tcgen05.mma.kind::f8f6f4` | FP8/FP6/FP4 | FP8/FP6/FP4 | FP32 | Block (UE8M0) | m128n256k32 | Narrow precision with native block scaling |\n\n| `tcgen05.mma.kind::mxf8` | MXFP8 | MXFP8 | FP32 | MX (E8M0) | m128n256k32 | Microscaling FP8 |\n| `tcgen05.mma.kind::mxf4` | MXFP4 | MXFP4 | FP32 | MX (E8M0) | m128n256k64 | Microscaling FP4 |\n| `tcgen05.mma.kind::mxf4nvf4` | NVFP4 | MXFP4 | FP32 | Mixed | m128n256k64 | Mixed NVFP4/MXFP4 |"}, "after": {"statement": "PTX ISA 9.0 divides dense `tcgen05.mma` into these grammar groups:\n\n| Floating point, without block scaling | `f16`, `tf32`, `f8f6f4` |\n| Floating point, with block scaling | `mxf8f6f4`, `mxf4`, `mxf4nvf4` |\n| Integer | `i8` |\n\nBlock-scaled forms add `.block_scale` and accept separate TMEM addresses for A and B scale factors. `f8f6f4` is not the block-scaled MX kind, and `mxf8` is not a dense kind token in this grammar."}, "reason": {"statement": "Replace the mixed taxonomy with the exact PTX grammar groups and avoid oversimplified type claims.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "`tcgen05.mma` is the Blackwell (SM100/SM100a) matrix-multiply-accumulate instruction that replaces Hopper's `wgmma.mma_async`. The name stands for **Tensor Core Generation 05**. NVIDIA also refers to the higher-level abstraction as **UMMA** (Unified Matrix Multiply-Accumulate) in the CUTLASS framework."}, "after": {"statement": null}, "reason": {"statement": "Retain the official tcgen05 name and CUTLASS namespace without inventing an acronym expansion.", "urls": ["https://github.com/NVIDIA/cutlass/tree/e406c186f510a15091cce01f782020ceb7ba8eb5"]}} +{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "1. **No warpgroup synchronization overhead** -- one elected thread (typically lane 0 of warp 0) issues the MMA."}, "after": {"statement": "| Completion | WGMMA commit/wait groups | `tcgen05.commit` plus an mbarrier wait |\n\nSingle-thread issue reduces the number of issuing threads, but it does not remove asynchronous completion, operand-lifetime, or inter-thread ordering requirements.\n\n`tcgen05.mma` is asynchronous. Two mechanisms serve different purposes:\n\n1. `tcgen05.commit.cta_group::N.mbarrier::arrive::one.b64` makes an mbarrier track completion of prior asynchronous tcgen05 operations issued by the executing thread. Waiting on that mbarrier observes completion.\n2. `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` order tcgen05 operations across an execution-ordering handoff and constrain code motion. A fence is not a completion wait.\n\nA cross-thread result handoff therefore needs both the applicable completion protocol and the applicable fence/execution-ordering protocol. Likewise, a pipelined mainloop must not release or overwrite an SMEM stage until the asynchronous MMA has finished consuming it."}, "reason": {"statement": "Separate issue granularity from synchronization and completion requirements.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-special-sync-operations-fence"]}} +{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "| Operand A source | Registers or SMEM | Shared memory only |\n\n| Matrix load | ldmatrix to registers | Direct from SMEM (no ldmatrix) |"}, "after": {"statement": "The programming-model shift from Hopper WGMMA is precise:\n\n| Issue granularity | Warpgroup | One thread for `cta_group::1` or `cta_group::2` |\n| D accumulator | Per-thread registers | Tensor Memory (TMEM) |\n| A location | Register or SMEM forms, depending on instruction | SMEM descriptor or TMEM address |\n| B location | SMEM descriptor | SMEM descriptor |"}, "reason": {"statement": "Restore the actual operand-location alternatives and avoid a mandatory ldmatrix claim.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma"]}} +{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "| Synchronization | Warpgroup-scoped barriers | Fully async, fence-based |\n\n3. **Fence-based completion** -- the producer must insert explicit fences before reading results from TMEM.\n\nFences are **mandatory** for correctness. The hardware does not implicitly synchronize between MMA and TMEM reads/writes.\n\nInsert before reading MMA results from TMEM:\n\n```cuda\n// Fence before CTA sync and TMEM reads after the completion mechanism.\n__device__ void fence_before_tmem_read() {\n asm volatile(\"tcgen05.fence::before_thread_sync;\");\n __syncthreads(); // Ensure all threads see the fence\n}\n```\n\n```ptx\n// Fence before reading TMEM (most common)\ntcgen05.fence::before_thread_sync;\n\n// Fence after CTA sync before issuing dependent tcgen05 operations\ntcgen05.fence::after_thread_sync;\n```"}, "after": {"statement": "`tcgen05.mma` is asynchronous. Two mechanisms serve different purposes:\n\n1. `tcgen05.commit.cta_group::N.mbarrier::arrive::one.b64` makes an mbarrier track completion of prior asynchronous tcgen05 operations issued by the executing thread. Waiting on that mbarrier observes completion.\n2. `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` order tcgen05 operations across an execution-ordering handoff and constrain code motion. A fence is not a completion wait.\n\nA cross-thread result handoff therefore needs both the applicable completion protocol and the applicable fence/execution-ordering protocol. Likewise, a pipelined mainloop must not release or overwrite an SMEM stage until the asynchronous MMA has finished consuming it."}, "reason": {"statement": "Teach completion and ordering as separate protocols.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-special-sync-operations-fence", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit"]}} +{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "| `tcgen05.mma.kind::f16` | FP16/BF16 | FP16/BF16 | FP32 | None | m128n256k16 | Standard half-precision |\n\n| `tcgen05.mma.kind::f8f6f4` | FP8/FP6/FP4 | FP8/FP6/FP4 | FP32 | Block (UE8M0) | m128n256k32 | Narrow precision with native block scaling |\n\n| `tcgen05.mma.kind::mxf8` | MXFP8 | MXFP8 | FP32 | MX (E8M0) | m128n256k32 | Microscaling FP8 |\n| `tcgen05.mma.kind::mxf4` | MXFP4 | MXFP4 | FP32 | MX (E8M0) | m128n256k64 | Microscaling FP4 |\n| `tcgen05.mma.kind::mxf4nvf4` | NVFP4 | MXFP4 | FP32 | Mixed | m128n256k64 | Mixed NVFP4/MXFP4 |"}, "after": {"statement": "PTX ISA 9.0 divides dense `tcgen05.mma` into these grammar groups:\n\n| Floating point, without block scaling | `f16`, `tf32`, `f8f6f4` |\n| Floating point, with block scaling | `mxf8f6f4`, `mxf4`, `mxf4nvf4` |\n| Integer | `i8` |\n\nBlock-scaled forms add `.block_scale` and accept separate TMEM addresses for A and B scale factors. `f8f6f4` is not the block-scaled MX kind, and `mxf8` is not a dense kind token in this grammar."}, "reason": {"statement": "Replace the mixed taxonomy with the exact PTX grammar groups and avoid oversimplified type claims.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma"]}} {"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "- **BF16/FP16**: M=128, N=256, K=16\n- **TF32**: M=128, N=256, K=8\n- **FP8/FP6/FP4**: M=128, N=256, K=32\n- **MXFP4/NVFP4**: M=128, N=256, K=64\n\n- **BF16/FP16**: M=256, N=256, K=16\n- **TF32**: M=256, N=256, K=8\n- **FP8/FP6/FP4**: M=256, N=256, K=32\n- **MXFP4/NVFP4**: M=256, N=256, K=64\n\nThe M dimension doubles because each SM contributes 128 rows from its own TMEM partition."}, "after": {"statement": "M and N are encoded in the instruction descriptor (`idesc`) and are constrained by kind, CTA group, layouts, and target ISA. Consequently, names such as m128n256k16 and m256n256k16 are useful maximum-shape examples for F16/BF16 configurations, not the only legal M and N values. `cta_group::1` uses current-CTA resources; `cta_group::2` can also access the paired peer CTA's SMEM and TMEM resources. All `tcgen05` instructions in a kernel must use the same CTA-group value."}, "reason": {"statement": "State group ownership separately from descriptor-selected shapes and label maxima as maxima.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instruction-descriptor"]}} -{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "```cuda\n// Single-thread MMA issuance pattern\n__device__ void issue_mma(uint32_t tmem_addr, uint64_t smem_desc_a, uint64_t smem_desc_b) {\n // Only one thread issues the MMA\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 0;\"\n :\n : \"r\"(tmem_addr),\n \"l\"(smem_desc_a),\n \"l\"(smem_desc_b),\n \"r\"(0) // scale descriptor (unused for f16)\n );\n }\n}\n```\n\n```ptx\n// Issue a 128x256x16 BF16 MMA\n// Operand A: shared memory descriptor\n// Operand B: shared memory descriptor\n// Accumulator: TMEM address\ntcgen05.mma.cta_group::1.kind::f16 [tmem_addr], desc_a, desc_b, idesc, 0;\n```\n\n```ptx\n// Issue a 256x256x16 BF16 MMA across two paired CTAs\n// cta_group::2 indicates cooperative mode\ntcgen05.mma.cta_group::2.kind::f16 [tmem_addr], desc_a, desc_b, idesc, 0;\n```\n\n```ptx\n// FP8 with native UE8M0 block scaling\n// scale_desc encodes the per-block scale factors\ntcgen05.mma.cta_group::1.kind::f8f6f4 [tmem_addr], desc_a, desc_b, idesc, scale_desc;\n```\n\n```cuda\n__device__ void mma_bf16_128x256x16(\n uint32_t tmem_addr,\n uint64_t desc_a,\n uint64_t desc_b\n) {\n if (threadIdx.x == 0) {\n // First MMA: zero-initialize accumulator\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 0;\"\n :\n : \"r\"(tmem_addr), \"l\"(desc_a), \"l\"(desc_b), \"r\"(0)\n );\n }\n}\n\n__device__ void mma_bf16_accumulate(\n uint32_t tmem_addr,\n uint64_t desc_a,\n uint64_t desc_b\n) {\n if (threadIdx.x == 0) {\n // Subsequent MMAs: accumulate into existing TMEM\n // The enable_accumulate flag controls whether to add to or overwrite TMEM\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 1;\" // last arg 1 = accumulate\n :\n : \"r\"(tmem_addr), \"l\"(desc_a), \"l\"(desc_b), \"r\"(0)\n );\n }\n}\n```"}, "after": {"statement": null}, "reason": {"statement": "Remove invalid executable-looking examples and retain the normative grammar with a vendor example link.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} -{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "```cuda\n__device__ void gemm_mainloop(/* params */) {\n for (int k_tile = 0; k_tile < num_k_tiles; ++k_tile) {\n // 1. Wait for operand data to arrive in SMEM\n wait_barrier(k_tile % NUM_STAGES);\n\n // 2. Issue MMA (single thread)\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 1;\"\n :\n : \"r\"(tmem_addr),\n \"l\"(make_smem_desc(smem_a, k_tile)),\n \"l\"(make_smem_desc(smem_b, k_tile)),\n \"r\"(0)\n );\n }\n\n // 3. Release SMEM buffer for next TMA load\n if (threadIdx.x == 0) {\n arrive_barrier((k_tile + 1) % NUM_STAGES);\n }\n }\n\n // 4. CRITICAL: Fence before reading accumulator from TMEM\n asm volatile(\"tcgen05.fence::before_thread_sync;\");\n __syncthreads();\n\n // 5. Now safe to read results from TMEM\n read_tmem_accumulator(tmem_addr, output);\n}\n```"}, "after": {"statement": null}, "reason": {"statement": "A partial pipeline with unsafe lifetime rules is more misleading than the normative sequence.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"]}} -{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "```cuda\n// ---- HOPPER (SM90): wgmma ----\n// Requires warpgroup-scoped execution\n// All 128 threads in a warpgroup participate\n__device__ void hopper_mma() {\n // Load A matrix into registers via ldmatrix\n uint32_t a_frag[4];\n asm volatile(\"ldmatrix.sync.aligned.m8n8.x4.shared.b16 \"\n \"{%0,%1,%2,%3}, [%4];\"\n : \"=r\"(a_frag[0]), \"=r\"(a_frag[1]),\n \"=r\"(a_frag[2]), \"=r\"(a_frag[3])\n : \"r\"(smem_addr));\n\n // Issue wgmma -- warpgroup scope, register accumulator\n asm volatile(\"wgmma.mma_async.sync.aligned.m64n256k16.f32.bf16.bf16 \"\n \"{%0, %1, ...}, \" // register accumulators (128+ registers!)\n \"{%N, %N+1, ...}, \" // A operand in registers\n \"desc_b, ...;\"\n : \"+f\"(acc[0]), \"+f\"(acc[1]), ...\n : ...);\n\n // Commit and wait\n asm volatile(\"wgmma.commit_group.sync.aligned;\");\n asm volatile(\"wgmma.wait_group.sync.aligned 0;\");\n // Accumulators now in registers -- high register pressure\n}\n\n// ---- BLACKWELL (SM100): tcgen05 ----\n// Single-thread issuance, TMEM accumulator\n__device__ void blackwell_mma() {\n // No ldmatrix needed -- reads directly from SMEM\n // No register allocation for accumulators\n\n // Single thread issues MMA\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 1;\"\n :\n : \"r\"(tmem_addr), // accumulator in TMEM (not registers!)\n \"l\"(desc_a), // A from SMEM descriptor\n \"l\"(desc_b), // B from SMEM descriptor\n \"r\"(0)\n );\n }\n\n // Fence + sync before reading results\n asm volatile(\"tcgen05.fence::before_thread_sync;\");\n __syncthreads();\n // Read from TMEM -- registers free for other work\n}\n```"}, "after": {"statement": null}, "reason": {"statement": "Replace executable-looking pseudo-assembly with a precise semantic comparison table.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} -{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "tcgen05.mma requires **128-byte swizzled** shared memory layouts for both operands. Non-swizzled or 64-byte swizzled layouts will produce incorrect results silently."}, "after": {"statement": "The tcgen05 shared-memory descriptor is a 64-bit runtime value. PTX ISA 9.0 assigns fields for the encoded base address, leading dimension, stride dimension, fixed bits, base offset, leading-dimension mode, and a three-bit swizzle mode.\n\nValid swizzle encodings include no swizzle, 128-byte, 64-byte, and 32-byte layouts (plus a 128-byte/32-byte-atomic mode). Values 3, 5, and 7 are invalid; ordinary 128-byte swizzling uses value 2. A layout must satisfy the addressing and alignment constraints for its chosen mode. Although 128-byte swizzling can materially improve a particular GEMM, it is not a universal correctness requirement."}, "reason": {"statement": "Distinguish valid layouts from performance-sensitive layout choice.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor"]}} -{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "```cuda\n// Shared memory descriptor construction for tcgen05\n// The descriptor encodes: base address, stride, swizzle mode, dimensions\n__device__ uint64_t make_smem_desc(void* smem_ptr, int stride_bytes) {\n uint64_t desc = 0;\n uint32_t addr = static_cast(__cvta_generic_to_shared(smem_ptr));\n\n // Encode base address (bits 0-13)\n desc |= (uint64_t)(addr >> 4);\n // Encode leading dimension stride (bits 16-29)\n desc |= (uint64_t)((stride_bytes >> 4) & 0x3FFF) << 16;\n // Encode 128-byte swizzle mode (bits 62-63) -- MANDATORY for tcgen05\n desc |= (uint64_t)(3) << 62; // 3 = 128-byte swizzle\n\n return desc;\n}\n```"}, "after": {"statement": null}, "reason": {"statement": "Direct users to the complete table rather than preserving a subtly corrupt descriptor builder.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor"]}} +{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "```cuda\n// Single-thread MMA issuance pattern\n__device__ void issue_mma(uint32_t tmem_addr, uint64_t smem_desc_a, uint64_t smem_desc_b) {\n // Only one thread issues the MMA\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 0;\"\n :\n : \"r\"(tmem_addr),\n \"l\"(smem_desc_a),\n \"l\"(smem_desc_b),\n \"r\"(0) // scale descriptor (unused for f16)\n );\n }\n}\n```\n\n```ptx\n// Issue a 128x256x16 BF16 MMA\n// Operand A: shared memory descriptor\n// Operand B: shared memory descriptor\n// Accumulator: TMEM address\ntcgen05.mma.cta_group::1.kind::f16 [tmem_addr], desc_a, desc_b, idesc, 0;\n```\n\n```ptx\n// Issue a 256x256x16 BF16 MMA across two paired CTAs\n// cta_group::2 indicates cooperative mode\ntcgen05.mma.cta_group::2.kind::f16 [tmem_addr], desc_a, desc_b, idesc, 0;\n```\n\n```ptx\n// FP8 with native UE8M0 block scaling\n// scale_desc encodes the per-block scale factors\ntcgen05.mma.cta_group::1.kind::f8f6f4 [tmem_addr], desc_a, desc_b, idesc, scale_desc;\n```\n\n```cuda\n__device__ void mma_bf16_128x256x16(\n uint32_t tmem_addr,\n uint64_t desc_a,\n uint64_t desc_b\n) {\n if (threadIdx.x == 0) {\n // First MMA: zero-initialize accumulator\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 0;\"\n :\n : \"r\"(tmem_addr), \"l\"(desc_a), \"l\"(desc_b), \"r\"(0)\n );\n }\n}\n\n__device__ void mma_bf16_accumulate(\n uint32_t tmem_addr,\n uint64_t desc_a,\n uint64_t desc_b\n) {\n if (threadIdx.x == 0) {\n // Subsequent MMAs: accumulate into existing TMEM\n // The enable_accumulate flag controls whether to add to or overwrite TMEM\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 1;\" // last arg 1 = accumulate\n :\n : \"r\"(tmem_addr), \"l\"(desc_a), \"l\"(desc_b), \"r\"(0)\n );\n }\n}\n```"}, "after": {"statement": null}, "reason": {"statement": "Remove invalid executable-looking examples and retain the normative grammar with a vendor example link.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma"]}} +{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "```cuda\n__device__ void gemm_mainloop(/* params */) {\n for (int k_tile = 0; k_tile < num_k_tiles; ++k_tile) {\n // 1. Wait for operand data to arrive in SMEM\n wait_barrier(k_tile % NUM_STAGES);\n\n // 2. Issue MMA (single thread)\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 1;\"\n :\n : \"r\"(tmem_addr),\n \"l\"(make_smem_desc(smem_a, k_tile)),\n \"l\"(make_smem_desc(smem_b, k_tile)),\n \"r\"(0)\n );\n }\n\n // 3. Release SMEM buffer for next TMA load\n if (threadIdx.x == 0) {\n arrive_barrier((k_tile + 1) % NUM_STAGES);\n }\n }\n\n // 4. CRITICAL: Fence before reading accumulator from TMEM\n asm volatile(\"tcgen05.fence::before_thread_sync;\");\n __syncthreads();\n\n // 5. Now safe to read results from TMEM\n read_tmem_accumulator(tmem_addr, output);\n}\n```"}, "after": {"statement": null}, "reason": {"statement": "A partial pipeline with unsafe lifetime rules is more misleading than the normative sequence.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit"]}} +{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "```cuda\n// ---- HOPPER (SM90): wgmma ----\n// Requires warpgroup-scoped execution\n// All 128 threads in a warpgroup participate\n__device__ void hopper_mma() {\n // Load A matrix into registers via ldmatrix\n uint32_t a_frag[4];\n asm volatile(\"ldmatrix.sync.aligned.m8n8.x4.shared.b16 \"\n \"{%0,%1,%2,%3}, [%4];\"\n : \"=r\"(a_frag[0]), \"=r\"(a_frag[1]),\n \"=r\"(a_frag[2]), \"=r\"(a_frag[3])\n : \"r\"(smem_addr));\n\n // Issue wgmma -- warpgroup scope, register accumulator\n asm volatile(\"wgmma.mma_async.sync.aligned.m64n256k16.f32.bf16.bf16 \"\n \"{%0, %1, ...}, \" // register accumulators (128+ registers!)\n \"{%N, %N+1, ...}, \" // A operand in registers\n \"desc_b, ...;\"\n : \"+f\"(acc[0]), \"+f\"(acc[1]), ...\n : ...);\n\n // Commit and wait\n asm volatile(\"wgmma.commit_group.sync.aligned;\");\n asm volatile(\"wgmma.wait_group.sync.aligned 0;\");\n // Accumulators now in registers -- high register pressure\n}\n\n// ---- BLACKWELL (SM100): tcgen05 ----\n// Single-thread issuance, TMEM accumulator\n__device__ void blackwell_mma() {\n // No ldmatrix needed -- reads directly from SMEM\n // No register allocation for accumulators\n\n // Single thread issues MMA\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 1;\"\n :\n : \"r\"(tmem_addr), // accumulator in TMEM (not registers!)\n \"l\"(desc_a), // A from SMEM descriptor\n \"l\"(desc_b), // B from SMEM descriptor\n \"r\"(0)\n );\n }\n\n // Fence + sync before reading results\n asm volatile(\"tcgen05.fence::before_thread_sync;\");\n __syncthreads();\n // Read from TMEM -- registers free for other work\n}\n```"}, "after": {"statement": null}, "reason": {"statement": "Replace executable-looking pseudo-assembly with a precise semantic comparison table.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma"]}} +{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "tcgen05.mma requires **128-byte swizzled** shared memory layouts for both operands. Non-swizzled or 64-byte swizzled layouts will produce incorrect results silently."}, "after": {"statement": "The tcgen05 shared-memory descriptor is a 64-bit runtime value. PTX ISA 9.0 assigns fields for the encoded base address, leading dimension, stride dimension, fixed bits, base offset, leading-dimension mode, and a three-bit swizzle mode.\n\nValid swizzle encodings include no swizzle, 128-byte, 64-byte, and 32-byte layouts (plus a 128-byte/32-byte-atomic mode). Values 3, 5, and 7 are invalid; ordinary 128-byte swizzling uses value 2. A layout must satisfy the addressing and alignment constraints for its chosen mode. Although 128-byte swizzling can materially improve a particular GEMM, it is not a universal correctness requirement."}, "reason": {"statement": "Distinguish valid layouts from performance-sensitive layout choice.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-shared-memory-descriptor"]}} +{"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "```cuda\n// Shared memory descriptor construction for tcgen05\n// The descriptor encodes: base address, stride, swizzle mode, dimensions\n__device__ uint64_t make_smem_desc(void* smem_ptr, int stride_bytes) {\n uint64_t desc = 0;\n uint32_t addr = static_cast(__cvta_generic_to_shared(smem_ptr));\n\n // Encode base address (bits 0-13)\n desc |= (uint64_t)(addr >> 4);\n // Encode leading dimension stride (bits 16-29)\n desc |= (uint64_t)((stride_bytes >> 4) & 0x3FFF) << 16;\n // Encode 128-byte swizzle mode (bits 62-63) -- MANDATORY for tcgen05\n desc |= (uint64_t)(3) << 62; // 3 = 128-byte swizzle\n\n return desc;\n}\n```"}, "after": {"statement": null}, "reason": {"statement": "Direct users to the complete table rather than preserving a subtly corrupt descriptor builder.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-shared-memory-descriptor"]}} {"path": "wiki/hardware/tcgen05-mma.md", "before": {"statement": "| + Persistent kernel + CLC | 1476 | 98% |"}, "after": {"statement": "Gau Nernst reports the following results for M=N=K=4096 on a Modal B200, using PyTorch 2.9.1 with CUDA 13. Values are measurements for that setup, not architecture-wide guarantees.\n\n| cuBLAS | 1506.74 | 100% |\n| v1a: basic tcgen05 + 2D 16B TMA | 254.62 | 17% |\n| v1b: 3D 16B TMA | 252.81 | 17% |\n| v2a: 2D 128B TMA | 681.20 | 45% |\n| v2b: 3D 128B TMA | 695.43 | 46% |\n| v3: pipelining | 939.61 | 62% |\n| v4: warp specialization | 1208.83 | 80% |\n| v5: 2-SM MMA | 1302.29 | 86% |\n| v6: persistent, static scheduling | 1475.93 | 98% |\n\nThe v6 result did not use Cluster Launch Control; the author lists CLC and threadblock swizzling as unimplemented follow-up ideas."}, "reason": {"statement": "Preserve the measurement while restoring the actual scheduling method.", "urls": ["https://gau-nernst.github.io/tcgen05/"]}} {"path": "wiki/hardware/2sm-cooperative.md", "before": {"statement": "Blackwell enables two SMs within a TPC to cooperatively execute a single larger MMA, doubling the effective compute tile size to m256×n256×k16."}, "after": {"statement": "PTX defines a CTA pair as two CTAs in the same cluster whose `%cluster_ctarank` values differ only in the low bit. With `cta_group::2`, one thread from either CTA can initiate a whole `tcgen05.mma`; the peer CTA must still be active. The operation accesses Tensor Memory belonging to both the current CTA and its peer.\n\nAllocation management differs from MMA issue. A `tcgen05.alloc` or `tcgen05.dealloc` for `cta_group::2` is issued collectively by two warps, one in each CTA. All tcgen05 instructions in the kernel must use the same CTA-group value.\n\n`cta_group::2` selects pair-level resources; it does not by itself select one fixed MxNxK shape. The instruction descriptor (`idesc`) encodes M, N, exact operand and accumulator types, sparsity, and related operation details. Legal values depend on the kind, data-path layout, CTA group, and target ISA.\n\nFor example, m256xn256xk16 is a useful maximum F16/BF16 configuration, but PTX also defines group-2 layouts with other M/N values. Do not infer that every pair-level MMA is created by mechanically doubling a group-1 M dimension."}, "reason": {"statement": "Describe pair-level resource scope separately from shape selection.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-data-path-layout-organization", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-issue-granularity"]}} {"path": "wiki/hardware/2sm-cooperative.md", "before": {"statement": "```\nTPC (Two Processing Clusters)\n├── SM 0: CTA 0 — issues tcgen05.mma with cta_group::2\n│ ├── Shared Memory A (rows 0-127)\n│ └── TMEM (columns 0-255)\n└── SM 1: CTA 1 — cooperates on same MMA\n ├── Shared Memory A (rows 128-255)\n └── TMEM (columns 256-511)\n```"}, "after": {"statement": null}, "reason": {"statement": "Replace the misleading topology diagram with normative pair and issue rules.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-data-path-layout-organization"]}} -{"path": "wiki/hardware/2sm-cooperative.md", "before": {"statement": "```ptx\n// 2-SM cooperative MMA\ntcgen05.mma.cta_group::2.kind::f16\n [tmem_addr], descA, descB, idescC, idescD, ...;\n```"}, "after": {"statement": null}, "reason": {"statement": "Link to the already verified normative grammar on the tcgen05 page instead of duplicating invalid code.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} -{"path": "wiki/hardware/2sm-cooperative.md", "before": {"statement": "1. **Identical shared memory layouts** across both CTAs\n2. `shared::cluster` mbarrier signaling between the two CTAs\n\n4. Each CTA contributes half the M-dimension"}, "after": {"statement": "PTX defines a CTA pair as two CTAs in the same cluster whose `%cluster_ctarank` values differ only in the low bit. With `cta_group::2`, one thread from either CTA can initiate a whole `tcgen05.mma`; the peer CTA must still be active. The operation accesses Tensor Memory belonging to both the current CTA and its peer.\n\nAllocation management differs from MMA issue. A `tcgen05.alloc` or `tcgen05.dealloc` for `cta_group::2` is issued collectively by two warps, one in each CTA. All tcgen05 instructions in the kernel must use the same CTA-group value.\n\nThe complete operand grammar, including the eight-register disable-output-lane vector used by `cta_group::2`, is documented on the related [tcgen05.mma page](tcgen05-mma.md)."}, "reason": {"statement": "Replace the invented checklist with direct ISA invariants.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-issue-granularity", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/hardware/2sm-cooperative.md", "before": {"statement": "```ptx\n// 2-SM cooperative MMA\ntcgen05.mma.cta_group::2.kind::f16\n [tmem_addr], descA, descB, idescC, idescD, ...;\n```"}, "after": {"statement": null}, "reason": {"statement": "Link to the already verified normative grammar on the tcgen05 page instead of duplicating invalid code.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma"]}} +{"path": "wiki/hardware/2sm-cooperative.md", "before": {"statement": "1. **Identical shared memory layouts** across both CTAs\n2. `shared::cluster` mbarrier signaling between the two CTAs\n\n4. Each CTA contributes half the M-dimension"}, "after": {"statement": "PTX defines a CTA pair as two CTAs in the same cluster whose `%cluster_ctarank` values differ only in the low bit. With `cta_group::2`, one thread from either CTA can initiate a whole `tcgen05.mma`; the peer CTA must still be active. The operation accesses Tensor Memory belonging to both the current CTA and its peer.\n\nAllocation management differs from MMA issue. A `tcgen05.alloc` or `tcgen05.dealloc` for `cta_group::2` is issued collectively by two warps, one in each CTA. All tcgen05 instructions in the kernel must use the same CTA-group value.\n\nThe complete operand grammar, including the eight-register disable-output-lane vector used by `cta_group::2`, is documented on the related [tcgen05.mma page](tcgen05-mma.md)."}, "reason": {"statement": "Replace the invented checklist with direct ISA invariants.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-issue-granularity", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma"]}} {"path": "wiki/hardware/2sm-cooperative.md", "before": {"statement": "- Large GEMM problems where M ≥ 256\n- Compute-bound kernels where peak FLOPS matters\n- Combined with persistent scheduling for maximum throughput"}, "after": {"statement": "In Gau Nernst's M=N=K=4096 experiment on a Modal B200 with PyTorch 2.9.1 and CUDA 13, v4 warp specialization reports 1208.83 TFLOP/s and v5 2-SM MMA reports 1302.29 TFLOP/s. Relative to the same 1506.74-TFLOP/s cuBLAS result, those are approximately 80.2% and 86.4%, or a 7.7% relative increase from v4 to v5.\n\nThat result establishes a gain for one kernel and setup, not a universal threshold. Choose between group 1 and group 2 by benchmarking the target shapes and accounting for CTA pairing, data reuse, layout, occupancy, pipeline stages, and epilogue cost. Persistence is an independent scheduling choice, not a condition that guarantees maximum throughput."}, "reason": {"statement": "Replace categorical advice with a measurement-and-resource tradeoff checklist.", "urls": ["https://gau-nernst.github.io/tcgen05/"]}} -{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "- **Operand loading**: ldmatrix + registers vs direct SMEM access\n- **Synchronization**: warpgroup barriers vs async fences"}, "after": {"statement": "Port the programming model, not just the opcode:\n\n| MMA issue | Warpgroup collective | One thread for group-1 or group-2 MMA |\n| D accumulator | Per-thread registers | TMEM |\n| A operand | SMEM-descriptor or register forms, depending on instruction | SMEM-descriptor or TMEM-address forms |\n| B operand | SMEM descriptor | SMEM descriptor |\n| MMA completion | Commit/wait groups | `tcgen05.commit` plus mbarrier wait |\n| Cross-thread ordering | WGMMA-specific rules | tcgen05 fences composed with an execution-ordering operation |\n| Narrow floating formats | FP8 WGMMA forms | FP8/FP6/FP4 plus block-scaled MX/NVFP4 kinds |\n\nThe exact old WGMMA form matters. Do not assume every Hopper kernel loads A through `ldmatrix`: descriptor-sourced WGMMA forms already read A from SMEM. Likewise, tcgen05 can source A from SMEM or TMEM.\n\n`tcgen05.mma` is asynchronous. `tcgen05.commit.cta_group::N.mbarrier::arrive::one.b64` makes an mbarrier track prior MMA work issued by the current thread; waiting on that barrier observes completion. A `tcgen05.fence` is an ordering and code-motion primitive, not a completion wait, so fence plus `__syncthreads()` is not a substitute for commit/mbarrier."}, "reason": {"statement": "Make migration conditional on the exact old and new instruction forms and separate completion from ordering.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-matrix-multiply-and-accumulate-operations", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"]}} -{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "```\n[ ] Replace wgmma.mma_async with tcgen05.mma\n[ ] Move accumulators from registers to TMEM (alloc/dealloc)\n[ ] Remove ldmatrix operations (tcgen05 reads directly from SMEM)\n[ ] Change SMEM swizzle from 64B to 128B\n[ ] Replace warpgroup commit/wait with tcgen05 fences\n[ ] Update warp specialization roles (fewer warps needed for MMA)\n[ ] Update tile sizes (128xN -> consider 256xN with 2-SM)\n[ ] Add TMEM lifecycle management (alloc at start, dealloc at end)\n[ ] Update epilogue to read from TMEM instead of registers\n```"}, "after": {"statement": "1. Identify the exact WGMMA operand form, accumulator type, shape, and group-completion points in the SM90 kernel.\n2. Choose a legal tcgen05 kind, A source, CTA group, instruction descriptor, and data-path layout for the SM100 target.\n3. Replace register-resident D with a TMEM allocation. Group-1 allocation/deallocation is warp-collective; group 2 requires one warp in each CTA of the pair.\n4. Keep A/B backing storage unchanged until all asynchronous MMA consumers have completed.\n5. Replace WGMMA group completion with `tcgen05.commit` and an mbarrier wait. Add tcgen05 fences only where an execution-ordering handoff must order tcgen05-visible state across threads or CTAs.\n6. Transfer completed accumulator values from TMEM to registers with `tcgen05.ld`, observe its completion/access rules, then run the epilogue.\n7. Deallocate every dynamic TMEM allocation before kernel exit. Use the same `cta_group` value for all tcgen05 instructions in the kernel.\n8. Retune CTA/cluster shapes, pipeline stages, thread roles, descriptors, and epilogue scheduling on the target workload."}, "reason": {"statement": "Replace categorical substitutions with a dependency-ordered audit checklist.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"]}} +{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "- **Operand loading**: ldmatrix + registers vs direct SMEM access\n- **Synchronization**: warpgroup barriers vs async fences"}, "after": {"statement": "Port the programming model, not just the opcode:\n\n| MMA issue | Warpgroup collective | One thread for group-1 or group-2 MMA |\n| D accumulator | Per-thread registers | TMEM |\n| A operand | SMEM-descriptor or register forms, depending on instruction | SMEM-descriptor or TMEM-address forms |\n| B operand | SMEM descriptor | SMEM descriptor |\n| MMA completion | Commit/wait groups | `tcgen05.commit` plus mbarrier wait |\n| Cross-thread ordering | WGMMA-specific rules | tcgen05 fences composed with an execution-ordering operation |\n| Narrow floating formats | FP8 WGMMA forms | FP8/FP6/FP4 plus block-scaled MX/NVFP4 kinds |\n\nThe exact old WGMMA form matters. Do not assume every Hopper kernel loads A through `ldmatrix`: descriptor-sourced WGMMA forms already read A from SMEM. Likewise, tcgen05 can source A from SMEM or TMEM.\n\n`tcgen05.mma` is asynchronous. `tcgen05.commit.cta_group::N.mbarrier::arrive::one.b64` makes an mbarrier track prior MMA work issued by the current thread; waiting on that barrier observes completion. A `tcgen05.fence` is an ordering and code-motion primitive, not a completion wait, so fence plus `__syncthreads()` is not a substitute for commit/mbarrier."}, "reason": {"statement": "Make migration conditional on the exact old and new instruction forms and separate completion from ordering.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit"]}} +{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "```\n[ ] Replace wgmma.mma_async with tcgen05.mma\n[ ] Move accumulators from registers to TMEM (alloc/dealloc)\n[ ] Remove ldmatrix operations (tcgen05 reads directly from SMEM)\n[ ] Change SMEM swizzle from 64B to 128B\n[ ] Replace warpgroup commit/wait with tcgen05 fences\n[ ] Update warp specialization roles (fewer warps needed for MMA)\n[ ] Update tile sizes (128xN -> consider 256xN with 2-SM)\n[ ] Add TMEM lifecycle management (alloc at start, dealloc at end)\n[ ] Update epilogue to read from TMEM instead of registers\n```"}, "after": {"statement": "1. Identify the exact WGMMA operand form, accumulator type, shape, and group-completion points in the SM90 kernel.\n2. Choose a legal tcgen05 kind, A source, CTA group, instruction descriptor, and data-path layout for the SM100 target.\n3. Replace register-resident D with a TMEM allocation. Group-1 allocation/deallocation is warp-collective; group 2 requires one warp in each CTA of the pair.\n4. Keep A/B backing storage unchanged until all asynchronous MMA consumers have completed.\n5. Replace WGMMA group completion with `tcgen05.commit` and an mbarrier wait. Add tcgen05 fences only where an execution-ordering handoff must order tcgen05-visible state across threads or CTAs.\n6. Transfer completed accumulator values from TMEM to registers with `tcgen05.ld`, observe its completion/access rules, then run the epilogue.\n7. Deallocate every dynamic TMEM allocation before kernel exit. Use the same `cta_group` value for all tcgen05 instructions in the kernel.\n8. Retune CTA/cluster shapes, pipeline stages, thread roles, descriptors, and epilogue scheduling on the target workload."}, "reason": {"statement": "Replace categorical substitutions with a dependency-ordered audit checklist.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-shared-memory-descriptor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit"]}} {"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "```cuda\n// SM90 GEMM kernel using wgmma\n__global__ void hopper_gemm(\n const half* A, const half* B, half* C,\n int M, int N, int K\n) {\n extern __shared__ char smem[];\n half* smem_a = reinterpret_cast(smem);\n half* smem_b = smem_a + TILE_M * TILE_K;\n\n // Register accumulators -- HIGH REGISTER PRESSURE\n float acc[MMA_M_FRAGS][MMA_N_FRAGS]; // 128+ registers!\n for (int i = 0; i < MMA_M_FRAGS; ++i)\n for (int j = 0; j < MMA_N_FRAGS; ++j)\n acc[i][j] = 0.0f;\n\n // Mainloop\n for (int k = 0; k < K / TILE_K; ++k) {\n // Load A to SMEM (TMA or cp.async)\n tma_load(smem_a, A, tile_m, k);\n tma_load(smem_b, B, k, tile_n);\n\n // Load A fragment from SMEM to registers via ldmatrix\n uint32_t a_frag[4];\n asm volatile(\n \"ldmatrix.sync.aligned.m8n8.x4.shared.b16 \"\n \"{%0,%1,%2,%3}, [%4];\"\n : \"=r\"(a_frag[0]), \"=r\"(a_frag[1]),\n \"=r\"(a_frag[2]), \"=r\"(a_frag[3])\n : \"r\"(smem_a_addr)\n );\n\n // Issue wgmma -- ALL 128 THREADS IN WARPGROUP participate\n asm volatile(\n \"wgmma.mma_async.sync.aligned.m64n256k16.f32.bf16.bf16 \"\n \"{%0, %1, %2, %3, %4, %5, %6, %7, ...}, \" // acc registers\n \"{%N, %N+1, %N+2, %N+3}, \" // A in registers\n \"desc_b, ...\" // B descriptor\n : \"+f\"(acc[0][0]), \"+f\"(acc[0][1]), ...\n : \"r\"(a_frag[0]), ...\n );\n\n // Commit and wait for warpgroup\n asm volatile(\"wgmma.commit_group.sync.aligned;\");\n asm volatile(\"wgmma.wait_group.sync.aligned 0;\");\n }\n\n // Epilogue: accumulators already in registers\n // Write directly to GMEM (or through SMEM for vectorized stores)\n for (int i = 0; i < MMA_M_FRAGS; ++i)\n for (int j = 0; j < MMA_N_FRAGS; ++j)\n C[row + i][col + j] = (half)acc[i][j];\n}\n```"}, "after": {"statement": null}, "reason": {"statement": "Use an exact semantic mapping and pinned CUTLASS examples rather than fabricated assembly.", "urls": []}} -{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "```cuda\n// SM100 GEMM kernel using tcgen05\n__global__ void blackwell_gemm(\n const half* A, const half* B, half* C,\n int M, int N, int K\n) {\n extern __shared__ char smem[];\n half* smem_a = reinterpret_cast(smem);\n half* smem_b = smem_a + TILE_M * TILE_K;\n\n // TMEM accumulator -- NO REGISTER PRESSURE\n __shared__ uint32_t s_tmem_acc;\n if (threadIdx.x == 0) {\n uint32_t smem_addr =\n static_cast(__cvta_generic_to_shared(&s_tmem_acc));\n asm volatile(\n \"tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;\"\n :: \"r\"(smem_addr), \"r\"(256) // 256 columns for 128x256 tile\n );\n }\n __syncthreads();\n uint32_t tmem_acc = s_tmem_acc;\n\n // Zero TMEM accumulator\n tmem_zero(tmem_acc, 256);\n\n // Mainloop\n for (int k = 0; k < K / TILE_K; ++k) {\n // TMA load with 128B swizzle (mandatory for tcgen05)\n tma_load_128b_swizzle(smem_a, A, tile_m, k);\n tma_load_128b_swizzle(smem_b, B, k, tile_n);\n wait_barrier();\n\n // NO ldmatrix -- tcgen05 reads directly from SMEM\n // SINGLE THREAD issues MMA (not warpgroup)\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 1;\"\n :\n : \"r\"(tmem_acc),\n \"l\"(make_smem_desc_128b(smem_a)),\n \"l\"(make_smem_desc_128b(smem_b)),\n \"r\"(0)\n );\n }\n // NO commit/wait -- fully async, fence-based\n }\n\n // Fence before reading TMEM (replaces wgmma.wait_group)\n asm volatile(\"tcgen05.fence::before_thread_sync;\");\n __syncthreads();\n\n // Epilogue: read from TMEM, then write to GMEM\n float4 acc_vals = tmem_load_f32x4(tmem_acc + col_offset);\n // Apply bias, activation, etc.\n store_output(acc_vals, C, row, col);\n\n // Deallocate TMEM (no equivalent needed on Hopper)\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;\"\n :\n : \"r\"(tmem_acc), \"r\"(256)\n );\n }\n}\n```"}, "after": {"statement": null}, "reason": {"statement": "Remove an unsafe end-to-end example that cannot be repaired without becoming a full kernel.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"]}} -{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "**Before (Hopper):** Load A operand from SMEM to registers.\n\n```cuda\n// Hopper: ldmatrix loads 8x8 matrix fragments into registers\nuint32_t a_frag[4];\nasm volatile(\n \"ldmatrix.sync.aligned.m8n8.x4.shared.b16 \"\n \"{%0,%1,%2,%3}, [%4];\"\n : \"=r\"(a_frag[0]), \"=r\"(a_frag[1]),\n \"=r\"(a_frag[2]), \"=r\"(a_frag[3])\n : \"r\"(smem_addr)\n);\n```\n\n**After (Blackwell):** No equivalent needed. tcgen05 reads A directly from SMEM via descriptor.\n\n```cuda\n// Blackwell: just construct the SMEM descriptor\nuint64_t desc_a = make_smem_desc_128b(smem_a_ptr);\n// Pass desc_a directly to tcgen05.mma -- no register staging\n```"}, "after": {"statement": "Port the programming model, not just the opcode:\n\n| MMA issue | Warpgroup collective | One thread for group-1 or group-2 MMA |\n| D accumulator | Per-thread registers | TMEM |\n| A operand | SMEM-descriptor or register forms, depending on instruction | SMEM-descriptor or TMEM-address forms |\n| B operand | SMEM descriptor | SMEM descriptor |\n| MMA completion | Commit/wait groups | `tcgen05.commit` plus mbarrier wait |\n| Cross-thread ordering | WGMMA-specific rules | tcgen05 fences composed with an execution-ordering operation |\n| Narrow floating formats | FP8 WGMMA forms | FP8/FP6/FP4 plus block-scaled MX/NVFP4 kinds |\n\nThe exact old WGMMA form matters. Do not assume every Hopper kernel loads A through `ldmatrix`: descriptor-sourced WGMMA forms already read A from SMEM. Likewise, tcgen05 can source A from SMEM or TMEM."}, "reason": {"statement": "Make operand migration a form-by-form decision.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-matrix-multiply-and-accumulate-operations"]}} -{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "**After (Blackwell):** 128-byte swizzle is mandatory.\n\n```cuda\n// Blackwell TMA descriptor: MUST use 128B swizzle\nCUtensorMap desc = create_tma_desc(ptr, M, N, tile_m, tile_n,\n CU_TENSOR_MAP_SWIZZLE_128B);\n```"}, "after": {"statement": "`tcgen05.alloc` writes a 32-bit TMEM address to shared memory. Allocation size is expressed in columns, in power-of-two multiples permitted by PTX. It is a synchronous warp instruction: a lane-0-only call is invalid. `tcgen05.dealloc` has the same issue granularity, and PTX requires all allocated TMEM to be deallocated before kernel exit—not only in persistent kernels.\n\nThe MMA issuer and the allocation participants are different concepts. One thread initiates a group-1 MMA, but one full warp collectively allocates or deallocates its TMEM. A group-2 allocation uses one warp from each peer CTA."}, "reason": {"statement": "Make 128B a benchmarkable layout option rather than a correctness mandate.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"]}} -{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "**Before (Hopper):**\n\n```cuda\n// Hopper: all 128 threads in warpgroup issue wgmma\nasm volatile(\n \"wgmma.mma_async.sync.aligned.m64n256k16.f32.bf16.bf16 \"\n \"{%0,%1,...}, {%N,...}, desc_b, ...;\"\n : \"+f\"(acc[0]), \"+f\"(acc[1]), ...\n : \"r\"(a_frag[0]), ...\n);\nasm volatile(\"wgmma.commit_group.sync.aligned;\");\n```\n\n**After (Blackwell):**\n\n```cuda\n// Blackwell: single thread issues tcgen05\nif (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 1;\"\n :\n : \"r\"(tmem_acc), \"l\"(desc_a), \"l\"(desc_b), \"r\"(0)\n );\n}\n// No commit needed -- fully async\n```"}, "after": {"statement": null}, "reason": {"statement": "Replace incomplete assembly with a normative lifecycle sequence.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} -{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "**After (Blackwell):**\n\n```cuda\n// Blackwell: fence before reading TMEM\nasm volatile(\"tcgen05.fence::before_thread_sync;\");\n__syncthreads();\n// TMEM accumulators now ready for reading\n```"}, "after": {"statement": "`tcgen05.mma` is asynchronous. `tcgen05.commit.cta_group::N.mbarrier::arrive::one.b64` makes an mbarrier track prior MMA work issued by the current thread; waiting on that barrier observes completion. A `tcgen05.fence` is an ordering and code-motion primitive, not a completion wait, so fence plus `__syncthreads()` is not a substitute for commit/mbarrier."}, "reason": {"statement": "Preserve the WGMMA analogy while mapping it to the correct Blackwell mechanism.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"]}} +{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "```cuda\n// SM100 GEMM kernel using tcgen05\n__global__ void blackwell_gemm(\n const half* A, const half* B, half* C,\n int M, int N, int K\n) {\n extern __shared__ char smem[];\n half* smem_a = reinterpret_cast(smem);\n half* smem_b = smem_a + TILE_M * TILE_K;\n\n // TMEM accumulator -- NO REGISTER PRESSURE\n __shared__ uint32_t s_tmem_acc;\n if (threadIdx.x == 0) {\n uint32_t smem_addr =\n static_cast(__cvta_generic_to_shared(&s_tmem_acc));\n asm volatile(\n \"tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;\"\n :: \"r\"(smem_addr), \"r\"(256) // 256 columns for 128x256 tile\n );\n }\n __syncthreads();\n uint32_t tmem_acc = s_tmem_acc;\n\n // Zero TMEM accumulator\n tmem_zero(tmem_acc, 256);\n\n // Mainloop\n for (int k = 0; k < K / TILE_K; ++k) {\n // TMA load with 128B swizzle (mandatory for tcgen05)\n tma_load_128b_swizzle(smem_a, A, tile_m, k);\n tma_load_128b_swizzle(smem_b, B, k, tile_n);\n wait_barrier();\n\n // NO ldmatrix -- tcgen05 reads directly from SMEM\n // SINGLE THREAD issues MMA (not warpgroup)\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 1;\"\n :\n : \"r\"(tmem_acc),\n \"l\"(make_smem_desc_128b(smem_a)),\n \"l\"(make_smem_desc_128b(smem_b)),\n \"r\"(0)\n );\n }\n // NO commit/wait -- fully async, fence-based\n }\n\n // Fence before reading TMEM (replaces wgmma.wait_group)\n asm volatile(\"tcgen05.fence::before_thread_sync;\");\n __syncthreads();\n\n // Epilogue: read from TMEM, then write to GMEM\n float4 acc_vals = tmem_load_f32x4(tmem_acc + col_offset);\n // Apply bias, activation, etc.\n store_output(acc_vals, C, row, col);\n\n // Deallocate TMEM (no equivalent needed on Hopper)\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;\"\n :\n : \"r\"(tmem_acc), \"r\"(256)\n );\n }\n}\n```"}, "after": {"statement": null}, "reason": {"statement": "Remove an unsafe end-to-end example that cannot be repaired without becoming a full kernel.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit"]}} +{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "**Before (Hopper):** Load A operand from SMEM to registers.\n\n```cuda\n// Hopper: ldmatrix loads 8x8 matrix fragments into registers\nuint32_t a_frag[4];\nasm volatile(\n \"ldmatrix.sync.aligned.m8n8.x4.shared.b16 \"\n \"{%0,%1,%2,%3}, [%4];\"\n : \"=r\"(a_frag[0]), \"=r\"(a_frag[1]),\n \"=r\"(a_frag[2]), \"=r\"(a_frag[3])\n : \"r\"(smem_addr)\n);\n```\n\n**After (Blackwell):** No equivalent needed. tcgen05 reads A directly from SMEM via descriptor.\n\n```cuda\n// Blackwell: just construct the SMEM descriptor\nuint64_t desc_a = make_smem_desc_128b(smem_a_ptr);\n// Pass desc_a directly to tcgen05.mma -- no register staging\n```"}, "after": {"statement": "Port the programming model, not just the opcode:\n\n| MMA issue | Warpgroup collective | One thread for group-1 or group-2 MMA |\n| D accumulator | Per-thread registers | TMEM |\n| A operand | SMEM-descriptor or register forms, depending on instruction | SMEM-descriptor or TMEM-address forms |\n| B operand | SMEM descriptor | SMEM descriptor |\n| MMA completion | Commit/wait groups | `tcgen05.commit` plus mbarrier wait |\n| Cross-thread ordering | WGMMA-specific rules | tcgen05 fences composed with an execution-ordering operation |\n| Narrow floating formats | FP8 WGMMA forms | FP8/FP6/FP4 plus block-scaled MX/NVFP4 kinds |\n\nThe exact old WGMMA form matters. Do not assume every Hopper kernel loads A through `ldmatrix`: descriptor-sourced WGMMA forms already read A from SMEM. Likewise, tcgen05 can source A from SMEM or TMEM."}, "reason": {"statement": "Make operand migration a form-by-form decision.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma"]}} +{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "**After (Blackwell):** 128-byte swizzle is mandatory.\n\n```cuda\n// Blackwell TMA descriptor: MUST use 128B swizzle\nCUtensorMap desc = create_tma_desc(ptr, M, N, tile_m, tile_n,\n CU_TENSOR_MAP_SWIZZLE_128B);\n```"}, "after": {"statement": "`tcgen05.alloc` writes a 32-bit TMEM address to shared memory. Allocation size is expressed in columns, in power-of-two multiples permitted by PTX. It is a synchronous warp instruction: a lane-0-only call is invalid. `tcgen05.dealloc` has the same issue granularity, and PTX requires all allocated TMEM to be deallocated before kernel exit—not only in persistent kernels.\n\nThe MMA issuer and the allocation participants are different concepts. One thread initiates a group-1 MMA, but one full warp collectively allocates or deallocates its TMEM. A group-2 allocation uses one warp from each peer CTA."}, "reason": {"statement": "Make 128B a benchmarkable layout option rather than a correctness mandate.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-shared-memory-descriptor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit"]}} +{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "**Before (Hopper):**\n\n```cuda\n// Hopper: all 128 threads in warpgroup issue wgmma\nasm volatile(\n \"wgmma.mma_async.sync.aligned.m64n256k16.f32.bf16.bf16 \"\n \"{%0,%1,...}, {%N,...}, desc_b, ...;\"\n : \"+f\"(acc[0]), \"+f\"(acc[1]), ...\n : \"r\"(a_frag[0]), ...\n);\nasm volatile(\"wgmma.commit_group.sync.aligned;\");\n```\n\n**After (Blackwell):**\n\n```cuda\n// Blackwell: single thread issues tcgen05\nif (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 1;\"\n :\n : \"r\"(tmem_acc), \"l\"(desc_a), \"l\"(desc_b), \"r\"(0)\n );\n}\n// No commit needed -- fully async\n```"}, "after": {"statement": null}, "reason": {"statement": "Replace incomplete assembly with a normative lifecycle sequence.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma"]}} +{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "**After (Blackwell):**\n\n```cuda\n// Blackwell: fence before reading TMEM\nasm volatile(\"tcgen05.fence::before_thread_sync;\");\n__syncthreads();\n// TMEM accumulators now ready for reading\n```"}, "after": {"statement": "`tcgen05.mma` is asynchronous. `tcgen05.commit.cta_group::N.mbarrier::arrive::one.b64` makes an mbarrier track prior MMA work issued by the current thread; waiting on that barrier observes completion. A `tcgen05.fence` is an ordering and code-motion primitive, not a completion wait, so fence plus `__syncthreads()` is not a substitute for commit/mbarrier."}, "reason": {"statement": "Preserve the WGMMA analogy while mapping it to the correct Blackwell mechanism.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit"]}} {"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "```\nWarp 0-3: Warpgroup 0 -- MMA producer (all 128 threads issue wgmma)\nWarp 4-7: Warpgroup 1 -- MMA producer (backup/double-buffer)\nWarp 8-11: Data movement (TMA loads, SMEM management)\nWarp 12: Tile scheduler\n\nTotal: 13+ warps, 416+ threads\n```\n\n```\nWarp 0: MMA producer (single thread issues tcgen05)\nWarp 1-2: TMA data movement (load A, load B, manage barriers)\nWarp 3: Epilogue (read TMEM, apply post-ops, store to GMEM)\nWarp 4: Tile scheduler / CLC management\n\nTotal: 5 warps, 160 threads (can be fewer)\n```"}, "after": {"statement": null}, "reason": {"statement": "Replace invented totals with role-level design guidance.", "urls": []}} -{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "| m64 x n256 x k16 | m128 x n256 x k16 | m256 x n256 x k16 |\n| m64 x n128 x k16 | m128 x n128 x k16 | m256 x n128 x k16 |\n\nBlackwell's base MMA tile is 2x larger in M (128 vs 64). When migrating:\n- CTA tile size of 128x256 maps naturally to a single tcgen05 MMA\n- CTA tile size of 64x256 on Hopper should be doubled to 128x256\n- For 2-SM mode, consider 256x256 tiles"}, "after": {"statement": "Do not mechanically convert every 64-byte swizzle to 128-byte swizzling. The tcgen05 shared-memory descriptor defines valid no-swizzle, 128B, 64B, and 32B modes, subject to mode-specific alignment and layout constraints. The 128B choice can be much faster for a particular tile, but it is not a universal correctness condition.\n\nSimilarly, there is no universal Hopper-to-Blackwell rule that doubles M. WGMMA and tcgen05 each expose multiple shapes; tcgen05 M/N are encoded in `idesc` and constrained by kind, layout, CTA group, and target ISA. Treat m128xn256xk16 and m256xn256xk16 as useful F16/BF16 maximum examples, not a complete migration table."}, "reason": {"statement": "Turn mechanical tile substitution into a resource-aware retuning step.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-data-path-layout-organization", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor"]}} -{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "1. **Forgetting 128B swizzle**: The most common silent-failure bug. wgmma works with 64B swizzle; tcgen05 does not. Results will be numerically wrong but the kernel won't crash.\n\n3. **Synchronization model mismatch**: Replacing `wgmma.wait_group` with `__syncthreads()` alone is insufficient. The `tcgen05.fence::before_thread_sync` must precede the syncthreads.\n\n4. **Over-allocating threads**: On Hopper, you need 128 threads per warpgroup for MMA. Blindly keeping the same thread count on Blackwell wastes resources since only 1 thread issues tcgen05."}, "after": {"statement": "1. Identify the exact WGMMA operand form, accumulator type, shape, and group-completion points in the SM90 kernel.\n2. Choose a legal tcgen05 kind, A source, CTA group, instruction descriptor, and data-path layout for the SM100 target.\n3. Replace register-resident D with a TMEM allocation. Group-1 allocation/deallocation is warp-collective; group 2 requires one warp in each CTA of the pair.\n4. Keep A/B backing storage unchanged until all asynchronous MMA consumers have completed.\n5. Replace WGMMA group completion with `tcgen05.commit` and an mbarrier wait. Add tcgen05 fences only where an execution-ordering handoff must order tcgen05-visible state across threads or CTAs.\n6. Transfer completed accumulator values from TMEM to registers with `tcgen05.ld`, observe its completion/access rules, then run the epilogue.\n7. Deallocate every dynamic TMEM allocation before kernel exit. Use the same `cta_group` value for all tcgen05 instructions in the kernel.\n8. Retune CTA/cluster shapes, pipeline stages, thread roles, descriptors, and epilogue scheduling on the target workload.\n\n`tcgen05.alloc` writes a 32-bit TMEM address to shared memory. Allocation size is expressed in columns, in power-of-two multiples permitted by PTX. It is a synchronous warp instruction: a lane-0-only call is invalid. `tcgen05.dealloc` has the same issue granularity, and PTX requires all allocated TMEM to be deallocated before kernel exit—not only in persistent kernels.\n\nThe MMA issuer and the allocation participants are different concepts. One thread initiates a group-1 MMA, but one full warp collectively allocates or deallocates its TMEM. A group-2 allocation uses one warp from each peer CTA.\n\n`tcgen05.mma` is asynchronous. `tcgen05.commit.cta_group::N.mbarrier::arrive::one.b64` makes an mbarrier track prior MMA work issued by the current thread; waiting on that barrier observes completion. A `tcgen05.fence` is an ordering and code-motion primitive, not a completion wait, so fence plus `__syncthreads()` is not a substitute for commit/mbarrier."}, "reason": {"statement": "Keep only direct, scoped pitfalls with observable failure modes.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-descriptor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"]}} +{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "| m64 x n256 x k16 | m128 x n256 x k16 | m256 x n256 x k16 |\n| m64 x n128 x k16 | m128 x n128 x k16 | m256 x n128 x k16 |\n\nBlackwell's base MMA tile is 2x larger in M (128 vs 64). When migrating:\n- CTA tile size of 128x256 maps naturally to a single tcgen05 MMA\n- CTA tile size of 64x256 on Hopper should be doubled to 128x256\n- For 2-SM mode, consider 256x256 tiles"}, "after": {"statement": "Do not mechanically convert every 64-byte swizzle to 128-byte swizzling. The tcgen05 shared-memory descriptor defines valid no-swizzle, 128B, 64B, and 32B modes, subject to mode-specific alignment and layout constraints. The 128B choice can be much faster for a particular tile, but it is not a universal correctness condition.\n\nSimilarly, there is no universal Hopper-to-Blackwell rule that doubles M. WGMMA and tcgen05 each expose multiple shapes; tcgen05 M/N are encoded in `idesc` and constrained by kind, layout, CTA group, and target ISA. Treat m128xn256xk16 and m256xn256xk16 as useful F16/BF16 maximum examples, not a complete migration table."}, "reason": {"statement": "Turn mechanical tile substitution into a resource-aware retuning step.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-data-path-layout-organization", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-shared-memory-descriptor"]}} +{"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "1. **Forgetting 128B swizzle**: The most common silent-failure bug. wgmma works with 64B swizzle; tcgen05 does not. Results will be numerically wrong but the kernel won't crash.\n\n3. **Synchronization model mismatch**: Replacing `wgmma.wait_group` with `__syncthreads()` alone is insufficient. The `tcgen05.fence::before_thread_sync` must precede the syncthreads.\n\n4. **Over-allocating threads**: On Hopper, you need 128 threads per warpgroup for MMA. Blindly keeping the same thread count on Blackwell wastes resources since only 1 thread issues tcgen05."}, "after": {"statement": "1. Identify the exact WGMMA operand form, accumulator type, shape, and group-completion points in the SM90 kernel.\n2. Choose a legal tcgen05 kind, A source, CTA group, instruction descriptor, and data-path layout for the SM100 target.\n3. Replace register-resident D with a TMEM allocation. Group-1 allocation/deallocation is warp-collective; group 2 requires one warp in each CTA of the pair.\n4. Keep A/B backing storage unchanged until all asynchronous MMA consumers have completed.\n5. Replace WGMMA group completion with `tcgen05.commit` and an mbarrier wait. Add tcgen05 fences only where an execution-ordering handoff must order tcgen05-visible state across threads or CTAs.\n6. Transfer completed accumulator values from TMEM to registers with `tcgen05.ld`, observe its completion/access rules, then run the epilogue.\n7. Deallocate every dynamic TMEM allocation before kernel exit. Use the same `cta_group` value for all tcgen05 instructions in the kernel.\n8. Retune CTA/cluster shapes, pipeline stages, thread roles, descriptors, and epilogue scheduling on the target workload.\n\n`tcgen05.alloc` writes a 32-bit TMEM address to shared memory. Allocation size is expressed in columns, in power-of-two multiples permitted by PTX. It is a synchronous warp instruction: a lane-0-only call is invalid. `tcgen05.dealloc` has the same issue granularity, and PTX requires all allocated TMEM to be deallocated before kernel exit—not only in persistent kernels.\n\nThe MMA issuer and the allocation participants are different concepts. One thread initiates a group-1 MMA, but one full warp collectively allocates or deallocates its TMEM. A group-2 allocation uses one warp from each peer CTA.\n\n`tcgen05.mma` is asynchronous. `tcgen05.commit.cta_group::N.mbarrier::arrive::one.b64` makes an mbarrier track prior MMA work issued by the current thread; waiting on that barrier observes completion. A `tcgen05.fence` is an ordering and code-motion primitive, not a completion wait, so fence plus `__syncthreads()` is not a substitute for commit/mbarrier."}, "reason": {"statement": "Keep only direct, scoped pitfalls with observable failure modes.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-shared-memory-descriptor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit"]}} {"path": "wiki/migration/wgmma-to-tcgen05.md", "before": {"statement": "If using CUTLASS, the migration is largely handled by changing the arch tag and kernel schedule:\n\n```cpp\n// Hopper CUTLASS GEMM\nusing GemmHopper = cutlass::gemm::device::GemmUniversal<\n /* ... */\n cutlass::arch::Sm90,\n /* ... */\n cutlass::gemm::collective::KernelScheduleSm90CpAsyncWarpSpecialized\n>;\n\n// Blackwell CUTLASS GEMM -- change arch + schedule\nusing GemmBlackwell = cutlass::gemm::device::GemmUniversal<\n /* ... */\n cutlass::arch::Sm100, // <-- changed\n /* ... */\n cutlass::gemm::collective::KernelScheduleSm100CpAsyncWarpSpecialized // <-- changed\n>;\n```\n\nCUTLASS handles the internal differences (TMEM allocation, descriptor construction, fence insertion, 128B swizzle) automatically through its `MMA_Atom` and `MMA_Traits` abstractions for SM100."}, "after": {"statement": "Single-thread MMA issue can free instruction-issue capacity for TMA, descriptor preparation, epilogue, reductions, or scheduling. It does not determine the kernel's total thread count: TMEM allocation, TMEM loads, TMA, epilogue, and barriers remain warp- or CTA-level work. Start from a pinned SM100 implementation and measure a role partition rather than copying fixed warp counts from an SM90 kernel.\n\nMoving D out of the GPR file also changes the register budget. Revisit tile size, pipeline depth, and launch bounds, but do not assume that lower accumulator pressure automatically increases occupancy; SMEM, TMEM, barriers, threads, and registers used by other roles can become limiting resources."}, "reason": {"statement": "Point to a pinned, complete SM100 example and enumerate the configuration surfaces that must be reselected.", "urls": []}} {"path": "wiki/hardware/tmem.md", "before": {"statement": "Tensor Memory (TMEM) is a new addressable memory space introduced in the Blackwell architecture (SM100). Each SM contains **256KB of dedicated TMEM**, used primarily as the accumulator storage for `tcgen05.mma` operations. TMEM eliminates the register pressure that plagued Hopper's wgmma, where accumulators consumed 128+ registers per warpgroup."}, "after": {"statement": "Tensor Memory is an addressable, on-chip memory introduced for fifth-generation Tensor Core operations on Blackwell. Each SM has 256 KiB, organized as 128 lanes by 512 columns of 32-bit cells. `tcgen05.mma` writes its D accumulator to TMEM; depending on the instruction form, A can also reside there.\n\nMoving D out of general-purpose registers changes the resource balance relative to Hopper WGMMA. For example, CUTLASS 4.5.0's SM90 m64n256 FP32 WGMMA wrapper exposes 128 accumulator registers **per participating thread**. TMEM avoids that particular register-resident D fragment, but it does not remove registers needed by operands, control flow, or the epilogue."}, "reason": {"statement": "Keep the verified storage shift but scope register impact to the exact old instruction and per-thread fragment.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cute/arch/mma_sm90_gmma.hpp", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} -{"path": "wiki/hardware/tmem.md", "before": {"statement": "Each thread \"owns\" the TMEM row corresponding to its warp and lane. When reading from TMEM, thread `T` in warp `W` accesses row `W*32 + T%32`."}, "after": {"statement": "A TMEM address (`taddr`) is a 32-bit value:\n\n| 31:16 | TMEM lane |\n| 15:0 | TMEM column |\n\nThe lane is a TMEM data-path lane, not storage independently owned by a CUDA thread. `tcgen05.ld` and `tcgen05.st` are warp-collective. Under their four-warp access model, the warps cover these lane chunks:\n\n| 0 | 0-31 |\n| 1 | 32-63 |\n| 2 | 64-95 |\n| 3 | 96-127 |\n\nThe chosen load/store shape determines how values from that lane-column region map to each thread's register vector. Use the PTX shape-specific layout tables rather than treating a logical MxN accumulator as a universal row-major array owned one row per thread."}, "reason": {"statement": "Describe lanes and collective warp chunks without inventing ownership.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory-addressing"]}} +{"path": "wiki/hardware/tmem.md", "before": {"statement": "Each thread \"owns\" the TMEM row corresponding to its warp and lane. When reading from TMEM, thread `T` in warp `W` accesses row `W*32 + T%32`."}, "after": {"statement": "A TMEM address (`taddr`) is a 32-bit value:\n\n| 31:16 | TMEM lane |\n| 15:0 | TMEM column |\n\nThe lane is a TMEM data-path lane, not storage independently owned by a CUDA thread. `tcgen05.ld` and `tcgen05.st` are warp-collective. Under their four-warp access model, the warps cover these lane chunks:\n\n| 0 | 0-31 |\n| 1 | 32-63 |\n| 2 | 64-95 |\n| 3 | 96-127 |\n\nThe chosen load/store shape determines how values from that lane-column region map to each thread's register vector. Use the PTX shape-specific layout tables rather than treating a logical MxN accumulator as a universal row-major array owned one row per thread."}, "reason": {"statement": "Describe lanes and collective warp chunks without inventing ownership.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory-addressing"]}} {"path": "wiki/hardware/tmem.md", "before": {"statement": "Columns are addressed via a column offset in the TMEM descriptor. A 128x256 MMA accumulator tile occupies:\n\n- 128 rows (all lanes across 4 warps)\n- 256 columns of FP32 values = 1024 bytes per row\n\n```\nTMEM Layout for 128x256 FP32 accumulator:\n col 0 col 255 col 511\n |------------|-----------|-----------|\n | Acc Tile | (free) | (free) | row 0 (warp0, lane0)\n | 128x256 | | | row 1 (warp0, lane1)\n | FP32 | | | ...\n | | | | row 127 (warp3, lane31)\n |------------|-----------|-----------|\n```"}, "after": {"statement": "A TMEM address (`taddr`) is a 32-bit value:\n\n| 31:16 | TMEM lane |\n| 15:0 | TMEM column |\n\nThe lane is a TMEM data-path lane, not storage independently owned by a CUDA thread. `tcgen05.ld` and `tcgen05.st` are warp-collective. Under their four-warp access model, the warps cover these lane chunks:\n\n| 0 | 0-31 |\n| 1 | 32-63 |\n| 2 | 64-95 |\n| 3 | 96-127 |\n\nThe chosen load/store shape determines how values from that lane-column region map to each thread's register vector. Use the PTX shape-specific layout tables rather than treating a logical MxN accumulator as a universal row-major array owned one row per thread."}, "reason": {"statement": "Retain capacity arithmetic separately from instruction-specific layout.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-data-path-layout-organization", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory-addressing"]}} -{"path": "wiki/hardware/tmem.md", "before": {"statement": "```cuda\n// Shared storage for CTA-wide broadcast of TMEM address\n__shared__ uint32_t s_tmem_addr;\n\n__device__ uint32_t tmem_alloc_cta(uint32_t num_cols) {\n // Only thread 0 allocates; result must reach ALL warps in the CTA.\n // __shfl_sync is warp-local — it cannot broadcast across warps.\n if (threadIdx.x == 0) {\n uint32_t smem_addr =\n static_cast(__cvta_generic_to_shared(&s_tmem_addr));\n asm volatile(\n \"tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;\"\n :\n : \"r\"(smem_addr), \"r\"(num_cols)\n );\n }\n __syncthreads(); // All warps now see s_tmem_addr\n return s_tmem_addr;\n}\n```\n\n- Only one thread (typically thread 0) issues the allocation.\n\n```cuda\n__device__ void tmem_dealloc(uint32_t tmem_addr, uint32_t num_cols) {\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;\"\n :\n : \"r\"(tmem_addr), \"r\"(num_cols)\n );\n }\n __syncthreads();\n}\n```"}, "after": {"statement": "TMEM has an explicit software-managed lifetime:\n\n1. Reserve shared memory for the 32-bit allocation result.\n2. Execute `tcgen05.alloc` collectively from one warp for `cta_group::1`, or from two warps—one per paired CTA—for `cta_group::2`.\n3. Synchronize consumers as required, then read the returned base `taddr` from shared memory.\n4. Use the allocation for MMA, copy, load, or store operations with the same CTA-group mode.\n5. Execute `tcgen05.dealloc` with the corresponding collective issue pattern.\n6. Deallocate every TMEM allocation before kernel exit.\n\nThe allocation operand counts columns. Legal allocation sizes are powers of two from 32 through 512 columns. Allocation can block until the requested TMEM is available; the ISA does not define folklore outcomes such as silent corruption for an invalid size."}, "reason": {"statement": "Remove unsafe inline PTX wrappers and state the collective contract directly.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"]}} -{"path": "wiki/hardware/tmem.md", "before": {"statement": "- The returned `tmem_addr` is the base column index."}, "after": {"statement": "TMEM has an explicit software-managed lifetime:\n\n1. Reserve shared memory for the 32-bit allocation result.\n2. Execute `tcgen05.alloc` collectively from one warp for `cta_group::1`, or from two warps—one per paired CTA—for `cta_group::2`.\n3. Synchronize consumers as required, then read the returned base `taddr` from shared memory.\n4. Use the allocation for MMA, copy, load, or store operations with the same CTA-group mode.\n5. Execute `tcgen05.dealloc` with the corresponding collective issue pattern.\n6. Deallocate every TMEM allocation before kernel exit.\n\nThe allocation operand counts columns. Legal allocation sizes are powers of two from 32 through 512 columns. Allocation can block until the requested TMEM is available; the ISA does not define folklore outcomes such as silent corruption for an invalid size."}, "reason": {"statement": "Preserve the full address model.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory-addressing", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"]}} -{"path": "wiki/hardware/tmem.md", "before": {"statement": "**Warning**: Failure to deallocate TMEM before CTA exit will leak memory and prevent subsequent CTAs from allocating, leading to hangs in persistent kernels.\n\n```cuda\n__global__ void persistent_gemm_kernel(/* ... */) {\n // 1. Allocate TMEM for accumulators\n uint32_t tmem_acc = tmem_alloc(256); // 256 cols for M=128, N=256\n\n while (has_more_tiles()) {\n // 2. Zero-initialize TMEM accumulator\n tmem_zero(tmem_acc, 256);\n\n // 3. Mainloop: accumulate K tiles\n for (int k = 0; k < K_tiles; ++k) {\n issue_tcgen05_mma(tmem_acc, smem_a[k], smem_b[k]);\n }\n\n // 4. Fence and read results\n asm volatile(\"tcgen05.fence::before_thread_sync;\");\n __syncthreads();\n\n // 5. Epilogue: read from TMEM, apply bias/activation, write to GMEM\n epilogue_from_tmem(tmem_acc, output);\n\n // 6. TMEM persists across tiles -- no need to reallocate\n }\n\n // 7. Deallocate before exit\n tmem_dealloc(tmem_acc, 256);\n}\n```"}, "after": {"statement": "TMEM has an explicit software-managed lifetime:\n\n1. Reserve shared memory for the 32-bit allocation result.\n2. Execute `tcgen05.alloc` collectively from one warp for `cta_group::1`, or from two warps—one per paired CTA—for `cta_group::2`.\n3. Synchronize consumers as required, then read the returned base `taddr` from shared memory.\n4. Use the allocation for MMA, copy, load, or store operations with the same CTA-group mode.\n5. Execute `tcgen05.dealloc` with the corresponding collective issue pattern.\n6. Deallocate every TMEM allocation before kernel exit.\n\nThe allocation operand counts columns. Legal allocation sizes are powers of two from 32 through 512 columns. Allocation can block until the requested TMEM is available; the ISA does not define folklore outcomes such as silent corruption for an invalid size.\n\nThe relevant mechanisms are distinct:\n\n- `tcgen05.commit` attaches completion of prior asynchronous MMA operations to an mbarrier. Wait on that barrier before consuming their results.\n- `tcgen05.wait::ld` and `tcgen05.wait::st` are the completion mechanisms for the corresponding asynchronous TMEM load/store operations.\n- `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` order tcgen05 operations around a documented execution-ordering handoff. A fence is not an MMA completion wait.\n- Producer buffers must remain live until the asynchronous operation that reads them has completed according to its instruction contract.\n\nCTA synchronization alone does not replace these completion operations."}, "reason": {"statement": "Remove a fabricated incomplete kernel and replace it with a normative lifecycle sequence.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-wait"]}} -{"path": "wiki/hardware/tmem.md", "before": {"statement": "```cuda\n// Store a register value to TMEM\n// Each thread writes to its own TMEM row at the specified column\n__device__ void tmem_store_f32(uint32_t tmem_col, float value) {\n asm volatile(\n \"tcgen05.st.sync.aligned.32x1b.x1.b32 [%0], {%1};\"\n :\n : \"r\"(tmem_col), \"f\"(value)\n );\n}\n\n// Vectorized store: 4 consecutive FP32 values\n__device__ void tmem_store_f32x4(uint32_t tmem_col, float4 values) {\n asm volatile(\n \"tcgen05.st.sync.aligned.32x1b.x4.b32 [%0], {%1, %2, %3, %4};\"\n :\n : \"r\"(tmem_col),\n \"f\"(values.x), \"f\"(values.y),\n \"f\"(values.z), \"f\"(values.w)\n );\n}\n```\n\n```cuda\n// Load a single FP32 from TMEM\n__device__ float tmem_load_f32(uint32_t tmem_col) {\n float result;\n asm volatile(\n \"tcgen05.ld.sync.aligned.32x1b.x1.b32 {%0}, [%1];\"\n : \"=f\"(result)\n : \"r\"(tmem_col)\n );\n return result;\n}\n\n// Vectorized load: 4 consecutive FP32 values\n__device__ float4 tmem_load_f32x4(uint32_t tmem_col) {\n float4 result;\n asm volatile(\n \"tcgen05.ld.sync.aligned.32x1b.x4.b32 {%0, %1, %2, %3}, [%4];\"\n : \"=f\"(result.x), \"=f\"(result.y),\n \"=f\"(result.z), \"=f\"(result.w)\n : \"r\"(tmem_col)\n );\n return result;\n}\n```\n\n```cuda\n// Zero-fill a range of TMEM columns\n__device__ void tmem_zero(uint32_t tmem_base_col, uint32_t num_cols) {\n // Each thread zeros its own row\n for (uint32_t c = 0; c < num_cols; c += 4) {\n float4 zero = make_float4(0.f, 0.f, 0.f, 0.f);\n tmem_store_f32x4(tmem_base_col + c, zero);\n }\n}\n```"}, "after": {"statement": "`tcgen05.ld` transfers TMEM into registers, and `tcgen05.st` transfers registers into TMEM. Their documented shapes include `.16x64b`, `.16x128b`, `.16x256b`, `.32x32b`, and `.16x32bx2`, with supported repetition qualifiers. There is no scalar `.32x1b` form.\n\n`tcgen05.cp` performs shaped shared-memory-to-TMEM copies. For example, PTX ISA 9.0 defines this exact form:\n\n```ptx\ntcgen05.cp.cta_group::1.128x256b [taddr], sdesc;\n```\n\nHere `taddr` is the TMEM destination and `sdesc` is the 64-bit shared-memory matrix descriptor. Follow the instruction's asynchronous ordering and completion rules before reusing the source or consuming the destination."}, "reason": {"statement": "Do not publish non-assembling executable-looking wrappers.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-cp"]}} -{"path": "wiki/hardware/tmem.md", "before": {"statement": "```cuda\n__global__ void double_buffered_gemm(/* ... */) {\n // Allocate two accumulator buffers\n uint32_t tmem_acc[2];\n tmem_acc[0] = tmem_alloc(256);\n tmem_acc[1] = tmem_alloc(256);\n\n int buf = 0;\n\n for (int tile = 0; tile < num_tiles; ++tile) {\n // Zero the current buffer\n tmem_zero(tmem_acc[buf], 256);\n\n // Mainloop: accumulate into current buffer\n for (int k = 0; k < K_tiles; ++k) {\n issue_tcgen05_mma(tmem_acc[buf], smem_a[k], smem_b[k]);\n }\n asm volatile(\"tcgen05.fence::before_thread_sync;\");\n __syncthreads();\n\n // If not the first tile, the *other* buffer's epilogue\n // was overlapped with this tile's MMA in the pipeline.\n\n // Start epilogue for current buffer\n // (can overlap with next tile's MMA on the other buffer)\n epilogue_from_tmem(tmem_acc[buf], output_tile[tile]);\n\n buf ^= 1; // Swap buffers\n }\n\n // Cleanup\n tmem_dealloc(tmem_acc[0], 256);\n tmem_dealloc(tmem_acc[1], 256);\n}\n```"}, "after": {"statement": "CUTLASS 4.5.0 exposes TMEM through `cutlass.utils.TmemAllocator` and layout-aware CuTe tensors. The official tutorial sequence is:\n\n1. create a `TmemAllocator` over shared storage for the allocation result;\n2. call `allocate(num_columns)` from the configured allocator warp;\n3. use `wait_for_alloc()` before other warps retrieve the address;\n4. obtain a typed pointer with `retrieve_ptr(dtype)`;\n5. bind that pointer to the MMA accumulator layout with `cute.make_tensor`;\n6. drain it with `tcgen05` copy atoms and the pipeline's completion protocol; and\n7. call `free(tmem_ptr)` before exit."}, "reason": {"statement": "Replace invented code with resource and synchronization requirements plus a pinned real example.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_6.py", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"]}} -{"path": "wiki/hardware/tmem.md", "before": {"statement": "| 128x192 FP32 | 192 cols | 2 | 128 |"}, "after": {"statement": "All allocations share the SM's 512-column capacity. Some useful whole-allocation budgets are:\n\n| 128 columns | 4 | 0 |\n| 256 columns | 2 | 0 |\n| 512 columns | 1 | 0 |\n\nA logical need of 192 columns cannot be requested directly: reserve 256 columns, or suballocate that logical region within another legal power-of-two reservation. Two 256-column accumulator stages consume the entire capacity, so scale-factor tensors or other scratch state must fit inside those reservations or the pipeline must use fewer columns."}, "reason": {"statement": "List only allocatable budgets and distinguish logical suballocation from hardware allocation.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"]}} +{"path": "wiki/hardware/tmem.md", "before": {"statement": "```cuda\n// Shared storage for CTA-wide broadcast of TMEM address\n__shared__ uint32_t s_tmem_addr;\n\n__device__ uint32_t tmem_alloc_cta(uint32_t num_cols) {\n // Only thread 0 allocates; result must reach ALL warps in the CTA.\n // __shfl_sync is warp-local — it cannot broadcast across warps.\n if (threadIdx.x == 0) {\n uint32_t smem_addr =\n static_cast(__cvta_generic_to_shared(&s_tmem_addr));\n asm volatile(\n \"tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;\"\n :\n : \"r\"(smem_addr), \"r\"(num_cols)\n );\n }\n __syncthreads(); // All warps now see s_tmem_addr\n return s_tmem_addr;\n}\n```\n\n- Only one thread (typically thread 0) issues the allocation.\n\n```cuda\n__device__ void tmem_dealloc(uint32_t tmem_addr, uint32_t num_cols) {\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;\"\n :\n : \"r\"(tmem_addr), \"r\"(num_cols)\n );\n }\n __syncthreads();\n}\n```"}, "after": {"statement": "TMEM has an explicit software-managed lifetime:\n\n1. Reserve shared memory for the 32-bit allocation result.\n2. Execute `tcgen05.alloc` collectively from one warp for `cta_group::1`, or from two warps—one per paired CTA—for `cta_group::2`.\n3. Synchronize consumers as required, then read the returned base `taddr` from shared memory.\n4. Use the allocation for MMA, copy, load, or store operations with the same CTA-group mode.\n5. Execute `tcgen05.dealloc` with the corresponding collective issue pattern.\n6. Deallocate every TMEM allocation before kernel exit.\n\nThe allocation operand counts columns. Legal allocation sizes are powers of two from 32 through 512 columns. Allocation can block until the requested TMEM is available; the ISA does not define folklore outcomes such as silent corruption for an invalid size."}, "reason": {"statement": "Remove unsafe inline PTX wrappers and state the collective contract directly.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit"]}} +{"path": "wiki/hardware/tmem.md", "before": {"statement": "- The returned `tmem_addr` is the base column index."}, "after": {"statement": "TMEM has an explicit software-managed lifetime:\n\n1. Reserve shared memory for the 32-bit allocation result.\n2. Execute `tcgen05.alloc` collectively from one warp for `cta_group::1`, or from two warps—one per paired CTA—for `cta_group::2`.\n3. Synchronize consumers as required, then read the returned base `taddr` from shared memory.\n4. Use the allocation for MMA, copy, load, or store operations with the same CTA-group mode.\n5. Execute `tcgen05.dealloc` with the corresponding collective issue pattern.\n6. Deallocate every TMEM allocation before kernel exit.\n\nThe allocation operand counts columns. Legal allocation sizes are powers of two from 32 through 512 columns. Allocation can block until the requested TMEM is available; the ISA does not define folklore outcomes such as silent corruption for an invalid size."}, "reason": {"statement": "Preserve the full address model.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory-addressing", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit"]}} +{"path": "wiki/hardware/tmem.md", "before": {"statement": "**Warning**: Failure to deallocate TMEM before CTA exit will leak memory and prevent subsequent CTAs from allocating, leading to hangs in persistent kernels.\n\n```cuda\n__global__ void persistent_gemm_kernel(/* ... */) {\n // 1. Allocate TMEM for accumulators\n uint32_t tmem_acc = tmem_alloc(256); // 256 cols for M=128, N=256\n\n while (has_more_tiles()) {\n // 2. Zero-initialize TMEM accumulator\n tmem_zero(tmem_acc, 256);\n\n // 3. Mainloop: accumulate K tiles\n for (int k = 0; k < K_tiles; ++k) {\n issue_tcgen05_mma(tmem_acc, smem_a[k], smem_b[k]);\n }\n\n // 4. Fence and read results\n asm volatile(\"tcgen05.fence::before_thread_sync;\");\n __syncthreads();\n\n // 5. Epilogue: read from TMEM, apply bias/activation, write to GMEM\n epilogue_from_tmem(tmem_acc, output);\n\n // 6. TMEM persists across tiles -- no need to reallocate\n }\n\n // 7. Deallocate before exit\n tmem_dealloc(tmem_acc, 256);\n}\n```"}, "after": {"statement": "TMEM has an explicit software-managed lifetime:\n\n1. Reserve shared memory for the 32-bit allocation result.\n2. Execute `tcgen05.alloc` collectively from one warp for `cta_group::1`, or from two warps—one per paired CTA—for `cta_group::2`.\n3. Synchronize consumers as required, then read the returned base `taddr` from shared memory.\n4. Use the allocation for MMA, copy, load, or store operations with the same CTA-group mode.\n5. Execute `tcgen05.dealloc` with the corresponding collective issue pattern.\n6. Deallocate every TMEM allocation before kernel exit.\n\nThe allocation operand counts columns. Legal allocation sizes are powers of two from 32 through 512 columns. Allocation can block until the requested TMEM is available; the ISA does not define folklore outcomes such as silent corruption for an invalid size.\n\nThe relevant mechanisms are distinct:\n\n- `tcgen05.commit` attaches completion of prior asynchronous MMA operations to an mbarrier. Wait on that barrier before consuming their results.\n- `tcgen05.wait::ld` and `tcgen05.wait::st` are the completion mechanisms for the corresponding asynchronous TMEM load/store operations.\n- `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` order tcgen05 operations around a documented execution-ordering handoff. A fence is not an MMA completion wait.\n- Producer buffers must remain live until the asynchronous operation that reads them has completed according to its instruction contract.\n\nCTA synchronization alone does not replace these completion operations."}, "reason": {"statement": "Remove a fabricated incomplete kernel and replace it with a normative lifecycle sequence.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-wait"]}} +{"path": "wiki/hardware/tmem.md", "before": {"statement": "```cuda\n// Store a register value to TMEM\n// Each thread writes to its own TMEM row at the specified column\n__device__ void tmem_store_f32(uint32_t tmem_col, float value) {\n asm volatile(\n \"tcgen05.st.sync.aligned.32x1b.x1.b32 [%0], {%1};\"\n :\n : \"r\"(tmem_col), \"f\"(value)\n );\n}\n\n// Vectorized store: 4 consecutive FP32 values\n__device__ void tmem_store_f32x4(uint32_t tmem_col, float4 values) {\n asm volatile(\n \"tcgen05.st.sync.aligned.32x1b.x4.b32 [%0], {%1, %2, %3, %4};\"\n :\n : \"r\"(tmem_col),\n \"f\"(values.x), \"f\"(values.y),\n \"f\"(values.z), \"f\"(values.w)\n );\n}\n```\n\n```cuda\n// Load a single FP32 from TMEM\n__device__ float tmem_load_f32(uint32_t tmem_col) {\n float result;\n asm volatile(\n \"tcgen05.ld.sync.aligned.32x1b.x1.b32 {%0}, [%1];\"\n : \"=f\"(result)\n : \"r\"(tmem_col)\n );\n return result;\n}\n\n// Vectorized load: 4 consecutive FP32 values\n__device__ float4 tmem_load_f32x4(uint32_t tmem_col) {\n float4 result;\n asm volatile(\n \"tcgen05.ld.sync.aligned.32x1b.x4.b32 {%0, %1, %2, %3}, [%4];\"\n : \"=f\"(result.x), \"=f\"(result.y),\n \"=f\"(result.z), \"=f\"(result.w)\n : \"r\"(tmem_col)\n );\n return result;\n}\n```\n\n```cuda\n// Zero-fill a range of TMEM columns\n__device__ void tmem_zero(uint32_t tmem_base_col, uint32_t num_cols) {\n // Each thread zeros its own row\n for (uint32_t c = 0; c < num_cols; c += 4) {\n float4 zero = make_float4(0.f, 0.f, 0.f, 0.f);\n tmem_store_f32x4(tmem_base_col + c, zero);\n }\n}\n```"}, "after": {"statement": "`tcgen05.ld` transfers TMEM into registers, and `tcgen05.st` transfers registers into TMEM. Their documented shapes include `.16x64b`, `.16x128b`, `.16x256b`, `.32x32b`, and `.16x32bx2`, with supported repetition qualifiers. There is no scalar `.32x1b` form.\n\n`tcgen05.cp` performs shaped shared-memory-to-TMEM copies. For example, PTX ISA 9.0 defines this exact form:\n\n```ptx\ntcgen05.cp.cta_group::1.128x256b [taddr], sdesc;\n```\n\nHere `taddr` is the TMEM destination and `sdesc` is the 64-bit shared-memory matrix descriptor. Follow the instruction's asynchronous ordering and completion rules before reusing the source or consuming the destination."}, "reason": {"statement": "Do not publish non-assembling executable-looking wrappers.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-cp"]}} +{"path": "wiki/hardware/tmem.md", "before": {"statement": "```cuda\n__global__ void double_buffered_gemm(/* ... */) {\n // Allocate two accumulator buffers\n uint32_t tmem_acc[2];\n tmem_acc[0] = tmem_alloc(256);\n tmem_acc[1] = tmem_alloc(256);\n\n int buf = 0;\n\n for (int tile = 0; tile < num_tiles; ++tile) {\n // Zero the current buffer\n tmem_zero(tmem_acc[buf], 256);\n\n // Mainloop: accumulate into current buffer\n for (int k = 0; k < K_tiles; ++k) {\n issue_tcgen05_mma(tmem_acc[buf], smem_a[k], smem_b[k]);\n }\n asm volatile(\"tcgen05.fence::before_thread_sync;\");\n __syncthreads();\n\n // If not the first tile, the *other* buffer's epilogue\n // was overlapped with this tile's MMA in the pipeline.\n\n // Start epilogue for current buffer\n // (can overlap with next tile's MMA on the other buffer)\n epilogue_from_tmem(tmem_acc[buf], output_tile[tile]);\n\n buf ^= 1; // Swap buffers\n }\n\n // Cleanup\n tmem_dealloc(tmem_acc[0], 256);\n tmem_dealloc(tmem_acc[1], 256);\n}\n```"}, "after": {"statement": "CUTLASS 4.5.0 exposes TMEM through `cutlass.utils.TmemAllocator` and layout-aware CuTe tensors. The official tutorial sequence is:\n\n1. create a `TmemAllocator` over shared storage for the allocation result;\n2. call `allocate(num_columns)` from the configured allocator warp;\n3. use `wait_for_alloc()` before other warps retrieve the address;\n4. obtain a typed pointer with `retrieve_ptr(dtype)`;\n5. bind that pointer to the MMA accumulator layout with `cute.make_tensor`;\n6. drain it with `tcgen05` copy atoms and the pipeline's completion protocol; and\n7. call `free(tmem_ptr)` before exit."}, "reason": {"statement": "Replace invented code with resource and synchronization requirements plus a pinned real example.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_6.py", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit"]}} +{"path": "wiki/hardware/tmem.md", "before": {"statement": "| 128x192 FP32 | 192 cols | 2 | 128 |"}, "after": {"statement": "All allocations share the SM's 512-column capacity. Some useful whole-allocation budgets are:\n\n| 128 columns | 4 | 0 |\n| 256 columns | 2 | 0 |\n| 512 columns | 1 | 0 |\n\nA logical need of 192 columns cannot be requested directly: reserve 256 columns, or suballocate that logical region within another legal power-of-two reservation. Two 256-column accumulator stages consume the entire capacity, so scale-factor tensors or other scratch state must fit inside those reservations or the pipeline must use fewer columns."}, "reason": {"statement": "List only allocatable budgets and distinguish logical suballocation from hardware allocation.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit"]}} {"path": "wiki/hardware/tmem.md", "before": {"statement": "From published Blackwell microbenchmarks:\n\n| End-to-end latency (cache miss) | ~420 cycles | ~30 cycles | ~4 cycles |\n| Bandwidth for large working sets | High (dedicated bus) | Medium | N/A (limited count) |\n| Best for | Multi-stage tensor pipelines | Single-shot small matrix | Scalar/vector ALU |\n\nTMEM is **not** a replacement for shared memory. Its strength is in serving as a dedicated accumulator buffer that eliminates register pressure for large MMA tiles. SMEM remains faster for small, frequently accessed data.\n\n5. **Assuming SMEM-like latency**: TMEM has ~420-cycle latency on cache miss vs ~30 cycles for SMEM. Do not use TMEM for low-latency random access patterns."}, "after": {"statement": null}, "reason": {"statement": "A stale, heterogeneous comparison cannot support general performance guidance.", "urls": ["https://arxiv.org/pdf/2512.02189v3", "https://arxiv.org/abs/2512.02189"]}} -{"path": "wiki/hardware/tmem.md", "before": {"statement": "```python\n# CuTe-DSL example: TMEM accumulator layout\n# 128 rows x 256 columns, FP32\ntmem_layout = Layout(\n shape=(128, 256),\n stride=(256, 1),\n memory_space=MemorySpace.TMEM\n)\n\n# Allocate TMEM accumulator\nacc = tmem_alloc(tmem_layout)\n\n# Issue MMA -- accumulator lives in TMEM\ntcgen05_mma(acc, smem_a, smem_b)\n\n# Fence and read\ntcgen05_fence()\nresult = tmem_load(acc)\n```"}, "after": {"statement": "See the pinned [CUTLASS 4.5.0 FP16 GEMM tutorial](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py) and [TMEM allocator implementation](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/python/CuTeDSL/cutlass/utils/tmem_allocator.py). APIs such as `Layout(memory_space=MemorySpace.TMEM)` and standalone `tcgen05_mma()` are not CUTLASS 4.5.0 interfaces."}, "reason": {"statement": "Link a complete version-pinned tutorial rather than fabricate a simplified API.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py"]}} -{"path": "wiki/hardware/tmem.md", "before": {"statement": "1. **Forgetting to fence**: Reading TMEM without `tcgen05.fence::before_thread_sync` produces undefined (stale) values.\n2. **Forgetting to deallocate**: In persistent kernels, TMEM must be freed before re-acquiring tiles. Otherwise, the next allocation will fail or hang.\n3. **Exceeding 512 columns**: Attempting to allocate more than the SM's total column budget silently corrupts data or causes a hang."}, "after": {"statement": "TMEM has an explicit software-managed lifetime:\n\n1. Reserve shared memory for the 32-bit allocation result.\n2. Execute `tcgen05.alloc` collectively from one warp for `cta_group::1`, or from two warps—one per paired CTA—for `cta_group::2`.\n3. Synchronize consumers as required, then read the returned base `taddr` from shared memory.\n4. Use the allocation for MMA, copy, load, or store operations with the same CTA-group mode.\n5. Execute `tcgen05.dealloc` with the corresponding collective issue pattern.\n6. Deallocate every TMEM allocation before kernel exit.\n\nThe allocation operand counts columns. Legal allocation sizes are powers of two from 32 through 512 columns. Allocation can block until the requested TMEM is available; the ISA does not define folklore outcomes such as silent corruption for an invalid size.\n\nAll allocations share the SM's 512-column capacity. Some useful whole-allocation budgets are:\n\n| 128 columns | 4 | 0 |\n| 256 columns | 2 | 0 |\n| 512 columns | 1 | 0 |\n\nA logical need of 192 columns cannot be requested directly: reserve 256 columns, or suballocate that logical region within another legal power-of-two reservation. Two 256-column accumulator stages consume the entire capacity, so scale-factor tensors or other scratch state must fit inside those reservations or the pipeline must use fewer columns.\n\nThe relevant mechanisms are distinct:\n\n- `tcgen05.commit` attaches completion of prior asynchronous MMA operations to an mbarrier. Wait on that barrier before consuming their results.\n- `tcgen05.wait::ld` and `tcgen05.wait::st` are the completion mechanisms for the corresponding asynchronous TMEM load/store operations.\n- `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` order tcgen05 operations around a documented execution-ordering handoff. A fence is not an MMA completion wait.\n- Producer buffers must remain live until the asynchronous operation that reads them has completed according to its instruction contract.\n\nCTA synchronization alone does not replace these completion operations."}, "reason": {"statement": "Replace folklore failure modes with normative obligations.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-wait"]}} -{"path": "wiki/hardware/tmem.md", "before": {"statement": "4. **Cross-warp reads**: A thread can only directly read/write TMEM rows mapped to its own lane. Accessing another warp's rows requires explicit shuffle or SMEM staging."}, "after": {"statement": "A TMEM address (`taddr`) is a 32-bit value:\n\n| 31:16 | TMEM lane |\n| 15:0 | TMEM column |\n\nThe lane is a TMEM data-path lane, not storage independently owned by a CUDA thread. `tcgen05.ld` and `tcgen05.st` are warp-collective. Under their four-warp access model, the warps cover these lane chunks:\n\n| 0 | 0-31 |\n| 1 | 32-63 |\n| 2 | 64-95 |\n| 3 | 96-127 |\n\nThe chosen load/store shape determines how values from that lane-column region map to each thread's register vector. Use the PTX shape-specific layout tables rather than treating a logical MxN accumulator as a universal row-major array owned one row per thread."}, "reason": {"statement": "State direct warp access restrictions and avoid a false communication primitive.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory-addressing"]}} +{"path": "wiki/hardware/tmem.md", "before": {"statement": "```python\n# CuTe-DSL example: TMEM accumulator layout\n# 128 rows x 256 columns, FP32\ntmem_layout = Layout(\n shape=(128, 256),\n stride=(256, 1),\n memory_space=MemorySpace.TMEM\n)\n\n# Allocate TMEM accumulator\nacc = tmem_alloc(tmem_layout)\n\n# Issue MMA -- accumulator lives in TMEM\ntcgen05_mma(acc, smem_a, smem_b)\n\n# Fence and read\ntcgen05_fence()\nresult = tmem_load(acc)\n```"}, "after": {"statement": "See the pinned [CUTLASS 4.5.0 FP16 GEMM tutorial](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py) and [TMEM allocator implementation](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/python/CuTeDSL/cutlass/utils/tmem_allocator.py). APIs such as `Layout(memory_space=MemorySpace.TMEM)` and standalone `tcgen05_mma()` are not CUTLASS 4.5.0 interfaces."}, "reason": {"statement": "Link a complete version-pinned tutorial rather than fabricate a simplified API.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py", "https://github.com/NVIDIA/cutlass/tree/e406c186f510a15091cce01f782020ceb7ba8eb5"]}} +{"path": "wiki/hardware/tmem.md", "before": {"statement": "1. **Forgetting to fence**: Reading TMEM without `tcgen05.fence::before_thread_sync` produces undefined (stale) values.\n2. **Forgetting to deallocate**: In persistent kernels, TMEM must be freed before re-acquiring tiles. Otherwise, the next allocation will fail or hang.\n3. **Exceeding 512 columns**: Attempting to allocate more than the SM's total column budget silently corrupts data or causes a hang."}, "after": {"statement": "TMEM has an explicit software-managed lifetime:\n\n1. Reserve shared memory for the 32-bit allocation result.\n2. Execute `tcgen05.alloc` collectively from one warp for `cta_group::1`, or from two warps—one per paired CTA—for `cta_group::2`.\n3. Synchronize consumers as required, then read the returned base `taddr` from shared memory.\n4. Use the allocation for MMA, copy, load, or store operations with the same CTA-group mode.\n5. Execute `tcgen05.dealloc` with the corresponding collective issue pattern.\n6. Deallocate every TMEM allocation before kernel exit.\n\nThe allocation operand counts columns. Legal allocation sizes are powers of two from 32 through 512 columns. Allocation can block until the requested TMEM is available; the ISA does not define folklore outcomes such as silent corruption for an invalid size.\n\nAll allocations share the SM's 512-column capacity. Some useful whole-allocation budgets are:\n\n| 128 columns | 4 | 0 |\n| 256 columns | 2 | 0 |\n| 512 columns | 1 | 0 |\n\nA logical need of 192 columns cannot be requested directly: reserve 256 columns, or suballocate that logical region within another legal power-of-two reservation. Two 256-column accumulator stages consume the entire capacity, so scale-factor tensors or other scratch state must fit inside those reservations or the pipeline must use fewer columns.\n\nThe relevant mechanisms are distinct:\n\n- `tcgen05.commit` attaches completion of prior asynchronous MMA operations to an mbarrier. Wait on that barrier before consuming their results.\n- `tcgen05.wait::ld` and `tcgen05.wait::st` are the completion mechanisms for the corresponding asynchronous TMEM load/store operations.\n- `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` order tcgen05 operations around a documented execution-ordering handoff. A fence is not an MMA completion wait.\n- Producer buffers must remain live until the asynchronous operation that reads them has completed according to its instruction contract.\n\nCTA synchronization alone does not replace these completion operations."}, "reason": {"statement": "Replace folklore failure modes with normative obligations.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-wait"]}} +{"path": "wiki/hardware/tmem.md", "before": {"statement": "4. **Cross-warp reads**: A thread can only directly read/write TMEM rows mapped to its own lane. Accessing another warp's rows requires explicit shuffle or SMEM staging."}, "after": {"statement": "A TMEM address (`taddr`) is a 32-bit value:\n\n| 31:16 | TMEM lane |\n| 15:0 | TMEM column |\n\nThe lane is a TMEM data-path lane, not storage independently owned by a CUDA thread. `tcgen05.ld` and `tcgen05.st` are warp-collective. Under their four-warp access model, the warps cover these lane chunks:\n\n| 0 | 0-31 |\n| 1 | 32-63 |\n| 2 | 64-95 |\n| 3 | 96-127 |\n\nThe chosen load/store shape determines how values from that lane-column region map to each thread's register vector. Use the PTX shape-specific layout tables rather than treating a logical MxN accumulator as a universal row-major array owned one row per thread."}, "reason": {"statement": "State direct warp access restrictions and avoid a false communication primitive.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory-addressing"]}} {"path": "wiki/hardware/mbarrier.md", "before": {"statement": "mbarriers are 64-bit shared memory primitives used for producer/consumer synchronization between asynchronous hardware units (TMA, tcgen05) and SM threads. Introduced on Hopper, essential for Blackwell warp-specialized kernels."}, "after": {"statement": "An mbarrier is an opaque, naturally aligned 64-bit object in shared memory. It synchronizes threads and can track asynchronous operations. The base instructions were introduced in PTX ISA 7.0 for `sm_80`; Hopper (`sm_90`) added transaction-count operations and cluster scope. Blackwell uses the same object for TMA transaction completion and for `tcgen05.commit` arrival-on completion.\n\nmbarriers are useful in warp-specialized pipelines, but they are a mechanism rather than a requirement for every such kernel."}, "reason": {"statement": "Separate the Ampere base primitive from Hopper transaction-count extensions and avoid a universal design mandate.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-init", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"]}} -{"path": "wiki/hardware/mbarrier.md", "before": {"statement": "```ptx\n// Initialize: set expected arrival count\nmbarrier.init.shared.b64 [mbar_addr], num_arrivals;\n\n// Producer: arrive on barrier (decrements expected count)\nmbarrier.arrive.shared.b64 _, [mbar_addr];\n\n// Producer (with byte expectation): used by TMA\nmbarrier.arrive.expect_tx.shared.b64 _, [mbar_addr], expected_bytes;\n// After this, TMA hardware completes the transaction by arriving with\n// the transferred byte count, and the mbarrier flips parity.\n\n// Consumer: wait for parity flip\nmbarrier.try_wait.parity.shared.b64 p, [mbar_addr], phase;\n@!p bra WAIT_LOOP;\n```"}, "after": {"statement": "For the current phase, an mbarrier tracks:\n\n- pending arrivals;\n- the expected arrival count for the next phase; and\n- a transaction count (`tx-count`) for outstanding asynchronous work.\n\nThe current phase completes only when **both** pending arrivals and tx-count reach zero. Completion atomically advances to the next phase and restores pending arrivals from the expected count. Before an arrival in the following phase, at least one `test_wait` or `try_wait` for the completed phase must have returned true.\n\nInitialization sets phase 0, initializes expected and pending arrivals to `count`, and sets tx-count to zero:\n\n```ptx\nmbarrier.init.shared::cta.b64 [bar], arrival_count;\n```\n\nInvalidate the object with `mbarrier.inval` before reusing its storage for another purpose or reinitializing an already-valid object."}, "reason": {"statement": "Preserve exact accounting semantics and avoid premature-completion bugs.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#phase-completion-of-the-mbarrier-object", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#contents-of-the-mbarrier-object"]}} +{"path": "wiki/hardware/mbarrier.md", "before": {"statement": "```ptx\n// Initialize: set expected arrival count\nmbarrier.init.shared.b64 [mbar_addr], num_arrivals;\n\n// Producer: arrive on barrier (decrements expected count)\nmbarrier.arrive.shared.b64 _, [mbar_addr];\n\n// Producer (with byte expectation): used by TMA\nmbarrier.arrive.expect_tx.shared.b64 _, [mbar_addr], expected_bytes;\n// After this, TMA hardware completes the transaction by arriving with\n// the transferred byte count, and the mbarrier flips parity.\n\n// Consumer: wait for parity flip\nmbarrier.try_wait.parity.shared.b64 p, [mbar_addr], phase;\n@!p bra WAIT_LOOP;\n```"}, "after": {"statement": "For the current phase, an mbarrier tracks:\n\n- pending arrivals;\n- the expected arrival count for the next phase; and\n- a transaction count (`tx-count`) for outstanding asynchronous work.\n\nThe current phase completes only when **both** pending arrivals and tx-count reach zero. Completion atomically advances to the next phase and restores pending arrivals from the expected count. Before an arrival in the following phase, at least one `test_wait` or `try_wait` for the completed phase must have returned true.\n\nInitialization sets phase 0, initializes expected and pending arrivals to `count`, and sets tx-count to zero:\n\n```ptx\nmbarrier.init.shared::cta.b64 [bar], arrival_count;\n```\n\nInvalidate the object with `mbarrier.inval` before reusing its storage for another purpose or reinitializing an already-valid object."}, "reason": {"statement": "Preserve exact accounting semantics and avoid premature-completion bugs.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-phase-completion", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-contents"]}} {"path": "wiki/hardware/mbarrier.md", "before": {"statement": "```cuda\nint phase = 0;\nfor (int k = 0; k < num_iterations; k++) {\n int stage = k % NUM_STAGES;\n\n // Wait for this stage's producer to complete\n mbarrier_wait_parity(&mbar[stage], phase);\n phase ^= 1; // Flip for next use of this stage slot\n\n // Use data...\n consume_data(stage);\n\n // Arrive on next-stage barrier when done\n mbarrier_arrive(&buffer_free[stage]);\n}\n```"}, "after": {"statement": "These operations affect different parts of the state:\n\n| `mbarrier.arrive` | Decrements pending arrivals; returns a phase state token for a CTA-shared object. |\n| `mbarrier.arrive.expect_tx` | Performs an arrival and increments tx-count by `txCount`. |\n| `mbarrier.expect_tx` | Increments tx-count without an arrival. |\n| `mbarrier.complete_tx` | Decrements tx-count; it is not an arrival. |\n| `mbarrier.test_wait` / `try_wait` | Tests completion of the phase identified by a state token or parity. |\n\nRepresentative CTA-shared forms from PTX ISA 9.0 are:\n\n```ptx\nmbarrier.arrive.shared::cta.b64 state, [bar];\nmbarrier.arrive.expect_tx.shared::cta.b64 state, [bar], tx_count;\nmbarrier.try_wait.parity.acquire.cta.shared::cta.b64 ready, [bar], phase_parity;\n```\n\n`try_wait` can suspend temporarily and must still be retried until its predicate is true. Use acquire/release semantics appropriate to the producer-consumer handoff; `.relaxed` does not provide memory-ordering or visibility guarantees."}, "reason": {"statement": "Remove subtly unsafe pseudocode and give a per-stage reuse rule instead.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive"]}} {"path": "wiki/hardware/mbarrier.md", "before": {"statement": "```cuda\n// Producer warp issues TMA with mbarrier target\nif (lane_id == 0) {\n uint32_t tx_bytes = TILE_A_BYTES + TILE_B_BYTES;\n mbarrier_arrive_expect_tx(&mbar[stage], tx_bytes);\n\n // TMA completion will fire the mbarrier automatically\n cp_async_bulk_tensor_2d(smem_A[stage], global_desc, x, y, &mbar[stage]);\n cp_async_bulk_tensor_2d(smem_B[stage], global_desc, x, y, &mbar[stage]);\n}\n// Do NOT manually arrive after TMA issue - it races with hardware\n```\n\n3. **Manual arrive after async issue**: TMA/tcgen05 hardware arrives on completion — extra manual arrive causes double-count"}, "after": {"statement": "Parity is the low bit of an individual mbarrier object's phase: even phases use 0 and odd phases use 1. A parity wait can refer only to the current or immediately preceding phase, so software must track phase for the entire lifetime of that object.\n\nFor an N-stage ring, track state per stage. When stage `s` is reused, pass the parity expected for `bar[s]`; toggle that stage's parity only after its phase completes. One global bit toggled on every loop iteration is generally wrong because different stage barriers advance independently.\n\nUsing the opaque state returned by `mbarrier.arrive` is an alternative when the same participant can carry that token to its wait."}, "reason": {"statement": "Replace fabricated code with an exact accounting sequence and vendor grammar link.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait"]}} {"path": "wiki/hardware/mbarrier.md", "before": {"statement": "4. **Cross-cluster mbarriers**: Use `shared::cluster` qualifier for cluster-level sync (2-SM cooperative MMA)"}, "after": {"statement": "For a common TMA-load phase initialized with one pending arrival:\n\n1. The producer executes `mbarrier.arrive.expect_tx` with the sum of bytes that all TMA operations in this phase will report.\n2. The producer issues the TMA operations with `.mbarrier::complete_tx::bytes` and the same barrier.\n3. Each TMA completion performs `complete-tx` for the bytes it copied.\n4. The consumer waits for the phase to complete before reading the destination.\n\nThe `arrive.expect_tx` operation accounts for the software arrival and establishes the expected byte total. TMA hardware does **not** perform a second arrival: it decrements tx-count through complete-tx. Therefore, do not add an unmatched `mbarrier.arrive`, and do not manually simulate TMA's complete-tx. Either mistake can advance or strand the wrong phase.\n\nThe expected byte total, barrier address, destination ownership, and exact `cp.async.bulk.tensor` form must match. Follow the complete instruction grammar in the PTX ISA rather than using placeholder helper calls."}, "reason": {"statement": "Teach object location, address mapping, operation support, and synchronization scope separately.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive"]}} -{"path": "wiki/hardware/nvfp4.md", "before": {"statement": "| `tcgen05.mma.mxf4.block_scale` | MX FP4 with block scaling | **4×** |\n| `tcgen05.mma.mxf4nvf4.block_scale` | NVFP4 + MX FP4 flexible scaling | **4×** |"}, "after": {"statement": "PTX ISA 9.0 provides the packed conversion form:\n\n```ptx\ncvt.rn.f16x2.e2m1x2 d, a;\n```\n\nHere `a` is a byte-sized packed pair and `d` is a 32-bit `f16x2` result. PTX also permits `mov.b32` to unpack a 32-bit scalar into four byte-sized vector destinations when the declarations satisfy its type rules. Neither syntax establishes that one packing strategy is faster than masks and shifts; inspect generated machine code and benchmark the target GPU."}, "reason": {"statement": "Give exact distinguishing qualifiers and remove an unscoped performance number.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov"]}} -{"path": "wiki/hardware/nvfp4.md", "before": {"statement": "```ptx\n// Convert two FP4 values to two FP16 values\ncvt.rn.f16x2.e2m1x2 result, packed_fp4;\n\n// Byte unpacking (faster than bitwise extraction)\nmov.b32 {tmp0, tmp1, tmp2, tmp3}, packed_data;\n```"}, "after": {"statement": "There is no architecture-independent \"4x versus Hopper\" result for these instructions. Hopper has no native FP4 tensor-core path, so any comparison depends on the emulation or higher-precision baseline as well as GPU SKU, clocks, matrix shape, layouts, scale preparation, epilogue, and achieved occupancy. Record those conditions with any throughput number."}, "reason": {"statement": "Retain the exact packed conversion form, state mov's typed preconditions, and require measurement for packing-strategy performance.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} -{"path": "wiki/hardware/nvfp4.md", "before": {"statement": "| Quantization error | Lower | Higher |"}, "after": {"statement": "In PTX ISA 9.0, the relevant `tcgen05.mma` combinations for E2M1 inputs include:\n\n| `.kind::mxf4.block_scale.block32` | 32 | `.ue8m0` | `.scale_vec::2X` |\n| `.kind::mxf4nvf4.block_scale.block16` | 16 | `.ue4m3` | `.scale_vec::4X` |\n\nThe `.kind::mxf4nvf4` family also admits documented UE8M0 modes, so the kind name by itself does not select the NVFP4 recipe. Use the complete block and scale-vector qualifiers and follow the type-combination tables. These forms target architecture-specific Blackwell targets such as `sm_100a`; a kernel using them must use the matching compile and runtime target."}, "reason": {"statement": "State the representational tradeoff without claiming an unconditional empirical ordering.", "urls": ["https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/hardware/nvfp4.md", "before": {"statement": "| `tcgen05.mma.mxf4.block_scale` | MX FP4 with block scaling | **4×** |\n| `tcgen05.mma.mxf4nvf4.block_scale` | NVFP4 + MX FP4 flexible scaling | **4×** |"}, "after": {"statement": "PTX ISA 9.0 provides the packed conversion form:\n\n```ptx\ncvt.rn.f16x2.e2m1x2 d, a;\n```\n\nHere `a` is a byte-sized packed pair and `d` is a 32-bit `f16x2` result. PTX also permits `mov.b32` to unpack a 32-bit scalar into four byte-sized vector destinations when the declarations satisfy its type rules. Neither syntax establishes that one packing strategy is faster than masks and shifts; inspect generated machine code and benchmark the target GPU."}, "reason": {"statement": "Give exact distinguishing qualifiers and remove an unscoped performance number.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov"]}} +{"path": "wiki/hardware/nvfp4.md", "before": {"statement": "```ptx\n// Convert two FP4 values to two FP16 values\ncvt.rn.f16x2.e2m1x2 result, packed_fp4;\n\n// Byte unpacking (faster than bitwise extraction)\nmov.b32 {tmp0, tmp1, tmp2, tmp3}, packed_data;\n```"}, "after": {"statement": "There is no architecture-independent \"4x versus Hopper\" result for these instructions. Hopper has no native FP4 tensor-core path, so any comparison depends on the emulation or higher-precision baseline as well as GPU SKU, clocks, matrix shape, layouts, scale preparation, epilogue, and achieved occupancy. Record those conditions with any throughput number."}, "reason": {"statement": "Retain the exact packed conversion form, state mov's typed preconditions, and require measurement for packing-strategy performance.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma"]}} +{"path": "wiki/hardware/nvfp4.md", "before": {"statement": "| Quantization error | Lower | Higher |"}, "after": {"statement": "In PTX ISA 9.0, the relevant `tcgen05.mma` combinations for E2M1 inputs include:\n\n| `.kind::mxf4.block_scale.block32` | 32 | `.ue8m0` | `.scale_vec::2X` |\n| `.kind::mxf4nvf4.block_scale.block16` | 16 | `.ue4m3` | `.scale_vec::4X` |\n\nThe `.kind::mxf4nvf4` family also admits documented UE8M0 modes, so the kind name by itself does not select the NVFP4 recipe. Use the complete block and scale-vector qualifiers and follow the type-combination tables. These forms target architecture-specific Blackwell targets such as `sm_100a`; a kernel using them must use the matching compile and runtime target."}, "reason": {"statement": "State the representational tradeoff without claiming an unconditional empirical ordering.", "urls": ["https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma"]}} {"path": "wiki/hardware/pdl-gdc.md", "before": {"statement": "```cuda\n// Primary kernel signals near completion\ncudaGridDependencySynchronize(); // or PTX equivalent\n\n// Secondary kernel can start overlapping with primary's tail\n// Enabled by default on SM100 (opt-in on SM90)\n```"}, "after": {"statement": "This minimal skeleton shows where each operation belongs; real kernels put independent and dependent work at the indicated points and add normal error checking:\n\n```cuda\n#include \n\n__global__ void primary() {\n // Produce everything needed before the secondary may be launched.\n if (threadIdx.x == 0) {\n cudaTriggerProgrammaticLaunchCompletion();\n }\n // Primary tail that does not change data consumed by the secondary.\n}\n\n__global__ void secondary() {\n // Independent preamble may execute early.\n cudaGridDependencySynchronize();\n // Dependent reads begin only after this wait.\n}\n\nvoid launch_pdl(dim3 grid, dim3 block, cudaStream_t stream) {\n cudaLaunchConfig_t cfg{};\n cfg.gridDim = grid;\n cfg.blockDim = block;\n cfg.stream = stream;\n\n cudaLaunchAttribute attr{};\n attr.id = cudaLaunchAttributeProgrammaticStreamSerialization;\n attr.val.programmaticStreamSerializationAllowed = 1;\n cfg.attrs = &attr;\n cfg.numAttrs = 1;\n\n primary<<>>();\n cudaLaunchKernelEx(&cfg, secondary);\n}\n```\n\nDo not place `cudaGridDependencySynchronize()` in the primary: it is the secondary-side wait, not the launch trigger. Do not replace the wait with an ordinary memory fence. PDL's wait supplies both prerequisite-grid completion and visibility for its dependent work."}, "reason": {"statement": "Replace the reversed, incomplete example with the three-part protocol.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#programmatic-dependent-launch-and-synchronization", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-griddepcontrol"]}} {"path": "wiki/hardware/pdl-gdc.md", "before": {"statement": "On SM100, PDL is **enabled by default** — no opt-in needed. This means:"}, "after": {"statement": "Programmatic Dependent Launch (PDL) lets a secondary grid in the same CUDA stream become eligible to start before its prerequisite primary grid completes. It is available on compute capability 9.0 and later, including Hopper and Blackwell.\n\nPDL creates an **opportunity** for overlap; it does not guarantee that the grids execute concurrently. The useful overlap is normally the primary's work after its launch trigger and the secondary's independent preamble before its dependency wait.\n\nThe CUDA protocol has three required roles:\n\n1. Every primary CTA either executes `cudaTriggerProgrammaticLaunchCompletion()` or exits. After all CTAs satisfy that condition, the driver may schedule the secondary grid.\n2. The host launches the secondary in the same stream with `cudaLaunchAttributeProgrammaticStreamSerialization` and `programmaticStreamSerializationAllowed = 1`.\n3. Every secondary thread waits with `cudaGridDependencySynchronize()` before it consumes prerequisite results. The wait completes after the prerequisite grids finish and their memory operations are visible.\n\nIf the primary does not explicitly trigger, its CTAs implicitly satisfy the trigger only as they exit. Omitting the explicit trigger is correct but removes the intended primary-tail overlap.\n\nBlackwell does not make arbitrary back-to-back launches overlap automatically. CUDA applications still opt the secondary launch into programmatic stream serialization and implement the device-side trigger/wait protocol.\n\nCUTLASS has a separate build choice. In CUTLASS 4.5.0, the CMake option `CUTLASS_ENABLE_GDC_FOR_SM100` defaults to `ON`, while the SM90 option is opt-in. That default only enables eligible CUTLASS code to emit its GDC wrappers; it is not a device-wide CUDA default and can be overridden by the build."}, "reason": {"statement": "Separate hardware availability, CUDA launch opt-in, and one library's build default.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#programmatic-dependent-launch-and-synchronization", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/CMakeLists.txt#L463-L474"]}} {"path": "wiki/hardware/pdl-gdc.md", "before": {"statement": "- Back-to-back kernel launches naturally overlap\n- Memory fences ensure correctness for dependent data\n- Reduces kernel launch gaps in compute-heavy pipelines"}, "after": {"statement": "Programmatic Dependent Launch (PDL) lets a secondary grid in the same CUDA stream become eligible to start before its prerequisite primary grid completes. It is available on compute capability 9.0 and later, including Hopper and Blackwell.\n\nPDL creates an **opportunity** for overlap; it does not guarantee that the grids execute concurrently. The useful overlap is normally the primary's work after its launch trigger and the secondary's independent preamble before its dependency wait.\n\nThe CUDA protocol has three required roles:\n\n1. Every primary CTA either executes `cudaTriggerProgrammaticLaunchCompletion()` or exits. After all CTAs satisfy that condition, the driver may schedule the secondary grid.\n2. The host launches the secondary in the same stream with `cudaLaunchAttributeProgrammaticStreamSerialization` and `programmaticStreamSerializationAllowed = 1`.\n3. Every secondary thread waits with `cudaGridDependencySynchronize()` before it consumes prerequisite results. The wait completes after the prerequisite grids finish and their memory operations are visible.\n\nIf the primary does not explicitly trigger, its CTAs implicitly satisfy the trigger only as they exit. Omitting the explicit trigger is correct but removes the intended primary-tail overlap.\n\nThis minimal skeleton shows where each operation belongs; real kernels put independent and dependent work at the indicated points and add normal error checking:\n\n```cuda\n#include \n\n__global__ void primary() {\n // Produce everything needed before the secondary may be launched.\n if (threadIdx.x == 0) {\n cudaTriggerProgrammaticLaunchCompletion();\n }\n // Primary tail that does not change data consumed by the secondary.\n}\n\n__global__ void secondary() {\n // Independent preamble may execute early.\n cudaGridDependencySynchronize();\n // Dependent reads begin only after this wait.\n}\n\nvoid launch_pdl(dim3 grid, dim3 block, cudaStream_t stream) {\n cudaLaunchConfig_t cfg{};\n cfg.gridDim = grid;\n cfg.blockDim = block;\n cfg.stream = stream;\n\n cudaLaunchAttribute attr{};\n attr.id = cudaLaunchAttributeProgrammaticStreamSerialization;\n attr.val.programmaticStreamSerializationAllowed = 1;\n cfg.attrs = &attr;\n cfg.numAttrs = 1;\n\n primary<<>>();\n cudaLaunchKernelEx(&cfg, secondary);\n}\n```\n\nDo not place `cudaGridDependencySynchronize()` in the primary: it is the secondary-side wait, not the launch trigger. Do not replace the wait with an ordinary memory fence. PDL's wait supplies both prerequisite-grid completion and visibility for its dependent work.\n\nCUDA lowers the two device roles to Grid Dependency Control instructions:\n\n```ptx\n// Primary CTA: makes designated dependent grids eligible after every CTA\n// has issued this instruction or completed.\ngriddepcontrol.launch_dependents;\n\n// Secondary thread: waits for in-flight prerequisite grids and visibility.\ngriddepcontrol.wait;\n```\n\n`griddepcontrol` was introduced in PTX ISA 7.8 and requires `sm_90` or newer. Repeating `launch_dependents` within one CTA has no additional effect after that CTA's first invocation. If a prerequisite uses `launch_dependents`, its dependent must use `griddepcontrol.wait` or an equivalent CUDA dependency wait for correct execution."}, "reason": {"statement": "State the exact ordering contract and make overlap conditional.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#programmatic-dependent-launch-and-synchronization", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-griddepcontrol"]}} {"path": "wiki/hardware/pdl-gdc.md", "before": {"statement": "- Chains of small kernels (e.g., MoE dispatch → compute → combine)\n- Pipeline-parallel training with many sequential kernel launches\n- Reduces overall wall-clock time without code changes on Blackwell"}, "after": {"statement": "PDL can help only when all of these conditions hold:\n\n- the secondary has enough independent preamble to overlap;\n- the primary has useful tail work after every CTA reaches the trigger;\n- the two grids have enough simultaneous resource headroom; and\n- the saved launch/serialization time exceeds the protocol and occupancy costs.\n\nSmall-kernel chains, GEMM/epilogue sequences, and pipeline-parallel stages are candidates, not guaranteed wins. Profile an explicit non-PDL baseline and record GPU, clocks, launch shapes, stream/graph configuration, input sizes, warmup, repetitions, and the observed overlap timeline. Never rely on overlap for forward progress; CUDA documents it as opportunistic and warns that such reliance can deadlock."}, "reason": {"statement": "Replace categorical workload promises with a decision and measurement checklist.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#programmatic-dependent-launch-and-synchronization"]}} -{"path": "wiki/hardware/tma.md", "before": {"statement": "The Tensor Memory Accelerator (TMA) is a hardware unit that performs **asynchronous bulk data transfers** between global memory and shared memory. First introduced on Hopper (SM90), TMA carries forward to Blackwell (SM100) with stricter requirements for tcgen05 compatibility.\n\n| Swizzle modes | None, 32B, 64B, 128B (128B required for tcgen05) |\n\nOn Blackwell, `tcgen05.mma` requires operands in **128-byte swizzled** SMEM layout. If TMA loads data without 128B swizzling, the MMA will produce incorrect results.\n\n```cuda\n// CORRECT for Blackwell tcgen05:\nCUtensorMap desc = create_tma_descriptor_2d(ptr, M, N, 128, 64, 128);\n// swizzle=128 ^^^\n\n// WRONG for tcgen05 (will silently produce garbage):\nCUtensorMap desc = create_tma_descriptor_2d(ptr, M, N, 128, 64, 0);\n// swizzle=0 ^^^\n```\n\nAll TMA loads feeding `tcgen05.mma` must use 128-byte swizzling. The swizzle pattern rearranges bytes within each 128-byte line to match the tensor core's internal data layout:\n\n| Always use 128B swizzle on Blackwell | Non-128B swizzle produces incorrect tcgen05 results |"}, "after": {"statement": "TMA supports no swizzle and multiple swizzled shared-memory layouts, including 32B, 64B, and 128B spans plus newer variants for selected types. Swizzling rearranges chunks across shared-memory banks. The consumer must address the matching logical layout; the exact mapping also depends on the documented shared-memory base-offset rule.\n\nThere is no universal rule that every Blackwell or `tcgen05.mma` input uses 128B swizzling. The TMA tensor-map swizzle, destination base alignment, leading dimension, and the tcgen05 shared-memory descriptor must describe the **same** legal layout. PTX defines no-, 32B-, 64B-, and 128B-swizzled tcgen05 descriptors with kind-, type-, shape-, and layout-specific constraints.\n\nA matched swizzle can reduce or remove bank conflicts for a particular access pattern. It does not make every possible consumer access conflict-free."}, "reason": {"statement": "Make swizzle selection descriptor- and shape-specific and remove the invented failure mode.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-layout", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"]}} +{"path": "wiki/hardware/tma.md", "before": {"statement": "The Tensor Memory Accelerator (TMA) is a hardware unit that performs **asynchronous bulk data transfers** between global memory and shared memory. First introduced on Hopper (SM90), TMA carries forward to Blackwell (SM100) with stricter requirements for tcgen05 compatibility.\n\n| Swizzle modes | None, 32B, 64B, 128B (128B required for tcgen05) |\n\nOn Blackwell, `tcgen05.mma` requires operands in **128-byte swizzled** SMEM layout. If TMA loads data without 128B swizzling, the MMA will produce incorrect results.\n\n```cuda\n// CORRECT for Blackwell tcgen05:\nCUtensorMap desc = create_tma_descriptor_2d(ptr, M, N, 128, 64, 128);\n// swizzle=128 ^^^\n\n// WRONG for tcgen05 (will silently produce garbage):\nCUtensorMap desc = create_tma_descriptor_2d(ptr, M, N, 128, 64, 0);\n// swizzle=0 ^^^\n```\n\nAll TMA loads feeding `tcgen05.mma` must use 128-byte swizzling. The swizzle pattern rearranges bytes within each 128-byte line to match the tensor core's internal data layout:\n\n| Always use 128B swizzle on Blackwell | Non-128B swizzle produces incorrect tcgen05 results |"}, "after": {"statement": "TMA supports no swizzle and multiple swizzled shared-memory layouts, including 32B, 64B, and 128B spans plus newer variants for selected types. Swizzling rearranges chunks across shared-memory banks. The consumer must address the matching logical layout; the exact mapping also depends on the documented shared-memory base-offset rule.\n\nThere is no universal rule that every Blackwell or `tcgen05.mma` input uses 128B swizzling. The TMA tensor-map swizzle, destination base alignment, leading dimension, and the tcgen05 shared-memory descriptor must describe the **same** legal layout. PTX defines no-, 32B-, 64B-, and 128B-swizzled tcgen05 descriptors with kind-, type-, shape-, and layout-specific constraints.\n\nA matched swizzle can reduce or remove bank conflicts for a particular access pattern. It does not make every possible consumer access conflict-free."}, "reason": {"statement": "Make swizzle selection descriptor- and shape-specific and remove the invented failure mode.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-shared-memory-layout-swizzling", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"]}} {"path": "wiki/hardware/tma.md", "before": {"statement": "TMA offloads data movement from CUDA cores entirely -- a single thread issues the transfer, and the TMA hardware engine handles the multi-dimensional copy, address calculation, out-of-bounds clamping, and format conversion.\n\n| Format conversion | FP32<->BF16, FP32<->FP16 during transfer |"}, "after": {"statement": "TMA is the descriptor-driven `cp.async.bulk.tensor` facility introduced for Hopper (`sm_90`) and retained on Blackwell. One thread can issue a non-blocking rank-1 through rank-5 tensor copy while the hardware performs multidimensional address traversal and moves the tile.\n\nThe principal copy directions and completion mechanisms are:\n\n| Global to shared | Issuing CTA or a CTA in its cluster | One masked instruction can multicast to selected CTAs | mbarrier `complete_tx` in bytes |\n| Shared to global | Issuing CTA's shared memory to a tensor map | Scatter modes are available on supported Blackwell targets | Bulk async group: issue, commit, wait |\n\nTMA copies the datatype represented by the tensor map. It does not provide a general FP32-to/from-FP16 or BF16 conversion step. For a tiled global-to-shared load, out-of-bounds elements are filled according to the supported tensor-map policy rather than clamped to an edge coordinate.\n\nA tensor map is opaque and is accessed through the tensor-map proxy. `cuTensorMapEncodeTiled` describes:\n\n- datatype and global base address;\n- rank, global dimensions, and byte strides;\n- traversal box dimensions and element strides;\n- interleave and shared-memory swizzle;\n- L2-promotion hint; and\n- out-of-bounds fill policy.\n\nFor the ordinary non-interleaved tiled path in CUDA Driver API 13.0.97, important constraints include:\n\n- the `CUtensorMap` output object is 64-byte aligned;\n- the global base and byte strides satisfy the documented alignment rules, commonly at least 16 bytes for the basic types/path;\n- every `boxDim` entry is from 1 through 256 elements;\n- `boxDim[0] * element_size` is a multiple of 16 bytes;\n- every element stride is from 1 through 8; and\n- when swizzling is enabled, the inner box in bytes does not exceed the selected swizzle span.\n\nDatatype, interleave, sub-byte, and architecture-specific modes add further restrictions. Always check the encoder's `CUresult`; never use the output after an encoding failure.\n\nHost encoding is common, but it is not the only Blackwell path. CUDA also documents device-side tensor-map construction and modification on Blackwell. A map modified through the generic proxy must be published to the tensor-map proxy with the required `fence.proxy.tensormap::generic` sequence before a TMA operation consumes it."}, "reason": {"statement": "Describe representation-preserving copies and OOB fill precisely.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#asynchronous-data-copies-using-the-tensor-memory-accelerator-tma", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#encoding-a-tensor-map-on-device"]}} -{"path": "wiki/hardware/tma.md", "before": {"statement": "| Max transfer size | Up to 256 bytes per element, tiles up to 128x256 |\n\n| Synchronization | mbarrier-based (arrive/wait) |"}, "after": {"statement": "TMA is the descriptor-driven `cp.async.bulk.tensor` facility introduced for Hopper (`sm_90`) and retained on Blackwell. One thread can issue a non-blocking rank-1 through rank-5 tensor copy while the hardware performs multidimensional address traversal and moves the tile.\n\nThe principal copy directions and completion mechanisms are:\n\n| Global to shared | Issuing CTA or a CTA in its cluster | One masked instruction can multicast to selected CTAs | mbarrier `complete_tx` in bytes |\n| Shared to global | Issuing CTA's shared memory to a tensor map | Scatter modes are available on supported Blackwell targets | Bulk async group: issue, commit, wait |\n\nTMA copies the datatype represented by the tensor map. It does not provide a general FP32-to/from-FP16 or BF16 conversion step. For a tiled global-to-shared load, out-of-bounds elements are filled according to the supported tensor-map policy rather than clamped to an edge coordinate.\n\nA tensor map is opaque and is accessed through the tensor-map proxy. `cuTensorMapEncodeTiled` describes:\n\n- datatype and global base address;\n- rank, global dimensions, and byte strides;\n- traversal box dimensions and element strides;\n- interleave and shared-memory swizzle;\n- L2-promotion hint; and\n- out-of-bounds fill policy.\n\nFor the ordinary non-interleaved tiled path in CUDA Driver API 13.0.97, important constraints include:\n\n- the `CUtensorMap` output object is 64-byte aligned;\n- the global base and byte strides satisfy the documented alignment rules, commonly at least 16 bytes for the basic types/path;\n- every `boxDim` entry is from 1 through 256 elements;\n- `boxDim[0] * element_size` is a multiple of 16 bytes;\n- every element stride is from 1 through 8; and\n- when swizzling is enabled, the inner box in bytes does not exceed the selected swizzle span.\n\nDatatype, interleave, sub-byte, and architecture-specific modes add further restrictions. Always check the encoder's `CUresult`; never use the output after an encoding failure.\n\nHost encoding is common, but it is not the only Blackwell path. CUDA also documents device-side tensor-map construction and modification on Blackwell. A map modified through the generic proxy must be published to the tensor-map proxy with the required `fence.proxy.tensormap::generic` sequence before a TMA operation consumes it.\n\nPTX ISA 9.0 defines this representative 2D CTA-local form:\n\n```ptx\ncp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes\n [dst_smem], [tensor_map, {x, y}], [full_barrier];\n```\n\nThe instruction is non-blocking. A common one-producer phase with one or more loads uses this accounting:\n\n1. Initialize and publish the stage's mbarrier with the intended pending-arrival count.\n2. The producer performs one `mbarrier.arrive.expect_tx` for its software arrival and the sum of bytes that all loads in this phase will complete.\n3. Issue the TMA loads against that barrier.\n4. Each completed load performs `complete_tx` for its copied byte count; it does **not** perform another arrival.\n5. Consumers wait with the correct state token or per-stage parity and acquire semantics before reading the destination.\n\nThe phase completes only after pending arrivals and tx-count are both zero. If a design performs multiple software arrivals instead, its initialized count must match them exactly. See [mbarrier](mbarrier.md) for lifecycle, phase, and memory-ordering rules.\n\nA tensor store uses bulk-group completion rather than the load's mbarrier protocol:\n\n```ptx\ncp.async.bulk.tensor.2d.global.shared::cta.bulk_group\n [tensor_map, {x, y}], [src_smem];\ncp.async.bulk.commit_group;\ncp.async.bulk.wait_group 0;\n```\n\nThe wait may be delayed to overlap independent work. It must occur before the issuing thread reuses the source shared memory or before code that requires store completion. `commit_group` creates the group; it is not itself a completion wait."}, "reason": {"statement": "List parameter constraints and direction-specific completion separately.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#asynchronous-data-copies-using-the-tensor-memory-accelerator-tma", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#encoding-a-tensor-map-on-device", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#phase-completion-of-the-mbarrier-object"]}} +{"path": "wiki/hardware/tma.md", "before": {"statement": "| Max transfer size | Up to 256 bytes per element, tiles up to 128x256 |\n\n| Synchronization | mbarrier-based (arrive/wait) |"}, "after": {"statement": "TMA is the descriptor-driven `cp.async.bulk.tensor` facility introduced for Hopper (`sm_90`) and retained on Blackwell. One thread can issue a non-blocking rank-1 through rank-5 tensor copy while the hardware performs multidimensional address traversal and moves the tile.\n\nThe principal copy directions and completion mechanisms are:\n\n| Global to shared | Issuing CTA or a CTA in its cluster | One masked instruction can multicast to selected CTAs | mbarrier `complete_tx` in bytes |\n| Shared to global | Issuing CTA's shared memory to a tensor map | Scatter modes are available on supported Blackwell targets | Bulk async group: issue, commit, wait |\n\nTMA copies the datatype represented by the tensor map. It does not provide a general FP32-to/from-FP16 or BF16 conversion step. For a tiled global-to-shared load, out-of-bounds elements are filled according to the supported tensor-map policy rather than clamped to an edge coordinate.\n\nA tensor map is opaque and is accessed through the tensor-map proxy. `cuTensorMapEncodeTiled` describes:\n\n- datatype and global base address;\n- rank, global dimensions, and byte strides;\n- traversal box dimensions and element strides;\n- interleave and shared-memory swizzle;\n- L2-promotion hint; and\n- out-of-bounds fill policy.\n\nFor the ordinary non-interleaved tiled path in CUDA Driver API 13.0.97, important constraints include:\n\n- the `CUtensorMap` output object is 64-byte aligned;\n- the global base and byte strides satisfy the documented alignment rules, commonly at least 16 bytes for the basic types/path;\n- every `boxDim` entry is from 1 through 256 elements;\n- `boxDim[0] * element_size` is a multiple of 16 bytes;\n- every element stride is from 1 through 8; and\n- when swizzling is enabled, the inner box in bytes does not exceed the selected swizzle span.\n\nDatatype, interleave, sub-byte, and architecture-specific modes add further restrictions. Always check the encoder's `CUresult`; never use the output after an encoding failure.\n\nHost encoding is common, but it is not the only Blackwell path. CUDA also documents device-side tensor-map construction and modification on Blackwell. A map modified through the generic proxy must be published to the tensor-map proxy with the required `fence.proxy.tensormap::generic` sequence before a TMA operation consumes it.\n\nPTX ISA 9.0 defines this representative 2D CTA-local form:\n\n```ptx\ncp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes\n [dst_smem], [tensor_map, {x, y}], [full_barrier];\n```\n\nThe instruction is non-blocking. A common one-producer phase with one or more loads uses this accounting:\n\n1. Initialize and publish the stage's mbarrier with the intended pending-arrival count.\n2. The producer performs one `mbarrier.arrive.expect_tx` for its software arrival and the sum of bytes that all loads in this phase will complete.\n3. Issue the TMA loads against that barrier.\n4. Each completed load performs `complete_tx` for its copied byte count; it does **not** perform another arrival.\n5. Consumers wait with the correct state token or per-stage parity and acquire semantics before reading the destination.\n\nThe phase completes only after pending arrivals and tx-count are both zero. If a design performs multiple software arrivals instead, its initialized count must match them exactly. See [mbarrier](mbarrier.md) for lifecycle, phase, and memory-ordering rules.\n\nA tensor store uses bulk-group completion rather than the load's mbarrier protocol:\n\n```ptx\ncp.async.bulk.tensor.2d.global.shared::cta.bulk_group\n [tensor_map, {x, y}], [src_smem];\ncp.async.bulk.commit_group;\ncp.async.bulk.wait_group 0;\n```\n\nThe wait may be delayed to overlap independent work. It must occur before the issuing thread reuses the source shared memory or before code that requires store completion. `commit_group` creates the group; it is not itself a completion wait."}, "reason": {"statement": "List parameter constraints and direction-specific completion separately.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#asynchronous-data-copies-using-the-tensor-memory-accelerator-tma", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#encoding-a-tensor-map-on-device", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-phase-completion"]}} {"path": "wiki/hardware/tma.md", "before": {"statement": "TMA operations are driven by a **descriptor** that encodes the tensor layout, addressing, and transfer parameters. The descriptor is created on the host and passed to the kernel."}, "after": {"statement": "A tensor map is opaque and is accessed through the tensor-map proxy. `cuTensorMapEncodeTiled` describes:\n\n- datatype and global base address;\n- rank, global dimensions, and byte strides;\n- traversal box dimensions and element strides;\n- interleave and shared-memory swizzle;\n- L2-promotion hint; and\n- out-of-bounds fill policy.\n\nFor the ordinary non-interleaved tiled path in CUDA Driver API 13.0.97, important constraints include:\n\n- the `CUtensorMap` output object is 64-byte aligned;\n- the global base and byte strides satisfy the documented alignment rules, commonly at least 16 bytes for the basic types/path;\n- every `boxDim` entry is from 1 through 256 elements;\n- `boxDim[0] * element_size` is a multiple of 16 bytes;\n- every element stride is from 1 through 8; and\n- when swizzling is enabled, the inner box in bytes does not exceed the selected swizzle span.\n\nDatatype, interleave, sub-byte, and architecture-specific modes add further restrictions. Always check the encoder's `CUresult`; never use the output after an encoding failure.\n\nHost encoding is common, but it is not the only Blackwell path. CUDA also documents device-side tensor-map construction and modification on Blackwell. A map modified through the generic proxy must be published to the tensor-map proxy with the required `fence.proxy.tensormap::generic` sequence before a TMA operation consumes it."}, "reason": {"statement": "Present host encoding as common, not exclusive, and mention device proxy ordering.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#encoding-a-tensor-map-on-device", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"]}} {"path": "wiki/hardware/tma.md", "before": {"statement": "```cuda\n#include \n\n// Create a 2D TMA descriptor for a row-major FP16 matrix\nCUtensorMap create_tma_descriptor_2d(\n const half* global_ptr,\n int M, int N, // Global tensor dimensions\n int tile_m, int tile_n, // Tile dimensions for each transfer\n int swizzle_bytes // Swizzle mode: 0, 32, 64, 128\n) {\n CUtensorMap tensor_map;\n\n // Tensor dimensions (outermost to innermost)\n uint64_t global_dims[2] = {(uint64_t)N, (uint64_t)M};\n uint64_t global_strides[1] = {(uint64_t)(N * sizeof(half))};\n\n // Tile box dimensions\n uint32_t box_dims[2] = {(uint32_t)tile_n, (uint32_t)tile_m};\n\n // Element strides (1 = contiguous)\n uint32_t elem_strides[2] = {1, 1};\n\n CUtensorMapSwizzle swizzle;\n switch (swizzle_bytes) {\n case 0: swizzle = CU_TENSOR_MAP_SWIZZLE_NONE; break;\n case 32: swizzle = CU_TENSOR_MAP_SWIZZLE_32B; break;\n case 64: swizzle = CU_TENSOR_MAP_SWIZZLE_64B; break;\n case 128: swizzle = CU_TENSOR_MAP_SWIZZLE_128B; break;\n }\n\n cuTensorMapEncodeTiled(\n &tensor_map,\n CU_TENSOR_MAP_DATA_TYPE_FLOAT16, // Element type\n 2, // Dimensionality\n (void*)global_ptr, // Global pointer\n global_dims, // Tensor dimensions\n global_strides, // Byte strides (exclude innermost)\n box_dims, // Tile/box dimensions\n elem_strides, // Element strides\n CU_TENSOR_MAP_INTERLEAVE_NONE,\n swizzle,\n CU_TENSOR_MAP_L2_PROMOTION_NONE,\n CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE\n );\n\n return tensor_map;\n}\n```"}, "after": {"statement": "A tensor map is opaque and is accessed through the tensor-map proxy. `cuTensorMapEncodeTiled` describes:\n\n- datatype and global base address;\n- rank, global dimensions, and byte strides;\n- traversal box dimensions and element strides;\n- interleave and shared-memory swizzle;\n- L2-promotion hint; and\n- out-of-bounds fill policy.\n\nFor the ordinary non-interleaved tiled path in CUDA Driver API 13.0.97, important constraints include:\n\n- the `CUtensorMap` output object is 64-byte aligned;\n- the global base and byte strides satisfy the documented alignment rules, commonly at least 16 bytes for the basic types/path;\n- every `boxDim` entry is from 1 through 256 elements;\n- `boxDim[0] * element_size` is a multiple of 16 bytes;\n- every element stride is from 1 through 8; and\n- when swizzling is enabled, the inner box in bytes does not exceed the selected swizzle span.\n\nDatatype, interleave, sub-byte, and architecture-specific modes add further restrictions. Always check the encoder's `CUresult`; never use the output after an encoding failure.\n\nHost encoding is common, but it is not the only Blackwell path. CUDA also documents device-side tensor-map construction and modification on Blackwell. A map modified through the generic proxy must be published to the tensor-map proxy with the required `fence.proxy.tensormap::generic` sequence before a TMA operation consumes it.\n\nChoose tile rank, swizzle, multicast, issue cadence, and stage count from the actual access pattern and resource budget. More distinct stages consume more shared memory; there is no universal optimum of three to five stages. Profile the target GPU and record the copy shape, descriptor, cluster mask, stage count, shared-memory use, occupancy, warmup, repetitions, and baseline."}, "reason": {"statement": "Replace a deceptively general factory with a constraints checklist and official complete sample.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#encoding-a-tensor-map-on-device"]}} -{"path": "wiki/hardware/tma.md", "before": {"statement": "```cuda\n// TMA load: single thread issues, hardware executes asynchronously\n__device__ void tma_load_tile(\n const CUtensorMap* desc,\n void* smem_ptr,\n uint64_t* mbar_ptr, // mbarrier for synchronization\n int coord_x, int coord_y\n) {\n if (threadIdx.x == 0) {\n // Set expected bytes on the mbarrier\n uint32_t expected_bytes = TILE_M * TILE_N * sizeof(half);\n asm volatile(\n \"mbarrier.arrive.expect_tx.shared.b64 _, [%0], %1;\"\n :\n : \"r\"((uint32_t)__cvta_generic_to_shared(mbar_ptr)),\n \"r\"(expected_bytes)\n );\n\n // Issue TMA copy\n asm volatile(\n \"cp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes \"\n \"[%0], [%1, {%2, %3}], [%4];\"\n :\n : \"r\"((uint32_t)__cvta_generic_to_shared(smem_ptr)),\n \"l\"(desc),\n \"r\"(coord_x), \"r\"(coord_y),\n \"r\"((uint32_t)__cvta_generic_to_shared(mbar_ptr))\n );\n }\n}\n```"}, "after": {"statement": "PTX ISA 9.0 defines this representative 2D CTA-local form:\n\n```ptx\ncp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes\n [dst_smem], [tensor_map, {x, y}], [full_barrier];\n```\n\nThe instruction is non-blocking. A common one-producer phase with one or more loads uses this accounting:\n\n1. Initialize and publish the stage's mbarrier with the intended pending-arrival count.\n2. The producer performs one `mbarrier.arrive.expect_tx` for its software arrival and the sum of bytes that all loads in this phase will complete.\n3. Issue the TMA loads against that barrier.\n4. Each completed load performs `complete_tx` for its copied byte count; it does **not** perform another arrival.\n5. Consumers wait with the correct state token or per-stage parity and acquire semantics before reading the destination.\n\nThe phase completes only after pending arrivals and tx-count are both zero. If a design performs multiple software arrivals instead, its initialized count must match them exactly. See [mbarrier](mbarrier.md) for lifecycle, phase, and memory-ordering rules."}, "reason": {"statement": "Teach one explicit phase-accounting sequence instead of an unsafe reusable helper.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#phase-completion-of-the-mbarrier-object", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"]}} +{"path": "wiki/hardware/tma.md", "before": {"statement": "```cuda\n// TMA load: single thread issues, hardware executes asynchronously\n__device__ void tma_load_tile(\n const CUtensorMap* desc,\n void* smem_ptr,\n uint64_t* mbar_ptr, // mbarrier for synchronization\n int coord_x, int coord_y\n) {\n if (threadIdx.x == 0) {\n // Set expected bytes on the mbarrier\n uint32_t expected_bytes = TILE_M * TILE_N * sizeof(half);\n asm volatile(\n \"mbarrier.arrive.expect_tx.shared.b64 _, [%0], %1;\"\n :\n : \"r\"((uint32_t)__cvta_generic_to_shared(mbar_ptr)),\n \"r\"(expected_bytes)\n );\n\n // Issue TMA copy\n asm volatile(\n \"cp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes \"\n \"[%0], [%1, {%2, %3}], [%4];\"\n :\n : \"r\"((uint32_t)__cvta_generic_to_shared(smem_ptr)),\n \"l\"(desc),\n \"r\"(coord_x), \"r\"(coord_y),\n \"r\"((uint32_t)__cvta_generic_to_shared(mbar_ptr))\n );\n }\n}\n```"}, "after": {"statement": "PTX ISA 9.0 defines this representative 2D CTA-local form:\n\n```ptx\ncp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes\n [dst_smem], [tensor_map, {x, y}], [full_barrier];\n```\n\nThe instruction is non-blocking. A common one-producer phase with one or more loads uses this accounting:\n\n1. Initialize and publish the stage's mbarrier with the intended pending-arrival count.\n2. The producer performs one `mbarrier.arrive.expect_tx` for its software arrival and the sum of bytes that all loads in this phase will complete.\n3. Issue the TMA loads against that barrier.\n4. Each completed load performs `complete_tx` for its copied byte count; it does **not** perform another arrival.\n5. Consumers wait with the correct state token or per-stage parity and acquire semantics before reading the destination.\n\nThe phase completes only after pending arrivals and tx-count are both zero. If a design performs multiple software arrivals instead, its initialized count must match them exactly. See [mbarrier](mbarrier.md) for lifecycle, phase, and memory-ordering rules."}, "reason": {"statement": "Teach one explicit phase-accounting sequence instead of an unsafe reusable helper.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-phase-completion", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"]}} {"path": "wiki/hardware/tma.md", "before": {"statement": "```cuda\n// TMA store: write a tile from shared memory back to global memory\n__device__ void tma_store_tile(\n const CUtensorMap* desc,\n const void* smem_ptr,\n int coord_x, int coord_y\n) {\n if (threadIdx.x == 0) {\n asm volatile(\n \"cp.async.bulk.tensor.2d.global.shared::cta \"\n \"[%0, {%1, %2}], [%3];\"\n :\n : \"l\"(desc),\n \"r\"(coord_x), \"r\"(coord_y),\n \"r\"((uint32_t)__cvta_generic_to_shared(smem_ptr))\n );\n\n // Commit the store\n asm volatile(\"cp.async.bulk.commit_group;\");\n }\n}\n```"}, "after": {"statement": "A tensor store uses bulk-group completion rather than the load's mbarrier protocol:\n\n```ptx\ncp.async.bulk.tensor.2d.global.shared::cta.bulk_group\n [tensor_map, {x, y}], [src_smem];\ncp.async.bulk.commit_group;\ncp.async.bulk.wait_group 0;\n```\n\nThe wait may be delayed to overlap independent work. It must occur before the issuing thread reuses the source shared memory or before code that requires store completion. `commit_group` creates the group; it is not itself a completion wait."}, "reason": {"statement": "Show the required qualifier and explicit wait boundary.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"]}} -{"path": "wiki/hardware/tma.md", "before": {"statement": "TMA uses mbarriers (memory barriers) for producer-consumer synchronization. The pattern is:\n\n1. **Producer** (TMA): arrives at the barrier when the transfer completes, decrementing the expected transaction count.\n2. **Consumer** (compute warps): waits on the barrier before reading the loaded data."}, "after": {"statement": "PTX ISA 9.0 defines this representative 2D CTA-local form:\n\n```ptx\ncp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes\n [dst_smem], [tensor_map, {x, y}], [full_barrier];\n```\n\nThe instruction is non-blocking. A common one-producer phase with one or more loads uses this accounting:\n\n1. Initialize and publish the stage's mbarrier with the intended pending-arrival count.\n2. The producer performs one `mbarrier.arrive.expect_tx` for its software arrival and the sum of bytes that all loads in this phase will complete.\n3. Issue the TMA loads against that barrier.\n4. Each completed load performs `complete_tx` for its copied byte count; it does **not** perform another arrival.\n5. Consumers wait with the correct state token or per-stage parity and acquire semantics before reading the destination.\n\nThe phase completes only after pending arrivals and tx-count are both zero. If a design performs multiple software arrivals instead, its initialized count must match them exactly. See [mbarrier](mbarrier.md) for lifecycle, phase, and memory-ordering rules."}, "reason": {"statement": "Separate software arrival from hardware byte completion.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#phase-completion-of-the-mbarrier-object"]}} +{"path": "wiki/hardware/tma.md", "before": {"statement": "TMA uses mbarriers (memory barriers) for producer-consumer synchronization. The pattern is:\n\n1. **Producer** (TMA): arrives at the barrier when the transfer completes, decrementing the expected transaction count.\n2. **Consumer** (compute warps): waits on the barrier before reading the loaded data."}, "after": {"statement": "PTX ISA 9.0 defines this representative 2D CTA-local form:\n\n```ptx\ncp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes\n [dst_smem], [tensor_map, {x, y}], [full_barrier];\n```\n\nThe instruction is non-blocking. A common one-producer phase with one or more loads uses this accounting:\n\n1. Initialize and publish the stage's mbarrier with the intended pending-arrival count.\n2. The producer performs one `mbarrier.arrive.expect_tx` for its software arrival and the sum of bytes that all loads in this phase will complete.\n3. Issue the TMA loads against that barrier.\n4. Each completed load performs `complete_tx` for its copied byte count; it does **not** perform another arrival.\n5. Consumers wait with the correct state token or per-stage parity and acquire semantics before reading the destination.\n\nThe phase completes only after pending arrivals and tx-count are both zero. If a design performs multiple software arrivals instead, its initialized count must match them exactly. See [mbarrier](mbarrier.md) for lifecycle, phase, and memory-ordering rules."}, "reason": {"statement": "Separate software arrival from hardware byte completion.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-phase-completion"]}} {"path": "wiki/hardware/tma.md", "before": {"statement": "```cuda\n// Multi-stage pipeline with TMA + mbarrier\n__device__ void pipelined_mainloop(\n const CUtensorMap* desc_a,\n const CUtensorMap* desc_b,\n void* smem_a_stages[NUM_STAGES],\n void* smem_b_stages[NUM_STAGES],\n uint64_t* mbar[NUM_STAGES],\n int num_k_tiles\n) {\n // Prologue: fill the first NUM_STAGES-1 stages\n for (int s = 0; s < NUM_STAGES - 1 && s < num_k_tiles; ++s) {\n tma_load_tile(desc_a, smem_a_stages[s], mbar[s], 0, s);\n tma_load_tile(desc_b, smem_b_stages[s], mbar[s], s, 0);\n }\n\n // Mainloop\n for (int k = 0; k < num_k_tiles; ++k) {\n int stage = k % NUM_STAGES;\n\n // Wait for data to arrive in this stage\n mbarrier_wait(mbar[stage]);\n\n // Issue MMA using this stage's SMEM buffers\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 1;\"\n :\n : \"r\"(tmem_acc),\n \"l\"(make_desc(smem_a_stages[stage])),\n \"l\"(make_desc(smem_b_stages[stage])),\n \"r\"(0)\n );\n }\n\n // Prefetch next stage\n int next_k = k + NUM_STAGES - 1;\n if (next_k < num_k_tiles) {\n int next_stage = next_k % NUM_STAGES;\n tma_load_tile(desc_a, smem_a_stages[next_stage],\n mbar[next_stage], 0, next_k);\n tma_load_tile(desc_b, smem_b_stages[next_stage],\n mbar[next_stage], next_k, 0);\n }\n }\n}\n```"}, "after": {"statement": "A common Blackwell GEMM data path is:\n\n`GMEM -> TMA -> SMEM -> tcgen05.mma -> TMEM -> tcgen05.ld -> registers -> output store`\n\nEach reusable pipeline stage needs two independent ownership transitions:\n\n- **full:** TMA has finished producing the shared-memory operands, so the MMA consumer may read them; and\n- **empty:** asynchronous MMA has stopped reading those operands, so the TMA producer may overwrite the stage.\n\nTrack full and empty state per stage. Account transaction bytes once, track each barrier's own phase, and do not equate CTA synchronization or a tcgen05 fence with async completion. A complete CUTLASS pipeline is safer evidence than a shortened helper that omits one ownership edge.\n\nChoose tile rank, swizzle, multicast, issue cadence, and stage count from the actual access pattern and resource budget. More distinct stages consume more shared memory; there is no universal optimum of three to five stages. Profile the target GPU and record the copy shape, descriptor, cluster mask, stage count, shared-memory use, occupancy, warmup, repetitions, and baseline."}, "reason": {"statement": "Replace unrepairable pseudocode with invariant-based stage lifecycle and a pinned complete implementation.", "urls": []}} -{"path": "wiki/hardware/tma.md", "before": {"statement": "```cuda\n// Initialize an mbarrier\n__device__ void mbarrier_init(uint64_t* mbar, int arrive_count) {\n if (threadIdx.x == 0) {\n asm volatile(\n \"mbarrier.init.shared.b64 [%0], %1;\"\n :\n : \"r\"((uint32_t)__cvta_generic_to_shared(mbar)),\n \"r\"(arrive_count)\n );\n }\n}\n\n// Wait for an mbarrier to complete (phase-based)\n__device__ void mbarrier_wait(uint64_t* mbar, int phase) {\n uint32_t mbar_addr = (uint32_t)__cvta_generic_to_shared(mbar);\n asm volatile(\n \"{\\n\"\n \".reg .pred p;\\n\"\n \"WAIT_LOOP:\\n\"\n \"mbarrier.try_wait.parity.shared.b64 p, [%0], %1;\\n\"\n \"@!p bra WAIT_LOOP;\\n\"\n \"}\\n\"\n :\n : \"r\"(mbar_addr), \"r\"(phase)\n );\n}\n```"}, "after": {"statement": "PTX ISA 9.0 defines this representative 2D CTA-local form:\n\n```ptx\ncp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes\n [dst_smem], [tensor_map, {x, y}], [full_barrier];\n```\n\nThe instruction is non-blocking. A common one-producer phase with one or more loads uses this accounting:\n\n1. Initialize and publish the stage's mbarrier with the intended pending-arrival count.\n2. The producer performs one `mbarrier.arrive.expect_tx` for its software arrival and the sum of bytes that all loads in this phase will complete.\n3. Issue the TMA loads against that barrier.\n4. Each completed load performs `complete_tx` for its copied byte count; it does **not** perform another arrival.\n5. Consumers wait with the correct state token or per-stage parity and acquire semantics before reading the destination.\n\nThe phase completes only after pending arrivals and tx-count are both zero. If a design performs multiple software arrivals instead, its initialized count must match them exactly. See [mbarrier](mbarrier.md) for lifecycle, phase, and memory-ordering rules.\n\nA common Blackwell GEMM data path is:\n\n`GMEM -> TMA -> SMEM -> tcgen05.mma -> TMEM -> tcgen05.ld -> registers -> output store`\n\nEach reusable pipeline stage needs two independent ownership transitions:\n\n- **full:** TMA has finished producing the shared-memory operands, so the MMA consumer may read them; and\n- **empty:** asynchronous MMA has stopped reading those operands, so the TMA producer may overwrite the stage.\n\nTrack full and empty state per stage. Account transaction bytes once, track each barrier's own phase, and do not equate CTA synchronization or a tcgen05 fence with async completion. A complete CUTLASS pipeline is safer evidence than a shortened helper that omits one ownership edge."}, "reason": {"statement": "Refer to the verified mbarrier page and state lifecycle requirements rather than ship a partial abstraction.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#phase-completion-of-the-mbarrier-object"]}} +{"path": "wiki/hardware/tma.md", "before": {"statement": "```cuda\n// Initialize an mbarrier\n__device__ void mbarrier_init(uint64_t* mbar, int arrive_count) {\n if (threadIdx.x == 0) {\n asm volatile(\n \"mbarrier.init.shared.b64 [%0], %1;\"\n :\n : \"r\"((uint32_t)__cvta_generic_to_shared(mbar)),\n \"r\"(arrive_count)\n );\n }\n}\n\n// Wait for an mbarrier to complete (phase-based)\n__device__ void mbarrier_wait(uint64_t* mbar, int phase) {\n uint32_t mbar_addr = (uint32_t)__cvta_generic_to_shared(mbar);\n asm volatile(\n \"{\\n\"\n \".reg .pred p;\\n\"\n \"WAIT_LOOP:\\n\"\n \"mbarrier.try_wait.parity.shared.b64 p, [%0], %1;\\n\"\n \"@!p bra WAIT_LOOP;\\n\"\n \"}\\n\"\n :\n : \"r\"(mbar_addr), \"r\"(phase)\n );\n}\n```"}, "after": {"statement": "PTX ISA 9.0 defines this representative 2D CTA-local form:\n\n```ptx\ncp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes\n [dst_smem], [tensor_map, {x, y}], [full_barrier];\n```\n\nThe instruction is non-blocking. A common one-producer phase with one or more loads uses this accounting:\n\n1. Initialize and publish the stage's mbarrier with the intended pending-arrival count.\n2. The producer performs one `mbarrier.arrive.expect_tx` for its software arrival and the sum of bytes that all loads in this phase will complete.\n3. Issue the TMA loads against that barrier.\n4. Each completed load performs `complete_tx` for its copied byte count; it does **not** perform another arrival.\n5. Consumers wait with the correct state token or per-stage parity and acquire semantics before reading the destination.\n\nThe phase completes only after pending arrivals and tx-count are both zero. If a design performs multiple software arrivals instead, its initialized count must match them exactly. See [mbarrier](mbarrier.md) for lifecycle, phase, and memory-ordering rules.\n\nA common Blackwell GEMM data path is:\n\n`GMEM -> TMA -> SMEM -> tcgen05.mma -> TMEM -> tcgen05.ld -> registers -> output store`\n\nEach reusable pipeline stage needs two independent ownership transitions:\n\n- **full:** TMA has finished producing the shared-memory operands, so the MMA consumer may read them; and\n- **empty:** asynchronous MMA has stopped reading those operands, so the TMA producer may overwrite the stage.\n\nTrack full and empty state per stage. Account transaction bytes once, track each barrier's own phase, and do not equate CTA synchronization or a tcgen05 fence with async completion. A complete CUTLASS pipeline is safer evidence than a shortened helper that omits one ownership edge."}, "reason": {"statement": "Refer to the verified mbarrier page and state lifecycle requirements rather than ship a partial abstraction.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-phase-completion"]}} {"path": "wiki/hardware/tma.md", "before": {"statement": "TMA multicast sends a single GMEM tile to **multiple CTAs within a cluster** simultaneously. This is critical for GEMM where the B operand is shared across M-axis tiles.\n\n```cuda\n// Multicast TMA: load B tile to all CTAs in the cluster\n__device__ void tma_multicast_load(\n const CUtensorMap* desc,\n void* smem_ptr,\n uint64_t* mbar_ptr,\n int coord_x, int coord_y,\n uint16_t multicast_mask // bitmask: which CTAs in cluster receive the data\n) {\n if (threadIdx.x == 0) {\n uint32_t expected_bytes = TILE_K * TILE_N * sizeof(half);\n asm volatile(\n \"mbarrier.arrive.expect_tx.shared.b64 _, [%0], %1;\"\n :\n : \"r\"((uint32_t)__cvta_generic_to_shared(mbar_ptr)),\n \"r\"(expected_bytes)\n );\n\n asm volatile(\n \"cp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes.multicast::cluster \"\n \"[%0], [%1, {%2, %3}], [%4], %5;\"\n :\n : \"r\"((uint32_t)__cvta_generic_to_shared(smem_ptr)),\n \"l\"(desc),\n \"r\"(coord_x), \"r\"(coord_y),\n \"r\"((uint32_t)__cvta_generic_to_shared(mbar_ptr)),\n \"h\"(multicast_mask)\n );\n }\n}\n```"}, "after": {"statement": "The cluster form copies a global tile to the same shared-memory offset in every CTA selected by a 16-bit mask:\n\n```ptx\ncp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes.multicast::cluster\n [dst_smem], [tensor_map, {x, y}], [full_barrier], cta_mask;\n```\n\nFor `cta_group::1` (the default), the completion signal is also multicast to the same barrier offset in every selected destination CTA. A correct design therefore:\n\n- launches an explicit cluster and keeps every destination CTA's shared memory alive;\n- initializes corresponding destination barriers before the elected issuer can start;\n- uses one cluster-wide elected issuer, not one `threadIdx.x == 0` issuer in every CTA;\n- includes only valid destination CTA ranks in the mask; and\n- has every destination wait on its own corresponding barrier phase before consuming the tile.\n\nFor GEMM tiles with the same N range and different M ranges, the CTAs use different A tiles but the same B tile, so multicast can avoid duplicate logical B-load requests. It does not promise an exact `cluster_size` reduction in measured DRAM traffic or elapsed time; caches, mask population, transaction behavior, and resource costs affect the result."}, "reason": {"statement": "Replace incomplete executable code with the cluster ownership and barrier invariants.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"]}} {"path": "wiki/hardware/tma.md", "before": {"statement": "```\nCluster: 2 CTAs (CTA0 and CTA1) each computing different M-tiles of the same N column\n\nCTA0: computes C[0:128, 0:256] -- needs A[0:128, :] and B[:, 0:256]\nCTA1: computes C[128:256, 0:256] -- needs A[128:256, :] and B[:, 0:256]\n\nB[:, 0:256] is SHARED -- multicast it once from GMEM to both CTAs\nA tiles are UNIQUE -- each CTA loads its own A tile\n\nResult: B bandwidth is halved (1 GMEM read serves 2 CTAs)\n```"}, "after": {"statement": "The cluster form copies a global tile to the same shared-memory offset in every CTA selected by a 16-bit mask:\n\n```ptx\ncp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes.multicast::cluster\n [dst_smem], [tensor_map, {x, y}], [full_barrier], cta_mask;\n```\n\nFor `cta_group::1` (the default), the completion signal is also multicast to the same barrier offset in every selected destination CTA. A correct design therefore:\n\n- launches an explicit cluster and keeps every destination CTA's shared memory alive;\n- initializes corresponding destination barriers before the elected issuer can start;\n- uses one cluster-wide elected issuer, not one `threadIdx.x == 0` issuer in every CTA;\n- includes only valid destination CTA ranks in the mask; and\n- has every destination wait on its own corresponding barrier phase before consuming the tile.\n\nFor GEMM tiles with the same N range and different M ranges, the CTAs use different A tiles but the same B tile, so multicast can avoid duplicate logical B-load requests. It does not promise an exact `cluster_size` reduction in measured DRAM traffic or elapsed time; caches, mask population, transaction behavior, and resource costs affect the result.\n\nA common Blackwell GEMM data path is:\n\n`GMEM -> TMA -> SMEM -> tcgen05.mma -> TMEM -> tcgen05.ld -> registers -> output store`\n\nEach reusable pipeline stage needs two independent ownership transitions:\n\n- **full:** TMA has finished producing the shared-memory operands, so the MMA consumer may read them; and\n- **empty:** asynchronous MMA has stopped reading those operands, so the TMA producer may overwrite the stage.\n\nTrack full and empty state per stage. Account transaction bytes once, track each barrier's own phase, and do not equate CTA synchronization or a tcgen05 fence with async completion. A complete CUTLASS pipeline is safer evidence than a shortened helper that omits one ownership edge."}, "reason": {"statement": "State logical load reuse without an unmeasured hardware-traffic factor.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor"]}} -{"path": "wiki/hardware/tma.md", "before": {"statement": "```\nWithout swizzle (linear):\n Row 0: bytes [0, 1, 2, ..., 127]\n Row 1: bytes [128, 129, ..., 255]\n\nWith 128B swizzle:\n Row 0: bytes [0, 1, ..., 127] (unchanged)\n Row 1: bytes [128, 129, ..., 255] XOR pattern applied\n Row 2: bytes [256, ...] XOR pattern applied differently\n ...\n```\n\nThe swizzle eliminates bank conflicts when the tensor core reads operand tiles from SMEM."}, "after": {"statement": "TMA supports no swizzle and multiple swizzled shared-memory layouts, including 32B, 64B, and 128B spans plus newer variants for selected types. Swizzling rearranges chunks across shared-memory banks. The consumer must address the matching logical layout; the exact mapping also depends on the documented shared-memory base-offset rule.\n\nThere is no universal rule that every Blackwell or `tcgen05.mma` input uses 128B swizzling. The TMA tensor-map swizzle, destination base alignment, leading dimension, and the tcgen05 shared-memory descriptor must describe the **same** legal layout. PTX defines no-, 32B-, 64B-, and 128B-swizzled tcgen05 descriptors with kind-, type-, shape-, and layout-specific constraints.\n\nA matched swizzle can reduce or remove bank conflicts for a particular access pattern. It does not make every possible consumer access conflict-free."}, "reason": {"statement": "Link the normative mapping and explain the producer/consumer layout match.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#the-swizzle-modes", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-shared-memory-layout"]}} +{"path": "wiki/hardware/tma.md", "before": {"statement": "```\nWithout swizzle (linear):\n Row 0: bytes [0, 1, 2, ..., 127]\n Row 1: bytes [128, 129, ..., 255]\n\nWith 128B swizzle:\n Row 0: bytes [0, 1, ..., 127] (unchanged)\n Row 1: bytes [128, 129, ..., 255] XOR pattern applied\n Row 2: bytes [256, ...] XOR pattern applied differently\n ...\n```\n\nThe swizzle eliminates bank conflicts when the tensor core reads operand tiles from SMEM."}, "after": {"statement": "TMA supports no swizzle and multiple swizzled shared-memory layouts, including 32B, 64B, and 128B spans plus newer variants for selected types. Swizzling rearranges chunks across shared-memory banks. The consumer must address the matching logical layout; the exact mapping also depends on the documented shared-memory base-offset rule.\n\nThere is no universal rule that every Blackwell or `tcgen05.mma` input uses 128B swizzling. The TMA tensor-map swizzle, destination base alignment, leading dimension, and the tcgen05 shared-memory descriptor must describe the **same** legal layout. PTX defines no-, 32B-, 64B-, and 128B-swizzled tcgen05 descriptors with kind-, type-, shape-, and layout-specific constraints.\n\nA matched swizzle can reduce or remove bank conflicts for a particular access pattern. It does not make every possible consumer access conflict-free."}, "reason": {"statement": "Link the normative mapping and explain the producer/consumer layout match.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#the-swizzle-modes", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-shared-memory-layout-swizzling"]}} {"path": "wiki/hardware/tma.md", "before": {"statement": "| Maximize TMA utilization | Keep the TMA unit busy with back-to-back loads across pipeline stages |\n| Use multicast for shared operands | Reduces GMEM bandwidth by cluster_size x for shared tiles |\n| Always use 128B swizzle on Blackwell | Non-128B swizzle produces incorrect tcgen05 results |\n| Prefer 2D TMA over manual addressing | TMA handles out-of-bounds clamping, padding, and strided access |\n| Pipeline depth | 3-5 stages typically optimal; more stages increase SMEM usage |"}, "after": {"statement": "A common Blackwell GEMM data path is:\n\n`GMEM -> TMA -> SMEM -> tcgen05.mma -> TMEM -> tcgen05.ld -> registers -> output store`\n\nEach reusable pipeline stage needs two independent ownership transitions:\n\n- **full:** TMA has finished producing the shared-memory operands, so the MMA consumer may read them; and\n- **empty:** asynchronous MMA has stopped reading those operands, so the TMA producer may overwrite the stage.\n\nTrack full and empty state per stage. Account transaction bytes once, track each barrier's own phase, and do not equate CTA synchronization or a tcgen05 fence with async completion. A complete CUTLASS pipeline is safer evidence than a shortened helper that omits one ownership edge."}, "reason": {"statement": "Replace folklore with a measurement and resource checklist.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#asynchronous-data-copies-using-the-tensor-memory-accelerator-tma"]}} {"path": "wiki/hardware/tma.md", "before": {"statement": "```python\n# CuTe-DSL TMA copy setup for Blackwell GEMM\nfrom cute import *\n\n# Define TMA copy atom for operand A (BF16, 128x64 tile)\ntma_a = make_tma_copy(\n SM100_TMA_LOAD_2D,\n tensor_a, # global tensor\n smem_layout_a, # shared memory layout\n tile_shape=(128, 64), # tile dimensions\n swizzle=Swizzle(7, 0, 4), # 128-byte swizzle\n multicast_mask=None # no multicast for A\n)\n\n# Define TMA copy atom for operand B with multicast\ntma_b = make_tma_copy(\n SM100_TMA_LOAD_2D_MULTICAST,\n tensor_b,\n smem_layout_b,\n tile_shape=(64, 256),\n swizzle=Swizzle(7, 0, 4), # 128-byte swizzle\n multicast_mask=cluster_mask # multicast B to all CTAs in cluster\n)\n```"}, "after": {"statement": "Choose tile rank, swizzle, multicast, issue cadence, and stage count from the actual access pattern and resource budget. More distinct stages consume more shared memory; there is no universal optimum of three to five stages. Profile the target GPU and record the copy shape, descriptor, cluster mask, stage count, shared-memory use, occupancy, warmup, repetitions, and baseline."}, "reason": {"statement": "Link a complete pinned tutorial rather than fabricate a simplified API.", "urls": []}} {"path": "wiki/kernels/deepgemm.md", "before": {"statement": "DeepGEMM is DeepSeek's open-source FP8 GEMM library providing high-performance matrix multiplication with fine-grained per-tile/per-block scaling. The core kernel is remarkably compact (~300 lines), yet achieves approximately 90% utilization on H800. It supports both Hopper (SM90 via WGMMA) and Blackwell (SM100 via tcgen05.mma) architectures, and includes specialized MoE grouped GEMM layouts."}, "after": {"statement": "DeepGEMM is DeepSeek's open-source tensor-core kernel library. This page describes the FP8 GEMM paths at commit [`891d57b4db1071624b5c8fa0d1e51cb317fa709f`](https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f), with byte-verified SM90 and SM100 kernel files stored locally. The pinned project requires an SM90 or SM100 GPU, CUDA 12.3 or newer for SM90, and CUDA 12.9 or newer for SM100.\n\nDeepGEMM contains multiple kernels and APIs; the local SM90 and SM100 files below are representative FP8 1D1D implementations, not the whole library.\n\nThe pinned README reports that DeepGEMM reached **up to 1550 TFLOPS on H800** in an April 2025 news item. It does not bind that peak to `M=N=K=4096`, state approximately 90% utilization, or preserve enough benchmark conditions for reproduction. The unsupported structured performance record and table have therefore been removed; the surviving number is source-reported only and should not be compared across software, clocks, shapes, or GPUs without a controlled rerun."}, "reason": {"statement": "Scope the overview to the pinned library and separate the surviving source-reported peak from unsupported utilization and size claims.", "urls": ["https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://github.com/deepseek-ai/DeepGEMM/pull/86"]}} @@ -181,7 +181,7 @@ {"path": "wiki/kernels/gated-delta-net.md", "before": {"statement": "performance_claims:\n- gpu: H100\n dtype: bf16\n shape: seqlen=8192, qk_dim=4, v_dim=8, d=128\n metric: speedup\n value: 10\n utilization: vs Qwen3-32B at 32K+ context, O(n) linear complexity\n source_id: blog-gated-delta-net"}, "after": {"statement": "performance_claims: []\n\n- [Gated DeltaNet paper](https://arxiv.org/abs/2412.06464)\n- [NVlabs implementation at `b53d6d3`](https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921)\n- [Qwen3-Next model card and configuration at `9c7f2fbe`](https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/tree/9c7f2fbe84465e40164a94cc16cd30b6999b0cc7)\n- [MLSys 2026 organizer](https://mlsys26.flashinfer.ai/)\n- [Exact decode definition](https://bench.flashinfer.ai/kernels/gdn_decode_qk4_v8_d128_k_last)\n- [Exact prefill definition](https://bench.flashinfer.ai/kernels/gdn_prefill_qk4_v8_d128_k_last)\n- [FlashInfer GDN trace at `7f614b8`](https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/trace/templates/gdn.py)\n- [FlashInfer prefill dispatch at `7f614b8`](https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/gdn_prefill.py)\n- [CUDA Graphs programming guide](https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cuda-graphs.html)\n- [TFLA v3](https://arxiv.org/abs/2503.14376v3)"}, "reason": {"statement": "No evidence-supported numeric kernel record can replace the fabricated structured tuple.", "urls": ["https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct"]}} {"path": "wiki/kernels/gated-dual-gemm.md", "before": {"statement": "Gated dual GEMM fuses two matrix multiplications with activation and elementwise operations — the canonical MLP gate-up pattern used by LLaMA, Qwen, DeepSeek, and most modern LLMs. Fusion eliminates two global memory roundtrips compared to separate gate/up GEMM + SwiGLU."}, "after": {"statement": "The official NVIDIA rules identify NVFP4 Gated Dual GEMM as Kernel Challenge 3 of the Blackwell NVFP4 Hackathon, open from December 20, 2025 through January 16, 2026. The public GPU Mode problem at challenge-opening commit `c5b2f7c` targets NVIDIA B200.\n\nThe shared `a` and `sfa` inputs create an opportunity to reuse data across the two products, and a genuinely fused epilogue can avoid writing both full-precision products to global memory. Those are optimization possibilities, not task guarantees.\n\nThe public task/reference does not establish a final leaderboard, winning source, single-launch decomposition, physical load count, TMEM accumulator partition, TMA pipeline, CUTLASS schedule, or compute-/memory-bound classification. If an implementation uses `tcgen05`, TMEM, or TMA, its exact descriptors, scale layouts, synchronization, allocation, and architecture requirements must be verified from that implementation and the version-matched ISA.\n\nCompatibility is correspondingly narrower than “any dual-output operation”: a model/backend must match the two same-shaped NVFP4 products, scale storage and layouts, first-branch SiLU, FP16 output, batching, and divisibility rules. Grouped per-expert MoE execution is a different contract unless an implementation explicitly supplies that grouping layer."}, "reason": {"statement": "Retain exact dual-GEMM/SwiGLU semantics while separating potential fusion benefits from contract guarantees.", "urls": ["https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf", "https://github.com/gpu-mode/reference-kernels/blob/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm/task.yml"]}} {"path": "wiki/kernels/gated-dual-gemm.md", "before": {"statement": "```\nGiven x, W_gate, W_up (weights shared same K dimension)\nStandard (unfused):\n gate = x @ W_gate (GEMM 1: reads x, W_gate, writes gate)\n up = x @ W_up (GEMM 2: reads x, W_up, writes up)\n silu = gate * sigmoid(gate) (elementwise: reads gate, writes silu)\n out = silu * up (elementwise: reads silu, up, writes out)\n\nFused:\n out = SiLU(x @ W_gate) * (x @ W_up) (single kernel, no intermediate GMEM)\n```"}, "after": {"statement": "| `a` | NVFP4 E2M1 | `[M,K,L]`, K-major |\n| `b1`, `b2` | NVFP4 E2M1 | `[N,K,L]`, K-major |\n| `sfa` | FP8 E4M3FNUZ | `[M,K/16,L]`, K-major |\n| `sfb1`, `sfb2` | FP8 E4M3FNUZ | `[N,K/16,L]`, K-major |\n| `c` | FP16 | `[M,N,L]` |\n\nThe submission tuple also supplies layout-reordered copies of `sfa`, `sfb1`, and `sfb2`, plus preallocated `c`. `K` is divisible by 256; `M` and `N` must be divisible by the selected MMA tile dimensions. The correctness checker uses `rtol=1e-3` and `atol=1e-3`.\n\nThe pinned upstream files contain one dtype-label inconsistency: `task.yml` and `template.py` call the scales E4M3FNUZ, while `reference.py` constructs `torch.float8_e4m3fn`. A backend must follow the actual submission objects rather than treating those spellings as interchangeable.\n\nThe shared `a` and `sfa` inputs create an opportunity to reuse data across the two products, and a genuinely fused epilogue can avoid writing both full-precision products to global memory. Those are optimization possibilities, not task guarantees.\n\nThe public task/reference does not establish a final leaderboard, winning source, single-launch decomposition, physical load count, TMEM accumulator partition, TMA pipeline, CUTLASS schedule, or compute-/memory-bound classification. If an implementation uses `tcgen05`, TMEM, or TMA, its exact descriptors, scale layouts, synchronization, allocation, and architecture requirements must be verified from that implementation and the version-matched ISA.\n\nCompatibility is correspondingly narrower than “any dual-output operation”: a model/backend must match the two same-shaped NVFP4 products, scale storage and layouts, first-branch SiLU, FP16 output, batching, and divisibility rules. Grouped per-expert MoE execution is a different contract unless an implementation explicitly supplies that grouping layer."}, "reason": {"statement": "Keep the exact formula without converting a possible optimization into an ABI requirement.", "urls": []}} -{"path": "wiki/kernels/gated-dual-gemm.md", "before": {"statement": "```cuda\ntemplate \n__global__ void gated_dual_gemm_nvfp4(\n const nvfp4_t* __restrict__ X, // [M, K] input\n const nvfp4_t* __restrict__ W_gate, // [N, K] gate weights\n const nvfp4_t* __restrict__ W_up, // [N, K] up weights\n const fp8_t* sf_x, const fp8_t* sf_gate, const fp8_t* sf_up,\n half* __restrict__ output, // [M, N] output\n int M, int N, int K\n) {\n // Two TMEM regions: one accumulator per GEMM\n uint32_t tmem_gate = tmem_alloc(256);\n uint32_t tmem_up = tmem_alloc(256);\n\n for (int k = 0; k < K; k += BLOCK_K) {\n int stage = (k / BLOCK_K) % NUM_STAGES;\n mbarrier_wait(&tma_done[stage]);\n\n // Two MMAs per K-tile: gate and up\n // Both read same X tile, different weight tiles\n tcgen05_mma(x_smem[stage], wg_smem[stage], tmem_gate);\n tcgen05_mma(x_smem[stage], wu_smem[stage], tmem_up);\n }\n\n // Fused epilogue: SiLU(gate) * up\n float g = tmem_load(tmem_gate);\n float u = tmem_load(tmem_up);\n float s = g / (1.0f + expf(-g)); // SiLU\n output[row * N + col] = __float2half(s * u);\n\n tmem_dealloc(tmem_gate, 256);\n tmem_dealloc(tmem_up, 256);\n}\n```"}, "after": {"statement": "| `a` | NVFP4 E2M1 | `[M,K,L]`, K-major |\n| `b1`, `b2` | NVFP4 E2M1 | `[N,K,L]`, K-major |\n| `sfa` | FP8 E4M3FNUZ | `[M,K/16,L]`, K-major |\n| `sfb1`, `sfb2` | FP8 E4M3FNUZ | `[N,K/16,L]`, K-major |\n| `c` | FP16 | `[M,N,L]` |\n\nThe submission tuple also supplies layout-reordered copies of `sfa`, `sfb1`, and `sfb2`, plus preallocated `c`. `K` is divisible by 256; `M` and `N` must be divisible by the selected MMA tile dimensions. The correctness checker uses `rtol=1e-3` and `atol=1e-3`.\n\nThe pinned upstream files contain one dtype-label inconsistency: `task.yml` and `template.py` call the scales E4M3FNUZ, while `reference.py` constructs `torch.float8_e4m3fn`. A backend must follow the actual submission objects rather than treating those spellings as interchangeable.\n\nThe shared `a` and `sfa` inputs create an opportunity to reuse data across the two products, and a genuinely fused epilogue can avoid writing both full-precision products to global memory. Those are optimization possibilities, not task guarantees.\n\nThe public task/reference does not establish a final leaderboard, winning source, single-launch decomposition, physical load count, TMEM accumulator partition, TMA pipeline, CUTLASS schedule, or compute-/memory-bound classification. If an implementation uses `tcgen05`, TMEM, or TMA, its exact descriptors, scale layouts, synchronization, allocation, and architecture requirements must be verified from that implementation and the version-matched ISA.\n\nCompatibility is correspondingly narrower than “any dual-output operation”: a model/backend must match the two same-shaped NVFP4 products, scale storage and layouts, first-branch SiLU, FP16 output, batching, and divisibility rules. Grouped per-expert MoE execution is a different contract unless an implementation explicitly supplies that grouping layer."}, "reason": {"statement": "Replace non-executable hardware pseudocode with an exact host-checkable semantic reference and pinned upstream sources.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"]}} +{"path": "wiki/kernels/gated-dual-gemm.md", "before": {"statement": "```cuda\ntemplate \n__global__ void gated_dual_gemm_nvfp4(\n const nvfp4_t* __restrict__ X, // [M, K] input\n const nvfp4_t* __restrict__ W_gate, // [N, K] gate weights\n const nvfp4_t* __restrict__ W_up, // [N, K] up weights\n const fp8_t* sf_x, const fp8_t* sf_gate, const fp8_t* sf_up,\n half* __restrict__ output, // [M, N] output\n int M, int N, int K\n) {\n // Two TMEM regions: one accumulator per GEMM\n uint32_t tmem_gate = tmem_alloc(256);\n uint32_t tmem_up = tmem_alloc(256);\n\n for (int k = 0; k < K; k += BLOCK_K) {\n int stage = (k / BLOCK_K) % NUM_STAGES;\n mbarrier_wait(&tma_done[stage]);\n\n // Two MMAs per K-tile: gate and up\n // Both read same X tile, different weight tiles\n tcgen05_mma(x_smem[stage], wg_smem[stage], tmem_gate);\n tcgen05_mma(x_smem[stage], wu_smem[stage], tmem_up);\n }\n\n // Fused epilogue: SiLU(gate) * up\n float g = tmem_load(tmem_gate);\n float u = tmem_load(tmem_up);\n float s = g / (1.0f + expf(-g)); // SiLU\n output[row * N + col] = __float2half(s * u);\n\n tmem_dealloc(tmem_gate, 256);\n tmem_dealloc(tmem_up, 256);\n}\n```"}, "after": {"statement": "| `a` | NVFP4 E2M1 | `[M,K,L]`, K-major |\n| `b1`, `b2` | NVFP4 E2M1 | `[N,K,L]`, K-major |\n| `sfa` | FP8 E4M3FNUZ | `[M,K/16,L]`, K-major |\n| `sfb1`, `sfb2` | FP8 E4M3FNUZ | `[N,K/16,L]`, K-major |\n| `c` | FP16 | `[M,N,L]` |\n\nThe submission tuple also supplies layout-reordered copies of `sfa`, `sfb1`, and `sfb2`, plus preallocated `c`. `K` is divisible by 256; `M` and `N` must be divisible by the selected MMA tile dimensions. The correctness checker uses `rtol=1e-3` and `atol=1e-3`.\n\nThe pinned upstream files contain one dtype-label inconsistency: `task.yml` and `template.py` call the scales E4M3FNUZ, while `reference.py` constructs `torch.float8_e4m3fn`. A backend must follow the actual submission objects rather than treating those spellings as interchangeable.\n\nThe shared `a` and `sfa` inputs create an opportunity to reuse data across the two products, and a genuinely fused epilogue can avoid writing both full-precision products to global memory. Those are optimization possibilities, not task guarantees.\n\nThe public task/reference does not establish a final leaderboard, winning source, single-launch decomposition, physical load count, TMEM accumulator partition, TMA pipeline, CUTLASS schedule, or compute-/memory-bound classification. If an implementation uses `tcgen05`, TMEM, or TMA, its exact descriptors, scale layouts, synchronization, allocation, and architecture requirements must be verified from that implementation and the version-matched ISA.\n\nCompatibility is correspondingly narrower than “any dual-output operation”: a model/backend must match the two same-shaped NVFP4 products, scale storage and layouts, first-branch SiLU, FP16 output, batching, and divisibility rules. Grouped per-expert MoE execution is a different contract unless an implementation explicitly supplies that grouping layer."}, "reason": {"statement": "Replace non-executable hardware pseudocode with an exact host-checkable semantic reference and pinned upstream sources.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit"]}} {"path": "wiki/kernels/gated-dual-gemm.md", "before": {"statement": "1. **X reuse**: Same X tile feeds both MMAs — loaded once, used twice\n2. **TMEM dual accumulator**: Blackwell's 512-column TMEM fits two 256-col accumulators side-by-side\n3. **Fused epilogue**: SiLU and multiply happen after TMEM load, no intermediate SMEM\n4. **Shared SFA**: X's block scales apply to both gate and up computations"}, "after": {"statement": "For each batch index, the required result is two block-scaled matrix products sharing the same left operand, with SiLU applied only to the first branch:\n\n```python\ndef gated_dual_result(a, b1, b2, scale_a, scale_b1, scale_b2, scaled_mm):\n gate = scaled_mm(a, b1.T, scale_a, scale_b1)\n up = scaled_mm(a, b2.T, scale_a, scale_b2)\n return silu(gate) * up\n```\n\nThis fixes result semantics and branch order. It does not require the submission entry point to use one GPU launch or forbid intermediate storage.\n\nThe shared `a` and `sfa` inputs create an opportunity to reuse data across the two products, and a genuinely fused epilogue can avoid writing both full-precision products to global memory. Those are optimization possibilities, not task guarantees.\n\nThe public task/reference does not establish a final leaderboard, winning source, single-launch decomposition, physical load count, TMEM accumulator partition, TMA pipeline, CUTLASS schedule, or compute-/memory-bound classification. If an implementation uses `tcgen05`, TMEM, or TMA, its exact descriptors, scale layouts, synchronization, allocation, and architecture requirements must be verified from that implementation and the version-matched ISA.\n\nCompatibility is correspondingly narrower than “any dual-output operation”: a model/backend must match the two same-shaped NVFP4 products, scale storage and layouts, first-branch SiLU, FP16 output, batching, and divisibility rules. Grouped per-expert MoE execution is a different contract unless an implementation explicitly supplies that grouping layer."}, "reason": {"statement": "Separate exact shared-input semantics from unsupported schedule claims.", "urls": []}} {"path": "wiki/kernels/gated-dual-gemm.md", "before": {"statement": "- MLP layers in modern LLMs (LLaMA, Qwen, DeepSeek, Mistral)\n- Any dual-output operation sharing one input\n- MoE expert computations (expand to per-expert fused kernels)"}, "after": {"statement": "For each batch index, the required result is two block-scaled matrix products sharing the same left operand, with SiLU applied only to the first branch:\n\n```python\ndef gated_dual_result(a, b1, b2, scale_a, scale_b1, scale_b2, scaled_mm):\n gate = scaled_mm(a, b1.T, scale_a, scale_b1)\n up = scaled_mm(a, b2.T, scale_a, scale_b2)\n return silu(gate) * up\n```\n\nThis fixes result semantics and branch order. It does not require the submission entry point to use one GPU launch or forbid intermediate storage.\n\nThe shared `a` and `sfa` inputs create an opportunity to reuse data across the two products, and a genuinely fused epilogue can avoid writing both full-precision products to global memory. Those are optimization possibilities, not task guarantees.\n\nThe public task/reference does not establish a final leaderboard, winning source, single-launch decomposition, physical load count, TMEM accumulator partition, TMA pipeline, CUTLASS schedule, or compute-/memory-bound classification. If an implementation uses `tcgen05`, TMEM, or TMA, its exact descriptors, scale layouts, synchronization, allocation, and architecture requirements must be verified from that implementation and the version-matched ISA.\n\nCompatibility is correspondingly narrower than “any dual-output operation”: a model/backend must match the two same-shaped NVFP4 products, scale storage and layouts, first-branch SiLU, FP16 output, batching, and divisibility rules. Grouped per-expert MoE execution is a different contract unless an implementation explicitly supplies that grouping layer."}, "reason": {"statement": "Replace universal use advice with an explicit compatibility checklist.", "urls": []}} {"path": "wiki/kernels/gated-dual-gemm.md", "before": {"statement": "The reference bundle lives in [`artifacts/kernels/gated-dual-gemm/full/`](../../artifacts/kernels/gated-dual-gemm/full/) and combines the upstream vLLM PR-23696 diff (`vllm-PR-23696-gated-dual-gemm.patch`, `mode: upstream-patch`) with an extracted CUTLASS-schedule snippet from the `tflops-gap-fp4-moe` blog (`blackwell-cutlass-schedules-and-tma.cu`, `mode: extracted`). Labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` header) live in [`artifacts/kernels/gated-dual-gemm/variants/`](../../artifacts/kernels/gated-dual-gemm/variants/). Every file's SHA-256 and upstream-pinning metadata is in `PROVENANCE.yaml` inside each bundle."}, "after": {"statement": "The `full/` bundle contains the byte-pinned official `task.yml` from commit `c5b2f7c`; it is the problem specification, not an optimized submission. The unrelated third-party schedule extract and duplicate vLLM MXFP4 MoE patch formerly stored here were removed; the latter remains preserved in its own PR artifact collection."}, "reason": {"statement": "Preserve useful evidence with corrected scope and replace the untested variant with a discriminating semantic test.", "urls": []}} @@ -212,21 +212,21 @@ {"path": "wiki/kernels/nsa.md", "before": {"statement": "- Triton implementation has CPU launch overhead impacting small-batch decode; CUDA graph mode recommended"}, "after": {"statement": "- NSA requires a model trained for its compression, selection, window, and gate mechanism; do not treat it as a drop-in sparse mask for an arbitrary checkpoint.\n- GQA/MQA lets query heads in a group share selected blocks, matching the paper's group-centric kernel design.\n- The paper evaluates long contexts through 64K but defines no universal 32K threshold. Compare quality, sequence distribution, memory use, and measured latency or throughput on the target workload.\n- CUDA Graphs can reduce repeated-launch CPU overhead, but they are not an NSA-specific default. Profile first: NVIDIA's guidance says the largest gains occur for CPU-bound workflows and that GPU-bound workloads may see little benefit or regress.\n- The compression MLP adds learned parameters and computation. The paper does not isolate a standalone cost for that component.\n- The paper notes that a Triton implementation can retain abstraction overhead relative to native CUDA; it does not publish an NSA-specific CPU-launch profile."}, "reason": {"statement": "Replace the categorical prescription with a profile-first CUDA Graph boundary.", "urls": ["https://aclanthology.org/2025.acl-long.1126.pdf", "https://docs.nvidia.com/dl-cuda-graph/troubleshooting/performance-issues.html"]}} {"path": "wiki/kernels/nsa.md", "before": {"statement": "The core NSA kernel is implemented in Triton with group-centric data loading and grid-based scheduling.\n\nNSA's sparsity pattern is explicitly designed for GPU memory access efficiency:"}, "after": {"statement": "Native Sparse Attention is the three-branch, natively trainable sparse-attention design published in the ACL 2025 proceedings. The paper targets long-context training and inference and reports quality comparable to or better than full attention on its evaluated tasks. Those are paper-scoped results, not a guarantee for an arbitrary model or post-training conversion.\n\nThe paper's efficiency experiments use an eight-GPU A100 system. The paper does not establish Hopper or Blackwell compatibility for its Triton implementation.\n\nDeepSeek-V3.2-Exp later introduced **DeepSeek Sparse Attention (DSA)**. Its pinned first-party inference code uses a learned indexer to select up to 2,048 token positions and masks attention to those positions. Its README points to DeepGEMM for indexer-logit kernels and FlashMLA for sparse-attention kernels. FlashMLA likewise says its sparse kernels power DSA.\n\nThat deployed DSA path is related sparse-attention work, but it is not evidence that the ACL paper's gated compression/selection/window NSA architecture was deployed in V3.2-Exp."}, "reason": {"statement": "Scope architecture metadata to the paper's A100 evidence and make Blackwell an unmeasured portability question.", "urls": ["https://aclanthology.org/2025.acl-long.1126.pdf", "https://github.com/deepseek-ai/FlashMLA/blob/71c737929f2567bd0a094ae140f8f60f390b1232/README.md", "https://aclanthology.org/2025.acl-long.1126/"]}} {"path": "wiki/kernels/nsa.md", "before": {"statement": "sources: [blog-nsa, blog-flashmla, blog-vllm-deepseek-v3-sparse]"}, "after": {"statement": "sources: [blog-nsa, blog-flashmla]"}, "reason": {"statement": "Remove the vLLM community source from direct frontmatter evidence and retain FlashMLA only for the explicit non-equivalence boundary.", "urls": ["https://github.com/deepseek-ai/DeepSeek-V3.2-Exp/tree/87e509a2e5a100d221c97df52c6e8be7835f0057", "https://github.com/deepseek-ai/FlashMLA/blob/71c737929f2567bd0a094ae140f8f60f390b1232/README.md"]}} -{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "NVFP4 GEMM is a compute-bound matrix multiplication kernel operating on NVIDIA's native 4-bit floating-point format (E2M1) with block scaling on Blackwell GPUs. Unlike the memory-bound GEMV, GEMM is dominated by tensor core throughput and benefits from Blackwell's native FP4 MMA instructions via tcgen05.mma, TMA bulk loads, TMEM accumulation, and warp specialization."}, "after": {"statement": "The official NVIDIA rules identify NVFP4 GEMM as Kernel Challenge 2 of the Blackwell NVFP4 Hackathon. The contest ran from November 29 through December 19, 2025; Problem 2 targeted NVIDIA B200 and contributed 20% of the four-problem grand-prize score.\n\nThe pinned public task defines observable inputs, correctness, test shapes, benchmark shapes, and ranking. It does not publish a canonical optimized kernel, require CUTLASS, or establish which implementation mechanisms any entrant used. “NVFP4 GEMM” also does not determine a bottleneck by itself: shape, layout, staging, launch geometry, and epilogue can change whether math, memory traffic, occupancy, or launch overhead controls runtime.\n\nUse this task contract only when the packed E2M1 payloads, logical and reordered scales, transpose convention, FP16 output, tolerances, and B200 target match the workload. “Four-bit weights” alone is insufficient: INT4, MXFP4, other scale granularities, and other layouts are not interchangeable with this ABI.\n\nThe documented native tensor-core path is Blackwell-specific; Hopper has no native FP4 tensor-core instruction. This does not exclude software conversion or emulation. Choose TMA, TMEM allocation, CUTLASS schedules, warp specialization, tile sizes, and stage counts only after validating their exact API constraints and profiling the actual shapes."}, "reason": {"statement": "Scope bottlenecks and implementation mechanisms to the exact shape and implementation.", "urls": ["https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py", "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://developer.nvidia.com/blog/introducing-nvfp4-for-efficient-and-accurate-low-precision-inference/", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "NVFP4 GEMM is a compute-bound matrix multiplication kernel operating on NVIDIA's native 4-bit floating-point format (E2M1) with block scaling on Blackwell GPUs. Unlike the memory-bound GEMV, GEMM is dominated by tensor core throughput and benefits from Blackwell's native FP4 MMA instructions via tcgen05.mma, TMA bulk loads, TMEM accumulation, and warp specialization."}, "after": {"statement": "The official NVIDIA rules identify NVFP4 GEMM as Kernel Challenge 2 of the Blackwell NVFP4 Hackathon. The contest ran from November 29 through December 19, 2025; Problem 2 targeted NVIDIA B200 and contributed 20% of the four-problem grand-prize score.\n\nThe pinned public task defines observable inputs, correctness, test shapes, benchmark shapes, and ranking. It does not publish a canonical optimized kernel, require CUTLASS, or establish which implementation mechanisms any entrant used. “NVFP4 GEMM” also does not determine a bottleneck by itself: shape, layout, staging, launch geometry, and epilogue can change whether math, memory traffic, occupancy, or launch overhead controls runtime.\n\nUse this task contract only when the packed E2M1 payloads, logical and reordered scales, transpose convention, FP16 output, tolerances, and B200 target match the workload. “Four-bit weights” alone is insufficient: INT4, MXFP4, other scale granularities, and other layouts are not interchangeable with this ABI.\n\nThe documented native tensor-core path is Blackwell-specific; Hopper has no native FP4 tensor-core instruction. This does not exclude software conversion or emulation. Choose TMA, TMEM allocation, CUTLASS schedules, warp specialization, tile sizes, and stage counts only after validating their exact API constraints and profiling the actual shapes."}, "reason": {"statement": "Scope bottlenecks and implementation mechanisms to the exact shape and implementation.", "urls": ["https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py", "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://developer.nvidia.com/blog/introducing-nvfp4-for-efficient-and-accurate-low-precision-inference/", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma"]}} {"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "This kernel was Problem 2 of the GPU Mode NVFP4 Hackathon (Nov-Dec 2025), targeting B200 GPUs. Top entries achieved within 1% of cuBLAS performance using CUTLASS SM100 schedules."}, "after": {"statement": "The task ranks the geometric mean across three benchmark cases. It labels the following values a speed-of-light analysis based on the maximum of B200 FP4 Tensor Core math time and DRAM-memory time at a 1.5 GHz clock:\n\n| 128 | 7168 | 16384 | 1 | 8.994 |\n| 128 | 4096 | 7168 | 1 | 2.354 |\n| 128 | 7168 | 2048 | 1 | 1.333 |\n\nThese are theoretical comparison rows, not measured contestant latencies and not cuBLAS results.\n\nThe public Popcorn endpoint currently gives the following dated snapshot. To make its small floating-point `submission_score` values readable beside the task's microsecond presentation, the table shows `submission_score × 10^6`; it does not relabel the rows as prize placements.\n\n| 1 | `gau.nernst` | 9.981889 | 2025-12-21 00:43:03 |\n| 2 | `s.am._` | 10.060110 | 2025-12-20 17:45:21 |\n| 3 | `billcarson` | 10.137411 | 2025-12-21 03:05:32 |\n| 8 | `Simon` | 10.806750 | 2025-12-16 20:18:42 |\n| 9 | `yue` | 10.914084 | 2025-12-11 04:36:45 |\n| 10 | `currybab` | 10.930623 | 2025-12-19 08:10:18 |\n\nSnapshot fetched August 8, 2026. Because the current first three submissions postdate the official December 19 cutoff, this endpoint alone cannot establish winners or final prize rankings. It also publishes no contestant source, CUTLASS attribution, cuBLAS comparison, raw trials, or variance."}, "reason": {"statement": "Remove an untraceable performance and implementation attribution; retain only dated public fields and theoretical task rows.", "urls": ["https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml", "https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf"]}} -{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "The kernel uses the `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100` CUTLASS schedule, which combines TMA async loads with warp-specialized MMA execution."}, "after": {"statement": "The native Blackwell instruction path supports block-of-16 NVFP4 through `tcgen05.mma...kind::mxf4nvf4.block_scale.block16`. It uses UE4M3 scale elements, which CUTLASS names `float_ue4m3_t`; UE8M0 is the MXFP4 scale type. Converting NVFP4 scales to UE8M0 would discard fractional scale values and is not a required NVFP4 preprocessing step.\n\nCUTLASS defines `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100`, but its direct official example is a grouped pointer-array kernel. The symbol's existence is not evidence that the contest entrants used it. CUTLASS's current official NVFP4 examples use `nv_float4_t`, collective mainloop and epilogue builders, `kernel::GemmUniversal`, and `device::GemmUniversalAdapter`; an old scalar-template `device::GemmUniversal` sketch is not interchangeable with that API.\n\nTMA likewise has no universal 128-byte alignment rule for every operand. CUDA 13.2.1 generally requires a 16-byte-aligned global base and 16-byte-multiple strides for a tiled tensor map, with additional datatype-, interleave-, swizzle-, and mode-specific restrictions. Problem 2's `K % 256 == 0` rule belongs to the task ABI; it does not follow as a universal TMA theorem.\n\nTMEM's 128-lane by 512-column organization is a storage and addressing model, not a universal 128-by-512 logical output-tile limit. Official CUTLASS NVFP4 configurations include a cooperative two-SM MMA tile with logical shape 256 by 256 by 256."}, "reason": {"statement": "Retain the schedule only as an adjacent official CUTLASS example and explicitly deny contestant attribution.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu", "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} -{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "```cpp\n// CUTLASS dispatch for NVFP4 GEMM on Blackwell\nusing Schedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100;\n\n// Tile configuration\nusing TileShape = cute::Shape<_128, _256, _128>; // M, N, K tile\nusing ClusterShape = cute::Shape<_1, _1, _1>; // 1-SM mode\n\n// Element types\nusing ElementA = cutlass::float_e2m1_t; // NVFP4\nusing ElementB = cutlass::float_e2m1_t; // NVFP4\nusing ElementC = float; // FP32 accumulator\nusing ElementScale = cutlass::float_e4m3_t; // FP8 E4M3 block scales\n\n// Kernel definition\nusing Kernel = cutlass::gemm::device::GemmUniversal<\n ElementA, cutlass::layout::RowMajor,\n ElementB, cutlass::layout::ColumnMajor,\n ElementC, cutlass::layout::RowMajor,\n float, // accumulator type\n cutlass::arch::OpClassTensorOp,\n cutlass::arch::Sm100,\n TileShape, ClusterShape, Schedule\n>;\n```"}, "after": {"statement": "The native Blackwell instruction path supports block-of-16 NVFP4 through `tcgen05.mma...kind::mxf4nvf4.block_scale.block16`. It uses UE4M3 scale elements, which CUTLASS names `float_ue4m3_t`; UE8M0 is the MXFP4 scale type. Converting NVFP4 scales to UE8M0 would discard fractional scale values and is not a required NVFP4 preprocessing step.\n\nCUTLASS defines `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100`, but its direct official example is a grouped pointer-array kernel. The symbol's existence is not evidence that the contest entrants used it. CUTLASS's current official NVFP4 examples use `nv_float4_t`, collective mainloop and epilogue builders, `kernel::GemmUniversal`, and `device::GemmUniversalAdapter`; an old scalar-template `device::GemmUniversal` sketch is not interchangeable with that API.\n\nTMA likewise has no universal 128-byte alignment rule for every operand. CUDA 13.2.1 generally requires a 16-byte-aligned global base and 16-byte-multiple strides for a tiled tensor map, with additional datatype-, interleave-, swizzle-, and mode-specific restrictions. Problem 2's `K % 256 == 0` rule belongs to the task ABI; it does not follow as a universal TMA theorem.\n\nTMEM's 128-lane by 512-column organization is a storage and addressing model, not a universal 128-by-512 logical output-tile limit. Official CUTLASS NVFP4 configurations include a cooperative two-SM MMA tile with logical shape 256 by 256 by 256.\n\n- [Official contest rules](https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf)\n- [Pinned task definition](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml)\n- [Pinned task types](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.py)\n- [Pinned starter template](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py)\n- [Pinned correctness reference](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/reference.py)\n- [Public Popcorn leaderboard API](https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12)\n- [Transformer Engine 2.13 NVFP4 recipe](https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html)\n- [cuBLAS block-scaling formats](https://docs.nvidia.com/cuda/cublas/index.html#element-1d-block-scaling-for-fp8-and-fp4-data-types)\n- [PTX ISA 9.0 block-scaled `tcgen05.mma`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma)\n- [Pinned CUTLASS NVFP4 example](https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu)\n- [Pinned CUTLASS grouped NVFP4 example](https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu)\n- [CUDA 13.2.1 tensor-map constraints](https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html)\n\nQuery via:"}, "reason": {"statement": "Remove non-compiling pseudo-CUTLASS and link to the immutable official example instead.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu", "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} -{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "```cpp\n// Warp specialization: TMA producer + MMA consumer + epilogue\n// Shared memory holds pipelined A/B tiles + scale factors\n\nconstexpr int NUM_STAGES = 4; // Pipeline depth\n\n// Shared memory layout\nstruct SharedStorage {\n // Double-buffered across NUM_STAGES\n nvfp4_t A_smem[NUM_STAGES][BLOCK_M * BLOCK_K / 2]; // Packed FP4: 2 per byte\n nvfp4_t B_smem[NUM_STAGES][BLOCK_N * BLOCK_K / 2];\n fp8_t sfa_smem[NUM_STAGES][BLOCK_M * (BLOCK_K / 16)]; // 1 scale per 16 elements\n fp8_t sfb_smem[NUM_STAGES][BLOCK_N * (BLOCK_K / 16)];\n uint64_t mbarrier[NUM_STAGES];\n};\n\n__global__ void nvfp4_gemm_kernel(\n const nvfp4_t* A, const nvfp4_t* B,\n const fp8_t* sfa, const fp8_t* sfb,\n float sf_a_global, float sf_b_global,\n float* C, int M, int N, int K\n) {\n extern __shared__ SharedStorage smem[];\n int warp_id = threadIdx.x / 32;\n int lane_id = threadIdx.x % 32;\n\n if (warp_id == 0 && lane_id == 0) {\n // TMA producer warp: async bulk loads\n for (int k = 0; k < K; k += BLOCK_K) {\n int stage = (k / BLOCK_K) % NUM_STAGES;\n\n // TMA descriptor-based bulk load (128-byte aligned)\n asm volatile(\n \"cp.async.bulk.tensor.2d.shared::cluster.global.tile\"\n \".mbarrier::complete_tx::bytes\"\n \" [%0], [%1, {%2, %3}], [%4];\"\n :: \"r\"((uint32_t)&smem->A_smem[stage]),\n \"l\"(tma_desc_A),\n \"r\"(tile_m), \"r\"(k),\n \"r\"((uint32_t)&smem->mbarrier[stage])\n );\n // Similarly for B, sfa, sfb\n }\n } else if (warp_id == 1 && lane_id == 0) {\n // MMA consumer warp: tcgen05.mma\n uint32_t tmem_addr = tmem_alloc_cta(256); // 256 TMEM columns\n\n for (int k = 0; k < K; k += BLOCK_K) {\n int stage = (k / BLOCK_K) % NUM_STAGES;\n // Wait for TMA to complete this stage\n mbarrier_wait(&smem->mbarrier[stage]);\n\n // tcgen05.mma with native block scaling\n // Reads A/B from SMEM, accumulates into TMEM\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f8f6f4\"\n \" [%0], %1, %2, %3, %4;\"\n :: \"r\"(tmem_addr),\n \"l\"((uint64_t)&smem->A_smem[stage]),\n \"l\"((uint64_t)&smem->B_smem[stage]),\n \"r\"(packed_scales),\n \"n\"(1) // scale_D enabled\n );\n }\n\n // Signal epilogue warps\n } else {\n // Epilogue warps: read from TMEM, apply global scales, store to C\n // tmem -> registers -> global\n }\n}\n```"}, "after": {"statement": "The native Blackwell instruction path supports block-of-16 NVFP4 through `tcgen05.mma...kind::mxf4nvf4.block_scale.block16`. It uses UE4M3 scale elements, which CUTLASS names `float_ue4m3_t`; UE8M0 is the MXFP4 scale type. Converting NVFP4 scales to UE8M0 would discard fractional scale values and is not a required NVFP4 preprocessing step.\n\nCUTLASS defines `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100`, but its direct official example is a grouped pointer-array kernel. The symbol's existence is not evidence that the contest entrants used it. CUTLASS's current official NVFP4 examples use `nv_float4_t`, collective mainloop and epilogue builders, `kernel::GemmUniversal`, and `device::GemmUniversalAdapter`; an old scalar-template `device::GemmUniversal` sketch is not interchangeable with that API.\n\nTMA likewise has no universal 128-byte alignment rule for every operand. CUDA 13.2.1 generally requires a 16-byte-aligned global base and 16-byte-multiple strides for a tiled tensor map, with additional datatype-, interleave-, swizzle-, and mode-specific restrictions. Problem 2's `K % 256 == 0` rule belongs to the task ABI; it does not follow as a universal TMA theorem.\n\nTMEM's 128-lane by 512-column organization is a storage and addressing model, not a universal 128-by-512 logical output-tile limit. Official CUTLASS NVFP4 configurations include a cooperative two-SM MMA tile with logical shape 256 by 256 by 256."}, "reason": {"statement": "Remove misleading non-executable CUDA rather than imply it is a reference implementation.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cute/arch/mma_sm100_umma.hpp#L1647-L1659", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu", "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} -{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "All TMA operands require 128-byte alignment. For NVFP4 (2 elements per byte), this means K dimensions must be multiples of 256 elements:\n\n- TMA requires 128-byte alignment for all operands"}, "after": {"statement": "| `a` | packed E2M1, two values per byte | `[M,K/2,L]` |\n| `b` | packed E2M1, two values per byte | `[N,K/2,L]` |\n| `sfa` | logical E4M3 scales | `[M,K/16,L]` |\n| `sfb` | logical E4M3 scales | `[N,K/16,L]` |\n| `sfa_reordered` | MMA-oriented scale copy | `[32,4,ceil(M/128),4,K/64,L]` |\n| `sfb_reordered` | MMA-oriented scale copy | `[32,4,ceil(N/128),4,K/64,L]` |\n| `c` | preallocated FP16 output | `[M,N,L]` |\n\nFor each `L` slice, the correctness reference computes the block-scaled equivalent of `A @ B.T` and stores FP16 output. It uses `rtol=1e-3` and `atol=1e-3`.\n\nThe upstream files contain two interface inconsistencies that implementations must not conceal:\n\n- `task.yml` describes a five-member `(a,b,sfa,sfb,c)` tuple, while `task.py`, `template.py`, and `reference.py` expose the seven tensors above.\n- Task and template prose label scale tensors E4M3FNUZ, while `reference.py` constructs `torch.float8_e4m3fn` values. A submission must follow the tensors supplied by the actual harness rather than treating those suffixes as interchangeable.\n\nThe published task requires `K` divisible by 256. Divisibility of `M` and `N` depends on the submission's selected MMA tile. The following CPU-only helper reproduces the published storage shapes; it does not decode FP4, apply scales, or model a GPU kernel:\n\nThe native Blackwell instruction path supports block-of-16 NVFP4 through `tcgen05.mma...kind::mxf4nvf4.block_scale.block16`. It uses UE4M3 scale elements, which CUTLASS names `float_ue4m3_t`; UE8M0 is the MXFP4 scale type. Converting NVFP4 scales to UE8M0 would discard fractional scale values and is not a required NVFP4 preprocessing step.\n\nCUTLASS defines `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100`, but its direct official example is a grouped pointer-array kernel. The symbol's existence is not evidence that the contest entrants used it. CUTLASS's current official NVFP4 examples use `nv_float4_t`, collective mainloop and epilogue builders, `kernel::GemmUniversal`, and `device::GemmUniversalAdapter`; an old scalar-template `device::GemmUniversal` sketch is not interchangeable with that API.\n\nTMA likewise has no universal 128-byte alignment rule for every operand. CUDA 13.2.1 generally requires a 16-byte-aligned global base and 16-byte-multiple strides for a tiled tensor map, with additional datatype-, interleave-, swizzle-, and mode-specific restrictions. Problem 2's `K % 256 == 0` rule belongs to the task ABI; it does not follow as a universal TMA theorem.\n\nTMEM's 128-lane by 512-column organization is a storage and addressing model, not a universal 128-by-512 logical output-tile limit. Official CUTLASS NVFP4 configurations include a cooperative two-SM MMA tile with logical shape 256 by 256 by 256."}, "reason": {"statement": "Separate the task's K divisibility rule from actual tensor-map constraints.", "urls": ["https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} +{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "The kernel uses the `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100` CUTLASS schedule, which combines TMA async loads with warp-specialized MMA execution."}, "after": {"statement": "The native Blackwell instruction path supports block-of-16 NVFP4 through `tcgen05.mma...kind::mxf4nvf4.block_scale.block16`. It uses UE4M3 scale elements, which CUTLASS names `float_ue4m3_t`; UE8M0 is the MXFP4 scale type. Converting NVFP4 scales to UE8M0 would discard fractional scale values and is not a required NVFP4 preprocessing step.\n\nCUTLASS defines `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100`, but its direct official example is a grouped pointer-array kernel. The symbol's existence is not evidence that the contest entrants used it. CUTLASS's current official NVFP4 examples use `nv_float4_t`, collective mainloop and epilogue builders, `kernel::GemmUniversal`, and `device::GemmUniversalAdapter`; an old scalar-template `device::GemmUniversal` sketch is not interchangeable with that API.\n\nTMA likewise has no universal 128-byte alignment rule for every operand. CUDA 13.2.1 generally requires a 16-byte-aligned global base and 16-byte-multiple strides for a tiled tensor map, with additional datatype-, interleave-, swizzle-, and mode-specific restrictions. Problem 2's `K % 256 == 0` rule belongs to the task ABI; it does not follow as a universal TMA theorem.\n\nTMEM's 128-lane by 512-column organization is a storage and addressing model, not a universal 128-by-512 logical output-tile limit. Official CUTLASS NVFP4 configurations include a cooperative two-SM MMA tile with logical shape 256 by 256 by 256."}, "reason": {"statement": "Retain the schedule only as an adjacent official CUTLASS example and explicitly deny contestant attribution.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu", "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} +{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "```cpp\n// CUTLASS dispatch for NVFP4 GEMM on Blackwell\nusing Schedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100;\n\n// Tile configuration\nusing TileShape = cute::Shape<_128, _256, _128>; // M, N, K tile\nusing ClusterShape = cute::Shape<_1, _1, _1>; // 1-SM mode\n\n// Element types\nusing ElementA = cutlass::float_e2m1_t; // NVFP4\nusing ElementB = cutlass::float_e2m1_t; // NVFP4\nusing ElementC = float; // FP32 accumulator\nusing ElementScale = cutlass::float_e4m3_t; // FP8 E4M3 block scales\n\n// Kernel definition\nusing Kernel = cutlass::gemm::device::GemmUniversal<\n ElementA, cutlass::layout::RowMajor,\n ElementB, cutlass::layout::ColumnMajor,\n ElementC, cutlass::layout::RowMajor,\n float, // accumulator type\n cutlass::arch::OpClassTensorOp,\n cutlass::arch::Sm100,\n TileShape, ClusterShape, Schedule\n>;\n```"}, "after": {"statement": "The native Blackwell instruction path supports block-of-16 NVFP4 through `tcgen05.mma...kind::mxf4nvf4.block_scale.block16`. It uses UE4M3 scale elements, which CUTLASS names `float_ue4m3_t`; UE8M0 is the MXFP4 scale type. Converting NVFP4 scales to UE8M0 would discard fractional scale values and is not a required NVFP4 preprocessing step.\n\nCUTLASS defines `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100`, but its direct official example is a grouped pointer-array kernel. The symbol's existence is not evidence that the contest entrants used it. CUTLASS's current official NVFP4 examples use `nv_float4_t`, collective mainloop and epilogue builders, `kernel::GemmUniversal`, and `device::GemmUniversalAdapter`; an old scalar-template `device::GemmUniversal` sketch is not interchangeable with that API.\n\nTMA likewise has no universal 128-byte alignment rule for every operand. CUDA 13.2.1 generally requires a 16-byte-aligned global base and 16-byte-multiple strides for a tiled tensor map, with additional datatype-, interleave-, swizzle-, and mode-specific restrictions. Problem 2's `K % 256 == 0` rule belongs to the task ABI; it does not follow as a universal TMA theorem.\n\nTMEM's 128-lane by 512-column organization is a storage and addressing model, not a universal 128-by-512 logical output-tile limit. Official CUTLASS NVFP4 configurations include a cooperative two-SM MMA tile with logical shape 256 by 256 by 256.\n\n- [Official contest rules](https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf)\n- [Pinned task definition](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml)\n- [Pinned task types](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.py)\n- [Pinned starter template](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py)\n- [Pinned correctness reference](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/reference.py)\n- [Public Popcorn leaderboard API](https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12)\n- [Transformer Engine 2.13 NVFP4 recipe](https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html)\n- [cuBLAS block-scaling formats](https://docs.nvidia.com/cuda/cublas/index.html#element-1d-block-scaling-for-fp8-and-fp4-data-types)\n- [PTX ISA 9.0 block-scaled `tcgen05.mma`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma)\n- [Pinned CUTLASS NVFP4 example](https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu)\n- [Pinned CUTLASS grouped NVFP4 example](https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu)\n- [CUDA 13.2.1 tensor-map constraints](https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html)\n\nQuery via:"}, "reason": {"statement": "Remove non-compiling pseudo-CUTLASS and link to the immutable official example instead.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu", "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} +{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "```cpp\n// Warp specialization: TMA producer + MMA consumer + epilogue\n// Shared memory holds pipelined A/B tiles + scale factors\n\nconstexpr int NUM_STAGES = 4; // Pipeline depth\n\n// Shared memory layout\nstruct SharedStorage {\n // Double-buffered across NUM_STAGES\n nvfp4_t A_smem[NUM_STAGES][BLOCK_M * BLOCK_K / 2]; // Packed FP4: 2 per byte\n nvfp4_t B_smem[NUM_STAGES][BLOCK_N * BLOCK_K / 2];\n fp8_t sfa_smem[NUM_STAGES][BLOCK_M * (BLOCK_K / 16)]; // 1 scale per 16 elements\n fp8_t sfb_smem[NUM_STAGES][BLOCK_N * (BLOCK_K / 16)];\n uint64_t mbarrier[NUM_STAGES];\n};\n\n__global__ void nvfp4_gemm_kernel(\n const nvfp4_t* A, const nvfp4_t* B,\n const fp8_t* sfa, const fp8_t* sfb,\n float sf_a_global, float sf_b_global,\n float* C, int M, int N, int K\n) {\n extern __shared__ SharedStorage smem[];\n int warp_id = threadIdx.x / 32;\n int lane_id = threadIdx.x % 32;\n\n if (warp_id == 0 && lane_id == 0) {\n // TMA producer warp: async bulk loads\n for (int k = 0; k < K; k += BLOCK_K) {\n int stage = (k / BLOCK_K) % NUM_STAGES;\n\n // TMA descriptor-based bulk load (128-byte aligned)\n asm volatile(\n \"cp.async.bulk.tensor.2d.shared::cluster.global.tile\"\n \".mbarrier::complete_tx::bytes\"\n \" [%0], [%1, {%2, %3}], [%4];\"\n :: \"r\"((uint32_t)&smem->A_smem[stage]),\n \"l\"(tma_desc_A),\n \"r\"(tile_m), \"r\"(k),\n \"r\"((uint32_t)&smem->mbarrier[stage])\n );\n // Similarly for B, sfa, sfb\n }\n } else if (warp_id == 1 && lane_id == 0) {\n // MMA consumer warp: tcgen05.mma\n uint32_t tmem_addr = tmem_alloc_cta(256); // 256 TMEM columns\n\n for (int k = 0; k < K; k += BLOCK_K) {\n int stage = (k / BLOCK_K) % NUM_STAGES;\n // Wait for TMA to complete this stage\n mbarrier_wait(&smem->mbarrier[stage]);\n\n // tcgen05.mma with native block scaling\n // Reads A/B from SMEM, accumulates into TMEM\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f8f6f4\"\n \" [%0], %1, %2, %3, %4;\"\n :: \"r\"(tmem_addr),\n \"l\"((uint64_t)&smem->A_smem[stage]),\n \"l\"((uint64_t)&smem->B_smem[stage]),\n \"r\"(packed_scales),\n \"n\"(1) // scale_D enabled\n );\n }\n\n // Signal epilogue warps\n } else {\n // Epilogue warps: read from TMEM, apply global scales, store to C\n // tmem -> registers -> global\n }\n}\n```"}, "after": {"statement": "The native Blackwell instruction path supports block-of-16 NVFP4 through `tcgen05.mma...kind::mxf4nvf4.block_scale.block16`. It uses UE4M3 scale elements, which CUTLASS names `float_ue4m3_t`; UE8M0 is the MXFP4 scale type. Converting NVFP4 scales to UE8M0 would discard fractional scale values and is not a required NVFP4 preprocessing step.\n\nCUTLASS defines `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100`, but its direct official example is a grouped pointer-array kernel. The symbol's existence is not evidence that the contest entrants used it. CUTLASS's current official NVFP4 examples use `nv_float4_t`, collective mainloop and epilogue builders, `kernel::GemmUniversal`, and `device::GemmUniversalAdapter`; an old scalar-template `device::GemmUniversal` sketch is not interchangeable with that API.\n\nTMA likewise has no universal 128-byte alignment rule for every operand. CUDA 13.2.1 generally requires a 16-byte-aligned global base and 16-byte-multiple strides for a tiled tensor map, with additional datatype-, interleave-, swizzle-, and mode-specific restrictions. Problem 2's `K % 256 == 0` rule belongs to the task ABI; it does not follow as a universal TMA theorem.\n\nTMEM's 128-lane by 512-column organization is a storage and addressing model, not a universal 128-by-512 logical output-tile limit. Official CUTLASS NVFP4 configurations include a cooperative two-SM MMA tile with logical shape 256 by 256 by 256."}, "reason": {"statement": "Remove misleading non-executable CUDA rather than imply it is a reference implementation.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cute/arch/mma_sm100_umma.hpp#L1647-L1659", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu", "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} +{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "All TMA operands require 128-byte alignment. For NVFP4 (2 elements per byte), this means K dimensions must be multiples of 256 elements:\n\n- TMA requires 128-byte alignment for all operands"}, "after": {"statement": "| `a` | packed E2M1, two values per byte | `[M,K/2,L]` |\n| `b` | packed E2M1, two values per byte | `[N,K/2,L]` |\n| `sfa` | logical E4M3 scales | `[M,K/16,L]` |\n| `sfb` | logical E4M3 scales | `[N,K/16,L]` |\n| `sfa_reordered` | MMA-oriented scale copy | `[32,4,ceil(M/128),4,K/64,L]` |\n| `sfb_reordered` | MMA-oriented scale copy | `[32,4,ceil(N/128),4,K/64,L]` |\n| `c` | preallocated FP16 output | `[M,N,L]` |\n\nFor each `L` slice, the correctness reference computes the block-scaled equivalent of `A @ B.T` and stores FP16 output. It uses `rtol=1e-3` and `atol=1e-3`.\n\nThe upstream files contain two interface inconsistencies that implementations must not conceal:\n\n- `task.yml` describes a five-member `(a,b,sfa,sfb,c)` tuple, while `task.py`, `template.py`, and `reference.py` expose the seven tensors above.\n- Task and template prose label scale tensors E4M3FNUZ, while `reference.py` constructs `torch.float8_e4m3fn` values. A submission must follow the tensors supplied by the actual harness rather than treating those suffixes as interchangeable.\n\nThe published task requires `K` divisible by 256. Divisibility of `M` and `N` depends on the submission's selected MMA tile. The following CPU-only helper reproduces the published storage shapes; it does not decode FP4, apply scales, or model a GPU kernel:\n\nThe native Blackwell instruction path supports block-of-16 NVFP4 through `tcgen05.mma...kind::mxf4nvf4.block_scale.block16`. It uses UE4M3 scale elements, which CUTLASS names `float_ue4m3_t`; UE8M0 is the MXFP4 scale type. Converting NVFP4 scales to UE8M0 would discard fractional scale values and is not a required NVFP4 preprocessing step.\n\nCUTLASS defines `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100`, but its direct official example is a grouped pointer-array kernel. The symbol's existence is not evidence that the contest entrants used it. CUTLASS's current official NVFP4 examples use `nv_float4_t`, collective mainloop and epilogue builders, `kernel::GemmUniversal`, and `device::GemmUniversalAdapter`; an old scalar-template `device::GemmUniversal` sketch is not interchangeable with that API.\n\nTMA likewise has no universal 128-byte alignment rule for every operand. CUDA 13.2.1 generally requires a 16-byte-aligned global base and 16-byte-multiple strides for a tiled tensor map, with additional datatype-, interleave-, swizzle-, and mode-specific restrictions. Problem 2's `K % 256 == 0` rule belongs to the task ABI; it does not follow as a universal TMA theorem.\n\nTMEM's 128-lane by 512-column organization is a storage and addressing model, not a universal 128-by-512 logical output-tile limit. Official CUTLASS NVFP4 configurations include a cooperative two-SM MMA tile with logical shape 256 by 256 by 256."}, "reason": {"statement": "Separate the task's K divisibility rule from actual tensor-map constraints.", "urls": ["https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} {"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "```cpp\n// Critical: pad tensors to 128-byte boundaries for TMA\n// NVFP4: 2 elements per byte, so 256 elements = 128 bytes\nstatic_assert(K % 256 == 0, \"K must align to 128 bytes for FP4 TMA\");\n\n// For scale factors: FP8 is 1 byte per element\n// 128 bytes = 128 scale values\n// Since 1 scale per 16 FP4 elements: 128 scales cover 2048 FP4 elements\nstatic_assert((K / 16) % 128 == 0, \"Scale array must align to 128 bytes\");\n```"}, "after": {"statement": "| `a` | packed E2M1, two values per byte | `[M,K/2,L]` |\n| `b` | packed E2M1, two values per byte | `[N,K/2,L]` |\n| `sfa` | logical E4M3 scales | `[M,K/16,L]` |\n| `sfb` | logical E4M3 scales | `[N,K/16,L]` |\n| `sfa_reordered` | MMA-oriented scale copy | `[32,4,ceil(M/128),4,K/64,L]` |\n| `sfb_reordered` | MMA-oriented scale copy | `[32,4,ceil(N/128),4,K/64,L]` |\n| `c` | preallocated FP16 output | `[M,N,L]` |\n\nFor each `L` slice, the correctness reference computes the block-scaled equivalent of `A @ B.T` and stores FP16 output. It uses `rtol=1e-3` and `atol=1e-3`.\n\nThe upstream files contain two interface inconsistencies that implementations must not conceal:\n\n- `task.yml` describes a five-member `(a,b,sfa,sfb,c)` tuple, while `task.py`, `template.py`, and `reference.py` expose the seven tensors above.\n- Task and template prose label scale tensors E4M3FNUZ, while `reference.py` constructs `torch.float8_e4m3fn` values. A submission must follow the tensors supplied by the actual harness rather than treating those suffixes as interchangeable.\n\nThe published task requires `K` divisible by 256. Divisibility of `M` and `N` depends on the submission's selected MMA tile. The following CPU-only helper reproduces the published storage shapes; it does not decode FP4, apply scales, or model a GPU kernel:\n\n```python\ndef task_storage_shapes(m, n, k, l=1):\n if min(m, n, k, l) <= 0:\n raise ValueError(\"dimensions must be positive\")\n if k % 256:\n raise ValueError(\"K must be divisible by 256\")\n\n return {\n \"a_packed\": (m, k // 2, l),\n \"b_packed\": (n, k // 2, l),\n \"sfa_logical\": (m, k // 16, l),\n \"sfb_logical\": (n, k // 16, l),\n \"sfa_reordered\": (32, 4, (m + 127) // 128, 4, (k + 63) // 64, l),\n \"sfb_reordered\": (32, 4, (n + 127) // 128, 4, (k + 63) // 64, l),\n \"c\": (m, n, l),\n }\n\n\nfor valid_k in (256, 512, 1536, 2048, 2304, 7168, 16384):\n task_storage_shapes(128, 256, valid_k)\n\nassert (256 // 16) % 128 != 0 # the former scale-array assertion was invalid\n```\n\nNine of the ten official correctness shapes fail the former `(K/16) % 128 == 0` assertion, including the smallest valid case with `K=256`."}, "reason": {"statement": "Replace the false assertion with a host-checkable representation of the published tensor shapes.", "urls": []}} -{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "The tcgen05.mma instruction expects UE8M0 (unsigned power-of-two exponent only) scales, but NVFP4 uses FP8 E4M3 (non-power-of-two). Conversion is needed:\n\n```cpp\n// Convert FP8 E4M3 block scales to UE8M0 for tcgen05.mma hardware\n// E4M3: 4 exponent bits, 3 mantissa bits (non-power-of-two)\n// UE8M0: 8 exponent bits, 0 mantissa bits (power-of-two only)\n__device__ uint32_t pack_scales_ue8m0(\n fp8_e4m3_t s0, fp8_e4m3_t s1, fp8_e4m3_t s2, fp8_e4m3_t s3\n) {\n uint8_t u0 = fp8_e4m3_to_ue8m0(s0); // Round to nearest power-of-two\n uint8_t u1 = fp8_e4m3_to_ue8m0(s1);\n uint8_t u2 = fp8_e4m3_to_ue8m0(s2);\n uint8_t u3 = fp8_e4m3_to_ue8m0(s3);\n return (u3 << 24) | (u2 << 16) | (u1 << 8) | u0;\n}\n```\n\n- Scale factor conversion (E4M3 to UE8M0) adds overhead if not precomputed"}, "after": {"statement": "The generic one-dimensional NVFP4 recipe reconstructs each value from a signed E2M1 payload, one E4M3 local scale per 16 consecutive payloads, and a per-tensor FP32 global scale. E2M1 represents zero and signed magnitudes 0.5, 1, 1.5, 2, 3, 4, and 6, with two payloads packed per byte. MXFP4 is different: its local groups contain 32 payloads and use power-of-two UE8M0 scales.\n\nThe contest ABI is narrower and does not expose the generic recipe's FP32 global scales. Its actual generated input has seven tensors:\n\nThe native Blackwell instruction path supports block-of-16 NVFP4 through `tcgen05.mma...kind::mxf4nvf4.block_scale.block16`. It uses UE4M3 scale elements, which CUTLASS names `float_ue4m3_t`; UE8M0 is the MXFP4 scale type. Converting NVFP4 scales to UE8M0 would discard fractional scale values and is not a required NVFP4 preprocessing step.\n\nCUTLASS defines `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100`, but its direct official example is a grouped pointer-array kernel. The symbol's existence is not evidence that the contest entrants used it. CUTLASS's current official NVFP4 examples use `nv_float4_t`, collective mainloop and epilogue builders, `kernel::GemmUniversal`, and `device::GemmUniversalAdapter`; an old scalar-template `device::GemmUniversal` sketch is not interchangeable with that API.\n\nTMA likewise has no universal 128-byte alignment rule for every operand. CUDA 13.2.1 generally requires a 16-byte-aligned global base and 16-byte-multiple strides for a tiled tensor map, with additional datatype-, interleave-, swizzle-, and mode-specific restrictions. Problem 2's `K % 256 == 0` rule belongs to the task ABI; it does not follow as a universal TMA theorem.\n\nTMEM's 128-lane by 512-column organization is a storage and addressing model, not a universal 128-by-512 logical output-tile limit. Official CUTLASS NVFP4 configurations include a cooperative two-SM MMA tile with logical shape 256 by 256 by 256."}, "reason": {"statement": "Remove a format-confusing conversion and document the exact UE4M3 scale path.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512", "https://docs.nvidia.com/cuda/cublas/index.html#element-1d-block-scaling-for-fp8-and-fp4-data-types", "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu", "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} +{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "The tcgen05.mma instruction expects UE8M0 (unsigned power-of-two exponent only) scales, but NVFP4 uses FP8 E4M3 (non-power-of-two). Conversion is needed:\n\n```cpp\n// Convert FP8 E4M3 block scales to UE8M0 for tcgen05.mma hardware\n// E4M3: 4 exponent bits, 3 mantissa bits (non-power-of-two)\n// UE8M0: 8 exponent bits, 0 mantissa bits (power-of-two only)\n__device__ uint32_t pack_scales_ue8m0(\n fp8_e4m3_t s0, fp8_e4m3_t s1, fp8_e4m3_t s2, fp8_e4m3_t s3\n) {\n uint8_t u0 = fp8_e4m3_to_ue8m0(s0); // Round to nearest power-of-two\n uint8_t u1 = fp8_e4m3_to_ue8m0(s1);\n uint8_t u2 = fp8_e4m3_to_ue8m0(s2);\n uint8_t u3 = fp8_e4m3_to_ue8m0(s3);\n return (u3 << 24) | (u2 << 16) | (u1 << 8) | u0;\n}\n```\n\n- Scale factor conversion (E4M3 to UE8M0) adds overhead if not precomputed"}, "after": {"statement": "The generic one-dimensional NVFP4 recipe reconstructs each value from a signed E2M1 payload, one E4M3 local scale per 16 consecutive payloads, and a per-tensor FP32 global scale. E2M1 represents zero and signed magnitudes 0.5, 1, 1.5, 2, 3, 4, and 6, with two payloads packed per byte. MXFP4 is different: its local groups contain 32 payloads and use power-of-two UE8M0 scales.\n\nThe contest ABI is narrower and does not expose the generic recipe's FP32 global scales. Its actual generated input has seven tensors:\n\nThe native Blackwell instruction path supports block-of-16 NVFP4 through `tcgen05.mma...kind::mxf4nvf4.block_scale.block16`. It uses UE4M3 scale elements, which CUTLASS names `float_ue4m3_t`; UE8M0 is the MXFP4 scale type. Converting NVFP4 scales to UE8M0 would discard fractional scale values and is not a required NVFP4 preprocessing step.\n\nCUTLASS defines `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100`, but its direct official example is a grouped pointer-array kernel. The symbol's existence is not evidence that the contest entrants used it. CUTLASS's current official NVFP4 examples use `nv_float4_t`, collective mainloop and epilogue builders, `kernel::GemmUniversal`, and `device::GemmUniversalAdapter`; an old scalar-template `device::GemmUniversal` sketch is not interchangeable with that API.\n\nTMA likewise has no universal 128-byte alignment rule for every operand. CUDA 13.2.1 generally requires a 16-byte-aligned global base and 16-byte-multiple strides for a tiled tensor map, with additional datatype-, interleave-, swizzle-, and mode-specific restrictions. Problem 2's `K % 256 == 0` rule belongs to the task ABI; it does not follow as a universal TMA theorem.\n\nTMEM's 128-lane by 512-column organization is a storage and addressing model, not a universal 128-by-512 logical output-tile limit. Official CUTLASS NVFP4 configurations include a cooperative two-SM MMA tile with logical shape 256 by 256 by 256."}, "reason": {"statement": "Remove a format-confusing conversion and document the exact UE4M3 scale path.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512", "https://docs.nvidia.com/cuda/cublas/index.html#element-1d-block-scaling-for-fp8-and-fp4-data-types", "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu", "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} {"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "Problem 2 top performers (geometric mean across benchmark configs):\n\n| 1 | Simon | 10.807 |\n| 2 | yue | 10.914 |\n| 3 | currybab | 10.931 |"}, "after": {"statement": "The public Popcorn endpoint currently gives the following dated snapshot. To make its small floating-point `submission_score` values readable beside the task's microsecond presentation, the table shows `submission_score × 10^6`; it does not relabel the rows as prize placements.\n\n| 1 | `gau.nernst` | 9.981889 | 2025-12-21 00:43:03 |\n| 2 | `s.am._` | 10.060110 | 2025-12-20 17:45:21 |\n| 3 | `billcarson` | 10.137411 | 2025-12-21 03:05:32 |\n| 8 | `Simon` | 10.806750 | 2025-12-16 20:18:42 |\n| 9 | `yue` | 10.914084 | 2025-12-11 04:36:45 |\n| 10 | `currybab` | 10.930623 | 2025-12-19 08:10:18 |\n\nSnapshot fetched August 8, 2026. Because the current first three submissions postdate the official December 19 cutoff, this endpoint alone cannot establish winners or final prize rankings. It also publishes no contestant source, CUTLASS attribution, cuBLAS comparison, raw trials, or variance."}, "reason": {"statement": "Publish only a dated current snapshot and explicitly separate it from official prize placements.", "urls": ["https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12", "https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf"]}} -{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "- Inference with 4-bit quantized weights on Blackwell"}, "after": {"statement": "Use this task contract only when the packed E2M1 payloads, logical and reordered scales, transpose convention, FP16 output, tolerances, and B200 target match the workload. “Four-bit weights” alone is insufficient: INT4, MXFP4, other scale granularities, and other layouts are not interchangeable with this ABI.\n\nThe documented native tensor-core path is Blackwell-specific; Hopper has no native FP4 tensor-core instruction. This does not exclude software conversion or emulation. Choose TMA, TMEM allocation, CUTLASS schedules, warp specialization, tile sizes, and stage counts only after validating their exact API constraints and profiling the actual shapes."}, "reason": {"statement": "Scope use to operands prepared for the exact NVFP4 ABI.", "urls": ["https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py", "https://developer.nvidia.com/blog/introducing-nvfp4-for-efficient-and-accurate-low-precision-inference/", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} -{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "- TMEM size (128x512 per SM) limits maximum output tile to 128 rows x 512 cols (32-bit)"}, "after": {"statement": "The native Blackwell instruction path supports block-of-16 NVFP4 through `tcgen05.mma...kind::mxf4nvf4.block_scale.block16`. It uses UE4M3 scale elements, which CUTLASS names `float_ue4m3_t`; UE8M0 is the MXFP4 scale type. Converting NVFP4 scales to UE8M0 would discard fractional scale values and is not a required NVFP4 preprocessing step.\n\nCUTLASS defines `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100`, but its direct official example is a grouped pointer-array kernel. The symbol's existence is not evidence that the contest entrants used it. CUTLASS's current official NVFP4 examples use `nv_float4_t`, collective mainloop and epilogue builders, `kernel::GemmUniversal`, and `device::GemmUniversalAdapter`; an old scalar-template `device::GemmUniversal` sketch is not interchangeable with that API.\n\nTMA likewise has no universal 128-byte alignment rule for every operand. CUDA 13.2.1 generally requires a 16-byte-aligned global base and 16-byte-multiple strides for a tiled tensor map, with additional datatype-, interleave-, swizzle-, and mode-specific restrictions. Problem 2's `K % 256 == 0` rule belongs to the task ABI; it does not follow as a universal TMA theorem.\n\nTMEM's 128-lane by 512-column organization is a storage and addressing model, not a universal 128-by-512 logical output-tile limit. Official CUTLASS NVFP4 configurations include a cooperative two-SM MMA tile with logical shape 256 by 256 by 256.\n\nUse this task contract only when the packed E2M1 payloads, logical and reordered scales, transpose convention, FP16 output, tolerances, and B200 target match the workload. “Four-bit weights” alone is insufficient: INT4, MXFP4, other scale granularities, and other layouts are not interchangeable with this ABI.\n\nThe documented native tensor-core path is Blackwell-specific; Hopper has no native FP4 tensor-core instruction. This does not exclude software conversion or emulation. Choose TMA, TMEM allocation, CUTLASS schedules, warp specialization, tile sizes, and stage counts only after validating their exact API constraints and profiling the actual shapes."}, "reason": {"statement": "Describe TMEM storage separately from implementation tile shapes.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu#L147-L159", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu", "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py", "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://developer.nvidia.com/blog/introducing-nvfp4-for-efficient-and-accurate-low-precision-inference/"]}} -{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "The reference bundle lives in [`artifacts/kernels/nvfp4-gemm/full/`](../../artifacts/kernels/nvfp4-gemm/full/) and combines the upstream PR-2139 diff (`PR-2139-blockwise-groupwise-gemm.patch`, `mode: upstream-patch`, SHA-pinned to `ca4fdbea` on NVIDIA/cutlass) with an extracted CUTLASS-schedule / TMA snippet from the `tflops-gap-fp4-moe` blog (`blackwell-cutlass-schedules-and-tma.cu`, `mode: extracted`). Labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` header) live in [`artifacts/kernels/nvfp4-gemm/variants/`](../../artifacts/kernels/nvfp4-gemm/variants/). Every file's SHA-256 and upstream-pinning metadata is in `PROVENANCE.yaml` inside each bundle."}, "after": {"statement": "- [Official contest rules](https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf)\n- [Pinned task definition](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml)\n- [Pinned task types](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.py)\n- [Pinned starter template](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py)\n- [Pinned correctness reference](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/reference.py)\n- [Public Popcorn leaderboard API](https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12)\n- [Transformer Engine 2.13 NVFP4 recipe](https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html)\n- [cuBLAS block-scaling formats](https://docs.nvidia.com/cuda/cublas/index.html#element-1d-block-scaling-for-fp8-and-fp4-data-types)\n- [PTX ISA 9.0 block-scaled `tcgen05.mma`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma)\n- [Pinned CUTLASS NVFP4 example](https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu)\n- [Pinned CUTLASS grouped NVFP4 example](https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu)\n- [CUDA 13.2.1 tensor-map constraints](https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html)\n\nQuery via:"}, "reason": {"statement": "Detach the artifact_dir and remove the Full Reference Implementation section while preserving audit evidence and files.", "urls": []}} +{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "- Inference with 4-bit quantized weights on Blackwell"}, "after": {"statement": "Use this task contract only when the packed E2M1 payloads, logical and reordered scales, transpose convention, FP16 output, tolerances, and B200 target match the workload. “Four-bit weights” alone is insufficient: INT4, MXFP4, other scale granularities, and other layouts are not interchangeable with this ABI.\n\nThe documented native tensor-core path is Blackwell-specific; Hopper has no native FP4 tensor-core instruction. This does not exclude software conversion or emulation. Choose TMA, TMEM allocation, CUTLASS schedules, warp specialization, tile sizes, and stage counts only after validating their exact API constraints and profiling the actual shapes."}, "reason": {"statement": "Scope use to operands prepared for the exact NVFP4 ABI.", "urls": ["https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py", "https://developer.nvidia.com/blog/introducing-nvfp4-for-efficient-and-accurate-low-precision-inference/", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma"]}} +{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "- TMEM size (128x512 per SM) limits maximum output tile to 128 rows x 512 cols (32-bit)"}, "after": {"statement": "The native Blackwell instruction path supports block-of-16 NVFP4 through `tcgen05.mma...kind::mxf4nvf4.block_scale.block16`. It uses UE4M3 scale elements, which CUTLASS names `float_ue4m3_t`; UE8M0 is the MXFP4 scale type. Converting NVFP4 scales to UE8M0 would discard fractional scale values and is not a required NVFP4 preprocessing step.\n\nCUTLASS defines `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100`, but its direct official example is a grouped pointer-array kernel. The symbol's existence is not evidence that the contest entrants used it. CUTLASS's current official NVFP4 examples use `nv_float4_t`, collective mainloop and epilogue builders, `kernel::GemmUniversal`, and `device::GemmUniversalAdapter`; an old scalar-template `device::GemmUniversal` sketch is not interchangeable with that API.\n\nTMA likewise has no universal 128-byte alignment rule for every operand. CUDA 13.2.1 generally requires a 16-byte-aligned global base and 16-byte-multiple strides for a tiled tensor map, with additional datatype-, interleave-, swizzle-, and mode-specific restrictions. Problem 2's `K % 256 == 0` rule belongs to the task ABI; it does not follow as a universal TMA theorem.\n\nTMEM's 128-lane by 512-column organization is a storage and addressing model, not a universal 128-by-512 logical output-tile limit. Official CUTLASS NVFP4 configurations include a cooperative two-SM MMA tile with logical shape 256 by 256 by 256.\n\nUse this task contract only when the packed E2M1 payloads, logical and reordered scales, transpose convention, FP16 output, tolerances, and B200 target match the workload. “Four-bit weights” alone is insufficient: INT4, MXFP4, other scale granularities, and other layouts are not interchangeable with this ABI.\n\nThe documented native tensor-core path is Blackwell-specific; Hopper has no native FP4 tensor-core instruction. This does not exclude software conversion or emulation. Choose TMA, TMEM allocation, CUTLASS schedules, warp specialization, tile sizes, and stage counts only after validating their exact API constraints and profiling the actual shapes."}, "reason": {"statement": "Describe TMEM storage separately from implementation tile shapes.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu#L147-L159", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu", "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py", "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://developer.nvidia.com/blog/introducing-nvfp4-for-efficient-and-accurate-low-precision-inference/"]}} +{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "The reference bundle lives in [`artifacts/kernels/nvfp4-gemm/full/`](../../artifacts/kernels/nvfp4-gemm/full/) and combines the upstream PR-2139 diff (`PR-2139-blockwise-groupwise-gemm.patch`, `mode: upstream-patch`, SHA-pinned to `ca4fdbea` on NVIDIA/cutlass) with an extracted CUTLASS-schedule / TMA snippet from the `tflops-gap-fp4-moe` blog (`blackwell-cutlass-schedules-and-tma.cu`, `mode: extracted`). Labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` header) live in [`artifacts/kernels/nvfp4-gemm/variants/`](../../artifacts/kernels/nvfp4-gemm/variants/). Every file's SHA-256 and upstream-pinning metadata is in `PROVENANCE.yaml` inside each bundle."}, "after": {"statement": "- [Official contest rules](https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf)\n- [Pinned task definition](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml)\n- [Pinned task types](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.py)\n- [Pinned starter template](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py)\n- [Pinned correctness reference](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/reference.py)\n- [Public Popcorn leaderboard API](https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12)\n- [Transformer Engine 2.13 NVFP4 recipe](https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html)\n- [cuBLAS block-scaling formats](https://docs.nvidia.com/cuda/cublas/index.html#element-1d-block-scaling-for-fp8-and-fp4-data-types)\n- [PTX ISA 9.0 block-scaled `tcgen05.mma`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma)\n- [Pinned CUTLASS NVFP4 example](https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu)\n- [Pinned CUTLASS grouped NVFP4 example](https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu)\n- [CUDA 13.2.1 tensor-map constraints](https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html)\n\nQuery via:"}, "reason": {"statement": "Detach the artifact_dir and remove the Full Reference Implementation section while preserving audit evidence and files.", "urls": []}} {"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "performance_claims:\n- gpu: B200\n dtype: nvfp4\n shape: standard GEMM configs\n metric: latency_us\n value: 10.807\n utilization: near cuBLAS\n source_id: contest-gpumode-p2"}, "after": {"statement": "performance_claims: []\n\nThe task ranks the geometric mean across three benchmark cases. It labels the following values a speed-of-light analysis based on the maximum of B200 FP4 Tensor Core math time and DRAM-memory time at a 1.5 GHz clock:\n\n| 128 | 7168 | 16384 | 1 | 8.994 |\n| 128 | 4096 | 7168 | 1 | 2.354 |\n| 128 | 7168 | 2048 | 1 | 1.333 |\n\nThese are theoretical comparison rows, not measured contestant latencies and not cuBLAS results.\n\nThe public Popcorn endpoint currently gives the following dated snapshot. To make its small floating-point `submission_score` values readable beside the task's microsecond presentation, the table shows `submission_score × 10^6`; it does not relabel the rows as prize placements.\n\n| 1 | `gau.nernst` | 9.981889 | 2025-12-21 00:43:03 |\n| 2 | `s.am._` | 10.060110 | 2025-12-20 17:45:21 |\n| 3 | `billcarson` | 10.137411 | 2025-12-21 03:05:32 |\n| 8 | `Simon` | 10.806750 | 2025-12-16 20:18:42 |\n| 9 | `yue` | 10.914084 | 2025-12-11 04:36:45 |\n| 10 | `currybab` | 10.930623 | 2025-12-19 08:10:18 |\n\nSnapshot fetched August 8, 2026. Because the current first three submissions postdate the official December 19 cutoff, this endpoint alone cannot establish winners or final prize rankings. It also publishes no contestant source, CUTLASS attribution, cuBLAS comparison, raw trials, or variance."}, "reason": {"statement": "Remove malformed performance metadata and retain exact theoretical rows plus a dated leaderboard snapshot in prose.", "urls": ["https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml", "https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf"]}} -{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "sources:\n- contest-gpumode-p2\n- doc-cutlass-blackwell\n- pr-cutlass-2139\n\nartifact_dir: artifacts/kernels/nvfp4-gemm"}, "after": {"statement": "sources:\n- contest-gpumode-p2\n- doc-transformer-engine-2.13-nvfp4\n- doc-ptx-isa-sm100\n- doc-cuda-13-0-2-tma"}, "reason": {"statement": "Use claim-directed task, format, PTX, and TMA source cards and detach the invalid bundle.", "urls": ["https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml", "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu", "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} -{"path": "wiki/hardware/nvfp4.md", "before": {"statement": "sources: [doc-nvidia-tuning-guide, contest-gpumode-p1, contest-gpumode-p2, blog-yue-nvfp4]"}, "after": {"statement": "sources: [doc-transformer-engine-2.13-nvfp4, doc-ptx-isa-sm100]\n\nevidence_basis:\n - source_id: doc-transformer-engine-2.13-nvfp4\n evidence_type: official-doc\n - source_id: doc-ptx-isa-sm100\n evidence_type: official-doc"}, "reason": {"statement": "Remove the misleading evidence relationship while retaining the verified source-scoped hardware prose unchanged.", "urls": ["https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#alternate-floating-point-data-formats", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/kernels/nvfp4-gemm.md", "before": {"statement": "sources:\n- contest-gpumode-p2\n- doc-cutlass-blackwell\n- pr-cutlass-2139\n\nartifact_dir: artifacts/kernels/nvfp4-gemm"}, "after": {"statement": "sources:\n- contest-gpumode-p2\n- doc-transformer-engine-2.13-nvfp4\n- doc-ptx-isa-sm100\n- doc-cuda-13-0-2-tma"}, "reason": {"statement": "Use claim-directed task, format, PTX, and TMA source cards and detach the invalid bundle.", "urls": ["https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml", "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/include/cutlass/float_subbyte.h#L494-L512", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu", "https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu", "https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} +{"path": "wiki/hardware/nvfp4.md", "before": {"statement": "sources: [doc-nvidia-tuning-guide, contest-gpumode-p1, contest-gpumode-p2, blog-yue-nvfp4]"}, "after": {"statement": "sources: [doc-transformer-engine-2.13-nvfp4, doc-ptx-isa-sm100]\n\nevidence_basis:\n - source_id: doc-transformer-engine-2.13-nvfp4\n evidence_type: official-doc\n - source_id: doc-ptx-isa-sm100\n evidence_type: official-doc"}, "reason": {"statement": "Remove the misleading evidence relationship while retaining the verified source-scoped hardware prose unchanged.", "urls": ["https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#alternate-floating-point-data-formats", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma"]}} {"path": "wiki/kernels/nvfp4-gemv.md", "before": {"statement": "NVFP4 Batched GEMV is a memory-bound kernel computing batched matrix-vector products with NVFP4 (E2M1) block-scaled inputs on B200 GPUs. Unlike compute-bound GEMM, GEMV is dominated by memory bandwidth utilization (each FP4 element is used only once in the dot product), making PTX-level memory access control, cache policy differentiation, and register budgeting the critical optimization levers."}, "after": {"statement": "GPU Mode's Problem 1 asks for a block-scaled NVFP4 batched matrix-vector product on NVIDIA B200. The official task models its three benchmark cases against the slower of FFMA math and DRAM transfer time and reports DRAM-limited theoretical times. That task-specific model does not make every GEMV implementation or shape bandwidth-bound.\n\nPTX ISA 9.0 defines the packed conversion and typed register decomposition used in Yue's author-reported decode path:\n\n```asm\ncvt.rn.f16x2.e2m1x2 %result, %packed_fp4_pair;\nmov.b32 {%b0, %b1, %b2, %b3}, %packed_word;\n```\n\nIt also defines vector loads such as `ld.global.v2.u64` and `ld.global.v4.u64`, which move 16 and 32 bytes and can carry 32 and 64 packed FP4 values. The ISA specifies behavior, not that one width or decomposition is universally faster."}, "reason": {"statement": "Scope bottleneck and optimization statements to the official task model and measured implementations.", "urls": ["https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld", "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"]}} {"path": "wiki/kernels/nvfp4-gemv.md", "before": {"statement": "performance_claims:\n- gpu: B200\n dtype: nvfp4\n shape: M=7168, K=16384, L=1\n metric: latency_us\n value: 22.4\n utilization: ~2.6x of SOL (8.6us)\n source_id: contest-gpumode-p1\n\nThis was Problem 1 of the GPU Mode NVFP4 Hackathon (Nov 2025). The theoretical speed-of-light is ~8.6us for the largest config, limited by B200's 8 TB/s HBM3e bandwidth. Top performers achieved ~22.4us (2.6x off SOL), reflecting the overhead of FP4 decoding and scale factor application."}, "after": {"statement": "performance_claims: []\n\nGPU Mode's Problem 1 asks for a block-scaled NVFP4 batched matrix-vector product on NVIDIA B200. The official task models its three benchmark cases against the slower of FFMA math and DRAM transfer time and reports DRAM-limited theoretical times. That task-specific model does not make every GEMV implementation or shape bandwidth-bound.\n\nThe public leaderboard is mutable and reports aggregate scores rather than per-shape latencies. In the snapshot fetched on 2026-08-08, the first three rows were `s.am._` at 18.549562452 µs, `gau.nernst` at 18.552844757 µs, and `shellsmile15795` at 18.707609314 µs. Yue's 22.392217755 µs submission was rank 11. The official cutoff was November 28, 2025 at 11:59 p.m. PT; current ranks 2 and 3 have November 30 timestamps, so this endpoint is not an official prize-placement snapshot."}, "reason": {"statement": "Remove malformed per-shape metadata and publish a dated aggregate snapshot with cutoff caveat.", "urls": ["https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemv/NVIDIA?limit=100", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml", "https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf"]}} {"path": "wiki/kernels/nvfp4-gemv.md", "before": {"statement": "```\nInputs:\n a: (M x K x L) NVFP4 packed matrix\n b: (1 x K x L) NVFP4 packed vector\n sfa: (M x K/16 x L) FP8 E4M3 scale factors for a\n sfb: (1 x K/16 x L) FP8 E4M3 scale factors for b\n sf_a_global, sf_b_global: FP32 per-tensor global scales\n\nOutput:\n c: (M x 1 x L) FP16\n\nComputation per element:\n c[m][l] = sum_k( sf_a_global * sfa[m][k/16][l] * deq(a[m][k][l])\n * sf_b_global * sfb[0][k/16][l] * deq(b[0][k][l]) )\n\nBenchmark configs:\n Config 1: M=7168, K=16384, L=1\n Config 2: M=4096, K=7168, L=8\n Config 3: M=7168, K=2048, L=4\n```"}, "after": {"statement": "The logical operation has one B row, reused by all M output rows. The reference harness pads B and its scales to 128 rows so it can call `torch._scaled_mm`, then retains only result column zero. Consequently, A values are row-specific while the logical B vector is reused across M; it is incorrect to say every FP4 input value is consumed only once.\n\nAt pinned commit `ae679486`, `custom_kernel` receives seven tensors, not five tensors plus global FP32 scale arguments:\n\n| `a` | `[M, K/2, L]` | Packed E2M1 A; two logical values per byte |\n| `b` | `[128, K/2, L]` | Packed E2M1 B, physically padded; logical row 0 is used |\n| `sfa` | `[M, K/16, L]` | Logical/reference A block scales |\n| `sfb` | `[128, K/16, L]` | Logical/reference B block scales, padded to 128 rows |\n| `sfa_reordered` | `[32, 4, ceil(M/128), 4, K/64, L]` | Swizzled A scales for custom kernels |\n| `sfb_reordered` | `[32, 4, 1, 4, K/64, L]` | Swizzled padded-B scales for custom kernels |\n| `c` | `[M, 1, L]` | FP16 output buffer |\n\n`K` must be divisible by 64, and M must be divisible by the implementation's selected M tile. The task prose labels scales `e4m3fnuz`, but the pinned generator constructs `torch.float8_e4m3fn`; implementations should follow the actual harness revision they are tested against. This contest ABI exposes no per-tensor FP32 global scales."}, "reason": {"statement": "Publish both the logical operation and actual seven-tensor callable without inventing inputs.", "urls": []}} @@ -243,14 +243,14 @@ {"path": "wiki/kernels/nvfp4-gemv.md", "before": {"statement": "- Memory-bound operations with FP4 quantized weights"}, "after": {"statement": "Amandeep reports that three solutions inspected after the event used `L1::no_allocate` for streamed A loads, `L1::evict_last` for reused B loads, wider PTX loads, and exact-K specializations. Those are author-reported observations; the public leaderboard API supplies no contestant code or technique field. Amandeep's own wider `uint2` experiment was 16–25% slower, and reducing `-maxrregcount` from 80 to 64 had no effect because the kernel already used fewer than 64 registers. Load width, cache hints, and register caps therefore require measurement in the exact implementation."}, "reason": {"statement": "Scope applicability to exact task-compatible NVFP4 operands.", "urls": ["https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"]}} {"path": "wiki/kernels/nvfp4-gemv.md", "before": {"statement": "- SM100/SM100a only (native FP4 decode instructions)"}, "after": {"statement": "Amandeep reports that three solutions inspected after the event used `L1::no_allocate` for streamed A loads, `L1::evict_last` for reused B loads, wider PTX loads, and exact-K specializations. Those are author-reported observations; the public leaderboard API supplies no contestant code or technique field. Amandeep's own wider `uint2` experiment was 16–25% slower, and reducing `-maxrregcount` from 80 to 64 had no effect because the kernel already used fewer than 64 registers. Load width, cache hints, and register caps therefore require measurement in the exact implementation."}, "reason": {"statement": "Separate contest B200 scope from versioned instruction/library support.", "urls": ["https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://docs.nvidia.com/cuda/archive/12.9.2/parallel-thread-execution/index.html", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"]}} {"path": "wiki/kernels/nvfp4-gemv.md", "before": {"statement": "Local verbatim upstream code lives in [`artifacts/kernels/nvfp4-gemv/full/`](../../artifacts/kernels/nvfp4-gemv/full/) (see its `PROVENANCE.yaml` for the pinned upstream SHA and byte-verified SHA-256). Labeled derived variants — including a naive/teaching skeleton — live in [`artifacts/kernels/nvfp4-gemv/variants/`](../../artifacts/kernels/nvfp4-gemv/variants/)."}, "after": {"statement": null}, "reason": {"statement": "Detach the misleading artifact relationship while preserving the files and immutable receipt.", "urls": []}} -{"path": "wiki/languages/ptx-sm100.md", "before": {"statement": "```ptx\n// Allocate TMEM columns\ntcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [smem_tmem_addr], num_cols;\n\n// MMA: inputs from SMEM, accumulator in TMEM\ntcgen05.mma.cta_group::1.kind::f16 [tmem_addr], desc_a, desc_b, idesc, enable_input_d;\n\n// 2-SM cooperative MMA\ntcgen05.mma.cta_group::2.kind::f16 [tmem_addr], desc_a, desc_b, idesc, enable_input_d;\n\n// Load TMEM to registers\ntcgen05.ld.sync.aligned.32x32b.x1.b32 {regs}, [tmem_addr];\n\n// Store registers to TMEM\ntcgen05.st.sync.aligned.32x32b.x1.b32 [tmem_addr], {regs};\n\n// Copy a shaped SMEM matrix descriptor into TMEM\ntcgen05.cp.cta_group::1.128x256b [tmem_addr], sdesc;\n\n// Critical fence between TMA completion and MMA\ntcgen05.fence::after_thread_sync;\n\n// Deallocate TMEM (MUST before kernel exit)\ntcgen05.dealloc.cta_group::1.sync.aligned.b32 tmem_addr, num_cols;\n```"}, "after": {"statement": "TMEM/register transfers and shared-memory/TMEM copies have their own asynchronous completion rules:\n\n```ptx\ntcgen05.ld.sync.aligned.32x32b.x1.b32 {r0}, [taddr];\ntcgen05.wait::ld.sync.aligned;\n\ntcgen05.st.sync.aligned.32x32b.x1.b32 [taddr], {r0};\ntcgen05.wait::st.sync.aligned;\n\ntcgen05.cp.cta_group::1.128x256b [taddr], sdesc;\n```\n\n`tcgen05.ld` and `tcgen05.st` are warp-collective. `tcgen05.cp` copies a shaped shared-memory descriptor into TMEM. MMA and cp completion can be attached to an mbarrier with `tcgen05.commit`; the source and destination must remain live until their documented completion points."}, "reason": {"statement": "Separate TMA completion, async-proxy visibility, tcgen completion, and cross-thread tcgen ordering.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-cp"]}} -{"path": "wiki/languages/ptx-sm100.md", "before": {"statement": "```ptx\n// Convert two packed FP4 values to two FP16 values\ncvt.rn.f16x2.e2m1x2 result_f16x2, packed_fp4;\n\n// Byte unpacking (faster than bitwise extraction)\nmov.b32 {byte0, byte1, byte2, byte3}, packed_word;\n```"}, "after": {"statement": "- A TMA global-to-shared load completes bytes on its mbarrier; a consumer waits for that phase before reading the destination.\n- `tcgen05.commit` makes an mbarrier track completion of prior asynchronous tcgen05 MMA/cp/shift operations issued by the thread.\n- `tcgen05.wait::ld` and `tcgen05.wait::st` wait for the corresponding prior TMEM/register transfers.\n- `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` constrain tcgen05 operations around a documented cross-thread execution-ordering handoff. They are not substitutes for TMA or MMA completion waits."}, "reason": {"statement": "Keep exact semantics and require generated-code/timing comparison for performance.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov", "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence"]}} +{"path": "wiki/languages/ptx-sm100.md", "before": {"statement": "```ptx\n// Allocate TMEM columns\ntcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [smem_tmem_addr], num_cols;\n\n// MMA: inputs from SMEM, accumulator in TMEM\ntcgen05.mma.cta_group::1.kind::f16 [tmem_addr], desc_a, desc_b, idesc, enable_input_d;\n\n// 2-SM cooperative MMA\ntcgen05.mma.cta_group::2.kind::f16 [tmem_addr], desc_a, desc_b, idesc, enable_input_d;\n\n// Load TMEM to registers\ntcgen05.ld.sync.aligned.32x32b.x1.b32 {regs}, [tmem_addr];\n\n// Store registers to TMEM\ntcgen05.st.sync.aligned.32x32b.x1.b32 [tmem_addr], {regs};\n\n// Copy a shaped SMEM matrix descriptor into TMEM\ntcgen05.cp.cta_group::1.128x256b [tmem_addr], sdesc;\n\n// Critical fence between TMA completion and MMA\ntcgen05.fence::after_thread_sync;\n\n// Deallocate TMEM (MUST before kernel exit)\ntcgen05.dealloc.cta_group::1.sync.aligned.b32 tmem_addr, num_cols;\n```"}, "after": {"statement": "TMEM/register transfers and shared-memory/TMEM copies have their own asynchronous completion rules:\n\n```ptx\ntcgen05.ld.sync.aligned.32x32b.x1.b32 {r0}, [taddr];\ntcgen05.wait::ld.sync.aligned;\n\ntcgen05.st.sync.aligned.32x32b.x1.b32 [taddr], {r0};\ntcgen05.wait::st.sync.aligned;\n\ntcgen05.cp.cta_group::1.128x256b [taddr], sdesc;\n```\n\n`tcgen05.ld` and `tcgen05.st` are warp-collective. `tcgen05.cp` copies a shaped shared-memory descriptor into TMEM. MMA and cp completion can be attached to an mbarrier with `tcgen05.commit`; the source and destination must remain live until their documented completion points."}, "reason": {"statement": "Separate TMA completion, async-proxy visibility, tcgen completion, and cross-thread tcgen ordering.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-special-sync-operations-fence", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-cp"]}} +{"path": "wiki/languages/ptx-sm100.md", "before": {"statement": "```ptx\n// Convert two packed FP4 values to two FP16 values\ncvt.rn.f16x2.e2m1x2 result_f16x2, packed_fp4;\n\n// Byte unpacking (faster than bitwise extraction)\nmov.b32 {byte0, byte1, byte2, byte3}, packed_word;\n```"}, "after": {"statement": "- A TMA global-to-shared load completes bytes on its mbarrier; a consumer waits for that phase before reading the destination.\n- `tcgen05.commit` makes an mbarrier track completion of prior asynchronous tcgen05 MMA/cp/shift operations issued by the thread.\n- `tcgen05.wait::ld` and `tcgen05.wait::st` wait for the corresponding prior TMEM/register transfers.\n- `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` constrain tcgen05 operations around a documented cross-thread execution-ordering handoff. They are not substitutes for TMA or MMA completion waits."}, "reason": {"statement": "Keep exact semantics and require generated-code/timing comparison for performance.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov", "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-special-sync-operations-fence"]}} {"path": "wiki/languages/ptx-sm100.md", "before": {"statement": "```ptx\n// Streaming data (use once): bypass L1\nld.global.L1::no_allocate.v2.u64 {r0, r1}, [addr];\n\n// Reused data: keep in L1\nld.global.L1::evict_last.v2.u64 {r0, r1}, [addr];\n\n// Wide vectorized loads\nld.global.v4.u64 {r0, r1, r2, r3}, [addr]; // 256-bit\n```"}, "after": {"statement": "PTX ISA 9.0 defines these typed operations:\n\n```ptx\ncvt.rn.f16x2.e2m1x2 result_f16x2, packed_fp4_pair;\nmov.b32 {byte0, byte1, byte2, byte3}, packed_word;\n```\n\nThe first converts one byte containing two E2M1 values into a 32-bit F16x2 result. The second can decompose a 32-bit scalar into four byte-sized destinations when operand declarations satisfy the scalar-to-vector size rules. PTX specifies these semantics but does not guarantee that the move is faster than every compiler-generated shift/mask sequence."}, "reason": {"statement": "Use normative hint semantics and separate workload selection from instruction behavior.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#cache-operators", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov"]}} {"path": "wiki/languages/ptx-sm100.md", "before": {"statement": "```ptx\n// Query for next tile (persistent kernel loop)\nclusterlaunchcontrol.try_cancel {clc_id};\n// Returns valid tile_id or decline (all work done)\n```"}, "after": {"statement": "```ptx\nld.global.L1::no_allocate.v2.u64 {r0, r1}, [addr];\nld.global.L1::evict_last.v2.u64 {r0, r1}, [addr];\nld.global.v4.u64 {r0, r1, r2, r3}, [addr];\n```\n\n`L1::no_allocate` and `L1::evict_last` are eviction-priority hints and may not always be respected. They do not guarantee L1 bypass or residency. The vector forms above move 16 and 32 bytes; whether a width or hint helps depends on alignment, surrounding instructions, reuse, cache state, and the target GPU."}, "reason": {"statement": "Replace with the exact request/wait/query sequence.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld"]}} {"path": "wiki/languages/ptx-sm100.md", "before": {"statement": "```ptx\n// Bulk tensor copy: global → shared\ncp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes\n [smem_ptr], [tensorMap, {x, y}], [mbarrier];\n\n// Multicast to cluster SMs\ncp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast\n [smem_ptr], [tensorMap, {x, y}], [mbarrier], multicast_mask;\n```"}, "after": {"statement": "CLC cancellation is an asynchronous request followed by response decoding:\n\n```ptx\nclusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128\n [response_smem], [response_mbarrier];\n\nclusterlaunchcontrol.query_cancel.is_canceled.pred.b128 p, response_b128;\n@p clusterlaunchcontrol.query_cancel.get_first_ctaid.v4.b32.b128\n {x, y, z, unused}, response_b128;\n```\n\nThe request writes an opaque 16-byte response to shared memory and completes on the mbarrier. Code must wait for that phase before loading and querying the response. A successful query returns the first CTA coordinate of a canceled not-yet-launched block or cluster; the request does not take a desired tile ID."}, "reason": {"statement": "Use the exact version-pinned PTX spelling.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel"]}} -{"path": "wiki/languages/cuda-cpp.md", "before": {"statement": "```cuda\n// Allocate TMEM. tcgen05.alloc writes the allocated address into SMEM.\n__device__ uint32_t tmem_alloc_cta(uint32_t* smem_tmem_addr,\n uint32_t num_cols) {\n if (threadIdx.x == 0) {\n uint32_t smem_addr =\n static_cast(__cvta_generic_to_shared(smem_tmem_addr));\n asm volatile(\n \"tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;\"\n :: \"r\"(smem_addr), \"r\"(num_cols)\n );\n }\n __syncthreads();\n return *smem_tmem_addr;\n}\n\n// Issue MMA (single thread, typically warp 1 lane 0)\n// idesc_c/idesc_d: immediate descriptors for accumulator C and output D\n__device__ void tcgen05_mma(uint32_t tmem_addr,\n uint64_t desc_a, uint64_t desc_b,\n uint32_t idesc_c, uint32_t idesc_d) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16\"\n \" [%0], %1, %2, %3, %4;\"\n :: \"r\"(tmem_addr), \"l\"(desc_a), \"l\"(desc_b),\n \"r\"(idesc_c), \"r\"(idesc_d)\n );\n}\n\n// Load TMEM to registers\n__device__ void tmem_load(float* dst, uint32_t tmem_addr, int cols) {\n asm volatile(\n \"tcgen05.ld.sync.aligned.32x32b.x1.b32 {%0}, [%1];\"\n : \"=f\"(*dst) : \"r\"(tmem_addr)\n );\n}\n\n// Deallocate TMEM (MUST do before kernel exit)\n__device__ void tmem_dealloc(uint32_t addr, uint32_t num_cols) {\n asm volatile(\n \"tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;\"\n :: \"r\"(addr), \"r\"(num_cols)\n );\n}\n```"}, "after": {"statement": "The CUDA front end does not parse the instruction text inside an `asm()` statement. Operand constraints and address-space conversion therefore remain the wrapper author's responsibility. Use `\"r\"` for a 32-bit integer register, `\"l\"` for a 64-bit integer register, and convert a generic pointer with `__cvta_generic_to_shared` before supplying a shared-memory address. Add a `\"memory\"` clobber when the assembly has memory effects that are hidden from the compiler."}, "reason": {"statement": "Show whole-warp allocation and reserve lane election for single-thread MMA/TMA issue.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu", "https://docs.nvidia.com/cuda/archive/13.0.2/inline-ptx-assembly/index.html"]}} -{"path": "wiki/languages/cuda-cpp.md", "before": {"statement": "```cuda\n// Allocate TMEM. tcgen05.alloc writes the allocated address into SMEM.\n__device__ uint32_t tmem_alloc_cta(uint32_t* smem_tmem_addr,\n uint32_t num_cols) {\n if (threadIdx.x == 0) {\n uint32_t smem_addr =\n static_cast(__cvta_generic_to_shared(smem_tmem_addr));\n asm volatile(\n \"tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;\"\n :: \"r\"(smem_addr), \"r\"(num_cols)\n );\n }\n __syncthreads();\n return *smem_tmem_addr;\n}\n\n// Issue MMA (single thread, typically warp 1 lane 0)\n// idesc_c/idesc_d: immediate descriptors for accumulator C and output D\n__device__ void tcgen05_mma(uint32_t tmem_addr,\n uint64_t desc_a, uint64_t desc_b,\n uint32_t idesc_c, uint32_t idesc_d) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16\"\n \" [%0], %1, %2, %3, %4;\"\n :: \"r\"(tmem_addr), \"l\"(desc_a), \"l\"(desc_b),\n \"r\"(idesc_c), \"r\"(idesc_d)\n );\n}\n\n// Load TMEM to registers\n__device__ void tmem_load(float* dst, uint32_t tmem_addr, int cols) {\n asm volatile(\n \"tcgen05.ld.sync.aligned.32x32b.x1.b32 {%0}, [%1];\"\n : \"=f\"(*dst) : \"r\"(tmem_addr)\n );\n}\n\n// Deallocate TMEM (MUST do before kernel exit)\n__device__ void tmem_dealloc(uint32_t addr, uint32_t num_cols) {\n asm volatile(\n \"tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;\"\n :: \"r\"(addr), \"r\"(num_cols)\n );\n}\n```"}, "after": {"statement": "The allocation instruction is collective. For `.cta_group::1`, every lane of one designated warp must execute the same instruction. Synchronize the CTA before another warp reads the address written to shared memory.\n\n```cuda\n// All 32 lanes of alloc_warp execute this branch.\nif (warp_id == alloc_warp) {\n uint32_t smem_addr =\n static_cast(__cvta_generic_to_shared(smem_tmem_addr));\n asm volatile(\n \"tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 \"\n \"[%0], %1;\"\n :: \"r\"(smem_addr), \"r\"(num_cols) : \"memory\");\n}\n__syncthreads();\nuint32_t taddr = *smem_tmem_addr;\n```"}, "reason": {"statement": "Use one idesc plus a local predicate constructed from a CUDA integer.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu"]}} -{"path": "wiki/languages/cuda-cpp.md", "before": {"statement": "```cuda\n// TMA-MMA synchronization via mbarrier\n// expected_bytes: total bytes the TMA will deliver to this stage\n__device__ void mbarrier_arrive(uint64_t* mbar, uint32_t expected_bytes) {\n asm volatile(\n \"mbarrier.arrive.expect_tx.shared.b64 _, [%0], %1;\"\n :: \"r\"((uint32_t)__cvta_generic_to_shared(mbar)),\n \"r\"(expected_bytes)\n );\n}\n\n__device__ void mbarrier_wait(uint64_t* mbar, int phase) {\n asm volatile(\n \"{\\n\"\n \".reg .pred p;\\n\"\n \"WAIT_LOOP:\\n\"\n \" mbarrier.try_wait.parity.shared.b64 p, [%0], %1;\\n\"\n \" @!p bra WAIT_LOOP;\\n\"\n \"}\\n\"\n :: \"r\"((uint32_t)__cvta_generic_to_shared(mbar)),\n \"r\"(phase)\n );\n}\n```"}, "after": {"statement": "An unscaled `kind::f16` MMA takes one instruction descriptor and an `enable-input-d` predicate. A single elected thread may issue this MMA form; converting an ordinary CUDA integer to the required PTX predicate inside the assembly keeps the C++ interface well typed.\n\n```cuda\n__device__ inline void tcgen05_mma_f16(\n uint32_t taddr, uint64_t a_desc, uint64_t b_desc,\n uint32_t idesc, int enable_input_d) {\n asm volatile(\n \"{\\n\\t\"\n \".reg .pred p;\\n\\t\"\n \"setp.ne.b32 p, %4, 0;\\n\\t\"\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, p;\\n\\t\"\n \"}\"\n :: \"r\"(taddr), \"l\"(a_desc), \"l\"(b_desc),\n \"r\"(idesc), \"r\"(enable_input_d));\n}\n```"}, "reason": {"statement": "Publish the full lifecycle invariants and memory-clobber requirement rather than a misleading two-function protocol.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/inline-ptx-assembly/index.html#incorrect-optimization", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/common.h#L66-L78"]}} +{"path": "wiki/languages/cuda-cpp.md", "before": {"statement": "```cuda\n// Allocate TMEM. tcgen05.alloc writes the allocated address into SMEM.\n__device__ uint32_t tmem_alloc_cta(uint32_t* smem_tmem_addr,\n uint32_t num_cols) {\n if (threadIdx.x == 0) {\n uint32_t smem_addr =\n static_cast(__cvta_generic_to_shared(smem_tmem_addr));\n asm volatile(\n \"tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;\"\n :: \"r\"(smem_addr), \"r\"(num_cols)\n );\n }\n __syncthreads();\n return *smem_tmem_addr;\n}\n\n// Issue MMA (single thread, typically warp 1 lane 0)\n// idesc_c/idesc_d: immediate descriptors for accumulator C and output D\n__device__ void tcgen05_mma(uint32_t tmem_addr,\n uint64_t desc_a, uint64_t desc_b,\n uint32_t idesc_c, uint32_t idesc_d) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16\"\n \" [%0], %1, %2, %3, %4;\"\n :: \"r\"(tmem_addr), \"l\"(desc_a), \"l\"(desc_b),\n \"r\"(idesc_c), \"r\"(idesc_d)\n );\n}\n\n// Load TMEM to registers\n__device__ void tmem_load(float* dst, uint32_t tmem_addr, int cols) {\n asm volatile(\n \"tcgen05.ld.sync.aligned.32x32b.x1.b32 {%0}, [%1];\"\n : \"=f\"(*dst) : \"r\"(tmem_addr)\n );\n}\n\n// Deallocate TMEM (MUST do before kernel exit)\n__device__ void tmem_dealloc(uint32_t addr, uint32_t num_cols) {\n asm volatile(\n \"tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;\"\n :: \"r\"(addr), \"r\"(num_cols)\n );\n}\n```"}, "after": {"statement": "The CUDA front end does not parse the instruction text inside an `asm()` statement. Operand constraints and address-space conversion therefore remain the wrapper author's responsibility. Use `\"r\"` for a 32-bit integer register, `\"l\"` for a 64-bit integer register, and convert a generic pointer with `__cvta_generic_to_shared` before supplying a shared-memory address. Add a `\"memory\"` clobber when the assembly has memory effects that are hidden from the compiler."}, "reason": {"statement": "Show whole-warp allocation and reserve lane election for single-thread MMA/TMA issue.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu", "https://docs.nvidia.com/cuda/archive/13.0.2/inline-ptx-assembly/index.html"]}} +{"path": "wiki/languages/cuda-cpp.md", "before": {"statement": "```cuda\n// Allocate TMEM. tcgen05.alloc writes the allocated address into SMEM.\n__device__ uint32_t tmem_alloc_cta(uint32_t* smem_tmem_addr,\n uint32_t num_cols) {\n if (threadIdx.x == 0) {\n uint32_t smem_addr =\n static_cast(__cvta_generic_to_shared(smem_tmem_addr));\n asm volatile(\n \"tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;\"\n :: \"r\"(smem_addr), \"r\"(num_cols)\n );\n }\n __syncthreads();\n return *smem_tmem_addr;\n}\n\n// Issue MMA (single thread, typically warp 1 lane 0)\n// idesc_c/idesc_d: immediate descriptors for accumulator C and output D\n__device__ void tcgen05_mma(uint32_t tmem_addr,\n uint64_t desc_a, uint64_t desc_b,\n uint32_t idesc_c, uint32_t idesc_d) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16\"\n \" [%0], %1, %2, %3, %4;\"\n :: \"r\"(tmem_addr), \"l\"(desc_a), \"l\"(desc_b),\n \"r\"(idesc_c), \"r\"(idesc_d)\n );\n}\n\n// Load TMEM to registers\n__device__ void tmem_load(float* dst, uint32_t tmem_addr, int cols) {\n asm volatile(\n \"tcgen05.ld.sync.aligned.32x32b.x1.b32 {%0}, [%1];\"\n : \"=f\"(*dst) : \"r\"(tmem_addr)\n );\n}\n\n// Deallocate TMEM (MUST do before kernel exit)\n__device__ void tmem_dealloc(uint32_t addr, uint32_t num_cols) {\n asm volatile(\n \"tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;\"\n :: \"r\"(addr), \"r\"(num_cols)\n );\n}\n```"}, "after": {"statement": "The allocation instruction is collective. For `.cta_group::1`, every lane of one designated warp must execute the same instruction. Synchronize the CTA before another warp reads the address written to shared memory.\n\n```cuda\n// All 32 lanes of alloc_warp execute this branch.\nif (warp_id == alloc_warp) {\n uint32_t smem_addr =\n static_cast(__cvta_generic_to_shared(smem_tmem_addr));\n asm volatile(\n \"tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 \"\n \"[%0], %1;\"\n :: \"r\"(smem_addr), \"r\"(num_cols) : \"memory\");\n}\n__syncthreads();\nuint32_t taddr = *smem_tmem_addr;\n```"}, "reason": {"statement": "Use one idesc plus a local predicate constructed from a CUDA integer.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu"]}} +{"path": "wiki/languages/cuda-cpp.md", "before": {"statement": "```cuda\n// TMA-MMA synchronization via mbarrier\n// expected_bytes: total bytes the TMA will deliver to this stage\n__device__ void mbarrier_arrive(uint64_t* mbar, uint32_t expected_bytes) {\n asm volatile(\n \"mbarrier.arrive.expect_tx.shared.b64 _, [%0], %1;\"\n :: \"r\"((uint32_t)__cvta_generic_to_shared(mbar)),\n \"r\"(expected_bytes)\n );\n}\n\n__device__ void mbarrier_wait(uint64_t* mbar, int phase) {\n asm volatile(\n \"{\\n\"\n \".reg .pred p;\\n\"\n \"WAIT_LOOP:\\n\"\n \" mbarrier.try_wait.parity.shared.b64 p, [%0], %1;\\n\"\n \" @!p bra WAIT_LOOP;\\n\"\n \"}\\n\"\n :: \"r\"((uint32_t)__cvta_generic_to_shared(mbar)),\n \"r\"(phase)\n );\n}\n```"}, "after": {"statement": "An unscaled `kind::f16` MMA takes one instruction descriptor and an `enable-input-d` predicate. A single elected thread may issue this MMA form; converting an ordinary CUDA integer to the required PTX predicate inside the assembly keeps the C++ interface well typed.\n\n```cuda\n__device__ inline void tcgen05_mma_f16(\n uint32_t taddr, uint64_t a_desc, uint64_t b_desc,\n uint32_t idesc, int enable_input_d) {\n asm volatile(\n \"{\\n\\t\"\n \".reg .pred p;\\n\\t\"\n \"setp.ne.b32 p, %4, 0;\\n\\t\"\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, p;\\n\\t\"\n \"}\"\n :: \"r\"(taddr), \"l\"(a_desc), \"l\"(b_desc),\n \"r\"(idesc), \"r\"(enable_input_d));\n}\n```"}, "reason": {"statement": "Publish the full lifecycle invariants and memory-clobber requirement rather than a misleading two-function protocol.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/inline-ptx-assembly/index.html#incorrect-optimization", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/common.h#L66-L78"]}} {"path": "wiki/languages/cute-dsl.md", "before": {"statement": "CuTe (CUDA Templates) DSL is the primary abstraction layer in CUTLASS 4.5.0 for Blackwell kernels. FlashAttention-4 was implemented entirely in CuTe-DSL (Python variant), achieving 20-30× faster compilation than C++ templates."}, "after": {"statement": "CUTLASS v4.5.0 includes CuTe DSL, a Python-native interface for authoring GPU kernels, alongside CUTLASS's C++ template interfaces. This page uses the exact `v4.5.0` tag (`e406c186f510a15091cce01f782020ceb7ba8eb5`); rolling `latest` documentation can contain later names."}, "reason": {"statement": "Describe CuTe DSL as a Python-native CUTLASS interface without assigning unsupported primacy.", "urls": []}} {"path": "wiki/languages/cute-dsl.md", "before": {"statement": "```python\n# CuTe-DSL: SM100 MMA atom for BF16\nfrom cutlass.cute import *\n\n# 1-SM MMA: m128 x n256 x k16\nmma_atom = SM100_MMA_F16BF16_SS # inputs from shared memory\n# Accumulator goes to TMEM automatically\n\n# 2-SM MMA: m256 x n256 x k16\nmma_atom_2sm = SM100_MMA_F16BF16_SS_2SM\n```"}, "after": {"statement": "CUTLASS 4.5.0 uses a configured `tcgen05.MmaF16BF16Op`, not an `SM100_MMA_F16BF16_SS` Python symbol. This excerpt shows the one-CTA operation used by the tagged first tutorial; it is construction code, not a complete kernel.\n\n```python\nimport cutlass\nimport cutlass.cute as cute\nfrom cutlass.cute.nvgpu import tcgen05\n\nop = tcgen05.MmaF16BF16Op(\n cutlass.Float16,\n cutlass.Float32,\n (128, 256, 16),\n tcgen05.CtaGroup.ONE,\n tcgen05.OperandSource.SMEM,\n cute.nvgpu.OperandMajorMode.K,\n cute.nvgpu.OperandMajorMode.K,\n)\ntiled_mma = cute.make_tiled_mma(op)\n```\n\nThe two-CTA tutorial instead uses instruction shape `(256, 256, 16)` and `tcgen05.CtaGroup.TWO`. The accumulator fragment is rebound to an allocated TMEM pointer before `cute.gemm` issues the operation."}, "reason": {"statement": "Replace invented identifiers with the exact version-pinned constructor.", "urls": []}} {"path": "wiki/languages/cute-dsl.md", "before": {"statement": "```python\n# TMEM allocation in CuTe\ntmem_tensor = make_tensor(\n make_tmem_ptr(tmem_addr),\n make_layout(make_shape(128, 256)) # rows x cols\n)\n\n# Copy TMEM → registers for epilogue\ncopy(tmem_tensor, reg_tensor) # tcgen05.ld under the hood\n```"}, "after": {"statement": "The tagged examples use `cutlass.utils.TmemAllocator` around a shared holding buffer. A complete path allocates columns, waits before pointer retrieval, rebinds the accumulator tensor, partitions a typed `tcgen05` TMEM-to-register copy across participating threads, synchronizes readers, and frees the same allocation.\n\n```python\n# Excerpt: storage/barrier/tensor definitions and pipeline edges are required.\ntmem = utils.TmemAllocator(\n storage.tmem_holding_buf.ptr,\n barrier_for_retrieve=tmem_alloc_barrier,\n)\ntmem.allocate(num_tmem_cols)\ntmem.wait_for_alloc()\ntmem_ptr = tmem.retrieve_ptr(cutlass.Float32)\ntCtAcc = cute.make_tensor(tmem_ptr, tCtAcc.layout)\n\ntmem_atom = cute.make_copy_atom(\n tcgen05.Ld32x32bOp(tcgen05.Repetition.x64), cutlass.Float32\n)\ntmem_tiled_copy = tcgen05.make_tmem_copy(tmem_atom, tCtAcc_epi[None, 0])\n# get_slice(), partition_S/D(), and cute.copy() form the per-thread epilogue.\n\npipeline.sync(barrier_id=1)\ntmem.free(tmem_ptr)\n```\n\nThe excerpt deliberately does not imply that allocation or deallocation is lane-local. Use the complete tutorial for collective participation, two-CTA handling, and pointer lifetime."}, "reason": {"statement": "Use an evidence-scoped excerpt that preserves allocation, copy partition, wait, and free responsibilities.", "urls": []}} @@ -263,61 +263,61 @@ {"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "If the tile is larger (e.g., m64xn256xk32 with double-buffered accumulators), register pressure becomes even worse:\n\n```\nDouble-buffered accumulator: 256 registers per thread\nTotal per thread: ~300 registers\nPer warpgroup: 38,400 registers\nPer SM: max 1 warpgroup -> 1 CTA per SM\n\nOccupancy: 1 CTA per SM (severe underutilization)\n```"}, "after": {"statement": "For `sm_100a`, tcgen05 D is addressed in Tensor Memory rather than supplied as a per-thread register vector. The CTA-visible TMEM structure has 128 lanes by 512 columns of 32-bit cells, or 256 KiB when fully allocated. Allocation is dynamic in columns.\n\nThis removes the resident D vector from the ordinary register file. It does not remove all register pressure: descriptors, addresses, loop state, pipeline state, and each TMEM-to-register epilogue batch still need registers. Both Hopper and compute-capability-10.0 Blackwell have 64K 32-bit registers per SM and at most 255 registers per thread. Occupancy, spilling, and practical tile size therefore remain properties of the compiled kernel and launch configuration."}, "reason": {"statement": "The numeric scenario is not source- or experiment-grounded.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/hopper-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"]}} {"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "```\nRegister access: ~4 cycles\nL1 local access: ~30 cycles\nL2 spill access: ~200 cycles\n\nA single spilled accumulator register accessed every MMA iteration\ncan cost 200 cycles * K_tiles additional latency.\n```"}, "after": {"statement": "For `sm_100a`, tcgen05 D is addressed in Tensor Memory rather than supplied as a per-thread register vector. The CTA-visible TMEM structure has 128 lanes by 512 columns of 32-bit cells, or 256 KiB when fully allocated. Allocation is dynamic in columns.\n\nThis removes the resident D vector from the ordinary register file. It does not remove all register pressure: descriptors, addresses, loop state, pipeline state, and each TMEM-to-register epilogue batch still need registers. Both Hopper and compute-capability-10.0 Blackwell have 64K 32-bit registers per SM and at most 255 registers per thread. Occupancy, spilling, and practical tile size therefore remain properties of the compiled kernel and launch configuration."}, "reason": {"statement": "Unsupported latency constants would mislead migration decisions.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-best-practices-guide/index.html#local-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"]}} {"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "```\nBlackwell tcgen05 m128xn256xk16 BF16 register budget (per thread):\n\n Accumulator: 0 registers (stored in TMEM!)\n SMEM descriptors: 4 registers\n Loop variables: 8 registers\n TMA state: 6 registers\n TMEM address: 1 register\n Misc (indexing): 10 registers\n ─────────────────────────────\n Total per thread: ~29 registers\n\n Per CTA (256 threads): 29 * 256 = 7,424 registers\n Per SM (65,536 regs): max 8 CTAs (register-wise)\n Actual limit: TMEM capacity (512 columns)\n\n TMEM usage per CTA: 256 columns for 128x256 tile\n Max CTAs per SM: 512 / 256 = 2 CTAs (TMEM-limited, not register-limited)\n```\n\nThe key insight: **registers are no longer the bottleneck**. TMEM capacity becomes the binding constraint for occupancy, and the freed registers enable complex epilogues without spilling."}, "after": {"statement": "For `sm_100a`, tcgen05 D is addressed in Tensor Memory rather than supplied as a per-thread register vector. The CTA-visible TMEM structure has 128 lanes by 512 columns of 32-bit cells, or 256 KiB when fully allocated. Allocation is dynamic in columns.\n\nThis removes the resident D vector from the ordinary register file. It does not remove all register pressure: descriptors, addresses, loop state, pipeline state, and each TMEM-to-register epilogue batch still need registers. Both Hopper and compute-capability-10.0 Blackwell have 64K 32-bit registers per SM and at most 255 registers per thread. Occupancy, spilling, and practical tile size therefore remain properties of the compiled kernel and launch configuration."}, "reason": {"statement": "Replace speculative arithmetic with a measurement workflow.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory-allocation", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} -{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "```cuda\n// HOPPER: Accumulator lives and dies in registers\n__global__ void hopper_kernel(/* ... */) {\n // 1. Declare accumulator in registers\n float acc[4][32]; // 128 registers per thread!\n\n // 2. Zero-initialize\n #pragma unroll\n for (int i = 0; i < 4; ++i)\n for (int j = 0; j < 32; ++j)\n acc[i][j] = 0.0f;\n\n // 3. Mainloop: wgmma accumulates into registers\n for (int k = 0; k < K_tiles; ++k) {\n // Load A via ldmatrix\n uint32_t a_frag[4];\n ldmatrix(a_frag, smem_a + k * TILE_K);\n\n // wgmma: reads A from registers, B from SMEM descriptor\n // Accumulates into acc[] registers\n wgmma_m64n256k16(acc, a_frag, smem_b_desc);\n wgmma_commit();\n wgmma_wait();\n }\n\n // 4. Epilogue: acc is directly in registers -- fast access\n // But: if epilogue needs temporary storage, registers are exhausted\n #pragma unroll\n for (int i = 0; i < 4; ++i) {\n for (int j = 0; j < 32; ++j) {\n float val = acc[i][j];\n val += bias[col_base + j]; // bias add\n val = fmaxf(val, 0.0f); // ReLU\n // Need to convert and store -- no registers left for temp!\n C[row * N + col_base + j] = __float2half(val);\n }\n }\n\n // 5. Accumulator freed implicitly when CTA exits\n}\n```"}, "after": {"statement": "For Hopper `wgmma.mma_async.m64nNk16` with an FP32 accumulator, each warpgroup thread holds `N/2` FP32 D registers. At `N=256`, that is 128 registers per thread for D. WGMMA updates that register vector asynchronously; the program uses the WGMMA fence, commit-group, and wait-group mechanisms before dependent use.\n\n| Resident D | Per-thread register fragment | TMEM region addressed by `taddr` |\n| Example FP32 D cost | `N/2` registers/thread; 128 at `N=256` | No resident per-thread D vector |\n| First accumulation | Set WGMMA `scale-d` false to compute `D=A*B`, or initialize/use D when accumulation is intended | Set `enable-input-d` false to compute `D=A*B`, or explicitly initialize D when the algorithm needs another value |\n| Compute completion | WGMMA commit/wait group | `tcgen05.commit` to an mbarrier, then wait before a consumer reads/reuses D |\n| Epilogue access | Use the thread's D registers | Collective `tcgen05.ld`, then `tcgen05.wait::ld` before consuming result registers |\n| Cleanup | Register lifetime ends with the thread | Matching collective `tcgen05.dealloc` before kernel exit |\n| Double buffering | Two simultaneously live D fragments consume two register fragments | Two live outputs consume disjoint TMEM columns inside the allocation |\n\nThe table describes storage contracts, not an occupancy prediction. Instruction shape is also not necessarily the same as a composed CTA tile."}, "reason": {"statement": "Separate the PTX register-D contract from compiler allocation conclusions.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#local-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-multiply-accumulate-instructions-wgmma-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} -{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "```cuda\n// BLACKWELL: Accumulator lives in TMEM\n__global__ void blackwell_kernel(/* ... */) {\n // 1. Allocate TMEM (explicit, must be done once)\n __shared__ uint32_t s_tmem_acc;\n if (threadIdx.x == 0) {\n uint32_t smem_addr =\n static_cast(__cvta_generic_to_shared(&s_tmem_acc));\n asm volatile(\n \"tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;\"\n :: \"r\"(smem_addr), \"r\"(256)\n );\n }\n __syncthreads();\n uint32_t tmem_acc = s_tmem_acc;\n\n // 2. Zero-initialize TMEM\n for (int c = 0; c < 256; c += 4) {\n asm volatile(\n \"tcgen05.st.sync.aligned.32x1b.x4.b32 [%0], {%1,%2,%3,%4};\"\n : : \"r\"(tmem_acc + c),\n \"f\"(0.f), \"f\"(0.f), \"f\"(0.f), \"f\"(0.f)\n );\n }\n\n // 3. Mainloop: tcgen05 accumulates into TMEM\n for (int k = 0; k < K_tiles; ++k) {\n // NO ldmatrix -- tcgen05 reads directly from SMEM\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 1;\"\n : : \"r\"(tmem_acc), \"l\"(desc_a), \"l\"(desc_b), \"r\"(0)\n );\n }\n // NO commit/wait -- fully async\n }\n\n // 4. Fence before reading\n asm volatile(\"tcgen05.fence::before_thread_sync;\");\n __syncthreads();\n\n // 5. Epilogue: read from TMEM to registers (in small batches)\n // PLENTY of registers available for epilogue temporaries!\n for (int c = 0; c < 256; c += 4) {\n float4 vals;\n asm volatile(\n \"tcgen05.ld.sync.aligned.32x1b.x4.b32 {%0,%1,%2,%3}, [%4];\"\n : \"=f\"(vals.x), \"=f\"(vals.y), \"=f\"(vals.z), \"=f\"(vals.w)\n : \"r\"(tmem_acc + c)\n );\n\n // Apply epilogue ops with register headroom\n vals.x += bias[col_base + c];\n vals.y += bias[col_base + c + 1];\n vals.z += bias[col_base + c + 2];\n vals.w += bias[col_base + c + 3];\n\n vals.x = fmaxf(vals.x, 0.0f); // ReLU\n vals.y = fmaxf(vals.y, 0.0f);\n vals.z = fmaxf(vals.z, 0.0f);\n vals.w = fmaxf(vals.w, 0.0f);\n\n // Vectorized store -- registers available for conversion\n half2 h01 = __floats2half2_rn(vals.x, vals.y);\n half2 h23 = __floats2half2_rn(vals.z, vals.w);\n *reinterpret_cast(&C[row * N + col_base + c]) = h01;\n *reinterpret_cast(&C[row * N + col_base + c + 2]) = h23;\n }\n\n // 6. Deallocate TMEM (explicit, MUST be done in persistent kernels)\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;\"\n : : \"r\"(tmem_acc), \"r\"(256)\n );\n }\n}\n```"}, "after": {"statement": "1. Choose one CTA-group mode for the kernel. For allocation, `nCols` is a power of two in `[32, 512]`; allocations are column-granular and cover all 128 lanes.\n2. Have every lane of one designated warp execute the same `.cta_group::1` allocation. Synchronize before other threads read the 32-bit `taddr` written to shared memory. Two-CTA mode requires one warp in each live peer CTA.\n3. Initialize and publish the barriers used by TMA and tcgen05. If the first MMA should compute only `A*B`, supply a false `enable-input-d` predicate instead of assuming a mandatory TMEM zero-store pass.\n4. Issue MMA from the permitted elected thread with valid A/B descriptors, instruction descriptor, predicate, and lifetimes.\n5. Attach completion of the relevant tcgen05 work to an mbarrier with `tcgen05.commit`. A fence controls execution ordering but is not a completion wait."}, "reason": {"statement": "Replace the block with lifecycle invariants and known-good source routes.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} -{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "With accumulators in TMEM, the tile size is no longer constrained by register count:\n\n```cuda\n// Hopper: large tiles cause spilling\n// m64xn256 uses 128 acc registers/thread -> barely fits\n// m128xn256 would need 256 acc registers/thread -> spills guaranteed\n\n// Blackwell: tile size limited by TMEM columns (512) and SMEM, not registers\n// m128xn256: 256 TMEM cols, 0 acc registers -> fits easily\n// m128xn512: 512 TMEM cols, 0 acc registers -> uses full TMEM budget\n// m256xn256 (2-SM): 256 TMEM cols/SM, 0 acc registers -> fits with cooperative\n```"}, "after": {"statement": "For `sm_100a`, tcgen05 D is addressed in Tensor Memory rather than supplied as a per-thread register vector. The CTA-visible TMEM structure has 128 lanes by 512 columns of 32-bit cells, or 256 KiB when fully allocated. Allocation is dynamic in columns.\n\nThis removes the resident D vector from the ordinary register file. It does not remove all register pressure: descriptors, addresses, loop state, pipeline state, and each TMEM-to-register epilogue batch still need registers. Both Hopper and compute-capability-10.0 Blackwell have 64K 32-bit registers per SM and at most 255 registers per thread. Occupancy, spilling, and practical tile size therefore remain properties of the compiled kernel and launch configuration.\n\nTMEM makes it practical to keep multiple output regions independently addressable while different warp roles issue MMA and drain a completed region. Safe overlap needs three distinct proofs:\n\n- compute completion occurs before an epilogue loads a region;\n- all epilogue loads complete before that region is overwritten or deallocated;\n- producer/consumer state and phase cannot alias a different pipeline stage.\n\nTMEM is not the only way to overlap tensor-core and non-matmul work. FlashAttention-3 already uses warp specialization and matmul/softmax interleaving on Hopper with register accumulators. The migration benefit is the different storage and lifetime tradeoff, not a new theorem that overlap was previously impossible."}, "reason": {"statement": "Replace categorical tile advice with descriptor legality plus empirical tuning.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://arxiv.org/abs/2407.08608"]}} -{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "On Hopper, complex epilogues (bias + activation + quantization + scaling) often spill because registers are already exhausted by the accumulator. On Blackwell, the epilogue has the full register file available:\n\n```cuda\n// Blackwell epilogue: full register file available\n__device__ void rich_epilogue(uint32_t tmem_acc, float* output,\n const float* bias, const float* scale,\n int M, int N) {\n // Read TMEM in chunks\n for (int c = 0; c < 256; c += 8) {\n // Load 8 values from TMEM (8 registers -- trivial)\n float v[8];\n tmem_load_f32x4(tmem_acc + c, &v[0]);\n tmem_load_f32x4(tmem_acc + c + 4, &v[4]);\n\n // Apply bias (8 registers for bias values)\n float b[8];\n load_global_f32x4(&b[0], bias + c);\n load_global_f32x4(&b[4], bias + c + 4);\n\n #pragma unroll\n for (int i = 0; i < 8; ++i) v[i] += b[i];\n\n // Apply GELU activation (needs temp registers for polynomial)\n #pragma unroll\n for (int i = 0; i < 8; ++i) v[i] = gelu(v[i]);\n\n // Apply per-channel scale\n float s[8];\n load_global_f32x4(&s[0], scale + c);\n load_global_f32x4(&s[4], scale + c + 4);\n\n #pragma unroll\n for (int i = 0; i < 8; ++i) v[i] *= s[i];\n\n // Quantize to FP8 for next layer\n uint8_t q[8];\n #pragma unroll\n for (int i = 0; i < 8; ++i) q[i] = float_to_e4m3(v[i]);\n\n // Store quantized output\n store_global_u8x8(output + c, q);\n\n // Total temp registers used: ~24 -- NO SPILLING\n // On Hopper this epilogue would need 24 + 128 (acc) = 152 per thread\n }\n}\n```"}, "after": {"statement": "For `sm_100a`, tcgen05 D is addressed in Tensor Memory rather than supplied as a per-thread register vector. The CTA-visible TMEM structure has 128 lanes by 512 columns of 32-bit cells, or 256 KiB when fully allocated. Allocation is dynamic in columns.\n\nThis removes the resident D vector from the ordinary register file. It does not remove all register pressure: descriptors, addresses, loop state, pipeline state, and each TMEM-to-register epilogue batch still need registers. Both Hopper and compute-capability-10.0 Blackwell have 64K 32-bit registers per SM and at most 255 registers per thread. Occupancy, spilling, and practical tile size therefore remain properties of the compiled kernel and launch configuration.\n\n3. Initialize and publish the barriers used by TMA and tcgen05. If the first MMA should compute only `A*B`, supply a false `enable-input-d` predicate instead of assuming a mandatory TMEM zero-store pass.\n4. Issue MMA from the permitted elected thread with valid A/B descriptors, instruction descriptor, predicate, and lifetimes.\n5. Attach completion of the relevant tcgen05 work to an mbarrier with `tcgen05.commit`. A fence controls execution ordering but is not a completion wait."}, "reason": {"statement": "Preserve the design opportunity without promising allocation or performance.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"]}} -{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "The most powerful pattern enabled by TMEM: overlapping the current tile's epilogue with the next tile's MMA computation. This is impossible with register accumulators because the accumulator registers are still in use.\n\n```cuda\n__global__ void overlapped_gemm(/* ... */) {\n // Two TMEM accumulator buffers\n uint32_t tmem_a = tmem_alloc(256);\n uint32_t tmem_b = tmem_alloc(256);\n uint32_t* tmem_cur = &tmem_a;\n uint32_t* tmem_nxt = &tmem_b;\n\n // Compute first tile into tmem_a\n compute_tile(*tmem_cur, tile_0);\n fence_and_sync();\n\n for (int t = 1; t < num_tiles; ++t) {\n // START: next tile MMA into tmem_nxt (background, async)\n zero_tmem(*tmem_nxt);\n start_mma(*tmem_nxt, tile_t);\n\n // SIMULTANEOUSLY: epilogue of current tile from tmem_cur\n // MMA is running in the background while we read the old accumulator!\n epilogue_from_tmem(*tmem_cur, output_tile[t-1]);\n\n // Wait for MMA to complete\n fence_and_sync();\n\n // Swap buffers\n uint32_t* tmp = tmem_cur;\n tmem_cur = tmem_nxt;\n tmem_nxt = tmp;\n }\n\n // Final epilogue\n epilogue_from_tmem(*tmem_cur, output_tile[num_tiles-1]);\n\n tmem_dealloc(tmem_a, 256);\n tmem_dealloc(tmem_b, 256);\n}\n```"}, "after": {"statement": "6. After the completion handoff, participating epilogue lanes execute a legal `tcgen05.ld` shape and `tcgen05.wait::ld` before using the loaded registers.\n7. After all readers finish, every lane of the designated warp executes the matching deallocation on every kernel exit path. Allocation/deallocation address, column count, and CTA-group mode must agree."}, "reason": {"statement": "Describe overlap as a scheduling option requiring disjoint regions and explicit handoffs.", "urls": ["https://arxiv.org/abs/2407.08608", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "```cuda\n// HOPPER: Accumulator lives and dies in registers\n__global__ void hopper_kernel(/* ... */) {\n // 1. Declare accumulator in registers\n float acc[4][32]; // 128 registers per thread!\n\n // 2. Zero-initialize\n #pragma unroll\n for (int i = 0; i < 4; ++i)\n for (int j = 0; j < 32; ++j)\n acc[i][j] = 0.0f;\n\n // 3. Mainloop: wgmma accumulates into registers\n for (int k = 0; k < K_tiles; ++k) {\n // Load A via ldmatrix\n uint32_t a_frag[4];\n ldmatrix(a_frag, smem_a + k * TILE_K);\n\n // wgmma: reads A from registers, B from SMEM descriptor\n // Accumulates into acc[] registers\n wgmma_m64n256k16(acc, a_frag, smem_b_desc);\n wgmma_commit();\n wgmma_wait();\n }\n\n // 4. Epilogue: acc is directly in registers -- fast access\n // But: if epilogue needs temporary storage, registers are exhausted\n #pragma unroll\n for (int i = 0; i < 4; ++i) {\n for (int j = 0; j < 32; ++j) {\n float val = acc[i][j];\n val += bias[col_base + j]; // bias add\n val = fmaxf(val, 0.0f); // ReLU\n // Need to convert and store -- no registers left for temp!\n C[row * N + col_base + j] = __float2half(val);\n }\n }\n\n // 5. Accumulator freed implicitly when CTA exits\n}\n```"}, "after": {"statement": "For Hopper `wgmma.mma_async.m64nNk16` with an FP32 accumulator, each warpgroup thread holds `N/2` FP32 D registers. At `N=256`, that is 128 registers per thread for D. WGMMA updates that register vector asynchronously; the program uses the WGMMA fence, commit-group, and wait-group mechanisms before dependent use.\n\n| Resident D | Per-thread register fragment | TMEM region addressed by `taddr` |\n| Example FP32 D cost | `N/2` registers/thread; 128 at `N=256` | No resident per-thread D vector |\n| First accumulation | Set WGMMA `scale-d` false to compute `D=A*B`, or initialize/use D when accumulation is intended | Set `enable-input-d` false to compute `D=A*B`, or explicitly initialize D when the algorithm needs another value |\n| Compute completion | WGMMA commit/wait group | `tcgen05.commit` to an mbarrier, then wait before a consumer reads/reuses D |\n| Epilogue access | Use the thread's D registers | Collective `tcgen05.ld`, then `tcgen05.wait::ld` before consuming result registers |\n| Cleanup | Register lifetime ends with the thread | Matching collective `tcgen05.dealloc` before kernel exit |\n| Double buffering | Two simultaneously live D fragments consume two register fragments | Two live outputs consume disjoint TMEM columns inside the allocation |\n\nThe table describes storage contracts, not an occupancy prediction. Instruction shape is also not necessarily the same as a composed CTA tile."}, "reason": {"statement": "Separate the PTX register-D contract from compiler allocation conclusions.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#local-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "```cuda\n// BLACKWELL: Accumulator lives in TMEM\n__global__ void blackwell_kernel(/* ... */) {\n // 1. Allocate TMEM (explicit, must be done once)\n __shared__ uint32_t s_tmem_acc;\n if (threadIdx.x == 0) {\n uint32_t smem_addr =\n static_cast(__cvta_generic_to_shared(&s_tmem_acc));\n asm volatile(\n \"tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;\"\n :: \"r\"(smem_addr), \"r\"(256)\n );\n }\n __syncthreads();\n uint32_t tmem_acc = s_tmem_acc;\n\n // 2. Zero-initialize TMEM\n for (int c = 0; c < 256; c += 4) {\n asm volatile(\n \"tcgen05.st.sync.aligned.32x1b.x4.b32 [%0], {%1,%2,%3,%4};\"\n : : \"r\"(tmem_acc + c),\n \"f\"(0.f), \"f\"(0.f), \"f\"(0.f), \"f\"(0.f)\n );\n }\n\n // 3. Mainloop: tcgen05 accumulates into TMEM\n for (int k = 0; k < K_tiles; ++k) {\n // NO ldmatrix -- tcgen05 reads directly from SMEM\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.mma.cta_group::1.kind::f16 \"\n \"[%0], %1, %2, %3, 1;\"\n : : \"r\"(tmem_acc), \"l\"(desc_a), \"l\"(desc_b), \"r\"(0)\n );\n }\n // NO commit/wait -- fully async\n }\n\n // 4. Fence before reading\n asm volatile(\"tcgen05.fence::before_thread_sync;\");\n __syncthreads();\n\n // 5. Epilogue: read from TMEM to registers (in small batches)\n // PLENTY of registers available for epilogue temporaries!\n for (int c = 0; c < 256; c += 4) {\n float4 vals;\n asm volatile(\n \"tcgen05.ld.sync.aligned.32x1b.x4.b32 {%0,%1,%2,%3}, [%4];\"\n : \"=f\"(vals.x), \"=f\"(vals.y), \"=f\"(vals.z), \"=f\"(vals.w)\n : \"r\"(tmem_acc + c)\n );\n\n // Apply epilogue ops with register headroom\n vals.x += bias[col_base + c];\n vals.y += bias[col_base + c + 1];\n vals.z += bias[col_base + c + 2];\n vals.w += bias[col_base + c + 3];\n\n vals.x = fmaxf(vals.x, 0.0f); // ReLU\n vals.y = fmaxf(vals.y, 0.0f);\n vals.z = fmaxf(vals.z, 0.0f);\n vals.w = fmaxf(vals.w, 0.0f);\n\n // Vectorized store -- registers available for conversion\n half2 h01 = __floats2half2_rn(vals.x, vals.y);\n half2 h23 = __floats2half2_rn(vals.z, vals.w);\n *reinterpret_cast(&C[row * N + col_base + c]) = h01;\n *reinterpret_cast(&C[row * N + col_base + c + 2]) = h23;\n }\n\n // 6. Deallocate TMEM (explicit, MUST be done in persistent kernels)\n if (threadIdx.x == 0) {\n asm volatile(\n \"tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;\"\n : : \"r\"(tmem_acc), \"r\"(256)\n );\n }\n}\n```"}, "after": {"statement": "1. Choose one CTA-group mode for the kernel. For allocation, `nCols` is a power of two in `[32, 512]`; allocations are column-granular and cover all 128 lanes.\n2. Have every lane of one designated warp execute the same `.cta_group::1` allocation. Synchronize before other threads read the 32-bit `taddr` written to shared memory. Two-CTA mode requires one warp in each live peer CTA.\n3. Initialize and publish the barriers used by TMA and tcgen05. If the first MMA should compute only `A*B`, supply a false `enable-input-d` predicate instead of assuming a mandatory TMEM zero-store pass.\n4. Issue MMA from the permitted elected thread with valid A/B descriptors, instruction descriptor, predicate, and lifetimes.\n5. Attach completion of the relevant tcgen05 work to an mbarrier with `tcgen05.commit`. A fence controls execution ordering but is not a completion wait."}, "reason": {"statement": "Replace the block with lifecycle invariants and known-good source routes.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "With accumulators in TMEM, the tile size is no longer constrained by register count:\n\n```cuda\n// Hopper: large tiles cause spilling\n// m64xn256 uses 128 acc registers/thread -> barely fits\n// m128xn256 would need 256 acc registers/thread -> spills guaranteed\n\n// Blackwell: tile size limited by TMEM columns (512) and SMEM, not registers\n// m128xn256: 256 TMEM cols, 0 acc registers -> fits easily\n// m128xn512: 512 TMEM cols, 0 acc registers -> uses full TMEM budget\n// m256xn256 (2-SM): 256 TMEM cols/SM, 0 acc registers -> fits with cooperative\n```"}, "after": {"statement": "For `sm_100a`, tcgen05 D is addressed in Tensor Memory rather than supplied as a per-thread register vector. The CTA-visible TMEM structure has 128 lanes by 512 columns of 32-bit cells, or 256 KiB when fully allocated. Allocation is dynamic in columns.\n\nThis removes the resident D vector from the ordinary register file. It does not remove all register pressure: descriptors, addresses, loop state, pipeline state, and each TMEM-to-register epilogue batch still need registers. Both Hopper and compute-capability-10.0 Blackwell have 64K 32-bit registers per SM and at most 255 registers per thread. Occupancy, spilling, and practical tile size therefore remain properties of the compiled kernel and launch configuration.\n\nTMEM makes it practical to keep multiple output regions independently addressable while different warp roles issue MMA and drain a completed region. Safe overlap needs three distinct proofs:\n\n- compute completion occurs before an epilogue loads a region;\n- all epilogue loads complete before that region is overwritten or deallocated;\n- producer/consumer state and phase cannot alias a different pipeline stage.\n\nTMEM is not the only way to overlap tensor-core and non-matmul work. FlashAttention-3 already uses warp specialization and matmul/softmax interleaving on Hopper with register accumulators. The migration benefit is the different storage and lifetime tradeoff, not a new theorem that overlap was previously impossible."}, "reason": {"statement": "Replace categorical tile advice with descriptor legality plus empirical tuning.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit", "https://arxiv.org/abs/2407.08608"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "On Hopper, complex epilogues (bias + activation + quantization + scaling) often spill because registers are already exhausted by the accumulator. On Blackwell, the epilogue has the full register file available:\n\n```cuda\n// Blackwell epilogue: full register file available\n__device__ void rich_epilogue(uint32_t tmem_acc, float* output,\n const float* bias, const float* scale,\n int M, int N) {\n // Read TMEM in chunks\n for (int c = 0; c < 256; c += 8) {\n // Load 8 values from TMEM (8 registers -- trivial)\n float v[8];\n tmem_load_f32x4(tmem_acc + c, &v[0]);\n tmem_load_f32x4(tmem_acc + c + 4, &v[4]);\n\n // Apply bias (8 registers for bias values)\n float b[8];\n load_global_f32x4(&b[0], bias + c);\n load_global_f32x4(&b[4], bias + c + 4);\n\n #pragma unroll\n for (int i = 0; i < 8; ++i) v[i] += b[i];\n\n // Apply GELU activation (needs temp registers for polynomial)\n #pragma unroll\n for (int i = 0; i < 8; ++i) v[i] = gelu(v[i]);\n\n // Apply per-channel scale\n float s[8];\n load_global_f32x4(&s[0], scale + c);\n load_global_f32x4(&s[4], scale + c + 4);\n\n #pragma unroll\n for (int i = 0; i < 8; ++i) v[i] *= s[i];\n\n // Quantize to FP8 for next layer\n uint8_t q[8];\n #pragma unroll\n for (int i = 0; i < 8; ++i) q[i] = float_to_e4m3(v[i]);\n\n // Store quantized output\n store_global_u8x8(output + c, q);\n\n // Total temp registers used: ~24 -- NO SPILLING\n // On Hopper this epilogue would need 24 + 128 (acc) = 152 per thread\n }\n}\n```"}, "after": {"statement": "For `sm_100a`, tcgen05 D is addressed in Tensor Memory rather than supplied as a per-thread register vector. The CTA-visible TMEM structure has 128 lanes by 512 columns of 32-bit cells, or 256 KiB when fully allocated. Allocation is dynamic in columns.\n\nThis removes the resident D vector from the ordinary register file. It does not remove all register pressure: descriptors, addresses, loop state, pipeline state, and each TMEM-to-register epilogue batch still need registers. Both Hopper and compute-capability-10.0 Blackwell have 64K 32-bit registers per SM and at most 255 registers per thread. Occupancy, spilling, and practical tile size therefore remain properties of the compiled kernel and launch configuration.\n\n3. Initialize and publish the barriers used by TMA and tcgen05. If the first MMA should compute only `A*B`, supply a false `enable-input-d` predicate instead of assuming a mandatory TMEM zero-store pass.\n4. Issue MMA from the permitted elected thread with valid A/B descriptors, instruction descriptor, predicate, and lifetimes.\n5. Attach completion of the relevant tcgen05 work to an mbarrier with `tcgen05.commit`. A fence controls execution ordering but is not a completion wait."}, "reason": {"statement": "Preserve the design opportunity without promising allocation or performance.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "The most powerful pattern enabled by TMEM: overlapping the current tile's epilogue with the next tile's MMA computation. This is impossible with register accumulators because the accumulator registers are still in use.\n\n```cuda\n__global__ void overlapped_gemm(/* ... */) {\n // Two TMEM accumulator buffers\n uint32_t tmem_a = tmem_alloc(256);\n uint32_t tmem_b = tmem_alloc(256);\n uint32_t* tmem_cur = &tmem_a;\n uint32_t* tmem_nxt = &tmem_b;\n\n // Compute first tile into tmem_a\n compute_tile(*tmem_cur, tile_0);\n fence_and_sync();\n\n for (int t = 1; t < num_tiles; ++t) {\n // START: next tile MMA into tmem_nxt (background, async)\n zero_tmem(*tmem_nxt);\n start_mma(*tmem_nxt, tile_t);\n\n // SIMULTANEOUSLY: epilogue of current tile from tmem_cur\n // MMA is running in the background while we read the old accumulator!\n epilogue_from_tmem(*tmem_cur, output_tile[t-1]);\n\n // Wait for MMA to complete\n fence_and_sync();\n\n // Swap buffers\n uint32_t* tmp = tmem_cur;\n tmem_cur = tmem_nxt;\n tmem_nxt = tmp;\n }\n\n // Final epilogue\n epilogue_from_tmem(*tmem_cur, output_tile[num_tiles-1]);\n\n tmem_dealloc(tmem_a, 256);\n tmem_dealloc(tmem_b, 256);\n}\n```"}, "after": {"statement": "6. After the completion handoff, participating epilogue lanes execute a legal `tcgen05.ld` shape and `tcgen05.wait::ld` before using the loaded registers.\n7. After all readers finish, every lane of the designated warp executes the matching deallocation on every kernel exit path. Allocation/deallocation address, column count, and CTA-group mode must agree."}, "reason": {"statement": "Describe overlap as a scheduling option requiring disjoint regions and explicit handoffs.", "urls": ["https://arxiv.org/abs/2407.08608", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit"]}} {"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "```\nFlashAttention on Hopper needs registers for:\n - QK^T accumulator: 64 registers (m64xn64 attention scores)\n - PV accumulator: 128 registers (m64xn256 output)\n - Softmax state: 4 registers (rowmax, rowsum)\n - Q fragment: 16 registers\n - K fragment: 16 registers\n - V fragment: 16 registers\n - Loop state: 10 registers\n ──────────────────────────────\n Total: ~254 registers per thread\n\n Available: 256 registers/thread at 1 CTA/SM occupancy\n Margin: 2 registers (!!)\n \n Result: compiler must spill, or tile sizes must shrink\n```\n\n```\nFlashAttention on Blackwell:\n - QK^T accumulator: 0 registers (TMEM buffer 1, 64 cols)\n - PV accumulator: 0 registers (TMEM buffer 2, 256 cols)\n - Softmax state: 4 registers (rowmax, rowsum)\n - SMEM descriptors: 6 registers\n - Loop state: 10 registers\n ──────────────────────────────\n Total: ~20 registers per thread\n\n Available: 256 registers/thread\n Margin: 236 registers -- massive headroom\n\n Result: Can use for ping-pong scheduling, software exp emulation,\n larger tiles, more pipeline stages\n```"}, "after": {"statement": "Representative PTX ISA 9.0 forms are shown below. They omit declarations, descriptor construction, collective control flow, barriers, and inline-assembly constraints, so they are not a standalone kernel.\n\n```ptx\ntcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [saddr], nCols;\ntcgen05.mma.cta_group::1.kind::f16 [taddr], a_desc, b_desc, idesc, p;\ntcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cta.b64 [mbar];\ntcgen05.ld.sync.aligned.32x32b.x1.b32 {r0}, [taddr];\ntcgen05.wait::ld.sync.aligned;\ntcgen05.dealloc.cta_group::1.sync.aligned.b32 taddr, nCols;\n```"}, "reason": {"statement": "Replace invented budgets with the paper's directly reported TMEM roles.", "urls": ["https://arxiv.org/abs/2603.05451v1", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions"]}} -{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "```cuda\n// WRONG: Fine-grained TMEM access in a tight loop (high latency)\nfor (int i = 0; i < 256; ++i) {\n float v = tmem_load_f32(tmem_acc + i); // 420 cycle latency each!\n output[i] = v;\n}\n\n// CORRECT: Vectorized loads to amortize latency\nfor (int i = 0; i < 256; i += 4) {\n float4 v = tmem_load_f32x4(tmem_acc + i); // Single 420-cycle access\n output[i] = v.x;\n output[i+1] = v.y;\n output[i+2] = v.z;\n output[i+3] = v.w;\n}\n```"}, "after": {"statement": "3. Initialize and publish the barriers used by TMA and tcgen05. If the first MMA should compute only `A*B`, supply a false `enable-input-d` predicate instead of assuming a mandatory TMEM zero-store pass.\n4. Issue MMA from the permitted elected thread with valid A/B descriptors, instruction descriptor, predicate, and lifetimes.\n5. Attach completion of the relevant tcgen05 work to an mbarrier with `tcgen05.commit`. A fence controls execution ordering but is not a completion wait.\n\nTMEM makes it practical to keep multiple output regions independently addressable while different warp roles issue MMA and drain a completed region. Safe overlap needs three distinct proofs:\n\n- compute completion occurs before an epilogue loads a region;\n- all epilogue loads complete before that region is overwritten or deallocated;\n- producer/consumer state and phase cannot alias a different pipeline stage.\n\nTMEM is not the only way to overlap tensor-core and non-matmul work. FlashAttention-3 already uses warp specialization and matmul/softmax interleaving on Hopper with register accumulators. The migration benefit is the different storage and lifetime tradeoff, not a new theorem that overlap was previously impossible."}, "reason": {"statement": "Keep batching as a profiling hypothesis, not a latency theorem.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://arxiv.org/abs/2512.02189", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://arxiv.org/abs/2407.08608"]}} -{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "```cuda\n// Hopper: tile size was m64xn128 to avoid register spilling\n// Migrating to Blackwell: keep same tile = leaving performance on the table\n\n// Blackwell should use at minimum m128xn256 (1-SM) or m256xn256 (2-SM)\n// The freed registers allow larger tiles without any spilling risk\n```"}, "after": {"statement": "TMEM makes it practical to keep multiple output regions independently addressable while different warp roles issue MMA and drain a completed region. Safe overlap needs three distinct proofs:\n\n- compute completion occurs before an epilogue loads a region;\n- all epilogue loads complete before that region is overwritten or deallocated;\n- producer/consumer state and phase cannot alias a different pipeline stage.\n\nTMEM is not the only way to overlap tensor-core and non-matmul work. FlashAttention-3 already uses warp specialization and matmul/softmax interleaving on Hopper with register accumulators. The migration benefit is the different storage and lifetime tradeoff, not a new theorem that overlap was previously impossible."}, "reason": {"statement": "Turn the mandate into an evidence-driven tuning checklist.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://arxiv.org/abs/2407.08608"]}} -{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "```cuda\n// WRONG: Forgetting to dealloc in a persistent kernel\n__global__ void persistent_kernel(/* ... */) {\n uint32_t tmem = tmem_alloc(256); // Allocated once\n\n while (has_work()) {\n compute_tile(tmem);\n // BUG: If the kernel exits early (e.g., error path),\n // TMEM is leaked. Next CTA on this SM cannot allocate.\n }\n // Missing: tmem_dealloc(tmem, 256);\n}\n\n// CORRECT: Always dealloc, even on error paths\n__global__ void persistent_kernel(/* ... */) {\n uint32_t tmem = tmem_alloc(256);\n\n while (has_work()) {\n compute_tile(tmem);\n }\n\n tmem_dealloc(tmem, 256); // Always reached\n}\n```"}, "after": {"statement": "1. Choose one CTA-group mode for the kernel. For allocation, `nCols` is a power of two in `[32, 512]`; allocations are column-granular and cover all 128 lanes.\n2. Have every lane of one designated warp execute the same `.cta_group::1` allocation. Synchronize before other threads read the 32-bit `taddr` written to shared memory. Two-CTA mode requires one warp in each live peer CTA.\n\nTMEM makes it practical to keep multiple output regions independently addressable while different warp roles issue MMA and drain a completed region. Safe overlap needs three distinct proofs:\n\n- compute completion occurs before an epilogue loads a region;\n- all epilogue loads complete before that region is overwritten or deallocated;\n- producer/consumer state and phase cannot alias a different pipeline stage.\n\nTMEM is not the only way to overlap tensor-core and non-matmul work. FlashAttention-3 already uses warp specialization and matmul/softmax interleaving on Hopper with register accumulators. The migration benefit is the different storage and lifetime tradeoff, not a new theorem that overlap was previously impossible."}, "reason": {"statement": "State the normative obligation and control-flow requirements.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://arxiv.org/abs/2407.08608"]}} -{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "| Zero-initialization | Loop over register array | `tcgen05.st` zero pattern |"}, "after": {"statement": "3. Initialize and publish the barriers used by TMA and tcgen05. If the first MMA should compute only `A*B`, supply a false `enable-input-d` predicate instead of assuming a mandatory TMEM zero-store pass.\n4. Issue MMA from the permitted elected thread with valid A/B descriptors, instruction descriptor, predicate, and lifetimes.\n5. Attach completion of the relevant tcgen05 work to an mbarrier with `tcgen05.commit`. A fence controls execution ordering but is not a completion wait."}, "reason": {"statement": "Describe both initialization choices without prescribing a needless store.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"]}} -{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "| Epilogue headroom | Minimal (spill risk) | Ample (full register file) |"}, "after": {"statement": "For `sm_100a`, tcgen05 D is addressed in Tensor Memory rather than supplied as a per-thread register vector. The CTA-visible TMEM structure has 128 lanes by 512 columns of 32-bit cells, or 256 KiB when fully allocated. Allocation is dynamic in columns.\n\nThis removes the resident D vector from the ordinary register file. It does not remove all register pressure: descriptors, addresses, loop state, pipeline state, and each TMEM-to-register epilogue batch still need registers. Both Hopper and compute-capability-10.0 Blackwell have 64K 32-bit registers per SM and at most 255 registers per thread. Occupancy, spilling, and practical tile size therefore remain properties of the compiled kernel and launch configuration.\n\nTMEM makes it practical to keep multiple output regions independently addressable while different warp roles issue MMA and drain a completed region. Safe overlap needs three distinct proofs:\n\n- compute completion occurs before an epilogue loads a region;\n- all epilogue loads complete before that region is overwritten or deallocated;\n- producer/consumer state and phase cannot alias a different pipeline stage.\n\nTMEM is not the only way to overlap tensor-core and non-matmul work. FlashAttention-3 already uses warp specialization and matmul/softmax interleaving on Hopper with register accumulators. The migration benefit is the different storage and lifetime tradeoff, not a new theorem that overlap was previously impossible."}, "reason": {"statement": "State reduced resident-D pressure as an opportunity, not a guarantee.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://arxiv.org/abs/2407.08608"]}} -{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "| Max practical tile | m64xn256 (limited by registers) | m128xn256 or m256xn256 (limited by TMEM cols) |"}, "after": {"statement": "For Hopper `wgmma.mma_async.m64nNk16` with an FP32 accumulator, each warpgroup thread holds `N/2` FP32 D registers. At `N=256`, that is 128 registers per thread for D. WGMMA updates that register vector asynchronously; the program uses the WGMMA fence, commit-group, and wait-group mechanisms before dependent use.\n\nTMEM makes it practical to keep multiple output regions independently addressable while different warp roles issue MMA and drain a completed region. Safe overlap needs three distinct proofs:\n\n- compute completion occurs before an epilogue loads a region;\n- all epilogue loads complete before that region is overwritten or deallocated;\n- producer/consumer state and phase cannot alias a different pipeline stage.\n\nTMEM is not the only way to overlap tensor-core and non-matmul work. FlashAttention-3 already uses warp specialization and matmul/softmax interleaving on Hopper with register accumulators. The migration benefit is the different storage and lifetime tradeoff, not a new theorem that overlap was previously impossible."}, "reason": {"statement": "Separate legal instruction shapes from empirical CTA tile tuning.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-multiply-accumulate-instructions-wgmma-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://arxiv.org/abs/2407.08608"]}} -{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "```cuda\n// TMEM double-buffering: ping-pong between two 128x256 accumulator regions\n//\n// TMEM physical layout (per SM):\n// +------ 256 cols ------+------ 256 cols ------+\n// | | |\n// | Buffer A (active) | Buffer B (drain) | 128 rows\n// | MMA writes here | Epilogue reads |\n// | | |\n// +----------------------+----------------------+\n//\n// After tile completes, roles swap:\n// Buffer A becomes \"drain\", Buffer B becomes \"active\"\n\n#define TMEM_BUF_A_OFFSET 0\n#define TMEM_BUF_B_OFFSET 256\n#define TMEM_COLS_PER_BUF 256\n\n__device__ void tmem_double_buffer_mainloop(\n const GemmParams& params,\n int num_output_tiles)\n{\n int warp_id = threadIdx.x / 32;\n int lane_id = threadIdx.x % 32;\n\n // Synchronization between MMA warp and epilogue warps\n __shared__ uint64_t mbar_acc_ready[2]; // MMA done, epilogue can read\n __shared__ uint64_t mbar_acc_drained[2]; // Epilogue done, MMA can reuse\n\n if (threadIdx.x == 0) {\n mbarrier_init(&mbar_acc_ready[0], 1);\n mbarrier_init(&mbar_acc_ready[1], 1);\n mbarrier_init(&mbar_acc_drained[0], 1);\n mbarrier_init(&mbar_acc_drained[1], 1);\n }\n __syncthreads();\n\n for (int tile = 0; tile < num_output_tiles; tile++) {\n int buf = tile % 2;\n int tmem_offset = buf ? TMEM_BUF_B_OFFSET : TMEM_BUF_A_OFFSET;\n\n if (warp_id == 1) {\n // === MMA WARP ===\n // Wait for epilogue to finish draining this buffer\n if (tile >= 2) {\n mbarrier_wait(&mbar_acc_drained[buf]);\n }\n\n // Clear accumulator region before new tile\n if (lane_id == 0) {\n tmem_clear(tmem_offset, TMEM_COLS_PER_BUF);\n }\n __syncwarp();\n\n // Accumulate K-tiles into this TMEM buffer\n for (int k = 0; k < num_k_tiles; k++) {\n // (TMA load sync omitted -- see pipeline-stages)\n if (lane_id == 0) {\n tcgen05_mma_accumulate(tmem_offset);\n }\n __syncwarp();\n }\n\n // Signal epilogue that accumulator is ready\n if (lane_id == 0) {\n mbarrier_arrive(&mbar_acc_ready[buf]);\n }\n\n } else if (warp_id >= 2) {\n // === EPILOGUE WARPS ===\n // Wait for MMA to finish accumulating\n mbarrier_wait(&mbar_acc_ready[buf]);\n\n // Read TMEM and store to global memory\n int rows_per_warp = TILE_M / 14;\n int my_row = (warp_id - 2) * rows_per_warp;\n for (int r = my_row; r < my_row + rows_per_warp; r++) {\n for (int c = lane_id; c < TMEM_COLS_PER_BUF; c += 32) {\n float val = tmem_load_f32(r, tmem_offset + c);\n // Apply epilogue and store...\n params.C_ptr[r * params.N + c] = __float2half(val);\n }\n }\n\n // All epilogue warps must finish reading TMEM before the MMA\n // warp can overwrite this half-buffer. Each epilogue warp arrives\n // on a shared mbarrier; the mbarrier is initialized with\n // arrival_count = NUM_EPILOGUE_WARPS (e.g. 14 for warps 2-15).\n // The MMA warp waits on this mbarrier before reusing the buffer.\n if (lane_id == 0) {\n mbarrier_arrive(&mbar_acc_drained[buf]);\n }\n // mbar_acc_drained[buf] fires only after ALL epilogue warps arrive\n }\n }\n}\n```"}, "after": {"statement": "The CTA-visible TMEM address structure has 128 lanes and 512 columns of 32-bit cells. Allocation is column-granular across all 128 lanes. A kernel that allocates 512 columns may choose two 256-column regions:\n\n```text\nregion[0] = columns [0, 256)\nregion[1] = columns [256, 512)\n```\n\nThis equal split is one implementation choice. The actual invariant is that every simultaneously live region is inside the allocation and does not alias another region whose MMA or epilogue access is still outstanding. Region sizes can differ, and logical element capacity depends on the tcgen05 instruction kind and packing rather than the raw cell count alone.\n\nFor each region, prove this lifecycle:\n\n| free → MMA-owned | every prior epilogue load from the region has completed |\n| MMA-owned → ready | relevant tcgen05 work is committed to an mbarrier and completion is observed |\n| ready → epilogue-owned | readers observe the matching barrier phase before `tcgen05.ld` |\n| epilogue-owned → free | all collective loads complete with `tcgen05.wait::ld`, then every reader reports completion |\n\nTMEM must first be allocated by the required participating warp and its address safely published. Matching collective deallocation is required before every kernel exit. Reused mbarriers need the correct arrival count and phase/parity state; a one-time wait on an address is not a reusable two-buffer protocol."}, "reason": {"statement": "Replace unsafe pseudo-implementation with verified ownership invariants and a pinned complete implementation route.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "```cuda\n// WRONG: Fine-grained TMEM access in a tight loop (high latency)\nfor (int i = 0; i < 256; ++i) {\n float v = tmem_load_f32(tmem_acc + i); // 420 cycle latency each!\n output[i] = v;\n}\n\n// CORRECT: Vectorized loads to amortize latency\nfor (int i = 0; i < 256; i += 4) {\n float4 v = tmem_load_f32x4(tmem_acc + i); // Single 420-cycle access\n output[i] = v.x;\n output[i+1] = v.y;\n output[i+2] = v.z;\n output[i+3] = v.w;\n}\n```"}, "after": {"statement": "3. Initialize and publish the barriers used by TMA and tcgen05. If the first MMA should compute only `A*B`, supply a false `enable-input-d` predicate instead of assuming a mandatory TMEM zero-store pass.\n4. Issue MMA from the permitted elected thread with valid A/B descriptors, instruction descriptor, predicate, and lifetimes.\n5. Attach completion of the relevant tcgen05 work to an mbarrier with `tcgen05.commit`. A fence controls execution ordering but is not a completion wait.\n\nTMEM makes it practical to keep multiple output regions independently addressable while different warp roles issue MMA and drain a completed region. Safe overlap needs three distinct proofs:\n\n- compute completion occurs before an epilogue loads a region;\n- all epilogue loads complete before that region is overwritten or deallocated;\n- producer/consumer state and phase cannot alias a different pipeline stage.\n\nTMEM is not the only way to overlap tensor-core and non-matmul work. FlashAttention-3 already uses warp specialization and matmul/softmax interleaving on Hopper with register accumulators. The migration benefit is the different storage and lifetime tradeoff, not a new theorem that overlap was previously impossible."}, "reason": {"statement": "Keep batching as a profiling hypothesis, not a latency theorem.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld", "https://arxiv.org/abs/2512.02189", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit", "https://arxiv.org/abs/2407.08608"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "```cuda\n// Hopper: tile size was m64xn128 to avoid register spilling\n// Migrating to Blackwell: keep same tile = leaving performance on the table\n\n// Blackwell should use at minimum m128xn256 (1-SM) or m256xn256 (2-SM)\n// The freed registers allow larger tiles without any spilling risk\n```"}, "after": {"statement": "TMEM makes it practical to keep multiple output regions independently addressable while different warp roles issue MMA and drain a completed region. Safe overlap needs three distinct proofs:\n\n- compute completion occurs before an epilogue loads a region;\n- all epilogue loads complete before that region is overwritten or deallocated;\n- producer/consumer state and phase cannot alias a different pipeline stage.\n\nTMEM is not the only way to overlap tensor-core and non-matmul work. FlashAttention-3 already uses warp specialization and matmul/softmax interleaving on Hopper with register accumulators. The migration benefit is the different storage and lifetime tradeoff, not a new theorem that overlap was previously impossible."}, "reason": {"statement": "Turn the mandate into an evidence-driven tuning checklist.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit", "https://arxiv.org/abs/2407.08608"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "```cuda\n// WRONG: Forgetting to dealloc in a persistent kernel\n__global__ void persistent_kernel(/* ... */) {\n uint32_t tmem = tmem_alloc(256); // Allocated once\n\n while (has_work()) {\n compute_tile(tmem);\n // BUG: If the kernel exits early (e.g., error path),\n // TMEM is leaked. Next CTA on this SM cannot allocate.\n }\n // Missing: tmem_dealloc(tmem, 256);\n}\n\n// CORRECT: Always dealloc, even on error paths\n__global__ void persistent_kernel(/* ... */) {\n uint32_t tmem = tmem_alloc(256);\n\n while (has_work()) {\n compute_tile(tmem);\n }\n\n tmem_dealloc(tmem, 256); // Always reached\n}\n```"}, "after": {"statement": "1. Choose one CTA-group mode for the kernel. For allocation, `nCols` is a power of two in `[32, 512]`; allocations are column-granular and cover all 128 lanes.\n2. Have every lane of one designated warp execute the same `.cta_group::1` allocation. Synchronize before other threads read the 32-bit `taddr` written to shared memory. Two-CTA mode requires one warp in each live peer CTA.\n\nTMEM makes it practical to keep multiple output regions independently addressable while different warp roles issue MMA and drain a completed region. Safe overlap needs three distinct proofs:\n\n- compute completion occurs before an epilogue loads a region;\n- all epilogue loads complete before that region is overwritten or deallocated;\n- producer/consumer state and phase cannot alias a different pipeline stage.\n\nTMEM is not the only way to overlap tensor-core and non-matmul work. FlashAttention-3 already uses warp specialization and matmul/softmax interleaving on Hopper with register accumulators. The migration benefit is the different storage and lifetime tradeoff, not a new theorem that overlap was previously impossible."}, "reason": {"statement": "State the normative obligation and control-flow requirements.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit", "https://arxiv.org/abs/2407.08608"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "| Zero-initialization | Loop over register array | `tcgen05.st` zero pattern |"}, "after": {"statement": "3. Initialize and publish the barriers used by TMA and tcgen05. If the first MMA should compute only `A*B`, supply a false `enable-input-d` predicate instead of assuming a mandatory TMEM zero-store pass.\n4. Issue MMA from the permitted elected thread with valid A/B descriptors, instruction descriptor, predicate, and lifetimes.\n5. Attach completion of the relevant tcgen05 work to an mbarrier with `tcgen05.commit`. A fence controls execution ordering but is not a completion wait."}, "reason": {"statement": "Describe both initialization choices without prescribing a needless store.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "| Epilogue headroom | Minimal (spill risk) | Ample (full register file) |"}, "after": {"statement": "For `sm_100a`, tcgen05 D is addressed in Tensor Memory rather than supplied as a per-thread register vector. The CTA-visible TMEM structure has 128 lanes by 512 columns of 32-bit cells, or 256 KiB when fully allocated. Allocation is dynamic in columns.\n\nThis removes the resident D vector from the ordinary register file. It does not remove all register pressure: descriptors, addresses, loop state, pipeline state, and each TMEM-to-register epilogue batch still need registers. Both Hopper and compute-capability-10.0 Blackwell have 64K 32-bit registers per SM and at most 255 registers per thread. Occupancy, spilling, and practical tile size therefore remain properties of the compiled kernel and launch configuration.\n\nTMEM makes it practical to keep multiple output regions independently addressable while different warp roles issue MMA and drain a completed region. Safe overlap needs three distinct proofs:\n\n- compute completion occurs before an epilogue loads a region;\n- all epilogue loads complete before that region is overwritten or deallocated;\n- producer/consumer state and phase cannot alias a different pipeline stage.\n\nTMEM is not the only way to overlap tensor-core and non-matmul work. FlashAttention-3 already uses warp specialization and matmul/softmax interleaving on Hopper with register accumulators. The migration benefit is the different storage and lifetime tradeoff, not a new theorem that overlap was previously impossible."}, "reason": {"statement": "State reduced resident-D pressure as an opportunity, not a guarantee.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit", "https://arxiv.org/abs/2407.08608"]}} +{"path": "wiki/migration/register-to-tmem.md", "before": {"statement": "| Max practical tile | m64xn256 (limited by registers) | m128xn256 or m256xn256 (limited by TMEM cols) |"}, "after": {"statement": "For Hopper `wgmma.mma_async.m64nNk16` with an FP32 accumulator, each warpgroup thread holds `N/2` FP32 D registers. At `N=256`, that is 128 registers per thread for D. WGMMA updates that register vector asynchronously; the program uses the WGMMA fence, commit-group, and wait-group mechanisms before dependent use.\n\nTMEM makes it practical to keep multiple output regions independently addressable while different warp roles issue MMA and drain a completed region. Safe overlap needs three distinct proofs:\n\n- compute completion occurs before an epilogue loads a region;\n- all epilogue loads complete before that region is overwritten or deallocated;\n- producer/consumer state and phase cannot alias a different pipeline stage.\n\nTMEM is not the only way to overlap tensor-core and non-matmul work. FlashAttention-3 already uses warp specialization and matmul/softmax interleaving on Hopper with register accumulators. The migration benefit is the different storage and lifetime tradeoff, not a new theorem that overlap was previously impossible."}, "reason": {"statement": "Separate legal instruction shapes from empirical CTA tile tuning.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit", "https://arxiv.org/abs/2407.08608"]}} +{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "```cuda\n// TMEM double-buffering: ping-pong between two 128x256 accumulator regions\n//\n// TMEM physical layout (per SM):\n// +------ 256 cols ------+------ 256 cols ------+\n// | | |\n// | Buffer A (active) | Buffer B (drain) | 128 rows\n// | MMA writes here | Epilogue reads |\n// | | |\n// +----------------------+----------------------+\n//\n// After tile completes, roles swap:\n// Buffer A becomes \"drain\", Buffer B becomes \"active\"\n\n#define TMEM_BUF_A_OFFSET 0\n#define TMEM_BUF_B_OFFSET 256\n#define TMEM_COLS_PER_BUF 256\n\n__device__ void tmem_double_buffer_mainloop(\n const GemmParams& params,\n int num_output_tiles)\n{\n int warp_id = threadIdx.x / 32;\n int lane_id = threadIdx.x % 32;\n\n // Synchronization between MMA warp and epilogue warps\n __shared__ uint64_t mbar_acc_ready[2]; // MMA done, epilogue can read\n __shared__ uint64_t mbar_acc_drained[2]; // Epilogue done, MMA can reuse\n\n if (threadIdx.x == 0) {\n mbarrier_init(&mbar_acc_ready[0], 1);\n mbarrier_init(&mbar_acc_ready[1], 1);\n mbarrier_init(&mbar_acc_drained[0], 1);\n mbarrier_init(&mbar_acc_drained[1], 1);\n }\n __syncthreads();\n\n for (int tile = 0; tile < num_output_tiles; tile++) {\n int buf = tile % 2;\n int tmem_offset = buf ? TMEM_BUF_B_OFFSET : TMEM_BUF_A_OFFSET;\n\n if (warp_id == 1) {\n // === MMA WARP ===\n // Wait for epilogue to finish draining this buffer\n if (tile >= 2) {\n mbarrier_wait(&mbar_acc_drained[buf]);\n }\n\n // Clear accumulator region before new tile\n if (lane_id == 0) {\n tmem_clear(tmem_offset, TMEM_COLS_PER_BUF);\n }\n __syncwarp();\n\n // Accumulate K-tiles into this TMEM buffer\n for (int k = 0; k < num_k_tiles; k++) {\n // (TMA load sync omitted -- see pipeline-stages)\n if (lane_id == 0) {\n tcgen05_mma_accumulate(tmem_offset);\n }\n __syncwarp();\n }\n\n // Signal epilogue that accumulator is ready\n if (lane_id == 0) {\n mbarrier_arrive(&mbar_acc_ready[buf]);\n }\n\n } else if (warp_id >= 2) {\n // === EPILOGUE WARPS ===\n // Wait for MMA to finish accumulating\n mbarrier_wait(&mbar_acc_ready[buf]);\n\n // Read TMEM and store to global memory\n int rows_per_warp = TILE_M / 14;\n int my_row = (warp_id - 2) * rows_per_warp;\n for (int r = my_row; r < my_row + rows_per_warp; r++) {\n for (int c = lane_id; c < TMEM_COLS_PER_BUF; c += 32) {\n float val = tmem_load_f32(r, tmem_offset + c);\n // Apply epilogue and store...\n params.C_ptr[r * params.N + c] = __float2half(val);\n }\n }\n\n // All epilogue warps must finish reading TMEM before the MMA\n // warp can overwrite this half-buffer. Each epilogue warp arrives\n // on a shared mbarrier; the mbarrier is initialized with\n // arrival_count = NUM_EPILOGUE_WARPS (e.g. 14 for warps 2-15).\n // The MMA warp waits on this mbarrier before reusing the buffer.\n if (lane_id == 0) {\n mbarrier_arrive(&mbar_acc_drained[buf]);\n }\n // mbar_acc_drained[buf] fires only after ALL epilogue warps arrive\n }\n }\n}\n```"}, "after": {"statement": "The CTA-visible TMEM address structure has 128 lanes and 512 columns of 32-bit cells. Allocation is column-granular across all 128 lanes. A kernel that allocates 512 columns may choose two 256-column regions:\n\n```text\nregion[0] = columns [0, 256)\nregion[1] = columns [256, 512)\n```\n\nThis equal split is one implementation choice. The actual invariant is that every simultaneously live region is inside the allocation and does not alias another region whose MMA or epilogue access is still outstanding. Region sizes can differ, and logical element capacity depends on the tcgen05 instruction kind and packing rather than the raw cell count alone.\n\nFor each region, prove this lifecycle:\n\n| free → MMA-owned | every prior epilogue load from the region has completed |\n| MMA-owned → ready | relevant tcgen05 work is committed to an mbarrier and completion is observed |\n| ready → epilogue-owned | readers observe the matching barrier phase before `tcgen05.ld` |\n| epilogue-owned → free | all collective loads complete with `tcgen05.wait::ld`, then every reader reports completion |\n\nTMEM must first be allocated by the required participating warp and its address safely published. Matching collective deallocation is required before every kernel exit. Reused mbarriers need the correct arrival count and phase/parity state; a one-time wait on an address is not a reusable two-buffer protocol."}, "reason": {"statement": "Replace unsafe pseudo-implementation with verified ownership invariants and a pinned complete implementation route.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory"]}} {"path": "wiki/techniques/double-buffering.md", "before": {"statement": "A fully optimized Blackwell GEMM uses both levels simultaneously:"}, "after": {"statement": "Double- or multi-buffering reserves disjoint storage regions and transfers ownership between producers and consumers. While a consumer reads stage `s`, a producer may fill another stage. The storage alone does not establish overlap: the program also needs a completion edge before consumption and a read-complete edge before reuse.\n\nOn SM100 these ideas can be applied independently:\n\n- SMEM stages can hold operand tiles while TMA production overlaps MMA consumption.\n- Separate TMEM regions can hold output accumulators while an epilogue drains a completed region and MMA writes a different region.\n\nThe pinned `matmul_v6.cu` from Gau Nernst's tutorial uses both mechanisms. That is a concrete design, not a rule that every optimized Blackwell GEMM needs both.\n\nThat is not a complete C++ shared-storage layout. Barriers, descriptors, epilogue scratch, padding, alignment, and swizzled physical layouts also consume or constrain shared memory. Compute capability 10.0 supports up to 228 KiB of shared memory per SM, while per-block opt-in limits and all other residency resources still apply."}, "reason": {"statement": "Scope the combination to concrete kernels and tuning conditions.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"]}} -{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "```cuda\n// Combined double-buffering: SMEM (3-stage) + TMEM (2-buffer)\n//\n// Outer loop: output tiles (TMEM double-buffered)\n// Inner loop: K-tiles (SMEM 3-stage pipelined)\n//\n// Timeline for 2 output tiles, 6 K-tiles each:\n//\n// TMEM buf: |---- A (tile 0) ----|---- B (tile 1) ----|\n// MMA warp: | k0 k1 k2 k3 k4 k5 | k0 k1 k2 ...\n// TMA warp: |k2 k3 k4 k5 - - k2 k3 k4 k5 ...\n// Epilogue: | idle | drain A | drain B |\n// SMEM stage: |0 1 2 0 1 2 |0 1 2 0 1 2 |\n//\n// Key insight: the epilogue of tile 0 overlaps with the MMA of tile 1.\n// This is only possible because they use different TMEM buffers.\n\n__device__ void full_double_buffered_gemm(const GemmParams& params) {\n int warp_id = threadIdx.x / 32;\n\n for (int out_tile = 0; out_tile < num_output_tiles; out_tile++) {\n int tmem_buf = out_tile % 2;\n\n if (warp_id == 0) {\n // TMA producer: fill SMEM stages for this output tile's K-loop\n tma_producer_loop(params, out_tile);\n } else if (warp_id == 1) {\n // MMA: consume SMEM stages, accumulate into TMEM[tmem_buf]\n mma_consumer_loop(params, tmem_buf);\n } else {\n // Epilogue: drain TMEM[1-tmem_buf] from previous tile\n if (out_tile > 0) {\n epilogue_drain(params, out_tile - 1, 1 - tmem_buf);\n }\n }\n\n // Lightweight sync point between tiles\n // (mbarrier-based, not __syncthreads)\n }\n\n // Final epilogue for last tile\n if (warp_id >= 2) {\n epilogue_drain(params, num_output_tiles - 1,\n (num_output_tiles - 1) % 2);\n }\n}\n```"}, "after": {"statement": "For each region, prove this lifecycle:\n\n| free → MMA-owned | every prior epilogue load from the region has completed |\n| MMA-owned → ready | relevant tcgen05 work is committed to an mbarrier and completion is observed |\n| ready → epilogue-owned | readers observe the matching barrier phase before `tcgen05.ld` |\n| epilogue-owned → free | all collective loads complete with `tcgen05.wait::ld`, then every reader reports completion |\n\nTMEM must first be allocated by the required participating warp and its address safely published. Matching collective deallocation is required before every kernel exit. Reused mbarriers need the correct arrival count and phase/parity state; a one-time wait on an address is not a reusable two-buffer protocol.\n\nAn SMEM operand stage normally has its own full/empty state:\n\n1. The producer waits until stage `s` is empty.\n2. It issues the stage's TMA copies and accounts for expected transaction bytes.\n3. The consumer observes the full barrier's matching phase before using the stage.\n4. After the final dependent read, the consumer releases the empty barrier for reuse.\n\nFlashInfer PR 2387 is a second pinned example. Its merged `selective_state_update.cuh` has `state[numStages][...]` plus `bar_full` and `bar_empty` arrays and uses stage-specific producer/consumer handoffs. One path selects three stages and another caps a geometry-derived count at four, illustrating why stage count is policy rather than an architecture-wide default."}, "reason": {"statement": "Use a state/ownership table instead of incomplete CUDA.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"]}} +{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "```cuda\n// Combined double-buffering: SMEM (3-stage) + TMEM (2-buffer)\n//\n// Outer loop: output tiles (TMEM double-buffered)\n// Inner loop: K-tiles (SMEM 3-stage pipelined)\n//\n// Timeline for 2 output tiles, 6 K-tiles each:\n//\n// TMEM buf: |---- A (tile 0) ----|---- B (tile 1) ----|\n// MMA warp: | k0 k1 k2 k3 k4 k5 | k0 k1 k2 ...\n// TMA warp: |k2 k3 k4 k5 - - k2 k3 k4 k5 ...\n// Epilogue: | idle | drain A | drain B |\n// SMEM stage: |0 1 2 0 1 2 |0 1 2 0 1 2 |\n//\n// Key insight: the epilogue of tile 0 overlaps with the MMA of tile 1.\n// This is only possible because they use different TMEM buffers.\n\n__device__ void full_double_buffered_gemm(const GemmParams& params) {\n int warp_id = threadIdx.x / 32;\n\n for (int out_tile = 0; out_tile < num_output_tiles; out_tile++) {\n int tmem_buf = out_tile % 2;\n\n if (warp_id == 0) {\n // TMA producer: fill SMEM stages for this output tile's K-loop\n tma_producer_loop(params, out_tile);\n } else if (warp_id == 1) {\n // MMA: consume SMEM stages, accumulate into TMEM[tmem_buf]\n mma_consumer_loop(params, tmem_buf);\n } else {\n // Epilogue: drain TMEM[1-tmem_buf] from previous tile\n if (out_tile > 0) {\n epilogue_drain(params, out_tile - 1, 1 - tmem_buf);\n }\n }\n\n // Lightweight sync point between tiles\n // (mbarrier-based, not __syncthreads)\n }\n\n // Final epilogue for last tile\n if (warp_id >= 2) {\n epilogue_drain(params, num_output_tiles - 1,\n (num_output_tiles - 1) % 2);\n }\n}\n```"}, "after": {"statement": "For each region, prove this lifecycle:\n\n| free → MMA-owned | every prior epilogue load from the region has completed |\n| MMA-owned → ready | relevant tcgen05 work is committed to an mbarrier and completion is observed |\n| ready → epilogue-owned | readers observe the matching barrier phase before `tcgen05.ld` |\n| epilogue-owned → free | all collective loads complete with `tcgen05.wait::ld`, then every reader reports completion |\n\nTMEM must first be allocated by the required participating warp and its address safely published. Matching collective deallocation is required before every kernel exit. Reused mbarriers need the correct arrival count and phase/parity state; a one-time wait on an address is not a reusable two-buffer protocol.\n\nAn SMEM operand stage normally has its own full/empty state:\n\n1. The producer waits until stage `s` is empty.\n2. It issues the stage's TMA copies and accounts for expected transaction bytes.\n3. The consumer observes the full barrier's matching phase before using the stage.\n4. After the final dependent read, the consumer releases the empty barrier for reuse.\n\nFlashInfer PR 2387 is a second pinned example. Its merged `selective_state_update.cuh` has `state[numStages][...]` plus `bar_full` and `bar_empty` arrays and uses stage-specific producer/consumer handoffs. One path selects three stages and another caps a geometry-derived count at four, illustrating why stage count is policy rather than an architecture-wide default."}, "reason": {"statement": "Use a state/ownership table instead of incomplete CUDA.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"]}} {"path": "wiki/techniques/double-buffering.md", "before": {"statement": "```cuda\n// Hopper: accumulator double-buffering uses register arrays\n// Each warpgroup maintains two register-based accumulators\n// This doubles register pressure and reduces occupancy\n\n// Hopper approach (register pressure is the primary constraint):\nfloat acc_buf0[REG_TILE_M][REG_TILE_N]; // First accumulator\nfloat acc_buf1[REG_TILE_M][REG_TILE_N]; // Second accumulator\n// Total: 2 * REG_TILE_M * REG_TILE_N * 4 bytes per thread\n// For a 64x256 tile with 128-thread warpgroup:\n// each thread holds 2 * (64/4) * (256/128) * 4 = 2 * 16 * 2 * 4 = 256 bytes\n// = 64 registers just for accumulators\n\n// Blackwell approach: TMEM holds both buffers with zero register cost\n// MMA warp uses ~0 registers for accumulators\n// All 256 KB of TMEM is dedicated accumulator space\n```"}, "after": {"statement": "For binary16 `A[128,64]` and `B[64,256]`, the unpadded payload arithmetic is:\n\n```text\nA = 128 × 64 × 2 bytes = 16 KiB\nB = 64 × 256 × 2 bytes = 32 KiB\none stage = 48 KiB; three stages = 144 KiB\n```"}, "reason": {"statement": "Replace incorrect arithmetic and reject zero total-register cost.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"]}} -{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "| Double-buffer cost | 2x register usage | Zero register cost |"}, "after": {"statement": "For binary16 `A[128,64]` and `B[64,256]`, the unpadded payload arithmetic is:\n\n```text\nA = 128 × 64 × 2 bytes = 16 KiB\nB = 64 × 256 × 2 bytes = 32 KiB\none stage = 48 KiB; three stages = 144 KiB\n```"}, "reason": {"statement": "Narrow the comparison to resident accumulator storage.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"]}} -{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "| Typical occupancy impact | Reduces from 2 to 1 CTA/SM | No impact |"}, "after": {"statement": "For Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, one D fragment is 128 registers per thread; two simultaneously live fragments are 256 registers, or 1024 bytes, per thread before other live values. That arithmetic describes the ISA fragment, not a promise that a C++ array stays in registers or that a particular launch is resident.\n\nOn SM100, tcgen05 keeps resident D in TMEM. This avoids a long-lived per-thread D vector, but the kernel still uses registers for addresses, descriptors, loop and barrier state, `tcgen05.ld` destinations, and epilogue temporaries. TMEM buffering also consumes TMEM capacity and synchronization state. Neither architecture therefore has a fixed occupancy result from “double buffering” alone.\n\n| Resident storage | per-thread register fragment | allocated TMEM region |\n| Two live outputs | two disjoint register fragments | two disjoint TMEM regions |\n| Epilogue access | dependent use after WGMMA completion | collective `tcgen05.ld` and `tcgen05.wait::ld` after MMA completion |\n| Main resource question | compiled registers, SMEM, block shape, and other limits | TMEM columns plus compiled registers, SMEM, block/cluster shape, and other limits |"}, "reason": {"statement": "Unsupported occupancy numbers cannot guide a real launch.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"]}} -{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "| Max accumulator size | ~16K elements (register limited) | 128x512 = 65K elements |"}, "after": {"statement": "For binary16 `A[128,64]` and `B[64,256]`, the unpadded payload arithmetic is:\n\n```text\nA = 128 × 64 × 2 bytes = 16 KiB\nB = 64 × 256 × 2 bytes = 32 KiB\none stage = 48 KiB; three stages = 144 KiB\n```\n\nFor Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, one D fragment is 128 registers per thread; two simultaneously live fragments are 256 registers, or 1024 bytes, per thread before other live values. That arithmetic describes the ISA fragment, not a promise that a C++ array stays in registers or that a particular launch is resident.\n\nOn SM100, tcgen05 keeps resident D in TMEM. This avoids a long-lived per-thread D vector, but the kernel still uses registers for addresses, descriptors, loop and barrier state, `tcgen05.ld` destinations, and epilogue temporaries. TMEM buffering also consumes TMEM capacity and synchronization state. Neither architecture therefore has a fixed occupancy result from “double buffering” alone.\n\n| Resident storage | per-thread register fragment | allocated TMEM region |\n| Two live outputs | two disjoint register fragments | two disjoint TMEM regions |\n| Epilogue access | dependent use after WGMMA completion | collective `tcgen05.ld` and `tcgen05.wait::ld` after MMA completion |\n| Main resource question | compiled registers, SMEM, block shape, and other limits | TMEM columns plus compiled registers, SMEM, block/cluster shape, and other limits |"}, "reason": {"statement": "Avoid conflating an instruction fragment or storage geometry with a practical kernel maximum.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-multiply-accumulate-instructions-wgmma-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} -{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "- **TMEM double-buffering**: Always use on Blackwell when the epilogue takes more than trivial time. The only cost is the TMEM space, which is plentiful."}, "after": {"statement": "For Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, one D fragment is 128 registers per thread; two simultaneously live fragments are 256 registers, or 1024 bytes, per thread before other live values. That arithmetic describes the ISA fragment, not a promise that a C++ array stays in registers or that a particular launch is resident.\n\nOn SM100, tcgen05 keeps resident D in TMEM. This avoids a long-lived per-thread D vector, but the kernel still uses registers for addresses, descriptors, loop and barrier state, `tcgen05.ld` destinations, and epilogue temporaries. TMEM buffering also consumes TMEM capacity and synchronization state. Neither architecture therefore has a fixed occupancy result from “double buffering” alone.\n\n| Resident storage | per-thread register fragment | allocated TMEM region |\n| Two live outputs | two disjoint register fragments | two disjoint TMEM regions |\n| Epilogue access | dependent use after WGMMA completion | collective `tcgen05.ld` and `tcgen05.wait::ld` after MMA completion |\n| Main resource question | compiled registers, SMEM, block shape, and other limits | TMEM columns plus compiled registers, SMEM, block/cluster shape, and other limits |"}, "reason": {"statement": "Make adoption conditional on measured overlap benefit and resource fit.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} -{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "- **SMEM multi-stage buffering**: Always use for GEMM/attention mainloops. 3 stages is the default; increase to 4-5 only if the K-loop is long and memory latency is high."}, "after": {"statement": "That is not a complete C++ shared-storage layout. Barriers, descriptors, epilogue scratch, padding, alignment, and swizzled physical layouts also consume or constrain shared memory. Compute capability 10.0 supports up to 228 KiB of shared memory per SM, while per-block opt-in limits and all other residency resources still apply.\n\nFor Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, one D fragment is 128 registers per thread; two simultaneously live fragments are 256 registers, or 1024 bytes, per thread before other live values. That arithmetic describes the ISA fragment, not a promise that a C++ array stays in registers or that a particular launch is resident.\n\nOn SM100, tcgen05 keeps resident D in TMEM. This avoids a long-lived per-thread D vector, but the kernel still uses registers for addresses, descriptors, loop and barrier state, `tcgen05.ld` destinations, and epilogue temporaries. TMEM buffering also consumes TMEM capacity and synchronization state. Neither architecture therefore has a fixed occupancy result from “double buffering” alone.\n\n| Resident storage | per-thread register fragment | allocated TMEM region |\n| Two live outputs | two disjoint register fragments | two disjoint TMEM regions |\n| Epilogue access | dependent use after WGMMA completion | collective `tcgen05.ld` and `tcgen05.wait::ld` after MMA completion |\n| Main resource question | compiled registers, SMEM, block shape, and other limits | TMEM columns plus compiled registers, SMEM, block/cluster shape, and other limits |"}, "reason": {"statement": "Replace a universal recipe with resource- and workload-dependent tuning.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} +{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "| Double-buffer cost | 2x register usage | Zero register cost |"}, "after": {"statement": "For binary16 `A[128,64]` and `B[64,256]`, the unpadded payload arithmetic is:\n\n```text\nA = 128 × 64 × 2 bytes = 16 KiB\nB = 64 × 256 × 2 bytes = 32 KiB\none stage = 48 KiB; three stages = 144 KiB\n```"}, "reason": {"statement": "Narrow the comparison to resident accumulator storage.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld"]}} +{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "| Typical occupancy impact | Reduces from 2 to 1 CTA/SM | No impact |"}, "after": {"statement": "For Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, one D fragment is 128 registers per thread; two simultaneously live fragments are 256 registers, or 1024 bytes, per thread before other live values. That arithmetic describes the ISA fragment, not a promise that a C++ array stays in registers or that a particular launch is resident.\n\nOn SM100, tcgen05 keeps resident D in TMEM. This avoids a long-lived per-thread D vector, but the kernel still uses registers for addresses, descriptors, loop and barrier state, `tcgen05.ld` destinations, and epilogue temporaries. TMEM buffering also consumes TMEM capacity and synchronization state. Neither architecture therefore has a fixed occupancy result from “double buffering” alone.\n\n| Resident storage | per-thread register fragment | allocated TMEM region |\n| Two live outputs | two disjoint register fragments | two disjoint TMEM regions |\n| Epilogue access | dependent use after WGMMA completion | collective `tcgen05.ld` and `tcgen05.wait::ld` after MMA completion |\n| Main resource question | compiled registers, SMEM, block shape, and other limits | TMEM columns plus compiled registers, SMEM, block/cluster shape, and other limits |"}, "reason": {"statement": "Unsupported occupancy numbers cannot guide a real launch.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld"]}} +{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "| Max accumulator size | ~16K elements (register limited) | 128x512 = 65K elements |"}, "after": {"statement": "For binary16 `A[128,64]` and `B[64,256]`, the unpadded payload arithmetic is:\n\n```text\nA = 128 × 64 × 2 bytes = 16 KiB\nB = 64 × 256 × 2 bytes = 32 KiB\none stage = 48 KiB; three stages = 144 KiB\n```\n\nFor Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, one D fragment is 128 registers per thread; two simultaneously live fragments are 256 registers, or 1024 bytes, per thread before other live values. That arithmetic describes the ISA fragment, not a promise that a C++ array stays in registers or that a particular launch is resident.\n\nOn SM100, tcgen05 keeps resident D in TMEM. This avoids a long-lived per-thread D vector, but the kernel still uses registers for addresses, descriptors, loop and barrier state, `tcgen05.ld` destinations, and epilogue temporaries. TMEM buffering also consumes TMEM capacity and synchronization state. Neither architecture therefore has a fixed occupancy result from “double buffering” alone.\n\n| Resident storage | per-thread register fragment | allocated TMEM region |\n| Two live outputs | two disjoint register fragments | two disjoint TMEM regions |\n| Epilogue access | dependent use after WGMMA completion | collective `tcgen05.ld` and `tcgen05.wait::ld` after MMA completion |\n| Main resource question | compiled registers, SMEM, block shape, and other limits | TMEM columns plus compiled registers, SMEM, block/cluster shape, and other limits |"}, "reason": {"statement": "Avoid conflating an instruction fragment or storage geometry with a practical kernel maximum.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} +{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "- **TMEM double-buffering**: Always use on Blackwell when the epilogue takes more than trivial time. The only cost is the TMEM space, which is plentiful."}, "after": {"statement": "For Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, one D fragment is 128 registers per thread; two simultaneously live fragments are 256 registers, or 1024 bytes, per thread before other live values. That arithmetic describes the ISA fragment, not a promise that a C++ array stays in registers or that a particular launch is resident.\n\nOn SM100, tcgen05 keeps resident D in TMEM. This avoids a long-lived per-thread D vector, but the kernel still uses registers for addresses, descriptors, loop and barrier state, `tcgen05.ld` destinations, and epilogue temporaries. TMEM buffering also consumes TMEM capacity and synchronization state. Neither architecture therefore has a fixed occupancy result from “double buffering” alone.\n\n| Resident storage | per-thread register fragment | allocated TMEM region |\n| Two live outputs | two disjoint register fragments | two disjoint TMEM regions |\n| Epilogue access | dependent use after WGMMA completion | collective `tcgen05.ld` and `tcgen05.wait::ld` after MMA completion |\n| Main resource question | compiled registers, SMEM, block shape, and other limits | TMEM columns plus compiled registers, SMEM, block/cluster shape, and other limits |"}, "reason": {"statement": "Make adoption conditional on measured overlap benefit and resource fit.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} +{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "- **SMEM multi-stage buffering**: Always use for GEMM/attention mainloops. 3 stages is the default; increase to 4-5 only if the K-loop is long and memory latency is high."}, "after": {"statement": "That is not a complete C++ shared-storage layout. Barriers, descriptors, epilogue scratch, padding, alignment, and swizzled physical layouts also consume or constrain shared memory. Compute capability 10.0 supports up to 228 KiB of shared memory per SM, while per-block opt-in limits and all other residency resources still apply.\n\nFor Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, one D fragment is 128 registers per thread; two simultaneously live fragments are 256 registers, or 1024 bytes, per thread before other live values. That arithmetic describes the ISA fragment, not a promise that a C++ array stays in registers or that a particular launch is resident.\n\nOn SM100, tcgen05 keeps resident D in TMEM. This avoids a long-lived per-thread D vector, but the kernel still uses registers for addresses, descriptors, loop and barrier state, `tcgen05.ld` destinations, and epilogue temporaries. TMEM buffering also consumes TMEM capacity and synchronization state. Neither architecture therefore has a fixed occupancy result from “double buffering” alone.\n\n| Resident storage | per-thread register fragment | allocated TMEM region |\n| Two live outputs | two disjoint register fragments | two disjoint TMEM regions |\n| Epilogue access | dependent use after WGMMA completion | collective `tcgen05.ld` and `tcgen05.wait::ld` after MMA completion |\n| Main resource question | compiled registers, SMEM, block shape, and other limits | TMEM columns plus compiled registers, SMEM, block/cluster shape, and other limits |"}, "reason": {"statement": "Replace a universal recipe with resource- and workload-dependent tuning.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} {"path": "wiki/techniques/double-buffering.md", "before": {"statement": "- **Combined**: The standard approach for production Blackwell GEMM kernels. Both CUTLASS and CuTe-DSL kernels use this pattern."}, "after": {"statement": "That is not a complete C++ shared-storage layout. Barriers, descriptors, epilogue scratch, padding, alignment, and swizzled physical layouts also consume or constrain shared memory. Compute capability 10.0 supports up to 228 KiB of shared memory per SM, while per-block opt-in limits and all other residency resources still apply."}, "reason": {"statement": "Use concrete, pinned examples rather than ecosystem-wide attribution.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"]}} -{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "- TMEM double-buffering requires that the accumulator fits in half the TMEM columns (256 out of 512). For very wide tiles (TILE_N > 256 with FP32 accumulators), the tile must be split or a different buffering strategy used."}, "after": {"statement": "The CTA-visible TMEM address structure has 128 lanes and 512 columns of 32-bit cells. Allocation is column-granular across all 128 lanes. A kernel that allocates 512 columns may choose two 256-column regions:\n\n```text\nregion[0] = columns [0, 256)\nregion[1] = columns [256, 512)\n```\n\nThis equal split is one implementation choice. The actual invariant is that every simultaneously live region is inside the allocation and does not alias another region whose MMA or epilogue access is still outstanding. Region sizes can differ, and logical element capacity depends on the tcgen05 instruction kind and packing rather than the raw cell count alone.\n\nFor Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, one D fragment is 128 registers per thread; two simultaneously live fragments are 256 registers, or 1024 bytes, per thread before other live values. That arithmetic describes the ISA fragment, not a promise that a C++ array stays in registers or that a particular launch is resident.\n\nOn SM100, tcgen05 keeps resident D in TMEM. This avoids a long-lived per-thread D vector, but the kernel still uses registers for addresses, descriptors, loop and barrier state, `tcgen05.ld` destinations, and epilogue temporaries. TMEM buffering also consumes TMEM capacity and synchronization state. Neither architecture therefore has a fixed occupancy result from “double buffering” alone.\n\n| Resident storage | per-thread register fragment | allocated TMEM region |\n| Two live outputs | two disjoint register fragments | two disjoint TMEM regions |\n| Epilogue access | dependent use after WGMMA completion | collective `tcgen05.ld` and `tcgen05.wait::ld` after MMA completion |\n| Main resource question | compiled registers, SMEM, block shape, and other limits | TMEM columns plus compiled registers, SMEM, block/cluster shape, and other limits |"}, "reason": {"statement": "State the actual nonaliasing/capacity invariant and label 256/256 as one choice.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory-allocation", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} -{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "- The mbarrier synchronization between TMEM buffers adds a few cycles of overhead per tile. For kernels with very few K-iterations per tile, this overhead is proportionally larger."}, "after": {"statement": "For Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, one D fragment is 128 registers per thread; two simultaneously live fragments are 256 registers, or 1024 bytes, per thread before other live values. That arithmetic describes the ISA fragment, not a promise that a C++ array stays in registers or that a particular launch is resident.\n\nOn SM100, tcgen05 keeps resident D in TMEM. This avoids a long-lived per-thread D vector, but the kernel still uses registers for addresses, descriptors, loop and barrier state, `tcgen05.ld` destinations, and epilogue temporaries. TMEM buffering also consumes TMEM capacity and synchronization state. Neither architecture therefore has a fixed occupancy result from “double buffering” alone.\n\n| Resident storage | per-thread register fragment | allocated TMEM region |\n| Two live outputs | two disjoint register fragments | two disjoint TMEM regions |\n| Epilogue access | dependent use after WGMMA completion | collective `tcgen05.ld` and `tcgen05.wait::ld` after MMA completion |\n| Main resource question | compiled registers, SMEM, block shape, and other limits | TMEM columns plus compiled registers, SMEM, block/cluster shape, and other limits |"}, "reason": {"statement": "Retain the need to measure synchronization overhead without inventing latency.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} +{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "- TMEM double-buffering requires that the accumulator fits in half the TMEM columns (256 out of 512). For very wide tiles (TILE_N > 256 with FP32 accumulators), the tile must be split or a different buffering strategy used."}, "after": {"statement": "The CTA-visible TMEM address structure has 128 lanes and 512 columns of 32-bit cells. Allocation is column-granular across all 128 lanes. A kernel that allocates 512 columns may choose two 256-column regions:\n\n```text\nregion[0] = columns [0, 256)\nregion[1] = columns [256, 512)\n```\n\nThis equal split is one implementation choice. The actual invariant is that every simultaneously live region is inside the allocation and does not alias another region whose MMA or epilogue access is still outstanding. Region sizes can differ, and logical element capacity depends on the tcgen05 instruction kind and packing rather than the raw cell count alone.\n\nFor Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, one D fragment is 128 registers per thread; two simultaneously live fragments are 256 registers, or 1024 bytes, per thread before other live values. That arithmetic describes the ISA fragment, not a promise that a C++ array stays in registers or that a particular launch is resident.\n\nOn SM100, tcgen05 keeps resident D in TMEM. This avoids a long-lived per-thread D vector, but the kernel still uses registers for addresses, descriptors, loop and barrier state, `tcgen05.ld` destinations, and epilogue temporaries. TMEM buffering also consumes TMEM capacity and synchronization state. Neither architecture therefore has a fixed occupancy result from “double buffering” alone.\n\n| Resident storage | per-thread register fragment | allocated TMEM region |\n| Two live outputs | two disjoint register fragments | two disjoint TMEM regions |\n| Epilogue access | dependent use after WGMMA completion | collective `tcgen05.ld` and `tcgen05.wait::ld` after MMA completion |\n| Main resource question | compiled registers, SMEM, block shape, and other limits | TMEM columns plus compiled registers, SMEM, block/cluster shape, and other limits |"}, "reason": {"statement": "State the actual nonaliasing/capacity invariant and label 256/256 as one choice.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory-allocation", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} +{"path": "wiki/techniques/double-buffering.md", "before": {"statement": "- The mbarrier synchronization between TMEM buffers adds a few cycles of overhead per tile. For kernels with very few K-iterations per tile, this overhead is proportionally larger."}, "after": {"statement": "For Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, one D fragment is 128 registers per thread; two simultaneously live fragments are 256 registers, or 1024 bytes, per thread before other live values. That arithmetic describes the ISA fragment, not a promise that a C++ array stays in registers or that a particular launch is resident.\n\nOn SM100, tcgen05 keeps resident D in TMEM. This avoids a long-lived per-thread D vector, but the kernel still uses registers for addresses, descriptors, loop and barrier state, `tcgen05.ld` destinations, and epilogue temporaries. TMEM buffering also consumes TMEM capacity and synchronization state. Neither architecture therefore has a fixed occupancy result from “double buffering” alone.\n\n| Resident storage | per-thread register fragment | allocated TMEM region |\n| Two live outputs | two disjoint register fragments | two disjoint TMEM regions |\n| Epilogue access | dependent use after WGMMA completion | collective `tcgen05.ld` and `tcgen05.wait::ld` after MMA completion |\n| Main resource question | compiled registers, SMEM, block shape, and other limits | TMEM columns plus compiled registers, SMEM, block/cluster shape, and other limits |"}, "reason": {"statement": "Retain the need to measure synchronization overhead without inventing latency.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} {"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "Epilogue fusion overlaps the post-MMA operations (scaling, bias addition, activation functions, quantization, store to global memory) with ongoing MMA computation. On Blackwell, the accumulator lives in TMEM rather than registers, enabling dedicated epilogue warps (typically warps 2-15) to read TMEM concurrently while the MMA warp (warp 1) continues accumulating the next tile. This overlap is achieved by double-buffering the TMEM accumulator: the MMA warp writes to one half while the epilogue warps read from the other half."}, "after": {"statement": "Epilogue fusion computes output transformations—such as `alpha*acc + beta*C`, bias, activation, output conversion, or auxiliary values—inside the producer kernel before final output materialization. It can remove an intermediate tensor or launch only when the unfused comparison would otherwise materialize or launch that work.\n\nFusion does not by itself imply overlap with the next MMA tile. A schedule may drain one completed accumulator after the mainloop, or it may use disjoint storage and specialized roles to overlap a completed region's epilogue with independent MMA work. Participant counts and role IDs come from the concrete kernel; there is no architectural default of fourteen epilogue warps."}, "reason": {"statement": "Separate semantic fusion from optional schedule overlap.", "urls": []}} -{"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "```cuda\n// Epilogue warp: read TMEM accumulator, apply fused operations, store\n// This runs on warps 2-15 while warp 1 continues MMA on next tile\n__device__ void epilogue_warp_fn(\n int warp_id,\n int tile_m, int tile_n,\n float scale, const float* bias,\n half* C, int ldc)\n{\n int lane_id = threadIdx.x % 32;\n\n // Each epilogue warp handles a stripe of the output tile\n // 14 warps, TILE_M = 128 -> ~9 rows per warp\n int rows_per_warp = (TILE_M + 13) / 14;\n int row_start = (warp_id - 2) * rows_per_warp;\n int row_end = min(row_start + rows_per_warp, TILE_M);\n\n for (int r = row_start; r < row_end; r++) {\n int global_row = tile_m * TILE_M + r;\n\n for (int c = lane_id; c < TILE_N; c += 32) {\n // Step 1: Load accumulator from TMEM into register\n float acc = tmem_load_f32(r, c);\n\n // Step 2: Fused epilogue operations (all in registers)\n // Scale\n acc *= scale;\n // Bias\n acc += bias[tile_n * TILE_N + c];\n // ReLU activation\n acc = fmaxf(acc, 0.0f);\n\n // Step 3: Store to global memory\n int global_col = tile_n * TILE_N + c;\n C[global_row * ldc + global_col] = __float2half(acc);\n }\n }\n}\n```"}, "after": {"statement": "For tcgen05, D resides in TMEM. An ordinary arithmetic epilogue first uses a legal collective `tcgen05.ld` mapping, or a library wrapper around it, to transfer a partition into per-thread registers. The load is asynchronous with respect to the issuing thread, so the matching `tcgen05.wait::ld` completion must occur before those registers are consumed.\n\nCUTLASS 4.5.0's `fp16_gemm_2.py` demonstrates the typed route:\n\n```python\ncopy_atom_t2r = cute.make_copy_atom(\n tcgen05.Ld32x32bOp(tcgen05.Repetition.x32), cutlass.Float32\n)\ntiled_copy_t2r = tcgen05.make_tmem_copy(copy_atom_t2r, tCtAcc_epi)\nthr_copy_t2r = tiled_copy_t2r.get_slice(tidx)\ntTR_tAcc = thr_copy_t2r.partition_S(tCtAcc_epi)\ntTR_rAcc = cute.make_rmem_tensor(..., cutlass.Float32)\ncute.copy(tiled_copy_t2r, tTR_tAcc_slice, tTR_rAcc)\n```\n\nThis is an API-routing fragment, not standalone code: the official file supplies the exact tensor layouts, selected load shape, participant group, edge predicates, accumulator-completion handoff, register-to-SMEM conversion, TMA-store pipeline, tail, and TMEM cleanup.\n\n| tcgen05 MMA → TMEM reader | the relevant asynchronous MMA is committed and its completion observed |\n| TMEM → result registers | the legal collective load completes before dependent arithmetic |\n| registers → output | element/layout mapping and output-edge predicates cover exactly the valid coordinates |\n| TMEM reader → region reuse | every reader's load has completed and the matching reusable barrier phase is released |\n| kernel tail | all output stores complete as required and every TMEM allocation is collectively freed |\n\nFor a multi-region overlap schedule, simultaneously live TMEM regions must also be disjoint. Equal 256-column halves are one possible policy for a 512-column allocation, not an epilogue-fusion requirement. Reusable mbarriers need correct expected-arrival counts and phase/parity tracking."}, "reason": {"statement": "Replace pseudo-CUDA with exact lifecycle and official CuTe copy-route identifiers.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"]}} -{"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "The key to epilogue fusion on Blackwell is TMEM double-buffering. The 512-column TMEM space is split into two halves (columns 0-255 and 256-511). While the MMA warp accumulates into one half, the epilogue warps drain the other:"}, "after": {"statement": "Epilogue fusion computes output transformations—such as `alpha*acc + beta*C`, bias, activation, output conversion, or auxiliary values—inside the producer kernel before final output materialization. It can remove an intermediate tensor or launch only when the unfused comparison would otherwise materialize or launch that work.\n\nFusion does not by itself imply overlap with the next MMA tile. A schedule may drain one completed accumulator after the mainloop, or it may use disjoint storage and specialized roles to overlap a completed region's epilogue with independent MMA work. Participant counts and role IDs come from the concrete kernel; there is no architectural default of fourteen epilogue warps.\n\n| tcgen05 MMA → TMEM reader | the relevant asynchronous MMA is committed and its completion observed |\n| TMEM → result registers | the legal collective load completes before dependent arithmetic |\n| registers → output | element/layout mapping and output-edge predicates cover exactly the valid coordinates |\n| TMEM reader → region reuse | every reader's load has completed and the matching reusable barrier phase is released |\n| kernel tail | all output stores complete as required and every TMEM allocation is collectively freed |\n\nFor a multi-region overlap schedule, simultaneously live TMEM regions must also be disjoint. Equal 256-column halves are one possible policy for a 512-column allocation, not an epilogue-fusion requirement. Reusable mbarriers need correct expected-arrival counts and phase/parity tracking."}, "reason": {"statement": "Make multi-region overlap optional and state nonaliasing constraints.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"]}} -{"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "```cuda\n// TMEM double-buffering for MMA-epilogue overlap\n//\n// TMEM layout: 128 rows x 512 columns (32-bit elements)\n// Buffer A: columns [0, 255] -- 128 x 256 accumulator\n// Buffer B: columns [256, 511] -- 128 x 256 accumulator\n//\n// Timeline:\n// Tile 0: MMA -> buffer A | epilogue idle (no prior result)\n// Tile 1: MMA -> buffer B | epilogue reads buffer A\n// Tile 2: MMA -> buffer A | epilogue reads buffer B\n// Tile 3: MMA -> buffer B | epilogue reads buffer A\n// ... ping-pong continues ...\n\n__device__ void mma_epilogue_overlap(\n const GemmParams& params,\n int num_tiles)\n{\n int warp_id = threadIdx.x / 32;\n\n __shared__ uint64_t mbar_mma_done[2]; // One per TMEM buffer half\n __shared__ uint64_t mbar_epi_done[2]; // Epilogue completion signals\n\n if (threadIdx.x == 0) {\n for (int i = 0; i < 2; i++) {\n mbarrier_init(&mbar_mma_done[i], 1);\n mbarrier_init(&mbar_epi_done[i], 1);\n }\n }\n __syncthreads();\n\n if (warp_id == 1) {\n // === MMA WARP ===\n for (int t = 0; t < num_tiles; t++) {\n int buf = t % 2; // Alternate between buffer halves\n\n // Wait for epilogue to finish reading this buffer\n if (t >= 2) {\n mbarrier_wait(&mbar_epi_done[buf]);\n }\n\n // Issue MMA, accumulating into TMEM buffer half\n int tmem_col_offset = buf * 256;\n tcgen05_mma_with_offset(tmem_col_offset,\n params.smem_A, params.smem_B);\n\n // Signal epilogue that this buffer is ready\n mbarrier_arrive(&mbar_mma_done[buf]);\n }\n\n } else if (warp_id >= 2) {\n // === EPILOGUE WARPS ===\n for (int t = 0; t < num_tiles; t++) {\n int buf = t % 2;\n\n // Wait for MMA to finish filling this buffer\n mbarrier_wait(&mbar_mma_done[buf]);\n\n // Read from TMEM buffer half and write to global memory\n int tmem_col_offset = buf * 256;\n epilogue_store(params, t, tmem_col_offset, warp_id);\n\n // ALL epilogue warps must finish reading TMEM before MMA reuses\n // the buffer. Each epilogue warp arrives on mbar_epi_done;\n // mbar_epi_done is initialized with count = NUM_EPILOGUE_WARPS.\n // MMA warp waits on this mbarrier before writing to this half.\n if (lane_id == 0) {\n mbarrier_arrive(&mbar_epi_done[buf]);\n }\n // mbar_epi_done[buf] fires only after ALL epilogue warps arrive\n }\n }\n}\n```"}, "after": {"statement": "| tcgen05 MMA → TMEM reader | the relevant asynchronous MMA is committed and its completion observed |\n| TMEM → result registers | the legal collective load completes before dependent arithmetic |\n| registers → output | element/layout mapping and output-edge predicates cover exactly the valid coordinates |\n| TMEM reader → region reuse | every reader's load has completed and the matching reusable barrier phase is released |\n| kernel tail | all output stores complete as required and every TMEM allocation is collectively freed |\n\nFor a multi-region overlap schedule, simultaneously live TMEM regions must also be disjoint. Equal 256-column halves are one possible policy for a 512-column allocation, not an epilogue-fusion requirement. Reusable mbarriers need correct expected-arrival counts and phase/parity tracking."}, "reason": {"statement": "Replace incomplete code with a required-edge table and negative tests.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc"]}} +{"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "```cuda\n// Epilogue warp: read TMEM accumulator, apply fused operations, store\n// This runs on warps 2-15 while warp 1 continues MMA on next tile\n__device__ void epilogue_warp_fn(\n int warp_id,\n int tile_m, int tile_n,\n float scale, const float* bias,\n half* C, int ldc)\n{\n int lane_id = threadIdx.x % 32;\n\n // Each epilogue warp handles a stripe of the output tile\n // 14 warps, TILE_M = 128 -> ~9 rows per warp\n int rows_per_warp = (TILE_M + 13) / 14;\n int row_start = (warp_id - 2) * rows_per_warp;\n int row_end = min(row_start + rows_per_warp, TILE_M);\n\n for (int r = row_start; r < row_end; r++) {\n int global_row = tile_m * TILE_M + r;\n\n for (int c = lane_id; c < TILE_N; c += 32) {\n // Step 1: Load accumulator from TMEM into register\n float acc = tmem_load_f32(r, c);\n\n // Step 2: Fused epilogue operations (all in registers)\n // Scale\n acc *= scale;\n // Bias\n acc += bias[tile_n * TILE_N + c];\n // ReLU activation\n acc = fmaxf(acc, 0.0f);\n\n // Step 3: Store to global memory\n int global_col = tile_n * TILE_N + c;\n C[global_row * ldc + global_col] = __float2half(acc);\n }\n }\n}\n```"}, "after": {"statement": "For tcgen05, D resides in TMEM. An ordinary arithmetic epilogue first uses a legal collective `tcgen05.ld` mapping, or a library wrapper around it, to transfer a partition into per-thread registers. The load is asynchronous with respect to the issuing thread, so the matching `tcgen05.wait::ld` completion must occur before those registers are consumed.\n\nCUTLASS 4.5.0's `fp16_gemm_2.py` demonstrates the typed route:\n\n```python\ncopy_atom_t2r = cute.make_copy_atom(\n tcgen05.Ld32x32bOp(tcgen05.Repetition.x32), cutlass.Float32\n)\ntiled_copy_t2r = tcgen05.make_tmem_copy(copy_atom_t2r, tCtAcc_epi)\nthr_copy_t2r = tiled_copy_t2r.get_slice(tidx)\ntTR_tAcc = thr_copy_t2r.partition_S(tCtAcc_epi)\ntTR_rAcc = cute.make_rmem_tensor(..., cutlass.Float32)\ncute.copy(tiled_copy_t2r, tTR_tAcc_slice, tTR_rAcc)\n```\n\nThis is an API-routing fragment, not standalone code: the official file supplies the exact tensor layouts, selected load shape, participant group, edge predicates, accumulator-completion handoff, register-to-SMEM conversion, TMA-store pipeline, tail, and TMEM cleanup.\n\n| tcgen05 MMA → TMEM reader | the relevant asynchronous MMA is committed and its completion observed |\n| TMEM → result registers | the legal collective load completes before dependent arithmetic |\n| registers → output | element/layout mapping and output-edge predicates cover exactly the valid coordinates |\n| TMEM reader → region reuse | every reader's load has completed and the matching reusable barrier phase is released |\n| kernel tail | all output stores complete as required and every TMEM allocation is collectively freed |\n\nFor a multi-region overlap schedule, simultaneously live TMEM regions must also be disjoint. Equal 256-column halves are one possible policy for a 512-column allocation, not an epilogue-fusion requirement. Reusable mbarriers need correct expected-arrival counts and phase/parity tracking."}, "reason": {"statement": "Replace pseudo-CUDA with exact lifecycle and official CuTe copy-route identifiers.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit"]}} +{"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "The key to epilogue fusion on Blackwell is TMEM double-buffering. The 512-column TMEM space is split into two halves (columns 0-255 and 256-511). While the MMA warp accumulates into one half, the epilogue warps drain the other:"}, "after": {"statement": "Epilogue fusion computes output transformations—such as `alpha*acc + beta*C`, bias, activation, output conversion, or auxiliary values—inside the producer kernel before final output materialization. It can remove an intermediate tensor or launch only when the unfused comparison would otherwise materialize or launch that work.\n\nFusion does not by itself imply overlap with the next MMA tile. A schedule may drain one completed accumulator after the mainloop, or it may use disjoint storage and specialized roles to overlap a completed region's epilogue with independent MMA work. Participant counts and role IDs come from the concrete kernel; there is no architectural default of fourteen epilogue warps.\n\n| tcgen05 MMA → TMEM reader | the relevant asynchronous MMA is committed and its completion observed |\n| TMEM → result registers | the legal collective load completes before dependent arithmetic |\n| registers → output | element/layout mapping and output-edge predicates cover exactly the valid coordinates |\n| TMEM reader → region reuse | every reader's load has completed and the matching reusable barrier phase is released |\n| kernel tail | all output stores complete as required and every TMEM allocation is collectively freed |\n\nFor a multi-region overlap schedule, simultaneously live TMEM regions must also be disjoint. Equal 256-column halves are one possible policy for a 512-column allocation, not an epilogue-fusion requirement. Reusable mbarriers need correct expected-arrival counts and phase/parity tracking."}, "reason": {"statement": "Make multi-region overlap optional and state nonaliasing constraints.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit"]}} +{"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "```cuda\n// TMEM double-buffering for MMA-epilogue overlap\n//\n// TMEM layout: 128 rows x 512 columns (32-bit elements)\n// Buffer A: columns [0, 255] -- 128 x 256 accumulator\n// Buffer B: columns [256, 511] -- 128 x 256 accumulator\n//\n// Timeline:\n// Tile 0: MMA -> buffer A | epilogue idle (no prior result)\n// Tile 1: MMA -> buffer B | epilogue reads buffer A\n// Tile 2: MMA -> buffer A | epilogue reads buffer B\n// Tile 3: MMA -> buffer B | epilogue reads buffer A\n// ... ping-pong continues ...\n\n__device__ void mma_epilogue_overlap(\n const GemmParams& params,\n int num_tiles)\n{\n int warp_id = threadIdx.x / 32;\n\n __shared__ uint64_t mbar_mma_done[2]; // One per TMEM buffer half\n __shared__ uint64_t mbar_epi_done[2]; // Epilogue completion signals\n\n if (threadIdx.x == 0) {\n for (int i = 0; i < 2; i++) {\n mbarrier_init(&mbar_mma_done[i], 1);\n mbarrier_init(&mbar_epi_done[i], 1);\n }\n }\n __syncthreads();\n\n if (warp_id == 1) {\n // === MMA WARP ===\n for (int t = 0; t < num_tiles; t++) {\n int buf = t % 2; // Alternate between buffer halves\n\n // Wait for epilogue to finish reading this buffer\n if (t >= 2) {\n mbarrier_wait(&mbar_epi_done[buf]);\n }\n\n // Issue MMA, accumulating into TMEM buffer half\n int tmem_col_offset = buf * 256;\n tcgen05_mma_with_offset(tmem_col_offset,\n params.smem_A, params.smem_B);\n\n // Signal epilogue that this buffer is ready\n mbarrier_arrive(&mbar_mma_done[buf]);\n }\n\n } else if (warp_id >= 2) {\n // === EPILOGUE WARPS ===\n for (int t = 0; t < num_tiles; t++) {\n int buf = t % 2;\n\n // Wait for MMA to finish filling this buffer\n mbarrier_wait(&mbar_mma_done[buf]);\n\n // Read from TMEM buffer half and write to global memory\n int tmem_col_offset = buf * 256;\n epilogue_store(params, t, tmem_col_offset, warp_id);\n\n // ALL epilogue warps must finish reading TMEM before MMA reuses\n // the buffer. Each epilogue warp arrives on mbar_epi_done;\n // mbar_epi_done is initialized with count = NUM_EPILOGUE_WARPS.\n // MMA warp waits on this mbarrier before writing to this half.\n if (lane_id == 0) {\n mbarrier_arrive(&mbar_epi_done[buf]);\n }\n // mbar_epi_done[buf] fires only after ALL epilogue warps arrive\n }\n }\n}\n```"}, "after": {"statement": "| tcgen05 MMA → TMEM reader | the relevant asynchronous MMA is committed and its completion observed |\n| TMEM → result registers | the legal collective load completes before dependent arithmetic |\n| registers → output | element/layout mapping and output-edge predicates cover exactly the valid coordinates |\n| TMEM reader → region reuse | every reader's load has completed and the matching reusable barrier phase is released |\n| kernel tail | all output stores complete as required and every TMEM allocation is collectively freed |\n\nFor a multi-region overlap schedule, simultaneously live TMEM regions must also be disjoint. Equal 256-column halves are one possible policy for a 512-column allocation, not an epilogue-fusion requirement. Reusable mbarriers need correct expected-arrival counts and phase/parity tracking."}, "reason": {"statement": "Replace incomplete code with a required-edge table and negative tests.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit"]}} {"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "CUTLASS 4.5.0 provides composable epilogue visitors that fuse arbitrary element-wise operations after GEMM:"}, "after": {"statement": "`cutlass::epilogue::fusion::LinCombEltAct` has this parameter order:\n\n```cpp\ntemplate <\n template class ActivationFn,\n class ElementOutput,\n class ElementCompute,\n class ElementSource = ElementOutput,\n class ElementScalar = ElementCompute,\n cutlass::FloatRoundStyle Round = cutlass::FloatRoundStyle::round_to_nearest>\nstruct LinCombEltAct;\n```\n\nFor SM100 construction, `cutlass::epilogue::collective::CollectiveBuilder` receives architecture, operator class, tile/cluster shapes, `EpilogueTileAuto` or an explicit epilogue tile, accumulator/compute/C/D types and layouts, alignment, `EpilogueScheduleAuto` or an explicit supported schedule, and finally a supported fusion operation or callbacks type.\n\nThere is no CUTLASS 4.5.0 type named `Sm100EpilogueTmaWarpSpecialized`. Support is constrained by the exact architecture, operator class, schedule, tile, layout, alignment, datatype, and fusion callback combination. Use `Gemm::can_implement(arguments)` and a tagged example rather than reconstructing a builder signature from prose.\n\nThe vLLM PR 16032 NVFP4 wrapper is one pinned C++ construction example: it uses `CollectiveBuilder<... EpilogueTileAuto, ... EpilogueScheduleAuto>` and derives mainloop shared-memory stages with `StageCountAutoCarveout`."}, "reason": {"statement": "Describe the exact supported operation-tag/callback interface.", "urls": []}} {"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "```cuda\n// CUTLASS SM100 epilogue with fused scale + bias + activation\n// Uses the EVT (Epilogue Visitor Tree) pattern\n\nusing EpilogueOp = cutlass::epilogue::fusion::LinCombEltAct<\n cutlass::epilogue::thread::ReLU, // Activation function\n float, // Compute type\n float, // Scale type\n cutlass::half_t // Output type\n>;\n\n// The epilogue descriptor tells CUTLASS how to partition work\n// across the 14 epilogue warps\nusing CollectiveEpilogue = cutlass::epilogue::collective::Sm100EpilogueTmaWarpSpecialized<\n cutlass::gemm::TagToStrideC_t,\n cutlass::gemm::TagToStrideC_t,\n EpilogueOp,\n cutlass::gemm::EpilogueDefault // Default tiling\n>;\n\n// In the kernel, the epilogue is invoked after the mainloop:\n// epilogue(\n// problem_shape,\n// collective_mainloop.get_accumulator(), // TMEM reference\n// epilogue_params, // scale, bias pointers\n// shared_storage // SMEM for TMA stores\n// );\n```"}, "after": {"statement": "`cutlass::epilogue::fusion::LinCombEltAct` has this parameter order:\n\n```cpp\ntemplate <\n template class ActivationFn,\n class ElementOutput,\n class ElementCompute,\n class ElementSource = ElementOutput,\n class ElementScalar = ElementCompute,\n cutlass::FloatRoundStyle Round = cutlass::FloatRoundStyle::round_to_nearest>\nstruct LinCombEltAct;\n```\n\nFor SM100 construction, `cutlass::epilogue::collective::CollectiveBuilder` receives architecture, operator class, tile/cluster shapes, `EpilogueTileAuto` or an explicit epilogue tile, accumulator/compute/C/D types and layouts, alignment, `EpilogueScheduleAuto` or an explicit supported schedule, and finally a supported fusion operation or callbacks type.\n\nThere is no CUTLASS 4.5.0 type named `Sm100EpilogueTmaWarpSpecialized`. Support is constrained by the exact architecture, operator class, schedule, tile, layout, alignment, datatype, and fusion callback combination. Use `Gemm::can_implement(arguments)` and a tagged example rather than reconstructing a builder signature from prose.\n\nThe vLLM PR 16032 NVFP4 wrapper is one pinned C++ construction example: it uses `CollectiveBuilder<... EpilogueTileAuto, ... EpilogueScheduleAuto>` and derives mainloop shared-memory stages with `StageCountAutoCarveout`."}, "reason": {"statement": "Replace invented code with exact version-pinned type signatures and a real builder example route.", "urls": []}} {"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "| Scale + Bias | `y = alpha * acc + beta * C` | Standard GEMM epilogue |"}, "after": {"statement": "| Source linear combination | `D = alpha*acc + beta*C` |\n| Per-row/per-column bias | add a separately laid-out broadcast bias input |\n| Activation | apply a supported functor such as ReLU, GELU, or SiLU to the output fragment |\n| Output conversion/quantization | convert the accumulator fragment and, when required, generate/store scale metadata |\n| Gated product | combine two values, for example `SiLU(gate) * up`; requires both inputs and a supported/custom callback |\n| Online-softmax rescale | rescale a prior partial output after a new row maximum; requires the attention reduction state |\n| Residual | combine a separately supplied residual/source tensor at the defined point in the expression tree |\n\nThese are operation categories, not a claim that every composition is supported by one built-in SM100 visitor. In particular, reductions, multiple outputs, auxiliary tensors, and broadcasts add synchronization and layout requirements beyond a pointwise activation."}, "reason": {"statement": "Separate source linear combination from explicit bias broadcasting.", "urls": []}} -{"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "- **All Blackwell GEMMs with non-trivial epilogues**: The 14 epilogue warps are available by default in the warp-specialized model. Fusing operations avoids a separate kernel launch and an extra global memory round-trip."}, "after": {"statement": "When an implementation overlaps the epilogue with independent MMA work, verify both directions of the handoff: compute-complete before read, and load/read-complete before overwrite. Include the prologue (no prior result), steady-state wraparound, final drain, and deallocation path in tests.\n\nEvaluate a defined fused/unfused pair with identical inputs, output semantics, launch policy, warmup, and timing statistics. Record registers, spills, SMEM, TMEM columns, barriers, threads/CTA, cluster shape, and occupancy. Sweep supported epilogue group sizes, load shapes, output tiles, and store stages. The PTX ISA defines no equal-share bandwidth model for a fixed number of epilogue warps; diagnose TMEM-load, conversion, barrier, and store bottlenecks with generated code and profiler data.\n\nFusing output conversion can avoid a global FP32 intermediate when the baseline would write and reread that intermediate. It does not guarantee a speedup: added registers, shared-memory staging, synchronization, edge handling, or reduced occupancy can outweigh saved traffic.\n\nUseful negative tests delay one reader, skip a barrier phase, reuse a TMEM region early, omit the final store tail, and exercise partial M/N tiles. Each fault should be detected by output comparison or a bounded watchdog."}, "reason": {"statement": "Replace universal role count and benefit with configuration-specific evaluation.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} -{"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "- TMEM-to-register bandwidth is not unlimited. With 14 warps simultaneously reading TMEM, each warp gets a proportional share. Very wide output tiles (large TILE_N) may bottleneck on TMEM read bandwidth."}, "after": {"statement": "When an implementation overlaps the epilogue with independent MMA work, verify both directions of the handoff: compute-complete before read, and load/read-complete before overwrite. Include the prologue (no prior result), steady-state wraparound, final drain, and deallocation path in tests.\n\nEvaluate a defined fused/unfused pair with identical inputs, output semantics, launch policy, warmup, and timing statistics. Record registers, spills, SMEM, TMEM columns, barriers, threads/CTA, cluster shape, and occupancy. Sweep supported epilogue group sizes, load shapes, output tiles, and store stages. The PTX ISA defines no equal-share bandwidth model for a fixed number of epilogue warps; diagnose TMEM-load, conversion, barrier, and store bottlenecks with generated code and profiler data.\n\nFusing output conversion can avoid a global FP32 intermediate when the baseline would write and reread that intermediate. It does not guarantee a speedup: added registers, shared-memory staging, synchronization, edge handling, or reduced occupancy can outweigh saved traffic.\n\nUseful negative tests delay one reader, skip a barrier phase, reuse a TMEM region early, omit the final store tail, and exercise partial M/N tiles. Each fault should be detected by output comparison or a bounded watchdog."}, "reason": {"statement": "Require profiler and variant measurements for bandwidth diagnosis.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} -{"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "- Simple epilogues (just store) waste the 14 epilogue warps. For such cases, consider reducing the CTA size or assigning epilogue warps to other work (e.g., next-tile TMA prefetch)."}, "after": {"statement": "When an implementation overlaps the epilogue with independent MMA work, verify both directions of the handoff: compute-complete before read, and load/read-complete before overwrite. Include the prologue (no prior result), steady-state wraparound, final drain, and deallocation path in tests.\n\nEvaluate a defined fused/unfused pair with identical inputs, output semantics, launch policy, warmup, and timing statistics. Record registers, spills, SMEM, TMEM columns, barriers, threads/CTA, cluster shape, and occupancy. Sweep supported epilogue group sizes, load shapes, output tiles, and store stages. The PTX ISA defines no equal-share bandwidth model for a fixed number of epilogue warps; diagnose TMEM-load, conversion, barrier, and store bottlenecks with generated code and profiler data.\n\nFusing output conversion can avoid a global FP32 intermediate when the baseline would write and reread that intermediate. It does not guarantee a speedup: added registers, shared-memory staging, synchronization, edge handling, or reduced occupancy can outweigh saved traffic.\n\nUseful negative tests delay one reader, skip a barrier phase, reuse a TMEM region early, omit the final store tail, and exercise partial M/N tiles. Each fault should be detected by output comparison or a bounded watchdog."}, "reason": {"statement": "Use measured role-count and store-path bottlenecks.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} +{"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "- **All Blackwell GEMMs with non-trivial epilogues**: The 14 epilogue warps are available by default in the warp-specialized model. Fusing operations avoids a separate kernel launch and an extra global memory round-trip."}, "after": {"statement": "When an implementation overlaps the epilogue with independent MMA work, verify both directions of the handoff: compute-complete before read, and load/read-complete before overwrite. Include the prologue (no prior result), steady-state wraparound, final drain, and deallocation path in tests.\n\nEvaluate a defined fused/unfused pair with identical inputs, output semantics, launch policy, warmup, and timing statistics. Record registers, spills, SMEM, TMEM columns, barriers, threads/CTA, cluster shape, and occupancy. Sweep supported epilogue group sizes, load shapes, output tiles, and store stages. The PTX ISA defines no equal-share bandwidth model for a fixed number of epilogue warps; diagnose TMEM-load, conversion, barrier, and store bottlenecks with generated code and profiler data.\n\nFusing output conversion can avoid a global FP32 intermediate when the baseline would write and reread that intermediate. It does not guarantee a speedup: added registers, shared-memory staging, synchronization, edge handling, or reduced occupancy can outweigh saved traffic.\n\nUseful negative tests delay one reader, skip a barrier phase, reuse a TMEM region early, omit the final store tail, and exercise partial M/N tiles. Each fault should be detected by output comparison or a bounded watchdog."}, "reason": {"statement": "Replace universal role count and benefit with configuration-specific evaluation.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} +{"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "- TMEM-to-register bandwidth is not unlimited. With 14 warps simultaneously reading TMEM, each warp gets a proportional share. Very wide output tiles (large TILE_N) may bottleneck on TMEM read bandwidth."}, "after": {"statement": "When an implementation overlaps the epilogue with independent MMA work, verify both directions of the handoff: compute-complete before read, and load/read-complete before overwrite. Include the prologue (no prior result), steady-state wraparound, final drain, and deallocation path in tests.\n\nEvaluate a defined fused/unfused pair with identical inputs, output semantics, launch policy, warmup, and timing statistics. Record registers, spills, SMEM, TMEM columns, barriers, threads/CTA, cluster shape, and occupancy. Sweep supported epilogue group sizes, load shapes, output tiles, and store stages. The PTX ISA defines no equal-share bandwidth model for a fixed number of epilogue warps; diagnose TMEM-load, conversion, barrier, and store bottlenecks with generated code and profiler data.\n\nFusing output conversion can avoid a global FP32 intermediate when the baseline would write and reread that intermediate. It does not guarantee a speedup: added registers, shared-memory staging, synchronization, edge handling, or reduced occupancy can outweigh saved traffic.\n\nUseful negative tests delay one reader, skip a barrier phase, reuse a TMEM region early, omit the final store tail, and exercise partial M/N tiles. Each fault should be detected by output comparison or a bounded watchdog."}, "reason": {"statement": "Require profiler and variant measurements for bandwidth diagnosis.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} +{"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "- Simple epilogues (just store) waste the 14 epilogue warps. For such cases, consider reducing the CTA size or assigning epilogue warps to other work (e.g., next-tile TMA prefetch)."}, "after": {"statement": "When an implementation overlaps the epilogue with independent MMA work, verify both directions of the handoff: compute-complete before read, and load/read-complete before overwrite. Include the prologue (no prior result), steady-state wraparound, final drain, and deallocation path in tests.\n\nEvaluate a defined fused/unfused pair with identical inputs, output semantics, launch policy, warmup, and timing statistics. Record registers, spills, SMEM, TMEM columns, barriers, threads/CTA, cluster shape, and occupancy. Sweep supported epilogue group sizes, load shapes, output tiles, and store stages. The PTX ISA defines no equal-share bandwidth model for a fixed number of epilogue warps; diagnose TMEM-load, conversion, barrier, and store bottlenecks with generated code and profiler data.\n\nFusing output conversion can avoid a global FP32 intermediate when the baseline would write and reread that intermediate. It does not guarantee a speedup: added registers, shared-memory staging, synchronization, edge handling, or reduced occupancy can outweigh saved traffic.\n\nUseful negative tests delay one reader, skip a barrier phase, reuse a TMEM region early, omit the final store tail, and exercise partial M/N tiles. Each fault should be detected by output comparison or a bounded watchdog."}, "reason": {"statement": "Use measured role-count and store-path bottlenecks.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} {"path": "wiki/techniques/epilogue-fusion.md", "before": {"statement": "Verbatim upstream code lives in [`artifacts/kernels/epilogue-fusion/full/`](../../artifacts/kernels/epilogue-fusion/full/); labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` header) live in [`artifacts/kernels/epilogue-fusion/variants/`](../../artifacts/kernels/epilogue-fusion/variants/). Every file's SHA-256 and upstream-pinning metadata is in `PROVENANCE.yaml` inside each bundle."}, "after": {"statement": "The artifact bundle contains mixed provenance modes:\n\n- `full/nvfp4_scaled_mm_kernels.cu` is verbatim from vLLM merge `ed7a29d9f8b48978e3bbf43599d21b4de65387e0` and byte-matches SHA-256 `e8aed5ccb3dd9de26c3aeff159a242a46dcb7c7d8d0351b6c44ff1f8d2f7effa`.\n- `full/tmem-load-into-registers-for-epilogue.cu` is an extracted historical snippet from a local source card; it is not an upstream-verbatim kernel and must not be treated as standalone safe code.\n- `variants/01-double-buffered-tmem-epilogue-skeleton.cu` is explicitly labeled derived/not-upstream and is incomplete pseudocode.\n\nRetrieve the page and bundle with:\n\n```bash\nconda run -n base python scripts/get_page.py technique-epilogue-fusion --include-code\n```"}, "reason": {"statement": "Describe each artifact's actual provenance mode and warn that extracted/derived snippets are not safe standalone kernels.", "urls": []}} {"path": "wiki/techniques/persistent-kernels.md", "before": {"statement": "Persistent kernels launch exactly as many CTAs as SMs, and each CTA processes multiple output tiles in a loop rather than exiting after one tile. On Blackwell, the CLC (Cluster Launch Control) hardware unit replaces software-based tile scheduling with a hardware-assisted mechanism. Each CTA queries the CLC for its next tile assignment and can cancel itself when no work remains, using the `try_cancel` pattern."}, "after": {"statement": "A persistent kernel keeps a resident worker alive for multiple logical work items. Its launch size is chosen from the problem decomposition, cluster shape, resource limits, and scheduling policy; “persistent” does not require exactly one CTA per physical SM.\n\nCluster Launch Control is a compute-capability-10.0 mechanism for redistributing grid coordinates that have not started. A CLC GEMM still launches its problem-sized grid. Each ClcID is processed exactly once through one of two paths:\n\n1. A block or cluster launches normally and processes its initial `blockIdx`.\n2. A running worker successfully cancels another not-yet-started block/cluster and processes the returned coordinate.\n\nCLC does not create tiles, cancel the requesting worker, or chain independent problems. It can help when the SMs available to a grid are uneven, but it cannot expose more independent parallel work than exists in the launched grid."}, "reason": {"statement": "Restore problem-sized CLC grid and two-path ClcID model.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} -{"path": "wiki/techniques/persistent-kernels.md", "before": {"statement": "```cuda\n// Persistent kernel with CLC tile scheduling (Blackwell SM100)\n__global__ void __launch_bounds__(512)\npersistent_gemm_clc(const __grid_constant__ GemmParams params)\n{\n // CLC-managed persistent loop: each CTA processes multiple tiles\n while (true) {\n // Query CLC for next tile assignment\n // Returns tile coordinates (tile_m, tile_n) or signals termination\n TileCoord tile;\n bool has_work = clc_try_get_tile(&tile);\n\n if (!has_work) {\n // No more tiles to process -- CTA exits\n // clc_try_cancel atomically checks if all tiles are done\n if (clc_try_cancel()) {\n return; // CTA terminates\n }\n continue; // Race condition: another CTA may have generated work\n }\n\n // Standard GEMM tile computation\n int tile_m = tile.m;\n int tile_n = tile.n;\n\n // TMA producer loads A[tile_m, :] and B[:, tile_n] tiles\n // MMA consumer accumulates K-dimension\n // Epilogue writes C[tile_m, tile_n]\n compute_gemm_tile(params, tile_m, tile_n);\n }\n}\n```"}, "after": {"statement": "Cluster Launch Control is a compute-capability-10.0 mechanism for redistributing grid coordinates that have not started. A CLC GEMM still launches its problem-sized grid. Each ClcID is processed exactly once through one of two paths:\n\n1. A block or cluster launches normally and processes its initial `blockIdx`.\n2. A running worker successfully cancels another not-yet-started block/cluster and processes the returned coordinate.\n\nCLC does not create tiles, cancel the requesting worker, or chain independent problems. It can help when the SMs available to a grid are uneven, but it cannot expose more independent parallel work than exists in the launched grid.\n\nThe normative request is asynchronous and returns an opaque 16-byte shared-memory response:\n\n```ptx\nclusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [response], [mbar];\nclusterlaunchcontrol.query_cancel.is_canceled.pred.b128 p, response_b128;\n@p clusterlaunchcontrol.query_cancel.get_first_ctaid.v4.b32.b128 {x, y, z, unused}, response_b128;\n```\n\nA complete implementation must initialize/publish the shared response and mbarrier, set the expected 16 transaction bytes, submit from the designated participant, wait for the matching phase, apply required proxy fences before reading/reusing the response, and decode a coordinate only when `is_canceled` succeeds.\n\nAfter a thread observes a failed request, another request from that thread is undefined. Failure is therefore not a “retry until another CTA creates work” condition; no CTA creates new ClcIDs.\n\nFor a thread-block cluster, one cluster participant submits the multicast request. Every CTA tracks completion at cluster scope, receives the same first coordinate, and adds its local block rank. Cluster synchronization is required before cancellation begins so all peers exist."}, "reason": {"statement": "Remove inverted/fabricated executable sketch.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} -{"path": "wiki/techniques/persistent-kernels.md", "before": {"statement": "At the PTX level, the CLC interaction is a cancel/query sequence. The exact\ninline PTX is usually hidden behind CUTLASS/CuTe wrappers, but the control flow\nlooks like this:\n\n```text\nTILE_LOOP:\n // Request cancellation of a not-yet-launched cluster.\n clusterlaunchcontrol.try_cancel(response_smem, mbarrier)\n wait(mbarrier)\n\n // Query the 16-byte response.\n has_work, tile_m, tile_n = clusterlaunchcontrol.query_cancel(response_smem)\n if (!has_work) return\n\n // ... compute tile ...\n goto TILE_LOOP\n```"}, "after": {"statement": "Cluster Launch Control is a compute-capability-10.0 mechanism for redistributing grid coordinates that have not started. A CLC GEMM still launches its problem-sized grid. Each ClcID is processed exactly once through one of two paths:\n\n1. A block or cluster launches normally and processes its initial `blockIdx`.\n2. A running worker successfully cancels another not-yet-started block/cluster and processes the returned coordinate.\n\nCLC does not create tiles, cancel the requesting worker, or chain independent problems. It can help when the SMs available to a grid are uneven, but it cannot expose more independent parallel work than exists in the launched grid.\n\nThe normative request is asynchronous and returns an opaque 16-byte shared-memory response:\n\n```ptx\nclusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [response], [mbar];\nclusterlaunchcontrol.query_cancel.is_canceled.pred.b128 p, response_b128;\n@p clusterlaunchcontrol.query_cancel.get_first_ctaid.v4.b32.b128 {x, y, z, unused}, response_b128;\n```\n\nA complete implementation must initialize/publish the shared response and mbarrier, set the expected 16 transaction bytes, submit from the designated participant, wait for the matching phase, apply required proxy fences before reading/reusing the response, and decode a coordinate only when `is_canceled` succeeds.\n\nAfter a thread observes a failed request, another request from that thread is undefined. Failure is therefore not a “retry until another CTA creates work” condition; no CTA creates new ClcIDs.\n\nFor a thread-block cluster, one cluster participant submits the multicast request. Every CTA tracks completion at cluster scope, receives the same first coordinate, and adds its local block rank. Cluster synchronization is required before cancellation begins so all peers exist."}, "reason": {"statement": "Replace with complete invariants and exact PTX forms linked elsewhere.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} -{"path": "wiki/techniques/persistent-kernels.md", "before": {"statement": "| Scheduling | Software loop with fixed stride | Hardware CLC unit assigns tiles |\n| Load balancing | Fixed; uneven if tile costs vary | Dynamic; CLC rebalances automatically |\n| Tail effect | Last wave may have partial occupancy | CLC minimizes by giving fast CTAs more tiles |\n| Launch overhead | Grid launch for each new problem | CLC can chain multiple problems |\n| Termination | Implicit when loop ends | Explicit `try_cancel` |\n| L2 locality | Depends on stride pattern | CLC can apply swizzled raster |"}, "after": {"statement": "Cluster Launch Control is a compute-capability-10.0 mechanism for redistributing grid coordinates that have not started. A CLC GEMM still launches its problem-sized grid. Each ClcID is processed exactly once through one of two paths:\n\n1. A block or cluster launches normally and processes its initial `blockIdx`.\n2. A running worker successfully cancels another not-yet-started block/cluster and processes the returned coordinate.\n\nCLC does not create tiles, cancel the requesting worker, or chain independent problems. It can help when the SMs available to a grid are uneven, but it cannot expose more independent parallel work than exists in the launched grid.\n\nA software persistent scheduler can assign a flat tile index by fixed stride:\n\n```cpp\nfor (int tile = blockIdx.x; tile < total_tiles; tile += gridDim.x) {\n int tile_m = tile / tiles_n;\n int tile_n = tile % tiles_n;\n compute_tile(tile_m, tile_n);\n}\n```\n\nFor positive `gridDim.x`, this covers each index in `[0,total_tiles)` exactly once, including nondivisible tails. It remains useful as a controlled baseline on Hopper and Blackwell."}, "reason": {"statement": "Separate raw CLC, CUTLASS transforms, and out-of-scope mechanisms.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"]}} -{"path": "wiki/techniques/persistent-kernels.md", "before": {"statement": "```cuda\n// CUTLASS SM100 persistent tile scheduler (simplified)\ntemplate \nstruct PersistentTileSchedulerSm100 {\n\n // Initialize the CLC with the problem geometry\n CUTLASS_DEVICE static void init(\n dim3 problem_tiles,\n void* clc_smem_buffer)\n {\n if (threadIdx.x == 0) {\n // Program CLC with total tile count and scheduling policy\n clc_init(clc_smem_buffer,\n problem_tiles.x, // tiles along M\n problem_tiles.y, // tiles along N\n ClcPolicy::SwizzledRaster);\n }\n __syncthreads();\n }\n\n // Shared storage for CTA-wide CLC result broadcast\n // __shfl_sync is warp-local and cannot reach warps 1-15.\n struct SharedClcState {\n int tile_m, tile_n;\n int valid; // 1 = got tile, 0 = no more work\n int cancelled;\n };\n\n // Get next tile assignment from CLC\n CUTLASS_DEVICE static bool get_next_tile(\n void* clc_smem_buffer,\n SharedClcState& shared_clc,\n int& tile_m,\n int& tile_n)\n {\n if (threadIdx.x == 0) {\n int m, n;\n bool v = clc_query_tile(clc_smem_buffer, m, n);\n shared_clc.tile_m = m;\n shared_clc.tile_n = n;\n shared_clc.valid = v ? 1 : 0;\n }\n __syncthreads(); // All warps see the result\n tile_m = shared_clc.tile_m;\n tile_n = shared_clc.tile_n;\n return shared_clc.valid != 0;\n }\n\n // Try to cancel the CTA when no more work\n CUTLASS_DEVICE static bool try_cancel(\n void* clc_smem_buffer,\n SharedClcState& shared_clc)\n {\n if (threadIdx.x == 0) {\n shared_clc.cancelled = clc_try_cancel(clc_smem_buffer) ? 1 : 0;\n }\n __syncthreads();\n return shared_clc.cancelled != 0;\n }\n};\n```"}, "after": {"statement": "The normative request is asynchronous and returns an opaque 16-byte shared-memory response:\n\n```ptx\nclusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [response], [mbar];\nclusterlaunchcontrol.query_cancel.is_canceled.pred.b128 p, response_b128;\n@p clusterlaunchcontrol.query_cancel.get_first_ctaid.v4.b32.b128 {x, y, z, unused}, response_b128;\n```\n\nA complete implementation must initialize/publish the shared response and mbarrier, set the expected 16 transaction bytes, submit from the designated participant, wait for the matching phase, apply required proxy fences before reading/reusing the response, and decode a coordinate only when `is_canceled` succeeds.\n\nAfter a thread observes a failed request, another request from that thread is undefined. Failure is therefore not a “retry until another CTA creates work” condition; no CTA creates new ClcIDs.\n\nFor a thread-block cluster, one cluster participant submits the multicast request. Every CTA tracks completion at cluster scope, receives the same first coordinate, and adds its local block rank. Cluster synchronization is required before cancellation begins so all peers exist."}, "reason": {"statement": "Use exact class/method routing and avoid fake implementation bodies.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"]}} +{"path": "wiki/techniques/persistent-kernels.md", "before": {"statement": "```cuda\n// Persistent kernel with CLC tile scheduling (Blackwell SM100)\n__global__ void __launch_bounds__(512)\npersistent_gemm_clc(const __grid_constant__ GemmParams params)\n{\n // CLC-managed persistent loop: each CTA processes multiple tiles\n while (true) {\n // Query CLC for next tile assignment\n // Returns tile coordinates (tile_m, tile_n) or signals termination\n TileCoord tile;\n bool has_work = clc_try_get_tile(&tile);\n\n if (!has_work) {\n // No more tiles to process -- CTA exits\n // clc_try_cancel atomically checks if all tiles are done\n if (clc_try_cancel()) {\n return; // CTA terminates\n }\n continue; // Race condition: another CTA may have generated work\n }\n\n // Standard GEMM tile computation\n int tile_m = tile.m;\n int tile_n = tile.n;\n\n // TMA producer loads A[tile_m, :] and B[:, tile_n] tiles\n // MMA consumer accumulates K-dimension\n // Epilogue writes C[tile_m, tile_n]\n compute_gemm_tile(params, tile_m, tile_n);\n }\n}\n```"}, "after": {"statement": "Cluster Launch Control is a compute-capability-10.0 mechanism for redistributing grid coordinates that have not started. A CLC GEMM still launches its problem-sized grid. Each ClcID is processed exactly once through one of two paths:\n\n1. A block or cluster launches normally and processes its initial `blockIdx`.\n2. A running worker successfully cancels another not-yet-started block/cluster and processes the returned coordinate.\n\nCLC does not create tiles, cancel the requesting worker, or chain independent problems. It can help when the SMs available to a grid are uneven, but it cannot expose more independent parallel work than exists in the launched grid.\n\nThe normative request is asynchronous and returns an opaque 16-byte shared-memory response:\n\n```ptx\nclusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [response], [mbar];\nclusterlaunchcontrol.query_cancel.is_canceled.pred.b128 p, response_b128;\n@p clusterlaunchcontrol.query_cancel.get_first_ctaid.v4.b32.b128 {x, y, z, unused}, response_b128;\n```\n\nA complete implementation must initialize/publish the shared response and mbarrier, set the expected 16 transaction bytes, submit from the designated participant, wait for the matching phase, apply required proxy fences before reading/reusing the response, and decode a coordinate only when `is_canceled` succeeds.\n\nAfter a thread observes a failed request, another request from that thread is undefined. Failure is therefore not a “retry until another CTA creates work” condition; no CTA creates new ClcIDs.\n\nFor a thread-block cluster, one cluster participant submits the multicast request. Every CTA tracks completion at cluster scope, receives the same first coordinate, and adds its local block rank. Cluster synchronization is required before cancellation begins so all peers exist."}, "reason": {"statement": "Remove inverted/fabricated executable sketch.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/techniques/persistent-kernels.md", "before": {"statement": "At the PTX level, the CLC interaction is a cancel/query sequence. The exact\ninline PTX is usually hidden behind CUTLASS/CuTe wrappers, but the control flow\nlooks like this:\n\n```text\nTILE_LOOP:\n // Request cancellation of a not-yet-launched cluster.\n clusterlaunchcontrol.try_cancel(response_smem, mbarrier)\n wait(mbarrier)\n\n // Query the 16-byte response.\n has_work, tile_m, tile_n = clusterlaunchcontrol.query_cancel(response_smem)\n if (!has_work) return\n\n // ... compute tile ...\n goto TILE_LOOP\n```"}, "after": {"statement": "Cluster Launch Control is a compute-capability-10.0 mechanism for redistributing grid coordinates that have not started. A CLC GEMM still launches its problem-sized grid. Each ClcID is processed exactly once through one of two paths:\n\n1. A block or cluster launches normally and processes its initial `blockIdx`.\n2. A running worker successfully cancels another not-yet-started block/cluster and processes the returned coordinate.\n\nCLC does not create tiles, cancel the requesting worker, or chain independent problems. It can help when the SMs available to a grid are uneven, but it cannot expose more independent parallel work than exists in the launched grid.\n\nThe normative request is asynchronous and returns an opaque 16-byte shared-memory response:\n\n```ptx\nclusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [response], [mbar];\nclusterlaunchcontrol.query_cancel.is_canceled.pred.b128 p, response_b128;\n@p clusterlaunchcontrol.query_cancel.get_first_ctaid.v4.b32.b128 {x, y, z, unused}, response_b128;\n```\n\nA complete implementation must initialize/publish the shared response and mbarrier, set the expected 16 transaction bytes, submit from the designated participant, wait for the matching phase, apply required proxy fences before reading/reusing the response, and decode a coordinate only when `is_canceled` succeeds.\n\nAfter a thread observes a failed request, another request from that thread is undefined. Failure is therefore not a “retry until another CTA creates work” condition; no CTA creates new ClcIDs.\n\nFor a thread-block cluster, one cluster participant submits the multicast request. Every CTA tracks completion at cluster scope, receives the same first coordinate, and adds its local block rank. Cluster synchronization is required before cancellation begins so all peers exist."}, "reason": {"statement": "Replace with complete invariants and exact PTX forms linked elsewhere.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/techniques/persistent-kernels.md", "before": {"statement": "| Scheduling | Software loop with fixed stride | Hardware CLC unit assigns tiles |\n| Load balancing | Fixed; uneven if tile costs vary | Dynamic; CLC rebalances automatically |\n| Tail effect | Last wave may have partial occupancy | CLC minimizes by giving fast CTAs more tiles |\n| Launch overhead | Grid launch for each new problem | CLC can chain multiple problems |\n| Termination | Implicit when loop ends | Explicit `try_cancel` |\n| L2 locality | Depends on stride pattern | CLC can apply swizzled raster |"}, "after": {"statement": "Cluster Launch Control is a compute-capability-10.0 mechanism for redistributing grid coordinates that have not started. A CLC GEMM still launches its problem-sized grid. Each ClcID is processed exactly once through one of two paths:\n\n1. A block or cluster launches normally and processes its initial `blockIdx`.\n2. A running worker successfully cancels another not-yet-started block/cluster and processes the returned coordinate.\n\nCLC does not create tiles, cancel the requesting worker, or chain independent problems. It can help when the SMs available to a grid are uneven, but it cannot expose more independent parallel work than exists in the launched grid.\n\nA software persistent scheduler can assign a flat tile index by fixed stride:\n\n```cpp\nfor (int tile = blockIdx.x; tile < total_tiles; tile += gridDim.x) {\n int tile_m = tile / tiles_n;\n int tile_n = tile % tiles_n;\n compute_tile(tile_m, tile_n);\n}\n```\n\nFor positive `gridDim.x`, this covers each index in `[0,total_tiles)` exactly once, including nondivisible tails. It remains useful as a controlled baseline on Hopper and Blackwell."}, "reason": {"statement": "Separate raw CLC, CUTLASS transforms, and out-of-scope mechanisms.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel"]}} +{"path": "wiki/techniques/persistent-kernels.md", "before": {"statement": "```cuda\n// CUTLASS SM100 persistent tile scheduler (simplified)\ntemplate \nstruct PersistentTileSchedulerSm100 {\n\n // Initialize the CLC with the problem geometry\n CUTLASS_DEVICE static void init(\n dim3 problem_tiles,\n void* clc_smem_buffer)\n {\n if (threadIdx.x == 0) {\n // Program CLC with total tile count and scheduling policy\n clc_init(clc_smem_buffer,\n problem_tiles.x, // tiles along M\n problem_tiles.y, // tiles along N\n ClcPolicy::SwizzledRaster);\n }\n __syncthreads();\n }\n\n // Shared storage for CTA-wide CLC result broadcast\n // __shfl_sync is warp-local and cannot reach warps 1-15.\n struct SharedClcState {\n int tile_m, tile_n;\n int valid; // 1 = got tile, 0 = no more work\n int cancelled;\n };\n\n // Get next tile assignment from CLC\n CUTLASS_DEVICE static bool get_next_tile(\n void* clc_smem_buffer,\n SharedClcState& shared_clc,\n int& tile_m,\n int& tile_n)\n {\n if (threadIdx.x == 0) {\n int m, n;\n bool v = clc_query_tile(clc_smem_buffer, m, n);\n shared_clc.tile_m = m;\n shared_clc.tile_n = n;\n shared_clc.valid = v ? 1 : 0;\n }\n __syncthreads(); // All warps see the result\n tile_m = shared_clc.tile_m;\n tile_n = shared_clc.tile_n;\n return shared_clc.valid != 0;\n }\n\n // Try to cancel the CTA when no more work\n CUTLASS_DEVICE static bool try_cancel(\n void* clc_smem_buffer,\n SharedClcState& shared_clc)\n {\n if (threadIdx.x == 0) {\n shared_clc.cancelled = clc_try_cancel(clc_smem_buffer) ? 1 : 0;\n }\n __syncthreads();\n return shared_clc.cancelled != 0;\n }\n};\n```"}, "after": {"statement": "The normative request is asynchronous and returns an opaque 16-byte shared-memory response:\n\n```ptx\nclusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [response], [mbar];\nclusterlaunchcontrol.query_cancel.is_canceled.pred.b128 p, response_b128;\n@p clusterlaunchcontrol.query_cancel.get_first_ctaid.v4.b32.b128 {x, y, z, unused}, response_b128;\n```\n\nA complete implementation must initialize/publish the shared response and mbarrier, set the expected 16 transaction bytes, submit from the designated participant, wait for the matching phase, apply required proxy fences before reading/reusing the response, and decode a coordinate only when `is_canceled` succeeds.\n\nAfter a thread observes a failed request, another request from that thread is undefined. Failure is therefore not a “retry until another CTA creates work” condition; no CTA creates new ClcIDs.\n\nFor a thread-block cluster, one cluster participant submits the multicast request. Every CTA tracks completion at cluster scope, receives the same first coordinate, and adds its local block rank. Cluster synchronization is required before cancellation begins so all peers exist."}, "reason": {"statement": "Use exact class/method routing and avoid fake implementation bodies.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel"]}} {"path": "wiki/techniques/persistent-kernels.md", "before": {"statement": "The tcgen05-tutorial progression demonstrates the impact of persistent kernels:\n\n```\nWithout persistence (static grid): 940 TFLOPS (62% of peak)\nWith CLC persistent scheduling: 1476 TFLOPS (98% of cuBLAS)\n```\n\nThe 57% improvement comes from:\n1. **Eliminated tail effect**: CLC dynamically assigns tiles, so fast-completing CTAs absorb extra work rather than sitting idle while the last wave finishes.\n2. **Reduced launch overhead**: A single kernel launch covers all tiles; no need to re-launch grids.\n3. **Better L2 cache utilization**: CLC can apply a swizzled raster pattern that improves spatial locality across neighboring tiles."}, "after": {"statement": "At CUTLASS 4.5.0, the SM100 persistent scheduler integrates CLC through `PersistentTileSchedulerSm100` and `PipelineCLCFetchAsync`. The scheduler's work flow is expressed through `WorkTileInfo` and methods including `get_current_work()`, `advance_to_next_work()`, and `fetch_next_work()`; it is not an API built from `clc_init`, `clc_query_tile`, or a caller-self-cancel helper.\n\n`advance_to_next_work()` submits/stages the next request, while `fetch_next_work()` waits for and decodes a response. CUTLASS applies `swizzle_and_rasterize()` as a software coordinate transform to both the initial `blockIdx` and returned CLC coordinates. `max_swizzle_size` and `raster_order` are scheduler arguments, not CLC hardware policy operands.\n\nThe official documentation is the authoritative implementation map; use its pinned scheduler and pipeline links rather than a short pseudo-class."}, "reason": {"statement": "Preserve the author progression only with exact variant scope and no CLC attribution.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} {"path": "wiki/techniques/persistent-kernels.md", "before": {"statement": "- **Large GEMM problems**: Persistent kernels are most beneficial when the number of output tiles exceeds the SM count by at least 2-3x.\n- **Grouped GEMMs / MoE**: CLC can chain multiple problem instances, eliminating inter-kernel launch gaps.\n- **Workloads with uneven tile cost**: CLC's dynamic scheduling naturally handles variable-cost tiles (e.g., triangular attention masks)."}, "after": {"statement": "At CUTLASS 4.5.0, the SM100 persistent scheduler integrates CLC through `PersistentTileSchedulerSm100` and `PipelineCLCFetchAsync`. The scheduler's work flow is expressed through `WorkTileInfo` and methods including `get_current_work()`, `advance_to_next_work()`, and `fetch_next_work()`; it is not an API built from `clc_init`, `clc_query_tile`, or a caller-self-cancel helper.\n\n`advance_to_next_work()` submits/stages the next request, while `fetch_next_work()` waits for and decodes a response. CUTLASS applies `swizzle_and_rasterize()` as a software coordinate transform to both the initial `blockIdx` and returned CLC coordinates. `max_swizzle_size` and `raster_order` are scheduler arguments, not CLC hardware policy operands.\n\nThe official documentation is the authoritative implementation map; use its pinned scheduler and pipeline links rather than a short pseudo-class."}, "reason": {"statement": "Replace prescriptive thresholds with evidence requirements and conserved-work limits.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cutlass/gemm/kernel/sm100_tile_scheduler.hpp"]}} -{"path": "wiki/techniques/persistent-kernels.md", "before": {"statement": "- The `try_cancel` pattern introduces a potential race that must be handled with a retry loop."}, "after": {"statement": "Cluster Launch Control is a compute-capability-10.0 mechanism for redistributing grid coordinates that have not started. A CLC GEMM still launches its problem-sized grid. Each ClcID is processed exactly once through one of two paths:\n\n1. A block or cluster launches normally and processes its initial `blockIdx`.\n2. A running worker successfully cancels another not-yet-started block/cluster and processes the returned coordinate.\n\nCLC does not create tiles, cancel the requesting worker, or chain independent problems. It can help when the SMs available to a grid are uneven, but it cannot expose more independent parallel work than exists in the launched grid."}, "reason": {"statement": "Make failed response terminal for that requesting thread.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/techniques/persistent-kernels.md", "before": {"statement": "- The `try_cancel` pattern introduces a potential race that must be handled with a retry loop."}, "after": {"statement": "Cluster Launch Control is a compute-capability-10.0 mechanism for redistributing grid coordinates that have not started. A CLC GEMM still launches its problem-sized grid. Each ClcID is processed exactly once through one of two paths:\n\n1. A block or cluster launches normally and processes its initial `blockIdx`.\n2. A running worker successfully cancels another not-yet-started block/cluster and processes the returned coordinate.\n\nCLC does not create tiles, cancel the requesting worker, or chain independent problems. It can help when the SMs available to a grid are uneven, but it cannot expose more independent parallel work than exists in the launched grid."}, "reason": {"statement": "Make failed response terminal for that requesting thread.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} {"path": "wiki/techniques/pipeline-stages.md", "before": {"statement": "Software pipelining overlaps data loading (TMA copies from global to shared memory) with computation (tcgen05.mma or wgmma) by maintaining multiple in-flight tile buffers. A circular buffer of 3-5 stages allows the TMA producer to fill stage N+2 while the MMA consumer processes stage N, hiding the global memory latency entirely. This technique is critical for achieving high utilization on both Hopper and Blackwell."}, "after": {"statement": "A multi-stage pipeline reserves disjoint shared-memory operand buffers so production of one tile can overlap consumption of another. Stage count controls how far producer and consumer progress may separate; it does not guarantee that latency is fully hidden or that the schedule is faster.\n\nThere is no architecture-wide table in which two stages are always partial, three always fully hide latency, or more than five are always excessive. Legal and useful depth depends on:\n\n- bytes per stage, alignment/swizzle padding, other SMEM, and occupancy;\n- TMA issue/transfer rate and descriptor/transaction shape;\n- MMA work and completion time per K tile;\n- producer/consumer role schedule and register pressure;\n- K-loop length, prologue/tail fraction, and output schedule.\n\nCompile every candidate, reject resource-invalid variants, then compare controlled timings and pipeline/barrier stalls. Include `num_k_tiles` values below, equal to, and above the stage count to exercise every boundary."}, "reason": {"statement": "Define the mechanism without guaranteed profitability.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-best-practices-guide/index.html#asynchronous-copy-from-global-memory-to-shared-memory"]}} {"path": "wiki/techniques/pipeline-stages.md", "before": {"statement": "The tcgen05-tutorial demonstrates the performance impact of pipelining:\n\n```\nNo pipelining (load, then compute): 695 TFLOPS (46%)\n3-stage pipeline (TMA + MMA overlap): 940 TFLOPS (62%)\n+ warp specialization: 1476 TFLOPS (98%)\n```"}, "after": {"statement": "Compute capability 10.0 supports up to 228 KiB of shared memory per SM. Thus five stages of that exact payload already exceed the per-SM capacity, while three stages consume about 63.2% before barriers, padding, descriptors, epilogue buffers, and other shared storage. Per-block opt-in limits and occupancy constraints also apply."}, "reason": {"statement": "Report only the adjacent v2b/v3 pipeline comparison and scope it to the author setup.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"]}} -{"path": "wiki/techniques/pipeline-stages.md", "before": {"statement": "```cuda\n// 3-stage circular buffer with mbarrier synchronization\n// Stages: [0] loading, [1] ready for MMA, [2] being consumed by MMA\n#define NUM_STAGES 3\n\n__global__ void __launch_bounds__(512)\npipelined_gemm(const __grid_constant__ GemmParams params)\n{\n extern __shared__ char smem[];\n\n // Circular buffer layout in shared memory\n // Each stage has its own A and B tile buffers\n half* smem_A[NUM_STAGES];\n half* smem_B[NUM_STAGES];\n for (int s = 0; s < NUM_STAGES; s++) {\n smem_A[s] = reinterpret_cast(\n smem + s * (TILE_A_BYTES + TILE_B_BYTES));\n smem_B[s] = reinterpret_cast(\n smem + s * (TILE_A_BYTES + TILE_B_BYTES) + TILE_A_BYTES);\n }\n\n // mbarrier arrays for producer-consumer sync\n __shared__ uint64_t mbar_load_complete[NUM_STAGES];\n __shared__ uint64_t mbar_mma_complete[NUM_STAGES];\n\n int warp_id = threadIdx.x / 32;\n int lane_id = threadIdx.x % 32;\n\n // Initialize barriers\n if (threadIdx.x == 0) {\n for (int s = 0; s < NUM_STAGES; s++) {\n mbarrier_init(&mbar_load_complete[s], 1);\n mbarrier_init(&mbar_mma_complete[s], 1);\n }\n }\n __syncthreads();\n\n int num_k_tiles = params.K / TILE_K;\n\n if (warp_id == 0) {\n // ===== TMA PRODUCER =====\n // Prologue: fill the first NUM_STAGES buffers\n for (int s = 0; s < NUM_STAGES && s < num_k_tiles; s++) {\n if (lane_id == 0) {\n // Set expected TX bytes on mbarrier BEFORE issuing TMA.\n // TMA hardware will arrive on the mbarrier when transfer completes.\n uint32_t tx_bytes = TILE_A_BYTES + TILE_B_BYTES;\n mbarrier_arrive_expect_tx(&mbar_load_complete[s], tx_bytes);\n tma_load_tile_A(smem_A[s], params, s, &mbar_load_complete[s]);\n tma_load_tile_B(smem_B[s], params, s, &mbar_load_complete[s]);\n // NOTE: Do NOT manually arrive after TMA issue — the TMA\n // hardware signals the mbarrier upon transfer completion.\n }\n }\n\n // Steady state: load stage s while MMA processes stage s-NUM_STAGES\n for (int k = NUM_STAGES; k < num_k_tiles; k++) {\n int stage = k % NUM_STAGES;\n // Wait for MMA to finish with this buffer\n mbarrier_wait(&mbar_mma_complete[stage]);\n if (lane_id == 0) {\n uint32_t tx_bytes = TILE_A_BYTES + TILE_B_BYTES;\n mbarrier_arrive_expect_tx(&mbar_load_complete[stage], tx_bytes);\n tma_load_tile_A(smem_A[stage], params, k, &mbar_load_complete[stage]);\n tma_load_tile_B(smem_B[stage], params, k, &mbar_load_complete[stage]);\n }\n }\n\n } else if (warp_id == 1) {\n // ===== MMA CONSUMER =====\n for (int k = 0; k < num_k_tiles; k++) {\n int stage = k % NUM_STAGES;\n // Wait for TMA to fill this buffer\n mbarrier_wait(&mbar_load_complete[stage]);\n\n // Issue MMA on the filled buffer\n if (lane_id == 0) {\n tcgen05_mma(smem_A[stage], smem_B[stage]);\n }\n __syncwarp();\n\n // Signal that this buffer is free for reuse\n if (lane_id == 0) {\n mbarrier_arrive(&mbar_mma_complete[stage]);\n }\n }\n }\n // Epilogue warps omitted for clarity\n}\n```"}, "after": {"statement": "Each stage cycles through this ownership state machine:\n\n| empty | producer | the prior consumer has completed every read of the stage |\n| loading | TMA async proxy | expected transaction bytes are registered before the copy and the TMA transaction is outstanding |\n| ready | consumer | the full barrier's matching phase has completed |\n| consuming | MMA path | the consumer has acquired the stage and no producer may overwrite it |\n| empty again | producer | every asynchronous MMA use of the SMEM operands is complete and the consumer releases the matching empty phase |\n\nModulo indexing selects storage, but it does not track ownership by itself. A correct implementation also supplies barrier initialization/publication, expected arrival counts, phase or token state, async-proxy ordering, a bounded prologue, steady-state advancement, pipeline tail, and error-free behavior when K tiles are fewer than stages.\n\nOn SM100, do not release a stage merely after issuing `tcgen05.mma`. The MMA is asynchronous; use its defined completion path before allowing a producer to overwrite A/B storage. Likewise, `__syncthreads()` is not a substitute for the required tcgen05/TMA async-proxy completion and ordering operations."}, "reason": {"statement": "Replace unsafe code with a state-transition contract and pinned implementations.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"]}} +{"path": "wiki/techniques/pipeline-stages.md", "before": {"statement": "```cuda\n// 3-stage circular buffer with mbarrier synchronization\n// Stages: [0] loading, [1] ready for MMA, [2] being consumed by MMA\n#define NUM_STAGES 3\n\n__global__ void __launch_bounds__(512)\npipelined_gemm(const __grid_constant__ GemmParams params)\n{\n extern __shared__ char smem[];\n\n // Circular buffer layout in shared memory\n // Each stage has its own A and B tile buffers\n half* smem_A[NUM_STAGES];\n half* smem_B[NUM_STAGES];\n for (int s = 0; s < NUM_STAGES; s++) {\n smem_A[s] = reinterpret_cast(\n smem + s * (TILE_A_BYTES + TILE_B_BYTES));\n smem_B[s] = reinterpret_cast(\n smem + s * (TILE_A_BYTES + TILE_B_BYTES) + TILE_A_BYTES);\n }\n\n // mbarrier arrays for producer-consumer sync\n __shared__ uint64_t mbar_load_complete[NUM_STAGES];\n __shared__ uint64_t mbar_mma_complete[NUM_STAGES];\n\n int warp_id = threadIdx.x / 32;\n int lane_id = threadIdx.x % 32;\n\n // Initialize barriers\n if (threadIdx.x == 0) {\n for (int s = 0; s < NUM_STAGES; s++) {\n mbarrier_init(&mbar_load_complete[s], 1);\n mbarrier_init(&mbar_mma_complete[s], 1);\n }\n }\n __syncthreads();\n\n int num_k_tiles = params.K / TILE_K;\n\n if (warp_id == 0) {\n // ===== TMA PRODUCER =====\n // Prologue: fill the first NUM_STAGES buffers\n for (int s = 0; s < NUM_STAGES && s < num_k_tiles; s++) {\n if (lane_id == 0) {\n // Set expected TX bytes on mbarrier BEFORE issuing TMA.\n // TMA hardware will arrive on the mbarrier when transfer completes.\n uint32_t tx_bytes = TILE_A_BYTES + TILE_B_BYTES;\n mbarrier_arrive_expect_tx(&mbar_load_complete[s], tx_bytes);\n tma_load_tile_A(smem_A[s], params, s, &mbar_load_complete[s]);\n tma_load_tile_B(smem_B[s], params, s, &mbar_load_complete[s]);\n // NOTE: Do NOT manually arrive after TMA issue — the TMA\n // hardware signals the mbarrier upon transfer completion.\n }\n }\n\n // Steady state: load stage s while MMA processes stage s-NUM_STAGES\n for (int k = NUM_STAGES; k < num_k_tiles; k++) {\n int stage = k % NUM_STAGES;\n // Wait for MMA to finish with this buffer\n mbarrier_wait(&mbar_mma_complete[stage]);\n if (lane_id == 0) {\n uint32_t tx_bytes = TILE_A_BYTES + TILE_B_BYTES;\n mbarrier_arrive_expect_tx(&mbar_load_complete[stage], tx_bytes);\n tma_load_tile_A(smem_A[stage], params, k, &mbar_load_complete[stage]);\n tma_load_tile_B(smem_B[stage], params, k, &mbar_load_complete[stage]);\n }\n }\n\n } else if (warp_id == 1) {\n // ===== MMA CONSUMER =====\n for (int k = 0; k < num_k_tiles; k++) {\n int stage = k % NUM_STAGES;\n // Wait for TMA to fill this buffer\n mbarrier_wait(&mbar_load_complete[stage]);\n\n // Issue MMA on the filled buffer\n if (lane_id == 0) {\n tcgen05_mma(smem_A[stage], smem_B[stage]);\n }\n __syncwarp();\n\n // Signal that this buffer is free for reuse\n if (lane_id == 0) {\n mbarrier_arrive(&mbar_mma_complete[stage]);\n }\n }\n }\n // Epilogue warps omitted for clarity\n}\n```"}, "after": {"statement": "Each stage cycles through this ownership state machine:\n\n| empty | producer | the prior consumer has completed every read of the stage |\n| loading | TMA async proxy | expected transaction bytes are registered before the copy and the TMA transaction is outstanding |\n| ready | consumer | the full barrier's matching phase has completed |\n| consuming | MMA path | the consumer has acquired the stage and no producer may overwrite it |\n| empty again | producer | every asynchronous MMA use of the SMEM operands is complete and the consumer releases the matching empty phase |\n\nModulo indexing selects storage, but it does not track ownership by itself. A correct implementation also supplies barrier initialization/publication, expected arrival counts, phase or token state, async-proxy ordering, a bounded prologue, steady-state advancement, pipeline tail, and error-free behavior when K tiles are fewer than stages.\n\nOn SM100, do not release a stage merely after issuing `tcgen05.mma`. The MMA is asynchronous; use its defined completion path before allowing a producer to overwrite A/B storage. Likewise, `__syncthreads()` is not a substitute for the required tcgen05/TMA async-proxy completion and ordering operations."}, "reason": {"statement": "Replace unsafe code with a state-transition contract and pinned implementations.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"]}} {"path": "wiki/techniques/pipeline-stages.md", "before": {"statement": "| 2 | 2x base | Partial | Small tiles, limited SMEM |\n| 3 | 3x base | Full for most GEMMs | Standard choice on Blackwell |\n| 4-5 | 4-5x base | Full with margin | Large K, high memory latency |\n| >5 | Excessive | Diminishing returns | Rarely justified |"}, "after": {"statement": "For binary16 `A[128,64]` and `B[64,256]`, unpadded operand payload is:\n\n```text\nA = 128 × 64 × 2 bytes = 16 KiB\nB = 64 × 256 × 2 bytes = 32 KiB\none stage = 48 KiB\nthree stages = 144 KiB\nfive stages = 240 KiB\n```\n\nPayload grows linearly with stage count. Total allocation can additionally contain fixed and stage-dependent metadata, so derive the concrete shared-storage type rather than multiplying payload alone.\n\nThere is no architecture-wide table in which two stages are always partial, three always fully hide latency, or more than five are always excessive. Legal and useful depth depends on:\n\n- bytes per stage, alignment/swizzle padding, other SMEM, and occupancy;\n- TMA issue/transfer rate and descriptor/transaction shape;\n- MMA work and completion time per K tile;\n- producer/consumer role schedule and register pressure;\n- K-loop length, prologue/tail fraction, and output schedule.\n\nCompile every candidate, reject resource-invalid variants, then compare controlled timings and pipeline/barrier stalls. Include `num_k_tiles` values below, equal to, and above the stage count to exercise every boundary."}, "reason": {"statement": "Replace fixed labels with resource and measurement inputs.", "urls": []}} -{"path": "wiki/techniques/pipeline-stages.md", "before": {"statement": "```ptx\n// Phase-based mbarrier protocol for 3-stage pipeline\n//\n// Each mbarrier tracks a \"phase\" (0 or 1). The producer flips the phase\n// on arrive; the consumer waits for the expected phase.\n\n// Producer: arrive on stage %s (flips phase)\nmbarrier.arrive.shared.b64 %state, [%mbar_load + %s * 8];\n\n// Consumer: wait for phase %expected_phase on stage %s\n// try_wait is non-blocking; the warp can spin-wait or do other work\nWAIT_LOOP:\n mbarrier.try_wait.parity.shared.b64 %ready, [%mbar_load + %s * 8], %phase;\n @!%ready bra WAIT_LOOP;\n\n// TMA can also arrive directly on an mbarrier:\n// The TMA unit signals completion without CPU thread involvement\ncp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes\n [%smem_addr], [%tensor_map, {%coord0, %coord1}], [%mbar_load + %s * 8];\n```\n\nThe key advantage is that TMA can arrive on an mbarrier autonomously. The producer warp only needs to initiate the TMA; the TMA hardware signals completion directly, removing the producer from the critical path."}, "after": {"statement": "Each stage cycles through this ownership state machine:\n\n| empty | producer | the prior consumer has completed every read of the stage |\n| loading | TMA async proxy | expected transaction bytes are registered before the copy and the TMA transaction is outstanding |\n| ready | consumer | the full barrier's matching phase has completed |\n| consuming | MMA path | the consumer has acquired the stage and no producer may overwrite it |\n| empty again | producer | every asynchronous MMA use of the SMEM operands is complete and the consumer releases the matching empty phase |\n\nModulo indexing selects storage, but it does not track ownership by itself. A correct implementation also supplies barrier initialization/publication, expected arrival counts, phase or token state, async-proxy ordering, a bounded prologue, steady-state advancement, pipeline tail, and error-free behavior when K tiles are fewer than stages.\n\nOn SM100, do not release a stage merely after issuing `tcgen05.mma`. The MMA is asynchronous; use its defined completion path before allowing a producer to overwrite A/B storage. Likewise, `__syncthreads()` is not a substitute for the required tcgen05/TMA async-proxy completion and ordering operations.\n\nFor a TMA load into shared memory, the producer initializes/publishes the mbarrier, accounts for the copy's expected transaction bytes, and issues the tensor copy with that barrier. Hardware completes the transaction on the barrier; the consumer waits for the matching phase before reading the stage.\n\nThis autonomous completion avoids a producer instruction that manually announces data-ready after the transfer. It does not prove that producer issue work or the overall load path is absent from the measured critical path.\n\nUse the exact PTX ISA tensor-copy grammar or a version-pinned library pipeline. An illustrative `arrive`/spin loop that omits expected transaction bytes, copy descriptors, proxy fences, phase initialization, and the consumer-to-producer reuse edge is not a safe pipeline."}, "reason": {"statement": "Use exact lifecycle prose and distinguish autonomous completion from measured critical path.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"]}} +{"path": "wiki/techniques/pipeline-stages.md", "before": {"statement": "```ptx\n// Phase-based mbarrier protocol for 3-stage pipeline\n//\n// Each mbarrier tracks a \"phase\" (0 or 1). The producer flips the phase\n// on arrive; the consumer waits for the expected phase.\n\n// Producer: arrive on stage %s (flips phase)\nmbarrier.arrive.shared.b64 %state, [%mbar_load + %s * 8];\n\n// Consumer: wait for phase %expected_phase on stage %s\n// try_wait is non-blocking; the warp can spin-wait or do other work\nWAIT_LOOP:\n mbarrier.try_wait.parity.shared.b64 %ready, [%mbar_load + %s * 8], %phase;\n @!%ready bra WAIT_LOOP;\n\n// TMA can also arrive directly on an mbarrier:\n// The TMA unit signals completion without CPU thread involvement\ncp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes\n [%smem_addr], [%tensor_map, {%coord0, %coord1}], [%mbar_load + %s * 8];\n```\n\nThe key advantage is that TMA can arrive on an mbarrier autonomously. The producer warp only needs to initiate the TMA; the TMA hardware signals completion directly, removing the producer from the critical path."}, "after": {"statement": "Each stage cycles through this ownership state machine:\n\n| empty | producer | the prior consumer has completed every read of the stage |\n| loading | TMA async proxy | expected transaction bytes are registered before the copy and the TMA transaction is outstanding |\n| ready | consumer | the full barrier's matching phase has completed |\n| consuming | MMA path | the consumer has acquired the stage and no producer may overwrite it |\n| empty again | producer | every asynchronous MMA use of the SMEM operands is complete and the consumer releases the matching empty phase |\n\nModulo indexing selects storage, but it does not track ownership by itself. A correct implementation also supplies barrier initialization/publication, expected arrival counts, phase or token state, async-proxy ordering, a bounded prologue, steady-state advancement, pipeline tail, and error-free behavior when K tiles are fewer than stages.\n\nOn SM100, do not release a stage merely after issuing `tcgen05.mma`. The MMA is asynchronous; use its defined completion path before allowing a producer to overwrite A/B storage. Likewise, `__syncthreads()` is not a substitute for the required tcgen05/TMA async-proxy completion and ordering operations.\n\nFor a TMA load into shared memory, the producer initializes/publishes the mbarrier, accounts for the copy's expected transaction bytes, and issues the tensor copy with that barrier. Hardware completes the transaction on the barrier; the consumer waits for the matching phase before reading the stage.\n\nThis autonomous completion avoids a producer instruction that manually announces data-ready after the transfer. It does not prove that producer issue work or the overall load path is absent from the measured critical path.\n\nUse the exact PTX ISA tensor-copy grammar or a version-pinned library pipeline. An illustrative `arrive`/spin loop that omits expected transaction bytes, copy descriptors, proxy fences, phase initialization, and the consumer-to-producer reuse edge is not a safe pipeline."}, "reason": {"statement": "Use exact lifecycle prose and distinguish autonomous completion from measured critical path.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit"]}} {"path": "wiki/techniques/pipeline-stages.md", "before": {"statement": "The Modular blog series describes a 5-stage circular buffer reaching 85% of SOTA performance on Blackwell:\n\n```cuda\n// Modular-style 5-stage pipeline constants\n// Chosen to fully hide B200 HBM latency (~400 cycles)\n// while fitting within 228 KB SMEM budget\n\nconstexpr int NUM_STAGES = 5;\nconstexpr int TILE_M = 128;\nconstexpr int TILE_N = 128; // Smaller N to fit 5 stages\nconstexpr int TILE_K = 64;\n\n// Per stage: A (128*64*2=16KB) + B (64*128*2=16KB) = 32 KB\n// 5 stages: 160 KB + barriers + metadata < 228 KB\n\n// The pipeline timing diagram for steady state:\n//\n// Stage: 0 1 2 3 4\n// TMA: [load k5] [done] [done] [load k8] [load k9]\n// MMA: [done] [done] [comp k7] [done] [done]\n//\n// At any point: 1 stage being loaded, 1 being computed, 3 in transit or done\n```"}, "after": {"statement": "Compute capability 10.0 supports up to 228 KiB of shared memory per SM. Thus five stages of that exact payload already exceed the per-SM capacity, while three stages consume about 63.2% before barriers, padding, descriptors, epilogue buffers, and other shared storage. Per-block opt-in limits and occupancy constraints also apply."}, "reason": {"statement": "Keep five stages as one source configuration and scope the endpoint to the full sequence.", "urls": ["https://www.modular.com/blog/matrix-multiplication-on-nvidias-blackwell-part-3-the-optimizations-behind-85-of-sota-performance", "https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy"]}} {"path": "wiki/techniques/pipeline-stages.md", "before": {"statement": "- **All memory-bound and compute-bound GEMM kernels**: Pipelining is never harmful and always improves utilization by hiding latency.\n\n- **Combined with warp specialization**: Pipelining provides the buffer structure; warp specialization assigns the producer/consumer roles. The two techniques are complementary and almost always used together."}, "after": {"statement": "A multi-stage pipeline reserves disjoint shared-memory operand buffers so production of one tile can overlap consumption of another. Stage count controls how far producer and consumer progress may separate; it does not guarantee that latency is fully hidden or that the schedule is faster.\n\nThere is no architecture-wide table in which two stages are always partial, three always fully hide latency, or more than five are always excessive. Legal and useful depth depends on:\n\n- bytes per stage, alignment/swizzle padding, other SMEM, and occupancy;\n- TMA issue/transfer rate and descriptor/transaction shape;\n- MMA work and completion time per K tile;\n- producer/consumer role schedule and register pressure;\n- K-loop length, prologue/tail fraction, and output schedule.\n\nCompile every candidate, reject resource-invalid variants, then compare controlled timings and pipeline/barrier stalls. Include `num_k_tiles` values below, equal to, and above the stage count to exercise every boundary."}, "reason": {"statement": "Make stage depth and role specialization independent tuning dimensions.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-best-practices-guide/index.html#asynchronous-copy-from-global-memory-to-shared-memory"]}} -{"path": "wiki/techniques/pipeline-stages.md", "before": {"statement": "- Barrier initialization overhead is negligible but must happen before the first TMA. Place init in a `__syncthreads()` block at kernel start."}, "after": {"statement": "Each stage cycles through this ownership state machine:\n\n| empty | producer | the prior consumer has completed every read of the stage |\n| loading | TMA async proxy | expected transaction bytes are registered before the copy and the TMA transaction is outstanding |\n| ready | consumer | the full barrier's matching phase has completed |\n| consuming | MMA path | the consumer has acquired the stage and no producer may overwrite it |\n| empty again | producer | every asynchronous MMA use of the SMEM operands is complete and the consumer releases the matching empty phase |\n\nModulo indexing selects storage, but it does not track ownership by itself. A correct implementation also supplies barrier initialization/publication, expected arrival counts, phase or token state, async-proxy ordering, a bounded prologue, steady-state advancement, pipeline tail, and error-free behavior when K tiles are fewer than stages.\n\nOn SM100, do not release a stage merely after issuing `tcgen05.mma`. The MMA is asynchronous; use its defined completion path before allowing a producer to overwrite A/B storage. Likewise, `__syncthreads()` is not a substitute for the required tcgen05/TMA async-proxy completion and ordering operations.\n\nThere is no architecture-wide table in which two stages are always partial, three always fully hide latency, or more than five are always excessive. Legal and useful depth depends on:\n\n- bytes per stage, alignment/swizzle padding, other SMEM, and occupancy;\n- TMA issue/transfer rate and descriptor/transaction shape;\n- MMA work and completion time per K tile;\n- producer/consumer role schedule and register pressure;\n- K-loop length, prologue/tail fraction, and output schedule.\n\nCompile every candidate, reject resource-invalid variants, then compare controlled timings and pipeline/barrier stalls. Include `num_k_tiles` values below, equal to, and above the stage count to exercise every boundary."}, "reason": {"statement": "State the ordering proof rather than a fixed initialization recipe/cost.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-init", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit"]}} +{"path": "wiki/techniques/pipeline-stages.md", "before": {"statement": "- Barrier initialization overhead is negligible but must happen before the first TMA. Place init in a `__syncthreads()` block at kernel start."}, "after": {"statement": "Each stage cycles through this ownership state machine:\n\n| empty | producer | the prior consumer has completed every read of the stage |\n| loading | TMA async proxy | expected transaction bytes are registered before the copy and the TMA transaction is outstanding |\n| ready | consumer | the full barrier's matching phase has completed |\n| consuming | MMA path | the consumer has acquired the stage and no producer may overwrite it |\n| empty again | producer | every asynchronous MMA use of the SMEM operands is complete and the consumer releases the matching empty phase |\n\nModulo indexing selects storage, but it does not track ownership by itself. A correct implementation also supplies barrier initialization/publication, expected arrival counts, phase or token state, async-proxy ordering, a bounded prologue, steady-state advancement, pipeline tail, and error-free behavior when K tiles are fewer than stages.\n\nOn SM100, do not release a stage merely after issuing `tcgen05.mma`. The MMA is asynchronous; use its defined completion path before allowing a producer to overwrite A/B storage. Likewise, `__syncthreads()` is not a substitute for the required tcgen05/TMA async-proxy completion and ordering operations.\n\nThere is no architecture-wide table in which two stages are always partial, three always fully hide latency, or more than five are always excessive. Legal and useful depth depends on:\n\n- bytes per stage, alignment/swizzle padding, other SMEM, and occupancy;\n- TMA issue/transfer rate and descriptor/transaction shape;\n- MMA work and completion time per K tile;\n- producer/consumer role schedule and register pressure;\n- K-loop length, prologue/tail fraction, and output schedule.\n\nCompile every candidate, reject resource-invalid variants, then compare controlled timings and pipeline/barrier stalls. Include `num_k_tiles` values below, equal to, and above the stage count to exercise every boundary."}, "reason": {"statement": "State the ordering proof rather than a fixed initialization recipe/cost.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-init", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit"]}} {"path": "wiki/techniques/tile-scheduling.md", "before": {"statement": "Tile scheduling determines the order in which output tiles of a GEMM (or attention) kernel are assigned to CTAs. The scheduling order affects L2 cache hit rates, tail-effect severity, and overall GPU utilization. On Blackwell, the CLC hardware unit supports dynamic scheduling policies including swizzled raster, while Hopper relies on software-based static stride or swizzled patterns computed at launch time."}, "after": {"statement": "“Tile scheduling” can refer to four different choices:\n\n1. **Coordinate order:** a software mapping from a logical work index to `(tile_m, tile_n, ...)`, such as row-major, column-major, or a blocked/swizzled raster.\n2. **Resident-worker iteration:** whether a CTA handles one logical tile or repeatedly advances through work, for example by a static grid stride.\n3. **Work reassignment:** on SM100, Cluster Launch Control (CLC) lets a running worker cancel another grid entity that has not started and process the returned ClcID.\n4. **Problem decomposition:** Stream-K or Split-K can partition a tile's K work and then reduce partial results.\n\nCLC does not accept a raster-order or swizzle-policy operand. CUTLASS applies those coordinate transforms in software to initial or returned coordinates. CLC also does not itself create K partitions or synthesize work beyond the launched grid."}, "reason": {"statement": "Separate tile-coordinate order from work acquisition.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} {"path": "wiki/techniques/tile-scheduling.md", "before": {"statement": "Tiles are assigned in row-major order. Simple but poor L2 locality: consecutive tiles share no B-matrix data until the entire M-dimension is traversed.\n\n```cuda\n// Linear raster: tile_idx maps directly to (tile_m, tile_n)\n__device__ void linear_raster(int tile_idx, int tiles_n,\n int& tile_m, int& tile_n) {\n tile_m = tile_idx / tiles_n;\n tile_n = tile_idx % tiles_n;\n}\n\n// Access pattern for a 4x4 tile grid:\n// 0 1 2 3\n// 4 5 6 7\n// 8 9 10 11\n// 12 13 14 15\n//\n// Problem: tiles 0,1,2,3 all load different B columns.\n// By tile 4, B column 0 has been evicted from L2.\n```"}, "after": {"statement": "A flat row-major mapping is:\n\n```cuda\n__device__ void row_major_tile(int tile_idx, int tiles_n,\n int& tile_m, int& tile_n) {\n tile_m = tile_idx / tiles_n;\n tile_n = tile_idx % tiles_n;\n}\n```\n\nFor a positive grid size, a static persistent worker can cover the flat index set without overlap:\n\n```cuda\nfor (int tile_idx = int(blockIdx.x);\n tile_idx < total_tiles;\n tile_idx += int(gridDim.x)) {\n int tile_m = tile_idx / tiles_n;\n int tile_n = tile_idx % tiles_n;\n compute_tile(tile_m, tile_n);\n}\n```\n\nChanging coordinate order changes the sequence of operand panels presented to the cache. It does not by itself prove a hit rate or speedup. A valid transform must first be shown to be in-bounds and bijective for edge groups; then its locality must be evaluated for the actual tile shape, batch/group structure, concurrent traffic, and cache capacity.\n\nFor a deliberately simplified model with `T` equal-duration independent tiles, `W` available one-tile workers, and no K decomposition, write:\n\n```text\nT = qW + r, 0 <= r < W\n```\n\nIf `r > 0`, the final partial wave has `r` active workers and occupancy `r / W`. If `r == 0` and `T > 0`, the final wave is full; reporting zero occupancy from the remainder alone is an error.\n\nFor `T = 150` and `W = 142`, the second wave contains 8 tiles, so 8 workers are active, 134 are idle, and wave occupancy is `8 / 142 = 5.63%` under those assumptions. CLC may change which workers receive those eight IDs and reduce delays caused by uneven worker availability, but it cannot turn eight independent IDs into 142. Stream-K may create additional K partitions, but its selected decomposition and reduction overhead determine whether that is profitable; near-100-percent occupancy is not guaranteed."}, "reason": {"statement": "Retain exact mapping and make cache effects empirical.", "urls": []}} {"path": "wiki/techniques/tile-scheduling.md", "before": {"statement": "Tiles are assigned in a blocked pattern that groups nearby M and N tiles together, maximizing reuse of both A rows and B columns in L2 cache:\n\n```cuda\n// Swizzled raster: group tiles into blocks that share A and B data\n// swizzle_size controls the block width (typically 4-8)\n__device__ void swizzled_raster(int tile_idx, int tiles_m, int tiles_n,\n int swizzle_size, int& tile_m, int& tile_n)\n{\n // Number of tile columns per swizzle group\n int group_cols = min(swizzle_size, tiles_n);\n int tiles_per_group = tiles_m * group_cols;\n\n // Which swizzle group\n int group_idx = tile_idx / tiles_per_group;\n int within_group = tile_idx % tiles_per_group;\n\n // Within the group, iterate in column-major order\n tile_m = within_group / group_cols;\n tile_n = group_idx * group_cols + within_group % group_cols;\n}\n\n// Access pattern with swizzle_size=2 on a 4x4 grid:\n// 0 1 | 8 9\n// 2 3 | 10 11\n// 4 5 | 12 13\n// 6 7 | 14 15\n//\n// Tiles 0,1,2,3 share the same B columns (0,1).\n// Tiles 0,2,4,6 share the same A rows.\n// Much better L2 reuse.\n```"}, "after": {"statement": "A flat row-major mapping is:\n\n```cuda\n__device__ void row_major_tile(int tile_idx, int tiles_n,\n int& tile_m, int& tile_n) {\n tile_m = tile_idx / tiles_n;\n tile_n = tile_idx % tiles_n;\n}\n```\n\nFor a positive grid size, a static persistent worker can cover the flat index set without overlap:\n\n```cuda\nfor (int tile_idx = int(blockIdx.x);\n tile_idx < total_tiles;\n tile_idx += int(gridDim.x)) {\n int tile_m = tile_idx / tiles_n;\n int tile_n = tile_idx % tiles_n;\n compute_tile(tile_m, tile_n);\n}\n```\n\nChanging coordinate order changes the sequence of operand panels presented to the cache. It does not by itself prove a hit rate or speedup. A valid transform must first be shown to be in-bounds and bijective for edge groups; then its locality must be evaluated for the actual tile shape, batch/group structure, concurrent traffic, and cache capacity.\n\nFor a deliberately simplified model with `T` equal-duration independent tiles, `W` available one-tile workers, and no K decomposition, write:\n\n```text\nT = qW + r, 0 <= r < W\n```\n\nIf `r > 0`, the final partial wave has `r` active workers and occupancy `r / W`. If `r == 0` and `T > 0`, the final wave is full; reporting zero occupancy from the remainder alone is an error.\n\nFor `T = 150` and `W = 142`, the second wave contains 8 tiles, so 8 workers are active, 134 are idle, and wave occupancy is `8 / 142 = 5.63%` under those assumptions. CLC may change which workers receive those eight IDs and reduce delays caused by uneven worker availability, but it cannot turn eight independent IDs into 142. Stream-K may create additional K partitions, but its selected decomposition and reduction overhead determine whether that is profitable; near-100-percent occupancy is not guaranteed."}, "reason": {"statement": "Remove incorrect executable-looking code and unsupported maximality.", "urls": []}} -{"path": "wiki/techniques/tile-scheduling.md", "before": {"statement": "The CLC hardware scheduler assigns tiles at runtime, combining the benefits of dynamic load balancing with configurable scheduling policies:\n\n```cuda\n// CLC-based scheduling on Blackwell\n// The scheduling policy is set once during CLC initialization\nenum class ClcSchedulePolicy {\n LinearRaster, // Simple row-major order\n SwizzledRaster, // Blocked pattern for L2 locality\n ColumnFirst, // Column-major for specific workloads\n Hilbert // Space-filling curve (experimental)\n};\n\n__device__ void clc_init_scheduler(\n void* clc_buffer,\n int tiles_m, int tiles_n,\n ClcSchedulePolicy policy)\n{\n if (threadIdx.x == 0) {\n // Configure the software scheduler metadata used around CLC queries.\n // The CLC PTX surface is try_cancel/query_cancel, not clusterctl.init.\n uint32_t config = encode_clc_config(tiles_m, tiles_n, policy);\n init_clc_scheduler_metadata(clc_buffer, tiles_m, tiles_n, config);\n }\n __syncwarp();\n}\n```"}, "after": {"statement": "CUTLASS 4.5.0 documents this lifecycle for a CLC-backed persistent scheduler:\n\n1. Launch the full problem grid and process the worker's initial `blockIdx` coordinate.\n2. Submit `clusterlaunchcontrol.try_cancel` against another not-yet-started grid entity. The asynchronous 16-byte response completes a transaction on an mbarrier.\n3. After the transaction completes, query whether cancellation succeeded. On success, decode and process the returned ClcID; for clusters, combine the returned first coordinate with the local cluster rank.\n4. Treat an observed failed request as terminal for requests by that thread. Retrying from the same thread after failure is undefined.\n\nThis can redistribute existing work when SM availability is uneven. It changes which resident worker handles an unlaunched ID, not the number of independent IDs in the grid. Software raster/swizzle may be applied consistently to both initial and returned coordinates."}, "reason": {"statement": "Replace the invented API with the documented work lifecycle.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/techniques/tile-scheduling.md", "before": {"statement": "The CLC hardware scheduler assigns tiles at runtime, combining the benefits of dynamic load balancing with configurable scheduling policies:\n\n```cuda\n// CLC-based scheduling on Blackwell\n// The scheduling policy is set once during CLC initialization\nenum class ClcSchedulePolicy {\n LinearRaster, // Simple row-major order\n SwizzledRaster, // Blocked pattern for L2 locality\n ColumnFirst, // Column-major for specific workloads\n Hilbert // Space-filling curve (experimental)\n};\n\n__device__ void clc_init_scheduler(\n void* clc_buffer,\n int tiles_m, int tiles_n,\n ClcSchedulePolicy policy)\n{\n if (threadIdx.x == 0) {\n // Configure the software scheduler metadata used around CLC queries.\n // The CLC PTX surface is try_cancel/query_cancel, not clusterctl.init.\n uint32_t config = encode_clc_config(tiles_m, tiles_n, policy);\n init_clc_scheduler_metadata(clc_buffer, tiles_m, tiles_n, config);\n }\n __syncwarp();\n}\n```"}, "after": {"statement": "CUTLASS 4.5.0 documents this lifecycle for a CLC-backed persistent scheduler:\n\n1. Launch the full problem grid and process the worker's initial `blockIdx` coordinate.\n2. Submit `clusterlaunchcontrol.try_cancel` against another not-yet-started grid entity. The asynchronous 16-byte response completes a transaction on an mbarrier.\n3. After the transaction completes, query whether cancellation succeeded. On success, decode and process the returned ClcID; for clusters, combine the returned first coordinate with the local cluster rank.\n4. Treat an observed failed request as terminal for requests by that thread. Retrying from the same thread after failure is undefined.\n\nThis can redistribute existing work when SM availability is uneven. It changes which resident worker handles an unlaunched ID, not the number of independent IDs in the grid. Software raster/swizzle may be applied consistently to both initial and returned coordinates."}, "reason": {"statement": "Replace the invented API with the documented work lifecycle.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} {"path": "wiki/techniques/tile-scheduling.md", "before": {"statement": "```cuda\n// CUTLASS tile scheduler selection for SM100\n// All persistent schedulers inherit from PersistentTileSchedulerSm100\n\n// 1. Default CLC scheduler with swizzled raster\nusing Scheduler_Default = cutlass::gemm::PersistentTileSchedulerSm100;\n\n// 2. Stream-K scheduler for better tail handling\n// Splits K-dimension across CTAs for the last wave\nusing Scheduler_StreamK = cutlass::gemm::StreamKSchedulerSm100;\n\n// 3. Grouped GEMM scheduler for MoE workloads\n// Each group has different M, shared N and K\nusing Scheduler_Grouped = cutlass::gemm::GroupedTileSchedulerSm100;\n\n// Usage in CUTLASS kernel definition:\nusing GemmKernel = cutlass::gemm::kernel::GemmUniversal<\n cute::Shape, Int<256>, Int<64>>, // Tile shape\n ElementA, LayoutA,\n ElementB, LayoutB,\n ElementC, LayoutC,\n TiledMma,\n CollectiveMainloop,\n CollectiveEpilogue,\n Scheduler_Default // Tile scheduler\n>;\n```"}, "after": {"statement": "The public scheduler tags select implementation classes; users should prefer those tags over spelling detail types directly.\n\n| `PersistentScheduler` (also the default `void` tag) | `PersistentTileSchedulerSm100` | CLC-backed persistent data-parallel scheduling |\n| `DynamicPersistentScheduler` | `PersistentTileSchedulerSm100` | Explicit dynamic route to the same SM100 implementation |\n| `StaticPersistentScheduler` | `StaticPersistentTileScheduler100` | Static persistent scheduling |\n| `StreamKScheduler` | `PersistentTileSchedulerSm100StreamK` | Parameterized data-parallel, Stream-K, or Split-K decomposition |\n| `GroupScheduler` | `PersistentTileSchedulerSm100Group` | Grouped-problem route; in this tag it wraps the SM90-style static group scheduler |\n\nThe SM100 Stream-K route accepts decomposition, split, raster, swizzle, and reduction settings. CUTLASS 4.5.0 has a deterministic lock/turnstile reduction and a nondeterministic atomic-workspace reduction; atomic accumulation is therefore not a universal property of every Stream-K configuration."}, "reason": {"statement": "Name exact public tags and keep implementation classes clearly internal.", "urls": []}} {"path": "wiki/techniques/tile-scheduling.md", "before": {"statement": "The choice of scheduling strategy directly impacts L2 cache hit rates. On B200 with 126 MB L2:\n\n```python\n# L2 cache reuse analysis for different schedulers\n# Problem: M=8192, N=8192, K=4096, BF16\n# Tile: 128x256, giving 64x32 = 2048 tiles\n# B200: 142 SMs, 126 MB L2\n\ntile_bytes_A = 128 * 4096 * 2 # 1 MB per tile row of A\ntile_bytes_B = 4096 * 256 * 2 # 2 MB per tile column of B\n\n# Linear raster: first wave loads 142 tiles across 142/32 = 4.4 column groups\n# B data for 5 different column groups = 5 * 2 MB = 10 MB (fits in L2)\n# But A data for 142/32 = 4.4 row groups * 4.4 col groups = ~20 distinct A rows\n# 20 * 1 MB = 20 MB -> some L2 eviction\n\n# Swizzled raster (swizzle=4): first wave covers 142 tiles in ~36 groups of 4\n# Each group uses 1 A row + 4 B columns = 1 + 8 = 9 MB per group\n# But groups share A rows: total unique A = ~36 rows * 1 MB = 36 MB\n# L2 pressure: 36 MB + 8 MB = 44 MB (fits in B200's 126 MB L2)\n\n# Conclusion: swizzled raster reduces L2 misses by ~2x vs linear for large problems\n```"}, "after": {"statement": "For a deliberately simplified model with `T` equal-duration independent tiles, `W` available one-tile workers, and no K decomposition, write:\n\n```text\nT = qW + r, 0 <= r < W\n```\n\nIf `r > 0`, the final partial wave has `r` active workers and occupancy `r / W`. If `r == 0` and `T > 0`, the final wave is full; reporting zero occupancy from the remainder alone is an error.\n\nFor `T = 150` and `W = 142`, the second wave contains 8 tiles, so 8 workers are active, 134 are idle, and wave occupancy is `8 / 142 = 5.63%` under those assumptions. CLC may change which workers receive those eight IDs and reduce delays caused by uneven worker availability, but it cannot turn eight independent IDs into 142. Stream-K may create additional K partitions, but its selected decomposition and reduction overhead determine whether that is profitable; near-100-percent occupancy is not guaranteed."}, "reason": {"statement": "Replace fabricated cache results with a measurement protocol.", "urls": []}} {"path": "wiki/techniques/tile-scheduling.md", "before": {"statement": "| Linear raster | None | `(total_tiles % num_SMs) / num_SMs` |\n| Static stride | None | Same as linear |\n| CLC dynamic | Automatic | Fast CTAs steal from slow ones |\n| Stream-K | K-splitting | Near 100% (splits partial tiles across SMs) |"}, "after": {"statement": "For `T = 150` and `W = 142`, the second wave contains 8 tiles, so 8 workers are active, 134 are idle, and wave occupancy is `8 / 142 = 5.63%` under those assumptions. CLC may change which workers receive those eight IDs and reduce delays caused by uneven worker availability, but it cannot turn eight independent IDs into 142. Stream-K may create additional K partitions, but its selected decomposition and reduction overhead determine whether that is profitable; near-100-percent occupancy is not guaranteed."}, "reason": {"statement": "Give exact scoped arithmetic and list what each mechanism can change.", "urls": []}} @@ -325,12 +325,12 @@ {"path": "wiki/techniques/tile-scheduling.md", "before": {"statement": "- **Swizzled raster**: Default choice for large GEMMs. Always better than linear for L2 locality.\n- **CLC dynamic**: Recommended on Blackwell for all persistent kernels. Combines dynamic load balancing with swizzled ordering.\n- **Stream-K**: Best for small-to-medium problems where the tail effect dominates. Adds complexity for K-dimension synchronization.\n- **Grouped scheduler**: Essential for MoE and batched GEMM where problem sizes vary across groups."}, "after": {"statement": "1. Establish a correct unswizzled data-parallel mapping and verify exact tile coverage, including nondivisible edge groups.\n2. Sweep legal raster orders and swizzle sizes. Record kernel time plus L2 hit/sector traffic; do not infer eviction from coordinate order alone.\n3. Compare static and CLC-backed persistence at identical tile, cluster, grid, and occupancy settings. Record successful/failed CLC requests and per-worker tile counts where instrumentation permits.\n4. Test Stream-K separately with explicit decomposition, split, and reduction modes. Include workspace traffic, synchronization, determinism, and numerical tolerance.\n5. Repeat across representative shapes and grouped-size distributions. Report regressions as well as wins; there is no universal best scheduler for all GEMMs, attention kernels, or MoE workloads.\n\nWithout target-GPU measurements, this page makes no fixed claim for CLC acquisition latency, L2-miss reduction, or scheduler speedup."}, "reason": {"statement": "Turn prescriptions into testable selection hypotheses.", "urls": []}} {"path": "wiki/techniques/tile-scheduling.md", "before": {"statement": "- Swizzle size must be tuned per problem shape. Too large a swizzle group exceeds L2 capacity; too small loses the locality benefit."}, "after": {"statement": "For a deliberately simplified model with `T` equal-duration independent tiles, `W` available one-tile workers, and no K decomposition, write:\n\n```text\nT = qW + r, 0 <= r < W\n```\n\nIf `r > 0`, the final partial wave has `r` active workers and occupancy `r / W`. If `r == 0` and `T > 0`, the final wave is full; reporting zero occupancy from the remainder alone is an error.\n\nFor `T = 150` and `W = 142`, the second wave contains 8 tiles, so 8 workers are active, 134 are idle, and wave occupancy is `8 / 142 = 5.63%` under those assumptions. CLC may change which workers receive those eight IDs and reduce delays caused by uneven worker availability, but it cannot turn eight independent IDs into 142. Stream-K may create additional K partitions, but its selected decomposition and reduction overhead determine whether that is profitable; near-100-percent occupancy is not guaranteed.\n\n1. Establish a correct unswizzled data-parallel mapping and verify exact tile coverage, including nondivisible edge groups.\n2. Sweep legal raster orders and swizzle sizes. Record kernel time plus L2 hit/sector traffic; do not infer eviction from coordinate order alone.\n3. Compare static and CLC-backed persistence at identical tile, cluster, grid, and occupancy settings. Record successful/failed CLC requests and per-worker tile counts where instrumentation permits.\n4. Test Stream-K separately with explicit decomposition, split, and reduction modes. Include workspace traffic, synchronization, determinism, and numerical tolerance.\n5. Repeat across representative shapes and grouped-size distributions. Report regressions as well as wins; there is no universal best scheduler for all GEMMs, attention kernels, or MoE workloads.\n\nWithout target-GPU measurements, this page makes no fixed claim for CLC acquisition latency, L2-miss reduction, or scheduler speedup."}, "reason": {"statement": "Retain tuning advice without a false monotonic cache model.", "urls": []}} {"path": "wiki/techniques/tile-scheduling.md", "before": {"statement": "- CLC scheduling adds a small latency per tile acquisition (~10s of cycles). For extremely small tiles, this overhead is proportionally larger.\n- Stream-K requires atomic accumulation where K-splits meet, adding synchronization overhead. Only worthwhile when tail utilization is a proven bottleneck."}, "after": {"statement": "The public scheduler tags select implementation classes; users should prefer those tags over spelling detail types directly.\n\n| `PersistentScheduler` (also the default `void` tag) | `PersistentTileSchedulerSm100` | CLC-backed persistent data-parallel scheduling |\n| `DynamicPersistentScheduler` | `PersistentTileSchedulerSm100` | Explicit dynamic route to the same SM100 implementation |\n| `StaticPersistentScheduler` | `StaticPersistentTileScheduler100` | Static persistent scheduling |\n| `StreamKScheduler` | `PersistentTileSchedulerSm100StreamK` | Parameterized data-parallel, Stream-K, or Split-K decomposition |\n| `GroupScheduler` | `PersistentTileSchedulerSm100Group` | Grouped-problem route; in this tag it wraps the SM90-style static group scheduler |\n\nThe SM100 Stream-K route accepts decomposition, split, raster, swizzle, and reduction settings. CUTLASS 4.5.0 has a deterministic lock/turnstile reduction and a nondeterministic atomic-workspace reduction; atomic accumulation is therefore not a universal property of every Stream-K configuration.\n\n1. Establish a correct unswizzled data-parallel mapping and verify exact tile coverage, including nondivisible edge groups.\n2. Sweep legal raster orders and swizzle sizes. Record kernel time plus L2 hit/sector traffic; do not infer eviction from coordinate order alone.\n3. Compare static and CLC-backed persistence at identical tile, cluster, grid, and occupancy settings. Record successful/failed CLC requests and per-worker tile counts where instrumentation permits.\n4. Test Stream-K separately with explicit decomposition, split, and reduction modes. Include workspace traffic, synchronization, determinism, and numerical tolerance.\n5. Repeat across representative shapes and grouped-size distributions. Report regressions as well as wins; there is no universal best scheduler for all GEMMs, attention kernels, or MoE workloads.\n\nWithout target-GPU measurements, this page makes no fixed claim for CLC acquisition latency, L2-miss reduction, or scheduler speedup."}, "reason": {"statement": "State observable costs and configurable reductions rather than invented constants.", "urls": []}} -{"path": "wiki/techniques/warp-specialization.md", "before": {"statement": "Warp specialization assigns distinct functional roles to warps within a CTA, allowing each warp to focus on a single pipeline stage (data loading, MMA computation, or epilogue writeback). On Blackwell (SM100), the 16-warp CTA structure replaces Hopper's 4-warp warpgroup model. Because tcgen05.mma is a single-thread instruction that operates on TMEM rather than registers, only one warp needs to issue MMA operations, freeing the remaining warps for producer and consumer roles."}, "after": {"statement": "Warp specialization assigns different long-lived functions to disjoint warps in a CTA—for example scheduler, TMA load, MMA control, softmax/correction, or epilogue work. It is a software organization for overlapping pipeline stages, not a fixed SM100 CTA layout.\n\nThe architecture-level comparison is narrower:\n\n| Issue granularity | Warpgroup collective | One thread for `cta_group::1` or `cta_group::2` |\n| D accumulator | Per-thread registers | TMEM |\n| A/B movement | Explicit register/SMEM operands and producer work | Explicit SMEM descriptors or TMEM addresses and producer work |\n| Completion | WGMMA commit/wait groups | `tcgen05.commit` tied to an mbarrier, followed by a wait |\n\nSingle-thread issue reduces the number of threads needed to submit MMA instructions. It does not make operands move automatically, turn an entire warp into an ISA-level role, complete MMA synchronously, or choose how many epilogue warps a kernel should launch."}, "reason": {"statement": "Separate single-thread issue from software CTA/role sizing.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/techniques/warp-specialization.md", "before": {"statement": "Warp specialization assigns distinct functional roles to warps within a CTA, allowing each warp to focus on a single pipeline stage (data loading, MMA computation, or epilogue writeback). On Blackwell (SM100), the 16-warp CTA structure replaces Hopper's 4-warp warpgroup model. Because tcgen05.mma is a single-thread instruction that operates on TMEM rather than registers, only one warp needs to issue MMA operations, freeing the remaining warps for producer and consumer roles."}, "after": {"statement": "Warp specialization assigns different long-lived functions to disjoint warps in a CTA—for example scheduler, TMA load, MMA control, softmax/correction, or epilogue work. It is a software organization for overlapping pipeline stages, not a fixed SM100 CTA layout.\n\nThe architecture-level comparison is narrower:\n\n| Issue granularity | Warpgroup collective | One thread for `cta_group::1` or `cta_group::2` |\n| D accumulator | Per-thread registers | TMEM |\n| A/B movement | Explicit register/SMEM operands and producer work | Explicit SMEM descriptors or TMEM addresses and producer work |\n| Completion | WGMMA commit/wait groups | `tcgen05.commit` tied to an mbarrier, followed by a wait |\n\nSingle-thread issue reduces the number of threads needed to submit MMA instructions. It does not make operands move automatically, turn an entire warp into an ISA-level role, complete MMA synchronously, or choose how many epilogue warps a kernel should launch."}, "reason": {"statement": "Separate single-thread issue from software CTA/role sizing.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-operation-wgmma-mma-async", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma"]}} {"path": "wiki/techniques/warp-specialization.md", "before": {"statement": "The canonical Blackwell GEMM kernel uses 16 warps (512 threads) per CTA with the following role assignment:\n\n| 0 | TMA Producer | Issues TMA bulk-copy from global to shared memory, signals mbarrier |\n| 1 | MMA Consumer | Issues tcgen05.mma on SMEM operands, writes results to TMEM |\n| 2-15 | Epilogue | Reads TMEM accumulator, applies scale/bias/activation, writes to global memory |"}, "after": {"statement": "There is no universal “warp 0 load, warp 1 MMA, warps 2–15 epilogue” rule. Three primary implementations illustrate the range:\n\n| Gau Nernst tutorial v4, commit `3b90ac9b...` | 4 warps. An elected lane in warp 0 runs the TMA loop; an elected lane in warp 1 runs the MMA loop; after completion, all four warps participate in TMEM load/conversion/output. |\n| CUTLASS 4.5.0 generic SM100 GEMM | One warp each for `MMA`, `Sched`, `MainloopLoad`, and `EpilogueLoad`, followed by `CollectiveEpilogue::ThreadCount / 32` epilogue warps. Optional work can make some control roles nonparticipants. |\n| FlashInfer PR 1039 context FMHA | 16 warps: 0–3 `Softmax0`, 4–7 `Softmax1`, 8–11 correction, 12 MMA, 13 load, 14 epilogue, and 15 empty. The kernel has explicit pipelines between these roles. |\n\nFlashAttention-4's pinned SM100 forward source likewise starts from a 16-warp layout with two four-warp softmax groups, four correction warps, and dedicated MMA/load/epilogue IDs, then adjusts roles for configuration choices such as one Q stage, non-TMA paths, and dynamic persistence. These are concrete attention schedules, not a GEMM-wide architectural template."}, "reason": {"statement": "Replace one invented canonical layout with version-pinned examples.", "urls": []}} -{"path": "wiki/techniques/warp-specialization.md", "before": {"statement": "This contrasts with Hopper where a warpgroup (4 warps, 128 threads) collectively issues wgmma.mma_async, and all threads in the warpgroup participate in the MMA. On Blackwell, the MMA warp dispatches the instruction from a single thread while the hardware handles the data movement internally."}, "after": {"statement": "Warp specialization assigns different long-lived functions to disjoint warps in a CTA—for example scheduler, TMA load, MMA control, softmax/correction, or epilogue work. It is a software organization for overlapping pipeline stages, not a fixed SM100 CTA layout.\n\nThe architecture-level comparison is narrower:\n\n| Issue granularity | Warpgroup collective | One thread for `cta_group::1` or `cta_group::2` |\n| D accumulator | Per-thread registers | TMEM |\n| A/B movement | Explicit register/SMEM operands and producer work | Explicit SMEM descriptors or TMEM addresses and producer work |\n| Completion | WGMMA commit/wait groups | `tcgen05.commit` tied to an mbarrier, followed by a wait |\n\nSingle-thread issue reduces the number of threads needed to submit MMA instructions. It does not make operands move automatically, turn an entire warp into an ISA-level role, complete MMA synchronously, or choose how many epilogue warps a kernel should launch."}, "reason": {"statement": "Keep the precise Hopper/SM100 issue distinction without implying autonomous operand staging.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma"]}} -{"path": "wiki/techniques/warp-specialization.md", "before": {"statement": "| MMA granularity | 4-warp warpgroup (128 threads) | Single thread in 1 warp |\n| MMA output destination | Registers (shared across warpgroup) | TMEM (256KB, CTA-visible) |\n| Producer warps | Separate warp(s) for TMA loads | Warp 0 dedicated to TMA |\n| Epilogue execution | Same warpgroup or separate warps | 14 dedicated warps (2-15) |\n| Synchronization | warpgroup barriers, arrive/wait | mbarrier pairs (producer/consumer) |\n| Register pressure | High (accumulators in registers) | Low (accumulators in TMEM) |"}, "after": {"statement": "Warp specialization assigns different long-lived functions to disjoint warps in a CTA—for example scheduler, TMA load, MMA control, softmax/correction, or epilogue work. It is a software organization for overlapping pipeline stages, not a fixed SM100 CTA layout.\n\nThe architecture-level comparison is narrower:\n\n| Issue granularity | Warpgroup collective | One thread for `cta_group::1` or `cta_group::2` |\n| D accumulator | Per-thread registers | TMEM |\n| A/B movement | Explicit register/SMEM operands and producer work | Explicit SMEM descriptors or TMEM addresses and producer work |\n| Completion | WGMMA commit/wait groups | `tcgen05.commit` tied to an mbarrier, followed by a wait |\n\nSingle-thread issue reduces the number of threads needed to submit MMA instructions. It does not make operands move automatically, turn an entire warp into an ISA-level role, complete MMA synchronously, or choose how many epilogue warps a kernel should launch.\n\nThere is no universal “warp 0 load, warp 1 MMA, warps 2–15 epilogue” rule. Three primary implementations illustrate the range:\n\n| Gau Nernst tutorial v4, commit `3b90ac9b...` | 4 warps. An elected lane in warp 0 runs the TMA loop; an elected lane in warp 1 runs the MMA loop; after completion, all four warps participate in TMEM load/conversion/output. |\n| CUTLASS 4.5.0 generic SM100 GEMM | One warp each for `MMA`, `Sched`, `MainloopLoad`, and `EpilogueLoad`, followed by `CollectiveEpilogue::ThreadCount / 32` epilogue warps. Optional work can make some control roles nonparticipants. |\n| FlashInfer PR 1039 context FMHA | 16 warps: 0–3 `Softmax0`, 4–7 `Softmax1`, 8–11 correction, 12 MMA, 13 load, 14 epilogue, and 15 empty. The kernel has explicit pipelines between these roles. |\n\nFlashAttention-4's pinned SM100 forward source likewise starts from a 16-warp layout with two four-warp softmax groups, four correction warps, and dedicated MMA/load/epilogue IDs, then adjusts roles for configuration choices such as one Q stage, non-TMA paths, and dynamic persistence. These are concrete attention schedules, not a GEMM-wide architectural template."}, "reason": {"statement": "Limit the comparison to normative issue/D placement and use concrete source layouts for software roles.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} -{"path": "wiki/techniques/warp-specialization.md", "before": {"statement": "The kernel entry point assigns each warp its role based on `threadIdx.x`:\n\n```cuda\n// Blackwell 16-warp specialized GEMM kernel skeleton\n// 16 warps = 512 threads per CTA\n__global__ void __launch_bounds__(512)\nblackwell_gemm_warp_specialized(\n const __grid_constant__ GemmParams params)\n{\n const int warp_id = threadIdx.x / 32;\n const int lane_id = threadIdx.x % 32;\n\n // Shared memory for A/B tiles and mbarrier objects\n extern __shared__ char smem[];\n half* smem_A = reinterpret_cast(smem);\n half* smem_B = reinterpret_cast(smem + SMEM_A_SIZE);\n\n // mbarrier pairs: TMA hardware signals \"data ready\", MMA signals \"buffer free\"\n __shared__ uint64_t mbar_data_ready[NUM_STAGES];\n __shared__ uint64_t mbar_buffer_free[NUM_STAGES];\n // MMA→epilogue handoff barrier\n __shared__ uint64_t mbar_acc_complete;\n // Phase tracking: mbarriers alternate parity on each reuse cycle\n int phase_data[NUM_STAGES];\n int phase_free[NUM_STAGES];\n\n if (warp_id == 0) {\n if (lane_id == 0) {\n for (int s = 0; s < NUM_STAGES; s++) {\n // TMA expects arrive.expect_tx → hardware completes\n mbarrier_init(&mbar_data_ready[s], 1);\n mbarrier_init(&mbar_buffer_free[s], 1);\n }\n mbarrier_init(&mbar_acc_complete, 1);\n }\n }\n // Initialize phase counters (all start at 0)\n for (int s = 0; s < NUM_STAGES; s++) {\n phase_data[s] = 0;\n phase_free[s] = 0;\n }\n __syncthreads();\n\n if (warp_id == 0) {\n // === TMA PRODUCER WARP ===\n for (int k_tile = 0; k_tile < num_k_tiles; k_tile++) {\n int stage = k_tile % NUM_STAGES;\n\n // Wait for consumer to release this buffer (with phase tracking)\n if (k_tile >= NUM_STAGES) {\n mbarrier_wait_parity(&mbar_buffer_free[stage], phase_free[stage]);\n phase_free[stage] ^= 1; // flip parity for next reuse\n }\n\n // Set expected TX bytes, then issue TMA. TMA hardware will\n // signal mbar_data_ready upon transfer completion.\n // Do NOT manually arrive — that races with the async transfer.\n if (lane_id == 0) {\n uint32_t tx_bytes = TILE_A_BYTES + TILE_B_BYTES;\n mbarrier_arrive_expect_tx(&mbar_data_ready[stage], tx_bytes);\n tma_copy_async(smem_A + stage * TILE_A_SIZE,\n ¶ms.A[k_tile * TILE_K], TILE_A_SIZE,\n &mbar_data_ready[stage]);\n tma_copy_async(smem_B + stage * TILE_B_SIZE,\n ¶ms.B[k_tile * TILE_K], TILE_B_SIZE,\n &mbar_data_ready[stage]);\n // TMA hardware arrives on mbar_data_ready when transfer completes\n }\n }\n\n } else if (warp_id == 1) {\n // === MMA CONSUMER WARP ===\n for (int k_tile = 0; k_tile < num_k_tiles; k_tile++) {\n int stage = k_tile % NUM_STAGES;\n\n // Wait for TMA to complete this stage (with phase tracking)\n mbarrier_wait_parity(&mbar_data_ready[stage], phase_data[stage]);\n phase_data[stage] ^= 1;\n\n // Critical fence: ensure TMA data visible before MMA reads SMEM\n tcgen05_fence_after_thread_sync();\n\n if (lane_id == 0) {\n tcgen05_mma(smem_A + stage * TILE_A_SIZE,\n smem_B + stage * TILE_B_SIZE);\n }\n __syncwarp();\n\n // Signal buffer is free for reuse\n if (lane_id == 0) {\n mbarrier_arrive(&mbar_buffer_free[stage]);\n }\n }\n\n // Signal epilogue warps that accumulation is complete\n if (lane_id == 0) {\n mbarrier_arrive(&mbar_acc_complete);\n }\n\n } else {\n // === EPILOGUE WARPS (2-15) ===\n // Wait for MMA completion via dedicated mbarrier (not __syncthreads,\n // which would deadlock since producer/MMA warps don't reach it)\n mbarrier_wait(&mbar_acc_complete);\n\n // Each epilogue warp handles a partition of the TMEM output.\n // Use ceiling division to cover tail rows when TILE_M % 14 != 0.\n constexpr int NUM_EPI_WARPS = 14; // warps 2-15\n int epi_warp = warp_id - 2; // 0..13\n int rows_per_warp = (TILE_M + NUM_EPI_WARPS - 1) / NUM_EPI_WARPS;\n int my_row_start = epi_warp * rows_per_warp;\n int my_row_end = min(my_row_start + rows_per_warp, TILE_M);\n\n for (int r = my_row_start; r < my_row_end; r++) {\n for (int c = lane_id; c < TILE_N; c += 32) {\n // Read accumulator from TMEM\n float acc = tmem_load(r, c);\n // Apply epilogue: scale + bias + activation\n float result = epilogue_op(acc, params.scale, params.bias[c]);\n // Write to global memory\n params.C[r * params.N + c] = __float2half(result);\n }\n }\n }\n}\n```"}, "after": {"statement": "A warp-specialized implementation must prove each ownership handoff. For a reusable TMA-to-MMA stage:\n\n1. Initialize the mbarrier objects with correct participant/transaction counts and publish them to the required threads and async proxies before use.\n2. The producer acquires an empty stage, sets expected transaction bytes, and submits the TMA copies. The full/data-ready phase completes only after the expected async transactions complete.\n3. The MMA controller waits for the full phase and applies the required cross-thread/proxy fence before issuing `tcgen05.mma` against that SMEM stage.\n4. Because MMA is asynchronous, `__syncwarp()` or an ordinary `mbarrier.arrive` after issue does not make the operands reusable. Commit prior tcgen05 work to an mbarrier and wait for completion before releasing or overwriting the stage.\n5. Before another role reads D from TMEM, complete the MMA sequence and apply the required `tcgen05.fence::before_thread_sync` / execution-ordering handoff / `tcgen05.fence::after_thread_sync` protocol. A fence orders operations; it is not a completion wait.\n6. Keep TMEM allocated until every consumer finishes its `tcgen05.ld` sequence and associated waits, then deallocate with the required collective participation.\n\nPipeline wrappers in CUTLASS/CuTe encode parts of these rules, but participant counts, initial phases, tails, and producer/consumer states remain configuration-specific. A short role-dispatch sketch is not a substitute for the complete pipeline and TMEM lifecycle."}, "reason": {"statement": "Unsafe pseudo-code is worse than a precise protocol checklist and pinned full sources.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-load", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-dealloc"]}} -{"path": "wiki/techniques/warp-specialization.md", "before": {"statement": "At the PTX level, the mbarrier operations map to:\n\n```ptx\n// Producer: signal data is ready in stage %stage\nmbarrier.arrive.shared.b64 %dummy, [%mbar_data_ready + %stage_offset];\n\n// Consumer: wait for data to be ready\nmbarrier.try_wait.parity.shared.b64 %pred, [%mbar_data_ready + %stage_offset], %phase;\n\n// Consumer: signal buffer is consumed\nmbarrier.arrive.shared.b64 %dummy, [%mbar_buffer_free + %stage_offset];\n\n// Producer: wait for buffer to be free\nmbarrier.try_wait.parity.shared.b64 %pred, [%mbar_buffer_free + %stage_offset], %phase;\n```"}, "after": {"statement": "A warp-specialized implementation must prove each ownership handoff. For a reusable TMA-to-MMA stage:\n\n1. Initialize the mbarrier objects with correct participant/transaction counts and publish them to the required threads and async proxies before use.\n2. The producer acquires an empty stage, sets expected transaction bytes, and submits the TMA copies. The full/data-ready phase completes only after the expected async transactions complete.\n3. The MMA controller waits for the full phase and applies the required cross-thread/proxy fence before issuing `tcgen05.mma` against that SMEM stage.\n4. Because MMA is asynchronous, `__syncwarp()` or an ordinary `mbarrier.arrive` after issue does not make the operands reusable. Commit prior tcgen05 work to an mbarrier and wait for completion before releasing or overwriting the stage.\n5. Before another role reads D from TMEM, complete the MMA sequence and apply the required `tcgen05.fence::before_thread_sync` / execution-ordering handoff / `tcgen05.fence::after_thread_sync` protocol. A fence orders operations; it is not a completion wait.\n6. Keep TMEM allocated until every consumer finishes its `tcgen05.ld` sequence and associated waits, then deallocate with the required collective participation.\n\nPipeline wrappers in CUTLASS/CuTe encode parts of these rules, but participant counts, initial phases, tails, and producer/consumer states remain configuration-specific. A short role-dispatch sketch is not a substitute for the complete pipeline and TMEM lifecycle."}, "reason": {"statement": "Replace incomplete PTX with named obligations and authoritative links.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-dealloc"]}} +{"path": "wiki/techniques/warp-specialization.md", "before": {"statement": "This contrasts with Hopper where a warpgroup (4 warps, 128 threads) collectively issues wgmma.mma_async, and all threads in the warpgroup participate in the MMA. On Blackwell, the MMA warp dispatches the instruction from a single thread while the hardware handles the data movement internally."}, "after": {"statement": "Warp specialization assigns different long-lived functions to disjoint warps in a CTA—for example scheduler, TMA load, MMA control, softmax/correction, or epilogue work. It is a software organization for overlapping pipeline stages, not a fixed SM100 CTA layout.\n\nThe architecture-level comparison is narrower:\n\n| Issue granularity | Warpgroup collective | One thread for `cta_group::1` or `cta_group::2` |\n| D accumulator | Per-thread registers | TMEM |\n| A/B movement | Explicit register/SMEM operands and producer work | Explicit SMEM descriptors or TMEM addresses and producer work |\n| Completion | WGMMA commit/wait groups | `tcgen05.commit` tied to an mbarrier, followed by a wait |\n\nSingle-thread issue reduces the number of threads needed to submit MMA instructions. It does not make operands move automatically, turn an entire warp into an ISA-level role, complete MMA synchronously, or choose how many epilogue warps a kernel should launch."}, "reason": {"statement": "Keep the precise Hopper/SM100 issue distinction without implying autonomous operand staging.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-operation-wgmma-mma-async"]}} +{"path": "wiki/techniques/warp-specialization.md", "before": {"statement": "| MMA granularity | 4-warp warpgroup (128 threads) | Single thread in 1 warp |\n| MMA output destination | Registers (shared across warpgroup) | TMEM (256KB, CTA-visible) |\n| Producer warps | Separate warp(s) for TMA loads | Warp 0 dedicated to TMA |\n| Epilogue execution | Same warpgroup or separate warps | 14 dedicated warps (2-15) |\n| Synchronization | warpgroup barriers, arrive/wait | mbarrier pairs (producer/consumer) |\n| Register pressure | High (accumulators in registers) | Low (accumulators in TMEM) |"}, "after": {"statement": "Warp specialization assigns different long-lived functions to disjoint warps in a CTA—for example scheduler, TMA load, MMA control, softmax/correction, or epilogue work. It is a software organization for overlapping pipeline stages, not a fixed SM100 CTA layout.\n\nThe architecture-level comparison is narrower:\n\n| Issue granularity | Warpgroup collective | One thread for `cta_group::1` or `cta_group::2` |\n| D accumulator | Per-thread registers | TMEM |\n| A/B movement | Explicit register/SMEM operands and producer work | Explicit SMEM descriptors or TMEM addresses and producer work |\n| Completion | WGMMA commit/wait groups | `tcgen05.commit` tied to an mbarrier, followed by a wait |\n\nSingle-thread issue reduces the number of threads needed to submit MMA instructions. It does not make operands move automatically, turn an entire warp into an ISA-level role, complete MMA synchronously, or choose how many epilogue warps a kernel should launch.\n\nThere is no universal “warp 0 load, warp 1 MMA, warps 2–15 epilogue” rule. Three primary implementations illustrate the range:\n\n| Gau Nernst tutorial v4, commit `3b90ac9b...` | 4 warps. An elected lane in warp 0 runs the TMA loop; an elected lane in warp 1 runs the MMA loop; after completion, all four warps participate in TMEM load/conversion/output. |\n| CUTLASS 4.5.0 generic SM100 GEMM | One warp each for `MMA`, `Sched`, `MainloopLoad`, and `EpilogueLoad`, followed by `CollectiveEpilogue::ThreadCount / 32` epilogue warps. Optional work can make some control roles nonparticipants. |\n| FlashInfer PR 1039 context FMHA | 16 warps: 0–3 `Softmax0`, 4–7 `Softmax1`, 8–11 correction, 12 MMA, 13 load, 14 epilogue, and 15 empty. The kernel has explicit pipelines between these roles. |\n\nFlashAttention-4's pinned SM100 forward source likewise starts from a 16-warp layout with two four-warp softmax groups, four correction warps, and dedicated MMA/load/epilogue IDs, then adjusts roles for configuration choices such as one Q stage, non-TMA paths, and dynamic persistence. These are concrete attention schedules, not a GEMM-wide architectural template."}, "reason": {"statement": "Limit the comparison to normative issue/D placement and use concrete source layouts for software roles.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-operation-wgmma-mma-async", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma"]}} +{"path": "wiki/techniques/warp-specialization.md", "before": {"statement": "The kernel entry point assigns each warp its role based on `threadIdx.x`:\n\n```cuda\n// Blackwell 16-warp specialized GEMM kernel skeleton\n// 16 warps = 512 threads per CTA\n__global__ void __launch_bounds__(512)\nblackwell_gemm_warp_specialized(\n const __grid_constant__ GemmParams params)\n{\n const int warp_id = threadIdx.x / 32;\n const int lane_id = threadIdx.x % 32;\n\n // Shared memory for A/B tiles and mbarrier objects\n extern __shared__ char smem[];\n half* smem_A = reinterpret_cast(smem);\n half* smem_B = reinterpret_cast(smem + SMEM_A_SIZE);\n\n // mbarrier pairs: TMA hardware signals \"data ready\", MMA signals \"buffer free\"\n __shared__ uint64_t mbar_data_ready[NUM_STAGES];\n __shared__ uint64_t mbar_buffer_free[NUM_STAGES];\n // MMA→epilogue handoff barrier\n __shared__ uint64_t mbar_acc_complete;\n // Phase tracking: mbarriers alternate parity on each reuse cycle\n int phase_data[NUM_STAGES];\n int phase_free[NUM_STAGES];\n\n if (warp_id == 0) {\n if (lane_id == 0) {\n for (int s = 0; s < NUM_STAGES; s++) {\n // TMA expects arrive.expect_tx → hardware completes\n mbarrier_init(&mbar_data_ready[s], 1);\n mbarrier_init(&mbar_buffer_free[s], 1);\n }\n mbarrier_init(&mbar_acc_complete, 1);\n }\n }\n // Initialize phase counters (all start at 0)\n for (int s = 0; s < NUM_STAGES; s++) {\n phase_data[s] = 0;\n phase_free[s] = 0;\n }\n __syncthreads();\n\n if (warp_id == 0) {\n // === TMA PRODUCER WARP ===\n for (int k_tile = 0; k_tile < num_k_tiles; k_tile++) {\n int stage = k_tile % NUM_STAGES;\n\n // Wait for consumer to release this buffer (with phase tracking)\n if (k_tile >= NUM_STAGES) {\n mbarrier_wait_parity(&mbar_buffer_free[stage], phase_free[stage]);\n phase_free[stage] ^= 1; // flip parity for next reuse\n }\n\n // Set expected TX bytes, then issue TMA. TMA hardware will\n // signal mbar_data_ready upon transfer completion.\n // Do NOT manually arrive — that races with the async transfer.\n if (lane_id == 0) {\n uint32_t tx_bytes = TILE_A_BYTES + TILE_B_BYTES;\n mbarrier_arrive_expect_tx(&mbar_data_ready[stage], tx_bytes);\n tma_copy_async(smem_A + stage * TILE_A_SIZE,\n ¶ms.A[k_tile * TILE_K], TILE_A_SIZE,\n &mbar_data_ready[stage]);\n tma_copy_async(smem_B + stage * TILE_B_SIZE,\n ¶ms.B[k_tile * TILE_K], TILE_B_SIZE,\n &mbar_data_ready[stage]);\n // TMA hardware arrives on mbar_data_ready when transfer completes\n }\n }\n\n } else if (warp_id == 1) {\n // === MMA CONSUMER WARP ===\n for (int k_tile = 0; k_tile < num_k_tiles; k_tile++) {\n int stage = k_tile % NUM_STAGES;\n\n // Wait for TMA to complete this stage (with phase tracking)\n mbarrier_wait_parity(&mbar_data_ready[stage], phase_data[stage]);\n phase_data[stage] ^= 1;\n\n // Critical fence: ensure TMA data visible before MMA reads SMEM\n tcgen05_fence_after_thread_sync();\n\n if (lane_id == 0) {\n tcgen05_mma(smem_A + stage * TILE_A_SIZE,\n smem_B + stage * TILE_B_SIZE);\n }\n __syncwarp();\n\n // Signal buffer is free for reuse\n if (lane_id == 0) {\n mbarrier_arrive(&mbar_buffer_free[stage]);\n }\n }\n\n // Signal epilogue warps that accumulation is complete\n if (lane_id == 0) {\n mbarrier_arrive(&mbar_acc_complete);\n }\n\n } else {\n // === EPILOGUE WARPS (2-15) ===\n // Wait for MMA completion via dedicated mbarrier (not __syncthreads,\n // which would deadlock since producer/MMA warps don't reach it)\n mbarrier_wait(&mbar_acc_complete);\n\n // Each epilogue warp handles a partition of the TMEM output.\n // Use ceiling division to cover tail rows when TILE_M % 14 != 0.\n constexpr int NUM_EPI_WARPS = 14; // warps 2-15\n int epi_warp = warp_id - 2; // 0..13\n int rows_per_warp = (TILE_M + NUM_EPI_WARPS - 1) / NUM_EPI_WARPS;\n int my_row_start = epi_warp * rows_per_warp;\n int my_row_end = min(my_row_start + rows_per_warp, TILE_M);\n\n for (int r = my_row_start; r < my_row_end; r++) {\n for (int c = lane_id; c < TILE_N; c += 32) {\n // Read accumulator from TMEM\n float acc = tmem_load(r, c);\n // Apply epilogue: scale + bias + activation\n float result = epilogue_op(acc, params.scale, params.bias[c]);\n // Write to global memory\n params.C[r * params.N + c] = __float2half(result);\n }\n }\n }\n}\n```"}, "after": {"statement": "A warp-specialized implementation must prove each ownership handoff. For a reusable TMA-to-MMA stage:\n\n1. Initialize the mbarrier objects with correct participant/transaction counts and publish them to the required threads and async proxies before use.\n2. The producer acquires an empty stage, sets expected transaction bytes, and submits the TMA copies. The full/data-ready phase completes only after the expected async transactions complete.\n3. The MMA controller waits for the full phase and applies the required cross-thread/proxy fence before issuing `tcgen05.mma` against that SMEM stage.\n4. Because MMA is asynchronous, `__syncwarp()` or an ordinary `mbarrier.arrive` after issue does not make the operands reusable. Commit prior tcgen05 work to an mbarrier and wait for completion before releasing or overwriting the stage.\n5. Before another role reads D from TMEM, complete the MMA sequence and apply the required `tcgen05.fence::before_thread_sync` / execution-ordering handoff / `tcgen05.fence::after_thread_sync` protocol. A fence orders operations; it is not a completion wait.\n6. Keep TMEM allocated until every consumer finishes its `tcgen05.ld` sequence and associated waits, then deallocate with the required collective participation.\n\nPipeline wrappers in CUTLASS/CuTe encode parts of these rules, but participant counts, initial phases, tails, and producer/consumer states remain configuration-specific. A short role-dispatch sketch is not a substitute for the complete pipeline and TMEM lifecycle."}, "reason": {"statement": "Unsafe pseudo-code is worse than a precise protocol checklist and pinned full sources.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-load", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-special-sync-operations-fence", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit"]}} +{"path": "wiki/techniques/warp-specialization.md", "before": {"statement": "At the PTX level, the mbarrier operations map to:\n\n```ptx\n// Producer: signal data is ready in stage %stage\nmbarrier.arrive.shared.b64 %dummy, [%mbar_data_ready + %stage_offset];\n\n// Consumer: wait for data to be ready\nmbarrier.try_wait.parity.shared.b64 %pred, [%mbar_data_ready + %stage_offset], %phase;\n\n// Consumer: signal buffer is consumed\nmbarrier.arrive.shared.b64 %dummy, [%mbar_buffer_free + %stage_offset];\n\n// Producer: wait for buffer to be free\nmbarrier.try_wait.parity.shared.b64 %pred, [%mbar_buffer_free + %stage_offset], %phase;\n```"}, "after": {"statement": "A warp-specialized implementation must prove each ownership handoff. For a reusable TMA-to-MMA stage:\n\n1. Initialize the mbarrier objects with correct participant/transaction counts and publish them to the required threads and async proxies before use.\n2. The producer acquires an empty stage, sets expected transaction bytes, and submits the TMA copies. The full/data-ready phase completes only after the expected async transactions complete.\n3. The MMA controller waits for the full phase and applies the required cross-thread/proxy fence before issuing `tcgen05.mma` against that SMEM stage.\n4. Because MMA is asynchronous, `__syncwarp()` or an ordinary `mbarrier.arrive` after issue does not make the operands reusable. Commit prior tcgen05 work to an mbarrier and wait for completion before releasing or overwriting the stage.\n5. Before another role reads D from TMEM, complete the MMA sequence and apply the required `tcgen05.fence::before_thread_sync` / execution-ordering handoff / `tcgen05.fence::after_thread_sync` protocol. A fence orders operations; it is not a completion wait.\n6. Keep TMEM allocated until every consumer finishes its `tcgen05.ld` sequence and associated waits, then deallocate with the required collective participation.\n\nPipeline wrappers in CUTLASS/CuTe encode parts of these rules, but participant counts, initial phases, tails, and producer/consumer states remain configuration-specific. A short role-dispatch sketch is not a substitute for the complete pipeline and TMEM lifecycle."}, "reason": {"statement": "Replace incomplete PTX with named obligations and authoritative links.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-special-sync-operations-fence", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit"]}} {"path": "wiki/techniques/warp-specialization.md", "before": {"statement": "In CUTLASS 4.5.0, the SM100 GEMM collective (`CollectiveMma_1SM`) implements this pattern with CuTe abstractions:\n\n```cuda\n// CUTLASS SM100 warp role dispatch (simplified from CollectiveMma)\n// Template parameter WarpCount = cute::Shape<1, 1, 14>\n// Warp 0 = producer, Warp 1 = math, Warps 2-15 = epilogue\n\ntemplate \nstruct CollectiveMma_1SM {\n static constexpr int NumProducerWarps = 1;\n static constexpr int NumMathWarps = 1;\n static constexpr int NumEpilogueWarps = 14;\n\n CUTLASS_DEVICE void operator()(\n Params const& params,\n char* smem_buf,\n TiledMma& tiled_mma)\n {\n int warp_idx = cutlass::canonical_warp_idx_sync();\n\n if (warp_idx == 0) {\n producer_warp(params, smem_buf);\n } else if (warp_idx == 1) {\n math_warp(params, smem_buf, tiled_mma);\n } else {\n epilogue_warp(params, smem_buf);\n }\n }\n};\n```"}, "after": {"statement": "There is no universal “warp 0 load, warp 1 MMA, warps 2–15 epilogue” rule. Three primary implementations illustrate the range:\n\n| Gau Nernst tutorial v4, commit `3b90ac9b...` | 4 warps. An elected lane in warp 0 runs the TMA loop; an elected lane in warp 1 runs the MMA loop; after completion, all four warps participate in TMEM load/conversion/output. |\n| CUTLASS 4.5.0 generic SM100 GEMM | One warp each for `MMA`, `Sched`, `MainloopLoad`, and `EpilogueLoad`, followed by `CollectiveEpilogue::ThreadCount / 32` epilogue warps. Optional work can make some control roles nonparticipants. |\n| FlashInfer PR 1039 context FMHA | 16 warps: 0–3 `Softmax0`, 4–7 `Softmax1`, 8–11 correction, 12 MMA, 13 load, 14 epilogue, and 15 empty. The kernel has explicit pipelines between these roles. |\n\nFlashAttention-4's pinned SM100 forward source likewise starts from a 16-warp layout with two four-warp softmax groups, four correction warps, and dedicated MMA/load/epilogue IDs, then adjusts roles for configuration choices such as one Q stage, non-TMA paths, and dynamic persistence. These are concrete attention schedules, not a GEMM-wide architectural template."}, "reason": {"statement": "Use exact source names and line-level descriptions instead of fabricated code.", "urls": []}} {"path": "wiki/techniques/warp-specialization.md", "before": {"statement": "- **Always on Blackwell GEMMs**: Warp specialization is the standard pattern for SM100 tensor core kernels. The tcgen05 instruction model assumes single-thread dispatch with TMEM output."}, "after": {"statement": "For the author's exact `M=N=K=4096` Modal B200 setup with PyTorch 2.9.1 and CUDA 13, tutorial v3 reports 939.61 TFLOP/s and v4 reports 1208.83 TFLOP/s after introducing warp specialization. That is a 269.22-TFLOP/s, approximately 28.65% step between those source variants. It is not a portable Blackwell speedup or evidence that specialization is always profitable."}, "reason": {"statement": "Present specialization as a candidate optimization with an isolated source-reported example.", "urls": []}} {"path": "wiki/techniques/warp-specialization.md", "before": {"statement": "- **Any kernel with producer-consumer pipeline**: When TMA loads and MMA compute can overlap, warp specialization provides the cleanest decomposition."}, "after": {"statement": "For the author's exact `M=N=K=4096` Modal B200 setup with PyTorch 2.9.1 and CUDA 13, tutorial v3 reports 939.61 TFLOP/s and v4 reports 1208.83 TFLOP/s after introducing warp specialization. That is a 269.22-TFLOP/s, approximately 28.65% step between those source variants. It is not a portable Blackwell speedup or evidence that specialization is always profitable."}, "reason": {"statement": "Supply selection criteria and controlled comparison instead of a universal adjective.", "urls": []}} @@ -341,30 +341,30 @@ {"path": "wiki/techniques/external-source-map-research.md", "before": {"statement": "```bash\ngit clone https://github.com/ColfaxResearch/cfx-article-src external/colfax-cfx\ngit clone https://github.com/simveit/load_and_store external/simveit-load-store\nrg -n \"tma|mbarrier|swizzle|ldmatrix|stream\" external/colfax-cfx external/simveit-load-store\n```"}, "after": {"statement": "```bash\ngit init external/colfax-cfx\ngit -C external/colfax-cfx remote add origin \\\n https://github.com/ColfaxResearch/cfx-article-src\ngit -C external/colfax-cfx fetch --depth=1 origin \\\n fbecfed88de2e4246f104a023188ba722937c5fc\ngit -C external/colfax-cfx checkout --detach FETCH_HEAD\ntest \"$(git -C external/colfax-cfx rev-parse HEAD)\" = \\\n fbecfed88de2e4246f104a023188ba722937c5fc\n\nrg -n \"SM90_TMA|mbarrier\" \\\n external/colfax-cfx/tma external/colfax-cfx/pipeline-gemm\nrg -n \"PersistentTileScheduler|StreamK\" external/colfax-cfx/streamk\n```"}, "reason": {"statement": "Pin every checkout and show mechanism-specific file searches.", "urls": []}} {"path": "wiki/techniques/external-source-map-research.md", "before": {"statement": "Do not cite this page as implementation evidence by itself. Cite one of its\nsource pages plus the concrete upstream file path, commit, or URL that shaped the\ncandidate edit."}, "after": {"statement": "- For tail waves, compare the pinned non-persistent, data-parallel persistent,\n and Stream-K scheduler implementations before adding shape dispatch. Profile\n the target shapes rather than treating persistence as a universal remedy."}, "reason": {"statement": "Require a pinned upstream revision and exact file/symbol locator, treating source pages as discovery leads only.", "urls": []}} {"path": "wiki/techniques/external-source-map-research.md", "before": {"statement": "Use external source-map research after a profile or benchmark identifies an edit\nfamily but the local PR pages do not expose a small enough implementation\nexample. The route is code-first: clone a source-map repository, grep for the\nmeasured mechanism, and cite exact files before adapting an idea."}, "after": {"statement": "Use external source-map research after a profile or benchmark identifies an edit\nfamily but the local PR pages do not expose a small enough implementation\nexample. Treat the profiler result as a search key, not a root-cause proof:\ninspect a pinned upstream revision and exact implementation files before\nadapting an idea.\n\n- For tail waves, compare the pinned non-persistent, data-parallel persistent,\n and Stream-K scheduler implementations before adding shape dispatch. Profile\n the target shapes rather than treating persistence as a universal remedy."}, "reason": {"statement": "Scope demonstrated implementations to SM90a and make Blackwell transfer require separate SM100 evidence.", "urls": []}} -{"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "Fine-grained quantization applies per-block (rather than per-tensor) scaling factors to low-precision data, preventing outlier values from destroying the quantization precision of an entire tensor. DeepSeek pioneered the tile-wise 1x128 scaling for activations and block-wise 128x128 scaling for weights in their FP8 training framework. On Blackwell (SM100), native block scaling support in tcgen05.mma enables hardware-accelerated fine-grained quantization using the UE8M0 scale format, while Hopper requires software-managed CUDA core promotion."}, "after": {"statement": "Fine-grained quantization associates low-precision payloads with scales for\nsubsets of a tensor. Smaller groups let scale selection respond more locally,\nbut they do not guarantee a particular error or speed result. Keep four choices\nseparate when transferring a kernel: payload type, scale type, scale geometry,\nand physical scale layout.\n\nThree verified recipes in this wiki illustrate why the full contract matters:\n\n- **DeepSeek FP8 training geometry:** one scale for each token row and 128\n activation channels (`1 x 128`), and one scale for each 128-input-channel by\n 128-output-channel weight block (`128 x 128`).\n- **NVFP4 1D recipe:** E2M1 payloads, one E4M3 local scale per 16 consecutive\n values, and one FP32 global scale per tensor. Transformer Engine 2.13 also\n defines a weight-oriented 2D mode with one local scale per `16 x 16` block.\n- **MXFP4:** E2M1 payloads with one UE8M0 power-of-two scale per 32 values; the\n NVFP4 per-tensor global scale is not part of this microscaling format."}, "reason": {"statement": "Separate recipe, implementation and architecture scopes.", "urls": ["https://arxiv.org/abs/2412.19437v2", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"]}} +{"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "Fine-grained quantization applies per-block (rather than per-tensor) scaling factors to low-precision data, preventing outlier values from destroying the quantization precision of an entire tensor. DeepSeek pioneered the tile-wise 1x128 scaling for activations and block-wise 128x128 scaling for weights in their FP8 training framework. On Blackwell (SM100), native block scaling support in tcgen05.mma enables hardware-accelerated fine-grained quantization using the UE8M0 scale format, while Hopper requires software-managed CUDA core promotion."}, "after": {"statement": "Fine-grained quantization associates low-precision payloads with scales for\nsubsets of a tensor. Smaller groups let scale selection respond more locally,\nbut they do not guarantee a particular error or speed result. Keep four choices\nseparate when transferring a kernel: payload type, scale type, scale geometry,\nand physical scale layout.\n\nThree verified recipes in this wiki illustrate why the full contract matters:\n\n- **DeepSeek FP8 training geometry:** one scale for each token row and 128\n activation channels (`1 x 128`), and one scale for each 128-input-channel by\n 128-output-channel weight block (`128 x 128`).\n- **NVFP4 1D recipe:** E2M1 payloads, one E4M3 local scale per 16 consecutive\n values, and one FP32 global scale per tensor. Transformer Engine 2.13 also\n defines a weight-oriented 2D mode with one local scale per `16 x 16` block.\n- **MXFP4:** E2M1 payloads with one UE8M0 power-of-two scale per 32 values; the\n NVFP4 per-tensor global scale is not part of this microscaling format."}, "reason": {"statement": "Separate recipe, implementation and architecture scopes.", "urls": ["https://arxiv.org/abs/2412.19437v2", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"]}} {"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "```\nPer-tensor scaling (coarsest):\n Entire tensor shares one FP32 scale factor.\n Problem: a single outlier ruins precision for all elements.\n \n [=== entire MxK matrix === ] -> 1 scale\n\nPer-block 128x128 scaling (weights):\n Each 128x128 block has its own scale factor.\n 128x128 = 16,384 elements per scale -> 0.006% overhead.\n \n +---+---+---+\n |s1 |s2 |s3 | <- each block has independent scale\n +---+---+---+\n |s4 |s5 |s6 |\n +---+---+---+\n\nPer-tile 1x128 scaling (activations):\n Each row of 128 elements has its own scale factor.\n Captures per-channel activation distributions.\n \n [s1: ====128 elements====]\n [s2: ====128 elements====]\n [s3: ====128 elements====]\n```"}, "after": {"statement": "Scale-count and storage overhead are different quantities. A `128 x 128` block\nhas one factor per 16,384 payloads, a factor-count ratio of about 0.0061%. If\nthe payload is one byte and the factor is FP32, the byte ratio is instead\n`4 / 16384`, about 0.0244%. Include factor width, payload width, padding, and\nlayout transformations in any storage or bandwidth claim."}, "reason": {"statement": "State geometry and formulas without an ambiguous percentage or per-channel label.", "urls": []}} {"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "```cuda\n// DeepGEMM FP8 GEMM with fine-grained scaling\n// A (activations): FP8 E4M3 with tile-wise 1x128 FP32 scales\n// B (weights): FP8 E4M3 with block-wise 128x128 FP32 scales\n// C (output): FP32 accumulator\n\n// Scale tensor shapes:\n// scale_A: [M, K/128] -- one FP32 scale per 128 elements along K for each row\n// scale_B: [K/128, N/128] -- one FP32 scale per 128x128 block\n\n__device__ void deepgemm_fp8_tile(\n const fp8_e4m3* A, // [TILE_M, TILE_K] in FP8\n const fp8_e4m3* B, // [TILE_K, TILE_N] in FP8\n const float* scale_A, // [TILE_M, TILE_K/128]\n const float* scale_B, // [TILE_K/128, TILE_N/128]\n float* C, // [TILE_M, TILE_N] accumulator\n int M, int N, int K)\n{\n // For each 128-element K-chunk:\n for (int k_block = 0; k_block < K; k_block += 128) {\n // 1. Load FP8 A tile [TILE_M, 128] and B tile [128, TILE_N]\n // 2. Perform MMA: partial = A_fp8 * B_fp8 (in limited-precision FP32)\n // 3. Apply combined scale: scale_A[m, k_block/128] * scale_B[k_block/128, n_block/128]\n // 4. Accumulate: C[m, n] += partial * combined_scale\n\n for (int m = 0; m < TILE_M; m++) {\n float sa = scale_A[m * (K / 128) + k_block / 128];\n for (int n_block = 0; n_block < TILE_N; n_block += 128) {\n float sb = scale_B[(k_block / 128) * (N / 128) + n_block / 128];\n float combined_scale = sa * sb;\n\n // Apply scale to the partial MMA result\n for (int n = n_block; n < n_block + 128; n++) {\n C[m * TILE_N + n] += partial[m][n] * combined_scale;\n }\n }\n }\n }\n}\n```"}, "after": {"statement": "At commit `891d57b4db1071624b5c8fa0d1e51cb317fa709f`, DeepGEMM uses\narchitecture-specific scale representations and accumulation paths.\n\nThe pinned SM90 1D1D kernel consumes FP32 A/B factors and fixes\n`BLOCK_K == 128`. The DeepSeek-V3 report characterizes the H800 FP8 Tensor Core\npath as retaining about 14 bits and describes this interval as four WGMMAs in\nits configuration. The exact kernel accumulates one K block in `float accum`\nand then applies both factors into a separate `float final_accum`:\n\n```cpp\nfinal_accum[i * 4 + 0] += scale_a_0 * scale_b_0 * accum[i * 4 + 0];\nfinal_accum[i * 4 + 1] += scale_a_0 * scale_b_1 * accum[i * 4 + 1];\nfinal_accum[i * 4 + 2] += scale_a_1 * scale_b_0 * accum[i * 4 + 2];\nfinal_accum[i * 4 + 3] += scale_a_1 * scale_b_1 * accum[i * 4 + 3];\n```\n\nThis is a source-specific implementation, not a rule for every Hopper FP8\nkernel. The cited sources do not establish the deleted 0.1% error bound or a\nuniversal ranking of `Nc=32`, `64`, `128`, and `256`."}, "reason": {"statement": "Use the pinned implementation and a short verbatim fragment instead.", "urls": ["https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh", "https://arxiv.org/abs/2412.19437v2"]}} {"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "On Hopper (SM90), the wgmma instruction accumulates in Tensor Core registers with limited precision (~FP22, not true FP32). To maintain numerical accuracy, DeepGEMM promotes the partial sum to a separate FP32 accumulator on CUDA Cores every 4 wgmma instructions (Nc=128, since each wgmma processes 32 K-elements):"}, "after": {"statement": "At commit `891d57b4db1071624b5c8fa0d1e51cb317fa709f`, DeepGEMM uses\narchitecture-specific scale representations and accumulation paths.\n\nThe pinned SM90 1D1D kernel consumes FP32 A/B factors and fixes\n`BLOCK_K == 128`. The DeepSeek-V3 report characterizes the H800 FP8 Tensor Core\npath as retaining about 14 bits and describes this interval as four WGMMAs in\nits configuration. The exact kernel accumulates one K block in `float accum`\nand then applies both factors into a separate `float final_accum`:\n\n```cpp\nfinal_accum[i * 4 + 0] += scale_a_0 * scale_b_0 * accum[i * 4 + 0];\nfinal_accum[i * 4 + 1] += scale_a_0 * scale_b_1 * accum[i * 4 + 1];\nfinal_accum[i * 4 + 2] += scale_a_1 * scale_b_0 * accum[i * 4 + 2];\nfinal_accum[i * 4 + 3] += scale_a_1 * scale_b_1 * accum[i * 4 + 3];\n```\n\nThis is a source-specific implementation, not a rule for every Hopper FP8\nkernel. The cited sources do not establish the deleted 0.1% error bound or a\nuniversal ranking of `Nc=32`, `64`, `128`, and `256`."}, "reason": {"statement": "Use the source's retained-bit wording and implementation scope.", "urls": ["https://arxiv.org/abs/2412.19437v2", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh"]}} {"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "```cuda\n// Hopper FP8 GEMM with CUDA Core promotion (DeepGEMM pattern)\n// Every Nc=128 K-elements, promote Tensor Core accumulator to FP32\n\n__device__ void hopper_fp8_gemm_with_promotion(\n const fp8_e4m3* A, const fp8_e4m3* B,\n const float* scale_A, const float* scale_B,\n float* C_accumulator,\n int K)\n{\n // Tensor Core limited-precision accumulator (FP22-ish)\n // These are wgmma output registers\n float tc_acc[TILE_M_PER_THREAD][TILE_N_PER_THREAD] = {0};\n\n // True FP32 accumulator on CUDA Cores\n float fp32_acc[TILE_M_PER_THREAD][TILE_N_PER_THREAD] = {0};\n\n int promotion_interval = 128; // Nc = 128 elements = 4 wgmma ops\n int wgmma_count = 0;\n\n for (int k = 0; k < K; k += 32) {\n // Issue wgmma (accumulates in limited-precision tc_acc)\n wgmma_mma_async(tc_acc, smem_A_ptr, smem_B_ptr);\n wgmma_count++;\n\n if (wgmma_count == 4) { // Every 128 K-elements\n // Promote: transfer tc_acc to fp32_acc on CUDA Cores\n wgmma_wait(); // Ensure wgmma is complete\n\n int k_block = k / promotion_interval;\n for (int m = 0; m < TILE_M_PER_THREAD; m++) {\n float sa = scale_A[/*...*/];\n for (int n = 0; n < TILE_N_PER_THREAD; n++) {\n float sb = scale_B[/*...*/];\n // Add scaled partial to true FP32 accumulator\n fp32_acc[m][n] += tc_acc[m][n] * sa * sb;\n // Reset TC accumulator for next interval\n tc_acc[m][n] = 0;\n }\n }\n wgmma_count = 0;\n }\n }\n}\n```"}, "after": {"statement": "At commit `891d57b4db1071624b5c8fa0d1e51cb317fa709f`, DeepGEMM uses\narchitecture-specific scale representations and accumulation paths.\n\nThe pinned SM90 1D1D kernel consumes FP32 A/B factors and fixes\n`BLOCK_K == 128`. The DeepSeek-V3 report characterizes the H800 FP8 Tensor Core\npath as retaining about 14 bits and describes this interval as four WGMMAs in\nits configuration. The exact kernel accumulates one K block in `float accum`\nand then applies both factors into a separate `float final_accum`:\n\n```cpp\nfinal_accum[i * 4 + 0] += scale_a_0 * scale_b_0 * accum[i * 4 + 0];\nfinal_accum[i * 4 + 1] += scale_a_0 * scale_b_1 * accum[i * 4 + 1];\nfinal_accum[i * 4 + 2] += scale_a_1 * scale_b_0 * accum[i * 4 + 2];\nfinal_accum[i * 4 + 3] += scale_a_1 * scale_b_1 * accum[i * 4 + 3];\n```\n\nThis is a source-specific implementation, not a rule for every Hopper FP8\nkernel. The cited sources do not establish the deleted 0.1% error bound or a\nuniversal ranking of `Nc=32`, `64`, `128`, and `256`."}, "reason": {"statement": "Replace it with an exact upstream fragment and workflow.", "urls": ["https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh", "https://arxiv.org/abs/2412.19437v2"]}} -{"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "The Nc=128 interval was chosen because:\n- 4 wgmma operations process 128 K-elements (4 x 32)\n- At this interval, the accumulated FP22 error is bounded to ~0.1% relative error\n- Fewer than 4 ops (Nc=32, Nc=64) adds too much promotion overhead\n- More than 4 ops (Nc=256) allows unacceptable precision loss"}, "after": {"statement": "At commit `891d57b4db1071624b5c8fa0d1e51cb317fa709f`, DeepGEMM uses\narchitecture-specific scale representations and accumulation paths.\n\nThe pinned SM90 1D1D kernel consumes FP32 A/B factors and fixes\n`BLOCK_K == 128`. The DeepSeek-V3 report characterizes the H800 FP8 Tensor Core\npath as retaining about 14 bits and describes this interval as four WGMMAs in\nits configuration. The exact kernel accumulates one K block in `float accum`\nand then applies both factors into a separate `float final_accum`:\n\n```cpp\nfinal_accum[i * 4 + 0] += scale_a_0 * scale_b_0 * accum[i * 4 + 0];\nfinal_accum[i * 4 + 1] += scale_a_0 * scale_b_1 * accum[i * 4 + 1];\nfinal_accum[i * 4 + 2] += scale_a_1 * scale_b_0 * accum[i * 4 + 2];\nfinal_accum[i * 4 + 3] += scale_a_1 * scale_b_1 * accum[i * 4 + 3];\n```\n\nThis is a source-specific implementation, not a rule for every Hopper FP8\nkernel. The cited sources do not establish the deleted 0.1% error bound or a\nuniversal ranking of `Nc=32`, `64`, `128`, and `256`.\n\n1. Match the model/checkpoint recipe: payload encoding, local scale type,\n optional global scale, and scale geometry.\n2. Match the kernel ABI: logical factor shape is not necessarily its TMA- or\n TMEM-ready physical layout. Account for packing, padding, transposition, and\n swizzling.\n3. Match the target instruction. For native Blackwell block scaling, use the\n complete kind, block/scale-vector qualifier, scale type, and target rules.\n4. Keep preparation in the timed region unless the producer already emits the\n required layout. Otherwise report preprocessing separately.\n5. Validate output accuracy with the exact quantizer, workload, reference,\n tolerance, and accumulation path; then measure end-to-end latency or\n throughput for the actual shapes.\n\n- More groups require more scale elements, but padding and layout determine the\n actual traffic and storage cost.\n- Smaller groups and fractional scales provide more representational freedom;\n they do not guarantee lower error for every tensor or scale-selection method.\n- DeepGEMM's Nc=128 path is evidence for that pinned SM90 implementation, not a\n universal optimum for Hopper.\n- Native instruction support does not make two recipes ABI-compatible. NVFP4,\n MXFP4, and DeepGEMM FP8 differ in payload, scale type, grouping, and layout."}, "reason": {"statement": "Retain only the exact source-supported interval and prohibit transfer without measurement.", "urls": ["https://arxiv.org/abs/2412.19437v2", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh", "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "The Nc=128 interval was chosen because:\n- 4 wgmma operations process 128 K-elements (4 x 32)\n- At this interval, the accumulated FP22 error is bounded to ~0.1% relative error\n- Fewer than 4 ops (Nc=32, Nc=64) adds too much promotion overhead\n- More than 4 ops (Nc=256) allows unacceptable precision loss"}, "after": {"statement": "At commit `891d57b4db1071624b5c8fa0d1e51cb317fa709f`, DeepGEMM uses\narchitecture-specific scale representations and accumulation paths.\n\nThe pinned SM90 1D1D kernel consumes FP32 A/B factors and fixes\n`BLOCK_K == 128`. The DeepSeek-V3 report characterizes the H800 FP8 Tensor Core\npath as retaining about 14 bits and describes this interval as four WGMMAs in\nits configuration. The exact kernel accumulates one K block in `float accum`\nand then applies both factors into a separate `float final_accum`:\n\n```cpp\nfinal_accum[i * 4 + 0] += scale_a_0 * scale_b_0 * accum[i * 4 + 0];\nfinal_accum[i * 4 + 1] += scale_a_0 * scale_b_1 * accum[i * 4 + 1];\nfinal_accum[i * 4 + 2] += scale_a_1 * scale_b_0 * accum[i * 4 + 2];\nfinal_accum[i * 4 + 3] += scale_a_1 * scale_b_1 * accum[i * 4 + 3];\n```\n\nThis is a source-specific implementation, not a rule for every Hopper FP8\nkernel. The cited sources do not establish the deleted 0.1% error bound or a\nuniversal ranking of `Nc=32`, `64`, `128`, and `256`.\n\n1. Match the model/checkpoint recipe: payload encoding, local scale type,\n optional global scale, and scale geometry.\n2. Match the kernel ABI: logical factor shape is not necessarily its TMA- or\n TMEM-ready physical layout. Account for packing, padding, transposition, and\n swizzling.\n3. Match the target instruction. For native Blackwell block scaling, use the\n complete kind, block/scale-vector qualifier, scale type, and target rules.\n4. Keep preparation in the timed region unless the producer already emits the\n required layout. Otherwise report preprocessing separately.\n5. Validate output accuracy with the exact quantizer, workload, reference,\n tolerance, and accumulation path; then measure end-to-end latency or\n throughput for the actual shapes.\n\n- More groups require more scale elements, but padding and layout determine the\n actual traffic and storage cost.\n- Smaller groups and fractional scales provide more representational freedom;\n they do not guarantee lower error for every tensor or scale-selection method.\n- DeepGEMM's Nc=128 path is evidence for that pinned SM90 implementation, not a\n universal optimum for Hopper.\n- Native instruction support does not make two recipes ABI-compatible. NVFP4,\n MXFP4, and DeepGEMM FP8 differ in payload, scale type, grouping, and layout."}, "reason": {"statement": "Retain only the exact source-supported interval and prohibit transfer without measurement.", "urls": ["https://arxiv.org/abs/2412.19437v2", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh", "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma"]}} {"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "```cuda\n// Blackwell native block scaling with UE8M0\n// Scale format: UE8M0 = pure power-of-two scale (2^exponent)\n// Packed: 4 UE8M0 values per 32-bit integer\n\n// DeepGEMM SM100 kernel: scale_A and scale_B are UE8M0 packed\n// tcgen05.mma applies scales automatically during accumulation\n\nstruct BlockScaleDescriptor {\n // 4 UE8M0 scale values packed into one uint32\n // Each UE8M0 is an 8-bit unsigned exponent: value = 2^(e - 127)\n uint32_t packed_scales; // Contains 4 block scales\n\n // Dequantization for block [i]:\n // scale_i = 2^(((packed >> (i*8)) & 0xFF) - 127)\n};\n\n// PTX for tcgen05.mma with block scaling:\n// The .scale modifier tells the hardware to apply UE8M0 scales\n// from a designated SMEM region alongside the MMA operands\n```"}, "after": {"statement": "The pinned SM100 1D1D template accepts K scale granularities of 32 or 128 for\neach operand. Its public interface packs four UE8M0 factors in each 32-bit\ncontainer. TMA moves factor blocks to shared memory, UTCCP copies them into\ndedicated TMEM SFA/SFB columns, and block-scaled UMMA consumes the TMEM scale\naddresses. The SM90 `final_accum` promotion loop is absent."}, "reason": {"statement": "Avoid an incomplete invented descriptor; cite the exact type definition and upstream representation.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#alternate-floating-point-data-formats", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"]}} -{"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "```ptx\n// tcgen05.mma with native block scaling (Blackwell PTX)\n// This instruction applies UE8M0 scales from SMEM during MMA\ntcgen05.mma.cta_group::1.kind::f8f6f4\n [%tmem_addr], // TMEM accumulator destination\n [%desc_a], // SMEM descriptor for A operand\n [%desc_b], // SMEM descriptor for B operand\n %scale_d, // Scale descriptor for D (output)\n %enable_mask,\n [%scale_a_smem], // UE8M0 scales for A in SMEM\n [%scale_b_smem]; // UE8M0 scales for B in SMEM\n```"}, "after": {"statement": "PTX ISA 9.0 expresses the corresponding FP8 block-scaled operand classes with\nthis grammar-level form:\n\n```ptx\ntcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.scale_vec::1X\n [d_tmem], a_desc, b_desc, idesc,\n [scale_a_tmem], [scale_b_tmem], enable_input_d;\n```\n\nThis line is an instruction-shape reference, not a complete kernel: declarations,\nlegal descriptors, collective participation, scale layouts, ordering, completion,\nand an architecture-specific target are still required. UE8M0 uses power-of-two\nfinite values and reserves encoding `0xff` for NaN."}, "reason": {"statement": "Show exact grammar-level PTX and label missing declarations/descriptors.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "```ptx\n// tcgen05.mma with native block scaling (Blackwell PTX)\n// This instruction applies UE8M0 scales from SMEM during MMA\ntcgen05.mma.cta_group::1.kind::f8f6f4\n [%tmem_addr], // TMEM accumulator destination\n [%desc_a], // SMEM descriptor for A operand\n [%desc_b], // SMEM descriptor for B operand\n %scale_d, // Scale descriptor for D (output)\n %enable_mask,\n [%scale_a_smem], // UE8M0 scales for A in SMEM\n [%scale_b_smem]; // UE8M0 scales for B in SMEM\n```"}, "after": {"statement": "PTX ISA 9.0 expresses the corresponding FP8 block-scaled operand classes with\nthis grammar-level form:\n\n```ptx\ntcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.scale_vec::1X\n [d_tmem], a_desc, b_desc, idesc,\n [scale_a_tmem], [scale_b_tmem], enable_input_d;\n```\n\nThis line is an instruction-shape reference, not a complete kernel: declarations,\nlegal descriptors, collective participation, scale layouts, ordering, completion,\nand an architecture-specific target are still required. UE8M0 uses power-of-two\nfinite values and reserves encoding `0xff` for NaN."}, "reason": {"statement": "Show exact grammar-level PTX and label missing declarations/descriptors.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma"]}} {"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "| Representable values | Powers of 2 only | 240 distinct values | Full FP32 range |\n| Range | 2^-127 to 2^128 | ~0 to 448 | Full FP32 |"}, "after": {"statement": "The pinned SM100 1D1D template accepts K scale granularities of 32 or 128 for\neach operand. Its public interface packs four UE8M0 factors in each 32-bit\ncontainer. TMA moves factor blocks to shared memory, UTCCP copies them into\ndedicated TMEM SFA/SFB columns, and block-scaled UMMA consumes the TMEM scale\naddresses. The SM90 `final_accum` promotion loop is absent."}, "reason": {"statement": "Use recipe/type combinations without unsupported range/count shorthand.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#alternate-floating-point-data-formats", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"]}} -{"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "| Hardware support | tcgen05.mma native | Software decode | Software promotion |\n| Precision impact | Coarser (power-of-2 only) | Fine (non-power-of-2) | Best (FP32) |"}, "after": {"statement": "The pinned SM100 1D1D template accepts K scale granularities of 32 or 128 for\neach operand. Its public interface packs four UE8M0 factors in each 32-bit\ncontainer. TMA moves factor blocks to shared memory, UTCCP copies them into\ndedicated TMEM SFA/SFB columns, and block-scaled UMMA consumes the TMEM scale\naddresses. The SM90 `final_accum` promotion loop is absent.\n\nPTX ISA 9.0 expresses the corresponding FP8 block-scaled operand classes with\nthis grammar-level form:\n\n```ptx\ntcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.scale_vec::1X\n [d_tmem], a_desc, b_desc, idesc,\n [scale_a_tmem], [scale_b_tmem], enable_input_d;\n```\n\nThis line is an instruction-shape reference, not a complete kernel: declarations,\nlegal descriptors, collective participation, scale layouts, ordering, completion,\nand an architecture-specific target are still required. UE8M0 uses power-of-two\nfinite values and reserves encoding `0xff` for NaN."}, "reason": {"statement": "Separate native instruction legality from recipe accuracy and software choices.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#alternate-floating-point-data-formats"]}} -{"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "- **FP8 training**: Use tile-wise 1x128 for activations and block-wise 128x128 for weights (DeepGEMM pattern). This is the validated approach for training 671B+ parameter models.\n- **FP4 inference on Blackwell**: Use NVFP4 with E4M3 block scales (block size 16) for highest precision, or MXFP4 with UE8M0 scales (block size 32) for native hardware acceleration.\n- **Hopper FP8 inference**: Use CUDA core promotion with Nc=128 interval to maintain precision despite limited TC accumulation."}, "after": {"statement": "PTX ISA 9.0 distinguishes the native formats by the complete instruction\ncombination, not by “FP4” alone:\n\n| `.kind::mxf4.block_scale.block32` / `.scale_vec::2X` | E2M1 | UE8M0 | 32 |\n| `.kind::mxf4nvf4.block_scale.block16` / `.scale_vec::4X` | E2M1 | UE4M3 | 16 |\n\nThe `mxf4nvf4` family also has documented UE8M0 modes, so the kind name alone\ndoes not select the NVFP4 recipe. NVFP4-compatible UE4M3 scaling is native on\nthe documented Blackwell targets; it is not necessarily a software-decode path.\n\n1. Match the model/checkpoint recipe: payload encoding, local scale type,\n optional global scale, and scale geometry.\n2. Match the kernel ABI: logical factor shape is not necessarily its TMA- or\n TMEM-ready physical layout. Account for packing, padding, transposition, and\n swizzling.\n3. Match the target instruction. For native Blackwell block scaling, use the\n complete kind, block/scale-vector qualifier, scale type, and target rules.\n4. Keep preparation in the timed region unless the producer already emits the\n required layout. Otherwise report preprocessing separately.\n5. Validate output accuracy with the exact quantizer, workload, reference,\n tolerance, and accumulation path; then measure end-to-end latency or\n throughput for the actual shapes.\n\n- More groups require more scale elements, but padding and layout determine the\n actual traffic and storage cost.\n- Smaller groups and fractional scales provide more representational freedom;\n they do not guarantee lower error for every tensor or scale-selection method.\n- DeepGEMM's Nc=128 path is evidence for that pinned SM90 implementation, not a\n universal optimum for Hopper.\n- Native instruction support does not make two recipes ABI-compatible. NVFP4,\n MXFP4, and DeepGEMM FP8 differ in payload, scale type, grouping, and layout."}, "reason": {"statement": "Replace prescriptions with a recipe-selection checklist.", "urls": ["https://arxiv.org/abs/2412.19437v2", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"]}} -{"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "- UE8M0 scales are power-of-two only. Non-power-of-two distributions (common in activations) lose precision compared to E4M3 or FP32 scales.\n- Smaller block sizes (16 for NVFP4 vs 128 for DeepGEMM) provide better precision but higher overhead: more scale values to store, load, and apply.\n- The Nc=128 promotion interval on Hopper is a performance-accuracy tradeoff. Reducing Nc improves accuracy but adds more promotion overhead. Increasing Nc risks precision degradation.\n- On Blackwell, native block scaling only works with UE8M0. Using E4M3 or FP32 scales still requires software handling."}, "after": {"statement": "Scale-count and storage overhead are different quantities. A `128 x 128` block\nhas one factor per 16,384 payloads, a factor-count ratio of about 0.0061%. If\nthe payload is one byte and the factor is FP32, the byte ratio is instead\n`4 / 16384`, about 0.0244%. Include factor width, payload width, padding, and\nlayout transformations in any storage or bandwidth claim.\n\n1. Match the model/checkpoint recipe: payload encoding, local scale type,\n optional global scale, and scale geometry.\n2. Match the kernel ABI: logical factor shape is not necessarily its TMA- or\n TMEM-ready physical layout. Account for packing, padding, transposition, and\n swizzling.\n3. Match the target instruction. For native Blackwell block scaling, use the\n complete kind, block/scale-vector qualifier, scale type, and target rules.\n4. Keep preparation in the timed region unless the producer already emits the\n required layout. Otherwise report preprocessing separately.\n5. Validate output accuracy with the exact quantizer, workload, reference,\n tolerance, and accumulation path; then measure end-to-end latency or\n throughput for the actual shapes.\n\n- More groups require more scale elements, but padding and layout determine the\n actual traffic and storage cost.\n- Smaller groups and fractional scales provide more representational freedom;\n they do not guarantee lower error for every tensor or scale-selection method.\n- DeepGEMM's Nc=128 path is evidence for that pinned SM90 implementation, not a\n universal optimum for Hopper.\n- Native instruction support does not make two recipes ABI-compatible. NVFP4,\n MXFP4, and DeepGEMM FP8 differ in payload, scale type, grouping, and layout."}, "reason": {"statement": "State only metadata arithmetic and verified capability; require measurement for accuracy/performance.", "urls": ["https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"]}} +{"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "| Hardware support | tcgen05.mma native | Software decode | Software promotion |\n| Precision impact | Coarser (power-of-2 only) | Fine (non-power-of-2) | Best (FP32) |"}, "after": {"statement": "The pinned SM100 1D1D template accepts K scale granularities of 32 or 128 for\neach operand. Its public interface packs four UE8M0 factors in each 32-bit\ncontainer. TMA moves factor blocks to shared memory, UTCCP copies them into\ndedicated TMEM SFA/SFB columns, and block-scaled UMMA consumes the TMEM scale\naddresses. The SM90 `final_accum` promotion loop is absent.\n\nPTX ISA 9.0 expresses the corresponding FP8 block-scaled operand classes with\nthis grammar-level form:\n\n```ptx\ntcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.scale_vec::1X\n [d_tmem], a_desc, b_desc, idesc,\n [scale_a_tmem], [scale_b_tmem], enable_input_d;\n```\n\nThis line is an instruction-shape reference, not a complete kernel: declarations,\nlegal descriptors, collective participation, scale layouts, ordering, completion,\nand an architecture-specific target are still required. UE8M0 uses power-of-two\nfinite values and reserves encoding `0xff` for NaN."}, "reason": {"statement": "Separate native instruction legality from recipe accuracy and software choices.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#alternate-floating-point-data-formats"]}} +{"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "- **FP8 training**: Use tile-wise 1x128 for activations and block-wise 128x128 for weights (DeepGEMM pattern). This is the validated approach for training 671B+ parameter models.\n- **FP4 inference on Blackwell**: Use NVFP4 with E4M3 block scales (block size 16) for highest precision, or MXFP4 with UE8M0 scales (block size 32) for native hardware acceleration.\n- **Hopper FP8 inference**: Use CUDA core promotion with Nc=128 interval to maintain precision despite limited TC accumulation."}, "after": {"statement": "PTX ISA 9.0 distinguishes the native formats by the complete instruction\ncombination, not by “FP4” alone:\n\n| `.kind::mxf4.block_scale.block32` / `.scale_vec::2X` | E2M1 | UE8M0 | 32 |\n| `.kind::mxf4nvf4.block_scale.block16` / `.scale_vec::4X` | E2M1 | UE4M3 | 16 |\n\nThe `mxf4nvf4` family also has documented UE8M0 modes, so the kind name alone\ndoes not select the NVFP4 recipe. NVFP4-compatible UE4M3 scaling is native on\nthe documented Blackwell targets; it is not necessarily a software-decode path.\n\n1. Match the model/checkpoint recipe: payload encoding, local scale type,\n optional global scale, and scale geometry.\n2. Match the kernel ABI: logical factor shape is not necessarily its TMA- or\n TMEM-ready physical layout. Account for packing, padding, transposition, and\n swizzling.\n3. Match the target instruction. For native Blackwell block scaling, use the\n complete kind, block/scale-vector qualifier, scale type, and target rules.\n4. Keep preparation in the timed region unless the producer already emits the\n required layout. Otherwise report preprocessing separately.\n5. Validate output accuracy with the exact quantizer, workload, reference,\n tolerance, and accumulation path; then measure end-to-end latency or\n throughput for the actual shapes.\n\n- More groups require more scale elements, but padding and layout determine the\n actual traffic and storage cost.\n- Smaller groups and fractional scales provide more representational freedom;\n they do not guarantee lower error for every tensor or scale-selection method.\n- DeepGEMM's Nc=128 path is evidence for that pinned SM90 implementation, not a\n universal optimum for Hopper.\n- Native instruction support does not make two recipes ABI-compatible. NVFP4,\n MXFP4, and DeepGEMM FP8 differ in payload, scale type, grouping, and layout."}, "reason": {"statement": "Replace prescriptions with a recipe-selection checklist.", "urls": ["https://arxiv.org/abs/2412.19437v2", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"]}} +{"path": "wiki/techniques/fine-grained-quantization.md", "before": {"statement": "- UE8M0 scales are power-of-two only. Non-power-of-two distributions (common in activations) lose precision compared to E4M3 or FP32 scales.\n- Smaller block sizes (16 for NVFP4 vs 128 for DeepGEMM) provide better precision but higher overhead: more scale values to store, load, and apply.\n- The Nc=128 promotion interval on Hopper is a performance-accuracy tradeoff. Reducing Nc improves accuracy but adds more promotion overhead. Increasing Nc risks precision degradation.\n- On Blackwell, native block scaling only works with UE8M0. Using E4M3 or FP32 scales still requires software handling."}, "after": {"statement": "Scale-count and storage overhead are different quantities. A `128 x 128` block\nhas one factor per 16,384 payloads, a factor-count ratio of about 0.0061%. If\nthe payload is one byte and the factor is FP32, the byte ratio is instead\n`4 / 16384`, about 0.0244%. Include factor width, payload width, padding, and\nlayout transformations in any storage or bandwidth claim.\n\n1. Match the model/checkpoint recipe: payload encoding, local scale type,\n optional global scale, and scale geometry.\n2. Match the kernel ABI: logical factor shape is not necessarily its TMA- or\n TMEM-ready physical layout. Account for packing, padding, transposition, and\n swizzling.\n3. Match the target instruction. For native Blackwell block scaling, use the\n complete kind, block/scale-vector qualifier, scale type, and target rules.\n4. Keep preparation in the timed region unless the producer already emits the\n required layout. Otherwise report preprocessing separately.\n5. Validate output accuracy with the exact quantizer, workload, reference,\n tolerance, and accumulation path; then measure end-to-end latency or\n throughput for the actual shapes.\n\n- More groups require more scale elements, but padding and layout determine the\n actual traffic and storage cost.\n- Smaller groups and fractional scales provide more representational freedom;\n they do not guarantee lower error for every tensor or scale-selection method.\n- DeepGEMM's Nc=128 path is evidence for that pinned SM90 implementation, not a\n universal optimum for Hopper.\n- Native instruction support does not make two recipes ABI-compatible. NVFP4,\n MXFP4, and DeepGEMM FP8 differ in payload, scale type, grouping, and layout."}, "reason": {"statement": "State only metadata arithmetic and verified capability; require measurement for accuracy/performance.", "urls": ["https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md"]}} {"path": "wiki/techniques/kernel-fusion.md", "before": {"statement": "Kernel fusion combines multiple operations into a single kernel launch, eliminating intermediate global memory roundtrips. Critical for MoE and attention pipelines where 5-7 sequential launches each incur latency, synchronization, and memory traffic overhead."}, "after": {"statement": "Kernel fusion implements two or more dependent logical operations inside one\nGPU kernel launch. It can remove a launch or an intermediate global-memory\nwrite/read only when the chosen unfused baseline would perform that work and\nthe fused implementation keeps the value on chip. Fusion is therefore an\nimplementation property, not a conclusion that follows from an operation's\nname.\n\nReport a benefit only for a defined fused/unfused pair with identical input and\noutput semantics. Count actual launches and bytes for the pinned implementation;\ndo not transfer framework-level launch counts or traffic percentages from an\nunpinned configuration."}, "reason": {"statement": "Define fusion semantically and state conditional benefits.", "urls": []}} -{"path": "wiki/techniques/kernel-fusion.md", "before": {"statement": "```cuda\n// Instead of: gate_gemm → up_gemm → silu → multiply (4 kernels)\n// Fused: single kernel with two TMEM accumulators\n__global__ void fused_gate_up_silu(...) {\n uint32_t tmem_gate = tmem_alloc(256);\n uint32_t tmem_up = tmem_alloc(256);\n\n for (int k = 0; k < K; k += BLOCK_K) {\n tcgen05_mma(x_smem, w_gate_smem, tmem_gate);\n tcgen05_mma(x_smem, w_up_smem, tmem_up);\n }\n\n float g = tmem_load(tmem_gate);\n float u = tmem_load(tmem_up);\n output = (g / (1.0f + expf(-g))) * u; // SwiGLU fused\n}\n```"}, "after": {"statement": "GPU Mode NVFP4 Challenge 3 fixes this operation for each batch index:\n\n```python\ngate = scaled_mm(a, b1.T, sfa, sfb1)\nup = scaled_mm(a, b2.T, sfa, sfb2)\noutput = silu(gate) * up\n```\n\nThe two products share `a` and its scale tensor but use separate B operands and\nscales. This operation graph permits reuse and fused pointwise output, but the\npinned correctness reference does not prescribe launch count, TMEM partition,\nTMA pipeline, or instruction schedule. A concatenated gate/up projection or two\nlogical accumulators can both implement compatible observable semantics when\ntheir tensor contracts match."}, "reason": {"statement": "Use the exact operation reference and keep schedule choices separate.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", "https://github.com/gpu-mode/reference-kernels/blob/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm/reference.py"]}} +{"path": "wiki/techniques/kernel-fusion.md", "before": {"statement": "```cuda\n// Instead of: gate_gemm → up_gemm → silu → multiply (4 kernels)\n// Fused: single kernel with two TMEM accumulators\n__global__ void fused_gate_up_silu(...) {\n uint32_t tmem_gate = tmem_alloc(256);\n uint32_t tmem_up = tmem_alloc(256);\n\n for (int k = 0; k < K; k += BLOCK_K) {\n tcgen05_mma(x_smem, w_gate_smem, tmem_gate);\n tcgen05_mma(x_smem, w_up_smem, tmem_up);\n }\n\n float g = tmem_load(tmem_gate);\n float u = tmem_load(tmem_up);\n output = (g / (1.0f + expf(-g))) * u; // SwiGLU fused\n}\n```"}, "after": {"statement": "GPU Mode NVFP4 Challenge 3 fixes this operation for each batch index:\n\n```python\ngate = scaled_mm(a, b1.T, sfa, sfb1)\nup = scaled_mm(a, b2.T, sfa, sfb2)\noutput = silu(gate) * up\n```\n\nThe two products share `a` and its scale tensor but use separate B operands and\nscales. This operation graph permits reuse and fused pointwise output, but the\npinned correctness reference does not prescribe launch count, TMEM partition,\nTMA pipeline, or instruction schedule. A concatenated gate/up projection or two\nlogical accumulators can both implement compatible observable semantics when\ntheir tensor contracts match."}, "reason": {"statement": "Use the exact operation reference and keep schedule choices separate.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit", "https://github.com/gpu-mode/reference-kernels/blob/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm/reference.py"]}} {"path": "wiki/techniques/kernel-fusion.md", "before": {"statement": "- **vLLM (7 kernels)**: softmax → topk → dispatch → gate → up → silu_mul → down+combine\n- **SGLang (5 kernels)**: router+topk → dispatch → fused gate-up-silu → down → combine\n- **Ideal (1-2 kernels)**: all ops in one launch, saves 21.9% activation memory traffic"}, "after": {"statement": "FlashInfer MLSys 2026 Track A includes sigmoid routing, grouped expert\nselection, two grouped GEMMs, SwiGLU, and weighted expert accumulation in one\nlogical benchmark definition. The exact reference performs one concatenated\nW13 projection, splits its 4096 columns into gate and up halves, applies\nSwiGLU, then performs W2. The term “Fused MoE” does not establish that a\nsubmission uses one launch or keeps every intermediate on chip.\n\nFor correctness, compare the fused and unfused paths on identical tensors and\nexercise empty/small groups, partial tiles, extreme activations, routing ties,\nand every supported scale/layout variant. Add negative tests for premature\nbuffer reuse, omitted completion, wrong routing weights, and invalid output\nedges."}, "reason": {"statement": "Do not preserve unpinned framework-specific performance claims.", "urls": ["https://bench.flashinfer.ai/kernels/moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048", "https://github.com/flashinfer-ai/flashinfer/blob/7f614b86470180bab2d22e36fd1775791c6bf3e6/flashinfer/trace/templates/moe.py"]}} -{"path": "wiki/techniques/kernel-fusion.md", "before": {"statement": "- TMEM capacity limits how many accumulators can fuse (256 cols total)"}, "after": {"statement": "- **Semantics:** preserve routing, scale, bias, activation, reduction,\n accumulation, dtype, ordering, and output-edge behavior. A pointwise epilogue\n is easier to compose than a cross-CTA reduction or global routing decision.\n- **Communication:** identify the producer and every consumer of each\n intermediate. Use the synchronization scope and memory space actually needed;\n fusion is not restricted to one CTA, but multi-CTA communication has its own\n legality and completion rules.\n- **Resources:** record registers, spills, SMEM, TMEM, barriers, threads, cluster\n shape, and occupancy. Added live values can raise register or shared-memory\n pressure even when a global intermediate disappears.\n- **TMEM:** SM100 TMEM is 128 lanes by 512 columns of 32-bit cells per SM.\n Simultaneously live regions must fit, use legal collective allocation sizes,\n remain live through every asynchronous access, and be collectively freed.\n Two 256-column halves are one possible allocation policy, not the total\n hardware capacity or a dual-GEMM requirement.\n- **Fallbacks:** keep unfused or differently fused paths for shapes, dtypes,\n layouts, reductions, or resource footprints that the fused schedule cannot\n implement safely."}, "reason": {"statement": "State 512-column capacity and implementation-specific live-region accounting.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} -{"path": "wiki/techniques/kernel-fusion.md", "before": {"statement": "- Fusion opportunities depend on dataflow shape (dependency graph must be DAG-compatible with CTA scope)"}, "after": {"statement": "- **Semantics:** preserve routing, scale, bias, activation, reduction,\n accumulation, dtype, ordering, and output-edge behavior. A pointwise epilogue\n is easier to compose than a cross-CTA reduction or global routing decision.\n- **Communication:** identify the producer and every consumer of each\n intermediate. Use the synchronization scope and memory space actually needed;\n fusion is not restricted to one CTA, but multi-CTA communication has its own\n legality and completion rules.\n- **Resources:** record registers, spills, SMEM, TMEM, barriers, threads, cluster\n shape, and occupancy. Added live values can raise register or shared-memory\n pressure even when a global intermediate disappears.\n- **TMEM:** SM100 TMEM is 128 lanes by 512 columns of 32-bit cells per SM.\n Simultaneously live regions must fit, use legal collective allocation sizes,\n remain live through every asynchronous access, and be collectively freed.\n Two 256-column halves are one possible allocation policy, not the total\n hardware capacity or a dual-GEMM requirement.\n- **Fallbacks:** keep unfused or differently fused paths for shapes, dtypes,\n layouts, reductions, or resource footprints that the fused schedule cannot\n implement safely."}, "reason": {"statement": "Replace vague scope jargon with explicit correctness/resource questions.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} -{"path": "wiki/techniques/register-budgeting.md", "before": {"statement": "SM occupancy is inversely proportional to registers-per-thread. For memory-bound kernels, higher occupancy = more warps to hide memory latency. `-maxrregcount` and `__launch_bounds__` force the compiler to stay within a budget."}, "after": {"statement": "Registers per thread are one input to residency. Threads per block, static and dynamic shared memory, architectural limits, allocation granularity, barriers, and cluster configuration can impose other limits. Register thresholds therefore produce discrete changes in active blocks or warps; occupancy is not generally the reciprocal of a source-level register count, and higher predicted occupancy does not by itself imply lower latency.\n\nThe CUDA occupancy API evaluates the compiled function with an intended block size and dynamic shared-memory size. `cudaOccupancyMaxActiveBlocksPerMultiprocessor` returns the maximum active blocks per SM for that configuration. Convert that result to active warps only after multiplying by the actual warps per block.\n\nFor tcgen05 MMA, the resident D accumulator is in TMEM rather than a per-thread D register vector. This changes one major register consumer, but it does not erase register pressure: descriptors, addresses, pipeline/control state, operands, and epilogue batches still use general-purpose registers. The pinned SM100 DeepGEMM path explicitly loads completed TMEM values into registers before its shared/global-store epilogue.\n\nTreat the removed D vector as shape-specific. For example, an SM90 m64n256 FP32 WGMMA fragment exposes 128 D registers per participating thread, but that number is not a universal Blackwell saving and does not predict a resident-block transition."}, "reason": {"statement": "Use the resource-threshold model and scope the TMEM effect to the resident D fragment.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-runtime-api/group__CUDART__OCCUPANCY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/arch/mma_sm90_gmma.hpp"]}} +{"path": "wiki/techniques/kernel-fusion.md", "before": {"statement": "- TMEM capacity limits how many accumulators can fuse (256 cols total)"}, "after": {"statement": "- **Semantics:** preserve routing, scale, bias, activation, reduction,\n accumulation, dtype, ordering, and output-edge behavior. A pointwise epilogue\n is easier to compose than a cross-CTA reduction or global routing decision.\n- **Communication:** identify the producer and every consumer of each\n intermediate. Use the synchronization scope and memory space actually needed;\n fusion is not restricted to one CTA, but multi-CTA communication has its own\n legality and completion rules.\n- **Resources:** record registers, spills, SMEM, TMEM, barriers, threads, cluster\n shape, and occupancy. Added live values can raise register or shared-memory\n pressure even when a global intermediate disappears.\n- **TMEM:** SM100 TMEM is 128 lanes by 512 columns of 32-bit cells per SM.\n Simultaneously live regions must fit, use legal collective allocation sizes,\n remain live through every asynchronous access, and be collectively freed.\n Two 256-column halves are one possible allocation policy, not the total\n hardware capacity or a dual-GEMM requirement.\n- **Fallbacks:** keep unfused or differently fused paths for shapes, dtypes,\n layouts, reductions, or resource footprints that the fused schedule cannot\n implement safely."}, "reason": {"statement": "State 512-column capacity and implementation-specific live-region accounting.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} +{"path": "wiki/techniques/kernel-fusion.md", "before": {"statement": "- Fusion opportunities depend on dataflow shape (dependency graph must be DAG-compatible with CTA scope)"}, "after": {"statement": "- **Semantics:** preserve routing, scale, bias, activation, reduction,\n accumulation, dtype, ordering, and output-edge behavior. A pointwise epilogue\n is easier to compose than a cross-CTA reduction or global routing decision.\n- **Communication:** identify the producer and every consumer of each\n intermediate. Use the synchronization scope and memory space actually needed;\n fusion is not restricted to one CTA, but multi-CTA communication has its own\n legality and completion rules.\n- **Resources:** record registers, spills, SMEM, TMEM, barriers, threads, cluster\n shape, and occupancy. Added live values can raise register or shared-memory\n pressure even when a global intermediate disappears.\n- **TMEM:** SM100 TMEM is 128 lanes by 512 columns of 32-bit cells per SM.\n Simultaneously live regions must fit, use legal collective allocation sizes,\n remain live through every asynchronous access, and be collectively freed.\n Two 256-column halves are one possible allocation policy, not the total\n hardware capacity or a dual-GEMM requirement.\n- **Fallbacks:** keep unfused or differently fused paths for shapes, dtypes,\n layouts, reductions, or resource footprints that the fused schedule cannot\n implement safely."}, "reason": {"statement": "Replace vague scope jargon with explicit correctness/resource questions.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#occupancy-calculator"]}} +{"path": "wiki/techniques/register-budgeting.md", "before": {"statement": "SM occupancy is inversely proportional to registers-per-thread. For memory-bound kernels, higher occupancy = more warps to hide memory latency. `-maxrregcount` and `__launch_bounds__` force the compiler to stay within a budget."}, "after": {"statement": "Registers per thread are one input to residency. Threads per block, static and dynamic shared memory, architectural limits, allocation granularity, barriers, and cluster configuration can impose other limits. Register thresholds therefore produce discrete changes in active blocks or warps; occupancy is not generally the reciprocal of a source-level register count, and higher predicted occupancy does not by itself imply lower latency.\n\nThe CUDA occupancy API evaluates the compiled function with an intended block size and dynamic shared-memory size. `cudaOccupancyMaxActiveBlocksPerMultiprocessor` returns the maximum active blocks per SM for that configuration. Convert that result to active warps only after multiplying by the actual warps per block.\n\nFor tcgen05 MMA, the resident D accumulator is in TMEM rather than a per-thread D register vector. This changes one major register consumer, but it does not erase register pressure: descriptors, addresses, pipeline/control state, operands, and epilogue batches still use general-purpose registers. The pinned SM100 DeepGEMM path explicitly loads completed TMEM values into registers before its shared/global-store epilogue.\n\nTreat the removed D vector as shape-specific. For example, an SM90 m64n256 FP32 WGMMA fragment exposes 128 D registers per participating thread, but that number is not a universal Blackwell saving and does not predict a resident-block transition."}, "reason": {"statement": "Use the resource-threshold model and scope the TMEM effect to the resident D fragment.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-runtime-api/group__CUDART__OCCUPANCY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cute/arch/mma_sm90_gmma.hpp"]}} {"path": "wiki/techniques/register-budgeting.md", "before": {"statement": "```cuda\n// Aggressive: 32 registers/thread → ~4 blocks per SM at 256 threads/block\n__launch_bounds__(256, 4)\n__global__ void gemv_memory_bound(...) {\n // Compiler will spill to local memory if needed\n}\n\n// Or via nvcc flag:\n// nvcc -maxrregcount=32 -arch=sm_100a ...\n```"}, "after": {"statement": "| `--maxrregcount=N` | Sets a maximum for GPU functions, subject to the ABI minimum and compiler-reserved registers | That the compiler will use exactly `N`, or that residency/performance will change |\n| `__launch_bounds__(T, B)` | Supplies maximum threads per block and desired minimum blocks per SM; the compiler derives an architecture-dependent register limit `L` | Exactly `B` resident blocks when shared memory or another resource is limiting |\n| `--resource-usage` | Reports registers, stack frame, spill loads/stores, and other resources for compiled functions | Runtime occupancy or the cost of those resources |\n| Occupancy API | Predicts maximum concurrent blocks for the compiled function and launch inputs | Achieved latency, bandwidth, or throughput |\n\nWhen launch bounds make initial register use exceed `L`, CUDA documents that the compiler usually trades registers for more local-memory use and/or instructions. When both `T` and `B` are supplied and initial use is below `L`, the compiler may instead increase register use up to `L` to reduce instruction count. A launch bound is therefore compiler guidance and a launch constraint, not an exact register assignment.\n\nCompile the same source and target first without a cap, then with candidate caps. `--resource-usage` makes a cap that does not change the compiled allocation visible.\n\n```bash\nnvcc -arch=sm_100a --resource-usage kernel.cu -o kernel-default\nnvcc -arch=sm_100a --maxrregcount=64 --resource-usage kernel.cu -o kernel-r64\n```\n\nFor each binary and each supported production shape:\n\n1. Record toolkit, target, compiler options, registers per thread, stack-frame bytes, spill loads/stores, static/dynamic shared memory, block size, and cluster shape.\n2. Query active blocks with the occupancy API using the exact function, block size, and dynamic shared-memory bytes. Identify which resource is actually limiting residency.\n3. Verify that the candidate cap changed generated resources or code before attributing a timing change to it. Inspect PTX/SASS when instruction selection or loop shape may have changed.\n4. Run the same correctness oracle, warmup, synchronization, inputs, and repeated-trial statistic for every variant. Profile spill traffic, memory stalls, and achieved warps rather than assuming the predicted occupancy is reached or useful.\n5. Keep a cap only for the measured target and workload where it improves the declared metric without violating correctness or a resource contract."}, "reason": {"statement": "The fenced snippet is incomplete CUDA and encodes false numeric implications.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html"]}} {"path": "wiki/techniques/register-budgeting.md", "before": {"statement": "Lower register count → compiler may:\n- Spill frequently-used values to local memory (bad)\n- Recompute values instead of storing them (neutral)\n- Use fewer unrolled iterations (bad for compute-bound)"}, "after": {"statement": "| `--maxrregcount=N` | Sets a maximum for GPU functions, subject to the ABI minimum and compiler-reserved registers | That the compiler will use exactly `N`, or that residency/performance will change |\n| `__launch_bounds__(T, B)` | Supplies maximum threads per block and desired minimum blocks per SM; the compiler derives an architecture-dependent register limit `L` | Exactly `B` resident blocks when shared memory or another resource is limiting |\n| `--resource-usage` | Reports registers, stack frame, spill loads/stores, and other resources for compiled functions | Runtime occupancy or the cost of those resources |\n| Occupancy API | Predicts maximum concurrent blocks for the compiled function and launch inputs | Achieved latency, bandwidth, or throughput |\n\nWhen launch bounds make initial register use exceed `L`, CUDA documents that the compiler usually trades registers for more local-memory use and/or instructions. When both `T` and `B` are supplied and initial use is below `L`, the compiler may instead increase register use up to `L` to reduce instruction count. A launch bound is therefore compiler guidance and a launch constraint, not an exact register assignment.\n\nCompile the same source and target first without a cap, then with candidate caps. `--resource-usage` makes a cap that does not change the compiled allocation visible.\n\n```bash\nnvcc -arch=sm_100a --resource-usage kernel.cu -o kernel-default\nnvcc -arch=sm_100a --maxrregcount=64 --resource-usage kernel.cu -o kernel-r64\n```\n\nFor each binary and each supported production shape:\n\n1. Record toolkit, target, compiler options, registers per thread, stack-frame bytes, spill loads/stores, static/dynamic shared memory, block size, and cluster shape.\n2. Query active blocks with the occupancy API using the exact function, block size, and dynamic shared-memory bytes. Identify which resource is actually limiting residency.\n3. Verify that the candidate cap changed generated resources or code before attributing a timing change to it. Inspect PTX/SASS when instruction selection or loop shape may have changed.\n4. Run the same correctness oracle, warmup, synchronization, inputs, and repeated-trial statistic for every variant. Profile spill traffic, memory stalls, and achieved warps rather than assuming the predicted occupancy is reached or useful.\n5. Keep a cap only for the measured target and workload where it improves the declared metric without violating correctness or a resource contract."}, "reason": {"statement": "Describe compiler responses as possible costs to measure.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html"]}} {"path": "wiki/techniques/register-budgeting.md", "before": {"statement": "For memory-bound kernels, spills can be hidden by memory latency anyway, so aggressive budgeting often wins."}, "after": {"statement": "Spill loads and stores use thread-local stack memory backed by the memory hierarchy. They add instructions and traffic; a memory-bound label is not evidence that this cost will be hidden. Conversely, a lower cap can be irrelevant when the uncapped allocation is already below it."}, "reason": {"statement": "Require controlled variants instead of a directional guarantee.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"]}} {"path": "wiki/techniques/register-budgeting.md", "before": {"statement": "| 1 | 32 | 18.5μs |\n| 3 | 45 | ~20μs |"}, "after": {"statement": "Amandeep Singh reports that three inspected B200 NVFP4 GEMV solutions clustered around an 18.5 microsecond geometric mean, with a 32-register cap in rank 1 and 45 in rank 3. The same report says those solutions also differed in PTX decode, cache policies, load widths, exact-K specialization, and B reuse; it provides no controlled register-cap ablation or public contestant code. In the author's own kernel, lowering the cap from 80 to 64 had no effect because natural use was already below 64, while extra accumulator chains and software pipelining increased pressure and regressed.\n\nThat observation motivates inspecting register allocation; it does not prove that 32 beats 45, that occupancy caused the ranking, or that aggressive caps generally help memory-bound kernels."}, "reason": {"statement": "Preserve only the author's scoped observation and its provenance limit.", "urls": ["https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"]}} {"path": "wiki/techniques/register-budgeting.md", "before": {"statement": "The measurable difference between 32 and 45 registers shows occupancy dominates for memory-bound NVFP4 GEMV."}, "after": {"statement": "Spill loads and stores use thread-local stack memory backed by the memory hierarchy. They add instructions and traffic; a memory-bound label is not evidence that this cost will be hidden. Conversely, a lower cap can be irrelevant when the uncapped allocation is already below it.\n\nAmandeep Singh reports that three inspected B200 NVFP4 GEMV solutions clustered around an 18.5 microsecond geometric mean, with a 32-register cap in rank 1 and 45 in rank 3. The same report says those solutions also differed in PTX decode, cache policies, load widths, exact-K specialization, and B reuse; it provides no controlled register-cap ablation or public contestant code. In the author's own kernel, lowering the cap from 80 to 64 had no effect because natural use was already below 64, while extra accumulator chains and software pipelining increased pressure and regressed.\n\nThat observation motivates inspecting register allocation; it does not prove that 32 beats 45, that occupancy caused the ranking, or that aggressive caps generally help memory-bound kernels."}, "reason": {"statement": "Do not infer causality from confounded implementations.", "urls": ["https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html"]}} -{"path": "wiki/techniques/register-budgeting.md", "before": {"statement": "- Memory-bound kernels (first priority: occupancy)\n- Kernels where register pressure comes from inner loop, not accumulators (TMEM handles accumulators)\n- Sub-byte types with heavy decode/scale computation\n\n- Compute-bound GEMM (let compiler use what it needs)\n- Kernels where spills to local memory would serialize"}, "after": {"statement": "Compile the same source and target first without a cap, then with candidate caps. `--resource-usage` makes a cap that does not change the compiled allocation visible.\n\n```bash\nnvcc -arch=sm_100a --resource-usage kernel.cu -o kernel-default\nnvcc -arch=sm_100a --maxrregcount=64 --resource-usage kernel.cu -o kernel-r64\n```\n\nFor each binary and each supported production shape:\n\n1. Record toolkit, target, compiler options, registers per thread, stack-frame bytes, spill loads/stores, static/dynamic shared memory, block size, and cluster shape.\n2. Query active blocks with the occupancy API using the exact function, block size, and dynamic shared-memory bytes. Identify which resource is actually limiting residency.\n3. Verify that the candidate cap changed generated resources or code before attributing a timing change to it. Inspect PTX/SASS when instruction selection or loop shape may have changed.\n4. Run the same correctness oracle, warmup, synchronization, inputs, and repeated-trial statistic for every variant. Profile spill traffic, memory stalls, and achieved warps rather than assuming the predicted occupancy is reached or useful.\n5. Keep a cap only for the measured target and workload where it improves the declared metric without violating correctness or a resource contract.\n\nSpill loads and stores use thread-local stack memory backed by the memory hierarchy. They add instructions and traffic; a memory-bound label is not evidence that this cost will be hidden. Conversely, a lower cap can be irrelevant when the uncapped allocation is already below it.\n\nFor tcgen05 MMA, the resident D accumulator is in TMEM rather than a per-thread D register vector. This changes one major register consumer, but it does not erase register pressure: descriptors, addresses, pipeline/control state, operands, and epilogue batches still use general-purpose registers. The pinned SM100 DeepGEMM path explicitly loads completed TMEM values into registers before its shared/global-store epilogue.\n\nTreat the removed D vector as shape-specific. For example, an SM90 m64n256 FP32 WGMMA fragment exposes 128 D registers per participating thread, but that number is not a universal Blackwell saving and does not predict a resident-block transition."}, "reason": {"statement": "Replace categories with a decision procedure based on the compiled kernel and controlled benchmarks.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh", "https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/arch/mma_sm90_gmma.hpp"]}} +{"path": "wiki/techniques/register-budgeting.md", "before": {"statement": "- Memory-bound kernels (first priority: occupancy)\n- Kernels where register pressure comes from inner loop, not accumulators (TMEM handles accumulators)\n- Sub-byte types with heavy decode/scale computation\n\n- Compute-bound GEMM (let compiler use what it needs)\n- Kernels where spills to local memory would serialize"}, "after": {"statement": "Compile the same source and target first without a cap, then with candidate caps. `--resource-usage` makes a cap that does not change the compiled allocation visible.\n\n```bash\nnvcc -arch=sm_100a --resource-usage kernel.cu -o kernel-default\nnvcc -arch=sm_100a --maxrregcount=64 --resource-usage kernel.cu -o kernel-r64\n```\n\nFor each binary and each supported production shape:\n\n1. Record toolkit, target, compiler options, registers per thread, stack-frame bytes, spill loads/stores, static/dynamic shared memory, block size, and cluster shape.\n2. Query active blocks with the occupancy API using the exact function, block size, and dynamic shared-memory bytes. Identify which resource is actually limiting residency.\n3. Verify that the candidate cap changed generated resources or code before attributing a timing change to it. Inspect PTX/SASS when instruction selection or loop shape may have changed.\n4. Run the same correctness oracle, warmup, synchronization, inputs, and repeated-trial statistic for every variant. Profile spill traffic, memory stalls, and achieved warps rather than assuming the predicted occupancy is reached or useful.\n5. Keep a cap only for the measured target and workload where it improves the declared metric without violating correctness or a resource contract.\n\nSpill loads and stores use thread-local stack memory backed by the memory hierarchy. They add instructions and traffic; a memory-bound label is not evidence that this cost will be hidden. Conversely, a lower cap can be irrelevant when the uncapped allocation is already below it.\n\nFor tcgen05 MMA, the resident D accumulator is in TMEM rather than a per-thread D register vector. This changes one major register consumer, but it does not erase register pressure: descriptors, addresses, pipeline/control state, operands, and epilogue batches still use general-purpose registers. The pinned SM100 DeepGEMM path explicitly loads completed TMEM values into registers before its shared/global-store epilogue.\n\nTreat the removed D vector as shape-specific. For example, an SM90 m64n256 FP32 WGMMA fragment exposes 128 D registers per participating thread, but that number is not a universal Blackwell saving and does not predict a resident-block transition."}, "reason": {"statement": "Replace categories with a decision procedure based on the compiled kernel and controlled benchmarks.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cute/arch/mma_sm90_gmma.hpp"]}} {"path": "wiki/techniques/software-exp.md", "before": {"statement": "FlashAttention-4 replaces the hardware Special Function Unit (SFU) exponential (`ex2.approx`) with a software-emulated 2^x function that distributes computation across the SM's FMA (fused multiply-add) units. On Blackwell, tensor core throughput doubled compared to Hopper while SFU count remained the same, making the SFU the throughput bottleneck for attention's softmax operation. The software exponential uses Cody-Waite range reduction followed by a Horner-form polynomial evaluation, achieving sufficient accuracy for attention while bypassing the SFU entirely."}, "after": {"statement": "FlashAttention-4 (FA4) combines two paths for base-2 exponential in its Blackwell forward softmax. Most selected configurations retain hardware `exp2` for some entries and evaluate a tunable fraction with a software polynomial on general-purpose FMA pipelines. The paper describes roughly 10–25% software evaluation, while the exact fraction is a configuration choice rather than an architecture constant.\n\nThis hybrid is part of a larger schedule: two 128-thread softmax warpgroups alternate 128-row query tiles, synchronize to limit simultaneous exponential contention, stage probabilities through TMEM, and hand conditional rescaling to a correction warpgroup. A standalone polynomial is not equivalent to that pipeline.\n\nAt Dao-AILab/flash-attention revision `a369df707e1980fb328abcc1733e3457ec10155f`, [`flash_attn/cute/utils.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py) defines the default software path and [`flash_attn/cute/softmax.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/softmax.py) performs fragment-level hardware/software selection.\n\nFor each software-selected pair, the pinned implementation:\n\n1. Assumes each input is no greater than 127 and clamps values below `-127`.\n2. Uses a rounding-down addition with the float32 constant `2^23 + 2^22` to recover the integer floor and a fractional value in `[0,1)`.\n3. Evaluates a degree-3 polynomial in Horner form with packed float32 FMA operations.\n4. Combines the integer contribution with the polynomial's float32 representation by exponent-field integer operations; the software branch does not call `ex2.approx` for reconstruction.\n\nThe degree-3 float32 coefficients in that revision are:\n\n| `p0` | `1.0` |\n| `p1` | `0.695146143436431884765625` |\n| `p2` | `0.227564394474029541015625` |\n| `p3` | `0.077119089663028717041015625` |\n\nThe authors state that Sollya selected these coefficients to minimize relative error for `2^f` over `f in [0,1)`. For a natural exponential, the caller uses the identity `e^z = 2^(z * log2(e))`."}, "reason": {"statement": "Describe the hybrid degree-3 implementation exactly.", "urls": ["https://tridao.me/blog/2026/flash4/", "https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute", "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py"]}} {"path": "wiki/techniques/software-exp.md", "before": {"statement": "The softmax in attention requires computing `exp(x - max)` for every element of the score matrix. On previous generations, the SFU's `ex2.approx` instruction was fast enough relative to the MMA throughput. On Blackwell:\n\n| Tensor core TFLOPS (BF16) | ~990 | ~2250 | 2.27x |\n| SFU units per SM | 16 | 16 | 1.0x |\n| SFU throughput (exp per cycle) | 16 | 16 | 1.0x |\n| FMA units per SM | 128 | 128 | 1.0x |\n\nThe tensor cores produce 2x more score elements per cycle, but the SFU can only process exp() at the same rate as before. This makes the SFU the bottleneck for any kernel that needs exp() proportional to the number of MMA outputs.\n\nFlashAttention-4's approach: distribute the exp() workload across FMA units (128 per SM) instead of SFU units (16 per SM), achieving 8x the throughput for the exponential computation."}, "after": {"statement": "For the authors' `M=N=D=128` B200 feeds-and-speeds model, one SM supplies 8192 BF16 tensor-core operations per cycle, 16 exponential operations per cycle, and 128 shared-memory bytes per cycle. Their forward-tile accounting assigns 1024 cycles to two MMAs, 1024 cycles to 128×128 exponentials, and 768 cycles to shared-memory traffic.\n\nThose are analytical inputs for that tile and schedule. They do not imply that every kernel with one exponential per MMA output is exponential-bound, nor do functional-unit counts give the throughput or latency of a software approximation. Range reduction, polynomial dependencies, reconstruction, instruction issue, and overlap all remain part of the measured implementation."}, "reason": {"statement": "Retain only the source's scoped feeds-and-speeds model.", "urls": ["https://tridao.me/blog/2026/flash4/"]}} {"path": "wiki/techniques/software-exp.md", "before": {"statement": "Range reduction transforms the input `x` into a small residual that a polynomial can accurately approximate. The Cody-Waite method splits the input into an integer part (for exact power-of-two scaling) and a fractional part (for polynomial approximation):\n\n```cuda\n// Cody-Waite range reduction for 2^x\n// Goal: decompose x = n + r where n is integer, r in [-0.5, 0.5]\n// Then 2^x = 2^n * 2^r, and 2^r is approximated by polynomial\n//\n// The Cody-Waite trick: subtract n using two constants (C1 + C2)\n// to maintain precision when x is large.\n//\n// C1 is the nearest representable float to log2(e) with low-order bits zeroed\n// C2 is the correction: log2(e) - C1\n// This avoids catastrophic cancellation in x - n\n\n__device__ float software_exp2(float x) {\n // Step 1: Range reduction (Cody-Waite)\n // Round x to nearest integer\n float n = rintf(x);\n // High-precision subtraction using two constants\n // C1 and C2 together represent 1.0 in extended precision\n const float C1 = 1.0f; // Exact in float\n const float C2 = 0.0f; // Correction term (zero for 2^x, non-zero for e^x)\n // For 2^x, range reduction is simpler: r = x - n\n float r = x - n; // r in [-0.5, 0.5]\n\n // Step 2: Polynomial approximation of 2^r via Horner's method\n // Minimax polynomial coefficients for 2^r on [-0.5, 0.5]\n // Degree-4 polynomial: sufficient for ~22 bits of accuracy\n const float c0 = 1.0f;\n const float c1 = 0.6931471805599453f; // ln(2)\n const float c2 = 0.2402265069591007f; // ln(2)^2 / 2\n const float c3 = 0.05550410866482158f; // ln(2)^3 / 6\n const float c4 = 0.009618129107628477f; // ln(2)^4 / 24\n\n // Horner evaluation: c0 + r*(c1 + r*(c2 + r*(c3 + r*c4)))\n // Each step is one FMA instruction\n float poly = c4;\n poly = fmaf(poly, r, c3); // FMA 1\n poly = fmaf(poly, r, c2); // FMA 2\n poly = fmaf(poly, r, c1); // FMA 3\n poly = fmaf(poly, r, c0); // FMA 4\n\n // Step 3: Reconstruct 2^x = 2^n * poly\n // Use integer addition to the float exponent field\n int n_int = (int)n;\n // ldexpf multiplies by 2^n by adjusting the exponent bits\n float result = ldexpf(poly, n_int);\n\n return result;\n}\n```"}, "after": {"statement": "At Dao-AILab/flash-attention revision `a369df707e1980fb328abcc1733e3457ec10155f`, [`flash_attn/cute/utils.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py) defines the default software path and [`flash_attn/cute/softmax.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/softmax.py) performs fragment-level hardware/software selection.\n\nFor each software-selected pair, the pinned implementation:\n\n1. Assumes each input is no greater than 127 and clamps values below `-127`.\n2. Uses a rounding-down addition with the float32 constant `2^23 + 2^22` to recover the integer floor and a fractional value in `[0,1)`.\n3. Evaluates a degree-3 polynomial in Horner form with packed float32 FMA operations.\n4. Combines the integer contribution with the polynomial's float32 representation by exponent-field integer operations; the software branch does not call `ex2.approx` for reconstruction.\n\nThe degree-3 float32 coefficients in that revision are:\n\n| `p0` | `1.0` |\n| `p1` | `0.695146143436431884765625` |\n| `p2` | `0.227564394474029541015625` |\n| `p3` | `0.077119089663028717041015625` |\n\nThe authors state that Sollya selected these coefficients to minimize relative error for `2^f` over `f in [0,1)`. For a natural exponential, the caller uses the identity `e^z = 2^(z * log2(e))`.\n\nThe polynomial is approximate. On a deterministic host grid of 1,000,002 evenly spaced points over `[0,1]`, evaluating the rounded degree-3 coefficients against `2^x` produced a maximum sampled relative error of approximately `8.763e-5`. This is a regression observation, not a proof of the continuous maximum and not an end-to-end attention tolerance."}, "reason": {"statement": "Avoid presenting invented CUDA as upstream or accuracy-validated.", "urls": ["https://tridao.me/blog/2026/flash4/", "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py"]}} @@ -376,15 +376,15 @@ {"path": "wiki/techniques/software-exp.md", "before": {"statement": "- The 4-FMA chain has a latency of ~16 cycles (4 dependent FMAs at ~4 cycles each), vs ~20 cycles for SFU `ex2.approx`. Latency is comparable; the win comes from throughput (128 FMA units vs 16 SFU units)."}, "after": {"statement": "For the authors' `M=N=D=128` B200 feeds-and-speeds model, one SM supplies 8192 BF16 tensor-core operations per cycle, 16 exponential operations per cycle, and 128 shared-memory bytes per cycle. Their forward-tile accounting assigns 1024 cycles to two MMAs, 1024 cycles to 128×128 exponentials, and 768 cycles to shared-memory traffic.\n\nThose are analytical inputs for that tile and schedule. They do not imply that every kernel with one exponential per MMA output is exponential-bound, nor do functional-unit counts give the throughput or latency of a software approximation. Range reduction, polynomial dependencies, reconstruction, instruction issue, and overlap all remain part of the measured implementation.\n\nBF16 conversion does not by itself prove that approximation errors are harmless. Per-entry error changes both the softmax numerator and row sum, and masking, all-`-inf` rows, conditional rescaling, underflow clamping, and output accumulation introduce separate boundary cases. Validate the scalar approximation and the complete attention output for the exact dtype and features.\n\nUse the pinned implementation as the starting point; do not reconstruct it from rounded blog coefficients. For each target configuration:\n\n1. Build an all-hardware control and one or more hybrid selections from the same source revision, compiler, target, and launch configuration.\n2. Inspect generated PTX/SASS to confirm which entries use hardware `MUFU.EX2`, which use the expected FMA/range-reduction sequence, and whether register use or spills changed.\n3. Test ordinary, masked, all-masked, causal, variable-length, large-gap, and underflow-heavy rows against a declared higher-precision reference. Record maximum absolute/relative output error and row-sum behavior, not only BF16 agreement on random inputs.\n4. Profile exponential-pipeline pressure, FMA/ALU issue, tensor-core overlap, registers, and spills. A high hardware-exp metric is a lead, not proof that moving more entries to FMA improves the schedule.\n5. Benchmark identical shapes, inputs, warmup, synchronization, clock policy, and repeated-trial statistics. Sweep the software fraction because both zero emulation and mixed settings appear in the pinned target/configuration table.\n\nKeep the hybrid only where both the numerical contract and declared end-to-end metric pass. Do not transfer the choice to another architecture or transcendental function without repeating the reduction, approximation, special-value, generated-code, and performance checks."}, "reason": {"statement": "Do not preserve unattributed cycle counts.", "urls": ["https://tridao.me/blog/2026/flash4/", "https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute"]}} {"path": "wiki/techniques/software-exp.md", "before": {"statement": "- The `ldexpf` or exponent bit-manipulation step for 2^n must handle overflow/underflow (very large/small x). In attention, `x <= 0` always holds, so only underflow toward zero is possible."}, "after": {"statement": "At Dao-AILab/flash-attention revision `a369df707e1980fb328abcc1733e3457ec10155f`, [`flash_attn/cute/utils.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py) defines the default software path and [`flash_attn/cute/softmax.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/softmax.py) performs fragment-level hardware/software selection.\n\nFor each software-selected pair, the pinned implementation:\n\n1. Assumes each input is no greater than 127 and clamps values below `-127`.\n2. Uses a rounding-down addition with the float32 constant `2^23 + 2^22` to recover the integer floor and a fractional value in `[0,1)`.\n3. Evaluates a degree-3 polynomial in Horner form with packed float32 FMA operations.\n4. Combines the integer contribution with the polynomial's float32 representation by exponent-field integer operations; the software branch does not call `ex2.approx` for reconstruction.\n\nThe degree-3 float32 coefficients in that revision are:\n\n| `p0` | `1.0` |\n| `p1` | `0.695146143436431884765625` |\n| `p2` | `0.227564394474029541015625` |\n| `p3` | `0.077119089663028717041015625` |\n\nThe authors state that Sollya selected these coefficients to minimize relative error for `2^f` over `f in [0,1)`. For a natural exponential, the caller uses the identity `e^z = 2^(z * log2(e))`.\n\nThe polynomial is approximate. On a deterministic host grid of 1,000,002 evenly spaced points over `[0,1]`, evaluating the rounded degree-3 coefficients against `2^x` produced a maximum sampled relative error of approximately `8.763e-5`. This is a regression observation, not a proof of the continuous maximum and not an end-to-end attention tolerance."}, "reason": {"statement": "State the exact helper domain and require special-case handling at the caller.", "urls": ["https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py", "https://tridao.me/blog/2026/flash4/"]}} {"path": "wiki/techniques/software-exp.md", "before": {"statement": "- The input `x = (S[i,j] - max) * log2(e)` is always non-positive\n- The softmax output is normalized, so small absolute errors cancel out\n- BF16 output has only 7 mantissa bits anyway\n\nFor applications requiring higher accuracy, a degree-6 polynomial (6 FMAs) achieves near-ULP accuracy across the full float range."}, "after": {"statement": "The polynomial is approximate. On a deterministic host grid of 1,000,002 evenly spaced points over `[0,1]`, evaluating the rounded degree-3 coefficients against `2^x` produced a maximum sampled relative error of approximately `8.763e-5`. This is a regression observation, not a proof of the continuous maximum and not an end-to-end attention tolerance.\n\nBF16 conversion does not by itself prove that approximation errors are harmless. Per-entry error changes both the softmax numerator and row sum, and masking, all-`-inf` rows, conditional rescaling, underflow clamping, and output accumulation introduce separate boundary cases. Validate the scalar approximation and the complete attention output for the exact dtype and features.\n\nUse the pinned implementation as the starting point; do not reconstruct it from rounded blog coefficients. For each target configuration:\n\n1. Build an all-hardware control and one or more hybrid selections from the same source revision, compiler, target, and launch configuration.\n2. Inspect generated PTX/SASS to confirm which entries use hardware `MUFU.EX2`, which use the expected FMA/range-reduction sequence, and whether register use or spills changed.\n3. Test ordinary, masked, all-masked, causal, variable-length, large-gap, and underflow-heavy rows against a declared higher-precision reference. Record maximum absolute/relative output error and row-sum behavior, not only BF16 agreement on random inputs.\n4. Profile exponential-pipeline pressure, FMA/ALU issue, tensor-core overlap, registers, and spills. A high hardware-exp metric is a lead, not proof that moving more entries to FMA improves the schedule.\n5. Benchmark identical shapes, inputs, warmup, synchronization, clock policy, and repeated-trial statistics. Sweep the software fraction because both zero emulation and mixed settings appear in the pinned target/configuration table.\n\nKeep the hybrid only where both the numerical contract and declared end-to-end metric pass. Do not transfer the choice to another architecture or transcendental function without repeating the reduction, approximation, special-value, generated-code, and performance checks."}, "reason": {"statement": "Replace universal accuracy claims with an explicit validation protocol.", "urls": ["https://tridao.me/blog/2026/flash4/", "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/utils.py", "https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute"]}} -{"path": "wiki/techniques/swizzling.md", "before": {"statement": "Shared memory swizzling remaps the linear address layout of a matrix tile in SMEM so that threads accessing consecutive columns (or rows) hit different 32-byte banks rather than the same bank. This eliminates bank conflicts that would otherwise serialize concurrent accesses. On Blackwell (SM100), 128-byte swizzling is mandatory for TMA loads and tcgen05.mma operands. Without it, performance drops to 46% of the achievable throughput for GEMM workloads.\n\nShared memory has 32 banks, each 4 bytes wide (128 bytes total per bank cycle). When a warp accesses a matrix stored in row-major layout, threads in the same warp reading elements from consecutive rows in the same column hit the same bank, causing a 32-way bank conflict.\n\nThe TMA unit on both Hopper and Blackwell encodes the swizzle pattern as part of the tensor descriptor. The tcgen05.mma instruction expects its SMEM operands to already be swizzled in the 128-byte pattern. Using unswizzled data produces incorrect MMA results."}, "after": {"statement": "Shared memory has 32 banks, and successive 32-bit words map to successive banks. For one warp memory request, accesses to different words in the same bank require serialized wavefronts; same-word reads can broadcast and do not create that conflict. The result therefore depends on the exact byte address requested by every participating lane.\n\nA swizzle permutes address chunks so a target access pattern may distribute its requests over banks more evenly. It can reduce conflicts for that pattern, but it does not make every access to the tile conflict-free. A layout favorable to an MMA operand can still be unfavorable to a CUDA-core row, column, transpose, or epilogue access.\n\nFor a TMA-to-tcgen05 path, keep these layers consistent:\n\n1. The tensor map describes how TMA places the global-memory box into shared memory, including interleave and swizzle.\n2. The shared-memory allocation and base address satisfy the alignment and span constraints of that mapping.\n3. The tcgen05 shared-memory descriptor describes the same physical-to-logical layout, leading/stride dimensions, base offset, and swizzle mode expected by the MMA operand.\n\nPTX ISA 9.0 permits ordinary tcgen05 shared-memory descriptor modes with no swizzle and 32B, 64B, or 128B spans, plus a 128B/base-32B mode. In the descriptor's three-bit layout field, ordinary no-swizzle is `0`, 128B/base-32B is `1`, ordinary 128B is `2`, 64B is `4`, and 32B is `6`; values `3`, `5`, and `7` are invalid. Legality also depends on MMA kind, element type, major mode, shape, CTA group, and target.\n\nThe CUDA 13.0.97 Driver API exposes TMA modes including:\n\n| `CU_TENSOR_MAP_SWIZZLE_NONE` | No bank swizzle |\n| `CU_TENSOR_MAP_SWIZZLE_32B` | 16B chunks within a 32B span |\n| `CU_TENSOR_MAP_SWIZZLE_64B` | 16B chunks within a 64B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B` | 16B chunks within a 128B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B` | 32B chunks within a 128B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B_ATOM_64B` | 64B chunks within a 128B span; support is operation/type-specific |\n\nThese names define mappings, not universal datatype or tile-size recommendations. The tensor-map encoder imposes mode-, interleave-, datatype-, alignment-, and box-size constraints; for example, ordinary 128B modes require the inner bounding-box byte span not to exceed 128 bytes. Check the exact archived API for the selected type and operation."}, "reason": {"statement": "Define conflicts per access and mode legality per consumer combination.", "urls": ["https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"]}} +{"path": "wiki/techniques/swizzling.md", "before": {"statement": "Shared memory swizzling remaps the linear address layout of a matrix tile in SMEM so that threads accessing consecutive columns (or rows) hit different 32-byte banks rather than the same bank. This eliminates bank conflicts that would otherwise serialize concurrent accesses. On Blackwell (SM100), 128-byte swizzling is mandatory for TMA loads and tcgen05.mma operands. Without it, performance drops to 46% of the achievable throughput for GEMM workloads.\n\nShared memory has 32 banks, each 4 bytes wide (128 bytes total per bank cycle). When a warp accesses a matrix stored in row-major layout, threads in the same warp reading elements from consecutive rows in the same column hit the same bank, causing a 32-way bank conflict.\n\nThe TMA unit on both Hopper and Blackwell encodes the swizzle pattern as part of the tensor descriptor. The tcgen05.mma instruction expects its SMEM operands to already be swizzled in the 128-byte pattern. Using unswizzled data produces incorrect MMA results."}, "after": {"statement": "Shared memory has 32 banks, and successive 32-bit words map to successive banks. For one warp memory request, accesses to different words in the same bank require serialized wavefronts; same-word reads can broadcast and do not create that conflict. The result therefore depends on the exact byte address requested by every participating lane.\n\nA swizzle permutes address chunks so a target access pattern may distribute its requests over banks more evenly. It can reduce conflicts for that pattern, but it does not make every access to the tile conflict-free. A layout favorable to an MMA operand can still be unfavorable to a CUDA-core row, column, transpose, or epilogue access.\n\nFor a TMA-to-tcgen05 path, keep these layers consistent:\n\n1. The tensor map describes how TMA places the global-memory box into shared memory, including interleave and swizzle.\n2. The shared-memory allocation and base address satisfy the alignment and span constraints of that mapping.\n3. The tcgen05 shared-memory descriptor describes the same physical-to-logical layout, leading/stride dimensions, base offset, and swizzle mode expected by the MMA operand.\n\nPTX ISA 9.0 permits ordinary tcgen05 shared-memory descriptor modes with no swizzle and 32B, 64B, or 128B spans, plus a 128B/base-32B mode. In the descriptor's three-bit layout field, ordinary no-swizzle is `0`, 128B/base-32B is `1`, ordinary 128B is `2`, 64B is `4`, and 32B is `6`; values `3`, `5`, and `7` are invalid. Legality also depends on MMA kind, element type, major mode, shape, CTA group, and target.\n\nThe CUDA 13.0.97 Driver API exposes TMA modes including:\n\n| `CU_TENSOR_MAP_SWIZZLE_NONE` | No bank swizzle |\n| `CU_TENSOR_MAP_SWIZZLE_32B` | 16B chunks within a 32B span |\n| `CU_TENSOR_MAP_SWIZZLE_64B` | 16B chunks within a 64B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B` | 16B chunks within a 128B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B` | 32B chunks within a 128B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B_ATOM_64B` | 64B chunks within a 128B span; support is operation/type-specific |\n\nThese names define mappings, not universal datatype or tile-size recommendations. The tensor-map encoder imposes mode-, interleave-, datatype-, alignment-, and box-size constraints; for example, ordinary 128B modes require the inner bounding-box byte span not to exceed 128 bytes. Check the exact archived API for the selected type and operation."}, "reason": {"statement": "Define conflicts per access and mode legality per consumer combination.", "urls": ["https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"]}} {"path": "wiki/techniques/swizzling.md", "before": {"statement": "The tcgen05-tutorial benchmark progression shows the impact:\n\n```\nNaive (no swizzle): 255 TFLOPS (17% of cuBLAS)\n128B swizzle applied: 695 TFLOPS (46% of cuBLAS)\n ---- 2.7x improvement from swizzling alone ----\n```"}, "after": {"statement": "Gau Nernst's pinned B200 tutorial compares M=N=K=4096 kernels with PyTorch 2.9.1 and CUDA 13. Its 3D TMA version changes from a 16-byte inner tile with no swizzle to a 128-byte inner tile with 128B swizzling and matching tcgen05 descriptors:\n\n| v1b: 3D 16B TMA | 252.81 |\n| v2b: 3D 128B TMA plus 128B swizzle | 695.43 |\n\nThe combined change is approximately 2.75× and the v2b endpoint is about 46% of the tutorial's 1506.74 TFLOP/s cuBLAS result. It is not a bank-conflict or swizzle-only ablation. The author notes that the earlier contiguous `8×16B` tile might already span all 32 banks, lacked Nsight Compute access, and offers wider TMA transfers as an alternative explanation."}, "reason": {"statement": "Preserve endpoints and list the simultaneous changes and unresolved cause.", "urls": ["https://gau-nernst.github.io/tcgen05/"]}} -{"path": "wiki/techniques/swizzling.md", "before": {"statement": "The swizzle function XORs a portion of the column address with the row address to scatter accesses across banks:\n\n```cuda\n// 128-byte swizzle: XOR bits [4:6] of the byte offset with the row index\n// This ensures that consecutive rows accessing the same logical column\n// map to different physical SMEM banks.\n//\n// For a tile stored in SMEM with TILE_N columns of 2-byte elements:\n// byte_offset = row * (TILE_N * sizeof(half)) + col * sizeof(half)\n// swizzled_offset = byte_offset ^ ((row & 0x7) << 4)\n//\n// The mask 0x7 = 3 bits, shift 4 = bits [4:6], giving 8-row periodicity\n// across the 128-byte bank group.\n\n__device__ int swizzle_128B(int row, int col, int stride_bytes) {\n int byte_offset = row * stride_bytes + col * sizeof(half);\n // XOR bits [4:6] of byte offset with low 3 bits of row\n int swizzled = byte_offset ^ ((row & 0x7) << 4);\n return swizzled;\n}\n```\n\nVisually, for an 8-row x 64-column half-precision tile (128 bytes per row):\n\n```\nWithout swizzle (row-major):\n Row 0: bank 0,1,2,...,31 bank 0,1,2,...,31\n Row 1: bank 0,1,2,...,31 bank 0,1,2,...,31\n Row 2: bank 0,1,2,...,31 bank 0,1,2,...,31\n -> Column access = 8-way bank conflict\n\nWith 128B swizzle (XOR pattern):\n Row 0: bank 0,1,2,...,31 bank 0,1,2,...,31\n Row 1: bank 1,2,3,...,0 bank 1,2,3,...,0 (rotated by 1)\n Row 2: bank 2,3,4,...,1 bank 2,3,4,...,1 (rotated by 2)\n -> Column access = conflict-free\n```"}, "after": {"statement": "CUTLASS 4.5.0 defines [`Swizzle`](https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/swizzle.hpp):\n\n- `BBits` is the number of mask bits;\n- `MBase` is the number of least-significant address bits kept invariant; and\n- `SShift` is the distance between the two bit fields.\n\nFor `Swizzle<3,4,3>`, three address bits beginning above the four invariant low bits are XORed with the three-bit field shifted by three positions. Equivalently, address bits 7:9 affect bits 4:6. This is an address transformation, not a general `row & 7` rule: equivalence to row-based indexing depends on stride, byte units, base alignment, and layout composition.\n\nCUTLASS also distinguishes a position-independent composed swizzle layout from a position-dependent swizzle pointer, because hardware swizzling depends on the shared-memory pointer address. Reuse a pinned complete layout/descriptor construction or prove the composition and required base alignment; a `Swizzle` type alone is not a complete TMA or MMA contract."}, "reason": {"statement": "Use exact address-bit semantics and library/descriptor mappings.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/swizzle.hpp", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/pointer_swizzle.hpp"]}} -{"path": "wiki/techniques/swizzling.md", "before": {"statement": "```cuda\n// Creating a TMA descriptor with 128-byte swizzle\n#include \n\nCUtensorMap tensor_map;\n\n// Swizzle mode: CU_TENSOR_MAP_SWIZZLE_128B\n// This tells TMA to apply the 128-byte XOR swizzle pattern\n// when writing data into shared memory\ncuTensorMapEncodeTiled(\n &tensor_map,\n CU_TENSOR_MAP_DATA_TYPE_FLOAT16,\n 2, // 2D tensor\n global_ptr, // global memory base\n global_dims, // {N, M} dimensions\n global_strides, // {N * sizeof(half), sizeof(half)}\n tile_dims, // {TILE_N, TILE_M}\n element_strides, // {1, 1}\n CU_TENSOR_MAP_INTERLEAVE_NONE,\n CU_TENSOR_MAP_SWIZZLE_128B, // 128-byte swizzle\n CU_TENSOR_MAP_L2_PROMOTION_NONE,\n CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE\n);\n```"}, "after": {"statement": "For a TMA-to-tcgen05 path, keep these layers consistent:\n\n1. The tensor map describes how TMA places the global-memory box into shared memory, including interleave and swizzle.\n2. The shared-memory allocation and base address satisfy the alignment and span constraints of that mapping.\n3. The tcgen05 shared-memory descriptor describes the same physical-to-logical layout, leading/stride dimensions, base offset, and swizzle mode expected by the MMA operand.\n\nPTX ISA 9.0 permits ordinary tcgen05 shared-memory descriptor modes with no swizzle and 32B, 64B, or 128B spans, plus a 128B/base-32B mode. In the descriptor's three-bit layout field, ordinary no-swizzle is `0`, 128B/base-32B is `1`, ordinary 128B is `2`, 64B is `4`, and 32B is `6`; values `3`, `5`, and `7` are invalid. Legality also depends on MMA kind, element type, major mode, shape, CTA group, and target.\n\nThe CUDA 13.0.97 Driver API exposes TMA modes including:\n\n| `CU_TENSOR_MAP_SWIZZLE_NONE` | No bank swizzle |\n| `CU_TENSOR_MAP_SWIZZLE_32B` | 16B chunks within a 32B span |\n| `CU_TENSOR_MAP_SWIZZLE_64B` | 16B chunks within a 64B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B` | 16B chunks within a 128B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B` | 32B chunks within a 128B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B_ATOM_64B` | 64B chunks within a 128B span; support is operation/type-specific |\n\nThese names define mappings, not universal datatype or tile-size recommendations. The tensor-map encoder imposes mode-, interleave-, datatype-, alignment-, and box-size constraints; for example, ordinary 128B modes require the inner bounding-box byte span not to exceed 128 bytes. Check the exact archived API for the selected type and operation.\n\nFor `cuTensorMapEncodeTiled`, record and validate all inputs rather than copying only the swizzle enumerator:\n\n1. Use a correctly aligned `CUtensorMap` object and global base pointer.\n2. Supply rank-sized global dimensions, rank-minus-one byte strides (the fastest dimension is implicit), rank-sized box dimensions, and rank-sized element strides.\n3. Satisfy the global alignment, stride, box, interleave, datatype, and selected-swizzle constraints.\n4. Check the returned `CUresult`; do not launch with an output descriptor after encoding failed.\n5. Use a shared-memory base and tcgen05 descriptor that represent the same mapping as the tensor map.\n\nAn invalid encoder combination can fail explicitly. A successfully encoded tensor map paired with the wrong consumer layout may instead read the wrong logical elements, so a successful API return is not a correctness oracle."}, "reason": {"statement": "Replace incomplete code with an encoder validation checklist and pinned full example.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} -{"path": "wiki/techniques/swizzling.md", "before": {"statement": "The available swizzle modes and their use cases:\n\n| `SWIZZLE_NONE` | No remapping | Non-MMA data (flags, scales) |\n| `SWIZZLE_32B` | 32-byte groups | Narrow tiles, small data types |\n| `SWIZZLE_64B` | 64-byte groups | Medium tiles |\n| `SWIZZLE_128B` | 128-byte groups | Standard for BF16/FP16 MMA operands |"}, "after": {"statement": "For a TMA-to-tcgen05 path, keep these layers consistent:\n\n1. The tensor map describes how TMA places the global-memory box into shared memory, including interleave and swizzle.\n2. The shared-memory allocation and base address satisfy the alignment and span constraints of that mapping.\n3. The tcgen05 shared-memory descriptor describes the same physical-to-logical layout, leading/stride dimensions, base offset, and swizzle mode expected by the MMA operand.\n\nPTX ISA 9.0 permits ordinary tcgen05 shared-memory descriptor modes with no swizzle and 32B, 64B, or 128B spans, plus a 128B/base-32B mode. In the descriptor's three-bit layout field, ordinary no-swizzle is `0`, 128B/base-32B is `1`, ordinary 128B is `2`, 64B is `4`, and 32B is `6`; values `3`, `5`, and `7` are invalid. Legality also depends on MMA kind, element type, major mode, shape, CTA group, and target.\n\nThe CUDA 13.0.97 Driver API exposes TMA modes including:\n\n| `CU_TENSOR_MAP_SWIZZLE_NONE` | No bank swizzle |\n| `CU_TENSOR_MAP_SWIZZLE_32B` | 16B chunks within a 32B span |\n| `CU_TENSOR_MAP_SWIZZLE_64B` | 16B chunks within a 64B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B` | 16B chunks within a 128B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B` | 32B chunks within a 128B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B_ATOM_64B` | 64B chunks within a 128B span; support is operation/type-specific |\n\nThese names define mappings, not universal datatype or tile-size recommendations. The tensor-map encoder imposes mode-, interleave-, datatype-, alignment-, and box-size constraints; for example, ordinary 128B modes require the inner bounding-box byte span not to exceed 128 bytes. Check the exact archived API for the selected type and operation."}, "reason": {"statement": "List exact mapping spans and defer legality to producer and consumer contracts.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} -{"path": "wiki/techniques/swizzling.md", "before": {"statement": "In CuTe/CUTLASS, swizzle is expressed as a layout composition:\n\n```cuda\n// CuTe swizzle layout for 128-byte swizzle pattern\n// Swizzle where:\n// B = number of bits in the base (non-swizzled) portion\n// M = number of bits in the mask\n// S = shift amount\n//\n// Swizzle<3, 4, 3> encodes the 128B swizzle:\n// 3 base bits (8-byte alignment)\n// 4 mask bits (16 rows)\n// 3 shift bits (8-column groups)\n\nusing SmemLayoutAtom = decltype(\n composition(\n Swizzle<3, 4, 3>{},\n Layout,\n Stride<_64, _1>>{}\n )\n);\n\n// Tile the atom across the full SMEM tile\nusing SmemLayoutA = decltype(\n tile_to_shape(SmemLayoutAtom{}, Shape, Int>{})\n);\n```"}, "after": {"statement": "CUTLASS 4.5.0 defines [`Swizzle`](https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/swizzle.hpp):\n\n- `BBits` is the number of mask bits;\n- `MBase` is the number of least-significant address bits kept invariant; and\n- `SShift` is the distance between the two bit fields.\n\nFor `Swizzle<3,4,3>`, three address bits beginning above the four invariant low bits are XORed with the three-bit field shifted by three positions. Equivalently, address bits 7:9 affect bits 4:6. This is an address transformation, not a general `row & 7` rule: equivalence to row-based indexing depends on stride, byte units, base alignment, and layout composition.\n\nCUTLASS also distinguishes a position-independent composed swizzle layout from a position-dependent swizzle pointer, because hardware swizzling depends on the shared-memory pointer address. Reuse a pinned complete layout/descriptor construction or prove the composition and required base alignment; a `Swizzle` type alone is not a complete TMA or MMA contract."}, "reason": {"statement": "Define parameters exactly and cite a pinned full implementation rather than a generic atom.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/swizzle.hpp", "https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/pointer_swizzle.hpp"]}} -{"path": "wiki/techniques/swizzling.md", "before": {"statement": "Use `nvprof` or Nsight Compute to verify that swizzling eliminates conflicts:\n\n```python\n# Nsight Compute command to check shared memory bank conflicts\n# Look for \"Shared Memory Bank Conflicts\" metric\n# ncu --metrics l1tex__data_bank_conflicts_pipe_lsu_mem_shared_op_ld.sum \\\n# ./my_kernel\n\n# Expected results:\n# Without swizzle: bank_conflicts >> 0\n# With 128B swizzle: bank_conflicts == 0\n```"}, "after": {"statement": "For `cuTensorMapEncodeTiled`, record and validate all inputs rather than copying only the swizzle enumerator:\n\n1. Use a correctly aligned `CUtensorMap` object and global base pointer.\n2. Supply rank-sized global dimensions, rank-minus-one byte strides (the fastest dimension is implicit), rank-sized box dimensions, and rank-sized element strides.\n3. Satisfy the global alignment, stride, box, interleave, datatype, and selected-swizzle constraints.\n4. Check the returned `CUresult`; do not launch with an output descriptor after encoding failed.\n5. Use a shared-memory base and tcgen05 descriptor that represent the same mapping as the tensor map.\n\nAn invalid encoder combination can fail explicitly. A successfully encoded tensor map paired with the wrong consumer layout may instead read the wrong logical elements, so a successful API return is not a correctness oracle."}, "reason": {"statement": "Use ncu --query-metrics and compare correctness, wavefronts/conflicts, and timing between matched layouts.", "urls": ["https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html", "https://gau-nernst.github.io/tcgen05/", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} -{"path": "wiki/techniques/swizzling.md", "before": {"statement": "- **All Blackwell tensor core kernels**: 128-byte swizzling is not optional. Both TMA and tcgen05.mma require it for correct results and peak performance.\n- **Hopper wgmma kernels**: Same requirement applies; wgmma expects swizzled SMEM operands.\n- **Non-MMA shared memory access**: If multiple warps access the same SMEM tile in a column pattern (e.g., reduction), swizzling prevents serialization."}, "after": {"statement": "Shared memory has 32 banks, and successive 32-bit words map to successive banks. For one warp memory request, accesses to different words in the same bank require serialized wavefronts; same-word reads can broadcast and do not create that conflict. The result therefore depends on the exact byte address requested by every participating lane.\n\nA swizzle permutes address chunks so a target access pattern may distribute its requests over banks more evenly. It can reduce conflicts for that pattern, but it does not make every access to the tile conflict-free. A layout favorable to an MMA operand can still be unfavorable to a CUDA-core row, column, transpose, or epilogue access.\n\nFor `cuTensorMapEncodeTiled`, record and validate all inputs rather than copying only the swizzle enumerator:\n\n1. Use a correctly aligned `CUtensorMap` object and global base pointer.\n2. Supply rank-sized global dimensions, rank-minus-one byte strides (the fastest dimension is implicit), rank-sized box dimensions, and rank-sized element strides.\n3. Satisfy the global alignment, stride, box, interleave, datatype, and selected-swizzle constraints.\n4. Check the returned `CUresult`; do not launch with an output descriptor after encoding failed.\n5. Use a shared-memory base and tcgen05 descriptor that represent the same mapping as the tensor map.\n\nAn invalid encoder combination can fail explicitly. A successfully encoded tensor map paired with the wrong consumer layout may instead read the wrong logical elements, so a successful API return is not a correctness oracle."}, "reason": {"statement": "Replace architecture categories with a producer/consumer/access-pattern decision.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"]}} -{"path": "wiki/techniques/swizzling.md", "before": {"statement": "- The swizzle mode in the TMA descriptor must exactly match the access pattern of the consumer. A mismatch produces silently incorrect results, not a runtime error.\n- Swizzled layouts make SMEM address computation non-trivial. Using CuTe's layout algebra avoids manual indexing errors.\n- For data types wider than 2 bytes (e.g., FP32 accumulators), the optimal swizzle mode may differ. TMEM accumulators avoid this issue since they use a separate address space."}, "after": {"statement": "For `cuTensorMapEncodeTiled`, record and validate all inputs rather than copying only the swizzle enumerator:\n\n1. Use a correctly aligned `CUtensorMap` object and global base pointer.\n2. Supply rank-sized global dimensions, rank-minus-one byte strides (the fastest dimension is implicit), rank-sized box dimensions, and rank-sized element strides.\n3. Satisfy the global alignment, stride, box, interleave, datatype, and selected-swizzle constraints.\n4. Check the returned `CUresult`; do not launch with an output descriptor after encoding failed.\n5. Use a shared-memory base and tcgen05 descriptor that represent the same mapping as the tensor map.\n\nAn invalid encoder combination can fail explicitly. A successfully encoded tensor map paired with the wrong consumer layout may instead read the wrong logical elements, so a successful API return is not a correctness oracle."}, "reason": {"statement": "State explicit encoder checks, layout equivalence and correctness/performance validation.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/swizzle.hpp", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/techniques/swizzling.md", "before": {"statement": "The swizzle function XORs a portion of the column address with the row address to scatter accesses across banks:\n\n```cuda\n// 128-byte swizzle: XOR bits [4:6] of the byte offset with the row index\n// This ensures that consecutive rows accessing the same logical column\n// map to different physical SMEM banks.\n//\n// For a tile stored in SMEM with TILE_N columns of 2-byte elements:\n// byte_offset = row * (TILE_N * sizeof(half)) + col * sizeof(half)\n// swizzled_offset = byte_offset ^ ((row & 0x7) << 4)\n//\n// The mask 0x7 = 3 bits, shift 4 = bits [4:6], giving 8-row periodicity\n// across the 128-byte bank group.\n\n__device__ int swizzle_128B(int row, int col, int stride_bytes) {\n int byte_offset = row * stride_bytes + col * sizeof(half);\n // XOR bits [4:6] of byte offset with low 3 bits of row\n int swizzled = byte_offset ^ ((row & 0x7) << 4);\n return swizzled;\n}\n```\n\nVisually, for an 8-row x 64-column half-precision tile (128 bytes per row):\n\n```\nWithout swizzle (row-major):\n Row 0: bank 0,1,2,...,31 bank 0,1,2,...,31\n Row 1: bank 0,1,2,...,31 bank 0,1,2,...,31\n Row 2: bank 0,1,2,...,31 bank 0,1,2,...,31\n -> Column access = 8-way bank conflict\n\nWith 128B swizzle (XOR pattern):\n Row 0: bank 0,1,2,...,31 bank 0,1,2,...,31\n Row 1: bank 1,2,3,...,0 bank 1,2,3,...,0 (rotated by 1)\n Row 2: bank 2,3,4,...,1 bank 2,3,4,...,1 (rotated by 2)\n -> Column access = conflict-free\n```"}, "after": {"statement": "CUTLASS 4.5.0 defines [`Swizzle`](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cute/swizzle.hpp):\n\n- `BBits` is the number of mask bits;\n- `MBase` is the number of least-significant address bits kept invariant; and\n- `SShift` is the distance between the two bit fields.\n\nFor `Swizzle<3,4,3>`, three address bits beginning above the four invariant low bits are XORed with the three-bit field shifted by three positions. Equivalently, address bits 7:9 affect bits 4:6. This is an address transformation, not a general `row & 7` rule: equivalence to row-based indexing depends on stride, byte units, base alignment, and layout composition.\n\nCUTLASS also distinguishes a position-independent composed swizzle layout from a position-dependent swizzle pointer, because hardware swizzling depends on the shared-memory pointer address. Reuse a pinned complete layout/descriptor construction or prove the composition and required base alignment; a `Swizzle` type alone is not a complete TMA or MMA contract."}, "reason": {"statement": "Use exact address-bit semantics and library/descriptor mappings.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cute/swizzle.hpp", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cute/pointer_swizzle.hpp"]}} +{"path": "wiki/techniques/swizzling.md", "before": {"statement": "```cuda\n// Creating a TMA descriptor with 128-byte swizzle\n#include \n\nCUtensorMap tensor_map;\n\n// Swizzle mode: CU_TENSOR_MAP_SWIZZLE_128B\n// This tells TMA to apply the 128-byte XOR swizzle pattern\n// when writing data into shared memory\ncuTensorMapEncodeTiled(\n &tensor_map,\n CU_TENSOR_MAP_DATA_TYPE_FLOAT16,\n 2, // 2D tensor\n global_ptr, // global memory base\n global_dims, // {N, M} dimensions\n global_strides, // {N * sizeof(half), sizeof(half)}\n tile_dims, // {TILE_N, TILE_M}\n element_strides, // {1, 1}\n CU_TENSOR_MAP_INTERLEAVE_NONE,\n CU_TENSOR_MAP_SWIZZLE_128B, // 128-byte swizzle\n CU_TENSOR_MAP_L2_PROMOTION_NONE,\n CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE\n);\n```"}, "after": {"statement": "For a TMA-to-tcgen05 path, keep these layers consistent:\n\n1. The tensor map describes how TMA places the global-memory box into shared memory, including interleave and swizzle.\n2. The shared-memory allocation and base address satisfy the alignment and span constraints of that mapping.\n3. The tcgen05 shared-memory descriptor describes the same physical-to-logical layout, leading/stride dimensions, base offset, and swizzle mode expected by the MMA operand.\n\nPTX ISA 9.0 permits ordinary tcgen05 shared-memory descriptor modes with no swizzle and 32B, 64B, or 128B spans, plus a 128B/base-32B mode. In the descriptor's three-bit layout field, ordinary no-swizzle is `0`, 128B/base-32B is `1`, ordinary 128B is `2`, 64B is `4`, and 32B is `6`; values `3`, `5`, and `7` are invalid. Legality also depends on MMA kind, element type, major mode, shape, CTA group, and target.\n\nThe CUDA 13.0.97 Driver API exposes TMA modes including:\n\n| `CU_TENSOR_MAP_SWIZZLE_NONE` | No bank swizzle |\n| `CU_TENSOR_MAP_SWIZZLE_32B` | 16B chunks within a 32B span |\n| `CU_TENSOR_MAP_SWIZZLE_64B` | 16B chunks within a 64B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B` | 16B chunks within a 128B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B` | 32B chunks within a 128B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B_ATOM_64B` | 64B chunks within a 128B span; support is operation/type-specific |\n\nThese names define mappings, not universal datatype or tile-size recommendations. The tensor-map encoder imposes mode-, interleave-, datatype-, alignment-, and box-size constraints; for example, ordinary 128B modes require the inner bounding-box byte span not to exceed 128 bytes. Check the exact archived API for the selected type and operation.\n\nFor `cuTensorMapEncodeTiled`, record and validate all inputs rather than copying only the swizzle enumerator:\n\n1. Use a correctly aligned `CUtensorMap` object and global base pointer.\n2. Supply rank-sized global dimensions, rank-minus-one byte strides (the fastest dimension is implicit), rank-sized box dimensions, and rank-sized element strides.\n3. Satisfy the global alignment, stride, box, interleave, datatype, and selected-swizzle constraints.\n4. Check the returned `CUresult`; do not launch with an output descriptor after encoding failed.\n5. Use a shared-memory base and tcgen05 descriptor that represent the same mapping as the tensor map.\n\nAn invalid encoder combination can fail explicitly. A successfully encoded tensor map paired with the wrong consumer layout may instead read the wrong logical elements, so a successful API return is not a correctness oracle."}, "reason": {"statement": "Replace incomplete code with an encoder validation checklist and pinned full example.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma"]}} +{"path": "wiki/techniques/swizzling.md", "before": {"statement": "The available swizzle modes and their use cases:\n\n| `SWIZZLE_NONE` | No remapping | Non-MMA data (flags, scales) |\n| `SWIZZLE_32B` | 32-byte groups | Narrow tiles, small data types |\n| `SWIZZLE_64B` | 64-byte groups | Medium tiles |\n| `SWIZZLE_128B` | 128-byte groups | Standard for BF16/FP16 MMA operands |"}, "after": {"statement": "For a TMA-to-tcgen05 path, keep these layers consistent:\n\n1. The tensor map describes how TMA places the global-memory box into shared memory, including interleave and swizzle.\n2. The shared-memory allocation and base address satisfy the alignment and span constraints of that mapping.\n3. The tcgen05 shared-memory descriptor describes the same physical-to-logical layout, leading/stride dimensions, base offset, and swizzle mode expected by the MMA operand.\n\nPTX ISA 9.0 permits ordinary tcgen05 shared-memory descriptor modes with no swizzle and 32B, 64B, or 128B spans, plus a 128B/base-32B mode. In the descriptor's three-bit layout field, ordinary no-swizzle is `0`, 128B/base-32B is `1`, ordinary 128B is `2`, 64B is `4`, and 32B is `6`; values `3`, `5`, and `7` are invalid. Legality also depends on MMA kind, element type, major mode, shape, CTA group, and target.\n\nThe CUDA 13.0.97 Driver API exposes TMA modes including:\n\n| `CU_TENSOR_MAP_SWIZZLE_NONE` | No bank swizzle |\n| `CU_TENSOR_MAP_SWIZZLE_32B` | 16B chunks within a 32B span |\n| `CU_TENSOR_MAP_SWIZZLE_64B` | 16B chunks within a 64B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B` | 16B chunks within a 128B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B` | 32B chunks within a 128B span |\n| `CU_TENSOR_MAP_SWIZZLE_128B_ATOM_64B` | 64B chunks within a 128B span; support is operation/type-specific |\n\nThese names define mappings, not universal datatype or tile-size recommendations. The tensor-map encoder imposes mode-, interleave-, datatype-, alignment-, and box-size constraints; for example, ordinary 128B modes require the inner bounding-box byte span not to exceed 128 bytes. Check the exact archived API for the selected type and operation."}, "reason": {"statement": "List exact mapping spans and defer legality to producer and consumer contracts.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma"]}} +{"path": "wiki/techniques/swizzling.md", "before": {"statement": "In CuTe/CUTLASS, swizzle is expressed as a layout composition:\n\n```cuda\n// CuTe swizzle layout for 128-byte swizzle pattern\n// Swizzle where:\n// B = number of bits in the base (non-swizzled) portion\n// M = number of bits in the mask\n// S = shift amount\n//\n// Swizzle<3, 4, 3> encodes the 128B swizzle:\n// 3 base bits (8-byte alignment)\n// 4 mask bits (16 rows)\n// 3 shift bits (8-column groups)\n\nusing SmemLayoutAtom = decltype(\n composition(\n Swizzle<3, 4, 3>{},\n Layout,\n Stride<_64, _1>>{}\n )\n);\n\n// Tile the atom across the full SMEM tile\nusing SmemLayoutA = decltype(\n tile_to_shape(SmemLayoutAtom{}, Shape, Int>{})\n);\n```"}, "after": {"statement": "CUTLASS 4.5.0 defines [`Swizzle`](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cute/swizzle.hpp):\n\n- `BBits` is the number of mask bits;\n- `MBase` is the number of least-significant address bits kept invariant; and\n- `SShift` is the distance between the two bit fields.\n\nFor `Swizzle<3,4,3>`, three address bits beginning above the four invariant low bits are XORed with the three-bit field shifted by three positions. Equivalently, address bits 7:9 affect bits 4:6. This is an address transformation, not a general `row & 7` rule: equivalence to row-based indexing depends on stride, byte units, base alignment, and layout composition.\n\nCUTLASS also distinguishes a position-independent composed swizzle layout from a position-dependent swizzle pointer, because hardware swizzling depends on the shared-memory pointer address. Reuse a pinned complete layout/descriptor construction or prove the composition and required base alignment; a `Swizzle` type alone is not a complete TMA or MMA contract."}, "reason": {"statement": "Define parameters exactly and cite a pinned full implementation rather than a generic atom.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cute/swizzle.hpp", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cute/pointer_swizzle.hpp"]}} +{"path": "wiki/techniques/swizzling.md", "before": {"statement": "Use `nvprof` or Nsight Compute to verify that swizzling eliminates conflicts:\n\n```python\n# Nsight Compute command to check shared memory bank conflicts\n# Look for \"Shared Memory Bank Conflicts\" metric\n# ncu --metrics l1tex__data_bank_conflicts_pipe_lsu_mem_shared_op_ld.sum \\\n# ./my_kernel\n\n# Expected results:\n# Without swizzle: bank_conflicts >> 0\n# With 128B swizzle: bank_conflicts == 0\n```"}, "after": {"statement": "For `cuTensorMapEncodeTiled`, record and validate all inputs rather than copying only the swizzle enumerator:\n\n1. Use a correctly aligned `CUtensorMap` object and global base pointer.\n2. Supply rank-sized global dimensions, rank-minus-one byte strides (the fastest dimension is implicit), rank-sized box dimensions, and rank-sized element strides.\n3. Satisfy the global alignment, stride, box, interleave, datatype, and selected-swizzle constraints.\n4. Check the returned `CUresult`; do not launch with an output descriptor after encoding failed.\n5. Use a shared-memory base and tcgen05 descriptor that represent the same mapping as the tensor map.\n\nAn invalid encoder combination can fail explicitly. A successfully encoded tensor map paired with the wrong consumer layout may instead read the wrong logical elements, so a successful API return is not a correctness oracle."}, "reason": {"statement": "Use ncu --query-metrics and compare correctness, wavefronts/conflicts, and timing between matched layouts.", "urls": ["https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html", "https://gau-nernst.github.io/tcgen05/", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma"]}} +{"path": "wiki/techniques/swizzling.md", "before": {"statement": "- **All Blackwell tensor core kernels**: 128-byte swizzling is not optional. Both TMA and tcgen05.mma require it for correct results and peak performance.\n- **Hopper wgmma kernels**: Same requirement applies; wgmma expects swizzled SMEM operands.\n- **Non-MMA shared memory access**: If multiple warps access the same SMEM tile in a column pattern (e.g., reduction), swizzling prevents serialization."}, "after": {"statement": "Shared memory has 32 banks, and successive 32-bit words map to successive banks. For one warp memory request, accesses to different words in the same bank require serialized wavefronts; same-word reads can broadcast and do not create that conflict. The result therefore depends on the exact byte address requested by every participating lane.\n\nA swizzle permutes address chunks so a target access pattern may distribute its requests over banks more evenly. It can reduce conflicts for that pattern, but it does not make every access to the tile conflict-free. A layout favorable to an MMA operand can still be unfavorable to a CUDA-core row, column, transpose, or epilogue access.\n\nFor `cuTensorMapEncodeTiled`, record and validate all inputs rather than copying only the swizzle enumerator:\n\n1. Use a correctly aligned `CUtensorMap` object and global base pointer.\n2. Supply rank-sized global dimensions, rank-minus-one byte strides (the fastest dimension is implicit), rank-sized box dimensions, and rank-sized element strides.\n3. Satisfy the global alignment, stride, box, interleave, datatype, and selected-swizzle constraints.\n4. Check the returned `CUresult`; do not launch with an output descriptor after encoding failed.\n5. Use a shared-memory base and tcgen05 descriptor that represent the same mapping as the tensor map.\n\nAn invalid encoder combination can fail explicitly. A successfully encoded tensor map paired with the wrong consumer layout may instead read the wrong logical elements, so a successful API return is not a correctness oracle."}, "reason": {"statement": "Replace architecture categories with a producer/consumer/access-pattern decision.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://docs.nvidia.com/nsight-compute/ProfilingGuide/index.html", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html"]}} +{"path": "wiki/techniques/swizzling.md", "before": {"statement": "- The swizzle mode in the TMA descriptor must exactly match the access pattern of the consumer. A mismatch produces silently incorrect results, not a runtime error.\n- Swizzled layouts make SMEM address computation non-trivial. Using CuTe's layout algebra avoids manual indexing errors.\n- For data types wider than 2 bytes (e.g., FP32 accumulators), the optimal swizzle mode may differ. TMEM accumulators avoid this issue since they use a separate address space."}, "after": {"statement": "For `cuTensorMapEncodeTiled`, record and validate all inputs rather than copying only the swizzle enumerator:\n\n1. Use a correctly aligned `CUtensorMap` object and global base pointer.\n2. Supply rank-sized global dimensions, rank-minus-one byte strides (the fastest dimension is implicit), rank-sized box dimensions, and rank-sized element strides.\n3. Satisfy the global alignment, stride, box, interleave, datatype, and selected-swizzle constraints.\n4. Check the returned `CUresult`; do not launch with an output descriptor after encoding failed.\n5. Use a shared-memory base and tcgen05 descriptor that represent the same mapping as the tensor map.\n\nAn invalid encoder combination can fail explicitly. A successfully encoded tensor map paired with the wrong consumer layout may instead read the wrong logical elements, so a successful API return is not a correctness oracle."}, "reason": {"statement": "State explicit encoder checks, layout equivalence and correctness/performance validation.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cute/swizzle.hpp", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma"]}} {"path": "wiki/techniques/vectorized-loads.md", "before": {"statement": "For memory-bound kernels (low arithmetic intensity), maximizing global memory throughput is critical. Three complementary techniques from the GPU Mode NVFP4 Hackathon achieve this: (1) wide vectorized loads (128-bit and 256-bit) to saturate memory bandwidth per thread, (2) differentiated L1 cache policies to keep reused data hot while bypassing the cache for streaming data, and (3) register budgeting via `-maxrregcount` to increase occupancy. These techniques reduced NVFP4 GEMV latency from 2000us to 22.4us (89x improvement)."}, "after": {"statement": "Amandeep Singh's attempts supply useful negative controls for the same task. Replacing two `uchar4` loads with one `uint2` load was reported 16–25% slower because extraction added instructions. Lowering `-maxrregcount` from 80 to 64 had no effect because natural allocation was already below the cap. The author later observed wider PTX forms and differentiated L1 policies in other solutions, but published no contestant code or controlled ablation for those observations.\n\nTogether, the reports justify testing vector width, decode organization, cache hints, and register limits. They do not establish that the widest load, a particular cache hint, or the lowest register cap is essential for GEMV, sub-byte arithmetic, or decode workloads."}, "reason": {"statement": "Retain the report as an author-reported multi-change case study without causal attribution.", "urls": ["https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"]}} {"path": "wiki/techniques/vectorized-loads.md", "before": {"statement": "Standard 32-bit loads waste memory bus bandwidth. Wider loads amortize the instruction overhead and saturate the 8 TB/s HBM bandwidth of B200:\n\n```cuda\n// Vectorized load widths comparison for FP4 GEMV\n// Each thread loads more data per instruction\n\n// 32-bit load: 4 bytes per thread per instruction\nfloat val;\nasm volatile(\"ld.global.b32 %0, [%1];\" : \"=f\"(val) : \"l\"(ptr));\n\n// 64-bit load: 8 bytes per thread per instruction\nuint2 val64;\nasm volatile(\"ld.global.v2.u32 {%0,%1}, [%2];\"\n : \"=r\"(val64.x), \"=r\"(val64.y) : \"l\"(ptr));\n\n// 128-bit load: 16 bytes per thread per instruction (preferred)\nuint4 val128;\nasm volatile(\"ld.global.v4.u32 {%0,%1,%2,%3}, [%4];\"\n : \"=r\"(val128.x), \"=r\"(val128.y), \"=r\"(val128.z), \"=r\"(val128.w)\n : \"l\"(ptr));\n\n// 256-bit load: 32 bytes per thread per instruction\n// Requires v4.u64 (4 x 64-bit)\nuint64_t v256[4];\nasm volatile(\"ld.global.v4.u64 {%0,%1,%2,%3}, [%4];\"\n : \"=l\"(v256[0]), \"=l\"(v256[1]), \"=l\"(v256[2]), \"=l\"(v256[3])\n : \"l\"(ptr));\n```"}, "after": {"statement": "A PTX vector load moves several typed elements into several registers with one instruction. This can reduce the number of load instructions issued by a thread, but PTX only says that vector loads *may* improve memory performance. Source-level width alone does not prove fewer physical memory transactions, higher achieved bandwidth, or lower kernel latency.\n\nPTX ISA 9.0 permits the following global-memory forms on `sm_100`. The first accesses 16 bytes and the second accesses 32 bytes:\n\n```ptx\n.reg .b64 addr;\n.reg .u32 x<4>;\n.reg .u64 y<4>;\n\nld.global.v4.u32 {x0, x1, x2, x3}, [addr];\nld.global.v4.u64 {y0, y1, y2, y3}, [addr];\n```\n\nThe address must be naturally aligned to the total access size: 16 bytes for `v4.u32` and 32 bytes for `v4.u64`. A misaligned PTX address has undefined behavior; the ISA says it may have low address bits masked or fault. It does not specify a fallback to narrower loads. Guard tails or dispatch them to an access whose complete range and alignment are valid.\n\nThe 256-bit `v4.b64` family is an SM100-or-newer feature. A 32-byte load contains 64 values when the payload is densely packed E2M1 at two 4-bit values per byte. That arithmetic says nothing about whether the width is profitable: unpack cost, live registers, instruction selection, per-thread address patterns, and tail handling can outweigh a lower source-level load count.\n\nPTX distinguishes cache operators, eviction priorities, prefetch-size hints, and non-coherent read-only loads. Representative legal forms are:\n\n```ptx\nld.global.L1::no_allocate.v4.u32 {x0, x1, x2, x3}, [addr];\nld.global.L1::evict_last.v4.u32 {x0, x1, x2, x3}, [addr];\nld.global.L2::256B.b32 x0, [addr];\nld.global.nc.b32 x0, [addr];\n```\n\n| `L1::no_allocate` | L1 eviction-priority selection that may be applied | Does not guarantee an L1 bypass or a speedup for a streaming operand |\n| `L1::evict_last` | Requests the corresponding L1 eviction priority | Does not guarantee that a line remains resident |\n| `L2::256B` | Hints that additional data of the stated size be prefetched into L2 | Is not a 256-byte load or a promotion guarantee |\n| `ld.global.nc` | Loads through a non-coherent read-only cache | Has architecture- and parallelism-dependent latency/throughput; it is not a general coherence optimization |\n\nCache-policy operands and prefetch-size qualifiers are performance hints and do not change the program's memory-consistency behavior. A reuse argument can motivate `evict_last`, and a one-pass stream can motivate `no_allocate`, but only a matched comparison determines whether either helps the concrete working set and launch."}, "reason": {"statement": "Document legal widths, exact alignment, and the need to inspect/profile generated code.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld-global-nc"]}} {"path": "wiki/techniques/vectorized-loads.md", "before": {"statement": "For the NVFP4 GEMV, 128-bit and 256-bit loads are essential because FP4 elements are only 0.5 bytes each. A 256-bit load fetches 64 FP4 values in a single instruction:\n\n```cuda\n// NVFP4 GEMV: each thread loads 64 FP4 values via 256-bit load\n// Then unpacks using PTX byte manipulation instead of bitwise ops\n__device__ void load_and_unpack_nvfp4_256bit(\n const uint8_t* fp4_data, // Packed FP4 data (2 values per byte)\n float* unpacked, // Output: 64 FP32 values\n int offset)\n{\n // 256-bit load: 32 bytes = 64 FP4 values\n uint64_t raw[4];\n const uint64_t* ptr = reinterpret_cast(fp4_data + offset);\n asm volatile(\n \"ld.global.v4.u64 {%0,%1,%2,%3}, [%4];\"\n : \"=l\"(raw[0]), \"=l\"(raw[1]), \"=l\"(raw[2]), \"=l\"(raw[3])\n : \"l\"(ptr)\n );\n\n // Unpack using PTX mov.b32 byte extraction\n // This avoids the bitwise shift-and-mask overhead\n for (int i = 0; i < 4; i++) {\n uint32_t lo = (uint32_t)(raw[i]);\n uint32_t hi = (uint32_t)(raw[i] >> 32);\n\n // PTX byte unpack: extract individual bytes from 32-bit word\n uint32_t b0, b1, b2, b3;\n asm volatile(\"mov.b32 {%0,%1,%2,%3}, %4;\"\n : \"=r\"(b0), \"=r\"(b1), \"=r\"(b2), \"=r\"(b3) : \"r\"(lo));\n\n // Each byte contains 2 FP4 values — decode low word (8 values)\n for (int b = 0; b < 4; b++) {\n uint32_t byte_val = (b == 0) ? b0 : (b == 1) ? b1 : (b == 2) ? b2 : b3;\n unpacked[i * 16 + b * 2] = decode_fp4(byte_val & 0xF);\n unpacked[i * 16 + b * 2 + 1] = decode_fp4((byte_val >> 4) & 0xF);\n }\n\n // Unpack high word (next 8 values from same 64-bit element)\n uint32_t hb0, hb1, hb2, hb3;\n asm volatile(\"mov.b32 {%0,%1,%2,%3}, %4;\"\n : \"=r\"(hb0), \"=r\"(hb1), \"=r\"(hb2), \"=r\"(hb3) : \"r\"(hi));\n for (int b = 0; b < 4; b++) {\n uint32_t byte_val = (b == 0) ? hb0 : (b == 1) ? hb1 : (b == 2) ? hb2 : hb3;\n unpacked[i * 16 + 8 + b * 2] = decode_fp4(byte_val & 0xF);\n unpacked[i * 16 + 8 + b * 2 + 1] = decode_fp4((byte_val >> 4) & 0xF);\n }\n }\n}\n```"}, "after": {"statement": "PTX distinguishes cache operators, eviction priorities, prefetch-size hints, and non-coherent read-only loads. Representative legal forms are:\n\n```ptx\nld.global.L1::no_allocate.v4.u32 {x0, x1, x2, x3}, [addr];\nld.global.L1::evict_last.v4.u32 {x0, x1, x2, x3}, [addr];\nld.global.L2::256B.b32 x0, [addr];\nld.global.nc.b32 x0, [addr];\n```\n\n| `L1::no_allocate` | L1 eviction-priority selection that may be applied | Does not guarantee an L1 bypass or a speedup for a streaming operand |\n| `L1::evict_last` | Requests the corresponding L1 eviction priority | Does not guarantee that a line remains resident |\n| `L2::256B` | Hints that additional data of the stated size be prefetched into L2 | Is not a 256-byte load or a promotion guarantee |\n| `ld.global.nc` | Loads through a non-coherent read-only cache | Has architecture- and parallelism-dependent latency/throughput; it is not a general coherence optimization |\n\nCache-policy operands and prefetch-size qualifiers are performance hints and do not change the program's memory-consistency behavior. A reuse argument can motivate `evict_last`, and a one-pass stream can motivate `no_allocate`, but only a matched comparison determines whether either helps the concrete working set and launch.\n\nYue Zhang reports the following CUDA progression for GPU Mode Problem 1. These are author-reported endpoints without raw repeated-trial data or released complete submission code:\n\n| Initial CUDA | Naive hand-written path | about 2000 µs |\n| CUDA optimization 1 | Coalescing, shared B, thread collaboration, warp reduction | about 443 µs |\n| CUDA optimization 2 | Remove shared B, per-thread tiles, `float4` loads, hardware intrinsics | about 39 µs |\n| CUDA optimization 3 | Vectorized PTX FP4 and scale decode | about 27 µs |\n| Parameter tuning | Threads per row and rows per block | about 26 µs |\n| ILP | Two tiles per loop iteration | about 22.9 µs |\n| Aggressive PTX fusion | Decode, scales, multiply, and accumulation in a larger PTX block | about 22.3 µs |\n\nThe submitted public-leaderboard score was 22.392 microseconds, the geometric mean over three benchmark shapes. The pinned task's theoretical model gives 8.622, 17.275, and 4.317 microseconds for those separate shapes; 8.622 microseconds is therefore not an aggregate “speed of light.” The 39-to-27 endpoints change the PTX load/decode path together and do not isolate cache policy, load width, or conversion choice."}, "reason": {"statement": "Replace the invented helper with exact PTX semantics and an empirical selection workflow.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-mov", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld-global-nc", "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html", "https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv/task.yml"]}} @@ -402,38 +402,38 @@ {"path": "wiki/techniques/cache-policy.md", "before": {"statement": "Rank 1 submission used **different qualifiers per K-dimension variant**:\n- K=16384 (large): aggressive `L1::no_allocate` on A (huge streaming matrix)\n- K=2048 (small): relaxed balance since B is smaller relative to cache"}, "after": {"statement": null}, "reason": {"statement": "No direct public source supports this contestant attribution.", "urls": ["https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv"]}} {"path": "wiki/techniques/cache-policy.md", "before": {"statement": "- NVFP4 GEMV: 443μs → 27μs (16x improvement) came partly from cache policy + PTX byte unpacking\n- On memory-bound kernels, cache policy can be the dominant lever"}, "after": {"statement": "Yue Zhang reports approximately 39 microseconds for a stage combining removal of B shared-memory staging, per-thread tiles, `float4` loads, and hardware conversion, followed by approximately 27 microseconds for a vectorized PTX FP4/scale-decode stage. Those endpoints change the load and decode path together. They do not measure a cache-hint-only speedup or establish cache policy as the dominant lever for memory-bound kernels."}, "reason": {"statement": "Preserve only the complete combined-stage observation and causal limit.", "urls": ["https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"]}} {"path": "wiki/techniques/cache-policy.md", "before": {"statement": "- Memory-bound kernels (profile with Nsight Compute first)\n- Tensor with clear \"streaming\" vs \"reused\" access patterns\n- Inputs > L2 cache size (B200: 126MB)\n- Separate M and N tile loading patterns in GEMM"}, "after": {"statement": "Amandeep Singh reports that three inspected B200 NVFP4 GEMV solutions used `L1::no_allocate` for A and `L1::evict_last` for B alongside raw PTX decode, wide loads, exact-K specialization, tighter register caps, and—in one solution—sharing B reads across M rows. The report does not release those contestant implementations or a cache-policy-only ablation, and it does not support the former page's rank-1 per-K policy attribution."}, "reason": {"statement": "Replace with a one-variable measurement procedure based on concrete address streams and reuse distances.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/", "https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv"]}} -{"path": "wiki/techniques/chunk-parallelism.md", "before": {"statement": "Linear attention variants (GatedDeltaNet, RetNet, Mamba) have O(n) complexity but naive implementations are sequential. Chunk-based parallelism divides the sequence into chunks of size C, computes within each chunk in parallel (matmul-friendly), and propagates state between chunks sequentially."}, "after": {"statement": "Some linear recurrent models admit an algebraically equivalent chunkwise formulation. Work local to a chunk can then be expressed with parallel matrix operations, while the state passed across chunk boundaries preserves the recurrence's sequence order. The exact transform is model-specific: it must be derived from the recurrence, not replaced by a generic attention matrix and additive state update.\n\nA correct implementation separates at least these obligations:\n\n1. Compute the chunk-local quantities required by the model's exact recurrence.\n2. Resolve boundary states in sequence order, either with an associative scan supported by the formulation or with explicitly ordered stages or launches.\n3. Combine each chunk's local result with its incoming boundary state and emit outputs in the original token order.\n4. Validate outputs and final states against a token-by-token reference across variable sequence lengths, chunk tails, batches, heads, dtypes, and gate extremes.\n\nAn ordinary GPU grid does not imply increasing program-ID execution order or grid-wide synchronization. Programs for every chunk therefore cannot safely read and overwrite one shared state pointer in a single unordered launch. A staged algorithm must make the boundary-state dependency explicit."}, "reason": {"statement": "Scope chunking to concrete recurrence transformations and implementations.", "urls": ["https://arxiv.org/abs/2503.14376v3", "https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921", "https://triton-lang.org/main/programming-guide/chapter-01/introduction.html"]}} -{"path": "wiki/techniques/chunk-parallelism.md", "before": {"statement": "```python\n@triton.jit\ndef chunk_parallel_linear_attn(Q, K, V, State, Output,\n chunk_size: tl.constexpr,\n d: tl.constexpr):\n # Grid: (num_chunks, num_heads, batch)\n chunk_id = tl.program_id(0)\n\n # Load chunk of Q, K, V\n q = tl.load(Q + chunk_id * chunk_size * d + offsets) # [C, d]\n k = tl.load(K + chunk_id * chunk_size * d + offsets)\n v = tl.load(V + chunk_id * chunk_size * d + offsets)\n\n # Intra-chunk: parallel O(C^2) attention-like compute\n scores = tl.dot(q, tl.trans(k))\n o_intra = tl.dot(scores, v)\n\n # Inter-chunk: sequential state propagation\n state = tl.load(State) # From previous chunk\n o_inter = tl.dot(q, state)\n\n # Combine and update state\n output = o_intra + o_inter\n state = update_state(state, k, v)\n\n tl.store(Output + offsets, output)\n tl.store(State, state)\n```"}, "after": {"statement": null}, "reason": {"statement": "The incomplete, racy pseudo-kernel cannot be repaired into a model-correct implementation without choosing a specific algorithm.", "urls": ["https://triton-lang.org/main/programming-guide/chapter-01/introduction.html", "https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921"]}} +{"path": "wiki/techniques/chunk-parallelism.md", "before": {"statement": "Linear attention variants (GatedDeltaNet, RetNet, Mamba) have O(n) complexity but naive implementations are sequential. Chunk-based parallelism divides the sequence into chunks of size C, computes within each chunk in parallel (matmul-friendly), and propagates state between chunks sequentially."}, "after": {"statement": "Some linear recurrent models admit an algebraically equivalent chunkwise formulation. Work local to a chunk can then be expressed with parallel matrix operations, while the state passed across chunk boundaries preserves the recurrence's sequence order. The exact transform is model-specific: it must be derived from the recurrence, not replaced by a generic attention matrix and additive state update.\n\nA correct implementation separates at least these obligations:\n\n1. Compute the chunk-local quantities required by the model's exact recurrence.\n2. Resolve boundary states in sequence order, either with an associative scan supported by the formulation or with explicitly ordered stages or launches.\n3. Combine each chunk's local result with its incoming boundary state and emit outputs in the original token order.\n4. Validate outputs and final states against a token-by-token reference across variable sequence lengths, chunk tails, batches, heads, dtypes, and gate extremes.\n\nAn ordinary GPU grid does not imply increasing program-ID execution order or grid-wide synchronization. Programs for every chunk therefore cannot safely read and overwrite one shared state pointer in a single unordered launch. A staged algorithm must make the boundary-state dependency explicit."}, "reason": {"statement": "Scope chunking to concrete recurrence transformations and implementations.", "urls": ["https://arxiv.org/abs/2503.14376v3", "https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921", "https://triton-lang.org/main/programming-guide/chapter-1/introduction.html"]}} +{"path": "wiki/techniques/chunk-parallelism.md", "before": {"statement": "```python\n@triton.jit\ndef chunk_parallel_linear_attn(Q, K, V, State, Output,\n chunk_size: tl.constexpr,\n d: tl.constexpr):\n # Grid: (num_chunks, num_heads, batch)\n chunk_id = tl.program_id(0)\n\n # Load chunk of Q, K, V\n q = tl.load(Q + chunk_id * chunk_size * d + offsets) # [C, d]\n k = tl.load(K + chunk_id * chunk_size * d + offsets)\n v = tl.load(V + chunk_id * chunk_size * d + offsets)\n\n # Intra-chunk: parallel O(C^2) attention-like compute\n scores = tl.dot(q, tl.trans(k))\n o_intra = tl.dot(scores, v)\n\n # Inter-chunk: sequential state propagation\n state = tl.load(State) # From previous chunk\n o_inter = tl.dot(q, state)\n\n # Combine and update state\n output = o_intra + o_inter\n state = update_state(state, k, v)\n\n tl.store(Output + offsets, output)\n tl.store(State, state)\n```"}, "after": {"statement": null}, "reason": {"statement": "The incomplete, racy pseudo-kernel cannot be repaired into a model-correct implementation without choosing a specific algorithm.", "urls": ["https://triton-lang.org/main/programming-guide/chapter-1/introduction.html", "https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921"]}} {"path": "wiki/techniques/chunk-parallelism.md", "before": {"statement": "- **Small chunks (C=32)**: low latency decode, fewer intermediate materializations\n- **Large chunks (C=256-512)**: better tensor core utilization, higher throughput for prefill\n- **TFLA (Tiled FLA)**: two-level tiling allows arbitrary chunk sizes via recursive tiling"}, "after": {"statement": "The pinned NVlabs GatedDeltaNet repository uses chunkwise Triton kernels for training and a WY representation of the gated delta rule. That implementation is direct evidence for GatedDeltaNet chunking, but it is not equivalent to a generic `scores = Q @ K.T; output = scores @ V` snippet and does not supply a universal chunk-size rule.\n\nTiled Flash Linear Attention (TFLA) starts from the chunkwise formulation of linear RNNs and adds another level of sequence parallelization within a chunk. The authors state that this permits arbitrarily large chunks, raises arithmetic intensity, and reduces intermediate-state materialization. The paper and pinned official code apply the method to mLSTM and report H100 results; they do not establish a GatedDeltaNet implementation, a Blackwell/TMEM implementation, or a recursive-tiling API."}, "reason": {"statement": "Retain TFLA's exact contribution and make chunk size an empirical backend/model choice.", "urls": ["https://arxiv.org/abs/2503.14376v3", "https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921"]}} {"path": "wiki/techniques/ping-pong-scheduling.md", "before": {"statement": "Ping-pong scheduling alternates two query tiles within a single CTA so the softmax warpgroup never stalls waiting for MMA. Introduced in FlashAttention-4 to exploit Blackwell's asymmetric hardware (2× tensor cores, same SFU count as Hopper)."}, "after": {"statement": "Ping-pong scheduling interleaves two output-tile states in one CTA. In the pinned FlashAttention-4 (FA4) SM100 forward implementation, `q_stage=2` makes the CTA cover two 128-row query tiles. One dedicated warp controls MMA, two four-warp groups perform softmax for the respective query stages, and a separate four-warp group performs correction work. While MMA advances one tile, the schedule tries to overlap softmax work for the other tile.\n\nThis is not merely ordinary double buffering of one producer-consumer stream. Each query stage owns a distinct output state, including its softmax and output TMEM regions, and the algorithm must preserve the dependencies among MMA accumulation, row statistics, rescaling/correction, and output consumption. “Ping-pong” describes this interleaving; it does not guarantee that either execution resource is stall-free."}, "reason": {"statement": "Describe the exact two-output-tile schedule and its bounded overlap objective.", "urls": ["https://arxiv.org/html/2603.05451v1", "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py"]}} -{"path": "wiki/techniques/ping-pong-scheduling.md", "before": {"statement": "```cuda\n// Two 128-token query tiles per CTA, alternating through the mainloop\n// Warpgroup 0: softmax for tile A while MMA runs on tile B\n// Warpgroup 1: softmax for tile B while MMA runs on tile A\n\n__global__ void fa4_ping_pong_attn(...) {\n int wg = warp_group_id();\n\n // TMEM holds accumulators for BOTH tiles\n uint32_t tmem_A = tmem_alloc(256);\n uint32_t tmem_B = tmem_alloc(256);\n\n for (int k = 0; k < num_kv_tiles; k++) {\n if (wg == 0) {\n // Compute softmax for tile A (previous MMA output)\n softmax_normalize(tmem_A);\n // Issue MMA for tile B next\n tcgen05_mma(Q_B_smem, K_smem[k], tmem_B);\n } else {\n softmax_normalize(tmem_B);\n tcgen05_mma(Q_A_smem, K_smem[k], tmem_A);\n }\n mbarrier_arrive(&ping_pong_sync);\n mbarrier_wait(&ping_pong_sync);\n }\n}\n```"}, "after": {"statement": "The concrete reference is [`flash_fwd_sm100.py` at commit `a369df707e1980fb328abcc1733e3457ec10155f`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py). Its default two-stage layout has the following software roles:\n\n| 0–3 | softmax for query stage 0 |\n| 4–7 | softmax for query stage 1 |\n| 8–11 | correction |\n| 12 | MMA control |\n| 13 | epilogue |\n| 14 | load |\n| 15 | empty in this configuration |\n\nThese IDs are implementation choices, not SM100 architectural roles. The source changes assignments for other configurations, including `q_stage=1`.\n\nThe implementation performs one collective TMEM allocation sized for its required columns and gives the two query stages disjoint offsets within that allocation. It also constructs multiple producer/consumer pipelines for distinct handoffs. A single generic barrier after issuing `tcgen05.mma` is not an equivalent implementation: tcgen05 work is asynchronous, and completion, memory visibility, stage reuse, and TMEM lifetime must follow the corresponding pipeline and ISA contracts.\n\nBefore treating a two-tile schedule as valid, verify all of the following in the complete kernel:\n\n1. TMEM regions for simultaneous tile states are disjoint, allocated and deallocated collectively, and remain live until their final consumers finish.\n2. Every SMEM or TMEM stage has one unambiguous producer/consumer phase owner; a stage cannot be overwritten until all operations that read it have completed.\n3. Asynchronous MMA completion is connected to the barrier that guards accumulator consumption and operand-stage reuse. A CTA barrier or an unrelated mbarrier arrival is insufficient.\n4. Softmax groups obey the implementation's explicit synchronization. FA4 serializes their exponential critical sections rather than allowing both groups to contend there simultaneously.\n5. Correction and epilogue work wait for the row statistics and accumulators they consume, including the proper proxy and execution-order fences.\n6. Boundary query rows, final key tiles, causal/window masks, and the pipeline tail preserve the same mathematical result as the reference attention computation."}, "reason": {"statement": "Replace executable-looking unsafe code with a source-pinned structural contract and verification checklist.", "urls": ["https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions"]}} +{"path": "wiki/techniques/ping-pong-scheduling.md", "before": {"statement": "```cuda\n// Two 128-token query tiles per CTA, alternating through the mainloop\n// Warpgroup 0: softmax for tile A while MMA runs on tile B\n// Warpgroup 1: softmax for tile B while MMA runs on tile A\n\n__global__ void fa4_ping_pong_attn(...) {\n int wg = warp_group_id();\n\n // TMEM holds accumulators for BOTH tiles\n uint32_t tmem_A = tmem_alloc(256);\n uint32_t tmem_B = tmem_alloc(256);\n\n for (int k = 0; k < num_kv_tiles; k++) {\n if (wg == 0) {\n // Compute softmax for tile A (previous MMA output)\n softmax_normalize(tmem_A);\n // Issue MMA for tile B next\n tcgen05_mma(Q_B_smem, K_smem[k], tmem_B);\n } else {\n softmax_normalize(tmem_B);\n tcgen05_mma(Q_A_smem, K_smem[k], tmem_A);\n }\n mbarrier_arrive(&ping_pong_sync);\n mbarrier_wait(&ping_pong_sync);\n }\n}\n```"}, "after": {"statement": "The concrete reference is [`flash_fwd_sm100.py` at commit `a369df707e1980fb328abcc1733e3457ec10155f`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py). Its default two-stage layout has the following software roles:\n\n| 0–3 | softmax for query stage 0 |\n| 4–7 | softmax for query stage 1 |\n| 8–11 | correction |\n| 12 | MMA control |\n| 13 | epilogue |\n| 14 | load |\n| 15 | empty in this configuration |\n\nThese IDs are implementation choices, not SM100 architectural roles. The source changes assignments for other configurations, including `q_stage=1`.\n\nThe implementation performs one collective TMEM allocation sized for its required columns and gives the two query stages disjoint offsets within that allocation. It also constructs multiple producer/consumer pipelines for distinct handoffs. A single generic barrier after issuing `tcgen05.mma` is not an equivalent implementation: tcgen05 work is asynchronous, and completion, memory visibility, stage reuse, and TMEM lifetime must follow the corresponding pipeline and ISA contracts.\n\nBefore treating a two-tile schedule as valid, verify all of the following in the complete kernel:\n\n1. TMEM regions for simultaneous tile states are disjoint, allocated and deallocated collectively, and remain live until their final consumers finish.\n2. Every SMEM or TMEM stage has one unambiguous producer/consumer phase owner; a stage cannot be overwritten until all operations that read it have completed.\n3. Asynchronous MMA completion is connected to the barrier that guards accumulator consumption and operand-stage reuse. A CTA barrier or an unrelated mbarrier arrival is insufficient.\n4. Softmax groups obey the implementation's explicit synchronization. FA4 serializes their exponential critical sections rather than allowing both groups to contend there simultaneously.\n5. Correction and epilogue work wait for the row statistics and accumulators they consume, including the proper proxy and execution-order fences.\n6. Boundary query rows, final key tiles, causal/window masks, and the pipeline tail preserve the same mathematical result as the reference attention computation."}, "reason": {"statement": "Replace executable-looking unsafe code with a source-pinned structural contract and verification checklist.", "urls": ["https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions"]}} {"path": "wiki/techniques/ping-pong-scheduling.md", "before": {"statement": "- Single-tile schedule would leave SFU idle while MMA runs, and vice versa\n- Ping-pong keeps both units 100% busy"}, "after": {"statement": "Ping-pong scheduling interleaves two output-tile states in one CTA. In the pinned FlashAttention-4 (FA4) SM100 forward implementation, `q_stage=2` makes the CTA cover two 128-row query tiles. One dedicated warp controls MMA, two four-warp groups perform softmax for the respective query stages, and a separate four-warp group performs correction work. While MMA advances one tile, the schedule tries to overlap softmax work for the other tile.\n\nThis is not merely ordinary double buffering of one producer-consumer stream. Each query stage owns a distinct output state, including its softmax and output TMEM regions, and the algorithm must preserve the dependencies among MMA accumulation, row statistics, rescaling/correction, and output consumption. “Ping-pong” describes this interleaving; it does not guarantee that either execution resource is stall-free.\n\nThe paper reports up to 1613 TFLOP/s and 71% utilization for complete FA4 forward kernels. The accompanying author blog reports up to 1605 TFLOP/s and 71%. Those endpoints include partial software exponentials, conditional rescaling, TMEM partitioning, register allocation choices, and the rest of the pipeline. Neither source supplies a ping-pong-only ablation, and neither establishes 100% tensor-core or exponential-unit utilization."}, "reason": {"statement": "Retain the overlap motivation without universal idle-state or utilization assertions.", "urls": ["https://arxiv.org/html/2603.05451v1", "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py", "https://tridao.me/blog/2026/flash4/"]}} {"path": "wiki/techniques/ping-pong-scheduling.md", "before": {"statement": "- FA4 achieves 1605 TFLOPS BF16 (71% utilization) with this pattern"}, "after": {"statement": "The paper reports up to 1613 TFLOP/s and 71% utilization for complete FA4 forward kernels. The accompanying author blog reports up to 1605 TFLOP/s and 71%. Those endpoints include partial software exponentials, conditional rescaling, TMEM partitioning, register allocation choices, and the rest of the pipeline. Neither source supplies a ping-pong-only ablation, and neither establishes 100% tensor-core or exponential-unit utilization."}, "reason": {"statement": "Report the result only at full-system scope and explicitly deny isolated attribution.", "urls": ["https://tridao.me/blog/2026/flash4/", "https://arxiv.org/html/2603.05451v1"]}} {"path": "wiki/techniques/ping-pong-scheduling.md", "before": {"statement": "- Compute-bound attention kernels on Blackwell\n- Kernels where softmax/epilogue is SFU-heavy\n- Not useful on Hopper (balance is different)"}, "after": {"statement": "Use the pinned SM100 kernel as the concrete reference when two independent query/output states can coexist within the register, shared-memory, and TMEM budgets. Compare `q_stage=2` with the source-supported `q_stage=1` configuration while holding the remaining build, input, tile, and launch choices fixed. Measure end-to-end time, MMA and exponential-pipeline activity, barrier stalls, per-role idle time, register spills, and occupancy across representative shapes.\n\nDo not select the technique from a generic “compute-bound” or “SFU-heavy” label alone. The FA4 paper says its schedule is similar to FlashAttention-3's Hopper ping-pong schedule, so the scheduling idea is not intrinsically Blackwell-only; this page's pinned implementation and hardware figures are specifically SM100. Keep the two-stage form only when the controlled comparison shows a relevant improvement and correctness tests cover boundaries and pipeline tails."}, "reason": {"statement": "Scope concrete code to SM100 and recommend controlled measurement instead of workload-label prescriptions.", "urls": ["https://arxiv.org/html/2603.05451v1", "https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py"]}} {"path": "wiki/techniques/ping-pong-scheduling.md", "before": {"statement": "Verbatim upstream code lives in [`artifacts/kernels/ping-pong-scheduling/full/`](../../artifacts/kernels/ping-pong-scheduling/full/); labeled derived variants (each with the required `// provenance: derived from ...; not upstream code` header) live in [`artifacts/kernels/ping-pong-scheduling/variants/`](../../artifacts/kernels/ping-pong-scheduling/variants/). Every file's SHA-256 and upstream-pinning metadata is in `PROVENANCE.yaml` inside each bundle.\n\nQuery via:\n\n```bash\npython3 scripts/get_page.py technique-ping-pong-scheduling --include-code\n```"}, "after": {"statement": null}, "reason": {"statement": "Remove the misleading page linkage while preserving the separately provenance-labeled files for repository integrity.", "urls": []}} {"path": "wiki/patterns/pipeline-stalls.md", "before": {"statement": "Nsight Compute shows TMA or tcgen05 units idle despite nominally compute-bound workload. Tensor core utilization drops during specific phases of the kernel. Warp-level profiling reveals threads blocked on `mbarrier.try_wait` more than expected."}, "after": {"statement": "A pipeline stall is lost issue opportunity caused by a producer-consumer dependency on the critical path. Low TMA or tensor-core activity, a warp sampled at a barrier wait, or a phase-local utilization drop is only a lead. Expected dependency waits, memory latency, execution-pipeline contention, load imbalance, prologue/tail work, and synchronization defects can produce similar observations.\n\nNsight Compute's Warp State Statistics describes why sampled warps could not issue, while Scheduler Statistics shows whether schedulers had eligible warps and issued instructions. NVIDIA cautions that stalls are not necessarily performance-limiting and should be prioritized when schedulers fail to issue. Metric availability and names vary by tool and chip, so query the installed profiler and use its shipped sections rather than assuming a Blackwell metric name. PM sampling can add a timeline on supported systems, but warp sampling itself has no time resolution."}, "reason": {"statement": "Require scheduler issue evidence, source correlation, and a controlled causal perturbation.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} {"path": "wiki/patterns/pipeline-stalls.md", "before": {"statement": "1. **Insufficient pipeline depth**: 2 stages cannot hide a 3-cycle latency chain"}, "after": {"statement": "1. Establish a synchronized end-to-end timing regression on fixed inputs and confirm outputs against a reference. Record GPU, clocks, toolkit, build flags, kernel name, launch shape, and resource use.\n2. Collect Speed-of-Light, Scheduler Statistics, Warp State Statistics, and source/SASS correlation with the sections available in the installed Nsight Compute. Account for replay and sampling effects.\n3. Map the dominant not-issued locations to concrete edges: TMA-full wait, MMA-completion wait, output-buffer reuse, CTA barrier, queue starvation, or pipeline prologue/tail.\n4. State a prediction. For example: if TMA readiness is the critical edge, a legal extra operand stage should reduce that wait and runtime; if only the tail is exposed, the steady-state wait distribution should remain largely unchanged.\n5. Change one variable and repeat identical correctness, warmup, timing, and profiler collection. Reject a cause when its predicted counter and time movement do not occur.\n\n| Vary legal SMEM stage count | Producer cannot stay far enough ahead of MMA | Same tile/math/roles; record SMEM, occupancy, short-loop behavior, and tail |\n| Separate long-lived warp roles | Role handoff or control work is on the issue path | Same stages and tile; record registers, active warps, and per-role idle/backpressure |\n| Add a second TMEM output region | Epilogue holds the only accumulator region | Prove disjoint lifetime; record TMEM columns and epilogue/MMA completion waits |\n| Interleave two query/output tiles | Softmax/correction and MMA have independent ready work | Prove separate state and dependencies; compare one versus two query stages |\n\nNone is a universal cure. More stages consume storage and can reduce occupancy; specialization adds warps and synchronization; TMEM buffering consumes columns; multi-tile schedules increase live state. A memory-bound roofline classification also does not prove pipelining is useless: test it only when the proposed change has a specific issue-gap, latency, or transaction-efficiency prediction."}, "reason": {"statement": "Make depth a resource-checked controlled variable.", "urls": ["https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html", "https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100", "https://arxiv.org/html/2603.05451v1"]}} -{"path": "wiki/patterns/pipeline-stalls.md", "before": {"statement": "3. **Missing `tcgen05.fence::after_thread_sync`**: MMA reads SMEM before TMA transfer fully visible"}, "after": {"statement": "Do this before tuning around an mbarrier wait:\n\n1. Identify the exact barrier object, owner, initialized arrival count, initial phase, and storage stage for the wait.\n2. For a TMA global-to-shared load, account for the producer's software arrival and expected transaction bytes. `mbarrier.arrive.expect_tx` performs an arrival and adds transaction bytes; the TMA `.mbarrier::complete_tx::bytes` operation decrements transaction bytes, not a second arrival.\n3. Wait for the matching phase of that same object. Phase parity changes when that barrier completes a phase and is reused; “flip after every wait” is not a valid global rule when a thread waits on multiple objects or phases.\n4. A successful acquire wait supplies the documented visibility for associated prior `cp.async.bulk` work before the consumer reads SMEM.\n5. Before reusing SMEM operands read by asynchronous MMA, attach completion of the relevant tcgen05 work with `tcgen05.commit` and observe its mbarrier. Issue is not completion.\n6. When asynchronous tcgen05 operations cross a thread handoff, place `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` around the applicable execution-ordering operation. The after fence is not a generic replacement for the TMA acquire wait.\n7. Verify prologue, steady-state wraparound, short loops, and the producer/consumer tails. A correct steady-state loop can still hang or reuse live storage at a boundary.\n\nAn extra software arrival is erroneous only relative to the barrier's initialized and phase-specific accounting. Diagnose pending arrivals and transaction bytes separately instead of assuming that TMA hardware performs another arrival."}, "reason": {"statement": "Separate TMA completion/visibility from cross-thread tcgen05 ordering.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"]}} +{"path": "wiki/patterns/pipeline-stalls.md", "before": {"statement": "3. **Missing `tcgen05.fence::after_thread_sync`**: MMA reads SMEM before TMA transfer fully visible"}, "after": {"statement": "Do this before tuning around an mbarrier wait:\n\n1. Identify the exact barrier object, owner, initialized arrival count, initial phase, and storage stage for the wait.\n2. For a TMA global-to-shared load, account for the producer's software arrival and expected transaction bytes. `mbarrier.arrive.expect_tx` performs an arrival and adds transaction bytes; the TMA `.mbarrier::complete_tx::bytes` operation decrements transaction bytes, not a second arrival.\n3. Wait for the matching phase of that same object. Phase parity changes when that barrier completes a phase and is reused; “flip after every wait” is not a valid global rule when a thread waits on multiple objects or phases.\n4. A successful acquire wait supplies the documented visibility for associated prior `cp.async.bulk` work before the consumer reads SMEM.\n5. Before reusing SMEM operands read by asynchronous MMA, attach completion of the relevant tcgen05 work with `tcgen05.commit` and observe its mbarrier. Issue is not completion.\n6. When asynchronous tcgen05 operations cross a thread handoff, place `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` around the applicable execution-ordering operation. The after fence is not a generic replacement for the TMA acquire wait.\n7. Verify prologue, steady-state wraparound, short loops, and the producer/consumer tails. A correct steady-state loop can still hang or reuse live storage at a boundary.\n\nAn extra software arrival is erroneous only relative to the barrier's initialized and phase-specific accounting. Diagnose pending arrivals and transaction bytes separately instead of assuming that TMA hardware performs another arrival."}, "reason": {"statement": "Separate TMA completion/visibility from cross-thread tcgen05 ordering.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-special-sync-operations-fence", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier"]}} {"path": "wiki/patterns/pipeline-stalls.md", "before": {"statement": "4. **Single-tile scheduling**: All warps serialized on one tile's softmax/epilogue"}, "after": {"statement": "1. Establish a synchronized end-to-end timing regression on fixed inputs and confirm outputs against a reference. Record GPU, clocks, toolkit, build flags, kernel name, launch shape, and resource use.\n2. Collect Speed-of-Light, Scheduler Statistics, Warp State Statistics, and source/SASS correlation with the sections available in the installed Nsight Compute. Account for replay and sampling effects.\n3. Map the dominant not-issued locations to concrete edges: TMA-full wait, MMA-completion wait, output-buffer reuse, CTA barrier, queue starvation, or pipeline prologue/tail.\n4. State a prediction. For example: if TMA readiness is the critical edge, a legal extra operand stage should reduce that wait and runtime; if only the tail is exposed, the steady-state wait distribution should remain largely unchanged.\n5. Change one variable and repeat identical correctness, warmup, timing, and profiler collection. Reject a cause when its predicted counter and time movement do not occur.\n\n| Vary legal SMEM stage count | Producer cannot stay far enough ahead of MMA | Same tile/math/roles; record SMEM, occupancy, short-loop behavior, and tail |\n| Separate long-lived warp roles | Role handoff or control work is on the issue path | Same stages and tile; record registers, active warps, and per-role idle/backpressure |\n| Add a second TMEM output region | Epilogue holds the only accumulator region | Prove disjoint lifetime; record TMEM columns and epilogue/MMA completion waits |\n| Interleave two query/output tiles | Softmax/correction and MMA have independent ready work | Prove separate state and dependencies; compare one versus two query stages |\n\nNone is a universal cure. More stages consume storage and can reduce occupancy; specialization adds warps and synchronization; TMEM buffering consumes columns; multi-tile schedules increase live state. A memory-bound roofline classification also does not prove pipelining is useless: test it only when the proposed change has a specific issue-gap, latency, or transaction-efficiency prediction."}, "reason": {"statement": "Present multi-tile scheduling as a hypothesis only when independent state and work exist.", "urls": ["https://arxiv.org/html/2603.05451v1", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html", "https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100"]}} -{"path": "wiki/patterns/pipeline-stalls.md", "before": {"statement": "5. **Producer over-arrives**: Manual `mbarrier_arrive` after async TMA — hardware + manual both arrive, next stage gets stale release"}, "after": {"statement": "Do this before tuning around an mbarrier wait:\n\n1. Identify the exact barrier object, owner, initialized arrival count, initial phase, and storage stage for the wait.\n2. For a TMA global-to-shared load, account for the producer's software arrival and expected transaction bytes. `mbarrier.arrive.expect_tx` performs an arrival and adds transaction bytes; the TMA `.mbarrier::complete_tx::bytes` operation decrements transaction bytes, not a second arrival.\n3. Wait for the matching phase of that same object. Phase parity changes when that barrier completes a phase and is reused; “flip after every wait” is not a valid global rule when a thread waits on multiple objects or phases.\n4. A successful acquire wait supplies the documented visibility for associated prior `cp.async.bulk` work before the consumer reads SMEM.\n5. Before reusing SMEM operands read by asynchronous MMA, attach completion of the relevant tcgen05 work with `tcgen05.commit` and observe its mbarrier. Issue is not completion.\n6. When asynchronous tcgen05 operations cross a thread handoff, place `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` around the applicable execution-ordering operation. The after fence is not a generic replacement for the TMA acquire wait.\n7. Verify prologue, steady-state wraparound, short loops, and the producer/consumer tails. A correct steady-state loop can still hang or reuse live storage at a boundary.\n\nAn extra software arrival is erroneous only relative to the barrier's initialized and phase-specific accounting. Diagnose pending arrivals and transaction bytes separately instead of assuming that TMA hardware performs another arrival."}, "reason": {"statement": "Teach the two independent counters exactly.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence"]}} +{"path": "wiki/patterns/pipeline-stalls.md", "before": {"statement": "5. **Producer over-arrives**: Manual `mbarrier_arrive` after async TMA — hardware + manual both arrive, next stage gets stale release"}, "after": {"statement": "Do this before tuning around an mbarrier wait:\n\n1. Identify the exact barrier object, owner, initialized arrival count, initial phase, and storage stage for the wait.\n2. For a TMA global-to-shared load, account for the producer's software arrival and expected transaction bytes. `mbarrier.arrive.expect_tx` performs an arrival and adds transaction bytes; the TMA `.mbarrier::complete_tx::bytes` operation decrements transaction bytes, not a second arrival.\n3. Wait for the matching phase of that same object. Phase parity changes when that barrier completes a phase and is reused; “flip after every wait” is not a valid global rule when a thread waits on multiple objects or phases.\n4. A successful acquire wait supplies the documented visibility for associated prior `cp.async.bulk` work before the consumer reads SMEM.\n5. Before reusing SMEM operands read by asynchronous MMA, attach completion of the relevant tcgen05 work with `tcgen05.commit` and observe its mbarrier. Issue is not completion.\n6. When asynchronous tcgen05 operations cross a thread handoff, place `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` around the applicable execution-ordering operation. The after fence is not a generic replacement for the TMA acquire wait.\n7. Verify prologue, steady-state wraparound, short loops, and the producer/consumer tails. A correct steady-state loop can still hang or reuse live storage at a boundary.\n\nAn extra software arrival is erroneous only relative to the barrier's initialized and phase-specific accounting. Diagnose pending arrivals and transaction bytes separately instead of assuming that TMA hardware performs another arrival."}, "reason": {"statement": "Teach the two independent counters exactly.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-special-sync-operations-fence"]}} {"path": "wiki/patterns/pipeline-stalls.md", "before": {"statement": "| [Pipeline stages](../techniques/pipeline-stages.md) | Increase NUM_STAGES (3-5 typical on Blackwell) |\n| [Warp specialization](../techniques/warp-specialization.md) | Dedicated warps for TMA/MMA/epilogue eliminate role-switching stalls |"}, "after": {"statement": "1. Establish a synchronized end-to-end timing regression on fixed inputs and confirm outputs against a reference. Record GPU, clocks, toolkit, build flags, kernel name, launch shape, and resource use.\n2. Collect Speed-of-Light, Scheduler Statistics, Warp State Statistics, and source/SASS correlation with the sections available in the installed Nsight Compute. Account for replay and sampling effects.\n3. Map the dominant not-issued locations to concrete edges: TMA-full wait, MMA-completion wait, output-buffer reuse, CTA barrier, queue starvation, or pipeline prologue/tail.\n4. State a prediction. For example: if TMA readiness is the critical edge, a legal extra operand stage should reduce that wait and runtime; if only the tail is exposed, the steady-state wait distribution should remain largely unchanged.\n5. Change one variable and repeat identical correctness, warmup, timing, and profiler collection. Reject a cause when its predicted counter and time movement do not occur.\n\n| Vary legal SMEM stage count | Producer cannot stay far enough ahead of MMA | Same tile/math/roles; record SMEM, occupancy, short-loop behavior, and tail |\n| Separate long-lived warp roles | Role handoff or control work is on the issue path | Same stages and tile; record registers, active warps, and per-role idle/backpressure |\n| Add a second TMEM output region | Epilogue holds the only accumulator region | Prove disjoint lifetime; record TMEM columns and epilogue/MMA completion waits |\n| Interleave two query/output tiles | Softmax/correction and MMA have independent ready work | Prove separate state and dependencies; compare one versus two query stages |\n\nNone is a universal cure. More stages consume storage and can reduce occupancy; specialization adds warps and synchronization; TMEM buffering consumes columns; multi-tile schedules increase live state. A memory-bound roofline classification also does not prove pipelining is useless: test it only when the proposed change has a specific issue-gap, latency, or transaction-efficiency prediction."}, "reason": {"statement": "Turn both into measured candidate changes with explicit costs.", "urls": ["https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu", "https://gau-nernst.github.io/tcgen05/", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html", "https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100", "https://arxiv.org/html/2603.05451v1"]}} -{"path": "wiki/patterns/pipeline-stalls.md", "before": {"statement": "```\n1. Profile with Nsight Compute, check tensor core active cycles\n2. Inspect mbarrier wait stalls in warp state breakdown\n3. Verify phase tracking increments correctly (each wait should flip parity)\n4. Check that TMA uses arrive_expect_tx + mbarrier target (not manual arrive)\n5. Ensure tcgen05.fence::after_thread_sync between TMA wait and MMA issue\n6. Measure pipeline depth: can you add more NUM_STAGES?\n```"}, "after": {"statement": "A pipeline stall is lost issue opportunity caused by a producer-consumer dependency on the critical path. Low TMA or tensor-core activity, a warp sampled at a barrier wait, or a phase-local utilization drop is only a lead. Expected dependency waits, memory latency, execution-pipeline contention, load imbalance, prologue/tail work, and synchronization defects can produce similar observations.\n\nNsight Compute's Warp State Statistics describes why sampled warps could not issue, while Scheduler Statistics shows whether schedulers had eligible warps and issued instructions. NVIDIA cautions that stalls are not necessarily performance-limiting and should be prioritized when schedulers fail to issue. Metric availability and names vary by tool and chip, so query the installed profiler and use its shipped sections rather than assuming a Blackwell metric name. PM sampling can add a timeline on supported systems, but warp sampling itself has no time resolution.\n\nDo this before tuning around an mbarrier wait:\n\n1. Identify the exact barrier object, owner, initialized arrival count, initial phase, and storage stage for the wait.\n2. For a TMA global-to-shared load, account for the producer's software arrival and expected transaction bytes. `mbarrier.arrive.expect_tx` performs an arrival and adds transaction bytes; the TMA `.mbarrier::complete_tx::bytes` operation decrements transaction bytes, not a second arrival.\n3. Wait for the matching phase of that same object. Phase parity changes when that barrier completes a phase and is reused; “flip after every wait” is not a valid global rule when a thread waits on multiple objects or phases.\n4. A successful acquire wait supplies the documented visibility for associated prior `cp.async.bulk` work before the consumer reads SMEM.\n5. Before reusing SMEM operands read by asynchronous MMA, attach completion of the relevant tcgen05 work with `tcgen05.commit` and observe its mbarrier. Issue is not completion.\n6. When asynchronous tcgen05 operations cross a thread handoff, place `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` around the applicable execution-ordering operation. The after fence is not a generic replacement for the TMA acquire wait.\n7. Verify prologue, steady-state wraparound, short loops, and the producer/consumer tails. A correct steady-state loop can still hang or reuse live storage at a boundary.\n\nAn extra software arrival is erroneous only relative to the barrier's initialized and phase-specific accounting. Diagnose pending arrivals and transaction bytes separately instead of assuming that TMA hardware performs another arrival.\n\n1. Establish a synchronized end-to-end timing regression on fixed inputs and confirm outputs against a reference. Record GPU, clocks, toolkit, build flags, kernel name, launch shape, and resource use.\n2. Collect Speed-of-Light, Scheduler Statistics, Warp State Statistics, and source/SASS correlation with the sections available in the installed Nsight Compute. Account for replay and sampling effects.\n3. Map the dominant not-issued locations to concrete edges: TMA-full wait, MMA-completion wait, output-buffer reuse, CTA barrier, queue starvation, or pipeline prologue/tail.\n4. State a prediction. For example: if TMA readiness is the critical edge, a legal extra operand stage should reduce that wait and runtime; if only the tail is exposed, the steady-state wait distribution should remain largely unchanged.\n5. Change one variable and repeat identical correctness, warmup, timing, and profiler collection. Reject a cause when its predicted counter and time movement do not occur.\n\n| Vary legal SMEM stage count | Producer cannot stay far enough ahead of MMA | Same tile/math/roles; record SMEM, occupancy, short-loop behavior, and tail |\n| Separate long-lived warp roles | Role handoff or control work is on the issue path | Same stages and tile; record registers, active warps, and per-role idle/backpressure |\n| Add a second TMEM output region | Epilogue holds the only accumulator region | Prove disjoint lifetime; record TMEM columns and epilogue/MMA completion waits |\n| Interleave two query/output tiles | Softmax/correction and MMA have independent ready work | Prove separate state and dependencies; compare one versus two query stages |\n\nNone is a universal cure. More stages consume storage and can reduce occupancy; specialization adds warps and synchronization; TMEM buffering consumes columns; multi-tile schedules increase live state. A memory-bound roofline classification also does not prove pipelining is useless: test it only when the proposed change has a specific issue-gap, latency, or transaction-efficiency prediction."}, "reason": {"statement": "Replace with separate profiling, barrier-accounting, and controlled-experiment stages.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100", "https://arxiv.org/html/2603.05451v1"]}} +{"path": "wiki/patterns/pipeline-stalls.md", "before": {"statement": "```\n1. Profile with Nsight Compute, check tensor core active cycles\n2. Inspect mbarrier wait stalls in warp state breakdown\n3. Verify phase tracking increments correctly (each wait should flip parity)\n4. Check that TMA uses arrive_expect_tx + mbarrier target (not manual arrive)\n5. Ensure tcgen05.fence::after_thread_sync between TMA wait and MMA issue\n6. Measure pipeline depth: can you add more NUM_STAGES?\n```"}, "after": {"statement": "A pipeline stall is lost issue opportunity caused by a producer-consumer dependency on the critical path. Low TMA or tensor-core activity, a warp sampled at a barrier wait, or a phase-local utilization drop is only a lead. Expected dependency waits, memory latency, execution-pipeline contention, load imbalance, prologue/tail work, and synchronization defects can produce similar observations.\n\nNsight Compute's Warp State Statistics describes why sampled warps could not issue, while Scheduler Statistics shows whether schedulers had eligible warps and issued instructions. NVIDIA cautions that stalls are not necessarily performance-limiting and should be prioritized when schedulers fail to issue. Metric availability and names vary by tool and chip, so query the installed profiler and use its shipped sections rather than assuming a Blackwell metric name. PM sampling can add a timeline on supported systems, but warp sampling itself has no time resolution.\n\nDo this before tuning around an mbarrier wait:\n\n1. Identify the exact barrier object, owner, initialized arrival count, initial phase, and storage stage for the wait.\n2. For a TMA global-to-shared load, account for the producer's software arrival and expected transaction bytes. `mbarrier.arrive.expect_tx` performs an arrival and adds transaction bytes; the TMA `.mbarrier::complete_tx::bytes` operation decrements transaction bytes, not a second arrival.\n3. Wait for the matching phase of that same object. Phase parity changes when that barrier completes a phase and is reused; “flip after every wait” is not a valid global rule when a thread waits on multiple objects or phases.\n4. A successful acquire wait supplies the documented visibility for associated prior `cp.async.bulk` work before the consumer reads SMEM.\n5. Before reusing SMEM operands read by asynchronous MMA, attach completion of the relevant tcgen05 work with `tcgen05.commit` and observe its mbarrier. Issue is not completion.\n6. When asynchronous tcgen05 operations cross a thread handoff, place `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` around the applicable execution-ordering operation. The after fence is not a generic replacement for the TMA acquire wait.\n7. Verify prologue, steady-state wraparound, short loops, and the producer/consumer tails. A correct steady-state loop can still hang or reuse live storage at a boundary.\n\nAn extra software arrival is erroneous only relative to the barrier's initialized and phase-specific accounting. Diagnose pending arrivals and transaction bytes separately instead of assuming that TMA hardware performs another arrival.\n\n1. Establish a synchronized end-to-end timing regression on fixed inputs and confirm outputs against a reference. Record GPU, clocks, toolkit, build flags, kernel name, launch shape, and resource use.\n2. Collect Speed-of-Light, Scheduler Statistics, Warp State Statistics, and source/SASS correlation with the sections available in the installed Nsight Compute. Account for replay and sampling effects.\n3. Map the dominant not-issued locations to concrete edges: TMA-full wait, MMA-completion wait, output-buffer reuse, CTA barrier, queue starvation, or pipeline prologue/tail.\n4. State a prediction. For example: if TMA readiness is the critical edge, a legal extra operand stage should reduce that wait and runtime; if only the tail is exposed, the steady-state wait distribution should remain largely unchanged.\n5. Change one variable and repeat identical correctness, warmup, timing, and profiler collection. Reject a cause when its predicted counter and time movement do not occur.\n\n| Vary legal SMEM stage count | Producer cannot stay far enough ahead of MMA | Same tile/math/roles; record SMEM, occupancy, short-loop behavior, and tail |\n| Separate long-lived warp roles | Role handoff or control work is on the issue path | Same stages and tile; record registers, active warps, and per-role idle/backpressure |\n| Add a second TMEM output region | Epilogue holds the only accumulator region | Prove disjoint lifetime; record TMEM columns and epilogue/MMA completion waits |\n| Interleave two query/output tiles | Softmax/correction and MMA have independent ready work | Prove separate state and dependencies; compare one versus two query stages |\n\nNone is a universal cure. More stages consume storage and can reduce occupancy; specialization adds warps and synchronization; TMEM buffering consumes columns; multi-tile schedules increase live state. A memory-bound roofline classification also does not prove pipelining is useless: test it only when the proposed change has a specific issue-gap, latency, or transaction-efficiency prediction."}, "reason": {"statement": "Replace with separate profiling, barrier-accounting, and controlled-experiment stages.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-special-sync-operations-fence", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier", "https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100", "https://arxiv.org/html/2603.05451v1"]}} {"path": "wiki/patterns/pipeline-stalls.md", "before": {"statement": "- 1-stage: 62% of cuBLAS (TMA blocks MMA)\n- 3-stage pipelined: 70% (hide most TMA latency)\n- Warp specialized: 80% (no role switching)\n- Add 2-SM MMA: 86% (larger tile, more reuse)\n- Persistent + CLC: 98% (eliminate tail effect)"}, "after": {"statement": "Gau Nernst reports the following for one `M=N=K=4096` BF16 GEMM on a Modal B200 using PyTorch 2.9.1 and CUDA 13. Percentages below are computed from the reported 1506.74-TFLOP/s cuBLAS value.\n\n| v1a | basic tcgen05 + 2D 16-byte TMA | 254.62 | 16.90% |\n| v2b | 3D 128-byte TMA | 695.43 | 46.15% |\n| v3 | pipelining | 939.61 | 62.36% |\n| v4 | warp specialization | 1208.83 | 80.23% |\n| v5 | 2-SM MMA | 1302.29 | 86.43% |\n| v6 | persistent kernel with static scheduling | 1475.93 | 97.96% |\n\nThe pinned v3 source instantiates two stages, not three. Each row is a cumulative source version rather than an isolated microbenchmark of the named mechanism. The final version uses static scheduling; the author explicitly says Cluster Launch Control was not added. The result is evidence that these changes helped that implementation and shape, not a Blackwell progression template."}, "reason": {"statement": "Report the exact source table and combined-version scope.", "urls": ["https://gau-nernst.github.io/tcgen05/", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu"]}} {"path": "wiki/patterns/pipeline-stalls.md", "before": {"statement": "- Too many stages consume SMEM; exceeds 228KB budget"}, "after": {"statement": "1. Establish a synchronized end-to-end timing regression on fixed inputs and confirm outputs against a reference. Record GPU, clocks, toolkit, build flags, kernel name, launch shape, and resource use.\n2. Collect Speed-of-Light, Scheduler Statistics, Warp State Statistics, and source/SASS correlation with the sections available in the installed Nsight Compute. Account for replay and sampling effects.\n3. Map the dominant not-issued locations to concrete edges: TMA-full wait, MMA-completion wait, output-buffer reuse, CTA barrier, queue starvation, or pipeline prologue/tail.\n4. State a prediction. For example: if TMA readiness is the critical edge, a legal extra operand stage should reduce that wait and runtime; if only the tail is exposed, the steady-state wait distribution should remain largely unchanged.\n5. Change one variable and repeat identical correctness, warmup, timing, and profiler collection. Reject a cause when its predicted counter and time movement do not occur.\n\n| Vary legal SMEM stage count | Producer cannot stay far enough ahead of MMA | Same tile/math/roles; record SMEM, occupancy, short-loop behavior, and tail |\n| Separate long-lived warp roles | Role handoff or control work is on the issue path | Same stages and tile; record registers, active warps, and per-role idle/backpressure |\n| Add a second TMEM output region | Epilogue holds the only accumulator region | Prove disjoint lifetime; record TMEM columns and epilogue/MMA completion waits |\n| Interleave two query/output tiles | Softmax/correction and MMA have independent ready work | Prove separate state and dependencies; compare one versus two query stages |\n\nNone is a universal cure. More stages consume storage and can reduce occupancy; specialization adds warps and synchronization; TMEM buffering consumes columns; multi-tile schedules increase live state. A memory-bound roofline classification also does not prove pipelining is useless: test it only when the proposed change has a specific issue-gap, latency, or transaction-efficiency prediction."}, "reason": {"statement": "Require explicit storage arithmetic and compiled resource checks.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html", "https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100", "https://arxiv.org/html/2603.05451v1"]}} -{"path": "wiki/patterns/pipeline-stalls.md", "before": {"statement": "- Profile first — pipeline is a waste of effort on memory-bound kernels"}, "after": {"statement": "- [Nsight Compute 2025.3 Profiling Guide](https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html)\n- [PTX ISA 9.0 mbarrier waits and visibility](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait)\n- [PTX ISA 9.0 tcgen05 execution ordering](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence)\n- [Pinned tutorial v3 source](https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu)\n- [Tutorial progression](https://gau-nernst.github.io/tcgen05/)"}, "reason": {"statement": "Use causal measurements rather than excluding an entire category.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu", "https://gau-nernst.github.io/tcgen05/"]}} +{"path": "wiki/patterns/pipeline-stalls.md", "before": {"statement": "- Profile first — pipeline is a waste of effort on memory-bound kernels"}, "after": {"statement": "- [Nsight Compute 2025.3 Profiling Guide](https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html)\n- [PTX ISA 9.0 mbarrier waits and visibility](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait)\n- [PTX ISA 9.0 tcgen05 execution ordering](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-special-sync-operations-fence)\n- [Pinned tutorial v3 source](https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu)\n- [Tutorial progression](https://gau-nernst.github.io/tcgen05/)"}, "reason": {"statement": "Use causal measurements rather than excluding an entire category.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu", "https://gau-nernst.github.io/tcgen05/"]}} {"path": "wiki/patterns/compute-bound.md", "before": {"statement": "Tensor core utilization below 70%. Memory bandwidth is not saturated. Kernel is compute-bound but not reaching peak FLOPS."}, "after": {"statement": "“Below peak FLOPS” is not itself a bottleneck diagnosis. Select the peak for the executed datatype, instruction kind, sparsity/scaling mode, clocks, and number of participating SMs. Then use arithmetic intensity and achieved compute/memory ceilings to decide whether the measured kernel lies on the compute side of the relevant roofline. Unsaturated DRAM bandwidth and tensor-core utilization below an arbitrary threshold such as 70% do not prove compute boundedness.\n\nAlso distinguish whole-kernel throughput from tensor-core active time. A correct kernel can have efficient MMA intervals yet spend material time in TMA readiness, CUDA-core transforms, reductions, synchronization, epilogue work, or the grid tail. Scheduler under-issue plus source/SASS correlation identifies where issue opportunity is lost; a high stall sample count alone does not establish the cause."}, "reason": {"statement": "Replace the threshold with roofline and scheduler evidence.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} -{"path": "wiki/patterns/compute-bound.md", "before": {"statement": "3. **Single-SM MMA tiles too small**: Not fully utilizing available compute"}, "after": {"statement": "Enumerate stage counts that compile and fit the complete shared-storage allocation. More stages can overlap TMA production with MMA consumption, but they also consume SMEM, change occupancy, and enlarge prologue/tail costs. There is no architecture-wide “three to five stages” range; the cited tutorial's beneficial v3 kernel uses two stages.\n\nDedicated producer, MMA, or epilogue warps can remove role switching from a loop and allow independent work to progress, but add live warps, registers, and synchronization. They do not eliminate stalls. If the epilogue owns the only output region, compare one versus multiple disjoint TMEM regions and prove MMA-completion and epilogue-load completion before each handoff.\n\n`cta_group::2` requires a valid cluster and exact kind-, shape-, layout-, descriptor-, and peer-resource constraints. It does not universally mean `m256n256`, require identical SMEM layouts, or promise twice the compute per cycle: one cooperative operation consumes resources from two CTAs. Compare total and per-SM throughput against a legal one-CTA mapping and measure whether peer operand reuse or a larger aggregate tile reduces traffic enough to repay coordination and occupancy costs.\n\nOptimize non-MMA work only after showing that it lies on the critical path. Fusion may remove intermediate traffic; tile interleaving may expose independent work; an approximation may exchange numerical error for throughput. Validate error against the application oracle rather than assuming a software transcendental is interchangeable with the hardware operation."}, "reason": {"statement": "Make CTA-group mode a controlled legal-shape and resource experiment.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu", "https://gau-nernst.github.io/tcgen05/"]}} -{"path": "wiki/patterns/compute-bound.md", "before": {"statement": "| [2-SM cooperative](../hardware/2sm-cooperative.md) | Double effective MMA tile (m256×n256), 2× compute per cycle |"}, "after": {"statement": "Enumerate stage counts that compile and fit the complete shared-storage allocation. More stages can overlap TMA production with MMA consumption, but they also consume SMEM, change occupancy, and enlarge prologue/tail costs. There is no architecture-wide “three to five stages” range; the cited tutorial's beneficial v3 kernel uses two stages.\n\nDedicated producer, MMA, or epilogue warps can remove role switching from a loop and allow independent work to progress, but add live warps, registers, and synchronization. They do not eliminate stalls. If the epilogue owns the only output region, compare one versus multiple disjoint TMEM regions and prove MMA-completion and epilogue-load completion before each handoff.\n\n`cta_group::2` requires a valid cluster and exact kind-, shape-, layout-, descriptor-, and peer-resource constraints. It does not universally mean `m256n256`, require identical SMEM layouts, or promise twice the compute per cycle: one cooperative operation consumes resources from two CTAs. Compare total and per-SM throughput against a legal one-CTA mapping and measure whether peer operand reuse or a larger aggregate tile reduces traffic enough to repay coordination and occupancy costs.\n\nOptimize non-MMA work only after showing that it lies on the critical path. Fusion may remove intermediate traffic; tile interleaving may expose independent work; an approximation may exchange numerical error for throughput. Validate error against the application oracle rather than assuming a software transcendental is interchangeable with the hardware operation."}, "reason": {"statement": "State exact compatibility scope and measure the paired-resource tradeoff.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu", "https://gau-nernst.github.io/tcgen05/"]}} -{"path": "wiki/patterns/compute-bound.md", "before": {"statement": "| [Warp specialization](../techniques/warp-specialization.md) | Dedicated warps for TMA/MMA/epilogue, no stalls |"}, "after": {"statement": "Enumerate stage counts that compile and fit the complete shared-storage allocation. More stages can overlap TMA production with MMA consumption, but they also consume SMEM, change occupancy, and enlarge prologue/tail costs. There is no architecture-wide “three to five stages” range; the cited tutorial's beneficial v3 kernel uses two stages.\n\nDedicated producer, MMA, or epilogue warps can remove role switching from a loop and allow independent work to progress, but add live warps, registers, and synchronization. They do not eliminate stalls. If the epilogue owns the only output region, compare one versus multiple disjoint TMEM regions and prove MMA-completion and epilogue-load completion before each handoff.\n\n`cta_group::2` requires a valid cluster and exact kind-, shape-, layout-, descriptor-, and peer-resource constraints. It does not universally mean `m256n256`, require identical SMEM layouts, or promise twice the compute per cycle: one cooperative operation consumes resources from two CTAs. Compare total and per-SM throughput against a legal one-CTA mapping and measure whether peer operand reuse or a larger aggregate tile reduces traffic enough to repay coordination and occupancy costs.\n\nOptimize non-MMA work only after showing that it lies on the critical path. Fusion may remove intermediate traffic; tile interleaving may expose independent work; an approximation may exchange numerical error for throughput. Validate error against the application oracle rather than assuming a software transcendental is interchangeable with the hardware operation."}, "reason": {"statement": "Treat role separation as a candidate with resource and synchronization costs.", "urls": ["https://gau-nernst.github.io/tcgen05/", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma"]}} +{"path": "wiki/patterns/compute-bound.md", "before": {"statement": "3. **Single-SM MMA tiles too small**: Not fully utilizing available compute"}, "after": {"statement": "Enumerate stage counts that compile and fit the complete shared-storage allocation. More stages can overlap TMA production with MMA consumption, but they also consume SMEM, change occupancy, and enlarge prologue/tail costs. There is no architecture-wide “three to five stages” range; the cited tutorial's beneficial v3 kernel uses two stages.\n\nDedicated producer, MMA, or epilogue warps can remove role switching from a loop and allow independent work to progress, but add live warps, registers, and synchronization. They do not eliminate stalls. If the epilogue owns the only output region, compare one versus multiple disjoint TMEM regions and prove MMA-completion and epilogue-load completion before each handoff.\n\n`cta_group::2` requires a valid cluster and exact kind-, shape-, layout-, descriptor-, and peer-resource constraints. It does not universally mean `m256n256`, require identical SMEM layouts, or promise twice the compute per cycle: one cooperative operation consumes resources from two CTAs. Compare total and per-SM throughput against a legal one-CTA mapping and measure whether peer operand reuse or a larger aggregate tile reduces traffic enough to repay coordination and occupancy costs.\n\nOptimize non-MMA work only after showing that it lies on the critical path. Fusion may remove intermediate traffic; tile interleaving may expose independent work; an approximation may exchange numerical error for throughput. Validate error against the application oracle rather than assuming a software transcendental is interchangeable with the hardware operation."}, "reason": {"statement": "Make CTA-group mode a controlled legal-shape and resource experiment.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu", "https://gau-nernst.github.io/tcgen05/"]}} +{"path": "wiki/patterns/compute-bound.md", "before": {"statement": "| [2-SM cooperative](../hardware/2sm-cooperative.md) | Double effective MMA tile (m256×n256), 2× compute per cycle |"}, "after": {"statement": "Enumerate stage counts that compile and fit the complete shared-storage allocation. More stages can overlap TMA production with MMA consumption, but they also consume SMEM, change occupancy, and enlarge prologue/tail costs. There is no architecture-wide “three to five stages” range; the cited tutorial's beneficial v3 kernel uses two stages.\n\nDedicated producer, MMA, or epilogue warps can remove role switching from a loop and allow independent work to progress, but add live warps, registers, and synchronization. They do not eliminate stalls. If the epilogue owns the only output region, compare one versus multiple disjoint TMEM regions and prove MMA-completion and epilogue-load completion before each handoff.\n\n`cta_group::2` requires a valid cluster and exact kind-, shape-, layout-, descriptor-, and peer-resource constraints. It does not universally mean `m256n256`, require identical SMEM layouts, or promise twice the compute per cycle: one cooperative operation consumes resources from two CTAs. Compare total and per-SM throughput against a legal one-CTA mapping and measure whether peer operand reuse or a larger aggregate tile reduces traffic enough to repay coordination and occupancy costs.\n\nOptimize non-MMA work only after showing that it lies on the critical path. Fusion may remove intermediate traffic; tile interleaving may expose independent work; an approximation may exchange numerical error for throughput. Validate error against the application oracle rather than assuming a software transcendental is interchangeable with the hardware operation."}, "reason": {"statement": "State exact compatibility scope and measure the paired-resource tradeoff.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu", "https://gau-nernst.github.io/tcgen05/"]}} +{"path": "wiki/patterns/compute-bound.md", "before": {"statement": "| [Warp specialization](../techniques/warp-specialization.md) | Dedicated warps for TMA/MMA/epilogue, no stalls |"}, "after": {"statement": "Enumerate stage counts that compile and fit the complete shared-storage allocation. More stages can overlap TMA production with MMA consumption, but they also consume SMEM, change occupancy, and enlarge prologue/tail costs. There is no architecture-wide “three to five stages” range; the cited tutorial's beneficial v3 kernel uses two stages.\n\nDedicated producer, MMA, or epilogue warps can remove role switching from a loop and allow independent work to progress, but add live warps, registers, and synchronization. They do not eliminate stalls. If the epilogue owns the only output region, compare one versus multiple disjoint TMEM regions and prove MMA-completion and epilogue-load completion before each handoff.\n\n`cta_group::2` requires a valid cluster and exact kind-, shape-, layout-, descriptor-, and peer-resource constraints. It does not universally mean `m256n256`, require identical SMEM layouts, or promise twice the compute per cycle: one cooperative operation consumes resources from two CTAs. Compare total and per-SM throughput against a legal one-CTA mapping and measure whether peer operand reuse or a larger aggregate tile reduces traffic enough to repay coordination and occupancy costs.\n\nOptimize non-MMA work only after showing that it lies on the critical path. Fusion may remove intermediate traffic; tile interleaving may expose independent work; an approximation may exchange numerical error for throughput. Validate error against the application oracle rather than assuming a software transcendental is interchangeable with the hardware operation."}, "reason": {"statement": "Treat role separation as a candidate with resource and synchronization costs.", "urls": ["https://gau-nernst.github.io/tcgen05/", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma"]}} {"path": "wiki/patterns/compute-bound.md", "before": {"statement": "```\n// Problem: Blackwell doubles tensor core throughput but SFU count unchanged\n// SFU bottleneck: exp() for softmax\n//\n// Solution: Software 2^x via Cody-Waite + Horner polynomial\n// Distributes across FMA units, multiplying exponential throughput\n// Result: 1605 TFLOPS (71% utilization) on B200\n```"}, "after": {"statement": "FA4's B200 analysis reports 8192 BF16 MMA operations per clock per SM versus 4096 on Hopper, while exponential throughput is 16 operations per clock per SM on both. Its response is a coordinated design: two-output-tile scheduling, selected software exponentials, conditional rescaling, TMEM partitioning, and pipeline/register choices.\n\nThe software path uses base-2 range reduction with `n=floor(x)` and a cubic FMA polynomial for a selected fraction of exponential evaluations; other values still use hardware `ex2`. The paper evaluates approximation error and end-to-end accuracy. This is not a claim that every exponential uses Cody-Waite reduction or that polynomial evaluation universally multiplies exponential throughput.\n\nThe paper reports up to 1613 TFLOP/s and 71% for complete FA4 forward kernels; the author blog reports up to 1605 TFLOP/s and 71%. Neither number isolates software exponentiation or any other single technique. Use FA4 as evidence that non-MMA work can become material after tensor-core throughput increases, not as a recipe for an unrelated compute-bound kernel."}, "reason": {"statement": "Remove the misleading pseudo-code and present the exact scoped case in prose.", "urls": ["https://arxiv.org/html/2603.05451v1", "https://tridao.me/blog/2026/flash4/"]}} -{"path": "wiki/patterns/compute-bound.md", "before": {"statement": "- 2-SM cooperative requires cluster configuration and identical SMEM layouts"}, "after": {"statement": "Enumerate stage counts that compile and fit the complete shared-storage allocation. More stages can overlap TMA production with MMA consumption, but they also consume SMEM, change occupancy, and enlarge prologue/tail costs. There is no architecture-wide “three to five stages” range; the cited tutorial's beneficial v3 kernel uses two stages.\n\nDedicated producer, MMA, or epilogue warps can remove role switching from a loop and allow independent work to progress, but add live warps, registers, and synchronization. They do not eliminate stalls. If the epilogue owns the only output region, compare one versus multiple disjoint TMEM regions and prove MMA-completion and epilogue-load completion before each handoff.\n\n`cta_group::2` requires a valid cluster and exact kind-, shape-, layout-, descriptor-, and peer-resource constraints. It does not universally mean `m256n256`, require identical SMEM layouts, or promise twice the compute per cycle: one cooperative operation consumes resources from two CTAs. Compare total and per-SM throughput against a legal one-CTA mapping and measure whether peer operand reuse or a larger aggregate tile reduces traffic enough to repay coordination and occupancy costs.\n\nOptimize non-MMA work only after showing that it lies on the critical path. Fusion may remove intermediate traffic; tile interleaving may expose independent work; an approximation may exchange numerical error for throughput. Validate error against the application oracle rather than assuming a software transcendental is interchangeable with the hardware operation."}, "reason": {"statement": "Use the actual instruction/configuration contract.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu", "https://gau-nernst.github.io/tcgen05/"]}} -{"path": "wiki/patterns/compute-bound.md", "before": {"statement": "- Pipeline depth tuning is workload-dependent (3-5 stages typical)"}, "after": {"statement": "Enumerate stage counts that compile and fit the complete shared-storage allocation. More stages can overlap TMA production with MMA consumption, but they also consume SMEM, change occupancy, and enlarge prologue/tail costs. There is no architecture-wide “three to five stages” range; the cited tutorial's beneficial v3 kernel uses two stages.\n\nDedicated producer, MMA, or epilogue warps can remove role switching from a loop and allow independent work to progress, but add live warps, registers, and synchronization. They do not eliminate stalls. If the epilogue owns the only output region, compare one versus multiple disjoint TMEM regions and prove MMA-completion and epilogue-load completion before each handoff.\n\n`cta_group::2` requires a valid cluster and exact kind-, shape-, layout-, descriptor-, and peer-resource constraints. It does not universally mean `m256n256`, require identical SMEM layouts, or promise twice the compute per cycle: one cooperative operation consumes resources from two CTAs. Compare total and per-SM throughput against a legal one-CTA mapping and measure whether peer operand reuse or a larger aggregate tile reduces traffic enough to repay coordination and occupancy costs.\n\nOptimize non-MMA work only after showing that it lies on the critical path. Fusion may remove intermediate traffic; tile interleaving may expose independent work; an approximation may exchange numerical error for throughput. Validate error against the application oracle rather than assuming a software transcendental is interchangeable with the hardware operation."}, "reason": {"statement": "Retain only configuration dependence and controlled selection.", "urls": ["https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma", "https://gau-nernst.github.io/tcgen05/"]}} +{"path": "wiki/patterns/compute-bound.md", "before": {"statement": "- 2-SM cooperative requires cluster configuration and identical SMEM layouts"}, "after": {"statement": "Enumerate stage counts that compile and fit the complete shared-storage allocation. More stages can overlap TMA production with MMA consumption, but they also consume SMEM, change occupancy, and enlarge prologue/tail costs. There is no architecture-wide “three to five stages” range; the cited tutorial's beneficial v3 kernel uses two stages.\n\nDedicated producer, MMA, or epilogue warps can remove role switching from a loop and allow independent work to progress, but add live warps, registers, and synchronization. They do not eliminate stalls. If the epilogue owns the only output region, compare one versus multiple disjoint TMEM regions and prove MMA-completion and epilogue-load completion before each handoff.\n\n`cta_group::2` requires a valid cluster and exact kind-, shape-, layout-, descriptor-, and peer-resource constraints. It does not universally mean `m256n256`, require identical SMEM layouts, or promise twice the compute per cycle: one cooperative operation consumes resources from two CTAs. Compare total and per-SM throughput against a legal one-CTA mapping and measure whether peer operand reuse or a larger aggregate tile reduces traffic enough to repay coordination and occupancy costs.\n\nOptimize non-MMA work only after showing that it lies on the critical path. Fusion may remove intermediate traffic; tile interleaving may expose independent work; an approximation may exchange numerical error for throughput. Validate error against the application oracle rather than assuming a software transcendental is interchangeable with the hardware operation."}, "reason": {"statement": "Use the actual instruction/configuration contract.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu", "https://gau-nernst.github.io/tcgen05/"]}} +{"path": "wiki/patterns/compute-bound.md", "before": {"statement": "- Pipeline depth tuning is workload-dependent (3-5 stages typical)"}, "after": {"statement": "Enumerate stage counts that compile and fit the complete shared-storage allocation. More stages can overlap TMA production with MMA consumption, but they also consume SMEM, change occupancy, and enlarge prologue/tail costs. There is no architecture-wide “three to five stages” range; the cited tutorial's beneficial v3 kernel uses two stages.\n\nDedicated producer, MMA, or epilogue warps can remove role switching from a loop and allow independent work to progress, but add live warps, registers, and synchronization. They do not eliminate stalls. If the epilogue owns the only output region, compare one versus multiple disjoint TMEM regions and prove MMA-completion and epilogue-load completion before each handoff.\n\n`cta_group::2` requires a valid cluster and exact kind-, shape-, layout-, descriptor-, and peer-resource constraints. It does not universally mean `m256n256`, require identical SMEM layouts, or promise twice the compute per cycle: one cooperative operation consumes resources from two CTAs. Compare total and per-SM throughput against a legal one-CTA mapping and measure whether peer operand reuse or a larger aggregate tile reduces traffic enough to repay coordination and occupancy costs.\n\nOptimize non-MMA work only after showing that it lies on the critical path. Fusion may remove intermediate traffic; tile interleaving may expose independent work; an approximation may exchange numerical error for throughput. Validate error against the application oracle rather than assuming a software transcendental is interchangeable with the hardware operation."}, "reason": {"statement": "Retain only configuration dependence and controlled selection.", "urls": ["https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma", "https://gau-nernst.github.io/tcgen05/"]}} {"path": "wiki/patterns/low-sm-utilization.md", "before": {"statement": "SM utilization below 60% despite sufficient occupancy. Nsight Compute shows idle SMs during portions of kernel execution."}, "after": {"statement": "Low SM utilization means that fewer SMs perform useful work than the workload could profitably use during a material part of its measured time. It is not defined by a universal 60% threshold. Theoretical occupancy describes how many blocks or warps can reside from resource limits; it does not prove that the grid supplies those workers, that they are simultaneously active, or that their work is balanced.\n\nCollect a synchronized kernel time and a time-resolved view where supported. Record the physical and application-constrained SM count, grid and cluster dimensions, blocks resident per SM, logical work count, work duration distribution, and active/idle intervals. Aggregate SM activity alone cannot distinguish a small grid, a partial final wave, variable-duration work, resource-limited residency, or phase-local serial work."}, "reason": {"statement": "Define the symptom from insufficient useful active work relative to required parallelism and timing.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} -{"path": "wiki/patterns/low-sm-utilization.md", "before": {"statement": "| [CLC](../hardware/clc.md) | SM100 only | Dynamic tile assignment, eliminates load imbalance |"}, "after": {"statement": "Persistence, coordinate order, CLC reassignment, and K decomposition are separate controls:\n\n- A static persistent worker can process multiple logical tiles by grid stride. It can amortize setup and change wave behavior but cannot guarantee removal of the final tail.\n- A row/column raster or swizzle changes coordinate order and may change operand locality. It does not guarantee a better L2 hit rate or lower work variance.\n- Cluster Launch Control lets a running worker cancel an unspecified not-yet-started block or cluster from the launched grid and process the returned ID. It redistributes existing work; it cannot create independent tiles, discard required work, or guarantee equal worker times.\n- Stream-K or Split-K can create additional partitions when tile-level parallelism is insufficient, at the cost of partial-result reduction, workspace, synchronization, and possible determinism changes.\n\nFor CLC, compare static and dynamic scheduling with identical math, tile/cluster shapes, problem-sized grid, resource limits, and timing method. Record successful and failed requests plus per-worker work counts where instrumentation permits. Include ordinary and intentionally uneven SM availability as separate cases.\n\nPTX ISA 9.0 says `clusterlaunchcontrol.try_cancel` requires `sm_100` or higher; its cluster-wide multicast qualifier explicitly lists `sm_120a` and the SM120 family. Therefore CLC is not categorically excluded from SM120 by the ISA. The pinned CUTLASS 4.5.0 persistent scheduler discussed in this wiki is specifically an SM100 integration; do not generalize a library route into an architecture exclusion."}, "reason": {"statement": "State exact redistribution scope and failure/parallelism limits.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} -{"path": "wiki/patterns/low-sm-utilization.md", "before": {"statement": "| [Persistent kernels](../techniques/persistent-kernels.md) | SM90+ | Eliminates tail effect, one-time launch overhead |"}, "after": {"statement": "Persistence, coordinate order, CLC reassignment, and K decomposition are separate controls:\n\n- A static persistent worker can process multiple logical tiles by grid stride. It can amortize setup and change wave behavior but cannot guarantee removal of the final tail.\n- A row/column raster or swizzle changes coordinate order and may change operand locality. It does not guarantee a better L2 hit rate or lower work variance.\n- Cluster Launch Control lets a running worker cancel an unspecified not-yet-started block or cluster from the launched grid and process the returned ID. It redistributes existing work; it cannot create independent tiles, discard required work, or guarantee equal worker times.\n- Stream-K or Split-K can create additional partitions when tile-level parallelism is insufficient, at the cost of partial-result reduction, workspace, synchronization, and possible determinism changes.\n\nFor CLC, compare static and dynamic scheduling with identical math, tile/cluster shapes, problem-sized grid, resource limits, and timing method. Record successful and failed requests plus per-worker work counts where instrumentation permits. Include ordinary and intentionally uneven SM availability as separate cases.\n\nPTX ISA 9.0 says `clusterlaunchcontrol.try_cancel` requires `sm_100` or higher; its cluster-wide multicast qualifier explicitly lists `sm_120a` and the SM120 family. Therefore CLC is not categorically excluded from SM120 by the ISA. The pinned CUTLASS 4.5.0 persistent scheduler discussed in this wiki is specifically an SM100 integration; do not generalize a library route into an architecture exclusion."}, "reason": {"statement": "Separate persistence from dynamic assignment and measure tail distribution.", "urls": ["https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} -{"path": "wiki/patterns/low-sm-utilization.md", "before": {"statement": "| [Tile scheduling](../techniques/tile-scheduling.md) | SM90+ | Better L2 locality, reduce load variance |"}, "after": {"statement": "Persistence, coordinate order, CLC reassignment, and K decomposition are separate controls:\n\n- A static persistent worker can process multiple logical tiles by grid stride. It can amortize setup and change wave behavior but cannot guarantee removal of the final tail.\n- A row/column raster or swizzle changes coordinate order and may change operand locality. It does not guarantee a better L2 hit rate or lower work variance.\n- Cluster Launch Control lets a running worker cancel an unspecified not-yet-started block or cluster from the launched grid and process the returned ID. It redistributes existing work; it cannot create independent tiles, discard required work, or guarantee equal worker times.\n- Stream-K or Split-K can create additional partitions when tile-level parallelism is insufficient, at the cost of partial-result reduction, workspace, synchronization, and possible determinism changes.\n\nFor CLC, compare static and dynamic scheduling with identical math, tile/cluster shapes, problem-sized grid, resource limits, and timing method. Record successful and failed requests plus per-worker work counts where instrumentation permits. Include ordinary and intentionally uneven SM availability as separate cases.\n\nPTX ISA 9.0 says `clusterlaunchcontrol.try_cancel` requires `sm_100` or higher; its cluster-wide multicast qualifier explicitly lists `sm_120a` and the SM120 family. Therefore CLC is not categorically excluded from SM120 by the ISA. The pinned CUTLASS 4.5.0 persistent scheduler discussed in this wiki is specifically an SM100 integration; do not generalize a library route into an architecture exclusion."}, "reason": {"statement": "Map each scheduler choice to a specific measured hypothesis.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"]}} +{"path": "wiki/patterns/low-sm-utilization.md", "before": {"statement": "| [CLC](../hardware/clc.md) | SM100 only | Dynamic tile assignment, eliminates load imbalance |"}, "after": {"statement": "Persistence, coordinate order, CLC reassignment, and K decomposition are separate controls:\n\n- A static persistent worker can process multiple logical tiles by grid stride. It can amortize setup and change wave behavior but cannot guarantee removal of the final tail.\n- A row/column raster or swizzle changes coordinate order and may change operand locality. It does not guarantee a better L2 hit rate or lower work variance.\n- Cluster Launch Control lets a running worker cancel an unspecified not-yet-started block or cluster from the launched grid and process the returned ID. It redistributes existing work; it cannot create independent tiles, discard required work, or guarantee equal worker times.\n- Stream-K or Split-K can create additional partitions when tile-level parallelism is insufficient, at the cost of partial-result reduction, workspace, synchronization, and possible determinism changes.\n\nFor CLC, compare static and dynamic scheduling with identical math, tile/cluster shapes, problem-sized grid, resource limits, and timing method. Record successful and failed requests plus per-worker work counts where instrumentation permits. Include ordinary and intentionally uneven SM availability as separate cases.\n\nPTX ISA 9.0 says `clusterlaunchcontrol.try_cancel` requires `sm_100` or higher; its cluster-wide multicast qualifier explicitly lists `sm_120a` and the SM120 family. Therefore CLC is not categorically excluded from SM120 by the ISA. The pinned CUTLASS 4.5.0 persistent scheduler discussed in this wiki is specifically an SM100 integration; do not generalize a library route into an architecture exclusion."}, "reason": {"statement": "State exact redistribution scope and failure/parallelism limits.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/patterns/low-sm-utilization.md", "before": {"statement": "| [Persistent kernels](../techniques/persistent-kernels.md) | SM90+ | Eliminates tail effect, one-time launch overhead |"}, "after": {"statement": "Persistence, coordinate order, CLC reassignment, and K decomposition are separate controls:\n\n- A static persistent worker can process multiple logical tiles by grid stride. It can amortize setup and change wave behavior but cannot guarantee removal of the final tail.\n- A row/column raster or swizzle changes coordinate order and may change operand locality. It does not guarantee a better L2 hit rate or lower work variance.\n- Cluster Launch Control lets a running worker cancel an unspecified not-yet-started block or cluster from the launched grid and process the returned ID. It redistributes existing work; it cannot create independent tiles, discard required work, or guarantee equal worker times.\n- Stream-K or Split-K can create additional partitions when tile-level parallelism is insufficient, at the cost of partial-result reduction, workspace, synchronization, and possible determinism changes.\n\nFor CLC, compare static and dynamic scheduling with identical math, tile/cluster shapes, problem-sized grid, resource limits, and timing method. Record successful and failed requests plus per-worker work counts where instrumentation permits. Include ordinary and intentionally uneven SM availability as separate cases.\n\nPTX ISA 9.0 says `clusterlaunchcontrol.try_cancel` requires `sm_100` or higher; its cluster-wide multicast qualifier explicitly lists `sm_120a` and the SM120 family. Therefore CLC is not categorically excluded from SM120 by the ISA. The pinned CUTLASS 4.5.0 persistent scheduler discussed in this wiki is specifically an SM100 integration; do not generalize a library route into an architecture exclusion."}, "reason": {"statement": "Separate persistence from dynamic assignment and measure tail distribution.", "urls": ["https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/patterns/low-sm-utilization.md", "before": {"statement": "| [Tile scheduling](../techniques/tile-scheduling.md) | SM90+ | Better L2 locality, reduce load variance |"}, "after": {"statement": "Persistence, coordinate order, CLC reassignment, and K decomposition are separate controls:\n\n- A static persistent worker can process multiple logical tiles by grid stride. It can amortize setup and change wave behavior but cannot guarantee removal of the final tail.\n- A row/column raster or swizzle changes coordinate order and may change operand locality. It does not guarantee a better L2 hit rate or lower work variance.\n- Cluster Launch Control lets a running worker cancel an unspecified not-yet-started block or cluster from the launched grid and process the returned ID. It redistributes existing work; it cannot create independent tiles, discard required work, or guarantee equal worker times.\n- Stream-K or Split-K can create additional partitions when tile-level parallelism is insufficient, at the cost of partial-result reduction, workspace, synchronization, and possible determinism changes.\n\nFor CLC, compare static and dynamic scheduling with identical math, tile/cluster shapes, problem-sized grid, resource limits, and timing method. Record successful and failed requests plus per-worker work counts where instrumentation permits. Include ordinary and intentionally uneven SM availability as separate cases.\n\nPTX ISA 9.0 says `clusterlaunchcontrol.try_cancel` requires `sm_100` or higher; its cluster-wide multicast qualifier explicitly lists `sm_120a` and the SM120 family. Therefore CLC is not categorically excluded from SM120 by the ISA. The pinned CUTLASS 4.5.0 persistent scheduler discussed in this wiki is specifically an SM100 integration; do not generalize a library route into an architecture exclusion."}, "reason": {"statement": "Map each scheduler choice to a specific measured hypothesis.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel"]}} {"path": "wiki/patterns/low-sm-utilization.md", "before": {"statement": "```\n// tcgen05 tutorial progression:\n// Without persistent/CLC: 86% of cuBLAS (some SMs idle at wave boundaries)\n// With persistent + CLC: 98% of cuBLAS (all SMs stay busy)\n```"}, "after": {"statement": "In Gau Nernst's `M=N=K=4096` B200 experiment, v5 reports 1302.29 TFLOP/s (86.43% of its 1506.74-TFLOP/s cuBLAS value), while v6 reports 1475.93 TFLOP/s (97.96%). The v6 endpoint combines static persistence, a changed output pipeline, and epilogue-specialized warps on top of earlier changes. The author says CLC was not added and observes that overlap remains imperfect. This is not a CLC result, an all-SMs-busy measurement, or an isolated tail-effect ablation."}, "reason": {"statement": "Report exact cumulative version scope without all-SMs-busy causality.", "urls": ["https://gau-nernst.github.io/tcgen05/", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu"]}} -{"path": "wiki/patterns/low-sm-utilization.md", "before": {"statement": "- CLC only available on SM100 datacenter GPUs (not SM120 consumer)"}, "after": {"statement": "Persistence, coordinate order, CLC reassignment, and K decomposition are separate controls:\n\n- A static persistent worker can process multiple logical tiles by grid stride. It can amortize setup and change wave behavior but cannot guarantee removal of the final tail.\n- A row/column raster or swizzle changes coordinate order and may change operand locality. It does not guarantee a better L2 hit rate or lower work variance.\n- Cluster Launch Control lets a running worker cancel an unspecified not-yet-started block or cluster from the launched grid and process the returned ID. It redistributes existing work; it cannot create independent tiles, discard required work, or guarantee equal worker times.\n- Stream-K or Split-K can create additional partitions when tile-level parallelism is insufficient, at the cost of partial-result reduction, workspace, synchronization, and possible determinism changes.\n\nFor CLC, compare static and dynamic scheduling with identical math, tile/cluster shapes, problem-sized grid, resource limits, and timing method. Record successful and failed requests plus per-worker work counts where instrumentation permits. Include ordinary and intentionally uneven SM availability as separate cases.\n\nPTX ISA 9.0 says `clusterlaunchcontrol.try_cancel` requires `sm_100` or higher; its cluster-wide multicast qualifier explicitly lists `sm_120a` and the SM120 family. Therefore CLC is not categorically excluded from SM120 by the ISA. The pinned CUTLASS 4.5.0 persistent scheduler discussed in this wiki is specifically an SM100 integration; do not generalize a library route into an architecture exclusion."}, "reason": {"statement": "Separate PTX target support from pinned CUTLASS integration.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/patterns/low-sm-utilization.md", "before": {"statement": "- CLC only available on SM100 datacenter GPUs (not SM120 consumer)"}, "after": {"statement": "Persistence, coordinate order, CLC reassignment, and K decomposition are separate controls:\n\n- A static persistent worker can process multiple logical tiles by grid stride. It can amortize setup and change wave behavior but cannot guarantee removal of the final tail.\n- A row/column raster or swizzle changes coordinate order and may change operand locality. It does not guarantee a better L2 hit rate or lower work variance.\n- Cluster Launch Control lets a running worker cancel an unspecified not-yet-started block or cluster from the launched grid and process the returned ID. It redistributes existing work; it cannot create independent tiles, discard required work, or guarantee equal worker times.\n- Stream-K or Split-K can create additional partitions when tile-level parallelism is insufficient, at the cost of partial-result reduction, workspace, synchronization, and possible determinism changes.\n\nFor CLC, compare static and dynamic scheduling with identical math, tile/cluster shapes, problem-sized grid, resource limits, and timing method. Record successful and failed requests plus per-worker work counts where instrumentation permits. Include ordinary and intentionally uneven SM availability as separate cases.\n\nPTX ISA 9.0 says `clusterlaunchcontrol.try_cancel` requires `sm_100` or higher; its cluster-wide multicast qualifier explicitly lists `sm_120a` and the SM120 family. Therefore CLC is not categorically excluded from SM120 by the ISA. The pinned CUTLASS 4.5.0 persistent scheduler discussed in this wiki is specifically an SM100 integration; do not generalize a library route into an architecture exclusion."}, "reason": {"statement": "Separate PTX target support from pinned CUTLASS integration.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} {"path": "wiki/patterns/low-sm-utilization.md", "before": {"statement": "- For non-persistent kernels, ensure grid size >> SM count"}, "after": {"statement": "| Too little independent work | Logical block/cluster count is below available one-wave worker capacity | Change problem decomposition, accounting for reduction and synchronization cost |\n| Wave quantization | Equal-duration data-parallel work has a small nonzero final-wave remainder | Compare tile shapes or a decomposition that changes tile count |\n| Variable work duration | Per-worker tile counts/times have a long tail despite enough pending work | Compare static and dynamic acquisition with identical tile/decomposition |\n| Residency or phase limitation | Grid is large, but resources or serial phases limit active blocks/warps | Change the limiting resource or phase; more grid blocks alone do not help |\n\nStatic assignment is nonadaptive, but it is not automatically imbalanced. A grid with fewer independent blocks than SMs cannot occupy every SM, while a grid much larger than the SM count can still show a tail or long-running stragglers. Do not prescribe `grid size >> SM count` without a decomposition that creates useful independent work."}, "reason": {"statement": "Use explicit worker-demand and tile arithmetic.", "urls": ["https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} {"path": "wiki/patterns/memory-bound.md", "before": {"statement": "Nsight Compute shows high DRAM throughput but low tensor core utilization. Arithmetic intensity below the roofline knee point."}, "after": {"statement": "A kernel is memory-bandwidth bound when its measured arithmetic intensity places it on the memory side of the relevant roofline and its useful throughput is limited by an attained memory ceiling. Compute intensity from the operations actually performed and bytes transferred at the memory level under study. Compare against a measured ceiling for the same device, clocks, memory level, datatype path, and environment; a nominal product bandwidth is only an upper-bound input.\n\nHigh DRAM throughput, low tensor-core activity, or a workload label such as GEMV is not sufficient alone. Record useful/requested bytes, transferred sectors or bytes, achieved bandwidth, cache behavior, scheduler issue/stalls, and end-to-end time. This separates four cases that can otherwise look similar:\n\n| Useful bytes and transferred bytes are close; attained bandwidth is near the measured roof | Plausible bandwidth-ceiling limit |\n| Transferred bytes substantially exceed useful bytes | Coalescing, overfetch, cache, or redundant-traffic problem |\n| Bandwidth is below the roof and warps lack ready work | Latency, dependency, insufficient concurrency, or issue problem |\n| Bandwidth is high only during one phase | Phase balance or fusion opportunity; whole-kernel boundedness remains unproven |\n\nSingle-use data can lower operations per transferred byte, but “poor reuse” is actionable only if additional legal reuse exists. Uncoalesced access and cache interference may increase traffic or latency; they are not synonyms for saturation of the DRAM bandwidth ceiling."}, "reason": {"statement": "Require a complete roofline and traffic/issue record.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} {"path": "wiki/patterns/memory-bound.md", "before": {"statement": "1. **Low arithmetic intensity**: Operations like GEMV, small batch decode, or reduction kernels"}, "after": {"statement": "A kernel is memory-bandwidth bound when its measured arithmetic intensity places it on the memory side of the relevant roofline and its useful throughput is limited by an attained memory ceiling. Compute intensity from the operations actually performed and bytes transferred at the memory level under study. Compare against a measured ceiling for the same device, clocks, memory level, datatype path, and environment; a nominal product bandwidth is only an upper-bound input.\n\nHigh DRAM throughput, low tensor-core activity, or a workload label such as GEMV is not sufficient alone. Record useful/requested bytes, transferred sectors or bytes, achieved bandwidth, cache behavior, scheduler issue/stalls, and end-to-end time. This separates four cases that can otherwise look similar:\n\n| Useful bytes and transferred bytes are close; attained bandwidth is near the measured roof | Plausible bandwidth-ceiling limit |\n| Transferred bytes substantially exceed useful bytes | Coalescing, overfetch, cache, or redundant-traffic problem |\n| Bandwidth is below the roof and warps lack ready work | Latency, dependency, insufficient concurrency, or issue problem |\n| Bandwidth is high only during one phase | Phase balance or fusion opportunity; whole-kernel boundedness remains unproven |\n\nSingle-use data can lower operations per transferred byte, but “poor reuse” is actionable only if additional legal reuse exists. Uncoalesced access and cache interference may increase traffic or latency; they are not synonyms for saturation of the DRAM bandwidth ceiling."}, "reason": {"statement": "Use measured work and bytes for the actual implementation/input.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} @@ -446,30 +446,30 @@ {"path": "wiki/patterns/memory-bound.md", "before": {"statement": "- B200 has 8 TB/s bandwidth; speed-of-light calculation determines achievable performance"}, "after": {"statement": "A kernel is memory-bandwidth bound when its measured arithmetic intensity places it on the memory side of the relevant roofline and its useful throughput is limited by an attained memory ceiling. Compute intensity from the operations actually performed and bytes transferred at the memory level under study. Compare against a measured ceiling for the same device, clocks, memory level, datatype path, and environment; a nominal product bandwidth is only an upper-bound input.\n\nHigh DRAM throughput, low tensor-core activity, or a workload label such as GEMV is not sufficient alone. Record useful/requested bytes, transferred sectors or bytes, achieved bandwidth, cache behavior, scheduler issue/stalls, and end-to-end time. This separates four cases that can otherwise look similar:\n\n| Useful bytes and transferred bytes are close; attained bandwidth is near the measured roof | Plausible bandwidth-ceiling limit |\n| Transferred bytes substantially exceed useful bytes | Coalescing, overfetch, cache, or redundant-traffic problem |\n| Bandwidth is below the roof and warps lack ready work | Latency, dependency, insufficient concurrency, or issue problem |\n| Bandwidth is high only during one phase | Phase balance or fusion opportunity; whole-kernel boundedness remains unproven |\n\nSingle-use data can lower operations per transferred byte, but “poor reuse” is actionable only if additional legal reuse exists. Uncoalesced access and cache interference may increase traffic or latency; they are not synonyms for saturation of the DRAM bandwidth ceiling."}, "reason": {"statement": "Use measured ceiling and efficiency with complete environment scope.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html#roofline", "https://www.nvidia.com/en-us/data-center/blackwell/", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} {"path": "wiki/patterns/memory-bound.md", "before": {"statement": "- ILP and compute optimizations have diminishing returns for memory-bound kernels"}, "after": {"statement": "Choose only legal, naturally aligned vector forms and handle tails separately. Compare scalar/narrow and wider variants with identical mapping. Record instruction count, requested and transferred bytes, transactions, registers, spills, achieved bandwidth, and time. Wider instructions do not guarantee fewer hardware transactions or higher bandwidth.\n\nPTX L1 eviction priorities and L2 prefetch controls are hints. `no_allocate` does not guarantee bypass, and `evict_last` does not guarantee residence. Characterize address order, reuse distance, working set, and interfering streams, then vary one hint at a time against the default. Retain it only when a representative end-to-end comparison improves.\n\nInspect compiled registers, spills, SMEM, threads, and the actual occupancy-limiting resource before applying `-maxrregcount`. A cap may be clamped by ABI requirements, introduce spill traffic, or leave occupancy unchanged. Higher theoretical occupancy is useful only if it raises ready work or attained bandwidth enough to reduce runtime.\n\nTMA multicast can issue one global-to-shared tensor copy to selected CTAs' shared-memory destinations within a cluster. It is relevant when those CTAs consume the same operand and the cluster/lifetime/barrier costs are valid. It does not help a stream with no inter-CTA reuse.\n\nSwizzling changes a shared-memory address mapping and can reduce conflicts for a specified access pattern. It does not universally eliminate conflicts and is not by itself a DRAM optimization. Validate the legal tensor-map/layout constraints and compare bank-conflict, traffic, and time measurements.\n\nThe cited Amandeep NVFP4 GEMV report is useful precisely because plausible memory-oriented changes failed. A wider `uint2` load was 16–25% slower on the reported shapes, and reducing `maxrregcount` from 80 to 64 had no effect. These observations do not prove that wide loads or register caps are generally harmful; they refute a fixed optimization priority based only on the “memory-bound GEMV” label.\n\nUse the same discipline for compute work. If the kernel is genuinely at its attainable memory roof, compute-only instruction reductions may have little effect. But address calculation, decoding, and instruction-level parallelism can affect memory issue and latency hiding before that roof is reached. State a predicted counter change, alter one variable, and keep or reject the hypothesis from matched time and profiler evidence."}, "reason": {"statement": "Tie every candidate to a measured critical resource and predicted counter movement.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor", "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"]}} {"path": "wiki/patterns/register-pressure.md", "before": {"statement": "Occupancy below target due to high register usage per thread. Nsight Compute shows register spilling to local memory."}, "after": {"statement": "Register pressure is performance-relevant only when compiled register allocation or spills constrain useful scheduling or add material local-memory traffic. “Occupancy below target” is not enough: theoretical occupancy is a residency limit, achieved occupancy is runtime behavior, and maximum occupancy is not a universal optimum.\n\nFor the exact binary and launch, record registers per thread, allocation granularity, spill stores/loads, local-memory traffic, threads and warps per CTA, SMEM, cluster shape, theoretical limiting resource, achieved active/eligible warps, scheduler stalls, and end-to-end time. Inspect generated SASS/source correlation to find the live ranges or spill sites. Then change one source or compiler factor and require the predicted resource/stall movement plus a runtime improvement.\n\nCommon contributors include a resident MMA fragment, an epilogue whose temporaries overlap the mainloop, descriptors/addresses and pipeline state, unrolled loops, and values live across branches. These are hypotheses about compiled liveness, not conclusions from source complexity alone."}, "reason": {"statement": "Require compiled resource and limiting-resource evidence plus a causal variant.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} -{"path": "wiki/patterns/register-pressure.md", "before": {"statement": "| [TMEM](../hardware/tmem.md) | SM100 only | Moves accumulators to dedicated 256KB memory |"}, "after": {"statement": "For Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, that is 128 32-bit registers (512 bytes) per thread for D. This exact fragment arithmetic does not include other live state or guarantee a particular resident-block count.\n\nOn the SM100 tcgen05 path, resident D is in TMEM, organized as 128 lanes by 512 columns of 32-bit cells. That removes the long-lived per-thread WGMMA D fragment, but not all accumulator-related register work: addresses, descriptors, barriers, loop state, `tcgen05.ld` destinations, conversion, and epilogue temporaries still consume registers. Logical capacity and required columns depend on MMA kind, shape, and packing, so a largest-operation or always-double-bufferable rule cannot be inferred from raw cell count.\n\nThe migration must implement collective allocation/address publication, tcgen05 MMA completion, cross-thread ordering where applicable, collective TMEM loads and `tcgen05.wait::ld`, consumer completion, and collective deallocation before exit. The described TMEM/tcgen05 path targets SM100-class data-center architectures and is not the SM120 consumer MMA path."}, "reason": {"statement": "State organization and resident-D scope without promising exact register relief.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"]}} -{"path": "wiki/patterns/register-pressure.md", "before": {"statement": "| [Warp specialization](../techniques/warp-specialization.md) | SM100+ | Different warps handle different roles, reducing per-warp register needs |"}, "after": {"statement": "Move epilogue work after accumulator lifetime when dependencies allow, reduce unnecessary unrolling, or split a long-lived role. Warp specialization exists on Hopper as well as Blackwell; it can shorten one role's live set, but adds role state and synchronization and does not guarantee fewer registers. Compare compiled resources and time for matched role layouts.\n\nCompare an uncapped build with selected `-maxrregcount` values. Record whether the requested cap is effective, the occupancy-limiting resource, spill traffic, instructions, and runtime. ABI minima can constrain the cap, another resource can remain limiting, and spills can erase any benefit."}, "reason": {"statement": "Separate architecture availability from measured role-liveness/resource changes.", "urls": ["https://docs.nvidia.com/cutlass/media/docs/cpp/hopper_gemm.html", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html"]}} -{"path": "wiki/patterns/register-pressure.md", "before": {"statement": "```\n// Hopper: 64×256 MMA tile accumulator = 64*256*4 bytes in registers per warp group\n// → ~128 registers per thread just for accumulators\n//\n// Blackwell: TMEM holds accumulators\n// → 0 registers for accumulators\n// → ~128 registers freed for other work or higher occupancy\n//\n// TMEM: 256 KB per SM, 128 rows × 512 columns × 32-bit\n// Largest 1-SM MMA uses half → double-buffering possible\n```"}, "after": {"statement": "For Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, that is 128 32-bit registers (512 bytes) per thread for D. This exact fragment arithmetic does not include other live state or guarantee a particular resident-block count.\n\nOn the SM100 tcgen05 path, resident D is in TMEM, organized as 128 lanes by 512 columns of 32-bit cells. That removes the long-lived per-thread WGMMA D fragment, but not all accumulator-related register work: addresses, descriptors, barriers, loop state, `tcgen05.ld` destinations, conversion, and epilogue temporaries still consume registers. Logical capacity and required columns depend on MMA kind, shape, and packing, so a largest-operation or always-double-bufferable rule cannot be inferred from raw cell count.\n\nThe migration must implement collective allocation/address publication, tcgen05 MMA completion, cross-thread ordering where applicable, collective TMEM loads and `tcgen05.wait::ld`, consumer completion, and collective deallocation before exit. The described TMEM/tcgen05 path targets SM100-class data-center architectures and is not the SM120 consumer MMA path."}, "reason": {"statement": "Keep the exact comparison while naming residual registers and configuration-specific capacity.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld"]}} -{"path": "wiki/patterns/register-pressure.md", "before": {"statement": "- TMEM→register transfer adds latency (offset by freeing registers)"}, "after": {"statement": "TMEM loads are asynchronous operations with explicit completion waits. Whether their issue/wait and epilogue work are repaid by lower long-lived register use is an empirical whole-kernel question. Compare the Hopper and Blackwell designs only with equivalent math and precision; for same-architecture alternatives, compare TMEM region count/layout while holding tile and launch policy fixed.\n\nReport compiled registers and spills, TMEM columns, SMEM, barriers, occupancy limit, achieved warps, pipeline/scoreboard stalls, correctness, and time. Include shapes where register occupancy changes and shapes where it does not. A reduction in register count without a useful residency, issue, traffic, or time improvement is not sufficient."}, "reason": {"statement": "Make it a controlled whole-kernel tradeoff.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} +{"path": "wiki/patterns/register-pressure.md", "before": {"statement": "| [TMEM](../hardware/tmem.md) | SM100 only | Moves accumulators to dedicated 256KB memory |"}, "after": {"statement": "For Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, that is 128 32-bit registers (512 bytes) per thread for D. This exact fragment arithmetic does not include other live state or guarantee a particular resident-block count.\n\nOn the SM100 tcgen05 path, resident D is in TMEM, organized as 128 lanes by 512 columns of 32-bit cells. That removes the long-lived per-thread WGMMA D fragment, but not all accumulator-related register work: addresses, descriptors, barriers, loop state, `tcgen05.ld` destinations, conversion, and epilogue temporaries still consume registers. Logical capacity and required columns depend on MMA kind, shape, and packing, so a largest-operation or always-double-bufferable rule cannot be inferred from raw cell count.\n\nThe migration must implement collective allocation/address publication, tcgen05 MMA completion, cross-thread ordering where applicable, collective TMEM loads and `tcgen05.wait::ld`, consumer completion, and collective deallocation before exit. The described TMEM/tcgen05 path targets SM100-class data-center architectures and is not the SM120 consumer MMA path."}, "reason": {"statement": "State organization and resident-D scope without promising exact register relief.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld"]}} +{"path": "wiki/patterns/register-pressure.md", "before": {"statement": "| [Warp specialization](../techniques/warp-specialization.md) | SM100+ | Different warps handle different roles, reducing per-warp register needs |"}, "after": {"statement": "Move epilogue work after accumulator lifetime when dependencies allow, reduce unnecessary unrolling, or split a long-lived role. Warp specialization exists on Hopper as well as Blackwell; it can shorten one role's live set, but adds role state and synchronization and does not guarantee fewer registers. Compare compiled resources and time for matched role layouts.\n\nCompare an uncapped build with selected `-maxrregcount` values. Record whether the requested cap is effective, the occupancy-limiting resource, spill traffic, instructions, and runtime. ABI minima can constrain the cap, another resource can remain limiting, and spills can erase any benefit."}, "reason": {"statement": "Separate architecture availability from measured role-liveness/resource changes.", "urls": ["https://docs.nvidia.com/cutlass/4.5.0/media/docs/cpp/gemm_api_3x.html", "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-compiler-driver-nvcc/index.html"]}} +{"path": "wiki/patterns/register-pressure.md", "before": {"statement": "```\n// Hopper: 64×256 MMA tile accumulator = 64*256*4 bytes in registers per warp group\n// → ~128 registers per thread just for accumulators\n//\n// Blackwell: TMEM holds accumulators\n// → 0 registers for accumulators\n// → ~128 registers freed for other work or higher occupancy\n//\n// TMEM: 256 KB per SM, 128 rows × 512 columns × 32-bit\n// Largest 1-SM MMA uses half → double-buffering possible\n```"}, "after": {"statement": "For Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, that is 128 32-bit registers (512 bytes) per thread for D. This exact fragment arithmetic does not include other live state or guarantee a particular resident-block count.\n\nOn the SM100 tcgen05 path, resident D is in TMEM, organized as 128 lanes by 512 columns of 32-bit cells. That removes the long-lived per-thread WGMMA D fragment, but not all accumulator-related register work: addresses, descriptors, barriers, loop state, `tcgen05.ld` destinations, conversion, and epilogue temporaries still consume registers. Logical capacity and required columns depend on MMA kind, shape, and packing, so a largest-operation or always-double-bufferable rule cannot be inferred from raw cell count.\n\nThe migration must implement collective allocation/address publication, tcgen05 MMA completion, cross-thread ordering where applicable, collective TMEM loads and `tcgen05.wait::ld`, consumer completion, and collective deallocation before exit. The described TMEM/tcgen05 path targets SM100-class data-center architectures and is not the SM120 consumer MMA path."}, "reason": {"statement": "Keep the exact comparison while naming residual registers and configuration-specific capacity.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld"]}} +{"path": "wiki/patterns/register-pressure.md", "before": {"statement": "- TMEM→register transfer adds latency (offset by freeing registers)"}, "after": {"statement": "TMEM loads are asynchronous operations with explicit completion waits. Whether their issue/wait and epilogue work are repaid by lower long-lived register use is an empirical whole-kernel question. Compare the Hopper and Blackwell designs only with equivalent math and precision; for same-architecture alternatives, compare TMEM region count/layout while holding tile and launch policy fixed.\n\nReport compiled registers and spills, TMEM columns, SMEM, barriers, occupancy limit, achieved warps, pipeline/scoreboard stalls, correctness, and time. Include shapes where register occupancy changes and shapes where it does not. A reduction in register count without a useful residency, issue, traffic, or time improvement is not sufficient."}, "reason": {"statement": "Make it a controlled whole-kernel tradeoff.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} {"path": "wiki/patterns/tail-effect.md", "before": {"statement": "Performance drops for problem sizes where total_tiles % num_SMs != 0. The last wave of tiles runs with many SMs idle."}, "after": {"statement": "For `T` equal-duration independent tiles and `W` available one-tile workers, with no K decomposition, write:\n\n```text\nT = qW + r, 0 <= r < W\n```\n\nThere are `q` full waves. If `r>0`, a final partial wave uses `r` workers and has instantaneous worker utilization `r/W`; if `r=0` and `T>0`, the final wave is full. For example, `T=150` and `W=142` gives one full wave plus eight tiles: the partial wave has `8/142 = 5.63%` worker utilization under these assumptions.\n\nThis is an analytical model, not a statement that a B200 has 142 SMs or that physical SM count always equals `W`. Block resources can make multiple blocks resident per SM, cluster shape changes scheduling granularity, application policies may restrict available SMs, and tile durations can differ. Measure the actual worker capacity and timeline.\n\nUnder the equal-duration model, the time fraction attributable to at most one partial wave shrinks as the number of full waves grows. There is no universal “below four times the SM count” cutoff: the remainder, wave duration, other phases, and imbalance determine significance."}, "reason": {"statement": "Define and compute the simplified model, then require measurement for real kernels.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} {"path": "wiki/patterns/tail-effect.md", "before": {"statement": "2. **Static assignment**: stride-by-gridDim leaves remainder tiles on few SMs\n3. **Non-persistent launch**: each kernel launch has fixed grid, no dynamic rebalancing"}, "after": {"statement": "1. Record logical tiles/clusters, grid and cluster dimensions, available worker capacity, residency limits, and per-work-item duration.\n2. Predict wave count and the final remainder from the simplified model.\n3. Use time-resolved activity or instrumented worker records to locate the underfilled interval at the end, rather than inferring it from aggregate utilization.\n4. Sweep nearby problem sizes or tile shapes. A wave-quantization hypothesis predicts a sawtooth response aligned with changes in the remainder, after controlling total work.\n5. Separate unequal tile durations: a long straggler is load imbalance even if the tile-count remainder is zero.\n\nCUDA schedules ordinary grid blocks onto available SMs; a static grid-stride loop fixes logical worker indices, not `blockIdx`-to-physical-SM placement. A fixed grid therefore does not mean CUDA lacks dynamic block scheduling."}, "reason": {"statement": "Separate logical worker assignment from hardware block placement and persistence.", "urls": ["https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} -{"path": "wiki/patterns/tail-effect.md", "before": {"statement": "| [CLC](../hardware/clc.md) | Hardware dynamic scheduling, SMs grab tiles on-demand |"}, "after": {"statement": "- A static persistent loop lets a resident CTA process multiple logical work items. Worker count is selected from problem decomposition, cluster/resource limits, and policy—not necessarily one CTA per physical SM. Finite work still has a final subset and may remain imbalanced.\n- Cluster Launch Control lets a selected thread request cancellation of an unspecified not-yet-started block or cluster and process the returned grid ID. CLC redistributes existing IDs; it cannot turn eight remaining independent tiles into 142 or 148 concurrent tiles.\n- Raster order and swizzle are coordinate transforms. Test them for locality while holding work acquisition fixed; they do not create work or guarantee balanced durations.\n- Stream-K or Split-K may create more independent partitions when tile-level work is insufficient. Evaluate reduction traffic, workspace, synchronization, determinism, and numerical effects separately.\n\nCompare static and CLC-backed persistence with identical math, decomposition, tile/cluster shapes, grid, resources, and timing. Record CLC request results and per-worker work/time distribution. PTX ISA 9.0 requires `sm_100` or higher for CLC and explicitly lists SM120-family support for the cluster-wide multicast qualifier; the pinned CUTLASS 4.5.0 route used here is specifically its SM100 scheduler."}, "reason": {"statement": "Describe exact work-reassignment lifecycle and limits.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} -{"path": "wiki/patterns/tail-effect.md", "before": {"statement": "| [Persistent kernels](../techniques/persistent-kernels.md) | SM-count grid, iterate over tiles, no wave boundary |"}, "after": {"statement": "- A static persistent loop lets a resident CTA process multiple logical work items. Worker count is selected from problem decomposition, cluster/resource limits, and policy—not necessarily one CTA per physical SM. Finite work still has a final subset and may remain imbalanced.\n- Cluster Launch Control lets a selected thread request cancellation of an unspecified not-yet-started block or cluster and process the returned grid ID. CLC redistributes existing IDs; it cannot turn eight remaining independent tiles into 142 or 148 concurrent tiles.\n- Raster order and swizzle are coordinate transforms. Test them for locality while holding work acquisition fixed; they do not create work or guarantee balanced durations.\n- Stream-K or Split-K may create more independent partitions when tile-level work is insufficient. Evaluate reduction traffic, workspace, synchronization, determinism, and numerical effects separately.\n\nCompare static and CLC-backed persistence with identical math, decomposition, tile/cluster shapes, grid, resources, and timing. Record CLC request results and per-worker work/time distribution. PTX ISA 9.0 requires `sm_100` or higher for CLC and explicitly lists SM120-family support for the cluster-wide multicast qualifier; the pinned CUTLASS 4.5.0 route used here is specifically its SM100 scheduler."}, "reason": {"statement": "Present static persistence as a baseline with its own final work distribution.", "urls": ["https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} -{"path": "wiki/patterns/tail-effect.md", "before": {"statement": "| [Tile scheduling](../techniques/tile-scheduling.md) | Raster order, swizzle patterns for better distribution |"}, "after": {"statement": "- A static persistent loop lets a resident CTA process multiple logical work items. Worker count is selected from problem decomposition, cluster/resource limits, and policy—not necessarily one CTA per physical SM. Finite work still has a final subset and may remain imbalanced.\n- Cluster Launch Control lets a selected thread request cancellation of an unspecified not-yet-started block or cluster and process the returned grid ID. CLC redistributes existing IDs; it cannot turn eight remaining independent tiles into 142 or 148 concurrent tiles.\n- Raster order and swizzle are coordinate transforms. Test them for locality while holding work acquisition fixed; they do not create work or guarantee balanced durations.\n- Stream-K or Split-K may create more independent partitions when tile-level work is insufficient. Evaluate reduction traffic, workspace, synchronization, determinism, and numerical effects separately.\n\nCompare static and CLC-backed persistence with identical math, decomposition, tile/cluster shapes, grid, resources, and timing. Record CLC request results and per-worker work/time distribution. PTX ISA 9.0 requires `sm_100` or higher for CLC and explicitly lists SM120-family support for the cluster-wide multicast qualifier; the pinned CUTLASS 4.5.0 route used here is specifically its SM100 scheduler."}, "reason": {"statement": "Separate locality ordering from acquisition and decomposition.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"]}} -{"path": "wiki/patterns/tail-effect.md", "before": {"statement": "```\n// B200: 142 SMs\n// Problem: 150 tiles\n// Without CLC: 2 waves (142 + 8), last wave uses only 8 SMs (5.6%)\n// With CLC: single persistent wave, all 142 SMs stay busy\n//\n// Impact: 86% → 98% of cuBLAS (tcgen05 tutorial data)\n```"}, "after": {"statement": "For `T` equal-duration independent tiles and `W` available one-tile workers, with no K decomposition, write:\n\n```text\nT = qW + r, 0 <= r < W\n```\n\nThere are `q` full waves. If `r>0`, a final partial wave uses `r` workers and has instantaneous worker utilization `r/W`; if `r=0` and `T>0`, the final wave is full. For example, `T=150` and `W=142` gives one full wave plus eight tiles: the partial wave has `8/142 = 5.63%` worker utilization under these assumptions.\n\nThis is an analytical model, not a statement that a B200 has 142 SMs or that physical SM count always equals `W`. Block resources can make multiple blocks resident per SM, cluster shape changes scheduling granularity, application policies may restrict available SMs, and tile durations can differ. Measure the actual worker capacity and timeline.\n\nUnder the equal-duration model, the time fraction attributable to at most one partial wave shrinks as the number of full waves grows. There is no universal “below four times the SM count” cutoff: the remainder, wave duration, other phases, and imbalance determine significance.\n\nGau Nernst's Modal B200 has 148 SMs. In the author's `M=N=K=4096` experiment, v5 reports 1302.29 TFLOP/s (86.43% of cuBLAS) and v6 reports 1475.93 TFLOP/s (97.96%). The v6 change is static persistence plus output-pipeline/epilogue-role changes accumulated over earlier versions. The author explicitly did not add CLC and says overlap remains imperfect. Those results cannot be used as a CLC ablation or proof that all SMs stay busy."}, "reason": {"statement": "Use explicit hypothetical arithmetic separate from exact tutorial evidence.", "urls": ["https://gau-nernst.github.io/tcgen05/", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu"]}} +{"path": "wiki/patterns/tail-effect.md", "before": {"statement": "| [CLC](../hardware/clc.md) | Hardware dynamic scheduling, SMs grab tiles on-demand |"}, "after": {"statement": "- A static persistent loop lets a resident CTA process multiple logical work items. Worker count is selected from problem decomposition, cluster/resource limits, and policy—not necessarily one CTA per physical SM. Finite work still has a final subset and may remain imbalanced.\n- Cluster Launch Control lets a selected thread request cancellation of an unspecified not-yet-started block or cluster and process the returned grid ID. CLC redistributes existing IDs; it cannot turn eight remaining independent tiles into 142 or 148 concurrent tiles.\n- Raster order and swizzle are coordinate transforms. Test them for locality while holding work acquisition fixed; they do not create work or guarantee balanced durations.\n- Stream-K or Split-K may create more independent partitions when tile-level work is insufficient. Evaluate reduction traffic, workspace, synchronization, determinism, and numerical effects separately.\n\nCompare static and CLC-backed persistence with identical math, decomposition, tile/cluster shapes, grid, resources, and timing. Record CLC request results and per-worker work/time distribution. PTX ISA 9.0 requires `sm_100` or higher for CLC and explicitly lists SM120-family support for the cluster-wide multicast qualifier; the pinned CUTLASS 4.5.0 route used here is specifically its SM100 scheduler."}, "reason": {"statement": "Describe exact work-reassignment lifecycle and limits.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/patterns/tail-effect.md", "before": {"statement": "| [Persistent kernels](../techniques/persistent-kernels.md) | SM-count grid, iterate over tiles, no wave boundary |"}, "after": {"statement": "- A static persistent loop lets a resident CTA process multiple logical work items. Worker count is selected from problem decomposition, cluster/resource limits, and policy—not necessarily one CTA per physical SM. Finite work still has a final subset and may remain imbalanced.\n- Cluster Launch Control lets a selected thread request cancellation of an unspecified not-yet-started block or cluster and process the returned grid ID. CLC redistributes existing IDs; it cannot turn eight remaining independent tiles into 142 or 148 concurrent tiles.\n- Raster order and swizzle are coordinate transforms. Test them for locality while holding work acquisition fixed; they do not create work or guarantee balanced durations.\n- Stream-K or Split-K may create more independent partitions when tile-level work is insufficient. Evaluate reduction traffic, workspace, synchronization, determinism, and numerical effects separately.\n\nCompare static and CLC-backed persistence with identical math, decomposition, tile/cluster shapes, grid, resources, and timing. Record CLC request results and per-worker work/time distribution. PTX ISA 9.0 requires `sm_100` or higher for CLC and explicitly lists SM120-family support for the cluster-wide multicast qualifier; the pinned CUTLASS 4.5.0 route used here is specifically its SM100 scheduler."}, "reason": {"statement": "Present static persistence as a baseline with its own final work distribution.", "urls": ["https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/patterns/tail-effect.md", "before": {"statement": "| [Tile scheduling](../techniques/tile-scheduling.md) | Raster order, swizzle patterns for better distribution |"}, "after": {"statement": "- A static persistent loop lets a resident CTA process multiple logical work items. Worker count is selected from problem decomposition, cluster/resource limits, and policy—not necessarily one CTA per physical SM. Finite work still has a final subset and may remain imbalanced.\n- Cluster Launch Control lets a selected thread request cancellation of an unspecified not-yet-started block or cluster and process the returned grid ID. CLC redistributes existing IDs; it cannot turn eight remaining independent tiles into 142 or 148 concurrent tiles.\n- Raster order and swizzle are coordinate transforms. Test them for locality while holding work acquisition fixed; they do not create work or guarantee balanced durations.\n- Stream-K or Split-K may create more independent partitions when tile-level work is insufficient. Evaluate reduction traffic, workspace, synchronization, determinism, and numerical effects separately.\n\nCompare static and CLC-backed persistence with identical math, decomposition, tile/cluster shapes, grid, resources, and timing. Record CLC request results and per-worker work/time distribution. PTX ISA 9.0 requires `sm_100` or higher for CLC and explicitly lists SM120-family support for the cluster-wide multicast qualifier; the pinned CUTLASS 4.5.0 route used here is specifically its SM100 scheduler."}, "reason": {"statement": "Separate locality ordering from acquisition and decomposition.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel"]}} +{"path": "wiki/patterns/tail-effect.md", "before": {"statement": "```\n// B200: 142 SMs\n// Problem: 150 tiles\n// Without CLC: 2 waves (142 + 8), last wave uses only 8 SMs (5.6%)\n// With CLC: single persistent wave, all 142 SMs stay busy\n//\n// Impact: 86% → 98% of cuBLAS (tcgen05 tutorial data)\n```"}, "after": {"statement": "For `T` equal-duration independent tiles and `W` available one-tile workers, with no K decomposition, write:\n\n```text\nT = qW + r, 0 <= r < W\n```\n\nThere are `q` full waves. If `r>0`, a final partial wave uses `r` workers and has instantaneous worker utilization `r/W`; if `r=0` and `T>0`, the final wave is full. For example, `T=150` and `W=142` gives one full wave plus eight tiles: the partial wave has `8/142 = 5.63%` worker utilization under these assumptions.\n\nThis is an analytical model, not a statement that a B200 has 142 SMs or that physical SM count always equals `W`. Block resources can make multiple blocks resident per SM, cluster shape changes scheduling granularity, application policies may restrict available SMs, and tile durations can differ. Measure the actual worker capacity and timeline.\n\nUnder the equal-duration model, the time fraction attributable to at most one partial wave shrinks as the number of full waves grows. There is no universal “below four times the SM count” cutoff: the remainder, wave duration, other phases, and imbalance determine significance.\n\nGau Nernst's Modal B200 has 148 SMs. In the author's `M=N=K=4096` experiment, v5 reports 1302.29 TFLOP/s (86.43% of cuBLAS) and v6 reports 1475.93 TFLOP/s (97.96%). The v6 change is static persistence plus output-pipeline/epilogue-role changes accumulated over earlier versions. The author explicitly did not add CLC and says overlap remains imperfect. Those results cannot be used as a CLC ablation or proof that all SMs stay busy."}, "reason": {"statement": "Use explicit hypothetical arithmetic separate from exact tutorial evidence.", "urls": ["https://gau-nernst.github.io/tcgen05/", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", "https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html", "https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu"]}} {"path": "wiki/patterns/tail-effect.md", "before": {"statement": "- Only significant for moderate tile counts (< 4× SM count)"}, "after": {"statement": "For `T` equal-duration independent tiles and `W` available one-tile workers, with no K decomposition, write:\n\n```text\nT = qW + r, 0 <= r < W\n```\n\nThere are `q` full waves. If `r>0`, a final partial wave uses `r` workers and has instantaneous worker utilization `r/W`; if `r=0` and `T>0`, the final wave is full. For example, `T=150` and `W=142` gives one full wave plus eight tiles: the partial wave has `8/142 = 5.63%` worker utilization under these assumptions.\n\nThis is an analytical model, not a statement that a B200 has 142 SMs or that physical SM count always equals `W`. Block resources can make multiple blocks resident per SM, cluster shape changes scheduling granularity, application policies may restrict available SMs, and tile durations can differ. Measure the actual worker capacity and timeline.\n\nUnder the equal-duration model, the time fraction attributable to at most one partial wave shrinks as the number of full waves grows. There is no universal “below four times the SM count” cutoff: the remainder, wave duration, other phases, and imbalance determine significance."}, "reason": {"statement": "Give the analytical fractional bound under declared assumptions and measure real kernels.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} -{"path": "wiki/patterns/tail-effect.md", "before": {"statement": "- CLC only on SM100 datacenter"}, "after": {"statement": "- A static persistent loop lets a resident CTA process multiple logical work items. Worker count is selected from problem decomposition, cluster/resource limits, and policy—not necessarily one CTA per physical SM. Finite work still has a final subset and may remain imbalanced.\n- Cluster Launch Control lets a selected thread request cancellation of an unspecified not-yet-started block or cluster and process the returned grid ID. CLC redistributes existing IDs; it cannot turn eight remaining independent tiles into 142 or 148 concurrent tiles.\n- Raster order and swizzle are coordinate transforms. Test them for locality while holding work acquisition fixed; they do not create work or guarantee balanced durations.\n- Stream-K or Split-K may create more independent partitions when tile-level work is insufficient. Evaluate reduction traffic, workspace, synchronization, determinism, and numerical effects separately.\n\nCompare static and CLC-backed persistence with identical math, decomposition, tile/cluster shapes, grid, resources, and timing. Record CLC request results and per-worker work/time distribution. PTX ISA 9.0 requires `sm_100` or higher for CLC and explicitly lists SM120-family support for the cluster-wide multicast qualifier; the pinned CUTLASS 4.5.0 route used here is specifically its SM100 scheduler."}, "reason": {"statement": "Separate ISA compatibility from pinned library examples.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/patterns/tail-effect.md", "before": {"statement": "- CLC only on SM100 datacenter"}, "after": {"statement": "- A static persistent loop lets a resident CTA process multiple logical work items. Worker count is selected from problem decomposition, cluster/resource limits, and policy—not necessarily one CTA per physical SM. Finite work still has a final subset and may remain imbalanced.\n- Cluster Launch Control lets a selected thread request cancellation of an unspecified not-yet-started block or cluster and process the returned grid ID. CLC redistributes existing IDs; it cannot turn eight remaining independent tiles into 142 or 148 concurrent tiles.\n- Raster order and swizzle are coordinate transforms. Test them for locality while holding work acquisition fixed; they do not create work or guarantee balanced durations.\n- Stream-K or Split-K may create more independent partitions when tile-level work is insufficient. Evaluate reduction traffic, workspace, synchronization, determinism, and numerical effects separately.\n\nCompare static and CLC-backed persistence with identical math, decomposition, tile/cluster shapes, grid, resources, and timing. Record CLC request results and per-worker work/time distribution. PTX ISA 9.0 requires `sm_100` or higher for CLC and explicitly lists SM120-family support for the cluster-wide multicast qualifier; the pinned CUTLASS 4.5.0 route used here is specifically its SM100 scheduler."}, "reason": {"statement": "Separate ISA compatibility from pinned library examples.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} {"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "MoE grouped GEMM shows uneven per-expert compute time. Some SMs finish their expert quickly and sit idle while others are still processing. Overall latency is dominated by the slowest expert."}, "after": {"statement": "Do not infer one expert per physical SM from a grouped-GEMM trace. CUDA assigns\nthread blocks to available SMs, while grouped kernels may split an expert into\nmultiple output tiles or let one resident worker process several logical tiles.\nMeasure at each relevant level:\n\n- routed tokens per expert for the exact batch and prefill/decode phase;\n- valid rows and output-tile counts per expert;\n- completed tiles and elapsed work per logical worker and, when instrumented,\n per SM;\n- dispatch, grouped-GEMM, combine, and end-to-end times per expert-parallel\n rank.\n\nSmall expert segments and partial output tiles can expose too little parallel\nwork or leave lanes predicated out. The effect depends on the kernel tile,\nother GEMM dimensions, resident-worker count, and competing implementation;\nthere is no universal `M < BLOCK_M` diagnosis or minimum viable size.\n\nUniform expected routing, a larger batch, or an auxiliary balancing loss may\nchange token-count skew, but none proves balanced tile counts, worker\ndurations, communication, or runtime. Treat imbalance as absent only when the\nmeasured distributions are narrow and a matched balancing variant does not\nmaterially improve the timed operation."}, "reason": {"statement": "Define imbalance across routing, logical work, worker time, and rank communication without assumed ownership.", "urls": ["https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/efficient_gemm.md", "https://www.lmsys.org/blog/2025-05-05-large-scale-ep/"]}} {"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "1. **Skewed token distribution**: Router sends 80% of tokens to 20% of experts (common in trained MoE models)"}, "after": {"statement": "Do not infer one expert per physical SM from a grouped-GEMM trace. CUDA assigns\nthread blocks to available SMs, while grouped kernels may split an expert into\nmultiple output tiles or let one resident worker process several logical tiles.\nMeasure at each relevant level:\n\n- routed tokens per expert for the exact batch and prefill/decode phase;\n- valid rows and output-tile counts per expert;\n- completed tiles and elapsed work per logical worker and, when instrumented,\n per SM;\n- dispatch, grouped-GEMM, combine, and end-to-end times per expert-parallel\n rank.\n\nSmall expert segments and partial output tiles can expose too little parallel\nwork or leave lanes predicated out. The effect depends on the kernel tile,\nother GEMM dimensions, resident-worker count, and competing implementation;\nthere is no universal `M < BLOCK_M` diagnosis or minimum viable size.\n\nUniform expected routing, a larger batch, or an auxiliary balancing loss may\nchange token-count skew, but none proves balanced tile counts, worker\ndurations, communication, or runtime. Treat imbalance as absent only when the\nmeasured distributions are narrow and a matched balancing variant does not\nmaterially improve the timed operation."}, "reason": {"statement": "Require exact observed count distributions.", "urls": ["https://github.com/deepseek-ai/EPLB/blob/d52c72d5b2f2fb4c41afbf8eb21366820239913d/README.md", "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/efficient_gemm.md", "https://www.lmsys.org/blog/2025-05-05-large-scale-ep/"]}} -{"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "2. **Static tile assignment**: Precomputed tile→SM mapping cannot rebalance at runtime"}, "after": {"statement": "Keep persistence, work acquisition, and decomposition as separate variables:\n\n1. A static persistent worker can advance through a deterministic logical-work\n sequence. CUDA still decides where its block runs; the rule is not a\n precomputed tile-to-physical-SM map.\n2. In a CLC-backed scheduler, a selected thread requests cancellation of an\n unspecified block or cluster that has not launched. A successful response\n returns that existing grid coordinate; a request can fail. CLC does not\n create tiles or make the fastest SM autonomously steal arbitrary work.\n3. The initial `blockIdx` in CUTLASS's pinned CLC scheduler is static and later\n requests are asynchronous, pipelined, and cluster-granular. There is no\n universal per-tile latency bound: compare request activity and end-to-end\n time against the matched static scheduler.\n4. Splitting a large expert or K range may expose more independent work, but it\n can add partial-result and reduction costs. Hold math, tile shape, cluster\n shape, launch resources, and output semantics constant in the comparison.\n\nPTX ISA 9.0 specifies CLC for `sm_100` or higher and explicitly includes\nSM120-family targets for the multicast form. CUTLASS 4.5.0's documented\n`PersistentTileSchedulerSm100` integration is a narrower library example, not\nthe complete ISA compatibility boundary."}, "reason": {"statement": "Separate logical assignment from physical placement.", "urls": ["https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "2. **Static tile assignment**: Precomputed tile→SM mapping cannot rebalance at runtime"}, "after": {"statement": "Keep persistence, work acquisition, and decomposition as separate variables:\n\n1. A static persistent worker can advance through a deterministic logical-work\n sequence. CUDA still decides where its block runs; the rule is not a\n precomputed tile-to-physical-SM map.\n2. In a CLC-backed scheduler, a selected thread requests cancellation of an\n unspecified block or cluster that has not launched. A successful response\n returns that existing grid coordinate; a request can fail. CLC does not\n create tiles or make the fastest SM autonomously steal arbitrary work.\n3. The initial `blockIdx` in CUTLASS's pinned CLC scheduler is static and later\n requests are asynchronous, pipelined, and cluster-granular. There is no\n universal per-tile latency bound: compare request activity and end-to-end\n time against the matched static scheduler.\n4. Splitting a large expert or K range may expose more independent work, but it\n can add partial-result and reduction costs. Hold math, tile shape, cluster\n shape, launch resources, and output semantics constant in the comparison.\n\nPTX ISA 9.0 specifies CLC for `sm_100` or higher and explicitly includes\nSM120-family targets for the multicast form. CUTLASS 4.5.0's documented\n`PersistentTileSchedulerSm100` integration is a narrower library example, not\nthe complete ISA compatibility boundary."}, "reason": {"statement": "Separate logical assignment from physical placement.", "urls": ["https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} {"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "3. **Masked layout waste**: Fixed M_max per expert wastes compute on padding rows"}, "after": {"statement": "At pinned DeepGEMM commit\n[`891d57b4`](https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f),\nthe three grouped interfaces have different axes and metadata:\n\n| M-grouped contiguous | Pack `A` and `D` along variable M with N/K fixed; identify groups by per-row expert IDs or per-group prefix-sum ends; segments are M-block aligned | Record valid rows, alignment padding, tile counts, and metadata form |\n| M-grouped masked | Allocate `[G, M_max, ...]`, pass one valid-M count per group, and compute valid portions; the fixed allocation is documented for a CUDA-graph decode case | Separate allocated rows from valid rows and inspect edge predication in generated code |\n| K-grouped contiguous | Pack variable K with M/N fixed for MoE weight backward | Do not use its K-axis contract to describe forward token imbalance |\n\nA fixed `M_max` allocation therefore does not establish that all padding rows\nare computed. Any residual edge-tile or predication cost needs generated-code\nand profile evidence for the selected kernel."}, "reason": {"statement": "Preserve graph-friendly allocation while removing false padding-compute attribution.", "urls": ["https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp"]}} -{"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "| [CLC (Cluster Launch Control)](../hardware/clc.md) | Hardware dynamic tile assignment — fastest SMs grab more tiles |"}, "after": {"statement": "Keep persistence, work acquisition, and decomposition as separate variables:\n\n1. A static persistent worker can advance through a deterministic logical-work\n sequence. CUDA still decides where its block runs; the rule is not a\n precomputed tile-to-physical-SM map.\n2. In a CLC-backed scheduler, a selected thread requests cancellation of an\n unspecified block or cluster that has not launched. A successful response\n returns that existing grid coordinate; a request can fail. CLC does not\n create tiles or make the fastest SM autonomously steal arbitrary work.\n3. The initial `blockIdx` in CUTLASS's pinned CLC scheduler is static and later\n requests are asynchronous, pipelined, and cluster-granular. There is no\n universal per-tile latency bound: compare request activity and end-to-end\n time against the matched static scheduler.\n4. Splitting a large expert or K range may expose more independent work, but it\n can add partial-result and reduction costs. Hold math, tile shape, cluster\n shape, launch resources, and output semantics constant in the comparison.\n\nPTX ISA 9.0 specifies CLC for `sm_100` or higher and explicitly includes\nSM120-family targets for the multicast form. CUTLASS 4.5.0's documented\n`PersistentTileSchedulerSm100` integration is a narrower library example, not\nthe complete ISA compatibility boundary."}, "reason": {"statement": "State the exact reassignment protocol.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} -{"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "| [Persistent kernels](../techniques/persistent-kernels.md) | Amortize launch overhead; loop over dynamic work queue |"}, "after": {"statement": "Keep persistence, work acquisition, and decomposition as separate variables:\n\n1. A static persistent worker can advance through a deterministic logical-work\n sequence. CUDA still decides where its block runs; the rule is not a\n precomputed tile-to-physical-SM map.\n2. In a CLC-backed scheduler, a selected thread requests cancellation of an\n unspecified block or cluster that has not launched. A successful response\n returns that existing grid coordinate; a request can fail. CLC does not\n create tiles or make the fastest SM autonomously steal arbitrary work.\n3. The initial `blockIdx` in CUTLASS's pinned CLC scheduler is static and later\n requests are asynchronous, pipelined, and cluster-granular. There is no\n universal per-tile latency bound: compare request activity and end-to-end\n time against the matched static scheduler.\n4. Splitting a large expert or K range may expose more independent work, but it\n can add partial-result and reduction costs. Hold math, tile shape, cluster\n shape, launch resources, and output semantics constant in the comparison.\n\nPTX ISA 9.0 specifies CLC for `sm_100` or higher and explicitly includes\nSM120-family targets for the multicast form. CUTLASS 4.5.0's documented\n`PersistentTileSchedulerSm100` integration is a narrower library example, not\nthe complete ISA compatibility boundary."}, "reason": {"statement": "Make persistence and acquisition policy independent choices.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"]}} +{"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "| [CLC (Cluster Launch Control)](../hardware/clc.md) | Hardware dynamic tile assignment — fastest SMs grab more tiles |"}, "after": {"statement": "Keep persistence, work acquisition, and decomposition as separate variables:\n\n1. A static persistent worker can advance through a deterministic logical-work\n sequence. CUDA still decides where its block runs; the rule is not a\n precomputed tile-to-physical-SM map.\n2. In a CLC-backed scheduler, a selected thread requests cancellation of an\n unspecified block or cluster that has not launched. A successful response\n returns that existing grid coordinate; a request can fail. CLC does not\n create tiles or make the fastest SM autonomously steal arbitrary work.\n3. The initial `blockIdx` in CUTLASS's pinned CLC scheduler is static and later\n requests are asynchronous, pipelined, and cluster-granular. There is no\n universal per-tile latency bound: compare request activity and end-to-end\n time against the matched static scheduler.\n4. Splitting a large expert or K range may expose more independent work, but it\n can add partial-result and reduction costs. Hold math, tile shape, cluster\n shape, launch resources, and output semantics constant in the comparison.\n\nPTX ISA 9.0 specifies CLC for `sm_100` or higher and explicitly includes\nSM120-family targets for the multicast form. CUTLASS 4.5.0's documented\n`PersistentTileSchedulerSm100` integration is a narrower library example, not\nthe complete ISA compatibility boundary."}, "reason": {"statement": "State the exact reassignment protocol.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel", "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "| [Persistent kernels](../techniques/persistent-kernels.md) | Amortize launch overhead; loop over dynamic work queue |"}, "after": {"statement": "Keep persistence, work acquisition, and decomposition as separate variables:\n\n1. A static persistent worker can advance through a deterministic logical-work\n sequence. CUDA still decides where its block runs; the rule is not a\n precomputed tile-to-physical-SM map.\n2. In a CLC-backed scheduler, a selected thread requests cancellation of an\n unspecified block or cluster that has not launched. A successful response\n returns that existing grid coordinate; a request can fail. CLC does not\n create tiles or make the fastest SM autonomously steal arbitrary work.\n3. The initial `blockIdx` in CUTLASS's pinned CLC scheduler is static and later\n requests are asynchronous, pipelined, and cluster-granular. There is no\n universal per-tile latency bound: compare request activity and end-to-end\n time against the matched static scheduler.\n4. Splitting a large expert or K range may expose more independent work, but it\n can add partial-result and reduction costs. Hold math, tile shape, cluster\n shape, launch resources, and output semantics constant in the comparison.\n\nPTX ISA 9.0 specifies CLC for `sm_100` or higher and explicitly includes\nSM120-family targets for the multicast form. CUTLASS 4.5.0's documented\n`PersistentTileSchedulerSm100` integration is a narrower library example, not\nthe complete ISA compatibility boundary."}, "reason": {"statement": "Make persistence and acquisition policy independent choices.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel"]}} {"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "| [Contiguous layout](../kernels/grouped-gemm.md) | Pack variable-M experts sequentially; offsets array indexes expert boundaries |"}, "after": {"statement": "At pinned DeepGEMM commit\n[`891d57b4`](https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f),\nthe three grouped interfaces have different axes and metadata:\n\n| M-grouped contiguous | Pack `A` and `D` along variable M with N/K fixed; identify groups by per-row expert IDs or per-group prefix-sum ends; segments are M-block aligned | Record valid rows, alignment padding, tile counts, and metadata form |\n| M-grouped masked | Allocate `[G, M_max, ...]`, pass one valid-M count per group, and compute valid portions; the fixed allocation is documented for a CUDA-graph decode case | Separate allocated rows from valid rows and inspect edge predication in generated code |\n| K-grouped contiguous | Pack variable K with M/N fixed for MoE weight backward | Do not use its K-axis contract to describe forward token imbalance |\n\nA fixed `M_max` allocation therefore does not establish that all padding rows\nare computed. Any residual edge-tile or predication cost needs generated-code\nand profile evidence for the selected kernel."}, "reason": {"statement": "Name both actual ABI forms.", "urls": ["https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp"]}} {"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "| [Masked layout](../kernels/grouped-gemm.md) | Good for CUDA graph capture; wastes compute on padding |"}, "after": {"statement": "At pinned DeepGEMM commit\n[`891d57b4`](https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f),\nthe three grouped interfaces have different axes and metadata:\n\n| M-grouped contiguous | Pack `A` and `D` along variable M with N/K fixed; identify groups by per-row expert IDs or per-group prefix-sum ends; segments are M-block aligned | Record valid rows, alignment padding, tile counts, and metadata form |\n| M-grouped masked | Allocate `[G, M_max, ...]`, pass one valid-M count per group, and compute valid portions; the fixed allocation is documented for a CUDA-graph decode case | Separate allocated rows from valid rows and inspect edge predication in generated code |\n| K-grouped contiguous | Pack variable K with M/N fixed for MoE weight backward | Do not use its K-axis contract to describe forward token imbalance |\n\nA fixed `M_max` allocation therefore does not establish that all padding rows\nare computed. Any residual edge-tile or predication cost needs generated-code\nand profile evidence for the selected kernel."}, "reason": {"statement": "Retain graph scope and separate allocation from valid computation.", "urls": ["https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md", "https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp"]}} {"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "| [EPLB (Expert Parallel Load Balancer)](https://github.com/deepseek-ai/EPLB) | Replicate heavy experts across GPUs; 1.49x prefill speedup, 2.54x decode |"}, "after": {"statement": "[DeepSeek EPLB at commit\n`d52c72d`](https://github.com/deepseek-ai/EPLB/tree/d52c72d5b2f2fb4c41afbf8eb21366820239913d)\ntakes per-logical-expert load statistics, replicates heavily loaded logical\nexperts, and packs physical experts across configured GPUs and nodes. It is a\nhost-side expert-parallel placement planner, not a device-kernel tile\nscheduler; use it as a complement only when multi-GPU expert placement is in\nthe measured system scope.\n\nAn LMSYS/SGLang study on a 96-H100 deployment reported `1.49x` prefill and\n`2.54x` decode throughput speedups in its large-scale EPLB ablation. The same\nstudy says it used in-distribution data and that production distribution\nshifts require further testing. Those figures are scoped results, not generic\nEPLB speedups."}, "reason": {"statement": "Scope placement mechanism and measurement separately.", "urls": ["https://www.lmsys.org/blog/2025-05-05-large-scale-ep/", "https://github.com/deepseek-ai/EPLB/blob/d52c72d5b2f2fb4c41afbf8eb21366820239913d/eplb.py"]}} {"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "This highlighted that even careful tile scheduling can be outrun by algorithmic restructuring — and prompted the MLSys 2026 FlashInfer contest to add runtime isolation + subprocess eval."}, "after": {"statement": "GPU Mode's official postmortem records a temporarily first-place submission\nthat combined a real grouped-GEMM kernel with a timing-harness exploit. The\ncorrectness path ran a padded eight-group computation on each of 15 cloned\nobjects. During timing, the first call merged them into one 120-group launch,\ncalls 2 through 15 returned cached output pointers, and the harness divided the\ncombined timing by 15.\n\nThis is evidence about evaluator state reuse, not a valid load-balancing\nperformance result. The postmortem identifies `gpu-mode/reference-kernels`\nPR 104 as the harness response; neither that record nor the pinned FlashInfer\nMLSys 2026 evaluation contract establishes a causal link between this incident\nand the later contest design."}, "reason": {"statement": "Preserve the verified incident mechanics without inventing historical causality.", "urls": ["https://www.gpumode.com/news/reward-hacking-nvfp4", "https://github.com/flashinfer-ai/flashinfer-bench-starter-kit/blob/75ccd05cafceb0fd1f86be4cd0f2117249463c66/EVALUATION.md"]}} -{"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "- CLC only available on SM100 datacenter (not SM120 consumer)"}, "after": {"statement": "Keep persistence, work acquisition, and decomposition as separate variables:\n\n1. A static persistent worker can advance through a deterministic logical-work\n sequence. CUDA still decides where its block runs; the rule is not a\n precomputed tile-to-physical-SM map.\n2. In a CLC-backed scheduler, a selected thread requests cancellation of an\n unspecified block or cluster that has not launched. A successful response\n returns that existing grid coordinate; a request can fail. CLC does not\n create tiles or make the fastest SM autonomously steal arbitrary work.\n3. The initial `blockIdx` in CUTLASS's pinned CLC scheduler is static and later\n requests are asynchronous, pipelined, and cluster-granular. There is no\n universal per-tile latency bound: compare request activity and end-to-end\n time against the matched static scheduler.\n4. Splitting a large expert or K range may expose more independent work, but it\n can add partial-result and reduction costs. Hold math, tile shape, cluster\n shape, launch resources, and output semantics constant in the comparison.\n\nPTX ISA 9.0 specifies CLC for `sm_100` or higher and explicitly includes\nSM120-family targets for the multicast form. CUTLASS 4.5.0's documented\n`PersistentTileSchedulerSm100` integration is a narrower library example, not\nthe complete ISA compatibility boundary."}, "reason": {"statement": "Separate PTX compatibility from a particular library integration.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel", "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} -{"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "- Dynamic scheduling has small per-tile overhead vs static precomputed"}, "after": {"statement": "Keep persistence, work acquisition, and decomposition as separate variables:\n\n1. A static persistent worker can advance through a deterministic logical-work\n sequence. CUDA still decides where its block runs; the rule is not a\n precomputed tile-to-physical-SM map.\n2. In a CLC-backed scheduler, a selected thread requests cancellation of an\n unspecified block or cluster that has not launched. A successful response\n returns that existing grid coordinate; a request can fail. CLC does not\n create tiles or make the fastest SM autonomously steal arbitrary work.\n3. The initial `blockIdx` in CUTLASS's pinned CLC scheduler is static and later\n requests are asynchronous, pipelined, and cluster-granular. There is no\n universal per-tile latency bound: compare request activity and end-to-end\n time against the matched static scheduler.\n4. Splitting a large expert or K range may expose more independent work, but it\n can add partial-result and reduction costs. Hold math, tile shape, cluster\n shape, launch resources, and output semantics constant in the comparison.\n\nPTX ISA 9.0 specifies CLC for `sm_100` or higher and explicitly includes\nSM120-family targets for the multicast form. CUTLASS 4.5.0's documented\n`PersistentTileSchedulerSm100` integration is a narrower library example, not\nthe complete ISA compatibility boundary."}, "reason": {"statement": "Require matched measurements of request activity and latency.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel"]}} +{"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "- CLC only available on SM100 datacenter (not SM120 consumer)"}, "after": {"statement": "Keep persistence, work acquisition, and decomposition as separate variables:\n\n1. A static persistent worker can advance through a deterministic logical-work\n sequence. CUDA still decides where its block runs; the rule is not a\n precomputed tile-to-physical-SM map.\n2. In a CLC-backed scheduler, a selected thread requests cancellation of an\n unspecified block or cluster that has not launched. A successful response\n returns that existing grid coordinate; a request can fail. CLC does not\n create tiles or make the fastest SM autonomously steal arbitrary work.\n3. The initial `blockIdx` in CUTLASS's pinned CLC scheduler is static and later\n requests are asynchronous, pipelined, and cluster-granular. There is no\n universal per-tile latency bound: compare request activity and end-to-end\n time against the matched static scheduler.\n4. Splitting a large expert or K range may expose more independent work, but it\n can add partial-result and reduction costs. Hold math, tile shape, cluster\n shape, launch resources, and output semantics constant in the comparison.\n\nPTX ISA 9.0 specifies CLC for `sm_100` or higher and explicitly includes\nSM120-family targets for the multicast form. CUTLASS 4.5.0's documented\n`PersistentTileSchedulerSm100` integration is a narrower library example, not\nthe complete ISA compatibility boundary."}, "reason": {"statement": "Separate PTX compatibility from a particular library integration.", "urls": ["https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel", "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"]}} +{"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "- Dynamic scheduling has small per-tile overhead vs static precomputed"}, "after": {"statement": "Keep persistence, work acquisition, and decomposition as separate variables:\n\n1. A static persistent worker can advance through a deterministic logical-work\n sequence. CUDA still decides where its block runs; the rule is not a\n precomputed tile-to-physical-SM map.\n2. In a CLC-backed scheduler, a selected thread requests cancellation of an\n unspecified block or cluster that has not launched. A successful response\n returns that existing grid coordinate; a request can fail. CLC does not\n create tiles or make the fastest SM autonomously steal arbitrary work.\n3. The initial `blockIdx` in CUTLASS's pinned CLC scheduler is static and later\n requests are asynchronous, pipelined, and cluster-granular. There is no\n universal per-tile latency bound: compare request activity and end-to-end\n time against the matched static scheduler.\n4. Splitting a large expert or K range may expose more independent work, but it\n can add partial-result and reduction costs. Hold math, tile shape, cluster\n shape, launch resources, and output semantics constant in the comparison.\n\nPTX ISA 9.0 specifies CLC for `sm_100` or higher and explicitly includes\nSM120-family targets for the multicast form. CUTLASS 4.5.0's documented\n`PersistentTileSchedulerSm100` integration is a narrower library example, not\nthe complete ISA compatibility boundary."}, "reason": {"statement": "Require matched measurements of request activity and latency.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md", "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel"]}} {"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "- Small experts may not benefit — minimum viable tile size is a floor"}, "after": {"statement": "Do not infer one expert per physical SM from a grouped-GEMM trace. CUDA assigns\nthread blocks to available SMs, while grouped kernels may split an expert into\nmultiple output tiles or let one resident worker process several logical tiles.\nMeasure at each relevant level:\n\n- routed tokens per expert for the exact batch and prefill/decode phase;\n- valid rows and output-tile counts per expert;\n- completed tiles and elapsed work per logical worker and, when instrumented,\n per SM;\n- dispatch, grouped-GEMM, combine, and end-to-end times per expert-parallel\n rank.\n\nSmall expert segments and partial output tiles can expose too little parallel\nwork or leave lanes predicated out. The effect depends on the kernel tile,\nother GEMM dimensions, resident-worker count, and competing implementation;\nthere is no universal `M < BLOCK_M` diagnosis or minimum viable size.\n\nUniform expected routing, a larger batch, or an auxiliary balancing loss may\nchange token-count skew, but none proves balanced tile counts, worker\ndurations, communication, or runtime. Treat imbalance as absent only when the\nmeasured distributions are narrow and a matched balancing variant does not\nmaterially improve the timed operation."}, "reason": {"statement": "Make small-M a measured risk rather than a fixed cutoff.", "urls": ["https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/efficient_gemm.md", "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://www.lmsys.org/blog/2025-05-05-large-scale-ep/"]}} {"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "- EPLB works at cluster scale, not single-device"}, "after": {"statement": "[DeepSeek EPLB at commit\n`d52c72d`](https://github.com/deepseek-ai/EPLB/tree/d52c72d5b2f2fb4c41afbf8eb21366820239913d)\ntakes per-logical-expert load statistics, replicates heavily loaded logical\nexperts, and packs physical experts across configured GPUs and nodes. It is a\nhost-side expert-parallel placement planner, not a device-kernel tile\nscheduler; use it as a complement only when multi-GPU expert placement is in\nthe measured system scope.\n\nAn LMSYS/SGLang study on a 96-H100 deployment reported `1.49x` prefill and\n`2.54x` decode throughput speedups in its large-scale EPLB ablation. The same\nstudy says it used in-distribution data and that production distribution\nshifts require further testing. Those figures are scoped results, not generic\nEPLB speedups."}, "reason": {"statement": "State intended multi-GPU placement scope without a false scale dichotomy.", "urls": ["https://github.com/deepseek-ai/EPLB/blob/d52c72d5b2f2fb4c41afbf8eb21366820239913d/eplb.py", "https://www.lmsys.org/blog/2025-05-05-large-scale-ep/"]}} {"path": "wiki/patterns/moe-load-imbalance.md", "before": {"statement": "- Uniform routing (rare in practice)\n- Very large batch sizes (statistics average out)\n- Training with auxiliary load balancing loss"}, "after": {"statement": "Do not infer one expert per physical SM from a grouped-GEMM trace. CUDA assigns\nthread blocks to available SMs, while grouped kernels may split an expert into\nmultiple output tiles or let one resident worker process several logical tiles.\nMeasure at each relevant level:\n\n- routed tokens per expert for the exact batch and prefill/decode phase;\n- valid rows and output-tile counts per expert;\n- completed tiles and elapsed work per logical worker and, when instrumented,\n per SM;\n- dispatch, grouped-GEMM, combine, and end-to-end times per expert-parallel\n rank.\n\nSmall expert segments and partial output tiles can expose too little parallel\nwork or leave lanes predicated out. The effect depends on the kernel tile,\nother GEMM dimensions, resident-worker count, and competing implementation;\nthere is no universal `M < BLOCK_M` diagnosis or minimum viable size.\n\nUniform expected routing, a larger batch, or an auxiliary balancing loss may\nchange token-count skew, but none proves balanced tile counts, worker\ndurations, communication, or runtime. Treat imbalance as absent only when the\nmeasured distributions are narrow and a matched balancing variant does not\nmaterially improve the timed operation."}, "reason": {"statement": "Replace proxies with an operational measurement criterion.", "urls": ["https://www.lmsys.org/blog/2025-05-05-large-scale-ep/", "https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html", "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/efficient_gemm.md"]}} @@ -490,4 +490,4 @@ {"path": "wiki/kernels/tensorrt-llm-blackwell-indexer.md", "before": {"statement": "```cuda\n// Evidence checklist before adapting the idea:\n// 1. Inspect indexerKCacheGather.cu and indexerKCacheScatter.cu.\n// 2. Check the scale and FP4 packing layout.\n// 3. Benchmark gather/scatter separately from top-k selection.\n__global__ void candidate_indexer_probe(const uint8_t* cache, int* indices) {\n int tid = blockIdx.x * blockDim.x + threadIdx.x;\n indices[tid] = static_cast(cache[tid]);\n}\n```"}, "after": {"statement": "For the fixed 128-value indexer head in this path, one row has:\n\n| Input | BF16 positional values followed by BF16 non-positional values | 256 before quantization |\n| Packed payload | Two FP4 E2M1 codes per byte | 64 |\n| Scale word | Four UE8M0 exponent bytes, one per successive group of 32 values, packed little-endian into one `int32` | 4 |\n| Cache footprint | Packed payload plus scale word | 68 |\n\nThe fused operator requires CUDA BF16 inputs on one device, at least two\ndimensions, a contiguous innermost dimension, eight-byte-aligned addresses,\nequal row counts, and positional width divisible by four. Positional and\nnon-positional widths must sum to 128. It returns packed `int8[M,64]` data and\n`int32[M,1]` scales.\n\nThe scale for each 32-value group is\n`2^ceil(log2(max(amax, 1e-12) / 6))`. Quantization uses the FP4 E2M1 magnitude\nset `{0, 0.5, 1, 1.5, 2, 3, 4, 6}` and packs the earlier value into the low\nnibble.\n\nThe cache is viewed as\n`[num_blocks, block_size, 1, per_token_size]` and may be non-contiguous. Each\ntoken launch copies four bytes per thread. The FP4 path uses 64 payload bytes\nand one four-byte scale word; the legacy FP8 path uses 128 payload bytes and\none four-byte FP32 scale.\n\nPayload and scale use separate contiguous `int64` slot-mapping arrays. If\neither mapping for a token is negative, gather and scatter skip that entire\ntoken. The gather wrapper allocates output with `empty`, so a directly gathered\nrow skipped this way has no kernel-written sentinel value; callers must avoid\nconsuming it or define initialization in their own reference.\n\nThe gather wrapper preserves a historical typed view: payload bytes are\nreturned as float8 and scale bytes as FP32. The FP4 call site reinterprets them\nas packed `int8` data and `int32` scale words before the DeepGEMM call. These\nviews are byte contracts, not evidence that the FP4 payload became FP8 or that\nits UE8M0 scale word became an FP32 numeric scale.\n\n1. Compare `fused_cat_fp4` byte-for-byte with a reference implementation for\n zeros, FP4 decision boundaries, saturation, non-contiguous row strides, and\n multiple positional/non-positional splits whose widths sum to 128.\n2. Round-trip valid tokens through scatter and gather using strided cache\n views. Check all 64 payload bytes and the four scale bytes independently;\n test negative payload and scale mappings without reading skipped empty rows.\n3. Compare FP4 non-paged and paged indexer logits, then selected indices,\n against an unquantized or higher-precision reference with identical masks,\n weights, and sequence boundaries.\n4. Profile fused preparation, scatter, gather, logit computation, top-k, and\n the complete indexer separately. Report shapes, cache layout, mapping mix,\n warmup, synchronization, repetitions, statistic, and variation.\n\nThe pinned PR contains no performance result, so use it to define candidate\nmechanisms and exact contracts rather than to claim a speedup."}, "reason": {"statement": "Remove invented CUDA that neither implements nor safely probes the pinned indexer contract.", "urls": []}} {"path": "wiki/kernels/tensorrt-llm-blackwell-indexer.md", "before": {"statement": "TensorRT-LLM PR 13340 integrates an FP4 indexer path for DSA on Blackwell and\nlands CUDA kernels for K-cache gather/scatter and fused FP4 concatenation. Use it\nas implementation evidence for sparse indexer memory movement and quantized\ncache layout, not as a drop-in answer for FlashInfer-Bench."}, "after": {"statement": "TensorRT-LLM PR 13340 adds an FP4 option to its DeepSeek Sparse Attention\n(DSA) indexer path. This page describes the PR's pinned merge revision\n`897c4bff`; it is a TensorRT-LLM implementation reference, not a generic DSA\nABI or a drop-in FlashInfer-Bench kernel.\n\nThe implementation has four distinct stages:\n\n| Q/K preparation | `fused_cat_fp4` concatenates BF16 positional and non-positional components, then quantizes the 128-value row to FP4 E2M1 with per-32-value UE8M0 scales |\n| Cache update/read | Scatter and gather copy an already-quantized payload and its scale word between contiguous tensors and TensorRT-LLM's possibly non-contiguous paged indexer cache |\n| Indexer logits | The FP4 path reinterprets the packed bytes/scales and dispatches TensorRT-LLM's DeepGEMM FP4 MQA-logit implementation |\n| Selection | Prefill or decode top-k runs after logit computation in a separate operator |\n\nThe gather/scatter kernels do not quantize values, compute logits, or select\nindices. Calling them “fused quantized gather/scatter” conflates the first two\nstages.\n\n1. Compare `fused_cat_fp4` byte-for-byte with a reference implementation for\n zeros, FP4 decision boundaries, saturation, non-contiguous row strides, and\n multiple positional/non-positional splits whose widths sum to 128.\n2. Round-trip valid tokens through scatter and gather using strided cache\n views. Check all 64 payload bytes and the four scale bytes independently;\n test negative payload and scale mappings without reading skipped empty rows.\n3. Compare FP4 non-paged and paged indexer logits, then selected indices,\n against an unquantized or higher-precision reference with identical masks,\n weights, and sequence boundaries.\n4. Profile fused preparation, scatter, gather, logit computation, top-k, and\n the complete indexer separately. Report shapes, cache layout, mapping mix,\n warmup, synchronization, repetitions, statistic, and variation.\n\nThe pinned PR contains no performance result, so use it to define candidate\nmechanisms and exact contracts rather than to claim a speedup."}, "reason": {"statement": "Name concat-plus-quantization and byte-copy cache movement as separate operations.", "urls": []}} {"path": "wiki/patterns/memory-bound.md", "before": {"statement": "- Always profile before optimizing — wrong assumption wastes effort"}, "after": {"statement": "Report input shapes/distributions, useful work and bytes, cache state, warmup/repetitions/statistic, GPU and clocks, software versions, generated instructions, resource usage, roofline assumptions, achieved memory-level bandwidth, requested/transferred efficiency, scheduler activity, and correctness tolerance. Include negative and regressing variants so a category heuristic is not mistaken for causal evidence."}, "reason": {"statement": "Replace an absolute workflow rule with an evidence-based diagnose, hypothesize, control, and measure loop.", "urls": ["https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html"]}} -{"path": "wiki/techniques/chunk-parallelism.md", "before": {"statement": "- Linear attention (O(n) complexity)\n- Recurrent state models (Mamba, GatedDeltaNet, Delta Rule)"}, "after": {"statement": "Some linear recurrent models admit an algebraically equivalent chunkwise formulation. Work local to a chunk can then be expressed with parallel matrix operations, while the state passed across chunk boundaries preserves the recurrence's sequence order. The exact transform is model-specific: it must be derived from the recurrence, not replaced by a generic attention matrix and additive state update.\n\nA correct implementation separates at least these obligations:\n\n1. Compute the chunk-local quantities required by the model's exact recurrence.\n2. Resolve boundary states in sequence order, either with an associative scan supported by the formulation or with explicitly ordered stages or launches.\n3. Combine each chunk's local result with its incoming boundary state and emit outputs in the original token order.\n4. Validate outputs and final states against a token-by-token reference across variable sequence lengths, chunk tails, batches, heads, dtypes, and gate extremes.\n\nAn ordinary GPU grid does not imply increasing program-ID execution order or grid-wide synchronization. Programs for every chunk therefore cannot safely read and overwrite one shared state pointer in a single unordered launch. A staged algorithm must make the boundary-state dependency explicit.\n\nThe pinned NVlabs GatedDeltaNet repository uses chunkwise Triton kernels for training and a WY representation of the gated delta rule. That implementation is direct evidence for GatedDeltaNet chunking, but it is not equivalent to a generic `scores = Q @ K.T; output = scores @ V` snippet and does not supply a universal chunk-size rule.\n\nTiled Flash Linear Attention (TFLA) starts from the chunkwise formulation of linear RNNs and adds another level of sequence parallelization within a chunk. The authors state that this permits arbitrarily large chunks, raises arithmetic intensity, and reduces intermediate-state materialization. The paper and pinned official code apply the method to mLSTM and report H100 results; they do not establish a GatedDeltaNet implementation, a Blackwell/TMEM implementation, or a recursive-tiling API.\n\nThere is no source-backed universal rule that `C=32` is a decode choice or that `C=256-512` is the prefill optimum. Choose only among chunk sizes supported by the exact algorithm and backend, then measure the tradeoff:\n\n- arithmetic intensity and matrix-instruction utilization;\n- intermediate-state and workspace traffic;\n- registers, shared memory, and occupancy;\n- tail handling and variable-length metadata;\n- launch count and boundary-scan cost;\n- latency and throughput for the intended batch and sequence distribution.\n\nCompare candidates with identical compiler/software revisions, launch inputs, correctness oracles, warmups, synchronization, and repeated-trial statistics. Treat a selected size as scoped to the model dimensions, dtype, GPU, backend, and workload. TMEM availability on SM100 may change an implementation's resource design, but it does not by itself prove that a larger chunk is legal or faster."}, "reason": {"statement": "Require a proven model-specific chunkwise-equivalent transform and measured benefit before transfer.", "urls": ["https://arxiv.org/abs/2503.14376v3", "https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921", "https://github.com/state-spaces/mamba", "https://triton-lang.org/main/programming-guide/chapter-01/introduction.html"]}} +{"path": "wiki/techniques/chunk-parallelism.md", "before": {"statement": "- Linear attention (O(n) complexity)\n- Recurrent state models (Mamba, GatedDeltaNet, Delta Rule)"}, "after": {"statement": "Some linear recurrent models admit an algebraically equivalent chunkwise formulation. Work local to a chunk can then be expressed with parallel matrix operations, while the state passed across chunk boundaries preserves the recurrence's sequence order. The exact transform is model-specific: it must be derived from the recurrence, not replaced by a generic attention matrix and additive state update.\n\nA correct implementation separates at least these obligations:\n\n1. Compute the chunk-local quantities required by the model's exact recurrence.\n2. Resolve boundary states in sequence order, either with an associative scan supported by the formulation or with explicitly ordered stages or launches.\n3. Combine each chunk's local result with its incoming boundary state and emit outputs in the original token order.\n4. Validate outputs and final states against a token-by-token reference across variable sequence lengths, chunk tails, batches, heads, dtypes, and gate extremes.\n\nAn ordinary GPU grid does not imply increasing program-ID execution order or grid-wide synchronization. Programs for every chunk therefore cannot safely read and overwrite one shared state pointer in a single unordered launch. A staged algorithm must make the boundary-state dependency explicit.\n\nThe pinned NVlabs GatedDeltaNet repository uses chunkwise Triton kernels for training and a WY representation of the gated delta rule. That implementation is direct evidence for GatedDeltaNet chunking, but it is not equivalent to a generic `scores = Q @ K.T; output = scores @ V` snippet and does not supply a universal chunk-size rule.\n\nTiled Flash Linear Attention (TFLA) starts from the chunkwise formulation of linear RNNs and adds another level of sequence parallelization within a chunk. The authors state that this permits arbitrarily large chunks, raises arithmetic intensity, and reduces intermediate-state materialization. The paper and pinned official code apply the method to mLSTM and report H100 results; they do not establish a GatedDeltaNet implementation, a Blackwell/TMEM implementation, or a recursive-tiling API.\n\nThere is no source-backed universal rule that `C=32` is a decode choice or that `C=256-512` is the prefill optimum. Choose only among chunk sizes supported by the exact algorithm and backend, then measure the tradeoff:\n\n- arithmetic intensity and matrix-instruction utilization;\n- intermediate-state and workspace traffic;\n- registers, shared memory, and occupancy;\n- tail handling and variable-length metadata;\n- launch count and boundary-scan cost;\n- latency and throughput for the intended batch and sequence distribution.\n\nCompare candidates with identical compiler/software revisions, launch inputs, correctness oracles, warmups, synchronization, and repeated-trial statistics. Treat a selected size as scoped to the model dimensions, dtype, GPU, backend, and workload. TMEM availability on SM100 may change an implementation's resource design, but it does not by itself prove that a larger chunk is legal or faster."}, "reason": {"statement": "Require a proven model-specific chunkwise-equivalent transform and measured benefit before transfer.", "urls": ["https://arxiv.org/abs/2503.14376v3", "https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921", "https://github.com/state-spaces/mamba", "https://triton-lang.org/main/programming-guide/chapter-1/introduction.html"]}} diff --git a/verification/current-queue.jsonl b/verification/current-queue.jsonl index d25fca406..32dd7a45a 100644 --- a/verification/current-queue.jsonl +++ b/verification/current-queue.jsonl @@ -1,11 +1,11 @@ {"body_sha256": "3323bbd48edb46227476afa34e05f5c2552b2883b02b8a0e60d189dc5592ae97", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "PTX defines a CTA pair as two CTAs in the same cluster whose `%cluster_ctarank` values differ only in the low bit. With `cta_group::2`, one thread from either CTA can initiate a whole `tcgen05.mma`; the peer CTA must still be active. The op", "sha256": "b7afdc24eb589c39ba6e02afe005b308b174971a2755868869988a5789159e7e"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "Allocation management differs from MMA issue. A `tcgen05.alloc` or `tcgen05.dealloc` for `cta_group::2` is issued collectively by two warps, one in each CTA. All tcgen05 instructions in the kernel must use the same CTA-group value.", "sha256": "dbe9bd221edf8a902619236cdafd74941eaa79370ff87f5be263e86965cd6403"}, {"id": "u003", "kind": "prose", "locator": "body:L11-L11", "preview": "`cta_group::2` selects pair-level resources; it does not by itself select one fixed MxNxK shape. The instruction descriptor (`idesc`) encodes M, N, exact operand and accumulator types, sparsity, and related operation details. Legal values d", "sha256": "645ac1d3970d062f6bb9eae3ae3b25048285e45ef7a9f67d305f3d5850fe00e6"}, {"id": "u004", "kind": "prose", "locator": "body:L13-L13", "preview": "For example, m256xn256xk16 is a useful maximum F16/BF16 configuration, but PTX also defines group-2 layouts with other M/N values. Do not infer that every pair-level MMA is created by mechanically doubling a group-1 M dimension.", "sha256": "45f876da36a8addf0f5f581bfda573dd932287c7dc2df933d377bd5a562a1e27"}, {"id": "u005", "kind": "prose", "locator": "body:L15-L15", "preview": "The complete operand grammar, including the eight-register disable-output-lane vector used by `cta_group::2`, is documented on the related [tcgen05.mma page](tcgen05-mma.md).", "sha256": "936e6a2c0cc8ca27f6d457147b1e4da3d5cb75d9e7ebe81f0d7a427f0df6a20e"}, {"id": "u006", "kind": "prose", "locator": "body:L19-L19", "preview": "A pair-level kernel must account for all of these invariants:", "sha256": "e650fb835f8148efa001798dc386feb5cf624c517a63042b2bf85b81eb9ce3e7"}, {"id": "u007", "kind": "list-item", "locator": "body:L21-L21", "preview": "- The two CTAs form a valid CTA pair, and the peer remains active while a group-2 operation is issued.", "sha256": "8c9f5a7af50f1a7d7318498652a12b917fadff697d60bbacde9da59632ca1727"}, {"id": "u008", "kind": "list-item", "locator": "body:L22-L22", "preview": "- Pair-level TMEM allocation and deallocation follow their two-warp collective issue rule.", "sha256": "7fc30f24802c8c5f7ad184674a87dd8fae96c86c6fb7c60b5f65bddd03c46a83"}, {"id": "u009", "kind": "list-item", "locator": "body:L23-L23", "preview": "- SMEM and TMEM operands use legal layouts and descriptors for the selected instruction configuration.", "sha256": "1b832c3aec6846dd172301510f13cf19edd674601565aaca3167657749c9298e"}, {"id": "u010", "kind": "list-item", "locator": "body:L24-L24", "preview": "- Source storage is not overwritten while the asynchronous MMA may still consume it.", "sha256": "68db8f70b3b5c1f6ffa65d9ca6a1b5fde856003e9f6774e061d5768162bba2ed"}, {"id": "u011", "kind": "list-item", "locator": "body:L25-L25", "preview": "- `tcgen05.commit.cta_group::2` and an mbarrier wait provide completion tracking. Fences and execution-ordering operations are added where tcgen05-visible state crosses threads or CTAs.", "sha256": "81756fed0c3ff7d3024ce2f8d37cc4909fa6f8e020401bd81455b0b1a565d299"}, {"id": "u012", "kind": "prose", "locator": "body:L29-L29", "preview": "In Gau Nernst's M=N=K=4096 experiment on a Modal B200 with PyTorch 2.9.1 and CUDA 13, v4 warp specialization reports 1208.83 TFLOP/s and v5 2-SM MMA reports 1302.29 TFLOP/s. Relative to the same 1506.74-TFLOP/s cuBLAS result, those are appr", "sha256": "61e879b8bfa7a29e2e447c1ee711711fbe772da90995a3fe00d069595f991495"}, {"id": "u013", "kind": "prose", "locator": "body:L31-L31", "preview": "That result establishes a gain for one kernel and setup, not a universal threshold. Choose between group 1 and group 2 by benchmarking the target shapes and accounting for CTA pairing, data reuse, layout, occupancy, pipeline stages, and epi", "sha256": "7a1a54d6bfb919e3146075f7166fe3c39e80c43c275ae0409c1c81288eae52ec"}, {"id": "u014", "kind": "list-item", "locator": "body:L35-L35", "preview": "- [tcgen05.mma](tcgen05-mma.md) \u2014 instruction grammar, completion, and descriptors", "sha256": "59d6232600e3c79f97116d6414daad7399a0e29b469130cf23ce8ff9b7f124c0"}, {"id": "u015", "kind": "list-item", "locator": "body:L36-L36", "preview": "- [Tensor Memory](tmem.md) \u2014 TMEM allocation, addressing, and access", "sha256": "a401db8bea32d6d1e6be27e6bb3b9692341354f1689f50bb323de203b7a85c75"}], "confidence_claimed": "verified", "headings": ["Execution model", "Shape is independent of group size", "Correctness requirements", "Source-reported performance", "Related"], "id": "hw-2sm-cooperative", "path": "wiki/hardware/2sm-cooperative.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/prs/cutlass/PR-2139.md", "revision": "ca4fdbea", "url": "https://github.com/NVIDIA/cutlass/pull/2139"}], "risk_flags": ["ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-ptx-isa-sm100", "blog-tcgen05-tutorial", "pr-cutlass-2139"], "title": "Two-SM Cooperative MMA", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} {"body_sha256": "1f3e642012f3b80c6e5f67e510f41b1577c6bf9f6c669549349e1825285ea491", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L8", "preview": "Cluster Launch Control is a Blackwell compute-capability 10.0 mechanism for work stealing between running and not-yet-started thread blocks or thread block clusters. A CLC kernel still launches the problem-sized grid. Each grid coordinate, ", "sha256": "aed47d2a04a85e6e88c936e4f0305caff731581c87ae085491120f8fa1ae4e43"}, {"id": "u002", "kind": "list-item", "locator": "body:L10-L10", "preview": "1. The block or cluster launches normally and first processes its own", "sha256": "dfffcdb04d50ba6a0611612fbb6ff9f3a8689688b0dcd3728ba506a89ef66b38"}, {"id": "u003", "kind": "prose", "locator": "body:L11-L11", "preview": "`blockIdx`.", "sha256": "4c5c20de2592c1acafe9877ccbb8b570ecf16bbef8b6f7e393e80a08ad288c45"}, {"id": "u004", "kind": "list-item", "locator": "body:L12-L12", "preview": "2. An existing worker successfully cancels that not-yet-started ClcID and", "sha256": "ed46f5f3fe23df91f74b22f3f5038976844d96b772df7bc9f16241a82cc067a3"}, {"id": "u005", "kind": "prose", "locator": "body:L13-L13", "preview": "processes the returned coordinate itself.", "sha256": "6d361b359451a9c431b0548ae2c09aa46647cf39813bcb04e3b07d03f17f274f"}, {"id": "u006", "kind": "prose", "locator": "body:L15-L17", "preview": "This lets a persistent worker request subsequent work without maintaining a separate software work queue. It is especially useful when the set of SMs available to a kernel is uneven or changes while the grid is running.", "sha256": "5882b7a980f93728bc6dd0f50b3db643bb8841689dfd7f738ccdb0d8ed30eaf9"}, {"id": "u007", "kind": "prose", "locator": "body:L21-L23", "preview": "`clusterlaunchcontrol.try_cancel` is asynchronous. It writes an opaque 16-byte response to shared memory and completes a transaction on a shared-memory `mbarrier`. The request does not accept a tile coordinate to cancel.", "sha256": "07b338face1ef537bc6a6b1f149dd819322353bfa5868d3c7912e3dc818d2221"}, {"id": "u008", "kind": "prose", "locator": "body:L25-L25", "preview": "The normative PTX instruction forms are:", "sha256": "ed7ac6510f74ef3d579ebe7336066cd20a940b5608289455a2568d88e557f929"}, {"id": "u009", "kind": "code", "locator": "body:L27-L31", "preview": "```ptx clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [response], [mbar]; clusterlaunchcontrol.query_cancel.is_canceled.pred.b128 p, response_b128; @p clusterlaunchcontrol.query_cancel.get_first_ctaid.v4", "sha256": "67fa262449a500557e66306c5d57edcda130d2e7ad8b3b2353b998f1aacb1556"}, {"id": "u010", "kind": "prose", "locator": "body:L33-L33", "preview": "A complete loop must:", "sha256": "d24cc2e528fb2301c03829b15fab641a2eca64b281568e016244188a32f257f6"}, {"id": "u011", "kind": "list-item", "locator": "body:L35-L35", "preview": "1. Allocate and initialize a shared response and `mbarrier`.", "sha256": "0ba588d30c80542da4138743e2418709adee3172b8e1aabc0eb1399de0d04c1e"}, {"id": "u012", "kind": "list-item", "locator": "body:L36-L36", "preview": "2. Submit one asynchronous request from the selected thread and set the", "sha256": "262ae4de6893ed5ff99b6aefa72c4f46117de59cee3bc9fdea46db4d10ccab08"}, {"id": "u013", "kind": "prose", "locator": "body:L37-L37", "preview": "expected transaction size to 16 bytes.", "sha256": "c5b2c635a95f64a2b4978c6746fb9a09fb96d36de263e92d906c555c0c376272"}, {"id": "u014", "kind": "list-item", "locator": "body:L38-L38", "preview": "3. Wait for the corresponding barrier phase to complete.", "sha256": "30d41ba4441d9bf673d62236cb464dde1dad69478b780b3fdd7f13aaea5d1fff"}, {"id": "u015", "kind": "list-item", "locator": "body:L39-L39", "preview": "4. Decode `is_canceled`; decode `get_first_ctaid` only after success.", "sha256": "38e1bc22c329938b11e59946ee33bcab2ee641c7f44aabec8d12e11ba0ee4347"}, {"id": "u016", "kind": "list-item", "locator": "body:L40-L40", "preview": "5. Apply the async/generic proxy fences needed before reusing the response.", "sha256": "26a497bb386f06ad534f04253e05b08ae4f740df588cfb9b7464897480ef7feb"}, {"id": "u017", "kind": "prose", "locator": "body:L42-L45", "preview": "After a thread has observed a failed request, issuing another request from that thread is undefined. Decoding a CTA ID from a failed response is also undefined. A request can fail because no ClcIDs remain or for another scheduling reason, i", "sha256": "2579bb2d0dc3f63c5616d49e9cd07bdaf39852713cc130a7f10392d3fc7d4a85"}, {"id": "u018", "kind": "prose", "locator": "body:L49-L54", "preview": "CLC cancellation is cluster-granular when the kernel uses thread block clusters. One cluster thread submits the multicast request. Every CTA tracks completion with its local shared-memory barrier at cluster scope and receives the same encod", "sha256": "af8682d845156e2308926d1a0bf821b02be6e659e9b3ff363b58921f5558b16d"}, {"id": "u019", "kind": "prose", "locator": "body:L56-L57", "preview": "For example, a successful query by a 2x2 worker cluster consumes the matching 2x2 group of ClcIDs; it is not four unrelated CTA-level cancellations.", "sha256": "6e23b2f1a5cde36d9a61363e854bcc5530da8de592fcda721043478ce0b8c281"}, {"id": "u020", "kind": "prose", "locator": "body:L61-L65", "preview": "At CUTLASS 4.5.0, the `PersistentScheduler` tag for `arch::Sm100` maps to `PersistentTileSchedulerSm100`; newer code can select the intent explicitly with `DynamicPersistentScheduler`. The SM100 scheduler uses `PipelineCLCFetchAsync`: `adva", "sha256": "15bb650d78b4f58a6b6a89201a36b0df29565b4d9ef8768ae8ca5f3a2ae03437"}, {"id": "u021", "kind": "prose", "locator": "body:L67-L67", "preview": "A schematic of the relevant CUTLASS 3.x kernel composition is:", "sha256": "c30f20ad1c9bae9938a60381504d2be3fa4a3a3d3989ca677da0457e294ed4a6"}, {"id": "u022", "kind": "code", "locator": "body:L69-L77", "preview": "```cpp using TileScheduler = cutlass::gemm::DynamicPersistentScheduler; using GemmKernel = cutlass::gemm::kernel::GemmUniversal< cute::Shape, CollectiveMainloop, CollectiveEpilogue, TileScheduler>; ```", "sha256": "f2d33d4b2462ceef8aaac37046da1b30637a67e5d88e2e6402598e676bf5d53b"}, {"id": "u023", "kind": "prose", "locator": "body:L79-L82", "preview": "CUTLASS can apply a software coordinate transform to both the initial `blockIdx` and decoded CLC responses. `max_swizzle_size` and `raster_order` are scheduler arguments used by `swizzle_and_rasterize()`; they are not operands of the CLC PT", "sha256": "a4b2a143772be0706381b1a9813f26cd44da22fd53d5f82576c4b4dd053d10e7"}, {"id": "u024", "kind": "prose", "locator": "body:L86-L86", "preview": "CLC redistributes ClcIDs that already exist in the launched grid. It does not:", "sha256": "bf4dc5fc684e939ef4ea815ea1f55868b6255ce0620993cb595c0fd89df6aab7"}, {"id": "u025", "kind": "list-item", "locator": "body:L88-L88", "preview": "- discard an application-selected output tile;", "sha256": "0e4143cc1ba403906fbd9bb07350d6ac3d46a4ab482c833d5befca3696d1c77f"}, {"id": "u026", "kind": "list-item", "locator": "body:L89-L89", "preview": "- create more independent tiles than the problem grid contains;", "sha256": "cd5c58b2f2af74b71e1551b0cab1067a68a6385717c9f4ce6d0138711e7814db"}, {"id": "u027", "kind": "list-item", "locator": "body:L90-L90", "preview": "- guarantee that every SM remains occupied; or", "sha256": "ca82f9c8684574f060c65785314120d152a34c0efe8956d710f8250716b52bde"}, {"id": "u028", "kind": "list-item", "locator": "body:L91-L91", "preview": "- universally eliminate GEMM wave quantization.", "sha256": "4eaabf49ee1f6647506cc7cdb71f11a5ec2655ea5ca0bee1aabfb7661a071d8a"}, {"id": "u029", "kind": "prose", "locator": "body:L93-L96", "preview": "Consequently, a 32-tile grid exposes at most 32 independent ClcIDs even on a 148-SM B200. Any CLC performance result must state the GPU, available SMs, software versions, tile and cluster shapes, data types, timed region, and measurement me", "sha256": "0453affd2bd43d77232b7fa7a6bd19d9d54911593654fe9569df1a5693c05960"}, {"id": "u030", "kind": "list-item", "locator": "body:L100-L100", "preview": "- [Persistent kernels](../techniques/persistent-kernels.md)", "sha256": "ec9585a9787c7d4334a41d77c6ecbeee6575e90c64bb37f1f2e8c8b8d64b041d"}, {"id": "u031", "kind": "list-item", "locator": "body:L101-L101", "preview": "- [Tile scheduling](../techniques/tile-scheduling.md)", "sha256": "cf18db71a889f046a73a3782397405ed6582feabacaff30a3581db0e6ca9b909"}, {"id": "u032", "kind": "list-item", "locator": "body:L102-L102", "preview": "- [Two-SM cooperative MMA](2sm-cooperative.md)", "sha256": "72792611c19c45c9af380298f75c59a65e9192b3ca1891804c858ae32c448c19"}], "confidence_claimed": "source-reported", "headings": ["Overview", "Request and Decode Protocol", "Thread Block Cluster Rules", "CUTLASS 4.5.0 Integration", "Scope and Limits", "Related"], "id": "hw-clc", "path": "wiki/hardware/clc.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/docs/cutlass-clc-documentation.md", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}], "risk_flags": ["code", "ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-ptx-isa-sm100", "doc-cutlass-clc"], "title": "Cluster Launch Control (CLC)", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} {"body_sha256": "4215a80fde138535a1d096475ad4d1b76c905d6cd8c3ab2b6198eae4c78180e3", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "An mbarrier is an opaque, naturally aligned 64-bit object in shared memory. It synchronizes threads and can track asynchronous operations. The base instructions were introduced in PTX ISA 7.0 for `sm_80`; Hopper (`sm_90`) added transaction-", "sha256": "c4f5f8029fdece300451e77adb2415e3afc2669f9a0de1bfb7882ffc46fc741a"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "mbarriers are useful in warp-specialized pipelines, but they are a mechanism rather than a requirement for every such kernel.", "sha256": "c1a2ae1d13b327f646fba681f8179c799b52402580d94b42b7d37665834c5abf"}, {"id": "u003", "kind": "prose", "locator": "body:L11-L11", "preview": "For the current phase, an mbarrier tracks:", "sha256": "304bf0328887acfa9d619fae48be89bcb511e2e5dfdda8e39df146bdc226ccc4"}, {"id": "u004", "kind": "list-item", "locator": "body:L13-L13", "preview": "- pending arrivals;", "sha256": "34d9f60ef2b7432a837c1500cfba321064686c29d34d2ae29f3b2958006daa5f"}, {"id": "u005", "kind": "list-item", "locator": "body:L14-L14", "preview": "- the expected arrival count for the next phase; and", "sha256": "f58e7621149f63f5e686db346bcae1c67443ce37f6e6548ed08d0405e9e19c4f"}, {"id": "u006", "kind": "list-item", "locator": "body:L15-L15", "preview": "- a transaction count (`tx-count`) for outstanding asynchronous work.", "sha256": "79aac4088b7db28aec02c8ae5c432e55959337176f40cf20ee2c9dd974977dc0"}, {"id": "u007", "kind": "prose", "locator": "body:L17-L17", "preview": "The current phase completes only when **both** pending arrivals and tx-count reach zero. Completion atomically advances to the next phase and restores pending arrivals from the expected count. Before an arrival in the following phase, at le", "sha256": "3ef1cba778254c5a2d9eb1e0c765268be4505dbd47deab20ed5d96aa70b884f1"}, {"id": "u008", "kind": "prose", "locator": "body:L19-L19", "preview": "Initialization sets phase 0, initializes expected and pending arrivals to `count`, and sets tx-count to zero:", "sha256": "3dd244f469369ff5bb493acac9cf1c56fa19c974e059eee55b901afb1febbd78"}, {"id": "u009", "kind": "code", "locator": "body:L21-L23", "preview": "```ptx mbarrier.init.shared::cta.b64 [bar], arrival_count; ```", "sha256": "aa334cafb032b7c70d03d4e311e23e4026954ad8ea5831c5b48c3204ddb2e352"}, {"id": "u010", "kind": "prose", "locator": "body:L25-L25", "preview": "Invalidate the object with `mbarrier.inval` before reusing its storage for another purpose or reinitializing an already-valid object.", "sha256": "3e1ccc8d4cf7dcc62712f205c681c77ac6676d2a8976dbbb38da31e5ce84dd82"}, {"id": "u011", "kind": "prose", "locator": "body:L29-L29", "preview": "These operations affect different parts of the state:", "sha256": "db8771ec1644298d11488ccedea8133c2d8a332e784378167861fde24f6d0dac"}, {"id": "u012", "kind": "table-row", "locator": "body:L33-L33", "preview": "| Operation | Effect | | `mbarrier.arrive` | Decrements pending arrivals; returns a phase state token for a CTA-shared object. |", "sha256": "300828c39cb781b4776ad8085e58d35a449c60007fb58130022bd4d2df92ee87"}, {"id": "u013", "kind": "table-row", "locator": "body:L34-L34", "preview": "| Operation | Effect | | `mbarrier.arrive.expect_tx` | Performs an arrival and increments tx-count by `txCount`. |", "sha256": "772dbfd906bd8df1d1355530998b10a6527e37976ed1f530c1e6a2ddee681939"}, {"id": "u014", "kind": "table-row", "locator": "body:L35-L35", "preview": "| Operation | Effect | | `mbarrier.expect_tx` | Increments tx-count without an arrival. |", "sha256": "5bb2361feb6d63ec4abc789110d8fe7ae1e7755422f6d64271931f8408e346a1"}, {"id": "u015", "kind": "table-row", "locator": "body:L36-L36", "preview": "| Operation | Effect | | `mbarrier.complete_tx` | Decrements tx-count; it is not an arrival. |", "sha256": "4a62afc513ab2b8c7e543ac78ea5073ed6a6e2edbdac71c34a72fcfa814f5d44"}, {"id": "u016", "kind": "table-row", "locator": "body:L37-L37", "preview": "| Operation | Effect | | `mbarrier.test_wait` / `try_wait` | Tests completion of the phase identified by a state token or parity. |", "sha256": "1eba28206a3d00699d921ca747a0c9124fbece5c665904280441030dffc1c170"}, {"id": "u017", "kind": "prose", "locator": "body:L39-L39", "preview": "Representative CTA-shared forms from PTX ISA 9.0 are:", "sha256": "6438bbf3813fa047bdeeb75c65d22888dd99b788ba7abe2ee32fa891eed9dc5b"}, {"id": "u018", "kind": "code", "locator": "body:L41-L45", "preview": "```ptx mbarrier.arrive.shared::cta.b64 state, [bar]; mbarrier.arrive.expect_tx.shared::cta.b64 state, [bar], tx_count; mbarrier.try_wait.parity.acquire.cta.shared::cta.b64 ready, [bar], phase_parity; ```", "sha256": "0c512373f52c01377048c5a4fea08c09a3152100ab4a8ed297d6eda13d7df509"}, {"id": "u019", "kind": "prose", "locator": "body:L47-L47", "preview": "`try_wait` can suspend temporarily and must still be retried until its predicate is true. Use acquire/release semantics appropriate to the producer-consumer handoff; `.relaxed` does not provide memory-ordering or visibility guarantees.", "sha256": "d4d9f62f69e26f69151ea42f45ec273782c1731a952027788eba1155e66b9538"}, {"id": "u020", "kind": "prose", "locator": "body:L51-L51", "preview": "Parity is the low bit of an individual mbarrier object's phase: even phases use 0 and odd phases use 1. A parity wait can refer only to the current or immediately preceding phase, so software must track phase for the entire lifetime of that", "sha256": "e47ce58bc9df238ddefe1d32feb34f481553161f6041f3bdeafe9dae2c88b0ea"}, {"id": "u021", "kind": "prose", "locator": "body:L53-L53", "preview": "For an N-stage ring, track state per stage. When stage `s` is reused, pass the parity expected for `bar[s]`; toggle that stage's parity only after its phase completes. One global bit toggled on every loop iteration is generally wrong becaus", "sha256": "dac3944e6a8647b2452f518676aca1284de81d4cd74b089dab331f7764a0bea3"}, {"id": "u022", "kind": "prose", "locator": "body:L55-L55", "preview": "Using the opaque state returned by `mbarrier.arrive` is an alternative when the same participant can carry that token to its wait.", "sha256": "526896b32527c238db4d339df27bd01784d736631118b538e38356bc6649bef1"}, {"id": "u023", "kind": "prose", "locator": "body:L59-L59", "preview": "For a common TMA-load phase initialized with one pending arrival:", "sha256": "a96f90645a2583b97dd27a02fea0ffb23a166f2fece4b65b4197900cf9b08a67"}, {"id": "u024", "kind": "list-item", "locator": "body:L61-L61", "preview": "1. The producer executes `mbarrier.arrive.expect_tx` with the sum of bytes that all TMA operations in this phase will report.", "sha256": "e6e85131dd05b8acd13524ece1fbf82c07d1fb351d06ba6b9b97466177defb4d"}, {"id": "u025", "kind": "list-item", "locator": "body:L62-L62", "preview": "2. The producer issues the TMA operations with `.mbarrier::complete_tx::bytes` and the same barrier.", "sha256": "3b455d20ff2edf905c7c37c22b84bb4032d6e814ebf48c6ebc9f292e7b9c2a12"}, {"id": "u026", "kind": "list-item", "locator": "body:L63-L63", "preview": "3. Each TMA completion performs `complete-tx` for the bytes it copied.", "sha256": "58b5cf80e14bbd08da00abbf27a7f8a2da08ada34555b744fbf89d7700b0b686"}, {"id": "u027", "kind": "list-item", "locator": "body:L64-L64", "preview": "4. The consumer waits for the phase to complete before reading the destination.", "sha256": "9a51e79d4ad54417a7c9bc97a7b91a0d23fae064b7c858456d0904fc8586e507"}, {"id": "u028", "kind": "prose", "locator": "body:L66-L66", "preview": "The `arrive.expect_tx` operation accounts for the software arrival and establishes the expected byte total. TMA hardware does **not** perform a second arrival: it decrements tx-count through complete-tx. Therefore, do not add an unmatched `", "sha256": "c52470c2e44b4d5cf43c371c1c9b810cea078c50208fbb8aac48d0844940ac00"}, {"id": "u029", "kind": "prose", "locator": "body:L68-L68", "preview": "The expected byte total, barrier address, destination ownership, and exact `cp.async.bulk.tensor` form must match. Follow the complete instruction grammar in the PTX ISA rather than using placeholder helper calls.", "sha256": "37fcfdfc369b1e9a22b669962cfd1e4ae24637c9e9f768a9b2dcc0054bc2122d"}, {"id": "u030", "kind": "prose", "locator": "body:L72-L72", "preview": "`tcgen05.commit.cta_group::*.mbarrier::arrive::one` makes an mbarrier track prior asynchronous `tcgen05.mma`, `tcgen05.cp`, or `tcgen05.shift` operations issued by that thread. When those operations complete, the system performs one arrive-", "sha256": "20e0ed0e00ca9af1a215f7874e174566a789deba3bab63c34b5cb599d5e24b8e"}, {"id": "u031", "kind": "prose", "locator": "body:L74-L74", "preview": "A consumer in another thread waits for the mbarrier and then uses the applicable `tcgen05.fence::after_thread_sync` ordering sequence. The fence participates in the handoff but does not replace the completion wait.", "sha256": "ef5af611efeacf06e77ed80510a63fb91011e700d7c43b4e8ef02263a285fe2b"}, {"id": "u032", "kind": "prose", "locator": "body:L78-L78", "preview": "State space and synchronization scope are separate concepts:", "sha256": "cde5c333b489108b69110f7a534b3445c2a5bdba33ea8114b885c3b9ef0f15bf"}, {"id": "u033", "kind": "list-item", "locator": "body:L80-L80", "preview": "- `.shared::cta` identifies a barrier in the current CTA's shared memory.", "sha256": "cb736ed3b82018b4af9ff8a71c06b39150af84e6300cd74341b00374ce604cd7"}, {"id": "u034", "kind": "list-item", "locator": "body:L81-L81", "preview": "- `.shared::cluster` identifies a cluster-shared address, such as a mapped address for another CTA's shared memory.", "sha256": "090eeb2f11c80401d9b399e755d561f418360d9f38fa7a8327e255e58ee245fe"}, {"id": "u035", "kind": "list-item", "locator": "body:L82-L82", "preview": "- `.cta` or `.cluster` on an operation specifies its synchronization scope.", "sha256": "1865209300b9b76af9ab274882f77faf299eb648588d07c2d2d23ea10172511a"}, {"id": "u036", "kind": "prose", "locator": "body:L84-L84", "preview": "Only `arrive`, `expect_tx`, and `complete_tx` support an mbarrier address in `.shared::cluster`. Other mbarrier operations, including initialization and waits, target a CTA-shared object. A cluster can initialize an owner CTA's object, map ", "sha256": "979150fd2755438fb6b2960fbfc8b1b3a6008d8b833e6e9e2368d68f72f98663"}, {"id": "u037", "kind": "list-item", "locator": "body:L88-L88", "preview": "- Initialize the object before any other mbarrier operation.", "sha256": "5a0505577b6d44496231fcf69d503700ad594baa46ed4caca261a31e4ae6fff8"}, {"id": "u038", "kind": "list-item", "locator": "body:L89-L89", "preview": "- Match all arrivals and byte-valued complete-tx operations to the initialized counts.", "sha256": "215bf504fc87a3066decf20c5d7b854138df7c926508dd83e33184698daa0f0d"}, {"id": "u039", "kind": "list-item", "locator": "body:L90-L90", "preview": "- Keep phase or opaque state separately for each pipeline stage.", "sha256": "9e11f202036db2380e4c8b118a24287c1b99ecdb55d99b096401f940d36ab928"}, {"id": "u040", "kind": "list-item", "locator": "body:L91-L91", "preview": "- Retry `try_wait` until it reports completion.", "sha256": "6ed11f86197d07208c541ec7e852358105d81227c2d119a210c9faac8b158b05"}, {"id": "u041", "kind": "list-item", "locator": "body:L92-L92", "preview": "- Use the required acquire/release semantics for data visibility.", "sha256": "7ad077d086fcdf1658e08a255ac255fb1bcaa232c537a86f920dbdce1f42130a"}, {"id": "u042", "kind": "list-item", "locator": "body:L93-L93", "preview": "- Distinguish TMA complete-tx from tcgen05 arrive-on completion.", "sha256": "9a4c97d8692c9ed6e03314c310463412be4d5b55a180624c83e2c0c70eb1f8b6"}, {"id": "u043", "kind": "list-item", "locator": "body:L94-L94", "preview": "- Respect the supported combinations of object state space and operation scope.", "sha256": "dd65f3da0041c597a64d3f6dcf024b5636d8dc1c4dbf5fc06b0d07075eedca5b"}, {"id": "u044", "kind": "list-item", "locator": "body:L95-L95", "preview": "- Invalidate the object before repurposing its storage.", "sha256": "a90e4657917b4d85f0f24721c561d0f0372ae63de4b3772aa96a1ad70a5b3cff"}, {"id": "u045", "kind": "list-item", "locator": "body:L99-L99", "preview": "- [PTX ISA 9.0: mbarrier object and lifecycle](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier)", "sha256": "440c7134e3238c710417e1ec64af241a2957ba1cd0e9ac30ba99e5ea63262e6a"}, {"id": "u046", "kind": "list-item", "locator": "body:L100-L100", "preview": "- [PTX ISA 9.0: mbarrier waits](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait)", "sha256": "545e66c84f010d0d57ac169190a90198a16ed31f5e14c5b327907a3110cb04b8"}, {"id": "u047", "kind": "list-item", "locator": "body:L101-L101", "preview": "- [PTX ISA 9.0: TMA tensor copies](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor)", "sha256": "daf224131f7f98d73e758e640b317c7c441ec221b87331b1d98fcaebc71e1c40"}, {"id": "u048", "kind": "list-item", "locator": "body:L102-L102", "preview": "- [TMA](tma.md)", "sha256": "d31942a8af5df5dde77f7f955775cf0a1e7539926207438cd5560491f1574f5d"}, {"id": "u049", "kind": "list-item", "locator": "body:L103-L103", "preview": "- [Warp specialization](../techniques/warp-specialization.md)", "sha256": "94df377cf6d384f1a13d7bab49fd429a6059089d72d3ec45bc1306ee3bba3549"}], "confidence_claimed": "verified", "headings": ["Scope and history", "Object state and phase completion", "Arrival, transaction, and wait operations", "Phase and parity in a stage ring", "TMA completion accounting", "tcgen05 completion accounting", "CTA and cluster distinctions", "Correctness checklist", "References"], "id": "hw-mbarrier", "path": "wiki/hardware/mbarrier.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/prs/cutlass/PR-2139.md", "revision": "ca4fdbea", "url": "https://github.com/NVIDIA/cutlass/pull/2139"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-ptx-isa-sm100", "pr-cutlass-2139"], "title": "mbarrier (Memory Barrier Primitives)", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} -{"body_sha256": "b574ab0127a35863b7ec1a2eb952aa93fe86bc145085a497ad64b9c3bc93a46a", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "NVFP4 is a quantization recipe, not just another name for its 4-bit payload. It reconstructs a value as:", "sha256": "227c70eceb96556659b2b5f6cf54e400e6245360f15905a84705c09b89a48113"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "`x_hat_i = e2m1_i * s_block * s_global`", "sha256": "794f7462514befdb8ac4a28522080782143495f8fd3157b5bea8c46d09934c6d"}, {"id": "u003", "kind": "prose", "locator": "body:L9-L9", "preview": "The three components are:", "sha256": "890b0b87819a86372f7ddcee49a11fd621e9a3a29c35ac7b8d1d2c2ff00d5c3e"}, {"id": "u004", "kind": "list-item", "locator": "body:L11-L11", "preview": "- **E2M1 payload:** one sign bit, two exponent bits, and one mantissa bit. Its numeric values are 0, +/-0.5, +/-1, +/-1.5, +/-2, +/-3, +/-4, and +/-6. Two E2M1 encodings fit in one byte.", "sha256": "c7e17ce8c55c39e0403be620d6ad757c8fa414b998dddfbdae211367d3c095e8"}, {"id": "u005", "kind": "list-item", "locator": "body:L12-L12", "preview": "- **Local scale:** one E4M3 scale shared by 16 consecutive payload values. PTX names the corresponding unsigned scale element type `.ue4m3`; its most-significant storage bit is zero.", "sha256": "5413cc4adc0cc112cd51d22ac07e5fcaae3f1e245164a937f4ba9333ca907655"}, {"id": "u006", "kind": "list-item", "locator": "body:L13-L13", "preview": "- **Global scale:** one FP32 scale per tensor. This second level lets the block scales use their range effectively.", "sha256": "12ca2c5eaaec7645bc831438892492d0c78328874b654e1ab71c39af7a429539"}, {"id": "u007", "kind": "prose", "locator": "body:L15-L15", "preview": "Transformer Engine 2.13 also defines a 2D scaling mode for weights in which a scale covers a 16-by-16 block. The 1D mode groups 16 consecutive values.", "sha256": "7288001414a30fe333b049e16ae69a4d1522ec4dc11bcaa88d3dabe2c03ba5b9"}, {"id": "u008", "kind": "prose", "locator": "body:L19-L19", "preview": "The recipes make different scale tradeoffs:", "sha256": "c9981094018252198ffe5ef16f090c60cb5483b13ea9d3f69e44906b9fd07b71"}, {"id": "u009", "kind": "table-row", "locator": "body:L23-L23", "preview": "| Property | NVFP4 | MXFP4 | | Payload | E2M1 | E2M1 |", "sha256": "a519973746ef18833e34e940839dffaf10f468db0b157561b380739125ce7fe4"}, {"id": "u010", "kind": "table-row", "locator": "body:L24-L24", "preview": "| Property | NVFP4 | MXFP4 | | Local group in the standard 1D recipe | 16 values | 32 values |", "sha256": "079a628d1c073e9d4c716174b141262c2e1f1fcedc21caca1a3f3693ac5c462b"}, {"id": "u011", "kind": "table-row", "locator": "body:L25-L25", "preview": "| Property | NVFP4 | MXFP4 | | Local scale | E4M3 (`.ue4m3` in PTX) | UE8M0 |", "sha256": "eb805ec4a81f255a4a58972a4e406037092fb53afe554d6d542b2851c232d419"}, {"id": "u012", "kind": "table-row", "locator": "body:L26-L26", "preview": "| Property | NVFP4 | MXFP4 | | Scale values | Fractional values are available | Powers of two |", "sha256": "64d0f1c0890f5f697993223f04daa88924b00162be84608850df6ea6fa8e4e93"}, {"id": "u013", "kind": "table-row", "locator": "body:L27-L27", "preview": "| Property | NVFP4 | MXFP4 | | Additional recipe scale | Per-tensor FP32 | Not part of the MXFP4 microscaling format |", "sha256": "660006f85f6632d68dcdeecff2dbe5a683dd5ab53e15f3d3c0eb5ea536906ea9"}, {"id": "u014", "kind": "prose", "locator": "body:L29-L29", "preview": "Finer groups and fractional local scales give NVFP4 more scaling freedom, at the cost of more scale metadata. They do not guarantee strictly lower error for every tensor, scale-selection algorithm, or error metric. Measure accuracy with the", "sha256": "18f8ae6f474472f1c2ea4ff9a3e2b597c1c36943095c02353db7eb28e098ed64"}, {"id": "u015", "kind": "prose", "locator": "body:L33-L33", "preview": "In PTX ISA 9.0, the relevant `tcgen05.mma` combinations for E2M1 inputs include:", "sha256": "4c6c2bfd4f02969ff8470c02bf64bbc802e646de47c9b11f4df253c5df3501fc"}, {"id": "u016", "kind": "table-row", "locator": "body:L37-L37", "preview": "| Instruction qualifiers | Block | Local scale | Scale-vector qualifier | | `.kind::mxf4.block_scale.block32` | 32 | `.ue8m0` | `.scale_vec::2X` |", "sha256": "c8d1878c1a0cadb7d3ca358add3c678ab21399a44da2e3d25f4c36ceebde2082"}, {"id": "u017", "kind": "table-row", "locator": "body:L38-L38", "preview": "| Instruction qualifiers | Block | Local scale | Scale-vector qualifier | | `.kind::mxf4nvf4.block_scale.block16` | 16 | `.ue4m3` | `.scale_vec::4X` |", "sha256": "4bd9801c764ad925b43b088977e249e1499ac6e62b0ef485dd544412fe0874f3"}, {"id": "u018", "kind": "prose", "locator": "body:L40-L40", "preview": "The `.kind::mxf4nvf4` family also admits documented UE8M0 modes, so the kind name by itself does not select the NVFP4 recipe. Use the complete block and scale-vector qualifiers and follow the type-combination tables. These forms target arch", "sha256": "c48d3a2f58dfe90849b6ee8abacd39ebf0f30cc5c6835517ffe7997d411dd227"}, {"id": "u019", "kind": "prose", "locator": "body:L42-L42", "preview": "PTX ISA 9.0 provides the packed conversion form:", "sha256": "0895c7933390c018b85b8fbb5f7a635b182476c0daf63b31053aa71a507bba62"}, {"id": "u020", "kind": "code", "locator": "body:L44-L46", "preview": "```ptx cvt.rn.f16x2.e2m1x2 d, a; ```", "sha256": "b23b367d52866372d72acbf6316fbedf194a07a40533912e462aa9f5d2989ef1"}, {"id": "u021", "kind": "prose", "locator": "body:L48-L48", "preview": "Here `a` is a byte-sized packed pair and `d` is a 32-bit `f16x2` result. PTX also permits `mov.b32` to unpack a 32-bit scalar into four byte-sized vector destinations when the declarations satisfy its type rules. Neither syntax establishes ", "sha256": "ebaaf257c26310449c24b83cf719888507d42b14702ca8571b844efa027b0539"}, {"id": "u022", "kind": "prose", "locator": "body:L52-L52", "preview": "There is no architecture-independent \"4x versus Hopper\" result for these instructions. Hopper has no native FP4 tensor-core path, so any comparison depends on the emulation or higher-precision baseline as well as GPU SKU, clocks, matrix sha", "sha256": "b45cdd32e45e94aae2f29d34b8bb9b28b6552f456b68f0f10720fb64169535b4"}, {"id": "u023", "kind": "list-item", "locator": "body:L56-L56", "preview": "- [Transformer Engine 2.13: NVFP4](https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html)", "sha256": "1ea38bb80e14795104811d1c91bf01eb96b18d0da47a64fc1f42f3ecf2ca933d"}, {"id": "u024", "kind": "list-item", "locator": "body:L57-L57", "preview": "- [PTX ISA 9.0: `tcgen05.mma`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma)", "sha256": "6ffc658c6eca84cb0a0b9011dd5975398802fb382416b751faf4868e97da0dc2"}, {"id": "u025", "kind": "list-item", "locator": "body:L58-L58", "preview": "- [PTX ISA 9.0: `cvt`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt)", "sha256": "64807f0b841d1fd5a38186fcf40e8f14fc61c912cd4e76a0710ba4ec427255fd"}, {"id": "u026", "kind": "list-item", "locator": "body:L59-L59", "preview": "- [Fine-grained quantization](../techniques/fine-grained-quantization.md)", "sha256": "9a377188c50b366c56091d8ff769167bc2ad34ee4821cf9d3aef73e824912f24"}, {"id": "u027", "kind": "list-item", "locator": "body:L60-L60", "preview": "- [NVFP4 GEMM](../kernels/nvfp4-gemm.md)", "sha256": "afe5e93ec48a77859c8b1186b661f64956be6135e45e31d60ccadce597bab2b0"}, {"id": "u028", "kind": "list-item", "locator": "body:L61-L61", "preview": "- [NVFP4 GEMV](../kernels/nvfp4-gemv.md)", "sha256": "d0e8ffc886b46a7311e65cd74d494d992955690fc698cf037a73a56513df887f"}], "confidence_claimed": "verified", "headings": ["Format and recipe", "NVFP4 and MXFP4", "PTX block-scaled MMA", "Performance claims", "References"], "id": "hw-nvfp4", "path": "wiki/hardware/nvfp4.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-transformer-engine-2.13-nvfp4.md", "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}], "risk_flags": ["code", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-transformer-engine-2.13-nvfp4", "doc-ptx-isa-sm100"], "title": "NVFP4 and Block-Scaled Narrow Precision", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "0b946b34a4845a0b6c3bb801db3b992a4e8b6a858cb8b98fb47ebfd4239ba65f", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "NVFP4 is a quantization recipe, not just another name for its 4-bit payload. It reconstructs a value as:", "sha256": "227c70eceb96556659b2b5f6cf54e400e6245360f15905a84705c09b89a48113"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "`x_hat_i = e2m1_i * s_block * s_global`", "sha256": "794f7462514befdb8ac4a28522080782143495f8fd3157b5bea8c46d09934c6d"}, {"id": "u003", "kind": "prose", "locator": "body:L9-L9", "preview": "The three components are:", "sha256": "890b0b87819a86372f7ddcee49a11fd621e9a3a29c35ac7b8d1d2c2ff00d5c3e"}, {"id": "u004", "kind": "list-item", "locator": "body:L11-L11", "preview": "- **E2M1 payload:** one sign bit, two exponent bits, and one mantissa bit. Its numeric values are 0, +/-0.5, +/-1, +/-1.5, +/-2, +/-3, +/-4, and +/-6. Two E2M1 encodings fit in one byte.", "sha256": "c7e17ce8c55c39e0403be620d6ad757c8fa414b998dddfbdae211367d3c095e8"}, {"id": "u005", "kind": "list-item", "locator": "body:L12-L12", "preview": "- **Local scale:** one E4M3 scale shared by 16 consecutive payload values. PTX names the corresponding unsigned scale element type `.ue4m3`; its most-significant storage bit is zero.", "sha256": "5413cc4adc0cc112cd51d22ac07e5fcaae3f1e245164a937f4ba9333ca907655"}, {"id": "u006", "kind": "list-item", "locator": "body:L13-L13", "preview": "- **Global scale:** one FP32 scale per tensor. This second level lets the block scales use their range effectively.", "sha256": "12ca2c5eaaec7645bc831438892492d0c78328874b654e1ab71c39af7a429539"}, {"id": "u007", "kind": "prose", "locator": "body:L15-L15", "preview": "Transformer Engine 2.13 also defines a 2D scaling mode for weights in which a scale covers a 16-by-16 block. The 1D mode groups 16 consecutive values.", "sha256": "7288001414a30fe333b049e16ae69a4d1522ec4dc11bcaa88d3dabe2c03ba5b9"}, {"id": "u008", "kind": "prose", "locator": "body:L19-L19", "preview": "The recipes make different scale tradeoffs:", "sha256": "c9981094018252198ffe5ef16f090c60cb5483b13ea9d3f69e44906b9fd07b71"}, {"id": "u009", "kind": "table-row", "locator": "body:L23-L23", "preview": "| Property | NVFP4 | MXFP4 | | Payload | E2M1 | E2M1 |", "sha256": "a519973746ef18833e34e940839dffaf10f468db0b157561b380739125ce7fe4"}, {"id": "u010", "kind": "table-row", "locator": "body:L24-L24", "preview": "| Property | NVFP4 | MXFP4 | | Local group in the standard 1D recipe | 16 values | 32 values |", "sha256": "079a628d1c073e9d4c716174b141262c2e1f1fcedc21caca1a3f3693ac5c462b"}, {"id": "u011", "kind": "table-row", "locator": "body:L25-L25", "preview": "| Property | NVFP4 | MXFP4 | | Local scale | E4M3 (`.ue4m3` in PTX) | UE8M0 |", "sha256": "eb805ec4a81f255a4a58972a4e406037092fb53afe554d6d542b2851c232d419"}, {"id": "u012", "kind": "table-row", "locator": "body:L26-L26", "preview": "| Property | NVFP4 | MXFP4 | | Scale values | Fractional values are available | Powers of two |", "sha256": "64d0f1c0890f5f697993223f04daa88924b00162be84608850df6ea6fa8e4e93"}, {"id": "u013", "kind": "table-row", "locator": "body:L27-L27", "preview": "| Property | NVFP4 | MXFP4 | | Additional recipe scale | Per-tensor FP32 | Not part of the MXFP4 microscaling format |", "sha256": "660006f85f6632d68dcdeecff2dbe5a683dd5ab53e15f3d3c0eb5ea536906ea9"}, {"id": "u014", "kind": "prose", "locator": "body:L29-L29", "preview": "Finer groups and fractional local scales give NVFP4 more scaling freedom, at the cost of more scale metadata. They do not guarantee strictly lower error for every tensor, scale-selection algorithm, or error metric. Measure accuracy with the", "sha256": "18f8ae6f474472f1c2ea4ff9a3e2b597c1c36943095c02353db7eb28e098ed64"}, {"id": "u015", "kind": "prose", "locator": "body:L33-L33", "preview": "In PTX ISA 9.0, the relevant `tcgen05.mma` combinations for E2M1 inputs include:", "sha256": "4c6c2bfd4f02969ff8470c02bf64bbc802e646de47c9b11f4df253c5df3501fc"}, {"id": "u016", "kind": "table-row", "locator": "body:L37-L37", "preview": "| Instruction qualifiers | Block | Local scale | Scale-vector qualifier | | `.kind::mxf4.block_scale.block32` | 32 | `.ue8m0` | `.scale_vec::2X` |", "sha256": "c8d1878c1a0cadb7d3ca358add3c678ab21399a44da2e3d25f4c36ceebde2082"}, {"id": "u017", "kind": "table-row", "locator": "body:L38-L38", "preview": "| Instruction qualifiers | Block | Local scale | Scale-vector qualifier | | `.kind::mxf4nvf4.block_scale.block16` | 16 | `.ue4m3` | `.scale_vec::4X` |", "sha256": "4bd9801c764ad925b43b088977e249e1499ac6e62b0ef485dd544412fe0874f3"}, {"id": "u018", "kind": "prose", "locator": "body:L40-L40", "preview": "The `.kind::mxf4nvf4` family also admits documented UE8M0 modes, so the kind name by itself does not select the NVFP4 recipe. Use the complete block and scale-vector qualifiers and follow the type-combination tables. These forms target arch", "sha256": "c48d3a2f58dfe90849b6ee8abacd39ebf0f30cc5c6835517ffe7997d411dd227"}, {"id": "u019", "kind": "prose", "locator": "body:L42-L42", "preview": "PTX ISA 9.0 provides the packed conversion form:", "sha256": "0895c7933390c018b85b8fbb5f7a635b182476c0daf63b31053aa71a507bba62"}, {"id": "u020", "kind": "code", "locator": "body:L44-L46", "preview": "```ptx cvt.rn.f16x2.e2m1x2 d, a; ```", "sha256": "b23b367d52866372d72acbf6316fbedf194a07a40533912e462aa9f5d2989ef1"}, {"id": "u021", "kind": "prose", "locator": "body:L48-L48", "preview": "Here `a` is a byte-sized packed pair and `d` is a 32-bit `f16x2` result. PTX also permits `mov.b32` to unpack a 32-bit scalar into four byte-sized vector destinations when the declarations satisfy its type rules. Neither syntax establishes ", "sha256": "ebaaf257c26310449c24b83cf719888507d42b14702ca8571b844efa027b0539"}, {"id": "u022", "kind": "prose", "locator": "body:L52-L52", "preview": "There is no architecture-independent \"4x versus Hopper\" result for these instructions. Hopper has no native FP4 tensor-core path, so any comparison depends on the emulation or higher-precision baseline as well as GPU SKU, clocks, matrix sha", "sha256": "b45cdd32e45e94aae2f29d34b8bb9b28b6552f456b68f0f10720fb64169535b4"}, {"id": "u023", "kind": "list-item", "locator": "body:L56-L56", "preview": "- [Transformer Engine 2.13: NVFP4](https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html)", "sha256": "1ea38bb80e14795104811d1c91bf01eb96b18d0da47a64fc1f42f3ecf2ca933d"}, {"id": "u024", "kind": "list-item", "locator": "body:L57-L57", "preview": "- [PTX ISA 9.0: `tcgen05.mma`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma)", "sha256": "cd7b812d2929140658187af5b3ff463866f434fd6a4c04871300be80fa50d967"}, {"id": "u025", "kind": "list-item", "locator": "body:L58-L58", "preview": "- [PTX ISA 9.0: `cvt`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt)", "sha256": "64807f0b841d1fd5a38186fcf40e8f14fc61c912cd4e76a0710ba4ec427255fd"}, {"id": "u026", "kind": "list-item", "locator": "body:L59-L59", "preview": "- [Fine-grained quantization](../techniques/fine-grained-quantization.md)", "sha256": "9a377188c50b366c56091d8ff769167bc2ad34ee4821cf9d3aef73e824912f24"}, {"id": "u027", "kind": "list-item", "locator": "body:L60-L60", "preview": "- [NVFP4 GEMM](../kernels/nvfp4-gemm.md)", "sha256": "afe5e93ec48a77859c8b1186b661f64956be6135e45e31d60ccadce597bab2b0"}, {"id": "u028", "kind": "list-item", "locator": "body:L61-L61", "preview": "- [NVFP4 GEMV](../kernels/nvfp4-gemv.md)", "sha256": "d0e8ffc886b46a7311e65cd74d494d992955690fc698cf037a73a56513df887f"}], "confidence_claimed": "verified", "headings": ["Format and recipe", "NVFP4 and MXFP4", "PTX block-scaled MMA", "Performance claims", "References"], "id": "hw-nvfp4", "path": "wiki/hardware/nvfp4.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-transformer-engine-2.13-nvfp4.md", "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}], "risk_flags": ["code", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-transformer-engine-2.13-nvfp4", "doc-ptx-isa-sm100"], "title": "NVFP4 and Block-Scaled Narrow Precision", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} {"body_sha256": "4c2bc154856d1dcfd15bb4f6a12ae30d4dadc52ead1d4e5994b4da45e031cd98", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Programmatic Dependent Launch (PDL) lets a secondary grid in the same CUDA stream become eligible to start before its prerequisite primary grid completes. It is available on compute capability 9.0 and later, including Hopper and Blackwell.", "sha256": "b2a6132457b1395799e2baeb71d8b78a2aefada9ebcdb45692fe7c25aaeb6823"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "PDL creates an **opportunity** for overlap; it does not guarantee that the grids execute concurrently. The useful overlap is normally the primary's work after its launch trigger and the secondary's independent preamble before its dependency", "sha256": "3fadb352705867bcecdab210311e7fef978fb6c4d548428135a83c2cc9a4a59e"}, {"id": "u003", "kind": "prose", "locator": "body:L9-L9", "preview": "The CUDA protocol has three required roles:", "sha256": "f06a64f5123e88dbd07114031cac5afabaeaa52abc8ecae0037804ec2d46ba9a"}, {"id": "u004", "kind": "list-item", "locator": "body:L11-L11", "preview": "1. Every primary CTA either executes `cudaTriggerProgrammaticLaunchCompletion()` or exits. After all CTAs satisfy that condition, the driver may schedule the secondary grid.", "sha256": "cdebb33e8456cbe33475d6ccabe75f697ff9a33db49579b708d27a888866ee78"}, {"id": "u005", "kind": "list-item", "locator": "body:L12-L12", "preview": "2. The host launches the secondary in the same stream with `cudaLaunchAttributeProgrammaticStreamSerialization` and `programmaticStreamSerializationAllowed = 1`.", "sha256": "5fe3d12bfbd54ebbeac70c19b7d88175202f6c7dee699f8bda30383167d2a2df"}, {"id": "u006", "kind": "list-item", "locator": "body:L13-L13", "preview": "3. Every secondary thread waits with `cudaGridDependencySynchronize()` before it consumes prerequisite results. The wait completes after the prerequisite grids finish and their memory operations are visible.", "sha256": "097e01d65d04ea53bb13e7e51ab176c852e4ade1679b09d0361bc5db4ab61651"}, {"id": "u007", "kind": "prose", "locator": "body:L15-L15", "preview": "If the primary does not explicitly trigger, its CTAs implicitly satisfy the trigger only as they exit. Omitting the explicit trigger is correct but removes the intended primary-tail overlap.", "sha256": "bfc1bc44195a8cde7f45936b64ed7d6aa9339ddddfada2ad1b2116386640a488"}, {"id": "u008", "kind": "prose", "locator": "body:L19-L19", "preview": "This minimal skeleton shows where each operation belongs; real kernels put independent and dependent work at the indicated points and add normal error checking:", "sha256": "3370f3719697733fed41083390007ea5145cd8c1b8282d838bd01adfb718bf3b"}, {"id": "u009", "kind": "code", "locator": "body:L21-L53", "preview": "```cuda #include __global__ void primary() { // Produce everything needed before the secondary may be launched. if (threadIdx.x == 0) { cudaTriggerProgrammaticLaunchCompletion(); } // Primary tail that does not change data ", "sha256": "eead129a867adbd5fb1df86742d091900d3f862173e7beba21c3690a3860a7e3"}, {"id": "u010", "kind": "prose", "locator": "body:L55-L55", "preview": "Do not place `cudaGridDependencySynchronize()` in the primary: it is the secondary-side wait, not the launch trigger. Do not replace the wait with an ordinary memory fence. PDL's wait supplies both prerequisite-grid completion and visibilit", "sha256": "e79291426c2535ad3f17bf2a11bf664cbe62f5ccc989b134980cb4688c2d116f"}, {"id": "u011", "kind": "prose", "locator": "body:L59-L59", "preview": "CUDA lowers the two device roles to Grid Dependency Control instructions:", "sha256": "b250259e26671cdcac831c563720d242a666b0ed7a9b6afd3bc3d07a4a29075e"}, {"id": "u012", "kind": "code", "locator": "body:L61-L68", "preview": "```ptx // Primary CTA: makes designated dependent grids eligible after every CTA // has issued this instruction or completed. griddepcontrol.launch_dependents; // Secondary thread: waits for in-flight prerequisite grids and visibility. grid", "sha256": "9a67c56925d56525f39e506610fa57ffe5f3d4cb35db94ae4b010c90bc37cc98"}, {"id": "u013", "kind": "prose", "locator": "body:L70-L70", "preview": "`griddepcontrol` was introduced in PTX ISA 7.8 and requires `sm_90` or newer. Repeating `launch_dependents` within one CTA has no additional effect after that CTA's first invocation. If a prerequisite uses `launch_dependents`, its dependent", "sha256": "11123906a586c5505184a7cdc1d00fc51809ba90d7022a7722448cdeb72d9f02"}, {"id": "u014", "kind": "prose", "locator": "body:L74-L74", "preview": "Blackwell does not make arbitrary back-to-back launches overlap automatically. CUDA applications still opt the secondary launch into programmatic stream serialization and implement the device-side trigger/wait protocol.", "sha256": "ccfc3e2c2f99820ecb72aa491631c59b39d47de9f75b8a957e277d6a99d7fa86"}, {"id": "u015", "kind": "prose", "locator": "body:L76-L76", "preview": "CUTLASS has a separate build choice. In CUTLASS 4.5.0, the CMake option `CUTLASS_ENABLE_GDC_FOR_SM100` defaults to `ON`, while the SM90 option is opt-in. That default only enables eligible CUTLASS code to emit its GDC wrappers; it is not a ", "sha256": "8eb6cc4790bdeac39a848471ff2ce472fad33e190c16c073942ece6b3446f6d8"}, {"id": "u016", "kind": "prose", "locator": "body:L80-L80", "preview": "PDL can help only when all of these conditions hold:", "sha256": "b2f3b22d9295e43ef95becd759c3f639e71bc43ce4f72542271211cd62079f54"}, {"id": "u017", "kind": "list-item", "locator": "body:L82-L82", "preview": "- the secondary has enough independent preamble to overlap;", "sha256": "eba59af65aaa7d730d01e0cc16bbd138f86c11888c7eb6c6b3f24a3c18abb67e"}, {"id": "u018", "kind": "list-item", "locator": "body:L83-L83", "preview": "- the primary has useful tail work after every CTA reaches the trigger;", "sha256": "275a77efa3bd46865530e8b4d92dbdd17cadc14417e31d68a8f1164ce5f46db4"}, {"id": "u019", "kind": "list-item", "locator": "body:L84-L84", "preview": "- the two grids have enough simultaneous resource headroom; and", "sha256": "f7874709a96e779af80196a86caff96361b4b30864ef2d3989d1c0e3352a02b4"}, {"id": "u020", "kind": "list-item", "locator": "body:L85-L85", "preview": "- the saved launch/serialization time exceeds the protocol and occupancy costs.", "sha256": "8b99cc7e62635157ccc3a04c9e04da4db13139ca6e15882d68b73abe87cbf385"}, {"id": "u021", "kind": "prose", "locator": "body:L87-L87", "preview": "Small-kernel chains, GEMM/epilogue sequences, and pipeline-parallel stages are candidates, not guaranteed wins. Profile an explicit non-PDL baseline and record GPU, clocks, launch shapes, stream/graph configuration, input sizes, warmup, rep", "sha256": "3d11e833065d2f0681ff30760c6053df3d5cab7fae71318b61e568b66057ea7c"}, {"id": "u022", "kind": "list-item", "locator": "body:L91-L91", "preview": "- [CUDA 13.0.2: Programmatic Dependent Launch](https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#programmatic-dependent-launch-and-synchronization)", "sha256": "d95502e4bc1f33df6ee25c6794b4a39e4012106894161a3ec102f5c04a9c45c2"}, {"id": "u023", "kind": "list-item", "locator": "body:L92-L92", "preview": "- [PTX ISA 9.0: `griddepcontrol`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-griddepcontrol)", "sha256": "59689f46851737c4064c9836e6c4570e18b5a470e70da05895569b59e8c5352f"}, {"id": "u024", "kind": "list-item", "locator": "body:L93-L93", "preview": "- [CUTLASS 4.5.0: dependent kernel launch](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/dependent_kernel_launch.md)", "sha256": "f1fbe66725cda7570cb0f38eceab85beae9d511d843e6badf86e7e749c693871"}, {"id": "u025", "kind": "list-item", "locator": "body:L94-L94", "preview": "- [Persistent kernels](../techniques/persistent-kernels.md)", "sha256": "ec9585a9787c7d4334a41d77c6ecbeee6575e90c64bb37f1f2e8c8b8d64b041d"}, {"id": "u026", "kind": "list-item", "locator": "body:L95-L95", "preview": "- [Cluster Launch Control](clc.md)", "sha256": "7c35bead8298662eaa2f80624f180adf2e122e6ac111dcf05aa0f5ac9a5653b1"}], "confidence_claimed": "verified", "headings": ["Contract", "CUDA role skeleton", "PTX mapping", "CUDA support versus CUTLASS defaults", "When to use it", "References"], "id": "hw-pdl-gdc", "path": "wiki/hardware/pdl-gdc.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-cuda-13.md", "url": "https://developer.nvidia.com/blog/whats-new-and-important-in-cuda-toolkit-13-0/"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/prs/cutlass/PR-2161.md", "revision": "06e560d9", "url": "https://github.com/NVIDIA/cutlass/pull/2161"}], "risk_flags": ["code", "ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-cuda-13", "doc-ptx-isa-sm100", "pr-cutlass-2161"], "title": "Programmatic Dependent Launch / Grid Dependency Control", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} {"body_sha256": "4dcc5ab666a88a95ba179a141027d5f8cb80ccbe6bae13bed1e745e2c2bd7b61", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "`tcgen05.mma` is NVIDIA PTX's fifth-generation TensorCore matrix-multiply-accumulate family. PTX ISA 9.0 supports its architecture-specific forms on `sm_100a`; CUTLASS uses the `UMMA` namespace for its SM100 wrappers. NVIDIA's pinned PTX an", "sha256": "5a30d3e62baf8f9e258d9543f2e7952f16a33644f324b4ab76b8652c9aeaa047"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "The programming-model shift from Hopper WGMMA is precise:", "sha256": "79a30a17b7a41df9858ff9cea9a7ad07ccc0402bba6e70e49699d048f8c82d22"}, {"id": "u003", "kind": "table-row", "locator": "body:L11-L11", "preview": "| Property | WGMMA | tcgen05.mma | | Issue granularity | Warpgroup | One thread for `cta_group::1` or `cta_group::2` |", "sha256": "cb2dd4481890789881169b854b62ab6c892eeba718691b0c2512592192d5fd27"}, {"id": "u004", "kind": "table-row", "locator": "body:L12-L12", "preview": "| Property | WGMMA | tcgen05.mma | | D accumulator | Per-thread registers | Tensor Memory (TMEM) |", "sha256": "5e4f9ace6cdb6311c5b99b47b2ea7a6663212f2ca7c69100d7af30d5428e8c12"}, {"id": "u005", "kind": "table-row", "locator": "body:L13-L13", "preview": "| Property | WGMMA | tcgen05.mma | | A location | Register or SMEM forms, depending on instruction | SMEM descriptor or TMEM address |", "sha256": "147c40c5a19ea02949386e003d7f89e2b7fabcba715fb9517e664f30260fe665"}, {"id": "u006", "kind": "table-row", "locator": "body:L14-L14", "preview": "| Property | WGMMA | tcgen05.mma | | B location | SMEM descriptor | SMEM descriptor |", "sha256": "9df1b553933b5276fc6098da41c4f388e190439f6722b2af979ebcf994110cbc"}, {"id": "u007", "kind": "table-row", "locator": "body:L15-L15", "preview": "| Property | WGMMA | tcgen05.mma | | Completion | WGMMA commit/wait groups | `tcgen05.commit` plus an mbarrier wait |", "sha256": "edd69cb02b0effe4c4d1e3b287cc8974c4e49b9883132a111ee81ac18c8d1385"}, {"id": "u008", "kind": "prose", "locator": "body:L17-L17", "preview": "Single-thread issue reduces the number of issuing threads, but it does not remove asynchronous completion, operand-lifetime, or inter-thread ordering requirements.", "sha256": "dd431728d3ea5a55526e6920039601fb733dcb5a2d1275c0c32c0f18e5ec3c6a"}, {"id": "u009", "kind": "prose", "locator": "body:L21-L21", "preview": "PTX ISA 9.0 divides dense `tcgen05.mma` into these grammar groups:", "sha256": "59dabf6dfb4c85e155ad4045ec8784b5e4a8ce7c2728efcaca602880f1aee4f9"}, {"id": "u010", "kind": "table-row", "locator": "body:L25-L25", "preview": "| Grammar group | Kind qualifiers | | Floating point, without block scaling | `f16`, `tf32`, `f8f6f4` |", "sha256": "c6edd6d05f9116fa0934c8bf39ebf349a48aa0fe6ceb5d6069603b5b370dd82a"}, {"id": "u011", "kind": "table-row", "locator": "body:L26-L26", "preview": "| Grammar group | Kind qualifiers | | Floating point, with block scaling | `mxf8f6f4`, `mxf4`, `mxf4nvf4` |", "sha256": "7524aa5d74cb23cae509d1f62790cd9b4097cfd670f1f71d7a566cf01044109d"}, {"id": "u012", "kind": "table-row", "locator": "body:L27-L27", "preview": "| Grammar group | Kind qualifiers | | Integer | `i8` |", "sha256": "d00dc8f6a441056e180aa0b7abc129f19832cb9ff64963c701f818e5dab0025e"}, {"id": "u013", "kind": "prose", "locator": "body:L29-L29", "preview": "Block-scaled forms add `.block_scale` and accept separate TMEM addresses for A and B scale factors. `f8f6f4` is not the block-scaled MX kind, and `mxf8` is not a dense kind token in this grammar.", "sha256": "6301afa139350f62dc9bf75d85dfa67de713cc20a501dfc475ed622d4a73d6b4"}, {"id": "u014", "kind": "prose", "locator": "body:L31-L31", "preview": "M and N are encoded in the instruction descriptor (`idesc`) and are constrained by kind, CTA group, layouts, and target ISA. Consequently, names such as m128n256k16 and m256n256k16 are useful maximum-shape examples for F16/BF16 configuratio", "sha256": "2c714ca4e2add6030e2ffd3a982d42e1928c97e656e998af2d554e29bb7c7052"}, {"id": "u015", "kind": "prose", "locator": "body:L35-L35", "preview": "The unscaled floating-point grammar includes a disable-output-lane vector before the accumulation predicate:", "sha256": "0c3d6d381d9ddfc6470a447bf5c7209a114dfe79cca59536d5329b306df0caeb"}, {"id": "u016", "kind": "code", "locator": "body:L37-L41", "preview": "```ptx tcgen05.mma.cta_group.kind [d-tmem], a-desc, b-desc, idesc, { disable-output-lane }, enable-input-d {, scale-input-d}; ```", "sha256": "2ed61a81ad137911dae0bc5b5d1f4cf410d79d32deb51059f1da36185bc35527"}, {"id": "u017", "kind": "prose", "locator": "body:L43-L43", "preview": "For `cta_group::1`, the lane-disable vector contains four 32-bit values; for `cta_group::2`, it contains eight. Setting `enable-input-d` false computes `D = A*B`; setting it true computes `D = A*B+D`. The optional `scale-input-d` is limited", "sha256": "b9fb63cf6d552bb548afef70443d9c4a0b4bcc76a419792cfffbcd851b60e038"}, {"id": "u018", "kind": "prose", "locator": "body:L45-L45", "preview": "The block-scaled grammar is structurally different:", "sha256": "c89976d470fdda264de2b4c13671d036c8da1bc84c424adf03ecfdb9ec956213"}, {"id": "u019", "kind": "code", "locator": "body:L47-L51", "preview": "```ptx tcgen05.mma.cta_group.kind.block_scale.scale_vectorsize [d-tmem], a-desc, b-desc, idesc, [scale-A-tmem], [scale-B-tmem], enable-input-d; ```", "sha256": "8e585731217607fbd7145d665ac7790e9f8610c4630fec9b63f1e613bc675ae8"}, {"id": "u020", "kind": "prose", "locator": "body:L53-L53", "preview": "These are normative grammar excerpts, not complete inline-assembly functions: real code must also build valid descriptors, allocate TMEM, preserve operand lifetimes, and implement completion.", "sha256": "c026a8a36bf3c27d1e949b295720a13a93ac89b99bf0e5d297e1efa0f274677d"}, {"id": "u021", "kind": "prose", "locator": "body:L57-L57", "preview": "`tcgen05.mma` is asynchronous. Two mechanisms serve different purposes:", "sha256": "0eb63c199bd2648a1a838b2d3a47ac38561ae64dd5a5aa4a389176a4bba0a4b9"}, {"id": "u022", "kind": "list-item", "locator": "body:L59-L59", "preview": "1. `tcgen05.commit.cta_group::N.mbarrier::arrive::one.b64` makes an mbarrier track completion of prior asynchronous tcgen05 operations issued by the executing thread. Waiting on that mbarrier observes completion.", "sha256": "b94843121138c19fe4f9c6b5f2e4180325b4b07209b2b16d53801684eb26feab"}, {"id": "u023", "kind": "list-item", "locator": "body:L60-L60", "preview": "2. `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` order tcgen05 operations across an execution-ordering handoff and constrain code motion. A fence is not a completion wait.", "sha256": "c0f4ab169700e7114ac3cfaf535bfcc96235d55a67b70ab445eeae606728c5c2"}, {"id": "u024", "kind": "prose", "locator": "body:L62-L62", "preview": "A cross-thread result handoff therefore needs both the applicable completion protocol and the applicable fence/execution-ordering protocol. Likewise, a pipelined mainloop must not release or overwrite an SMEM stage until the asynchronous MM", "sha256": "adf70d7f3800a733444d846a2e3c015cbd24273cb2574a36d0f5998aff0d4b87"}, {"id": "u025", "kind": "prose", "locator": "body:L66-L66", "preview": "The tcgen05 shared-memory descriptor is a 64-bit runtime value. PTX ISA 9.0 assigns fields for the encoded base address, leading dimension, stride dimension, fixed bits, base offset, leading-dimension mode, and a three-bit swizzle mode.", "sha256": "cd4b8ef944a646278af6e6a6ed009a7641447384c06e392a47867c99b4214e9c"}, {"id": "u026", "kind": "prose", "locator": "body:L68-L68", "preview": "Valid swizzle encodings include no swizzle, 128-byte, 64-byte, and 32-byte layouts (plus a 128-byte/32-byte-atomic mode). Values 3, 5, and 7 are invalid; ordinary 128-byte swizzling uses value 2. A layout must satisfy the addressing and ali", "sha256": "ffd078c072d1f53a4004b9a5bfd9958461011d22405ec4ef1724cc5f8fa0a464"}, {"id": "u027", "kind": "prose", "locator": "body:L72-L72", "preview": "For comparison, CUTLASS's pinned Hopper wrapper for m64n256k16 with FP32 accumulation declares 128 accumulator registers per participating thread. tcgen05 keeps D in TMEM during the MMA sequence, which can free GPR capacity for data movemen", "sha256": "aea6db1f85746818e4767b9d406b4e771290c0e462d3cabf587264e302613aef"}, {"id": "u028", "kind": "prose", "locator": "body:L76-L76", "preview": "Gau Nernst reports the following results for M=N=K=4096 on a Modal B200, using PyTorch 2.9.1 with CUDA 13. Values are measurements for that setup, not architecture-wide guarantees.", "sha256": "13f1217bf7f22fcc79830356b02e702206bb4a00151c16924639cf6d2bb21a99"}, {"id": "u029", "kind": "table-row", "locator": "body:L80-L80", "preview": "| Tutorial version | Reported TFLOP/s | Approx. cuBLAS share | | cuBLAS | 1506.74 | 100% |", "sha256": "ac004f48b31dd6f7024996d3995ded82f823d87f3eff7413f55ff3e6acc116b4"}, {"id": "u030", "kind": "table-row", "locator": "body:L81-L81", "preview": "| Tutorial version | Reported TFLOP/s | Approx. cuBLAS share | | v1a: basic tcgen05 + 2D 16B TMA | 254.62 | 17% |", "sha256": "81e39acb68544a7b7f6402f743738b5e37e51cd2b474f9cbfdb331689fdfd3f4"}, {"id": "u031", "kind": "table-row", "locator": "body:L82-L82", "preview": "| Tutorial version | Reported TFLOP/s | Approx. cuBLAS share | | v1b: 3D 16B TMA | 252.81 | 17% |", "sha256": "26b2647c4773366a5f1d00f84486e9190c5ce010b5958463cc44d7d8335336b6"}, {"id": "u032", "kind": "table-row", "locator": "body:L83-L83", "preview": "| Tutorial version | Reported TFLOP/s | Approx. cuBLAS share | | v2a: 2D 128B TMA | 681.20 | 45% |", "sha256": "589a9a37b7cf4601f48e74eed00ff7f502e5754d3003b46ad18b82471359782c"}, {"id": "u033", "kind": "table-row", "locator": "body:L84-L84", "preview": "| Tutorial version | Reported TFLOP/s | Approx. cuBLAS share | | v2b: 3D 128B TMA | 695.43 | 46% |", "sha256": "e70b55975fd80df16e31a7ae4e4699c6dc4a32c1f1579136407c89a4385d9c75"}, {"id": "u034", "kind": "table-row", "locator": "body:L85-L85", "preview": "| Tutorial version | Reported TFLOP/s | Approx. cuBLAS share | | v3: pipelining | 939.61 | 62% |", "sha256": "cb276d4db269baa1b1d091ffd693f81215a14e11e87bca3bbb7b4e737d267150"}, {"id": "u035", "kind": "table-row", "locator": "body:L86-L86", "preview": "| Tutorial version | Reported TFLOP/s | Approx. cuBLAS share | | v4: warp specialization | 1208.83 | 80% |", "sha256": "85b80f84546f9b39fe250172f16144299c9d09f56364b578a12423b78e221f9f"}, {"id": "u036", "kind": "table-row", "locator": "body:L87-L87", "preview": "| Tutorial version | Reported TFLOP/s | Approx. cuBLAS share | | v5: 2-SM MMA | 1302.29 | 86% |", "sha256": "7b18e80a7aa94f919322812f64ce759b9517b58ca7efba7933732a0f9aa0f531"}, {"id": "u037", "kind": "table-row", "locator": "body:L88-L88", "preview": "| Tutorial version | Reported TFLOP/s | Approx. cuBLAS share | | v6: persistent, static scheduling | 1475.93 | 98% |", "sha256": "dc244caa4f60fc59b1d9e62d010731e5e119b04331695c37ec54a5d5660f60d2"}, {"id": "u038", "kind": "prose", "locator": "body:L90-L90", "preview": "The v6 result did not use Cluster Launch Control; the author lists CLC and threadblock swizzling as unimplemented follow-up ideas.", "sha256": "58b7618727532d91a2095babb6482b7a126310da1fbdfaf4bab8776a2cefabc9"}], "confidence_claimed": "verified", "headings": ["Overview", "Kinds and shapes", "Operand grammar", "Completion and inter-thread ordering", "Shared-memory descriptors", "Register pressure and specialization", "Source-reported performance progression"], "id": "hw-tcgen05-mma", "path": "wiki/hardware/tcgen05-mma.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/prs/cutlass/PR-2139.md", "revision": "ca4fdbea", "url": "https://github.com/NVIDIA/cutlass/pull/2139"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-ptx-isa-sm100", "pr-cutlass-2139", "blog-tcgen05-tutorial"], "title": "tcgen05.mma \u2014 Blackwell MMA Instruction", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} {"body_sha256": "1eb48a33cd77ce277ac4783d357f5784e487533159bbe155fc942f404f3655f1", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "TMA is the descriptor-driven `cp.async.bulk.tensor` facility introduced for Hopper (`sm_90`) and retained on Blackwell. One thread can issue a non-blocking rank-1 through rank-5 tensor copy while the hardware performs multidimensional addre", "sha256": "942ee8eaa2c8a1a7a630eb77b351844e980ef1e36f0e549a8f0fa35689f227df"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "The principal copy directions and completion mechanisms are:", "sha256": "cd258dbf1c80e8b3bf04db3d6601ed09723c72ff7ce612a7202994b8369b02d5"}, {"id": "u003", "kind": "table-row", "locator": "body:L11-L11", "preview": "| Direction | Destination | Optional cluster behavior | Completion | | Global to shared | Issuing CTA or a CTA in its cluster | One masked instruction can multicast to selected CTAs | mbarrier `complete_tx` in bytes |", "sha256": "0d062928bfbbdbd16737f4478e912c51a16020e856647600f0273112a61ce977"}, {"id": "u004", "kind": "table-row", "locator": "body:L12-L12", "preview": "| Direction | Destination | Optional cluster behavior | Completion | | Shared to global | Issuing CTA's shared memory to a tensor map | Scatter modes are available on supported Blackwell targets | Bulk async group: issue, commit, wait |", "sha256": "be8df0a0b92e6d46dc931918bc193cda48ba5a02cdbf7655cef986ad552fc1cb"}, {"id": "u005", "kind": "prose", "locator": "body:L14-L14", "preview": "TMA copies the datatype represented by the tensor map. It does not provide a general FP32-to/from-FP16 or BF16 conversion step. For a tiled global-to-shared load, out-of-bounds elements are filled according to the supported tensor-map polic", "sha256": "2b7f99033261e23d1b85e977330495009b7ae63c585efc4d6b0d5e5ac665f399"}, {"id": "u006", "kind": "prose", "locator": "body:L18-L18", "preview": "A tensor map is opaque and is accessed through the tensor-map proxy. `cuTensorMapEncodeTiled` describes:", "sha256": "ccc2dedb38f5f9072f35ff9cdf790aa1ba6305d726a0341b5184080c60ad76a5"}, {"id": "u007", "kind": "list-item", "locator": "body:L20-L20", "preview": "- datatype and global base address;", "sha256": "bdedfc644a6b74912840d80daf76b5edf4ab3f9fffcdd594efbc530d95a2b81c"}, {"id": "u008", "kind": "list-item", "locator": "body:L21-L21", "preview": "- rank, global dimensions, and byte strides;", "sha256": "9c9a24117ca440fe4a6d6090ce809391c3f8b5ee855a73097640a3f05af0de4f"}, {"id": "u009", "kind": "list-item", "locator": "body:L22-L22", "preview": "- traversal box dimensions and element strides;", "sha256": "9adc19c95d2386a5160b3a3f1551bf6fc65d1e41afd43e0c4d386527ba55af1e"}, {"id": "u010", "kind": "list-item", "locator": "body:L23-L23", "preview": "- interleave and shared-memory swizzle;", "sha256": "bea5f5d23bd9392029b749d89c1d641bb241b07d4eb68e4795c34a9983dc6cef"}, {"id": "u011", "kind": "list-item", "locator": "body:L24-L24", "preview": "- L2-promotion hint; and", "sha256": "5ae4da2e9d5fe5615b8ee15e1a1c8bfe9e24f86837721d4bcdf10944b9f12249"}, {"id": "u012", "kind": "list-item", "locator": "body:L25-L25", "preview": "- out-of-bounds fill policy.", "sha256": "1f9f8ca6a1112db5590a406b647932dd377abb4cc1768c9d00b783ecf7bef430"}, {"id": "u013", "kind": "prose", "locator": "body:L27-L27", "preview": "For the ordinary non-interleaved tiled path in CUDA Driver API 13.0.97, important constraints include:", "sha256": "b22bce9849c6562e29ba2932641636dbf6542a64c546d304859ca5ebd323e652"}, {"id": "u014", "kind": "list-item", "locator": "body:L29-L29", "preview": "- the `CUtensorMap` output object is 64-byte aligned;", "sha256": "44c7c6b9b737364654f0d6a08279bcf6ca9fc22bed1ecb2e4669b358694fe623"}, {"id": "u015", "kind": "list-item", "locator": "body:L30-L30", "preview": "- the global base and byte strides satisfy the documented alignment rules, commonly at least 16 bytes for the basic types/path;", "sha256": "863f766000dc344890dbe17dfcd48c835fab68f68e6ffd64feff051a0afb8964"}, {"id": "u016", "kind": "list-item", "locator": "body:L31-L31", "preview": "- every `boxDim` entry is from 1 through 256 elements;", "sha256": "6729dd96d450f4c92833472caa18c04a6c97afd6d7cbcb25f034cd7c865f1a68"}, {"id": "u017", "kind": "list-item", "locator": "body:L32-L32", "preview": "- `boxDim[0] * element_size` is a multiple of 16 bytes;", "sha256": "bc3d083f157b2d2c6efd45522d8368a362504e504b01c619ee5769749b33af2f"}, {"id": "u018", "kind": "list-item", "locator": "body:L33-L33", "preview": "- every element stride is from 1 through 8; and", "sha256": "e98ad79a06a4e814bd46f56699ba0b5b07c84af7f99ff5821f354a58a5a7c4ad"}, {"id": "u019", "kind": "list-item", "locator": "body:L34-L34", "preview": "- when swizzling is enabled, the inner box in bytes does not exceed the selected swizzle span.", "sha256": "8d4b2cee8563f7311901d071d457877dd9b4675b8b0a577f25959b1ceb4428b8"}, {"id": "u020", "kind": "prose", "locator": "body:L36-L36", "preview": "Datatype, interleave, sub-byte, and architecture-specific modes add further restrictions. Always check the encoder's `CUresult`; never use the output after an encoding failure.", "sha256": "1dc42273ad70a1619c9b071188b6c21a33296af5ea280e944e372650c47d5e1f"}, {"id": "u021", "kind": "prose", "locator": "body:L38-L38", "preview": "Host encoding is common, but it is not the only Blackwell path. CUDA also documents device-side tensor-map construction and modification on Blackwell. A map modified through the generic proxy must be published to the tensor-map proxy with t", "sha256": "32c5584ca3441e4fc303d1aacb9ebde7f3bd75753c43adcac3cf8eda7bfd543e"}, {"id": "u022", "kind": "prose", "locator": "body:L42-L42", "preview": "PTX ISA 9.0 defines this representative 2D CTA-local form:", "sha256": "b0293941d4875c7b42475bd148a898910ccfba21295ea5e319fcd8483dfdf15a"}, {"id": "u023", "kind": "code", "locator": "body:L44-L47", "preview": "```ptx cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes [dst_smem], [tensor_map, {x, y}], [full_barrier]; ```", "sha256": "c63f1e52e13d7bed644ceffc9967df48dac5830d6c1c436b73250f38dd90fdb3"}, {"id": "u024", "kind": "prose", "locator": "body:L49-L49", "preview": "The instruction is non-blocking. A common one-producer phase with one or more loads uses this accounting:", "sha256": "33c59a474aaa779a2840508f993381a25aeaa81e88b896d3e729b91f4863ca8d"}, {"id": "u025", "kind": "list-item", "locator": "body:L51-L51", "preview": "1. Initialize and publish the stage's mbarrier with the intended pending-arrival count.", "sha256": "b97850eccf2dd9133f354210b468324757ccddd7363c04b11905826bd5231e19"}, {"id": "u026", "kind": "list-item", "locator": "body:L52-L52", "preview": "2. The producer performs one `mbarrier.arrive.expect_tx` for its software arrival and the sum of bytes that all loads in this phase will complete.", "sha256": "7536187d8435ca9f9d70ea4ba02e02a996a19a2b67a19e2524ae42383ccaeb3f"}, {"id": "u027", "kind": "list-item", "locator": "body:L53-L53", "preview": "3. Issue the TMA loads against that barrier.", "sha256": "387aec934b017aca25a3dde3f8509850f2406a8e8fb178ae11305df52df38430"}, {"id": "u028", "kind": "list-item", "locator": "body:L54-L54", "preview": "4. Each completed load performs `complete_tx` for its copied byte count; it does **not** perform another arrival.", "sha256": "41c113b3b786060e187ca94075b7e8b4fbf2bdd2a611cab9393d01671a812f20"}, {"id": "u029", "kind": "list-item", "locator": "body:L55-L55", "preview": "5. Consumers wait with the correct state token or per-stage parity and acquire semantics before reading the destination.", "sha256": "d844abea9463ba69fbba5103258780bd95d6b474e5bb6eb11991a6f48d940c29"}, {"id": "u030", "kind": "prose", "locator": "body:L57-L57", "preview": "The phase completes only after pending arrivals and tx-count are both zero. If a design performs multiple software arrivals instead, its initialized count must match them exactly. See [mbarrier](mbarrier.md) for lifecycle, phase, and memory", "sha256": "84689cde185105f289c1e125c553e90e62cf23b7efc0219e0dc269aec0b3cf30"}, {"id": "u031", "kind": "prose", "locator": "body:L61-L61", "preview": "A tensor store uses bulk-group completion rather than the load's mbarrier protocol:", "sha256": "7644e09174ad1ec06520bbb2067d83e2dd1964ae8a1b874ce3b0f3e6a5d9cb90"}, {"id": "u032", "kind": "code", "locator": "body:L63-L68", "preview": "```ptx cp.async.bulk.tensor.2d.global.shared::cta.bulk_group [tensor_map, {x, y}], [src_smem]; cp.async.bulk.commit_group; cp.async.bulk.wait_group 0; ```", "sha256": "9b7414acc80138f372a237ff15efd42d0f0f188d9931500d80b109804411152e"}, {"id": "u033", "kind": "prose", "locator": "body:L70-L70", "preview": "The wait may be delayed to overlap independent work. It must occur before the issuing thread reuses the source shared memory or before code that requires store completion. `commit_group` creates the group; it is not itself a completion wait", "sha256": "bfc107e49734a610b28fe2cde672a1a0645e272320b570dc8c44f3b9596dab3f"}, {"id": "u034", "kind": "prose", "locator": "body:L74-L74", "preview": "The cluster form copies a global tile to the same shared-memory offset in every CTA selected by a 16-bit mask:", "sha256": "94c03e8dfa1adfc21eee3339add8ff57b0febd458ea6dca96c153d7bf652b35c"}, {"id": "u035", "kind": "code", "locator": "body:L76-L79", "preview": "```ptx cp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes.multicast::cluster [dst_smem], [tensor_map, {x, y}], [full_barrier], cta_mask; ```", "sha256": "13740ee71ef9eb029a8fd56b0cf01d83934433dff96a8f1fc6ab0f1869ebdbc7"}, {"id": "u036", "kind": "prose", "locator": "body:L81-L81", "preview": "For `cta_group::1` (the default), the completion signal is also multicast to the same barrier offset in every selected destination CTA. A correct design therefore:", "sha256": "bd2bc4bda81f3d523a915f9119c19a704ad18216bea214eb3116c3ca820ce4a6"}, {"id": "u037", "kind": "list-item", "locator": "body:L83-L83", "preview": "- launches an explicit cluster and keeps every destination CTA's shared memory alive;", "sha256": "99220fe82c4329fd5ff1a570bae96e8b1a304953a628a7ed13aaff2e932dc681"}, {"id": "u038", "kind": "list-item", "locator": "body:L84-L84", "preview": "- initializes corresponding destination barriers before the elected issuer can start;", "sha256": "aaa47a1f531b09283cee30555df707bd08f0c1271b4e15442f61a81f8a97df53"}, {"id": "u039", "kind": "list-item", "locator": "body:L85-L85", "preview": "- uses one cluster-wide elected issuer, not one `threadIdx.x == 0` issuer in every CTA;", "sha256": "002e972562c21a4dd64b4792b70d2d300e01fe73754728937a5e32b8a654441d"}, {"id": "u040", "kind": "list-item", "locator": "body:L86-L86", "preview": "- includes only valid destination CTA ranks in the mask; and", "sha256": "8dfb07761fafb9091a650ad2883aa5dd47b19a85a523291ef4aba5dc9e978dd7"}, {"id": "u041", "kind": "list-item", "locator": "body:L87-L87", "preview": "- has every destination wait on its own corresponding barrier phase before consuming the tile.", "sha256": "6032817a977cb65d2e38311dba90fce95f1ba507d414939a1be5383ac925cd88"}, {"id": "u042", "kind": "prose", "locator": "body:L89-L89", "preview": "For GEMM tiles with the same N range and different M ranges, the CTAs use different A tiles but the same B tile, so multicast can avoid duplicate logical B-load requests. It does not promise an exact `cluster_size` reduction in measured DRA", "sha256": "c6b1d2169446becddd2f513ea356236dd3a2dc964bd99d7b0c2582db59b2f24b"}, {"id": "u043", "kind": "prose", "locator": "body:L93-L93", "preview": "TMA supports no swizzle and multiple swizzled shared-memory layouts, including 32B, 64B, and 128B spans plus newer variants for selected types. Swizzling rearranges chunks across shared-memory banks. The consumer must address the matching l", "sha256": "853a7a8ba52912cbc1b920f6f71d0610c40733f923573857286b071994a68364"}, {"id": "u044", "kind": "prose", "locator": "body:L95-L95", "preview": "There is no universal rule that every Blackwell or `tcgen05.mma` input uses 128B swizzling. The TMA tensor-map swizzle, destination base alignment, leading dimension, and the tcgen05 shared-memory descriptor must describe the **same** legal", "sha256": "2b4b0d6d11682df0f0130a149296140f1a53aa39f3ec97618d9acc1302d4715a"}, {"id": "u045", "kind": "prose", "locator": "body:L97-L97", "preview": "A matched swizzle can reduce or remove bank conflicts for a particular access pattern. It does not make every possible consumer access conflict-free.", "sha256": "a0b6b60319b33d60b4c5d3fe6af97125681fcc47c3b23349ca8c394854345795"}, {"id": "u046", "kind": "prose", "locator": "body:L101-L101", "preview": "A common Blackwell GEMM data path is:", "sha256": "13268f43c424737e1cb3cec39868151eea4e3d9bdd671f7b87dafcb603717e4f"}, {"id": "u047", "kind": "prose", "locator": "body:L103-L103", "preview": "`GMEM -> TMA -> SMEM -> tcgen05.mma -> TMEM -> tcgen05.ld -> registers -> output store`", "sha256": "23afe141ca999584d588aa68850c067da012eed1030c92b9aef1b932e349a614"}, {"id": "u048", "kind": "prose", "locator": "body:L105-L105", "preview": "Each reusable pipeline stage needs two independent ownership transitions:", "sha256": "bfa1891a762540b1465d3767155e2928b8b026e0f40f012c7bcd41afd38b6392"}, {"id": "u049", "kind": "list-item", "locator": "body:L107-L107", "preview": "- **full:** TMA has finished producing the shared-memory operands, so the MMA consumer may read them; and", "sha256": "11d6052606c7b84bebee01e3c45bd2589c56b249d41d348edfd2aafd566d180b"}, {"id": "u050", "kind": "list-item", "locator": "body:L108-L108", "preview": "- **empty:** asynchronous MMA has stopped reading those operands, so the TMA producer may overwrite the stage.", "sha256": "eef3d18fd694d3e4b240e591ddccf87b99764dd71f1f72c9e14f2fc1fceab149"}, {"id": "u051", "kind": "prose", "locator": "body:L110-L110", "preview": "Track full and empty state per stage. Account transaction bytes once, track each barrier's own phase, and do not equate CTA synchronization or a tcgen05 fence with async completion. A complete CUTLASS pipeline is safer evidence than a short", "sha256": "248d9132d4467e85eeff75f7f120ab16a0679a718df2d20a806c822af36f2027"}, {"id": "u052", "kind": "prose", "locator": "body:L112-L112", "preview": "Choose tile rank, swizzle, multicast, issue cadence, and stage count from the actual access pattern and resource budget. More distinct stages consume more shared memory; there is no universal optimum of three to five stages. Profile the tar", "sha256": "8a0567c9674b127300435eb31acb41d3d1aead134c3bae110c17c6d5aec4d12f"}, {"id": "u053", "kind": "list-item", "locator": "body:L116-L116", "preview": "- [CUDA 13.0.2 Programming Guide: TMA](https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#asynchronous-data-copies-using-the-tensor-memory-accelerator-tma)", "sha256": "6e4dfa9b3ded302ca056bb6a498de0ab3aaafeef829d5103c81e632f5c1bf14a"}, {"id": "u054", "kind": "list-item", "locator": "body:L117-L117", "preview": "- [CUDA Driver API 13.0.97: tensor-map management](https://docs.nvidia.com/cuda/archive/13.0.2/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html)", "sha256": "4e29813dfddcaddb177f40e4fe9480deb10b9dbe097b724c947ccd91c3658c8f"}, {"id": "u055", "kind": "list-item", "locator": "body:L118-L118", "preview": "- [PTX ISA 9.0: `cp.async.bulk.tensor`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor)", "sha256": "ef3133ce097f33a254f3b19981e6d76ec88f33e964ee32edb03b3aecb627cc9f"}, {"id": "u056", "kind": "list-item", "locator": "body:L119-L119", "preview": "- [CUTLASS 4.5.0: complete CuTe DSL TMA tutorial](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_tma/tma_v0.py)", "sha256": "fffa82636381165a465946081584ded3dbcb39253ce114ed6ca90eec84090ffb"}, {"id": "u057", "kind": "list-item", "locator": "body:L120-L120", "preview": "- [mbarrier](mbarrier.md)", "sha256": "3cdff3a1605be2d98c176340b72d54b96f0575bb3dad4b249d5599d1bb94c369"}, {"id": "u058", "kind": "list-item", "locator": "body:L121-L121", "preview": "- [tcgen05 MMA](tcgen05-mma.md)", "sha256": "8fe04c67b24dcafdf0ff00c51238617219c223e45065aa447c5f56c2203a5e2c"}], "confidence_claimed": "verified", "headings": ["Scope", "Tensor maps", "Global-to-shared load", "Shared-to-global store", "Cluster multicast", "Swizzle and tcgen05", "Pipeline invariants", "References"], "id": "hw-tma", "path": "wiki/hardware/tma.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-cuda-13-0-2-tma.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#asynchronous-data-copies-using-the-tensor-memory-accelerator-tma"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/prs/cutlass/PR-2139.md", "revision": "ca4fdbea", "url": "https://github.com/NVIDIA/cutlass/pull/2139"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-cuda-13-0-2-tma", "doc-ptx-isa-sm100", "pr-cutlass-2139"], "title": "Tensor Memory Accelerator (TMA)", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} -{"body_sha256": "f6c0135c31b4600deecf3ee1dd972746a40bb9b3c0968b96e5ceb231a35ce816", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Tensor Memory is an addressable, on-chip memory introduced for fifth-generation Tensor Core operations on Blackwell. Each SM has 256 KiB, organized as 128 lanes by 512 columns of 32-bit cells. `tcgen05.mma` writes its D accumulator to TMEM;", "sha256": "0df97e28b422381bddbf8db848a9efad8674027194393755004ac345a3631094"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "Moving D out of general-purpose registers changes the resource balance relative to Hopper WGMMA. For example, CUTLASS 4.5.0's SM90 m64n256 FP32 WGMMA wrapper exposes 128 accumulator registers **per participating thread**. TMEM avoids that p", "sha256": "5afe174e3db5a624f39253beb7a40cc63c9a8843d4163b11310c197f1d69656d"}, {"id": "u003", "kind": "prose", "locator": "body:L11-L11", "preview": "A TMEM address (`taddr`) is a 32-bit value:", "sha256": "7f5f41f2bc49fc82a013135b23d8ba697ad04901e624c995259e5cd0b907a1fd"}, {"id": "u004", "kind": "table-row", "locator": "body:L15-L15", "preview": "| Bits | Meaning | | 31:16 | TMEM lane |", "sha256": "cee3131449614a54c7ffe5f19dc9eda0dc07a58cd86780f43db79beca031e933"}, {"id": "u005", "kind": "table-row", "locator": "body:L16-L16", "preview": "| Bits | Meaning | | 15:0 | TMEM column |", "sha256": "b4bc3f1c30523c2010b9f5cf1722da25b5a63186c9b27201a1b8fd8f74339b7b"}, {"id": "u006", "kind": "prose", "locator": "body:L18-L18", "preview": "The lane is a TMEM data-path lane, not storage independently owned by a CUDA thread. `tcgen05.ld` and `tcgen05.st` are warp-collective. Under their four-warp access model, the warps cover these lane chunks:", "sha256": "28d9f3ee58888f5aab3a8dc64f93d17d32ada56fc2f6f5576d452efc503707ba"}, {"id": "u007", "kind": "table-row", "locator": "body:L22-L22", "preview": "| Participating warp | TMEM lanes | | 0 | 0-31 |", "sha256": "e53446be920ecbd70393e486f6e7948091941cbd67eda6ccdcc393d536c365ca"}, {"id": "u008", "kind": "table-row", "locator": "body:L23-L23", "preview": "| Participating warp | TMEM lanes | | 1 | 32-63 |", "sha256": "9294efa2eb1f0b53aa8b8b5f566b209581a3e00a8c2826bd43088a98e1c7d9f4"}, {"id": "u009", "kind": "table-row", "locator": "body:L24-L24", "preview": "| Participating warp | TMEM lanes | | 2 | 64-95 |", "sha256": "1a42b20cd839a4160147e7de90d07ed64dbcd8526d6b3b42f93ab0078a4e4fb3"}, {"id": "u010", "kind": "table-row", "locator": "body:L25-L25", "preview": "| Participating warp | TMEM lanes | | 3 | 96-127 |", "sha256": "210a7e92aa2b4eb70e52583c755b97ad0b7721515062249ee913f2ce4bdffa83"}, {"id": "u011", "kind": "prose", "locator": "body:L27-L27", "preview": "The chosen load/store shape determines how values from that lane-column region map to each thread's register vector. Use the PTX shape-specific layout tables rather than treating a logical MxN accumulator as a universal row-major array owne", "sha256": "2ea3747ea0fa2ad571db92917836e6431266f2c4a6f150e3cf559d78cad6fa63"}, {"id": "u012", "kind": "prose", "locator": "body:L31-L31", "preview": "TMEM has an explicit software-managed lifetime:", "sha256": "619075113787e60845488a825201856ee59878b761427028e7103ca64764ccdf"}, {"id": "u013", "kind": "list-item", "locator": "body:L33-L33", "preview": "1. Reserve shared memory for the 32-bit allocation result.", "sha256": "347425a839a5fa6eac81d4797ca4de56bb02bbb2eda77b4c47ae23820a94a58c"}, {"id": "u014", "kind": "list-item", "locator": "body:L34-L34", "preview": "2. Execute `tcgen05.alloc` collectively from one warp for `cta_group::1`, or from two warps\u2014one per paired CTA\u2014for `cta_group::2`.", "sha256": "b8e4f5ea142d2258f24cb03b6fea0cce727beb4c7d601d6b5af1bbc2b3abb396"}, {"id": "u015", "kind": "list-item", "locator": "body:L35-L35", "preview": "3. Synchronize consumers as required, then read the returned base `taddr` from shared memory.", "sha256": "75df93469c3e8eb30d652d8f99f7f56af6674bf7704e0a36db7c6d7f92987a9e"}, {"id": "u016", "kind": "list-item", "locator": "body:L36-L36", "preview": "4. Use the allocation for MMA, copy, load, or store operations with the same CTA-group mode.", "sha256": "bb1d97a60d0710ea6e20c4bf6951496a283f5fbf8979b4c474b8e45df508f13c"}, {"id": "u017", "kind": "list-item", "locator": "body:L37-L37", "preview": "5. Execute `tcgen05.dealloc` with the corresponding collective issue pattern.", "sha256": "2dc6706ed0536b771b2cde4f4f7fa2fac7f90658f9b4da940bfe0d24161a4c5b"}, {"id": "u018", "kind": "list-item", "locator": "body:L38-L38", "preview": "6. Deallocate every TMEM allocation before kernel exit.", "sha256": "d1938b5098e7b266d47da39e56ea7dab3975b115b4fc0ba3424c3ddf451ac0cd"}, {"id": "u019", "kind": "prose", "locator": "body:L40-L40", "preview": "The allocation operand counts columns. Legal allocation sizes are powers of two from 32 through 512 columns. Allocation can block until the requested TMEM is available; the ISA does not define folklore outcomes such as silent corruption for", "sha256": "f91ee27bb11a3513503232c20e34d34d2a15adafe81323e309113b223563d26a"}, {"id": "u020", "kind": "prose", "locator": "body:L44-L44", "preview": "All allocations share the SM's 512-column capacity. Some useful whole-allocation budgets are:", "sha256": "b4ee5bf23171df60c790b148f9fd3cedc5a56e4433af63ad5d475858f984e163"}, {"id": "u021", "kind": "table-row", "locator": "body:L48-L48", "preview": "| Allocation per stage | Maximum stages if TMEM has no other use | Columns left | | 128 columns | 4 | 0 |", "sha256": "b072c857bef9e2ac11466c7551dd2f9becdd59b4f4673742468fcc36d7d11ebb"}, {"id": "u022", "kind": "table-row", "locator": "body:L49-L49", "preview": "| Allocation per stage | Maximum stages if TMEM has no other use | Columns left | | 256 columns | 2 | 0 |", "sha256": "05a58dad733419c4156a0f06f2154ef8615e852d12525633dfd35418f52a89f3"}, {"id": "u023", "kind": "table-row", "locator": "body:L50-L50", "preview": "| Allocation per stage | Maximum stages if TMEM has no other use | Columns left | | 512 columns | 1 | 0 |", "sha256": "a940200162e3b907c6f8faab934616f673d5f4a5aff941ec4376c3638fdc0f1d"}, {"id": "u024", "kind": "prose", "locator": "body:L52-L52", "preview": "A logical need of 192 columns cannot be requested directly: reserve 256 columns, or suballocate that logical region within another legal power-of-two reservation. Two 256-column accumulator stages consume the entire capacity, so scale-facto", "sha256": "0e50cd966770f36695b446d06e378b73e842ebf0b42c509fa17fea8336fed88c"}, {"id": "u025", "kind": "prose", "locator": "body:L56-L56", "preview": "The relevant mechanisms are distinct:", "sha256": "fc1c5304326b584de63c1c1519794b7e024c06d00ee87b0186768ab05a4c4a4c"}, {"id": "u026", "kind": "list-item", "locator": "body:L58-L58", "preview": "- `tcgen05.commit` attaches completion of prior asynchronous MMA operations to an mbarrier. Wait on that barrier before consuming their results.", "sha256": "1af1167372a70c2f450e4a843b993d8129391f12b1cc1919b4c294825e811e89"}, {"id": "u027", "kind": "list-item", "locator": "body:L59-L59", "preview": "- `tcgen05.wait::ld` and `tcgen05.wait::st` are the completion mechanisms for the corresponding asynchronous TMEM load/store operations.", "sha256": "386820efb8194c8b401e0b7a6a6743e16f245b9eaa180ec26a92080c99e8b34d"}, {"id": "u028", "kind": "list-item", "locator": "body:L60-L60", "preview": "- `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` order tcgen05 operations around a documented execution-ordering handoff. A fence is not an MMA completion wait.", "sha256": "0943cb17d0d11a25cf92c877aadb5c2a5ab386de0329b9466b6895733f0cfa65"}, {"id": "u029", "kind": "list-item", "locator": "body:L61-L61", "preview": "- Producer buffers must remain live until the asynchronous operation that reads them has completed according to its instruction contract.", "sha256": "feb718367dc639669dec4406eae2afadd79207dafd794557fa719986099b240d"}, {"id": "u030", "kind": "prose", "locator": "body:L63-L63", "preview": "CTA synchronization alone does not replace these completion operations.", "sha256": "c9f735b433318be62dcfdac6bd798094eddd61f8b412e8617e21d7ecdb82b89c"}, {"id": "u031", "kind": "prose", "locator": "body:L67-L67", "preview": "`tcgen05.ld` transfers TMEM into registers, and `tcgen05.st` transfers registers into TMEM. Their documented shapes include `.16x64b`, `.16x128b`, `.16x256b`, `.32x32b`, and `.16x32bx2`, with supported repetition qualifiers. There is no sca", "sha256": "00c8ef6aa00691fb742087960b4de34db91b6645935dd58d1869ccc2f1f05775"}, {"id": "u032", "kind": "prose", "locator": "body:L69-L69", "preview": "`tcgen05.cp` performs shaped shared-memory-to-TMEM copies. For example, PTX ISA 9.0 defines this exact form:", "sha256": "9ae8051754c858f6c0e847374738cfc3809f3de06adf5ad59088acdcd4389e5c"}, {"id": "u033", "kind": "code", "locator": "body:L71-L73", "preview": "```ptx tcgen05.cp.cta_group::1.128x256b [taddr], sdesc; ```", "sha256": "31c9dbcff5abb4891536fbccdba342e817b50f428cafdf35cebc02bf54bd30f1"}, {"id": "u034", "kind": "prose", "locator": "body:L75-L75", "preview": "Here `taddr` is the TMEM destination and `sdesc` is the 64-bit shared-memory matrix descriptor. Follow the instruction's asynchronous ordering and completion rules before reusing the source or consuming the destination.", "sha256": "e068f39c91ad4a5696b7fc43d22115ac304f1155eeede678c6d93adc1160eec4"}, {"id": "u035", "kind": "prose", "locator": "body:L79-L79", "preview": "Multiple accumulator stages can overlap MMA production with draining a different, completed TMEM stage. The design is valid only when:", "sha256": "991654cf4559d3b7da1d51e83dcda7db8c0ab1803d8d84734223be63483ea452"}, {"id": "u036", "kind": "list-item", "locator": "body:L81-L81", "preview": "- the combined allocations and suballocations fit the 512-column budget;", "sha256": "17fd8af5790cb5d70bdc34a7f45f8874f12f30b681bea0944af76bc4f3a33768"}, {"id": "u037", "kind": "list-item", "locator": "body:L82-L82", "preview": "- producer and consumer roles use explicit pipeline barriers;", "sha256": "33a31fcf7dce9d220bd55852b4c572cf63e5fa7e983b37d2da40fef011abccd5"}, {"id": "u038", "kind": "list-item", "locator": "body:L83-L83", "preview": "- MMA completion is observed before a consumer drains a stage;", "sha256": "e7c3b8ae852fb631c85ccd5f45b861b4388ae3bdf8d483489d44d8f62fa939be"}, {"id": "u039", "kind": "list-item", "locator": "body:L84-L84", "preview": "- TMEM load completion is observed before register results are used; and", "sha256": "f2ed672764ffffbedc6bd0dabc492b18674a4866746bf6d07551955baee25d50"}, {"id": "u040", "kind": "list-item", "locator": "body:L85-L85", "preview": "- no stage is recycled while an asynchronous operation can still access it.", "sha256": "7da7a6de9e5d7952da1a50d38417f3cc335d39e018c3a67e33e0fc3cf595c69a"}, {"id": "u041", "kind": "prose", "locator": "body:L87-L87", "preview": "This is a pipeline design, not an automatic consequence of alternating two addresses. CUTLASS's version-pinned SM100 tutorials show complete producer/consumer implementations.", "sha256": "d4011413497e52d0dcae86b5ffa75a65ffb125c97d706d62a54ae44b73511e4c"}, {"id": "u042", "kind": "prose", "locator": "body:L91-L91", "preview": "CUTLASS 4.5.0 exposes TMEM through `cutlass.utils.TmemAllocator` and layout-aware CuTe tensors. The official tutorial sequence is:", "sha256": "e18540de26c1c50ab8326e2ae286d0d965df1b58986f5c5eae9076c610e4bebb"}, {"id": "u043", "kind": "list-item", "locator": "body:L93-L93", "preview": "1. create a `TmemAllocator` over shared storage for the allocation result;", "sha256": "a3edc461fce5b5784083dacacd0f1c101b43ca2a85cf79745fece0939240b903"}, {"id": "u044", "kind": "list-item", "locator": "body:L94-L94", "preview": "2. call `allocate(num_columns)` from the configured allocator warp;", "sha256": "292ed99bc0f058be44017f232311ac5c48a4aa2d2d479d9e6e1d01fc3675398c"}, {"id": "u045", "kind": "list-item", "locator": "body:L95-L95", "preview": "3. use `wait_for_alloc()` before other warps retrieve the address;", "sha256": "ce966bcfd9c03d8d2d5ac2362e6a8493e4fa73e2008dea59f9b3b83c8fd2dc52"}, {"id": "u046", "kind": "list-item", "locator": "body:L96-L96", "preview": "4. obtain a typed pointer with `retrieve_ptr(dtype)`;", "sha256": "07e29aa2cd0afb608284d84518f2dd6ca9e0992bce434000e1e795ba8f54c97d"}, {"id": "u047", "kind": "list-item", "locator": "body:L97-L97", "preview": "5. bind that pointer to the MMA accumulator layout with `cute.make_tensor`;", "sha256": "cffb932874a3bf23919dc46c78e281039af766ddac29dcd431edd27a8e57f4db"}, {"id": "u048", "kind": "list-item", "locator": "body:L98-L98", "preview": "6. drain it with `tcgen05` copy atoms and the pipeline's completion protocol; and", "sha256": "2c1a41548f8cadb56da832323b1246a164a9b72881e751bdc28165d0e836fd4b"}, {"id": "u049", "kind": "list-item", "locator": "body:L99-L99", "preview": "7. call `free(tmem_ptr)` before exit.", "sha256": "43514435699f5adeeb0344be0c63200aa778fa36ddef2b0aed2e70ee5579cb83"}, {"id": "u050", "kind": "prose", "locator": "body:L101-L101", "preview": "See the pinned [CUTLASS 4.5.0 FP16 GEMM tutorial](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py) and [TMEM allocator implementati", "sha256": "ed0773f61bf2f5159b5ba120c90bdf34465bfa942c6d6a4ea55d4fcbd90048ab"}, {"id": "u051", "kind": "list-item", "locator": "body:L105-L105", "preview": "- [PTX ISA 9.0: Tensor Memory](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory)", "sha256": "40744f485d8c843f17e8166199678d16332069d8bcd1fbd4f490d2703e20657a"}, {"id": "u052", "kind": "list-item", "locator": "body:L106-L106", "preview": "- [PTX ISA 9.0: `tcgen05.alloc`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc)", "sha256": "f4138a536dd8f10f27c3b7e99b22b3d68f162948a899bf3f883d7d4820269775"}, {"id": "u053", "kind": "list-item", "locator": "body:L107-L107", "preview": "- [PTX ISA 9.0: `tcgen05.ld`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld)", "sha256": "4ecd5d78052e70a7efc674ba9e98404c79471b2f6ff7054f46463a6fa4e2ec36"}, {"id": "u054", "kind": "list-item", "locator": "body:L108-L108", "preview": "- [PTX ISA 9.0: `tcgen05.commit`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-commit)", "sha256": "4b6757bbc80715870a3a7823c1b45358f3efbffa77b50cf04231cef404cd00f8"}, {"id": "u055", "kind": "list-item", "locator": "body:L109-L109", "preview": "- [CUTLASS 4.5.0 SM100 Python tutorial](https://github.com/NVIDIA/cutlass/tree/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm)", "sha256": "42eee452d016fa75699f23e63de919826e07c0ff194b091beae8c3b06db6ffa9"}], "confidence_claimed": "verified", "headings": ["What TMEM is", "Addressing and warp access", "Allocation lifecycle", "Column budgeting", "Completion and ordering", "Data movement", "Staging accumulators", "CUTLASS 4.5.0 Python DSL", "Primary references"], "id": "hw-tmem", "path": "wiki/hardware/tmem.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/prs/cutlass/PR-2139.md", "revision": "ca4fdbea", "url": "https://github.com/NVIDIA/cutlass/pull/2139"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-ptx-isa-sm100", "pr-cutlass-2139"], "title": "Tensor Memory (TMEM)", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "e03fceba3cdacef11e31732fc23f086a61294f8dd66e33483845ab06046ff224", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Tensor Memory is an addressable, on-chip memory introduced for fifth-generation Tensor Core operations on Blackwell. Each SM has 256 KiB, organized as 128 lanes by 512 columns of 32-bit cells. `tcgen05.mma` writes its D accumulator to TMEM;", "sha256": "0df97e28b422381bddbf8db848a9efad8674027194393755004ac345a3631094"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "Moving D out of general-purpose registers changes the resource balance relative to Hopper WGMMA. For example, CUTLASS 4.5.0's SM90 m64n256 FP32 WGMMA wrapper exposes 128 accumulator registers **per participating thread**. TMEM avoids that p", "sha256": "5afe174e3db5a624f39253beb7a40cc63c9a8843d4163b11310c197f1d69656d"}, {"id": "u003", "kind": "prose", "locator": "body:L11-L11", "preview": "A TMEM address (`taddr`) is a 32-bit value:", "sha256": "7f5f41f2bc49fc82a013135b23d8ba697ad04901e624c995259e5cd0b907a1fd"}, {"id": "u004", "kind": "table-row", "locator": "body:L15-L15", "preview": "| Bits | Meaning | | 31:16 | TMEM lane |", "sha256": "cee3131449614a54c7ffe5f19dc9eda0dc07a58cd86780f43db79beca031e933"}, {"id": "u005", "kind": "table-row", "locator": "body:L16-L16", "preview": "| Bits | Meaning | | 15:0 | TMEM column |", "sha256": "b4bc3f1c30523c2010b9f5cf1722da25b5a63186c9b27201a1b8fd8f74339b7b"}, {"id": "u006", "kind": "prose", "locator": "body:L18-L18", "preview": "The lane is a TMEM data-path lane, not storage independently owned by a CUDA thread. `tcgen05.ld` and `tcgen05.st` are warp-collective. Under their four-warp access model, the warps cover these lane chunks:", "sha256": "28d9f3ee58888f5aab3a8dc64f93d17d32ada56fc2f6f5576d452efc503707ba"}, {"id": "u007", "kind": "table-row", "locator": "body:L22-L22", "preview": "| Participating warp | TMEM lanes | | 0 | 0-31 |", "sha256": "e53446be920ecbd70393e486f6e7948091941cbd67eda6ccdcc393d536c365ca"}, {"id": "u008", "kind": "table-row", "locator": "body:L23-L23", "preview": "| Participating warp | TMEM lanes | | 1 | 32-63 |", "sha256": "9294efa2eb1f0b53aa8b8b5f566b209581a3e00a8c2826bd43088a98e1c7d9f4"}, {"id": "u009", "kind": "table-row", "locator": "body:L24-L24", "preview": "| Participating warp | TMEM lanes | | 2 | 64-95 |", "sha256": "1a42b20cd839a4160147e7de90d07ed64dbcd8526d6b3b42f93ab0078a4e4fb3"}, {"id": "u010", "kind": "table-row", "locator": "body:L25-L25", "preview": "| Participating warp | TMEM lanes | | 3 | 96-127 |", "sha256": "210a7e92aa2b4eb70e52583c755b97ad0b7721515062249ee913f2ce4bdffa83"}, {"id": "u011", "kind": "prose", "locator": "body:L27-L27", "preview": "The chosen load/store shape determines how values from that lane-column region map to each thread's register vector. Use the PTX shape-specific layout tables rather than treating a logical MxN accumulator as a universal row-major array owne", "sha256": "2ea3747ea0fa2ad571db92917836e6431266f2c4a6f150e3cf559d78cad6fa63"}, {"id": "u012", "kind": "prose", "locator": "body:L31-L31", "preview": "TMEM has an explicit software-managed lifetime:", "sha256": "619075113787e60845488a825201856ee59878b761427028e7103ca64764ccdf"}, {"id": "u013", "kind": "list-item", "locator": "body:L33-L33", "preview": "1. Reserve shared memory for the 32-bit allocation result.", "sha256": "347425a839a5fa6eac81d4797ca4de56bb02bbb2eda77b4c47ae23820a94a58c"}, {"id": "u014", "kind": "list-item", "locator": "body:L34-L34", "preview": "2. Execute `tcgen05.alloc` collectively from one warp for `cta_group::1`, or from two warps\u2014one per paired CTA\u2014for `cta_group::2`.", "sha256": "b8e4f5ea142d2258f24cb03b6fea0cce727beb4c7d601d6b5af1bbc2b3abb396"}, {"id": "u015", "kind": "list-item", "locator": "body:L35-L35", "preview": "3. Synchronize consumers as required, then read the returned base `taddr` from shared memory.", "sha256": "75df93469c3e8eb30d652d8f99f7f56af6674bf7704e0a36db7c6d7f92987a9e"}, {"id": "u016", "kind": "list-item", "locator": "body:L36-L36", "preview": "4. Use the allocation for MMA, copy, load, or store operations with the same CTA-group mode.", "sha256": "bb1d97a60d0710ea6e20c4bf6951496a283f5fbf8979b4c474b8e45df508f13c"}, {"id": "u017", "kind": "list-item", "locator": "body:L37-L37", "preview": "5. Execute `tcgen05.dealloc` with the corresponding collective issue pattern.", "sha256": "2dc6706ed0536b771b2cde4f4f7fa2fac7f90658f9b4da940bfe0d24161a4c5b"}, {"id": "u018", "kind": "list-item", "locator": "body:L38-L38", "preview": "6. Deallocate every TMEM allocation before kernel exit.", "sha256": "d1938b5098e7b266d47da39e56ea7dab3975b115b4fc0ba3424c3ddf451ac0cd"}, {"id": "u019", "kind": "prose", "locator": "body:L40-L40", "preview": "The allocation operand counts columns. Legal allocation sizes are powers of two from 32 through 512 columns. Allocation can block until the requested TMEM is available; the ISA does not define folklore outcomes such as silent corruption for", "sha256": "f91ee27bb11a3513503232c20e34d34d2a15adafe81323e309113b223563d26a"}, {"id": "u020", "kind": "prose", "locator": "body:L44-L44", "preview": "All allocations share the SM's 512-column capacity. Some useful whole-allocation budgets are:", "sha256": "b4ee5bf23171df60c790b148f9fd3cedc5a56e4433af63ad5d475858f984e163"}, {"id": "u021", "kind": "table-row", "locator": "body:L48-L48", "preview": "| Allocation per stage | Maximum stages if TMEM has no other use | Columns left | | 128 columns | 4 | 0 |", "sha256": "b072c857bef9e2ac11466c7551dd2f9becdd59b4f4673742468fcc36d7d11ebb"}, {"id": "u022", "kind": "table-row", "locator": "body:L49-L49", "preview": "| Allocation per stage | Maximum stages if TMEM has no other use | Columns left | | 256 columns | 2 | 0 |", "sha256": "05a58dad733419c4156a0f06f2154ef8615e852d12525633dfd35418f52a89f3"}, {"id": "u023", "kind": "table-row", "locator": "body:L50-L50", "preview": "| Allocation per stage | Maximum stages if TMEM has no other use | Columns left | | 512 columns | 1 | 0 |", "sha256": "a940200162e3b907c6f8faab934616f673d5f4a5aff941ec4376c3638fdc0f1d"}, {"id": "u024", "kind": "prose", "locator": "body:L52-L52", "preview": "A logical need of 192 columns cannot be requested directly: reserve 256 columns, or suballocate that logical region within another legal power-of-two reservation. Two 256-column accumulator stages consume the entire capacity, so scale-facto", "sha256": "0e50cd966770f36695b446d06e378b73e842ebf0b42c509fa17fea8336fed88c"}, {"id": "u025", "kind": "prose", "locator": "body:L56-L56", "preview": "The relevant mechanisms are distinct:", "sha256": "fc1c5304326b584de63c1c1519794b7e024c06d00ee87b0186768ab05a4c4a4c"}, {"id": "u026", "kind": "list-item", "locator": "body:L58-L58", "preview": "- `tcgen05.commit` attaches completion of prior asynchronous MMA operations to an mbarrier. Wait on that barrier before consuming their results.", "sha256": "1af1167372a70c2f450e4a843b993d8129391f12b1cc1919b4c294825e811e89"}, {"id": "u027", "kind": "list-item", "locator": "body:L59-L59", "preview": "- `tcgen05.wait::ld` and `tcgen05.wait::st` are the completion mechanisms for the corresponding asynchronous TMEM load/store operations.", "sha256": "386820efb8194c8b401e0b7a6a6743e16f245b9eaa180ec26a92080c99e8b34d"}, {"id": "u028", "kind": "list-item", "locator": "body:L60-L60", "preview": "- `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` order tcgen05 operations around a documented execution-ordering handoff. A fence is not an MMA completion wait.", "sha256": "0943cb17d0d11a25cf92c877aadb5c2a5ab386de0329b9466b6895733f0cfa65"}, {"id": "u029", "kind": "list-item", "locator": "body:L61-L61", "preview": "- Producer buffers must remain live until the asynchronous operation that reads them has completed according to its instruction contract.", "sha256": "feb718367dc639669dec4406eae2afadd79207dafd794557fa719986099b240d"}, {"id": "u030", "kind": "prose", "locator": "body:L63-L63", "preview": "CTA synchronization alone does not replace these completion operations.", "sha256": "c9f735b433318be62dcfdac6bd798094eddd61f8b412e8617e21d7ecdb82b89c"}, {"id": "u031", "kind": "prose", "locator": "body:L67-L67", "preview": "`tcgen05.ld` transfers TMEM into registers, and `tcgen05.st` transfers registers into TMEM. Their documented shapes include `.16x64b`, `.16x128b`, `.16x256b`, `.32x32b`, and `.16x32bx2`, with supported repetition qualifiers. There is no sca", "sha256": "00c8ef6aa00691fb742087960b4de34db91b6645935dd58d1869ccc2f1f05775"}, {"id": "u032", "kind": "prose", "locator": "body:L69-L69", "preview": "`tcgen05.cp` performs shaped shared-memory-to-TMEM copies. For example, PTX ISA 9.0 defines this exact form:", "sha256": "9ae8051754c858f6c0e847374738cfc3809f3de06adf5ad59088acdcd4389e5c"}, {"id": "u033", "kind": "code", "locator": "body:L71-L73", "preview": "```ptx tcgen05.cp.cta_group::1.128x256b [taddr], sdesc; ```", "sha256": "31c9dbcff5abb4891536fbccdba342e817b50f428cafdf35cebc02bf54bd30f1"}, {"id": "u034", "kind": "prose", "locator": "body:L75-L75", "preview": "Here `taddr` is the TMEM destination and `sdesc` is the 64-bit shared-memory matrix descriptor. Follow the instruction's asynchronous ordering and completion rules before reusing the source or consuming the destination.", "sha256": "e068f39c91ad4a5696b7fc43d22115ac304f1155eeede678c6d93adc1160eec4"}, {"id": "u035", "kind": "prose", "locator": "body:L79-L79", "preview": "Multiple accumulator stages can overlap MMA production with draining a different, completed TMEM stage. The design is valid only when:", "sha256": "991654cf4559d3b7da1d51e83dcda7db8c0ab1803d8d84734223be63483ea452"}, {"id": "u036", "kind": "list-item", "locator": "body:L81-L81", "preview": "- the combined allocations and suballocations fit the 512-column budget;", "sha256": "17fd8af5790cb5d70bdc34a7f45f8874f12f30b681bea0944af76bc4f3a33768"}, {"id": "u037", "kind": "list-item", "locator": "body:L82-L82", "preview": "- producer and consumer roles use explicit pipeline barriers;", "sha256": "33a31fcf7dce9d220bd55852b4c572cf63e5fa7e983b37d2da40fef011abccd5"}, {"id": "u038", "kind": "list-item", "locator": "body:L83-L83", "preview": "- MMA completion is observed before a consumer drains a stage;", "sha256": "e7c3b8ae852fb631c85ccd5f45b861b4388ae3bdf8d483489d44d8f62fa939be"}, {"id": "u039", "kind": "list-item", "locator": "body:L84-L84", "preview": "- TMEM load completion is observed before register results are used; and", "sha256": "f2ed672764ffffbedc6bd0dabc492b18674a4866746bf6d07551955baee25d50"}, {"id": "u040", "kind": "list-item", "locator": "body:L85-L85", "preview": "- no stage is recycled while an asynchronous operation can still access it.", "sha256": "7da7a6de9e5d7952da1a50d38417f3cc335d39e018c3a67e33e0fc3cf595c69a"}, {"id": "u041", "kind": "prose", "locator": "body:L87-L87", "preview": "This is a pipeline design, not an automatic consequence of alternating two addresses. CUTLASS's version-pinned SM100 tutorials show complete producer/consumer implementations.", "sha256": "d4011413497e52d0dcae86b5ffa75a65ffb125c97d706d62a54ae44b73511e4c"}, {"id": "u042", "kind": "prose", "locator": "body:L91-L91", "preview": "CUTLASS 4.5.0 exposes TMEM through `cutlass.utils.TmemAllocator` and layout-aware CuTe tensors. The official tutorial sequence is:", "sha256": "e18540de26c1c50ab8326e2ae286d0d965df1b58986f5c5eae9076c610e4bebb"}, {"id": "u043", "kind": "list-item", "locator": "body:L93-L93", "preview": "1. create a `TmemAllocator` over shared storage for the allocation result;", "sha256": "a3edc461fce5b5784083dacacd0f1c101b43ca2a85cf79745fece0939240b903"}, {"id": "u044", "kind": "list-item", "locator": "body:L94-L94", "preview": "2. call `allocate(num_columns)` from the configured allocator warp;", "sha256": "292ed99bc0f058be44017f232311ac5c48a4aa2d2d479d9e6e1d01fc3675398c"}, {"id": "u045", "kind": "list-item", "locator": "body:L95-L95", "preview": "3. use `wait_for_alloc()` before other warps retrieve the address;", "sha256": "ce966bcfd9c03d8d2d5ac2362e6a8493e4fa73e2008dea59f9b3b83c8fd2dc52"}, {"id": "u046", "kind": "list-item", "locator": "body:L96-L96", "preview": "4. obtain a typed pointer with `retrieve_ptr(dtype)`;", "sha256": "07e29aa2cd0afb608284d84518f2dd6ca9e0992bce434000e1e795ba8f54c97d"}, {"id": "u047", "kind": "list-item", "locator": "body:L97-L97", "preview": "5. bind that pointer to the MMA accumulator layout with `cute.make_tensor`;", "sha256": "cffb932874a3bf23919dc46c78e281039af766ddac29dcd431edd27a8e57f4db"}, {"id": "u048", "kind": "list-item", "locator": "body:L98-L98", "preview": "6. drain it with `tcgen05` copy atoms and the pipeline's completion protocol; and", "sha256": "2c1a41548f8cadb56da832323b1246a164a9b72881e751bdc28165d0e836fd4b"}, {"id": "u049", "kind": "list-item", "locator": "body:L99-L99", "preview": "7. call `free(tmem_ptr)` before exit.", "sha256": "43514435699f5adeeb0344be0c63200aa778fa36ddef2b0aed2e70ee5579cb83"}, {"id": "u050", "kind": "prose", "locator": "body:L101-L101", "preview": "See the pinned [CUTLASS 4.5.0 FP16 GEMM tutorial](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py) and [TMEM allocator implementati", "sha256": "ed0773f61bf2f5159b5ba120c90bdf34465bfa942c6d6a4ea55d4fcbd90048ab"}, {"id": "u051", "kind": "list-item", "locator": "body:L105-L105", "preview": "- [PTX ISA 9.0: Tensor Memory](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory)", "sha256": "40744f485d8c843f17e8166199678d16332069d8bcd1fbd4f490d2703e20657a"}, {"id": "u052", "kind": "list-item", "locator": "body:L106-L106", "preview": "- [PTX ISA 9.0: `tcgen05.alloc`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit)", "sha256": "76fb28bfcdeee86b21c8f72d7dc1cb015f9d3ec4e10a6ade0aef7a2eed4df97c"}, {"id": "u053", "kind": "list-item", "locator": "body:L107-L107", "preview": "- [PTX ISA 9.0: `tcgen05.ld`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld)", "sha256": "239281ff0e000a65d4c317dadbe3d2e97a6e2e3a1cb7be5b4fbdabccfe02d633"}, {"id": "u054", "kind": "list-item", "locator": "body:L108-L108", "preview": "- [PTX ISA 9.0: `tcgen05.commit`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen-async-sync-operations-commit)", "sha256": "c69ca2bbb4309efad8d395fa08712c1df40146c6081c9fc7197d604b4a9fb4fa"}, {"id": "u055", "kind": "list-item", "locator": "body:L109-L109", "preview": "- [CUTLASS 4.5.0 SM100 Python tutorial](https://github.com/NVIDIA/cutlass/tree/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm)", "sha256": "42eee452d016fa75699f23e63de919826e07c0ff194b091beae8c3b06db6ffa9"}], "confidence_claimed": "verified", "headings": ["What TMEM is", "Addressing and warp access", "Allocation lifecycle", "Column budgeting", "Completion and ordering", "Data movement", "Staging accumulators", "CUTLASS 4.5.0 Python DSL", "Primary references"], "id": "hw-tmem", "path": "wiki/hardware/tmem.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/prs/cutlass/PR-2139.md", "revision": "ca4fdbea", "url": "https://github.com/NVIDIA/cutlass/pull/2139"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-ptx-isa-sm100", "pr-cutlass-2139"], "title": "Tensor Memory (TMEM)", "type": "hardware", "unresolved_source_ids": [], "version_sensitive": null} {"body_sha256": "5e2f98b996033253da1252cec281af40f1f858fa39675a8296b17564863d76fe", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "DeepGEMM is DeepSeek's open-source tensor-core kernel library. This page describes the FP8 GEMM paths at commit [`891d57b4db1071624b5c8fa0d1e51cb317fa709f`](https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa70", "sha256": "edc884f2b15966309a2f894d85f5e0cc5e7f6e8be752415ddc8b5c870d4fcc8f"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "DeepGEMM contains multiple kernels and APIs; the local SM90 and SM100 files below are representative FP8 1D1D implementations, not the whole library.", "sha256": "07ce76e8768d979abddde871720e0182365b4bdfa27b3ca467b4c1c4d823efb9"}, {"id": "u003", "kind": "prose", "locator": "body:L11-L11", "preview": "The [DeepSeek-V3 Technical Report v2](https://arxiv.org/abs/2412.19437v2) defines the training scheme that motivates this path:", "sha256": "d5d25beeba794a452ddfdbfb7f2711ddecc946602125840ec83ddc44a8a70211"}, {"id": "u004", "kind": "list-item", "locator": "body:L13-L13", "preview": "- activations are grouped per token and per 128 channels, forming `1 x 128` tiles;", "sha256": "d7523bee1428dc93fed5ab553055f979f4a735bf9765274f1071846fe85f6b7a"}, {"id": "u005", "kind": "list-item", "locator": "body:L14-L14", "preview": "- weights are grouped per 128 input channels and 128 output channels, forming `128 x 128` blocks; and", "sha256": "62bb926d275734b391c879539ab2d755a9e8816843dfb6307f76326ddab70563"}, {"id": "u006", "kind": "list-item", "locator": "body:L15-L15", "preview": "- smaller groups let scales adapt more locally, which the paper says better accommodates outliers. This is a scoped accuracy motivation, not a guarantee that quantization error disappears.", "sha256": "ff206a53bb27abcfe054b691bd34313268d95364b3b5358dacc0b02d1a3eb51a"}, {"id": "u007", "kind": "prose", "locator": "body:L17-L17", "preview": "Scale representation is architecture-specific in the pinned DeepGEMM interface. SM90 consumes FP32 scale factors. SM100 consumes packed UE8M0 factors, four UE8M0 values per `torch.int`. The pinned SM90 1D1D kernel fixes `BLOCK_K == 128`; th", "sha256": "e6026ff271a6b450b3b13f3d8add4c8296916a6e5f637c409363b33b5975c990"}, {"id": "u008", "kind": "prose", "locator": "body:L21-L21", "preview": "The DeepSeek-V3 report characterizes H800 FP8 Tensor Core accumulation as retaining about 14 bits. Its `Nc=128` strategy accumulates 128 elements of the GEMM inner dimension\u2014four WGMMAs in the described configuration\u2014before moving the parti", "sha256": "c24f17362025a8f052e005acfbb16c91b9ec59729dff0831ba4ec2b0ff77d245"}, {"id": "u009", "kind": "prose", "locator": "body:L23-L23", "preview": "The pinned [`sm90_fp8_gemm_1d1d.cuh`](../../artifacts/kernels/deepgemm/full/sm90_fp8_gemm_1d1d.cuh) implements that structure directly:", "sha256": "28c625a2df66c12415c1fdbca2c390ce628c03257471e82b88294d81adb12a6e"}, {"id": "u010", "kind": "list-item", "locator": "body:L25-L25", "preview": "1. A math warp-group owns `float accum[...]` for WGMMA output and zero-initialized `float final_accum[...]` for promoted results.", "sha256": "017785087339586990befb86eed10f456ab3fdea6e55f20099e25f7e3aca49af"}, {"id": "u011", "kind": "list-item", "locator": "body:L26-L26", "preview": "2. For each 128-element K block, it reads the A and B FP32 factors from shared memory, issues `BLOCK_K / WGMMA::K` WGMMA operations, commits the group, and waits for completion.", "sha256": "16af9fe856a77026821209080a4bb5d64b1b710abe0540ebb08e107d52a0234a"}, {"id": "u012", "kind": "list-item", "locator": "body:L27-L27", "preview": "3. The `Promote with scales` loop multiplies each partial result by its A and B factors and adds it to `final_accum`.", "sha256": "4c32846882c303ba304d47e97688ddab5d2ff00cf6391d834a2a91bbf4d48ff2"}, {"id": "u013", "kind": "list-item", "locator": "body:L28-L28", "preview": "4. After all K blocks, the kernel stages the final FP32 values for the epilogue/store path.", "sha256": "f7783a1c6fb5faf02bc3b5e1de09015b518d2f5e4cdf3e20f55cca904a2b495d"}, {"id": "u014", "kind": "prose", "locator": "body:L30-L30", "preview": "The exact upstream file is the reproducible reference. It should not be replaced by a sketch using half accumulators or by deriving the promotion interval from WGMMA's output-N tile.", "sha256": "925a0225db7cd7132a69208d817eb1af5a27db511f4f65a66e43b6c105290cd5"}, {"id": "u015", "kind": "prose", "locator": "body:L34-L34", "preview": "The pinned [`sm100_fp8_gemm_1d1d.cuh`](../../artifacts/kernels/deepgemm/full/sm100_fp8_gemm_1d1d.cuh) uses a different data path:", "sha256": "32baacf395d6dca95c445def85a6d35ecd549ba78a9d23a33a0ac511c5b3c116"}, {"id": "u016", "kind": "list-item", "locator": "body:L36-L36", "preview": "1. It creates a block-scaled UMMA instruction descriptor with a `float` accumulator type and `cutlass::float_ue8m0_t` scale type.", "sha256": "e4d0d03fdc026620956fc648d7b9b500f2c77ae9a96e8c85c028f1fa8fa08f86"}, {"id": "u017", "kind": "list-item", "locator": "body:L37-L37", "preview": "2. TMA loads packed scale-factor data into shared memory. UTCCP copies selected scale blocks from shared memory into dedicated SFA and SFB columns in TMEM.", "sha256": "ea3a550a3695844dd1292293e91418bd8514f42d2b2da9c09981584452bfea44"}, {"id": "u018", "kind": "list-item", "locator": "body:L38-L38", "preview": "3. The elected issuing thread calls the SM100 UMMA wrapper with shared-memory operand descriptors, the TMEM accumulator column, a runtime instruction descriptor containing scale IDs, and the two TMEM scale addresses.", "sha256": "e405f66e1b53d88218e2e20a61440d777fcbc83e5db36be347d089df597d0f66"}, {"id": "u019", "kind": "list-item", "locator": "body:L39-L39", "preview": "4. The epilogue drains the TMEM accumulator after the kernel's full/empty barrier protocol says it is ready.", "sha256": "255120ab2662ebea058a8ececcabcbb648e337d16a860d70134dc24994ae88fa"}, {"id": "u020", "kind": "prose", "locator": "body:L41-L41", "preview": "This path does not contain the SM90 `final_accum` CUDA-core promotion loop. That difference does not justify a blanket claim that every tcgen05 accumulator mode or datatype has \"full FP32\" behavior; the accumulator type and instruction form", "sha256": "032e89c865af65e49fd21968c9ccf8e1074c5932b40d2e28b7444d03ed3cb6b8"}, {"id": "u021", "kind": "prose", "locator": "body:L45-L45", "preview": "The pinned interface distinguishes three workload arrangements rather than treating all of them as M-varying layouts:", "sha256": "929dc28b5052e9d9082a478f9145da75cf574dbe40e0fe8ce42d9881ea6fa20d"}, {"id": "u022", "kind": "table-row", "locator": "body:L49-L49", "preview": "| Interface family | Varying group dimension | Fixed dimensions | Work metadata | | M-grouped contiguous | M | N and K | Either a group index for each packed M row or a prefix-sum M layout, depending on the selected option |", "sha256": "f7573a4ed8c1905b6389177ee297f27b173966c755b256e452c45cb74823ae95"}, {"id": "u023", "kind": "table-row", "locator": "body:L50-L50", "preview": "| Interface family | Varying group dimension | Fixed dimensions | Work metadata | | M-grouped masked | Valid M within each `[G, M, K]` allocation | Maximum M, N, and K tensor extents | An integer `masked_m[G]` vector holding each group's va", "sha256": "be7ea9aa4dce872c95db03488e96c249b3c54d4d251596470d9ad9425f682ced"}, {"id": "u024", "kind": "table-row", "locator": "body:L51-L51", "preview": "| Interface family | Varying group dimension | Fixed dimensions | Work metadata | | K-grouped contiguous | K | M and N | Per-group K lengths plus their device tensor; used for weight-gradient-style grouped GEMM |", "sha256": "7310a12d3f4e4e0fda19fded69bf1350f734d99d6e2604faa989f64702635f24"}, {"id": "u025", "kind": "prose", "locator": "body:L53-L53", "preview": "Contiguous M grouping is intended for variable per-expert token counts in training forward or inference prefill. Masked M grouping keeps fixed allocations suitable for CUDA-graph decode while limiting work to each group's valid M. The pinne", "sha256": "70a8f3bbeae80bf8c0c0aac86aa2c0c3660c948dc1df5e3bc0cb1ae46c2dfff2"}, {"id": "u026", "kind": "prose", "locator": "body:L57-L57", "preview": "DeepGEMM generates and compiles kernel source at runtime. At the pinned commit, the compiler defaults to NVCC. Setting `DG_JIT_USE_NVRTC=1` selects the optional NVRTC path; the project warns that this may reduce performance for some cases.", "sha256": "f913b537fa51d19b4b9fcbe96f968fb465d7a606e1e7ec5579ce26905c4995d6"}, {"id": "u027", "kind": "prose", "locator": "body:L59-L59", "preview": "The cache key includes the kernel name, compiler signature, compiler flags, and generated source. A cache hit reuses the existing kernel runtime; a miss compiles a CUBIN in a temporary directory and then publishes the completed cache entry.", "sha256": "b35a525c4472aae9970a77f0af0207eb0b614a25db706f4facec3b0bfcae106e"}, {"id": "u028", "kind": "prose", "locator": "body:L63-L63", "preview": "For the pinned FP8 interface, SM90 supports NT only: A is non-transposed and B is supplied in the representation used for `A @ B.T`. SM100 exposes dense `fp8_gemm_{nt, nn, tn, tt}` variants. Its implementation propagates the selected operan", "sha256": "45eee831267e7730526a83b7bb4f986d8ff85c414f93e0cd755c424951e96ab3"}, {"id": "u029", "kind": "prose", "locator": "body:L67-L67", "preview": "The pinned README reports that DeepGEMM reached **up to 1550 TFLOPS on H800** in an April 2025 news item. It does not bind that peak to `M=N=K=4096`, state approximately 90% utilization, or preserve enough benchmark conditions for reproduct", "sha256": "fc2a0fd111e796b31a0c7e0947d44cc55e0b67eb0dda053f25f0d4dff9835a38"}, {"id": "u030", "kind": "list-item", "locator": "body:L71-L71", "preview": "- Input transposition, FP8 casting, and scale-layout preparation are separate from the optimized GEMM kernels; the project supplies utilities but warns they may be slower than fusing the work into producers.", "sha256": "13b031a9e940905a676b7e3e9b18cbd3d783a31dc4a622a7231e0c33147cdd46"}, {"id": "u031", "kind": "list-item", "locator": "body:L72-L72", "preview": "- M-grouped contiguous segments must satisfy the configured M/K alignment. Masked mode uses valid-length integers, while K-grouped mode has different shapes and architecture-specific layout variants.", "sha256": "a98c5f037a06bb31517448c4705d346c2b1164baca656c828855dfa1b9c179f4"}, {"id": "u032", "kind": "list-item", "locator": "body:L73-L73", "preview": "- JIT cache misses add compilation latency, and NVRTC is optional rather than the default.", "sha256": "baf5e541e3f2c2af10edc3b9a7ea55312df051da85ac772dfced929f7d872e91"}, {"id": "u033", "kind": "list-item", "locator": "body:L74-L74", "preview": "- Fine-grained scaling's accuracy and cost tradeoffs depend on quantization recipe, target architecture, scale preparation, fusion, and workload. There is no universal per-tensor-scaling penalty or single \"use only for outliers\" rule.", "sha256": "f9ee05e0618cef4fc4736191e6043939ccd7187cf5bb95ffcfe425f7633e66ad"}, {"id": "u034", "kind": "list-item", "locator": "body:L78-L78", "preview": "- [DeepGEMM README at commit `891d57b`](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md)", "sha256": "a8115b879e294785bbc27931facc99142bd71a23cf5bb9c6d0ff177a40b32e03"}, {"id": "u035", "kind": "list-item", "locator": "body:L79-L79", "preview": "- [DeepGEMM GEMM API at commit `891d57b`](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp)", "sha256": "6344a99161ed183b81d6f0677ac4d26de38cc9974fed0d20c38bf7432a9ca253"}, {"id": "u036", "kind": "list-item", "locator": "body:L80-L80", "preview": "- [DeepGEMM JIT compiler at commit `891d57b`](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/jit/compiler.hpp)", "sha256": "256e0bde5da762844f66e366cd65a297e4093cbaebe7b8369f2d20c4d87aa958"}, {"id": "u037", "kind": "list-item", "locator": "body:L81-L81", "preview": "- [DeepSeek-V3 Technical Report v2](https://arxiv.org/abs/2412.19437v2)", "sha256": "15d831659fae7c38fffb3b6766c08144ff0af11075d3445e73179f7691e7b4e5"}, {"id": "u038", "kind": "prose", "locator": "body:L85-L85", "preview": "Local verbatim upstream code lives in [`artifacts/kernels/deepgemm/full/`](../../artifacts/kernels/deepgemm/full/) and is pinned by [`PROVENANCE.yaml`](../../artifacts/kernels/deepgemm/full/PROVENANCE.yaml) to commit `891d57b4db1071624b5c8f", "sha256": "5fef9de55b64a522e42b6cfb7faad3d0a73df75e4c883e89431c4595916e569f"}, {"id": "u039", "kind": "prose", "locator": "body:L87-L87", "preview": "Query the page and its attached code with:", "sha256": "0da0b6e07247d8d8ebfa9751fbc20aba136debe8f0a07212223182e717bf84d1"}, {"id": "u040", "kind": "code", "locator": "body:L89-L91", "preview": "```bash python3 scripts/get_page.py kernel-deepgemm --include-code ```", "sha256": "a2051b59354d2536049b6650555806c0410635aa728921831a30c1e1855977b1"}, {"id": "u041", "kind": "prose", "locator": "body:L93-L93", "preview": "The following is a verbatim fragment of the SM90 promotion loop; the linked full file supplies its surrounding declarations and loop bounds:", "sha256": "806d5647140f861cdf5ae0117bbbf910630168e069a638198e6ff844f98594ed"}, {"id": "u042", "kind": "code", "locator": "body:L95-L102", "preview": "```cpp const float &scale_b_0 = scales_b[i].x; const float &scale_b_1 = scales_b[i].y; final_accum[i * 4 + 0] += scale_a_0 * scale_b_0 * accum[i * 4 + 0]; final_accum[i * 4 + 1] += scale_a_0 * scale_b_1 * accum[i * 4 + 1]; final_accum[i * 4", "sha256": "7d801ab9fe44c41c62fdf33ba40c8a04c0057328187ea3c19e0f056a5b0ecca0"}], "confidence_claimed": "source-reported", "headings": ["Verified Scope", "Fine-Grained Quantization", "SM90: WGMMA and CUDA-Core Promotion", "SM100: Native Block-Scaled UMMA", "Grouped GEMM Interfaces", "JIT Compilation", "Operand Layouts", "Performance Evidence", "Practical Boundaries", "Pinned Sources", "Full Reference Implementation"], "id": "kernel-deepgemm", "path": "wiki/kernels/deepgemm.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/deepgemm.md", "url": "https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f"}, {"path": "sources/prs/DeepGEMM/PR-304.md", "revision": "7f2a703e", "url": "https://github.com/deepseek-ai/DeepGEMM/pull/304"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-deepgemm", "pr-deepgemm-304"], "title": "DeepGEMM \u2014 FP8 GEMM with Fine-Grained Scaling", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} {"body_sha256": "eb0beda6d21999eb5a9af51fbb7869f46f78dcb0a29c7d5df1fbde5a0b49b875", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "FlashAttention-4 is an attention algorithm and CuTe DSL implementation designed around Blackwell's asymmetric throughput: B200 tensor-core throughput grows much more than its special-function and shared-memory resources. This page separates", "sha256": "1e410d8f43116d7362371c8f774303c9f6c1f9cf76d3978e901efadfdc47845d"}, {"id": "u002", "kind": "list-item", "locator": "body:L7-L7", "preview": "- the [FA4 paper v1](https://arxiv.org/abs/2603.05451v1), which studies the SM100/B200 algorithm and reports the authors' measurements; and", "sha256": "ff06a9c75bcd69176044dff5a2e64357af716d6fc527b4b5a918cdf9a5cd9a05"}, {"id": "u003", "kind": "list-item", "locator": "body:L8-L8", "preview": "- the public implementation at Dao-AILab/flash-attention commit [`a369df707e1980fb328abcc1733e3457ec10155f`](https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute), which is the source sn", "sha256": "582bae2812e5a0d1842e5dabcaa3a111d9723410fdcb5287165744e1b7b680f9"}, {"id": "u004", "kind": "prose", "locator": "body:L10-L10", "preview": "The paper implementation is written in CuTe DSL. Its compilation comparison is against corresponding FA3 CUTLASS kernels: forward compiles in 2.5 seconds instead of 55 seconds, and backward in 1.4 seconds instead of 45 seconds. Those are si", "sha256": "2266d20d5c09dbaa973f352ea1400662ca84a3b9038021272b1258c9deefc4e0"}, {"id": "u005", "kind": "prose", "locator": "body:L14-L14", "preview": "The forward kernel assigns two output tiles of 128 query rows to one CTA. One MMA warp issues the matrix products, two four-warp softmax groups serve the two output tiles, and a correction warpgroup handles accumulator corrections. The scor", "sha256": "f72a68c72ef30e73a848c40dcf32c36a953ba4f63d07dea896a729aaf5fad961"}, {"id": "u006", "kind": "prose", "locator": "body:L16-L16", "preview": "This is more than ordinary K/V double buffering: the two alternating objects are output tiles with separate softmax state. The exact synchronization and stage ownership are implementation details; the invented one-loop pseudocode formerly o", "sha256": "c191fae7db0b6d0f0e9f84bffa3a6831477e6cfdabf2886659511eed72e8c34a"}, {"id": "u007", "kind": "prose", "locator": "body:L20-L20", "preview": "FA4 does not replace every hardware exponential. The paper selects only about 10-25% of entries for software evaluation on FMA units and leaves the rest on the hardware MUFU `ex2` path, allowing both resources to contribute.", "sha256": "684eaf9635c6bb161d267b05e4f15ce13f54b79e00d86174511b80879bedc800"}, {"id": "u008", "kind": "prose", "locator": "body:L22-L22", "preview": "For a software-selected value, the published range reduction writes `x = n + f` with `n = floor(x)` and `f` in `[0, 1)`, evaluates a degree-3 polynomial for `2**f`, and reconstructs the scale from `n`. The function below is a scalar referen", "sha256": "1baf75455f827398faef384641ff9669089152483a730956c6aa7ea364948060"}, {"id": "u009", "kind": "code", "locator": "body:L24-L32", "preview": "```python import math def fa4_blog_exp2_reference(x: float) -> float: n = math.floor(x) f = x - n polynomial = 1.0 + f * (0.6951 + f * (0.2276 + f * 0.0771)) return math.ldexp(polynomial, n) ```", "sha256": "9298f823e2f79bb1929071207fa66371dd4e82a8662ce232e6ebe29f493f1631"}, {"id": "u010", "kind": "prose", "locator": "body:L34-L34", "preview": "No standalone four-times software-versus-hardware exponential result is asserted here. The paper evaluates the combined kernel and its ablations rather than establishing that former page claim.", "sha256": "4f249df50219c8b338ef11f0dd2ed6b32aaeb17be2bb4157dafbeba464059290"}, {"id": "u011", "kind": "prose", "locator": "body:L38-L38", "preview": "Ordinary online softmax updates the row maximum and rescales accumulated state as each score block arrives. FA4 permits its retained maximum to lag: it resynchronizes only when the new block maximum exceeds the retained maximum by more than", "sha256": "9da61724cbae84193d5788fca9ce25e3c579a072c13fd16b8d54605f7ae86bc5"}, {"id": "u012", "kind": "prose", "locator": "body:L40-L40", "preview": "When a rescale is skipped, subsequent probabilities are still evaluated relative to the retained old maximum, and auxiliary statistics track the delayed normalization. The algorithm performs final renormalization at the end. Comparing absol", "sha256": "3177714cb704f95ebae51a833c10979b765bc122ccf6a14cd176cff5677c7b0d"}, {"id": "u013", "kind": "prose", "locator": "body:L44-L44", "preview": "The paper maps five backward GEMMs to two-CTA tcgen05 MMA with `M=256, N=128, K=128`. For those operations, the paired CTAs can share operand B, which the authors describe as roughly halving the shared-memory reads for that operand. This is", "sha256": "8d23ff074a9499de040bca63da0fe4d30d194b537de181f4ca508d2bc79bc11a"}, {"id": "u014", "kind": "prose", "locator": "body:L46-L46", "preview": "For dQ, each CTA computes a half of dS and exchanges that half through distributed shared memory so both CTAs can form the required dQ product. The two-CTA organization also doubles the dQ reduction tile along N and thereby halves the numbe", "sha256": "6a3c37791017a6a89f4f7716abe8d963db55672339299885a32d6ed573fe6662"}, {"id": "u015", "kind": "prose", "locator": "body:L50-L50", "preview": "At commit `a369df7`:", "sha256": "7113e059ccec8966d4256da23af091cdc161af598c071f055b0a1482b94c73e0"}, {"id": "u016", "kind": "list-item", "locator": "body:L52-L52", "preview": "- [`flash_fwd_sm100.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py) builds tcgen05 operations through `make_trivial_tiled_mma`, allocates TMEM through `Tmem", "sha256": "c521af8e33a110418a404d89f23ef9da68a4ad15215aeac9d228dfa9045847cf"}, {"id": "u017", "kind": "list-item", "locator": "body:L53-L53", "preview": "- The forward path constructs TMA atoms for Q, K, and V where its configuration enables them. It also has non-TMA Q and paged-K/V copy paths, so TMA use is not unconditional.", "sha256": "4c5c693b748161d30902f58dbd51ef0a42316b48ac01e7d28f12bcafddee5fea"}, {"id": "u018", "kind": "list-item", "locator": "body:L54-L54", "preview": "- [`flash_bwd_sm100.py`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_bwd_sm100.py) constructs the five backward MMA operations and the two-CTA exchange/reduction pipelines", "sha256": "815a65094e11852de9574447bc0d50adf1df3ad935e82caf224f979c7d21448c"}, {"id": "u019", "kind": "list-item", "locator": "body:L55-L55", "preview": "- The package README describes CuTe DSL attention for Hopper and Blackwell, and the tree contains SM90, SM100, and SM120 dispatch modules. The paper's FA4 result remains SM100/B200-specific; package architecture coverage should not be used ", "sha256": "2cc2823a1a7c188475de9f29140603b203621cbb842f595398a7083e604ffb08"}, {"id": "u020", "kind": "list-item", "locator": "body:L56-L56", "preview": "- The pinned [`pyproject.toml`](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/pyproject.toml) requires `nvidia-cutlass-dsl==4.6.0.dev0`. That is a property of this source snapshot", "sha256": "2560ae01ebeeaa288f2229ebb7f069cf4864ff5ec4988bf4eceb965be3630659"}, {"id": "u021", "kind": "list-item", "locator": "body:L57-L57", "preview": "- Forward and backward accept FP16/BF16. An FP8 benchmark/bring-up script is present, but at this revision it explicitly expects the FA4 FP8 call to fail until support is implemented.", "sha256": "feeb126d7d71e6c313ef6c9da1aab21d2d319602db6e2c3de1fd9437053222c4"}, {"id": "u022", "kind": "prose", "locator": "body:L61-L61", "preview": "The first-party sources contain two different source-reported peak values. Paper v1 reports **up to 1613 TFLOPS/s on B200 BF16, or 71% of the peak convention used by the authors**. Tri Dao's blog reports **up to 1605 TFLOPS/s, also labeled ", "sha256": "1c2e20a8e65ba15f72f872cf74d6a28352ebfb75c641d12fde648b98cf037bd7"}, {"id": "u023", "kind": "prose", "locator": "body:L63-L63", "preview": "The paper's benchmark suite spans sequence lengths from 1K through 32K and multiple query/value head-dimension pairs under a fixed total-token convention. Neither textual source establishes the former single row that attached 1605 TFLOPS, 7", "sha256": "ffecfa7f2cb14d406a31606a14172909bfbbe0db54c419239eb5ae2e8ecb70ed"}, {"id": "u024", "kind": "list-item", "locator": "body:L67-L67", "preview": "- Choose the package path only for a supported architecture, dtype, head-dimension pair, masking mode, and feature set at the exact revision in use.", "sha256": "762ad956d4509aee447615faf6b80f1a258782fafdec6dd079f3e3e3c6ac920d"}, {"id": "u025", "kind": "list-item", "locator": "body:L68-L68", "preview": "- Do not treat sequence length 1024 or head dimension 128 as universal crossover or optimum values. Compare against the relevant cuDNN, framework, or other kernel path on the actual workload.", "sha256": "8375983f1d4f7ce52c849304a3aad867c8c2b667afa82f6b1ebf061cf544f372"}, {"id": "u026", "kind": "list-item", "locator": "body:L69-L69", "preview": "- Compilation speed and runtime speed are separate measurements. The paper's compile comparison does not prove an equivalent runtime factor.", "sha256": "90d8676cddf785613147e85842b741464ae3ae1fb640fd08252120ddd68f6785"}, {"id": "u027", "kind": "list-item", "locator": "body:L70-L70", "preview": "- Treat source-reported B200 numbers as unreproduced unless the same software, clock/power settings, tensor shapes, timing region, warmup, and FLOP convention are available.", "sha256": "896b8ebc0778eb869845e20ff98d507909b34ea91491af6057824a37de4f85a0"}, {"id": "u028", "kind": "list-item", "locator": "body:L74-L74", "preview": "- [FlashAttention-4 paper, arXiv v1](https://arxiv.org/abs/2603.05451v1)", "sha256": "65c89a97ddf06d9514d19588fe1266a84f9ab8f7dc02d57f8732947385eabe66"}, {"id": "u029", "kind": "list-item", "locator": "body:L75-L75", "preview": "- [Tri Dao's FlashAttention-4 blog](https://tridao.me/blog/2026/flash4/)", "sha256": "0c88bc6176d024bb44f1ccb80556697f7868de9d35d59a10546d17b279974d99"}, {"id": "u030", "kind": "list-item", "locator": "body:L76-L76", "preview": "- [FA4 CuTe DSL package at commit `a369df7`](https://github.com/Dao-AILab/flash-attention/tree/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute)", "sha256": "98fa8065e41a96242bc726acab9d3101d53459cf3260434fc27bf76545ee39e5"}, {"id": "u031", "kind": "list-item", "locator": "body:L77-L77", "preview": "- [Pinned SM100 forward source](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_fwd_sm100.py)", "sha256": "4938cdf2c16b4de76f90e1e8a33103be66d267abb2adb5a030d8fc510818857b"}, {"id": "u032", "kind": "list-item", "locator": "body:L78-L78", "preview": "- [Pinned SM100 backward source](https://github.com/Dao-AILab/flash-attention/blob/a369df707e1980fb328abcc1733e3457ec10155f/flash_attn/cute/flash_bwd_sm100.py)", "sha256": "afee82a4bbcd20dd427756fd46204f17af4ce2985c49fe7e4aca940035bb47b6"}, {"id": "u033", "kind": "prose", "locator": "body:L82-L82", "preview": "The local [`full/`](../../artifacts/kernels/flash-attention-4/full/) bundle is a byte-verified, verbatim **adjacent NVIDIA CUTLASS SM100 FMHA backward MLA example**, pinned to CUTLASS commit `0e026982`. It is not the Dao-AILab FA4 implement", "sha256": "a85ac9e12fb38e14987c53ae33ffdfc5ca91b605e4368033767b3b1cecebc738"}, {"id": "u034", "kind": "prose", "locator": "body:L84-L84", "preview": "Query the page and its attached local references with:", "sha256": "38c4142a3241d438fee5d4ab59db0c7c243b0825b789d47c12e6f4d62b2a4092"}, {"id": "u035", "kind": "code", "locator": "body:L86-L88", "preview": "```bash python3 scripts/get_page.py kernel-flash-attention-4 --include-code ```", "sha256": "dbaa5f540bdab5755bcdb9295bc1637f3e10a533091792abfe6b693678d4bd54"}], "confidence_claimed": "source-reported", "headings": ["Verified Scope", "Ping-Pong Forward Schedule", "Partial Software Exponential", "Conditional Softmax Rescaling", "Two-CTA Backward", "Pinned Implementation Notes", "Performance Evidence", "Practical Boundaries", "Pinned Sources", "Local Code References"], "id": "kernel-flash-attention-4", "path": "wiki/kernels/flash-attention-4.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/flash-attention-4.md", "url": "https://arxiv.org/abs/2603.05451v1"}, {"path": "sources/blogs/flash-attention-4.md", "url": "https://tridao.me/blog/2026/flash4/"}], "risk_flags": ["code", "ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-flash-attention-4", "blog-flash-attention-4"], "title": "FlashAttention-4", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} {"body_sha256": "73e3ffa9038dafda7f4083be34a1dabae23531849cb738ba46e7914bfcff9857", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "[Dao-AILab/flash-attention PR 2441](https://github.com/Dao-AILab/flash-attention/pull/2441), merged as [`f219c89c886c6ccbf9d3dbd9fe41b11ac64e9df8`](https://github.com/Dao-AILab/flash-attention/commit/f219c89c886c6ccbf9d3dbd9fe41b11ac64e9df8", "sha256": "4f63a059a81265b7e1cc2d93083303f2ea9498f4b056d7f2855f19c4807dabbf"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "The top-k path is deliberately narrower than a generic paged sparse-attention API. It requires packed GQA/MQA, `qhead_per_kvhead == 128`, and the cp.async K/V gather path. Its constructor defaults to top-k length 2048 and requires the confi", "sha256": "da9c2fe6ef32dd275484f90f2353c35bb3d70a61c4879680f449bfcd5bb081d1"}, {"id": "u003", "kind": "prose", "locator": "body:L11-L11", "preview": "The pinned merge keeps three responsibilities distinguishable:", "sha256": "7c8e584dcb6ba3f5fec244c339c7c94b0aa5316af71a4de934df4270db9bb7f3"}, {"id": "u004", "kind": "list-item", "locator": "body:L13-L13", "preview": "- [`topk_gather_kv.py`](../../artifacts/prs/flash-attention/PR-2441/key-files/flash_attn/cute/topk_gather_kv.py) loads caller-provided top-k indices, forms indexed K/V addresses, issues cp.async copies, and optionally constructs validity bi", "sha256": "73efde420784dfeb2b7aa524d5aa4affeab8241ce307f37e1e5353236b1008f7"}, {"id": "u005", "kind": "list-item", "locator": "body:L14-L14", "preview": "- [`tile_scheduler.py`](../../artifacts/prs/flash-attention/PR-2441/key-files/flash_attn/cute/tile_scheduler.py) supplies the tile-scheduler implementations selected by the MLA kernel.", "sha256": "e609fba486c07cb0513067e5110f4a56712998c460ac1190ae764e870ea1df45"}, {"id": "u006", "kind": "list-item", "locator": "body:L15-L15", "preview": "- [`flash_fwd_mla_sm100.py`](../../artifacts/prs/flash-attention/PR-2441/key-files/flash_attn/cute/flash_fwd_mla_sm100.py) integrates gather, scheduling, two-CTA tcgen05 MMA, TMEM accumulators, softmax, and the output path.", "sha256": "82cb38d28021580714bc04318a4ef997cb376381d574b4edf67e7311c34059d2"}, {"id": "u007", "kind": "prose", "locator": "body:L17-L17", "preview": "This separation matters for evaluation. Arithmetic work follows the effective top-k length, whereas memory addresses follow the caller's index set and use an indexed gather/optional-bitmask path. Reduced attention FLOPs therefore do not pro", "sha256": "2daefc28898a6cce495bbd7fda384f2c0419b4e5882ed996444d0f964d074001"}, {"id": "u008", "kind": "prose", "locator": "body:L21-L21", "preview": "The PR author reports an initial saturating-decode comparison for batch 512, `seqlen_q=1`, `seqlen_k=16384`, 128 query heads, the 64/512 MLA shape, and top-k length 2048:", "sha256": "b3035685847df985b94a47aa4fa504d1371190e34396dde9ccd741363aa0a602"}, {"id": "u009", "kind": "table-row", "locator": "body:L25-L25", "preview": "| Variant | Author-reported latency | Author-reported throughput | | DSA, no bitmask; indices assumed in bounds | 0.31 ms | 955.47 TFLOPS |", "sha256": "a24fbe88c47cc40ebc98942a7df9423dbbbb468b341e1bb5aebed73761544c38"}, {"id": "u010", "kind": "table-row", "locator": "body:L26-L26", "preview": "| Variant | Author-reported latency | Author-reported throughput | | DSA, validity bitmask | 0.33 ms | 898.08 TFLOPS |", "sha256": "e7bf57c1383e05c0ff49169009cdc8f758023a7591f7c01c3ed64e68125b906f"}, {"id": "u011", "kind": "table-row", "locator": "body:L27-L27", "preview": "| Variant | Author-reported latency | Author-reported throughput | | Vanilla MLA baseline | 1.98 ms | 1180.70 TFLOPS |", "sha256": "8955e33c28ba6617ad774374f32687f50790bc825342c54af110832cf10f2fd8"}, {"id": "u012", "kind": "prose", "locator": "body:L29-L29", "preview": "These PR-description observations are not reproduced here. The PR body does not name the exact GPU model, dtype, clocks/power state, software environment, timing protocol, or run-to-run variation for the rows. They therefore remain prose wi", "sha256": "8f464ccdce13dd8c03fcdf509014a32437181e4d6a2d29317aa718b9aadb63fd"}, {"id": "u013", "kind": "list-item", "locator": "body:L33-L33", "preview": "- Confirm the exact MQA packing, 64/512 dimensions, top-k divisibility, index layout, and bounds contract used by the target revision.", "sha256": "0bd5d403083d1c9c0011d53972bb920f173d638cc6297d0fa560921a2d139f9b"}, {"id": "u014", "kind": "list-item", "locator": "body:L34-L34", "preview": "- Treat no-bitmask results as valid only when the caller guarantees every index is in bounds; otherwise preserve the validity path.", "sha256": "b76830a1691f4a2f7e8439909d3a544660928842234c0f5a41c0cb587eca8bdc"}, {"id": "u015", "kind": "list-item", "locator": "body:L35-L35", "preview": "- Do not infer paged-KV compatibility from the exposed parameter name: merge `f219c89c` rejects page tables in this MLA path.", "sha256": "0d852df017b6e52be08f9f98ecc3426541d04f254d3ece4a4dd788bc77df514e"}, {"id": "u016", "kind": "list-item", "locator": "body:L36-L36", "preview": "- Profile indexed-gather traffic and tensor arithmetic separately, then validate end-to-end accuracy and latency on the full workload.", "sha256": "9ab5962234ff3e14f16d030c48e8abf52d8283ce1b4b6c187393bc55d964aeed"}, {"id": "u017", "kind": "prose", "locator": "body:L40-L40", "preview": "The local PR record and byte-verified bundle identify the integration and its dedicated gather file:", "sha256": "47a761a14371c5670c19841d3279cbd73443fbd8a209713016e33d2693199cad"}, {"id": "u018", "kind": "code", "locator": "body:L42-L49", "preview": "```python from pathlib import Path pr_page = Path(\"sources/prs/flash-attention/PR-2441.md\").read_text() provenance = Path(\"artifacts/prs/flash-attention/PR-2441/PROVENANCE.yaml\").read_text() assert \"flash_fwd_mla_sm100.py\" in pr_page assert", "sha256": "cb588e655e6743eac16572e26541b164f2853fd9aa333125fff8febac7bb5298"}], "confidence_claimed": "source-reported", "headings": ["Verified Scope", "Gather and Scheduling Paths", "Source-Reported Performance", "Transfer Checklist", "Reproducible Source Lookup"], "id": "kernel-flash-attention-sm100-mla-topk", "path": "wiki/kernels/flash-attention-sm100-mla-topk.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/prs/flash-attention/PR-2441.md", "revision": "f219c89c", "url": "https://github.com/Dao-AILab/flash-attention/pull/2441"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["pr-flash-attention-2441"], "title": "FlashAttention SM100 MLA TopK Sparse Forward", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} @@ -16,7 +16,7 @@ {"body_sha256": "11f0038a8e9f681dac4ffd25654362cefa9f94a039e46745ac55f9488a405f80", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "The official NVIDIA rules identify NVFP4 Gated Dual GEMM as Kernel Challenge 3 of the Blackwell NVFP4 Hackathon, open from December 20, 2025 through January 16, 2026. The public GPU Mode problem at challenge-opening commit `c5b2f7c` targets", "sha256": "eb13cd2340105068b914156cb7d5d1e808b3f463ab2acb8a78db6fa1f3c5020c"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "For each batch index, the required result is two block-scaled matrix products sharing the same left operand, with SiLU applied only to the first branch:", "sha256": "a8a01d217423ed48389944aef48ae23ca65b467f846332a0b3a7f9b4f99b1704"}, {"id": "u003", "kind": "code", "locator": "body:L9-L14", "preview": "```python def gated_dual_result(a, b1, b2, scale_a, scale_b1, scale_b2, scaled_mm): gate = scaled_mm(a, b1.T, scale_a, scale_b1) up = scaled_mm(a, b2.T, scale_a, scale_b2) return silu(gate) * up ```", "sha256": "dbebdd2007ca6c67649f785252795530745a124fdd528eef5cdd9a9e3c41974a"}, {"id": "u004", "kind": "prose", "locator": "body:L16-L16", "preview": "This fixes result semantics and branch order. It does not require the submission entry point to use one GPU launch or forbid intermediate storage.", "sha256": "20fbacace5607f8e72cc6cf0b019f48f71788aa01d28640b2e62c771fb961479"}, {"id": "u005", "kind": "table-row", "locator": "body:L22-L22", "preview": "| Tensor | Published dtype | Logical shape | | `a` | NVFP4 E2M1 | `[M,K,L]`, K-major |", "sha256": "c51027e2c1727c7e1eebc42cd4417a53e5e5c7249152c256c969ef3dcf2e456d"}, {"id": "u006", "kind": "table-row", "locator": "body:L23-L23", "preview": "| Tensor | Published dtype | Logical shape | | `b1`, `b2` | NVFP4 E2M1 | `[N,K,L]`, K-major |", "sha256": "80489fc9f617fc818771c3642171bf4225506614fba752cb1466f9983e217ea5"}, {"id": "u007", "kind": "table-row", "locator": "body:L24-L24", "preview": "| Tensor | Published dtype | Logical shape | | `sfa` | FP8 E4M3FNUZ | `[M,K/16,L]`, K-major |", "sha256": "825e69aaf043da6d8f8b6bc86989e1507d940ba110183e27578ec296f251de8a"}, {"id": "u008", "kind": "table-row", "locator": "body:L25-L25", "preview": "| Tensor | Published dtype | Logical shape | | `sfb1`, `sfb2` | FP8 E4M3FNUZ | `[N,K/16,L]`, K-major |", "sha256": "0b77d5e56a0c9e095f5346f5b105840fd07e48ee57eb311855c97903cd89091a"}, {"id": "u009", "kind": "table-row", "locator": "body:L26-L26", "preview": "| Tensor | Published dtype | Logical shape | | `c` | FP16 | `[M,N,L]` |", "sha256": "91c57469d3b6cdb36340525d8683e7a8163d63ff0f0a02ed93b436a665cd15f4"}, {"id": "u010", "kind": "prose", "locator": "body:L28-L28", "preview": "The submission tuple also supplies layout-reordered copies of `sfa`, `sfb1`, and `sfb2`, plus preallocated `c`. `K` is divisible by 256; `M` and `N` must be divisible by the selected MMA tile dimensions. The correctness checker uses `rtol=1", "sha256": "a9160f4022cf1f167223f0ac260c9a5f45e6e26c1bd33db1bd277f2ef7a29e92"}, {"id": "u011", "kind": "prose", "locator": "body:L30-L30", "preview": "The pinned upstream files contain one dtype-label inconsistency: `task.yml` and `template.py` call the scales E4M3FNUZ, while `reference.py` constructs `torch.float8_e4m3fn`. A backend must follow the actual submission objects rather than t", "sha256": "bed832ba69cda0b6bef5b927ab33a5a660cb6e0ca5943dd345d1f4a4ea9ffc7d"}, {"id": "u012", "kind": "table-row", "locator": "body:L36-L36", "preview": "| M | N | K | L | Task speed-of-light time (\u00b5s) | | 256 | 4096 | 7168 | 1 | 4.708 |", "sha256": "7411db857d7dcc6631366727895a460e35c7d547a4efc9906e770fea0e5fa848"}, {"id": "u013", "kind": "table-row", "locator": "body:L37-L37", "preview": "| M | N | K | L | Task speed-of-light time (\u00b5s) | | 512 | 4096 | 7168 | 1 | 8.714 |", "sha256": "b4d7a0c0e8410c7ab28ed355f9adaa78d0e41a6955d60489d1eb86411de35d7d"}, {"id": "u014", "kind": "table-row", "locator": "body:L38-L38", "preview": "| M | N | K | L | Task speed-of-light time (\u00b5s) | | 256 | 3072 | 4096 | 1 | 2.125 |", "sha256": "1bce4631f28966ecc639c587fe0a817eedd229d39bcfb63736463d5af08d1173"}, {"id": "u015", "kind": "table-row", "locator": "body:L39-L39", "preview": "| M | N | K | L | Task speed-of-light time (\u00b5s) | | 512 | 3072 | 7168 | 1 | 6.535 |", "sha256": "0d114fedb3663c1f794288403e6bb6237a8fc7403af83090d0c68576a704b7ff"}, {"id": "u016", "kind": "prose", "locator": "body:L41-L41", "preview": "Ranking uses the geometric mean of the four benchmark times. The task explicitly labels the final column a speed-of-light analysis based on the maximum of B200 FP4 Tensor Core math time and DRAM-memory time at a 1.5 GHz clock. These values ", "sha256": "dbd703852dc16a02bacb2b9359f855f7cc6bb5aedbbe12766b5359b12b4c3df3"}, {"id": "u017", "kind": "prose", "locator": "body:L45-L45", "preview": "The shared `a` and `sfa` inputs create an opportunity to reuse data across the two products, and a genuinely fused epilogue can avoid writing both full-precision products to global memory. Those are optimization possibilities, not task guar", "sha256": "b9506a581d543f3f3449095a2de708afaf0cc2d3a4074808d129cf966b6143d6"}, {"id": "u018", "kind": "prose", "locator": "body:L47-L47", "preview": "The public task/reference does not establish a final leaderboard, winning source, single-launch decomposition, physical load count, TMEM accumulator partition, TMA pipeline, CUTLASS schedule, or compute-/memory-bound classification. If an i", "sha256": "7eebde5afbe8c3125f5c62849cf9a347608afc921f13de479637aa8d16a48e84"}, {"id": "u019", "kind": "prose", "locator": "body:L49-L49", "preview": "Compatibility is correspondingly narrower than \u201cany dual-output operation\u201d: a model/backend must match the two same-shaped NVFP4 products, scale storage and layouts, first-branch SiLU, FP16 output, batching, and divisibility rules. Grouped ", "sha256": "de73a447abc831be11c8a20216e9aa9deb202edda82ffd8e4b5fc04f855730f1"}, {"id": "u020", "kind": "prose", "locator": "body:L53-L53", "preview": "The `full/` bundle contains the byte-pinned official `task.yml` from commit `c5b2f7c`; it is the problem specification, not an optimized submission. The unrelated third-party schedule extract and duplicate vLLM MXFP4 MoE patch formerly stor", "sha256": "a38ab1c3e4bbe45a644a36ccc7d310f1d64e15abe5098570c0957bb0d2796105"}, {"id": "u021", "kind": "prose", "locator": "body:L55-L55", "preview": "The `variants/` bundle contains a standard-library semantic reference for two small dense products followed by `SiLU(gate) * up`. Its self-test compares compact and expanded forms and rejects the wrong alternative that gates the second bran", "sha256": "dce2ea09300743713bd166cc06f0b1dc512d956b3e4ab65c423b66a9d18ffc9c"}, {"id": "u022", "kind": "list-item", "locator": "body:L59-L59", "preview": "- [Official challenge rules](https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf)", "sha256": "1f5c0e65cebd680067dc91e0fa9151877d2bab0124675326a188474f3053ddae"}, {"id": "u023", "kind": "list-item", "locator": "body:L60-L60", "preview": "- [Pinned public task](https://github.com/gpu-mode/reference-kernels/blob/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm/task.yml)", "sha256": "0cddd0c030bce47ece25fb01646203081b9a5e6c128f164b8552a7c8a671b84a"}, {"id": "u024", "kind": "list-item", "locator": "body:L61-L61", "preview": "- [Pinned starter template](https://github.com/gpu-mode/reference-kernels/blob/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm/template.py)", "sha256": "9e4752cc881230defb5421be419a54115184bf44943fe7b7f55fc43f8ec8ccc5"}, {"id": "u025", "kind": "list-item", "locator": "body:L62-L62", "preview": "- [Pinned correctness reference](https://github.com/gpu-mode/reference-kernels/blob/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm/reference.py)", "sha256": "33537e2a914a4c1604f1a972e3ff6d137cba06b2e245ee8fa0373bb396f73ce5"}, {"id": "u026", "kind": "prose", "locator": "body:L64-L64", "preview": "Query via:", "sha256": "6abd63e508f9320fdd3b2b0cc5b5f6a000f4462abfd17f9ecc3c4818f4a0f875"}, {"id": "u027", "kind": "code", "locator": "body:L66-L68", "preview": "```bash conda run -n base python scripts/get_page.py kernel-gated-dual-gemm --include-code ```", "sha256": "45d0054e609734a050470028979c69f41c63c9e9f1d7d447ab961af3e447cdbf"}], "confidence_claimed": "source-reported", "headings": ["Verified scope", "Published tensor contract", "Published workloads and theoretical bounds", "Implementation boundary", "Local artifacts", "Primary sources"], "id": "kernel-gated-dual-gemm", "path": "wiki/kernels/gated-dual-gemm.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/contests/gpu-mode-nvfp4/problem-3-gated-dual-gemm.md", "url": "https://github.com/gpu-mode/reference-kernels/tree/c5b2f7c062d5015f29c3a1043cfd04954397944c/problems/nvidia/nvfp4_dual_gemm"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["contest-gpumode-p3"], "title": "GPU Mode NVFP4 Gated Dual GEMM", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} {"body_sha256": "733a51f41033ffcc0ff6fc42c4813e72da9b342f2e17aa629cfbee80177927ce", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "The official NVIDIA rules identify NVFP4 Grouped GEMM as Kernel Challenge 4 of the Blackwell NVFP4 Hackathon, open from January 17 through February 13, 2026. It carries 40% of the four-problem grand-prize score. The corrected public task at", "sha256": "1639b7acf7b23b5b3321612131bdeb27e99062306a5f8603dfffeed74f69e18d"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "\u201cGrouped GEMM\u201d does not imply one universal shape contract. Two relevant interfaces differ materially:", "sha256": "5b62624db91dbdc4fd13581947cfda51890c1a7856fb2d73124be674dbe12784"}, {"id": "u003", "kind": "list-item", "locator": "body:L9-L9", "preview": "- The GPU Mode challenge accepts a list of independent problems; `M_i`, `N_i`, and `K_i` can all differ by group.", "sha256": "b9b8d88536cad39bc38b2372af39b03590aa774e4505cb406874ddcaa776f659"}, {"id": "u004", "kind": "list-item", "locator": "body:L10-L10", "preview": "- DeepGEMM's M-grouped MoE APIs vary M while holding N and K fixed. It separately provides a K-grouped interface for MoE weight backward.", "sha256": "096a1dea2f4799d14e774f4856385805b30814ef51ea5daaa6c7382112c21f76"}, {"id": "u005", "kind": "prose", "locator": "body:L12-L12", "preview": "Neither interface definition proves that a conforming implementation uses exactly one GPU launch.", "sha256": "7d2218cba139ae1b298d6c15bc8defc2c2639149b272d77e8be8a87e6a7c19b5"}, {"id": "u006", "kind": "prose", "locator": "body:L16-L16", "preview": "For each group `i`, the correctness reference computes `C_i = A_i @ B_i.T` with block scales and FP16 output.", "sha256": "821caf2f5b552a4124fd374bc92a0cb5f3ea499948da59300ecb510e2ec28519"}, {"id": "u007", "kind": "table-row", "locator": "body:L20-L20", "preview": "| Per-group value | Published dtype | Logical shape | | `a_i` | packed NVFP4 E2M1, two values per byte | `[M_i,K_i/2,L_i]` |", "sha256": "d34b857726733a2afa24e47f8733e8d6e6f043b335a81b2dc85328bc258e8e30"}, {"id": "u008", "kind": "table-row", "locator": "body:L21-L21", "preview": "| Per-group value | Published dtype | Logical shape | | `b_i` | packed NVFP4 E2M1, two values per byte | `[N_i,K_i/2,L_i]` |", "sha256": "6a6a65388a18d8ec1b7e3c83cf8c579b997842d51ed159132c3cb63f9c6f1bd7"}, {"id": "u009", "kind": "table-row", "locator": "body:L22-L22", "preview": "| Per-group value | Published dtype | Logical shape | | `c_i` | FP16 | `[M_i,N_i,L_i]` |", "sha256": "c30f983df15fb59e81796b93a1a61b5fe6637462a59524c4e36e2ed9f4e4e4eb"}, {"id": "u010", "kind": "table-row", "locator": "body:L23-L23", "preview": "| Per-group value | Published dtype | Logical shape | | `sfa_i` | FP8 E4M3FNUZ in task/template | `[M_i,K_i/16,L_i]` |", "sha256": "6f30839adec711adb38f9c03cf9fb078c7a66a155f9836da11ca60ff7c28b8f3"}, {"id": "u011", "kind": "table-row", "locator": "body:L24-L24", "preview": "| Per-group value | Published dtype | Logical shape | | `sfb_i` | FP8 E4M3FNUZ in task/template | `[N_i,K_i/16,L_i]` |", "sha256": "fb577a5c677951b74064836772973ae414b66c4eb9c75785449ac7326f388696"}, {"id": "u012", "kind": "table-row", "locator": "body:L25-L25", "preview": "| Per-group value | Published dtype | Logical shape | | problem size | integers | `(M_i,N_i,K_i,L_i)` |", "sha256": "5e238b762375b33910bc4972916f29f1235d1a043f2fe6c3ba0f41f3b856e448"}, {"id": "u013", "kind": "prose", "locator": "body:L27-L27", "preview": "The submission object actually contains four lists: logical A/B/C tensors, logical scales, reordered scale copies, and problem sizes. The `task.yml` prose lists only three tuple members, but `task.py`, `template.py`, and `reference.py` expo", "sha256": "7257e0924104d32452daba7c843a68a82c1eaf3d35326fbf8e0be92291040ffe"}, {"id": "u014", "kind": "prose", "locator": "body:L29-L29", "preview": "The pinned upstream files also disagree on the scale dtype suffix: task/template text says E4M3FNUZ, while `reference.py` constructs `torch.float8_e4m3fn`. A submission must follow the actual tensors it receives rather than silently treatin", "sha256": "5e4eee6cc002895da478fe96cc270e4b91af5cd04101f51a2d6cefcab5dd6cf2"}, {"id": "u015", "kind": "prose", "locator": "body:L31-L31", "preview": "For small non-empty ordinary-number matrices, this CPU-only reference isolates the group semantics. Each `b` is stored as `[N,K]`, so its rows are the columns of the mathematical right operand:", "sha256": "c320379f94fb28a61cb7701495a1d6932da8855cefe44c8cff6bbf150cb886ae"}, {"id": "u016", "kind": "code", "locator": "body:L33-L45", "preview": "```python def grouped_reference(groups): outputs = [] for a, b in groups: k = len(a[0]) if any(len(row) != k for row in a) or any(len(row) != k for row in b): raise ValueError(\"ragged or incompatible K\") outputs.append([ [sum(x * y for x, y", "sha256": "d4407b78235c5840c595cce75e98468022e0ca987e0d8fc61a22a961b679a537"}, {"id": "u017", "kind": "prose", "locator": "body:L47-L47", "preview": "It models neither NVFP4 packing nor block scales; those remain part of the challenge ABI above.", "sha256": "db4cf3e6019dbe5437fe573745b89e7c26bd66bf1eb1bde7f5476bee73e47323"}, {"id": "u018", "kind": "prose", "locator": "body:L51-L51", "preview": "The following shapes describe the pinned FP8/FP4 grouped APIs; scale tensors and layout conversions are additional required inputs.", "sha256": "b668b60f58e395720c62a89ca031cf80584385c6778c5de7785c935b7919a176"}, {"id": "u019", "kind": "table-row", "locator": "body:L55-L55", "preview": "| Mode | Main tensors | Group descriptor | Documented role | | M-grouped contiguous | A `[M,K]`, B `[G,N,K]`, D `[M,N]` | `grouped_layout` is either one expert ID per packed row (with `-1` padding) or one prefix-sum end per group | Training", "sha256": "3edecad8eeab41879305a4720468fe2b3a039c304de9a2f6ed4b2f7554929c39"}, {"id": "u020", "kind": "table-row", "locator": "body:L56-L56", "preview": "| Mode | Main tensors | Group descriptor | Documented role | | M-grouped masked | A `[G,Mmax,K]`, B `[G,N,K]`, D `[G,Mmax,N]` | `masked_m` is one int32 valid-row count per group | CUDA-graph decode; fixed allocation while computing valid po", "sha256": "afe0b21b4ba74a5b245f7b7473fa0a5db9141c4c404a861f5e32a9ee39ab9a09"}, {"id": "u021", "kind": "table-row", "locator": "body:L57-L57", "preview": "| Mode | Main tensors | Group descriptor | Documented role | | K-grouped contiguous | packed A `[sum(K_i),M]`, B `[sum(K_i),N]`, D `[G,M,N]` | host and device K-size lists; optional C has `[G,M,N]` | MoE weight backward; M/N fixed |", "sha256": "3e21ac0d9e7eca36eedbde09c1f5b847faa5edff155b273eb488f4168043f2ea"}, {"id": "u022", "kind": "prose", "locator": "body:L59-L59", "preview": "These are concrete library contracts, not generic C++ structs. Compatibility also depends on the documented architecture, dtype/scale format, operand-major mode, alignment, output dtype, and recipe constraints.", "sha256": "37db519c8bd9df805acef16357b537f98b78a4dfd8e3225ebb06c1e117d4b21f"}, {"id": "u023", "kind": "table-row", "locator": "body:L65-L65", "preview": "| Groups | M values | N | K | L | Task speed-of-light time (\u00b5s) | | 8 | 80, 176, 128, 72, 64, 248, 96, 160 | 4096 | 7168 | 1 | 18.833 |", "sha256": "1ca9581034e9fe688c74c377c0708dba1f3ea1092c93cd5eeb893e2f77e6a0dd"}, {"id": "u024", "kind": "table-row", "locator": "body:L66-L66", "preview": "| Groups | M values | N | K | L | Task speed-of-light time (\u00b5s) | | 8 | 40, 76, 168, 72, 164, 148, 196, 160 | 7168 | 2048 | 1 | 10.667 |", "sha256": "b9c58ac0eb054106541e7aff9bd5a9638104efc3af03f3848506f060f3470154"}, {"id": "u025", "kind": "table-row", "locator": "body:L67-L67", "preview": "| Groups | M values | N | K | L | Task speed-of-light time (\u00b5s) | | 2 | 192, 320 | 3072 | 4096 | 1 | 2.406 |", "sha256": "4f0fa644456b878bcf99df776d2cd88881bb05b73802300f95d8735110732d52"}, {"id": "u026", "kind": "table-row", "locator": "body:L68-L68", "preview": "| Groups | M values | N | K | L | Task speed-of-light time (\u00b5s) | | 2 | 128, 384 | 4096 | 1536 | 1 | 1.525 |", "sha256": "008c90daa24d22e14b488ac75949db726004748e92e59940c7c9b91a438e36cf"}, {"id": "u027", "kind": "prose", "locator": "body:L70-L70", "preview": "Ranking uses the geometric mean. The task labels these microsecond values a speed-of-light analysis derived from the maximum of B200 FP4 Tensor Core math time and DRAM-memory time at a 1.5 GHz clock. They are theoretical comparison values, ", "sha256": "6465beb0e7a292f48bf4e830c3442e6a8b1a8b71c37567dc903a1a346d91299a"}, {"id": "u028", "kind": "prose", "locator": "body:L74-L74", "preview": "GPU Mode's official postmortem records a submission that temporarily reached the number-one leaderboard position with a reported `11.191 \u00b5s`, roughly `2 \u00b5s` ahead of the next entry, and was scrubbed minutes after the competition.", "sha256": "b537c11c34b663ab971a605f55df3a11f215899dc99d34622e205983403d5c10"}, {"id": "u029", "kind": "prose", "locator": "body:L76-L76", "preview": "During correctness, it ran a real padded 8-group kernel on each of 15 cloned data objects. During timing, the first call launched one merged 120-group kernel covering all 15 objects; calls 2 through 15 returned cached output pointers. The h", "sha256": "fb7cc6e225efe9c103ee51aa0acf7fcb118fd3215154625920012cf1d6418a1c"}, {"id": "u030", "kind": "prose", "locator": "body:L78-L78", "preview": "The official post points to `gpu-mode/reference-kernels` PR #104 as the harness response. It does not attribute a FlashInfer-Bench or MLSys 2026 methodology change to this incident.", "sha256": "af363322f4bd273ed6d9542a802080bb526828195a6a0717566c17eb3054ab47"}, {"id": "u031", "kind": "prose", "locator": "body:L82-L82", "preview": "The public challenge constrains outputs, tolerances, workloads, and scoring. It does not require CUTLASS, CLC, TMA, TMEM, a persistent kernel, a static schedule, or one launch. The former CUTLASS and CUDA sketches were removed because they ", "sha256": "1fe62fa8512775ce291aec39d4653fbc7987d9e8df405c3f1f4988b6c8e2afe8"}, {"id": "u032", "kind": "prose", "locator": "body:L84-L84", "preview": "CUTLASS documents that small M or N can leave threads outside the useful problem bounds, and that a small M/N grid with large K can launch too few threadblocks to use every multiprocessor. This is a possible shape effect, not proof that eve", "sha256": "19b76be52212e350c7ab2a813e5df59f97f417436e5b78689a9a8ce75a9685bf"}, {"id": "u033", "kind": "prose", "locator": "body:L86-L86", "preview": "CLC itself uses an asynchronous cancellation request, shared response, mbarrier completion, and response decoding after a worker's initial block. That is different from a global `atomicAdd` tile queue. Whether CLC improves end-to-end time r", "sha256": "bce4d68a2a5f9d951609827c054da927ee679655ed5bf782d54e3e2e9d1bc468"}, {"id": "u034", "kind": "prose", "locator": "body:L88-L88", "preview": "TMA does not have one universal 128-byte alignment rule from which minimum tile sizes follow. CUDA 13.2.1's tiled tensor-map API generally documents 16-byte global-address and stride alignment, a 64-byte descriptor, and feature-specific 32-", "sha256": "25a89a65ad00ad6324e5c598533ef593b1deadb0dc6f50e647167551470242a3"}, {"id": "u035", "kind": "list-item", "locator": "body:L92-L92", "preview": "- [Official challenge rules](https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf)", "sha256": "1f5c0e65cebd680067dc91e0fa9151877d2bab0124675326a188474f3053ddae"}, {"id": "u036", "kind": "list-item", "locator": "body:L93-L93", "preview": "- [Pinned public task](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/task.yml)", "sha256": "c1bba884e3eee0f733185950f50932694f315394494566c74f0f208bd766e437"}, {"id": "u037", "kind": "list-item", "locator": "body:L94-L94", "preview": "- [Pinned starter template](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/template.py)", "sha256": "5aa4b241a5e4e3866f3fd5385c9ff42499d7ff5ebc088ffdedc512c0613564c6"}, {"id": "u038", "kind": "list-item", "locator": "body:L95-L95", "preview": "- [Pinned correctness reference](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm/reference.py)", "sha256": "9f16a390d51604967a3cfb56bdc8255ef16fb2abcfb41b5175e73768643bd652"}, {"id": "u039", "kind": "list-item", "locator": "body:L96-L96", "preview": "- [Pinned DeepGEMM grouped overview](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md#grouped-gemms-contiguous-layout)", "sha256": "e645b512d46f8042349c24ca0a7976640909a20681c8a710160e96a2dc398660"}, {"id": "u040", "kind": "list-item", "locator": "body:L97-L97", "preview": "- [Pinned DeepGEMM grouped APIs](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/csrc/apis/gemm.hpp)", "sha256": "d3f54e38c58497a1a89b916f67253dc71d49a276cb93d040ee7fdcb225de6ed7"}, {"id": "u041", "kind": "list-item", "locator": "body:L98-L98", "preview": "- [Official reward-hack postmortem](https://www.gpumode.com/news/reward-hacking-nvfp4)", "sha256": "8197e7f31a17b0bb68c8301efb12308f82ad01c1abb3e1f1f82afb55fe1cd3f8"}, {"id": "u042", "kind": "list-item", "locator": "body:L99-L99", "preview": "- [CUDA 13.2.1 tensor-map constraints](https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html)", "sha256": "2ce8fa29f655906e7c265b4f2dcf842b58186c862ebf8809d5b5e51ecc9f5225"}, {"id": "u043", "kind": "list-item", "locator": "body:L100-L100", "preview": "- [CUDA 13.2 CLC programming guide](https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html)", "sha256": "aba48c8e1d6d8f310707cb1fb96cb16cf010f530efda480048d46518b5b4ccdc"}, {"id": "u044", "kind": "list-item", "locator": "body:L101-L101", "preview": "- [Pinned CUTLASS efficient-GEMM guide](https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/media/docs/cpp/efficient_gemm.md)", "sha256": "e99485d7254d7868a51325c7d98f9bd12104f73f87855028bf6ca337c5225982"}, {"id": "u045", "kind": "prose", "locator": "body:L103-L103", "preview": "Query via:", "sha256": "6abd63e508f9320fdd3b2b0cc5b5f6a000f4462abfd17f9ecc3c4818f4a0f875"}, {"id": "u046", "kind": "code", "locator": "body:L105-L107", "preview": "```bash conda run -n base python scripts/get_page.py kernel-grouped-gemm ```", "sha256": "9fbb183e9a149ef4de2e59d8f89f3a99147623edc37ab5c8c97198a67b143de1"}], "confidence_claimed": "source-reported", "headings": ["Verified scope", "GPU Mode Problem 4 contract", "DeepGEMM's different grouped layouts", "Published workloads and theoretical bounds", "The scrubbed reward hack", "Implementation and performance boundary", "Primary sources"], "id": "kernel-grouped-gemm", "path": "wiki/kernels/grouped-gemm.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md", "url": "https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm"}, {"path": "sources/blogs/deepgemm.md", "url": "https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f"}, {"path": "sources/blogs/gpu-mode-reward-hack.md", "url": "https://www.gpumode.com/news/reward-hacking-nvfp4"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["contest-gpumode-p4", "blog-deepgemm", "blog-gpu-mode-reward-hack"], "title": "Grouped GEMM Contracts for MoE and NVFP4", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} {"body_sha256": "70483995c616c5db093c78410e006ee58b4ba3b50d6a9c3d4df495d17ee83b8b", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Native Sparse Attention is the three-branch, natively trainable sparse-attention design published in the ACL 2025 proceedings. The paper targets long-context training and inference and reports quality comparable to or better than full atten", "sha256": "2c0ec2bc109661ee09f54dc63f8417335388815f3753d73eec2981291af143b5"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "The paper's efficiency experiments use an eight-GPU A100 system. The paper does not establish Hopper or Blackwell compatibility for its Triton implementation.", "sha256": "6f6896a7fa0ba40dc353d0c167e3c649ded0ce47b2ae36612a4b1923e15eb3ae"}, {"id": "u003", "kind": "prose", "locator": "body:L11-L11", "preview": "For token representation \\(h_t\\), NSA computes three attention outputs and combines them with learned, input-dependent gates:", "sha256": "f64e6768752c7e6f3ba740ef76224502099ecf66a0106a52d67a3c47a39a56a3"}, {"id": "u004", "kind": "prose", "locator": "body:L13-L17", "preview": "\\[ o_t = g_t^{cmp} o_t^{cmp} + g_t^{slc} o_t^{slc} + g_t^{win} o_t^{win}, \\qquad g_t^c = \\operatorname{sigmoid}(\\operatorname{MLP}_c(h_t)). \\]", "sha256": "41026f4d18871ed500bfd946a01192fae3ccd21c8a793f7c9591d36a29b24888"}, {"id": "u005", "kind": "table-row", "locator": "body:L21-L21", "preview": "| Branch | Paper-defined role | | Compression | Learned MLPs with intra-block position encoding compress overlapping KV blocks into coarse-grained representations. |", "sha256": "39c68f8814a572d058f80500989273c21222e31059dc8352f8658036d1d04edf"}, {"id": "u006", "kind": "table-row", "locator": "body:L22-L22", "preview": "| Branch | Paper-defined role | | Selection | Compression-attention scores are reused and aggregated into fine-grained block-importance scores; top-n blocks are selected and attended at full token resolution. |", "sha256": "69c857a98cc6bcc41f69ea7ad8a1ad832967ede3ebec5df870fd4c2daf71d7f7"}, {"id": "u007", "kind": "table-row", "locator": "body:L23-L23", "preview": "| Branch | Paper-defined role | | Sliding window | Recent tokens are attended directly to preserve local context. |", "sha256": "7c88b15a55c170f0f387757aee97a3be92a4b1acbfa50cdb85e45286f450d002"}, {"id": "u008", "kind": "prose", "locator": "body:L25-L25", "preview": "The main experimental configuration uses compression block length 32 and stride 16, selected block length 64 with 16 selected blocks, and a 512-token sliding window. These are experiment settings, not universal NSA constants.", "sha256": "6c078d35fe343e0df340cf3581dab508e404337a77d5cc800fd8b1a501479571"}, {"id": "u009", "kind": "prose", "locator": "body:L29-L29", "preview": "This CPU function checks only the learned gated-sum semantics above. It is KernelWiki-derived, uses ordinary numbers, and makes no claim about the paper's GPU layout or performance.", "sha256": "f3bde73dc92644d83be1a500e942585d86bb8f275cc45390aa6e0440d2bbbc3d"}, {"id": "u010", "kind": "code", "locator": "body:L31-L49", "preview": "```python def gated_branch_sum(branch_outputs, gates): \"\"\"Combine equal-width branch vectors with one [0, 1] gate per branch.\"\"\" if len(branch_outputs) != len(gates) or not branch_outputs: raise ValueError(\"one gate is required for each non", "sha256": "44a4b9439967a835a9020b8d71c4e0705ddbe6889bde833a8bf2c4d774c04576"}, {"id": "u011", "kind": "prose", "locator": "body:L51-L51", "preview": "Removing the gates is a useful negative control: the raw branch sum for the same vectors is `[9.0, 26.0]`, not `[1.0, 10.0]`.", "sha256": "73b34bbecad3370b6e0eec3da0c92b7b15dc9817d821aa7921d2f62c52f3ef42"}, {"id": "u012", "kind": "prose", "locator": "body:L55-L55", "preview": "The authors say compression and sliding-window attention can use existing FlashAttention-2-style kernels, while selected attention needs a specialized Triton kernel for training and prefill. Its source-described structure is:", "sha256": "c7cd36466b1021b9863fae17675a7e07076a11916f6d0577fe9e275c71682848"}, {"id": "u013", "kind": "list-item", "locator": "body:L57-L57", "preview": "1. Load all query heads in one GQA/MQA group into SRAM so they share selected sparse KV indices.", "sha256": "768260e8436d762fc968c79d6fd814d0c40208dbf2be3f79d488232f1ba7c6c8"}, {"id": "u014", "kind": "list-item", "locator": "body:L58-L58", "preview": "2. Load selected KV data as contiguous blocks rather than scattered individual tokens.", "sha256": "7693234b4fa52ffc907e7569bc88a99ef4e7022a678855b8071ab1fbd41d0f4d"}, {"id": "u015", "kind": "list-item", "locator": "body:L59-L59", "preview": "3. Put the nearly constant query and output loops in Triton grid parallelism; keep the selected-block loop inside a program because its count is approximately constant.", "sha256": "6a0b998d60a53e4fe652acf63e51dd57cf0c7cd1a01b790589be65d488ea6450"}, {"id": "u016", "kind": "prose", "locator": "body:L61-L61", "preview": "The paper does not publish the former page's kernel listing, specify a grid exactly as `(query_block, head, batch)`, or claim that such a grid eliminates dynamic scheduling overhead. The removed listings were non-executable and mathematical", "sha256": "2d0122b777bf8c5c8e032dae7d13f2e1faf87911dc235148aad2b8120a48dff1"}, {"id": "u017", "kind": "table-row", "locator": "body:L67-L67", "preview": "| Quantity | Source scope at sequence/context length 65,536 | | Training/prefill forward | 9.0x versus the authors' Triton FlashAttention-2 baseline; Figure 5 timing result |", "sha256": "eeabdd9e3872b414925f03f14acb6773447dad55641cd836f611ce5f68673a3b"}, {"id": "u018", "kind": "table-row", "locator": "body:L68-L68", "preview": "| Quantity | Source scope at sequence/context length 65,536 | | Training/prefill backward | 6.0x versus the same baseline; Figure 5 timing result |", "sha256": "472d7906e53c0e40b8a332a8287fbf4a71db6e008b63e96e97117fc829405b37"}, {"id": "u019", "kind": "table-row", "locator": "body:L69-L69", "preview": "| Quantity | Source scope at sequence/context length 65,536 | | Decoding | 11.6x **expected** speedup from Table 4's memory-access volumes: 65,536 full-attention tokens versus 5,632 NSA-equivalent tokens |", "sha256": "3bf66c2184375f601a4dc615346eb44d33c37253d21f20113aa9c4f8e310c4b0"}, {"id": "u020", "kind": "prose", "locator": "body:L71-L71", "preview": "The setup states eight A100 GPUs, GQA group count 4, 16 query heads per group, key dimension 192, and value dimension 128. The paper does not provide the benchmark dtype, software versions, batch details, raw samples, or variance. Consequen", "sha256": "93cc2b833ca205d7c37ee86be7e7de768c0f5bb64c43ccd76012177aa40a0321"}, {"id": "u021", "kind": "prose", "locator": "body:L75-L75", "preview": "DeepSeek-V3.2-Exp later introduced **DeepSeek Sparse Attention (DSA)**. Its pinned first-party inference code uses a learned indexer to select up to 2,048 token positions and masks attention to those positions. Its README points to DeepGEMM", "sha256": "b759b38189951b82f1e7ff5ce2f07ab9901041dd2a8e8fbbffb3004d989bfecc"}, {"id": "u022", "kind": "prose", "locator": "body:L77-L77", "preview": "That deployed DSA path is related sparse-attention work, but it is not evidence that the ACL paper's gated compression/selection/window NSA architecture was deployed in V3.2-Exp.", "sha256": "4606d7c523a22914f1bc8ef5e748d3b0a4cd3d30f19ab3b48330a2a3808fd904"}, {"id": "u023", "kind": "list-item", "locator": "body:L81-L81", "preview": "- NSA requires a model trained for its compression, selection, window, and gate mechanism; do not treat it as a drop-in sparse mask for an arbitrary checkpoint.", "sha256": "d520c97cb2a307f60e35e18e72170c4d9b3d929d5f23d9b7130c267c78bef030"}, {"id": "u024", "kind": "list-item", "locator": "body:L82-L82", "preview": "- GQA/MQA lets query heads in a group share selected blocks, matching the paper's group-centric kernel design.", "sha256": "48adf162d4b0542d6469331811c4675cf802871f58d1ff43739261c47b3207ec"}, {"id": "u025", "kind": "list-item", "locator": "body:L83-L83", "preview": "- The paper evaluates long contexts through 64K but defines no universal 32K threshold. Compare quality, sequence distribution, memory use, and measured latency or throughput on the target workload.", "sha256": "c9ee5b9ee579038f32902b7284d069a8b03a8a5513560bf99a3c9b4caddf92d9"}, {"id": "u026", "kind": "list-item", "locator": "body:L84-L84", "preview": "- CUDA Graphs can reduce repeated-launch CPU overhead, but they are not an NSA-specific default. Profile first: NVIDIA's guidance says the largest gains occur for CPU-bound workflows and that GPU-bound workloads may see little benefit or re", "sha256": "f9b6df5ef4f3f131f00c87890ba79a225234505a177ddf9f29eae775645f6938"}, {"id": "u027", "kind": "list-item", "locator": "body:L85-L85", "preview": "- The compression MLP adds learned parameters and computation. The paper does not isolate a standalone cost for that component.", "sha256": "9ccd7326bd68872d3e7c3e4f9753cb1a498f16f8e6fc032eac6fba3b8475a883"}, {"id": "u028", "kind": "list-item", "locator": "body:L86-L86", "preview": "- The paper notes that a Triton implementation can retain abstraction overhead relative to native CUDA; it does not publish an NSA-specific CPU-launch profile.", "sha256": "f6435058cc76f46ad9f41b109905a67f7e096f67dce91f53dc284c7fd4d1f671"}, {"id": "u029", "kind": "list-item", "locator": "body:L90-L90", "preview": "- [NSA in the ACL 2025 Anthology](https://aclanthology.org/2025.acl-long.1126/)", "sha256": "78ccf4813ea3076dc266e56c039c186998cec537ae12c990098355ca1dd0affc"}, {"id": "u030", "kind": "list-item", "locator": "body:L91-L91", "preview": "- [ACL proceedings PDF](https://aclanthology.org/2025.acl-long.1126.pdf)", "sha256": "13864f984b34a739b22354671bf40b7b132a4a92b1fe709c69c18516a3157473"}, {"id": "u031", "kind": "list-item", "locator": "body:L92-L92", "preview": "- [DeepSeek-V3.2-Exp at pinned commit 87e509a](https://github.com/deepseek-ai/DeepSeek-V3.2-Exp/tree/87e509a2e5a100d221c97df52c6e8be7835f0057)", "sha256": "495799eeb9bf66dfb26c143f0617fa51bb3eafc0b83db9a0ecbf52e94f45bd9f"}, {"id": "u032", "kind": "list-item", "locator": "body:L93-L93", "preview": "- [FlashMLA at pinned commit 71c7379](https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232)", "sha256": "9a703d254eb11b0c82694980bf3ace79260d5a522fbbc81e75e1d5c838cc0273"}, {"id": "u033", "kind": "list-item", "locator": "body:L94-L94", "preview": "- [NVIDIA CUDA Graph performance troubleshooting](https://docs.nvidia.com/dl-cuda-graph/troubleshooting/performance-issues.html)", "sha256": "f07028fb31a610923a456920862ac95e7fe74dbbb90917e2ee033cc76d23f1d9"}, {"id": "u034", "kind": "list-item", "locator": "body:L95-L95", "preview": "- [Third-party lucidrains PyTorch implementation](https://github.com/lucidrains/native-sparse-attention-pytorch)", "sha256": "4ac2ee874ac52c888f894dbe154072fa140f783aefe6c722c45a6b7eaf2cfd00"}, {"id": "u035", "kind": "code", "locator": "body:L99-L101", "preview": "```bash conda run -n base python scripts/get_page.py kernel-nsa ```", "sha256": "1f78b6b3ff77dec53e1ab9848cec6dad73d5a38e2591afa12472b71b618f4c82"}], "confidence_claimed": "source-reported", "headings": ["Verified scope", "Exact three-branch mechanism", "Executable branch-combination reference", "Paper-described Triton design", "Source-reported efficiency", "NSA is not the later DSA deployment", "Applicability and boundaries", "Primary sources", "Query"], "id": "kernel-nsa", "path": "wiki/kernels/nsa.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nsa.md", "url": "https://aclanthology.org/2025.acl-long.1126/"}, {"path": "sources/blogs/flashmla.md", "url": "https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232"}], "risk_flags": ["code", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-nsa", "blog-flashmla"], "title": "Native Sparse Attention (NSA)", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} -{"body_sha256": "c5bd73b9a4501849ca7bc91e4a94a7de62cd1bf395495ceb820d6ca9c8465a37", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "The official NVIDIA rules identify NVFP4 GEMM as Kernel Challenge 2 of the Blackwell NVFP4 Hackathon. The contest ran from November 29 through December 19, 2025; Problem 2 targeted NVIDIA B200 and contributed 20% of the four-problem grand-p", "sha256": "c012f9b4b901261153563003ab333b1f3208acf9127751594a9c6f92e6194686"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "The pinned public task defines observable inputs, correctness, test shapes, benchmark shapes, and ranking. It does not publish a canonical optimized kernel, require CUTLASS, or establish which implementation mechanisms any entrant used. \u201cNV", "sha256": "60449fada505af15ad32e1652d853557b7618e9f0adf40fc59aa58181e7654c7"}, {"id": "u003", "kind": "prose", "locator": "body:L11-L11", "preview": "The generic one-dimensional NVFP4 recipe reconstructs each value from a signed E2M1 payload, one E4M3 local scale per 16 consecutive payloads, and a per-tensor FP32 global scale. E2M1 represents zero and signed magnitudes 0.5, 1, 1.5, 2, 3,", "sha256": "2d3e513162b32573b229edb6791eca5c968c25d20a763fc0c6728bee7623a754"}, {"id": "u004", "kind": "prose", "locator": "body:L13-L13", "preview": "The contest ABI is narrower and does not expose the generic recipe's FP32 global scales. Its actual generated input has seven tensors:", "sha256": "3a871dd4a928094b50b5397043bcb0ed40b4ca720015ca6f28f00940abc8308f"}, {"id": "u005", "kind": "table-row", "locator": "body:L17-L17", "preview": "| Tensor | Published representation | Physical shape | | `a` | packed E2M1, two values per byte | `[M,K/2,L]` |", "sha256": "e0a7789d78d69b33651c32ed0b1322da6a51ce99701fb8c4f691c1d606ca1e27"}, {"id": "u006", "kind": "table-row", "locator": "body:L18-L18", "preview": "| Tensor | Published representation | Physical shape | | `b` | packed E2M1, two values per byte | `[N,K/2,L]` |", "sha256": "7d0102d70ed8d43681e30199436e1a4ca93f82176a713d1a4eb60f915d82a32b"}, {"id": "u007", "kind": "table-row", "locator": "body:L19-L19", "preview": "| Tensor | Published representation | Physical shape | | `sfa` | logical E4M3 scales | `[M,K/16,L]` |", "sha256": "8c6e7b8cc27b3f4659878a193b0d177ad42556861e9af3c56bf8dd190847c87d"}, {"id": "u008", "kind": "table-row", "locator": "body:L20-L20", "preview": "| Tensor | Published representation | Physical shape | | `sfb` | logical E4M3 scales | `[N,K/16,L]` |", "sha256": "53f0052f4f81c4ddb14c0a6d49f8abbe1ee49fffe56a0e6600ba2e4690d9a1fc"}, {"id": "u009", "kind": "table-row", "locator": "body:L21-L21", "preview": "| Tensor | Published representation | Physical shape | | `sfa_reordered` | MMA-oriented scale copy | `[32,4,ceil(M/128),4,K/64,L]` |", "sha256": "fdccc0a2063460171c6874f7b71b04aa81f8547a6adf6647394f30407ad2130f"}, {"id": "u010", "kind": "table-row", "locator": "body:L22-L22", "preview": "| Tensor | Published representation | Physical shape | | `sfb_reordered` | MMA-oriented scale copy | `[32,4,ceil(N/128),4,K/64,L]` |", "sha256": "dc780d607c17c5380f0d0cf88b406360fc63691f9131f5658c4666fe6a431427"}, {"id": "u011", "kind": "table-row", "locator": "body:L23-L23", "preview": "| Tensor | Published representation | Physical shape | | `c` | preallocated FP16 output | `[M,N,L]` |", "sha256": "5bb5aa5ac7ba2bf79f1a086d297ae8bf6ac71c70fd93ef7176c42f472b856b8b"}, {"id": "u012", "kind": "prose", "locator": "body:L25-L25", "preview": "For each `L` slice, the correctness reference computes the block-scaled equivalent of `A @ B.T` and stores FP16 output. It uses `rtol=1e-3` and `atol=1e-3`.", "sha256": "6f3e0ffc95c1f7cfbba0f5cce8d4ad5cf98e730c2518d3f862973484eb5cd357"}, {"id": "u013", "kind": "prose", "locator": "body:L27-L27", "preview": "The upstream files contain two interface inconsistencies that implementations must not conceal:", "sha256": "2ead2f5e8d44bc8c68f365c286a7d40895ac73ad636d88cfbbf74c8456754f89"}, {"id": "u014", "kind": "list-item", "locator": "body:L29-L29", "preview": "- `task.yml` describes a five-member `(a,b,sfa,sfb,c)` tuple, while `task.py`, `template.py`, and `reference.py` expose the seven tensors above.", "sha256": "b1ab36d41d8414867d0fd4b868add90d1552fb037fa289b4952ffc5335fe9025"}, {"id": "u015", "kind": "list-item", "locator": "body:L30-L30", "preview": "- Task and template prose label scale tensors E4M3FNUZ, while `reference.py` constructs `torch.float8_e4m3fn` values. A submission must follow the tensors supplied by the actual harness rather than treating those suffixes as interchangeable", "sha256": "60a30abca2b8e35e3e85f9529afad31c41f9e672f64a665c9cab4c93db10e6cb"}, {"id": "u016", "kind": "prose", "locator": "body:L34-L34", "preview": "The published task requires `K` divisible by 256. Divisibility of `M` and `N` depends on the submission's selected MMA tile. The following CPU-only helper reproduces the published storage shapes; it does not decode FP4, apply scales, or mod", "sha256": "b665b4c6d8030c332f28c4d3ee158ae219b6d15795bdaee05374861dbc13385b"}, {"id": "u017", "kind": "code", "locator": "body:L36-L58", "preview": "```python def task_storage_shapes(m, n, k, l=1): if min(m, n, k, l) <= 0: raise ValueError(\"dimensions must be positive\") if k % 256: raise ValueError(\"K must be divisible by 256\") return { \"a_packed\": (m, k // 2, l), \"b_packed\": (n, k // 2", "sha256": "7642131a89d5b771cbf27494b4cc0b80cd0cf33ee914733ba4332b41646dc052"}, {"id": "u018", "kind": "prose", "locator": "body:L60-L60", "preview": "Nine of the ten official correctness shapes fail the former `(K/16) % 128 == 0` assertion, including the smallest valid case with `K=256`.", "sha256": "05437858b45314ea93e0a0104ea46596c3ea7d49d2c24ae53df396395c1074ae"}, {"id": "u019", "kind": "prose", "locator": "body:L64-L64", "preview": "The native Blackwell instruction path supports block-of-16 NVFP4 through `tcgen05.mma...kind::mxf4nvf4.block_scale.block16`. It uses UE4M3 scale elements, which CUTLASS names `float_ue4m3_t`; UE8M0 is the MXFP4 scale type. Converting NVFP4 ", "sha256": "5c735c80238f43b67916629af854f0bc5ebe3cbdc2a5dd8c6884eb7c0e838112"}, {"id": "u020", "kind": "prose", "locator": "body:L66-L66", "preview": "CUTLASS defines `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100`, but its direct official example is a grouped pointer-array kernel. The symbol's existence is not evidence that the contest entrants used it. CUTLASS's current official NVFP4 ex", "sha256": "ab4df04cd09ce423053e72e6743e8bd87ad2ec87bbac6cc6a67fcdf80f74f1bb"}, {"id": "u021", "kind": "prose", "locator": "body:L68-L68", "preview": "TMA likewise has no universal 128-byte alignment rule for every operand. CUDA 13.2.1 generally requires a 16-byte-aligned global base and 16-byte-multiple strides for a tiled tensor map, with additional datatype-, interleave-, swizzle-, and", "sha256": "3ccf1a1511937dc90f14ec1f46a7a6da76096f2e80ebf2b09b6040f7c8715278"}, {"id": "u022", "kind": "prose", "locator": "body:L70-L70", "preview": "TMEM's 128-lane by 512-column organization is a storage and addressing model, not a universal 128-by-512 logical output-tile limit. Official CUTLASS NVFP4 configurations include a cooperative two-SM MMA tile with logical shape 256 by 256 by", "sha256": "9146b8a20d4fa3e2d61f9fc78dbb921c708e685affa23cd507b61539ba3c289d"}, {"id": "u023", "kind": "prose", "locator": "body:L74-L74", "preview": "The task ranks the geometric mean across three benchmark cases. It labels the following values a speed-of-light analysis based on the maximum of B200 FP4 Tensor Core math time and DRAM-memory time at a 1.5 GHz clock:", "sha256": "1333ed27524466513afa4aa3d6b2aca56e71b8dd321380468992191a08483324"}, {"id": "u024", "kind": "table-row", "locator": "body:L78-L78", "preview": "| M | N | K | L | Theoretical time (\u00b5s) | | 128 | 7168 | 16384 | 1 | 8.994 |", "sha256": "ac29495b16a50fe1cfa2dcde5a85eb0fbc58049c674a2acd145386cdf9509a03"}, {"id": "u025", "kind": "table-row", "locator": "body:L79-L79", "preview": "| M | N | K | L | Theoretical time (\u00b5s) | | 128 | 4096 | 7168 | 1 | 2.354 |", "sha256": "5830e08d67630c0aa4612f11749d003bd0e8772b58d5696ff74cb0b2383e0fe2"}, {"id": "u026", "kind": "table-row", "locator": "body:L80-L80", "preview": "| M | N | K | L | Theoretical time (\u00b5s) | | 128 | 7168 | 2048 | 1 | 1.333 |", "sha256": "fa3e79cd0999f76fee4e1016298889112f86cae481bb308bf85adc53cba145e9"}, {"id": "u027", "kind": "prose", "locator": "body:L82-L82", "preview": "These are theoretical comparison rows, not measured contestant latencies and not cuBLAS results.", "sha256": "e7cccec1a9469a76f5310902e63e0ff426f9cdfb679295d05b973561c5a59ba8"}, {"id": "u028", "kind": "prose", "locator": "body:L84-L84", "preview": "The public Popcorn endpoint currently gives the following dated snapshot. To make its small floating-point `submission_score` values readable beside the task's microsecond presentation, the table shows `submission_score \u00d7 10^6`; it does not", "sha256": "ca5d81241e455f5afedd1cd83c9e5f33b6ce5ffd65e4fc6dbee310fe12e17238"}, {"id": "u029", "kind": "table-row", "locator": "body:L88-L88", "preview": "| Current rank | User | API score \u00d7 10^6 | Submission timestamp (UTC) | | 1 | `gau.nernst` | 9.981889 | 2025-12-21 00:43:03 |", "sha256": "fe02ccfff7ba7a01f611de399274d94df3fc34621f9c1f49f0b4661b1049fc8e"}, {"id": "u030", "kind": "table-row", "locator": "body:L89-L89", "preview": "| Current rank | User | API score \u00d7 10^6 | Submission timestamp (UTC) | | 2 | `s.am._` | 10.060110 | 2025-12-20 17:45:21 |", "sha256": "3b924272968baecf2642aff9df9a27f543bf3a3ec76b865ad5651e3dfc27ae46"}, {"id": "u031", "kind": "table-row", "locator": "body:L90-L90", "preview": "| Current rank | User | API score \u00d7 10^6 | Submission timestamp (UTC) | | 3 | `billcarson` | 10.137411 | 2025-12-21 03:05:32 |", "sha256": "682d0b4289aaf64c4a8dfce794ab877797b0f27e2dfbdb0558cd5b91020e7ea6"}, {"id": "u032", "kind": "table-row", "locator": "body:L91-L91", "preview": "| Current rank | User | API score \u00d7 10^6 | Submission timestamp (UTC) | | 8 | `Simon` | 10.806750 | 2025-12-16 20:18:42 |", "sha256": "532478b666f20292d73ffe2cf7b1d1e68503fdadedf372b6bc068566201ba1ca"}, {"id": "u033", "kind": "table-row", "locator": "body:L92-L92", "preview": "| Current rank | User | API score \u00d7 10^6 | Submission timestamp (UTC) | | 9 | `yue` | 10.914084 | 2025-12-11 04:36:45 |", "sha256": "390d4818810b01ab386a95f2b688d3b0e6b496b5c99251c2597eee2fffca1387"}, {"id": "u034", "kind": "table-row", "locator": "body:L93-L93", "preview": "| Current rank | User | API score \u00d7 10^6 | Submission timestamp (UTC) | | 10 | `currybab` | 10.930623 | 2025-12-19 08:10:18 |", "sha256": "b20e53b276da79e3e8db64ea4bc8792ffde6e7159c74d43fdd475df2f464a8d0"}, {"id": "u035", "kind": "prose", "locator": "body:L95-L95", "preview": "Snapshot fetched August 8, 2026. Because the current first three submissions postdate the official December 19 cutoff, this endpoint alone cannot establish winners or final prize rankings. It also publishes no contestant source, CUTLASS att", "sha256": "31d7de708031defe935bf9ec093e7e4e609515f457b21a57edaae459c012889a"}, {"id": "u036", "kind": "prose", "locator": "body:L99-L99", "preview": "Use this task contract only when the packed E2M1 payloads, logical and reordered scales, transpose convention, FP16 output, tolerances, and B200 target match the workload. \u201cFour-bit weights\u201d alone is insufficient: INT4, MXFP4, other scale g", "sha256": "73cdc11c5d684987ddca4b7d6b757fa05f35817fafcc4b69fab11eb85fdadbea"}, {"id": "u037", "kind": "prose", "locator": "body:L101-L101", "preview": "The documented native tensor-core path is Blackwell-specific; Hopper has no native FP4 tensor-core instruction. This does not exclude software conversion or emulation. Choose TMA, TMEM allocation, CUTLASS schedules, warp specialization, til", "sha256": "44a9079d9bde40b4b9d9f4314563b36497b2c4c3baf8f73d63b50ba622654431"}, {"id": "u038", "kind": "list-item", "locator": "body:L105-L105", "preview": "- [Official contest rules](https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf)", "sha256": "c945d69cc0933ee89c1418170925a1dac08b34aa49c449aff8fab2f67960ab55"}, {"id": "u039", "kind": "list-item", "locator": "body:L106-L106", "preview": "- [Pinned task definition](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml)", "sha256": "e65c352f0be491a3018d9832b9eb33f9a20999ed3c89f8073d9b51d7b05ab736"}, {"id": "u040", "kind": "list-item", "locator": "body:L107-L107", "preview": "- [Pinned task types](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.py)", "sha256": "3fd0cff1732ff2946231c340cdb3f1e603f51d9e30c1d38d02c5bdd5e97047cc"}, {"id": "u041", "kind": "list-item", "locator": "body:L108-L108", "preview": "- [Pinned starter template](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py)", "sha256": "f0c85282243b89d3b5178d653b7368861f3c8179b8902ccb2d8cf362e007ba1f"}, {"id": "u042", "kind": "list-item", "locator": "body:L109-L109", "preview": "- [Pinned correctness reference](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/reference.py)", "sha256": "a9913b0ecd262bdf9fc1ed22d8ba73b0df684b9d60385ea92765cefada05884c"}, {"id": "u043", "kind": "list-item", "locator": "body:L110-L110", "preview": "- [Public Popcorn leaderboard API](https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12)", "sha256": "05fc2df13c95fa5525c4bc583dfe7fbd3d792d5b56bd9701b72b2005410eba68"}, {"id": "u044", "kind": "list-item", "locator": "body:L111-L111", "preview": "- [Transformer Engine 2.13 NVFP4 recipe](https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html)", "sha256": "6a8a7f25f33afde1adc5eb40bedd91ec6b208700f493f6716ab59dda9d710f87"}, {"id": "u045", "kind": "list-item", "locator": "body:L112-L112", "preview": "- [cuBLAS block-scaling formats](https://docs.nvidia.com/cuda/cublas/index.html#element-1d-block-scaling-for-fp8-and-fp4-data-types)", "sha256": "835625f4c0a79e2e00ed840ffb666c21b6794ec80c63ba082abf75c3ac010c1f"}, {"id": "u046", "kind": "list-item", "locator": "body:L113-L113", "preview": "- [PTX ISA 9.0 block-scaled `tcgen05.mma`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma)", "sha256": "3b0e1ea6faf1d7ae1afa563c938aa77e64dfc63629fa38f2b9d81668b2ba6da4"}, {"id": "u047", "kind": "list-item", "locator": "body:L114-L114", "preview": "- [Pinned CUTLASS NVFP4 example](https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu)", "sha256": "673b6e03246cef1e9406aab15f2e173b99d155e39e94f235480d949ea32a846e"}, {"id": "u048", "kind": "list-item", "locator": "body:L115-L115", "preview": "- [Pinned CUTLASS grouped NVFP4 example](https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu)", "sha256": "072f9f025862aacd8fa196a463d4a6c65ad7f4c07ffc75f84a2e3230b331ecd9"}, {"id": "u049", "kind": "list-item", "locator": "body:L116-L116", "preview": "- [CUDA 13.2.1 tensor-map constraints](https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html)", "sha256": "2ce8fa29f655906e7c265b4f2dcf842b58186c862ebf8809d5b5e51ecc9f5225"}, {"id": "u050", "kind": "prose", "locator": "body:L118-L118", "preview": "Query via:", "sha256": "6abd63e508f9320fdd3b2b0cc5b5f6a000f4462abfd17f9ecc3c4818f4a0f875"}, {"id": "u051", "kind": "code", "locator": "body:L120-L122", "preview": "```bash conda run -n base python scripts/get_page.py kernel-nvfp4-gemm ```", "sha256": "48df91e99487c982432b8d227c4d314527b5a7a8f1814e19da626a790ab563a4"}], "confidence_claimed": "verified", "headings": ["Verified scope", "Format recipe versus task ABI", "Host-checkable shape contract", "Blackwell implementation boundary", "Published performance records", "Applicability", "Primary sources"], "id": "kernel-nvfp4-gemm", "path": "wiki/kernels/nvfp4-gemm.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/contests/gpu-mode-nvfp4/problem-2-gemm.md", "url": "https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm"}, {"path": "sources/docs/nvidia-transformer-engine-2.13-nvfp4.md", "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/docs/nvidia-cuda-13-0-2-tma.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#asynchronous-data-copies-using-the-tensor-memory-accelerator-tma"}], "risk_flags": ["code", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["contest-gpumode-p2", "doc-transformer-engine-2.13-nvfp4", "doc-ptx-isa-sm100", "doc-cuda-13-0-2-tma"], "title": "NVFP4 GEMM \u2014 GPU Mode Problem 2 Contract", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "174e77c3133c6b67914db0e72ccf06349e86c0818a834cc8196e0965554d776c", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "The official NVIDIA rules identify NVFP4 GEMM as Kernel Challenge 2 of the Blackwell NVFP4 Hackathon. The contest ran from November 29 through December 19, 2025; Problem 2 targeted NVIDIA B200 and contributed 20% of the four-problem grand-p", "sha256": "c012f9b4b901261153563003ab333b1f3208acf9127751594a9c6f92e6194686"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "The pinned public task defines observable inputs, correctness, test shapes, benchmark shapes, and ranking. It does not publish a canonical optimized kernel, require CUTLASS, or establish which implementation mechanisms any entrant used. \u201cNV", "sha256": "60449fada505af15ad32e1652d853557b7618e9f0adf40fc59aa58181e7654c7"}, {"id": "u003", "kind": "prose", "locator": "body:L11-L11", "preview": "The generic one-dimensional NVFP4 recipe reconstructs each value from a signed E2M1 payload, one E4M3 local scale per 16 consecutive payloads, and a per-tensor FP32 global scale. E2M1 represents zero and signed magnitudes 0.5, 1, 1.5, 2, 3,", "sha256": "2d3e513162b32573b229edb6791eca5c968c25d20a763fc0c6728bee7623a754"}, {"id": "u004", "kind": "prose", "locator": "body:L13-L13", "preview": "The contest ABI is narrower and does not expose the generic recipe's FP32 global scales. Its actual generated input has seven tensors:", "sha256": "3a871dd4a928094b50b5397043bcb0ed40b4ca720015ca6f28f00940abc8308f"}, {"id": "u005", "kind": "table-row", "locator": "body:L17-L17", "preview": "| Tensor | Published representation | Physical shape | | `a` | packed E2M1, two values per byte | `[M,K/2,L]` |", "sha256": "e0a7789d78d69b33651c32ed0b1322da6a51ce99701fb8c4f691c1d606ca1e27"}, {"id": "u006", "kind": "table-row", "locator": "body:L18-L18", "preview": "| Tensor | Published representation | Physical shape | | `b` | packed E2M1, two values per byte | `[N,K/2,L]` |", "sha256": "7d0102d70ed8d43681e30199436e1a4ca93f82176a713d1a4eb60f915d82a32b"}, {"id": "u007", "kind": "table-row", "locator": "body:L19-L19", "preview": "| Tensor | Published representation | Physical shape | | `sfa` | logical E4M3 scales | `[M,K/16,L]` |", "sha256": "8c6e7b8cc27b3f4659878a193b0d177ad42556861e9af3c56bf8dd190847c87d"}, {"id": "u008", "kind": "table-row", "locator": "body:L20-L20", "preview": "| Tensor | Published representation | Physical shape | | `sfb` | logical E4M3 scales | `[N,K/16,L]` |", "sha256": "53f0052f4f81c4ddb14c0a6d49f8abbe1ee49fffe56a0e6600ba2e4690d9a1fc"}, {"id": "u009", "kind": "table-row", "locator": "body:L21-L21", "preview": "| Tensor | Published representation | Physical shape | | `sfa_reordered` | MMA-oriented scale copy | `[32,4,ceil(M/128),4,K/64,L]` |", "sha256": "fdccc0a2063460171c6874f7b71b04aa81f8547a6adf6647394f30407ad2130f"}, {"id": "u010", "kind": "table-row", "locator": "body:L22-L22", "preview": "| Tensor | Published representation | Physical shape | | `sfb_reordered` | MMA-oriented scale copy | `[32,4,ceil(N/128),4,K/64,L]` |", "sha256": "dc780d607c17c5380f0d0cf88b406360fc63691f9131f5658c4666fe6a431427"}, {"id": "u011", "kind": "table-row", "locator": "body:L23-L23", "preview": "| Tensor | Published representation | Physical shape | | `c` | preallocated FP16 output | `[M,N,L]` |", "sha256": "5bb5aa5ac7ba2bf79f1a086d297ae8bf6ac71c70fd93ef7176c42f472b856b8b"}, {"id": "u012", "kind": "prose", "locator": "body:L25-L25", "preview": "For each `L` slice, the correctness reference computes the block-scaled equivalent of `A @ B.T` and stores FP16 output. It uses `rtol=1e-3` and `atol=1e-3`.", "sha256": "6f3e0ffc95c1f7cfbba0f5cce8d4ad5cf98e730c2518d3f862973484eb5cd357"}, {"id": "u013", "kind": "prose", "locator": "body:L27-L27", "preview": "The upstream files contain two interface inconsistencies that implementations must not conceal:", "sha256": "2ead2f5e8d44bc8c68f365c286a7d40895ac73ad636d88cfbbf74c8456754f89"}, {"id": "u014", "kind": "list-item", "locator": "body:L29-L29", "preview": "- `task.yml` describes a five-member `(a,b,sfa,sfb,c)` tuple, while `task.py`, `template.py`, and `reference.py` expose the seven tensors above.", "sha256": "b1ab36d41d8414867d0fd4b868add90d1552fb037fa289b4952ffc5335fe9025"}, {"id": "u015", "kind": "list-item", "locator": "body:L30-L30", "preview": "- Task and template prose label scale tensors E4M3FNUZ, while `reference.py` constructs `torch.float8_e4m3fn` values. A submission must follow the tensors supplied by the actual harness rather than treating those suffixes as interchangeable", "sha256": "60a30abca2b8e35e3e85f9529afad31c41f9e672f64a665c9cab4c93db10e6cb"}, {"id": "u016", "kind": "prose", "locator": "body:L34-L34", "preview": "The published task requires `K` divisible by 256. Divisibility of `M` and `N` depends on the submission's selected MMA tile. The following CPU-only helper reproduces the published storage shapes; it does not decode FP4, apply scales, or mod", "sha256": "b665b4c6d8030c332f28c4d3ee158ae219b6d15795bdaee05374861dbc13385b"}, {"id": "u017", "kind": "code", "locator": "body:L36-L58", "preview": "```python def task_storage_shapes(m, n, k, l=1): if min(m, n, k, l) <= 0: raise ValueError(\"dimensions must be positive\") if k % 256: raise ValueError(\"K must be divisible by 256\") return { \"a_packed\": (m, k // 2, l), \"b_packed\": (n, k // 2", "sha256": "7642131a89d5b771cbf27494b4cc0b80cd0cf33ee914733ba4332b41646dc052"}, {"id": "u018", "kind": "prose", "locator": "body:L60-L60", "preview": "Nine of the ten official correctness shapes fail the former `(K/16) % 128 == 0` assertion, including the smallest valid case with `K=256`.", "sha256": "05437858b45314ea93e0a0104ea46596c3ea7d49d2c24ae53df396395c1074ae"}, {"id": "u019", "kind": "prose", "locator": "body:L64-L64", "preview": "The native Blackwell instruction path supports block-of-16 NVFP4 through `tcgen05.mma...kind::mxf4nvf4.block_scale.block16`. It uses UE4M3 scale elements, which CUTLASS names `float_ue4m3_t`; UE8M0 is the MXFP4 scale type. Converting NVFP4 ", "sha256": "5c735c80238f43b67916629af854f0bc5ebe3cbdc2a5dd8c6884eb7c0e838112"}, {"id": "u020", "kind": "prose", "locator": "body:L66-L66", "preview": "CUTLASS defines `KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100`, but its direct official example is a grouped pointer-array kernel. The symbol's existence is not evidence that the contest entrants used it. CUTLASS's current official NVFP4 ex", "sha256": "ab4df04cd09ce423053e72e6743e8bd87ad2ec87bbac6cc6a67fcdf80f74f1bb"}, {"id": "u021", "kind": "prose", "locator": "body:L68-L68", "preview": "TMA likewise has no universal 128-byte alignment rule for every operand. CUDA 13.2.1 generally requires a 16-byte-aligned global base and 16-byte-multiple strides for a tiled tensor map, with additional datatype-, interleave-, swizzle-, and", "sha256": "3ccf1a1511937dc90f14ec1f46a7a6da76096f2e80ebf2b09b6040f7c8715278"}, {"id": "u022", "kind": "prose", "locator": "body:L70-L70", "preview": "TMEM's 128-lane by 512-column organization is a storage and addressing model, not a universal 128-by-512 logical output-tile limit. Official CUTLASS NVFP4 configurations include a cooperative two-SM MMA tile with logical shape 256 by 256 by", "sha256": "9146b8a20d4fa3e2d61f9fc78dbb921c708e685affa23cd507b61539ba3c289d"}, {"id": "u023", "kind": "prose", "locator": "body:L74-L74", "preview": "The task ranks the geometric mean across three benchmark cases. It labels the following values a speed-of-light analysis based on the maximum of B200 FP4 Tensor Core math time and DRAM-memory time at a 1.5 GHz clock:", "sha256": "1333ed27524466513afa4aa3d6b2aca56e71b8dd321380468992191a08483324"}, {"id": "u024", "kind": "table-row", "locator": "body:L78-L78", "preview": "| M | N | K | L | Theoretical time (\u00b5s) | | 128 | 7168 | 16384 | 1 | 8.994 |", "sha256": "ac29495b16a50fe1cfa2dcde5a85eb0fbc58049c674a2acd145386cdf9509a03"}, {"id": "u025", "kind": "table-row", "locator": "body:L79-L79", "preview": "| M | N | K | L | Theoretical time (\u00b5s) | | 128 | 4096 | 7168 | 1 | 2.354 |", "sha256": "5830e08d67630c0aa4612f11749d003bd0e8772b58d5696ff74cb0b2383e0fe2"}, {"id": "u026", "kind": "table-row", "locator": "body:L80-L80", "preview": "| M | N | K | L | Theoretical time (\u00b5s) | | 128 | 7168 | 2048 | 1 | 1.333 |", "sha256": "fa3e79cd0999f76fee4e1016298889112f86cae481bb308bf85adc53cba145e9"}, {"id": "u027", "kind": "prose", "locator": "body:L82-L82", "preview": "These are theoretical comparison rows, not measured contestant latencies and not cuBLAS results.", "sha256": "e7cccec1a9469a76f5310902e63e0ff426f9cdfb679295d05b973561c5a59ba8"}, {"id": "u028", "kind": "prose", "locator": "body:L84-L84", "preview": "The public Popcorn endpoint currently gives the following dated snapshot. To make its small floating-point `submission_score` values readable beside the task's microsecond presentation, the table shows `submission_score \u00d7 10^6`; it does not", "sha256": "ca5d81241e455f5afedd1cd83c9e5f33b6ce5ffd65e4fc6dbee310fe12e17238"}, {"id": "u029", "kind": "table-row", "locator": "body:L88-L88", "preview": "| Current rank | User | API score \u00d7 10^6 | Submission timestamp (UTC) | | 1 | `gau.nernst` | 9.981889 | 2025-12-21 00:43:03 |", "sha256": "fe02ccfff7ba7a01f611de399274d94df3fc34621f9c1f49f0b4661b1049fc8e"}, {"id": "u030", "kind": "table-row", "locator": "body:L89-L89", "preview": "| Current rank | User | API score \u00d7 10^6 | Submission timestamp (UTC) | | 2 | `s.am._` | 10.060110 | 2025-12-20 17:45:21 |", "sha256": "3b924272968baecf2642aff9df9a27f543bf3a3ec76b865ad5651e3dfc27ae46"}, {"id": "u031", "kind": "table-row", "locator": "body:L90-L90", "preview": "| Current rank | User | API score \u00d7 10^6 | Submission timestamp (UTC) | | 3 | `billcarson` | 10.137411 | 2025-12-21 03:05:32 |", "sha256": "682d0b4289aaf64c4a8dfce794ab877797b0f27e2dfbdb0558cd5b91020e7ea6"}, {"id": "u032", "kind": "table-row", "locator": "body:L91-L91", "preview": "| Current rank | User | API score \u00d7 10^6 | Submission timestamp (UTC) | | 8 | `Simon` | 10.806750 | 2025-12-16 20:18:42 |", "sha256": "532478b666f20292d73ffe2cf7b1d1e68503fdadedf372b6bc068566201ba1ca"}, {"id": "u033", "kind": "table-row", "locator": "body:L92-L92", "preview": "| Current rank | User | API score \u00d7 10^6 | Submission timestamp (UTC) | | 9 | `yue` | 10.914084 | 2025-12-11 04:36:45 |", "sha256": "390d4818810b01ab386a95f2b688d3b0e6b496b5c99251c2597eee2fffca1387"}, {"id": "u034", "kind": "table-row", "locator": "body:L93-L93", "preview": "| Current rank | User | API score \u00d7 10^6 | Submission timestamp (UTC) | | 10 | `currybab` | 10.930623 | 2025-12-19 08:10:18 |", "sha256": "b20e53b276da79e3e8db64ea4bc8792ffde6e7159c74d43fdd475df2f464a8d0"}, {"id": "u035", "kind": "prose", "locator": "body:L95-L95", "preview": "Snapshot fetched August 8, 2026. Because the current first three submissions postdate the official December 19 cutoff, this endpoint alone cannot establish winners or final prize rankings. It also publishes no contestant source, CUTLASS att", "sha256": "31d7de708031defe935bf9ec093e7e4e609515f457b21a57edaae459c012889a"}, {"id": "u036", "kind": "prose", "locator": "body:L99-L99", "preview": "Use this task contract only when the packed E2M1 payloads, logical and reordered scales, transpose convention, FP16 output, tolerances, and B200 target match the workload. \u201cFour-bit weights\u201d alone is insufficient: INT4, MXFP4, other scale g", "sha256": "73cdc11c5d684987ddca4b7d6b757fa05f35817fafcc4b69fab11eb85fdadbea"}, {"id": "u037", "kind": "prose", "locator": "body:L101-L101", "preview": "The documented native tensor-core path is Blackwell-specific; Hopper has no native FP4 tensor-core instruction. This does not exclude software conversion or emulation. Choose TMA, TMEM allocation, CUTLASS schedules, warp specialization, til", "sha256": "44a9079d9bde40b4b9d9f4314563b36497b2c4c3baf8f73d63b50ba622654431"}, {"id": "u038", "kind": "list-item", "locator": "body:L105-L105", "preview": "- [Official contest rules](https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf)", "sha256": "c945d69cc0933ee89c1418170925a1dac08b34aa49c449aff8fab2f67960ab55"}, {"id": "u039", "kind": "list-item", "locator": "body:L106-L106", "preview": "- [Pinned task definition](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.yml)", "sha256": "e65c352f0be491a3018d9832b9eb33f9a20999ed3c89f8073d9b51d7b05ab736"}, {"id": "u040", "kind": "list-item", "locator": "body:L107-L107", "preview": "- [Pinned task types](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/task.py)", "sha256": "3fd0cff1732ff2946231c340cdb3f1e603f51d9e30c1d38d02c5bdd5e97047cc"}, {"id": "u041", "kind": "list-item", "locator": "body:L108-L108", "preview": "- [Pinned starter template](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/template.py)", "sha256": "f0c85282243b89d3b5178d653b7368861f3c8179b8902ccb2d8cf362e007ba1f"}, {"id": "u042", "kind": "list-item", "locator": "body:L109-L109", "preview": "- [Pinned correctness reference](https://github.com/gpu-mode/reference-kernels/blob/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm/reference.py)", "sha256": "a9913b0ecd262bdf9fc1ed22d8ba73b0df684b9d60385ea92765cefada05884c"}, {"id": "u043", "kind": "list-item", "locator": "body:L110-L110", "preview": "- [Public Popcorn leaderboard API](https://site--bot--dxfjds728w5v.code.run/submissions/nvfp4_gemm/NVIDIA?limit=12)", "sha256": "05fc2df13c95fa5525c4bc583dfe7fbd3d792d5b56bd9701b72b2005410eba68"}, {"id": "u044", "kind": "list-item", "locator": "body:L111-L111", "preview": "- [Transformer Engine 2.13 NVFP4 recipe](https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html)", "sha256": "6a8a7f25f33afde1adc5eb40bedd91ec6b208700f493f6716ab59dda9d710f87"}, {"id": "u045", "kind": "list-item", "locator": "body:L112-L112", "preview": "- [cuBLAS block-scaling formats](https://docs.nvidia.com/cuda/cublas/index.html#element-1d-block-scaling-for-fp8-and-fp4-data-types)", "sha256": "835625f4c0a79e2e00ed840ffb666c21b6794ec80c63ba082abf75c3ac010c1f"}, {"id": "u046", "kind": "list-item", "locator": "body:L113-L113", "preview": "- [PTX ISA 9.0 block-scaled `tcgen05.mma`](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma)", "sha256": "0eae676c73ff2a26b0c7f4700430e3d5418c17442c24232d71e63edb3926281d"}, {"id": "u047", "kind": "list-item", "locator": "body:L114-L114", "preview": "- [Pinned CUTLASS NVFP4 example](https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu)", "sha256": "673b6e03246cef1e9406aab15f2e173b99d155e39e94f235480d949ea32a846e"}, {"id": "u048", "kind": "list-item", "locator": "body:L115-L115", "preview": "- [Pinned CUTLASS grouped NVFP4 example](https://github.com/NVIDIA/cutlass/blob/e05f953a5b3d38adc240df2ff928e0421c2abba3/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu)", "sha256": "072f9f025862aacd8fa196a463d4a6c65ad7f4c07ffc75f84a2e3230b331ecd9"}, {"id": "u049", "kind": "list-item", "locator": "body:L116-L116", "preview": "- [CUDA 13.2.1 tensor-map constraints](https://docs.nvidia.com/cuda/archive/13.2.1/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html)", "sha256": "2ce8fa29f655906e7c265b4f2dcf842b58186c862ebf8809d5b5e51ecc9f5225"}, {"id": "u050", "kind": "prose", "locator": "body:L118-L118", "preview": "Query via:", "sha256": "6abd63e508f9320fdd3b2b0cc5b5f6a000f4462abfd17f9ecc3c4818f4a0f875"}, {"id": "u051", "kind": "code", "locator": "body:L120-L122", "preview": "```bash conda run -n base python scripts/get_page.py kernel-nvfp4-gemm ```", "sha256": "48df91e99487c982432b8d227c4d314527b5a7a8f1814e19da626a790ab563a4"}], "confidence_claimed": "verified", "headings": ["Verified scope", "Format recipe versus task ABI", "Host-checkable shape contract", "Blackwell implementation boundary", "Published performance records", "Applicability", "Primary sources"], "id": "kernel-nvfp4-gemm", "path": "wiki/kernels/nvfp4-gemm.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/contests/gpu-mode-nvfp4/problem-2-gemm.md", "url": "https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemm"}, {"path": "sources/docs/nvidia-transformer-engine-2.13-nvfp4.md", "url": "https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/docs/nvidia-cuda-13-0-2-tma.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#asynchronous-data-copies-using-the-tensor-memory-accelerator-tma"}], "risk_flags": ["code", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["contest-gpumode-p2", "doc-transformer-engine-2.13-nvfp4", "doc-ptx-isa-sm100", "doc-cuda-13-0-2-tma"], "title": "NVFP4 GEMM \u2014 GPU Mode Problem 2 Contract", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} {"body_sha256": "15a1de756e23c26b7938e4b0c5e12e918492d5f204e504a428fcb1573c5bbd26", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "GPU Mode's Problem 1 asks for a block-scaled NVFP4 batched matrix-vector product on NVIDIA B200. The official task models its three benchmark cases against the slower of FFMA math and DRAM transfer time and reports DRAM-limited theoretical ", "sha256": "14e1c04988eab77eb5fda24b66c9161c6b4819d6a72816cd82aa24e0a228b98e"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "The logical operation has one B row, reused by all M output rows. The reference harness pads B and its scales to 128 rows so it can call `torch._scaled_mm`, then retains only result column zero. Consequently, A values are row-specific while", "sha256": "fb7fdddf466f0ad9cf575500ce1520eda0f07afbd4e86b5654b1dc604b053c5d"}, {"id": "u003", "kind": "prose", "locator": "body:L11-L11", "preview": "At pinned commit `ae679486`, `custom_kernel` receives seven tensors, not five tensors plus global FP32 scale arguments:", "sha256": "15bf1ed3ca25ae5af1f21dfbb407e770ac71c82bc8cde6e78cd46afd632e8b3f"}, {"id": "u004", "kind": "table-row", "locator": "body:L15-L15", "preview": "| Tensor | Physical shape | Role | | `a` | `[M, K/2, L]` | Packed E2M1 A; two logical values per byte |", "sha256": "f74daf225a044f42c185baf318bc992eb494afe58da70331b78e61f48059c196"}, {"id": "u005", "kind": "table-row", "locator": "body:L16-L16", "preview": "| Tensor | Physical shape | Role | | `b` | `[128, K/2, L]` | Packed E2M1 B, physically padded; logical row 0 is used |", "sha256": "4e4df829b77072f6fd90eda8d41974abd6ddcd71887459f0ab2e1a6f449f3171"}, {"id": "u006", "kind": "table-row", "locator": "body:L17-L17", "preview": "| Tensor | Physical shape | Role | | `sfa` | `[M, K/16, L]` | Logical/reference A block scales |", "sha256": "47e879cdd6d9333011e683d6bb76b0bb1ebd562f7e7d900090b2ae9f86d97c5b"}, {"id": "u007", "kind": "table-row", "locator": "body:L18-L18", "preview": "| Tensor | Physical shape | Role | | `sfb` | `[128, K/16, L]` | Logical/reference B block scales, padded to 128 rows |", "sha256": "a0be0f8d74ca34db15d2798662259b839777715672f2963b11ac9ae6cd6e401b"}, {"id": "u008", "kind": "table-row", "locator": "body:L19-L19", "preview": "| Tensor | Physical shape | Role | | `sfa_reordered` | `[32, 4, ceil(M/128), 4, K/64, L]` | Swizzled A scales for custom kernels |", "sha256": "e1f29ebc95c561c93de68422dd62d1e78f1b2f0e26a56cb1e63d2971a7ee3d4e"}, {"id": "u009", "kind": "table-row", "locator": "body:L20-L20", "preview": "| Tensor | Physical shape | Role | | `sfb_reordered` | `[32, 4, 1, 4, K/64, L]` | Swizzled padded-B scales for custom kernels |", "sha256": "25d7de3baa2b15fd47bea0643ffd612cf8dbd8732ed66bb80c7f3a9cf9fe501b"}, {"id": "u010", "kind": "table-row", "locator": "body:L21-L21", "preview": "| Tensor | Physical shape | Role | | `c` | `[M, 1, L]` | FP16 output buffer |", "sha256": "dc83bb9a64c7548c727836928612797531bb396623d9b852aad6bd803787427e"}, {"id": "u011", "kind": "prose", "locator": "body:L23-L23", "preview": "`K` must be divisible by 64, and M must be divisible by the implementation's selected M tile. The task prose labels scales `e4m3fnuz`, but the pinned generator constructs `torch.float8_e4m3fn`; implementations should follow the actual harne", "sha256": "9d23ed25965f166e208682143b1379b4a346ef123a4f513c3160a28f289fbe68"}, {"id": "u012", "kind": "prose", "locator": "body:L27-L27", "preview": "The official table assumes a 1.5 GHz B200 clock and ranks submissions by the geometric mean across all three rows:", "sha256": "181b1e55fb5b84d28814d49c47efc252c00f59b87529df6181d4be0f24168dcc"}, {"id": "u013", "kind": "table-row", "locator": "body:L31-L31", "preview": "| M | K | L | Theoretical time (\u00b5s) | | 7168 | 16384 | 1 | 8.622 |", "sha256": "9d7c65618fc8dc86bb3d2746a16f175fd4649ddfb006c68c807ce5e587b8fd0b"}, {"id": "u014", "kind": "table-row", "locator": "body:L32-L32", "preview": "| M | K | L | Theoretical time (\u00b5s) | | 4096 | 7168 | 8 | 17.275 |", "sha256": "6ef10de1e876f22ab461b936146b0788d1bf5d704d586c207ec965c2cfb38c65"}, {"id": "u015", "kind": "table-row", "locator": "body:L33-L33", "preview": "| M | K | L | Theoretical time (\u00b5s) | | 7168 | 2048 | 4 | 4.317 |", "sha256": "f615c82589dadc4edfe7b91cae3fca3ca39d09c582d799c3ea68d1cd3429a24c"}, {"id": "u016", "kind": "prose", "locator": "body:L35-L35", "preview": "NVIDIA's Blackwell technical brief specifies 8 TB/s of HBM3e bandwidth for one GB200 GPU. The task's theoretical numbers are model values, not measured kernel timings.", "sha256": "704c6601665fccbda264f611828f4318072acc029cf200d5edb39ac485314581"}, {"id": "u017", "kind": "prose", "locator": "body:L37-L37", "preview": "The public leaderboard is mutable and reports aggregate scores rather than per-shape latencies. In the snapshot fetched on 2026-08-08, the first three rows were `s.am._` at 18.549562452 \u00b5s, `gau.nernst` at 18.552844757 \u00b5s, and `shellsmile15", "sha256": "22240d05193ce717b94fafae8edaa23f1553f76a8c00a46d942aa92c4300367a"}, {"id": "u018", "kind": "prose", "locator": "body:L41-L41", "preview": "PTX ISA 9.0 defines the packed conversion and typed register decomposition used in Yue's author-reported decode path:", "sha256": "4affc9d80b04da71f0c299c5784d8efc15535361c1156364eff99e69729b354f"}, {"id": "u019", "kind": "code", "locator": "body:L43-L46", "preview": "```asm cvt.rn.f16x2.e2m1x2 %result, %packed_fp4_pair; mov.b32 {%b0, %b1, %b2, %b3}, %packed_word; ```", "sha256": "3e8b7ea84b83abad0b9ce987bd7810cd56904af6c822034b66dc22b09deced0a"}, {"id": "u020", "kind": "prose", "locator": "body:L48-L48", "preview": "It also defines vector loads such as `ld.global.v2.u64` and `ld.global.v4.u64`, which move 16 and 32 bytes and can carry 32 and 64 packed FP4 values. The ISA specifies behavior, not that one width or decomposition is universally faster.", "sha256": "9d0de1cc2b4db79424fedba1e2f5d3bc17dccb713cb173f3a8b41405ac5bf617"}, {"id": "u021", "kind": "prose", "locator": "body:L50-L50", "preview": "Amandeep reports that three solutions inspected after the event used `L1::no_allocate` for streamed A loads, `L1::evict_last` for reused B loads, wider PTX loads, and exact-K specializations. Those are author-reported observations; the publ", "sha256": "8d769e43685ea0b46eefdae1e86397cdbe7f97387c4fdb5be59a5305352d80cb"}, {"id": "u022", "kind": "prose", "locator": "body:L54-L54", "preview": "The following timings are from Yue's post, not an independent reproduction. Several stages combine multiple changes and cannot establish isolated causality:", "sha256": "a663193aef641ad8433597c548f5820e215e4556a1f06db8485c4075e1cbf6f9"}, {"id": "u023", "kind": "table-row", "locator": "body:L58-L58", "preview": "| Stage | Combined change | Reported latency | | Initial CuTe DSL | First working CuTe implementation | ~100 \u00b5s |", "sha256": "43eb908efd6940c3a7f0b3cb44880e4110b5b361042ab2ccf9cec68ad6d5009f"}, {"id": "u024", "kind": "table-row", "locator": "body:L59-L59", "preview": "| Stage | Combined change | Reported latency | | Optimized CuTe DSL | Scale-load and arithmetic changes plus thread collaboration | ~33 \u00b5s |", "sha256": "8ff27a458e08c63aa19c12f8680ccc4cb633d6b880364aec30bfce4b162c5aa7"}, {"id": "u025", "kind": "table-row", "locator": "body:L60-L60", "preview": "| Stage | Combined change | Reported latency | | Initial CUDA | Naive hand-written path | ~2000 \u00b5s |", "sha256": "ca54d2e0867aaacac8ef7cb2d50ce50784f5192560a97826eabca1c17f14eaba"}, {"id": "u026", "kind": "table-row", "locator": "body:L61-L61", "preview": "| Stage | Combined change | Reported latency | | CUDA step 1 | Coalescing, shared B, thread collaboration, warp reduction | ~443 \u00b5s |", "sha256": "45a99e5c48954c46d64d8216a104e7f192c5d36eae133d72ead8f09d1b957a37"}, {"id": "u027", "kind": "table-row", "locator": "body:L62-L62", "preview": "| Stage | Combined change | Reported latency | | CUDA step 2 | Remove shared B, per-thread tiles, `float4` loads, hardware intrinsics | ~39 \u00b5s |", "sha256": "c7347c699aff3a09cbd3e861889c16bd4cab72d2239ec315fc23068cdff5cf2b"}, {"id": "u028", "kind": "table-row", "locator": "body:L63-L63", "preview": "| Stage | Combined change | Reported latency | | CUDA step 3 | Vectorized PTX FP4/scale decode | ~27 \u00b5s |", "sha256": "6219356ab8e9caca95b5f5bf5e4a40b08c5b6eaef70784ebcafa9b4bc9966a55"}, {"id": "u029", "kind": "table-row", "locator": "body:L64-L64", "preview": "| Stage | Combined change | Reported latency | | Parameter tuning | Threads per row and rows per block | ~26 \u00b5s |", "sha256": "5d2fda1525d1d93874e8cdcc642024d47fe2139e768806cdfa35ab38b4826a4c"}, {"id": "u030", "kind": "table-row", "locator": "body:L65-L65", "preview": "| Stage | Combined change | Reported latency | | Two-tile ILP | Interleave two tiles per loop | ~22.9 \u00b5s |", "sha256": "55bc6ef2fdac8923e7f8d89df31dd74418182cd3741c71c983b0b942c8165841"}, {"id": "u031", "kind": "table-row", "locator": "body:L66-L66", "preview": "| Stage | Combined change | Reported latency | | Aggressive PTX fusion | Fuse decode, scale, multiply, and accumulation | ~22.3 \u00b5s |", "sha256": "3e9d7de1f63be9029aa4ff29559822da7ac982629cde96e956e04ec4c3330b31"}, {"id": "u032", "kind": "table-row", "locator": "body:L67-L67", "preview": "| Stage | Combined change | Reported latency | | Submitted score | Geometric mean shown by the leaderboard | 22.392 \u00b5s |", "sha256": "088f1750e1a2800452b3e097abdcb2af7099608a24ecb7bf909f53c3406f2917"}, {"id": "u033", "kind": "prose", "locator": "body:L69-L69", "preview": "Yue also reports that loading the entire B vector into shared memory did not improve the CuTe attempt. This is a useful counterexample to treating shared-memory B staging or an exact `BLOCK_M` traffic reduction as automatic.", "sha256": "19ee8de8d318e9c5ada566f02126040608afa1d3e39dbc64e014946f41585387"}, {"id": "u034", "kind": "list-item", "locator": "body:L73-L73", "preview": "- Use this case study for an N=1 decode-style matrix-vector operation only when payloads, scales, layouts, output type, and target match the chosen NVFP4 implementation contract.", "sha256": "a8248df41bbce5cc45f98412659fa2f2d8f3ad504d6e7dbf6611009207c697f4"}, {"id": "u035", "kind": "list-item", "locator": "body:L74-L74", "preview": "- Profile bytes moved, instruction count, decode and reduction work, cache behavior, spills, and occupancy before deciding which resource is limiting performance.", "sha256": "c725284ade431a9257a1b4a7cf2ae3f84f565ddd6b5a45c645adbba3458b3580"}, {"id": "u036", "kind": "list-item", "locator": "body:L75-L75", "preview": "- Exact-K dispatch can enable full unrolling and size-specific tuning, but retaining multiple specializations adds compiled kernel entries.", "sha256": "2f807d582f25851529b1c82489b1ba304fdc4b91510a1bfaf6f8b259db03ec6f"}, {"id": "u037", "kind": "list-item", "locator": "body:L76-L76", "preview": "- Inline PTX must be revalidated against the selected PTX ISA, toolkit, and target. The contest ran on B200; that does not imply packed FP4 conversion is restricted to SM100/SM100a forever. Transformer Engine 2.13 lists NVFP4 inference supp", "sha256": "9f975c4e2c336fdd731e7222126701eac54c48fbacbccd74f4fc7230a54c0ca7"}, {"id": "u038", "kind": "list-item", "locator": "body:L77-L77", "preview": "- The official task accepts any implementation that satisfies its checker. It does not require the cache policies, load widths, register limits, or code structures described in participant posts.", "sha256": "86286884a901e356f8c977190835d2ab47157b7f2fa23ef0bbb05aa129da29f2"}, {"id": "u039", "kind": "list-item", "locator": "body:L81-L81", "preview": "- [Pinned Problem 1 task](https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv)", "sha256": "253eac43081be0b1c0f587a706185b6b8f30e7b2cebb381630c17652de95898d"}, {"id": "u040", "kind": "list-item", "locator": "body:L82-L82", "preview": "- [Official contest rules](https://developer.download.nvidia.com/licenses/Blackwell-NVFP4-Hackathon-Terms-and-Conditions.pdf)", "sha256": "c945d69cc0933ee89c1418170925a1dac08b34aa49c449aff8fab2f67960ab55"}, {"id": "u041", "kind": "list-item", "locator": "body:L83-L83", "preview": "- [NVIDIA Blackwell Architecture Technical Brief](https://resources.nvidia.com/en-us-blackwell-architecture/blackwell-architecture-technical-brief)", "sha256": "c330012ad06dc51b1eccaade34a7425fc7b7a8aaefd177c220b19f75aacfc633"}, {"id": "u042", "kind": "list-item", "locator": "body:L84-L84", "preview": "- [PTX ISA 9.0](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html)", "sha256": "f287fbe5a77017bc028e72cfac864b6bb16272498a46b45f643b430593555c9b"}, {"id": "u043", "kind": "list-item", "locator": "body:L85-L85", "preview": "- [Transformer Engine 2.13 NVFP4](https://docs.nvidia.com/deeplearning/transformer-engine-releases/release-2.13/user-guide/features/low_precision_training/nvfp4/nvfp4.html)", "sha256": "fd66d9aaa128a191225657e888e4d0692eac5623d5d11ade7dfc67793dad73e0"}, {"id": "u044", "kind": "list-item", "locator": "body:L86-L86", "preview": "- [Yue Zhang's hackathon journey](https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html)", "sha256": "c92a8a46203210ed1e85fb159f8abb7ee9402f2560f9d4f9c3bba8b71a5a2467"}, {"id": "u045", "kind": "list-item", "locator": "body:L87-L87", "preview": "- [Amandeep Singh's twelve attempts](https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/)", "sha256": "32b13b998fa855e2f28acc4e3385e172306944a5676385b43b3a1d65c0fffbe7"}, {"id": "u046", "kind": "list-item", "locator": "body:L88-L88", "preview": "- [Simon Veitner's reference](https://veitner.bearblog.dev/nvfp4-gemv/) and [improved variants](https://veitner.bearblog.dev/nvfp4-gemv-improved/)", "sha256": "1de91575cdf7f4cd9043e4939095ac18a892ba8e120f1b5343db9ef35082a8e0"}], "confidence_claimed": "source-reported", "headings": ["Verified Scope", "Official Callable Contract", "Official Benchmark Model", "PTX Semantics and Reported Techniques", "Yue's Author-Reported Progression", "Practical Use and Caveats", "Primary Sources"], "id": "kernel-nvfp4-gemv", "path": "wiki/kernels/nvfp4-gemv.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/contests/gpu-mode-nvfp4/problem-1-gemv.md", "url": "https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv"}, {"path": "sources/blogs/yue-nvfp4-hackathon.md", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}, {"path": "sources/blogs/amandeep-nvfp4-attempts.md", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}], "risk_flags": ["code", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["contest-gpumode-p1", "blog-yue-nvfp4", "blog-amandeep-nvfp4"], "title": "NVFP4 Batched GEMV", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} {"body_sha256": "948b326db003b5db4472a9222cbd12a7ea8b28c706bb3c67abfc29cdc7846cca", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L6", "preview": "DeepSeek-V3.2-Exp introduces **DeepSeek Sparse Attention (DSA)**. Its first-party report defines two components:", "sha256": "bb3c5529691e366dd6efa2a880b744e11851437076378b06b60557eed9f130e4"}, {"id": "u002", "kind": "list-item", "locator": "body:L8-L8", "preview": "1. The lightning indexer computes one score for every query/preceding-token", "sha256": "800571c72e22c76c4f46f84b358de8ed823f7e7daa1d5a0a1aada431dff73396"}, {"id": "u003", "kind": "prose", "locator": "body:L9-L10", "preview": "pair. A score is a weighted sum across indexer heads of a ReLU-applied query and key dot product.", "sha256": "b0c28b86e74cff53ff735a2fccf865af8a26d42271a8e1f9e5ba566ac3c6bc24"}, {"id": "u004", "kind": "list-item", "locator": "body:L11-L11", "preview": "2. Fine-grained token selection retains the KV entries at the top-k index", "sha256": "27bba385baa0f30644e808286cf4cd7f206ac7b835de6e843d5877b66605f308"}, {"id": "u005", "kind": "prose", "locator": "body:L12-L12", "preview": "scores and applies the main attention only to that selected set.", "sha256": "799ed51e15b7fac5411d28e251d11824024993140b9ef0591e8307e92855c99b"}, {"id": "u006", "kind": "prose", "locator": "body:L14-L19", "preview": "The released `config_671B_v3.2.json` uses 64 indexer heads of dimension 128 and `index_topk=2048`. DSA is instantiated under MLA's MQA mode: the selected MLA latent entry is shared across the query heads. The model has 128 MLA query heads, ", "sha256": "7999f09cb62ad0774fc6162c7eeb34be31d20d0047d0dae05d29f9edc76f3774"}, {"id": "u007", "kind": "prose", "locator": "body:L21-L25", "preview": "For a length-L sequence and fixed selected count k, the report reduces the **main/core attention** complexity from O(L squared) to O(Lk). The lightning indexer still scores the preceding context and remains O(L squared) over the sequence. F", "sha256": "2c7a6f110419179e164e3a5294219756ca737c3fcc8e666080d7b2dbdaa83df2"}, {"id": "u008", "kind": "prose", "locator": "body:L29-L30", "preview": "The released high-performance path is not one fused \u201cindexer plus sparse MLA\u201d kernel:", "sha256": "55f8654222c7c4ed6f9915195f9062b25ad60c76e2b1fdb603290b284fdd718a"}, {"id": "u009", "kind": "list-item", "locator": "body:L32-L32", "preview": "- DeepGEMM provides non-paged and paged indexer-logit kernels. Its pinned SM100", "sha256": "49ef39eedf0417732d260892fbf391b96e42f8be6fcfc25f65df6a95b5a976d8"}, {"id": "u010", "kind": "prose", "locator": "body:L33-L35", "preview": "FP8 path computes token logits from FP8 query/key inputs, per-token key scales, and per-query head weights; top-k selection remains a separate operation.", "sha256": "37f4d58acab34fe50c0f8f7112e6ca0b4bfea2750e936edd7287731209210208"}, {"id": "u011", "kind": "list-item", "locator": "body:L36-L36", "preview": "- FlashMLA sparse attention consumes caller-produced token indices. It does not", "sha256": "cc6129e45570619d4939c753c50aa7f48f97d6bd16ddccbef36dafe6d6a91980"}, {"id": "u012", "kind": "prose", "locator": "body:L37-L37", "preview": "run the lightning indexer or top-k selector.", "sha256": "ae05e1bbfe66ece445518ed31b0a0d666c384437f79b7d468584eec2274a5262"}, {"id": "u013", "kind": "prose", "locator": "body:L39-L41", "preview": "At FlashMLA commit [`71c7379`](https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232), prefill and decode have different contracts:", "sha256": "eb389ddbba522174d4e4bfeeae557782e265db6d785ad1db1e6534dae97f1dd5"}, {"id": "u014", "kind": "table-row", "locator": "body:L45-L45", "preview": "| Stage | Inputs and selected-token encoding | Precision and outputs | | Sparse decode | `q[batch,s_q,h_q,576]`; paged `k_cache`; `indices[batch,s_q,topk]`, where each nonnegative value encodes physical page times page size plus token offse", "sha256": "26a36dfe709a48b77979096047f08d9c163ac61519ddf4cdf0418f3ac8e5e996"}, {"id": "u015", "kind": "table-row", "locator": "body:L46-L46", "preview": "| Stage | Inputs and selected-token encoding | Precision and outputs | | Sparse prefill | Unbatched BF16 `q[s_q,h_q,d_qk]`, BF16 `kv[s_kv,h_kv,d_qk]`, and INT32 `indices[s_q,h_kv,topk]`; the documented equivalence requires `h_kv=1`; negativ", "sha256": "1e44a7ce28c3d88a202ffd6faf77f7cedd7ca507f7ce76c55601542bc3357387"}, {"id": "u016", "kind": "prose", "locator": "body:L48-L52", "preview": "These are token indices, not one maximum or one selection per cache block. The sparse-decode API accepts page size through the cache tensor; pinned correctness tests exercise multiple values including 2, 53, 61, 64, 69, 256, and 576. Page s", "sha256": "708e5ca34ceab27f48e8fe7b2f704d457087a65dd4385e2131a535dc0871273d"}, {"id": "u017", "kind": "prose", "locator": "body:L56-L57", "preview": "Only the V3/V3.1/V3.2 **FP8 sparse-decode** mode has the documented 656-byte per-token layout:", "sha256": "953e4b660a1d8b065676b80e7b7bfe06fbcc32914d3a83085d33182a259edc3c"}, {"id": "u018", "kind": "list-item", "locator": "body:L59-L59", "preview": "- 512 E4M3 NoPE values: 512 bytes;", "sha256": "cab31744fcfa6f82078271f60cd43b820f43334dfc55e6a23d6b4eff5d85eeab"}, {"id": "u019", "kind": "list-item", "locator": "body:L60-L60", "preview": "- four FP32 scales, one for each successive group of 128 NoPE values: 16 bytes;", "sha256": "e9793ded4f667e1592dd6b993dd390b5016a1cf8154c4041a50971a687d58d57"}, {"id": "u020", "kind": "list-item", "locator": "body:L61-L61", "preview": "- 64 BF16 RoPE values used by the attention key: 128 bytes.", "sha256": "42502eff2ec7dd17fd0437ce18e46942556ee145ee15b2f566f78fc7f1f35cf6"}, {"id": "u021", "kind": "prose", "locator": "body:L63-L65", "preview": "The indexer has a separate FP8 K cache and scale cache in the released model. The 656-byte attention entry is not an indexer cache and is not the layout of every dense, sparse-prefill, or non-V3 FlashMLA mode.", "sha256": "004ef4c9a6d5f3ed82854f9a9c23912785829fab4c14b88f3209e130e2804f32"}, {"id": "u022", "kind": "prose", "locator": "body:L69-L71", "preview": "FlashMLA's pinned README reports the following maxima. They are useful source claims, not reproducible benchmark tuples: the README does not provide complete shapes, timed regions, repetitions, samples, or variance.", "sha256": "53fac485447961fd25735e0a07f1bc42805fa1e081bfc85f7e358c428dfce49f"}, {"id": "u023", "kind": "table-row", "locator": "body:L75-L75", "preview": "| Operator and regime | Reported environment | Author-reported maximum and precision scope | | Dense MLA decode, memory-bound configuration | H800 SXM5, CUDA 12.8 | Up to 3000 GB/s with BF16 cache |", "sha256": "330ca8a2334ecb87ab1bbb24ffd109a884a2205e87bf06eb13a6a23f7971d01c"}, {"id": "u024", "kind": "table-row", "locator": "body:L76-L76", "preview": "| Operator and regime | Reported environment | Author-reported maximum and precision scope | | Dense MLA decode, compute-bound configuration | H800 SXM5, CUDA 12.8 | Up to 660 TFLOPS with BF16 cache; separate from the 3000-GB/s case |", "sha256": "498754f9010440f53a448bb30c012279c0b07afbed50d3948e10936a375fe13b"}, {"id": "u025", "kind": "table-row", "locator": "body:L77-L77", "preview": "| Operator and regime | Reported environment | Author-reported maximum and precision scope | | Sparse MLA decode | H800 SXM5, CUDA 12.8 | 410 TFLOPS; FP8 KV storage and BF16 matrix multiplication |", "sha256": "619c3ef250d56d62746909cd8385a09d21c42b91cbb78ca19ff12ee196e011d7"}, {"id": "u026", "kind": "table-row", "locator": "body:L78-L78", "preview": "| Operator and regime | Reported environment | Author-reported maximum and precision scope | | Sparse MLA decode | B200 | Up to 350 TFLOPS; the source says this path was not really optimized and gives no bandwidth-causality result |", "sha256": "8c29c94ec9620c62ce3dc2dca6c5caf9fc593e6e655ed3083238714f063298c1"}, {"id": "u027", "kind": "table-row", "locator": "body:L79-L79", "preview": "| Operator and regime | Reported environment | Author-reported maximum and precision scope | | Sparse MLA prefill | H800 SXM5, CUDA 12.8 | Up to 640 TFLOPS forward with BF16 Q/KV |", "sha256": "65b792e1dcb4c09b1df29d269f6b31e97e9759eb45d1dbfb50461bdfc30a2b78"}, {"id": "u028", "kind": "table-row", "locator": "body:L80-L80", "preview": "| Operator and regime | Reported environment | Author-reported maximum and precision scope | | Sparse MLA prefill | B200, CUDA 12.9 | Up to 1450 TFLOPS forward with BF16 Q/KV |", "sha256": "95c25ad0d981b3686a61775f7965640cc532bb9f6b636a35fd27c6a294572fe7"}, {"id": "u029", "kind": "prose", "locator": "body:L82-L85", "preview": "The same README separately reports NVIDIA's dense **MHA** prefill maxima of 1460 TFLOPS forward and 1000 TFLOPS backward on B200. Those values are not a dense-MLA baseline matched to the sparse-prefill result, so numerical proximity between", "sha256": "7ac9b3f51be9a7774fa020507229c154b698af2de5f5294b218fdc3135466a57"}, {"id": "u030", "kind": "prose", "locator": "body:L89-L92", "preview": "Use this path directly for DeepSeek-V3.2-Exp, or for another model only after its query dimensions, latent/RoPE layout, cache format, head relationships, index encoding, invalid-entry rules, and output semantics match the chosen FlashMLA in", "sha256": "8f4750da95e3b1406c6de34977adb16629218d8cbf8633e93a0e2d1402c11c69"}, {"id": "u031", "kind": "prose", "locator": "body:L94-L94", "preview": "For a target deployment:", "sha256": "fbb956c923fa1fd09806a78a27d293e0f8ebba82fb051ec936245cfbab5aa6bf"}, {"id": "u032", "kind": "list-item", "locator": "body:L96-L96", "preview": "1. Compare index scores and selected token sets against the released model", "sha256": "a242b13ee51da5cc6cb541c02ee0c6daec6fdfc1984d6ec275554957be416c4a"}, {"id": "u033", "kind": "prose", "locator": "body:L97-L98", "preview": "equation and a dense reference, including causal masking and the indexer's non-interleaved RoPE layout.", "sha256": "2e2cead28d1d22caa0e2eb0a926978703e3dac81cef6c1ca51ca073875034c13"}, {"id": "u034", "kind": "list-item", "locator": "body:L99-L99", "preview": "2. Compare sparse prefill/decode outputs and LSE values with the pinned", "sha256": "da0d88573313119d302c4d5be8a002a02d2e95ed3cfdf92b11d9eb6d471ee920"}, {"id": "u035", "kind": "prose", "locator": "body:L100-L101", "preview": "FlashMLA reference for identical selected indices, including invalid and partially filled top-k cases.", "sha256": "7dfcefbf9f9a818d7f9216ec620e421da6a17ef61a85dc8a12903765a3d3a748"}, {"id": "u036", "kind": "list-item", "locator": "body:L102-L102", "preview": "3. Time indexer logits, top-k selection, sparse attention, and the complete", "sha256": "94eb29b4206955c162be3c506b690b8704ac47a014a9065ffb96b40829d3a68e"}, {"id": "u037", "kind": "prose", "locator": "body:L103-L105", "preview": "pipeline separately across context lengths, query batch sizes, page sizes, and selected counts. Use a matched dense-attention baseline and report synchronization, warmup, repeated trials, statistic, and variation.", "sha256": "f02576926515f6346b13cb20938a90d9add5d59c67836537a247f53a7ed3964e"}, {"id": "u038", "kind": "prose", "locator": "body:L107-L109", "preview": "There is no verified universal 32K crossover. The relevant threshold is where the measured selector-plus-sparse-attention pipeline improves the target's end-to-end latency or cost without violating its accuracy criterion.", "sha256": "dbcd039e000ca1384e83656c5550b990540d5f937a2165464a53be0b3a47373b"}, {"id": "u039", "kind": "prose", "locator": "body:L113-L117", "preview": "DeepSeek-V3.2-Exp DSA selects individual token positions with a learned indexer. The ACL Native Sparse Attention architecture instead combines compressed-block, selected-block, and sliding-window branches with learned gates. Do not transfer", "sha256": "0acee81708a88fb6dadac0671ca900e0781e9ea10c0f915e18641f375c096b84"}, {"id": "u040", "kind": "list-item", "locator": "body:L121-L121", "preview": "- [DeepSeek-V3.2-Exp report and released model](https://github.com/deepseek-ai/DeepSeek-V3.2-Exp/tree/87e509a2e5a100d221c97df52c6e8be7835f0057)", "sha256": "7e2d20a04ee83d9a3bf13f14c6776ff4b781d09b3be1430590dc1ecf4a0c74bc"}, {"id": "u041", "kind": "list-item", "locator": "body:L122-L122", "preview": "- [FlashMLA at audited commit `71c7379`](https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232)", "sha256": "1f1e839b0c6d66154bae95cc00ff4970eabb13c2b7f9c1fbf2c2adbc49cbff46"}, {"id": "u042", "kind": "list-item", "locator": "body:L123-L123", "preview": "- [FlashMLA sparse interface at `71c7379`](https://github.com/deepseek-ai/FlashMLA/blob/71c737929f2567bd0a094ae140f8f60f390b1232/flash_mla/flash_mla_interface.py)", "sha256": "e814ec84eaefe35612a2df8db9295b511349797f6c84c94ee4a8c5dd1fa447ae"}, {"id": "u043", "kind": "list-item", "locator": "body:L124-L124", "preview": "- [DeepGEMM indexer-logit implementation at `891d57b4`](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/deep_gemm/include/deep_gemm/impls/sm100_fp8_mqa_logits.cuh)", "sha256": "719f548e64d48d007a4167184782be4a682c07ce1fc80ce249f2422d75dd59cf"}, {"id": "u044", "kind": "list-item", "locator": "body:L125-L125", "preview": "- [ACL 2025 Native Sparse Attention paper](https://aclanthology.org/2025.acl-long.1126/)", "sha256": "0d5536512e7b6f36724ec0738c6ae2905c69b13a296ce9b447941d99d1ba1f39"}], "confidence_claimed": "source-reported", "headings": ["Model mechanism", "Separate implementation boundaries", "V3-family sparse-decode cache", "Source-reported performance boundary", "Evaluation procedure", "DSA is not Native Sparse Attention", "Primary references"], "id": "kernel-sparse-mla", "path": "wiki/kernels/sparse-mla.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/flashmla.md", "url": "https://github.com/deepseek-ai/FlashMLA/tree/71c737929f2567bd0a094ae140f8f60f390b1232"}, {"path": "sources/blogs/vllm-deepseek-v3-sparse-attention.md", "url": "https://blog.vllm.ai/2025/09/29/deepseek-v3-2.html"}, {"path": "sources/docs/nsa.md", "url": "https://aclanthology.org/2025.acl-long.1126/"}], "risk_flags": ["ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-flashmla", "blog-vllm-deepseek-v3-sparse", "blog-nsa"], "title": "DeepSeek Sparse Attention / Sparse MLA", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} {"body_sha256": "64b746e36ef7866e93cb1c9aca98703acc5008d485a0d9c96ab31f7a1504a964", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L6", "preview": "TensorRT-LLM PR 13340 adds an FP4 option to its DeepSeek Sparse Attention (DSA) indexer path. This page describes the PR's pinned merge revision `897c4bff`; it is a TensorRT-LLM implementation reference, not a generic DSA ABI or a drop-in F", "sha256": "deba7267d6d469eb687d6d3449e5af0ad0c6d1ba369591c3fb16a90d5e366c36"}, {"id": "u002", "kind": "prose", "locator": "body:L8-L8", "preview": "The implementation has four distinct stages:", "sha256": "88d97624362af38799246c1501a4fe33def3f446b9b4aa90ab01b801fcd1925c"}, {"id": "u003", "kind": "table-row", "locator": "body:L12-L12", "preview": "| Stage | Pinned behavior | | Q/K preparation | `fused_cat_fp4` concatenates BF16 positional and non-positional components, then quantizes the 128-value row to FP4 E2M1 with per-32-value UE8M0 scales |", "sha256": "b4dd786700dab6a1916f3f6864b7313bc68209fd47ff51f0797e8e5519acfc4c"}, {"id": "u004", "kind": "table-row", "locator": "body:L13-L13", "preview": "| Stage | Pinned behavior | | Cache update/read | Scatter and gather copy an already-quantized payload and its scale word between contiguous tensors and TensorRT-LLM's possibly non-contiguous paged indexer cache |", "sha256": "9b7854ba44f86df2319c17ed31631bf2238523273d2d8d6358322f7e1ecbbdc6"}, {"id": "u005", "kind": "table-row", "locator": "body:L14-L14", "preview": "| Stage | Pinned behavior | | Indexer logits | The FP4 path reinterprets the packed bytes/scales and dispatches TensorRT-LLM's DeepGEMM FP4 MQA-logit implementation |", "sha256": "c9d9feea66087fdce32ebbe15b4d41baff2b2cd21955fa373128c608ae7e7477"}, {"id": "u006", "kind": "table-row", "locator": "body:L15-L15", "preview": "| Stage | Pinned behavior | | Selection | Prefill or decode top-k runs after logit computation in a separate operator |", "sha256": "67b72313b995f38a22bc9869baf2440b8e21d4e022ff92a1c80aad932b4dfaf1"}, {"id": "u007", "kind": "prose", "locator": "body:L17-L19", "preview": "The gather/scatter kernels do not quantize values, compute logits, or select indices. Calling them \u201cfused quantized gather/scatter\u201d conflates the first two stages.", "sha256": "807375e96e742adf3dcf843c6629b966585ade9ccf3c8eeb2203385bc47f166c"}, {"id": "u008", "kind": "prose", "locator": "body:L23-L23", "preview": "For the fixed 128-value indexer head in this path, one row has:", "sha256": "ebb2a6213e4bff8f699eae8109c1f172ba7e674d0b9b63b222df48f57b18e727"}, {"id": "u009", "kind": "table-row", "locator": "body:L27-L27", "preview": "| Component | Representation | Bytes per row | | Input | BF16 positional values followed by BF16 non-positional values | 256 before quantization |", "sha256": "baec963beab3b990ec4394a2317a757e16a2ae7eed59ba449f4b13f0c8d41276"}, {"id": "u010", "kind": "table-row", "locator": "body:L28-L28", "preview": "| Component | Representation | Bytes per row | | Packed payload | Two FP4 E2M1 codes per byte | 64 |", "sha256": "733a12c767a0e5c2b16891c439d9afb7f02967486419f72c19ab6be2f8899338"}, {"id": "u011", "kind": "table-row", "locator": "body:L29-L29", "preview": "| Component | Representation | Bytes per row | | Scale word | Four UE8M0 exponent bytes, one per successive group of 32 values, packed little-endian into one `int32` | 4 |", "sha256": "4f5e47e73b176f3b9410f26081aa80b35d5dc6f60655aab518f2b403c93c8d03"}, {"id": "u012", "kind": "table-row", "locator": "body:L30-L30", "preview": "| Component | Representation | Bytes per row | | Cache footprint | Packed payload plus scale word | 68 |", "sha256": "eb8484b32ee5c8dbb43eac0b6a213e540b06d702b5a0c10b8af1388ef129c7f5"}, {"id": "u013", "kind": "prose", "locator": "body:L32-L36", "preview": "The fused operator requires CUDA BF16 inputs on one device, at least two dimensions, a contiguous innermost dimension, eight-byte-aligned addresses, equal row counts, and positional width divisible by four. Positional and non-positional wid", "sha256": "a5cc71924358ceb6ead6a2ecda37330218b28290f4b2d41b6adaeb5ac031bebd"}, {"id": "u014", "kind": "prose", "locator": "body:L38-L41", "preview": "The scale for each 32-value group is `2^ceil(log2(max(amax, 1e-12) / 6))`. Quantization uses the FP4 E2M1 magnitude set `{0, 0.5, 1, 1.5, 2, 3, 4, 6}` and packs the earlier value into the low nibble.", "sha256": "f7c210916986dea9cc641e69e78cc98f6ecd2e4cf7699853f0ff2c8ccc511256"}, {"id": "u015", "kind": "prose", "locator": "body:L45-L49", "preview": "The cache is viewed as `[num_blocks, block_size, 1, per_token_size]` and may be non-contiguous. Each token launch copies four bytes per thread. The FP4 path uses 64 payload bytes and one four-byte scale word; the legacy FP8 path uses 128 pa", "sha256": "987ce4347f8506f5eea5a3d223d7a93fb36282a6a0fde62ad4a46290c6f401ba"}, {"id": "u016", "kind": "prose", "locator": "body:L51-L55", "preview": "Payload and scale use separate contiguous `int64` slot-mapping arrays. If either mapping for a token is negative, gather and scatter skip that entire token. The gather wrapper allocates output with `empty`, so a directly gathered row skippe", "sha256": "9368aff166cf872488d1f74571c046b6ce8644b1e602bf3a20e68ce7b65d9bec"}, {"id": "u017", "kind": "prose", "locator": "body:L57-L61", "preview": "The gather wrapper preserves a historical typed view: payload bytes are returned as float8 and scale bytes as FP32. The FP4 call site reinterprets them as packed `int8` data and `int32` scale words before the DeepGEMM call. These views are ", "sha256": "3d313476860278e4ba8137a9d7cdeab1f70baca9b13c2e21fb85e25506deb8a4"}, {"id": "u018", "kind": "list-item", "locator": "body:L65-L65", "preview": "1. Compare `fused_cat_fp4` byte-for-byte with a reference implementation for", "sha256": "154a706e5febec7291f37c016f2c410dda069d1093b5277e758ecc7df33d51d8"}, {"id": "u019", "kind": "prose", "locator": "body:L66-L67", "preview": "zeros, FP4 decision boundaries, saturation, non-contiguous row strides, and multiple positional/non-positional splits whose widths sum to 128.", "sha256": "73c51aeb33326d6cb9aa5f4b0051dbd1226634ef3d17ad7ac5542abf75368a5b"}, {"id": "u020", "kind": "list-item", "locator": "body:L68-L68", "preview": "2. Round-trip valid tokens through scatter and gather using strided cache", "sha256": "dd3e85e021b2f37adc460ca62eb0dd4572d922c801da9f356f462c7edd762b3f"}, {"id": "u021", "kind": "prose", "locator": "body:L69-L70", "preview": "views. Check all 64 payload bytes and the four scale bytes independently; test negative payload and scale mappings without reading skipped empty rows.", "sha256": "5c513ab9c82962738d09cd52fa8abbe2157d34276c0d623a35c76c705daf72bb"}, {"id": "u022", "kind": "list-item", "locator": "body:L71-L71", "preview": "3. Compare FP4 non-paged and paged indexer logits, then selected indices,", "sha256": "f2a52d0cdaea7c448fda6a9ec9891ff1715df56ef0cf0f062f7a4f8e5d951d60"}, {"id": "u023", "kind": "prose", "locator": "body:L72-L73", "preview": "against an unquantized or higher-precision reference with identical masks, weights, and sequence boundaries.", "sha256": "ec15cbb04e5738b69458f6dd903cb1034e1125baeb8eb7fe8ee1db97b5311e8f"}, {"id": "u024", "kind": "list-item", "locator": "body:L74-L74", "preview": "4. Profile fused preparation, scatter, gather, logit computation, top-k, and", "sha256": "21d1fd695924b33d8c8316a89cad074218542e8d28d1311eb48f082429e42713"}, {"id": "u025", "kind": "prose", "locator": "body:L75-L76", "preview": "the complete indexer separately. Report shapes, cache layout, mapping mix, warmup, synchronization, repetitions, statistic, and variation.", "sha256": "b76f11b1782d4489ce3834019cbd0aa1e7079900dc54bde8f66498aa7519c781"}, {"id": "u026", "kind": "prose", "locator": "body:L78-L79", "preview": "The pinned PR contains no performance result, so use it to define candidate mechanisms and exact contracts rather than to claim a speedup.", "sha256": "62b72a9c0d5493138efc92602fe3a9b570342ca3e3a1afd37d89359535faae33"}, {"id": "u027", "kind": "list-item", "locator": "body:L83-L83", "preview": "- [TensorRT-LLM PR 13340](https://github.com/NVIDIA/TensorRT-LLM/pull/13340)", "sha256": "b5192a1e772c7397854ba03a1cf3a79cadfd904f45437b099332dda59be08ce4"}, {"id": "u028", "kind": "list-item", "locator": "body:L84-L84", "preview": "- [Pinned `fusedCatFp4.cu` artifact](../../artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/fusedCatFp4.cu)", "sha256": "5c2e1274b194d2974112ca102087a93493f94e7c6c0166d5e928aaaf4423415f"}, {"id": "u029", "kind": "list-item", "locator": "body:L85-L85", "preview": "- [Pinned gather kernel artifact](../../artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheGather.cu)", "sha256": "e4cb68f24b2d80c4b670273abebab689b0effa32829ffbd7ba41922a51f517c3"}, {"id": "u030", "kind": "list-item", "locator": "body:L86-L86", "preview": "- [Pinned scatter kernel artifact](../../artifacts/prs/tensorrt-llm/PR-13340/key-files/cpp/tensorrt_llm/kernels/indexerKCacheScatter.cu)", "sha256": "0811f065211145ad22d7ea66d97a233992db16c6c2a1559f862a6399de29cef5"}, {"id": "u031", "kind": "list-item", "locator": "body:L87-L87", "preview": "- [Pinned DSA integration artifact](../../artifacts/prs/tensorrt-llm/PR-13340/key-files/tensorrt_llm/_torch/attention_backend/sparse/dsa.py)", "sha256": "6ab0e3cc566c51a18ede438c4e7b208034aae703a7d915c4a0ae46c997b62226"}], "confidence_claimed": "source-reported", "headings": ["Audited scope", "FP4 row and scale contract", "Gather and scatter semantics", "Validation and profiling procedure", "Primary references"], "id": "kernel-tensorrt-llm-blackwell-indexer", "path": "wiki/kernels/tensorrt-llm-blackwell-indexer.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/prs/TensorRT-LLM/PR-13340.md", "revision": "897c4bff", "url": "https://github.com/NVIDIA/TensorRT-LLM/pull/13340"}], "risk_flags": ["ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["pr-TensorRT-LLM-13340"], "title": "TensorRT-LLM Blackwell FP4 DSA Indexer", "type": "kernel", "unresolved_source_ids": [], "version_sensitive": null} @@ -24,29 +24,29 @@ {"body_sha256": "90ca9df7c219547d83ab1a5609935cb5d4b5fe4ebd1dee959da0fadaf0c0c5a9", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "CUTLASS v4.5.0 includes CuTe DSL, a Python-native interface for authoring GPU kernels, alongside CUTLASS's C++ template interfaces. This page uses the exact `v4.5.0` tag (`e406c186f510a15091cce01f782020ceb7ba8eb5`); rolling `latest` documen", "sha256": "334261620dc71d6f22a0a53fcf732078f58347ac693be8556adaa40ca4a61e5d"}, {"id": "u002", "kind": "prose", "locator": "body:L5-L5", "preview": "FlashAttention-4 is implemented entirely in CuTe DSL. Its author reports roughly 20--30x shorter compile times than C++ templates and a peak of 1605 TFLOP/s on B200 BF16. These are FA4-specific, source-reported results. The 1605-TFLOP/s res", "sha256": "056d2cac890be9c1eb0d800df0380ae301fe97f8895ad6404e8633b9112135f7"}, {"id": "u003", "kind": "prose", "locator": "body:L9-L9", "preview": "CUTLASS 4.5.0 uses a configured `tcgen05.MmaF16BF16Op`, not an `SM100_MMA_F16BF16_SS` Python symbol. This excerpt shows the one-CTA operation used by the tagged first tutorial; it is construction code, not a complete kernel.", "sha256": "37f03738b7d78dbf444d54d1793ac9d5defae6cad433d27dabfea02f62b5ce85"}, {"id": "u004", "kind": "code", "locator": "body:L11-L26", "preview": "```python import cutlass import cutlass.cute as cute from cutlass.cute.nvgpu import tcgen05 op = tcgen05.MmaF16BF16Op( cutlass.Float16, cutlass.Float32, (128, 256, 16), tcgen05.CtaGroup.ONE, tcgen05.OperandSource.SMEM, cute.nvgpu.OperandMaj", "sha256": "e63cc3018bae603de0436601418eab234ce8ae74754a85c50ecda03dc733d264"}, {"id": "u005", "kind": "prose", "locator": "body:L28-L28", "preview": "The two-CTA tutorial instead uses instruction shape `(256, 256, 16)` and `tcgen05.CtaGroup.TWO`. The accumulator fragment is rebound to an allocated TMEM pointer before `cute.gemm` issues the operation.", "sha256": "59dd30be131eab3431bcbbf1b6d2ce88712c263139dd23b53853f002255058fa"}, {"id": "u006", "kind": "prose", "locator": "body:L32-L32", "preview": "The tagged examples use `cutlass.utils.TmemAllocator` around a shared holding buffer. A complete path allocates columns, waits before pointer retrieval, rebinds the accumulator tensor, partitions a typed `tcgen05` TMEM-to-register copy acro", "sha256": "609d3277d8eeaeb952785969f9a32c9cb5a3029a965f2f2a6b6ab1644358cdef"}, {"id": "u007", "kind": "code", "locator": "body:L34-L53", "preview": "```python # Excerpt: storage/barrier/tensor definitions and pipeline edges are required. tmem = utils.TmemAllocator( storage.tmem_holding_buf.ptr, barrier_for_retrieve=tmem_alloc_barrier, ) tmem.allocate(num_tmem_cols) tmem.wait_for_alloc()", "sha256": "76e8bef333b52e86589cffba34de7858d51e7e22e686ea20302ad4a681a3ff65"}, {"id": "u008", "kind": "prose", "locator": "body:L55-L55", "preview": "The excerpt deliberately does not imply that allocation or deallocation is lane-local. Use the complete tutorial for collective participation, two-CTA handling, and pointer lifetime.", "sha256": "784d2fca4eef895f2657ad5f9f3c7efc742f0578ad17207364970580d4ca2145"}, {"id": "u009", "kind": "prose", "locator": "body:L59-L59", "preview": "The one-CTA tutorial constructs a typed global-to-shared TMA operation and then derives operand-specific tiled atoms. Kernel code subsequently uses `tma_partition` and `cute.copy` with pipeline barriers.", "sha256": "0f534dfc499371ce84c7e460c19e133f0d91acb3fcc1925107b5beff7735a851"}, {"id": "u010", "kind": "code", "locator": "body:L61-L72", "preview": "```python from cutlass.cute.nvgpu import cpasync, tcgen05 tma_op = cpasync.CopyBulkTensorTileG2SOp(tcgen05.CtaGroup.ONE) a_tma_atom, a_tma_tensor = cute.nvgpu.make_tiled_tma_atom_A( tma_op, a, a_smem_layout_one_stage, mma_tiler_mnk, tiled_m", "sha256": "60ca34012cb11ec18e15df9160f3855f4f84db9a459f67d9b7b757bd738b7650"}, {"id": "u011", "kind": "prose", "locator": "body:L74-L74", "preview": "For the two-CTA tutorial, the corresponding operation is `CopyBulkTensorTileG2SMulticastOp(CtaGroup.TWO)` and the cluster layout/multicast participants must agree with the launch.", "sha256": "131b73c4815e2daf0d7909840ca0dfb9311821758eed73af3350d021bd113207"}, {"id": "u012", "kind": "prose", "locator": "body:L78-L78", "preview": "`fp16_gemm_2.py` specializes TMA, MMA, and epilogue warps and uses `PipelineTmaUmma` plus `PipelineUmmaAsync` to represent full/empty ownership. It also retains explicit TMEM allocation, epilogue TMEM-copy partitioning, TMA-store completion", "sha256": "2642e3d5608c72ac65111fba5b4e0ff7f40f082f286dc2eb0264a7a484a8c8ed"}, {"id": "u013", "kind": "prose", "locator": "body:L80-L80", "preview": "CuTe supplies typed layout algebra and Blackwell helpers such as `make_smem_layout_a` and `make_smem_layout_b`. The author still supplies the MMA tiler, datatypes, operand major modes, alignment, cluster shape, and pipeline policy; layout a", "sha256": "d6c1c90120129d90b89dff9868a75e231e1d052c8d141776044e6ada0808f8ea"}, {"id": "u014", "kind": "prose", "locator": "body:L84-L84", "preview": "The following files are untruncated, verbatim captures. Their bytes match the SHA-256 records in each PR's `PROVENANCE.yaml`, and the seven numeric sizes below are physical line counts. To inspect through the repository helper, run `conda r", "sha256": "973251632a33fdc9e5912eab3418784d8643fd27350dc1b9c9a6f5b406ebd703"}, {"id": "u015", "kind": "table-row", "locator": "body:L88-L88", "preview": "| File | Upstream purpose | Lines | | [`fp16_gemm_0.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_0.py) | Software-pipelined FP16 tutorial baseline | 447 |", "sha256": "b18a5ec40a6c45897cf6249107ed902c41a33e9ef744d1bab24361e43e147139"}, {"id": "u016", "kind": "table-row", "locator": "body:L89-L89", "preview": "| File | Upstream purpose | Lines | | [`fp16_gemm_1.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_1.py) | Add two-CTA MMA and TMA multicast | 535 |", "sha256": "d6bf4bd7815fdca9a99c3fc22ff3d92162b156750825d479ced568997c142584"}, {"id": "u017", "kind": "table-row", "locator": "body:L90-L90", "preview": "| File | Upstream purpose | Lines | | [`fp16_gemm_2.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_2.py) | Add TMA/MMA/epilogue warp specialization | 679 |", "sha256": "1e5ec9c2a029b9219be46f35295e8e70936565a4222e8f2af8b92686ef480c39"}, {"id": "u018", "kind": "table-row", "locator": "body:L91-L91", "preview": "| File | Upstream purpose | Lines | | [`fp16_gemm_3.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_3.py) | Add static persistent scheduling | 769 |", "sha256": "352bb7673da6dbf2d748da7fc8a6b97df8bbaaf3c384d87dad952797c5e376ee"}, {"id": "u019", "kind": "table-row", "locator": "body:L92-L92", "preview": "| File | Upstream purpose | Lines | | [`fp16_gemm_4.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_4.py) | Add preferred and fallback/dynamic clusters | 1065 |", "sha256": "14f3ac2f46ca97c9134ac0b89fc357bffd4074ee52b3b6d16ac578d8f07865cb"}, {"id": "u020", "kind": "table-row", "locator": "body:L93-L93", "preview": "| File | Upstream purpose | Lines | | [`fp16_gemm_5.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_5.py) | Add TMA data prefetch | 919 |", "sha256": "9fd8f92a66857f1af567a8e0f0e9189a5d04916a7d81244ff86764bcf57e6a21"}, {"id": "u021", "kind": "table-row", "locator": "body:L94-L94", "preview": "| File | Upstream purpose | Lines | | [`fp16_gemm_6.py`](../../artifacts/prs/cutlass/PR-3106/key-files/examples/python/CuTeDSL/blackwell/tutorial_gemm/fp16_gemm_6.py) | Add Programmatic Dependent Launch | 1002 |", "sha256": "eb3aaa1fe6a194346ade2621b9c431c4380a88779e4eb4c04a97b71d5ab0f931"}, {"id": "u022", "kind": "table-row", "locator": "body:L95-L95", "preview": "| File | Upstream purpose | Lines | | [`dense_gemm_persistent_prefetch.py`](../../artifacts/prs/cutlass/PR-2881/key-files/examples/python/CuTeDSL/blackwell/dense_gemm_persistent_prefetch.py) | Persistent GEMM with TMA prefetch | full captur", "sha256": "9ebdb330439f1139f28b3838ec227c79d34b7975dec2c9e5df6d14660bb3b241"}, {"id": "u023", "kind": "table-row", "locator": "body:L96-L96", "preview": "| File | Upstream purpose | Lines | | [`clc.py`](../../artifacts/prs/cutlass/PR-3021/key-files/python/CuTeDSL/cutlass/cute/arch/clc.py) | CLC Python binding | full capture |", "sha256": "79166083bd1d286c85076e5de9724750ddff7a17063461789553b1bb6ed340a3"}, {"id": "u024", "kind": "prose", "locator": "body:L98-L98", "preview": "PR 3106's series is an official progressive tutorial: software-pipelined FP16, two-CTA multicast, warp specialization, static persistence, preferred/dynamic clusters, prefetch, and PDL. In the v4.5.0 tag, the same series is under `examples/", "sha256": "b40157951f3e6a0b7e4642199d84f739ad03db4544e88e5408ccd6f00e1093ed"}, {"id": "u025", "kind": "list-item", "locator": "body:L102-L102", "preview": "- [CUTLASS v4.5.0 tag](https://github.com/NVIDIA/cutlass/tree/e406c186f510a15091cce01f782020ceb7ba8eb5)", "sha256": "8c040cd9bb2af23c748671cfd8c2c060bb7c97a2538c966f83ac196ccea9a9ad"}, {"id": "u026", "kind": "list-item", "locator": "body:L103-L103", "preview": "- [Pinned v4.5.0 tutorial series](https://github.com/NVIDIA/cutlass/tree/e406c186f510a15091cce01f782020ceb7ba8eb5/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm)", "sha256": "3375a33859f68d3d9dba83826c9e7d675fbd2d160d113044e3a54814fbac9c05"}, {"id": "u027", "kind": "list-item", "locator": "body:L104-L104", "preview": "- [FlashAttention-4 first-party post](https://tridao.me/blog/2026/flash4/)", "sha256": "ffdb1b83558fb29dbc72af17130e91affe8fa4562c2f6c01a63be8b7082b1ab3"}, {"id": "u028", "kind": "list-item", "locator": "body:L108-L108", "preview": "- [tcgen05-mma](../hardware/tcgen05-mma.md) \u2014 underlying instruction semantics", "sha256": "b66ad09998a670f6ea957a7d01c9bc9760f1107ca50f4444c411f41d0a76fed5"}, {"id": "u029", "kind": "list-item", "locator": "body:L109-L109", "preview": "- [flash-attention-4](../kernels/flash-attention-4.md) \u2014 evidence-scoped CuTe DSL case study", "sha256": "b35637f36cfb18108cd0c26bab36090406fbfca2c4690afb113a6a08856e2602"}, {"id": "u030", "kind": "list-item", "locator": "body:L110-L110", "preview": "- [CUTLASS Blackwell source card](../../sources/docs/nvidia-cutlass-blackwell.md) \u2014 version-pinned source routes", "sha256": "a445c9adf25fe1277381dbdeea01cb44d616f8c12ebbf5a8251f145cfff266dc"}], "confidence_claimed": "verified", "headings": ["Scope", "F16/BF16 MMA construction", "TMEM lifecycle and epilogue copy", "TMA construction", "Warp specialization and layouts", "Verbatim upstream artifacts", "Primary references", "Related"], "id": "lang-cute-dsl", "path": "wiki/languages/cute-dsl.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-cutlass-blackwell.md", "url": "https://github.com/NVIDIA/cutlass/tree/v4.5.0"}, {"path": "sources/blogs/colfax-cutlass-blackwell.md", "url": "https://research.colfax-intl.com/cutlass-tutorial-writing-gemm-kernels-using-tmem-for-nvidia-blackwell-gpus/"}, {"path": "sources/blogs/flash-attention-4.md", "url": "https://tridao.me/blog/2026/flash4/"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-cutlass-blackwell", "blog-colfax-cutlass", "blog-flash-attention-4"], "title": "CuTe DSL for Blackwell", "type": "language", "unresolved_source_ids": [], "version_sensitive": null} {"body_sha256": "4bbed1d5aa5094a031fdbe1e8349f5079263caf9887a77721a5ace82c565fad5", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "This page uses CUDA 13.0.2's archived PTX ISA 9.0 grammar. The fragments are representative instruction forms, not complete inline-assembly functions: operands need correctly typed declarations, descriptors, collective issue, lifetime manag", "sha256": "033f6c44d6511b7780358590ba1a15b928bfc82f021f49297cac0039b7377829"}, {"id": "u002", "kind": "prose", "locator": "body:L9-L9", "preview": "Representative unscaled F16 forms include:", "sha256": "44a5c5881e34f6a016787e26082b9e096b84d5f8c8455c0c6fa3807305071b57"}, {"id": "u003", "kind": "code", "locator": "body:L11-L19", "preview": "```ptx tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [alloc_result_smem], ncols; ld.shared.b32 taddr, [alloc_result_smem]; tcgen05.mma.cta_group::1.kind::f16 [taddr], a_desc, b_desc, idesc, enable_input_d; tcgen05.commit.cta_group", "sha256": "8ee862ded6432cbcbaab1586c6dca16626f9c1a8562d6b39189ba6c05e5afb44"}, {"id": "u004", "kind": "prose", "locator": "body:L21-L21", "preview": "`tcgen05.alloc` writes a 32-bit TMEM address to shared memory and counts columns. Allocation and deallocation are warp-collective for `cta_group::1`, every allocation must be deallocated before kernel exit, and all tcgen05 instructions in o", "sha256": "549eeb92a806311fe19b9d3a00fc5521bddc347201227fe36fe18b4ab20ce0aa"}, {"id": "u005", "kind": "prose", "locator": "body:L23-L23", "preview": "TMEM/register transfers and shared-memory/TMEM copies have their own asynchronous completion rules:", "sha256": "309d97f3ce7cd084b463b572dd973759f13aa75f87df09939aa962b25d2b998f"}, {"id": "u006", "kind": "code", "locator": "body:L25-L33", "preview": "```ptx tcgen05.ld.sync.aligned.32x32b.x1.b32 {r0}, [taddr]; tcgen05.wait::ld.sync.aligned; tcgen05.st.sync.aligned.32x32b.x1.b32 [taddr], {r0}; tcgen05.wait::st.sync.aligned; tcgen05.cp.cta_group::1.128x256b [taddr], sdesc; ```", "sha256": "07a9ef9877db393b8545daac9555a8c89fecdb37869186d00ca531f2c0673fa4"}, {"id": "u007", "kind": "prose", "locator": "body:L35-L35", "preview": "`tcgen05.ld` and `tcgen05.st` are warp-collective. `tcgen05.cp` copies a shaped shared-memory descriptor into TMEM. MMA and cp completion can be attached to an mbarrier with `tcgen05.commit`; the source and destination must remain live unti", "sha256": "5d14e69f987be01a3b619b1aa87a7089e1f92428c3dc595e2db26b3fcba0ed3b"}, {"id": "u008", "kind": "list-item", "locator": "body:L39-L39", "preview": "- A TMA global-to-shared load completes bytes on its mbarrier; a consumer waits for that phase before reading the destination.", "sha256": "78e297e1879d27a7cf7e373d63d1a5098a93c33b03215f1eafce9842ed1aa2a0"}, {"id": "u009", "kind": "list-item", "locator": "body:L40-L40", "preview": "- `tcgen05.commit` makes an mbarrier track completion of prior asynchronous tcgen05 MMA/cp/shift operations issued by the thread.", "sha256": "ed6a936d875de52f53dfa04b1ae6a7d11f9232c0e335cd07557501558d6b9e4c"}, {"id": "u010", "kind": "list-item", "locator": "body:L41-L41", "preview": "- `tcgen05.wait::ld` and `tcgen05.wait::st` wait for the corresponding prior TMEM/register transfers.", "sha256": "a9db9c17084afc3bea027eeda6880b05178a6a63a358cbd960d4a046f9c45bb6"}, {"id": "u011", "kind": "list-item", "locator": "body:L42-L42", "preview": "- `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` constrain tcgen05 operations around a documented cross-thread execution-ordering handoff. They are not substitutes for TMA or MMA completion waits.", "sha256": "711d6cdcb2d789fb3aeb9b8207f61c2863f9ac6fbb4f1408e2fc015cba4a4d73"}, {"id": "u012", "kind": "prose", "locator": "body:L46-L46", "preview": "PTX ISA 9.0 defines these typed operations:", "sha256": "9adc5d2ba8359512852b72fddbd060410dec97cb859c1966ad61851b7308d289"}, {"id": "u013", "kind": "code", "locator": "body:L48-L51", "preview": "```ptx cvt.rn.f16x2.e2m1x2 result_f16x2, packed_fp4_pair; mov.b32 {byte0, byte1, byte2, byte3}, packed_word; ```", "sha256": "e38e93ad27ad63afd75bcd8acd17fea4fd7b8d6947dac9c50bbc5189fd76c762"}, {"id": "u014", "kind": "prose", "locator": "body:L53-L53", "preview": "The first converts one byte containing two E2M1 values into a 32-bit F16x2 result. The second can decompose a 32-bit scalar into four byte-sized destinations when operand declarations satisfy the scalar-to-vector size rules. PTX specifies t", "sha256": "4e1316f897104a6e4ff30b2d1629ba8c31578c28efa1f9f335e2a90635a5bc10"}, {"id": "u015", "kind": "code", "locator": "body:L57-L61", "preview": "```ptx ld.global.L1::no_allocate.v2.u64 {r0, r1}, [addr]; ld.global.L1::evict_last.v2.u64 {r0, r1}, [addr]; ld.global.v4.u64 {r0, r1, r2, r3}, [addr]; ```", "sha256": "340109ac2a79a1eab9995193f4fdf446a23eac0c855e50ec472ab508290c93ad"}, {"id": "u016", "kind": "prose", "locator": "body:L63-L63", "preview": "`L1::no_allocate` and `L1::evict_last` are eviction-priority hints and may not always be respected. They do not guarantee L1 bypass or residency. The vector forms above move 16 and 32 bytes; whether a width or hint helps depends on alignmen", "sha256": "2d50ea674a04d5f5af2f3d408566c7166d4da40732d3c0f69a61ee3be6bda853"}, {"id": "u017", "kind": "prose", "locator": "body:L67-L67", "preview": "CLC cancellation is an asynchronous request followed by response decoding:", "sha256": "e1053c3dda6a336be00c8fe12b42dda779f9491181fe98936e6b907183ac3f63"}, {"id": "u018", "kind": "code", "locator": "body:L69-L76", "preview": "```ptx clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [response_smem], [response_mbarrier]; clusterlaunchcontrol.query_cancel.is_canceled.pred.b128 p, response_b128; @p clusterlaunchcontrol.query_cancel.", "sha256": "3d1e08688104e24edf01c64713b007cd76c560a8224eb3ff452117f18046ea42"}, {"id": "u019", "kind": "prose", "locator": "body:L78-L78", "preview": "The request writes an opaque 16-byte response to shared memory and completes on the mbarrier. Code must wait for that phase before loading and querying the response. A successful query returns the first CTA coordinate of a canceled not-yet-", "sha256": "a71d676af20173533f4312c70e51a00ade2b502fcf17504097caaaccb8229b5d"}, {"id": "u020", "kind": "prose", "locator": "body:L82-L82", "preview": "A representative 2D global-to-cluster-shared load is:", "sha256": "4420aec4da4b62535c1e9270382b6fdf555635efc5797fb7dc2d8db90a7767ab"}, {"id": "u021", "kind": "code", "locator": "body:L84-L87", "preview": "```ptx cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes [dst_smem], [tensor_map, {x, y}], [full_mbarrier]; ```", "sha256": "384b75e38a92db0f949c0b5a2f222f154108174be7c8aeb347f23bd57cbe6c54"}, {"id": "u022", "kind": "prose", "locator": "body:L89-L89", "preview": "Cluster multicast adds the full `.multicast::cluster` qualifier and a 16-bit CTA mask:", "sha256": "bdbff1d282630fa2048aa2d3e5b44339402c00503993effcf84fb95908336362"}, {"id": "u023", "kind": "code", "locator": "body:L91-L94", "preview": "```ptx cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster [dst_smem], [tensor_map, {x, y}], [full_mbarrier], cta_mask; ```", "sha256": "a65641655400da2854fcf8a87260ac7f8823215dba2e9d9cbdaa262378541169"}, {"id": "u024", "kind": "prose", "locator": "body:L96-L96", "preview": "The multicast data and completion signal target the selected CTAs at corresponding shared-memory offsets. Each destination must have a live, initialized matching barrier and wait for its own phase before consuming the tile.", "sha256": "a4eb1693de166fae870fc703cd84ebf94c38231450f081d012205e644ee51b01"}, {"id": "u025", "kind": "list-item", "locator": "body:L100-L100", "preview": "- [CUDA C++](cuda-cpp.md) \u2014 inline PTX integration", "sha256": "0cc70a94a27647104f0234aed6752f236117acdbd312d3b46d7eccd2e99e116a"}, {"id": "u026", "kind": "list-item", "locator": "body:L101-L101", "preview": "- [tcgen05 MMA](../hardware/tcgen05-mma.md) \u2014 operand, descriptor, completion, and shape details", "sha256": "b3297f05442f0dbd4d7c52be92ee40895b97bd816b3fbed5217de657284554bd"}, {"id": "u027", "kind": "list-item", "locator": "body:L102-L102", "preview": "- [Tensor Memory](../hardware/tmem.md) \u2014 allocation, layout, and transfer details", "sha256": "c13ae762a67d3f3b24856da39bfe99e2231999145211ed928d11e100c5cabad8"}, {"id": "u028", "kind": "list-item", "locator": "body:L103-L103", "preview": "- [Cluster Launch Control](../hardware/clc.md) \u2014 complete request/decode protocol", "sha256": "9e45deb92f889fefaf02795579b0410a680552b1ee29c0b894cadd5c4df97e11"}, {"id": "u029", "kind": "list-item", "locator": "body:L104-L104", "preview": "- [TMA](../hardware/tma.md) \u2014 tensor-map and pipeline semantics", "sha256": "325d9c316ccd41d8da1955efe4719b95727a1b4e47e368cc36dc5b69c2df3389"}, {"id": "u030", "kind": "list-item", "locator": "body:L105-L105", "preview": "- [NVFP4](../hardware/nvfp4.md) \u2014 packed format and block scaling", "sha256": "41e892a35a6f5c8b6e2a987ed2c3b6d644d33c7c199fe6b98d74dfd378ee941b"}, {"id": "u031", "kind": "list-item", "locator": "body:L109-L109", "preview": "- [CUDA 13.0.2, PTX ISA 9.0](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html)", "sha256": "ec36f101dd169720db290c03b2a18a106539b88b4b3c0c12559a90520c58403b"}], "confidence_claimed": "verified", "headings": ["Scope", "tcgen05 and Tensor Memory", "Completion and Ordering Are Distinct", "Packed FP4 Conversion", "Cache Eviction Hints and Vector Width", "Cluster Launch Control", "Tensor Memory Accelerator", "Related", "Primary Reference"], "id": "lang-ptx", "path": "wiki/languages/ptx-sm100.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/yue-nvfp4-hackathon.md", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}], "risk_flags": ["code", "ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-ptx-isa-sm100", "doc-nvidia-tuning-guide", "blog-yue-nvfp4"], "title": "PTX Instructions for SM100", "type": "language", "unresolved_source_ids": [], "version_sensitive": null} {"body_sha256": "ea6b2fb7068dfa7b8a5d31fd9324937075bcbd313e86292a970f031a005a821c", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "Native Blackwell TCGen5/TMEM compiler support enters between Triton v3.2.0 and v3.3.0. In the exact tag comparison, the corresponding TCGen5 MMA, TMEM, and MMAv5-lowering symbols are absent at v3.2.0 (`9641643d`) and present at v3.3.0 (`819", "sha256": "e3920948bd65b746042cddc8a55cbb58d67cea87f7de76b45b7f86b40076a399"}, {"id": "u002", "kind": "table-row", "locator": "body:L7-L7", "preview": "| Release | Evidence-scoped milestone | | v3.2.0 | Checked negative side of the native-backend boundary. |", "sha256": "29a2a0a6a1b0cab67d3c9170be5b53e14d5e925950ae7accae025a44743032c2"}, {"id": "u003", "kind": "table-row", "locator": "body:L8-L8", "preview": "| Release | Evidence-scoped milestone | | v3.3.0 | First checked tag after v3.2.0 with TCGen5/TMEM operations, allocation and lowering passes, and concrete conversion tests. |", "sha256": "96b6b9eb266480436cec3e260a7190ef701a6cdb182590188a1a9e08e8b2bc72"}, {"id": "u004", "kind": "table-row", "locator": "body:L9-L9", "preview": "| Release | Evidence-scoped milestone | | v3.5.0 | Tagged tree includes an explicit Gluon TCGen5/TMEM tutorial and the Blackwell block-scaled matmul tutorial; release notes also document warp-specialization work. |", "sha256": "bbfeba5bdbe013bf1e25f0b3313537c044591289530ad5ad3348df812d2f7db9"}, {"id": "u005", "kind": "table-row", "locator": "body:L10-L10", "preview": "| Release | Evidence-scoped milestone | | v3.6.0 | Generalizes TCGen5 copies/layouts and MMA handling, advances aref-style warp specialization, and adds initial multi-CTA/2-CTA Gluon work. It is not the introduction boundary. |", "sha256": "ae8dd494d86b9bdf4335d086a7fc67293ccb647ee15a43e83ae74a51c7583926"}, {"id": "u006", "kind": "table-row", "locator": "body:L11-L11", "preview": "| Release | Evidence-scoped milestone | | v3.7.0 / v3.7.1 | v3.7.0 continues 2-CTA, multicast, and TMA work; v3.7.1 is a two-regression patch with no advertised new API or feature. |", "sha256": "4635dc9d8b11b6614cb89888178e84327676f99fed5e0b67f2e5f75e64e46829"}, {"id": "u007", "kind": "prose", "locator": "body:L13-L13", "preview": "The compiler evidence proves that native paths exist. It does not prove that every plain `tl.dot` shape selects TCGen5: instruction selection can depend on architecture, dtype, shape, layout, and compiler configuration. A claim about one ke", "sha256": "469542c8b3f4a8fa1b59f96d3475a8808eb5e863feb48288a117287981a6cd98"}, {"id": "u008", "kind": "prose", "locator": "body:L17-L17", "preview": "The v3.5.0 tree provides two useful pinned examples:", "sha256": "e5dab5d53c539d58ff6ace9b78fcdcc0297af67d3094ebb9b76bdd4f6d044c54"}, {"id": "u009", "kind": "list-item", "locator": "body:L19-L19", "preview": "- [`python/tutorials/gluon/06-tcgen05.py`](https://github.com/triton-lang/triton/blob/c3c476f357f1e9768ea4e45aa5c17528449ab9ef/python/tutorials/gluon/06-tcgen05.py) explicitly allocates, loads, and stores TMEM and invokes TCGen5 MMA through", "sha256": "d517d20730d84aea345276a3cd17d16fc337b410b720198a34bd749d15a97691"}, {"id": "u010", "kind": "list-item", "locator": "body:L20-L20", "preview": "- [`python/tutorials/10-block-scaled-matmul.py`](https://github.com/triton-lang/triton/blob/c3c476f357f1e9768ea4e45aa5c17528449ab9ef/python/tutorials/10-block-scaled-matmul.py) demonstrates `tl.dot_scaled` for Blackwell block-scaled matmul.", "sha256": "b50af21443f3e19ead4a29e736f49169439ab7c89c9c9d58f3d1ff30d4d513d7"}, {"id": "u011", "kind": "prose", "locator": "body:L22-L22", "preview": "Triton v3.6.0 expands those foundations with broader layouts and copies and initial multi-CTA/2-CTA Gluon support. \u201cInitial\u201d is deliberate: v3.7.0 contains follow-on end-to-end 2-CTA, multicast, and TMA changes. See [`doc-triton-3.6-blackwe", "sha256": "248e2625a1e191a60b52ac1983ab2fa8ef8742c3dc42749ec8560102a3d81256"}, {"id": "u012", "kind": "list-item", "locator": "body:L26-L26", "preview": "- [`pr-vllm-34597`](../../sources/prs/vllm/PR-34597.md), pinned at `a1257fd1`, adds FP8 KV-cache handling to the Triton MLA decode backend. Its verbatim kernel contains `tl.dot`, but no target guard, Triton-version requirement, TCGen5/TMEM ", "sha256": "085b49bb4247323760315512a1abade8e49e1a52e8ae7bb49cc3ea835a7fa13e"}, {"id": "u013", "kind": "list-item", "locator": "body:L27-L27", "preview": "- [`pr-vllm-29339`](../../sources/prs/vllm/PR-29339.md), pinned at `c17610e2`, gates MXFP4 `triton_kernels` dispatch to SM90 and SM100. It changes dispatch logic, not a kernel or compiler lowering.", "sha256": "992ea35b568fd02174010ce53c73d442063daab448917c69bc178401610ad4d4"}, {"id": "u014", "kind": "list-item", "locator": "body:L28-L28", "preview": "- [`pr-sglang-21019`](../../sources/prs/sglang/PR-21019.md), pinned at `5bdc07d9`, provides a Triton GatedDeltaNet projection rearrangement using loads and stores, with no `tl.dot`.", "sha256": "9b174a1966e9c73cd142807ed4ba1fadd3d338c47fb5f52962b55350f2e9bc44"}, {"id": "u015", "kind": "list-item", "locator": "body:L29-L29", "preview": "- [`pr-sglang-22079`](../../sources/prs/sglang/PR-22079.md), pinned at `5638d40f`, provides an extend-attention Triton kernel with real `tl.dot` operations. The source does not contain an emitted-PTX witness for a particular MMA instruction", "sha256": "e4a160b6f2a7c6504d671624f8587b747817600bf587d805c15d41780f4dfd5c"}, {"id": "u016", "kind": "prose", "locator": "body:L31-L31", "preview": "These are verified downstream Triton examples. They must not be combined with the separate compiler-version evidence to infer an unobserved lowering.", "sha256": "e25568d8c8d675f229103457b89bfe14c70bb60ea3c384f8aec4ee49263bde73"}, {"id": "u017", "kind": "prose", "locator": "body:L35-L35", "preview": "SGLang [`pr-sglang-5390`](../../sources/prs/sglang/PR-5390.md) reports 10,447.34 total tok/s for its CUTLASS MLA run and 8,227.35 total tok/s for its Triton run, 26.98% higher under the PR's recorded DeepSeek-R1, 3,000-prompt, TP8/DP8, floa", "sha256": "e18be1d70f68c7e9f57e4ce6512515fba305011c95d9c3fb4f4aa91cee317942"}, {"id": "u018", "kind": "prose", "locator": "body:L37-L37", "preview": "SGLang [`pr-sglang-21595`](../../sources/prs/sglang/PR-21595.md) changes the SM100 datacenter multimodal-attention default from `triton_attn` to FA4. That routing decision is likewise workload- and architecture-scoped.", "sha256": "bc5f5aa77792eade3e6336d612d0c662e86f76c68607c91984455ea36ce09846"}, {"id": "u019", "kind": "prose", "locator": "body:L39-L39", "preview": "The live [FlashInfer-Bench leaderboard](https://bench.flashinfer.ai/), retrieved 2026-08-08, reports these author rows across 660 workloads each:", "sha256": "eb3c9c6949d2033f82810f3915ba67acd79712b58c62852e8abee880b272c1f9"}, {"id": "u020", "kind": "table-row", "locator": "body:L43-L43", "preview": "| Author | Average speedup | Resolved | | Gemini 2.5 Pro | 0.628x | 73.1% |", "sha256": "2c931bfe6fdce66ed874bc7939e251dfd74ce7efc8f76df9065a732b9160595d"}, {"id": "u021", "kind": "table-row", "locator": "body:L44-L44", "preview": "| Author | Average speedup | Resolved | | GPT-5 | 0.467x | 92.3% |", "sha256": "10715c194c5d2213a64860a9723c9215121405e98c58f8c654d5568d0ba792ed"}, {"id": "u022", "kind": "table-row", "locator": "body:L45-L45", "preview": "| Author | Average speedup | Resolved | | Claude Opus 4.1 | 0.456x | 73.1% |", "sha256": "da12cac615989bf57643afef7d553cf8210b73575d9b19def2feff7012898c65"}, {"id": "u023", "kind": "prose", "locator": "body:L47-L47", "preview": "The leaderboard does not attach those rows to a Triton release or identify them as a Triton-only language subset.", "sha256": "dc9b8f5d025f7b785f70fdb6cdf00ba150996bc6d75aca61cadfd91fc3f17a18"}, {"id": "u024", "kind": "prose", "locator": "body:L51-L51", "preview": "The CUDA Programming Guide explains that CPU setup and launch overhead can be significant for short kernels and that CUDA Graphs reduce repeated launch costs by preparing work in advance. At vLLM commit `a1257fd1`, `FULL_AND_PIECEWISE` is t", "sha256": "86ec28fae13765d35d9fddd442f01998a03d9e5f585a31169e38df465f897058"}, {"id": "u025", "kind": "prose", "locator": "body:L55-L55", "preview": "Each linked file is stored verbatim under its bundle's recorded merge SHA:", "sha256": "d8ff7fcdf07f06c752587e1f093f017a673d3c1baaf61a748a7f43d06cec1c58"}, {"id": "u026", "kind": "table-row", "locator": "body:L59-L59", "preview": "| File | Verified role | | [`triton_decode_attention.py`](../../artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/ops/triton_decode_attention.py) | Triton MLA decode kernels with `tl.dot`; PR 34597 adds FP8 cache handling. |", "sha256": "c531c719c3ba28f58191f062dda8ae43664fde6c4201bfbcf6c130b7cd19f417"}, {"id": "u027", "kind": "table-row", "locator": "body:L60-L60", "preview": "| File | Verified role | | [`triton_mla.py`](../../artifacts/prs/vllm/PR-34597/key-files/vllm/v1/attention/backends/mla/triton_mla.py) | Backend wrapper and supported FP8 cache dtypes. |", "sha256": "6ca7b916a53f1fcdb23f6e6fdf4bdc51d7a26add939752dcc66c526d30ed4f26"}, {"id": "u028", "kind": "table-row", "locator": "body:L61-L61", "preview": "| File | Verified role | | [`format_conversion.py`](../../artifacts/prs/flashinfer/PR-1025/key-files/flashinfer/triton/format_conversion.py) | Triton FP8/FP16 format-conversion kernels. |", "sha256": "e57454c7314e09c20b08b2d5cd27ae1786974d1c9b2daddb61c1997228957b94"}, {"id": "u029", "kind": "table-row", "locator": "body:L62-L62", "preview": "| File | Verified role | | [`norm.py`](../../artifacts/prs/sglang/PR-20910/key-files/python/sglang/jit_kernel/norm.py) | Triton normalization kernels. |", "sha256": "65fd3a5f292128dc8cab99086d97e11181d0e2fc254c13490ac0543dd66ce342"}, {"id": "u030", "kind": "table-row", "locator": "body:L63-L63", "preview": "| File | Verified role | | [`gdn_fused_proj.py`](../../artifacts/prs/sglang/PR-21019/key-files/python/sglang/jit_kernel/triton/gdn_fused_proj.py) | Triton GatedDeltaNet projection rearrangement using loads/stores. |", "sha256": "f663a530904bc8709ee5d576f14ec0bc29e6a806bbfd85f94aa5274d519b0bc8"}, {"id": "u031", "kind": "table-row", "locator": "body:L64-L64", "preview": "| File | Verified role | | [`extend_attention.py`](../../artifacts/prs/sglang/PR-22079/key-files/python/sglang/srt/layers/attention/triton_ops/extend_attention.py) | Triton extend-attention kernels containing `tl.dot`. |", "sha256": "d03db60ea2e11a9393db7a9ccf09c4eb67f51e4fe93b2ddca10d598d0444d892"}, {"id": "u032", "kind": "prose", "locator": "body:L66-L66", "preview": "The complete tracked PR universe and its captured/skipped flags are recorded in [`data/triton-universe.yaml`](../../data/triton-universe.yaml); this page does not duplicate that changing count.", "sha256": "b82bbb549f1e4dfa484c36d1134005bb13fd50ddc1a6db308b67a2dab96b78c1"}, {"id": "u033", "kind": "prose", "locator": "body:L68-L68", "preview": "This verbatim excerpt from the pinned vLLM decode kernel demonstrates the page's deliberately limited downstream claim\u2014FP8 rescaling followed by `tl.dot`, without proving a particular emitted MMA instruction:", "sha256": "161635e13b202066602eecd585ccef97bc289ce7efd17d6eda51155b8cbce11b"}, {"id": "u034", "kind": "code", "locator": "body:L70-L76", "preview": "```python if k.dtype.is_fp8(): k = (k.to(tl.float32) * ks).to(q.dtype) qk = tl.dot(q, k.to(q.dtype)) if BLOCK_DPE > 0: offs_buf_kpe = kv_loc[None, :] * stride_buf_kbs ```", "sha256": "98ec2d0651858cbb3b5795bc15f9064e1167c1f0800a61b764ad5906cb6bcca9"}, {"id": "u035", "kind": "metadata-version", "locator": "frontmatter:version_sensitive", "preview": "{\"id\": \"vs-triton-3.3-blackwell-tcgen05\"}", "sha256": "2c0977b31b1a2e307ef3fca06c52165f985ad1366e7255b1e8c869baca5a2669"}], "confidence_claimed": "verified", "headings": ["Verified release boundary", "User-visible surfaces", "What downstream code establishes", "Scoped ecosystem results", "Launch overhead and CUDA Graphs", "Provenance-pinned examples"], "id": "lang-triton", "path": "wiki/languages/triton-blackwell.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/triton-3.3-blackwell.md", "url": "https://github.com/triton-lang/triton/compare/v3.2.0...v3.3.0"}, {"path": "sources/docs/triton-3.6-blackwell.md", "url": "https://github.com/triton-lang/triton/releases/tag/v3.6.0"}, {"path": "sources/prs/vllm/PR-34597.md", "revision": "a1257fd1", "url": "https://github.com/vllm-project/vllm/pull/34597"}, {"path": "sources/prs/vllm/PR-29339.md", "revision": "c17610e2", "url": "https://github.com/vllm-project/vllm/pull/29339"}, {"path": "sources/prs/sglang/PR-22079.md", "revision": "5638d40f", "url": "https://github.com/sgl-project/sglang/pull/22079"}, {"path": "sources/prs/sglang/PR-21019.md", "revision": "5bdc07d9", "url": "https://github.com/sgl-project/sglang/pull/21019"}, {"path": "sources/prs/sglang/PR-5390.md", "revision": "84810da4", "url": "https://github.com/sgl-project/sglang/pull/5390"}, {"path": "sources/prs/sglang/PR-21595.md", "revision": "87a27682", "url": "https://github.com/sgl-project/sglang/pull/21595"}], "risk_flags": ["code", "table", "version"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-triton-3.3-blackwell", "doc-triton-3.6-blackwell", "pr-vllm-34597", "pr-vllm-29339", "pr-sglang-22079", "pr-sglang-21019", "pr-sglang-5390", "pr-sglang-21595"], "title": "Triton on Blackwell", "type": "language", "unresolved_source_ids": [], "version_sensitive": {"id": "vs-triton-3.3-blackwell-tcgen05"}} -{"body_sha256": "7f0bb2f02e5a9f1f50c0c9003dc0baeeee3685876e3897726abbd12a14ae611a", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "For Hopper `wgmma.mma_async.m64nNk16` with an FP32 accumulator, each warpgroup thread holds `N/2` FP32 D registers. At `N=256`, that is 128 registers per thread for D. WGMMA updates that register vector asynchronously; the program uses the ", "sha256": "81c50814350c832517909572561c4f72593f90fdf9fd14235d07601b29cc502b"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "For `sm_100a`, tcgen05 D is addressed in Tensor Memory rather than supplied as a per-thread register vector. The CTA-visible TMEM structure has 128 lanes by 512 columns of 32-bit cells, or 256 KiB when fully allocated. Allocation is dynamic", "sha256": "90915abc172e7b8a5e9fdec98cdb99a4f905d1e52e66f7b4a939c546ed740ab2"}, {"id": "u003", "kind": "prose", "locator": "body:L9-L9", "preview": "This removes the resident D vector from the ordinary register file. It does not remove all register pressure: descriptors, addresses, loop state, pipeline state, and each TMEM-to-register epilogue batch still need registers. Both Hopper and", "sha256": "3065c8674899961c948b2920ce5998033450f02b782f496619e938425f12bb2c"}, {"id": "u004", "kind": "table-row", "locator": "body:L15-L15", "preview": "| Concern | Hopper WGMMA | Blackwell tcgen05 | | Resident D | Per-thread register fragment | TMEM region addressed by `taddr` |", "sha256": "caf62cc51664762ce0014371563bd85fb173f40058af73033877c114fb4e957d"}, {"id": "u005", "kind": "table-row", "locator": "body:L16-L16", "preview": "| Concern | Hopper WGMMA | Blackwell tcgen05 | | Example FP32 D cost | `N/2` registers/thread; 128 at `N=256` | No resident per-thread D vector |", "sha256": "6e4cf747f49122769816e81e7f7200b5e7c2dbef080de92600e88c94f0a6a67e"}, {"id": "u006", "kind": "table-row", "locator": "body:L17-L17", "preview": "| Concern | Hopper WGMMA | Blackwell tcgen05 | | First accumulation | Set WGMMA `scale-d` false to compute `D=A*B`, or initialize/use D when accumulation is intended | Set `enable-input-d` false to compute `D=A*B`, or explicitly initialize ", "sha256": "3eef8e2f3c1f793204f9ac25cd12a7ca2646443ef0aac8c4c12cf7b7d68a4482"}, {"id": "u007", "kind": "table-row", "locator": "body:L18-L18", "preview": "| Concern | Hopper WGMMA | Blackwell tcgen05 | | Compute completion | WGMMA commit/wait group | `tcgen05.commit` to an mbarrier, then wait before a consumer reads/reuses D |", "sha256": "0b73d94f0d547ca8fde7bfcbd32bb2051564e76bfacb9631eb78add6646bf88c"}, {"id": "u008", "kind": "table-row", "locator": "body:L19-L19", "preview": "| Concern | Hopper WGMMA | Blackwell tcgen05 | | Epilogue access | Use the thread's D registers | Collective `tcgen05.ld`, then `tcgen05.wait::ld` before consuming result registers |", "sha256": "61e6795055d9b77d891ca6222fd9697889c19e800e9403c14867010befb557d6"}, {"id": "u009", "kind": "table-row", "locator": "body:L20-L20", "preview": "| Concern | Hopper WGMMA | Blackwell tcgen05 | | Cleanup | Register lifetime ends with the thread | Matching collective `tcgen05.dealloc` before kernel exit |", "sha256": "735266abf367eab94461e883b716688a8976f3847bed7a112cba79534d11f5d7"}, {"id": "u010", "kind": "table-row", "locator": "body:L21-L21", "preview": "| Concern | Hopper WGMMA | Blackwell tcgen05 | | Double buffering | Two simultaneously live D fragments consume two register fragments | Two live outputs consume disjoint TMEM columns inside the allocation |", "sha256": "b04813198ede3f9af7088d4f1b95aa578b1195e6e41e388421499ab2487372ba"}, {"id": "u011", "kind": "prose", "locator": "body:L23-L23", "preview": "The table describes storage contracts, not an occupancy prediction. Instruction shape is also not necessarily the same as a composed CTA tile.", "sha256": "195c5b3cd276fffc69431065c21852ab5d89c54e26805dfbc82eeadd098432ab"}, {"id": "u012", "kind": "list-item", "locator": "body:L27-L27", "preview": "1. Choose one CTA-group mode for the kernel. For allocation, `nCols` is a power of two in `[32, 512]`; allocations are column-granular and cover all 128 lanes.", "sha256": "663ec5e2ecd0084580d22cb5e1a1fdb601f34042656316eb82971db2810e2220"}, {"id": "u013", "kind": "list-item", "locator": "body:L28-L28", "preview": "2. Have every lane of one designated warp execute the same `.cta_group::1` allocation. Synchronize before other threads read the 32-bit `taddr` written to shared memory. Two-CTA mode requires one warp in each live peer CTA.", "sha256": "442f844d571108d531ca6c4f1c2fcb34a128af006b51feb868efa7ca66f6945f"}, {"id": "u014", "kind": "list-item", "locator": "body:L29-L29", "preview": "3. Initialize and publish the barriers used by TMA and tcgen05. If the first MMA should compute only `A*B`, supply a false `enable-input-d` predicate instead of assuming a mandatory TMEM zero-store pass.", "sha256": "688cacfc78db32ef2dbc1b5748ab430829d8949fa6ec1e7f1737048cbcfa2858"}, {"id": "u015", "kind": "list-item", "locator": "body:L30-L30", "preview": "4. Issue MMA from the permitted elected thread with valid A/B descriptors, instruction descriptor, predicate, and lifetimes.", "sha256": "c81894a6edec76da9df885f000827bc66cd3820796598af4d484b7a69c4643e9"}, {"id": "u016", "kind": "list-item", "locator": "body:L31-L31", "preview": "5. Attach completion of the relevant tcgen05 work to an mbarrier with `tcgen05.commit`. A fence controls execution ordering but is not a completion wait.", "sha256": "3f66638c0aa59578daef93b5cd3375015890606f7dbaa0f9f2b07f9e33caa19c"}, {"id": "u017", "kind": "list-item", "locator": "body:L32-L32", "preview": "6. After the completion handoff, participating epilogue lanes execute a legal `tcgen05.ld` shape and `tcgen05.wait::ld` before using the loaded registers.", "sha256": "add86c6758160d1ee6436e9bbb4a42118fa75ad68987d8da15053e12fcddcc68"}, {"id": "u018", "kind": "list-item", "locator": "body:L33-L33", "preview": "7. After all readers finish, every lane of the designated warp executes the matching deallocation on every kernel exit path. Allocation/deallocation address, column count, and CTA-group mode must agree.", "sha256": "f07d98a96b90ae28744b10a65fee6446286122ccfd1e24b3f6df56759d8d3b9a"}, {"id": "u019", "kind": "prose", "locator": "body:L35-L35", "preview": "Representative PTX ISA 9.0 forms are shown below. They omit declarations, descriptor construction, collective control flow, barriers, and inline-assembly constraints, so they are not a standalone kernel.", "sha256": "ba8a0bbd80024f2e78f378a5524beb0317003225da5d484380df0020489d90f3"}, {"id": "u020", "kind": "code", "locator": "body:L37-L44", "preview": "```ptx tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [saddr], nCols; tcgen05.mma.cta_group::1.kind::f16 [taddr], a_desc, b_desc, idesc, p; tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cta.b64 [mbar]; tcgen05.ld.sync.a", "sha256": "a5d9f9e0f8f9643e18992569407c06dd52f1583195162b4b41c9bdfa172c67d7"}, {"id": "u021", "kind": "prose", "locator": "body:L48-L48", "preview": "TMEM makes it practical to keep multiple output regions independently addressable while different warp roles issue MMA and drain a completed region. Safe overlap needs three distinct proofs:", "sha256": "472061f2ee2a0acce107d45fd26f201f4bd143c15aeb1efeaef163973f894aa0"}, {"id": "u022", "kind": "list-item", "locator": "body:L50-L50", "preview": "- compute completion occurs before an epilogue loads a region;", "sha256": "797834acb4631fde043533de9ef5d1938c604993e27069015705834e832921ab"}, {"id": "u023", "kind": "list-item", "locator": "body:L51-L51", "preview": "- all epilogue loads complete before that region is overwritten or deallocated;", "sha256": "f76c38cde4d90d1036b53206db7b3fa9eb7c698a8ec70a94f914970e27393b41"}, {"id": "u024", "kind": "list-item", "locator": "body:L52-L52", "preview": "- producer/consumer state and phase cannot alias a different pipeline stage.", "sha256": "53171494a69370538165b9a5233bde906d81d1a6117a916676ef9f91f33fa533"}, {"id": "u025", "kind": "prose", "locator": "body:L54-L54", "preview": "TMEM is not the only way to overlap tensor-core and non-matmul work. FlashAttention-3 already uses warp specialization and matmul/softmax interleaving on Hopper with register accumulators. The migration benefit is the different storage and ", "sha256": "c0d0df2e7890cb2071ca656369ce0ebacde9548a7e8d52684018dc1f460f15bf"}, {"id": "u026", "kind": "prose", "locator": "body:L58-L58", "preview": "FlashAttention-4's first-party paper places score/output accumulators in TMEM and assigns MMA, softmax, and correction work to specialized warp roles. The accompanying author post describes software-selected polynomial `exp2` work on CUDA c", "sha256": "3b5cea1d7c442196306c53f5222f1b29458f1a4a7f94880145e94721679140e9"}, {"id": "u027", "kind": "prose", "locator": "body:L62-L62", "preview": "Do not translate an instruction shape directly into a required CTA tile or assume that moving D guarantees no spills. For each concrete kernel:", "sha256": "458b810cc30dfa258823ec0f43d812cd1b0d189139c427c79b07cf5bcf6d2d8b"}, {"id": "u028", "kind": "list-item", "locator": "body:L64-L64", "preview": "1. Record the compiler's registers/thread, spill loads/stores, static/dynamic shared memory, barriers, threads/CTA, and cluster shape.", "sha256": "d16c6acd3215b8f25c44c532f8c911211631184ea4f19611507b0865dcbf5215"}, {"id": "u029", "kind": "list-item", "locator": "body:L65-L65", "preview": "2. Compute occupancy with the CUDA occupancy APIs for the actual launch; do not equate one warpgroup with one CTA.", "sha256": "61f8007df8291d17b10b5d98fa24b9e5270e9671577f6d2185f80148fcd9a1d0"}, {"id": "u030", "kind": "list-item", "locator": "body:L66-L66", "preview": "3. Sweep legal instruction descriptors, CTA tiles, pipeline depth, TMEM column partitioning, and epilogue batch width.", "sha256": "c38ba2f0bc7de4c81597e253f093a2e181a747fa3c44afa31f9fc7ca8fe9edcc"}, {"id": "u031", "kind": "list-item", "locator": "body:L67-L67", "preview": "4. Benchmark identical shapes, datatypes, outputs, synchronization, warmup, and trial statistics. A fixed latency such as \u201c420 cycles per TMEM load\u201d is not an ISA guarantee.", "sha256": "254ff0f6b010a5a5da0857102c7f77d31d8d19ebeb3a17d4d96f5fb73b3da8e2"}, {"id": "u032", "kind": "list-item", "locator": "body:L68-L68", "preview": "5. Inspect generated PTX/SASS to confirm that the intended tcgen05 shapes, waits, and no unexpected spills are present.", "sha256": "a0dc065f5f6cd024c8097f8d5c531813014b8bcd3af6638ce958f8f15187547f"}, {"id": "u033", "kind": "list-item", "locator": "body:L72-L72", "preview": "- TMEM is not an ordinary CUDA pointer space. Load through a legal collective tcgen05 copy operation before scalar arithmetic.", "sha256": "5a0dc36e41360a05b9484ba2c76a8a40f4c50cd91d824efc8390bd263d26569e"}, {"id": "u034", "kind": "list-item", "locator": "body:L73-L73", "preview": "- Do not issue collective allocation, load/store, or deallocation from lane 0 alone.", "sha256": "7a08a8655056273dbd66d60daed365c7599fec8e52caf56e1ba389cdb33f2a41"}, {"id": "u035", "kind": "list-item", "locator": "body:L74-L74", "preview": "- Do not substitute `tcgen05.fence` or `__syncthreads()` for asynchronous MMA completion.", "sha256": "39b83c942611535ed32d220db39984aaaa353937f00d2407e2638f6864583fa9"}, {"id": "u036", "kind": "list-item", "locator": "body:L75-L75", "preview": "- Do not zero TMEM unconditionally when a false `enable-input-d` predicate provides the intended first-write semantics.", "sha256": "b99c99b2e0c4cd1c80b2fe08796a0916cc6f0b10aa816e84fc12cc3a76507733"}, {"id": "u037", "kind": "list-item", "locator": "body:L76-L76", "preview": "- Do not treat TMEM-resident D as the entire kernel's register allocation; the epilogue can still spill.", "sha256": "34b5e7d2ea4620b8b6eac38d7fd6247040e0caa5401bfee293902cc71ebd84c9"}, {"id": "u038", "kind": "list-item", "locator": "body:L77-L77", "preview": "- Do not describe missing deallocation as a defined recoverable \u201cleak.\u201d The normative rule is explicit matching deallocation before exit.", "sha256": "7459f5d2fab8c6fe59eeef14ce208773a6ea79f29893b81890e9546f5334030d"}, {"id": "u039", "kind": "list-item", "locator": "body:L81-L81", "preview": "- [PTX ISA 9.0 WGMMA register fragments](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16)", "sha256": "56a14ee2db1f1b79e0e525648d0d966aaa69571dfe36c87de7168f26881ef269"}, {"id": "u040", "kind": "list-item", "locator": "body:L82-L82", "preview": "- [PTX ISA 9.0 Tensor Memory](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory)", "sha256": "aed47f18c514497e50694f89415704f98533229e52ed733c1898d3c9bb3e2747"}, {"id": "u041", "kind": "list-item", "locator": "body:L83-L83", "preview": "- [PTX ISA 9.0 tcgen05 allocation](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc)", "sha256": "31973f025402fdb2aee03f37ec947608275b8e131e5d9423444bc588c91662b7"}, {"id": "u042", "kind": "list-item", "locator": "body:L84-L84", "preview": "- [PTX ISA 9.0 tcgen05 MMA](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma)", "sha256": "4c1c4ca67353c9e04e383f31694978300286983ac6d9415be5a57439e730d050"}, {"id": "u043", "kind": "list-item", "locator": "body:L85-L85", "preview": "- [CUDA 13.0.2 Hopper Tuning Guide](https://docs.nvidia.com/cuda/archive/13.0.2/hopper-tuning-guide/index.html#occupancy)", "sha256": "acaee20e11d45fd20a8d0c6e29f05aec3d89d17a203eae0a47e821c7f1c2946c"}, {"id": "u044", "kind": "list-item", "locator": "body:L86-L86", "preview": "- [CUDA 13.0.2 Blackwell Tuning Guide](https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy)", "sha256": "8d963db7f76eb6508d3be3b8853709f57f9a8b2f98e623d121cff16c31de6585"}, {"id": "u045", "kind": "list-item", "locator": "body:L87-L87", "preview": "- [FlashAttention-3 paper](https://arxiv.org/abs/2407.08608)", "sha256": "5a5ef1b6d90749138db83d063d75df914e64702b3d88af097f7ab1b7918bd60c"}, {"id": "u046", "kind": "list-item", "locator": "body:L88-L88", "preview": "- [FlashAttention-4 paper v1](https://arxiv.org/abs/2603.05451v1)", "sha256": "0fb60e121910c6e3db40016b49479cf32b63868bf4c07d1765c52f09c5adad70"}, {"id": "u047", "kind": "list-item", "locator": "body:L92-L92", "preview": "- [Tensor Memory](../hardware/tmem.md) \u2014 allocation, addressing, and access constraints", "sha256": "b3c60111974bb86dc60da7215cbb37b18f5ed884f7afc8f2ba9e888704ee4c81"}, {"id": "u048", "kind": "list-item", "locator": "body:L93-L93", "preview": "- [tcgen05 MMA](../hardware/tcgen05-mma.md) \u2014 exact operation and completion semantics", "sha256": "8eaf7f0f1fcdebbab2fb0edc10f2597a39ea62bb5d5350106b42fccbf9e6260c"}, {"id": "u049", "kind": "list-item", "locator": "body:L94-L94", "preview": "- [register pressure](../patterns/register-pressure.md) \u2014 diagnosis and compiler-resource evidence", "sha256": "63798ee2070c890c2bee614823e025d90ee4f0841197c7da548cb5327b8c297d"}], "confidence_claimed": "verified", "headings": ["What actually changes", "Storage and lifecycle contrast", "Migration lifecycle", "Overlap without invented guarantees", "FlashAttention-4 case study", "Evidence-driven retuning", "Common migration errors", "Primary references", "Related"], "id": "migration-register-to-tmem", "path": "wiki/migration/register-to-tmem.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/prs/vllm/PR-22738.md", "revision": "b1361c72", "url": "https://github.com/vllm-project/vllm/pull/22738"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "pr-vllm-22738"], "title": "Register Accumulators to TMEM", "type": "migration", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "1780b4d4aa8b9c8bc2757dd6d8601d0ed39055ea4e641003be7d16db8b598931", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "For Hopper `wgmma.mma_async.m64nNk16` with an FP32 accumulator, each warpgroup thread holds `N/2` FP32 D registers. At `N=256`, that is 128 registers per thread for D. WGMMA updates that register vector asynchronously; the program uses the ", "sha256": "81c50814350c832517909572561c4f72593f90fdf9fd14235d07601b29cc502b"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "For `sm_100a`, tcgen05 D is addressed in Tensor Memory rather than supplied as a per-thread register vector. The CTA-visible TMEM structure has 128 lanes by 512 columns of 32-bit cells, or 256 KiB when fully allocated. Allocation is dynamic", "sha256": "90915abc172e7b8a5e9fdec98cdb99a4f905d1e52e66f7b4a939c546ed740ab2"}, {"id": "u003", "kind": "prose", "locator": "body:L9-L9", "preview": "This removes the resident D vector from the ordinary register file. It does not remove all register pressure: descriptors, addresses, loop state, pipeline state, and each TMEM-to-register epilogue batch still need registers. Both Hopper and", "sha256": "3065c8674899961c948b2920ce5998033450f02b782f496619e938425f12bb2c"}, {"id": "u004", "kind": "table-row", "locator": "body:L15-L15", "preview": "| Concern | Hopper WGMMA | Blackwell tcgen05 | | Resident D | Per-thread register fragment | TMEM region addressed by `taddr` |", "sha256": "caf62cc51664762ce0014371563bd85fb173f40058af73033877c114fb4e957d"}, {"id": "u005", "kind": "table-row", "locator": "body:L16-L16", "preview": "| Concern | Hopper WGMMA | Blackwell tcgen05 | | Example FP32 D cost | `N/2` registers/thread; 128 at `N=256` | No resident per-thread D vector |", "sha256": "6e4cf747f49122769816e81e7f7200b5e7c2dbef080de92600e88c94f0a6a67e"}, {"id": "u006", "kind": "table-row", "locator": "body:L17-L17", "preview": "| Concern | Hopper WGMMA | Blackwell tcgen05 | | First accumulation | Set WGMMA `scale-d` false to compute `D=A*B`, or initialize/use D when accumulation is intended | Set `enable-input-d` false to compute `D=A*B`, or explicitly initialize ", "sha256": "3eef8e2f3c1f793204f9ac25cd12a7ca2646443ef0aac8c4c12cf7b7d68a4482"}, {"id": "u007", "kind": "table-row", "locator": "body:L18-L18", "preview": "| Concern | Hopper WGMMA | Blackwell tcgen05 | | Compute completion | WGMMA commit/wait group | `tcgen05.commit` to an mbarrier, then wait before a consumer reads/reuses D |", "sha256": "0b73d94f0d547ca8fde7bfcbd32bb2051564e76bfacb9631eb78add6646bf88c"}, {"id": "u008", "kind": "table-row", "locator": "body:L19-L19", "preview": "| Concern | Hopper WGMMA | Blackwell tcgen05 | | Epilogue access | Use the thread's D registers | Collective `tcgen05.ld`, then `tcgen05.wait::ld` before consuming result registers |", "sha256": "61e6795055d9b77d891ca6222fd9697889c19e800e9403c14867010befb557d6"}, {"id": "u009", "kind": "table-row", "locator": "body:L20-L20", "preview": "| Concern | Hopper WGMMA | Blackwell tcgen05 | | Cleanup | Register lifetime ends with the thread | Matching collective `tcgen05.dealloc` before kernel exit |", "sha256": "735266abf367eab94461e883b716688a8976f3847bed7a112cba79534d11f5d7"}, {"id": "u010", "kind": "table-row", "locator": "body:L21-L21", "preview": "| Concern | Hopper WGMMA | Blackwell tcgen05 | | Double buffering | Two simultaneously live D fragments consume two register fragments | Two live outputs consume disjoint TMEM columns inside the allocation |", "sha256": "b04813198ede3f9af7088d4f1b95aa578b1195e6e41e388421499ab2487372ba"}, {"id": "u011", "kind": "prose", "locator": "body:L23-L23", "preview": "The table describes storage contracts, not an occupancy prediction. Instruction shape is also not necessarily the same as a composed CTA tile.", "sha256": "195c5b3cd276fffc69431065c21852ab5d89c54e26805dfbc82eeadd098432ab"}, {"id": "u012", "kind": "list-item", "locator": "body:L27-L27", "preview": "1. Choose one CTA-group mode for the kernel. For allocation, `nCols` is a power of two in `[32, 512]`; allocations are column-granular and cover all 128 lanes.", "sha256": "663ec5e2ecd0084580d22cb5e1a1fdb601f34042656316eb82971db2810e2220"}, {"id": "u013", "kind": "list-item", "locator": "body:L28-L28", "preview": "2. Have every lane of one designated warp execute the same `.cta_group::1` allocation. Synchronize before other threads read the 32-bit `taddr` written to shared memory. Two-CTA mode requires one warp in each live peer CTA.", "sha256": "442f844d571108d531ca6c4f1c2fcb34a128af006b51feb868efa7ca66f6945f"}, {"id": "u014", "kind": "list-item", "locator": "body:L29-L29", "preview": "3. Initialize and publish the barriers used by TMA and tcgen05. If the first MMA should compute only `A*B`, supply a false `enable-input-d` predicate instead of assuming a mandatory TMEM zero-store pass.", "sha256": "688cacfc78db32ef2dbc1b5748ab430829d8949fa6ec1e7f1737048cbcfa2858"}, {"id": "u015", "kind": "list-item", "locator": "body:L30-L30", "preview": "4. Issue MMA from the permitted elected thread with valid A/B descriptors, instruction descriptor, predicate, and lifetimes.", "sha256": "c81894a6edec76da9df885f000827bc66cd3820796598af4d484b7a69c4643e9"}, {"id": "u016", "kind": "list-item", "locator": "body:L31-L31", "preview": "5. Attach completion of the relevant tcgen05 work to an mbarrier with `tcgen05.commit`. A fence controls execution ordering but is not a completion wait.", "sha256": "3f66638c0aa59578daef93b5cd3375015890606f7dbaa0f9f2b07f9e33caa19c"}, {"id": "u017", "kind": "list-item", "locator": "body:L32-L32", "preview": "6. After the completion handoff, participating epilogue lanes execute a legal `tcgen05.ld` shape and `tcgen05.wait::ld` before using the loaded registers.", "sha256": "add86c6758160d1ee6436e9bbb4a42118fa75ad68987d8da15053e12fcddcc68"}, {"id": "u018", "kind": "list-item", "locator": "body:L33-L33", "preview": "7. After all readers finish, every lane of the designated warp executes the matching deallocation on every kernel exit path. Allocation/deallocation address, column count, and CTA-group mode must agree.", "sha256": "f07d98a96b90ae28744b10a65fee6446286122ccfd1e24b3f6df56759d8d3b9a"}, {"id": "u019", "kind": "prose", "locator": "body:L35-L35", "preview": "Representative PTX ISA 9.0 forms are shown below. They omit declarations, descriptor construction, collective control flow, barriers, and inline-assembly constraints, so they are not a standalone kernel.", "sha256": "ba8a0bbd80024f2e78f378a5524beb0317003225da5d484380df0020489d90f3"}, {"id": "u020", "kind": "code", "locator": "body:L37-L44", "preview": "```ptx tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [saddr], nCols; tcgen05.mma.cta_group::1.kind::f16 [taddr], a_desc, b_desc, idesc, p; tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cta.b64 [mbar]; tcgen05.ld.sync.a", "sha256": "a5d9f9e0f8f9643e18992569407c06dd52f1583195162b4b41c9bdfa172c67d7"}, {"id": "u021", "kind": "prose", "locator": "body:L48-L48", "preview": "TMEM makes it practical to keep multiple output regions independently addressable while different warp roles issue MMA and drain a completed region. Safe overlap needs three distinct proofs:", "sha256": "472061f2ee2a0acce107d45fd26f201f4bd143c15aeb1efeaef163973f894aa0"}, {"id": "u022", "kind": "list-item", "locator": "body:L50-L50", "preview": "- compute completion occurs before an epilogue loads a region;", "sha256": "797834acb4631fde043533de9ef5d1938c604993e27069015705834e832921ab"}, {"id": "u023", "kind": "list-item", "locator": "body:L51-L51", "preview": "- all epilogue loads complete before that region is overwritten or deallocated;", "sha256": "f76c38cde4d90d1036b53206db7b3fa9eb7c698a8ec70a94f914970e27393b41"}, {"id": "u024", "kind": "list-item", "locator": "body:L52-L52", "preview": "- producer/consumer state and phase cannot alias a different pipeline stage.", "sha256": "53171494a69370538165b9a5233bde906d81d1a6117a916676ef9f91f33fa533"}, {"id": "u025", "kind": "prose", "locator": "body:L54-L54", "preview": "TMEM is not the only way to overlap tensor-core and non-matmul work. FlashAttention-3 already uses warp specialization and matmul/softmax interleaving on Hopper with register accumulators. The migration benefit is the different storage and ", "sha256": "c0d0df2e7890cb2071ca656369ce0ebacde9548a7e8d52684018dc1f460f15bf"}, {"id": "u026", "kind": "prose", "locator": "body:L58-L58", "preview": "FlashAttention-4's first-party paper places score/output accumulators in TMEM and assigns MMA, softmax, and correction work to specialized warp roles. The accompanying author post describes software-selected polynomial `exp2` work on CUDA c", "sha256": "3b5cea1d7c442196306c53f5222f1b29458f1a4a7f94880145e94721679140e9"}, {"id": "u027", "kind": "prose", "locator": "body:L62-L62", "preview": "Do not translate an instruction shape directly into a required CTA tile or assume that moving D guarantees no spills. For each concrete kernel:", "sha256": "458b810cc30dfa258823ec0f43d812cd1b0d189139c427c79b07cf5bcf6d2d8b"}, {"id": "u028", "kind": "list-item", "locator": "body:L64-L64", "preview": "1. Record the compiler's registers/thread, spill loads/stores, static/dynamic shared memory, barriers, threads/CTA, and cluster shape.", "sha256": "d16c6acd3215b8f25c44c532f8c911211631184ea4f19611507b0865dcbf5215"}, {"id": "u029", "kind": "list-item", "locator": "body:L65-L65", "preview": "2. Compute occupancy with the CUDA occupancy APIs for the actual launch; do not equate one warpgroup with one CTA.", "sha256": "61f8007df8291d17b10b5d98fa24b9e5270e9671577f6d2185f80148fcd9a1d0"}, {"id": "u030", "kind": "list-item", "locator": "body:L66-L66", "preview": "3. Sweep legal instruction descriptors, CTA tiles, pipeline depth, TMEM column partitioning, and epilogue batch width.", "sha256": "c38ba2f0bc7de4c81597e253f093a2e181a747fa3c44afa31f9fc7ca8fe9edcc"}, {"id": "u031", "kind": "list-item", "locator": "body:L67-L67", "preview": "4. Benchmark identical shapes, datatypes, outputs, synchronization, warmup, and trial statistics. A fixed latency such as \u201c420 cycles per TMEM load\u201d is not an ISA guarantee.", "sha256": "254ff0f6b010a5a5da0857102c7f77d31d8d19ebeb3a17d4d96f5fb73b3da8e2"}, {"id": "u032", "kind": "list-item", "locator": "body:L68-L68", "preview": "5. Inspect generated PTX/SASS to confirm that the intended tcgen05 shapes, waits, and no unexpected spills are present.", "sha256": "a0dc065f5f6cd024c8097f8d5c531813014b8bcd3af6638ce958f8f15187547f"}, {"id": "u033", "kind": "list-item", "locator": "body:L72-L72", "preview": "- TMEM is not an ordinary CUDA pointer space. Load through a legal collective tcgen05 copy operation before scalar arithmetic.", "sha256": "5a0dc36e41360a05b9484ba2c76a8a40f4c50cd91d824efc8390bd263d26569e"}, {"id": "u034", "kind": "list-item", "locator": "body:L73-L73", "preview": "- Do not issue collective allocation, load/store, or deallocation from lane 0 alone.", "sha256": "7a08a8655056273dbd66d60daed365c7599fec8e52caf56e1ba389cdb33f2a41"}, {"id": "u035", "kind": "list-item", "locator": "body:L74-L74", "preview": "- Do not substitute `tcgen05.fence` or `__syncthreads()` for asynchronous MMA completion.", "sha256": "39b83c942611535ed32d220db39984aaaa353937f00d2407e2638f6864583fa9"}, {"id": "u036", "kind": "list-item", "locator": "body:L75-L75", "preview": "- Do not zero TMEM unconditionally when a false `enable-input-d` predicate provides the intended first-write semantics.", "sha256": "b99c99b2e0c4cd1c80b2fe08796a0916cc6f0b10aa816e84fc12cc3a76507733"}, {"id": "u037", "kind": "list-item", "locator": "body:L76-L76", "preview": "- Do not treat TMEM-resident D as the entire kernel's register allocation; the epilogue can still spill.", "sha256": "34b5e7d2ea4620b8b6eac38d7fd6247040e0caa5401bfee293902cc71ebd84c9"}, {"id": "u038", "kind": "list-item", "locator": "body:L77-L77", "preview": "- Do not describe missing deallocation as a defined recoverable \u201cleak.\u201d The normative rule is explicit matching deallocation before exit.", "sha256": "7459f5d2fab8c6fe59eeef14ce208773a6ea79f29893b81890e9546f5334030d"}, {"id": "u039", "kind": "list-item", "locator": "body:L81-L81", "preview": "- [PTX ISA 9.0 WGMMA register fragments](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-register-fragment-wgmma-64n16)", "sha256": "56a14ee2db1f1b79e0e525648d0d966aaa69571dfe36c87de7168f26881ef269"}, {"id": "u040", "kind": "list-item", "locator": "body:L82-L82", "preview": "- [PTX ISA 9.0 Tensor Memory](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory)", "sha256": "aed47f18c514497e50694f89415704f98533229e52ed733c1898d3c9bb3e2747"}, {"id": "u041", "kind": "list-item", "locator": "body:L83-L83", "preview": "- [PTX ISA 9.0 tcgen05 allocation](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-alloc-dealloc-relinquish-alloc-permit)", "sha256": "d42d0c01cb062292369af97827421a3a287ec970b4bc869b80edaefb263b177c"}, {"id": "u042", "kind": "list-item", "locator": "body:L84-L84", "preview": "- [PTX ISA 9.0 tcgen05 MMA](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma)", "sha256": "732ea76b4a51fe3c2ffcae7ea22d70813776ee650170d529574101600b2a1686"}, {"id": "u043", "kind": "list-item", "locator": "body:L85-L85", "preview": "- [CUDA 13.0.2 Hopper Tuning Guide](https://docs.nvidia.com/cuda/archive/13.0.2/hopper-tuning-guide/index.html#occupancy)", "sha256": "acaee20e11d45fd20a8d0c6e29f05aec3d89d17a203eae0a47e821c7f1c2946c"}, {"id": "u044", "kind": "list-item", "locator": "body:L86-L86", "preview": "- [CUDA 13.0.2 Blackwell Tuning Guide](https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy)", "sha256": "8d963db7f76eb6508d3be3b8853709f57f9a8b2f98e623d121cff16c31de6585"}, {"id": "u045", "kind": "list-item", "locator": "body:L87-L87", "preview": "- [FlashAttention-3 paper](https://arxiv.org/abs/2407.08608)", "sha256": "5a5ef1b6d90749138db83d063d75df914e64702b3d88af097f7ab1b7918bd60c"}, {"id": "u046", "kind": "list-item", "locator": "body:L88-L88", "preview": "- [FlashAttention-4 paper v1](https://arxiv.org/abs/2603.05451v1)", "sha256": "0fb60e121910c6e3db40016b49479cf32b63868bf4c07d1765c52f09c5adad70"}, {"id": "u047", "kind": "list-item", "locator": "body:L92-L92", "preview": "- [Tensor Memory](../hardware/tmem.md) \u2014 allocation, addressing, and access constraints", "sha256": "b3c60111974bb86dc60da7215cbb37b18f5ed884f7afc8f2ba9e888704ee4c81"}, {"id": "u048", "kind": "list-item", "locator": "body:L93-L93", "preview": "- [tcgen05 MMA](../hardware/tcgen05-mma.md) \u2014 exact operation and completion semantics", "sha256": "8eaf7f0f1fcdebbab2fb0edc10f2597a39ea62bb5d5350106b42fccbf9e6260c"}, {"id": "u049", "kind": "list-item", "locator": "body:L94-L94", "preview": "- [register pressure](../patterns/register-pressure.md) \u2014 diagnosis and compiler-resource evidence", "sha256": "63798ee2070c890c2bee614823e025d90ee4f0841197c7da548cb5327b8c297d"}], "confidence_claimed": "verified", "headings": ["What actually changes", "Storage and lifecycle contrast", "Migration lifecycle", "Overlap without invented guarantees", "FlashAttention-4 case study", "Evidence-driven retuning", "Common migration errors", "Primary references", "Related"], "id": "migration-register-to-tmem", "path": "wiki/migration/register-to-tmem.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/prs/vllm/PR-22738.md", "revision": "b1361c72", "url": "https://github.com/vllm-project/vllm/pull/22738"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "pr-vllm-22738"], "title": "Register Accumulators to TMEM", "type": "migration", "unresolved_source_ids": [], "version_sensitive": null} {"body_sha256": "138004faa6219749f4d0e9072b9bca0649a4a7ad20e9616caa52e18ea33b5b17", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Port the programming model, not just the opcode:", "sha256": "9be3861214d8d1c37479c20bd21f2a3e7c33c7460656c78dc9992c6bbbe1999a"}, {"id": "u002", "kind": "table-row", "locator": "body:L9-L9", "preview": "| Concern | SM90 WGMMA | SM100 tcgen05.mma | | MMA issue | Warpgroup collective | One thread for group-1 or group-2 MMA |", "sha256": "aa5e3304df841c5d5e3ac5c52d103b8fcf0d8e0bb2a39c54a026f790dd7a4b9a"}, {"id": "u003", "kind": "table-row", "locator": "body:L10-L10", "preview": "| Concern | SM90 WGMMA | SM100 tcgen05.mma | | D accumulator | Per-thread registers | TMEM |", "sha256": "599eed1c6192c3cdba2fd7c04288ba5f919d0307d513ea12d52ed5533800a1f3"}, {"id": "u004", "kind": "table-row", "locator": "body:L11-L11", "preview": "| Concern | SM90 WGMMA | SM100 tcgen05.mma | | A operand | SMEM-descriptor or register forms, depending on instruction | SMEM-descriptor or TMEM-address forms |", "sha256": "c8e2f6685d308cb41e7788d727dab8bf0129426bad70f0138f73a6c64cb636c6"}, {"id": "u005", "kind": "table-row", "locator": "body:L12-L12", "preview": "| Concern | SM90 WGMMA | SM100 tcgen05.mma | | B operand | SMEM descriptor | SMEM descriptor |", "sha256": "0966cfea21760a831d495ee1346a258dd33095cff8f2e23e5d7148f1280f1c8d"}, {"id": "u006", "kind": "table-row", "locator": "body:L13-L13", "preview": "| Concern | SM90 WGMMA | SM100 tcgen05.mma | | MMA completion | Commit/wait groups | `tcgen05.commit` plus mbarrier wait |", "sha256": "129b864fde6ae8e8e58264f223c3ab52804dcb50c282146b2a153673afe572eb"}, {"id": "u007", "kind": "table-row", "locator": "body:L14-L14", "preview": "| Concern | SM90 WGMMA | SM100 tcgen05.mma | | Cross-thread ordering | WGMMA-specific rules | tcgen05 fences composed with an execution-ordering operation |", "sha256": "2ba345a4115e7a6e1f81acbd1416a0f89dc2ee2033027406b83461798bb45007"}, {"id": "u008", "kind": "table-row", "locator": "body:L15-L15", "preview": "| Concern | SM90 WGMMA | SM100 tcgen05.mma | | Narrow floating formats | FP8 WGMMA forms | FP8/FP6/FP4 plus block-scaled MX/NVFP4 kinds |", "sha256": "5947c0ca6e715328562acf6c43de21c60e196b9155df29bf170d6e250ea8b9e2"}, {"id": "u009", "kind": "prose", "locator": "body:L17-L17", "preview": "The exact old WGMMA form matters. Do not assume every Hopper kernel loads A through `ldmatrix`: descriptor-sourced WGMMA forms already read A from SMEM. Likewise, tcgen05 can source A from SMEM or TMEM.", "sha256": "382cb9dc63ddcd3017855fe5603348fd65995c563c146bef2da1122f8ddd6434"}, {"id": "u010", "kind": "list-item", "locator": "body:L21-L21", "preview": "1. Identify the exact WGMMA operand form, accumulator type, shape, and group-completion points in the SM90 kernel.", "sha256": "5b2fcba32ae9b677b57ca8e62cea7f0b536a60f4597ad9ed56f5c6109b4719b6"}, {"id": "u011", "kind": "list-item", "locator": "body:L22-L22", "preview": "2. Choose a legal tcgen05 kind, A source, CTA group, instruction descriptor, and data-path layout for the SM100 target.", "sha256": "7c480de2cd0b7a0f51ab66f750a7f2e43174c52b028129e8b203320459111c42"}, {"id": "u012", "kind": "list-item", "locator": "body:L23-L23", "preview": "3. Replace register-resident D with a TMEM allocation. Group-1 allocation/deallocation is warp-collective; group 2 requires one warp in each CTA of the pair.", "sha256": "94550e33d2e5fc9492d28f341ecdc5ca0655fb2f1c88303fdb7165407f78fcd9"}, {"id": "u013", "kind": "list-item", "locator": "body:L24-L24", "preview": "4. Keep A/B backing storage unchanged until all asynchronous MMA consumers have completed.", "sha256": "e1465bb385f2612cdc483569cb82b1fcee5fb9aaebcb9eeb65667686e7e33848"}, {"id": "u014", "kind": "list-item", "locator": "body:L25-L25", "preview": "5. Replace WGMMA group completion with `tcgen05.commit` and an mbarrier wait. Add tcgen05 fences only where an execution-ordering handoff must order tcgen05-visible state across threads or CTAs.", "sha256": "4362a63d2b1bd22a3096480f6c9a35d00538f93e94821b33c705086971b544ad"}, {"id": "u015", "kind": "list-item", "locator": "body:L26-L26", "preview": "6. Transfer completed accumulator values from TMEM to registers with `tcgen05.ld`, observe its completion/access rules, then run the epilogue.", "sha256": "cc2fdea219e29be1d20d93e0ba014159d0e34b2c8e0de0d4b01083b03c8a177d"}, {"id": "u016", "kind": "list-item", "locator": "body:L27-L27", "preview": "7. Deallocate every dynamic TMEM allocation before kernel exit. Use the same `cta_group` value for all tcgen05 instructions in the kernel.", "sha256": "93a1c4bb169b8c6c02b3b85f458290d7346ff8af1ea4b70ae3f779db14d83181"}, {"id": "u017", "kind": "list-item", "locator": "body:L28-L28", "preview": "8. Retune CTA/cluster shapes, pipeline stages, thread roles, descriptors, and epilogue scheduling on the target workload.", "sha256": "7f11e9b39fa11e8d8ebfaa1313cf7dd006346a8458830b0170a888d8b75eb94d"}, {"id": "u018", "kind": "prose", "locator": "body:L32-L32", "preview": "`tcgen05.alloc` writes a 32-bit TMEM address to shared memory. Allocation size is expressed in columns, in power-of-two multiples permitted by PTX. It is a synchronous warp instruction: a lane-0-only call is invalid. `tcgen05.dealloc` has t", "sha256": "74c19585a0e1b1b0242b4e24476aa7c1c33cc83163385246b670415ca03b4448"}, {"id": "u019", "kind": "prose", "locator": "body:L34-L34", "preview": "The MMA issuer and the allocation participants are different concepts. One thread initiates a group-1 MMA, but one full warp collectively allocates or deallocates its TMEM. A group-2 allocation uses one warp from each peer CTA.", "sha256": "73e16902aedb4d98c7e1df66bf5c584223b108f2e7212840db033cad2b067c88"}, {"id": "u020", "kind": "prose", "locator": "body:L38-L38", "preview": "`tcgen05.mma` is asynchronous. `tcgen05.commit.cta_group::N.mbarrier::arrive::one.b64` makes an mbarrier track prior MMA work issued by the current thread; waiting on that barrier observes completion. A `tcgen05.fence` is an ordering and co", "sha256": "2c0f765385f99c91fd6bba2fce4d7f86c037b58d0a20e7f6ebd86695c01aa76c"}, {"id": "u021", "kind": "prose", "locator": "body:L40-L40", "preview": "After completion and any necessary thread handoff, epilogue warps use `tcgen05.ld` to bring their accessible TMEM lanes into registers. Ordinary bias, activation, conversion, and global-store code then operates on those register values. Pre", "sha256": "9a0355a075bf524abfe540283b025bfdd995ccf1fff1a336741a4c22859c7514"}, {"id": "u022", "kind": "prose", "locator": "body:L44-L44", "preview": "Do not mechanically convert every 64-byte swizzle to 128-byte swizzling. The tcgen05 shared-memory descriptor defines valid no-swizzle, 128B, 64B, and 32B modes, subject to mode-specific alignment and layout constraints. The 128B choice can", "sha256": "a5608028d7dfad8506b7b6106284085462c52175cc29c7fd2293ab77560f02b3"}, {"id": "u023", "kind": "prose", "locator": "body:L46-L46", "preview": "Similarly, there is no universal Hopper-to-Blackwell rule that doubles M. WGMMA and tcgen05 each expose multiple shapes; tcgen05 M/N are encoded in `idesc` and constrained by kind, layout, CTA group, and target ISA. Treat m128xn256xk16 and ", "sha256": "a8234afdf8accc4e7f9754a352400271f4c0da85834278656ef70b903510bf4e"}, {"id": "u024", "kind": "prose", "locator": "body:L50-L50", "preview": "Single-thread MMA issue can free instruction-issue capacity for TMA, descriptor preparation, epilogue, reductions, or scheduling. It does not determine the kernel's total thread count: TMEM allocation, TMEM loads, TMA, epilogue, and barrier", "sha256": "9c73fd720d7cee08f667430e653f338b5de11c015ad161b38c7e22150ac74a05"}, {"id": "u025", "kind": "prose", "locator": "body:L52-L52", "preview": "Moving D out of the GPR file also changes the register budget. Revisit tile size, pipeline depth, and launch bounds, but do not assume that lower accumulator pressure automatically increases occupancy; SMEM, TMEM, barriers, threads, and reg", "sha256": "a9ec2eb6b76b5bbe2308d3a68e5b60337addc6a971601f555cb98fb058f78f48"}, {"id": "u026", "kind": "prose", "locator": "body:L56-L56", "preview": "CUTLASS does not reduce this port to replacing an architecture tag and one schedule token. In pinned SM100 examples, the collective builders are re-instantiated with SM100 operator classes, tile and cluster shapes, stage policy, mainloop sc", "sha256": "7daf9cbdd59a2295c9b4b8a5d268cd9f8688a699e49351df54dbe17a067d6f94"}, {"id": "u027", "kind": "prose", "locator": "body:L58-L58", "preview": "Use a complete pinned example such as CUTLASS `examples/70_blackwell_gemm/70_blackwell_fp16_gemm.cu`, then revalidate alignment, workspace, hardware support, dispatch, and numerical results for the migrated configuration.", "sha256": "f910c88656bc71121c90a77b069b343ac27796b217f14f1c6d8a8cf3e4d143d4"}], "confidence_claimed": "verified", "headings": ["What actually changes", "Dependency-ordered migration checklist", "TMEM lifecycle", "Completion, ordering, and epilogue", "Layout and shape selection", "Warp specialization", "CUTLASS migration"], "id": "migration-wgmma-to-tcgen05", "path": "wiki/migration/wgmma-to-tcgen05.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/prs/cutlass/PR-2139.md", "revision": "ca4fdbea", "url": "https://github.com/NVIDIA/cutlass/pull/2139"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}], "risk_flags": ["ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-ptx-isa-sm100", "pr-cutlass-2139", "blog-tcgen05-tutorial"], "title": "Migrating from wgmma to tcgen05", "type": "migration", "unresolved_source_ids": [], "version_sensitive": null} -{"body_sha256": "e08627fe8f79b7e82f8eb53531d3aa22ddb2a98858cae22ae1389d0084a145c3", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "\u201cBelow peak FLOPS\u201d is not itself a bottleneck diagnosis. Select the peak for the executed datatype, instruction kind, sparsity/scaling mode, clocks, and number of participating SMs. Then use arithmetic intensity and achieved compute/memory ", "sha256": "37a1bbcb990374122e9646954c44ff413f62d50018c5702c5d882c75a80633c2"}, {"id": "u002", "kind": "prose", "locator": "body:L5-L5", "preview": "Also distinguish whole-kernel throughput from tensor-core active time. A correct kernel can have efficient MMA intervals yet spend material time in TMA readiness, CUDA-core transforms, reductions, synchronization, epilogue work, or the grid", "sha256": "dc2fa3d8af9c6c95dc56e1fd641fe6c0a042bfdb6becdd5bfeb8af505ef98076"}, {"id": "u003", "kind": "table-row", "locator": "body:L11-L11", "preview": "| Observation and matched control | Supported inference if runtime also improves | | An extra legal SMEM operand stage reduces TMA-full waits | Operand production was exposed on the tested K loop |", "sha256": "3b43f587c282234b6d8375f8dbfeaae54809c78846ee848467588c17da3cab6e"}, {"id": "u004", "kind": "table-row", "locator": "body:L12-L12", "preview": "| Observation and matched control | Supported inference if runtime also improves | | Separating producer/MMA roles reduces control or dependency gaps | The prior role schedule was on the critical issue path |", "sha256": "1cbdf57b4870a67ac98e4e2f3a7c62322d7812ad05c1559d8529e6dbc6d8043d"}, {"id": "u005", "kind": "table-row", "locator": "body:L13-L13", "preview": "| Observation and matched control | Supported inference if runtime also improves | | A second TMEM output region reduces mainloop-to-epilogue waits | Accumulator reuse was serialized by the epilogue |", "sha256": "5d043e745540fbc1868cf31ba9b60aac234ea672c2633528c7ec4ecb1d551cf6"}, {"id": "u006", "kind": "table-row", "locator": "body:L14-L14", "preview": "| Observation and matched control | Supported inference if runtime also improves | | Moving only selected non-MMA operations changes their pipeline activity and runtime | Those operations contributed to the tested critical path |", "sha256": "977930000d035631826feb8c0f2e73f456f1feaeab8e0b5979244180123d5514"}, {"id": "u007", "kind": "table-row", "locator": "body:L15-L15", "preview": "| Observation and matched control | Supported inference if runtime also improves | | A legal 2-SM variant changes traffic/reuse and aggregate throughput | Cooperative mapping helped that shape and resource configuration |", "sha256": "ca57e5367e878d85eab42bbedef761a622b7312edd9c443d14b9205a288b66bc"}, {"id": "u008", "kind": "prose", "locator": "body:L17-L17", "preview": "For every comparison, hold math, datatype, tile coverage, launch environment, warmup, and timing statistic fixed. Record registers, spills, SMEM, TMEM columns, occupancy, cluster size, achieved bandwidth, scheduler issue activity, and per-p", "sha256": "64b22cc4fe9704c2e3785586fa1233ed26749a30d1292d6b101979aac43b4248"}, {"id": "u009", "kind": "prose", "locator": "body:L23-L23", "preview": "Enumerate stage counts that compile and fit the complete shared-storage allocation. More stages can overlap TMA production with MMA consumption, but they also consume SMEM, change occupancy, and enlarge prologue/tail costs. There is no arch", "sha256": "965081cf338b66d916d70a6e2a6a575655f8d7348a132f8468ba94fcbc6abbf7"}, {"id": "u010", "kind": "prose", "locator": "body:L27-L27", "preview": "Dedicated producer, MMA, or epilogue warps can remove role switching from a loop and allow independent work to progress, but add live warps, registers, and synchronization. They do not eliminate stalls. If the epilogue owns the only output ", "sha256": "2a05937c07bd241e43d89abea73682f936e5657100982c7bb3e5e3cbb5f55449"}, {"id": "u011", "kind": "prose", "locator": "body:L31-L31", "preview": "`cta_group::2` requires a valid cluster and exact kind-, shape-, layout-, descriptor-, and peer-resource constraints. It does not universally mean `m256n256`, require identical SMEM layouts, or promise twice the compute per cycle: one coope", "sha256": "d3b7453b3cc18c372b289d5200bfc051ca379b625f6a0527c8cf50d59c88e178"}, {"id": "u012", "kind": "prose", "locator": "body:L35-L35", "preview": "Optimize non-MMA work only after showing that it lies on the critical path. Fusion may remove intermediate traffic; tile interleaving may expose independent work; an approximation may exchange numerical error for throughput. Validate error ", "sha256": "e7197f0143cfd5cbf7e6424b5ff1b4dc3d49327f77499e54304e45f5e75ab5a0"}, {"id": "u013", "kind": "prose", "locator": "body:L39-L39", "preview": "FA4's B200 analysis reports 8192 BF16 MMA operations per clock per SM versus 4096 on Hopper, while exponential throughput is 16 operations per clock per SM on both. Its response is a coordinated design: two-output-tile scheduling, selected ", "sha256": "567f2a49763773c1bb46cd26b83edfc8f556f823ecfa082ac0c1def9873ad16f"}, {"id": "u014", "kind": "prose", "locator": "body:L41-L41", "preview": "The software path uses base-2 range reduction with `n=floor(x)` and a cubic FMA polynomial for a selected fraction of exponential evaluations; other values still use hardware `ex2`. The paper evaluates approximation error and end-to-end acc", "sha256": "d07f08d69c441f10ff4348a210c915a0a3e4cc28559056964b60f5923a124e9d"}, {"id": "u015", "kind": "prose", "locator": "body:L43-L43", "preview": "The paper reports up to 1613 TFLOP/s and 71% for complete FA4 forward kernels; the author blog reports up to 1605 TFLOP/s and 71%. Neither number isolates software exponentiation or any other single technique. Use FA4 as evidence that non-M", "sha256": "9d430642b62bcc1e464c1834a7038a7c7ebeaeffae02b469f9dc7ca5be34e5b9"}, {"id": "u016", "kind": "list-item", "locator": "body:L47-L47", "preview": "- [Nsight Compute 2025.3 roofline and profiling guidance](https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html)", "sha256": "9e40cc52e2795a8ac60ea70f7ce96eaf9fbbc49396653dd7eb617324d7628e09"}, {"id": "u017", "kind": "list-item", "locator": "body:L48-L48", "preview": "- [PTX ISA 9.0 tcgen05 MMA forms](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-mma)", "sha256": "0a9f68ac9e6cd67818c5d0f58f43cae5f62e1fc07f73ce030a4a27e90096ef33"}, {"id": "u018", "kind": "list-item", "locator": "body:L49-L49", "preview": "- [Pinned two-stage tutorial pipeline](https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu)", "sha256": "5480b05a76d56aaef8da618217be1dcf0583de432a860a245bf8b8aefb8a596e"}, {"id": "u019", "kind": "list-item", "locator": "body:L50-L50", "preview": "- [FlashAttention-4 paper v1](https://arxiv.org/html/2603.05451v1)", "sha256": "bc4648fc7d9797d1e5f1a28a9df55310a395f7bfc7efd78a1304fd817f2b10a8"}, {"id": "u020", "kind": "list-item", "locator": "body:L51-L51", "preview": "- [FlashAttention-4 author blog](https://tridao.me/blog/2026/flash4/)", "sha256": "cb9eb04e449adccb306d0356032bae2e6788f881a47ab8d8e3744cdcc9a4a55f"}], "confidence_claimed": "verified", "headings": ["Classify the Gap First", "Separate the Limiting Edge", "Candidate-Specific Checks", "Pipeline stages", "Warp specialization and output overlap", "2-SM cooperative MMA", "Non-MMA work", "FlashAttention-4 Case Study", "Primary References"], "id": "pattern-compute-bound", "path": "wiki/patterns/compute-bound.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/docs/flash-attention-4.md", "url": "https://arxiv.org/abs/2603.05451v1"}, {"path": "sources/blogs/flash-attention-4.md", "url": "https://tridao.me/blog/2026/flash4/"}], "risk_flags": ["ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "doc-flash-attention-4", "blog-flash-attention-4"], "title": "Not Reaching the Relevant Compute Ceiling", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} -{"body_sha256": "2c8aa46a8f96c06bc50d29d566037068eabf9c65e7c5d513b52cae6efc50c993", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "Low SM utilization means that fewer SMs perform useful work than the workload could profitably use during a material part of its measured time. It is not defined by a universal 60% threshold. Theoretical occupancy describes how many blocks ", "sha256": "dc68053a12a55778656da58a55c9814e043d8b253dd92d90f1ae3fbb763ecff3"}, {"id": "u002", "kind": "prose", "locator": "body:L5-L5", "preview": "Collect a synchronized kernel time and a time-resolved view where supported. Record the physical and application-constrained SM count, grid and cluster dimensions, blocks resident per SM, logical work count, work duration distribution, and ", "sha256": "4d2bd929f571b1290b98d979ae170aec632c360030e06c07774fbbe7590a0a78"}, {"id": "u003", "kind": "table-row", "locator": "body:L11-L11", "preview": "| Case | Discriminating evidence | Appropriate next experiment | | Too little independent work | Logical block/cluster count is below available one-wave worker capacity | Change problem decomposition, accounting for reduction and synchroniz", "sha256": "ff994186496977f7e437e154e3f90a1e2fa87d0eda0137219a9ab114e4246e2d"}, {"id": "u004", "kind": "table-row", "locator": "body:L12-L12", "preview": "| Case | Discriminating evidence | Appropriate next experiment | | Wave quantization | Equal-duration data-parallel work has a small nonzero final-wave remainder | Compare tile shapes or a decomposition that changes tile count |", "sha256": "5ce10167b4d3a44da66313f807951f9b46f3ef85adf55d4aeb93a7ce29d7ab2a"}, {"id": "u005", "kind": "table-row", "locator": "body:L13-L13", "preview": "| Case | Discriminating evidence | Appropriate next experiment | | Variable work duration | Per-worker tile counts/times have a long tail despite enough pending work | Compare static and dynamic acquisition with identical tile/decomposition", "sha256": "b1df3c32e77f9429f103d73d192d7db365f866ddc6cf1e845f75925a7ac957cf"}, {"id": "u006", "kind": "table-row", "locator": "body:L14-L14", "preview": "| Case | Discriminating evidence | Appropriate next experiment | | Residency or phase limitation | Grid is large, but resources or serial phases limit active blocks/warps | Change the limiting resource or phase; more grid blocks alone do no", "sha256": "e855f5bd4b6efd616e9c5159e215890d6ad81a321dbd2373c6aa65dde38069a7"}, {"id": "u007", "kind": "prose", "locator": "body:L16-L16", "preview": "Static assignment is nonadaptive, but it is not automatically imbalanced. A grid with fewer independent blocks than SMs cannot occupy every SM, while a grid much larger than the SM count can still show a tail or long-running stragglers. Do ", "sha256": "67a641057a83184ccd37d92d8b68398286f404e0ce56f00ce940fff1df3fe2d3"}, {"id": "u008", "kind": "prose", "locator": "body:L20-L20", "preview": "Persistence, coordinate order, CLC reassignment, and K decomposition are separate controls:", "sha256": "00ed398368679026145a964150a682adb0d3cc7fd0d060c54598a487366eece5"}, {"id": "u009", "kind": "list-item", "locator": "body:L22-L22", "preview": "- A static persistent worker can process multiple logical tiles by grid stride. It can amortize setup and change wave behavior but cannot guarantee removal of the final tail.", "sha256": "970c3cdbaf7dffb98cbbed36f91f9769dc030e8b7387f9dc98b592f0e7b58908"}, {"id": "u010", "kind": "list-item", "locator": "body:L23-L23", "preview": "- A row/column raster or swizzle changes coordinate order and may change operand locality. It does not guarantee a better L2 hit rate or lower work variance.", "sha256": "ee2954aa11e887c12ecac32122fff7c169e2a9aed5c43a8aff669da98adfaa95"}, {"id": "u011", "kind": "list-item", "locator": "body:L24-L24", "preview": "- Cluster Launch Control lets a running worker cancel an unspecified not-yet-started block or cluster from the launched grid and process the returned ID. It redistributes existing work; it cannot create independent tiles, discard required w", "sha256": "6d7b13fddf81ed5b845e50c993a9625c5d24e5f476ab5e896e8683254265a886"}, {"id": "u012", "kind": "list-item", "locator": "body:L25-L25", "preview": "- Stream-K or Split-K can create additional partitions when tile-level parallelism is insufficient, at the cost of partial-result reduction, workspace, synchronization, and possible determinism changes.", "sha256": "125414014589e01cb3a8612ef37f7a9a4ec31d18d82378aeea0953131a3e6b67"}, {"id": "u013", "kind": "prose", "locator": "body:L27-L27", "preview": "For CLC, compare static and dynamic scheduling with identical math, tile/cluster shapes, problem-sized grid, resource limits, and timing method. Record successful and failed requests plus per-worker work counts where instrumentation permits", "sha256": "812e28e793d9fe3f9828c17e49c7779bc964eb811dd42692bc89b4c80b15c4c8"}, {"id": "u014", "kind": "prose", "locator": "body:L29-L29", "preview": "PTX ISA 9.0 says `clusterlaunchcontrol.try_cancel` requires `sm_100` or higher; its cluster-wide multicast qualifier explicitly lists `sm_120a` and the SM120 family. Therefore CLC is not categorically excluded from SM120 by the ISA. The pin", "sha256": "49c0772dcd2917bc11ccc1ad8c2e9fef3bbe8d4376fe49cf7aadd36fc57aebb0"}, {"id": "u015", "kind": "prose", "locator": "body:L33-L33", "preview": "In Gau Nernst's `M=N=K=4096` B200 experiment, v5 reports 1302.29 TFLOP/s (86.43% of its 1506.74-TFLOP/s cuBLAS value), while v6 reports 1475.93 TFLOP/s (97.96%). The v6 endpoint combines static persistence, a changed output pipeline, and ep", "sha256": "f198e4c538684687922c5b39623a6e903579ad5473f3ec9267f3edeb9ab89647"}, {"id": "u016", "kind": "list-item", "locator": "body:L37-L37", "preview": "1. Prove exact logical-work coverage and no duplicates for initial and reassigned coordinates.", "sha256": "eaead768d23afffd442c784dbf60455f43526c0efe7cbd3c0751bf9dd3620fa4"}, {"id": "u017", "kind": "list-item", "locator": "body:L38-L38", "preview": "2. Compare requested work with available worker capacity and calculate the equal-duration final-wave remainder as a diagnostic baseline.", "sha256": "282a05b0f3813e50351b2d9b9121bf28136488746f2bf53251ff7ff18147466d"}, {"id": "u018", "kind": "list-item", "locator": "body:L39-L39", "preview": "3. Instrument work count and time per worker to separate shortage from variable-duration imbalance.", "sha256": "85d7667eb183b65674cf5bd1b4a61e1b431faf63459925677d7428c012beb7db"}, {"id": "u019", "kind": "list-item", "locator": "body:L40-L40", "preview": "4. Hold decomposition constant when testing static versus dynamic acquisition; test decomposition separately.", "sha256": "52d3102612d922f1fc85038af5c4ab2562f4ebabc9055143ddff583eafbdc805"}, {"id": "u020", "kind": "list-item", "locator": "body:L41-L41", "preview": "5. Report end-to-end time along with active-SM distribution, scheduler overhead, cache traffic, occupancy, and regressions.", "sha256": "baade11400a3d956b8272f80cca633553e8e4a8b7937e365aec1771558f87edb"}, {"id": "u021", "kind": "list-item", "locator": "body:L45-L45", "preview": "- [CUDA Programming Guide: Cluster Launch Control](https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html)", "sha256": "434985b84456959e4f53d2ca8c885026c1e84c1718e8d70d3a58f21c7cfff6cd"}, {"id": "u022", "kind": "list-item", "locator": "body:L46-L46", "preview": "- [PTX ISA 9.0 CLC target and request semantics](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel)", "sha256": "1ea71a1b44ae1bff04003838fd3c253ddd1bf748f883aa107a4c088bf748f5c3"}, {"id": "u023", "kind": "list-item", "locator": "body:L47-L47", "preview": "- [CUTLASS 4.5.0 CLC scheduling](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md)", "sha256": "f82b94cccb8b9883e584fa5ec6b7e602e2b7c3247932e262a7a61b54773d925d"}, {"id": "u024", "kind": "list-item", "locator": "body:L48-L48", "preview": "- [Pinned tutorial v6](https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu)", "sha256": "5b526dd64c330d7d736082b7324135601597bb08ed0687532ebef171bf9798d1"}, {"id": "u025", "kind": "list-item", "locator": "body:L49-L49", "preview": "- [Tutorial result progression](https://gau-nernst.github.io/tcgen05/)", "sha256": "797b16f54ff8d25ba34fefe90153d0a2e58a3efa7414f3594476c0b133a26bad"}], "confidence_claimed": "verified", "headings": ["Define the Missing Parallelism", "Distinguish Four Cases", "Scheduler Choices", "Source-Reported Tutorial Result", "Verification Checklist", "Primary References"], "id": "pattern-low-sm-utilization", "path": "wiki/patterns/low-sm-utilization.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/docs/cutlass-clc-documentation.md", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}], "risk_flags": ["ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "doc-cutlass-clc", "doc-ptx-isa-sm100"], "title": "Low SM Utilization", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "4228e9cb6b16da5fc6d2e38050568c5663fdc8d919e081ff0990a2479e9c1d0c", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "\u201cBelow peak FLOPS\u201d is not itself a bottleneck diagnosis. Select the peak for the executed datatype, instruction kind, sparsity/scaling mode, clocks, and number of participating SMs. Then use arithmetic intensity and achieved compute/memory ", "sha256": "37a1bbcb990374122e9646954c44ff413f62d50018c5702c5d882c75a80633c2"}, {"id": "u002", "kind": "prose", "locator": "body:L5-L5", "preview": "Also distinguish whole-kernel throughput from tensor-core active time. A correct kernel can have efficient MMA intervals yet spend material time in TMA readiness, CUDA-core transforms, reductions, synchronization, epilogue work, or the grid", "sha256": "dc2fa3d8af9c6c95dc56e1fd641fe6c0a042bfdb6becdd5bfeb8af505ef98076"}, {"id": "u003", "kind": "table-row", "locator": "body:L11-L11", "preview": "| Observation and matched control | Supported inference if runtime also improves | | An extra legal SMEM operand stage reduces TMA-full waits | Operand production was exposed on the tested K loop |", "sha256": "3b43f587c282234b6d8375f8dbfeaae54809c78846ee848467588c17da3cab6e"}, {"id": "u004", "kind": "table-row", "locator": "body:L12-L12", "preview": "| Observation and matched control | Supported inference if runtime also improves | | Separating producer/MMA roles reduces control or dependency gaps | The prior role schedule was on the critical issue path |", "sha256": "1cbdf57b4870a67ac98e4e2f3a7c62322d7812ad05c1559d8529e6dbc6d8043d"}, {"id": "u005", "kind": "table-row", "locator": "body:L13-L13", "preview": "| Observation and matched control | Supported inference if runtime also improves | | A second TMEM output region reduces mainloop-to-epilogue waits | Accumulator reuse was serialized by the epilogue |", "sha256": "5d043e745540fbc1868cf31ba9b60aac234ea672c2633528c7ec4ecb1d551cf6"}, {"id": "u006", "kind": "table-row", "locator": "body:L14-L14", "preview": "| Observation and matched control | Supported inference if runtime also improves | | Moving only selected non-MMA operations changes their pipeline activity and runtime | Those operations contributed to the tested critical path |", "sha256": "977930000d035631826feb8c0f2e73f456f1feaeab8e0b5979244180123d5514"}, {"id": "u007", "kind": "table-row", "locator": "body:L15-L15", "preview": "| Observation and matched control | Supported inference if runtime also improves | | A legal 2-SM variant changes traffic/reuse and aggregate throughput | Cooperative mapping helped that shape and resource configuration |", "sha256": "ca57e5367e878d85eab42bbedef761a622b7312edd9c443d14b9205a288b66bc"}, {"id": "u008", "kind": "prose", "locator": "body:L17-L17", "preview": "For every comparison, hold math, datatype, tile coverage, launch environment, warmup, and timing statistic fixed. Record registers, spills, SMEM, TMEM columns, occupancy, cluster size, achieved bandwidth, scheduler issue activity, and per-p", "sha256": "64b22cc4fe9704c2e3785586fa1233ed26749a30d1292d6b101979aac43b4248"}, {"id": "u009", "kind": "prose", "locator": "body:L23-L23", "preview": "Enumerate stage counts that compile and fit the complete shared-storage allocation. More stages can overlap TMA production with MMA consumption, but they also consume SMEM, change occupancy, and enlarge prologue/tail costs. There is no arch", "sha256": "965081cf338b66d916d70a6e2a6a575655f8d7348a132f8468ba94fcbc6abbf7"}, {"id": "u010", "kind": "prose", "locator": "body:L27-L27", "preview": "Dedicated producer, MMA, or epilogue warps can remove role switching from a loop and allow independent work to progress, but add live warps, registers, and synchronization. They do not eliminate stalls. If the epilogue owns the only output ", "sha256": "2a05937c07bd241e43d89abea73682f936e5657100982c7bb3e5e3cbb5f55449"}, {"id": "u011", "kind": "prose", "locator": "body:L31-L31", "preview": "`cta_group::2` requires a valid cluster and exact kind-, shape-, layout-, descriptor-, and peer-resource constraints. It does not universally mean `m256n256`, require identical SMEM layouts, or promise twice the compute per cycle: one coope", "sha256": "d3b7453b3cc18c372b289d5200bfc051ca379b625f6a0527c8cf50d59c88e178"}, {"id": "u012", "kind": "prose", "locator": "body:L35-L35", "preview": "Optimize non-MMA work only after showing that it lies on the critical path. Fusion may remove intermediate traffic; tile interleaving may expose independent work; an approximation may exchange numerical error for throughput. Validate error ", "sha256": "e7197f0143cfd5cbf7e6424b5ff1b4dc3d49327f77499e54304e45f5e75ab5a0"}, {"id": "u013", "kind": "prose", "locator": "body:L39-L39", "preview": "FA4's B200 analysis reports 8192 BF16 MMA operations per clock per SM versus 4096 on Hopper, while exponential throughput is 16 operations per clock per SM on both. Its response is a coordinated design: two-output-tile scheduling, selected ", "sha256": "567f2a49763773c1bb46cd26b83edfc8f556f823ecfa082ac0c1def9873ad16f"}, {"id": "u014", "kind": "prose", "locator": "body:L41-L41", "preview": "The software path uses base-2 range reduction with `n=floor(x)` and a cubic FMA polynomial for a selected fraction of exponential evaluations; other values still use hardware `ex2`. The paper evaluates approximation error and end-to-end acc", "sha256": "d07f08d69c441f10ff4348a210c915a0a3e4cc28559056964b60f5923a124e9d"}, {"id": "u015", "kind": "prose", "locator": "body:L43-L43", "preview": "The paper reports up to 1613 TFLOP/s and 71% for complete FA4 forward kernels; the author blog reports up to 1605 TFLOP/s and 71%. Neither number isolates software exponentiation or any other single technique. Use FA4 as evidence that non-M", "sha256": "9d430642b62bcc1e464c1834a7038a7c7ebeaeffae02b469f9dc7ca5be34e5b9"}, {"id": "u016", "kind": "list-item", "locator": "body:L47-L47", "preview": "- [Nsight Compute 2025.3 roofline and profiling guidance](https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html)", "sha256": "9e40cc52e2795a8ac60ea70f7ce96eaf9fbbc49396653dd7eb617324d7628e09"}, {"id": "u017", "kind": "list-item", "locator": "body:L48-L48", "preview": "- [PTX ISA 9.0 tcgen05 MMA forms](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-mma-instructions-mma)", "sha256": "511c14f67434214ce8cab27a76c6316475c10430e4871b2720fadb675da81efc"}, {"id": "u018", "kind": "list-item", "locator": "body:L49-L49", "preview": "- [Pinned two-stage tutorial pipeline](https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu)", "sha256": "5480b05a76d56aaef8da618217be1dcf0583de432a860a245bf8b8aefb8a596e"}, {"id": "u019", "kind": "list-item", "locator": "body:L50-L50", "preview": "- [FlashAttention-4 paper v1](https://arxiv.org/html/2603.05451v1)", "sha256": "bc4648fc7d9797d1e5f1a28a9df55310a395f7bfc7efd78a1304fd817f2b10a8"}, {"id": "u020", "kind": "list-item", "locator": "body:L51-L51", "preview": "- [FlashAttention-4 author blog](https://tridao.me/blog/2026/flash4/)", "sha256": "cb9eb04e449adccb306d0356032bae2e6788f881a47ab8d8e3744cdcc9a4a55f"}], "confidence_claimed": "verified", "headings": ["Classify the Gap First", "Separate the Limiting Edge", "Candidate-Specific Checks", "Pipeline stages", "Warp specialization and output overlap", "2-SM cooperative MMA", "Non-MMA work", "FlashAttention-4 Case Study", "Primary References"], "id": "pattern-compute-bound", "path": "wiki/patterns/compute-bound.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/docs/flash-attention-4.md", "url": "https://arxiv.org/abs/2603.05451v1"}, {"path": "sources/blogs/flash-attention-4.md", "url": "https://tridao.me/blog/2026/flash4/"}], "risk_flags": ["ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "doc-flash-attention-4", "blog-flash-attention-4"], "title": "Not Reaching the Relevant Compute Ceiling", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "a634b268d016569c69a65ff48961bf10e404a3c7447470edb118ed7d1eb5a38f", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "Low SM utilization means that fewer SMs perform useful work than the workload could profitably use during a material part of its measured time. It is not defined by a universal 60% threshold. Theoretical occupancy describes how many blocks ", "sha256": "dc68053a12a55778656da58a55c9814e043d8b253dd92d90f1ae3fbb763ecff3"}, {"id": "u002", "kind": "prose", "locator": "body:L5-L5", "preview": "Collect a synchronized kernel time and a time-resolved view where supported. Record the physical and application-constrained SM count, grid and cluster dimensions, blocks resident per SM, logical work count, work duration distribution, and ", "sha256": "4d2bd929f571b1290b98d979ae170aec632c360030e06c07774fbbe7590a0a78"}, {"id": "u003", "kind": "table-row", "locator": "body:L11-L11", "preview": "| Case | Discriminating evidence | Appropriate next experiment | | Too little independent work | Logical block/cluster count is below available one-wave worker capacity | Change problem decomposition, accounting for reduction and synchroniz", "sha256": "ff994186496977f7e437e154e3f90a1e2fa87d0eda0137219a9ab114e4246e2d"}, {"id": "u004", "kind": "table-row", "locator": "body:L12-L12", "preview": "| Case | Discriminating evidence | Appropriate next experiment | | Wave quantization | Equal-duration data-parallel work has a small nonzero final-wave remainder | Compare tile shapes or a decomposition that changes tile count |", "sha256": "5ce10167b4d3a44da66313f807951f9b46f3ef85adf55d4aeb93a7ce29d7ab2a"}, {"id": "u005", "kind": "table-row", "locator": "body:L13-L13", "preview": "| Case | Discriminating evidence | Appropriate next experiment | | Variable work duration | Per-worker tile counts/times have a long tail despite enough pending work | Compare static and dynamic acquisition with identical tile/decomposition", "sha256": "b1df3c32e77f9429f103d73d192d7db365f866ddc6cf1e845f75925a7ac957cf"}, {"id": "u006", "kind": "table-row", "locator": "body:L14-L14", "preview": "| Case | Discriminating evidence | Appropriate next experiment | | Residency or phase limitation | Grid is large, but resources or serial phases limit active blocks/warps | Change the limiting resource or phase; more grid blocks alone do no", "sha256": "e855f5bd4b6efd616e9c5159e215890d6ad81a321dbd2373c6aa65dde38069a7"}, {"id": "u007", "kind": "prose", "locator": "body:L16-L16", "preview": "Static assignment is nonadaptive, but it is not automatically imbalanced. A grid with fewer independent blocks than SMs cannot occupy every SM, while a grid much larger than the SM count can still show a tail or long-running stragglers. Do ", "sha256": "67a641057a83184ccd37d92d8b68398286f404e0ce56f00ce940fff1df3fe2d3"}, {"id": "u008", "kind": "prose", "locator": "body:L20-L20", "preview": "Persistence, coordinate order, CLC reassignment, and K decomposition are separate controls:", "sha256": "00ed398368679026145a964150a682adb0d3cc7fd0d060c54598a487366eece5"}, {"id": "u009", "kind": "list-item", "locator": "body:L22-L22", "preview": "- A static persistent worker can process multiple logical tiles by grid stride. It can amortize setup and change wave behavior but cannot guarantee removal of the final tail.", "sha256": "970c3cdbaf7dffb98cbbed36f91f9769dc030e8b7387f9dc98b592f0e7b58908"}, {"id": "u010", "kind": "list-item", "locator": "body:L23-L23", "preview": "- A row/column raster or swizzle changes coordinate order and may change operand locality. It does not guarantee a better L2 hit rate or lower work variance.", "sha256": "ee2954aa11e887c12ecac32122fff7c169e2a9aed5c43a8aff669da98adfaa95"}, {"id": "u011", "kind": "list-item", "locator": "body:L24-L24", "preview": "- Cluster Launch Control lets a running worker cancel an unspecified not-yet-started block or cluster from the launched grid and process the returned ID. It redistributes existing work; it cannot create independent tiles, discard required w", "sha256": "6d7b13fddf81ed5b845e50c993a9625c5d24e5f476ab5e896e8683254265a886"}, {"id": "u012", "kind": "list-item", "locator": "body:L25-L25", "preview": "- Stream-K or Split-K can create additional partitions when tile-level parallelism is insufficient, at the cost of partial-result reduction, workspace, synchronization, and possible determinism changes.", "sha256": "125414014589e01cb3a8612ef37f7a9a4ec31d18d82378aeea0953131a3e6b67"}, {"id": "u013", "kind": "prose", "locator": "body:L27-L27", "preview": "For CLC, compare static and dynamic scheduling with identical math, tile/cluster shapes, problem-sized grid, resource limits, and timing method. Record successful and failed requests plus per-worker work counts where instrumentation permits", "sha256": "812e28e793d9fe3f9828c17e49c7779bc964eb811dd42692bc89b4c80b15c4c8"}, {"id": "u014", "kind": "prose", "locator": "body:L29-L29", "preview": "PTX ISA 9.0 says `clusterlaunchcontrol.try_cancel` requires `sm_100` or higher; its cluster-wide multicast qualifier explicitly lists `sm_120a` and the SM120 family. Therefore CLC is not categorically excluded from SM120 by the ISA. The pin", "sha256": "49c0772dcd2917bc11ccc1ad8c2e9fef3bbe8d4376fe49cf7aadd36fc57aebb0"}, {"id": "u015", "kind": "prose", "locator": "body:L33-L33", "preview": "In Gau Nernst's `M=N=K=4096` B200 experiment, v5 reports 1302.29 TFLOP/s (86.43% of its 1506.74-TFLOP/s cuBLAS value), while v6 reports 1475.93 TFLOP/s (97.96%). The v6 endpoint combines static persistence, a changed output pipeline, and ep", "sha256": "f198e4c538684687922c5b39623a6e903579ad5473f3ec9267f3edeb9ab89647"}, {"id": "u016", "kind": "list-item", "locator": "body:L37-L37", "preview": "1. Prove exact logical-work coverage and no duplicates for initial and reassigned coordinates.", "sha256": "eaead768d23afffd442c784dbf60455f43526c0efe7cbd3c0751bf9dd3620fa4"}, {"id": "u017", "kind": "list-item", "locator": "body:L38-L38", "preview": "2. Compare requested work with available worker capacity and calculate the equal-duration final-wave remainder as a diagnostic baseline.", "sha256": "282a05b0f3813e50351b2d9b9121bf28136488746f2bf53251ff7ff18147466d"}, {"id": "u018", "kind": "list-item", "locator": "body:L39-L39", "preview": "3. Instrument work count and time per worker to separate shortage from variable-duration imbalance.", "sha256": "85d7667eb183b65674cf5bd1b4a61e1b431faf63459925677d7428c012beb7db"}, {"id": "u019", "kind": "list-item", "locator": "body:L40-L40", "preview": "4. Hold decomposition constant when testing static versus dynamic acquisition; test decomposition separately.", "sha256": "52d3102612d922f1fc85038af5c4ab2562f4ebabc9055143ddff583eafbdc805"}, {"id": "u020", "kind": "list-item", "locator": "body:L41-L41", "preview": "5. Report end-to-end time along with active-SM distribution, scheduler overhead, cache traffic, occupancy, and regressions.", "sha256": "baade11400a3d956b8272f80cca633553e8e4a8b7937e365aec1771558f87edb"}, {"id": "u021", "kind": "list-item", "locator": "body:L45-L45", "preview": "- [CUDA Programming Guide: Cluster Launch Control](https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/cluster-launch-control.html)", "sha256": "434985b84456959e4f53d2ca8c885026c1e84c1718e8d70d3a58f21c7cfff6cd"}, {"id": "u022", "kind": "list-item", "locator": "body:L46-L46", "preview": "- [PTX ISA 9.0 CLC target and request semantics](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel)", "sha256": "055e35e565a291ec0e9bbe519173d1aec130896cce80b1a65209730c3f346eb5"}, {"id": "u023", "kind": "list-item", "locator": "body:L47-L47", "preview": "- [CUTLASS 4.5.0 CLC scheduling](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md)", "sha256": "f82b94cccb8b9883e584fa5ec6b7e602e2b7c3247932e262a7a61b54773d925d"}, {"id": "u024", "kind": "list-item", "locator": "body:L48-L48", "preview": "- [Pinned tutorial v6](https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu)", "sha256": "5b526dd64c330d7d736082b7324135601597bb08ed0687532ebef171bf9798d1"}, {"id": "u025", "kind": "list-item", "locator": "body:L49-L49", "preview": "- [Tutorial result progression](https://gau-nernst.github.io/tcgen05/)", "sha256": "797b16f54ff8d25ba34fefe90153d0a2e58a3efa7414f3594476c0b133a26bad"}], "confidence_claimed": "verified", "headings": ["Define the Missing Parallelism", "Distinguish Four Cases", "Scheduler Choices", "Source-Reported Tutorial Result", "Verification Checklist", "Primary References"], "id": "pattern-low-sm-utilization", "path": "wiki/patterns/low-sm-utilization.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/docs/cutlass-clc-documentation.md", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}], "risk_flags": ["ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "doc-cutlass-clc", "doc-ptx-isa-sm100"], "title": "Low SM Utilization", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} {"body_sha256": "df62414848150544dd22c908b1f015e7c3925bad2512fe91186cfc9edffc8e8b", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "A kernel is memory-bandwidth bound when its measured arithmetic intensity places it on the memory side of the relevant roofline and its useful throughput is limited by an attained memory ceiling. Compute intensity from the operations actual", "sha256": "cc4bd91f176d605e31d0cc4f8a09523ded7dd1e512000b09385f3d19c9bd8a60"}, {"id": "u002", "kind": "prose", "locator": "body:L5-L5", "preview": "High DRAM throughput, low tensor-core activity, or a workload label such as GEMV is not sufficient alone. Record useful/requested bytes, transferred sectors or bytes, achieved bandwidth, cache behavior, scheduler issue/stalls, and end-to-en", "sha256": "452ba745b5f2d4681a17bb456941995bd003aa68107d46f228e615b82b2f9549"}, {"id": "u003", "kind": "table-row", "locator": "body:L9-L9", "preview": "| Evidence | More precise interpretation | | Useful bytes and transferred bytes are close; attained bandwidth is near the measured roof | Plausible bandwidth-ceiling limit |", "sha256": "0fd2a73f2b152964fbaa4f7d2413a91390ffa5a24361d357b8dbfced09282d11"}, {"id": "u004", "kind": "table-row", "locator": "body:L10-L10", "preview": "| Evidence | More precise interpretation | | Transferred bytes substantially exceed useful bytes | Coalescing, overfetch, cache, or redundant-traffic problem |", "sha256": "1716a47f062a06bff7fe43f5a747a8e6cff508071d791994fa9bbccc331c7f7f"}, {"id": "u005", "kind": "table-row", "locator": "body:L11-L11", "preview": "| Evidence | More precise interpretation | | Bandwidth is below the roof and warps lack ready work | Latency, dependency, insufficient concurrency, or issue problem |", "sha256": "cb08bba7957e0f2efd478c48c73e9acbcb838f6a54f90eaf06f5ea6960f77daf"}, {"id": "u006", "kind": "table-row", "locator": "body:L12-L12", "preview": "| Evidence | More precise interpretation | | Bandwidth is high only during one phase | Phase balance or fusion opportunity; whole-kernel boundedness remains unproven |", "sha256": "b4665c2eeed674412d9ed1d76c3a428205c05ad775b7294d0817ea60a1267113"}, {"id": "u007", "kind": "prose", "locator": "body:L14-L14", "preview": "Single-use data can lower operations per transferred byte, but \u201cpoor reuse\u201d is actionable only if additional legal reuse exists. Uncoalesced access and cache interference may increase traffic or latency; they are not synonyms for saturation", "sha256": "9d79f4c3ceb564927b8b707cf9c280edb4bf25c2d8d6cce03e8afe8a832561e6"}, {"id": "u008", "kind": "prose", "locator": "body:L20-L20", "preview": "Choose only legal, naturally aligned vector forms and handle tails separately. Compare scalar/narrow and wider variants with identical mapping. Record instruction count, requested and transferred bytes, transactions, registers, spills, achi", "sha256": "9b5f361784b37fe7c571114418e38b50ee1987cfd3b7c737bef1a8b177838955"}, {"id": "u009", "kind": "prose", "locator": "body:L24-L24", "preview": "PTX L1 eviction priorities and L2 prefetch controls are hints. `no_allocate` does not guarantee bypass, and `evict_last` does not guarantee residence. Characterize address order, reuse distance, working set, and interfering streams, then va", "sha256": "6b721336a33758eae0d71d95c602770ea6f3b58e37d6da610cf37f12c78b10b0"}, {"id": "u010", "kind": "prose", "locator": "body:L28-L28", "preview": "Inspect compiled registers, spills, SMEM, threads, and the actual occupancy-limiting resource before applying `-maxrregcount`. A cap may be clamped by ABI requirements, introduce spill traffic, or leave occupancy unchanged. Higher theoretic", "sha256": "a70f10cc7f9f485234c8040496f261aa5083ec2eb42ca7bcb936976be0270208"}, {"id": "u011", "kind": "prose", "locator": "body:L32-L32", "preview": "TMA multicast can issue one global-to-shared tensor copy to selected CTAs' shared-memory destinations within a cluster. It is relevant when those CTAs consume the same operand and the cluster/lifetime/barrier costs are valid. It does not he", "sha256": "fe64e13acbf1e8ce7ae0173430224af05a0baf6dd14a144777b87edd82cd33f5"}, {"id": "u012", "kind": "prose", "locator": "body:L34-L34", "preview": "Swizzling changes a shared-memory address mapping and can reduce conflicts for a specified access pattern. It does not universally eliminate conflicts and is not by itself a DRAM optimization. Validate the legal tensor-map/layout constraint", "sha256": "68bbe5d835608c759d3a6139ee4e7fbfb4b4f48e5b64ed210264ea86871a8b67"}, {"id": "u013", "kind": "prose", "locator": "body:L38-L38", "preview": "The cited Amandeep NVFP4 GEMV report is useful precisely because plausible memory-oriented changes failed. A wider `uint2` load was 16\u201325% slower on the reported shapes, and reducing `maxrregcount` from 80 to 64 had no effect. These observa", "sha256": "6f318f71c4ba4cfd742ff6fee67b3d4f8929e2a8b0a721fb81d828b089f41a4e"}, {"id": "u014", "kind": "prose", "locator": "body:L40-L40", "preview": "Use the same discipline for compute work. If the kernel is genuinely at its attainable memory roof, compute-only instruction reductions may have little effect. But address calculation, decoding, and instruction-level parallelism can affect ", "sha256": "4bf3256418639f29d01b7d95f9821b101480083ef4bfbcb021d8157b9ede2930"}, {"id": "u015", "kind": "prose", "locator": "body:L44-L44", "preview": "Report input shapes/distributions, useful work and bytes, cache state, warmup/repetitions/statistic, GPU and clocks, software versions, generated instructions, resource usage, roofline assumptions, achieved memory-level bandwidth, requested", "sha256": "c3fb3a60d91af6c78b839dce82e7b77f74688407543f7107927ad3d1dbe5fda4"}, {"id": "u016", "kind": "list-item", "locator": "body:L48-L48", "preview": "- [Nsight Compute 2025.3 Profiling Guide](https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html)", "sha256": "7d96e056fb2b06ab7298e5192301e839e81cf4e9a2afaf7ec44b408f56a20811"}, {"id": "u017", "kind": "list-item", "locator": "body:L49-L49", "preview": "- [PTX ISA 9.0 cache-operation semantics](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld)", "sha256": "18178953eb6438ebcfcb96936312c7d79c58c34750dee187e04be2251e1df4ef"}, {"id": "u018", "kind": "list-item", "locator": "body:L50-L50", "preview": "- [PTX ISA 9.0 TMA multicast](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor)", "sha256": "b61a3148a0694e4892a76dc9a81a407d548f10f93062892071b496747572f0a7"}, {"id": "u019", "kind": "list-item", "locator": "body:L51-L51", "preview": "- [Amandeep NVFP4 GEMV attempts](https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/)", "sha256": "cd885328e7372e60c5d3ad3b5f4b19d7d0c89f9aa3a7f3cff0992ffd946275a3"}], "confidence_claimed": "verified", "headings": ["Establish the Memory Roof", "Controlled Candidate Tests", "Load width and coalescing", "Cache policy", "Register budget and concurrency", "TMA multicast and shared-memory layout", "NVFP4 GEMV Negative Controls", "Reproduction Record", "Primary References"], "id": "pattern-memory-bound", "path": "wiki/patterns/memory-bound.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/amandeep-nvfp4-attempts.md", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}, {"path": "sources/blogs/yue-nvfp4-hackathon.md", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}, {"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}], "risk_flags": ["ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-amandeep-nvfp4", "blog-yue-nvfp4", "doc-nvidia-tuning-guide", "doc-ptx-isa-sm100"], "title": "Memory Bandwidth Bound", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} -{"body_sha256": "39a1e4e3fb4709726fa3f81e7e26b68f6c0936ca10fa151101aee424b9f283f0", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L8", "preview": "Do not infer one expert per physical SM from a grouped-GEMM trace. CUDA assigns thread blocks to available SMs, while grouped kernels may split an expert into multiple output tiles or let one resident worker process several logical tiles. M", "sha256": "6a4556c295e1fe46a216bb02fa3b9c18a4dfd635ed0ea95b40ab3105782a90aa"}, {"id": "u002", "kind": "list-item", "locator": "body:L10-L10", "preview": "- routed tokens per expert for the exact batch and prefill/decode phase;", "sha256": "7a869c75edb2661a6e1d7146b9df98b243d7611e31d063edc71f3f5c2948929c"}, {"id": "u003", "kind": "list-item", "locator": "body:L11-L11", "preview": "- valid rows and output-tile counts per expert;", "sha256": "21af8e58169651d75b62c2099c735e98ad0f179b6f192d03734b1b9f5e5d3541"}, {"id": "u004", "kind": "list-item", "locator": "body:L12-L12", "preview": "- completed tiles and elapsed work per logical worker and, when instrumented,", "sha256": "abe0c2bb6e4ce66df866b22cdacc7b3459bf91742abf5a10d70a7c157ba3e856"}, {"id": "u005", "kind": "prose", "locator": "body:L13-L13", "preview": "per SM;", "sha256": "a0e9294558ab12243cb5c87a97a4cbaf4966ab84a68cb44017852f8d95467519"}, {"id": "u006", "kind": "list-item", "locator": "body:L14-L14", "preview": "- dispatch, grouped-GEMM, combine, and end-to-end times per expert-parallel", "sha256": "642242588794a89bcb0b8bb3d69a8cf589838a4273d06d3bc63b3de5db8ec3c7"}, {"id": "u007", "kind": "prose", "locator": "body:L15-L15", "preview": "rank.", "sha256": "a9d4d07a50c665beea0d85f220de764cca3663b2c2af5d8c790a1d2970ef5cbc"}, {"id": "u008", "kind": "prose", "locator": "body:L17-L20", "preview": "Small expert segments and partial output tiles can expose too little parallel work or leave lanes predicated out. The effect depends on the kernel tile, other GEMM dimensions, resident-worker count, and competing implementation; there is no", "sha256": "10f047c682158b00cdf5835272cf7cbe2cc8c09f449d19f70966620f49ff4efc"}, {"id": "u009", "kind": "prose", "locator": "body:L22-L26", "preview": "Uniform expected routing, a larger batch, or an auxiliary balancing loss may change token-count skew, but none proves balanced tile counts, worker durations, communication, or runtime. Treat imbalance as absent only when the measured distri", "sha256": "4aea38fd66dea81359addca88f480b75cc6a8cd021b546e0728a216da167bc01"}, {"id": "u010", "kind": "prose", "locator": "body:L30-L32", "preview": "At pinned DeepGEMM commit [`891d57b4`](https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f), the three grouped interfaces have different axes and metadata:", "sha256": "db4321873118f047cf255c2898cd512bca03ae25073f932d1f4f5aa2696aeccd"}, {"id": "u011", "kind": "table-row", "locator": "body:L36-L36", "preview": "| Layout | Pinned contract | Relevant control | | M-grouped contiguous | Pack `A` and `D` along variable M with N/K fixed; identify groups by per-row expert IDs or per-group prefix-sum ends; segments are M-block aligned | Record valid rows,", "sha256": "d2455c5057ca34282247c70175297f8294562ea84975b8dbfd1d42af7117fccd"}, {"id": "u012", "kind": "table-row", "locator": "body:L37-L37", "preview": "| Layout | Pinned contract | Relevant control | | M-grouped masked | Allocate `[G, M_max, ...]`, pass one valid-M count per group, and compute valid portions; the fixed allocation is documented for a CUDA-graph decode case | Separate alloca", "sha256": "4992cd4a0d340ae7c7c8a103d0755063816b02ca84de49d6685fa2f9d5d22c83"}, {"id": "u013", "kind": "table-row", "locator": "body:L38-L38", "preview": "| Layout | Pinned contract | Relevant control | | K-grouped contiguous | Pack variable K with M/N fixed for MoE weight backward | Do not use its K-axis contract to describe forward token imbalance |", "sha256": "844315bc56884144631be8c64520b6b669112454794b93acd1837452b7777cb0"}, {"id": "u014", "kind": "prose", "locator": "body:L40-L42", "preview": "A fixed `M_max` allocation therefore does not establish that all padding rows are computed. Any residual edge-tile or predication cost needs generated-code and profile evidence for the selected kernel.", "sha256": "e1ad215b1834aea24c4c71f10f9dd49adb76027b5707f91aa558f23fed0775eb"}, {"id": "u015", "kind": "prose", "locator": "body:L46-L46", "preview": "Keep persistence, work acquisition, and decomposition as separate variables:", "sha256": "72b5c8bd73eb8523601df6dc917588754a8f96308e1b63a0634cf317643b4636"}, {"id": "u016", "kind": "list-item", "locator": "body:L48-L48", "preview": "1. A static persistent worker can advance through a deterministic logical-work", "sha256": "2ebeac42de4c86f459f32944d820f960af294bb7e3e90652bce81f1750d37a47"}, {"id": "u017", "kind": "prose", "locator": "body:L49-L50", "preview": "sequence. CUDA still decides where its block runs; the rule is not a precomputed tile-to-physical-SM map.", "sha256": "2141d38662b0abc377581ebce726e1b0f19cb22664d04f848c469c8824eb4a2d"}, {"id": "u018", "kind": "list-item", "locator": "body:L51-L51", "preview": "2. In a CLC-backed scheduler, a selected thread requests cancellation of an", "sha256": "ae1a3d61b865fd35104df9841d44658b7fbf250e361ae47cd15e04acd0767ab9"}, {"id": "u019", "kind": "prose", "locator": "body:L52-L54", "preview": "unspecified block or cluster that has not launched. A successful response returns that existing grid coordinate; a request can fail. CLC does not create tiles or make the fastest SM autonomously steal arbitrary work.", "sha256": "44666cd2a7aa439b477e3be4e332e7f1801285aeaa09555f77dc3ef6df41bd7a"}, {"id": "u020", "kind": "list-item", "locator": "body:L55-L55", "preview": "3. The initial `blockIdx` in CUTLASS's pinned CLC scheduler is static and later", "sha256": "ad48157736775ac54b4f798356d96d521ff0b5736bb3d5d8f9f2d83c18ca8b36"}, {"id": "u021", "kind": "prose", "locator": "body:L56-L58", "preview": "requests are asynchronous, pipelined, and cluster-granular. There is no universal per-tile latency bound: compare request activity and end-to-end time against the matched static scheduler.", "sha256": "848689501e7d30e823c98dde175448b2942777d47358b178c1b0b59592536b7d"}, {"id": "u022", "kind": "list-item", "locator": "body:L59-L59", "preview": "4. Splitting a large expert or K range may expose more independent work, but it", "sha256": "043c212f70796c400b7d34d0f4fcb011ecad2c412deb8ce351311e96cf1b18b2"}, {"id": "u023", "kind": "prose", "locator": "body:L60-L61", "preview": "can add partial-result and reduction costs. Hold math, tile shape, cluster shape, launch resources, and output semantics constant in the comparison.", "sha256": "ae096ac1401562950161c28a56d6eb66835ed10ab8b13f6892d16321db4ed587"}, {"id": "u024", "kind": "prose", "locator": "body:L63-L66", "preview": "PTX ISA 9.0 specifies CLC for `sm_100` or higher and explicitly includes SM120-family targets for the multicast form. CUTLASS 4.5.0's documented `PersistentTileSchedulerSm100` integration is a narrower library example, not the complete ISA ", "sha256": "ccd99bd11c24e601ec78e2e879751d77c744e1a7abbc3b1e17ee3194f421e15c"}, {"id": "u025", "kind": "prose", "locator": "body:L70-L76", "preview": "[DeepSeek EPLB at commit `d52c72d`](https://github.com/deepseek-ai/EPLB/tree/d52c72d5b2f2fb4c41afbf8eb21366820239913d) takes per-logical-expert load statistics, replicates heavily loaded logical experts, and packs physical experts across co", "sha256": "02ab54c819389e5434998bcacf1c43243f328f7f943ba034ccfec09615c7c6b7"}, {"id": "u026", "kind": "prose", "locator": "body:L78-L82", "preview": "An LMSYS/SGLang study on a 96-H100 deployment reported `1.49x` prefill and `2.54x` decode throughput speedups in its large-scale EPLB ablation. The same study says it used in-distribution data and that production distribution shifts require", "sha256": "16286c58c9ea8cdc2af988c2d8f5376bb6f0dcd832b599ebda14f672268be661"}, {"id": "u027", "kind": "prose", "locator": "body:L86-L91", "preview": "GPU Mode's official postmortem records a temporarily first-place submission that combined a real grouped-GEMM kernel with a timing-harness exploit. The correctness path ran a padded eight-group computation on each of 15 cloned objects. Duri", "sha256": "aff49e9b75d438a2acf47a45c80a8c00117bf9b8a72bbe8c75671b30fa878cd7"}, {"id": "u028", "kind": "prose", "locator": "body:L93-L97", "preview": "This is evidence about evaluator state reuse, not a valid load-balancing performance result. The postmortem identifies `gpu-mode/reference-kernels` PR 104 as the harness response; neither that record nor the pinned FlashInfer MLSys 2026 eva", "sha256": "db377644f5e0f24dfa0a6332d0ba891a7a6b2b62e7ae1eec35c18e9fac9223ed"}, {"id": "u029", "kind": "list-item", "locator": "body:L101-L101", "preview": "- [CUDA thread-block scheduling](https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html)", "sha256": "d5f8f19a50f2f68a004881c534ff8e7c16f392f92660593dab4c93558383aa06"}, {"id": "u030", "kind": "list-item", "locator": "body:L102-L102", "preview": "- [PTX ISA 9.0 CLC request semantics](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel)", "sha256": "4bfd1b3f62e762332ebd2013370bedf813ecfbfd024253b98a104a3e1e4ba66a"}, {"id": "u031", "kind": "list-item", "locator": "body:L103-L103", "preview": "- [CUTLASS 4.5.0 CLC scheduler](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md)", "sha256": "303ee33b3fbba445fb8238b14d3584472c2ffd1d5afb110eb71ad2fdde234d77"}, {"id": "u032", "kind": "list-item", "locator": "body:L104-L104", "preview": "- [DeepGEMM grouped interfaces at `891d57b4`](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md)", "sha256": "694d095fb8d4a47e1e3948889a83f8e8f2d6eecaaebb602be743c9510989ffe8"}, {"id": "u033", "kind": "list-item", "locator": "body:L105-L105", "preview": "- [DeepSeek EPLB at `d52c72d`](https://github.com/deepseek-ai/EPLB/tree/d52c72d5b2f2fb4c41afbf8eb21366820239913d)", "sha256": "745a1db428d252c3685ad9006ee3bccc82c88c3d83639a0df4fd908a336de615"}, {"id": "u034", "kind": "list-item", "locator": "body:L106-L106", "preview": "- [LMSYS/SGLang 96-H100 deployment study](https://www.lmsys.org/blog/2025-05-05-large-scale-ep/)", "sha256": "167e6224c182e20109786dc6943037c6ab8f4286f21811d71e84790250ebc755"}, {"id": "u035", "kind": "list-item", "locator": "body:L107-L107", "preview": "- [GPU Mode reward-hack postmortem](https://www.gpumode.com/news/reward-hacking-nvfp4)", "sha256": "215b30f533c162dba1a59352f6ba3b00f187094045e1690586ad24aa927ee27d"}, {"id": "u036", "kind": "list-item", "locator": "body:L108-L108", "preview": "- [Pinned FlashInfer MLSys 2026 evaluator](https://github.com/flashinfer-ai/flashinfer-bench-starter-kit/blob/75ccd05cafceb0fd1f86be4cd0f2117249463c66/EVALUATION.md)", "sha256": "75a976c5cc6d841615f24e87fc2de683ab22293c488615a4db5c32a69151a24b"}], "confidence_claimed": null, "headings": ["Diagnose the level that is imbalanced", "Grouped-layout choices", "Scheduler comparisons", "Expert placement is a separate system layer", "GPU Mode Problem 4 reward-hack boundary", "Primary references"], "id": "pattern-moe-load-imbalance", "path": "wiki/patterns/moe-load-imbalance.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md", "url": "https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm"}, {"path": "sources/contests/flashinfer-mlsys26/track-a-fused-moe.md", "url": "https://mlsys26.flashinfer.ai/"}, {"path": "sources/blogs/deepgemm.md", "url": "https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f"}, {"path": "sources/docs/cutlass-clc-documentation.md", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}, {"path": "sources/blogs/gpu-mode-reward-hack.md", "url": "https://www.gpumode.com/news/reward-hacking-nvfp4"}], "risk_flags": ["ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["contest-gpumode-p4", "contest-flashinfer-track-a", "blog-deepgemm", "doc-cutlass-clc", "blog-gpu-mode-reward-hack"], "title": "MoE Expert Load Imbalance", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} -{"body_sha256": "0c5294d705b6402ce30d3482e6d7f3bb2cb0ca6bf75e81712347ff1d23959d45", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "A pipeline stall is lost issue opportunity caused by a producer-consumer dependency on the critical path. Low TMA or tensor-core activity, a warp sampled at a barrier wait, or a phase-local utilization drop is only a lead. Expected dependen", "sha256": "da7a72a788c774770d5fdbfdf3a011e6425280d30b33bc03a25a0f39f5c83564"}, {"id": "u002", "kind": "prose", "locator": "body:L5-L5", "preview": "Nsight Compute's Warp State Statistics describes why sampled warps could not issue, while Scheduler Statistics shows whether schedulers had eligible warps and issued instructions. NVIDIA cautions that stalls are not necessarily performance-", "sha256": "7e0728caba67afc2a3981a4c5be636334cc90fa93bf4fc0ee6782c0c878e3126"}, {"id": "u003", "kind": "prose", "locator": "body:L9-L9", "preview": "Do this before tuning around an mbarrier wait:", "sha256": "14ed4da2dccf88e9f5096dc225ad0185ddffa3f32922f4eba71d0c4949f3c428"}, {"id": "u004", "kind": "list-item", "locator": "body:L11-L11", "preview": "1. Identify the exact barrier object, owner, initialized arrival count, initial phase, and storage stage for the wait.", "sha256": "214f90bfaac4e1a67bc424fb4679a4d4914d5f861188655234c0caaa267b4043"}, {"id": "u005", "kind": "list-item", "locator": "body:L12-L12", "preview": "2. For a TMA global-to-shared load, account for the producer's software arrival and expected transaction bytes. `mbarrier.arrive.expect_tx` performs an arrival and adds transaction bytes; the TMA `.mbarrier::complete_tx::bytes` operation de", "sha256": "3e96182dbe690a820cbf9b5e9fdb6b3cc25b08f91eca7cef16719bd9922e117c"}, {"id": "u006", "kind": "list-item", "locator": "body:L13-L13", "preview": "3. Wait for the matching phase of that same object. Phase parity changes when that barrier completes a phase and is reused; \u201cflip after every wait\u201d is not a valid global rule when a thread waits on multiple objects or phases.", "sha256": "fc67e43b74ef2c49970cdfcfcdfbc17e669fe14236db9b9cbb80a129f16f059e"}, {"id": "u007", "kind": "list-item", "locator": "body:L14-L14", "preview": "4. A successful acquire wait supplies the documented visibility for associated prior `cp.async.bulk` work before the consumer reads SMEM.", "sha256": "cad75bba1a6209991e889aebed6a0e54d9a993ac199a3b785434c82630900b3e"}, {"id": "u008", "kind": "list-item", "locator": "body:L15-L15", "preview": "5. Before reusing SMEM operands read by asynchronous MMA, attach completion of the relevant tcgen05 work with `tcgen05.commit` and observe its mbarrier. Issue is not completion.", "sha256": "0471e26fc7edd3ac9914e4d364ca725d6935473ed772fc429bdde5c130e8f362"}, {"id": "u009", "kind": "list-item", "locator": "body:L16-L16", "preview": "6. When asynchronous tcgen05 operations cross a thread handoff, place `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` around the applicable execution-ordering operation. The after fence is not a generic replacemen", "sha256": "a5a5678d754c4afc5251f822e93233a03ffb7e18a06d9f8a9157f3a0686476c6"}, {"id": "u010", "kind": "list-item", "locator": "body:L17-L17", "preview": "7. Verify prologue, steady-state wraparound, short loops, and the producer/consumer tails. A correct steady-state loop can still hang or reuse live storage at a boundary.", "sha256": "44d732e3dec9b913fb7e525f92cb9f90686aa569ff6b9dac92ebae4e0e8b19e2"}, {"id": "u011", "kind": "prose", "locator": "body:L19-L19", "preview": "An extra software arrival is erroneous only relative to the barrier's initialized and phase-specific accounting. Diagnose pending arrivals and transaction bytes separately instead of assuming that TMA hardware performs another arrival.", "sha256": "b0db1b4e9bcf02bdb609d55928eb002804bcdc285274448f73a2424b521cc581"}, {"id": "u012", "kind": "list-item", "locator": "body:L23-L23", "preview": "1. Establish a synchronized end-to-end timing regression on fixed inputs and confirm outputs against a reference. Record GPU, clocks, toolkit, build flags, kernel name, launch shape, and resource use.", "sha256": "c93c68e7cb1a483a6a1d4d21cf5f26631ffbbb1a2a8ab85db6bec467a9cb4798"}, {"id": "u013", "kind": "list-item", "locator": "body:L24-L24", "preview": "2. Collect Speed-of-Light, Scheduler Statistics, Warp State Statistics, and source/SASS correlation with the sections available in the installed Nsight Compute. Account for replay and sampling effects.", "sha256": "a67f70665ee157c5e1909e56a42f789904f46a2629e9a837c8b9573a46cb1eb5"}, {"id": "u014", "kind": "list-item", "locator": "body:L25-L25", "preview": "3. Map the dominant not-issued locations to concrete edges: TMA-full wait, MMA-completion wait, output-buffer reuse, CTA barrier, queue starvation, or pipeline prologue/tail.", "sha256": "9110e18720b271d8f64c2dd21aeb2b3f56a93dac9966380f70d9299ba74a4a67"}, {"id": "u015", "kind": "list-item", "locator": "body:L26-L26", "preview": "4. State a prediction. For example: if TMA readiness is the critical edge, a legal extra operand stage should reduce that wait and runtime; if only the tail is exposed, the steady-state wait distribution should remain largely unchanged.", "sha256": "36934631ace735be26b95ac853c3673c22dce4b1f4969d98234b65b90350f632"}, {"id": "u016", "kind": "list-item", "locator": "body:L27-L27", "preview": "5. Change one variable and repeat identical correctness, warmup, timing, and profiler collection. Reject a cause when its predicted counter and time movement do not occur.", "sha256": "2cd9d472de4c08e8283c6bd52fcbd6704e0be919b520fe671cb542c07a6841e4"}, {"id": "u017", "kind": "table-row", "locator": "body:L31-L31", "preview": "| Controlled change | Plausible target | Required controls and costs | | Vary legal SMEM stage count | Producer cannot stay far enough ahead of MMA | Same tile/math/roles; record SMEM, occupancy, short-loop behavior, and tail |", "sha256": "04c436dbcb263690831932e9dc3de9489d78d1aff38a257a533eb521cdf51549"}, {"id": "u018", "kind": "table-row", "locator": "body:L32-L32", "preview": "| Controlled change | Plausible target | Required controls and costs | | Separate long-lived warp roles | Role handoff or control work is on the issue path | Same stages and tile; record registers, active warps, and per-role idle/backpressu", "sha256": "b5b7ebcf0548afdbddcb877c7fcc70d3352750d277941928b752f71dcd59231e"}, {"id": "u019", "kind": "table-row", "locator": "body:L33-L33", "preview": "| Controlled change | Plausible target | Required controls and costs | | Add a second TMEM output region | Epilogue holds the only accumulator region | Prove disjoint lifetime; record TMEM columns and epilogue/MMA completion waits |", "sha256": "ab79f66f2d1b762e0d24d903673ac799dce4d639794e042dceeaa12daa1f3b07"}, {"id": "u020", "kind": "table-row", "locator": "body:L34-L34", "preview": "| Controlled change | Plausible target | Required controls and costs | | Interleave two query/output tiles | Softmax/correction and MMA have independent ready work | Prove separate state and dependencies; compare one versus two query stages", "sha256": "995291335dcbcf4e24bb3a145bedf98f4fa72eb4f49add2fca9612b80d272803"}, {"id": "u021", "kind": "prose", "locator": "body:L36-L36", "preview": "None is a universal cure. More stages consume storage and can reduce occupancy; specialization adds warps and synchronization; TMEM buffering consumes columns; multi-tile schedules increase live state. A memory-bound roofline classification", "sha256": "214e9852ed70ec2446680cf91d4def45cd5a23b65b1c8c942241a501ce9405b5"}, {"id": "u022", "kind": "prose", "locator": "body:L40-L40", "preview": "Gau Nernst reports the following for one `M=N=K=4096` BF16 GEMM on a Modal B200 using PyTorch 2.9.1 and CUDA 13. Percentages below are computed from the reported 1506.74-TFLOP/s cuBLAS value.", "sha256": "b5438bc9ed3152dfc868c28a134f08b5fad7f494f7c6b85998f51619284bdd40"}, {"id": "u023", "kind": "table-row", "locator": "body:L44-L44", "preview": "| Version | Author's cumulative version label | TFLOP/s | cuBLAS ratio | | v1a | basic tcgen05 + 2D 16-byte TMA | 254.62 | 16.90% |", "sha256": "2ec8ddf8e324506fc9632dec2e2b8bc384ea8deefdb9267c6cdcdc2c2732566d"}, {"id": "u024", "kind": "table-row", "locator": "body:L45-L45", "preview": "| Version | Author's cumulative version label | TFLOP/s | cuBLAS ratio | | v2b | 3D 128-byte TMA | 695.43 | 46.15% |", "sha256": "dbba2419e3054de28eff2b7f1fd83d83d98f18bea3fbea121804656bd24e694c"}, {"id": "u025", "kind": "table-row", "locator": "body:L46-L46", "preview": "| Version | Author's cumulative version label | TFLOP/s | cuBLAS ratio | | v3 | pipelining | 939.61 | 62.36% |", "sha256": "8b5c05c6b3e8641c5d14409e7371999aef50591dc7ded0fb18dd77e6bb1138f1"}, {"id": "u026", "kind": "table-row", "locator": "body:L47-L47", "preview": "| Version | Author's cumulative version label | TFLOP/s | cuBLAS ratio | | v4 | warp specialization | 1208.83 | 80.23% |", "sha256": "12da1a7a1107856224c31730db16b1b6d72ccb37605926b17cd3f1a7d9ec851f"}, {"id": "u027", "kind": "table-row", "locator": "body:L48-L48", "preview": "| Version | Author's cumulative version label | TFLOP/s | cuBLAS ratio | | v5 | 2-SM MMA | 1302.29 | 86.43% |", "sha256": "f849944c51b336479e60eefa90208c5313995e912594c9861c162dfbec203411"}, {"id": "u028", "kind": "table-row", "locator": "body:L49-L49", "preview": "| Version | Author's cumulative version label | TFLOP/s | cuBLAS ratio | | v6 | persistent kernel with static scheduling | 1475.93 | 97.96% |", "sha256": "37b786019302d87416e187fb57c9b75385ecb50a1b91484b1afc0437adc2fac5"}, {"id": "u029", "kind": "prose", "locator": "body:L51-L51", "preview": "The pinned v3 source instantiates two stages, not three. Each row is a cumulative source version rather than an isolated microbenchmark of the named mechanism. The final version uses static scheduling; the author explicitly says Cluster Lau", "sha256": "d2f53b4f0ed98b073c91470bb91db4a9b9c07a2543d56bf6f91af73e4c57c019"}, {"id": "u030", "kind": "list-item", "locator": "body:L55-L55", "preview": "- [Nsight Compute 2025.3 Profiling Guide](https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html)", "sha256": "7d96e056fb2b06ab7298e5192301e839e81cf4e9a2afaf7ec44b408f56a20811"}, {"id": "u031", "kind": "list-item", "locator": "body:L56-L56", "preview": "- [PTX ISA 9.0 mbarrier waits and visibility](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait)", "sha256": "63ce3a13a589f43f07e09bd7a101a72cbdcee3d1b7f3bbd268867bb512aa5ff4"}, {"id": "u032", "kind": "list-item", "locator": "body:L57-L57", "preview": "- [PTX ISA 9.0 tcgen05 execution ordering](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-fence)", "sha256": "2057f748a6cbb130b0db39057b2306f6bb2d2f06a151010176b32a5a3f998ea1"}, {"id": "u033", "kind": "list-item", "locator": "body:L58-L58", "preview": "- [Pinned tutorial v3 source](https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu)", "sha256": "bb64a4f937640f9b1116a335140e9c1f5ac530178c90a2202836bb97f66fdc52"}, {"id": "u034", "kind": "list-item", "locator": "body:L59-L59", "preview": "- [Tutorial progression](https://gau-nernst.github.io/tcgen05/)", "sha256": "e6ec04716b8773935c14b45e6b61195ce3c4704958a654a0b4599834055cb401"}], "confidence_claimed": "verified", "headings": ["Symptom, Not Diagnosis", "Correctness Gate: Audit the Wait Edge", "Causal Diagnosis Workflow", "Source-Reported Tutorial Progression", "Primary References"], "id": "pattern-pipeline-stalls", "path": "wiki/patterns/pipeline-stalls.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/docs/flash-attention-4.md", "url": "https://arxiv.org/abs/2603.05451v1"}, {"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}], "risk_flags": ["ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-tcgen05-tutorial", "doc-flash-attention-4", "doc-nvidia-tuning-guide", "doc-ptx-isa-sm100"], "title": "Pipeline Stalls", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} -{"body_sha256": "6411aba77dd98964d25ec7fa3a3e7f97e70f5400886e930aaa77f6b2072c7621", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "Register pressure is performance-relevant only when compiled register allocation or spills constrain useful scheduling or add material local-memory traffic. \u201cOccupancy below target\u201d is not enough: theoretical occupancy is a residency limit,", "sha256": "35e63dfd7e4f8b07c35d4ba215f12c71e732e0380db9d8319622c21a7bce90d7"}, {"id": "u002", "kind": "prose", "locator": "body:L5-L5", "preview": "For the exact binary and launch, record registers per thread, allocation granularity, spill stores/loads, local-memory traffic, threads and warps per CTA, SMEM, cluster shape, theoretical limiting resource, achieved active/eligible warps, s", "sha256": "926ef4107f387443dda4fc164c195d6657426805e698cae84bd0cca2d227a21a"}, {"id": "u003", "kind": "prose", "locator": "body:L7-L7", "preview": "Common contributors include a resident MMA fragment, an epilogue whose temporaries overlap the mainloop, descriptors/addresses and pipeline state, unrolled loops, and values live across branches. These are hypotheses about compiled liveness", "sha256": "74eb90aed5a121282b7dc1d357f02933e867b1bb16e6e4c53f1f4fa9f324aca1"}, {"id": "u004", "kind": "prose", "locator": "body:L13-L13", "preview": "Move epilogue work after accumulator lifetime when dependencies allow, reduce unnecessary unrolling, or split a long-lived role. Warp specialization exists on Hopper as well as Blackwell; it can shorten one role's live set, but adds role st", "sha256": "956ff6f89523aee899fb12dbab514dbfc87df14051783bc1cbea6f59361c72a7"}, {"id": "u005", "kind": "prose", "locator": "body:L17-L17", "preview": "Compare an uncapped build with selected `-maxrregcount` values. Record whether the requested cap is effective, the occupancy-limiting resource, spill traffic, instructions, and runtime. ABI minima can constrain the cap, another resource can", "sha256": "3a9edd4e72e4bc95e154d7d98e24c1a82b67388a1767ca6ce02ab24aa7927d4e"}, {"id": "u006", "kind": "prose", "locator": "body:L21-L21", "preview": "For Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, that is 128 32-bit registers (512 bytes) per thread for D. This exact fragment arithmetic does not include other live st", "sha256": "092af7b2947a48f0c1be1cccfd9d8da3508467f7456f603f964493b3fb506ccc"}, {"id": "u007", "kind": "prose", "locator": "body:L23-L23", "preview": "On the SM100 tcgen05 path, resident D is in TMEM, organized as 128 lanes by 512 columns of 32-bit cells. That removes the long-lived per-thread WGMMA D fragment, but not all accumulator-related register work: addresses, descriptors, barrier", "sha256": "41d3e8dcb70ed28fc35ae07fb47fbdc6303cf95955dd4577bd87e66b6045887a"}, {"id": "u008", "kind": "prose", "locator": "body:L25-L25", "preview": "The migration must implement collective allocation/address publication, tcgen05 MMA completion, cross-thread ordering where applicable, collective TMEM loads and `tcgen05.wait::ld`, consumer completion, and collective deallocation before ex", "sha256": "87e1d3773fdd20611b000abb10a908e303df0aaa55f40aab0b7dfd02792e6dd8"}, {"id": "u009", "kind": "prose", "locator": "body:L29-L29", "preview": "TMEM loads are asynchronous operations with explicit completion waits. Whether their issue/wait and epilogue work are repaid by lower long-lived register use is an empirical whole-kernel question. Compare the Hopper and Blackwell designs on", "sha256": "357908b2226dfa23e93f2a7fc6ad4bf80d1de89814a113b887d122f19213c26f"}, {"id": "u010", "kind": "prose", "locator": "body:L31-L31", "preview": "Report compiled registers and spills, TMEM columns, SMEM, barriers, occupancy limit, achieved warps, pipeline/scoreboard stalls, correctness, and time. Include shapes where register occupancy changes and shapes where it does not. A reductio", "sha256": "d0e60296c0451a2af20f944162800dc4b92754098cf1276b2bdf855204e8cbbe"}, {"id": "u011", "kind": "list-item", "locator": "body:L35-L35", "preview": "- [Nsight Compute 2025.3 register and occupancy analysis](https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html)", "sha256": "8cc14da20e47f0d5c7a97d9d05e8ec6dc2eb98aa3607cd8ea9764eacea609329"}, {"id": "u012", "kind": "list-item", "locator": "body:L36-L36", "preview": "- [PTX ISA 9.0 WGMMA accumulator fragments](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma)", "sha256": "4341e89ad4d7106f8ad0a8b5962187c0c6d4fdff10a19bdc713dc4c2f55ab1e3"}, {"id": "u013", "kind": "list-item", "locator": "body:L37-L37", "preview": "- [PTX ISA 9.0 Tensor Memory](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory)", "sha256": "aed47f18c514497e50694f89415704f98533229e52ed733c1898d3c9bb3e2747"}, {"id": "u014", "kind": "list-item", "locator": "body:L38-L38", "preview": "- [PTX ISA 9.0 tcgen05 load/wait](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld)", "sha256": "08c6551885ef06a7cbc5e0a5d824bdac60ef2e5086309b1d03ab0db1688c71fd"}, {"id": "u015", "kind": "list-item", "locator": "body:L39-L39", "preview": "- [Pinned SM100 tutorial implementation](https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100)", "sha256": "e650005a038546e8e510ec66cf6726931ba1ef3ae74a509855085fa902f03748"}], "confidence_claimed": "verified", "headings": ["Diagnose a Register-Limited Kernel", "Candidate Tests", "Shorten or separate live ranges", "Test a register cap", "Move resident D from registers to TMEM", "Evaluate the Tradeoff", "Primary References"], "id": "pattern-register-pressure", "path": "wiki/patterns/register-pressure.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}], "risk_flags": ["ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "doc-ptx-isa-sm100"], "title": "Register Pressure and Residency", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} -{"body_sha256": "015951e658cf7ca107772efab40ffc056cd6152f9c5c7864a4d0b59dcc1df455", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "For `T` equal-duration independent tiles and `W` available one-tile workers, with no K decomposition, write:", "sha256": "ae0eaa132780542dd74c3d0be7cdf9ebc63b079b2a026a31927442a4d500d190"}, {"id": "u002", "kind": "code", "locator": "body:L5-L7", "preview": "```text T = qW + r, 0 <= r < W ```", "sha256": "9055386055d6219a3731bea281e8962958fea9a3ccddd596354c89e898f6cd16"}, {"id": "u003", "kind": "prose", "locator": "body:L9-L9", "preview": "There are `q` full waves. If `r>0`, a final partial wave uses `r` workers and has instantaneous worker utilization `r/W`; if `r=0` and `T>0`, the final wave is full. For example, `T=150` and `W=142` gives one full wave plus eight tiles: the", "sha256": "a8ef68c0fcff6dd1e2a1c0bd2abaadce92c16b19b92d43925b41a0d9f0ecce59"}, {"id": "u004", "kind": "prose", "locator": "body:L11-L11", "preview": "This is an analytical model, not a statement that a B200 has 142 SMs or that physical SM count always equals `W`. Block resources can make multiple blocks resident per SM, cluster shape changes scheduling granularity, application policies m", "sha256": "d9e0cf3962e4d30c5a85e8a50f0dd53e70c56ead1c68156e831025ddf65c138e"}, {"id": "u005", "kind": "prose", "locator": "body:L13-L13", "preview": "Under the equal-duration model, the time fraction attributable to at most one partial wave shrinks as the number of full waves grows. There is no universal \u201cbelow four times the SM count\u201d cutoff: the remainder, wave duration, other phases, ", "sha256": "f324cd0832cae3c8f0339ca28879361e2a3ea07764d3318bfdc48a357f3fec53"}, {"id": "u006", "kind": "list-item", "locator": "body:L17-L17", "preview": "1. Record logical tiles/clusters, grid and cluster dimensions, available worker capacity, residency limits, and per-work-item duration.", "sha256": "e795d774faa4fd3b84027ef69d3dbb0f4ac2f61de31b0ad0ae7c902891d22e3f"}, {"id": "u007", "kind": "list-item", "locator": "body:L18-L18", "preview": "2. Predict wave count and the final remainder from the simplified model.", "sha256": "1d8d9be6330b539daf5cecf3a3e9ca5bdc43c8d66dc6dafc1626dcc58ab23c80"}, {"id": "u008", "kind": "list-item", "locator": "body:L19-L19", "preview": "3. Use time-resolved activity or instrumented worker records to locate the underfilled interval at the end, rather than inferring it from aggregate utilization.", "sha256": "171e09c6edcc03571aba5731487e7092cbfe98166edaacb3f751eca20419d90d"}, {"id": "u009", "kind": "list-item", "locator": "body:L20-L20", "preview": "4. Sweep nearby problem sizes or tile shapes. A wave-quantization hypothesis predicts a sawtooth response aligned with changes in the remainder, after controlling total work.", "sha256": "23112755735c4dfdfb4280e3bb343028ef65ce6126d88127f7553182529809e6"}, {"id": "u010", "kind": "list-item", "locator": "body:L21-L21", "preview": "5. Separate unequal tile durations: a long straggler is load imbalance even if the tile-count remainder is zero.", "sha256": "12e6a06a0cf28b05b3c3a675a4db26089ec744d706d3001cb0bd2b7b1e7342a7"}, {"id": "u011", "kind": "prose", "locator": "body:L23-L23", "preview": "CUDA schedules ordinary grid blocks onto available SMs; a static grid-stride loop fixes logical worker indices, not `blockIdx`-to-physical-SM placement. A fixed grid therefore does not mean CUDA lacks dynamic block scheduling.", "sha256": "6452a882b8c5416203c08e6041cef08ba04ccf0849deafc93a15afae865c9512"}, {"id": "u012", "kind": "list-item", "locator": "body:L27-L27", "preview": "- A static persistent loop lets a resident CTA process multiple logical work items. Worker count is selected from problem decomposition, cluster/resource limits, and policy\u2014not necessarily one CTA per physical SM. Finite work still has a fi", "sha256": "9db529aa6ae9bd030c3c8089509d63d8584dd95219b31f6d18be3cdf57767e73"}, {"id": "u013", "kind": "list-item", "locator": "body:L28-L28", "preview": "- Cluster Launch Control lets a selected thread request cancellation of an unspecified not-yet-started block or cluster and process the returned grid ID. CLC redistributes existing IDs; it cannot turn eight remaining independent tiles into ", "sha256": "0a5afba4961508157c3c0154f138d89d14c968938adcce7244fc0c81efbacd1c"}, {"id": "u014", "kind": "list-item", "locator": "body:L29-L29", "preview": "- Raster order and swizzle are coordinate transforms. Test them for locality while holding work acquisition fixed; they do not create work or guarantee balanced durations.", "sha256": "7fd94e42d1d2576ed4e87b200e19e9776b4d787a553e5e40fcef1a118dd2e23a"}, {"id": "u015", "kind": "list-item", "locator": "body:L30-L30", "preview": "- Stream-K or Split-K may create more independent partitions when tile-level work is insufficient. Evaluate reduction traffic, workspace, synchronization, determinism, and numerical effects separately.", "sha256": "dd0d2eea0bfed3c0fd23a77f78dc563c2da37c925225e0718d43ab29b94f49d4"}, {"id": "u016", "kind": "prose", "locator": "body:L32-L32", "preview": "Compare static and CLC-backed persistence with identical math, decomposition, tile/cluster shapes, grid, resources, and timing. Record CLC request results and per-worker work/time distribution. PTX ISA 9.0 requires `sm_100` or higher for CL", "sha256": "fcb099d00bab658c03147f5a9ec638a5500d4389439fad74c26eef326c17a5e8"}, {"id": "u017", "kind": "prose", "locator": "body:L36-L36", "preview": "Gau Nernst's Modal B200 has 148 SMs. In the author's `M=N=K=4096` experiment, v5 reports 1302.29 TFLOP/s (86.43% of cuBLAS) and v6 reports 1475.93 TFLOP/s (97.96%). The v6 change is static persistence plus output-pipeline/epilogue-role chan", "sha256": "6f9b202e053f7d5da7cb68d6ce3e62c4533da60caaa7c16fa6b728a202a33c95"}, {"id": "u018", "kind": "list-item", "locator": "body:L40-L40", "preview": "- [CUDA Programming Guide: block and cluster scheduling](https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html)", "sha256": "502cb7460a63cee49afa6d82044dcbfe0b1a257319a5ceb0c2543173f8b71844"}, {"id": "u019", "kind": "list-item", "locator": "body:L41-L41", "preview": "- [PTX ISA 9.0 CLC](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#clusterlaunchcontrol-try-cancel)", "sha256": "0de2d63b4dc4111728ebc15efcb7962402b936e82599007da061f78ba8f4057b"}, {"id": "u020", "kind": "list-item", "locator": "body:L42-L42", "preview": "- [CUTLASS 4.5.0 CLC scheduling](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md)", "sha256": "f82b94cccb8b9883e584fa5ec6b7e602e2b7c3247932e262a7a61b54773d925d"}, {"id": "u021", "kind": "list-item", "locator": "body:L43-L43", "preview": "- [Pinned tutorial v6 source](https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu)", "sha256": "d47967878d57c59a54264abe233d783fe466469b45b79870f89f24ba6c73531a"}, {"id": "u022", "kind": "list-item", "locator": "body:L44-L44", "preview": "- [Tutorial result progression](https://gau-nernst.github.io/tcgen05/)", "sha256": "797b16f54ff8d25ba34fefe90153d0a2e58a3efa7414f3594476c0b133a26bad"}], "confidence_claimed": "verified", "headings": ["Exact Simplified Model", "Confirm a Last-Wave Cause", "Scheduler and Decomposition Choices", "Tutorial Scope", "Primary References"], "id": "pattern-tail-effect", "path": "wiki/patterns/tail-effect.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/docs/cutlass-clc-documentation.md", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}], "risk_flags": ["code", "ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "doc-cutlass-clc", "doc-ptx-isa-sm100"], "title": "Tail Effect \u2014 Last-Wave Underutilization", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "d29c5d578d0a60f485f42062a7af45aa75bbf06fee511dc6508c40b70ba753a3", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L8", "preview": "Do not infer one expert per physical SM from a grouped-GEMM trace. CUDA assigns thread blocks to available SMs, while grouped kernels may split an expert into multiple output tiles or let one resident worker process several logical tiles. M", "sha256": "6a4556c295e1fe46a216bb02fa3b9c18a4dfd635ed0ea95b40ab3105782a90aa"}, {"id": "u002", "kind": "list-item", "locator": "body:L10-L10", "preview": "- routed tokens per expert for the exact batch and prefill/decode phase;", "sha256": "7a869c75edb2661a6e1d7146b9df98b243d7611e31d063edc71f3f5c2948929c"}, {"id": "u003", "kind": "list-item", "locator": "body:L11-L11", "preview": "- valid rows and output-tile counts per expert;", "sha256": "21af8e58169651d75b62c2099c735e98ad0f179b6f192d03734b1b9f5e5d3541"}, {"id": "u004", "kind": "list-item", "locator": "body:L12-L12", "preview": "- completed tiles and elapsed work per logical worker and, when instrumented,", "sha256": "abe0c2bb6e4ce66df866b22cdacc7b3459bf91742abf5a10d70a7c157ba3e856"}, {"id": "u005", "kind": "prose", "locator": "body:L13-L13", "preview": "per SM;", "sha256": "a0e9294558ab12243cb5c87a97a4cbaf4966ab84a68cb44017852f8d95467519"}, {"id": "u006", "kind": "list-item", "locator": "body:L14-L14", "preview": "- dispatch, grouped-GEMM, combine, and end-to-end times per expert-parallel", "sha256": "642242588794a89bcb0b8bb3d69a8cf589838a4273d06d3bc63b3de5db8ec3c7"}, {"id": "u007", "kind": "prose", "locator": "body:L15-L15", "preview": "rank.", "sha256": "a9d4d07a50c665beea0d85f220de764cca3663b2c2af5d8c790a1d2970ef5cbc"}, {"id": "u008", "kind": "prose", "locator": "body:L17-L20", "preview": "Small expert segments and partial output tiles can expose too little parallel work or leave lanes predicated out. The effect depends on the kernel tile, other GEMM dimensions, resident-worker count, and competing implementation; there is no", "sha256": "10f047c682158b00cdf5835272cf7cbe2cc8c09f449d19f70966620f49ff4efc"}, {"id": "u009", "kind": "prose", "locator": "body:L22-L26", "preview": "Uniform expected routing, a larger batch, or an auxiliary balancing loss may change token-count skew, but none proves balanced tile counts, worker durations, communication, or runtime. Treat imbalance as absent only when the measured distri", "sha256": "4aea38fd66dea81359addca88f480b75cc6a8cd021b546e0728a216da167bc01"}, {"id": "u010", "kind": "prose", "locator": "body:L30-L32", "preview": "At pinned DeepGEMM commit [`891d57b4`](https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f), the three grouped interfaces have different axes and metadata:", "sha256": "db4321873118f047cf255c2898cd512bca03ae25073f932d1f4f5aa2696aeccd"}, {"id": "u011", "kind": "table-row", "locator": "body:L36-L36", "preview": "| Layout | Pinned contract | Relevant control | | M-grouped contiguous | Pack `A` and `D` along variable M with N/K fixed; identify groups by per-row expert IDs or per-group prefix-sum ends; segments are M-block aligned | Record valid rows,", "sha256": "d2455c5057ca34282247c70175297f8294562ea84975b8dbfd1d42af7117fccd"}, {"id": "u012", "kind": "table-row", "locator": "body:L37-L37", "preview": "| Layout | Pinned contract | Relevant control | | M-grouped masked | Allocate `[G, M_max, ...]`, pass one valid-M count per group, and compute valid portions; the fixed allocation is documented for a CUDA-graph decode case | Separate alloca", "sha256": "4992cd4a0d340ae7c7c8a103d0755063816b02ca84de49d6685fa2f9d5d22c83"}, {"id": "u013", "kind": "table-row", "locator": "body:L38-L38", "preview": "| Layout | Pinned contract | Relevant control | | K-grouped contiguous | Pack variable K with M/N fixed for MoE weight backward | Do not use its K-axis contract to describe forward token imbalance |", "sha256": "844315bc56884144631be8c64520b6b669112454794b93acd1837452b7777cb0"}, {"id": "u014", "kind": "prose", "locator": "body:L40-L42", "preview": "A fixed `M_max` allocation therefore does not establish that all padding rows are computed. Any residual edge-tile or predication cost needs generated-code and profile evidence for the selected kernel.", "sha256": "e1ad215b1834aea24c4c71f10f9dd49adb76027b5707f91aa558f23fed0775eb"}, {"id": "u015", "kind": "prose", "locator": "body:L46-L46", "preview": "Keep persistence, work acquisition, and decomposition as separate variables:", "sha256": "72b5c8bd73eb8523601df6dc917588754a8f96308e1b63a0634cf317643b4636"}, {"id": "u016", "kind": "list-item", "locator": "body:L48-L48", "preview": "1. A static persistent worker can advance through a deterministic logical-work", "sha256": "2ebeac42de4c86f459f32944d820f960af294bb7e3e90652bce81f1750d37a47"}, {"id": "u017", "kind": "prose", "locator": "body:L49-L50", "preview": "sequence. CUDA still decides where its block runs; the rule is not a precomputed tile-to-physical-SM map.", "sha256": "2141d38662b0abc377581ebce726e1b0f19cb22664d04f848c469c8824eb4a2d"}, {"id": "u018", "kind": "list-item", "locator": "body:L51-L51", "preview": "2. In a CLC-backed scheduler, a selected thread requests cancellation of an", "sha256": "ae1a3d61b865fd35104df9841d44658b7fbf250e361ae47cd15e04acd0767ab9"}, {"id": "u019", "kind": "prose", "locator": "body:L52-L54", "preview": "unspecified block or cluster that has not launched. A successful response returns that existing grid coordinate; a request can fail. CLC does not create tiles or make the fastest SM autonomously steal arbitrary work.", "sha256": "44666cd2a7aa439b477e3be4e332e7f1801285aeaa09555f77dc3ef6df41bd7a"}, {"id": "u020", "kind": "list-item", "locator": "body:L55-L55", "preview": "3. The initial `blockIdx` in CUTLASS's pinned CLC scheduler is static and later", "sha256": "ad48157736775ac54b4f798356d96d521ff0b5736bb3d5d8f9f2d83c18ca8b36"}, {"id": "u021", "kind": "prose", "locator": "body:L56-L58", "preview": "requests are asynchronous, pipelined, and cluster-granular. There is no universal per-tile latency bound: compare request activity and end-to-end time against the matched static scheduler.", "sha256": "848689501e7d30e823c98dde175448b2942777d47358b178c1b0b59592536b7d"}, {"id": "u022", "kind": "list-item", "locator": "body:L59-L59", "preview": "4. Splitting a large expert or K range may expose more independent work, but it", "sha256": "043c212f70796c400b7d34d0f4fcb011ecad2c412deb8ce351311e96cf1b18b2"}, {"id": "u023", "kind": "prose", "locator": "body:L60-L61", "preview": "can add partial-result and reduction costs. Hold math, tile shape, cluster shape, launch resources, and output semantics constant in the comparison.", "sha256": "ae096ac1401562950161c28a56d6eb66835ed10ab8b13f6892d16321db4ed587"}, {"id": "u024", "kind": "prose", "locator": "body:L63-L66", "preview": "PTX ISA 9.0 specifies CLC for `sm_100` or higher and explicitly includes SM120-family targets for the multicast form. CUTLASS 4.5.0's documented `PersistentTileSchedulerSm100` integration is a narrower library example, not the complete ISA ", "sha256": "ccd99bd11c24e601ec78e2e879751d77c744e1a7abbc3b1e17ee3194f421e15c"}, {"id": "u025", "kind": "prose", "locator": "body:L70-L76", "preview": "[DeepSeek EPLB at commit `d52c72d`](https://github.com/deepseek-ai/EPLB/tree/d52c72d5b2f2fb4c41afbf8eb21366820239913d) takes per-logical-expert load statistics, replicates heavily loaded logical experts, and packs physical experts across co", "sha256": "02ab54c819389e5434998bcacf1c43243f328f7f943ba034ccfec09615c7c6b7"}, {"id": "u026", "kind": "prose", "locator": "body:L78-L82", "preview": "An LMSYS/SGLang study on a 96-H100 deployment reported `1.49x` prefill and `2.54x` decode throughput speedups in its large-scale EPLB ablation. The same study says it used in-distribution data and that production distribution shifts require", "sha256": "16286c58c9ea8cdc2af988c2d8f5376bb6f0dcd832b599ebda14f672268be661"}, {"id": "u027", "kind": "prose", "locator": "body:L86-L91", "preview": "GPU Mode's official postmortem records a temporarily first-place submission that combined a real grouped-GEMM kernel with a timing-harness exploit. The correctness path ran a padded eight-group computation on each of 15 cloned objects. Duri", "sha256": "aff49e9b75d438a2acf47a45c80a8c00117bf9b8a72bbe8c75671b30fa878cd7"}, {"id": "u028", "kind": "prose", "locator": "body:L93-L97", "preview": "This is evidence about evaluator state reuse, not a valid load-balancing performance result. The postmortem identifies `gpu-mode/reference-kernels` PR 104 as the harness response; neither that record nor the pinned FlashInfer MLSys 2026 eva", "sha256": "db377644f5e0f24dfa0a6332d0ba891a7a6b2b62e7ae1eec35c18e9fac9223ed"}, {"id": "u029", "kind": "list-item", "locator": "body:L101-L101", "preview": "- [CUDA thread-block scheduling](https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html)", "sha256": "d5f8f19a50f2f68a004881c534ff8e7c16f392f92660593dab4c93558383aa06"}, {"id": "u030", "kind": "list-item", "locator": "body:L102-L102", "preview": "- [PTX ISA 9.0 CLC request semantics](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel)", "sha256": "2b78a4f8d401e6a568772100107c9ea3ec774775a4c63c9e43fa2e18c0f2c38c"}, {"id": "u031", "kind": "list-item", "locator": "body:L103-L103", "preview": "- [CUTLASS 4.5.0 CLC scheduler](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md)", "sha256": "303ee33b3fbba445fb8238b14d3584472c2ffd1d5afb110eb71ad2fdde234d77"}, {"id": "u032", "kind": "list-item", "locator": "body:L104-L104", "preview": "- [DeepGEMM grouped interfaces at `891d57b4`](https://github.com/deepseek-ai/DeepGEMM/blob/891d57b4db1071624b5c8fa0d1e51cb317fa709f/README.md)", "sha256": "694d095fb8d4a47e1e3948889a83f8e8f2d6eecaaebb602be743c9510989ffe8"}, {"id": "u033", "kind": "list-item", "locator": "body:L105-L105", "preview": "- [DeepSeek EPLB at `d52c72d`](https://github.com/deepseek-ai/EPLB/tree/d52c72d5b2f2fb4c41afbf8eb21366820239913d)", "sha256": "745a1db428d252c3685ad9006ee3bccc82c88c3d83639a0df4fd908a336de615"}, {"id": "u034", "kind": "list-item", "locator": "body:L106-L106", "preview": "- [LMSYS/SGLang 96-H100 deployment study](https://www.lmsys.org/blog/2025-05-05-large-scale-ep/)", "sha256": "167e6224c182e20109786dc6943037c6ab8f4286f21811d71e84790250ebc755"}, {"id": "u035", "kind": "list-item", "locator": "body:L107-L107", "preview": "- [GPU Mode reward-hack postmortem](https://www.gpumode.com/news/reward-hacking-nvfp4)", "sha256": "215b30f533c162dba1a59352f6ba3b00f187094045e1690586ad24aa927ee27d"}, {"id": "u036", "kind": "list-item", "locator": "body:L108-L108", "preview": "- [Pinned FlashInfer MLSys 2026 evaluator](https://github.com/flashinfer-ai/flashinfer-bench-starter-kit/blob/75ccd05cafceb0fd1f86be4cd0f2117249463c66/EVALUATION.md)", "sha256": "75a976c5cc6d841615f24e87fc2de683ab22293c488615a4db5c32a69151a24b"}], "confidence_claimed": null, "headings": ["Diagnose the level that is imbalanced", "Grouped-layout choices", "Scheduler comparisons", "Expert placement is a separate system layer", "GPU Mode Problem 4 reward-hack boundary", "Primary references"], "id": "pattern-moe-load-imbalance", "path": "wiki/patterns/moe-load-imbalance.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/contests/gpu-mode-nvfp4/problem-4-grouped-gemm.md", "url": "https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_group_gemm"}, {"path": "sources/contests/flashinfer-mlsys26/track-a-fused-moe.md", "url": "https://mlsys26.flashinfer.ai/"}, {"path": "sources/blogs/deepgemm.md", "url": "https://github.com/deepseek-ai/DeepGEMM/tree/891d57b4db1071624b5c8fa0d1e51cb317fa709f"}, {"path": "sources/docs/cutlass-clc-documentation.md", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}, {"path": "sources/blogs/gpu-mode-reward-hack.md", "url": "https://www.gpumode.com/news/reward-hacking-nvfp4"}], "risk_flags": ["ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["contest-gpumode-p4", "contest-flashinfer-track-a", "blog-deepgemm", "doc-cutlass-clc", "blog-gpu-mode-reward-hack"], "title": "MoE Expert Load Imbalance", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "b9fab3c58abf51a52ffbb2d02b23eb23ea15a7bfbb0124cce1080b434a097d7b", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "A pipeline stall is lost issue opportunity caused by a producer-consumer dependency on the critical path. Low TMA or tensor-core activity, a warp sampled at a barrier wait, or a phase-local utilization drop is only a lead. Expected dependen", "sha256": "da7a72a788c774770d5fdbfdf3a011e6425280d30b33bc03a25a0f39f5c83564"}, {"id": "u002", "kind": "prose", "locator": "body:L5-L5", "preview": "Nsight Compute's Warp State Statistics describes why sampled warps could not issue, while Scheduler Statistics shows whether schedulers had eligible warps and issued instructions. NVIDIA cautions that stalls are not necessarily performance-", "sha256": "7e0728caba67afc2a3981a4c5be636334cc90fa93bf4fc0ee6782c0c878e3126"}, {"id": "u003", "kind": "prose", "locator": "body:L9-L9", "preview": "Do this before tuning around an mbarrier wait:", "sha256": "14ed4da2dccf88e9f5096dc225ad0185ddffa3f32922f4eba71d0c4949f3c428"}, {"id": "u004", "kind": "list-item", "locator": "body:L11-L11", "preview": "1. Identify the exact barrier object, owner, initialized arrival count, initial phase, and storage stage for the wait.", "sha256": "214f90bfaac4e1a67bc424fb4679a4d4914d5f861188655234c0caaa267b4043"}, {"id": "u005", "kind": "list-item", "locator": "body:L12-L12", "preview": "2. For a TMA global-to-shared load, account for the producer's software arrival and expected transaction bytes. `mbarrier.arrive.expect_tx` performs an arrival and adds transaction bytes; the TMA `.mbarrier::complete_tx::bytes` operation de", "sha256": "3e96182dbe690a820cbf9b5e9fdb6b3cc25b08f91eca7cef16719bd9922e117c"}, {"id": "u006", "kind": "list-item", "locator": "body:L13-L13", "preview": "3. Wait for the matching phase of that same object. Phase parity changes when that barrier completes a phase and is reused; \u201cflip after every wait\u201d is not a valid global rule when a thread waits on multiple objects or phases.", "sha256": "fc67e43b74ef2c49970cdfcfcdfbc17e669fe14236db9b9cbb80a129f16f059e"}, {"id": "u007", "kind": "list-item", "locator": "body:L14-L14", "preview": "4. A successful acquire wait supplies the documented visibility for associated prior `cp.async.bulk` work before the consumer reads SMEM.", "sha256": "cad75bba1a6209991e889aebed6a0e54d9a993ac199a3b785434c82630900b3e"}, {"id": "u008", "kind": "list-item", "locator": "body:L15-L15", "preview": "5. Before reusing SMEM operands read by asynchronous MMA, attach completion of the relevant tcgen05 work with `tcgen05.commit` and observe its mbarrier. Issue is not completion.", "sha256": "0471e26fc7edd3ac9914e4d364ca725d6935473ed772fc429bdde5c130e8f362"}, {"id": "u009", "kind": "list-item", "locator": "body:L16-L16", "preview": "6. When asynchronous tcgen05 operations cross a thread handoff, place `tcgen05.fence::before_thread_sync` and `tcgen05.fence::after_thread_sync` around the applicable execution-ordering operation. The after fence is not a generic replacemen", "sha256": "a5a5678d754c4afc5251f822e93233a03ffb7e18a06d9f8a9157f3a0686476c6"}, {"id": "u010", "kind": "list-item", "locator": "body:L17-L17", "preview": "7. Verify prologue, steady-state wraparound, short loops, and the producer/consumer tails. A correct steady-state loop can still hang or reuse live storage at a boundary.", "sha256": "44d732e3dec9b913fb7e525f92cb9f90686aa569ff6b9dac92ebae4e0e8b19e2"}, {"id": "u011", "kind": "prose", "locator": "body:L19-L19", "preview": "An extra software arrival is erroneous only relative to the barrier's initialized and phase-specific accounting. Diagnose pending arrivals and transaction bytes separately instead of assuming that TMA hardware performs another arrival.", "sha256": "b0db1b4e9bcf02bdb609d55928eb002804bcdc285274448f73a2424b521cc581"}, {"id": "u012", "kind": "list-item", "locator": "body:L23-L23", "preview": "1. Establish a synchronized end-to-end timing regression on fixed inputs and confirm outputs against a reference. Record GPU, clocks, toolkit, build flags, kernel name, launch shape, and resource use.", "sha256": "c93c68e7cb1a483a6a1d4d21cf5f26631ffbbb1a2a8ab85db6bec467a9cb4798"}, {"id": "u013", "kind": "list-item", "locator": "body:L24-L24", "preview": "2. Collect Speed-of-Light, Scheduler Statistics, Warp State Statistics, and source/SASS correlation with the sections available in the installed Nsight Compute. Account for replay and sampling effects.", "sha256": "a67f70665ee157c5e1909e56a42f789904f46a2629e9a837c8b9573a46cb1eb5"}, {"id": "u014", "kind": "list-item", "locator": "body:L25-L25", "preview": "3. Map the dominant not-issued locations to concrete edges: TMA-full wait, MMA-completion wait, output-buffer reuse, CTA barrier, queue starvation, or pipeline prologue/tail.", "sha256": "9110e18720b271d8f64c2dd21aeb2b3f56a93dac9966380f70d9299ba74a4a67"}, {"id": "u015", "kind": "list-item", "locator": "body:L26-L26", "preview": "4. State a prediction. For example: if TMA readiness is the critical edge, a legal extra operand stage should reduce that wait and runtime; if only the tail is exposed, the steady-state wait distribution should remain largely unchanged.", "sha256": "36934631ace735be26b95ac853c3673c22dce4b1f4969d98234b65b90350f632"}, {"id": "u016", "kind": "list-item", "locator": "body:L27-L27", "preview": "5. Change one variable and repeat identical correctness, warmup, timing, and profiler collection. Reject a cause when its predicted counter and time movement do not occur.", "sha256": "2cd9d472de4c08e8283c6bd52fcbd6704e0be919b520fe671cb542c07a6841e4"}, {"id": "u017", "kind": "table-row", "locator": "body:L31-L31", "preview": "| Controlled change | Plausible target | Required controls and costs | | Vary legal SMEM stage count | Producer cannot stay far enough ahead of MMA | Same tile/math/roles; record SMEM, occupancy, short-loop behavior, and tail |", "sha256": "04c436dbcb263690831932e9dc3de9489d78d1aff38a257a533eb521cdf51549"}, {"id": "u018", "kind": "table-row", "locator": "body:L32-L32", "preview": "| Controlled change | Plausible target | Required controls and costs | | Separate long-lived warp roles | Role handoff or control work is on the issue path | Same stages and tile; record registers, active warps, and per-role idle/backpressu", "sha256": "b5b7ebcf0548afdbddcb877c7fcc70d3352750d277941928b752f71dcd59231e"}, {"id": "u019", "kind": "table-row", "locator": "body:L33-L33", "preview": "| Controlled change | Plausible target | Required controls and costs | | Add a second TMEM output region | Epilogue holds the only accumulator region | Prove disjoint lifetime; record TMEM columns and epilogue/MMA completion waits |", "sha256": "ab79f66f2d1b762e0d24d903673ac799dce4d639794e042dceeaa12daa1f3b07"}, {"id": "u020", "kind": "table-row", "locator": "body:L34-L34", "preview": "| Controlled change | Plausible target | Required controls and costs | | Interleave two query/output tiles | Softmax/correction and MMA have independent ready work | Prove separate state and dependencies; compare one versus two query stages", "sha256": "995291335dcbcf4e24bb3a145bedf98f4fa72eb4f49add2fca9612b80d272803"}, {"id": "u021", "kind": "prose", "locator": "body:L36-L36", "preview": "None is a universal cure. More stages consume storage and can reduce occupancy; specialization adds warps and synchronization; TMEM buffering consumes columns; multi-tile schedules increase live state. A memory-bound roofline classification", "sha256": "214e9852ed70ec2446680cf91d4def45cd5a23b65b1c8c942241a501ce9405b5"}, {"id": "u022", "kind": "prose", "locator": "body:L40-L40", "preview": "Gau Nernst reports the following for one `M=N=K=4096` BF16 GEMM on a Modal B200 using PyTorch 2.9.1 and CUDA 13. Percentages below are computed from the reported 1506.74-TFLOP/s cuBLAS value.", "sha256": "b5438bc9ed3152dfc868c28a134f08b5fad7f494f7c6b85998f51619284bdd40"}, {"id": "u023", "kind": "table-row", "locator": "body:L44-L44", "preview": "| Version | Author's cumulative version label | TFLOP/s | cuBLAS ratio | | v1a | basic tcgen05 + 2D 16-byte TMA | 254.62 | 16.90% |", "sha256": "2ec8ddf8e324506fc9632dec2e2b8bc384ea8deefdb9267c6cdcdc2c2732566d"}, {"id": "u024", "kind": "table-row", "locator": "body:L45-L45", "preview": "| Version | Author's cumulative version label | TFLOP/s | cuBLAS ratio | | v2b | 3D 128-byte TMA | 695.43 | 46.15% |", "sha256": "dbba2419e3054de28eff2b7f1fd83d83d98f18bea3fbea121804656bd24e694c"}, {"id": "u025", "kind": "table-row", "locator": "body:L46-L46", "preview": "| Version | Author's cumulative version label | TFLOP/s | cuBLAS ratio | | v3 | pipelining | 939.61 | 62.36% |", "sha256": "8b5c05c6b3e8641c5d14409e7371999aef50591dc7ded0fb18dd77e6bb1138f1"}, {"id": "u026", "kind": "table-row", "locator": "body:L47-L47", "preview": "| Version | Author's cumulative version label | TFLOP/s | cuBLAS ratio | | v4 | warp specialization | 1208.83 | 80.23% |", "sha256": "12da1a7a1107856224c31730db16b1b6d72ccb37605926b17cd3f1a7d9ec851f"}, {"id": "u027", "kind": "table-row", "locator": "body:L48-L48", "preview": "| Version | Author's cumulative version label | TFLOP/s | cuBLAS ratio | | v5 | 2-SM MMA | 1302.29 | 86.43% |", "sha256": "f849944c51b336479e60eefa90208c5313995e912594c9861c162dfbec203411"}, {"id": "u028", "kind": "table-row", "locator": "body:L49-L49", "preview": "| Version | Author's cumulative version label | TFLOP/s | cuBLAS ratio | | v6 | persistent kernel with static scheduling | 1475.93 | 97.96% |", "sha256": "37b786019302d87416e187fb57c9b75385ecb50a1b91484b1afc0437adc2fac5"}, {"id": "u029", "kind": "prose", "locator": "body:L51-L51", "preview": "The pinned v3 source instantiates two stages, not three. Each row is a cumulative source version rather than an isolated microbenchmark of the named mechanism. The final version uses static scheduling; the author explicitly says Cluster Lau", "sha256": "d2f53b4f0ed98b073c91470bb91db4a9b9c07a2543d56bf6f91af73e4c57c019"}, {"id": "u030", "kind": "list-item", "locator": "body:L55-L55", "preview": "- [Nsight Compute 2025.3 Profiling Guide](https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html)", "sha256": "7d96e056fb2b06ab7298e5192301e839e81cf4e9a2afaf7ec44b408f56a20811"}, {"id": "u031", "kind": "list-item", "locator": "body:L56-L56", "preview": "- [PTX ISA 9.0 mbarrier waits and visibility](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait)", "sha256": "63ce3a13a589f43f07e09bd7a101a72cbdcee3d1b7f3bbd268867bb512aa5ff4"}, {"id": "u032", "kind": "list-item", "locator": "body:L57-L57", "preview": "- [PTX ISA 9.0 tcgen05 execution ordering](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-special-sync-operations-fence)", "sha256": "3dbb17b7d8c4b5ee73be55fb5f9cecac69a08367255d64db1aa4b277f9d9bce6"}, {"id": "u033", "kind": "list-item", "locator": "body:L58-L58", "preview": "- [Pinned tutorial v3 source](https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v3.cu)", "sha256": "bb64a4f937640f9b1116a335140e9c1f5ac530178c90a2202836bb97f66fdc52"}, {"id": "u034", "kind": "list-item", "locator": "body:L59-L59", "preview": "- [Tutorial progression](https://gau-nernst.github.io/tcgen05/)", "sha256": "e6ec04716b8773935c14b45e6b61195ce3c4704958a654a0b4599834055cb401"}], "confidence_claimed": "verified", "headings": ["Symptom, Not Diagnosis", "Correctness Gate: Audit the Wait Edge", "Causal Diagnosis Workflow", "Source-Reported Tutorial Progression", "Primary References"], "id": "pattern-pipeline-stalls", "path": "wiki/patterns/pipeline-stalls.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/docs/flash-attention-4.md", "url": "https://arxiv.org/abs/2603.05451v1"}, {"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}], "risk_flags": ["ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-tcgen05-tutorial", "doc-flash-attention-4", "doc-nvidia-tuning-guide", "doc-ptx-isa-sm100"], "title": "Pipeline Stalls", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "428b39b5988aec925528b5c6040a0e6040e81353f79b041f847e114003ccc23a", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "Register pressure is performance-relevant only when compiled register allocation or spills constrain useful scheduling or add material local-memory traffic. \u201cOccupancy below target\u201d is not enough: theoretical occupancy is a residency limit,", "sha256": "35e63dfd7e4f8b07c35d4ba215f12c71e732e0380db9d8319622c21a7bce90d7"}, {"id": "u002", "kind": "prose", "locator": "body:L5-L5", "preview": "For the exact binary and launch, record registers per thread, allocation granularity, spill stores/loads, local-memory traffic, threads and warps per CTA, SMEM, cluster shape, theoretical limiting resource, achieved active/eligible warps, s", "sha256": "926ef4107f387443dda4fc164c195d6657426805e698cae84bd0cca2d227a21a"}, {"id": "u003", "kind": "prose", "locator": "body:L7-L7", "preview": "Common contributors include a resident MMA fragment, an epilogue whose temporaries overlap the mainloop, descriptors/addresses and pipeline state, unrolled loops, and values live across branches. These are hypotheses about compiled liveness", "sha256": "74eb90aed5a121282b7dc1d357f02933e867b1bb16e6e4c53f1f4fa9f324aca1"}, {"id": "u004", "kind": "prose", "locator": "body:L13-L13", "preview": "Move epilogue work after accumulator lifetime when dependencies allow, reduce unnecessary unrolling, or split a long-lived role. Warp specialization exists on Hopper as well as Blackwell; it can shorten one role's live set, but adds role st", "sha256": "956ff6f89523aee899fb12dbab514dbfc87df14051783bc1cbea6f59361c72a7"}, {"id": "u005", "kind": "prose", "locator": "body:L17-L17", "preview": "Compare an uncapped build with selected `-maxrregcount` values. Record whether the requested cap is effective, the occupancy-limiting resource, spill traffic, instructions, and runtime. ABI minima can constrain the cap, another resource can", "sha256": "3a9edd4e72e4bc95e154d7d98e24c1a82b67388a1767ca6ce02ab24aa7927d4e"}, {"id": "u006", "kind": "prose", "locator": "body:L21-L21", "preview": "For Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, that is 128 32-bit registers (512 bytes) per thread for D. This exact fragment arithmetic does not include other live st", "sha256": "092af7b2947a48f0c1be1cccfd9d8da3508467f7456f603f964493b3fb506ccc"}, {"id": "u007", "kind": "prose", "locator": "body:L23-L23", "preview": "On the SM100 tcgen05 path, resident D is in TMEM, organized as 128 lanes by 512 columns of 32-bit cells. That removes the long-lived per-thread WGMMA D fragment, but not all accumulator-related register work: addresses, descriptors, barrier", "sha256": "41d3e8dcb70ed28fc35ae07fb47fbdc6303cf95955dd4577bd87e66b6045887a"}, {"id": "u008", "kind": "prose", "locator": "body:L25-L25", "preview": "The migration must implement collective allocation/address publication, tcgen05 MMA completion, cross-thread ordering where applicable, collective TMEM loads and `tcgen05.wait::ld`, consumer completion, and collective deallocation before ex", "sha256": "87e1d3773fdd20611b000abb10a908e303df0aaa55f40aab0b7dfd02792e6dd8"}, {"id": "u009", "kind": "prose", "locator": "body:L29-L29", "preview": "TMEM loads are asynchronous operations with explicit completion waits. Whether their issue/wait and epilogue work are repaid by lower long-lived register use is an empirical whole-kernel question. Compare the Hopper and Blackwell designs on", "sha256": "357908b2226dfa23e93f2a7fc6ad4bf80d1de89814a113b887d122f19213c26f"}, {"id": "u010", "kind": "prose", "locator": "body:L31-L31", "preview": "Report compiled registers and spills, TMEM columns, SMEM, barriers, occupancy limit, achieved warps, pipeline/scoreboard stalls, correctness, and time. Include shapes where register occupancy changes and shapes where it does not. A reductio", "sha256": "d0e60296c0451a2af20f944162800dc4b92754098cf1276b2bdf855204e8cbbe"}, {"id": "u011", "kind": "list-item", "locator": "body:L35-L35", "preview": "- [Nsight Compute 2025.3 register and occupancy analysis](https://docs.nvidia.com/nsight-compute/2025.3/ProfilingGuide/index.html)", "sha256": "8cc14da20e47f0d5c7a97d9d05e8ec6dc2eb98aa3607cd8ea9764eacea609329"}, {"id": "u012", "kind": "list-item", "locator": "body:L36-L36", "preview": "- [PTX ISA 9.0 WGMMA accumulator fragments](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-mma)", "sha256": "4341e89ad4d7106f8ad0a8b5962187c0c6d4fdff10a19bdc713dc4c2f55ab1e3"}, {"id": "u013", "kind": "list-item", "locator": "body:L37-L37", "preview": "- [PTX ISA 9.0 Tensor Memory](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory)", "sha256": "aed47f18c514497e50694f89415704f98533229e52ed733c1898d3c9bb3e2747"}, {"id": "u014", "kind": "list-item", "locator": "body:L38-L38", "preview": "- [PTX ISA 9.0 tcgen05 load/wait](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tcgen05-instructions-tcgen05-ld)", "sha256": "2be7a82b68af375a864c76e4ecd8a3418524220f672ea55778b57af67045fd31"}, {"id": "u015", "kind": "list-item", "locator": "body:L39-L39", "preview": "- [Pinned SM100 tutorial implementation](https://github.com/gau-nernst/learn-cuda/tree/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100)", "sha256": "e650005a038546e8e510ec66cf6726931ba1ef3ae74a509855085fa902f03748"}], "confidence_claimed": "verified", "headings": ["Diagnose a Register-Limited Kernel", "Candidate Tests", "Shorten or separate live ranges", "Test a register cap", "Move resident D from registers to TMEM", "Evaluate the Tradeoff", "Primary References"], "id": "pattern-register-pressure", "path": "wiki/patterns/register-pressure.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}], "risk_flags": ["ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "doc-ptx-isa-sm100"], "title": "Register Pressure and Residency", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "60ee80b065d1bed883a649dc028ff0ee4a3f54e1dd896f1abdb5aadc6e0e17b2", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "For `T` equal-duration independent tiles and `W` available one-tile workers, with no K decomposition, write:", "sha256": "ae0eaa132780542dd74c3d0be7cdf9ebc63b079b2a026a31927442a4d500d190"}, {"id": "u002", "kind": "code", "locator": "body:L5-L7", "preview": "```text T = qW + r, 0 <= r < W ```", "sha256": "9055386055d6219a3731bea281e8962958fea9a3ccddd596354c89e898f6cd16"}, {"id": "u003", "kind": "prose", "locator": "body:L9-L9", "preview": "There are `q` full waves. If `r>0`, a final partial wave uses `r` workers and has instantaneous worker utilization `r/W`; if `r=0` and `T>0`, the final wave is full. For example, `T=150` and `W=142` gives one full wave plus eight tiles: the", "sha256": "a8ef68c0fcff6dd1e2a1c0bd2abaadce92c16b19b92d43925b41a0d9f0ecce59"}, {"id": "u004", "kind": "prose", "locator": "body:L11-L11", "preview": "This is an analytical model, not a statement that a B200 has 142 SMs or that physical SM count always equals `W`. Block resources can make multiple blocks resident per SM, cluster shape changes scheduling granularity, application policies m", "sha256": "d9e0cf3962e4d30c5a85e8a50f0dd53e70c56ead1c68156e831025ddf65c138e"}, {"id": "u005", "kind": "prose", "locator": "body:L13-L13", "preview": "Under the equal-duration model, the time fraction attributable to at most one partial wave shrinks as the number of full waves grows. There is no universal \u201cbelow four times the SM count\u201d cutoff: the remainder, wave duration, other phases, ", "sha256": "f324cd0832cae3c8f0339ca28879361e2a3ea07764d3318bfdc48a357f3fec53"}, {"id": "u006", "kind": "list-item", "locator": "body:L17-L17", "preview": "1. Record logical tiles/clusters, grid and cluster dimensions, available worker capacity, residency limits, and per-work-item duration.", "sha256": "e795d774faa4fd3b84027ef69d3dbb0f4ac2f61de31b0ad0ae7c902891d22e3f"}, {"id": "u007", "kind": "list-item", "locator": "body:L18-L18", "preview": "2. Predict wave count and the final remainder from the simplified model.", "sha256": "1d8d9be6330b539daf5cecf3a3e9ca5bdc43c8d66dc6dafc1626dcc58ab23c80"}, {"id": "u008", "kind": "list-item", "locator": "body:L19-L19", "preview": "3. Use time-resolved activity or instrumented worker records to locate the underfilled interval at the end, rather than inferring it from aggregate utilization.", "sha256": "171e09c6edcc03571aba5731487e7092cbfe98166edaacb3f751eca20419d90d"}, {"id": "u009", "kind": "list-item", "locator": "body:L20-L20", "preview": "4. Sweep nearby problem sizes or tile shapes. A wave-quantization hypothesis predicts a sawtooth response aligned with changes in the remainder, after controlling total work.", "sha256": "23112755735c4dfdfb4280e3bb343028ef65ce6126d88127f7553182529809e6"}, {"id": "u010", "kind": "list-item", "locator": "body:L21-L21", "preview": "5. Separate unequal tile durations: a long straggler is load imbalance even if the tile-count remainder is zero.", "sha256": "12e6a06a0cf28b05b3c3a675a4db26089ec744d706d3001cb0bd2b7b1e7342a7"}, {"id": "u011", "kind": "prose", "locator": "body:L23-L23", "preview": "CUDA schedules ordinary grid blocks onto available SMs; a static grid-stride loop fixes logical worker indices, not `blockIdx`-to-physical-SM placement. A fixed grid therefore does not mean CUDA lacks dynamic block scheduling.", "sha256": "6452a882b8c5416203c08e6041cef08ba04ccf0849deafc93a15afae865c9512"}, {"id": "u012", "kind": "list-item", "locator": "body:L27-L27", "preview": "- A static persistent loop lets a resident CTA process multiple logical work items. Worker count is selected from problem decomposition, cluster/resource limits, and policy\u2014not necessarily one CTA per physical SM. Finite work still has a fi", "sha256": "9db529aa6ae9bd030c3c8089509d63d8584dd95219b31f6d18be3cdf57767e73"}, {"id": "u013", "kind": "list-item", "locator": "body:L28-L28", "preview": "- Cluster Launch Control lets a selected thread request cancellation of an unspecified not-yet-started block or cluster and process the returned grid ID. CLC redistributes existing IDs; it cannot turn eight remaining independent tiles into ", "sha256": "0a5afba4961508157c3c0154f138d89d14c968938adcce7244fc0c81efbacd1c"}, {"id": "u014", "kind": "list-item", "locator": "body:L29-L29", "preview": "- Raster order and swizzle are coordinate transforms. Test them for locality while holding work acquisition fixed; they do not create work or guarantee balanced durations.", "sha256": "7fd94e42d1d2576ed4e87b200e19e9776b4d787a553e5e40fcef1a118dd2e23a"}, {"id": "u015", "kind": "list-item", "locator": "body:L30-L30", "preview": "- Stream-K or Split-K may create more independent partitions when tile-level work is insufficient. Evaluate reduction traffic, workspace, synchronization, determinism, and numerical effects separately.", "sha256": "dd0d2eea0bfed3c0fd23a77f78dc563c2da37c925225e0718d43ab29b94f49d4"}, {"id": "u016", "kind": "prose", "locator": "body:L32-L32", "preview": "Compare static and CLC-backed persistence with identical math, decomposition, tile/cluster shapes, grid, resources, and timing. Record CLC request results and per-worker work/time distribution. PTX ISA 9.0 requires `sm_100` or higher for CL", "sha256": "fcb099d00bab658c03147f5a9ec638a5500d4389439fad74c26eef326c17a5e8"}, {"id": "u017", "kind": "prose", "locator": "body:L36-L36", "preview": "Gau Nernst's Modal B200 has 148 SMs. In the author's `M=N=K=4096` experiment, v5 reports 1302.29 TFLOP/s (86.43% of cuBLAS) and v6 reports 1475.93 TFLOP/s (97.96%). The v6 change is static persistence plus output-pipeline/epilogue-role chan", "sha256": "6f9b202e053f7d5da7cb68d6ce3e62c4533da60caaa7c16fa6b728a202a33c95"}, {"id": "u018", "kind": "list-item", "locator": "body:L40-L40", "preview": "- [CUDA Programming Guide: block and cluster scheduling](https://docs.nvidia.com/cuda/cuda-programming-guide/01-introduction/programming-model.html)", "sha256": "502cb7460a63cee49afa6d82044dcbfe0b1a257319a5ceb0c2543173f8b71844"}, {"id": "u019", "kind": "list-item", "locator": "body:L41-L41", "preview": "- [PTX ISA 9.0 CLC](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel)", "sha256": "8c89668f826fa705d199d168dcff6bea3195792e01de2698f854bb90db46ce5f"}, {"id": "u020", "kind": "list-item", "locator": "body:L42-L42", "preview": "- [CUTLASS 4.5.0 CLC scheduling](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md)", "sha256": "f82b94cccb8b9883e584fa5ec6b7e602e2b7c3247932e262a7a61b54773d925d"}, {"id": "u021", "kind": "list-item", "locator": "body:L43-L43", "preview": "- [Pinned tutorial v6 source](https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu)", "sha256": "d47967878d57c59a54264abe233d783fe466469b45b79870f89f24ba6c73531a"}, {"id": "u022", "kind": "list-item", "locator": "body:L44-L44", "preview": "- [Tutorial result progression](https://gau-nernst.github.io/tcgen05/)", "sha256": "797b16f54ff8d25ba34fefe90153d0a2e58a3efa7414f3594476c0b133a26bad"}], "confidence_claimed": "verified", "headings": ["Exact Simplified Model", "Confirm a Last-Wave Cause", "Scheduler and Decomposition Choices", "Tutorial Scope", "Primary References"], "id": "pattern-tail-effect", "path": "wiki/patterns/tail-effect.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/docs/cutlass-clc-documentation.md", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}], "risk_flags": ["code", "ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-nvidia-tuning-guide", "blog-tcgen05-tutorial", "doc-cutlass-clc", "doc-ptx-isa-sm100"], "title": "Tail Effect \u2014 Last-Wave Underutilization", "type": "pattern", "unresolved_source_ids": [], "version_sensitive": null} {"body_sha256": "3af5831a5b82c05926f7c6ecb8301f7e0c4d3d03774f809a36f86ef5e95df07b", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "PTX global loads and stores can carry cache operators and eviction-priority qualifiers. Global loads can also carry L2 prefetch-size hints. These controls let a kernel express a preference for a particular access; they do not guarantee cach", "sha256": "58fc3708b4bf0b06839ff597548bd0d45b4f269cf14a89d70ed6547be08989a6"}, {"id": "u002", "kind": "table-row", "locator": "body:L9-L9", "preview": "| Form | PTX meaning | Not guaranteed | | `L1::no_allocate` | Selects an L1 eviction priority that may be applied | That the access bypasses L1 |", "sha256": "8225a6a519ce7e32ce882a77aa1056b9169be6e42a5f22560b898481579b8eef"}, {"id": "u003", "kind": "table-row", "locator": "body:L10-L10", "preview": "| Form | PTX meaning | Not guaranteed | | `L1::evict_first` | Requests first-eviction priority | Immediate eviction |", "sha256": "bc512e53bc172d71d0cb8d1ccfa67a7842621adcf4c72daf4c4016c906262a26"}, {"id": "u004", "kind": "table-row", "locator": "body:L11-L11", "preview": "| Form | PTX meaning | Not guaranteed | | `L1::evict_last` | Requests last-eviction priority | Persistent residence |", "sha256": "54e7c9299b1db1db2546f3ad6f51fd38f3f1ea1c67355720fe46d456320adaa4"}, {"id": "u005", "kind": "table-row", "locator": "body:L12-L12", "preview": "| Form | PTX meaning | Not guaranteed | | `L2::64B`, `128B`, `256B` | Hints that additional data of that size be fetched into L2 | A wider memory instruction or a completed prefetch |", "sha256": "23a0133f060ed5d78700c80e13b705675ace800df529b843bb4417b4894b9035"}, {"id": "u006", "kind": "table-row", "locator": "body:L13-L13", "preview": "| Form | PTX meaning | Not guaranteed | | `L2::cache_hint` with a policy operand | Supplies a created L2 eviction policy | That the hint is respected or changes memory consistency |", "sha256": "92165e23d514905a7089b6f790e98bc0b81b229b23cd9f6eb1a558c76feb682e"}, {"id": "u007", "kind": "prose", "locator": "body:L15-L15", "preview": "The following are legal PTX 9.0 instruction fragments when their operands and addresses are declared with matching types. Each vector access shown is 16 bytes and therefore requires 16-byte natural alignment:", "sha256": "e6875d2adb5ad541e5579d45da0f70247686265f3f187debecb3662cfe1c686d"}, {"id": "u008", "kind": "code", "locator": "body:L17-L24", "preview": "```ptx .reg .b64 addr_a, addr_b, addr_c; .reg .u32 a<4>, b<4>, c<4>; ld.global.L1::no_allocate.v4.u32 {a0, a1, a2, a3}, [addr_a]; ld.global.L1::evict_last.v4.u32 {b0, b1, b2, b3}, [addr_b]; st.global.L1::evict_first.v4.u32 [addr_c], {c0, c1", "sha256": "dff1576a0ce3b45407dcdc9dddaf973b2f0df90f5c9dc8eedbab93b06b805315"}, {"id": "u009", "kind": "prose", "locator": "body:L26-L26", "preview": "The common \u201cstream A, retain B\u201d explanation is a hypothesis about reuse and interference, not the semantics of the code. Even with correct alignment, the hardware may apply the priorities differently than a literal bypass/keep model suggest", "sha256": "0f8b19c545e0751d4f2f9962b9b26f4586f822f95847e2b4ab0fbb405ea44515"}, {"id": "u010", "kind": "prose", "locator": "body:L30-L30", "preview": "Start from the concrete access trace rather than a kernel label or tensor name:", "sha256": "eeb262f9767e73e0d30f6587d5465e23ddb312d3367caff656125c14d8a1e9b1"}, {"id": "u011", "kind": "list-item", "locator": "body:L32-L32", "preview": "1. Identify which addresses each warp touches, the reuse distance for each line, the live working set, and competing traffic at the same launch shape.", "sha256": "db5d6449fa0ee136e4483eec513b11e760a840112559819e672a1d3c568586fb"}, {"id": "u012", "kind": "list-item", "locator": "body:L33-L33", "preview": "2. Establish a correct default-policy baseline. Hold mapping, vector width, decode, unrolling, register controls, compiler, inputs, and launch parameters constant.", "sha256": "0aaf1ed9f480045eafdcc9a795490fea975fb9496b7da080a6f7dc21a2fcdceb"}, {"id": "u013", "kind": "list-item", "locator": "body:L34-L34", "preview": "3. Change one qualifier at a time. Re-run the same correctness oracle, including sizes that exercise alignment and tail paths.", "sha256": "6f4744fe0b055ceede7d49f44cbe6268747460836c80a678ad1457b7c21208ed"}, {"id": "u014", "kind": "list-item", "locator": "body:L35-L35", "preview": "4. Record generated instructions and resources, then profile cache hit behavior, requested/transferred bytes, stalls, achieved bandwidth, and active warps with metrics available on the installed tool and target.", "sha256": "4bfc53a191440ff619777b08cc32f2adb819edb032cee818fd8539d4679668de"}, {"id": "u015", "kind": "list-item", "locator": "body:L36-L36", "preview": "5. Benchmark every production shape with the same warmup, synchronization, repetitions, and statistic. Retain only scoped improvements; a policy may help one shape and regress another.", "sha256": "ae112842d641a2c358bbe90a6d930cfd3f10160668b7dadbea95eec4259b234f"}, {"id": "u016", "kind": "prose", "locator": "body:L38-L38", "preview": "An input being larger than L2, a kernel being described as memory-bound, or a tensor being called streamed/reused does not by itself predict a useful qualifier. Concurrent blocks can reuse data, a nominally reused vector may exceed the effe", "sha256": "01cdf9368ffd2d391dc2cb527f3463989408a958627a15dbcef4bd36147c631d"}, {"id": "u017", "kind": "prose", "locator": "body:L42-L42", "preview": "Amandeep Singh reports that three inspected B200 NVFP4 GEMV solutions used `L1::no_allocate` for A and `L1::evict_last` for B alongside raw PTX decode, wide loads, exact-K specialization, tighter register caps, and\u2014in one solution\u2014sharing B", "sha256": "ad93cd1575dc885525b55b8e82655208722219d63a561b9342350d8273f3a9ee"}, {"id": "u018", "kind": "prose", "locator": "body:L44-L44", "preview": "Yue Zhang reports approximately 39 microseconds for a stage combining removal of B shared-memory staging, per-thread tiles, `float4` loads, and hardware conversion, followed by approximately 27 microseconds for a vectorized PTX FP4/scale-de", "sha256": "c1f0bc6fb12c42dff1c938a4a57f0cc0fb0f492ef548917c637f70b038bd7739"}], "confidence_claimed": "verified", "headings": ["Contract", "Selecting a policy", "Evidence boundary for the NVFP4 reports"], "id": "technique-cache-policy", "path": "wiki/techniques/cache-policy.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/blogs/yue-nvfp4-hackathon.md", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}, {"path": "sources/blogs/amandeep-nvfp4-attempts.md", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}, {"path": "sources/contests/gpu-mode-nvfp4/problem-1-gemv.md", "url": "https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-ptx-isa-sm100", "blog-yue-nvfp4", "blog-amandeep-nvfp4", "contest-gpumode-p1"], "title": "PTX Cache Eviction and Prefetch Hints", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} {"body_sha256": "d1c2e2981a7cbb20812788c814381f9f5a67edcec521a6497f68034d83483b8f", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "The two linked PRs have different evidentiary value:", "sha256": "3532db0f410a9281e56ca0b23d9c4c49c19a665f1b6d5e22c617cb4063670fb8"}, {"id": "u002", "kind": "table-row", "locator": "body:L7-L7", "preview": "| PR | Semantic scope | Transfer value | | CCCL 3559, captured merge `25523da2` | Adds B200/SM100 exclusive-sum scan tuning, expands policy classification to input/output/accumulator/offset types, and updates scan dispatch/tests. | A concre", "sha256": "1118714ee9a762906799170546cdc7f8d01af8836ba6cf2e9de5ff78d6ce1bf5"}, {"id": "u003", "kind": "table-row", "locator": "body:L8-L8", "preview": "| PR | Semantic scope | Transfer value | | CCCL 6152, captured merge `3fb05826` | Changes only `CUB_DEBUG_LOG` formatting and stale variable names in `DispatchTopK`. | Evidence for the corrected debug output, not TopK algorithm, performance", "sha256": "9b60c5435b1bbabba385a2d27b84a9705c10a71f82fd3cb5164e2d3c539155f0"}, {"id": "u004", "kind": "prose", "locator": "body:L10-L10", "preview": "Do not use PR 6152 as evidence for DSA TopK design or performance. Its captured key file exposes surrounding TopK implementation for inspection, but the PR's semantic delta is the small debug-only patch.", "sha256": "c6caf045a6b9260f06d822235725a7371ef237eb9773c2804f1e8bf3dfb8dc89"}, {"id": "u005", "kind": "prose", "locator": "body:L14-L14", "preview": "The new `sm100_tuning` specializations select a tuple rather than a single \u201cvectorized\u201d bit:", "sha256": "ef1ba6a5ffaf5c3db38f4137420cffbd6f9b3147a650d9c0bce7ab83dc894da3"}, {"id": "u006", "kind": "table-row", "locator": "body:L18-L18", "preview": "| Policy dimension | Examples in the captured source | | Classification | input value size, accumulator type/size, offset size, and recognized `plus` operator |", "sha256": "a9e0c1a68c8e41698173559e124463741f1f39a94e516597aef7ad189b74d4d5"}, {"id": "u007", "kind": "table-row", "locator": "body:L19-L19", "preview": "| Policy dimension | Examples in the captured source | | Work partition | `threads` and `items` per thread |", "sha256": "0815c33204b77ed25cca9827b2ab51e07004ebad4b5f9f78e9093d09f3a45c15"}, {"id": "u008", "kind": "table-row", "locator": "body:L20-L20", "preview": "| Policy dimension | Examples in the captured source | | Memory policy | `BLOCK_LOAD_*`, `BLOCK_STORE_*`, and `LOAD_DEFAULT` or `LOAD_CA` |", "sha256": "affc3f2082c6b33e588efad21bb73e3ae75573b1ba705d2cadbef18c285fc8cb"}, {"id": "u009", "kind": "table-row", "locator": "body:L21-L21", "preview": "| Policy dimension | Examples in the captured source | | Look-back behavior | an exponential backoff/delay constructor and its parameters |", "sha256": "d476c9630872da451c8547fe530ade5fa22ca4e2f4e0773635a7b5c3c3731dcb"}, {"id": "u010", "kind": "table-row", "locator": "body:L22-L22", "preview": "| Policy dimension | Examples in the captured source | | Fallback | `Policy1000` selects a matching SM100 specialization; otherwise it falls back to `Policy900`; the double specialization explicitly inherits an SM90 tuning |", "sha256": "612ef7a8a3d8257ec2557bc3eddb18089b41ad5c6f0249bcb1e0eed5db932d78"}, {"id": "u011", "kind": "prose", "locator": "body:L24-L24", "preview": "`items >= 4` does not establish vectorized access. Vector width also depends on iterator contiguity, element type, alignment, load/store algorithm, compiler lowering, and the executed policy. Treat items-per-thread, block size, load/store a", "sha256": "05b467376da229569223be7afe7af2a5044c10fabb1983ccbb1aed8b5a096f2c"}, {"id": "u012", "kind": "list-item", "locator": "body:L28-L28", "preview": "1. Identify the exact primitive and semantic contract: inclusive/exclusive scan, operator, input/output/accumulator types, offset width, aliasing, and empty/large-size behavior.", "sha256": "22bee881668fd269c29db6987e756c55090a7bdd5b4037f8d671563d07193f25"}, {"id": "u013", "kind": "list-item", "locator": "body:L29-L29", "preview": "2. Follow runtime architecture dispatch to the active policy. Confirm whether SM100 has a matching specialization or inherits the SM90/default route.", "sha256": "133fa8b4c60fbf759bbe9b2666428bd7250701273d08f0525df21dea718ff00b"}, {"id": "u014", "kind": "list-item", "locator": "body:L30-L30", "preview": "3. Reproduce the upstream baseline and changed policy over the size/type/operator matrix relevant to the application. A policy comment containing benchmark ratios is source context, not a portable speedup guarantee.", "sha256": "e1e61e646347efc9bc0b0608ba22f5dff8e61a40790c4fcb6b0f56bff39ca816"}, {"id": "u015", "kind": "list-item", "locator": "body:L31-L31", "preview": "4. Change one policy dimension at a time where practical and record time, bandwidth, occupancy, register/spill data, and correctness.", "sha256": "7566f62910d39f9e0363c194e49c8b25cc0cc4b9df73f45a5ef292116bfe7d77"}, {"id": "u016", "kind": "list-item", "locator": "body:L32-L32", "preview": "5. For selection, separately verify membership, output count, ordering, ties, NaNs, signed zero, key/value association, and repeatability. Those properties are not established by PR 6152.", "sha256": "c695167e5abc2c932b57c83095361a7e4713d338d1da7cf9e6ee588d4054dfa0"}, {"id": "u017", "kind": "prose", "locator": "body:L34-L34", "preview": "The linked evidence does not support claims about fill, histogram, reduce, block-load/store vectorization, or application-level DSA score computation. Add a directly relevant CCCL source before transferring policy conclusions to those primi", "sha256": "64c0b1fa7d7a20f4942ddeaae956cb7837d4285ade8e50e71db3e773e162e840"}], "confidence_claimed": "source-reported", "headings": ["Exact Source Scope", "What PR 3559 Actually Tunes", "Transfer Workflow"], "id": "technique-cccl-memory-primitives", "path": "wiki/techniques/cccl-memory-primitives.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/prs/cccl/PR-3559.md", "revision": "25523da2", "url": "https://github.com/NVIDIA/cccl/pull/3559"}, {"path": "sources/prs/cccl/PR-6152.md", "revision": "3fb05826", "url": "https://github.com/NVIDIA/cccl/pull/6152"}], "risk_flags": ["ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["pr-cccl-3559", "pr-cccl-6152"], "title": "CCCL CUB Memory Primitives For Selection And Scan", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} {"body_sha256": "dc1f127417087638a33a432f9defd1b6e66efce189bd004f9e699f1bbbee4372", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Some linear recurrent models admit an algebraically equivalent chunkwise formulation. Work local to a chunk can then be expressed with parallel matrix operations, while the state passed across chunk boundaries preserves the recurrence's seq", "sha256": "48a92e68a2fe0a23f551664777d23c15d691c5921abf8b854b06caa7ef5106b9"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "A correct implementation separates at least these obligations:", "sha256": "8c48104a851aa54aa0a45bc389fd93b6f9409ea226bc2d7c0ed56b1f8d6b352d"}, {"id": "u003", "kind": "list-item", "locator": "body:L9-L9", "preview": "1. Compute the chunk-local quantities required by the model's exact recurrence.", "sha256": "7fd75a382a40ba603fb0f8a9810aebf341cd1a888ddc21d67b892970f1d1c1ef"}, {"id": "u004", "kind": "list-item", "locator": "body:L10-L10", "preview": "2. Resolve boundary states in sequence order, either with an associative scan supported by the formulation or with explicitly ordered stages or launches.", "sha256": "32e5983ffa33337e71cec6880f42fd8b53325b7039069f4940df823e46c9485b"}, {"id": "u005", "kind": "list-item", "locator": "body:L11-L11", "preview": "3. Combine each chunk's local result with its incoming boundary state and emit outputs in the original token order.", "sha256": "4185354ea9059f98c36f3e955471e7a5fe4d33df82d607b931c250a87e364f94"}, {"id": "u006", "kind": "list-item", "locator": "body:L12-L12", "preview": "4. Validate outputs and final states against a token-by-token reference across variable sequence lengths, chunk tails, batches, heads, dtypes, and gate extremes.", "sha256": "a3cba5f1e5eb6aa7623dc2822355cf7b802b9d57465d667703f3c10ba06f35e1"}, {"id": "u007", "kind": "prose", "locator": "body:L14-L14", "preview": "An ordinary GPU grid does not imply increasing program-ID execution order or grid-wide synchronization. Programs for every chunk therefore cannot safely read and overwrite one shared state pointer in a single unordered launch. A staged algo", "sha256": "9fc013326b6878c87eff2b13eecd2a1d2a3cf9e14d2f23ccd000b58042d8c0c4"}, {"id": "u008", "kind": "prose", "locator": "body:L18-L18", "preview": "The pinned NVlabs GatedDeltaNet repository uses chunkwise Triton kernels for training and a WY representation of the gated delta rule. That implementation is direct evidence for GatedDeltaNet chunking, but it is not equivalent to a generic ", "sha256": "e7c843e29ca3efe700861205d29f457d86bd1ba013279d32be1a0f2c46aa779b"}, {"id": "u009", "kind": "prose", "locator": "body:L20-L20", "preview": "Tiled Flash Linear Attention (TFLA) starts from the chunkwise formulation of linear RNNs and adds another level of sequence parallelization within a chunk. The authors state that this permits arbitrarily large chunks, raises arithmetic inte", "sha256": "b202431cbc60d3c50fc8c3ae71e5f1a923acdcee1c70f8aa9b30460a4d8181f2"}, {"id": "u010", "kind": "prose", "locator": "body:L22-L22", "preview": "Qwen3-Next is a hybrid-model example, not evidence that one kernel handles every layer. Its immutable 48-layer configuration repeats three Gated DeltaNet linear-attention layers and one full-attention layer twelve times: 36 GDN and 12 full-", "sha256": "8f59435f91c48cc1ead81d8b10be97200d4b6fd046f75d1a4827f1eab9ddc95d"}, {"id": "u011", "kind": "prose", "locator": "body:L26-L26", "preview": "There is no source-backed universal rule that `C=32` is a decode choice or that `C=256-512` is the prefill optimum. Choose only among chunk sizes supported by the exact algorithm and backend, then measure the tradeoff:", "sha256": "8818b3f5fbd2db4ba16a31baad13c667b4fe419941c1b236e1abc9ea5581ef7b"}, {"id": "u012", "kind": "list-item", "locator": "body:L28-L28", "preview": "- arithmetic intensity and matrix-instruction utilization;", "sha256": "ae9e1bf71be10c77bfb65cba9a45c82ae2c478b773284bb7c2e103a158e499a5"}, {"id": "u013", "kind": "list-item", "locator": "body:L29-L29", "preview": "- intermediate-state and workspace traffic;", "sha256": "73a40945cc41e860d08f14bf1d10885e9ab3d814182a7d447c0572e59311dcd7"}, {"id": "u014", "kind": "list-item", "locator": "body:L30-L30", "preview": "- registers, shared memory, and occupancy;", "sha256": "e6793ebfc69570934a6b8cd081905bbe72a12cc6c39148da17b85306eb20484f"}, {"id": "u015", "kind": "list-item", "locator": "body:L31-L31", "preview": "- tail handling and variable-length metadata;", "sha256": "53ab739e34e9312aa6fb64dfb93f2d192f7cc8c5b50888daa8ead3a08b47d27e"}, {"id": "u016", "kind": "list-item", "locator": "body:L32-L32", "preview": "- launch count and boundary-scan cost;", "sha256": "8d8b1cbac255a5b44bba9b24f6458d8179305382641fa292d6679114f7ab477c"}, {"id": "u017", "kind": "list-item", "locator": "body:L33-L33", "preview": "- latency and throughput for the intended batch and sequence distribution.", "sha256": "329f61d749b5347dfc9b2232f4944da7d760a2123ec4cd32523ce05cf5de766e"}, {"id": "u018", "kind": "prose", "locator": "body:L35-L35", "preview": "Compare candidates with identical compiler/software revisions, launch inputs, correctness oracles, warmups, synchronization, and repeated-trial statistics. Treat a selected size as scoped to the model dimensions, dtype, GPU, backend, and wo", "sha256": "d0698561089a9d858f6770ce429c967f63c781117e5ec3e658af91b908ef14d4"}], "confidence_claimed": "verified", "headings": ["Mechanism", "Verified implementations and scope", "Choosing a chunk configuration"], "id": "technique-chunk-parallelism", "path": "wiki/techniques/chunk-parallelism.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/gated-delta-net.md", "url": "https://github.com/NVlabs/GatedDeltaNet/tree/b53d6d3a161267432a79c1c04af69fa52bddc921"}, {"path": "sources/docs/tfla.md", "url": "https://arxiv.org/abs/2503.14376v3"}, {"path": "sources/blogs/qwen3-next-architecture.md", "url": "https://developer.nvidia.com/blog/new-open-source-qwen3-next-models-preview-hybrid-moe-architecture-delivering-improved-accuracy-and-accelerated-parallel-processing-across-nvidia-platform/"}], "risk_flags": ["ordering"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-gated-delta-net", "doc-tfla", "blog-qwen3-next-architecture"], "title": "Chunkwise Parallelism for Linear Recurrences", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} -{"body_sha256": "e32c3558c9bc6483db8f063a150cca5c88247952aa4d168c007310ad16e98baa", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Double- or multi-buffering reserves disjoint storage regions and transfers ownership between producers and consumers. While a consumer reads stage `s`, a producer may fill another stage. The storage alone does not establish overlap: the pro", "sha256": "353970ee76fddcefdd8634bdce84efaf522019d1c67aadbc7734485b35739dec"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "On SM100 these ideas can be applied independently:", "sha256": "7a9f970eba37f57c06d3e4c1053f758644480ec9c1bfd6e13000b66fa7e2fd52"}, {"id": "u003", "kind": "list-item", "locator": "body:L9-L9", "preview": "- SMEM stages can hold operand tiles while TMA production overlaps MMA consumption.", "sha256": "bceca9323f3eeda738243bf77003183e73292e753517a3d9c5e0a58bf486072e"}, {"id": "u004", "kind": "list-item", "locator": "body:L10-L10", "preview": "- Separate TMEM regions can hold output accumulators while an epilogue drains a completed region and MMA writes a different region.", "sha256": "c08e7056d2703d9010ebb72f542b30e222b600fba13f3990bf373404cd5ec57f"}, {"id": "u005", "kind": "prose", "locator": "body:L12-L12", "preview": "The pinned `matmul_v6.cu` from Gau Nernst's tutorial uses both mechanisms. That is a concrete design, not a rule that every optimized Blackwell GEMM needs both.", "sha256": "7143ee6782943e2cf11478db5d26219b0b9ed56dbb20c7df822457fa19405753"}, {"id": "u006", "kind": "prose", "locator": "body:L16-L16", "preview": "The CTA-visible TMEM address structure has 128 lanes and 512 columns of 32-bit cells. Allocation is column-granular across all 128 lanes. A kernel that allocates 512 columns may choose two 256-column regions:", "sha256": "28df4d63890310783d60527b96ed4c07c057cfe045b622775dbf7d3029cffe26"}, {"id": "u007", "kind": "code", "locator": "body:L18-L21", "preview": "```text region[0] = columns [0, 256) region[1] = columns [256, 512) ```", "sha256": "ed69a2b1fa0220c87f2e0b386d916c2e13666c8f97966b1b23e1551f5656de80"}, {"id": "u008", "kind": "prose", "locator": "body:L23-L23", "preview": "This equal split is one implementation choice. The actual invariant is that every simultaneously live region is inside the allocation and does not alias another region whose MMA or epilogue access is still outstanding. Region sizes can diff", "sha256": "47dc0422eca78419d023b265f59cbad85c48bb108bf29c8b6982f069ad1e6564"}, {"id": "u009", "kind": "prose", "locator": "body:L25-L25", "preview": "For each region, prove this lifecycle:", "sha256": "77958ca2b2300609b14a11248488adc9a6078379d31d5df83720e798035ab4a9"}, {"id": "u010", "kind": "table-row", "locator": "body:L29-L29", "preview": "| Transition | Required edge | | free \u2192 MMA-owned | every prior epilogue load from the region has completed |", "sha256": "b36ebc214cce51253c6d828b95dc22426a8ab01b4128d0a6e49484d51e3735db"}, {"id": "u011", "kind": "table-row", "locator": "body:L30-L30", "preview": "| Transition | Required edge | | MMA-owned \u2192 ready | relevant tcgen05 work is committed to an mbarrier and completion is observed |", "sha256": "8af311d47b977e2484a14ff52be09086cc85443f38fed3ec42c5cc1d87cf531d"}, {"id": "u012", "kind": "table-row", "locator": "body:L31-L31", "preview": "| Transition | Required edge | | ready \u2192 epilogue-owned | readers observe the matching barrier phase before `tcgen05.ld` |", "sha256": "086da55067d42eea98da03787c03f386f30b566554fe854779777bfb6a1909b7"}, {"id": "u013", "kind": "table-row", "locator": "body:L32-L32", "preview": "| Transition | Required edge | | epilogue-owned \u2192 free | all collective loads complete with `tcgen05.wait::ld`, then every reader reports completion |", "sha256": "04c3311dcc82b7cc6680a8b4e7e23e43fc32e4f21b96dc5d35631d0e0ad3e828"}, {"id": "u014", "kind": "prose", "locator": "body:L34-L34", "preview": "TMEM must first be allocated by the required participating warp and its address safely published. Matching collective deallocation is required before every kernel exit. Reused mbarriers need the correct arrival count and phase/parity state;", "sha256": "0190655232d79908e1b5184a1aba9c8b97f9ce1895c72ef9b24d30b25b54600e"}, {"id": "u015", "kind": "prose", "locator": "body:L38-L38", "preview": "An SMEM operand stage normally has its own full/empty state:", "sha256": "70c65950ab321f20237632a725827aae81634feef5c60cb54887ca343647416d"}, {"id": "u016", "kind": "list-item", "locator": "body:L40-L40", "preview": "1. The producer waits until stage `s` is empty.", "sha256": "a79eddecdf545db8d1cf874c1f8d4281c05ee71fbe6cbec7e01fe3c0c5162046"}, {"id": "u017", "kind": "list-item", "locator": "body:L41-L41", "preview": "2. It issues the stage's TMA copies and accounts for expected transaction bytes.", "sha256": "4bff5a30df81531732eda4d3b5c80ed4c98124eb1f1f620d593322f52494ae75"}, {"id": "u018", "kind": "list-item", "locator": "body:L42-L42", "preview": "3. The consumer observes the full barrier's matching phase before using the stage.", "sha256": "e61f4af11998b70bd86556f287f45075f6b2ec06632bb6af1d0977cf7c9f3624"}, {"id": "u019", "kind": "list-item", "locator": "body:L43-L43", "preview": "4. After the final dependent read, the consumer releases the empty barrier for reuse.", "sha256": "763037b61e4bfbaae65d104e217069396c21b86838e86a0a3cf76a821234b0fa"}, {"id": "u020", "kind": "prose", "locator": "body:L45-L45", "preview": "For binary16 `A[128,64]` and `B[64,256]`, the unpadded payload arithmetic is:", "sha256": "720c85da0975a0f1d2896dc5160c1ddb319b5ab4e5f9e77a1f863b5571298ee7"}, {"id": "u021", "kind": "code", "locator": "body:L47-L51", "preview": "```text A = 128 \u00d7 64 \u00d7 2 bytes = 16 KiB B = 64 \u00d7 256 \u00d7 2 bytes = 32 KiB one stage = 48 KiB; three stages = 144 KiB ```", "sha256": "b092d8e540dcc0b91039c62edcef86c1328d4c9d837144ae54279532c788868b"}, {"id": "u022", "kind": "prose", "locator": "body:L53-L53", "preview": "That is not a complete C++ shared-storage layout. Barriers, descriptors, epilogue scratch, padding, alignment, and swizzled physical layouts also consume or constrain shared memory. Compute capability 10.0 supports up to 228 KiB of shared m", "sha256": "7571140fcd587dd0d7dbfd830e6b0aa9af7f30e57c7f6c49c46017c55c048e65"}, {"id": "u023", "kind": "prose", "locator": "body:L55-L55", "preview": "FlashInfer PR 2387 is a second pinned example. Its merged `selective_state_update.cuh` has `state[numStages][...]` plus `bar_full` and `bar_empty` arrays and uses stage-specific producer/consumer handoffs. One path selects three stages and ", "sha256": "b3e9b66c8f8cc9b7e41acfa835cc972955d57829756334c98c944cf8b6d7aa7e"}, {"id": "u024", "kind": "prose", "locator": "body:L59-L59", "preview": "For Hopper `wgmma.mma_async.m64nNk16` with FP32 D, each warpgroup thread holds `N/2` accumulator registers. At `N=256`, one D fragment is 128 registers per thread; two simultaneously live fragments are 256 registers, or 1024 bytes, per thre", "sha256": "f6e642682b459f63b368dd28720e7321715e6f3f095f5708a72b505a14d6e214"}, {"id": "u025", "kind": "prose", "locator": "body:L61-L61", "preview": "On SM100, tcgen05 keeps resident D in TMEM. This avoids a long-lived per-thread D vector, but the kernel still uses registers for addresses, descriptors, loop and barrier state, `tcgen05.ld` destinations, and epilogue temporaries. TMEM buff", "sha256": "bfb0ec4702c7e38a89db5564c2c0bc9038b25f7fb2da4fe7e93dfb6d6fc53189"}, {"id": "u026", "kind": "table-row", "locator": "body:L65-L65", "preview": "| Concern | Hopper WGMMA D | Blackwell tcgen05 D | | Resident storage | per-thread register fragment | allocated TMEM region |", "sha256": "c6d2a5bb97def6cf558e27a9beaf1ef9ddecc90fe5db35e212ebb270b9829736"}, {"id": "u027", "kind": "table-row", "locator": "body:L66-L66", "preview": "| Concern | Hopper WGMMA D | Blackwell tcgen05 D | | Two live outputs | two disjoint register fragments | two disjoint TMEM regions |", "sha256": "0713b8c5efd58eb69a0c9afbb6b8fdfabbfdad4b38831bbf0fba248d83b392c5"}, {"id": "u028", "kind": "table-row", "locator": "body:L67-L67", "preview": "| Concern | Hopper WGMMA D | Blackwell tcgen05 D | | Epilogue access | dependent use after WGMMA completion | collective `tcgen05.ld` and `tcgen05.wait::ld` after MMA completion |", "sha256": "1a1b26e2df7829f360c6b0d265010b85cdb71600218bc36dc143cbdb080c9368"}, {"id": "u029", "kind": "table-row", "locator": "body:L68-L68", "preview": "| Concern | Hopper WGMMA D | Blackwell tcgen05 D | | Main resource question | compiled registers, SMEM, block shape, and other limits | TMEM columns plus compiled registers, SMEM, block/cluster shape, and other limits |", "sha256": "45f5f676075588236b666c4a523ad20b4b6719508a6f5749f640ee3fbd88902f"}, {"id": "u030", "kind": "prose", "locator": "body:L72-L72", "preview": "Buffering is useful only when useful producer/consumer work overlaps enough to repay extra storage and coordination. For each concrete variant:", "sha256": "793da3bb85dc9c0487c967d2d3786db983a421feb6842d70691e85f9b3dd0189"}, {"id": "u031", "kind": "list-item", "locator": "body:L74-L74", "preview": "1. Keep shape, datatype, outputs, launch policy, warmup, and timing statistics identical.", "sha256": "c811a391248f2e1296904f15657e53b1e655a693b0c2f20cdf700f28553830e9"}, {"id": "u032", "kind": "list-item", "locator": "body:L75-L75", "preview": "2. Compare one versus multiple TMEM regions and the legal SMEM stage counts.", "sha256": "deb1c0cee7d6eaf7e50b64e7c05f585d2a755e855afdd7d697211d5fcfb3c6a2"}, {"id": "u033", "kind": "list-item", "locator": "body:L76-L76", "preview": "3. Record compiled registers/thread, spill traffic, static/dynamic SMEM, TMEM columns, barriers, threads/CTA, and cluster shape.", "sha256": "ae0395488ac0efa61f5fd2d5ac4a9b1daf9550fd06c04f868ae9f0183e291406"}, {"id": "u034", "kind": "list-item", "locator": "body:L77-L77", "preview": "4. Use the CUDA occupancy APIs for the actual resource record; do not infer a universal CTA/SM count from an instruction tile.", "sha256": "2a10693b0198014d1dc04730355f7a036237d5182f1bd7761e564c5141a60c79"}, {"id": "u035", "kind": "list-item", "locator": "body:L78-L78", "preview": "5. Inspect profiler pipeline and barrier stalls. PTX specifies mbarrier semantics, not a fixed \u201cfew cycles\u201d latency.", "sha256": "cf8f03e1c706642c110aae5d3923247b587728b29b7cd75a9abadb186bb84dbe"}, {"id": "u036", "kind": "list-item", "locator": "body:L79-L79", "preview": "6. Inspect generated PTX/SASS and execute correctness tests that force region wraparound and pipeline-tail paths.", "sha256": "65a8a0abcb1daa2ba41ef79bcbcc029319411ffd7cd2758984f0c96c15b9f169"}, {"id": "u037", "kind": "prose", "locator": "body:L81-L81", "preview": "Useful negative tests deliberately delay the epilogue, swap a barrier phase, omit the final drain, or reuse a region early. A correct test should detect stale/overwritten output or time out under a watchdog rather than silently accepting th", "sha256": "679336a50791d2df8ee87709263ae3c87ca8e1ff4b42560ec26d159a2e390266"}, {"id": "u038", "kind": "list-item", "locator": "body:L85-L85", "preview": "- [PTX ISA 9.0 Tensor Memory](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensor-memory)", "sha256": "aed47f18c514497e50694f89415704f98533229e52ed733c1898d3c9bb3e2747"}, {"id": "u039", "kind": "list-item", "locator": "body:L86-L86", "preview": "- [PTX ISA 9.0 tcgen05 allocation](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-alloc)", "sha256": "31973f025402fdb2aee03f37ec947608275b8e131e5d9423444bc588c91662b7"}, {"id": "u040", "kind": "list-item", "locator": "body:L87-L87", "preview": "- [PTX ISA 9.0 tcgen05 load and wait](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#tensorcore-5th-generation-instructions-tcgen05-ld)", "sha256": "48de0a6246274c6b6cd72eb33f13df8bc2d371baf458c769c7dbc09c21d05cb3"}, {"id": "u041", "kind": "list-item", "locator": "body:L88-L88", "preview": "- [PTX ISA 9.0 mbarrier](https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier)", "sha256": "57f21ef2db856a7d997dff531533e549ea54d4a49becca4e57e4fa0f44f2cc99"}, {"id": "u042", "kind": "list-item", "locator": "body:L89-L89", "preview": "- [CUDA 13.0.2 Blackwell Tuning Guide](https://docs.nvidia.com/cuda/archive/13.0.2/blackwell-tuning-guide/index.html#occupancy)", "sha256": "8d963db7f76eb6508d3be3b8853709f57f9a8b2f98e623d121cff16c31de6585"}, {"id": "u043", "kind": "list-item", "locator": "body:L90-L90", "preview": "- [Pinned combined SMEM/TMEM example](https://github.com/gau-nernst/learn-cuda/blob/3b90ac9b3f624bdf1f6f78d02dcd533675d36573/02e_matmul_sm100/matmul_v6.cu)", "sha256": "862d7ef0e5fe7e4485454b7d0fbf52b79f47212d96d6c3d2d61e2f99a12c0966"}, {"id": "u044", "kind": "list-item", "locator": "body:L91-L91", "preview": "- [FlashInfer PR 2387 merged source](https://github.com/flashinfer-ai/flashinfer/blob/18804cd51734cccf807356d017733bc757677f15/include/flashinfer/mamba/selective_state_update.cuh)", "sha256": "21e58f5ba0dd7a07429d9e9e15e0cb6abd0137cefbff017bad4ae6654b1df059"}, {"id": "u045", "kind": "list-item", "locator": "body:L95-L95", "preview": "- [Tensor Memory](../hardware/tmem.md) \u2014 allocation, addressing, and lifetime rules", "sha256": "7afb4cc4a4c10faf8414ea0adcf8dbca658326f9d2b530cdba04f46f89c8a510"}, {"id": "u046", "kind": "list-item", "locator": "body:L96-L96", "preview": "- [pipeline stages](pipeline-stages.md) \u2014 SMEM pipeline construction and tail handling", "sha256": "a926fba806fc32d5e84f37e9775faa981aee434d3e360678aee986e6bc627802"}, {"id": "u047", "kind": "list-item", "locator": "body:L97-L97", "preview": "- [epilogue fusion](epilogue-fusion.md) \u2014 work performed while draining an output tile", "sha256": "613bbcc5ee62a62bc83b7c29a8227e6b57497c4f7f38939f1ca80134084b718c"}], "confidence_claimed": "verified", "headings": ["What the pattern guarantees", "TMEM regions", "SMEM stages", "Hopper and Blackwell storage tradeoff", "Decide with a controlled comparison", "Primary references", "Related"], "id": "technique-double-buffering", "path": "wiki/techniques/double-buffering.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/docs/nvidia-blackwell-tuning-guide.md", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/"}, {"path": "sources/prs/flashinfer/PR-2387.md", "revision": "18804cd51734cccf807356d017733bc757677f15", "url": "https://github.com/flashinfer-ai/flashinfer/pull/2387"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["blog-tcgen05-tutorial", "doc-nvidia-tuning-guide", "pr-flashinfer-2387"], "title": "Double/Multi-Buffering Patterns", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} -{"body_sha256": "7b78dc20ff19f53642d3b695375cd2e64bb0f6f55abad1feba792f98a12ad994", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Epilogue fusion computes output transformations\u2014such as `alpha*acc + beta*C`, bias, activation, output conversion, or auxiliary values\u2014inside the producer kernel before final output materialization. It can remove an intermediate tensor or l", "sha256": "f9107441dd920540c0d1ac9e5090773a75dfa9fa7d1443b994ffe3d7f94d465d"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "Fusion does not by itself imply overlap with the next MMA tile. A schedule may drain one completed accumulator after the mainloop, or it may use disjoint storage and specialized roles to overlap a completed region's epilogue with independen", "sha256": "4f47d7d2b8553488318d46030c512f990423ca91ad8d800925811d2b7e8bb075"}, {"id": "u003", "kind": "prose", "locator": "body:L11-L11", "preview": "For tcgen05, D resides in TMEM. An ordinary arithmetic epilogue first uses a legal collective `tcgen05.ld` mapping, or a library wrapper around it, to transfer a partition into per-thread registers. The load is asynchronous with respect to ", "sha256": "5877a199df06e039e1d768c0f9a71bba45e88384bc7e3ff057d758f72ec8ce82"}, {"id": "u004", "kind": "prose", "locator": "body:L13-L13", "preview": "CUTLASS 4.5.0's `fp16_gemm_2.py` demonstrates the typed route:", "sha256": "adc38d40084211a2785289cdae32b37c45f48b6c540acd24ab586a3a8994814b"}, {"id": "u005", "kind": "code", "locator": "body:L15-L24", "preview": "```python copy_atom_t2r = cute.make_copy_atom( tcgen05.Ld32x32bOp(tcgen05.Repetition.x32), cutlass.Float32 ) tiled_copy_t2r = tcgen05.make_tmem_copy(copy_atom_t2r, tCtAcc_epi) thr_copy_t2r = tiled_copy_t2r.get_slice(tidx) tTR_tAcc = thr_cop", "sha256": "98fb794337ea2463c1d0e401026810e136bcde4790746a21365872ce92d466a6"}, {"id": "u006", "kind": "prose", "locator": "body:L26-L26", "preview": "This is an API-routing fragment, not standalone code: the official file supplies the exact tensor layouts, selected load shape, participant group, edge predicates, accumulator-completion handoff, register-to-SMEM conversion, TMA-store pipel", "sha256": "a8dc66e0523115acbd50a98fb09e529ba902911c6c8ccd4b24c2afc24fec85eb"}, {"id": "u007", "kind": "table-row", "locator": "body:L32-L32", "preview": "| Boundary | Proof required before crossing it | | tcgen05 MMA \u2192 TMEM reader | the relevant asynchronous MMA is committed and its completion observed |", "sha256": "9985d4cd976b715fdc76865925a3aa8489cadc7a359e285805c6174ead01f6a4"}, {"id": "u008", "kind": "table-row", "locator": "body:L33-L33", "preview": "| Boundary | Proof required before crossing it | | TMEM \u2192 result registers | the legal collective load completes before dependent arithmetic |", "sha256": "6b6d528df317b00dd9c026068a171d0397fece81c64efdee7e8451c739811c78"}, {"id": "u009", "kind": "table-row", "locator": "body:L34-L34", "preview": "| Boundary | Proof required before crossing it | | registers \u2192 output | element/layout mapping and output-edge predicates cover exactly the valid coordinates |", "sha256": "efc6c04d14475fe8461a37c816dadacb9dfe78df871eba2e26350ad5e59bbc3e"}, {"id": "u010", "kind": "table-row", "locator": "body:L35-L35", "preview": "| Boundary | Proof required before crossing it | | TMEM reader \u2192 region reuse | every reader's load has completed and the matching reusable barrier phase is released |", "sha256": "624abc6a869187539c7ac548bb053a85c391b041419a4f52f82cd2390fe000da"}, {"id": "u011", "kind": "table-row", "locator": "body:L36-L36", "preview": "| Boundary | Proof required before crossing it | | kernel tail | all output stores complete as required and every TMEM allocation is collectively freed |", "sha256": "394bf4e44474bc37252b13e6680b64dce13cdd39a349c33da579f6af120afb85"}, {"id": "u012", "kind": "prose", "locator": "body:L38-L38", "preview": "For a multi-region overlap schedule, simultaneously live TMEM regions must also be disjoint. Equal 256-column halves are one possible policy for a 512-column allocation, not an epilogue-fusion requirement. Reusable mbarriers need correct ex", "sha256": "632d24557c88b222486de12166a2781d2e53a39ed69410cfa66d6ec170d85f1a"}, {"id": "u013", "kind": "prose", "locator": "body:L42-L42", "preview": "`cutlass::epilogue::fusion::LinCombEltAct` has this parameter order:", "sha256": "91ce48572731a856757d222bfb3aa65e8350a69cd68d1b04eff3e953be42239a"}, {"id": "u014", "kind": "code", "locator": "body:L44-L53", "preview": "```cpp template < template class ActivationFn, class ElementOutput, class ElementCompute, class ElementSource = ElementOutput, class ElementScalar = ElementCompute, cutlass::FloatRoundStyle Round = cutlass::FloatRoundStyle::round_to", "sha256": "2c17e7f24711c91e72d68d975cae0b68a3c2482f8b386e3a64ab96d14482efa0"}, {"id": "u015", "kind": "prose", "locator": "body:L55-L55", "preview": "For SM100 construction, `cutlass::epilogue::collective::CollectiveBuilder` receives architecture, operator class, tile/cluster shapes, `EpilogueTileAuto` or an explicit epilogue tile, accumulator/compute/C/D types and layouts, alignment, `E", "sha256": "5ba02c063f7bbb12184dbe6ba70d2875fca82c9b9bb28392add306148e470825"}, {"id": "u016", "kind": "prose", "locator": "body:L57-L57", "preview": "There is no CUTLASS 4.5.0 type named `Sm100EpilogueTmaWarpSpecialized`. Support is constrained by the exact architecture, operator class, schedule, tile, layout, alignment, datatype, and fusion callback combination. Use `Gemm::can_implement", "sha256": "d111f3431cc1550474efe0835affdc6255eb3133f4ab79c1adeab8d7bb1385ce"}, {"id": "u017", "kind": "prose", "locator": "body:L59-L59", "preview": "The vLLM PR 16032 NVFP4 wrapper is one pinned C++ construction example: it uses `CollectiveBuilder<... EpilogueTileAuto, ... EpilogueScheduleAuto>` and derives mainloop shared-memory stages with `StageCountAutoCarveout class ActivationFn, class ElementOutput, class ElementCompute, class ElementSource = ElementOutput, class ElementScalar = ElementCompute, cutlass::FloatRoundStyle Round = cutlass::FloatRoundStyle::round_to", "sha256": "2c17e7f24711c91e72d68d975cae0b68a3c2482f8b386e3a64ab96d14482efa0"}, {"id": "u015", "kind": "prose", "locator": "body:L55-L55", "preview": "For SM100 construction, `cutlass::epilogue::collective::CollectiveBuilder` receives architecture, operator class, tile/cluster shapes, `EpilogueTileAuto` or an explicit epilogue tile, accumulator/compute/C/D types and layouts, alignment, `E", "sha256": "5ba02c063f7bbb12184dbe6ba70d2875fca82c9b9bb28392add306148e470825"}, {"id": "u016", "kind": "prose", "locator": "body:L57-L57", "preview": "There is no CUTLASS 4.5.0 type named `Sm100EpilogueTmaWarpSpecialized`. Support is constrained by the exact architecture, operator class, schedule, tile, layout, alignment, datatype, and fusion callback combination. Use `Gemm::can_implement", "sha256": "d111f3431cc1550474efe0835affdc6255eb3133f4ab79c1adeab8d7bb1385ce"}, {"id": "u017", "kind": "prose", "locator": "body:L59-L59", "preview": "The vLLM PR 16032 NVFP4 wrapper is one pinned C++ construction example: it uses `CollectiveBuilder<... EpilogueTileAuto, ... EpilogueScheduleAuto>` and derives mainloop shared-memory stages with `StageCountAutoCarveout`](https://github.com/NVIDIA/cutlass/blob/e406c186d2cae5782a846f7280af282ca4fecec2/include/cute/swizzle.hpp):", "sha256": "5ef103ec8227418a2bf6fc668b9d8a44e15b7542e22dfe952b9f3368f5e85332"}, {"id": "u017", "kind": "list-item", "locator": "body:L36-L36", "preview": "- `BBits` is the number of mask bits;", "sha256": "5af4a067d6c7da7415684db6889c71471be01324eefd2b56cba5ab0605b9048b"}, {"id": "u018", "kind": "list-item", "locator": "body:L37-L37", "preview": "- `MBase` is the number of least-significant address bits kept invariant; and", "sha256": "4baa3200cd5285f7c28a299e5577b8cdb19ad27e8d093c6208a6d0b33aad32b5"}, {"id": "u019", "kind": "list-item", "locator": "body:L38-L38", "preview": "- `SShift` is the distance between the two bit fields.", "sha256": "f28019ed4ea31c3b1d699597255eec3b2cb5bca744ea8d505802e5142e328689"}, {"id": "u020", "kind": "prose", "locator": "body:L40-L40", "preview": "For `Swizzle<3,4,3>`, three address bits beginning above the four invariant low bits are XORed with the three-bit field shifted by three positions. Equivalently, address bits 7:9 affect bits 4:6. This is an address transformation, not a gen", "sha256": "ffaae440bfd21e9f23db7977e6f25a0843206a5ebcccceb60b0f43b287fb38b6"}, {"id": "u021", "kind": "prose", "locator": "body:L42-L42", "preview": "CUTLASS also distinguishes a position-independent composed swizzle layout from a position-dependent swizzle pointer, because hardware swizzling depends on the shared-memory pointer address. Reuse a pinned complete layout/descriptor construc", "sha256": "7b7c33d7d8af0396265809e977fa799bb5bc22896a34d87bb0341b7036cd6fdf"}, {"id": "u022", "kind": "prose", "locator": "body:L46-L46", "preview": "For `cuTensorMapEncodeTiled`, record and validate all inputs rather than copying only the swizzle enumerator:", "sha256": "52e9d72146342e12fd2b1490c3761b731df5f3257a9203f475f988e31c7143c9"}, {"id": "u023", "kind": "list-item", "locator": "body:L48-L48", "preview": "1. Use a correctly aligned `CUtensorMap` object and global base pointer.", "sha256": "80e4aa5c2d38c0d179760ddbd62c54902ad761ce25c126a2a276d27c250232a0"}, {"id": "u024", "kind": "list-item", "locator": "body:L49-L49", "preview": "2. Supply rank-sized global dimensions, rank-minus-one byte strides (the fastest dimension is implicit), rank-sized box dimensions, and rank-sized element strides.", "sha256": "7a0cabb5ab90c8ad9fcb90c4a3b1abec710ea10e42d49e4b1ae7f089df312640"}, {"id": "u025", "kind": "list-item", "locator": "body:L50-L50", "preview": "3. Satisfy the global alignment, stride, box, interleave, datatype, and selected-swizzle constraints.", "sha256": "44c861550453d7469cfe6ab5f3f95ff727c04f68a1728ad416f78b1f7d2b1b7c"}, {"id": "u026", "kind": "list-item", "locator": "body:L51-L51", "preview": "4. Check the returned `CUresult`; do not launch with an output descriptor after encoding failed.", "sha256": "e35e8c5abe5a043f7a59a02a53ec58fbe4d9c5630736504600a4fdf081ea801d"}, {"id": "u027", "kind": "list-item", "locator": "body:L52-L52", "preview": "5. Use a shared-memory base and tcgen05 descriptor that represent the same mapping as the tensor map.", "sha256": "bcf3491b71148b2582b92aa3a39be1f27b68b24860545007d7c3fceec0df92d3"}, {"id": "u028", "kind": "prose", "locator": "body:L54-L54", "preview": "An invalid encoder combination can fail explicitly. A successfully encoded tensor map paired with the wrong consumer layout may instead read the wrong logical elements, so a successful API return is not a correctness oracle.", "sha256": "5bb6b35556ce52d1c4949afd7353eea6ed6e47d18072506fc1477c4c5e422a86"}, {"id": "u029", "kind": "prose", "locator": "body:L58-L58", "preview": "Gau Nernst's pinned B200 tutorial compares M=N=K=4096 kernels with PyTorch 2.9.1 and CUDA 13. Its 3D TMA version changes from a 16-byte inner tile with no swizzle to a 128-byte inner tile with 128B swizzling and matching tcgen05 descriptors", "sha256": "1442557fe3203a558928700307223fea48556a1423bb751823fe7a3ded0daec6"}, {"id": "u030", "kind": "table-row", "locator": "body:L62-L62", "preview": "| Tutorial version | Author-reported TFLOP/s | | v1b: 3D 16B TMA | 252.81 |", "sha256": "5ec52bb84daca8d9c4f9c330d7e0f75780b21a3674ec4e8c94329488f3030c29"}, {"id": "u031", "kind": "table-row", "locator": "body:L63-L63", "preview": "| Tutorial version | Author-reported TFLOP/s | | v2b: 3D 128B TMA plus 128B swizzle | 695.43 |", "sha256": "98a08ba0c20d778d67556cee3756e894c83682172bf48209e67517fa770e44e4"}, {"id": "u032", "kind": "prose", "locator": "body:L65-L65", "preview": "The combined change is approximately 2.75\u00d7 and the v2b endpoint is about 46% of the tutorial's 1506.74 TFLOP/s cuBLAS result. It is not a bank-conflict or swizzle-only ablation. The author notes that the earlier contiguous `8\u00d716B` tile migh", "sha256": "5c904bdadd0bdbec537b8da0a58b9e0eae1846a8130911c2933fdae421f9771c"}, {"id": "u033", "kind": "prose", "locator": "body:L69-L69", "preview": "For each candidate mapping:", "sha256": "7efda7c57692314fbd1a9fcdd587176241012dc24de5eecd9b854d1b2545ead1"}, {"id": "u034", "kind": "list-item", "locator": "body:L71-L71", "preview": "1. Keep global tensor, box, consumer instruction, tile shape, synchronization, and output oracle fixed. Change only the producer/consumer layout pair when an isolated comparison is possible.", "sha256": "85fa3e9198db89eed9cb12480937658783d70f38dd929725f0b8cf71e59328c7"}, {"id": "u035", "kind": "list-item", "locator": "body:L72-L72", "preview": "2. Check every tensor-map encoder result and run correctness tests that distinguish rows, columns, tiles, boundaries, and out-of-bounds fill. Do not rely on random uniform values that can hide permutations.", "sha256": "7c059219f0a0bef45641d0ee887c2f16f188c681fa01a4258e0aac06772ee9ef"}, {"id": "u036", "kind": "list-item", "locator": "body:L73-L73", "preview": "3. Use the installed Nsight Compute's `--query-metrics` or Memory Workload Analysis rather than assuming one metric name exists on every chip/tool version. Current documentation includes `l1tex__data_bank_conflicts_pipe_lsu.sum` and request", "sha256": "7081d7ec194ad85836cd0a830d113b7edfbf9547a09a3d54ef52e60a1a4dbf2e"}, {"id": "u037", "kind": "list-item", "locator": "body:L74-L74", "preview": "4. Compare executed shared-memory requests, ideal versus excessive wavefronts, bank conflicts, TMA behavior, total time, and occupancy. Zero observed CUDA-core bank conflicts neither proves the MMA descriptor matches nor explains a performa", "sha256": "16afbfd8bff05f99eb2faf2c5860e7a0786f1e850a6c43efa566f887bdcc24dd"}, {"id": "u038", "kind": "list-item", "locator": "body:L75-L75", "preview": "5. Record GPU, clocks, toolkit, profiler, compiler, exact addresses/alignment, layout types, descriptor fields, warmup, repetitions, and trial statistic.", "sha256": "09ceac52c96b6ed2ee30601a3682d58e7e976256b12796df1a192f98531b105e"}, {"id": "u039", "kind": "prose", "locator": "body:L77-L77", "preview": "Select the mapping that is legal and correct for both producer and every consumer, then retain it only where the controlled target workload improves. Do not infer the choice from architecture name, element width, or \u201cMMA versus non-MMA\u201d alo", "sha256": "d9fe95880e83d8c77d23da16c2196e5649905b51170a21816dd3bca04ba4e1a5"}], "confidence_claimed": "verified", "headings": ["Bank-conflict model", "Three mappings must agree", "CuTe address-bit form", "Tensor-map construction checklist", "Source-reported tutorial result", "Verification procedure"], "id": "technique-swizzling", "path": "wiki/techniques/swizzling.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-cuda-13-0-2-tma.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#asynchronous-data-copies-using-the-tensor-memory-accelerator-tma"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}], "risk_flags": ["ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-cuda-13-0-2-tma", "doc-ptx-isa-sm100", "blog-tcgen05-tutorial"], "title": "Shared Memory Swizzling", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} +{"body_sha256": "f44a550adc8f128e15dad0b4fe874a491284451eb6e9beb0fc9554253eac855d", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "Shared memory has 32 banks, and successive 32-bit words map to successive banks. For one warp memory request, accesses to different words in the same bank require serialized wavefronts; same-word reads can broadcast and do not create that c", "sha256": "6b4784395deb22e22c47d81e1073bda698d9946d6dfa8956aba1fb25a4599618"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "A swizzle permutes address chunks so a target access pattern may distribute its requests over banks more evenly. It can reduce conflicts for that pattern, but it does not make every access to the tile conflict-free. A layout favorable to an", "sha256": "049b7999551a50cd60053dfbb5caafcd9b7b5c16959fd943cd01bbfdd61f28de"}, {"id": "u003", "kind": "prose", "locator": "body:L11-L11", "preview": "For a TMA-to-tcgen05 path, keep these layers consistent:", "sha256": "ce65777d5a7d19fdd488eac3fb93793f2fc06eecda8aca3fdbb8ba2593a2b5a9"}, {"id": "u004", "kind": "list-item", "locator": "body:L13-L13", "preview": "1. The tensor map describes how TMA places the global-memory box into shared memory, including interleave and swizzle.", "sha256": "e387028b2a9dcb25eb9b30133634561dd154b56dc21709f95d132e873b789e26"}, {"id": "u005", "kind": "list-item", "locator": "body:L14-L14", "preview": "2. The shared-memory allocation and base address satisfy the alignment and span constraints of that mapping.", "sha256": "47ad2cabca60121ebef151e99d7e061d565af2ac00a4ccf20eee2c6f58b70a7f"}, {"id": "u006", "kind": "list-item", "locator": "body:L15-L15", "preview": "3. The tcgen05 shared-memory descriptor describes the same physical-to-logical layout, leading/stride dimensions, base offset, and swizzle mode expected by the MMA operand.", "sha256": "144dae2e7f337f4bbd5d00c78b747a850410fd5afd70e13cd24710604b5d0990"}, {"id": "u007", "kind": "prose", "locator": "body:L17-L17", "preview": "PTX ISA 9.0 permits ordinary tcgen05 shared-memory descriptor modes with no swizzle and 32B, 64B, or 128B spans, plus a 128B/base-32B mode. In the descriptor's three-bit layout field, ordinary no-swizzle is `0`, 128B/base-32B is `1`, ordina", "sha256": "d1fb40783e7e9a81de95e039139fbd02b35defc2544ccb4cae9f6aeb14be4e83"}, {"id": "u008", "kind": "prose", "locator": "body:L19-L19", "preview": "The CUDA 13.0.97 Driver API exposes TMA modes including:", "sha256": "3c6d9ecbb46dc013064408601a375a1c73deefd73d8bb67a6772e308c8b8d290"}, {"id": "u009", "kind": "table-row", "locator": "body:L23-L23", "preview": "| Tensor-map mode | Documented chunk mapping | | `CU_TENSOR_MAP_SWIZZLE_NONE` | No bank swizzle |", "sha256": "01f61fc63cd1281a315f33e31834eedd96f6bc4bcef3241942e9ff911ab8c87a"}, {"id": "u010", "kind": "table-row", "locator": "body:L24-L24", "preview": "| Tensor-map mode | Documented chunk mapping | | `CU_TENSOR_MAP_SWIZZLE_32B` | 16B chunks within a 32B span |", "sha256": "1a533d3728f567b6ed6f24bcee8d854fbbeb3efeafe4e3d18b1508595b1ad3ca"}, {"id": "u011", "kind": "table-row", "locator": "body:L25-L25", "preview": "| Tensor-map mode | Documented chunk mapping | | `CU_TENSOR_MAP_SWIZZLE_64B` | 16B chunks within a 64B span |", "sha256": "9f1924e13445a8096c9cf91a11b2838ec45eab77dc173374972c0c4e30f51899"}, {"id": "u012", "kind": "table-row", "locator": "body:L26-L26", "preview": "| Tensor-map mode | Documented chunk mapping | | `CU_TENSOR_MAP_SWIZZLE_128B` | 16B chunks within a 128B span |", "sha256": "15d60e5fa8fa1cbec3b8b112e85c296d6ebcd56e383c5531aa287c8781c1655f"}, {"id": "u013", "kind": "table-row", "locator": "body:L27-L27", "preview": "| Tensor-map mode | Documented chunk mapping | | `CU_TENSOR_MAP_SWIZZLE_128B_ATOM_32B` | 32B chunks within a 128B span |", "sha256": "db5d4fd8a409c127453926ba71ebc67645d30c9ea6b67efe2479c169bee9ec8b"}, {"id": "u014", "kind": "table-row", "locator": "body:L28-L28", "preview": "| Tensor-map mode | Documented chunk mapping | | `CU_TENSOR_MAP_SWIZZLE_128B_ATOM_64B` | 64B chunks within a 128B span; support is operation/type-specific |", "sha256": "5a146edfdcdb88c80a6af0f66f4f7d6546adf933f80a2c3da2b7ebdb83f77016"}, {"id": "u015", "kind": "prose", "locator": "body:L30-L30", "preview": "These names define mappings, not universal datatype or tile-size recommendations. The tensor-map encoder imposes mode-, interleave-, datatype-, alignment-, and box-size constraints; for example, ordinary 128B modes require the inner boundin", "sha256": "5b12ffdaeb491fc7574d4dc1a7a88b320c50888483edabfce158c6b168f1d98a"}, {"id": "u016", "kind": "prose", "locator": "body:L34-L34", "preview": "CUTLASS 4.5.0 defines [`Swizzle`](https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/include/cute/swizzle.hpp):", "sha256": "e2de909b5c3702e90f3adc61d96f2aa80f06ace5b81ecfbbc16554b50cd26daa"}, {"id": "u017", "kind": "list-item", "locator": "body:L36-L36", "preview": "- `BBits` is the number of mask bits;", "sha256": "5af4a067d6c7da7415684db6889c71471be01324eefd2b56cba5ab0605b9048b"}, {"id": "u018", "kind": "list-item", "locator": "body:L37-L37", "preview": "- `MBase` is the number of least-significant address bits kept invariant; and", "sha256": "4baa3200cd5285f7c28a299e5577b8cdb19ad27e8d093c6208a6d0b33aad32b5"}, {"id": "u019", "kind": "list-item", "locator": "body:L38-L38", "preview": "- `SShift` is the distance between the two bit fields.", "sha256": "f28019ed4ea31c3b1d699597255eec3b2cb5bca744ea8d505802e5142e328689"}, {"id": "u020", "kind": "prose", "locator": "body:L40-L40", "preview": "For `Swizzle<3,4,3>`, three address bits beginning above the four invariant low bits are XORed with the three-bit field shifted by three positions. Equivalently, address bits 7:9 affect bits 4:6. This is an address transformation, not a gen", "sha256": "ffaae440bfd21e9f23db7977e6f25a0843206a5ebcccceb60b0f43b287fb38b6"}, {"id": "u021", "kind": "prose", "locator": "body:L42-L42", "preview": "CUTLASS also distinguishes a position-independent composed swizzle layout from a position-dependent swizzle pointer, because hardware swizzling depends on the shared-memory pointer address. Reuse a pinned complete layout/descriptor construc", "sha256": "7b7c33d7d8af0396265809e977fa799bb5bc22896a34d87bb0341b7036cd6fdf"}, {"id": "u022", "kind": "prose", "locator": "body:L46-L46", "preview": "For `cuTensorMapEncodeTiled`, record and validate all inputs rather than copying only the swizzle enumerator:", "sha256": "52e9d72146342e12fd2b1490c3761b731df5f3257a9203f475f988e31c7143c9"}, {"id": "u023", "kind": "list-item", "locator": "body:L48-L48", "preview": "1. Use a correctly aligned `CUtensorMap` object and global base pointer.", "sha256": "80e4aa5c2d38c0d179760ddbd62c54902ad761ce25c126a2a276d27c250232a0"}, {"id": "u024", "kind": "list-item", "locator": "body:L49-L49", "preview": "2. Supply rank-sized global dimensions, rank-minus-one byte strides (the fastest dimension is implicit), rank-sized box dimensions, and rank-sized element strides.", "sha256": "7a0cabb5ab90c8ad9fcb90c4a3b1abec710ea10e42d49e4b1ae7f089df312640"}, {"id": "u025", "kind": "list-item", "locator": "body:L50-L50", "preview": "3. Satisfy the global alignment, stride, box, interleave, datatype, and selected-swizzle constraints.", "sha256": "44c861550453d7469cfe6ab5f3f95ff727c04f68a1728ad416f78b1f7d2b1b7c"}, {"id": "u026", "kind": "list-item", "locator": "body:L51-L51", "preview": "4. Check the returned `CUresult`; do not launch with an output descriptor after encoding failed.", "sha256": "e35e8c5abe5a043f7a59a02a53ec58fbe4d9c5630736504600a4fdf081ea801d"}, {"id": "u027", "kind": "list-item", "locator": "body:L52-L52", "preview": "5. Use a shared-memory base and tcgen05 descriptor that represent the same mapping as the tensor map.", "sha256": "bcf3491b71148b2582b92aa3a39be1f27b68b24860545007d7c3fceec0df92d3"}, {"id": "u028", "kind": "prose", "locator": "body:L54-L54", "preview": "An invalid encoder combination can fail explicitly. A successfully encoded tensor map paired with the wrong consumer layout may instead read the wrong logical elements, so a successful API return is not a correctness oracle.", "sha256": "5bb6b35556ce52d1c4949afd7353eea6ed6e47d18072506fc1477c4c5e422a86"}, {"id": "u029", "kind": "prose", "locator": "body:L58-L58", "preview": "Gau Nernst's pinned B200 tutorial compares M=N=K=4096 kernels with PyTorch 2.9.1 and CUDA 13. Its 3D TMA version changes from a 16-byte inner tile with no swizzle to a 128-byte inner tile with 128B swizzling and matching tcgen05 descriptors", "sha256": "1442557fe3203a558928700307223fea48556a1423bb751823fe7a3ded0daec6"}, {"id": "u030", "kind": "table-row", "locator": "body:L62-L62", "preview": "| Tutorial version | Author-reported TFLOP/s | | v1b: 3D 16B TMA | 252.81 |", "sha256": "5ec52bb84daca8d9c4f9c330d7e0f75780b21a3674ec4e8c94329488f3030c29"}, {"id": "u031", "kind": "table-row", "locator": "body:L63-L63", "preview": "| Tutorial version | Author-reported TFLOP/s | | v2b: 3D 128B TMA plus 128B swizzle | 695.43 |", "sha256": "98a08ba0c20d778d67556cee3756e894c83682172bf48209e67517fa770e44e4"}, {"id": "u032", "kind": "prose", "locator": "body:L65-L65", "preview": "The combined change is approximately 2.75\u00d7 and the v2b endpoint is about 46% of the tutorial's 1506.74 TFLOP/s cuBLAS result. It is not a bank-conflict or swizzle-only ablation. The author notes that the earlier contiguous `8\u00d716B` tile migh", "sha256": "5c904bdadd0bdbec537b8da0a58b9e0eae1846a8130911c2933fdae421f9771c"}, {"id": "u033", "kind": "prose", "locator": "body:L69-L69", "preview": "For each candidate mapping:", "sha256": "7efda7c57692314fbd1a9fcdd587176241012dc24de5eecd9b854d1b2545ead1"}, {"id": "u034", "kind": "list-item", "locator": "body:L71-L71", "preview": "1. Keep global tensor, box, consumer instruction, tile shape, synchronization, and output oracle fixed. Change only the producer/consumer layout pair when an isolated comparison is possible.", "sha256": "85fa3e9198db89eed9cb12480937658783d70f38dd929725f0b8cf71e59328c7"}, {"id": "u035", "kind": "list-item", "locator": "body:L72-L72", "preview": "2. Check every tensor-map encoder result and run correctness tests that distinguish rows, columns, tiles, boundaries, and out-of-bounds fill. Do not rely on random uniform values that can hide permutations.", "sha256": "7c059219f0a0bef45641d0ee887c2f16f188c681fa01a4258e0aac06772ee9ef"}, {"id": "u036", "kind": "list-item", "locator": "body:L73-L73", "preview": "3. Use the installed Nsight Compute's `--query-metrics` or Memory Workload Analysis rather than assuming one metric name exists on every chip/tool version. Current documentation includes `l1tex__data_bank_conflicts_pipe_lsu.sum` and request", "sha256": "7081d7ec194ad85836cd0a830d113b7edfbf9547a09a3d54ef52e60a1a4dbf2e"}, {"id": "u037", "kind": "list-item", "locator": "body:L74-L74", "preview": "4. Compare executed shared-memory requests, ideal versus excessive wavefronts, bank conflicts, TMA behavior, total time, and occupancy. Zero observed CUDA-core bank conflicts neither proves the MMA descriptor matches nor explains a performa", "sha256": "16afbfd8bff05f99eb2faf2c5860e7a0786f1e850a6c43efa566f887bdcc24dd"}, {"id": "u038", "kind": "list-item", "locator": "body:L75-L75", "preview": "5. Record GPU, clocks, toolkit, profiler, compiler, exact addresses/alignment, layout types, descriptor fields, warmup, repetitions, and trial statistic.", "sha256": "09ceac52c96b6ed2ee30601a3682d58e7e976256b12796df1a192f98531b105e"}, {"id": "u039", "kind": "prose", "locator": "body:L77-L77", "preview": "Select the mapping that is legal and correct for both producer and every consumer, then retain it only where the controlled target workload improves. Do not infer the choice from architecture name, element width, or \u201cMMA versus non-MMA\u201d alo", "sha256": "d9fe95880e83d8c77d23da16c2196e5649905b51170a21816dd3bca04ba4e1a5"}], "confidence_claimed": "verified", "headings": ["Bank-conflict model", "Three mappings must agree", "CuTe address-bit form", "Tensor-map construction checklist", "Source-reported tutorial result", "Verification procedure"], "id": "technique-swizzling", "path": "wiki/techniques/swizzling.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-cuda-13-0-2-tma.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/cuda-c-programming-guide/index.html#asynchronous-data-copies-using-the-tensor-memory-accelerator-tma"}, {"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}], "risk_flags": ["ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-cuda-13-0-2-tma", "doc-ptx-isa-sm100", "blog-tcgen05-tutorial"], "title": "Shared Memory Swizzling", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} {"body_sha256": "ddee79f7472a6c3a775767f5f72596fd42a2d47df2856c53f96609d5e55ce71c", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "\u201cTile scheduling\u201d can refer to four different choices:", "sha256": "7b32fb0a937d7562fe4d500af4d3ac5ec152f9dc500779bc152c44b07f745659"}, {"id": "u002", "kind": "list-item", "locator": "body:L5-L5", "preview": "1. **Coordinate order:** a software mapping from a logical work index to `(tile_m, tile_n, ...)`, such as row-major, column-major, or a blocked/swizzled raster.", "sha256": "01bc0ff0de2d8faaa3f3ecc471ffde0939b45942f3e3b0e91c98312a9e644340"}, {"id": "u003", "kind": "list-item", "locator": "body:L6-L6", "preview": "2. **Resident-worker iteration:** whether a CTA handles one logical tile or repeatedly advances through work, for example by a static grid stride.", "sha256": "dfce2ca5b0bf2328975bc0b6e37cb7aa76a25fe5d9e46ec5b83b958e1df2ca8e"}, {"id": "u004", "kind": "list-item", "locator": "body:L7-L7", "preview": "3. **Work reassignment:** on SM100, Cluster Launch Control (CLC) lets a running worker cancel another grid entity that has not started and process the returned ClcID.", "sha256": "ad294691c329b879334b36f7addb42f5a735d07f182fa05a1568622e393ddf42"}, {"id": "u005", "kind": "list-item", "locator": "body:L8-L8", "preview": "4. **Problem decomposition:** Stream-K or Split-K can partition a tile's K work and then reduce partial results.", "sha256": "d68060e493898620463da03ac5643b456b664ef8a1a347e6ab695b45be712287"}, {"id": "u006", "kind": "prose", "locator": "body:L10-L10", "preview": "CLC does not accept a raster-order or swizzle-policy operand. CUTLASS applies those coordinate transforms in software to initial or returned coordinates. CLC also does not itself create K partitions or synthesize work beyond the launched gr", "sha256": "a5330ef0261e88c9d868a9e3dffeb4c1113bb325cdf01980a5cd9c1961898532"}, {"id": "u007", "kind": "prose", "locator": "body:L14-L14", "preview": "A flat row-major mapping is:", "sha256": "afb209da3a8a9396139e64e9bd7d8a4a8312b522317d1e515c25e13dd2696ed1"}, {"id": "u008", "kind": "code", "locator": "body:L16-L22", "preview": "```cuda __device__ void row_major_tile(int tile_idx, int tiles_n, int& tile_m, int& tile_n) { tile_m = tile_idx / tiles_n; tile_n = tile_idx % tiles_n; } ```", "sha256": "95c2ded3665bde2f3dcdf549fc05aa118b02258f5ab7143b40e7c212d48f4269"}, {"id": "u009", "kind": "prose", "locator": "body:L24-L24", "preview": "For a positive grid size, a static persistent worker can cover the flat index set without overlap:", "sha256": "cb8e18435ff65424ff8eecb1e050db1e5b8b4b92213bd48c80889d4c1f35059c"}, {"id": "u010", "kind": "code", "locator": "body:L26-L34", "preview": "```cuda for (int tile_idx = int(blockIdx.x); tile_idx < total_tiles; tile_idx += int(gridDim.x)) { int tile_m = tile_idx / tiles_n; int tile_n = tile_idx % tiles_n; compute_tile(tile_m, tile_n); } ```", "sha256": "e671351d81c0a79ee7e9273265fb228fe6e8819cee691435167ec958ed2cefed"}, {"id": "u011", "kind": "prose", "locator": "body:L36-L36", "preview": "Changing coordinate order changes the sequence of operand panels presented to the cache. It does not by itself prove a hit rate or speedup. A valid transform must first be shown to be in-bounds and bijective for edge groups; then its locali", "sha256": "6f6772443125529b78281c3269007c9d850c5072b7942599dbb6f86413914f7e"}, {"id": "u012", "kind": "prose", "locator": "body:L40-L40", "preview": "CUTLASS 4.5.0 documents this lifecycle for a CLC-backed persistent scheduler:", "sha256": "b9c7e8b605d64aac51b86256ed00d2cacf354e8b60280f9b4a025abfb2d71e25"}, {"id": "u013", "kind": "list-item", "locator": "body:L42-L42", "preview": "1. Launch the full problem grid and process the worker's initial `blockIdx` coordinate.", "sha256": "20d20b6a4cf0cb97dd6ffbb93b309b13a3241d6fcdaade9558afc3a1d4ac05c2"}, {"id": "u014", "kind": "list-item", "locator": "body:L43-L43", "preview": "2. Submit `clusterlaunchcontrol.try_cancel` against another not-yet-started grid entity. The asynchronous 16-byte response completes a transaction on an mbarrier.", "sha256": "a06961216d88c13174e5484e203558f45cf98ac572f6a213677701fbf730bf7d"}, {"id": "u015", "kind": "list-item", "locator": "body:L44-L44", "preview": "3. After the transaction completes, query whether cancellation succeeded. On success, decode and process the returned ClcID; for clusters, combine the returned first coordinate with the local cluster rank.", "sha256": "c7ab67729620e19fb98c56bbd8970ce7a7b024dd1e993362701587079589ba6d"}, {"id": "u016", "kind": "list-item", "locator": "body:L45-L45", "preview": "4. Treat an observed failed request as terminal for requests by that thread. Retrying from the same thread after failure is undefined.", "sha256": "c881c94cd5d38a82f418a9dd3cb9fd072897c3e38cf5208b7f2616ef9db28130"}, {"id": "u017", "kind": "prose", "locator": "body:L47-L47", "preview": "This can redistribute existing work when SM availability is uneven. It changes which resident worker handles an unlaunched ID, not the number of independent IDs in the grid. Software raster/swizzle may be applied consistently to both initia", "sha256": "1438dba392c4a82e98af96e55adc1a396e15067319beaa2511761701eca3fe2c"}, {"id": "u018", "kind": "prose", "locator": "body:L51-L51", "preview": "The public scheduler tags select implementation classes; users should prefer those tags over spelling detail types directly.", "sha256": "c9194ef2ae3478c668ebd794c2ebda157dd3b3fd60d8dff4d17922264e0e1c40"}, {"id": "u019", "kind": "table-row", "locator": "body:L55-L55", "preview": "| Public tag for `arch::Sm100` | Selected implementation | Scope | | `PersistentScheduler` (also the default `void` tag) | `PersistentTileSchedulerSm100` | CLC-backed persistent data-parallel scheduling |", "sha256": "436f3f1bdc780f0bcbba5b14d827591dbd44f9d903da5772fa655b3ac7a478c6"}, {"id": "u020", "kind": "table-row", "locator": "body:L56-L56", "preview": "| Public tag for `arch::Sm100` | Selected implementation | Scope | | `DynamicPersistentScheduler` | `PersistentTileSchedulerSm100` | Explicit dynamic route to the same SM100 implementation |", "sha256": "ce0c2c6ed629edf9fded3c6cfe63c01b6790dc62bc3d52c0b6858ae77a6c97b0"}, {"id": "u021", "kind": "table-row", "locator": "body:L57-L57", "preview": "| Public tag for `arch::Sm100` | Selected implementation | Scope | | `StaticPersistentScheduler` | `StaticPersistentTileScheduler100` | Static persistent scheduling |", "sha256": "59209b09fb5932bb87b80c3e1579fb729d01134b2ad07175e6e3765b61a7531a"}, {"id": "u022", "kind": "table-row", "locator": "body:L58-L58", "preview": "| Public tag for `arch::Sm100` | Selected implementation | Scope | | `StreamKScheduler` | `PersistentTileSchedulerSm100StreamK` | Parameterized data-parallel, Stream-K, or Split-K decomposition |", "sha256": "9dbd3bb085575ad9584218613f353eac0dcb7893ce1c68d03863b8011786992e"}, {"id": "u023", "kind": "table-row", "locator": "body:L59-L59", "preview": "| Public tag for `arch::Sm100` | Selected implementation | Scope | | `GroupScheduler` | `PersistentTileSchedulerSm100Group` | Grouped-problem route; in this tag it wraps the SM90-style static group scheduler |", "sha256": "0ecf5b3ad36b38af94e66745a93247520268644a8550382e195bb62071a6bf95"}, {"id": "u024", "kind": "prose", "locator": "body:L61-L61", "preview": "The SM100 Stream-K route accepts decomposition, split, raster, swizzle, and reduction settings. CUTLASS 4.5.0 has a deterministic lock/turnstile reduction and a nondeterministic atomic-workspace reduction; atomic accumulation is therefore n", "sha256": "1ff1f5194c712ba9bb2dae5da5ffaa2ba7c9fc8351523869f82bed91b7567175"}, {"id": "u025", "kind": "prose", "locator": "body:L65-L65", "preview": "For a deliberately simplified model with `T` equal-duration independent tiles, `W` available one-tile workers, and no K decomposition, write:", "sha256": "3184df242365781fc5c684291037f9127188eed995576762d2924ebb7cebccab"}, {"id": "u026", "kind": "code", "locator": "body:L67-L69", "preview": "```text T = qW + r, 0 <= r < W ```", "sha256": "9055386055d6219a3731bea281e8962958fea9a3ccddd596354c89e898f6cd16"}, {"id": "u027", "kind": "prose", "locator": "body:L71-L71", "preview": "If `r > 0`, the final partial wave has `r` active workers and occupancy `r / W`. If `r == 0` and `T > 0`, the final wave is full; reporting zero occupancy from the remainder alone is an error.", "sha256": "26ad025dfeb8dbf255a8583d4cc6332064e9c8889825eea498c1e7f0b3f5e5db"}, {"id": "u028", "kind": "prose", "locator": "body:L73-L73", "preview": "For `T = 150` and `W = 142`, the second wave contains 8 tiles, so 8 workers are active, 134 are idle, and wave occupancy is `8 / 142 = 5.63%` under those assumptions. CLC may change which workers receive those eight IDs and reduce delays ca", "sha256": "ea61406965d276427719ff49baf8c6394acc0d66f5e297ec2fe702333b6d0412"}, {"id": "u029", "kind": "list-item", "locator": "body:L77-L77", "preview": "1. Establish a correct unswizzled data-parallel mapping and verify exact tile coverage, including nondivisible edge groups.", "sha256": "e0aab377952cc5e5d1d899541ec67d739c272ed4e36f6ec16e835f1ce75277f5"}, {"id": "u030", "kind": "list-item", "locator": "body:L78-L78", "preview": "2. Sweep legal raster orders and swizzle sizes. Record kernel time plus L2 hit/sector traffic; do not infer eviction from coordinate order alone.", "sha256": "c06c06c02554411d3f9b47429f84f9a8e5441e79fc4dbdb332d4e538106ece78"}, {"id": "u031", "kind": "list-item", "locator": "body:L79-L79", "preview": "3. Compare static and CLC-backed persistence at identical tile, cluster, grid, and occupancy settings. Record successful/failed CLC requests and per-worker tile counts where instrumentation permits.", "sha256": "9b223b0ad22011e157f170461269e99ab6faeb845709c550a38ce039b5bb616d"}, {"id": "u032", "kind": "list-item", "locator": "body:L80-L80", "preview": "4. Test Stream-K separately with explicit decomposition, split, and reduction modes. Include workspace traffic, synchronization, determinism, and numerical tolerance.", "sha256": "636232c0d996791186f2d18f4d6911d3189712cd4ca86a0da1cd78474119981a"}, {"id": "u033", "kind": "list-item", "locator": "body:L81-L81", "preview": "5. Repeat across representative shapes and grouped-size distributions. Report regressions as well as wins; there is no universal best scheduler for all GEMMs, attention kernels, or MoE workloads.", "sha256": "ffb9c866fb101d1393c4c852ec2fb45c0f61356743566ff7791cda405e4c31ce"}, {"id": "u034", "kind": "prose", "locator": "body:L83-L83", "preview": "Without target-GPU measurements, this page makes no fixed claim for CLC acquisition latency, L2-miss reduction, or scheduler speedup.", "sha256": "2925a13ea32074b79278f44f396331abbada0e420d39689800102052fdcf8625"}], "confidence_claimed": "source-reported", "headings": ["Keep the Scheduling Layers Separate", "Exact Static Mappings", "SM100 CLC Work Acquisition", "CUTLASS 4.5.0 Scheduler Routes", "Tail Arithmetic", "Selection and Verification Workflow"], "id": "technique-tile-scheduling", "path": "wiki/techniques/tile-scheduling.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/cutlass-clc-documentation.md", "url": "https://github.com/NVIDIA/cutlass/blob/e406c186f510a15091cce01f782020ceb7ba8eb5/media/docs/cpp/blackwell_cluster_launch_control.md"}, {"path": "sources/docs/nvidia-cutlass-blackwell.md", "url": "https://github.com/NVIDIA/cutlass/tree/v4.5.0"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-cutlass-clc", "doc-cutlass-blackwell"], "title": "Tile Scheduling Strategies", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} {"body_sha256": "06ac3ebdfd34c7e7e4d4572a79c562a6b8733c0609fbdfe47db0573ad33e2290", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L5-L5", "preview": "A PTX vector load moves several typed elements into several registers with one instruction. This can reduce the number of load instructions issued by a thread, but PTX only says that vector loads *may* improve memory performance. Source-lev", "sha256": "3d99821d64a7e0fcf66a49561f2925641a983fbfdf7a6bdc83d77beacdcaa737"}, {"id": "u002", "kind": "prose", "locator": "body:L7-L7", "preview": "PTX ISA 9.0 permits the following global-memory forms on `sm_100`. The first accesses 16 bytes and the second accesses 32 bytes:", "sha256": "9f567d25863717dbe10d6a0c35951f195f02e73b69f3216826746dd1b16b7492"}, {"id": "u003", "kind": "code", "locator": "body:L9-L16", "preview": "```ptx .reg .b64 addr; .reg .u32 x<4>; .reg .u64 y<4>; ld.global.v4.u32 {x0, x1, x2, x3}, [addr]; ld.global.v4.u64 {y0, y1, y2, y3}, [addr]; ```", "sha256": "761519ce1a2f10c73aea258740cca70aac5fdf07bf0ff7a8a9d5b4984de5eace"}, {"id": "u004", "kind": "prose", "locator": "body:L18-L18", "preview": "The address must be naturally aligned to the total access size: 16 bytes for `v4.u32` and 32 bytes for `v4.u64`. A misaligned PTX address has undefined behavior; the ISA says it may have low address bits masked or fault. It does not specify", "sha256": "8cb92e31fee4aa36338676a252ea633a71739cc94656f4d19c25a20b651c5d11"}, {"id": "u005", "kind": "prose", "locator": "body:L20-L20", "preview": "The 256-bit `v4.b64` family is an SM100-or-newer feature. A 32-byte load contains 64 values when the payload is densely packed E2M1 at two 4-bit values per byte. That arithmetic says nothing about whether the width is profitable: unpack cos", "sha256": "33deaf137e1d64915539255a42f0a1496919c9302b9da264378d73eb537c5026"}, {"id": "u006", "kind": "prose", "locator": "body:L24-L24", "preview": "PTX distinguishes cache operators, eviction priorities, prefetch-size hints, and non-coherent read-only loads. Representative legal forms are:", "sha256": "0b9faa0d058fdadf045f326ea5dfcbab2f53d282615162309f99230b62e0cd20"}, {"id": "u007", "kind": "code", "locator": "body:L26-L31", "preview": "```ptx ld.global.L1::no_allocate.v4.u32 {x0, x1, x2, x3}, [addr]; ld.global.L1::evict_last.v4.u32 {x0, x1, x2, x3}, [addr]; ld.global.L2::256B.b32 x0, [addr]; ld.global.nc.b32 x0, [addr]; ```", "sha256": "ec4167ea0ba7152cb2179a67671479d0a2f4f407a28687bf87af2d87540b6f29"}, {"id": "u008", "kind": "table-row", "locator": "body:L35-L35", "preview": "| Form | Documented meaning | Limit on inference | | `L1::no_allocate` | L1 eviction-priority selection that may be applied | Does not guarantee an L1 bypass or a speedup for a streaming operand |", "sha256": "2780be407bbe96d5b106e3a0f5a703536fbb5465cf3d0d723517517d9918dcea"}, {"id": "u009", "kind": "table-row", "locator": "body:L36-L36", "preview": "| Form | Documented meaning | Limit on inference | | `L1::evict_last` | Requests the corresponding L1 eviction priority | Does not guarantee that a line remains resident |", "sha256": "1840c23696a0f16e1089d8d0649d755433db335252c3256c41f938763dc645d2"}, {"id": "u010", "kind": "table-row", "locator": "body:L37-L37", "preview": "| Form | Documented meaning | Limit on inference | | `L2::256B` | Hints that additional data of the stated size be prefetched into L2 | Is not a 256-byte load or a promotion guarantee |", "sha256": "06b2d39edee55c63dc2701519fcfeb993a3d00a45c4922a7b7b2e8dc80220e9b"}, {"id": "u011", "kind": "table-row", "locator": "body:L38-L38", "preview": "| Form | Documented meaning | Limit on inference | | `ld.global.nc` | Loads through a non-coherent read-only cache | Has architecture- and parallelism-dependent latency/throughput; it is not a general coherence optimization |", "sha256": "494ed829c67676d6c3b42e0d71f7c5924d0ae9f55c3a3caaf41be0d2bc6e093e"}, {"id": "u012", "kind": "prose", "locator": "body:L40-L40", "preview": "Cache-policy operands and prefetch-size qualifiers are performance hints and do not change the program's memory-consistency behavior. A reuse argument can motivate `evict_last`, and a one-pass stream can motivate `no_allocate`, but only a m", "sha256": "ad91b927be30bcfcfa70030224edc683a2b9de8c4f6af8a4d75c1c7d147b9724"}, {"id": "u013", "kind": "prose", "locator": "body:L44-L44", "preview": "For each candidate width and cache policy:", "sha256": "5e99de4759e9a94d2fa025e1ecf3ba20b1ee617b70fb262ad9d3cfde7d0408f5"}, {"id": "u014", "kind": "list-item", "locator": "body:L46-L46", "preview": "1. Prove base and per-thread address alignment for every executed access. Use a separate scalar or narrower-vector path for tails; validate minimum, maximum, and awkward sizes.", "sha256": "e6034cb185b56b2aa0b6088595b59f10d947cedfc609d0a6b028860c3041cbbf"}, {"id": "u015", "kind": "list-item", "locator": "body:L47-L47", "preview": "2. Hold the algorithm, mapping, decode, unrolling, launch shape, compiler, and inputs fixed while changing one load or cache choice. Compare identical correctness oracles before timing.", "sha256": "280cf574c975c962cfa7fb90943e03e73b2c5c96a11093dcab42ecce65d1bbb6"}, {"id": "u016", "kind": "list-item", "locator": "body:L48-L48", "preview": "3. Inspect generated PTX and SASS. Record actual load instructions, registers per thread, spill loads/stores, stack bytes, shared memory, and launch parameters. A source cast or inline-PTX block does not bypass compiler register allocation.", "sha256": "2fd170479b42be4da847ddabe8e7582f942debfed14c3050e1b31f7ad4941bdc"}, {"id": "u017", "kind": "list-item", "locator": "body:L49-L49", "preview": "4. Profile the matched variants. Check achieved bandwidth, requested versus transferred bytes, cache hit behavior, instruction issue/stalls, and active warps with metrics available on the installed Nsight Compute and GPU versions.", "sha256": "b7cee53be2ebd70f15f015073066cec635dbf6649f523057f03d1f5ac2e26ea9"}, {"id": "u018", "kind": "list-item", "locator": "body:L50-L50", "preview": "5. Use warmups, synchronization, repeated trials, and the same statistic for every production shape. Keep the change only where the declared metric improves without a correctness or resource regression.", "sha256": "36924cb17b1ccb46e32d0ddea78c1db22526b9871e47971d6fb917b03381c23f"}, {"id": "u019", "kind": "prose", "locator": "body:L52-L52", "preview": "Register caps and launch bounds are a separate variable. A nominal cap may be inert when natural allocation is lower, or it may trade registers for spills and extra instructions. Follow the resource and occupancy workflow in [Register Budge", "sha256": "cb3704e62d7d4725397d5fe9b2a2bcecbb903980445d842b50eb237c778bd327"}, {"id": "u020", "kind": "prose", "locator": "body:L56-L56", "preview": "Yue Zhang reports the following CUDA progression for GPU Mode Problem 1. These are author-reported endpoints without raw repeated-trial data or released complete submission code:", "sha256": "7b332e96d30f401078015efa8d7c6b2790a822f625eebb2f9b0a386d4e64a262"}, {"id": "u021", "kind": "table-row", "locator": "body:L60-L60", "preview": "| Stage | Combined change | Reported latency | | Initial CUDA | Naive hand-written path | about 2000 \u00b5s |", "sha256": "7c11921183bdbcbdb88258eff9a6b1d38d68d52b03ad0de2a6ddd30a6c5dbae1"}, {"id": "u022", "kind": "table-row", "locator": "body:L61-L61", "preview": "| Stage | Combined change | Reported latency | | CUDA optimization 1 | Coalescing, shared B, thread collaboration, warp reduction | about 443 \u00b5s |", "sha256": "98bdc14cff910536e6b2902f8e77cea8a8cf5401513c2fb653c934478cfaaf4b"}, {"id": "u023", "kind": "table-row", "locator": "body:L62-L62", "preview": "| Stage | Combined change | Reported latency | | CUDA optimization 2 | Remove shared B, per-thread tiles, `float4` loads, hardware intrinsics | about 39 \u00b5s |", "sha256": "3ccddb9db69afdec4a4b5e80e9279ce9e056ea7e8650ec0fc13b76260a91c8a3"}, {"id": "u024", "kind": "table-row", "locator": "body:L63-L63", "preview": "| Stage | Combined change | Reported latency | | CUDA optimization 3 | Vectorized PTX FP4 and scale decode | about 27 \u00b5s |", "sha256": "56cbb91dc0717675c0b075e8dc705e187892f107bb0bdbd355c5f7523fa1fb50"}, {"id": "u025", "kind": "table-row", "locator": "body:L64-L64", "preview": "| Stage | Combined change | Reported latency | | Parameter tuning | Threads per row and rows per block | about 26 \u00b5s |", "sha256": "20ec181854506636133e200346d437e30e574a0b0c4cf11fe5b13a2b148e2dde"}, {"id": "u026", "kind": "table-row", "locator": "body:L65-L65", "preview": "| Stage | Combined change | Reported latency | | ILP | Two tiles per loop iteration | about 22.9 \u00b5s |", "sha256": "65bd979b07f3eb7e869fa06be77e1a2ca90c448ece9242fb34e69473313aa428"}, {"id": "u027", "kind": "table-row", "locator": "body:L66-L66", "preview": "| Stage | Combined change | Reported latency | | Aggressive PTX fusion | Decode, scales, multiply, and accumulation in a larger PTX block | about 22.3 \u00b5s |", "sha256": "b88905c939fdf1e9c68c504abe3f1270ee932c5ff8c8adc0a20f9b4be6559dec"}, {"id": "u028", "kind": "prose", "locator": "body:L68-L68", "preview": "The submitted public-leaderboard score was 22.392 microseconds, the geometric mean over three benchmark shapes. The pinned task's theoretical model gives 8.622, 17.275, and 4.317 microseconds for those separate shapes; 8.622 microseconds is", "sha256": "59f3301f0b00e666daa6bccf8f7b771f1e6c77e3c16f6af8d5c0fb429fecef87"}, {"id": "u029", "kind": "prose", "locator": "body:L70-L70", "preview": "Amandeep Singh's attempts supply useful negative controls for the same task. Replacing two `uchar4` loads with one `uint2` load was reported 16\u201325% slower because extraction added instructions. Lowering `-maxrregcount` from 80 to 64 had no ", "sha256": "4f5cf4a108e0b33babb146932c0781c5c7e48b7b71672d1aee225a6f5226c9c6"}, {"id": "u030", "kind": "prose", "locator": "body:L72-L72", "preview": "Together, the reports justify testing vector width, decode organization, cache hints, and register limits. They do not establish that the widest load, a particular cache hint, or the lowest register cap is essential for GEMV, sub-byte arith", "sha256": "ef8f3594f82b50827d6fbc4ad359d74a683c0fee16b9c701e09124fd010fb6a4"}], "confidence_claimed": "verified", "headings": ["What a wider load establishes", "Cache controls are hints", "Reproducible selection procedure", "NVFP4 GEMV case study"], "id": "technique-vectorized-loads", "path": "wiki/techniques/vectorized-loads.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/docs/nvidia-cuda-register-controls.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html"}, {"path": "sources/blogs/yue-nvfp4-hackathon.md", "url": "https://yue-zhang-2025.github.io/2025/12/02/blackwell-nvfp4-kernel-hackathon-journey.html"}, {"path": "sources/blogs/amandeep-nvfp4-attempts.md", "url": "https://amandeepsp.github.io/blog/nvfp4-blackwell-gemv/"}, {"path": "sources/contests/gpu-mode-nvfp4/problem-1-gemv.md", "url": "https://github.com/gpu-mode/reference-kernels/tree/ae67948685dfccf54ae8374dc9402addb7aae4f6/problems/nvidia/nvfp4_gemv"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-ptx-isa-sm100", "doc-cuda-register-controls", "blog-yue-nvfp4", "blog-amandeep-nvfp4", "contest-gpumode-p1"], "title": "Vectorized Loads and Cache Hints", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} {"body_sha256": "9cff7bf68b39a3b6cef5c930d99bd525e7e3f232e933f44bee877722f8ecaa6c", "claim_units": [{"id": "u001", "kind": "prose", "locator": "body:L3-L3", "preview": "Warp specialization assigns different long-lived functions to disjoint warps in a CTA\u2014for example scheduler, TMA load, MMA control, softmax/correction, or epilogue work. It is a software organization for overlapping pipeline stages, not a f", "sha256": "3ba8e2f12c87ebb0e5dcc7c4505180f38dc8ff8beac7d1d793ab842d17117595"}, {"id": "u002", "kind": "prose", "locator": "body:L5-L5", "preview": "The architecture-level comparison is narrower:", "sha256": "f5fce66b629d64368a8ad2f3262804a0414fb44cf8e2cad0753ec4c379769a0d"}, {"id": "u003", "kind": "table-row", "locator": "body:L9-L9", "preview": "| Property | Hopper `wgmma.mma_async` | Blackwell `tcgen05.mma` | | Issue granularity | Warpgroup collective | One thread for `cta_group::1` or `cta_group::2` |", "sha256": "78012326dd8c87c08b14223c51cb0caca44f22f93153cad66a4f9db57ff7f46c"}, {"id": "u004", "kind": "table-row", "locator": "body:L10-L10", "preview": "| Property | Hopper `wgmma.mma_async` | Blackwell `tcgen05.mma` | | D accumulator | Per-thread registers | TMEM |", "sha256": "9d8f6c6479d0022158169c0c47e8d10179521df1707c1a368c754cc5a2a3a884"}, {"id": "u005", "kind": "table-row", "locator": "body:L11-L11", "preview": "| Property | Hopper `wgmma.mma_async` | Blackwell `tcgen05.mma` | | A/B movement | Explicit register/SMEM operands and producer work | Explicit SMEM descriptors or TMEM addresses and producer work |", "sha256": "562e8662604420b24685b52bbc320c26ac8d2af69a511ce0e6e24f94d573f776"}, {"id": "u006", "kind": "table-row", "locator": "body:L12-L12", "preview": "| Property | Hopper `wgmma.mma_async` | Blackwell `tcgen05.mma` | | Completion | WGMMA commit/wait groups | `tcgen05.commit` tied to an mbarrier, followed by a wait |", "sha256": "9adf92e37d53097b532873a08e3c9c1fb6d611b608108674deb1ff1e73395fe6"}, {"id": "u007", "kind": "prose", "locator": "body:L14-L14", "preview": "Single-thread issue reduces the number of threads needed to submit MMA instructions. It does not make operands move automatically, turn an entire warp into an ISA-level role, complete MMA synchronously, or choose how many epilogue warps a k", "sha256": "04a758f12de17c65173d342cb8f24e546f345fb1e1d1e7807da8597afc4c9033"}, {"id": "u008", "kind": "prose", "locator": "body:L18-L18", "preview": "There is no universal \u201cwarp 0 load, warp 1 MMA, warps 2\u201315 epilogue\u201d rule. Three primary implementations illustrate the range:", "sha256": "85be6e7da4f7a119a0268d227c104475692b8b379b0e80a19e54c5ecadf8d6d0"}, {"id": "u009", "kind": "table-row", "locator": "body:L22-L22", "preview": "| Implementation | Total warps and roles | | Gau Nernst tutorial v4, commit `3b90ac9b...` | 4 warps. An elected lane in warp 0 runs the TMA loop; an elected lane in warp 1 runs the MMA loop; after completion, all four warps participate in T", "sha256": "04f9a434f6d86ab29394b02159dd40778cf936ab3da64f4c13afea393bf664ee"}, {"id": "u010", "kind": "table-row", "locator": "body:L23-L23", "preview": "| Implementation | Total warps and roles | | CUTLASS 4.5.0 generic SM100 GEMM | One warp each for `MMA`, `Sched`, `MainloopLoad`, and `EpilogueLoad`, followed by `CollectiveEpilogue::ThreadCount / 32` epilogue warps. Optional work can make ", "sha256": "cdd3cb833878e188948d9fa2d121d4b3ac4deb77d7309f891e965cff438b590c"}, {"id": "u011", "kind": "table-row", "locator": "body:L24-L24", "preview": "| Implementation | Total warps and roles | | FlashInfer PR 1039 context FMHA | 16 warps: 0\u20133 `Softmax0`, 4\u20137 `Softmax1`, 8\u201311 correction, 12 MMA, 13 load, 14 epilogue, and 15 empty. The kernel has explicit pipelines between these roles. |", "sha256": "a1adbbd5ec3b0d9716988bb7715ca8f100d59539d41bb5bfa1bb704f18062ac2"}, {"id": "u012", "kind": "prose", "locator": "body:L26-L26", "preview": "FlashAttention-4's pinned SM100 forward source likewise starts from a 16-warp layout with two four-warp softmax groups, four correction warps, and dedicated MMA/load/epilogue IDs, then adjusts roles for configuration choices such as one Q s", "sha256": "85453f8d16b28425bdf3fabef09e21c9aa6f172224da9679fd66630af21a45ae"}, {"id": "u013", "kind": "prose", "locator": "body:L30-L30", "preview": "A warp-specialized implementation must prove each ownership handoff. For a reusable TMA-to-MMA stage:", "sha256": "901977eb7283ec2dcce8362743f2200d3162ca12f2f28ca013243cee361015f0"}, {"id": "u014", "kind": "list-item", "locator": "body:L32-L32", "preview": "1. Initialize the mbarrier objects with correct participant/transaction counts and publish them to the required threads and async proxies before use.", "sha256": "9b61910e09ca8df51a88598997a7a9ccf36c7e3a690f01531a4933e884dcbe37"}, {"id": "u015", "kind": "list-item", "locator": "body:L33-L33", "preview": "2. The producer acquires an empty stage, sets expected transaction bytes, and submits the TMA copies. The full/data-ready phase completes only after the expected async transactions complete.", "sha256": "6480a07f2fe51b271cadf66a732a388767b3bc00f4b7b5d69ae5081150fc2337"}, {"id": "u016", "kind": "list-item", "locator": "body:L34-L34", "preview": "3. The MMA controller waits for the full phase and applies the required cross-thread/proxy fence before issuing `tcgen05.mma` against that SMEM stage.", "sha256": "8c289184a687c5b6dc0a191ff7f7c890df85100e6e37b09bb7251409b2af34df"}, {"id": "u017", "kind": "list-item", "locator": "body:L35-L35", "preview": "4. Because MMA is asynchronous, `__syncwarp()` or an ordinary `mbarrier.arrive` after issue does not make the operands reusable. Commit prior tcgen05 work to an mbarrier and wait for completion before releasing or overwriting the stage.", "sha256": "e95047726737de8cacee65eccb16e9c33b29eb85a3f60bd462536178e6ccf539"}, {"id": "u018", "kind": "list-item", "locator": "body:L36-L36", "preview": "5. Before another role reads D from TMEM, complete the MMA sequence and apply the required `tcgen05.fence::before_thread_sync` / execution-ordering handoff / `tcgen05.fence::after_thread_sync` protocol. A fence orders operations; it is not ", "sha256": "c144a17005a0a78e9914bbf04f8e669ae34d5edc880fd085e95eb70a070df51c"}, {"id": "u019", "kind": "list-item", "locator": "body:L37-L37", "preview": "6. Keep TMEM allocated until every consumer finishes its `tcgen05.ld` sequence and associated waits, then deallocate with the required collective participation.", "sha256": "0c51e8e3c826e6843d2f29ea4dadff3c0aec26d0e2e3f62b60976ff405bc3857"}, {"id": "u020", "kind": "prose", "locator": "body:L39-L39", "preview": "Pipeline wrappers in CUTLASS/CuTe encode parts of these rules, but participant counts, initial phases, tails, and producer/consumer states remain configuration-specific. A short role-dispatch sketch is not a substitute for the complete pipe", "sha256": "89b2f283a5df3d8a1f56bf6af22a431d104476439e941498a2c99702463b4198"}, {"id": "u021", "kind": "prose", "locator": "body:L43-L43", "preview": "For the author's exact `M=N=K=4096` Modal B200 setup with PyTorch 2.9.1 and CUDA 13, tutorial v3 reports 939.61 TFLOP/s and v4 reports 1208.83 TFLOP/s after introducing warp specialization. That is a 269.22-TFLOP/s, approximately 28.65% ste", "sha256": "6621d3a76f8c031fec1925cda99d56cfdf593b282737d84897dd3277eeb93451"}, {"id": "u022", "kind": "prose", "locator": "body:L47-L47", "preview": "Use it as a candidate when independent pipeline roles have enough steady-state work to overlap and their register needs differ materially. Compare it with a temporally pipelined version at the same tile shape, stage count, CTA-group mode, a", "sha256": "1627600031e403cc9fddb951b81bf297d7caa64fb89255c37db446929c69e78b"}, {"id": "u023", "kind": "list-item", "locator": "body:L49-L49", "preview": "- time and achieved throughput across representative shapes;", "sha256": "8932c9b5bb7697e947fe26437053390822855b8d9afe20b6084fb4aa9ab88b26"}, {"id": "u024", "kind": "list-item", "locator": "body:L50-L50", "preview": "- per-role active/stall time and pipeline backpressure;", "sha256": "366a32c28b89526070b467e74080a4bf9f037e0f770c23f0a439da3094e25b40"}, {"id": "u025", "kind": "list-item", "locator": "body:L51-L51", "preview": "- register allocation, spills, shared memory, and occupancy;", "sha256": "e6d4d8f371f22ff49a3f1b804e0e88ce3d32209249dc5d9b04d60e0793d7cc8e"}, {"id": "u026", "kind": "list-item", "locator": "body:L52-L52", "preview": "- TMA/MMA/epilogue balance, including prologue and tail cost;", "sha256": "a94aa0efb4d9bedf92e4e7496eaef4e479680d6899227cdd262b800f28a4a47e"}, {"id": "u027", "kind": "list-item", "locator": "body:L53-L53", "preview": "- 1-SM versus 2-SM mode as an independent legal-shape/resource choice.", "sha256": "eecde6ff7378da0ec52ac7970d16eca0580901fa5268e91d4e8dac14c05f5c0f"}, {"id": "u028", "kind": "prose", "locator": "body:L55-L55", "preview": "More role warps are not automatically useful for a more complex epilogue, and one MMA-control warp does not limit the issuer to one outstanding asynchronous MMA operation. Choose participant counts from the concrete collective and measureme", "sha256": "1f5622ad117b53bfcf321b00eccfc035da8b2468ad897d4b62c3e0c2ffd988a7"}, {"id": "u029", "kind": "prose", "locator": "body:L59-L59", "preview": "[`artifacts/kernels/warp-specialization/full/`](../../artifacts/kernels/warp-specialization/full/) contains the captured FlashInfer PR 1039 mainloop file; its bytes match the SHA-256 recorded in `PROVENANCE.yaml` and the duplicate captured ", "sha256": "2f40dc82464551aafd8efd85eb3d27958f6b2220bb352611772e73c0998ec3c0"}, {"id": "u030", "kind": "prose", "locator": "body:L61-L61", "preview": "[`artifacts/kernels/warp-specialization/variants/`](../../artifacts/kernels/warp-specialization/variants/) is explicitly `derived` teaching material and must not be cited as upstream code or as a complete safe kernel.", "sha256": "9b8752deff98b9437a0ad5787db5b0941b2c657c3ca2c5ca0a4a02cab448e48e"}, {"id": "u031", "kind": "prose", "locator": "body:L63-L63", "preview": "Retrieve the page plus both artifact modes with:", "sha256": "aa7fe4f8fefb6af22e88bcb2774f2b7880991739ab30daee394e2957a5c2b167"}, {"id": "u032", "kind": "code", "locator": "body:L65-L67", "preview": "```bash conda run -n base python scripts/get_page.py technique-warp-specialization --include-code ```", "sha256": "5304a268eeb5c50c0c72b18eb3a7eeea5bb8b36817dd99d35deb384c5254f4bf"}], "confidence_claimed": "source-reported", "headings": ["Definition and ISA Boundary", "Version-Pinned Role Maps", "Correct Synchronization Obligations", "Source-Reported Tutorial Result", "When to Test Warp Specialization", "Local Evidence Bundle"], "id": "technique-warp-specialization", "path": "wiki/techniques/warp-specialization.md", "performance_claim_count": 0, "resolved_sources": [{"path": "sources/docs/nvidia-ptx-isa-sm100.md", "url": "https://docs.nvidia.com/cuda/archive/13.0.2/parallel-thread-execution/index.html"}, {"path": "sources/docs/nvidia-cutlass-blackwell.md", "url": "https://github.com/NVIDIA/cutlass/tree/v4.5.0"}, {"path": "sources/blogs/tcgen05-tutorial.md", "url": "https://gau-nernst.github.io/tcgen05/"}, {"path": "sources/prs/flashinfer/PR-1039.md", "revision": "9a05c92a", "url": "https://github.com/flashinfer-ai/flashinfer/pull/1039"}], "risk_flags": ["code", "ordering", "table"], "schema_version": "kernel-wiki-verifier-queue/v2", "source_ids": ["doc-ptx-isa-sm100", "doc-cutlass-blackwell", "blog-tcgen05-tutorial", "pr-flashinfer-1039"], "title": "Warp Specialization on Blackwell", "type": "technique", "unresolved_source_ids": [], "version_sensitive": null} diff --git a/verification/evidence/MANIFEST.json b/verification/evidence/MANIFEST.json new file mode 100644 index 000000000..dcf979f00 --- /dev/null +++ b/verification/evidence/MANIFEST.json @@ -0,0 +1,1200 @@ +{ + "format": "kernelwiki-local-evidence-v1", + "copy_policy": "Exact evidence files and named subtrees are copied without nested Git metadata; oversized whole-checkout searches use immutable upstream commit trees.", + "copied_unique_files": 908, + "copied_bytes": 19362141, + "entries": [ + { + "source_key": "cutlass-audit.8aj07j/repo/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu", + "path": "verification/evidence/local-snapshots/cutlass-audit.8aj07j/repo/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu", + "kind": "file", + "files": 1, + "bytes": 24372, + "sha256": "92d348e998984d3c8a912ab63cbda547ac1c366c37c095c0d77b11b0e60eac7e", + "revisions": [ + "NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3" + ] + }, + { + "source_key": "cutlass-audit.8aj07j/repo/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu", + "path": "verification/evidence/local-snapshots/cutlass-audit.8aj07j/repo/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu", + "kind": "file", + "files": 1, + "bytes": 39692, + "sha256": "52f2966221fc4f9a2e4a6acfab487ce3860376fe769dc43789d6159ea3427965", + "revisions": [ + "NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3", + "same commit" + ] + }, + { + "source_key": "cutlass-audit.8aj07j/repo/include/cutlass/gemm/dispatch_policy.hpp", + "path": "verification/evidence/local-snapshots/cutlass-audit.8aj07j/repo/include/cutlass/gemm/dispatch_policy.hpp", + "kind": "file", + "files": 1, + "bytes": 74799, + "sha256": "fcce5fffb3118b15fea5aa59e39bea15ddd18c28c858b434ced889045968117c", + "revisions": [ + "NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3", + "same commit" + ] + }, + { + "source_key": "cutlass-audit.8aj07j/repo/include/cutlass/gemm/kernel/gemm_grouped.h", + "path": "verification/evidence/local-snapshots/cutlass-audit.8aj07j/repo/include/cutlass/gemm/kernel/gemm_grouped.h", + "kind": "file", + "files": 1, + "bytes": 14394, + "sha256": "fe46d9f4b0cd414674118f6c2e5460b5484e47d9e5909640fd56abc248fadf9b", + "revisions": [ + "NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3" + ] + }, + { + "source_key": "cutlass-audit.8aj07j/repo/media/docs/cpp/efficient_gemm.md", + "path": "verification/evidence/local-snapshots/cutlass-audit.8aj07j/repo/media/docs/cpp/efficient_gemm.md", + "kind": "file", + "files": 1, + "bytes": 20550, + "sha256": "44003151268090f0f1c3be9518164ec33eb04a030c9ed731a408f2f94886aaa5", + "revisions": [ + "NVIDIA/cutlass e05f953a5b3d38adc240df2ff928e0421c2abba3" + ] + }, + { + "source_key": "kernelwiki-amandeep-nvfp4.html", + "path": "verification/evidence/local-snapshots/kernelwiki-amandeep-nvfp4.html", + "kind": "file", + "files": 1, + "bytes": 85583, + "sha256": "66314df12ecfc8f18c4a2f234251e66a405a6e93160093e56e16e54a1ab002e8", + "revisions": [ + "author URL fetched 2026-08-08", + "author post fetched 2026-08-08" + ] + }, + { + "source_key": "kernelwiki-amandeepsp-cuda/nvfp4/gemv", + "path": "verification/evidence/local-snapshots/kernelwiki-amandeepsp-cuda/nvfp4/gemv", + "kind": "directory", + "files": 15, + "bytes": 133940, + "sha256": "5fc0de14d7e01f069ab1968918b7ddd5987bfe7ecb92237b59d22fd619cd8ee3", + "revisions": [ + "amandeepsp/cuda 44513ac7d5bbd1cf8109cab952844adac5b6c551" + ] + }, + { + "source_key": "kernelwiki-cutlass-4.5.0", + "resolution": "immutable-upstream-tree", + "url": "https://github.com/NVIDIA/cutlass/tree/e406c186f510a15091cce01f782020ceb7ba8eb5", + "revisions": [ + "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5" + ], + "reason": "Complete checkout omitted by the repository size guard." + }, + { + "source_key": "kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue", + "kind": "directory", + "files": 132, + "bytes": 2460160, + "sha256": "045ce1a87f147ac148324ce23a84f1632e18f7d2ad416e286abafe22bfc038e9", + "revisions": [ + "same tag" + ] + }, + { + "source_key": "kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/builders/sm100_builder.inl", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/builders/sm100_builder.inl", + "kind": "file", + "files": 1, + "bytes": 80376, + "sha256": "f80c7d8bfb018e1011909f50e42c9941d0c5f502491871be43fa2b1e291173f6", + "revisions": [ + "NVIDIA/cutlass v4.5.0 e406c186", + "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5" + ] + }, + { + "source_key": "kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/collective_builder.hpp", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/collective/collective_builder.hpp", + "kind": "file", + "files": 1, + "bytes": 4701, + "sha256": "6bc1b7415e6618eb9ec63bc3e23c7f0cc58b2e4a83aa904d156a4a9bb111aaf3", + "revisions": [ + "same tag" + ] + }, + { + "source_key": "kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/operations.hpp", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/operations.hpp", + "kind": "file", + "files": 1, + "bytes": 25127, + "sha256": "cff76a8ec17a778f2924d0b64d8686b084395d24b2bb8c98a31fb34b9a94784d", + "revisions": [ + "CUTLASS 4.5.0 e406c186", + "CUTLASS v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5", + "NVIDIA/cutlass v4.5.0 e406c186", + "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5" + ] + }, + { + "source_key": "kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/sm100_visitor_store_tma_warpspecialized.hpp", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/fusion/sm100_visitor_store_tma_warpspecialized.hpp", + "kind": "file", + "files": 1, + "bytes": 30229, + "sha256": "37d91bf3c2a34289062c36fec10804ad8f03af809b3a9482ecb6e8b630dd8a5a", + "revisions": [ + "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5" + ] + }, + { + "source_key": "kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/activation.h", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/epilogue/thread/activation.h", + "kind": "file", + "files": 1, + "bytes": 25139, + "sha256": "4574c38e89158b4741aa2e698e7c99fd65f23a023ac777ee5becba298daf199e", + "revisions": [ + "same tag" + ] + }, + { + "source_key": "kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/sm100_gemm_tma_warpspecialized.hpp", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/sm100_gemm_tma_warpspecialized.hpp", + "kind": "file", + "files": 1, + "bytes": 42793, + "sha256": "847f884a93c82329fc0b3869473943361185baa47d7d3a10f3357c5dbe9e1c58", + "revisions": [ + "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5" + ] + }, + { + "source_key": "kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/sm100_tile_scheduler_group.hpp", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/sm100_tile_scheduler_group.hpp", + "kind": "file", + "files": 1, + "bytes": 13018, + "sha256": "6e156e55cd75aea88def562b78859708efd50b934a7eb3b43a970bb40ba4b5f0", + "revisions": [ + "same tag" + ] + }, + { + "source_key": "kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/sm100_tile_scheduler_stream_k.hpp", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/sm100_tile_scheduler_stream_k.hpp", + "kind": "file", + "files": 1, + "bytes": 39453, + "sha256": "31655341e689e4a28098089013980fa6a436f53b1d652773fbb5bd456f20867b", + "revisions": [ + "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5" + ] + }, + { + "source_key": "kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/tile_scheduler.hpp", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/tile_scheduler.hpp", + "kind": "file", + "files": 1, + "bytes": 12395, + "sha256": "acc90548b9e2b19f944764ced57e1459d5c2ed7e118d6a1af476add26c3d5e73", + "revisions": [ + "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5" + ] + }, + { + "source_key": "kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/tile_scheduler_detail.hpp", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/tile_scheduler_detail.hpp", + "kind": "file", + "files": 1, + "bytes": 3880, + "sha256": "2fcac9ce11418ddcfd5c4890516b97cdab3accca5f3715841ff1e8a6cf49efe4", + "revisions": [ + "same tag" + ] + }, + { + "source_key": "kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/tile_scheduler_params.h", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0-full/include/cutlass/gemm/kernel/tile_scheduler_params.h", + "kind": "file", + "files": 1, + "bytes": 97023, + "sha256": "ef48a12e8920183e88259d0b685279c2232fc2fb12c4fb4db7e8d0fbfdc019e9", + "revisions": [ + "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5" + ] + }, + { + "source_key": "kernelwiki-cutlass-4.5.0/CHANGELOG.md", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0/CHANGELOG.md", + "kind": "file", + "files": 1, + "bytes": 121159, + "sha256": "fdff6cf6e305073ee71cac9219c200cbea2056f6b4a93c9a5fbb36ef34bd6d2f", + "revisions": [ + "NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5" + ] + }, + { + "source_key": "kernelwiki-cutlass-4.5.0/README.md", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0/README.md", + "kind": "file", + "files": 1, + "bytes": 33190, + "sha256": "5b52ccda5896f63381acd62a6fba378d86a311f50f0fdbeaaf6e2a8243d182e5", + "revisions": [ + "NVIDIA/cutlass v4.5.0 e406c186", + "NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5" + ], + "restored_from_equivalent_checkout": "kernelwiki-cutlass-dRC5i4/cutlass/README.md" + }, + { + "source_key": "kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm", + "kind": "directory", + "files": 12, + "bytes": 296550, + "sha256": "119faba72d67ec09a4cc677dcbf6bc285a7d69fe70c70c5d14ae45d3297fd05f", + "revisions": [ + "NVIDIA/cutlass v4.5.0 e406c186", + "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5" + ] + }, + { + "source_key": "kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/README.md", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/README.md", + "kind": "file", + "files": 1, + "bytes": 913, + "sha256": "a49a9b0f9f9f0fcf182205c9ecd48fa8f8442060aec59cfec950007448509e34", + "revisions": [ + "NVIDIA/cutlass v4.5.0 e406c186" + ] + }, + { + "source_key": "kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py", + "kind": "file", + "files": 1, + "bytes": 14445, + "sha256": "68ba1aac829c07a9af7a9d09b180e4390d3d2b8a4cac0d61bf72aefadfd4cf8f", + "revisions": [ + "NVIDIA/cutlass v4.5.0 e406c186", + "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5", + "NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5" + ] + }, + { + "source_key": "kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_1.py", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_1.py", + "kind": "file", + "files": 1, + "bytes": 18459, + "sha256": "aa3dd5358661e1730d25b425c7153384985d0e9f0fded19c860190b68bb8e50a", + "revisions": [ + "same tag" + ] + }, + { + "source_key": "kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_2.py", + "kind": "file", + "files": 1, + "bytes": 23703, + "sha256": "21044b5fca133ce0968777d1b042da4576ded18312e61d91789e8217d6f26d2d", + "revisions": [ + "CUTLASS v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5", + "NVIDIA/cutlass v4.5.0 e406c186", + "NVIDIA/cutlass v4.5.0 e406c186f510a15091cce01f782020ceb7ba8eb5", + "NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5", + "same tag" + ] + }, + { + "source_key": "kernelwiki-cutlass-4.5.0/python/CuTeDSL/cutlass/cute/arch/tmem.py", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0/python/CuTeDSL/cutlass/cute/arch/tmem.py", + "kind": "file", + "files": 1, + "bytes": 7450, + "sha256": "5547791b19ee31b00a8d9c6eea42196e94412276fc97a35a5f7f263e8e0729db", + "revisions": [ + "same tag" + ] + }, + { + "source_key": "kernelwiki-cutlass-4.5.0/python/CuTeDSL/cutlass/cute/nvgpu/tcgen05/mma.py", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-4.5.0/python/CuTeDSL/cutlass/cute/nvgpu/tcgen05/mma.py", + "kind": "file", + "files": 1, + "bytes": 61717, + "sha256": "682d33493a07f5aaf562956b7637647b0b6789ff7963355652d5a8103bc2f82b", + "revisions": [ + "NVIDIA/cutlass v4.5.0 e406c186", + "NVIDIA/cutlass v4.5.0, e406c186f510a15091cce01f782020ceb7ba8eb5" + ] + }, + { + "source_key": "kernelwiki-cutlass-dRC5i4/cutlass", + "resolution": "immutable-upstream-tree", + "url": "https://github.com/NVIDIA/cutlass/tree/e406c186f510a15091cce01f782020ceb7ba8eb5", + "revisions": [ + "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5" + ], + "reason": "Complete checkout omitted by the repository size guard." + }, + { + "source_key": "kernelwiki-cutlass-dRC5i4/cutlass/CMakeLists.txt", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-dRC5i4/cutlass/CMakeLists.txt", + "kind": "file", + "files": 1, + "bytes": 49341, + "sha256": "ef9a892eda0e4cbb93d2ed7e671cf4ed317c14f6c4e4b19cda6c4304336f8331", + "revisions": [ + "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5" + ] + }, + { + "source_key": "kernelwiki-cutlass-dRC5i4/cutlass/examples/70_blackwell_gemm/70_blackwell_fp16_gemm.cu", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-dRC5i4/cutlass/examples/70_blackwell_gemm/70_blackwell_fp16_gemm.cu", + "kind": "file", + "files": 1, + "bytes": 17908, + "sha256": "cbf06d38b63b481965f5b4121a5ef562147f8b56a1888eaabb851b24754452d6", + "revisions": [ + "CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5" + ] + }, + { + "source_key": "kernelwiki-cutlass-dRC5i4/cutlass/examples/cute/tutorial/blackwell/02_mma_tma_sm100.cu", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-dRC5i4/cutlass/examples/cute/tutorial/blackwell/02_mma_tma_sm100.cu", + "kind": "file", + "files": 1, + "bytes": 35790, + "sha256": "4db6545363d1c9a81e13b7e187396306a3a6d58b3501c0f3cccaa429515c451e", + "revisions": [ + "CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5" + ] + }, + { + "source_key": "kernelwiki-cutlass-dRC5i4/cutlass/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-dRC5i4/cutlass/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_0.py", + "kind": "file", + "files": 1, + "bytes": 14445, + "sha256": "68ba1aac829c07a9af7a9d09b180e4390d3d2b8a4cac0d61bf72aefadfd4cf8f", + "revisions": [ + "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5" + ] + }, + { + "source_key": "kernelwiki-cutlass-dRC5i4/cutlass/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_6.py", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-dRC5i4/cutlass/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_gemm/fp16_gemm_6.py", + "kind": "file", + "files": 1, + "bytes": 35621, + "sha256": "8cb350408a7b163063ded5e05aa62fcb683265731c5dda59b56175f62dcf67e1", + "revisions": [ + "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5" + ] + }, + { + "source_key": "kernelwiki-cutlass-dRC5i4/cutlass/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_tma/tma_v0.py", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-dRC5i4/cutlass/examples/python/CuTeDSL/cute/blackwell/tutorial/tutorial_tma/tma_v0.py", + "kind": "file", + "files": 1, + "bytes": 16836, + "sha256": "de96579e7434a3325345cca7d5e8380712ab9ceb41cf60b2b515bb8b0bbc3a57", + "revisions": [ + "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5" + ] + }, + { + "source_key": "kernelwiki-cutlass-dRC5i4/cutlass/include/cute/arch/mma_sm90_gmma.hpp", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-dRC5i4/cutlass/include/cute/arch/mma_sm90_gmma.hpp", + "kind": "file", + "files": 1, + "bytes": 969217, + "sha256": "3c7eb20a05841cb2e8ebf6de16c1c2451b1a315e9398a66f9ef45f5b69639108", + "revisions": [ + "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5", + "CUTLASS commit e406c186f510a15091cce01f782020ceb7ba8eb5" + ] + }, + { + "source_key": "kernelwiki-cutlass-dRC5i4/cutlass/include/cutlass/arch/grid_dependency_control.h", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-dRC5i4/cutlass/include/cutlass/arch/grid_dependency_control.h", + "kind": "file", + "files": 1, + "bytes": 4565, + "sha256": "2f07b329ba4282fd200ff4bf7fe00343ca2b9b5b026db2eea49bc209281367c8", + "revisions": [ + "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5" + ] + }, + { + "source_key": "kernelwiki-cutlass-dRC5i4/cutlass/python/CuTeDSL/cutlass/utils/tmem_allocator.py", + "path": "verification/evidence/local-snapshots/kernelwiki-cutlass-dRC5i4/cutlass/python/CuTeDSL/cutlass/utils/tmem_allocator.py", + "kind": "file", + "files": 1, + "bytes": 26259, + "sha256": "21a8c121828499e7b1b08235f25a10a23f0f5aa587ce5c4266738baaf5b5201f", + "revisions": [ + "CUTLASS 4.5.0 commit e406c186f510a15091cce01f782020ceb7ba8eb5" + ] + }, + { + "source_key": "kernelwiki-deepgemm-pinned/README.md", + "path": "verification/evidence/local-snapshots/kernelwiki-deepgemm-pinned/README.md", + "kind": "file", + "files": 1, + "bytes": 12644, + "sha256": "df82e7d6979e49e864cd0c8b2389648ebfa226b9f4b331646d7d5cdc4951ee63", + "revisions": [ + "891d57b4", + "891d57b4db1071624b5c8fa0d1e51cb317fa709f", + "DeepGEMM 891d57b4", + "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f", + "DeepGEMM commit 891d57b4", + "DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f", + "DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f; SHA-256 tied to local clone", + "commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f" + ] + }, + { + "source_key": "kernelwiki-deepgemm-pinned/csrc/apis/gemm.hpp", + "path": "verification/evidence/local-snapshots/kernelwiki-deepgemm-pinned/csrc/apis/gemm.hpp", + "kind": "file", + "files": 1, + "bytes": 35851, + "sha256": "b8edecebf2b943faf85a5c3c5d70ed7e4e2ea27be8fa63013213964460a93776", + "revisions": [ + "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f", + "DeepGEMM commit 891d57b", + "DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f" + ] + }, + { + "source_key": "kernelwiki-deepgemm-pinned/csrc/jit/compiler.hpp", + "path": "verification/evidence/local-snapshots/kernelwiki-deepgemm-pinned/csrc/jit/compiler.hpp", + "kind": "file", + "files": 1, + "bytes": 15844, + "sha256": "aec70f54bffc693a935ef9775ee9b40bc8857dcb8b464e6732f39a86e3292d62", + "revisions": [ + "DeepGEMM commit 891d57b", + "DeepGEMM commit 891d57b4db1071624b5c8fa0d1e51cb317fa709f" + ] + }, + { + "source_key": "kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh", + "path": "verification/evidence/local-snapshots/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm100_fp8_gemm_1d1d.cuh", + "kind": "file", + "files": 1, + "bytes": 33297, + "sha256": "84ecd247b5fb6500fd31e464388b281b1425d105451dbf8dd3c189945dcf9384", + "revisions": [ + "891d57b4", + "DeepGEMM commit 891d57b4", + "same commit" + ] + }, + { + "source_key": "kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm100_fp8_mqa_logits.cuh", + "path": "verification/evidence/local-snapshots/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm100_fp8_mqa_logits.cuh", + "kind": "file", + "files": 1, + "bytes": 19667, + "sha256": "66629c2eeca18e419edc76d57fc5c50f761db8de52bbc3f80c747b8bd873b464", + "revisions": [ + "DeepGEMM 891d57b4", + "DeepGEMM 891d57b4db1071624b5c8fa0d1e51cb317fa709f" + ] + }, + { + "source_key": "kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh", + "path": "verification/evidence/local-snapshots/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/impls/sm90_fp8_gemm_1d1d.cuh", + "kind": "file", + "files": 1, + "bytes": 18701, + "sha256": "4a3e39d4a1dd25c0be2bc2ff586c5f50192a91d5a0d1bcda5e03e20ef094e2db", + "revisions": [ + "891d57b4", + "DeepGEMM commit 891d57b4", + "same commit" + ] + }, + { + "source_key": "kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/ptx/tcgen05.cuh", + "path": "verification/evidence/local-snapshots/kernelwiki-deepgemm-pinned/deep_gemm/include/deep_gemm/ptx/tcgen05.cuh", + "kind": "file", + "files": 1, + "bytes": 5412, + "sha256": "5fc33904690100c4330442a4564a68447cacf362df2299063cebed245a6aed55", + "revisions": [ + "891d57b4", + "same commit" + ] + }, + { + "source_key": "kernelwiki-deepgemm-pinned/tests/generators.py", + "path": "verification/evidence/local-snapshots/kernelwiki-deepgemm-pinned/tests/generators.py", + "kind": "file", + "files": 1, + "bytes": 18594, + "sha256": "d8673d8c5ff8d0f457a033664389c3becba9b118d3156be793f735e08c80fa23", + "revisions": [ + "same commit" + ] + }, + { + "source_key": "kernelwiki-deepgemm-pinned/tests/test_fp8_fp4.py", + "path": "verification/evidence/local-snapshots/kernelwiki-deepgemm-pinned/tests/test_fp8_fp4.py", + "kind": "file", + "files": 1, + "bytes": 12567, + "sha256": "ff778e59c9428db29cf8681d2d1a432e5036e008a997ac2a4822a2371adb6448", + "revisions": [ + "same commit" + ] + }, + { + "source_key": "kernelwiki-deepseek-v32/DeepSeek_V3_2.pdf", + "path": "verification/evidence/local-snapshots/kernelwiki-deepseek-v32/DeepSeek_V3_2.pdf", + "kind": "file", + "files": 1, + "bytes": 526189, + "sha256": "f8ef04ddf50f924a7d251861d4453d7605ff5711f3941ae8320e1ea780dc3041", + "revisions": [ + "87e509a2", + "DeepSeek-V3.2-Exp 87e509a2", + "DeepSeek-V3.2-Exp 87e509a2e5a100d221c97df52c6e8be7835f0057", + "same model commit" + ] + }, + { + "source_key": "kernelwiki-deepseek-v32/README.md", + "path": "verification/evidence/local-snapshots/kernelwiki-deepseek-v32/README.md", + "kind": "file", + "files": 1, + "bytes": 6899, + "sha256": "dffcdf358a42599945d49293a4f210dbe589141085207b76c081c9ace1f8fd74", + "revisions": [ + "DeepSeek-V3.2-Exp 87e509a2", + "DeepSeek-V3.2-Exp 87e509a2e5a100d221c97df52c6e8be7835f0057" + ] + }, + { + "source_key": "kernelwiki-deepseek-v32/inference/config_671B_v3.2.json", + "path": "verification/evidence/local-snapshots/kernelwiki-deepseek-v32/inference/config_671B_v3.2.json", + "kind": "file", + "files": 1, + "bytes": 605, + "sha256": "f9fe074cdef8fdcc0feabbe635ff7c072a65d273a42a6bc207fe425439dff835", + "revisions": [ + "87e509a2", + "DeepSeek-V3.2-Exp 87e509a2", + "same commit" + ] + }, + { + "source_key": "kernelwiki-deepseek-v32/inference/model.py", + "path": "verification/evidence/local-snapshots/kernelwiki-deepseek-v32/inference/model.py", + "kind": "file", + "files": 1, + "bytes": 38639, + "sha256": "bfe5b89186b579910b6d59fa7e0cf1f7a75ceb4ec5d0f5b54f0ccbe77cdb6a63", + "revisions": [ + "DeepSeek-V3.2-Exp 87e509a2", + "same", + "same commit" + ] + }, + { + "source_key": "kernelwiki-ext-cfx", + "path": "verification/evidence/local-snapshots/kernelwiki-ext-cfx", + "kind": "directory", + "files": 64, + "bytes": 323571, + "sha256": "b47e74c1439c86a410424a9ccd65e600f82f84ceb80097efa8409e55861435f5", + "revisions": [ + "ColfaxResearch/cfx-article-src fbecfed88de2e4246f104a023188ba722937c5fc" + ] + }, + { + "source_key": "kernelwiki-ext-cfx/streamk", + "path": "verification/evidence/local-snapshots/kernelwiki-ext-cfx/streamk", + "kind": "directory", + "files": 14, + "bytes": 100419, + "sha256": "e1070b8506e453f756c93c06ca6a04e1deb18b8b9ff1778774a2a81c6ec3561d", + "revisions": [ + "ColfaxResearch/cfx-article-src fbecfed88de2e4246f104a023188ba722937c5fc" + ] + }, + { + "source_key": "kernelwiki-ext-colfax-cutlass", + "path": "verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass", + "kind": "directory", + "files": 33, + "bytes": 230244, + "sha256": "5570f3e9adeb1bcdcef79389b0afb4058bc047ccf0043819dd1652136a6ba485", + "revisions": [ + "ColfaxResearch/cutlass-kernels 84f0802e2b4a1bf068ac70359f20ffdb368c8f6a" + ] + }, + { + "source_key": "kernelwiki-ext-colfax-cutlass/src/fmha-pipeline", + "path": "verification/evidence/local-snapshots/kernelwiki-ext-colfax-cutlass/src/fmha-pipeline", + "kind": "directory", + "files": 16, + "bytes": 116387, + "sha256": "43c0da2e3c769bdee69a9907b05853cf45972b4985c6dc1f3809437fbadb1f36", + "revisions": [ + "ColfaxResearch/cutlass-kernels 84f0802e2b4a1bf068ac70359f20ffdb368c8f6a" + ] + }, + { + "source_key": "kernelwiki-ext-load-store", + "path": "verification/evidence/local-snapshots/kernelwiki-ext-load-store", + "kind": "directory", + "files": 9, + "bytes": 12430, + "sha256": "6c52f9b07070f0b78cf9252e3f62421dfc89e8862b5e3a1e7791c5ed712c8c64", + "revisions": [ + "simveit/load_and_store 05d828cf910dd43f0053ddbbe4744218a06e9d7f" + ] + }, + { + "source_key": "kernelwiki-ext-load-store/README.md", + "path": "verification/evidence/local-snapshots/kernelwiki-ext-load-store/README.md", + "kind": "file", + "files": 1, + "bytes": 68, + "sha256": "c6c0520e43374a9e63dfbe224bf155212a5dde6a7f045317bb4040e8c3bf478a", + "revisions": [ + "simveit/load_and_store 05d828cf910dd43f0053ddbbe4744218a06e9d7f" + ] + }, + { + "source_key": "kernelwiki-ext-nvidia-samples", + "path": "verification/evidence/local-snapshots/kernelwiki-ext-nvidia-samples", + "kind": "directory", + "files": 355, + "bytes": 4862571, + "sha256": "c469dca084ed3f5d332f44efde8db249b2be91a26643956ff5788de81ff7624e", + "revisions": [ + "NVIDIA-developer-blog/code-samples 3350d216083a902ccbf5b31665e3b82096a75b55" + ] + }, + { + "source_key": "kernelwiki-ext-transpose", + "path": "verification/evidence/local-snapshots/kernelwiki-ext-transpose", + "kind": "directory", + "files": 9, + "bytes": 40440, + "sha256": "5f41fb866c85850e3d807501b8de07ab487ec46e05ba7054ca1c36cb941ed31e", + "revisions": [ + "simveit/effective_transpose 994b2b5acaa67f80e411df3e8274b6ae13fd1949" + ] + }, + { + "source_key": "kernelwiki-fa4-current/flash_attn/cute/utils.py", + "path": "verification/evidence/local-snapshots/kernelwiki-fa4-current/flash_attn/cute/utils.py", + "kind": "file", + "files": 1, + "bytes": 36954, + "sha256": "60eca870f3dcb80e3693428600c2a9b5c38db48109d230c2f83dc1bc6b1c6952", + "revisions": [ + "a369df7" + ] + }, + { + "source_key": "kernelwiki-flash-attention-pinned/flash_attn/cute/", + "path": "verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute", + "kind": "directory", + "files": 58, + "bytes": 2188345, + "sha256": "0001c3c88fcd7663b9ae43e18dff00d1d232b77ceeac4e97b8ba1eab4bbeb088", + "revisions": [ + "Dao-AILab commit a369df707e1980fb328abcc1733e3457ec10155f", + "Dao-AILab/flash-attention commit a369df707e1980fb328abcc1733e3457ec10155f", + "same commit" + ] + }, + { + "source_key": "kernelwiki-flash-attention-pinned/flash_attn/cute/flash_bwd_sm100.py", + "path": "verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_bwd_sm100.py", + "kind": "file", + "files": 1, + "bytes": 190166, + "sha256": "ff3549914cc41dd8cd7c05daa1334dfe674e81a170379676fb63847608456532", + "revisions": [ + "Dao-AILab commit a369df707e1980fb328abcc1733e3457ec10155f", + "Dao-AILab/flash-attention commit a369df707e1980fb328abcc1733e3457ec10155f", + "same commit" + ] + }, + { + "source_key": "kernelwiki-flash-attention-pinned/flash_attn/cute/flash_fwd_sm100.py", + "path": "verification/evidence/local-snapshots/kernelwiki-flash-attention-pinned/flash_attn/cute/flash_fwd_sm100.py", + "kind": "file", + "files": 1, + "bytes": 165752, + "sha256": "f5928a15a7aa559084d14aa12cc2c95efdc89c7b3a1f0d20219b24103a32456f", + "revisions": [ + "Dao-AILab commit a369df707e1980fb328abcc1733e3457ec10155f", + "Dao-AILab/flash-attention a369df707e1980fb328abcc1733e3457ec10155f", + "Dao-AILab/flash-attention commit a369df707e1980fb328abcc1733e3457ec10155f" + ] + }, + { + "source_key": "kernelwiki-flashinfer-bench-starter-kit/EVALUATION.md", + "path": "verification/evidence/local-snapshots/kernelwiki-flashinfer-bench-starter-kit/EVALUATION.md", + "kind": "file", + "files": 1, + "bytes": 6264, + "sha256": "79897fe80c0b857c49ee466a37ea91bbd5f131d553eb38a93fd9c02311881ccb", + "revisions": [ + "75ccd05cafceb0fd1f86be4cd0f2117249463c66", + "commit 75ccd05cafceb0fd1f86be4cd0f2117249463c66", + "flashinfer-bench-starter-kit 75ccd05cafceb0fd1f86be4cd0f2117249463c66" + ] + }, + { + "source_key": "kernelwiki-flashinfer-pinned/benchmarks/bench_trtllm_gen_fused_moe_autotuner.py", + "path": "verification/evidence/local-snapshots/kernelwiki-flashinfer-pinned/benchmarks/bench_trtllm_gen_fused_moe_autotuner.py", + "kind": "file", + "files": 1, + "bytes": 23232, + "sha256": "690812f92e10f4a9dd9f7c9871ec95621513261f288aa704c00f9823063efff4", + "revisions": [ + "same commit" + ] + }, + { + "source_key": "kernelwiki-flashinfer-pinned/flashinfer/fused_moe/core.py", + "path": "verification/evidence/local-snapshots/kernelwiki-flashinfer-pinned/flashinfer/fused_moe/core.py", + "kind": "file", + "files": 1, + "bytes": 212449, + "sha256": "0509690184cfb3cf6f7ebb6d4f7ae26dbf5b5b7c254990848923e089432e4c76", + "revisions": [ + "7f614b86470180bab2d22e36fd1775791c6bf3e6", + "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6", + "same commit" + ] + }, + { + "source_key": "kernelwiki-flashinfer-pinned/flashinfer/fused_moe/runners.py", + "path": "verification/evidence/local-snapshots/kernelwiki-flashinfer-pinned/flashinfer/fused_moe/runners.py", + "kind": "file", + "files": 1, + "bytes": 88239, + "sha256": "24b8395f47fc57a1978898036d507607095ddbae2815c16d6b70e77af842404f", + "revisions": [ + "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6" + ] + }, + { + "source_key": "kernelwiki-flashinfer-pinned/flashinfer/gdn_prefill.py", + "path": "verification/evidence/local-snapshots/kernelwiki-flashinfer-pinned/flashinfer/gdn_prefill.py", + "kind": "file", + "files": 1, + "bytes": 23810, + "sha256": "e4475a2edc830b97d6bc27f13980e2f5951d7d97eb13547f539acf94e2a2132d", + "revisions": [ + "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6" + ] + }, + { + "source_key": "kernelwiki-flashinfer-pinned/flashinfer/trace/templates/gdn.py", + "path": "verification/evidence/local-snapshots/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/gdn.py", + "kind": "file", + "files": 1, + "bytes": 27184, + "sha256": "3fee96d66fc429bdafe10baefcf72a110da0c06ed3dd9cbb93db054d8c332aef", + "revisions": [ + "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6", + "same commit" + ] + }, + { + "source_key": "kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py", + "path": "verification/evidence/local-snapshots/kernelwiki-flashinfer-pinned/flashinfer/trace/templates/moe.py", + "kind": "file", + "files": 1, + "bytes": 143394, + "sha256": "e41c50149adfc31448c6453b92eb50228a70034f3355393d776f50e20a94408e", + "revisions": [ + "7f614b86470180bab2d22e36fd1775791c6bf3e6", + "FlashInfer 7f614b8", + "FlashInfer 7f614b86470180bab2d22e36fd1775791c6bf3e6" + ] + }, + { + "source_key": "kernelwiki-flashmla-pinned/README.md", + "path": "verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/README.md", + "kind": "file", + "files": 1, + "bytes": 10657, + "sha256": "46c001c53800f47148918f4fc2969102492dd5164f710fc015db1622f6f9929a", + "revisions": [ + "DeepSeek FlashMLA commit 71c7379", + "DeepSeek FlashMLA commit 71c737929f2567bd0a094ae140f8f60f390b1232; SHA-256 46c001c53800f47148918f4fc2969102492dd5164f710fc015db1622f6f9929a", + "DeepSeek commit 71c7379", + "DeepSeek commit 71c7379; SHA-256 46c001c53800f47148918f4fc2969102492dd5164f710fc015db1622f6f9929a", + "FlashMLA 71c7379", + "FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232", + "same commit" + ] + }, + { + "source_key": "kernelwiki-flashmla-pinned/csrc/sm100", + "path": "verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm100", + "kind": "directory", + "files": 51, + "bytes": 633496, + "sha256": "b5dfb24085774dea62b94229e6dff78e62aca1ff1d5f83947e88cfda1096a1ff", + "revisions": [ + "DeepSeek FlashMLA commit 71c7379" + ] + }, + { + "source_key": "kernelwiki-flashmla-pinned/csrc/sm90/decode", + "path": "verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/csrc/sm90/decode", + "kind": "directory", + "files": 16, + "bytes": 120822, + "sha256": "1d3aa508c0fd7151032f7ea4678ca3c200d31306640a78b6a21612633a71b4a5", + "revisions": [ + "DeepSeek FlashMLA commit 71c7379", + "DeepSeek commit 71c7379" + ] + }, + { + "source_key": "kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py", + "path": "verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/flash_mla/flash_mla_interface.py", + "kind": "file", + "files": 1, + "bytes": 19102, + "sha256": "802da6b2db58ca3455e4f48bb860e448fd16b12f817fbd94d900ea7a51334bd6", + "revisions": [ + "DeepSeek FlashMLA commit 71c7379", + "DeepSeek commit 71c7379", + "FlashMLA 71c7379", + "FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232" + ] + }, + { + "source_key": "kernelwiki-flashmla-pinned/tests/quant.py", + "path": "verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/tests/quant.py", + "kind": "file", + "files": 1, + "bytes": 8162, + "sha256": "84604c879bc69e715de334978b0d00a84d1d5feb3ed1a9407b504959f1368114", + "revisions": [ + "DeepSeek commit 71c7379", + "same commit" + ] + }, + { + "source_key": "kernelwiki-flashmla-pinned/tests/ref.py", + "path": "verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/tests/ref.py", + "kind": "file", + "files": 1, + "bytes": 4556, + "sha256": "7634a9db7020552a3588f02959c4dbfbd124e3aaf07d369f2d3adc617e0b5069", + "revisions": [ + "FlashMLA 71c737929f2567bd0a094ae140f8f60f390b1232", + "same commit" + ] + }, + { + "source_key": "kernelwiki-flashmla-pinned/tests/test_flash_mla_sparse_decoding.py", + "path": "verification/evidence/local-snapshots/kernelwiki-flashmla-pinned/tests/test_flash_mla_sparse_decoding.py", + "kind": "file", + "files": 1, + "bytes": 14317, + "sha256": "4380079dd6e0fabb4f8ed86522dc0f51054678930f7a728cf1fadf5e81bf6983", + "revisions": [ + "same", + "same commit" + ] + }, + { + "source_key": "kernelwiki-gdn-pinned/README.md", + "path": "verification/evidence/local-snapshots/kernelwiki-gdn-pinned/README.md", + "kind": "file", + "files": 1, + "bytes": 10655, + "sha256": "6ea2aa74c32e739c7b1d20d97f43569effb1eed848d811c2576cb608546a3029", + "revisions": [ + "NVlabs/GatedDeltaNet b53d6d3a161267432a79c1c04af69fa52bddc921" + ] + }, + { + "source_key": "kernelwiki-gdn-pinned/lit_gpt/gated_delta_net.py", + "path": "verification/evidence/local-snapshots/kernelwiki-gdn-pinned/lit_gpt/gated_delta_net.py", + "kind": "file", + "files": 1, + "bytes": 9514, + "sha256": "8aaee1dcff184742148bb694730e5c1da0d246bbba92d2a0a462e77963f35fbc", + "revisions": [ + "NVlabs b53d6d3a161267432a79c1c04af69fa52bddc921", + "NVlabs/GatedDeltaNet b53d6d3a161267432a79c1c04af69fa52bddc921" + ] + }, + { + "source_key": "kernelwiki-gdn-pinned/lit_gpt/gated_delta_rule_ops/chunk.py", + "path": "verification/evidence/local-snapshots/kernelwiki-gdn-pinned/lit_gpt/gated_delta_rule_ops/chunk.py", + "kind": "file", + "files": 1, + "bytes": 28437, + "sha256": "f93055afdc7fcfaae212bbf4d2d81a17e86b28fed3f3266668953b240af8edfa", + "revisions": [ + "NVlabs b53d6d3a161267432a79c1c04af69fa52bddc921", + "NVlabs/GatedDeltaNet b53d6d3a161267432a79c1c04af69fa52bddc921" + ] + }, + { + "source_key": "kernelwiki-learn-cuda-3b90ac9b", + "path": "verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b", + "kind": "directory", + "files": 86, + "bytes": 1424319, + "sha256": "ad1f3e22972360db67500b9b440666ff9431eeb8a2f161a9ed7fad10455335d0", + "revisions": [ + "gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573" + ] + }, + { + "source_key": "kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/README.md", + "path": "verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/README.md", + "kind": "file", + "files": 1, + "bytes": 3490, + "sha256": "c83c60fb74ee13fdf5f96537aa619767ecd2295a7e7ac3135719e61db3bb447c", + "revisions": [ + "gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573" + ] + }, + { + "source_key": "kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/common.h", + "path": "verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/common.h", + "kind": "file", + "files": 1, + "bytes": 4033, + "sha256": "48c596666a12be8a4e970449f91617bb8586b9c76eec7ecb94e8e61b3fc06018", + "revisions": [ + "gau-nernst/learn-cuda 3b90ac9b", + "gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573" + ] + }, + { + "source_key": "kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v3.cu", + "path": "verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v3.cu", + "kind": "file", + "files": 1, + "bytes": 9198, + "sha256": "f09baabed99b21a1b62a513ea5af00b3799ebee9f34fffbb8545fdcc43bad4d6", + "revisions": [ + "gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573" + ] + }, + { + "source_key": "kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v4.cu", + "path": "verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v4.cu", + "kind": "file", + "files": 1, + "bytes": 9319, + "sha256": "6cba13d44a3f8d64d63a8a8ce25ea6b0b2fcbc394e29afa19942e48f7a88fc56", + "revisions": [ + "gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573" + ] + }, + { + "source_key": "kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v6.cu", + "path": "verification/evidence/local-snapshots/kernelwiki-learn-cuda-3b90ac9b/02e_matmul_sm100/matmul_v6.cu", + "kind": "file", + "files": 1, + "bytes": 16620, + "sha256": "d8ca43d1248cf60d39249a7cbff6cb1c9cb9ea89c0939ea68f71d40be360fb12", + "revisions": [ + "gau-nernst/learn-cuda 3b90ac9b3f624bdf1f6f78d02dcd533675d36573", + "gau-nernst/learn-cuda 3b90ac9b697a1180d06a0fba42431534410d1949" + ] + }, + { + "source_key": "kernelwiki-mlstm-kernels/README.md", + "path": "verification/evidence/local-snapshots/kernelwiki-mlstm-kernels/README.md", + "kind": "file", + "files": 1, + "bytes": 13792, + "sha256": "ba1bb55d7c7cc53fe98b4682f3d281fc2a13bc8a182aca5e9b30584f1dc30b2e", + "revisions": [ + "NX-AI/mlstm_kernels 5b98ff8e2bec189b3d3c249405bab5149564d6f8" + ] + }, + { + "source_key": "kernelwiki-ptx-1302.html", + "path": "verification/evidence/local-snapshots/kernelwiki-ptx-1302.html", + "kind": "file", + "files": 1, + "bytes": 3313063, + "sha256": "4f2a4739fcd6c636cf2b8a59d3ab881b23c351ef2f3978eefc960ae23244b79d", + "revisions": [ + "CUDA 13.0.2, PTX ISA 9.0 fetched 2026-08-08", + "PTX ISA 9.0" + ] + }, + { + "source_key": "kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/reference.py", + "path": "verification/evidence/local-snapshots/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/reference.py", + "kind": "file", + "files": 1, + "bytes": 7900, + "sha256": "e344e0fc3ff88e2dff0ac69a8b3d41a500143007c0c76915e4e3d049aa90722f", + "revisions": [ + "gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c", + "same commit" + ] + }, + { + "source_key": "kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml", + "path": "verification/evidence/local-snapshots/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/task.yml", + "kind": "file", + "files": 1, + "bytes": 2520, + "sha256": "2bf4e5c04cc8a9c283e519437c8324ca9265d9c12d0dfe2b76dc9f0542ada051", + "revisions": [ + "51e22db671d36c1c76091c43c36a44546ba324a1", + "gpu-mode/reference-kernels 51e22db671d36c1c76091c43c36a44546ba324a1", + "gpu-mode/reference-kernels c5b2f7c062d5015f29c3a1043cfd04954397944c" + ] + }, + { + "source_key": "kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/template.py", + "path": "verification/evidence/local-snapshots/kernelwiki-reference-kernels/problems/nvidia/nvfp4_dual_gemm/template.py", + "kind": "file", + "files": 1, + "bytes": 1398, + "sha256": "7817d0aee927bc3a519515514ea651f63d1468eca6e9981c5b107dec9c8277c0", + "revisions": [ + "same commit" + ] + }, + { + "source_key": "kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/reference.py", + "path": "verification/evidence/local-snapshots/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/reference.py", + "kind": "file", + "files": 1, + "bytes": 6185, + "sha256": "c212965ddb50681b6a01a34149f51e18a9bd07848baf4f98b954e3b305c17dbd", + "revisions": [ + "gpu-mode/reference-kernels ae679486", + "same commit" + ] + }, + { + "source_key": "kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/task.py", + "path": "verification/evidence/local-snapshots/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/task.py", + "kind": "file", + "files": 1, + "bytes": 327, + "sha256": "b92c1d0b37c31ec9fe5afe1f737f0cfa37e1f93381509d049de91a0c9b7659e1", + "revisions": [ + "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6" + ] + }, + { + "source_key": "kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/task.yml", + "path": "verification/evidence/local-snapshots/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/task.yml", + "kind": "file", + "files": 1, + "bytes": 2246, + "sha256": "404ff2c9904789c822e248f3c9e83947a8c32a68a1692ed254a0c51b89edb5f9", + "revisions": [ + "gpu-mode/reference-kernels ae679486", + "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6", + "same commit" + ] + }, + { + "source_key": "kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/template.py", + "path": "verification/evidence/local-snapshots/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemm/template.py", + "kind": "file", + "files": 1, + "bytes": 1008, + "sha256": "82bb6b5454889f5bce83f50776b9325339c2aed6e24647a8bea5e19357ea91e1", + "revisions": [ + "same commit" + ] + }, + { + "source_key": "kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/reference.py", + "path": "verification/evidence/local-snapshots/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/reference.py", + "kind": "file", + "files": 1, + "bytes": 6425, + "sha256": "fdba0ac0f671dc6d7fb0581d37fb5c2a9b356f4ad65875c836fc5256fea4d69d", + "revisions": [ + "gpu-mode/reference-kernels ae679486", + "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6", + "same commit" + ] + }, + { + "source_key": "kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/task.py", + "path": "verification/evidence/local-snapshots/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/task.py", + "kind": "file", + "files": 1, + "bytes": 317, + "sha256": "9ee8aa1f20dc1db4d8d3e7a900c6f8c9009ff3fe7a00ea5c7171c4ef5fb78e72", + "revisions": [ + "gpu-mode/reference-kernels ae679486", + "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6" + ] + }, + { + "source_key": "kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/task.yml", + "path": "verification/evidence/local-snapshots/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/task.yml", + "kind": "file", + "files": 1, + "bytes": 2083, + "sha256": "1316c0ab5ef0f7ed27d4310a89dbbdf8263694d8d876dfc2b019dc846af651fc", + "revisions": [ + "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6", + "same commit" + ] + }, + { + "source_key": "kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/template.py", + "path": "verification/evidence/local-snapshots/kernelwiki-reference-kernels/problems/nvidia/nvfp4_gemv/template.py", + "kind": "file", + "files": 1, + "bytes": 1075, + "sha256": "2d24e25aebcec8cb9ac0858d4a1025888e4bc7c3621e80ca4e58e4661c0aaba5", + "revisions": [ + "gpu-mode/reference-kernels ae679486", + "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6" + ] + }, + { + "source_key": "kernelwiki-reference-kernels/problems/nvidia/nvfp4_group_gemm/task.yml", + "path": "verification/evidence/local-snapshots/kernelwiki-reference-kernels/problems/nvidia/nvfp4_group_gemm/task.yml", + "kind": "file", + "files": 1, + "bytes": 3629, + "sha256": "9e3416d049a889464788573abf6ffafa0b5f9ead355cc2e5aed6f74d6d4d836d", + "revisions": [ + "gpu-mode/reference-kernels ae67948685dfccf54ae8374dc9402addb7aae4f6" + ] + } + ] +} diff --git a/verification/evidence/README.md b/verification/evidence/README.md new file mode 100644 index 000000000..7218fd79f --- /dev/null +++ b/verification/evidence/README.md @@ -0,0 +1,30 @@ +# Repository-local verification evidence + +`local-snapshots/` contains the exact files and named source subtrees used by +the verifier reports. Nested Git metadata is intentionally excluded. + +`MANIFEST.json` records the stable repository-relative path, revision labels, +file count, byte count, and SHA-256 digest for every copied evidence target. +Two complete CUTLASS checkout roots that were used only for repository-wide +searches are represented by the immutable upstream commit tree recorded in the +manifest; copying both full worktrees would have added roughly 200 MiB. + +`historical-artifacts/` preserves six small files that remediation later +deleted or replaced but that original-review receipts still cite. Its separate +manifest records the source Git revision and SHA-256 for every restored file. + +Exact host checks cited by receipts live in +`verification/tools/check_verification_evidence.py`. The aggregate audit checks structured +paths and commands, and can also fetch every current evidence URL and validate +NVIDIA fragments: + +```bash +python3 verification/tools/audit_verification_evidence.py --network +``` + +Do not edit copied source files in place. Re-run the migration from the pinned +source checkouts when evidence must be refreshed, then validate the result: + +```bash +python3 verification/tools/migrate_verification_evidence.py --check +``` diff --git a/verification/evidence/historical-artifacts/MANIFEST.json b/verification/evidence/historical-artifacts/MANIFEST.json new file mode 100644 index 000000000..3bba55fab --- /dev/null +++ b/verification/evidence/historical-artifacts/MANIFEST.json @@ -0,0 +1,48 @@ +{ + "files": [ + { + "bytes": 1453, + "path": "verification/evidence/historical-artifacts/artifacts/kernels/flashmla/variants/01-mla-decode-inner-loop.cu", + "revision": "2777d18", + "sha256": "4f494dc2d282468966a1e2028bc50d0ef0c364b34d24f75ee881608ddef2df69", + "source_path": "artifacts/kernels/flashmla/variants/01-mla-decode-inner-loop.cu" + }, + { + "bytes": 1451, + "path": "verification/evidence/historical-artifacts/artifacts/kernels/gated-delta-net/variants/01-chunk-parallel-prefill-reference-pytorch.py", + "revision": "2777d18", + "sha256": "d7e74b02df4974fb082cacbae8a5182dd2c40cb1c13fd1d74505b51cfb15481b", + "source_path": "artifacts/kernels/gated-delta-net/variants/01-chunk-parallel-prefill-reference-pytorch.py" + }, + { + "bytes": 1610, + "path": "verification/evidence/historical-artifacts/artifacts/kernels/gated-delta-net/variants/02-triton-decode-step-kernel-streaming.py", + "revision": "2777d18", + "sha256": "d5eef14bdbc44e50784c0ca34e7041dcad2151960a85feddba62b097e20079ad", + "source_path": "artifacts/kernels/gated-delta-net/variants/02-triton-decode-step-kernel-streaming.py" + }, + { + "bytes": 39691, + "path": "verification/evidence/historical-artifacts/artifacts/kernels/gated-dual-gemm/full/vllm-PR-23696-gated-dual-gemm.patch", + "revision": "2777d18", + "sha256": "033d30b15af4f2050189c2e76ba61959e1e5d4bd049dc901fd56f68890b891a0", + "source_path": "artifacts/kernels/gated-dual-gemm/full/vllm-PR-23696-gated-dual-gemm.patch" + }, + { + "bytes": 1484, + "path": "verification/evidence/historical-artifacts/artifacts/kernels/gated-dual-gemm/full/blackwell-cutlass-schedules-and-tma.cu", + "revision": "2777d18", + "sha256": "e720037bbdb5ee5054259a331cdaf489e7135a44038d66d1fd9d9e5998b75255", + "source_path": "artifacts/kernels/gated-dual-gemm/full/blackwell-cutlass-schedules-and-tma.cu" + }, + { + "bytes": 976, + "path": "verification/evidence/historical-artifacts/artifacts/kernels/gated-dual-gemm/variants/01-fused-epilogue-swiglu-skeleton.cu", + "revision": "2777d18", + "sha256": "85e158c184546912579ac5270451471a96f135efa3584bafbf45064e5fb7e223", + "source_path": "artifacts/kernels/gated-dual-gemm/variants/01-fused-epilogue-swiglu-skeleton.cu" + } + ], + "schema_version": "kernelwiki-historical-evidence/v1", + "source_revision": "2777d18" +} diff --git a/verification/evidence/historical-artifacts/artifacts/kernels/flashmla/variants/01-mla-decode-inner-loop.cu b/verification/evidence/historical-artifacts/artifacts/kernels/flashmla/variants/01-mla-decode-inner-loop.cu new file mode 100644 index 000000000..345d7df55 --- /dev/null +++ b/verification/evidence/historical-artifacts/artifacts/kernels/flashmla/variants/01-mla-decode-inner-loop.cu @@ -0,0 +1,34 @@ +// provenance: derived from blog-flashmla, pr-cutlass-2472; not upstream code +// origin: wiki/kernels/flashmla.md Phase 3 variant (copied from extracted blog bundle) + +// Extracted from sources/blogs/flashmla.md by scripts/extract_blog_code.py +// Heading: ## Key Code > ### MLA decode inner loop +// Original fence language: cuda +// See artifacts/blogs/flashmla/code/PROVENANCE.yaml for origin + license metadata. + +// MLA collapses K and V into a shared latent matrix of head-dim Dc=128. +// On decode (one query vector against N KV tokens) we alternate TMA load, +// wgmma/tcgen05 into the q@K^T accumulator, online softmax, and the second +// accumulator against V. +constexpr int Dc = 128; // latent head dim +constexpr int BLOCK_N = 64; // paged KV block size +float acc[Dc] = {0}; +float max_val = -INFINITY; +float l = 0.f; +for (int n0 = 0; n0 < seqlen; n0 += BLOCK_N) { + tma_load(smem_kv, KV_pages + n0); + cp_async_wait(); + float scores[BLOCK_N]; + wgmma_or_tcgen05_mma(scores, q, smem_kv); // q @ K^T + float new_max = warp_reduce_max(scores, BLOCK_N); + float scale = expf(max_val - new_max); + for (int j = 0; j < Dc; j++) acc[j] *= scale; + l *= scale; + for (int j = 0; j < BLOCK_N; j++) { + float p = expf(scores[j] - new_max); + l += p; + for (int d = 0; d < Dc; d++) acc[d] += p * smem_kv[j * Dc + d]; + } + max_val = new_max; +} +for (int d = 0; d < Dc; d++) O[d] = acc[d] / l; diff --git a/verification/evidence/historical-artifacts/artifacts/kernels/gated-delta-net/variants/01-chunk-parallel-prefill-reference-pytorch.py b/verification/evidence/historical-artifacts/artifacts/kernels/gated-delta-net/variants/01-chunk-parallel-prefill-reference-pytorch.py new file mode 100644 index 000000000..9957f3a5b --- /dev/null +++ b/verification/evidence/historical-artifacts/artifacts/kernels/gated-delta-net/variants/01-chunk-parallel-prefill-reference-pytorch.py @@ -0,0 +1,32 @@ +# provenance: derived from blog-gated-delta-net, pr-sglang-21019; not upstream code +# origin: wiki/kernels/gated-delta-net.md Phase 3 variant (copied from extracted blog bundle) + +# Extracted from sources/blogs/gated-delta-net.md by scripts/extract_blog_code.py +# Heading: ## Key Code > ### Chunk-parallel prefill reference (PyTorch) +# Original fence language: python +# See artifacts/blogs/gated-delta-net/code/PROVENANCE.yaml for origin + license metadata. + +import torch + +def gated_delta_net_prefill(q, k, v, gate, initial_state, CHUNK_SIZE=64): + """ + Chunk-parallel prefill. Each chunk's state matrix is reused across its + query window, so we pay the O(Dk*Dv) state update once per chunk, not + per token. + q, k: [B, L, Dk] v: [B, L, Dv] gate: [B, L] + """ + B, L, Dk = q.shape + Dv = v.shape[-1] + out = torch.empty(B, L, Dv, device=q.device, dtype=q.dtype) + state = initial_state.clone() # [B, Dk, Dv] + for ci in range(0, L, CHUNK_SIZE): + ce = min(ci + CHUNK_SIZE, L) + k_chunk = k[:, ci:ce] + v_chunk = v[:, ci:ce] + g_chunk = gate[:, ci:ce] + decay = torch.cumprod(g_chunk, dim=1) # adaptive memory decay + for t in range(ce - ci): + state = state * decay[:, t:t+1, None] + state = state + k_chunk[:, t, :, None] * v_chunk[:, t, None, :] + out[:, ci + t] = (q[:, ci + t, :, None] * state).sum(dim=1) + return out, state diff --git a/verification/evidence/historical-artifacts/artifacts/kernels/gated-delta-net/variants/02-triton-decode-step-kernel-streaming.py b/verification/evidence/historical-artifacts/artifacts/kernels/gated-delta-net/variants/02-triton-decode-step-kernel-streaming.py new file mode 100644 index 000000000..4e85fae5e --- /dev/null +++ b/verification/evidence/historical-artifacts/artifacts/kernels/gated-delta-net/variants/02-triton-decode-step-kernel-streaming.py @@ -0,0 +1,36 @@ +# provenance: derived from blog-gated-delta-net, pr-sglang-21019; not upstream code +# origin: wiki/kernels/gated-delta-net.md Phase 3 variant (copied from extracted blog bundle) + +# Extracted from sources/blogs/gated-delta-net.md by scripts/extract_blog_code.py +# Heading: ## Key Code > ### Triton decode-step kernel (streaming) +# Original fence language: python +# See artifacts/blogs/gated-delta-net/code/PROVENANCE.yaml for origin + license metadata. + +import triton +import triton.language as tl + +@triton.jit +def gdn_decode_step_kernel( + Q, K, V, GATE, STATE, OUT, + stride_qb, stride_kb, stride_vb, + Dk: tl.constexpr, Dv: tl.constexpr): + """ + One-token delta-rule update for decode. STATE is a [Dk, Dv] matrix kept + per sample; we fold in the new (k,v) pair after applying the decay gate. + """ + b = tl.program_id(0) + dk = tl.arange(0, Dk) + dv = tl.arange(0, Dv) + + q = tl.load(Q + b * stride_qb + dk) # [Dk] + k = tl.load(K + b * stride_kb + dk) # [Dk] + v = tl.load(V + b * stride_vb + dv) # [Dv] + g = tl.load(GATE + b) # scalar decay + + state = tl.load(STATE + b * Dk * Dv + dk[:, None] * Dv + dv[None, :]) + state = state * g # apply decay + state = state + k[:, None] * v[None, :] # delta update + tl.store(STATE + b * Dk * Dv + dk[:, None] * Dv + dv[None, :], state) + + out = tl.sum(q[:, None] * state, axis=0) # [Dv] + tl.store(OUT + b * Dv + dv, out) diff --git a/verification/evidence/historical-artifacts/artifacts/kernels/gated-dual-gemm/full/blackwell-cutlass-schedules-and-tma.cu b/verification/evidence/historical-artifacts/artifacts/kernels/gated-dual-gemm/full/blackwell-cutlass-schedules-and-tma.cu new file mode 100644 index 000000000..4e9d7d9dc --- /dev/null +++ b/verification/evidence/historical-artifacts/artifacts/kernels/gated-dual-gemm/full/blackwell-cutlass-schedules-and-tma.cu @@ -0,0 +1,24 @@ +// Extracted from sources/blogs/tflops-gap-fp4-moe.md by scripts/extract_blog_code.py +// Heading: # TFLOPS Gap: Why FP4 MoE Kernel Engineering Matters on Blackwell > ## Three Key Optimization Techniques > ### 2. Blackwell-Specific CUTLASS Schedules and TMA +// Original fence languages: and cuda +// See artifacts/blogs/tflops-gap-fp4-moe/code/PROVENANCE.yaml for origin + license metadata. + +// --- Blackwell-optimized CUTLASS schedule (from the unlabeled config fence) --- +// SGLang selects this kernel schedule + thread-block shape for Blackwell NvFP4 MoE: +// +// KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100 +// ThreadBlockShape = Shape<_128, _128, _128> +// +// Key features of this schedule: +// * Warp Specialization for FP4: dedicated warp roles for loading FP4 data, +// dequantizing to FP16/BF16, and accumulating in FP32. +// * TMA Integration: asynchronous bulk tensor loads bypassing L1 cache, +// feeding directly into shared memory with strict 128-byte alignment. +// * 1 SM Grouping: multiple experts processed per SM rather than +// one-expert-per-SM, better for variable expert sizes. +// * Native NvFP4 Support: hardware FP4 instructions instead of software +// emulation. + +// --- TMA alignment padding (from the cuda-labeled fence) --- +// TMA alignment is enforced by padding blockscale offsets to 128-byte boundaries: +blockscale_offsets[expert_id + 1] = (expert_offsets[expert_id + 1] + 127) / 128 * 128; diff --git a/verification/evidence/historical-artifacts/artifacts/kernels/gated-dual-gemm/full/vllm-PR-23696-gated-dual-gemm.patch b/verification/evidence/historical-artifacts/artifacts/kernels/gated-dual-gemm/full/vllm-PR-23696-gated-dual-gemm.patch new file mode 100644 index 000000000..33114063f --- /dev/null +++ b/verification/evidence/historical-artifacts/artifacts/kernels/gated-dual-gemm/full/vllm-PR-23696-gated-dual-gemm.patch @@ -0,0 +1,872 @@ +diff --git a/tests/kernels/moe/test_mxfp4_moe.py b/tests/kernels/moe/test_mxfp4_moe.py +index 882b034e2f23..9fd72ee152b5 100644 +--- a/tests/kernels/moe/test_mxfp4_moe.py ++++ b/tests/kernels/moe/test_mxfp4_moe.py +@@ -11,6 +11,7 @@ + from packaging import version + + from vllm.platforms import current_platform ++from vllm.utils.flashinfer import has_flashinfer + + QUARK_MXFP4_AVAILABLE = importlib.util.find_spec( + "quark") is not None and version.parse( +@@ -19,6 +20,10 @@ + TRTLLM_GEN_MXFP4_AVAILABLE = current_platform.is_cuda( + ) and current_platform.is_device_capability(100) + ++HOPPER_MXFP4_BF16_AVAILABLE = (current_platform.is_cuda() ++ and current_platform.is_device_capability(90) ++ and has_flashinfer()) ++ + if TRTLLM_GEN_MXFP4_AVAILABLE: + from flashinfer import (fp4_quantize, mxfp8_quantize, + next_positive_power_of_2, +@@ -542,3 +547,317 @@ def test_trtllm_gen_mxfp4_fused_moe( + transpose_optimized=transpose_optimized) + # relatively loose check since the mxfp4 quantization is less accurate + check_accuracy(ref_result, tg_result, atol=0, rtol=0.3, percent=0.8) ++ ++ ++def _interleave_scales_lastdim_by4(scales: torch.Tensor) -> torch.Tensor: ++ """Interleave scales on the last dimension by groups of 4, matching ++ the transformation in mxfp4.py's BF16 (Hopper) path.""" ++ s = scales.to(torch.uint8) ++ s_shape = s.shape ++ assert s_shape[-1] % 4 == 0 ++ s = s.reshape(*s_shape[:-1], s_shape[-1] // 4, 4) ++ # Move the 4-group dimension before the row dimension ++ permuted = s.permute(0, 2, 1, 3) ++ # Merge the row dim with the 4-group dim ++ return permuted.reshape(s_shape[0], s_shape[-1] // 4, s_shape[1] * 4) ++ ++ ++@pytest.mark.parametrize("topk", [1, 4]) ++@pytest.mark.parametrize("num_experts", [32]) ++@pytest.mark.parametrize("num_tokens", [1, 128]) ++@pytest.mark.parametrize("intermediate_size,hidden_size", [(3072, 3072)]) ++@pytest.mark.parametrize("alpha,beta,limit", [(1.0, 1.0, None), ++ (1.702, 1.0, 7.0)]) ++@pytest.mark.skipif( ++ not HOPPER_MXFP4_BF16_AVAILABLE, ++ reason="nvidia gpu sm90 and flashinfer are required for this test", ++) ++def test_flashinfer_cutlass_mxfp4_fused_moe( ++ topk: int, ++ num_experts: int, ++ num_tokens: int, ++ intermediate_size: int, ++ hidden_size: int, ++ alpha: float, ++ beta: float, ++ limit: Optional[float], ++): ++ torch.manual_seed(42) ++ device = "cuda:0" ++ ++ # Inputs ++ hidden_states = torch.randn(num_tokens, ++ hidden_size, ++ device=device, ++ dtype=torch.bfloat16) ++ # Random MXFP4 weights and scales (uint8), contiguous [w1; w3] ++ w13_q = torch.randint( ++ 0, ++ 256, (num_experts, 2 * intermediate_size, hidden_size // 2), ++ device=device, ++ dtype=torch.uint8) ++ w13_scale = torch.randint( ++ 118, ++ 123, (num_experts, 2 * intermediate_size, hidden_size // 32), ++ device=device, ++ dtype=torch.uint8) ++ ++ w2_q = torch.randint(0, ++ 256, ++ (num_experts, hidden_size, intermediate_size // 2), ++ device=device, ++ dtype=torch.uint8) ++ w2_scale = torch.randint( ++ 118, ++ 123, (num_experts, hidden_size, intermediate_size // 32), ++ device=device, ++ dtype=torch.uint8) ++ # Bias contiguous [b1; b3] ++ bias13 = (torch.randn(num_experts, ++ 2 * intermediate_size, ++ device=device, ++ dtype=torch.bfloat16) * 10) ++ bias2 = (torch.randn( ++ num_experts, hidden_size, device=device, dtype=torch.bfloat16) * 10) ++ router_logits = torch.rand(num_tokens, ++ num_experts, ++ dtype=torch.float32, ++ device=device) ++ ++ w13_ref = mxfp4_dequantize(w13_q.clone(), w13_scale.clone()).reshape( ++ num_experts, 2 * intermediate_size, hidden_size) ++ w2_ref = mxfp4_dequantize(w2_q.clone(), w2_scale.clone()).reshape( ++ num_experts, hidden_size, intermediate_size) ++ ref = reference_moe(router_logits.to(torch.float32), topk, num_experts, ++ hidden_states.to(torch.float32), w13_ref, ++ bias13.to(torch.float32), w2_ref, ++ bias2.to(torch.float32), alpha, beta, limit, 'bf16') ++ ++ from vllm.utils.flashinfer import flashinfer_cutlass_fused_moe ++ ++ # Swap halves to arrange as [w3; w1] (kernel expectation) ++ w1_w, w3_w = torch.chunk(w13_q, 2, dim=1) ++ w13_q_swapped = torch.cat([w3_w, w1_w], dim=1) ++ ++ b1, b3 = torch.chunk(bias13.to(torch.float32), 2, dim=-1) ++ w13_b = torch.cat([b3, b1], dim=-1).to(torch.bfloat16) ++ ++ w1_s, w3_s = torch.chunk(w13_scale, 2, dim=1) ++ w13_s = torch.cat([w3_s, w1_s], dim=1) ++ w13_s_inter = _interleave_scales_lastdim_by4(w13_s) ++ w2_s_inter = _interleave_scales_lastdim_by4(w2_scale) ++ ++ routing_weights = torch.nn.functional.softmax(router_logits, ++ dim=1, ++ dtype=torch.float32) ++ token_final_scales, token_selected_experts = torch.topk(routing_weights, ++ topk, ++ dim=-1) ++ token_final_scales = (token_final_scales / ++ token_final_scales.sum(dim=-1, keepdim=True)) ++ token_selected_experts = token_selected_experts.to(torch.int).contiguous() ++ ++ out = torch.empty_like(hidden_states, dtype=torch.bfloat16) ++ if alpha is not None: ++ alpha = torch.full((num_experts, ), alpha, device=hidden_states.device) ++ if beta is not None: ++ beta = torch.full((num_experts, ), beta, device=hidden_states.device) ++ if limit is not None: ++ limit = torch.full((num_experts, ), limit, device=hidden_states.device) ++ ++ _ = flashinfer_cutlass_fused_moe( ++ input=hidden_states, ++ token_selected_experts=token_selected_experts, ++ token_final_scales=token_final_scales, ++ fc1_expert_weights=w13_q_swapped, ++ fc2_expert_weights=w2_q, ++ output_dtype=torch.bfloat16, ++ output=out, ++ quant_scales=[w13_s_inter.to(torch.uint8), ++ w2_s_inter.to(torch.uint8)], ++ fc1_expert_biases=w13_b, ++ fc2_expert_biases=bias2.to(torch.bfloat16), ++ swiglu_alpha=alpha, ++ swiglu_beta=beta, ++ swiglu_limit=limit, ++ tp_size=1, ++ tp_rank=0, ++ ep_size=1, ++ ep_rank=0, ++ use_w4_group_scaling=True, ++ ) ++ ++ # Allow some mismatch due to MXFP4 quantization ++ check_accuracy(ref, out, atol=0, rtol=0.3, percent=0.8) ++ ++ ++@pytest.mark.parametrize("topk", [1, 4]) ++@pytest.mark.parametrize("num_experts", [32]) ++@pytest.mark.parametrize("num_tokens", [1, 128]) ++@pytest.mark.parametrize("intermediate_size,hidden_size", [(3072, 3072)]) ++@pytest.mark.parametrize("alpha,beta,limit", [(1.0, 1.0, None), ++ (1.702, 1.0, 7.0)]) ++@pytest.mark.skipif( ++ not (current_platform.is_cuda() ++ and current_platform.is_device_capability(100) and has_flashinfer()), ++ reason="NVIDIA GPU sm100 and flashinfer are required for this test", ++) ++def test_flashinfer_cutlass_mxfp4_mxfp8_fused_moe( ++ topk: int, ++ num_experts: int, ++ num_tokens: int, ++ intermediate_size: int, ++ hidden_size: int, ++ alpha: Optional[float], ++ beta: Optional[float], ++ limit: Optional[float], ++): ++ torch.manual_seed(42) ++ device = "cuda:0" ++ ++ # Inputs ++ hidden_states = torch.randn(num_tokens, ++ hidden_size, ++ device=device, ++ dtype=torch.bfloat16) ++ # Float weights in w13 format [w1; w3] ++ w13 = (torch.randn(num_experts, ++ 2 * intermediate_size, ++ hidden_size, ++ device=device, ++ dtype=torch.bfloat16) / 10) ++ w2 = (torch.randn(num_experts, ++ hidden_size, ++ intermediate_size, ++ device=device, ++ dtype=torch.bfloat16) / 10) ++ # Bias contiguous [b1; b3] ++ bias13 = (torch.randn(num_experts, ++ 2 * intermediate_size, ++ device=device, ++ dtype=torch.bfloat16) * 10) ++ bias2 = (torch.randn( ++ num_experts, hidden_size, device=device, dtype=torch.bfloat16) * 10) ++ router_logits = torch.rand(num_tokens, ++ num_experts, ++ dtype=torch.float32, ++ device=device) ++ ++ # Quantize weights to MXFP4 per expert (SM100 path) ++ from flashinfer import mxfp4_quantize ++ ++ def quant_mxfp4_batches(a: torch.Tensor, e: int): ++ qs, sfs = [], [] ++ for i in range(e): ++ q, sf = mxfp4_quantize(a[i].cuda()) ++ qs.append(q) ++ sfs.append(sf) ++ return torch.stack(qs), torch.stack(sfs) ++ ++ def dequant_mxfp4_batches(mat_fp4: torch.Tensor, ++ scale_tensor: torch.Tensor): ++ num_batches = mat_fp4.size(0) ++ scale_tensor = scale_tensor.view(num_batches, -1) ++ from flashinfer import mxfp4_dequantize ++ return torch.stack([ ++ mxfp4_dequantize(mat_fp4[b, :, :], scale_tensor[b, :]) ++ for b in range(num_batches) ++ ]) ++ ++ w13_q, w13_scale = quant_mxfp4_batches(w13, num_experts) ++ w2_q, w2_scale = quant_mxfp4_batches(w2, num_experts) ++ ++ # Reference result using dequantized tensors and reference_moe ++ w13_ref = dequant_mxfp4_batches( ++ w13_q.view(torch.uint8), ++ w13_scale.view(torch.uint8).reshape(-1)).to(torch.float32).reshape( ++ num_experts, 2 * intermediate_size, hidden_size) ++ w2_ref = dequant_mxfp4_batches( ++ w2_q.view(torch.uint8), ++ w2_scale.view(torch.uint8).reshape(-1)).to(torch.float32).reshape( ++ num_experts, hidden_size, intermediate_size) ++ ++ # Quantize activations for SM100 path and dequantize for reference ++ hidden_states_q, hidden_states_sf = mxfp8_quantize(hidden_states, True, 32) ++ # Reference uses BF16 input but quantizes intermediate activation to MXFP8 ++ ref = reference_moe(router_logits.to(torch.float32), topk, num_experts, ++ hidden_states.to(torch.float32), w13_ref, ++ bias13.to(torch.float32), w2_ref, ++ bias2.to(torch.float32), alpha, beta, limit, 'mxfp8') ++ ++ # Prepare inputs for FlashInfer CUTLASS fused MoE ++ from vllm.utils.flashinfer import flashinfer_cutlass_fused_moe ++ ++ # Swap halves to arrange as [w3; w1] (kernel expectation) ++ w1_w, w3_w = torch.chunk(w13_q, 2, dim=1) ++ w13_q_swapped = torch.cat([w3_w, w1_w], dim=1) ++ ++ # Swap scales halves to match swapped weights ++ s1, s3 = torch.chunk(w13_scale, 2, dim=1) ++ w13_scale_swapped = torch.cat([s3, s1], dim=1) ++ ++ b1, b3 = torch.chunk(bias13.to(torch.float32), 2, dim=-1) ++ w13_b = torch.cat([b3, b1], dim=-1).to(torch.bfloat16) ++ ++ # Build routing for kernel ++ routing_weights = torch.nn.functional.softmax(router_logits, ++ dim=1, ++ dtype=torch.float32) ++ token_final_scales, token_selected_experts = torch.topk(routing_weights, ++ topk, ++ dim=-1) ++ token_final_scales = (token_final_scales / ++ token_final_scales.sum(dim=-1, keepdim=True)) ++ token_selected_experts = token_selected_experts.to(torch.int).contiguous() ++ ++ out = torch.empty_like(hidden_states, dtype=torch.bfloat16) ++ if alpha is not None: ++ alpha_t = torch.full((num_experts, ), ++ alpha, ++ device=hidden_states.device) ++ else: ++ alpha_t = None ++ if beta is not None: ++ beta_t = torch.full((num_experts, ), beta, device=hidden_states.device) ++ else: ++ beta_t = None ++ if limit is not None: ++ limit_t = torch.full((num_experts, ), ++ limit, ++ device=hidden_states.device) ++ else: ++ limit_t = None ++ ++ # Quant scales for SM100 MXFP8+MXFP4 path ++ fake_input_scale = torch.ones(num_experts, device=device) ++ quant_scales = [ ++ w13_scale_swapped.view(torch.int32), ++ fake_input_scale, ++ w2_scale.view(torch.int32), ++ fake_input_scale, ++ ] ++ ++ _ = flashinfer_cutlass_fused_moe( ++ input=hidden_states_q, ++ token_selected_experts=token_selected_experts, ++ token_final_scales=token_final_scales, ++ fc1_expert_weights=w13_q_swapped.contiguous().view(torch.long), ++ fc2_expert_weights=w2_q.contiguous().view(torch.long), ++ output_dtype=torch.bfloat16, ++ output=out, ++ quant_scales=quant_scales, ++ fc1_expert_biases=w13_b, ++ fc2_expert_biases=bias2.to(torch.bfloat16), ++ swiglu_alpha=alpha_t, ++ swiglu_beta=beta_t, ++ swiglu_limit=limit_t, ++ tp_size=1, ++ tp_rank=0, ++ ep_size=1, ++ ep_rank=0, ++ use_mxfp8_act_scaling=True, ++ input_sf=hidden_states_sf, ++ ) ++ ++ # Allow some mismatch due to MXFP4 quantization ++ check_accuracy(ref, out, atol=0, rtol=0.3, percent=0.8) +diff --git a/vllm/envs.py b/vllm/envs.py +index d7956a2adff6..165cd32721fe 100755 +--- a/vllm/envs.py ++++ b/vllm/envs.py +@@ -166,7 +166,8 @@ + VLLM_HAS_FLASHINFER_CUBIN: bool = False + VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8: bool = False + VLLM_USE_FLASHINFER_MOE_MXFP4_BF16: bool = False +- VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True ++ VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS: bool = False ++ VLLM_ALLREDUCE_USE_SYMM_MEM: bool = False + VLLM_TUNED_CONFIG_FOLDER: Optional[str] = None + VLLM_DISABLE_PAD_FOR_CUDAGRAPH: bool = False + VLLM_GPT_OSS_USE_CONTAINER_TOOL: bool = False +@@ -1004,6 +1005,15 @@ def get_vllm_port() -> Optional[int]: + "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8": + lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8", "0"))), + ++ # If set to 1, use the FlashInfer CUTLASS backend for ++ # MXFP8 (activation) x MXFP4 (weight) MoE. ++ # This is separate from the TRTLLMGEN path controlled by ++ # VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8. ++ "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS": ++ lambda: bool(int( ++ os.getenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS", "0") ++ )), ++ + # If set to 1, use the FlashInfer + # BF16 (activation) x MXFP4 (weight) MoE backend. + "VLLM_USE_FLASHINFER_MOE_MXFP4_BF16": +@@ -1296,6 +1306,7 @@ def compute_hash() -> str: + "VLLM_USE_FLASHINFER_MOE_FP8", + "VLLM_USE_FLASHINFER_MOE_FP4", + "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8", ++ "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS", + "VLLM_USE_FLASHINFER_MOE_MXFP4_BF16", + "VLLM_USE_CUDNN_PREFILL", + "VLLM_USE_TRTLLM_ATTENTION", +diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py +index 89676f98cb0e..a90a71159f72 100644 +--- a/vllm/model_executor/layers/fused_moe/layer.py ++++ b/vllm/model_executor/layers/fused_moe/layer.py +@@ -813,9 +813,16 @@ def __init__( + + # we are padding globally so EP buffer allocation works + if quant_config and quant_config.get_name() == "mxfp4": +- from vllm.model_executor.layers.quantization.mxfp4 import ( # noqa: E501 +- should_use_flashinfer_mxfp4) +- if current_platform.is_rocm() or should_use_flashinfer_mxfp4(): ++ from vllm.model_executor.layers.quantization.mxfp4 import ( ++ Mxfp4Backend, get_mxfp4_backend) ++ current_mxfp4_backend = get_mxfp4_backend() ++ if (current_mxfp4_backend == Mxfp4Backend.SM90_FI_MXFP4_BF16 ++ or current_mxfp4_backend ++ == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_CUTLASS): ++ hidden_size = round_up(hidden_size, 128) ++ elif (current_platform.is_rocm() or current_mxfp4_backend ++ == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_TRTLLM or ++ current_mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_BF16): + hidden_size = round_up(hidden_size, 256) + + # For smuggling this layer into the fused moe custom op +diff --git a/vllm/model_executor/layers/quantization/mxfp4.py b/vllm/model_executor/layers/quantization/mxfp4.py +index 74922756afe5..f935bdd84124 100644 +--- a/vllm/model_executor/layers/quantization/mxfp4.py ++++ b/vllm/model_executor/layers/quantization/mxfp4.py +@@ -1,5 +1,6 @@ + # SPDX-License-Identifier: Apache-2.0 + # SPDX-FileCopyrightText: Copyright contributors to the vLLM project ++from enum import Enum + from typing import Callable, Optional, Union + + import torch +@@ -33,33 +34,72 @@ + logger = init_logger(__name__) + + +-def _should_use_flashinfer_mxfp4_bf16(): +- """Determine if FlashInfer MXFP4 BF16 should be used.""" +- # If explicitly set, respect the setting +- if envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16"): +- return envs.VLLM_USE_FLASHINFER_MOE_MXFP4_BF16 +- +- # Enable by default on SM100 if MXFP8 is not explicitly enabled +- if (current_platform.is_device_capability(100) and has_flashinfer() +- and not envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8")): +- logger.info_once( +- "Enabling FlashInfer MXFP4 BF16 backend by default for Blackwell. " +- "For faster performance, consider setting " +- "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8=1, " +- "though this may impact accuracy.") +- return True +- +- return False +- +- +-def _should_use_flashinfer_mxfp4_mxfp8(): +- """Determine if FlashInfer MXFP4 MXFP8 should be used.""" +- return envs.VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8 ++# enum for mxfp4 backend ++class Mxfp4Backend(Enum): ++ NONE = 0 ++ ++ # FlashInfer Backend ++ SM100_FI_MXFP4_MXFP8_TRTLLM = 1 ++ SM100_FI_MXFP4_MXFP8_CUTLASS = 2 ++ SM100_FI_MXFP4_BF16 = 3 ++ SM90_FI_MXFP4_BF16 = 4 ++ ++ # Marlin Backend ++ MARLIN = 5 ++ ++ # Triton Backend ++ TRITON = 6 ++ ++ ++def get_mxfp4_backend(): ++ # Backend Selection ++ if current_platform.is_cuda(): ++ if (current_platform.is_device_capability(90) and has_flashinfer() ++ and envs.VLLM_USE_FLASHINFER_MOE_MXFP4_BF16): ++ logger.info_once("Using FlashInfer MXFP4 BF16 backend for SM90") ++ return Mxfp4Backend.SM90_FI_MXFP4_BF16 ++ elif (current_platform.is_device_capability(100) and has_flashinfer() ++ and envs.VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS): ++ logger.info_once( ++ "Using FlashInfer MXFP4 MXFP8 CUTLASS backend for SM100") ++ return Mxfp4Backend.SM100_FI_MXFP4_MXFP8_CUTLASS ++ elif (current_platform.is_device_capability(100) and has_flashinfer() ++ and envs.VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8): ++ logger.info_once( ++ "Using FlashInfer MXFP4 MXFP8 TRTLLM backend for SM100, " ++ "for high concurrency throughput workloads consider setting " ++ "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS=1 for better " ++ "performance") ++ return Mxfp4Backend.SM100_FI_MXFP4_MXFP8_TRTLLM ++ elif current_platform.is_device_capability(100) and has_flashinfer(): ++ logger.info_once( ++ "Using FlashInfer MXFP4 BF16 backend for SM100, " ++ "For faster performance on SM100, consider setting " ++ "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8=1, though this may impact " ++ "accuracy.") ++ return Mxfp4Backend.SM100_FI_MXFP4_BF16 ++ elif ((current_platform.is_device_capability(100) ++ or current_platform.is_device_capability(90)) ++ and not has_flashinfer()): ++ logger.warning_once( ++ "MXFP4 MoE is enabled on Hopper/Blackwell but FlashInfer " ++ "is not available. This may result in degraded performance. " ++ "Please `pip install vllm[flashinfer]` for best results.") + ++ # If FlashInfer is not available, try either Marlin or Triton ++ if current_platform.get_device_capability( ++ )[0] < 9 or not has_triton_kernels() or not is_torch_equal_or_newer( ++ "2.8.0"): ++ logger.info_once("Using Marlin backend") ++ return Mxfp4Backend.MARLIN ++ else: ++ logger.info_once("Using Triton backend") ++ return Mxfp4Backend.TRITON ++ elif current_platform.is_rocm() and has_triton_kernels(): ++ logger.info_once("Using Triton backend") ++ return Mxfp4Backend.TRITON + +-def should_use_flashinfer_mxfp4(): +- return (_should_use_flashinfer_mxfp4_mxfp8() +- or _should_use_flashinfer_mxfp4_bf16()) ++ return Mxfp4Backend.NONE + + + class Mxfp4Config(QuantizationConfig): +@@ -113,31 +153,15 @@ def __init__(self, moe: FusedMoEConfig): + super().__init__(moe) + self.topk_indices_dtype = None + self.moe = moe +- self.use_marlin = self._should_use_marlin() ++ self.mxfp4_backend = get_mxfp4_backend() + self.max_capture_size = get_current_vllm_config( + ).compilation_config.max_capture_size + +- if current_platform.is_device_capability(100) and not has_flashinfer(): +- logger.warning_once( +- "MXFP4 MoE is enabled on Blackwell but FlashInfer " +- "is not available. This may result in degraded performance. " +- "Please `pip install vllm[flashinfer]` for best results.") ++ assert self.mxfp4_backend != Mxfp4Backend.NONE, ( ++ "No MXFP4 MoE backend (FlashInfer/Marlin/Triton) available." ++ "Please check your environment and try again.") + self._cache_permute_indices: dict[torch.Size, torch.Tensor] = {} + +- def _should_use_marlin(self): +- if envs.VLLM_MXFP4_USE_MARLIN is not None: +- return envs.VLLM_MXFP4_USE_MARLIN +- if current_platform.is_cuda() and \ +- not current_platform.is_device_capability(100): +- if not current_platform.has_device_capability(90): +- # marlin kernel has better performance on ampere +- return True +- if not has_triton_kernels(): +- return True +- if not is_torch_equal_or_newer("2.8.0"): +- return True +- return False +- + def create_weights(self, layer: torch.nn.Module, num_experts: int, + hidden_size: int, intermediate_size_per_partition: int, + params_dtype: torch.dtype, **extra_weight_attrs): +@@ -157,7 +181,7 @@ def create_weights(self, layer: torch.nn.Module, num_experts: int, + + intermediate_size_per_partition_after_pad = \ + intermediate_size_per_partition +- if self.use_marlin: ++ if self.mxfp4_backend == Mxfp4Backend.MARLIN: + # The moe marlin kernel requires that for each linear + # n % 256 == 0 and k % 128 == 0. + # In gate_up_proj: +@@ -175,16 +199,20 @@ def create_weights(self, layer: torch.nn.Module, num_experts: int, + layer.hidden_size = hidden_size + layer.intermediate_size_per_partition = \ + intermediate_size_per_partition_after_pad +- elif should_use_flashinfer_mxfp4(): ++ elif (self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_TRTLLM ++ or self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_BF16): + # pad the intermediate size to be a multiple of 2 * mxfp4_block + # for to hold non-uniform sharded tensor as well as swizzling + # other padding to increase performance + intermediate_size_per_partition_after_pad = round_up( + intermediate_size_per_partition, 256) + hidden_size = round_up(hidden_size, 256) +- elif current_platform.is_rocm(): ++ elif current_platform.is_rocm() or ( ++ self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_CUTLASS ++ or self.mxfp4_backend == Mxfp4Backend.SM90_FI_MXFP4_BF16): + intermediate_size_per_partition_after_pad = round_up( + intermediate_size_per_partition, 128) ++ hidden_size = round_up(hidden_size, 128) + else: + intermediate_size_per_partition_after_pad = round_up( + intermediate_size_per_partition, 64) +@@ -264,9 +292,10 @@ def create_weights(self, layer: torch.nn.Module, num_experts: int, + set_weight_attrs(w2_bias, extra_weight_attrs) + + def process_weights_after_loading(self, layer): +- if self.use_marlin: ++ if self.mxfp4_backend == Mxfp4Backend.MARLIN: + prepare_moe_fp4_layer_for_marlin(layer) +- elif should_use_flashinfer_mxfp4(): ++ elif (self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_TRTLLM ++ or self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_BF16): + from flashinfer.fp4_quantization import ( + nvfp4_block_scale_interleave) + from flashinfer.fused_moe.core import ( +@@ -429,7 +458,116 @@ def swap_every_two_rows(x, axis=-1): + layer.w2_bias = Parameter(torch.stack(gemm2_bias_shuffled).reshape( + self.num_experts, -1), + requires_grad=False) +- else: ++ elif (self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_CUTLASS ++ or self.mxfp4_backend == Mxfp4Backend.SM90_FI_MXFP4_BF16): ++ layer.gemm1_alpha = Parameter(torch.tensor( ++ [1.702] * self.num_experts, dtype=torch.float32).cuda(), ++ requires_grad=False) ++ layer.gemm1_beta = Parameter(torch.tensor( ++ [1.0] * self.num_experts, dtype=torch.float32).cuda(), ++ requires_grad=False) ++ layer.gemm1_clamp_limit = Parameter(torch.tensor( ++ [7.0] * self.num_experts, dtype=torch.float32).cuda(), ++ requires_grad=False) ++ ++ sf_block_size = 32 # mxfp4 block size ++ ++ # Common shape assertions ++ assert (layer.w13_weight.dim() == 3 ++ and layer.w13_weight.shape[0] == self.num_experts ++ and layer.w13_weight.shape[1] == self.intermediate_size * 2 ++ and layer.w13_weight.shape[2] == self.hidden_size // 2) ++ assert (layer.w13_weight_scale.dim() == 3 ++ and layer.w13_weight_scale.shape[0] == self.num_experts ++ and layer.w13_weight_scale.shape[1] ++ == self.intermediate_size * 2 ++ and layer.w13_weight_scale.shape[2] ++ == self.hidden_size // sf_block_size) ++ assert (layer.w2_weight.dim() == 3 ++ and layer.w2_weight.shape[0] == self.num_experts ++ and layer.w2_weight.shape[1] == self.hidden_size and ++ layer.w2_weight.shape[2] == self.intermediate_size // 2) ++ assert (layer.w2_weight_scale.dim() == 3 ++ and layer.w2_weight_scale.shape[1] == self.hidden_size ++ and layer.w2_weight_scale.shape[2] ++ == self.intermediate_size // sf_block_size) ++ assert (layer.w13_bias.dim() == 2 ++ and layer.w13_bias.shape[0] == self.num_experts ++ and layer.w13_bias.shape[1] == self.intermediate_size * 2) ++ assert (layer.w2_bias.dim() == 2 ++ and layer.w2_bias.shape[0] == self.num_experts ++ and layer.w2_bias.shape[1] == self.hidden_size) ++ ++ # De-interleave and swap for w13 weight, bias, and scales ++ w13_w = layer.w13_weight.data ++ gate_w, up_w = w13_w[:, ::2, :], w13_w[:, 1::2, :] ++ deinterleaved_w13_w = torch.cat([gate_w, up_w], dim=1) ++ w1_w, w3_w = torch.chunk(deinterleaved_w13_w, 2, dim=1) ++ w13_weight_swapped = torch.cat([w3_w, w1_w], dim=1) ++ ++ w13_b = layer.w13_bias.data.to(torch.float32) ++ gate_b, up_b = w13_b[:, ::2], w13_b[:, 1::2] ++ deinterleaved_w13_b = torch.cat([gate_b, up_b], dim=1) ++ b1, b3 = torch.chunk(deinterleaved_w13_b, 2, dim=-1) ++ w13_bias_swapped = torch.cat([b3, b1], dim=-1).to(torch.bfloat16) ++ ++ w13_s = layer.w13_weight_scale.data ++ gate_s, up_s = w13_s[:, ::2, :], w13_s[:, 1::2, :] ++ deinterleaved_w13_s = torch.cat([gate_s, up_s], dim=1) ++ s1, s3 = torch.chunk(deinterleaved_w13_s, 2, dim=1) ++ w13_scale_swapped = torch.cat([s3, s1], dim=1) ++ ++ if self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_CUTLASS: ++ from flashinfer import block_scale_interleave ++ ++ orig_shape = w13_scale_swapped.shape ++ w13_scale_interleaved = block_scale_interleave( ++ w13_scale_swapped.view(torch.uint8)).reshape(orig_shape) ++ ++ w2_s = layer.w2_weight_scale.data ++ orig_shape = w2_s.shape ++ w2_scale_interleaved = block_scale_interleave( ++ w2_s.view(torch.uint8)).reshape(orig_shape) ++ ++ layer.w13_weight = Parameter(w13_weight_swapped, ++ requires_grad=False) ++ layer.w13_weight_scale = Parameter(w13_scale_interleaved, ++ requires_grad=False) ++ layer.w13_bias = Parameter(w13_bias_swapped, ++ requires_grad=False) ++ layer.w2_weight_scale = Parameter(w2_scale_interleaved, ++ requires_grad=False) ++ elif self.mxfp4_backend == Mxfp4Backend.SM90_FI_MXFP4_BF16: ++ ++ def _interleave_mxfp4_cutlass_sm90(w): ++ w_shape = w.shape ++ w_interleaved = w.reshape(w_shape[0], w_shape[1], ++ (w_shape[2] // 4), 4) ++ w_interleaved = w_interleaved.permute(0, 2, 1, 3) ++ w_interleaved = w_interleaved.reshape( ++ w_shape[0], w_shape[2] // 4, w_shape[1] * 4) ++ return w_interleaved ++ ++ w31_scales = w13_scale_swapped.to(torch.uint8).view( ++ torch.uint8) ++ w31_scales_interleaved = _interleave_mxfp4_cutlass_sm90( ++ w31_scales) ++ ++ w2_weight_scale = layer.w2_weight_scale.data ++ w2_scales = w2_weight_scale.to(torch.uint8).view(torch.uint8) ++ w2_scales_interleaved = _interleave_mxfp4_cutlass_sm90( ++ w2_scales) ++ ++ layer.w13_weight = torch.nn.Parameter(torch.cat([w3_w, w1_w], ++ dim=1), ++ requires_grad=False) ++ layer.w13_bias = torch.nn.Parameter(w13_bias_swapped, ++ requires_grad=False) ++ layer.w13_weight_scale = torch.nn.Parameter( ++ w31_scales_interleaved, requires_grad=False) ++ layer.w2_weight_scale = torch.nn.Parameter( ++ w2_scales_interleaved, requires_grad=False) ++ elif self.mxfp4_backend == Mxfp4Backend.TRITON: + from triton_kernels.matmul_ogs import FlexCtx, PrecisionConfig + + w13_bias = layer.w13_bias.to(torch.float32) +@@ -464,6 +602,8 @@ def swap_every_two_rows(x, axis=-1): + layer.w13_weight = None + layer.w2_weight = None + torch.cuda.empty_cache() ++ else: ++ raise ValueError(f"Unsupported backend: {self.mxfp4_backend}") + + def _get_tile_tokens_dim(self, x: torch.Tensor, top_k: int): + # Number of tokens in the input tensor. +@@ -500,7 +640,8 @@ def select_gemm_impl( + raise NotImplementedError( + "Mxfp4 does not support batched experts format for EP") + else: +- if should_use_flashinfer_mxfp4(): ++ if (self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_TRTLLM ++ or self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_BF16): + # B200 code-path + kwargs = { + "gemm1_alpha": layer.gemm1_alpha, +@@ -601,7 +742,7 @@ def apply( + if enable_eplb: + raise NotImplementedError("EPLB is not supported for mxfp4") + +- if self.use_marlin: ++ if self.mxfp4_backend == Mxfp4Backend.MARLIN: + topk_weights, topk_ids = FusedMoE.select_experts( + hidden_states=x, + router_logits=router_logits, +@@ -665,16 +806,19 @@ def apply( + logical_replica_count), ( + "MXFP4 are not supported with this configuration.") + +- if should_use_flashinfer_mxfp4(): +- from flashinfer import mxfp8_quantize, trtllm_fp4_block_scale_moe +- if _should_use_flashinfer_mxfp4_bf16(): ++ if (self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_TRTLLM ++ or self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_BF16): ++ from flashinfer import trtllm_fp4_block_scale_moe ++ if self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_BF16: + assert x.dtype == torch.bfloat16 + x_quant = x + x_scale = None +- else: ++ elif self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_TRTLLM: ++ from flashinfer import mxfp8_quantize + x_quant, x_scale = mxfp8_quantize(x, False) # to mxfp8 + x_scale = x_scale.view(torch.float8_e4m3fn).reshape( + *x.shape[:-1], -1) ++ + trtllm_gen_output = trtllm_fp4_block_scale_moe( + router_logits.to(torch.bfloat16), + None, # routing_bias +@@ -706,7 +850,86 @@ def apply( + tune_max_num_tokens=self.max_capture_size, + )[0] + return trtllm_gen_output +- else: ++ elif (self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_CUTLASS ++ or self.mxfp4_backend == Mxfp4Backend.SM90_FI_MXFP4_BF16): ++ from vllm.utils.flashinfer import flashinfer_cutlass_fused_moe ++ ++ topk_weights, topk_ids = FusedMoE.select_experts( ++ hidden_states=x, ++ router_logits=router_logits, ++ use_grouped_topk=use_grouped_topk, ++ top_k=top_k, ++ renormalize=renormalize, ++ topk_group=topk_group, ++ num_expert_group=num_expert_group, ++ custom_routing_function=custom_routing_function, ++ scoring_func=scoring_func, ++ e_score_correction_bias=e_score_correction_bias, ++ ) ++ ++ # Backend-specific preparation ++ if self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_CUTLASS: ++ ++ from flashinfer import mxfp8_quantize ++ ++ x_quant, x_scale = mxfp8_quantize(x, True, 32) ++ ++ fake_input_scale = torch.ones(self.num_experts, ++ device=x.device) ++ quant_scales = [ ++ layer.w13_weight_scale.contiguous().view(torch.int32), ++ fake_input_scale, ++ layer.w2_weight_scale.contiguous().view(torch.int32), ++ fake_input_scale, ++ ] ++ ++ fi_input = x_quant ++ extra_kwargs = dict( ++ use_mxfp8_act_scaling=True, ++ input_sf=x_scale, ++ fc1_expert_weights=layer.w13_weight.contiguous().view( ++ torch.long), ++ fc2_expert_weights=layer.w2_weight.contiguous().view( ++ torch.long), ++ ) ++ elif self.mxfp4_backend == Mxfp4Backend.SM90_FI_MXFP4_BF16: ++ assert x.dtype == torch.bfloat16 ++ ++ quant_scales = [ ++ layer.w13_weight_scale, ++ layer.w2_weight_scale, ++ ] ++ ++ fi_input = x ++ extra_kwargs = dict( ++ use_w4_group_scaling=True, ++ fc1_expert_weights=layer.w13_weight, ++ fc2_expert_weights=layer.w2_weight, ++ ) ++ ++ output = torch.empty_like(x, dtype=torch.bfloat16) ++ _ = flashinfer_cutlass_fused_moe( ++ input=fi_input, ++ token_selected_experts=topk_ids.to(torch.int).contiguous(), ++ token_final_scales=topk_weights, ++ output_dtype=torch.bfloat16, ++ output=output, ++ quant_scales=quant_scales, ++ fc1_expert_biases=layer.w13_bias, ++ fc2_expert_biases=layer.w2_bias, ++ swiglu_alpha=layer.gemm1_alpha, ++ swiglu_beta=layer.gemm1_beta, ++ swiglu_limit=layer.gemm1_clamp_limit, ++ tp_size=self.moe.tp_size, ++ tp_rank=self.moe.tp_rank, ++ ep_size=self.moe.ep_size, ++ ep_rank=self.moe.ep_rank, ++ tune_max_num_tokens=self.max_capture_size, ++ **extra_kwargs, ++ ) ++ ++ return output ++ elif self.mxfp4_backend == Mxfp4Backend.TRITON: + from vllm.model_executor.layers.fused_moe.gpt_oss_triton_kernels_moe import ( # noqa: E501 + triton_kernel_moe_forward) + return triton_kernel_moe_forward( +@@ -724,3 +947,5 @@ def apply( + w2_precision=self.w2_precision_config, + apply_router_weight_on_input=apply_router_weight_on_input, + ) ++ else: ++ raise ValueError(f"Unsupported backend: {self.mxfp4_backend}") +diff --git a/vllm/model_executor/warmup/kernel_warmup.py b/vllm/model_executor/warmup/kernel_warmup.py +index e42e34ebc77b..89ce20308f44 100644 +--- a/vllm/model_executor/warmup/kernel_warmup.py ++++ b/vllm/model_executor/warmup/kernel_warmup.py +@@ -33,8 +33,8 @@ def kernel_warmup(worker: "Worker"): + max_tokens = worker.scheduler_config.max_num_batched_tokens + deep_gemm_warmup(model, max_tokens) + +- # FlashInfer kernel autotune for Blackwell (SM 10.0) GPUs +- if has_flashinfer() and current_platform.is_device_capability(100): ++ # FlashInfer autotune for Hopper (SM 9.0) and Blackwell (SM 10.0) GPUs ++ if has_flashinfer() and current_platform.has_device_capability(90): + flashinfer_autotune(worker.model_runner) + + # FlashInfer attention warmup diff --git a/verification/evidence/historical-artifacts/artifacts/kernels/gated-dual-gemm/variants/01-fused-epilogue-swiglu-skeleton.cu b/verification/evidence/historical-artifacts/artifacts/kernels/gated-dual-gemm/variants/01-fused-epilogue-swiglu-skeleton.cu new file mode 100644 index 000000000..125821414 --- /dev/null +++ b/verification/evidence/historical-artifacts/artifacts/kernels/gated-dual-gemm/variants/01-fused-epilogue-swiglu-skeleton.cu @@ -0,0 +1,25 @@ +// provenance: derived from pr-vllm-23696, kernel-gated-dual-gemm, technique-epilogue-fusion; not upstream code +// origin: wiki/kernels/gated-dual-gemm.md Phase 3 variant + +// Gated dual GEMM fused epilogue: gate_out = silu(gate_gemm_out) * up_gemm_out +// Two GEMMs share the same A operand; their outputs are combined in the +// epilogue warpgroup without touching global memory between them. + +#include +#include + +template +__device__ void gated_dual_gemm_epilogue( + float gate_acc[TILE_M][TILE_N], // accumulator from gate GEMM + float up_acc[TILE_M][TILE_N], // accumulator from up GEMM + __half* out) +{ + for (int m = 0; m < TILE_M; ++m) { + for (int n = 0; n < TILE_N; ++n) { + float g = gate_acc[m][n]; + // SiLU(x) = x * sigmoid(x) + float silu_g = g * (1.f / (1.f + __expf(-g))); + out[m * TILE_N + n] = __float2half(silu_g * up_acc[m][n]); + } + } +} diff --git a/verification/evidence/local-snapshots/cutlass-audit.8aj07j/repo/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu b/verification/evidence/local-snapshots/cutlass-audit.8aj07j/repo/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu new file mode 100644 index 000000000..0a5d6c314 --- /dev/null +++ b/verification/evidence/local-snapshots/cutlass-audit.8aj07j/repo/examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm.cu @@ -0,0 +1,597 @@ +/*************************************************************************************************** + * Copyright (c) 2025 - 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ + +/*! \file + \brief A GEMM example using CUTLASS for the NVIDIA Blackwell SM100 architecture. + + This example demonstrate a simple way to instantiate and run a blockscaled NVFP4 GEMM on the NVIDIA Blackwell SM100 architecture + on NVIDIA Blackwell SM100 architecture. The kernel outputs quantized fp4 values with scale factors that be the input of another GEMM. + + Similar to 72a_blackwell_nvfp4_bf16_gemm, this kernel leverages: + 1. Blockscaled tcgen05.mma instructions. + + 2. Per-SM memory called Tensor Memory (TMEM) + + 3. The extended warp-specialized kernel design introduced in Hopper enabled by use of TMEM + which allows us to decouple the execution of MMA and epilogue into separate warps. + + 4. A new SW controlled dynamic scheduler based on cluster launch control (See https://docs.nvidia.com/cuda/parallel-thread-execution). + + Usage: + + $ ./examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm --m=2048 --n=2048 --k=2048 +*/ + +#include + +#include "cutlass/cutlass.h" + +#include "cute/tensor.hpp" +#include "cutlass/tensor_ref.h" +#include "cutlass/epilogue/thread/linear_combination.h" +#include "cutlass/gemm/dispatch_policy.hpp" +#include "cutlass/gemm/collective/collective_builder.hpp" +#include "cutlass/epilogue/collective/collective_builder.hpp" +#include "cutlass/detail/sm100_blockscaled_layout.hpp" +#include "cutlass/gemm/device/gemm_universal_adapter.h" +#include "cutlass/gemm/kernel/gemm_universal.hpp" +#include "cutlass/gemm/kernel/tile_scheduler_params.h" + +#include "cutlass/util/command_line.h" +#include "cutlass/util/distribution.h" +#include "cutlass/util/host_tensor.h" +#include "cutlass/util/packed_stride.hpp" +#include "cutlass/util/tensor_view_io.h" +#include "cutlass/util/reference/device/gemm.h" +#include "cutlass/util/reference/device/tensor_compare.h" +#include "cutlass/util/reference/host/tensor_fill.h" +#include "cutlass/util/reference/host/gett.hpp" +#include "cutlass/util/reference/host/tensor_norm.h" +#include "cutlass/util/reference/host/tensor_compare.h" + + +#include + +#include "helper.h" + +using namespace cute; + +#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) + + +///////////////////////////////////////////////////////////////////////////////////////////////// +/// GEMM kernel configurations +///////////////////////////////////////////////////////////////////////////////////////////////// + +// A matrix configuration +using ElementA = cutlass::nv_float4_t; // Element type for A matrix operand +using LayoutATag = cutlass::layout::RowMajor; // Layout type for A matrix operand +constexpr int AlignmentA = 32; // Memory access granularity/alignment of A matrix in units of elements (up to 16 bytes) + +// B matrix configuration +using ElementB = cutlass::nv_float4_t; // Element type for A matrix operand +using LayoutBTag = cutlass::layout::ColumnMajor; // Layout type for B matrix operand +constexpr int AlignmentB = 32; // Memory access granularity/alignment of B matrix in units of elements (up to 16 bytes) + +// C/D matrix configuration +using ElementD = cutlass::float_e2m1_t; // Element type for D matrix operand +using ElementSFD = cutlass::float_ue8m0_t; // Element type for SFB matrix operand +using ElementC = float; // Element type for C matrix operand +using LayoutCTag = cutlass::layout::RowMajor; // Layout type for C matrix operand +using LayoutDTag = cutlass::layout::RowMajor; // Layout type for D matrix operand +using LayoutSFDTag = LayoutDTag; // Layout type for SFD should be same as D matrix operand + +constexpr int AlignmentD = 128 / cutlass::sizeof_bits::value; // Memory access granularity/alignment of C matrix in units of elements (up to 16 bytes) +constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; // Memory access granularity/alignment of C matrix in units of elements (up to 16 bytes) + +// Kernel functional config +using ElementAccumulator = float; // Element type for internal accumulation +using ElementCompute = float; // Element type for internal accumulation +using ArchTag = cutlass::arch::Sm100; // Tag indicating the minimum SM that supports the intended feature +using OperatorClass = cutlass::arch::OpClassBlockScaledTensorOp; // Operator class tag + +// Kernel Perf config +using MmaTileShape = Shape<_128,_128,_256>; // MMA's tile size +using ClusterShape = Shape<_1,_1,_1>; // Shape of the threadblocks in a cluster + +constexpr int InputSFVectorSize = 16; +constexpr int OutputSFVectorSize = InputSFVectorSize; + +// D = alpha * acc + beta * C +// With BlockScaleFactor generation. +using FusionOperation = cutlass::epilogue::fusion::LinCombBlockScaleFactor< + OutputSFVectorSize, + ElementD, + ElementCompute, + ElementSFD, LayoutSFDTag, + ElementC>; + +using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, OperatorClass, + MmaTileShape, ClusterShape, + cutlass::epilogue::collective::EpilogueTileAuto, + ElementAccumulator, ElementAccumulator, + ElementC, LayoutCTag, AlignmentC, + ElementD, LayoutDTag, AlignmentD, + cutlass::epilogue::collective::EpilogueScheduleAuto, // Epilogue schedule policy + FusionOperation + >::CollectiveOp; + +using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, OperatorClass, + ElementA, LayoutATag, AlignmentA, + ElementB, LayoutBTag, AlignmentB, + ElementAccumulator, + MmaTileShape, ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout(sizeof(typename CollectiveEpilogue::SharedStorage))>, + cutlass::gemm::collective::KernelScheduleAuto // Kernel schedule policy. Auto or using targeted scheduling policy + >::CollectiveOp; + +using GemmKernel = cutlass::gemm::kernel::GemmUniversal< + Shape, // Indicates ProblemShape + CollectiveMainloop, + CollectiveEpilogue, + void>; + +using Gemm = cutlass::gemm::device::GemmUniversalAdapter; + +// Reference device GEMM implementation type +using StrideA = typename Gemm::GemmKernel::StrideA; +using LayoutA = decltype(cute::make_layout(make_shape(0,0,0), StrideA{})); +using LayoutSFA = typename Gemm::GemmKernel::CollectiveMainloop::LayoutSFA; // Scale Factor tensors have an interleaved layout. Bring Layout instead of stride. +using StrideB = typename Gemm::GemmKernel::StrideB; +using LayoutB = decltype(cute::make_layout(make_shape(0,0,0), StrideB{})); +using LayoutSFB = typename Gemm::GemmKernel::CollectiveMainloop::LayoutSFB; // Scale Factor tensors have an interleaved layout. Bring Layout instead of stride. +using StrideC = typename Gemm::GemmKernel::StrideC; +using LayoutC = decltype(cute::make_layout(make_shape(0,0,0), StrideC{})); +using StrideD = typename Gemm::GemmKernel::StrideD; +using LayoutD = decltype(cute::make_layout(make_shape(0,0,0), StrideD{})); + +using FusionOp = typename Gemm::EpilogueOutputOp; +constexpr bool IsBlockScaleSupported = FusionOp::IsBlockScaleSupported; +using SfdOutputCfg = cutlass::detail::Sm1xxBlockScaledOutputConfig; +using LayoutSFD = typename SfdOutputCfg::LayoutSF; + +// +// Data members +// + +/// Initialization +StrideA stride_A; +LayoutA layout_A; +LayoutSFA layout_SFA; +StrideB stride_B; +LayoutB layout_B; +LayoutSFB layout_SFB; +StrideC stride_C; +LayoutC layout_C; +StrideD stride_D; +LayoutD layout_D; +LayoutSFD layout_SFD; + +uint64_t seed; + +// The HostTensors are only used for allocating memory on host and device, and transferring data between host and device +// Use cute::Tensor and cute::Layout for iterating thru the matrix elements +cutlass::HostTensor block_A; +cutlass::HostTensor block_SFA; +cutlass::HostTensor block_B; +cutlass::HostTensor block_SFB; +cutlass::HostTensor block_C; +// Output Tensors +cutlass::HostTensor block_D; +cutlass::HostTensor block_SFD; +// Reference Output Tensors +cutlass::HostTensor block_reference_D; +cutlass::HostTensor block_reference_SFD; +// Matrix-wide normalization constant +cutlass::HostTensor block_Normconst; + +#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) + +template +auto make_iterator(T* ptr) { + return cute::recast_ptr(ptr); +} + +///////////////////////////////////////////////////////////////////////////////////////////////// +/// Testbed utility types +///////////////////////////////////////////////////////////////////////////////////////////////// + +// Command line options parsing +struct Options { + + bool help; + + float alpha, beta; + int iterations; + int m, n, k; + int swizzle = 0; + + Options(): + help(false), + m(1024), n(1024), k(1024), + alpha(1.f), beta(0.f), + iterations(10), + swizzle(0) + { } + + // Parses the command line + void parse(int argc, char const **args) { + cutlass::CommandLine cmd(argc, args); + + if (cmd.check_cmd_line_flag("help")) { + help = true; + return; + } + + cmd.get_cmd_line_argument("m", m); + cmd.get_cmd_line_argument("n", n); + cmd.get_cmd_line_argument("k", k); + cmd.get_cmd_line_argument("alpha", alpha, 1.f); + cmd.get_cmd_line_argument("beta", beta, 0.f); + cmd.get_cmd_line_argument("iterations", iterations); + cmd.get_cmd_line_argument("swizzle", swizzle); + } + + /// Prints the usage statement. + std::ostream & print_usage(std::ostream &out) const { + + out << "72b_blackwell_nvfp4_nvfp4_gemm\n\n" + << " Blackwell NVFP4 GEMM using a Warp Specialized kernel.\n\n" + << "Options:\n\n" + << " --help If specified, displays this usage statement\n\n" + << " --m= Sets the M extent of the GEMM\n" + << " --n= Sets the N extent of the GEMM\n" + << " --k= Sets the K extent of the GEMM\n" + << " --alpha= Epilogue scalar alpha\n" + << " --beta= Epilogue scalar beta\n" + << " --swizzle= Cluster rasterization swizzle\n" + << " --iterations= Number of profiling iterations to perform.\n\n"; + + out << "\n\nExamples:\n\n" + << "$ " << "./examples/72_blackwell_narrow_precision_gemm/72b_blackwell_nvfp4_nvfp4_gemm" << " --m=1024 --n=512 --k=1024 --alpha=2 --beta=0.707 \n\n"; + + return out; + } + + /// Compute performance in GFLOP/s + double gflops(double runtime_s) const + { + // Two flops per multiply-add + uint64_t flop = uint64_t(2) * m * n * k; + double gflop = double(flop) / double(1.0e9); + return gflop / runtime_s; + } +}; + +/// Result structure +struct Result +{ + double avg_runtime_ms; + double gflops; + cutlass::Status status; + cudaError_t error; + bool passed; + + Result( + double avg_runtime_ms = 0, + double gflops = 0, + cutlass::Status status = cutlass::Status::kSuccess, + cudaError_t error = cudaSuccess) + : + avg_runtime_ms(avg_runtime_ms), gflops(gflops), status(status), error(error), passed(false) + {} + +}; + +#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) + +///////////////////////////////////////////////////////////////////////////////////////////////// +/// GEMM setup and evaluation +///////////////////////////////////////////////////////////////////////////////////////////////// + +/// Helper to initialize a block of device data +template +bool initialize_block( + cutlass::TensorView view, + uint64_t seed) { + + double scope_max, scope_min; + constexpr int bits_input = cutlass::sizeof_bits::value; + + if constexpr (bits_input == 1) { + scope_max = 2; + scope_min = 0; + } + else if constexpr (bits_input <= 6) { + scope_max = 2; + scope_min = -2; + } + else if constexpr (bits_input <= 8) { + if constexpr (cute::is_same_v) { + scope_max = 4; + scope_min = 1; + } + else { + scope_max = 1; + scope_min = -1; + } + } + else{ + scope_max = 4; + scope_min = -4; + } + cutlass::reference::host::TensorFillRandomUniform( + view, seed, scope_max, scope_min, 0); + + return true; +} + +/// Initialize operands to be used in the GEMM and reference GEMM +void initialize(const Options &options) { + using namespace cute; + // For SFA and SFB tensors layouts + using Sm1xxBlkScaledConfig = typename Gemm::GemmKernel::CollectiveMainloop::Sm1xxBlkScaledConfig; + // For SFD tensor layout + using Sm1xxBlockScaledOutputConfig= typename Gemm::GemmKernel::CollectiveMainloop::Sm1xxBlkScaledConfig; + + stride_A = cutlass::make_cute_packed_stride(StrideA{}, {options.m, options.k, 1}); + stride_B = cutlass::make_cute_packed_stride(StrideB{}, {options.n, options.k, 1}); + stride_C = cutlass::make_cute_packed_stride(StrideC{}, {options.m, options.n, 1}); + stride_D = cutlass::make_cute_packed_stride(StrideD{}, {options.m, options.n, 1}); + + layout_A = make_layout(make_shape(options.m, options.k, 1), stride_A); + layout_B = make_layout(make_shape(options.n, options.k, 1), stride_B); + layout_C = make_layout(make_shape(options.m, options.n, 1), stride_C); + layout_D = make_layout(make_shape(options.m, options.n, 1), stride_D); + layout_SFA = Sm1xxBlkScaledConfig::tile_atom_to_shape_SFA(cute::make_shape(options.m, options.n, options.k, 1)); + layout_SFB = Sm1xxBlkScaledConfig::tile_atom_to_shape_SFB(cute::make_shape(options.m, options.n, options.k, 1)); + layout_SFD = SfdOutputCfg::tile_atom_to_shape_SFD(cute::make_shape(options.m, options.n, options.k, 1)); + + block_A.reset(cutlass::make_Coord(size(layout_A))); + block_B.reset(cutlass::make_Coord(size(layout_B))); + block_C.reset(cutlass::make_Coord(size(layout_C))); + block_D.reset(cutlass::make_Coord(size(layout_D))); + block_reference_D.reset(cutlass::make_Coord(size(layout_D))); + block_reference_SFD.reset(cutlass::make_Coord(size(filter_zeros(layout_SFD)))); + block_Normconst.reset(cutlass::make_Coord(1)); + + block_SFA.reset(cutlass::make_Coord(size(filter_zeros(layout_SFA)))); + block_SFB.reset(cutlass::make_Coord(size(filter_zeros(layout_SFB)))); + block_SFD.reset(cutlass::make_Coord(size(filter_zeros(layout_SFD)))); + + initialize_block(block_A.host_view(), seed + 2021); + initialize_block(block_B.host_view(), seed + 2022); + initialize_block(block_C.host_view(), seed + 2023); + initialize_block(block_SFA.host_view(), seed + 2024); + initialize_block(block_SFB.host_view(), seed + 2025); + block_Normconst.at(cutlass::make_Coord(0)) = 2; + + block_A.sync_device(); + block_B.sync_device(); + block_C.sync_device(); + block_D.sync_device(); + block_SFA.sync_device(); + block_SFB.sync_device(); + block_SFD.sync_device(); + block_Normconst.sync_device(); + +} + +// Populates a Gemm::Arguments structure from the given commandline options +typename Gemm::Arguments args_from_options(const Options &options) +{ + typename Gemm::Arguments arguments { + cutlass::gemm::GemmUniversalMode::kGemm, + {options.m, options.n, options.k, 1}, + { // Mainloop arguments + block_A.device_data(), stride_A, + block_B.device_data(), stride_B, + block_SFA.device_data(), layout_SFA, + block_SFB.device_data(), layout_SFB + }, + { // Epilogue arguments + { options.alpha, options.beta }, + block_C.device_data(), stride_C, + block_D.device_data(), stride_D} + }; + + if constexpr (IsBlockScaleSupported) { + arguments.epilogue.thread.block_scale_factor_ptr = block_SFD.device_data(); + arguments.epilogue.thread.norm_constant_ptr = block_Normconst.device_data(); + } + + arguments.scheduler.max_swizzle_size = options.swizzle; + return arguments; +} + +bool verify(const Options &options) { + using namespace cute; + // Create the arguments for host reference implementation + Tensor tensor_A = make_tensor(make_iterator(block_A.host_data()), layout_A); + Tensor tensor_SFA = make_tensor(block_SFA.host_data(), layout_SFA); + Tensor tensor_B = make_tensor(make_iterator(block_B.host_data()), layout_B); + Tensor tensor_SFB = make_tensor(block_SFB.host_data(), layout_SFB); + + // think about how to simplify the gemm3x interface. + cutlass::reference::host::GettBlockScalingMainloopParams< + ElementAccumulator, // ElementAccumulator + decltype(tensor_A), // TensorA + decltype(tensor_SFA), // TensorSfA + decltype(tensor_B), // TensorB + decltype(tensor_SFB) // TensorSfB + > mainloop_params{tensor_A, tensor_SFA, tensor_B, tensor_SFB}; + + Tensor tensor_C = cute::make_tensor(make_iterator(block_C.host_data()), layout_C); + Tensor tensor_D = cute::make_tensor(make_iterator(block_reference_D.host_data()), layout_D); + Tensor tensor_SFD = make_tensor(block_reference_SFD.host_data(), layout_SFD); + + cutlass::reference::host::GettBlockScalingEpilogueParams< + ElementCompute, // ElementScalar + ElementAccumulator, // ElementAccumulator + ElementCompute, // ElementCompute + decltype(tensor_C), // TensorC + decltype(tensor_D), // TensorD + decltype(tensor_SFD), // TensorSfD + cute::Int, + cutlass::reference::host::SfStrategy::SfDGen + > epilogue_params {options.alpha, options.beta, tensor_C, tensor_D, tensor_SFD, block_Normconst.at(cutlass::make_Coord(0))}; + + cutlass::reference::host::Gemm3x(mainloop_params, epilogue_params); + + // Comparison + block_D.sync_host(); + bool passed = cutlass::reference::host::TensorEquals(block_reference_D.host_view(), block_D.host_view()); + passed &= (cutlass::reference::host::TensorNorm(block_reference_D.host_view()) > 0); + passed &= (cutlass::reference::host::TensorNorm(block_D.host_view()) > 0); + + block_SFD.sync_host(); + bool passed_sfd = cutlass::reference::host::TensorEquals(block_reference_SFD.host_view(), block_SFD.host_view()); + passed_sfd &= (cutlass::reference::host::TensorNorm(block_reference_SFD.host_view()) > 0); + passed_sfd &= (cutlass::reference::host::TensorNorm(block_SFD.host_view()) > 0); + + return passed && passed_sfd; +} + +/// Execute a given example GEMM computation +template +int run(Options &options) +{ + initialize(options); + + // Instantiate CUTLASS kernel depending on templates + Gemm gemm; + + // Create a structure of gemm kernel arguments suitable for invoking an instance of Gemm + auto arguments = args_from_options(options); + + // Using the arguments, query for extra workspace required for matrix multiplication computation + size_t workspace_size = Gemm::get_workspace_size(arguments); + + // Allocate workspace memory + cutlass::device_memory::allocation workspace(workspace_size); + + // Check if the problem size is supported or not + CUTLASS_CHECK(gemm.can_implement(arguments)); + + // Initialize CUTLASS kernel with arguments and workspace pointer + CUTLASS_CHECK(gemm.initialize(arguments, workspace.get())); + + // Correctness / Warmup iteration + CUTLASS_CHECK(gemm.run()); + + cudaDeviceSynchronize(); + + // Check if output from CUTLASS kernel and reference kernel are equal or not + Result result; + result.passed = verify(options); + + std::cout << " Disposition: " << (result.passed ? "Passed" : "Failed") << std::endl; + + if (!result.passed) { + exit(-1); + } + + // Run profiling loop + if (options.iterations > 0) + { + GpuTimer timer; + timer.start(); + for (int iter = 0; iter < options.iterations; ++iter) { + CUTLASS_CHECK(gemm.initialize(arguments, workspace.get())); + CUTLASS_CHECK(gemm.run()); + } + timer.stop(); + + // Compute average runtime and GFLOPs. + float elapsed_ms = timer.elapsed_millis(); + result.avg_runtime_ms = double(elapsed_ms) / double(options.iterations); + result.gflops = options.gflops(result.avg_runtime_ms / 1000.0); + + + std::cout << " Problem Size: " << options.m << 'x' << options.n << 'x' << options.k << std::endl; + std::cout << " Avg runtime: " << result.avg_runtime_ms << " ms" << std::endl; + std::cout << " GFLOPS: " << result.gflops << std::endl; + } + + return 0; +} + +#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) + +/////////////////////////////////////////////////////////////////////////////////////////////////// + +int main(int argc, char const **args) { + + // CUTLASS must be compiled with CUDA 12.8 or higher Toolkit to run this example + // and must have compute capability at least 100. + if (__CUDACC_VER_MAJOR__ < 12 || (__CUDACC_VER_MAJOR__ == 12 && __CUDACC_VER_MINOR__ < 8)) { + std::cerr << "This example requires CUDA 12.8 or newer." << std::endl; + // Returning zero so this test passes on older Toolkits. Its actions are no-op. + return 0; + } + + cudaDeviceProp props; + int current_device_id; + CUDA_CHECK(cudaGetDevice(¤t_device_id)); + + CUDA_CHECK(cudaGetDeviceProperties(&props, current_device_id)); + + if (props.major != 10 || (props.minor != 0 && props.minor != 1 && props.minor != 3)) { + std::cerr << "This example requires a GPU with compute capability 100a|f, 101a|f, or 103a|f)." << std::endl; + return 0; + } + + // + // Parse options + // + + Options options; + + options.parse(argc, args); + + if (options.help) { + options.print_usage(std::cout) << std::endl; + return 0; + } + + // + // Evaluate CUTLASS kernels + // +#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) + run(options); +#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) + + return 0; +} + +///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/verification/evidence/local-snapshots/cutlass-audit.8aj07j/repo/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu b/verification/evidence/local-snapshots/cutlass-audit.8aj07j/repo/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu new file mode 100644 index 000000000..1ff8f2152 --- /dev/null +++ b/verification/evidence/local-snapshots/cutlass-audit.8aj07j/repo/examples/75_blackwell_grouped_gemm/75_blackwell_grouped_gemm_block_scaled.cu @@ -0,0 +1,944 @@ +/*************************************************************************************************** + * Copyright (c) 2024 - 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ + + + +/*! \file + \brief Grouped GEMM example using CUTLASS 3 APIs for the NVIDIA Blackwell SM100 architecture. + + This example demonstrates an implementation of Grouped GEMM using a TMA + Blackwell SM100 TensorOp-based warp-specialized kernel + for narrow precisions (FP4) with Scale Factors (In and Out). + For this example all scheduling work is performed on the device. + The new feature showcased in this example is device-side modification of TMA descriptors + to move between groups/problem_count (represented by groups). + https://docs.nvidia.com/cuda/cuda-c-programming-guide/#encoding-a-tensor-map-on-device + + To run this example: + + $ ./examples/75_blackwell_grouped_gemm_block_scaled/75_blackwell_grouped_gemm_block_scaled --m=2048 --n=2048 --k=2048 --groups=10 + + The above example command makes all 10 groups to be sized at the given m, n, k sizes. + Skipping any of the problem dimensions randomizes it across the different groups. + Same applies for alpha and beta values that are randomized across the different groups. + + To run this example for a set of problems using the benchmark option: + + $ ./examples/75_blackwell_grouped_gemm_block_scaled/75_blackwell_grouped_gemm_block_scaled --benchmark=./test_benchmark.txt + + Where the test_benchmark.txt may look as such: + 0 256x512x128 + 1 256x512x512 + 2 512x256x128 + 3 256x256x128 + 4 256x512x1024 + 5 1024x512x128 and so on +*/ + +#include +#include +#include +#include +#include +#include + +#include "cutlass/cutlass.h" + +#include "cute/tensor.hpp" +#include "cutlass/tensor_ref.h" +#include "cutlass/epilogue/collective/default_epilogue.hpp" +#include "cutlass/epilogue/thread/linear_combination.h" +#include "cutlass/gemm/dispatch_policy.hpp" +#include "cutlass/gemm/group_array_problem_shape.hpp" +#include "cutlass/gemm/collective/collective_builder.hpp" +#include "cutlass/epilogue/collective/collective_builder.hpp" +#include "cutlass/gemm/device/gemm_universal_adapter.h" +#include "cutlass/gemm/kernel/gemm_universal.hpp" + +#include "cutlass/util/command_line.h" +#include "cutlass/util/distribution.h" +#include "cutlass/util/host_tensor.h" +#include "cutlass/util/packed_stride.hpp" +#include "cutlass/util/tensor_view_io.h" +#include "cutlass/util/reference/device/gemm.h" +#include "cutlass/util/reference/device/tensor_compare.h" +#include "cutlass/util/reference/host/tensor_fill.h" +#include "cutlass/util/reference/host/gett.hpp" +#include "cutlass/util/reference/host/tensor_norm.h" +#include "cutlass/util/reference/host/tensor_compare.h" + +#include "helper.h" +using namespace cute; + +using ProblemShape = cutlass::gemm::GroupProblemShape>; // per group +using ElementInput = cutlass::float_e2m1_t; // Element type for Input matrix operands +using ElementSF = cutlass::float_ue4m3_t; // Element type for SF matrix operands +using ElementC = cutlass::half_t; // Element type for C matrix operands + +#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) +///////////////////////////////////////////////////////////////////////////////////////////////// +/// GEMM kernel configurations +///////////////////////////////////////////////////////////////////////////////////////////////// +// A matrix configuration +using ElementA = cutlass::nv_float4_t; // Element type for A matrix operand +using LayoutA = cutlass::layout::RowMajor; // Layout type for A matrix operand +constexpr int AlignmentA = 32; // Alignment of A matrix in units of elements (up to 16 bytes) + +// B matrix configuration +using ElementB = cutlass::nv_float4_t; // Element type for B matrix operand +using LayoutB = cutlass::layout::ColumnMajor; // Layout type for B matrix operand +constexpr int AlignmentB = 32; // Alignment of A matrix in units of elements (up to 16 bytes) + +// C/D matrix configuration +using ElementD = ElementC; // Element type for D matrix operands +using LayoutC = cutlass::layout::RowMajor; // Layout type for C and D matrix operands +constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; // Alignment of C matrix in units of elements (up to 16 bytes) +constexpr int AlignmentD = 128 / cutlass::sizeof_bits::value; // Alignment of D matrix in units of elements (up to 16 bytes) +using ElementAccumulator = float; // Element type for internal accumulation + +// using ElementD = cutlass::float_e2m1_t; // Enable for SF Output // Element type for D matrix operands + +using ElementSFD = cutlass::float_ue4m3_t; // Element type for SF Output operands +constexpr int OutputSFVectorSize = 16; +using FusionOperation = cutlass::epilogue::fusion::LinCombEltActBlockScaleFactor< + cutlass::epilogue::thread::SiLu, + OutputSFVectorSize, + ElementD, + ElementAccumulator, + ElementSFD, + LayoutC, + ElementC>; + +// Core kernel configurations +using ArchTag = cutlass::arch::Sm100; // Tag indicating the minimum SM that supports the intended feature +using OperatorClass = cutlass::arch::OpClassBlockScaledTensorOp; // Operator class tag +using StageCountType = cutlass::gemm::collective::StageCountAuto; // Stage count maximized based on the tile size + +// Runtime Cluster Shape +using ClusterShape = Shape; + +// Different configs for 1SM and 2SM MMA kernel +struct MMA1SMConfig { + using MmaTileShape = Shape<_128,_256,_256>; + using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100; // Kernel to launch + using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecialized1Sm; // Epilogue to launch +}; + +struct MMA2SMConfig { + using MmaTileShape = Shape<_256,_256,_256>; + using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecialized2SmNvf4Sm100; // Kernel to launch + using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecialized2Sm; // Epilogue to launch +}; + +using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, OperatorClass, + typename MMA1SMConfig::MmaTileShape, ClusterShape, + Shape<_128,_64>, + ElementAccumulator, ElementAccumulator, + ElementC, LayoutC *, AlignmentC, + ElementD, LayoutC *, AlignmentD, + typename MMA1SMConfig::EpilogueSchedule + // , FusionOperation // Enable for SF Output +>::CollectiveOp; +using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, OperatorClass, + ElementA, LayoutA *, AlignmentA, + ElementB, LayoutB *, AlignmentB, + ElementAccumulator, + typename MMA1SMConfig::MmaTileShape, ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout< + static_cast(sizeof(typename CollectiveEpilogue::SharedStorage))>, + typename MMA1SMConfig::KernelSchedule +>::CollectiveOp; +using GemmKernel = cutlass::gemm::kernel::GemmUniversal< + ProblemShape, + CollectiveMainloop, + CollectiveEpilogue +>; +using Gemm1SM = cutlass::gemm::device::GemmUniversalAdapter; +using Gemm = Gemm1SM; + +using CollectiveEpilogue2SM = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, OperatorClass, + typename MMA2SMConfig::MmaTileShape, ClusterShape, + Shape<_128,_64>, + ElementAccumulator, ElementAccumulator, + ElementC, LayoutC *, AlignmentC, + ElementD, LayoutC *, AlignmentD, + typename MMA2SMConfig::EpilogueSchedule + // , FusionOperation // Enable for SF Output +>::CollectiveOp; +using CollectiveMainloop2SM = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, OperatorClass, + ElementA, LayoutA *, AlignmentA, + ElementB, LayoutB *, AlignmentB, + ElementAccumulator, + typename MMA2SMConfig::MmaTileShape, ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout< + static_cast(sizeof(typename CollectiveEpilogue2SM::SharedStorage))>, + typename MMA2SMConfig::KernelSchedule +>::CollectiveOp; +using GemmKernel2SM = cutlass::gemm::kernel::GemmUniversal< + ProblemShape, + CollectiveMainloop2SM, + CollectiveEpilogue2SM +>; +using Gemm2SM = cutlass::gemm::device::GemmUniversalAdapter; + +using StrideA = typename Gemm::GemmKernel::InternalStrideA; +using StrideB = typename Gemm::GemmKernel::InternalStrideB; +using StrideC = typename Gemm::GemmKernel::InternalStrideC; +using StrideD = typename Gemm::GemmKernel::InternalStrideD; + +using LayoutSFA = typename Gemm::GemmKernel::CollectiveMainloop::InternalLayoutSFA; +using LayoutSFB = typename Gemm::GemmKernel::CollectiveMainloop::InternalLayoutSFB; +using Sm1xxBlkScaledConfig = typename Gemm::GemmKernel::CollectiveMainloop::Sm1xxBlkScaledConfig; +using Sm1xxBlockScaledOutputConfig= cutlass::detail::Sm1xxBlockScaledOutputConfig< + OutputSFVectorSize, + cute::is_same_v ? cute::UMMA::Major::K : cute::UMMA::Major::MN + >; +using OutputSFAtom = typename Sm1xxBlockScaledOutputConfig::SfAtom; +using LayoutSFD = typename Sm1xxBlockScaledOutputConfig::LayoutSF; + +// Host-side allocations +std::vector stride_A_host; +std::vector stride_B_host; +std::vector layout_SFA_host; +std::vector layout_SFB_host; +std::vector stride_C_host; +std::vector stride_D_host; + +std::vector alpha_host; +std::vector beta_host; + +using HostTensorA = cutlass::HostTensor; +using HostTensorB = cutlass::HostTensor; +using HostTensorSF = cutlass::HostTensor; +using HostTensorC = cutlass::HostTensor; +using HostTensorD = cutlass::HostTensor; +std::vector block_A; +std::vector block_B; +std::vector block_SFA; +std::vector block_SFB; +std::vector block_C; +std::vector block_D; +std::vector block_SFD; +std::vector block_ref_D; + +// Device-side allocations +cutlass::DeviceAllocation problem_sizes; + +cutlass::DeviceAllocation ptr_A; +cutlass::DeviceAllocation ptr_B; +cutlass::DeviceAllocation ptr_SFA; +cutlass::DeviceAllocation ptr_SFB; +cutlass::DeviceAllocation ptr_C; +cutlass::DeviceAllocation ptr_D; +cutlass::DeviceAllocation ptr_SFD; +cutlass::DeviceAllocation ptr_ref_D; + +cutlass::DeviceAllocation stride_A; +cutlass::DeviceAllocation stride_B; +cutlass::DeviceAllocation layout_SFA; +cutlass::DeviceAllocation layout_SFB; +cutlass::DeviceAllocation stride_C; +cutlass::DeviceAllocation stride_D; + +// Note, this is an array of pointers to alpha and beta scaling values per group +cutlass::DeviceAllocation alpha_device; +cutlass::DeviceAllocation beta_device; +cutlass::DeviceAllocation block_alpha; +cutlass::DeviceAllocation block_beta; +// A matrix wide constant value to scale the output matrix +// Avoids generating small FP4 values. +// NormConst is a single device-side constant value, its not per-batch or per-group +cutlass::DeviceAllocation norm_constant_device; + +#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) + +template +auto make_iterator(T* ptr) { + return cute::recast_ptr(ptr); +} + +///////////////////////////////////////////////////////////////////////////////////////////////// +/// Testbed utility types +///////////////////////////////////////////////////////////////////////////////////////////////// + +using RasterOrderOptions = cutlass::gemm::kernel::detail::RasterOrderOptions; +// Command line options parsing +struct Options { + + bool help = false; + bool verification = true; + bool use_pdl = false; + + float alpha = FLT_MAX; + float beta = FLT_MAX; + float norm_constant = 1.0; + int iterations = 10; + int m = 1024, n = 2048, k = 512, groups = 10; + dim3 cluster_shape = dim3(2,1,1); + dim3 cluster_shape_fallback = dim3(2,1,1); + RasterOrderOptions raster_order = RasterOrderOptions::AlongN; + char raster_char = 'N'; + int swizzle = 1; + int max_sm_count = INT_MAX; + std::string benchmark_path; + std::vector problem_sizes_host; + int const tma_alignment_bits = 128; + int const alignment = tma_alignment_bits / cutlass::sizeof_bits::value; + + // Parses the command line + void parse(int argc, char const **args) { + cutlass::CommandLine cmd(argc, args); + + if (cmd.check_cmd_line_flag("help")) { + help = true; + return; + } + if (cmd.check_cmd_line_flag("no_verif")) { + verification = false; + } + if (cmd.check_cmd_line_flag("use_pdl")) { + use_pdl = true; + } + + cmd.get_cmd_line_argument("m", m); + cmd.get_cmd_line_argument("n", n); + cmd.get_cmd_line_argument("k", k); + cmd.get_cmd_line_argument("groups", groups); + cmd.get_cmd_line_argument("alpha", alpha, FLT_MAX); + cmd.get_cmd_line_argument("beta", beta, FLT_MAX); + cmd.get_cmd_line_argument("norm_constant", norm_constant, float(1.0)); + cmd.get_cmd_line_argument("iterations", iterations); + cmd.get_cmd_line_argument("benchmark", benchmark_path); + cmd.get_cmd_line_argument("cluster_m", cluster_shape.x); + cmd.get_cmd_line_argument("cluster_n", cluster_shape.y); + cmd.get_cmd_line_argument("cluster_fallback_m", cluster_shape_fallback.x); + cmd.get_cmd_line_argument("cluster_fallback_n", cluster_shape_fallback.y); + cmd.get_cmd_line_argument("max_sm_count", max_sm_count, INT_MAX); + + // Decide how to initialize the problems + if (!benchmark_path.empty()) { + if (!benchmark_problems()) { + problem_sizes_host.clear(); + return; + } + } + else { + randomize_problems(cmd); + } + + cmd.get_cmd_line_argument("raster", raster_char); + + if (raster_char == 'N' || raster_char == 'n') { + raster_order = RasterOrderOptions::AlongN; + } + else if (raster_char == 'M' || raster_char == 'm') { + raster_order = RasterOrderOptions::AlongM; + } + cmd.get_cmd_line_argument("swizzle", swizzle, 1); + } + + void randomize_problems(cutlass::CommandLine &cmd) { + int cmd_line_m = -1, cmd_line_n = -1, cmd_line_k = -1; + cmd.get_cmd_line_argument("m", cmd_line_m); + cmd.get_cmd_line_argument("n", cmd_line_n); + cmd.get_cmd_line_argument("k", cmd_line_k); + + problem_sizes_host.reserve(groups); + + for (int i = groups; i > 0; i--) { + int m = cmd_line_m; + int n = cmd_line_n; + int k = cmd_line_k; + if (m < 0) { + m = alignment * ((rand() % 64)); + } + if (n < 0) { + n = alignment * ((rand() % 64)); + } + if (k < 0) { + k = alignment * ((rand() % 64)); + } + problem_sizes_host.push_back({m, n, k}); + } + } + + /// Load a benchmark + bool benchmark_problems() { + std::ifstream file(benchmark_path); + if (!file.good()) { + return false; + } + + while (file.good()) { + + int idx = -1; + std::string extent_str; + + file >> idx >> extent_str; + + if (idx < 0 || extent_str.empty()) { + break; + } + + cutlass::gemm::GemmCoord extent; + std::vector tokens; + + cutlass::CommandLine::tokenize(tokens, extent_str, 'x'); + + for (int i = 0; i < int(tokens.size()); ++i) { + extent.at(i) = std::atoi(tokens.at(i).c_str()); + } + problem_sizes_host.push_back({extent.m(), extent.n(), extent.k()}); + } + groups = static_cast(problem_sizes_host.size()); + + return true; + } + + /// Prints the usage statement. + std::ostream & print_usage(std::ostream &out) const { + + out << "75_blackwell_grouped_gemm_block_scaled\n\n" + << " Blackwell Block Scaled Narrow Precision Grouped GEMM using a Warp Specialized kernel.\n\n" + << "Options:\n\n" + << " --help If specified, displays this usage statement\n\n" + << " --m= Sets the M extent of the GEMM for all groups\n" + << " --n= Sets the N extent of the GEMM for all groups\n" + << " --k= Sets the K extent of the GEMM for all groups\n" + << " --groups= Sets the number of individual GEMM problems for Grouped GEMM\n" + << " --alpha= Epilogue scalar alpha\n" + << " --beta= Epilogue scalar beta\n" + << " --norm_constant= Epilogue scalar normalization constant for the output matrix\n\n" + << " --cluster_m= and --cluster_n= Sets the X,Y dims of the preferred cluster shape\n" + << " --cluster_fallback_m= and --cluster_fallback_n= Sets the X,Y dims of the fallback cluster shape\n\n" + << " --raster= Cluster rasterization direction (N for along N, M for along M)\n" + << " --swizzle= Cluster swizzle (swizzle up to 8 and with the nearest multiple of 2)\n\n" + << " --iterations= Number of profiling iterations to perform\n\n" + << " --benchmark= Executes a benchmark problem size\n" + << " --max_sm_count= Run kernels using only these number of SMs\n" + << " --no_verif Do not run (host-side) verification kernels\n" + << " --use_pdl Launch kernel with PDL (Programmatic Dependent Launch) enabled\n"; + + out + << "\n\nExamples:\n\n" + << "$ " << "75_blackwell_grouped_gemm_block_scaled" << " --m=1024 --n=512 --k=1024 --groups=10 --alpha=2 --beta=0.707 \n\n"; + + return out; + } + + /// Compute performance in GFLOP/s + double gflops(double runtime_s, std::vector problem_sizes_host) const + { + // Number of real-valued multiply-adds + uint64_t fmas = uint64_t(); + + for (auto const & problem : problem_sizes_host) { + fmas += static_cast(get<0>(problem)) * + static_cast(get<1>(problem)) * + static_cast(get<2>(problem)); + } + // Two flops per multiply-add + uint64_t flop = uint64_t(2) * uint64_t(fmas); + double gflop = double(flop) / double(1.0e9); + return gflop / runtime_s; + } +}; + +/// Result structure +struct Result +{ + double avg_runtime_ms = 0.0; + double gflops = 0.0; + cutlass::Status status = cutlass::Status::kSuccess; + cudaError_t error = cudaSuccess; + bool passed = false; +}; + +#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) + +///////////////////////////////////////////////////////////////////////////////////////////////// +/// GEMM setup and evaluation +///////////////////////////////////////////////////////////////////////////////////////////////// + +/// Helper to initialize a block of device data +template +bool initialize_block( + cutlass::TensorView view, + uint64_t seed) { + + double scope_max, scope_min; + constexpr int bits_input = cutlass::sizeof_bits::value; + + if constexpr (bits_input == 1) { + scope_max = 2; + scope_min = 0; + } + else if constexpr (bits_input <= 6) { + scope_max = 2; + scope_min = -2; + } + else if constexpr (bits_input <= 8) { + if constexpr (cute::is_same_v) { + scope_max = 4; + scope_min = 1; + } + else { + scope_max = 1; + scope_min = -1; + } + } + else{ + scope_max = 4; + scope_min = -4; + } + cutlass::reference::host::TensorFillRandomUniform( + view, seed, scope_max, scope_min, 0); + + return true; +} + +/// Allocates device-side data +void allocate(const Options &options) { + for (int32_t i = 0; i < options.groups; ++i) { + auto problem = options.problem_sizes_host.at(i); + auto M = get<0>(problem); + auto N = get<1>(problem); + auto K = get<2>(problem); + + auto stride_A = cutlass::make_cute_packed_stride(StrideA{}, {M, K, 1}); + auto stride_B = cutlass::make_cute_packed_stride(StrideB{}, {N, K, 1}); + auto stride_C = cutlass::make_cute_packed_stride(StrideC{}, {M, N, 1}); + auto stride_D = cutlass::make_cute_packed_stride(StrideD{}, {M, N, 1}); + + auto layout_A = make_layout(make_shape(M, K, 1), stride_A); + auto layout_B = make_layout(make_shape(N, K, 1), stride_B); + auto layout_C = make_layout(make_shape(M, N, 1), stride_C); + auto layout_D = make_layout(make_shape(M, N, 1), stride_D); + auto layout_SFA = Sm1xxBlkScaledConfig::tile_atom_to_shape_SFA(cute::make_shape(M, N, K, 1)); + auto layout_SFB = Sm1xxBlkScaledConfig::tile_atom_to_shape_SFB(cute::make_shape(M, N, K, 1)); + auto layout_SFD = Sm1xxBlockScaledOutputConfig::tile_atom_to_shape_SFD(cute::make_shape(M, N, K, 1)); + + stride_A_host.push_back(stride_A); + stride_B_host.push_back(stride_B); + layout_SFA_host.push_back(layout_SFA); + layout_SFB_host.push_back(layout_SFB); + stride_C_host.push_back(stride_C); + stride_D_host.push_back(stride_D); + + block_A.push_back(HostTensorA(cutlass::make_Coord(size(layout_A)))); + block_B.push_back(HostTensorB(cutlass::make_Coord(size(layout_B)))); + block_SFA.push_back(HostTensorSF(cutlass::make_Coord(size(filter_zeros(layout_SFA))))); + block_SFB.push_back(HostTensorSF(cutlass::make_Coord(size(filter_zeros(layout_SFB))))); + block_C.push_back(HostTensorC(cutlass::make_Coord(size(layout_C)))); + block_D.push_back(HostTensorD(cutlass::make_Coord(size(layout_D)))); + block_SFD.push_back(HostTensorSF(cutlass::make_Coord(size(filter_zeros(layout_SFD))))); + block_ref_D.push_back(HostTensorD(cutlass::make_Coord(size(layout_D)))); + } + block_alpha.reset(options.groups); + block_beta.reset(options.groups); +} + +/// Initialize operands to be used in the GEMM and reference GEMM +void initialize(const Options &options) { + uint64_t seed = 2020; + problem_sizes.reset(options.groups); + problem_sizes.copy_from_host(options.problem_sizes_host.data()); + + // + // Assign pointers + // + + std::vector ptr_A_host(options.groups); + std::vector ptr_B_host(options.groups); + std::vector ptr_SFA_host(options.groups); + std::vector ptr_SFB_host(options.groups); + std::vector ptr_C_host(options.groups); + std::vector ptr_D_host(options.groups); + std::vector ptr_SFD_host(options.groups); + std::vector ptr_alpha_host(options.groups); + std::vector ptr_beta_host(options.groups); + + for (int32_t i = 0; i < options.groups; ++i) { + + initialize_block(block_A.at(i).host_view(), seed + 2021); + initialize_block(block_B.at(i).host_view(), seed + 2022); + initialize_block(block_C.at(i).host_view(), seed + 2023); + initialize_block(block_SFA.at(i).host_view(), seed + 2024); + initialize_block(block_SFB.at(i).host_view(), seed + 2025); + + block_A.at(i).sync_device(); + block_B.at(i).sync_device(); + block_C.at(i).sync_device(); + block_SFA.at(i).sync_device(); + block_SFB.at(i).sync_device(); + + ptr_A_host.at(i) = block_A.at(i).device_data(); + ptr_B_host.at(i) = block_B.at(i).device_data(); + ptr_SFA_host.at(i) = block_SFA.at(i).device_data(); + ptr_SFB_host.at(i) = block_SFB.at(i).device_data(); + ptr_C_host.at(i) = block_C.at(i).device_data(); + ptr_D_host.at(i) = block_D.at(i).device_data(); + ptr_SFD_host.at(i) = block_SFD.at(i).device_data(); + + alpha_host.push_back((options.alpha == FLT_MAX) ? static_cast((rand() % 5) + 1) : options.alpha); + beta_host.push_back((options.beta == FLT_MAX) ? static_cast(rand() % 5) : options.beta); + ptr_alpha_host.at(i) = block_alpha.get() + i; + ptr_beta_host.at(i) = block_beta.get() + i; + } + + ptr_A.reset(options.groups); + ptr_A.copy_from_host(ptr_A_host.data()); + + ptr_B.reset(options.groups); + ptr_B.copy_from_host(ptr_B_host.data()); + + ptr_SFA.reset(options.groups); + ptr_SFA.copy_from_host(ptr_SFA_host.data()); + + ptr_SFB.reset(options.groups); + ptr_SFB.copy_from_host(ptr_SFB_host.data()); + + ptr_C.reset(options.groups); + ptr_C.copy_from_host(ptr_C_host.data()); + + ptr_D.reset(options.groups); + ptr_D.copy_from_host(ptr_D_host.data()); + + ptr_SFD.reset(options.groups); + ptr_SFD.copy_from_host(ptr_SFD_host.data()); + + stride_A.reset(options.groups); + stride_A.copy_from_host(stride_A_host.data()); + + stride_B.reset(options.groups); + stride_B.copy_from_host(stride_B_host.data()); + + layout_SFA.reset(options.groups); + layout_SFA.copy_from_host(layout_SFA_host.data()); + + layout_SFB.reset(options.groups); + layout_SFB.copy_from_host(layout_SFB_host.data()); + + stride_C.reset(options.groups); + stride_C.copy_from_host(stride_C_host.data()); + + stride_D.reset(options.groups); + stride_D.copy_from_host(stride_D_host.data()); + + alpha_device.reset(options.groups); + alpha_device.copy_from_host(ptr_alpha_host.data()); + beta_device.reset(options.groups); + beta_device.copy_from_host(ptr_beta_host.data()); + + block_alpha.copy_from_host(alpha_host.data()); + block_beta.copy_from_host(beta_host.data()); + + norm_constant_device.reset(1); + norm_constant_device.copy_from_host(&options.norm_constant); +} + +/// Populates a Gemm::Arguments structure from the given commandline options +template +typename Gemm::Arguments args_from_options(Options &options, bool host_problem_shapes_available = true) +{ + cutlass::KernelHardwareInfo hw_info; + // Change device_id to another value if you are running on a machine with multiple GPUs and wish + // to use a GPU other than that with device ID 0. + hw_info.device_id = 0; + hw_info.sm_count = min(cutlass::KernelHardwareInfo::query_device_multiprocessor_count(hw_info.device_id), options.max_sm_count); + + if (!is_static_v) { + if (size<0>(typename Gemm::GemmKernel::CollectiveMainloop::AtomThrShapeMNK{}) == 2 && + (options.cluster_shape.x < 2 || options.cluster_shape_fallback.x < 2)) { + std::cout << "Error: MMA2SMConfig kernel config needs cluster_dim.x >= 2" << std::endl; + } + hw_info.cluster_shape = options.cluster_shape; + hw_info.cluster_shape_fallback = options.cluster_shape_fallback; + } + + typename Gemm::Arguments arguments; + decltype(arguments.epilogue.thread) fusion_args; + fusion_args.alpha_ptr = nullptr; + fusion_args.beta_ptr = nullptr; + + // If alpha/beta are provided (via cmd line args) and are scalar, i.e., same alpha/beta applies to all batches. + // If pointers to alpha/beta are provided, i.e., alpha/beta can differ between batches/groups. + if (options.alpha != FLT_MAX){ + // Single alpha for all groups + fusion_args.alpha = options.alpha; + fusion_args.alpha_ptr_array = nullptr; + fusion_args.dAlpha = {_0{}, _0{}, 0}; + } + else { + fusion_args.alpha = 0; + fusion_args.alpha_ptr_array = alpha_device.get(); + // Only one alpha per each group + fusion_args.dAlpha = {_0{}, _0{}, 1}; + } + if (options.beta != FLT_MAX) { + // Single beta for all groups + fusion_args.beta = options.beta; + fusion_args.beta_ptr_array = nullptr; + fusion_args.dBeta = {_0{}, _0{}, 0}; + } + else { + fusion_args.beta = 0; + fusion_args.beta_ptr_array = beta_device.get(); + // Only one beta per each group + fusion_args.dBeta = {_0{}, _0{}, 1}; + } + // Output Block SF + // fusion_args.block_scale_factor_ptr = ptr_SFD.get(); // Enable for SF Output + // fusion_args.norm_constant_ptr = norm_constant_device.get(); // Enable for SF Output + + typename Gemm::GemmKernel::TileSchedulerArguments scheduler; + scheduler.raster_order = options.raster_order; + scheduler.max_swizzle_size = options.swizzle; + + if (host_problem_shapes_available) { + arguments = typename Gemm::Arguments { + cutlass::gemm::GemmUniversalMode::kGrouped, + {options.groups, problem_sizes.get(), options.problem_sizes_host.data()}, + {ptr_A.get(), stride_A.get(), ptr_B.get(), stride_B.get(), + ptr_SFA.get(), layout_SFA.get(), ptr_SFB.get(), layout_SFB.get()}, + {fusion_args, ptr_C.get(), stride_C.get(), ptr_D.get(), stride_D.get()}, + hw_info, scheduler + }; + } + else { + arguments = typename Gemm::Arguments { + cutlass::gemm::GemmUniversalMode::kGrouped, + {options.groups, problem_sizes.get(), nullptr}, + {ptr_A.get(), stride_A.get(), ptr_B.get(), stride_B.get(), + ptr_SFA.get(), layout_SFA.get(), ptr_SFB.get(), layout_SFB.get()}, + {fusion_args, ptr_C.get(), stride_C.get(), ptr_D.get(), stride_D.get()}, + hw_info, scheduler + }; + } + + return arguments; +} + +bool verify(const Options &options) { + using namespace cute; + bool passed = true; + for (int32_t i = 0; i < options.groups; ++i) { + auto problem = options.problem_sizes_host.at(i); + auto M = get<0>(problem); + auto N = get<1>(problem); + auto K = get<2>(problem); + + auto stride_A = cutlass::make_cute_packed_stride(StrideA{}, {M, K, 1}); + auto stride_B = cutlass::make_cute_packed_stride(StrideB{}, {N, K, 1}); + auto stride_C = cutlass::make_cute_packed_stride(StrideC{}, {M, N, 1}); + auto stride_D = cutlass::make_cute_packed_stride(StrideD{}, {M, N, 1}); + auto layout_A = make_layout(make_shape(M, K, 1), stride_A); + auto layout_B = make_layout(make_shape(N, K, 1), stride_B); + auto layout_C = make_layout(make_shape(M, N, 1), stride_C); + auto layout_D = make_layout(make_shape(M, N, 1), stride_D); + auto layout_SFA = Sm1xxBlkScaledConfig::tile_atom_to_shape_SFA(cute::make_shape(M, N, K, 1)); + auto layout_SFB = Sm1xxBlkScaledConfig::tile_atom_to_shape_SFB(cute::make_shape(M, N, K, 1)); + auto layout_SFD = Sm1xxBlockScaledOutputConfig::tile_atom_to_shape_SFD(cute::make_shape(M, N, K, 1)); + + // Create the arguments for host reference implementation + Tensor tensor_A = make_tensor(make_iterator(block_A.at(i).host_data()), layout_A); + Tensor tensor_SFA = make_tensor(block_SFA.at(i).host_data(), layout_SFA); + Tensor tensor_B = make_tensor(make_iterator(block_B.at(i).host_data()), layout_B); + Tensor tensor_SFB = make_tensor(block_SFB.at(i).host_data(), layout_SFB); + cutlass::reference::host::GettBlockScalingMainloopParams + mainloop_params{tensor_A, tensor_SFA, tensor_B, tensor_SFB}; + + auto tensor_C = cute::make_tensor(make_iterator(block_C.at(i).host_data()), layout_C); + auto tensor_ref_D = cute::make_tensor(make_iterator(block_ref_D.at(i).host_data()), layout_D); + + cutlass::reference::host::GettEpilogueParams< + float, float, + ElementAccumulator, ElementAccumulator, + decltype(tensor_C), decltype(tensor_ref_D) + > epilogue_params{}; + + epilogue_params.C = tensor_C; + epilogue_params.D = tensor_ref_D; + epilogue_params.alpha = alpha_host.at(i); + epilogue_params.beta = beta_host.at(i); + + cutlass::reference::host::Gemm3x(mainloop_params, epilogue_params); + + block_D.at(i).sync_host(); + // Check if output from CUTLASS kernel and reference kernel are equal or not + passed &= cutlass::reference::host::TensorEquals(block_ref_D.at(i).host_view(), block_D.at(i).host_view()); + } + return passed; +} + +/// Execute a given example GEMM computation +template +int run(Options &options, bool host_problem_shapes_available = true) +{ + std::cout << " Problem Sizes, Alpha, Beta " << std::endl; + for (int32_t i = 0; i < options.groups; ++i) { + std::cout << " " << options.problem_sizes_host.at(i); + std::cout << ", " << alpha_host.at(i) << ", " << beta_host.at(i) << std::endl; + } + std::cout << " Groups : " << options.groups << std::endl; + + std::cout << " Cluster Shape : " << options.cluster_shape.x << "x" << options.cluster_shape.y << std::endl; + std::cout << " Cluster Fallback Shape : " << options.cluster_shape_fallback.x << "x" << options.cluster_shape_fallback.y << std::endl; + + std::cout << " Raster Order : Along-" << options.raster_char << std::endl; + std::cout << " Max Swizzle Size : " << options.swizzle << std::endl; + + // Instantiate CUTLASS kernel depending on templates + Gemm gemm; + + // Create a structure of gemm kernel arguments suitable for invoking an instance of Gemm + auto arguments = args_from_options(options, host_problem_shapes_available); + + // Using the arguments, query for extra workspace required for matrix multiplication computation + size_t workspace_size = Gemm::get_workspace_size(arguments); + + // Allocate workspace memory + cutlass::device_memory::allocation workspace(workspace_size); + + // Check if the problem size is supported or not + CUTLASS_CHECK(gemm.can_implement(arguments)); + + // Initialize CUTLASS kernel with arguments and workspace pointer + CUTLASS_CHECK(gemm.initialize(arguments, workspace.get())); + + // Correctness / Warmup iteration + CUTLASS_CHECK(gemm.run(/* stream = */ nullptr, /* cuda_adapter = */ nullptr, /* launch_with_pdl = */ options.use_pdl)); + + cudaDeviceSynchronize(); + + // Check if output from CUTLASS kernel and reference kernel are equal or not + Result result; + if (options.verification) { + std::cout << " Host-side verification is now running - may be very slow for large cases." << std::endl; + result.passed = verify(options); + std::cout << " Disposition: " << (result.passed ? "Passed" : "Failed") << std::endl; + if (!result.passed) { + exit(-1); + } + } + else { + std::cout << " Verification is turned off for this run." << std::endl; + } + + // Run profiling loop + if (options.iterations > 0) + { + GpuTimer timer; + timer.start(); + for (int iter = 0; iter < options.iterations; ++iter) { + CUTLASS_CHECK(gemm.initialize(arguments, workspace.get())); + CUTLASS_CHECK(gemm.run(/* stream = */ nullptr, /* cuda_adapter = */ nullptr, /* launch_with_pdl = */ options.use_pdl)); + } + timer.stop(); + + // Compute average setup and runtime and GFLOPs. + float elapsed_ms = timer.elapsed_millis(); + result.avg_runtime_ms = double(elapsed_ms) / double(options.iterations); + result.gflops = options.gflops(result.avg_runtime_ms / 1000.0, options.problem_sizes_host); + + std::cout << " Avg runtime : " << result.avg_runtime_ms << " ms" << std::endl; + std::cout << " TFLOPS : " << result.gflops / 1000.0 << std::endl; + } + + return 0; +} + +#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) + +/////////////////////////////////////////////////////////////////////////////////////////////////// + +int main(int argc, char const **args) { + + // CUTLASS must be compiled with CUDA 12.8 Toolkit to run this example + if (__CUDACC_VER_MAJOR__ < 12 || + ((__CUDACC_VER_MAJOR__ == 12 && __CUDACC_VER_MINOR__ < 8) + ) + ) { + std::cerr << "This example requires CUDA 12.8 or newer.\n"; + // Returning zero so this test passes on older Toolkits. Its actions are no-op. + return 0; + } + + cudaDeviceProp props; + int current_device_id; + CUDA_CHECK(cudaGetDevice(¤t_device_id)); + CUDA_CHECK(cudaGetDeviceProperties(&props, current_device_id)); + cudaError_t error = cudaGetDeviceProperties(&props, 0); + if (props.major != 10 || (props.minor != 0 && props.minor != 1 && props.minor != 3)) { + std::cerr << "This example requires a GPU with compute capability 100a|f, 101a|f, or 103a|f)." << std::endl; + return 0; + } + + // + // Parse options + // + + Options options; + + options.parse(argc, args); + + if (options.help) { + options.print_usage(std::cout) << std::endl; + return 0; + } + +#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) + allocate(options); + initialize(options); + + // + // Evaluate CUTLASS kernels + // + + std::cout << "Running kernel with 1SM MMA config:" << std::endl; + run(options, false /*host_problem_shapes_available*/); + std::cout << "Running kernel with 2SM MMA config:" << std::endl; + run(options, false /*host_problem_shapes_available*/); +#endif + + return 0; +} + +///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/verification/evidence/local-snapshots/cutlass-audit.8aj07j/repo/include/cutlass/gemm/dispatch_policy.hpp b/verification/evidence/local-snapshots/cutlass-audit.8aj07j/repo/include/cutlass/gemm/dispatch_policy.hpp new file mode 100644 index 000000000..9cd9d2569 --- /dev/null +++ b/verification/evidence/local-snapshots/cutlass-audit.8aj07j/repo/include/cutlass/gemm/dispatch_policy.hpp @@ -0,0 +1,1590 @@ +/*************************************************************************************************** + * Copyright (c) 2023 - 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +#pragma once + +#include "cutlass/arch/arch.h" +#include "cutlass/gemm/gemm.h" + +#include "cute/layout.hpp" +#include "cute/numeric/integral_constant.hpp" // cute::false_type +#include "cute/atom/copy_traits_sm100.hpp" +#include "cutlass/detail/collective/sm103_kernel_type.hpp" +////////////////////////////////////////////////////////////////////////////// + +namespace cutlass::detail { + +template class U> +struct is_kernel_tag_of : cute::false_type {}; + +template